def collect(path='/home/msherstennikov/db/', rev=None, force=False):
rev = rev or infra.xsim_nightly_revisions()[-2]
tests = []
print 'Revision:', rev
fname = '%s/r%d'%(path, rev)
if os.path.exists(fname) and not force:
print 'r%d already exists...'%rev
data = stor.load(open(fname))
if data['ver_tests'][0] == __VERSION__ and data['ver_tests'][1] == __HASH__:
print ' skip'
return
print ' outdated, force update!'
for f in infra.xsim_nightly_warmup_test_logs(rev):
print f
tests.append(load_test(f))
data = {
'rev': rev,
'ver_tests': [ __VERSION__, __HASH__],
'tests': tests
}
with open(fname, 'w') as fw:
fw.write(stor.dumps(data))
def collect_all(force=False):
db_path = '/home/msherstennikov/db/'
for rev in infra.xsim_nightly_revisions()[:-1]:
collect(path=db_path, rev=rev, force=force)
collect(path=db_path, rev=infra.xsim_nightly_last_revision(), force=True)
def add_all():
for rev in infra.xsim_nightly_revisions():
add_rev(rev)
def add_rev(rev = None):
client = MongoClient(r'mongodb://*****:*****@ds047315.mongolab.com:47315/smi')
db = client.get_default_database()
if rev is None:
rev = infra.xsim_nightly_revisions()[-1]
print 'Process %d revison...'%rev
if not os.path.exists('/sw/data/top100x/r{}'.format(rev)):
print "Revision doesn't exist!"
return
test_logs = [f for f in glob.glob('/sw/data/top100x/r{}/xsim_bmark*.log'.format(rev))
if ('_O2_' in f or '_PJIT_' in f) and not f.endswith('_dbg.log')]
failed_tests = []
fsim_tests = []
psim_tests = []
dec_part = len(test_logs)/10
for i, f in enumerate(test_logs):
if i % dec_part == 0:
sys.stdout.write('.%d'%(i/dec_part))
try:
with infra.RedirectStdout():
tst = test.load_test(f)
if isinstance(tst, test.TestFsim):
fsim_tests.append(tst)
elif isinstance(tst, test.TestPsim):
psim_tests.append(tst)
else:
raise RuntimeError('Invalid test')
except Exception as e:
failed_tests.append([f, str(e)])
print
print 'Fsim count:', len(fsim_tests)
print 'Psim count:', len(psim_tests)
print 'Fail count:', len(failed_tests)
tests = test.group_tests(fsim_tests, psim_tests)
db_rev = { 'rev': rev, 'tests': [] }
for tst in tests:
db_test = {}
db_test['res_fsim'] = None
db_test['res_psim'] = None
db_test['name'] = test_name = tst[0].test_name_base if tst[0] else tst[1].test_name_base
if tst[0]:
db_test.update(tst[0].to_dict())
if tst[1]:
db_test.update(tst[1].to_dict())
db_rev['tests'].append(db_test)
import json
with open('/local_disk/msherstennikov/dump.json', 'w') as f:
f.write(json.dumps(db_rev, indent=2))
if db.tests.find_one({'rev': rev}):
print 'Update document in DB...'
db.tests.update_one({'rev': rev}, { '$set': db_rev })
else:
print 'Insert document to DB...'
db.tests.insert_one(db_rev)
print "Finish"
