def collect(path='/home/msherstennikov/db/', rev=None, force=False): rev = rev or infra.xsim_nightly_revisions()[-2] tests = [] print 'Revision:', rev fname = '%s/r%d'%(path, rev) if os.path.exists(fname) and not force: print 'r%d already exists...'%rev data = stor.load(open(fname)) if data['ver_tests'][0] == __VERSION__ and data['ver_tests'][1] == __HASH__: print ' skip' return print ' outdated, force update!' for f in infra.xsim_nightly_warmup_test_logs(rev): print f tests.append(load_test(f)) data = { 'rev': rev, 'ver_tests': [ __VERSION__, __HASH__], 'tests': tests } with open(fname, 'w') as fw: fw.write(stor.dumps(data))
def collect_all(force=False): db_path = '/home/msherstennikov/db/' for rev in infra.xsim_nightly_revisions()[:-1]: collect(path=db_path, rev=rev, force=force) collect(path=db_path, rev=infra.xsim_nightly_last_revision(), force=True)
def add_all(): for rev in infra.xsim_nightly_revisions(): add_rev(rev)
def add_rev(rev = None): client = MongoClient(r'mongodb://*****:*****@ds047315.mongolab.com:47315/smi') db = client.get_default_database() if rev is None: rev = infra.xsim_nightly_revisions()[-1] print 'Process %d revison...'%rev if not os.path.exists('/sw/data/top100x/r{}'.format(rev)): print "Revision doesn't exist!" return test_logs = [f for f in glob.glob('/sw/data/top100x/r{}/xsim_bmark*.log'.format(rev)) if ('_O2_' in f or '_PJIT_' in f) and not f.endswith('_dbg.log')] failed_tests = [] fsim_tests = [] psim_tests = [] dec_part = len(test_logs)/10 for i, f in enumerate(test_logs): if i % dec_part == 0: sys.stdout.write('.%d'%(i/dec_part)) try: with infra.RedirectStdout(): tst = test.load_test(f) if isinstance(tst, test.TestFsim): fsim_tests.append(tst) elif isinstance(tst, test.TestPsim): psim_tests.append(tst) else: raise RuntimeError('Invalid test') except Exception as e: failed_tests.append([f, str(e)]) print print 'Fsim count:', len(fsim_tests) print 'Psim count:', len(psim_tests) print 'Fail count:', len(failed_tests) tests = test.group_tests(fsim_tests, psim_tests) db_rev = { 'rev': rev, 'tests': [] } for tst in tests: db_test = {} db_test['res_fsim'] = None db_test['res_psim'] = None db_test['name'] = test_name = tst[0].test_name_base if tst[0] else tst[1].test_name_base if tst[0]: db_test.update(tst[0].to_dict()) if tst[1]: db_test.update(tst[1].to_dict()) db_rev['tests'].append(db_test) import json with open('/local_disk/msherstennikov/dump.json', 'w') as f: f.write(json.dumps(db_rev, indent=2)) if db.tests.find_one({'rev': rev}): print 'Update document in DB...' db.tests.update_one({'rev': rev}, { '$set': db_rev }) else: print 'Insert document to DB...' db.tests.insert_one(db_rev) print "Finish"