def main(options): r = StrictRedis.from_url(REDIS_URL_OUT) for project, v in iter_versions(restrict_project=options.restrict_project, restrict_version=options.restrict_version): reasons = [] for qm in ['line', 'mutant', 'mutant-line']: key = mk_key('out', [qm, 'file', '0', 'B.F', project, v]) + ':info' info = json.loads(r.get(key)) reasons.append(info[1]) print '{project}:{v}'.format(**locals()), ' '.join(reasons)
def covers(r, tool, project, v, suite, idxes): lookup_key = mk_key('exec', [tool, project, v, suite]) lookup_results = [] if len(idxes) == 0 else r.hmget(lookup_key, *idxes) assert all(x is not None for x in lookup_results) assert len(lookup_results) == len(idxes) covers = map(int, [ idx for (idx, lookup_result) in zip(idxes, lookup_results) if not is_empty(tool, json.loads(lookup_result)) ]) return covers
def get_tg_map(tests, idx_tp, relevant_tg_set): get_suite = lambda t: idx_tp[t][0] get_tid = lambda t: idx_tp[t][1] tests_sorted = sorted(tests, key=get_suite) # get suite tg_map = {} for suite, suite_tests_iter in groupby(tests_sorted, key=get_suite): suite_tests = list(suite_tests_iter) vals = rr.hmget(mk_key('tgs', [project, version, suite]), map(get_tid, suite_tests)) vals_mapped = [set(msgpack.unpackb(v)) & relevant_tg_set for v in vals] tg_map.update(dict(zip(suite_tests, vals_mapped))) return {k: v for k, v in tg_map.iteritems() if len(v) > 0}
def get_fails(r, tool, project, v, suite, idxes, raise_fail_but_not_exec=True): check_int(idxes) [lookup_key_tool, lookup_key_exec ] = [mk_key('fail', [T, project, v, suite]) for T in (tool, 'exec')] [failed_tool, failed_exec] = [ set(map(int, list(r.smembers(K)))) for K in (lookup_key_tool, lookup_key_exec) ] failed_idxes = [idx for idx in idxes if idx in failed_tool] failed_but_not_exec = [ idx for idx in failed_idxes if idx not in failed_exec ] if failed_but_not_exec: pass_key = mk_key('passcnt', [project, v, suite]) passes = defaultdict( int, zip(failed_but_not_exec, [ 0 if x is None else int(x) for x in r.hmget(pass_key, *failed_but_not_exec) ])) failed_but_known_to_pass = [ idx for idx in failed_but_not_exec if passes[idx] > 0 ] if failed_but_known_to_pass: raise Straggler('FAIL_BUT_KNOWN_TO_PASS', [tool, project, v, suite], failed_but_known_to_pass, fix=('cvgmeasure.cvg.do_cvg', lambda bundle: bundle[1:], lambda bundle: b_pvs(bundle) + " -K cvg_tool -a {tool}".format(tool=tool))) if raise_fail_but_not_exec: raise Straggler('FAIL_BUT_NOT_EXEC', [tool, project, v, suite], failed_but_not_exec, fix=('cvgmeasure.cvg.run_tests', lambda bundle: bundle[1:], b_pvs)) return failed_idxes
def map_tgs(r, rr, work_dir, input): project = input['project'] version = input['version'] redo = input.get('redo', False) with check_key( r, 'tgs-mapped', [project, version], redo=redo, other_keys=[], ) as done: map_file_name = '{project}:{version}'.format(project=project, version=version) get_file_from_cache_or_s3('darioush-map-files', map_file_name, str(work_dir / 'map.txt')) for tool in ['cobertura', 'codecover', 'jmockit', 'major']: # - get a test name for suite_type in [ 'dev', 'randoop.1', 'evosuite-strongmutation-fse.5' ]: test_list = r.hkeys( mk_key('nonempty', [tool, project, version, suite_type]))[:1] if tool != 'major': assert len(test_list) == 1 if len(test_list) == 1: break if test_list: [ test_name, ] = i_tn_s(r, test_list, suite_type) print tool, test_name # - prep the tmp dir get_files(work_dir, tool, project, version, suite_type, test_name) # - invoke java if tool == 'major' and len(test_list) == 0: tgs = [tg for tg in ALL_TGS if not tg.endswith('major')] else: tgs = ALL_TGS result = jar()[work_dir](*tgs) result = result.rstrip() tgs = [s.partition(' ')[2] for s in result.split('\n')] tg_i_s(rr, tgs, project, version, allow_create=True) done(len(tgs)) return "Success (tgs={tgs})".format(tgs=len(tgs))
def check_test_list(r, project, v, suite): key = mk_key('tms', [project, v, suite]) test_list = r.llen(key) if test_list == 0 and suite != 'dev': ## empty test list should correspond to ## a bad fetch result fetch_result_key = mk_key('fetch', [project, v]) fetch_result = r.hget(fetch_result_key, suite) if fetch_result not in ['missing', 'empty']: print suite, fetch_result raise Straggler('-- {project}:{v}:{suite}'.format(project=project, v=v, suite=suite)) else: print '-- {project}:{v}:{suite} has fetch result {fetch}'.format( project=project, v=v, suite=suite, fetch=fetch_result) return [], [] print '{key}: {tl}'.format(key=key, tl=test_list) test_list = r.lrange(key, 0, -1) test_names = i_tn_s(r, test_list, suite) assert len(test_list) == len(test_names) return map(int, test_list), test_names
def main(): r = StrictRedis.from_url(get_property('redis_url')) parser = OptionParser() parser.add_option("-p", "--project", dest="restrict_project", action="append") parser.add_option("-v", "--version", dest="restrict_version", action="append") parser.add_option("-x", "--commit", dest="action", action="store_true", default=False) parser.add_option("-q", "--queue", dest="queue", action="store", default='default') (options, args) = parser.parse_args(sys.argv) TOOLS = [ 'cobertura', ] #'codecover', 'jmockit', 'major'] SUITES = ['evosuite-branch-fse.{i}'.format(i=i) for i in xrange(8, 9)] #['dev'] # ['randoop.{i}'.format(i=i) for i in xrange(1,11)] + \ # ['evosuite-branch.{i}'.format(i=i) for i in xrange(0,10)] for suite in SUITES: for project, v in iter_versions(options.restrict_project, options.restrict_version): for tool in TOOLS: result = r.hget(mk_key('fetch', [project, v]), suite) if result == 'ok' or suite == 'dev': single_enqueue('cvgmeasure.cvg.compile_cache', json.dumps({ "project": project, "version": v, "suite": suite, "cvg_tool": tool, }), print_only=not options.action, timeout=1800, queue_name=options.queue)
def register_suite(suite, ff, tp_idx=tp_idx, idx_tp=idx_tp): tests = map(int, rr.hkeys(mk_key('tgs', [project, version, suite]))) suite_triggers = get_triggers_from_results(r, project, version, suite) fails = get_fails(suite) # always remove failing tests tests_no_fail = filter(lambda x: x not in fails, tests) tests_filtered = filter(lambda x: ff(x, suite_triggers), tests_no_fail) triggers = filter(lambda x: x in suite_triggers, tests_filtered) print 'Suite: {0}, Tests: {1}, Suite Triggers:{2}, Selected Triggers: {3}, Fails: {4}, No fails: {5}, Filtered: {6}'.format( suite, *map(len, [tests, suite_triggers, triggers, fails, tests_no_fail, tests_filtered])) for test in tests_filtered: tp_idx[(suite, test)] = tp_idx.get((suite, test), len(idx_tp)) if len(idx_tp) != len(tp_idx): idx_tp.append((suite, test)) triggers = set(tp_idx[(suite, t)] for t in triggers) return triggers, [tp_idx[(suite, test)] for test in tests_filtered]
def non_empty_agree_includes_tt_methods(input, hostname, pid): project = input['project'] version = input['version'] work_dir, d4j_path, redis_url = map( lambda property: get_property(property, hostname, pid), ['work_dir', 'd4j_path', 'redis_url'] ) r = StrictRedis.from_url(redis_url) key = mk_key('test-methods-agree-cvg-nonempty', [project, version]) test_methods = set(r.lrange(key, 0, -1)) tts = get_tts(project, version) print tts missing_tms = [tm for tm in tts if tm not in test_methods] if len(missing_tms) > 0: raise MissingTT(' '.join(missing_tms)) return "Success"
def minimization(r, rr, qm, project, version): source_list_key = source_list_keys[qm] all_tests = r.lrange(mk_key(source_list_key, [project, version]), 0, -1) tts = get_tts(project, version) missing_tts = tts - set(all_tests) if len(missing_tts) > 0: raise MissingTTs(' '.join(missing_tts)) print "Total # of tts: ", len(tts), " of ", len(all_tests) tts_with_unique_goals = get_unique_goal_tts(rr, qm, project, version, tts, all_tests) print "Guaranteed: ", len(tts_with_unique_goals) if len(tts_with_unique_goals) == 0: print "Reading..." tgs_map = {test: rr.smembers(mk_data_key('tgs', [qm, project, version, test])) for test in all_tests} greedy_minimization("G ", all_tests, tts, tgs_map) greedy_minimization("GE ", all_tests, tts, tgs_map, essentials=get_essential_tests) greedy_minimization("GRE", all_tests, tts, tgs_map, redundants=get_redundant_tests, essentials=get_essential_tests)
def enqueue_bundles(fun_dotted, json_str, queue_name='default', timeout=1800, print_only=False, restrict_project=None, restrict_version=None, tail_keys=[], tail_key_descr=None, at_front=False, check_key=None, **kwargs): q = Queue(queue_name, connection=StrictRedis.from_url(REDIS_URL_RQ)) r = StrictRedis.from_url(get_property('redis_url')) for project, i in iter_versions(restrict_project, restrict_version): input = {'project': project, 'version': i} additionals = json.loads(json_str) input.update(additionals) input.update({'timeout': timeout}) if tail_keys == []: tail_keys_to_iterate = [ [] ] # run the forloop once, but don't add any tail_key else: tail_keys_to_iterate = [ [tk] for tk in tail_keys ] # each of the tk's now counts, but singly for tail_key in tail_keys_to_iterate: if check_key: if r.hget(mk_key(check_key, [project, i]), ':'.join(tail_key)) is not None: continue input.update({} if tail_key_descr is None else {tail_key_descr: ':'.join(tail_key)}) doQ(q, fun_dotted, json.dumps(input), timeout, print_only, at_front)
#! /usr/bin/env python import redis import re import json from rq import get_failed_queue from cvgmeasure.conf import REDIS_URL_RQ, get_property_defaults from cvgmeasure.common import connect_to_redis, mk_key r = redis.StrictRedis.from_url(REDIS_URL_RQ) fq = get_failed_queue(connection=r) with connect_to_redis(get_property_defaults('redis_url')) as r2: for job in fq.get_jobs(): exc_info = '' if job.exc_info is None else job.exc_info reason = exc_info.split('\n')[-2:-1] for r in reason: if re.match( r"""NameError: global name 'bucket_name' is not defined""", r): input = json.loads(job.args[0]) project, version, cvg_tool, suite = map( lambda x: input[x], 'project version cvg_tool suite'.split(' ')) key = mk_key('fetch-result', [project, version, suite]) if r2.get(key) in ['empty', 'missing']: print job.id if project == 'Time' and version in [28, 29, 30, 31, 32]: print job.id
def non_empty_match(input, hostname, pid): project = input['project'] version = input['version'] input_key = input['key'] key_all = input['key_all'] should_fail_job = input.get('should_fail_job', True) work_dir, d4j_path, redis_url = map( lambda property: get_property(property, hostname, pid), ['work_dir', 'd4j_path', 'redis_url'] ) r = StrictRedis.from_url(redis_url) key = mk_key(key_all, [project, version]) test_classes = r.llen(key) #r.lrange(key, 0, -1) cobertura, codecover, jmockit = [r.hkeys(mk_key(input_key, [tool, project, version])) for tool in ['cobertura', 'codecover', 'jmockit']] exclude_static_fields_from = [ ('Closure', 117), ('Closure', 100), ('Closure', 43), ('Closure', 47), ('Closure', 37), ('Math', 3), ('Math', 63), ('Lang', 6), ('Lang', 28), ('Lang', 17), ('Lang', 19)] exclude_static_fields = [t for t in cobertura if t not in codecover and t in jmockit and \ (project, version) in exclude_static_fields_from and plausable_static_field(project, version, t)] codecover_exception_from = [('Chart', 1), ('Lang', 64)] codecover_exception = [t for t in codecover if t not in jmockit and t not in cobertura and \ (project, version) in codecover_exception_from and plausable_codecover_field(project, version, t)] cobertura = [t for t in cobertura if t not in exclude_static_fields] jmockit = [t for t in jmockit if t not in exclude_static_fields] codecover = [t for t in codecover if t not in codecover_exception] core = set(cobertura) & set(codecover) & set(jmockit) cobertura_, codecover_, jmockit_ = [[t for t in l if t not in core] for l in (cobertura, codecover, jmockit)] print test_classes, '/', len(core), "Agreement" print len(exclude_static_fields), " Excluded from jmockit and cobertura as static field initializers" print len(codecover_exception), " Excluded from codecover as static field initializers" print "---" print len(cobertura_), sorted(cobertura_) print len(codecover_), sorted(codecover_) print len(jmockit_), sorted(jmockit_) fails = [] fails.extend(with_fails(lambda: check_sub(cobertura, 'cobertura', codecover, 'codecover'))) fails.extend(with_fails(lambda: check_sub(codecover, 'codecover', cobertura, 'cobertura'))) fails.extend(with_fails(lambda: check_sub(cobertura, 'cobertura', jmockit, 'jmockit'))) fails.extend(with_fails(lambda: check_sub(jmockit, 'jmockit', cobertura, 'cobertura'))) fails.extend(with_fails(lambda: check_sub(codecover, 'codecover', jmockit, 'jmockit'))) fails.extend(with_fails(lambda: check_sub(jmockit, 'jmockit', codecover, 'codecover'))) if fails and should_fail_job: raise SubMismatch(' AND '.join([str(ex) for ex in fails])) return "Success"
def method_list_matches(input, hostname, pid): project = input['project'] version = input['version'] work_dir, d4j_path, redis_url = map( lambda property: get_property(property, hostname, pid), ['work_dir', 'd4j_path', 'redis_url'] ) r = StrictRedis.from_url(redis_url) key = mk_key('test-methods', [project, version]) test_methods_from_redis = r.lrange(key, 0, -1) work_dir_path = local.path(work_dir) / ('child.%d' % os.getpid()) print work_dir_path with refresh_dir(work_dir_path, cleanup=True): with add_to_path(d4j_path): with checkout(project, version, work_dir_path / 'checkout'): d4()('compile') test_methods_from_d4 = d4()('list-tests').rstrip().split('\n') with local.env(SUCCESS_OUT="passing-tests.txt"): failing_tests = test() with open("passing-tests.txt") as f: test_methods_from_run = [x[len('--- '):] for x in f.read().rstrip().split('\n')] with open("count-of-tests.txt") as f: per_run_counts = [int(line.rstrip()) for line in f] count_of_tests_from_run = sum(per_run_counts) if project == 'Lang' and version >= 37: ## In this case, we know that some tests may fail ## this is really ugly, but I'm doing it. klass_name = 'org.apache.commons.%s.builder.ToStringBuilderTest' % ( 'lang' if version > 39 else 'lang3', ) expected_fails = [method for method in failing_tests if method.startswith(klass_name)] single_run_fails = test(['-t', klass_name]) if len(single_run_fails) > 0: raise TestFail('Single run failed: ' + ' '.join(single_run_fails)) elif project == 'Time': ## In this case, org.joda.time.chrono.gj.MainTest ## isn't really a jUnit test because it doesn't have a public ## constructor. We fix this during run by replacing it ## with two classes with a public constructor, each of which ## initializes the original class with parameters used during ## testing bad_class = 'org.joda.time.chrono.gj.MainTest' good_class1, good_class2 = ['edu.washington.cs.testfixer.time.GjMainTest' + s for s in ('1', '2')] tname = '::testChronology' tcs = [tc for tc, _, _ in [method.partition('::') for method in test_methods_from_run]] idx = tcs.index(bad_class) test_methods_from_run[idx] = good_class1 + tname test_methods_from_run[idx+1] = good_class2 + tname tcsd4 = [tc for tc, _, _ in [method.partition('::') for method in test_methods_from_d4]] idxd4 = tcsd4.index(bad_class) test_methods_from_d4 = test_methods_from_d4[:idxd4] + [good_class1 + tname, good_class2 + tname] + test_methods_from_d4[idxd4+1:] expected_fails = [] else: expected_fails = [] unexpected_fails = [method for method in failing_tests if method not in expected_fails] # Sanity check #0 -- check out the test fails if len(unexpected_fails) > 0: raise TestFail(' '.join(unexpected_fails)) # Sanity check #1 -- number of tests counted through the runner should equal # the length of the list of passing tests the runner outputs num_tests = len(test_methods_from_d4) if num_tests != count_of_tests_from_run: raise LenMismatch("Test methods from d4 (%d) don't match counter (%d)" % (num_tests, count_of_tests_from_run)) # Sanity check #2 -- we should not be running duplicate tests no_dups(test_methods_from_run, 'test methods from run') # Sanity check #3 -- we should not be list-outputting duplicate tests no_dups(test_methods_from_d4, 'test methods from d4') # Sanity check #4 -- we should not have duplicate tests in redis store no_dups(test_methods_from_redis, 'test methods from redis') # Sanity check #5 -- tests output from the runner should match the tests output from # d4 list-tests check_eq(test_methods_from_run, 'test methods from run', test_methods_from_d4, 'test methods from d4') # Sanity check #6 -- test methods from d4 should match ones in redis # # Preprocess step: We know that these methods were wrongly inserted: #lang_methods = [ # 'org.apache.commons.lang3.EnumUtilsTest::test_processBitVectors_longClass', # 'org.apache.commons.lang3.builder.ReflectionToStringBuilderConcurrencyTest::testLinkedList', # 'org.apache.commons.lang3.builder.ReflectionToStringBuilderConcurrencyTest::testArrayList', # 'org.apache.commons.lang3.builder.ReflectionToStringBuilderConcurrencyTest::testCopyOnWriteArrayList', # 'org.apache.commons.lang3.builder.ReflectionToStringBuilderMutateInspectConcurrencyTest::testConcurrency', #] #lang_methods_in_redis = [method for method in lang_methods if method in test_methods_from_redis] #for lang_method in lang_methods_in_redis: # print "Removing %s from redis:" % lang_method # print r.lrem(key, 1, lang_method) #if lang_methods_in_redis: # print "Redis store was modified, reloading list before testing" # test_methods_from_redis = r.lrange(key, 0, -1) check_eq(test_methods_from_redis, 'test methods from redis', test_methods_from_d4, 'test methods from d4') return "Success"
def check_cvg(r, tool, project, v, suite, t_idxs, ts): key = mk_key('exec', [tool, project, v, suite]) cvg_infos = r.hmget(key, *t_idxs) assert len(cvg_infos) == len(t_idxs) nils = [(t_idx, t) for (t_idx, t, cvg_info) in zip(t_idxs, ts, cvg_infos) if cvg_info is None] print len(nils), len(ts) if suite == 'dev': check_for_nil_classes = [ tc for tc, _, _ in [t.partition('::') for (t_idx, t) in nils] ] check_for_nil_classes = list(set(check_for_nil_classes)) check_for_nil_classes_idxes = tn_i_s(r, check_for_nil_classes, 'dev') non_nil_classes_idxes = set( cobertura_covers(r, project, v, suite, check_for_nil_classes_idxes)) nil_class_dict = { class_name: idx not in non_nil_classes_idxes for ( class_name, idx) in zip(check_for_nil_classes, check_for_nil_classes_idxes) } nil_idxes = [(t_idx_, tc, tm) for (t_idx_, (tc, _, tm)) in [(t_idx, t.partition('::')) for (t_idx, t) in nils] if nil_class_dict.get(tc) is False] # really only need the idxes nil_idxes = [t_idx for (t_idx, _, _) in nil_idxes] else: nil_idxes = [t_idx for (t_idx, _) in nils] if tool == 'cobertura' and nil_idxes: raise Straggler('COBERTURA_NOT_RUN', [tool, project, v, suite], idxes=nil_idxes, fix=('cvgmeasure.cvg.do_cvg', lambda bundle: bundle[1:], lambda bundle: b_pvs(bundle) + " -K cvg_tool -a {tool}".format(tool=tool))) cc = cobertura_covers(r, project, v, suite, nil_idxes) if cc != []: raise Straggler('CVG_NOT_RUN_COBERTURA_NONEMPTY', [tool, project, v, suite], idxes=cc, fix=('cvgmeasure.cvg.do_cvg', lambda bundle: bundle[1:], lambda bundle: b_pvs(bundle) + " -K cvg_tool -a {tool}".format(tool=tool))) # time to check s3 non_nils = [ (t_idx, t) for (t_idx, t, cvg_info) in zip(t_idxs, ts, cvg_infos) if cvg_info is not None and not is_empty(tool, json.loads(cvg_info)) ] print '- non-nil len: {0}'.format(len(non_nils)) if non_nils: s3_list = list_from_s3('cvg-files', [tool, project, v, suite]) s3_tname_list = set( [key.name.rpartition('/')[2] for key in s3_list if key.size > 0]) non_nils_missing_from_s3 = [(t_idx, t) for (t_idx, t) in non_nils if t not in s3_tname_list] if len(non_nils_missing_from_s3) != 0: raise Straggler( 'NON_NIL_CVG_BUT_NO_S3', [tool, project, v, suite], idxes=[t_idx for (t_idx, _) in non_nils_missing_from_s3], fix=('cvgmeasure.cvg.do_cvg', lambda bundle: bundle[1:], lambda bundle: b_pvs(bundle) + " -K cvg_tool -a {tool} -j '{{\"redo\": true}}'".format( tool=tool))) return "Cvg for in s3 : {0}".format(len(non_nils))
def minimization(r, rr, conn, qm_name, granularity, project, version, bases, augs): qm = QMS[qm_name] # 0. pull granularity info from cloud fn = '{project}:{version}.txt'.format(project=project, version=version) get_file_from_cache_or_s3('darioush-mm-files', '{project}:{version}'.format(project=project,version=version), fn) with open(fn) as f: granularity_info = json.loads(f.read()) # 1. Get the testing goals that match the qm all_tgs = rr.hkeys(mk_key('tg-i', [project, version])) relevant_tgs = [tg for tg in filter(qm['fun'], all_tgs) if GRANS[granularity](tg, granularity_info)] relevant_tg_is = tg_i_s(rr, relevant_tgs, project, version) relevant_tg_set = set(relevant_tg_is) tp_idx = {} idx_tp = [] fails = {} def get_fails(suite, fails=fails): if not fails.has_key(suite): fail_ids = set(map(int, r.smembers(mk_key('fail', ['exec', project, version, suite])))) fails[suite] = fail_ids return fails[suite] def register_suite(suite, ff, tp_idx=tp_idx, idx_tp=idx_tp): tests = map(int, rr.hkeys(mk_key('tgs', [project, version, suite]))) suite_triggers = get_triggers_from_results(r, project, version, suite) fails = get_fails(suite) # always remove failing tests tests_no_fail = filter(lambda x: x not in fails, tests) tests_filtered = filter(lambda x: ff(x, suite_triggers), tests_no_fail) triggers = filter(lambda x: x in suite_triggers, tests_filtered) print 'Suite: {0}, Tests: {1}, Suite Triggers:{2}, Selected Triggers: {3}, Fails: {4}, No fails: {5}, Filtered: {6}'.format( suite, *map(len, [tests, suite_triggers, triggers, fails, tests_no_fail, tests_filtered])) for test in tests_filtered: tp_idx[(suite, test)] = tp_idx.get((suite, test), len(idx_tp)) if len(idx_tp) != len(tp_idx): idx_tp.append((suite, test)) triggers = set(tp_idx[(suite, t)] for t in triggers) return triggers, [tp_idx[(suite, test)] for test in tests_filtered] # 2. Get the list of tests with some coverage goals from base / augmenting pools def flatten(l): return reduce(lambda a,b: a+b, l, []) def pool_to_tests(pools, tp_idx=tp_idx, idx_tp=idx_tp): tests = [] triggers = set([]) for pool in pools: pool_suites, pool_filter = POOLS[pool] for pool_suite in pool_suites: a, b = register_suite(pool_suite, pool_filter, tp_idx=tp_idx, idx_tp=idx_tp) tests.append(b) triggers.update(a) return triggers, flatten(tests) base_tp_idx = {} base_idx_tp = [] base_triggers, base_tests = pool_to_tests(bases, base_tp_idx, base_idx_tp) aug_triggers, aug_tests = pool_to_tests(augs, tp_idx, idx_tp) print "Triggers: Base {0}, Aug {1}".format(len(base_triggers), len(aug_triggers)) print "Reading..." tRead = Timer() tRead.start() def get_tg_map(tests, idx_tp, relevant_tg_set): get_suite = lambda t: idx_tp[t][0] get_tid = lambda t: idx_tp[t][1] tests_sorted = sorted(tests, key=get_suite) # get suite tg_map = {} for suite, suite_tests_iter in groupby(tests_sorted, key=get_suite): suite_tests = list(suite_tests_iter) vals = rr.hmget(mk_key('tgs', [project, version, suite]), map(get_tid, suite_tests)) vals_mapped = [set(msgpack.unpackb(v)) & relevant_tg_set for v in vals] tg_map.update(dict(zip(suite_tests, vals_mapped))) return {k: v for k, v in tg_map.iteritems() if len(v) > 0} base_tg_map = get_tg_map(base_tests, base_idx_tp, relevant_tg_set) get_all_tgs = lambda tg_map: reduce(lambda a, b: a | b, tg_map.values(), set([])) get_all_tests = lambda tg_map: set(tg_map.keys()) base_tgs = get_all_tgs(base_tg_map) print "Relevant_tgs {0}, Base tgs: {1}".format(*map(len, (relevant_tg_set, base_tgs))) base_relevant_tests = get_all_tests(base_tg_map) base_relevant_tts = base_triggers & base_relevant_tests aug_relevant_tg_map = get_tg_map(aug_tests, idx_tp, relevant_tg_set) aug_tgs = get_all_tgs(aug_relevant_tg_map) aug_relevant_tests = get_all_tests(aug_relevant_tg_map) aug_relevant_tts = aug_triggers & aug_relevant_tests aug_additional_tg_map = get_tg_map(aug_tests, idx_tp, relevant_tg_set - base_tgs) aug_additional_tgs = get_all_tgs(aug_relevant_tg_map) aug_additional_tests = get_all_tests(aug_additional_tg_map) aug_additional_tts = aug_triggers & set(aug_additional_tests) print "Triggers: Base_relevant {0}, Aug_relevant {1}, Aug_additional {2}".format(*map(len, (base_relevant_tts, aug_relevant_tts, aug_additional_tts))) tRead.stop() print "Reading complete {0} msecs.".format(tRead.msec) tts_with_unique_goals = get_unique_goal_tts(aug_additional_tts, aug_additional_tests, aug_additional_tg_map) print "Guaranteed: ", len(tts_with_unique_goals) print "Building bit vectors" all_tgs = { k: i for i, k in enumerate(sorted(get_all_tgs(aug_additional_tg_map)))} def to_bitvector(s): bv = bitarray(len(all_tgs)) bv.setall(False) for tg in s: bv[all_tgs[tg]] = 1 return bv bit_map = {k : to_bitvector(v) for k, v in aug_additional_tg_map.iteritems()} print "Built.." def timing(tests, idx_tp=idx_tp): tps = map(lambda t: idx_tp[t], tests) tps_sorted = sorted(tps, key=lambda (suite, i): suite) total_time = 0 for suite, i_it in groupby(tps_sorted, key=lambda(suite, i): suite): i_s = map(lambda (suite, i): i, i_it) if suite == 'dev': tns = i_tn_s(r, i_s, suite) tc_ns = set(tn.partition('::')[0] for tn in tns) tc_is = tn_i_s(r, list(tc_ns), suite, allow_create=False) all_is = tc_is + i_s else: all_is = i_s method_times = [msgpack.unpackb(b) for b in r.hmget(mk_key('time', [project, version, suite]), all_is)] bad_timings = [(time, i) for (time, i) in zip(method_times, all_is) if any(t < 0 for t in time)] if bad_timings: raise Exception('bad timing for tests: {project}, {version}, {suite} {idxs}'.format( project=project, version=version, suite=suite, idxs=' '.join(map(str, [i for (time, i) in bad_timings])))) def aggr(l): if any(x == -1 for x in l): raise Exception('bad timing for tests: {project}, {version}, {suite}'.format( project=project, version=version, suite=suite)) # let's go with average return reduce(lambda a, b: a+b, l)/len(l) method_times_aggregate = [aggr(timings) for timings in method_times] suite_time = reduce(lambda a, b: a+b, method_times_aggregate, 0) total_time += suite_time return total_time base_time = timing(base_tests, base_idx_tp) def reason(given): if len(tts_with_unique_goals) > 0: return 'U' elif len(aug_additional_tts) == 0: return 'X' else: return given # row independent info suite_schema = lambda x: ['{0} {1}'.format(x,i) for i in ('triggers', 'tests', 'tgs', 'time')] schema = [ 'Relevant tgs', 'Reason',] + \ suite_schema('Base') + suite_schema('Base relevant') + suite_schema('Aug') + suite_schema('Aug relevant') \ + suite_schema('Aug additional') info = ( len(relevant_tgs), reason('-'), len(base_triggers), len(base_tests), len(base_tgs), base_time, # all of base suite len(base_relevant_tts), len(base_relevant_tests), len(base_tgs), timing(base_relevant_tests, base_idx_tp), # relevant part of base suite len(aug_triggers), len(aug_tests), len(aug_tgs), timing(aug_tests), # all of augmentation pool len(aug_relevant_tts), len(aug_relevant_tests), len(aug_tgs), timing(aug_relevant_tests),# relevant part of aug pool len(aug_additional_tts), len(aug_additional_tests), len(aug_additional_tgs), timing(aug_additional_tests), # part of aug pool in addition to the base suite ) print ', '.join('{0}: {1}'.format(a,b) for a,b in (zip(schema, info))) print info key = mk_key('out', [qm['name'], granularity, '.'.join(sorted(bases)), '.'.join(sorted(augs)), project, version]) save_row_independent(conn, key, info) minimization = lambda **kwargs: get_do_one(aug_additional_tests, bit_map, **kwargs) redundants, eq_partition = get_redundant_and_equal_tests(bit_map, aug_additional_tests) #print redundants, eq_partition choice = lambda eq_partition=eq_partition: reduce(lambda a, b: a | b, map(lambda s: s - set(random.sample(s, 1)), eq_partition), set()) algo_params = [ ('G', {}), ('GE', {'essentials': get_essential_tests}), ('GRE', {'redundants': lambda redundants=redundants: redundants , 'essentials': get_essential_tests}), ('GREQ', {'redundants': lambda redundants=redundants: redundants | choice(), 'essentials': get_essential_tests}), ] algos = [(name, lambda algo_param=algo_param: minimization(**algo_param)) for (name, algo_param) in algo_params] for algo, fun in algos: info = {'nofd': 0, 'cnt': 0, 'cvrtg': None, 'times':[], 'cnts':[], 'reasons': {'S': 0, 'R': 0, 'E': 0}} label=lambda :'{algo:<4} {info[cvrtg]:>4}tgs {info[reasons][S]:>3}S {info[reasons][R]:>3}R {info[reasons][E]:>3}E {info[nofd]:>3}F {time:>6}ms'.format(info=info, algo=algo, time=''.join(map(str,info['times'][-1:]))) with click.progressbar(xrange(RUNS), label=label(), width=20) as bar: def pr(results): rr = process_results(results, aug_additional_tts, timing) determined_by, sel_tts, sel_cnt, sel_tgs, time = rr info['cnt'] += 1 if sel_tts == 0: info['nofd'] += 1 info['reasons'][determined_by] += 1 info['cnts'].append(sel_cnt) info['times'].append(time) if info['cvrtg'] is None: info['cvrtg'] = sel_tgs else: assert info['cvrtg'] == sel_tgs bar.label=label() return rr results = greedy_minimization(bar, fun(), pr) # vals should be a dict from {'GRE:1': (fault_detection, reason, triggers, count, tgs, time) } save_rows(conn, key, {'{algo}:{run}'.format(algo=algo, run=run_id+1): ( 1 if selected_triggers > 0 else 0, reason(determined_by), selected_triggers, selected_count, selected_tgs, selected_time ) for (run_id, (determined_by, selected_triggers, selected_count, selected_tgs, selected_time)) in enumerate(results)} )
def get_triggers_from_results(r, project, version, suite): result = json.loads(r.hget(mk_key('trigger', [project, version]), suite)) return set(result)
def get_fails(suite, fails=fails): if not fails.has_key(suite): fail_ids = set(map(int, r.smembers(mk_key('fail', ['exec', project, version, suite])))) fails[suite] = fail_ids return fails[suite]
def enqueue_bundles_sliced(fun_dotted, json_str, bundle_key, source_key, tail_keys=[], tail_key_descr=None, queue_name='default', timeout=1800, print_only=False, restrict_project=None, restrict_version=None, bundle_size=10, bundle_offset=0, bundle_max=None, alternates=None, alternate_key=None, check_key=None, filter_function=None, filter_arg=None, map_function=None, at_front=False, **kwargs): if bundle_key is None: raise Exception("bundle key not provided [-k]") if tail_keys == []: tail_keys_to_iterate = [ [] ] # run the forloop once, but don't add any tail_key else: tail_keys_to_iterate = [[tk] for tk in tail_keys ] # each of the tk's now counts, but singly q = Queue(queue_name, connection=StrictRedis.from_url(REDIS_URL_RQ)) r = StrictRedis.from_url(get_property('redis_url')) rr = StrictRedis.from_url(REDIS_URL_TG) R = defaultdict(lambda: r) R['tgs'] = rr key_type = None # such hack if source_key.startswith('file:'): key_type = 'file' _, _, fn = source_key.partition(':') source_key = 'file' file_data = defaultdict(list) with open(fn) as f: for line in f: line_data = json.loads(line) file_data.update(line_data) for tail_key in tail_keys_to_iterate: for project, i in iter_versions(restrict_project, restrict_version): key = mk_key(source_key, [project, i] + tail_key) if key_type != 'file': key_type = r.type(key) if key_type == 'list': size = r.llen(key) elif key_type == 'hash': size = r.hlen(key) elif key_type == 'file': size = len(file_data[key]) elif key_type == 'none': size = 0 else: raise Exception('-- Unexpected key type: {0}'.format(key_type)) if bundle_max is not None: size = min(size, bundle_max) mf = (lambda _1, x, _2: x ) if map_function is None else get_fun(map_function) already_computed = {} if alternate_key and check_key: for alternate in alternates: _key = mk_key(check_key, [alternate, project, i] + tail_key) already_computed[alternate] = set( mf(r, R[check_key].hkeys(_key), tail_key)) if check_key and not alternate_key: _key = mk_key(check_key, [project, i] + tail_key) already_computed = set( mf(r, R[check_key].hkeys(_key), tail_key)) else: already_computed = set() if key_type == 'hash': all_items = r.hkeys(key) elif key_type == 'file': all_items = file_data[key] elif key_type == 'list': all_items = r.lrange(key, 0, -1) elif key_type == 'none': #print key all_items = [] if filter_function is not None: ff = get_fun(filter_function) all_items = ff(r, project, i, tail_key, filter_arg, all_items) all_items = mf(r, all_items, tail_key) def bundle_it(l, more_dict={}): if len(l) == 0: return for j in xrange(bundle_offset, size, bundle_size): if key_type in ('hash', 'file', 'list'): bundle = l[j:j + bundle_size] elif key_type == 'none': bundle = [] if len(bundle) == 0: continue input = { 'project': project, 'version': i, bundle_key: bundle } tk_input = {} if tail_key_descr is None else { tail_key_descr: ':'.join(tail_key) } additionals = json.loads(json_str) input.update(tk_input) input.update(more_dict) input.update(additionals) input.update({'timeout': timeout}) doQ(q, fun_dotted, json.dumps(input), timeout, print_only, at_front) if alternate_key: for alternate in alternates: if check_key: all_items = [ item for item in all_items if item not in already_computed[alternate] ] bundle_it(all_items, {alternate_key: alternate}) else: if check_key: all_items = [ item for item in all_items if item not in already_computed ] bundle_it(all_items)