def mk_additional_test_info(): s, d = StrictRedis.from_url(SRC), StrictRedis.from_url(DST) for suite in ['evosuite-branch.{0}'.format(i) for i in xrange(0, 10) ] + ['randoop.{0}'.format(i + 1) for i in xrange(0, 10)]: for project, version in iter_versions(): for KN in ['passcnt']: fail_key = ':'.join( [PRE_SRC, KN, project, str(version), suite]) dst_fail_key = ':'.join([KN, project, str(version), suite]) fail_members = list(s.hkeys(fail_key)) if len(fail_members) > 0: fail_idxes = tn_i_s(d, fail_members, suite) results = s.hmget(fail_key, *fail_members) mapping = dict(zip(fail_idxes, results)) d.hmset(dst_fail_key, mapping) for tool in ['cobertura', 'codecover', 'jmockit', 'exec']: for KN in ['fail']: fail_key = ':'.join( [PRE_SRC, KN, tool, project, str(version), suite]) print fail_key dst_fail_key = ':'.join( [KN, tool, project, str(version), suite]) fail_members = list(s.smembers(fail_key)) fail_idxes = tn_i_s(d, fail_members, suite) if len(fail_members) > 0: d.sadd(dst_fail_key, fail_idxes)
def main(options): tool = options.tool bunzip2 = local['bunzip2'] bzip2 = local['bzip2'] tar = local['tar'] for project, v in iter_versions(options.restrict_project, options.restrict_version, old=True, minimum=True): tarname = get_name_of_tar(tool, project) with tarfile.open(str(tarname), "r") as t: def mapping(inv, v=v): assert inv == v _, _, result = d4()('match-commits', '-p', project, '-v', '{0}f'.format(inv), '-c', 'fse-commit-dbs').rstrip().partition('-> ') return int(result) for source, dest in get_extract_list(tool, project, v, mapping=mapping): alt_source = source[:-len('.bak')] try: extract(t, source, OUT_DIR / dest) # check for broken files fixed_file = t.extractfile(alt_source) with tarfile.open(fileobj=fixed_file) as t_fixed: broken_files = [name[:-len('.broken')] for name in t_fixed.getnames() if name.endswith('.broken')] fixed_file.close() # now we have to remove the broken files from the archive if broken_files: plain_name = str(OUT_DIR / dest)[:-len('.bz2')] bunzip2(str(OUT_DIR / dest)) # get number of .java currently in archive with tarfile.open(plain_name) as t_current: java_files = [name for name in t_current.getnames() if name.endswith('.java') and not name.endswith('_scaffolding.java')] if len(broken_files) == len(java_files): # we are going to remove them all. rm(plain_name) touch(plain_name[:-len('.tar')] + '.empty') else: for broken_file in broken_files: tar('--delete', '-f', plain_name, './' + broken_file) bzip2(plain_name) print "+ {source} -> {dest} ({broken} / {total} broken)".format( source=source, dest=dest, broken=len(broken_files), total=len(java_files) ) else: print "+ {source} -> {dest} (none broken)".format(source=source, dest=dest) except KeyError: try: # no .bak file was ever created, so we are good. extract(t, alt_source, OUT_DIR / dest) print "* {source} -> {dest} (as is)".format(source=alt_source, dest=dest) except KeyError: print "- {source} -> missing".format(source=alt_source) touch(str(OUT_DIR / dest)[:-len('.tar.bz2')] + '.missing')
def mk_bundles(): s, d = StrictRedis.from_url(SRC), StrictRedis.from_url(DST) for suite in ['evosuite-branch.{0}'.format(i) for i in xrange(0, 10) ] + ['randoop.{0}'.format(i + 1) for i in xrange(0, 10)]: for project, version in iter_versions(): for tool in ['cobertura', 'codecover', 'jmockit']: key = ':'.join([ PRE_SRC, 'test-methods-exec', tool, project, str(version), suite, 'bundles' ]) result_key = ':'.join( [PRE_SRC, 'cvg', tool, project, str(version), suite]) print key execed_bundles = s.hkeys(key) if len(execed_bundles) == 0: continue idxes = tn_i_s(d, execed_bundles, suite) results = s.hmget(result_key, *execed_bundles) results_ = [ result if result is not None else json.dumps(None) for result in results ] dst_key = ':'.join( ['exec', tool, project, str(version), suite]) mapping = dict(zip(idxes, results_)) #{idx: for idx in idxes} if len(mapping) > 0: d.hmset(dst_key, mapping)
def main(): r = StrictRedis.from_url(get_property('redis_url')) parser = OptionParser() parser.add_option("-p", "--project", dest="restrict_project", action="append") parser.add_option("-v", "--version", dest="restrict_version", action="append") parser.add_option("-x", "--commit", dest="action", action="store_true", default=False) parser.add_option("-q", "--queue", dest="queue", action="store", default='default') (options, args) = parser.parse_args(sys.argv) #TOOLS = ['cobertura', 'codecover', 'jmockit', 'major'] SUITES = ['evosuite-strongmutation-fse.{i}'.format(i=i) for i in xrange(1,2)] # ['randoop.{i}'.format(i=i) for i in xrange(1,11)] + \ # ['evosuite-branch.{i}'.format(i=i) for i in xrange(0,10)] for suite in SUITES: for project, v in iter_versions(options.restrict_project, options.restrict_version): #for tool in TOOLS: #result = r.get(mk_key('fetch', [project, v, suite])) #if result == 'ok': single_enqueue('cvgmeasure.cvg.test_lists_gen', json.dumps({ "project": project, "version": v, "suite": suite, }), print_only=not options.action, timeout=1800, queue_name=options.queue)
def main(options): r = redis.StrictRedis.from_url(get_property('redis_url')) rr = redis.StrictRedis.from_url(REDIS_URL_TG) rrr = redis.StrictRedis.from_url(REDIS_URL_OUT) for qm in options.qms: for gran in options.grans: for experiment in options.experiments: bases, _, pools = experiment.partition(',') if options.print_only: print '''./main.py qb cvgmeasure.select.m {project} {version} -j '{json}' {additional}'''.format( project=''.join('-p {0}'.format(rp) for rp in options.restrict_project), version=''.join('-v {0}'.format(rv) for rv in options.restrict_version), json=json.dumps({ 'granularity': gran, 'bases': bases, 'pools': pools, "qm": qm }), additional=options.print_only) else: for project, v in iter_versions( restrict_project=options.restrict_project, restrict_version=options.restrict_version): print "----( %s %d -- %s : %s)----" % (project, v, qm, gran) minimization(r, rr, rrr, qm, gran, project, v, bases.split('.'), pools.split('.')) print
def s3_files_for(r_from, r_to, projects=[], versions=[]): for project, version in iter_versions(projects, versions): key = 'results:test-methods:{project}:{version}:dev'.format( project=project, version=version ) test_methods = r_to.lrange(key, 0, -1) print "{project}:{version} has {tms} methods".format( project=project, version=version, tms=len(test_methods) ) for test_method in test_methods: c_name, _, m_name = test_method.partition('::') is_class_empty = [] for tool in ["codecover", "cobertura", "jmockit"]: class_key = mk_key('test-classes-cvg', [tool, project, version]) class_r = json.loads(r_from.hget(class_key, c_name)) is_class_empty.append(is_empty(tool, class_r)) if all(is_class_empty): continue is_method_nonempty = [] for tool in ["codecover", "cobertura", "jmockit"]: method_key = mk_key('test-methods-run-cvg', [tool, project, version]) try: method_r = json.loads(r_from.hget(method_key, test_method)) if not is_empty(tool, method_r): key = ':'.join([tool, project, str(version), test_method]) DIR = '/scratch/darioush/files' p = LocalPath(DIR) / '{key}.tar.gz'.format(key=key) assert p.exists() is_method_nonempty.append(True) with open(str(p)) as f: s3_up = put_into_s3('cvg-files', [tool, project, version, 'dev'], test_method, f) else: is_method_nonempty.append(False) except TypeError: print "--> Missing result {tool}:{project}:{version}:dev:{test}".format(tool=tool,test=test_method, project=project, version=version) except AssertionError: print "--> Missing file {tool}:{project}:{version}:dev:{test}".format(tool=tool,test=test_method, project=project, version=version) if any(is_method_nonempty): mut_key = mk_key('test-methods-run-cvg', ['major', project, version]) try: mut_r = json.loads(r_from.hget(mut_key, test_method)) if not is_empty('major', mut_r): key = ':'.join(['major', project, str(version), test_method]) DIR = '/scratch/darioush/files' p = LocalPath(DIR) / '{key}.tar.gz'.format(key=key) assert p.exists() with open(str(p)) as f: s3_up = put_into_s3('cvg-files', ['major', project, version, 'dev'], test_method, f) except TypeError: tool = 'major' print "--> Missing result {tool}:{project}:{version}:dev:{test}".format(tool=tool,test=test_method, project=project, version=version) except AssertionError: print "--> Missing file {tool}:{project}:{version}:dev:{test}".format(tool=tool,test=test_method, project=project, version=version)
def main(options): r = StrictRedis.from_url(REDIS_URL_OUT) for project, v in iter_versions(restrict_project=options.restrict_project, restrict_version=options.restrict_version): reasons = [] for qm in ['line', 'mutant', 'mutant-line']: key = mk_key('out', [qm, 'file', '0', 'B.F', project, v]) + ':info' info = json.loads(r.get(key)) reasons.append(info[1]) print '{project}:{v}'.format(**locals()), ' '.join(reasons)
def mk_index(): s, d = StrictRedis.from_url(SRC), StrictRedis.from_url(DST) for suite in ['dev' ] + ['evosuite-branch.{0}'.format(i) for i in xrange(0, 10)]: for project, version in iter_versions(): key = ':'.join( [PRE_SRC, 'test-methods', project, str(version), suite]) print key tm_list = s.lrange(key, 0, -1) idxes = tn_i_s(d, tm_list, suite) assert (all(type(idx) is type(0) for idx in idxes))
def mk_fetch_result(): s, d = StrictRedis.from_url(SRC), StrictRedis.from_url(DST) for suite in ['evosuite-branch.{0}'.format(i) for i in xrange(0, 10) ] + ['randoop.{0}'.format(i + 1) for i in xrange(0, 10)]: for project, version in iter_versions(): key = ':'.join( [PRE_SRC, 'fetch-result', project, str(version), suite]) dst_key = ':'.join(['fetch', project, str(version)]) res = s.get(key) if res is not None: d.hset(dst_key, suite, res)
def migrate_test_classes(r_from, r_to, projects=[], versions=[]): for project, version in iter_versions(projects, versions): print project, version tm_key = 'tms:{project}:{version}:dev'.format(project=project, version=version) tm_is = r_to.lrange(tm_key, 0, -1) tms = i_tn_s(r_to, tm_is, 'dev') unique_set = set([]) def is_unique(tt): retval = tt not in unique_set unique_set.add(tt) return retval tc_tns = [tc for tc, _, _ in [tm.partition('::') for tm in tms] if is_unique(tc)] tc_tis = tn_i_s(r_to, tc_tns, 'dev', allow_create=False) # for tool in ['codecover', 'jmockit']: # class_key = 'results:test-classes-cvg:{tool}:{project}:{version}'.format( # tool=tool,project=project,version=version # ) # class_cvgs = r_from.hmget(class_key, *tc_tns) # assert all(lambda x: x is not None for x in class_cvgs) # assert len(tc_tis) == len(class_cvgs) # to_key_class = 'exec:{tool}:{project}:{version}:dev'.format( # tool=tool, project=project, version=version # ) # r_to.hmset(to_key_class, {ck: cv for (ck, cv) in zip(tc_tis, class_cvgs)}) for tool in ['cobertura', 'codecover', 'jmockit', 'major']: method_key = 'results:test-methods-run-cvg:{tool}:{project}:{version}'.format( tool=tool, project=project, version=version ) res_dict = r_from.hgetall(method_key) assert(type(res_dict) == dict) res_list = res_dict.items() res_idxs = tn_i_s(r_to, [k for (k, v) in res_list], 'dev') res_vals = [v for (_, v) in res_list] assert len(res_vals) == len(res_idxs) res_map = {ki: v for (ki, v) in zip(res_idxs, res_vals)} to_key = 'exec:{tool}:{project}:{version}:dev'.format( tool=tool, project=project, version=version ) if res_map: r_to.hmset(to_key, res_map)
def mk_tms(): s, d = StrictRedis.from_url(SRC), StrictRedis.from_url(DST) for suite in ['dev'] + [ 'evosuite-branch.{0}'.format(i) for i in xrange(0, 10) ] + ['randoop.{0}'.format(i + 1) for i in xrange(0, 10)]: for project, version in iter_versions(): key = ':'.join( [PRE_SRC, 'test-methods', project, str(version), suite]) print key tm_list = s.lrange(key, 0, -1) idxes = tn_i_s(d, tm_list, suite) dst_key = ':'.join(['tms', project, str(version), suite]) assert (len(idxes) == len(tm_list)) for chunk in chunks(idxes, 100): if len(chunk) == 0: continue d.rpush(dst_key, *chunk)
def main(r): for p, v in iter_versions(): for tool in ['cobertura', 'codecover', 'jmockit', 'major']: m = r.hgetall('exec:{t}:{p}:{v}:dev'.format(p=p, v=v, t=tool)) keys = m.keys() tn_dict = { ti: tn for (ti, tn) in zip(keys, i_tn_s(r, keys, 'dev')) } mm = { ti: 1 for ti, result in m.iteritems() if (not is_empty(tool, json.loads(result))) and tn_dict[ti].find('::') != -1 } r.delete('nonempty:{t}:{p}:{v}:dev'.format(p=p, v=v, t=tool), mm) print p, v, tool, len(mm) if mm: r.hmset('nonempty:{t}:{p}:{v}:dev'.format(p=p, v=v, t=tool), mm)
def main(): parser = OptionParser() parser.add_option("-p", "--project", dest="restrict_project", action="append") parser.add_option("-v", "--version", dest="restrict_version", action="append") (options, args) = parser.parse_args(sys.argv) java = local['java']['-cp', JAR_PATH, 'edu.washington.cs.tgs.MapBuilder'] for p, v in iter_versions(options.restrict_project, options.restrict_version): print p, v src_dir, f_list = get_modified_sources(p, v) work_dir_path = '/tmp/work.{pid}'.format(pid=os.getpid()) with refresh_dir(work_dir_path, cleanup=True): with checkout(p, v, work_dir_path): with local.cwd(src_dir): (java > '/tmp/results/{p}:{v}'.format(p=p, v=v))(*f_list)
def enqueue_bundles(fun_dotted, json_str, queue_name='default', timeout=1800, print_only=False, restrict_project=None, restrict_version=None, tail_keys=[], tail_key_descr=None, at_front=False, check_key=None, **kwargs): q = Queue(queue_name, connection=StrictRedis.from_url(REDIS_URL_RQ)) r = StrictRedis.from_url(get_property('redis_url')) for project, i in iter_versions(restrict_project, restrict_version): input = {'project': project, 'version': i} additionals = json.loads(json_str) input.update(additionals) input.update({'timeout': timeout}) if tail_keys == []: tail_keys_to_iterate = [ [] ] # run the forloop once, but don't add any tail_key else: tail_keys_to_iterate = [ [tk] for tk in tail_keys ] # each of the tk's now counts, but singly for tail_key in tail_keys_to_iterate: if check_key: if r.hget(mk_key(check_key, [project, i]), ':'.join(tail_key)) is not None: continue input.update({} if tail_key_descr is None else {tail_key_descr: ':'.join(tail_key)}) doQ(q, fun_dotted, json.dumps(input), timeout, print_only, at_front)
def main(options): ts, ts_ri = connect_db() r = StrictRedis.from_url(REDIS_URL_OUT) for qm in options.qms: for granularity in options.grans: for experiment in options.experiments: bases, _, pools = experiment.partition(',') for project, v in iter_versions( restrict_project=options.restrict_project, restrict_version=options.restrict_version): bases_srtd = '.'.join(sorted(bases.split('.'))) pools_srtd = '.'.join(sorted(pools.split('.'))) fmt = '{qm}:{granularity}:{bases_srtd}:{pools_srtd}:{project}:{v}'.format( **locals()) if options.skip: print '{fmt}...'.format(fmt=fmt) exists = sel( ts_ri, { 'qm': qm, 'granularity': granularity, 'project': project, 'version': v, 'base': bases_srtd, 'select_from': pools_srtd }) if exists > 0: print "SKIP" continue session = Session() runs = { k: json.loads(v) for (k, v) in r.hgetall('out:{fmt}'.format( fmt=fmt)).iteritems() } keys = sorted([(alg, int(x)) for ( alg, x) in [tuple(key.split(':')) for key in runs.keys()]]) rows = [] fixed_info = { 'qm': qm, 'granularity': granularity, 'project': project, 'version': v, 'base': bases_srtd, 'select_from': pools_srtd, } suite_schema = lambda x: [ '{0}_{1}'.format(x, i) for i in ('triggers', 'count', 'tgs', 'time') ] schema = [ 'relevant_tgs', 'determined_by',] + \ suite_schema('base') + suite_schema('base_relevant') + suite_schema('aug') + suite_schema('aug_relevant') \ + suite_schema('aug_additional') ri_val = dict( zip( schema, json.loads(r.get( 'out:{fmt}:info'.format(fmt=fmt))))) for algo, it in groupby(keys, key=lambda (a, b): a): for algo, run_id in it: uniq_key = '{fmt}:{algo}:{run_id}'.format( **locals()) run_result = runs['{0}:{1}'.format(algo, run_id)] row_info = { 'algorithm': algo, 'run_id': run_id, 'fault_detection': run_result[0], 'determined_by': run_result[1], 'selected_triggers': run_result[2], 'selected_count': run_result[3], 'selected_tgs': run_result[4], 'selected_time': run_result[5], } row_info.update(fixed_info) rows.append(row_info) save_rows(ts, rows) ri_val.update(fixed_info) save_rows(ts_ri, [ri_val]) print '{0}: {1} records'.format(fmt, len(rows)) session.commit()
def enqueue_bundles_sliced(fun_dotted, json_str, bundle_key, source_key, tail_keys=[], tail_key_descr=None, queue_name='default', timeout=1800, print_only=False, restrict_project=None, restrict_version=None, bundle_size=10, bundle_offset=0, bundle_max=None, alternates=None, alternate_key=None, check_key=None, filter_function=None, filter_arg=None, map_function=None, at_front=False, **kwargs): if bundle_key is None: raise Exception("bundle key not provided [-k]") if tail_keys == []: tail_keys_to_iterate = [ [] ] # run the forloop once, but don't add any tail_key else: tail_keys_to_iterate = [[tk] for tk in tail_keys ] # each of the tk's now counts, but singly q = Queue(queue_name, connection=StrictRedis.from_url(REDIS_URL_RQ)) r = StrictRedis.from_url(get_property('redis_url')) rr = StrictRedis.from_url(REDIS_URL_TG) R = defaultdict(lambda: r) R['tgs'] = rr key_type = None # such hack if source_key.startswith('file:'): key_type = 'file' _, _, fn = source_key.partition(':') source_key = 'file' file_data = defaultdict(list) with open(fn) as f: for line in f: line_data = json.loads(line) file_data.update(line_data) for tail_key in tail_keys_to_iterate: for project, i in iter_versions(restrict_project, restrict_version): key = mk_key(source_key, [project, i] + tail_key) if key_type != 'file': key_type = r.type(key) if key_type == 'list': size = r.llen(key) elif key_type == 'hash': size = r.hlen(key) elif key_type == 'file': size = len(file_data[key]) elif key_type == 'none': size = 0 else: raise Exception('-- Unexpected key type: {0}'.format(key_type)) if bundle_max is not None: size = min(size, bundle_max) mf = (lambda _1, x, _2: x ) if map_function is None else get_fun(map_function) already_computed = {} if alternate_key and check_key: for alternate in alternates: _key = mk_key(check_key, [alternate, project, i] + tail_key) already_computed[alternate] = set( mf(r, R[check_key].hkeys(_key), tail_key)) if check_key and not alternate_key: _key = mk_key(check_key, [project, i] + tail_key) already_computed = set( mf(r, R[check_key].hkeys(_key), tail_key)) else: already_computed = set() if key_type == 'hash': all_items = r.hkeys(key) elif key_type == 'file': all_items = file_data[key] elif key_type == 'list': all_items = r.lrange(key, 0, -1) elif key_type == 'none': #print key all_items = [] if filter_function is not None: ff = get_fun(filter_function) all_items = ff(r, project, i, tail_key, filter_arg, all_items) all_items = mf(r, all_items, tail_key) def bundle_it(l, more_dict={}): if len(l) == 0: return for j in xrange(bundle_offset, size, bundle_size): if key_type in ('hash', 'file', 'list'): bundle = l[j:j + bundle_size] elif key_type == 'none': bundle = [] if len(bundle) == 0: continue input = { 'project': project, 'version': i, bundle_key: bundle } tk_input = {} if tail_key_descr is None else { tail_key_descr: ':'.join(tail_key) } additionals = json.loads(json_str) input.update(tk_input) input.update(more_dict) input.update(additionals) input.update({'timeout': timeout}) doQ(q, fun_dotted, json.dumps(input), timeout, print_only, at_front) if alternate_key: for alternate in alternates: if check_key: all_items = [ item for item in all_items if item not in already_computed[alternate] ] bundle_it(all_items, {alternate_key: alternate}) else: if check_key: all_items = [ item for item in all_items if item not in already_computed ] bundle_it(all_items)
def main(): r = StrictRedis.from_url(get_property_defaults('redis_url')) parser = OptionParser() parser.add_option("-p", "--project", dest="restrict_project", action="append") parser.add_option("-v", "--version", dest="restrict_version", action="append") parser.add_option("-T", "--tool", dest="tools", action="append", default=[]) parser.add_option("-s", "--suite", dest="suite", action="append", default=[]) parser.add_option("-S", "--suite-ids", dest="suite_ids", action="store", default=None) parser.add_option("-q", "--queue", dest="queue_name", action="store", type="string", default="default") parser.add_option("-t", "--timeout", dest="timeout", action="store", type="int", default=1800) parser.add_option("-b", "--bundle-size", dest="bundle_size", action="store", type="int", default=10) parser.add_option("-c", "--commit", dest="print_only", action="store_false", default=True) (options, args) = parser.parse_args(sys.argv) if options.suite_ids is None: suites = options.suite else: rmin, _, rmax = options.suite_ids.partition('-') rmin, rmax = int(rmin), int(rmax) suites = [ '{name}.{id}'.format(name=suite, id=id) for id in xrange(rmin, rmax + 1) for suite in options.suite ] if options.tools == []: tools = [[]] ## ?? else: tools = [[tool] for tool in options.tools] if suites == []: suites = ['dev'] total_tms = 0 stragglers = False for suite in suites: suite_tms = 0 for project, v in iter_versions(options.restrict_project, options.restrict_version): v_idxs, v_tns = check_test_list(r, project, v, suite) suite_tms += len(v_idxs) total_tms += len(v_idxs) if len(v_idxs) == 0: print ".. passing" continue for tool_ in tools: tool = ''.join(tool_) print "- {tool}:{project}:{v}:{suite}".format(tool=tool, project=project, v=v, suite=suite) wasOK = False with run_with_fixes(): fails = get_fails(r, tool, project, v, suite, v_idxs) passing = [(v_idx, v_tn) for (v_idx, v_tn) in zip(v_idxs, v_tns) if v_idx not in fails] print len(passing), len(fails) if len(passing) > 0: p_idxs, p_tns = zip(*passing) cvg_info = check_cvg(r, tool, project, v, suite, p_idxs, p_tns) print cvg_info else: print "No passing tests." wasOK = True stragglers |= (not wasOK) print 'suite {suite} tms: {tms}'.format(suite=suite, tms=suite_tms) print 'total tms', total_tms if not stragglers: print Fore.GREEN + "NO STRAGGLERS"