def main(options):
r = StrictRedis.from_url(REDIS_URL_OUT)
for project, v in iter_versions(restrict_project=options.restrict_project,
restrict_version=options.restrict_version):
reasons = []
for qm in ['line', 'mutant', 'mutant-line']:
key = mk_key('out', [qm, 'file', '0', 'B.F', project, v]) + ':info'
info = json.loads(r.get(key))
reasons.append(info[1])
print '{project}:{v}'.format(**locals()), ' '.join(reasons)
def covers(r, tool, project, v, suite, idxes):
lookup_key = mk_key('exec', [tool, project, v, suite])
lookup_results = [] if len(idxes) == 0 else r.hmget(lookup_key, *idxes)
assert all(x is not None for x in lookup_results)
assert len(lookup_results) == len(idxes)
covers = map(int, [
idx for (idx, lookup_result) in zip(idxes, lookup_results)
if not is_empty(tool, json.loads(lookup_result))
])
return covers
def get_tg_map(tests, idx_tp, relevant_tg_set):
get_suite = lambda t: idx_tp[t][0]
get_tid = lambda t: idx_tp[t][1]
tests_sorted = sorted(tests, key=get_suite) # get suite
tg_map = {}
for suite, suite_tests_iter in groupby(tests_sorted, key=get_suite):
suite_tests = list(suite_tests_iter)
vals = rr.hmget(mk_key('tgs', [project, version, suite]), map(get_tid, suite_tests))
vals_mapped = [set(msgpack.unpackb(v)) & relevant_tg_set for v in vals]
tg_map.update(dict(zip(suite_tests, vals_mapped)))
return {k: v for k, v in tg_map.iteritems() if len(v) > 0}
def get_fails(r, tool, project, v, suite, idxes, raise_fail_but_not_exec=True):
check_int(idxes)
[lookup_key_tool, lookup_key_exec
] = [mk_key('fail', [T, project, v, suite]) for T in (tool, 'exec')]
[failed_tool, failed_exec] = [
set(map(int, list(r.smembers(K))))
for K in (lookup_key_tool, lookup_key_exec)
]
failed_idxes = [idx for idx in idxes if idx in failed_tool]
failed_but_not_exec = [
idx for idx in failed_idxes if idx not in failed_exec
]
if failed_but_not_exec:
pass_key = mk_key('passcnt', [project, v, suite])
passes = defaultdict(
int,
zip(failed_but_not_exec, [
0 if x is None else int(x)
for x in r.hmget(pass_key, *failed_but_not_exec)
]))
failed_but_known_to_pass = [
idx for idx in failed_but_not_exec if passes[idx] > 0
]
if failed_but_known_to_pass:
raise Straggler('FAIL_BUT_KNOWN_TO_PASS',
[tool, project, v, suite],
failed_but_known_to_pass,
fix=('cvgmeasure.cvg.do_cvg',
lambda bundle: bundle[1:],
lambda bundle: b_pvs(bundle) +
" -K cvg_tool -a {tool}".format(tool=tool)))
if raise_fail_but_not_exec:
raise Straggler('FAIL_BUT_NOT_EXEC', [tool, project, v, suite],
failed_but_not_exec,
fix=('cvgmeasure.cvg.run_tests',
lambda bundle: bundle[1:], b_pvs))
return failed_idxes
def map_tgs(r, rr, work_dir, input):
project = input['project']
version = input['version']
redo = input.get('redo', False)
with check_key(
r,
'tgs-mapped',
[project, version],
redo=redo,
other_keys=[],
) as done:
map_file_name = '{project}:{version}'.format(project=project,
version=version)
get_file_from_cache_or_s3('darioush-map-files', map_file_name,
str(work_dir / 'map.txt'))
for tool in ['cobertura', 'codecover', 'jmockit', 'major']:
# - get a test name
for suite_type in [
'dev', 'randoop.1', 'evosuite-strongmutation-fse.5'
]:
test_list = r.hkeys(
mk_key('nonempty',
[tool, project, version, suite_type]))[:1]
if tool != 'major':
assert len(test_list) == 1
if len(test_list) == 1:
break
if test_list:
[
test_name,
] = i_tn_s(r, test_list, suite_type)
print tool, test_name
# - prep the tmp dir
get_files(work_dir, tool, project, version, suite_type,
test_name)
# - invoke java
if tool == 'major' and len(test_list) == 0:
tgs = [tg for tg in ALL_TGS if not tg.endswith('major')]
else:
tgs = ALL_TGS
result = jar()[work_dir](*tgs)
result = result.rstrip()
tgs = [s.partition(' ')[2] for s in result.split('\n')]
tg_i_s(rr, tgs, project, version, allow_create=True)
done(len(tgs))
return "Success (tgs={tgs})".format(tgs=len(tgs))
def check_test_list(r, project, v, suite):
key = mk_key('tms', [project, v, suite])
test_list = r.llen(key)
if test_list == 0 and suite != 'dev':
## empty test list should correspond to
## a bad fetch result
fetch_result_key = mk_key('fetch', [project, v])
fetch_result = r.hget(fetch_result_key, suite)
if fetch_result not in ['missing', 'empty']:
print suite, fetch_result
raise Straggler('-- {project}:{v}:{suite}'.format(project=project,
v=v,
suite=suite))
else:
print '-- {project}:{v}:{suite} has fetch result {fetch}'.format(
project=project, v=v, suite=suite, fetch=fetch_result)
return [], []
print '{key}: {tl}'.format(key=key, tl=test_list)
test_list = r.lrange(key, 0, -1)
test_names = i_tn_s(r, test_list, suite)
assert len(test_list) == len(test_names)
return map(int, test_list), test_names
def main():
r = StrictRedis.from_url(get_property('redis_url'))
parser = OptionParser()
parser.add_option("-p",
"--project",
dest="restrict_project",
action="append")
parser.add_option("-v",
"--version",
dest="restrict_version",
action="append")
parser.add_option("-x",
"--commit",
dest="action",
action="store_true",
default=False)
parser.add_option("-q",
"--queue",
dest="queue",
action="store",
default='default')
(options, args) = parser.parse_args(sys.argv)
TOOLS = [
'cobertura',
] #'codecover', 'jmockit', 'major']
SUITES = ['evosuite-branch-fse.{i}'.format(i=i) for i in xrange(8, 9)]
#['dev']
# ['randoop.{i}'.format(i=i) for i in xrange(1,11)] + \
# ['evosuite-branch.{i}'.format(i=i) for i in xrange(0,10)]
for suite in SUITES:
for project, v in iter_versions(options.restrict_project,
options.restrict_version):
for tool in TOOLS:
result = r.hget(mk_key('fetch', [project, v]), suite)
if result == 'ok' or suite == 'dev':
single_enqueue('cvgmeasure.cvg.compile_cache',
json.dumps({
"project": project,
"version": v,
"suite": suite,
"cvg_tool": tool,
}),
print_only=not options.action,
timeout=1800,
queue_name=options.queue)
def register_suite(suite, ff, tp_idx=tp_idx, idx_tp=idx_tp):
tests = map(int, rr.hkeys(mk_key('tgs', [project, version, suite])))
suite_triggers = get_triggers_from_results(r, project, version, suite)
fails = get_fails(suite)
# always remove failing tests
tests_no_fail = filter(lambda x: x not in fails, tests)
tests_filtered = filter(lambda x: ff(x, suite_triggers), tests_no_fail)
triggers = filter(lambda x: x in suite_triggers, tests_filtered)
print 'Suite: {0}, Tests: {1}, Suite Triggers:{2}, Selected Triggers: {3}, Fails: {4}, No fails: {5}, Filtered: {6}'.format(
suite, *map(len, [tests, suite_triggers, triggers, fails, tests_no_fail, tests_filtered]))
for test in tests_filtered:
tp_idx[(suite, test)] = tp_idx.get((suite, test), len(idx_tp))
if len(idx_tp) != len(tp_idx):
idx_tp.append((suite, test))
triggers = set(tp_idx[(suite, t)] for t in triggers)
return triggers, [tp_idx[(suite, test)] for test in tests_filtered]
def non_empty_agree_includes_tt_methods(input, hostname, pid):
project = input['project']
version = input['version']
work_dir, d4j_path, redis_url = map(
lambda property: get_property(property, hostname, pid),
['work_dir', 'd4j_path', 'redis_url']
)
r = StrictRedis.from_url(redis_url)
key = mk_key('test-methods-agree-cvg-nonempty', [project, version])
test_methods = set(r.lrange(key, 0, -1))
tts = get_tts(project, version)
print tts
missing_tms = [tm for tm in tts if tm not in test_methods]
if len(missing_tms) > 0:
raise MissingTT(' '.join(missing_tms))
return "Success"
def minimization(r, rr, qm, project, version):
source_list_key = source_list_keys[qm]
all_tests = r.lrange(mk_key(source_list_key, [project, version]), 0, -1)
tts = get_tts(project, version)
missing_tts = tts - set(all_tests)
if len(missing_tts) > 0:
raise MissingTTs(' '.join(missing_tts))
print "Total # of tts: ", len(tts), " of ", len(all_tests)
tts_with_unique_goals = get_unique_goal_tts(rr, qm, project, version, tts, all_tests)
print "Guaranteed: ", len(tts_with_unique_goals)
if len(tts_with_unique_goals) == 0:
print "Reading..."
tgs_map = {test: rr.smembers(mk_data_key('tgs', [qm, project, version, test]))
for test in all_tests}
greedy_minimization("G ", all_tests, tts, tgs_map)
greedy_minimization("GE ", all_tests, tts, tgs_map, essentials=get_essential_tests)
greedy_minimization("GRE", all_tests, tts, tgs_map, redundants=get_redundant_tests, essentials=get_essential_tests)
def enqueue_bundles(fun_dotted,
json_str,
queue_name='default',
timeout=1800,
print_only=False,
restrict_project=None,
restrict_version=None,
tail_keys=[],
tail_key_descr=None,
at_front=False,
check_key=None,
**kwargs):
q = Queue(queue_name, connection=StrictRedis.from_url(REDIS_URL_RQ))
r = StrictRedis.from_url(get_property('redis_url'))
for project, i in iter_versions(restrict_project, restrict_version):
input = {'project': project, 'version': i}
additionals = json.loads(json_str)
input.update(additionals)
input.update({'timeout': timeout})
if tail_keys == []:
tail_keys_to_iterate = [
[]
] # run the forloop once, but don't add any tail_key
else:
tail_keys_to_iterate = [
[tk] for tk in tail_keys
] # each of the tk's now counts, but singly
for tail_key in tail_keys_to_iterate:
if check_key:
if r.hget(mk_key(check_key, [project, i]),
':'.join(tail_key)) is not None:
continue
input.update({} if tail_key_descr is None else
{tail_key_descr: ':'.join(tail_key)})
doQ(q, fun_dotted, json.dumps(input), timeout, print_only,
at_front)
#! /usr/bin/env python
import redis
import re
import json
from rq import get_failed_queue
from cvgmeasure.conf import REDIS_URL_RQ, get_property_defaults
from cvgmeasure.common import connect_to_redis, mk_key
r = redis.StrictRedis.from_url(REDIS_URL_RQ)
fq = get_failed_queue(connection=r)
with connect_to_redis(get_property_defaults('redis_url')) as r2:
for job in fq.get_jobs():
exc_info = '' if job.exc_info is None else job.exc_info
reason = exc_info.split('\n')[-2:-1]
for r in reason:
if re.match(
r"""NameError: global name 'bucket_name' is not defined""",
r):
input = json.loads(job.args[0])
project, version, cvg_tool, suite = map(
lambda x: input[x],
'project version cvg_tool suite'.split(' '))
key = mk_key('fetch-result', [project, version, suite])
if r2.get(key) in ['empty', 'missing']:
print job.id
if project == 'Time' and version in [28, 29, 30, 31, 32]:
print job.id
def non_empty_match(input, hostname, pid):
project = input['project']
version = input['version']
input_key = input['key']
key_all = input['key_all']
should_fail_job = input.get('should_fail_job', True)
work_dir, d4j_path, redis_url = map(
lambda property: get_property(property, hostname, pid),
['work_dir', 'd4j_path', 'redis_url']
)
r = StrictRedis.from_url(redis_url)
key = mk_key(key_all, [project, version])
test_classes = r.llen(key) #r.lrange(key, 0, -1)
cobertura, codecover, jmockit = [r.hkeys(mk_key(input_key, [tool, project, version]))
for tool in ['cobertura', 'codecover', 'jmockit']]
exclude_static_fields_from = [
('Closure', 117), ('Closure', 100), ('Closure', 43), ('Closure', 47), ('Closure', 37),
('Math', 3), ('Math', 63), ('Lang', 6),
('Lang', 28), ('Lang', 17), ('Lang', 19)]
exclude_static_fields = [t for t in cobertura if t not in codecover and t in jmockit and \
(project, version) in exclude_static_fields_from and plausable_static_field(project, version, t)]
codecover_exception_from = [('Chart', 1), ('Lang', 64)]
codecover_exception = [t for t in codecover if t not in jmockit and t not in cobertura and \
(project, version) in codecover_exception_from and plausable_codecover_field(project, version, t)]
cobertura = [t for t in cobertura if t not in exclude_static_fields]
jmockit = [t for t in jmockit if t not in exclude_static_fields]
codecover = [t for t in codecover if t not in codecover_exception]
core = set(cobertura) & set(codecover) & set(jmockit)
cobertura_, codecover_, jmockit_ = [[t for t in l if t not in core] for l in (cobertura, codecover, jmockit)]
print test_classes, '/', len(core), "Agreement"
print len(exclude_static_fields), " Excluded from jmockit and cobertura as static field initializers"
print len(codecover_exception), " Excluded from codecover as static field initializers"
print "---"
print len(cobertura_), sorted(cobertura_)
print len(codecover_), sorted(codecover_)
print len(jmockit_), sorted(jmockit_)
fails = []
fails.extend(with_fails(lambda: check_sub(cobertura, 'cobertura', codecover, 'codecover')))
fails.extend(with_fails(lambda: check_sub(codecover, 'codecover', cobertura, 'cobertura')))
fails.extend(with_fails(lambda: check_sub(cobertura, 'cobertura', jmockit, 'jmockit')))
fails.extend(with_fails(lambda: check_sub(jmockit, 'jmockit', cobertura, 'cobertura')))
fails.extend(with_fails(lambda: check_sub(codecover, 'codecover', jmockit, 'jmockit')))
fails.extend(with_fails(lambda: check_sub(jmockit, 'jmockit', codecover, 'codecover')))
if fails and should_fail_job:
raise SubMismatch(' AND '.join([str(ex) for ex in fails]))
return "Success"
def method_list_matches(input, hostname, pid):
project = input['project']
version = input['version']
work_dir, d4j_path, redis_url = map(
lambda property: get_property(property, hostname, pid),
['work_dir', 'd4j_path', 'redis_url']
)
r = StrictRedis.from_url(redis_url)
key = mk_key('test-methods', [project, version])
test_methods_from_redis = r.lrange(key, 0, -1)
work_dir_path = local.path(work_dir) / ('child.%d' % os.getpid())
print work_dir_path
with refresh_dir(work_dir_path, cleanup=True):
with add_to_path(d4j_path):
with checkout(project, version, work_dir_path / 'checkout'):
d4()('compile')
test_methods_from_d4 = d4()('list-tests').rstrip().split('\n')
with local.env(SUCCESS_OUT="passing-tests.txt"):
failing_tests = test()
with open("passing-tests.txt") as f:
test_methods_from_run = [x[len('--- '):] for x in f.read().rstrip().split('\n')]
with open("count-of-tests.txt") as f:
per_run_counts = [int(line.rstrip()) for line in f]
count_of_tests_from_run = sum(per_run_counts)
if project == 'Lang' and version >= 37:
## In this case, we know that some tests may fail
## this is really ugly, but I'm doing it.
klass_name = 'org.apache.commons.%s.builder.ToStringBuilderTest' % (
'lang' if version > 39 else 'lang3',
)
expected_fails = [method for method in failing_tests if method.startswith(klass_name)]
single_run_fails = test(['-t', klass_name])
if len(single_run_fails) > 0:
raise TestFail('Single run failed: ' + ' '.join(single_run_fails))
elif project == 'Time':
## In this case, org.joda.time.chrono.gj.MainTest
## isn't really a jUnit test because it doesn't have a public
## constructor. We fix this during run by replacing it
## with two classes with a public constructor, each of which
## initializes the original class with parameters used during
## testing
bad_class = 'org.joda.time.chrono.gj.MainTest'
good_class1, good_class2 = ['edu.washington.cs.testfixer.time.GjMainTest' + s for s in ('1', '2')]
tname = '::testChronology'
tcs = [tc for tc, _, _ in [method.partition('::') for method in test_methods_from_run]]
idx = tcs.index(bad_class)
test_methods_from_run[idx] = good_class1 + tname
test_methods_from_run[idx+1] = good_class2 + tname
tcsd4 = [tc for tc, _, _ in [method.partition('::') for method in test_methods_from_d4]]
idxd4 = tcsd4.index(bad_class)
test_methods_from_d4 = test_methods_from_d4[:idxd4] + [good_class1 + tname,
good_class2 + tname] + test_methods_from_d4[idxd4+1:]
expected_fails = []
else:
expected_fails = []
unexpected_fails = [method for method in failing_tests if method not in expected_fails]
# Sanity check #0 -- check out the test fails
if len(unexpected_fails) > 0:
raise TestFail(' '.join(unexpected_fails))
# Sanity check #1 -- number of tests counted through the runner should equal
# the length of the list of passing tests the runner outputs
num_tests = len(test_methods_from_d4)
if num_tests != count_of_tests_from_run:
raise LenMismatch("Test methods from d4 (%d) don't match counter (%d)" %
(num_tests, count_of_tests_from_run))
# Sanity check #2 -- we should not be running duplicate tests
no_dups(test_methods_from_run, 'test methods from run')
# Sanity check #3 -- we should not be list-outputting duplicate tests
no_dups(test_methods_from_d4, 'test methods from d4')
# Sanity check #4 -- we should not have duplicate tests in redis store
no_dups(test_methods_from_redis, 'test methods from redis')
# Sanity check #5 -- tests output from the runner should match the tests output from
# d4 list-tests
check_eq(test_methods_from_run, 'test methods from run', test_methods_from_d4, 'test methods from d4')
# Sanity check #6 -- test methods from d4 should match ones in redis
#
# Preprocess step: We know that these methods were wrongly inserted:
#lang_methods = [
# 'org.apache.commons.lang3.EnumUtilsTest::test_processBitVectors_longClass',
# 'org.apache.commons.lang3.builder.ReflectionToStringBuilderConcurrencyTest::testLinkedList',
# 'org.apache.commons.lang3.builder.ReflectionToStringBuilderConcurrencyTest::testArrayList',
# 'org.apache.commons.lang3.builder.ReflectionToStringBuilderConcurrencyTest::testCopyOnWriteArrayList',
# 'org.apache.commons.lang3.builder.ReflectionToStringBuilderMutateInspectConcurrencyTest::testConcurrency',
#]
#lang_methods_in_redis = [method for method in lang_methods if method in test_methods_from_redis]
#for lang_method in lang_methods_in_redis:
# print "Removing %s from redis:" % lang_method
# print r.lrem(key, 1, lang_method)
#if lang_methods_in_redis:
# print "Redis store was modified, reloading list before testing"
# test_methods_from_redis = r.lrange(key, 0, -1)
check_eq(test_methods_from_redis, 'test methods from redis', test_methods_from_d4, 'test methods from d4')
return "Success"
def check_cvg(r, tool, project, v, suite, t_idxs, ts):
key = mk_key('exec', [tool, project, v, suite])
cvg_infos = r.hmget(key, *t_idxs)
assert len(cvg_infos) == len(t_idxs)
nils = [(t_idx, t) for (t_idx, t, cvg_info) in zip(t_idxs, ts, cvg_infos)
if cvg_info is None]
print len(nils), len(ts)
if suite == 'dev':
check_for_nil_classes = [
tc for tc, _, _ in [t.partition('::') for (t_idx, t) in nils]
]
check_for_nil_classes = list(set(check_for_nil_classes))
check_for_nil_classes_idxes = tn_i_s(r, check_for_nil_classes, 'dev')
non_nil_classes_idxes = set(
cobertura_covers(r, project, v, suite,
check_for_nil_classes_idxes))
nil_class_dict = {
class_name: idx not in non_nil_classes_idxes
for (
class_name,
idx) in zip(check_for_nil_classes, check_for_nil_classes_idxes)
}
nil_idxes = [(t_idx_, tc, tm)
for (t_idx_, (tc, _, tm)) in [(t_idx, t.partition('::'))
for (t_idx, t) in nils]
if nil_class_dict.get(tc) is False]
# really only need the idxes
nil_idxes = [t_idx for (t_idx, _, _) in nil_idxes]
else:
nil_idxes = [t_idx for (t_idx, _) in nils]
if tool == 'cobertura' and nil_idxes:
raise Straggler('COBERTURA_NOT_RUN', [tool, project, v, suite],
idxes=nil_idxes,
fix=('cvgmeasure.cvg.do_cvg',
lambda bundle: bundle[1:],
lambda bundle: b_pvs(bundle) +
" -K cvg_tool -a {tool}".format(tool=tool)))
cc = cobertura_covers(r, project, v, suite, nil_idxes)
if cc != []:
raise Straggler('CVG_NOT_RUN_COBERTURA_NONEMPTY',
[tool, project, v, suite],
idxes=cc,
fix=('cvgmeasure.cvg.do_cvg',
lambda bundle: bundle[1:],
lambda bundle: b_pvs(bundle) +
" -K cvg_tool -a {tool}".format(tool=tool)))
# time to check s3
non_nils = [
(t_idx, t) for (t_idx, t, cvg_info) in zip(t_idxs, ts, cvg_infos)
if cvg_info is not None and not is_empty(tool, json.loads(cvg_info))
]
print '- non-nil len: {0}'.format(len(non_nils))
if non_nils:
s3_list = list_from_s3('cvg-files', [tool, project, v, suite])
s3_tname_list = set(
[key.name.rpartition('/')[2] for key in s3_list if key.size > 0])
non_nils_missing_from_s3 = [(t_idx, t) for (t_idx, t) in non_nils
if t not in s3_tname_list]
if len(non_nils_missing_from_s3) != 0:
raise Straggler(
'NON_NIL_CVG_BUT_NO_S3', [tool, project, v, suite],
idxes=[t_idx for (t_idx, _) in non_nils_missing_from_s3],
fix=('cvgmeasure.cvg.do_cvg', lambda bundle: bundle[1:],
lambda bundle: b_pvs(bundle) +
" -K cvg_tool -a {tool} -j '{{\"redo\": true}}'".format(
tool=tool)))
return "Cvg for in s3 : {0}".format(len(non_nils))
def minimization(r, rr, conn, qm_name, granularity, project, version, bases, augs):
qm = QMS[qm_name]
# 0. pull granularity info from cloud
fn = '{project}:{version}.txt'.format(project=project, version=version)
get_file_from_cache_or_s3('darioush-mm-files', '{project}:{version}'.format(project=project,version=version), fn)
with open(fn) as f:
granularity_info = json.loads(f.read())
# 1. Get the testing goals that match the qm
all_tgs = rr.hkeys(mk_key('tg-i', [project, version]))
relevant_tgs = [tg for tg in filter(qm['fun'], all_tgs) if GRANS[granularity](tg, granularity_info)]
relevant_tg_is = tg_i_s(rr, relevant_tgs, project, version)
relevant_tg_set = set(relevant_tg_is)
tp_idx = {}
idx_tp = []
fails = {}
def get_fails(suite, fails=fails):
if not fails.has_key(suite):
fail_ids = set(map(int, r.smembers(mk_key('fail', ['exec', project, version, suite]))))
fails[suite] = fail_ids
return fails[suite]
def register_suite(suite, ff, tp_idx=tp_idx, idx_tp=idx_tp):
tests = map(int, rr.hkeys(mk_key('tgs', [project, version, suite])))
suite_triggers = get_triggers_from_results(r, project, version, suite)
fails = get_fails(suite)
# always remove failing tests
tests_no_fail = filter(lambda x: x not in fails, tests)
tests_filtered = filter(lambda x: ff(x, suite_triggers), tests_no_fail)
triggers = filter(lambda x: x in suite_triggers, tests_filtered)
print 'Suite: {0}, Tests: {1}, Suite Triggers:{2}, Selected Triggers: {3}, Fails: {4}, No fails: {5}, Filtered: {6}'.format(
suite, *map(len, [tests, suite_triggers, triggers, fails, tests_no_fail, tests_filtered]))
for test in tests_filtered:
tp_idx[(suite, test)] = tp_idx.get((suite, test), len(idx_tp))
if len(idx_tp) != len(tp_idx):
idx_tp.append((suite, test))
triggers = set(tp_idx[(suite, t)] for t in triggers)
return triggers, [tp_idx[(suite, test)] for test in tests_filtered]
# 2. Get the list of tests with some coverage goals from base / augmenting pools
def flatten(l):
return reduce(lambda a,b: a+b, l, [])
def pool_to_tests(pools, tp_idx=tp_idx, idx_tp=idx_tp):
tests = []
triggers = set([])
for pool in pools:
pool_suites, pool_filter = POOLS[pool]
for pool_suite in pool_suites:
a, b = register_suite(pool_suite, pool_filter, tp_idx=tp_idx, idx_tp=idx_tp)
tests.append(b)
triggers.update(a)
return triggers, flatten(tests)
base_tp_idx = {}
base_idx_tp = []
base_triggers, base_tests = pool_to_tests(bases, base_tp_idx, base_idx_tp)
aug_triggers, aug_tests = pool_to_tests(augs, tp_idx, idx_tp)
print "Triggers: Base {0}, Aug {1}".format(len(base_triggers), len(aug_triggers))
print "Reading..."
tRead = Timer()
tRead.start()
def get_tg_map(tests, idx_tp, relevant_tg_set):
get_suite = lambda t: idx_tp[t][0]
get_tid = lambda t: idx_tp[t][1]
tests_sorted = sorted(tests, key=get_suite) # get suite
tg_map = {}
for suite, suite_tests_iter in groupby(tests_sorted, key=get_suite):
suite_tests = list(suite_tests_iter)
vals = rr.hmget(mk_key('tgs', [project, version, suite]), map(get_tid, suite_tests))
vals_mapped = [set(msgpack.unpackb(v)) & relevant_tg_set for v in vals]
tg_map.update(dict(zip(suite_tests, vals_mapped)))
return {k: v for k, v in tg_map.iteritems() if len(v) > 0}
base_tg_map = get_tg_map(base_tests, base_idx_tp, relevant_tg_set)
get_all_tgs = lambda tg_map: reduce(lambda a, b: a | b, tg_map.values(), set([]))
get_all_tests = lambda tg_map: set(tg_map.keys())
base_tgs = get_all_tgs(base_tg_map)
print "Relevant_tgs {0}, Base tgs: {1}".format(*map(len, (relevant_tg_set, base_tgs)))
base_relevant_tests = get_all_tests(base_tg_map)
base_relevant_tts = base_triggers & base_relevant_tests
aug_relevant_tg_map = get_tg_map(aug_tests, idx_tp, relevant_tg_set)
aug_tgs = get_all_tgs(aug_relevant_tg_map)
aug_relevant_tests = get_all_tests(aug_relevant_tg_map)
aug_relevant_tts = aug_triggers & aug_relevant_tests
aug_additional_tg_map = get_tg_map(aug_tests, idx_tp, relevant_tg_set - base_tgs)
aug_additional_tgs = get_all_tgs(aug_relevant_tg_map)
aug_additional_tests = get_all_tests(aug_additional_tg_map)
aug_additional_tts = aug_triggers & set(aug_additional_tests)
print "Triggers: Base_relevant {0}, Aug_relevant {1}, Aug_additional {2}".format(*map(len,
(base_relevant_tts, aug_relevant_tts, aug_additional_tts)))
tRead.stop()
print "Reading complete {0} msecs.".format(tRead.msec)
tts_with_unique_goals = get_unique_goal_tts(aug_additional_tts, aug_additional_tests, aug_additional_tg_map)
print "Guaranteed: ", len(tts_with_unique_goals)
print "Building bit vectors"
all_tgs = { k: i for i, k in enumerate(sorted(get_all_tgs(aug_additional_tg_map)))}
def to_bitvector(s):
bv = bitarray(len(all_tgs))
bv.setall(False)
for tg in s:
bv[all_tgs[tg]] = 1
return bv
bit_map = {k : to_bitvector(v) for k, v in aug_additional_tg_map.iteritems()}
print "Built.."
def timing(tests, idx_tp=idx_tp):
tps = map(lambda t: idx_tp[t], tests)
tps_sorted = sorted(tps, key=lambda (suite, i): suite)
total_time = 0
for suite, i_it in groupby(tps_sorted, key=lambda(suite, i): suite):
i_s = map(lambda (suite, i): i, i_it)
if suite == 'dev':
tns = i_tn_s(r, i_s, suite)
tc_ns = set(tn.partition('::')[0] for tn in tns)
tc_is = tn_i_s(r, list(tc_ns), suite, allow_create=False)
all_is = tc_is + i_s
else:
all_is = i_s
method_times = [msgpack.unpackb(b) for b in r.hmget(mk_key('time', [project, version, suite]), all_is)]
bad_timings = [(time, i) for (time, i) in zip(method_times, all_is) if any(t < 0 for t in time)]
if bad_timings:
raise Exception('bad timing for tests: {project}, {version}, {suite} {idxs}'.format(
project=project, version=version, suite=suite, idxs=' '.join(map(str, [i for (time, i) in bad_timings]))))
def aggr(l):
if any(x == -1 for x in l):
raise Exception('bad timing for tests: {project}, {version}, {suite}'.format(
project=project, version=version, suite=suite))
# let's go with average
return reduce(lambda a, b: a+b, l)/len(l)
method_times_aggregate = [aggr(timings) for timings in method_times]
suite_time = reduce(lambda a, b: a+b, method_times_aggregate, 0)
total_time += suite_time
return total_time
base_time = timing(base_tests, base_idx_tp)
def reason(given):
if len(tts_with_unique_goals) > 0:
return 'U'
elif len(aug_additional_tts) == 0:
return 'X'
else:
return given
# row independent info
suite_schema = lambda x: ['{0} {1}'.format(x,i) for i in ('triggers', 'tests', 'tgs', 'time')]
schema = [ 'Relevant tgs', 'Reason',] + \
suite_schema('Base') + suite_schema('Base relevant') + suite_schema('Aug') + suite_schema('Aug relevant') \
+ suite_schema('Aug additional')
info = (
len(relevant_tgs), reason('-'),
len(base_triggers), len(base_tests), len(base_tgs), base_time, # all of base suite
len(base_relevant_tts), len(base_relevant_tests), len(base_tgs), timing(base_relevant_tests, base_idx_tp), # relevant part of base suite
len(aug_triggers), len(aug_tests), len(aug_tgs), timing(aug_tests), # all of augmentation pool
len(aug_relevant_tts), len(aug_relevant_tests), len(aug_tgs), timing(aug_relevant_tests),# relevant part of aug pool
len(aug_additional_tts), len(aug_additional_tests), len(aug_additional_tgs), timing(aug_additional_tests), # part of aug pool in addition to the base suite
)
print ', '.join('{0}: {1}'.format(a,b) for a,b in (zip(schema, info)))
print info
key = mk_key('out', [qm['name'], granularity, '.'.join(sorted(bases)), '.'.join(sorted(augs)), project, version])
save_row_independent(conn, key, info)
minimization = lambda **kwargs: get_do_one(aug_additional_tests, bit_map, **kwargs)
redundants, eq_partition = get_redundant_and_equal_tests(bit_map, aug_additional_tests)
#print redundants, eq_partition
choice = lambda eq_partition=eq_partition: reduce(lambda a, b: a | b, map(lambda s: s - set(random.sample(s, 1)), eq_partition), set())
algo_params = [
('G', {}),
('GE', {'essentials': get_essential_tests}),
('GRE', {'redundants': lambda redundants=redundants: redundants , 'essentials': get_essential_tests}),
('GREQ', {'redundants': lambda redundants=redundants: redundants | choice(), 'essentials': get_essential_tests}),
]
algos = [(name, lambda algo_param=algo_param: minimization(**algo_param)) for (name, algo_param) in algo_params]
for algo, fun in algos:
info = {'nofd': 0, 'cnt': 0, 'cvrtg': None, 'times':[], 'cnts':[], 'reasons': {'S': 0, 'R': 0, 'E': 0}}
label=lambda :'{algo:<4} {info[cvrtg]:>4}tgs {info[reasons][S]:>3}S {info[reasons][R]:>3}R {info[reasons][E]:>3}E {info[nofd]:>3}F {time:>6}ms'.format(info=info, algo=algo, time=''.join(map(str,info['times'][-1:])))
with click.progressbar(xrange(RUNS), label=label(), width=20) as bar:
def pr(results):
rr = process_results(results, aug_additional_tts, timing)
determined_by, sel_tts, sel_cnt, sel_tgs, time = rr
info['cnt'] += 1
if sel_tts == 0:
info['nofd'] += 1
info['reasons'][determined_by] += 1
info['cnts'].append(sel_cnt)
info['times'].append(time)
if info['cvrtg'] is None:
info['cvrtg'] = sel_tgs
else:
assert info['cvrtg'] == sel_tgs
bar.label=label()
return rr
results = greedy_minimization(bar, fun(), pr)
# vals should be a dict from {'GRE:1': (fault_detection, reason, triggers, count, tgs, time) }
save_rows(conn, key, {'{algo}:{run}'.format(algo=algo, run=run_id+1):
(
1 if selected_triggers > 0 else 0,
reason(determined_by),
selected_triggers, selected_count, selected_tgs, selected_time
)
for (run_id, (determined_by, selected_triggers, selected_count, selected_tgs, selected_time)) in enumerate(results)}
)
def get_triggers_from_results(r, project, version, suite):
result = json.loads(r.hget(mk_key('trigger', [project, version]), suite))
return set(result)
def get_fails(suite, fails=fails):
if not fails.has_key(suite):
fail_ids = set(map(int, r.smembers(mk_key('fail', ['exec', project, version, suite]))))
fails[suite] = fail_ids
return fails[suite]
def enqueue_bundles_sliced(fun_dotted,
json_str,
bundle_key,
source_key,
tail_keys=[],
tail_key_descr=None,
queue_name='default',
timeout=1800,
print_only=False,
restrict_project=None,
restrict_version=None,
bundle_size=10,
bundle_offset=0,
bundle_max=None,
alternates=None,
alternate_key=None,
check_key=None,
filter_function=None,
filter_arg=None,
map_function=None,
at_front=False,
**kwargs):
if bundle_key is None:
raise Exception("bundle key not provided [-k]")
if tail_keys == []:
tail_keys_to_iterate = [
[]
] # run the forloop once, but don't add any tail_key
else:
tail_keys_to_iterate = [[tk] for tk in tail_keys
] # each of the tk's now counts, but singly
q = Queue(queue_name, connection=StrictRedis.from_url(REDIS_URL_RQ))
r = StrictRedis.from_url(get_property('redis_url'))
rr = StrictRedis.from_url(REDIS_URL_TG)
R = defaultdict(lambda: r)
R['tgs'] = rr
key_type = None # such hack
if source_key.startswith('file:'):
key_type = 'file'
_, _, fn = source_key.partition(':')
source_key = 'file'
file_data = defaultdict(list)
with open(fn) as f:
for line in f:
line_data = json.loads(line)
file_data.update(line_data)
for tail_key in tail_keys_to_iterate:
for project, i in iter_versions(restrict_project, restrict_version):
key = mk_key(source_key, [project, i] + tail_key)
if key_type != 'file':
key_type = r.type(key)
if key_type == 'list':
size = r.llen(key)
elif key_type == 'hash':
size = r.hlen(key)
elif key_type == 'file':
size = len(file_data[key])
elif key_type == 'none':
size = 0
else:
raise Exception('-- Unexpected key type: {0}'.format(key_type))
if bundle_max is not None:
size = min(size, bundle_max)
mf = (lambda _1, x, _2: x
) if map_function is None else get_fun(map_function)
already_computed = {}
if alternate_key and check_key:
for alternate in alternates:
_key = mk_key(check_key,
[alternate, project, i] + tail_key)
already_computed[alternate] = set(
mf(r, R[check_key].hkeys(_key), tail_key))
if check_key and not alternate_key:
_key = mk_key(check_key, [project, i] + tail_key)
already_computed = set(
mf(r, R[check_key].hkeys(_key), tail_key))
else:
already_computed = set()
if key_type == 'hash':
all_items = r.hkeys(key)
elif key_type == 'file':
all_items = file_data[key]
elif key_type == 'list':
all_items = r.lrange(key, 0, -1)
elif key_type == 'none':
#print key
all_items = []
if filter_function is not None:
ff = get_fun(filter_function)
all_items = ff(r, project, i, tail_key, filter_arg, all_items)
all_items = mf(r, all_items, tail_key)
def bundle_it(l, more_dict={}):
if len(l) == 0:
return
for j in xrange(bundle_offset, size, bundle_size):
if key_type in ('hash', 'file', 'list'):
bundle = l[j:j + bundle_size]
elif key_type == 'none':
bundle = []
if len(bundle) == 0:
continue
input = {
'project': project,
'version': i,
bundle_key: bundle
}
tk_input = {} if tail_key_descr is None else {
tail_key_descr: ':'.join(tail_key)
}
additionals = json.loads(json_str)
input.update(tk_input)
input.update(more_dict)
input.update(additionals)
input.update({'timeout': timeout})
doQ(q, fun_dotted, json.dumps(input), timeout, print_only,
at_front)
if alternate_key:
for alternate in alternates:
if check_key:
all_items = [
item for item in all_items
if item not in already_computed[alternate]
]
bundle_it(all_items, {alternate_key: alternate})
else:
if check_key:
all_items = [
item for item in all_items
if item not in already_computed
]
bundle_it(all_items)