def get_file_from_cache_or_s3(bucket, fn, dst, cache=True):
hostname, _, _ = socket.gethostname().partition('.')
look_dirs = get_property('s3_cache', hostname)
if cache:
cache_dir = look_dirs[-1:]
else:
cache_dir = []
for d in look_dirs:
path = (LocalPath(d) / bucket / fn)
if path.exists():
mkdir_p(dst)
path.copy(dst)
break
else:
b = s3.lookup(bucket)
key = b.lookup(fn)
if key is None:
raise NoFileOnS3(
"Key missing from bucket {bucket}: {key_name}".format(
bucket=bucket, key_name=fn))
mkdir_p(dst)
with open(dst, 'w') as out_f:
out_f.write(key.read())
for d in cache_dir:
path = (LocalPath(d) / bucket / fn)
if not path.exists():
mkdir_p(path)
LocalPath(dst).copy(path)
def main():
r = StrictRedis.from_url(get_property('redis_url'))
parser = OptionParser()
parser.add_option("-p", "--project", dest="restrict_project", action="append")
parser.add_option("-v", "--version", dest="restrict_version", action="append")
parser.add_option("-x", "--commit", dest="action", action="store_true", default=False)
parser.add_option("-q", "--queue", dest="queue", action="store", default='default')
(options, args) = parser.parse_args(sys.argv)
#TOOLS = ['cobertura', 'codecover', 'jmockit', 'major']
SUITES = ['evosuite-strongmutation-fse.{i}'.format(i=i) for i in xrange(1,2)]
# ['randoop.{i}'.format(i=i) for i in xrange(1,11)] + \
# ['evosuite-branch.{i}'.format(i=i) for i in xrange(0,10)]
for suite in SUITES:
for project, v in iter_versions(options.restrict_project, options.restrict_version):
#for tool in TOOLS:
#result = r.get(mk_key('fetch', [project, v, suite]))
#if result == 'ok':
single_enqueue('cvgmeasure.cvg.test_lists_gen', json.dumps({
"project": project,
"version": v,
"suite": suite,
}), print_only=not options.action, timeout=1800, queue_name=options.queue)
def non_empty_includes_tt(input, hostname, pid):
project = input['project']
version = input['version']
work_dir, d4j_path, redis_url = map(
lambda property: get_property(property, hostname, pid),
['work_dir', 'd4j_path', 'redis_url']
)
r = StrictRedis.from_url(redis_url)
keys = [mk_key('test-classes-cvg-nonempty', [tool, project, version]) for tool in ('cobertura', 'codecover', 'jmockit')]
test_classes = [set(r.hkeys(key)) for key in keys]
print test_classes
test_classes_core = reduce(lambda a,b: a&b, test_classes)
tts = get_tts(project, version)
print tts
tcs = [tc for tc, _, _ in [tt.partition('::') for tt in tts]]
print tcs
print test_classes_core
missing_tcs = [tc for tc in tcs if tc not in test_classes_core]
if len(missing_tcs) > 0:
raise MissingTT(' '.join(missing_tcs))
return "Success"
def main(options):
r = redis.StrictRedis.from_url(get_property('redis_url'))
rr = redis.StrictRedis.from_url(REDIS_URL_TG)
rrr = redis.StrictRedis.from_url(REDIS_URL_OUT)
for qm in options.qms:
for gran in options.grans:
for experiment in options.experiments:
bases, _, pools = experiment.partition(',')
if options.print_only:
print '''./main.py qb cvgmeasure.select.m {project} {version} -j '{json}' {additional}'''.format(
project=''.join('-p {0}'.format(rp)
for rp in options.restrict_project),
version=''.join('-v {0}'.format(rv)
for rv in options.restrict_version),
json=json.dumps({
'granularity': gran,
'bases': bases,
'pools': pools,
"qm": qm
}),
additional=options.print_only)
else:
for project, v in iter_versions(
restrict_project=options.restrict_project,
restrict_version=options.restrict_version):
print "----( %s %d -- %s : %s)----" % (project, v, qm,
gran)
minimization(r, rr, rrr, qm, gran, project, v,
bases.split('.'), pools.split('.'))
print
def main():
r = redis.StrictRedis.from_url(get_property('redis_url'))
rr = redis.StrictRedis.from_url(REDIS_URL_TG)
qm = 'linecvg'
for project in ["Lang", "Chart", "Time"]:
for v in xrange(0, get_num_bugs(project)):
version = v + 1
print "----( %s %d -- %s )----" % (project, version, qm)
minimization(r, rr, qm, project, version)
print
def setup(machine):
rem = SshMachine(workers(machine)['hostname'], ssh_opts=SSH_OPTS)
dir = rem.path(workers(machine)['rqdir'])
if not dir.exists():
print "CLONING REPO..."
rem["git"]("clone", "http://github.com/darioush/rq-dist", dir)
print "CLONED..."
print "MAKING VIRTUAL ENV..."
with rem.cwd(dir):
rem["virtualenv"]("env")
print "MADE VIRTUAL ENV..."
with rem.cwd(dir):
print "UPDATING CODE ..."
rem["git"]("pull", "origin", "master")
print "UPDATING VENV ..."
rem["./update-venv.sh"]()
my_hostname, _, _ = socket.gethostname().partition('.')
if my_hostname == machine:
print "Not syncing master worker"
return
my_d4j = '/'.join(
get_property('d4j_path', my_hostname, 0)[0].split('/')[:-2])
dst_d4j = '/'.join(get_property('d4j_path', machine, 0)[0].split('/')[:-3])
print "RSYNCING FOR DEFECTS4J "
rsync = local['rsync']['-avz', '--exclude', '.git', '--exclude',
'project_repos'][my_d4j]
rsync('%s:%s' % (workers(machine)['hostname'], dst_d4j))
rem_d4j = rem.path(dst_d4j) / 'defects4j'
repos_dir = rem_d4j / 'project_repos'
if not repos_dir.exists():
with rem.cwd(rem_d4j):
print "GETTING REPOSITORIES..."
rem['./get-repos.sh']()
def setup_tgs(input, hostname, pid):
project = input['project']
version = input['version']
qm = input['qm']
tests = input['tests']
redo = input.get('redo', False)
verbose = input.get('verbose', False)
work_dir, d4j_path, redis_url = map(
lambda property: get_property(property, hostname, pid),
['work_dir', 'd4j_path', 'redis_url'])
work_dir_path = local.path(work_dir) / ('child.%d' % os.getpid())
print work_dir
directory, sources = get_modified_sources(project, version)
tools = ['cobertura', 'codecover', 'jmockit']
r = StrictRedis.from_url(redis_url)
rr = StrictRedis.from_url(REDIS_URL_TG)
d4j_location = '/'.join(which('defects4j').rstrip().split('/')[:-1])
with filter_key_list(
r,
key='qm-computed',
bundle=[qm, project, version],
list=tests,
redo=redo,
other_keys=[],
) as worklist:
files_i_will_want = [[tool, project, version, test] for tool in tools
for (test, _) in worklist]
prefetch(files_i_will_want)
for test, callback in worklist:
with refresh_dir(work_dir_path, cleanup=True):
print test
tgs = {
tool: get_tgs(d4j_location, tool, project, version, test)
for tool in tools
}
pp_tgs(rr, [qm, project, version],
test,
tgs,
tools,
verbose=verbose)
callback()
return "Success"
def main():
r = redis.StrictRedis.from_url(get_property('redis_url'))
sum = 0
cvr = 0
for p in PROJECTS:
for i in xrange(get_num_bugs(p)):
b = i + 1
cvred = r.hlen('results:test-classes-cvg-nonempty:jmockit:%s:%d' %
(p, i))
if cvred > 0:
sum += r.llen('results:test-classes:%s:%d' % (p, i))
cvr += cvred
print cvr
print sum
def method_list_matches_class_list(input, hostname, pid):
project = input['project']
version = input['version']
work_dir, d4j_path, redis_url = map(
lambda property: get_property(property, hostname, pid),
['work_dir', 'd4j_path', 'redis_url']
)
r = StrictRedis.from_url(redis_url)
key = mk_key('test-methods', [project, version])
test_methods_from_redis = r.lrange(key, 0, -1)
key2 = mk_key('test-classes', [project, version])
test_classes_from_redis = r.lrange(key2, 0, -1)
# Sanity check #1 -- no dups in test classes
no_dups(test_classes_from_redis, 'tcs from redis')
# Sanity check #2 -- no dups in test methods
no_dups(test_methods_from_redis, 'tms from redis')
tcs_as_according_to_tms = set(tc for tc, _, _ in [tm.partition('::') for tm in test_methods_from_redis])
# Sanity check #3 -- check that test classes and test methods match
# Preprocess step: We know that these classes were wrongly inserted:
#lang_classes = [
# 'org.apache.commons.lang3.builder.ReflectionToStringBuilderConcurrencyTest',
# 'org.apache.commons.lang3.builder.ReflectionToStringBuilderMutateInspectConcurrencyTest',
#]
#lang_classes_in_redis = [cl for cl in lang_classes if cl in test_classes_from_redis]
#for lang_class in lang_classes_in_redis:
# print "Removing %s from redis:" % lang_class
# print r.lrem(key2, 1, lang_class)
#if lang_classes_in_redis:
# print "Redis store was modified, reloading list before testing"
# test_classes_from_redis = r.lrange(key2, 0, -1)
check_eq(tcs_as_according_to_tms, 'tcs as according to tms', test_classes_from_redis, 'tcs from redis')
return "Success"
def non_empty_agree_includes_tt_methods(input, hostname, pid):
project = input['project']
version = input['version']
work_dir, d4j_path, redis_url = map(
lambda property: get_property(property, hostname, pid),
['work_dir', 'd4j_path', 'redis_url']
)
r = StrictRedis.from_url(redis_url)
key = mk_key('test-methods-agree-cvg-nonempty', [project, version])
test_methods = set(r.lrange(key, 0, -1))
tts = get_tts(project, version)
print tts
missing_tms = [tm for tm in tts if tm not in test_methods]
if len(missing_tms) > 0:
raise MissingTT(' '.join(missing_tms))
return "Success"
def enqueue_bundles(fun_dotted,
json_str,
queue_name='default',
timeout=1800,
print_only=False,
restrict_project=None,
restrict_version=None,
tail_keys=[],
tail_key_descr=None,
at_front=False,
check_key=None,
**kwargs):
q = Queue(queue_name, connection=StrictRedis.from_url(REDIS_URL_RQ))
r = StrictRedis.from_url(get_property('redis_url'))
for project, i in iter_versions(restrict_project, restrict_version):
input = {'project': project, 'version': i}
additionals = json.loads(json_str)
input.update(additionals)
input.update({'timeout': timeout})
if tail_keys == []:
tail_keys_to_iterate = [
[]
] # run the forloop once, but don't add any tail_key
else:
tail_keys_to_iterate = [
[tk] for tk in tail_keys
] # each of the tk's now counts, but singly
for tail_key in tail_keys_to_iterate:
if check_key:
if r.hget(mk_key(check_key, [project, i]),
':'.join(tail_key)) is not None:
continue
input.update({} if tail_key_descr is None else
{tail_key_descr: ':'.join(tail_key)})
doQ(q, fun_dotted, json.dumps(input), timeout, print_only,
at_front)
def main(project, version):
tools = ['cobertura', 'codecover', 'jmockit']
key = "%s:%d" % (project, version)
r = StrictRedis.from_url(get_property('redis_url'))
rkey = ':'.join(['results', 'test-methods-agree-cvg-nonempty', key])
tms = r.lrange(rkey, 0, -1)
for tool in tools:
missings = [
tm for (fn, tm) in [('/scratch/darioush/files/%s:%s:%s.tar.gz' %
(tool, key, tm), tm)
for tm in tms] if not LocalPath(fn).exists()
]
command = {
'redo': True,
'cvg_tool': tool,
'test_methods': missings,
'project': project,
'version': version
}
if missings:
print "python main.py q cvgmeasure.cvg.test_cvg_methods -j '%s' -t 1800 -q localH --commit" % json.dumps(
command)
def non_empty_match(input, hostname, pid):
project = input['project']
version = input['version']
input_key = input['key']
key_all = input['key_all']
should_fail_job = input.get('should_fail_job', True)
work_dir, d4j_path, redis_url = map(
lambda property: get_property(property, hostname, pid),
['work_dir', 'd4j_path', 'redis_url']
)
r = StrictRedis.from_url(redis_url)
key = mk_key(key_all, [project, version])
test_classes = r.llen(key) #r.lrange(key, 0, -1)
cobertura, codecover, jmockit = [r.hkeys(mk_key(input_key, [tool, project, version]))
for tool in ['cobertura', 'codecover', 'jmockit']]
exclude_static_fields_from = [
('Closure', 117), ('Closure', 100), ('Closure', 43), ('Closure', 47), ('Closure', 37),
('Math', 3), ('Math', 63), ('Lang', 6),
('Lang', 28), ('Lang', 17), ('Lang', 19)]
exclude_static_fields = [t for t in cobertura if t not in codecover and t in jmockit and \
(project, version) in exclude_static_fields_from and plausable_static_field(project, version, t)]
codecover_exception_from = [('Chart', 1), ('Lang', 64)]
codecover_exception = [t for t in codecover if t not in jmockit and t not in cobertura and \
(project, version) in codecover_exception_from and plausable_codecover_field(project, version, t)]
cobertura = [t for t in cobertura if t not in exclude_static_fields]
jmockit = [t for t in jmockit if t not in exclude_static_fields]
codecover = [t for t in codecover if t not in codecover_exception]
core = set(cobertura) & set(codecover) & set(jmockit)
cobertura_, codecover_, jmockit_ = [[t for t in l if t not in core] for l in (cobertura, codecover, jmockit)]
print test_classes, '/', len(core), "Agreement"
print len(exclude_static_fields), " Excluded from jmockit and cobertura as static field initializers"
print len(codecover_exception), " Excluded from codecover as static field initializers"
print "---"
print len(cobertura_), sorted(cobertura_)
print len(codecover_), sorted(codecover_)
print len(jmockit_), sorted(jmockit_)
fails = []
fails.extend(with_fails(lambda: check_sub(cobertura, 'cobertura', codecover, 'codecover')))
fails.extend(with_fails(lambda: check_sub(codecover, 'codecover', cobertura, 'cobertura')))
fails.extend(with_fails(lambda: check_sub(cobertura, 'cobertura', jmockit, 'jmockit')))
fails.extend(with_fails(lambda: check_sub(jmockit, 'jmockit', cobertura, 'cobertura')))
fails.extend(with_fails(lambda: check_sub(codecover, 'codecover', jmockit, 'jmockit')))
fails.extend(with_fails(lambda: check_sub(jmockit, 'jmockit', codecover, 'codecover')))
if fails and should_fail_job:
raise SubMismatch(' AND '.join([str(ex) for ex in fails]))
return "Success"
def enqueue_bundles_sliced(fun_dotted,
json_str,
bundle_key,
source_key,
tail_keys=[],
tail_key_descr=None,
queue_name='default',
timeout=1800,
print_only=False,
restrict_project=None,
restrict_version=None,
bundle_size=10,
bundle_offset=0,
bundle_max=None,
alternates=None,
alternate_key=None,
check_key=None,
filter_function=None,
filter_arg=None,
map_function=None,
at_front=False,
**kwargs):
if bundle_key is None:
raise Exception("bundle key not provided [-k]")
if tail_keys == []:
tail_keys_to_iterate = [
[]
] # run the forloop once, but don't add any tail_key
else:
tail_keys_to_iterate = [[tk] for tk in tail_keys
] # each of the tk's now counts, but singly
q = Queue(queue_name, connection=StrictRedis.from_url(REDIS_URL_RQ))
r = StrictRedis.from_url(get_property('redis_url'))
rr = StrictRedis.from_url(REDIS_URL_TG)
R = defaultdict(lambda: r)
R['tgs'] = rr
key_type = None # such hack
if source_key.startswith('file:'):
key_type = 'file'
_, _, fn = source_key.partition(':')
source_key = 'file'
file_data = defaultdict(list)
with open(fn) as f:
for line in f:
line_data = json.loads(line)
file_data.update(line_data)
for tail_key in tail_keys_to_iterate:
for project, i in iter_versions(restrict_project, restrict_version):
key = mk_key(source_key, [project, i] + tail_key)
if key_type != 'file':
key_type = r.type(key)
if key_type == 'list':
size = r.llen(key)
elif key_type == 'hash':
size = r.hlen(key)
elif key_type == 'file':
size = len(file_data[key])
elif key_type == 'none':
size = 0
else:
raise Exception('-- Unexpected key type: {0}'.format(key_type))
if bundle_max is not None:
size = min(size, bundle_max)
mf = (lambda _1, x, _2: x
) if map_function is None else get_fun(map_function)
already_computed = {}
if alternate_key and check_key:
for alternate in alternates:
_key = mk_key(check_key,
[alternate, project, i] + tail_key)
already_computed[alternate] = set(
mf(r, R[check_key].hkeys(_key), tail_key))
if check_key and not alternate_key:
_key = mk_key(check_key, [project, i] + tail_key)
already_computed = set(
mf(r, R[check_key].hkeys(_key), tail_key))
else:
already_computed = set()
if key_type == 'hash':
all_items = r.hkeys(key)
elif key_type == 'file':
all_items = file_data[key]
elif key_type == 'list':
all_items = r.lrange(key, 0, -1)
elif key_type == 'none':
#print key
all_items = []
if filter_function is not None:
ff = get_fun(filter_function)
all_items = ff(r, project, i, tail_key, filter_arg, all_items)
all_items = mf(r, all_items, tail_key)
def bundle_it(l, more_dict={}):
if len(l) == 0:
return
for j in xrange(bundle_offset, size, bundle_size):
if key_type in ('hash', 'file', 'list'):
bundle = l[j:j + bundle_size]
elif key_type == 'none':
bundle = []
if len(bundle) == 0:
continue
input = {
'project': project,
'version': i,
bundle_key: bundle
}
tk_input = {} if tail_key_descr is None else {
tail_key_descr: ':'.join(tail_key)
}
additionals = json.loads(json_str)
input.update(tk_input)
input.update(more_dict)
input.update(additionals)
input.update({'timeout': timeout})
doQ(q, fun_dotted, json.dumps(input), timeout, print_only,
at_front)
if alternate_key:
for alternate in alternates:
if check_key:
all_items = [
item for item in all_items
if item not in already_computed[alternate]
]
bundle_it(all_items, {alternate_key: alternate})
else:
if check_key:
all_items = [
item for item in all_items
if item not in already_computed
]
bundle_it(all_items)
def dl(r, key, hashkey, out):
with open(out, 'w') as f:
f.write(r.hget(key, hashkey))
def dlkey(r, key):
global i
_, _, tool, project, v = key.split(':')
DIR = '/scratch/darioush/files/'
for k in r.hkeys(key):
fn = "%s:%s:%s:%s.tar.gz" % (tool, project, v, k)
print i, DIR + fn
i += 1
dl(r, key, k, DIR + fn)
def keys(r):
#for key in r.keys("results:test-classes-cvg-files:*"):
for key in r.keys("results:test-methods-run-cvg-files:*"):
try:
dlkey(r, key)
r.delete(key)
except:
raise
if __name__ == "__main__":
r = redis.StrictRedis.from_url(get_property('redis_url'))
keys(r)
#dl(sys.argv[1], sys.argv[2], sys.argv[3])
def monitor_job(input, hostname, pid):
bundle = input['bundle'] # e.g., project:version:cvg_tool
bundle_keys = bundle.split(':')
check_key = input['check_key'] #e.g., test-methods-run
list_key = input['list_key'] # e.g., test_methods
monitor_queue, job_queue = input['monitor_queue'], input['job_queue']
job_name = input['job_name']
timeout = input['timeout']
monitor_name = u'cvgmeasure.monitor.monitor_job'
commit = input.get('commit', False)
downsize = input.get('downsize', None)
work_dir, d4j_path, redis_url = map(
lambda property: get_property(property, hostname, pid),
['work_dir', 'd4j_path', 'redis_url'])
r = StrictRedis.from_url(redis_url)
try:
with filter_key_list(
r,
bundle=[input[k] for k in bundle_keys],
key=check_key,
list=input[list_key],
) as worklist:
items = [item for (item, _) in worklist] # ignore the callback
except DuplicateBundleAttempt:
items = []
except:
raise MonitorFailException()
print items
if downsize is not None:
chunks_fun = lambda l, n: [l[x:x + n] for x in xrange(0, len(l), n)]
chunks = chunks_fun(items, downsize)
else:
chunks = [items]
for chunk in chunks:
if len(chunk) > 0:
r_rq = StrictRedis.from_url(REDIS_URL_RQ)
jq = Queue(job_queue, connection=r_rq)
monq = Queue(monitor_queue, connection=r_rq)
job_input = {
bundle_key: input[bundle_key]
for bundle_key in bundle_keys
}
job_input.update({list_key: chunk})
doQ(
jq,
job_name,
json.dumps(job_input),
timeout=timeout,
print_only=not commit,
)
mon_input = {}
mon_input.update(input)
mon_input.update({
list_key: chunk,
'monitor_queue': monitor_queue + '_'
})
doQ(
monq,
monitor_name,
json.dumps(mon_input),
timeout=get_current_job().timeout,
print_only=not commit,
)
return "Success ({0} / {1})".format(len(items), len(input[list_key]))
def method_list_matches(input, hostname, pid):
project = input['project']
version = input['version']
work_dir, d4j_path, redis_url = map(
lambda property: get_property(property, hostname, pid),
['work_dir', 'd4j_path', 'redis_url']
)
r = StrictRedis.from_url(redis_url)
key = mk_key('test-methods', [project, version])
test_methods_from_redis = r.lrange(key, 0, -1)
work_dir_path = local.path(work_dir) / ('child.%d' % os.getpid())
print work_dir_path
with refresh_dir(work_dir_path, cleanup=True):
with add_to_path(d4j_path):
with checkout(project, version, work_dir_path / 'checkout'):
d4()('compile')
test_methods_from_d4 = d4()('list-tests').rstrip().split('\n')
with local.env(SUCCESS_OUT="passing-tests.txt"):
failing_tests = test()
with open("passing-tests.txt") as f:
test_methods_from_run = [x[len('--- '):] for x in f.read().rstrip().split('\n')]
with open("count-of-tests.txt") as f:
per_run_counts = [int(line.rstrip()) for line in f]
count_of_tests_from_run = sum(per_run_counts)
if project == 'Lang' and version >= 37:
## In this case, we know that some tests may fail
## this is really ugly, but I'm doing it.
klass_name = 'org.apache.commons.%s.builder.ToStringBuilderTest' % (
'lang' if version > 39 else 'lang3',
)
expected_fails = [method for method in failing_tests if method.startswith(klass_name)]
single_run_fails = test(['-t', klass_name])
if len(single_run_fails) > 0:
raise TestFail('Single run failed: ' + ' '.join(single_run_fails))
elif project == 'Time':
## In this case, org.joda.time.chrono.gj.MainTest
## isn't really a jUnit test because it doesn't have a public
## constructor. We fix this during run by replacing it
## with two classes with a public constructor, each of which
## initializes the original class with parameters used during
## testing
bad_class = 'org.joda.time.chrono.gj.MainTest'
good_class1, good_class2 = ['edu.washington.cs.testfixer.time.GjMainTest' + s for s in ('1', '2')]
tname = '::testChronology'
tcs = [tc for tc, _, _ in [method.partition('::') for method in test_methods_from_run]]
idx = tcs.index(bad_class)
test_methods_from_run[idx] = good_class1 + tname
test_methods_from_run[idx+1] = good_class2 + tname
tcsd4 = [tc for tc, _, _ in [method.partition('::') for method in test_methods_from_d4]]
idxd4 = tcsd4.index(bad_class)
test_methods_from_d4 = test_methods_from_d4[:idxd4] + [good_class1 + tname,
good_class2 + tname] + test_methods_from_d4[idxd4+1:]
expected_fails = []
else:
expected_fails = []
unexpected_fails = [method for method in failing_tests if method not in expected_fails]
# Sanity check #0 -- check out the test fails
if len(unexpected_fails) > 0:
raise TestFail(' '.join(unexpected_fails))
# Sanity check #1 -- number of tests counted through the runner should equal
# the length of the list of passing tests the runner outputs
num_tests = len(test_methods_from_d4)
if num_tests != count_of_tests_from_run:
raise LenMismatch("Test methods from d4 (%d) don't match counter (%d)" %
(num_tests, count_of_tests_from_run))
# Sanity check #2 -- we should not be running duplicate tests
no_dups(test_methods_from_run, 'test methods from run')
# Sanity check #3 -- we should not be list-outputting duplicate tests
no_dups(test_methods_from_d4, 'test methods from d4')
# Sanity check #4 -- we should not have duplicate tests in redis store
no_dups(test_methods_from_redis, 'test methods from redis')
# Sanity check #5 -- tests output from the runner should match the tests output from
# d4 list-tests
check_eq(test_methods_from_run, 'test methods from run', test_methods_from_d4, 'test methods from d4')
# Sanity check #6 -- test methods from d4 should match ones in redis
#
# Preprocess step: We know that these methods were wrongly inserted:
#lang_methods = [
# 'org.apache.commons.lang3.EnumUtilsTest::test_processBitVectors_longClass',
# 'org.apache.commons.lang3.builder.ReflectionToStringBuilderConcurrencyTest::testLinkedList',
# 'org.apache.commons.lang3.builder.ReflectionToStringBuilderConcurrencyTest::testArrayList',
# 'org.apache.commons.lang3.builder.ReflectionToStringBuilderConcurrencyTest::testCopyOnWriteArrayList',
# 'org.apache.commons.lang3.builder.ReflectionToStringBuilderMutateInspectConcurrencyTest::testConcurrency',
#]
#lang_methods_in_redis = [method for method in lang_methods if method in test_methods_from_redis]
#for lang_method in lang_methods_in_redis:
# print "Removing %s from redis:" % lang_method
# print r.lrem(key, 1, lang_method)
#if lang_methods_in_redis:
# print "Redis store was modified, reloading list before testing"
# test_methods_from_redis = r.lrange(key, 0, -1)
check_eq(test_methods_from_redis, 'test methods from redis', test_methods_from_d4, 'test methods from d4')
return "Success"
