def mk_additional_test_info():
s, d = StrictRedis.from_url(SRC), StrictRedis.from_url(DST)
for suite in ['evosuite-branch.{0}'.format(i) for i in xrange(0, 10)
] + ['randoop.{0}'.format(i + 1) for i in xrange(0, 10)]:
for project, version in iter_versions():
for KN in ['passcnt']:
fail_key = ':'.join(
[PRE_SRC, KN, project,
str(version), suite])
dst_fail_key = ':'.join([KN, project, str(version), suite])
fail_members = list(s.hkeys(fail_key))
if len(fail_members) > 0:
fail_idxes = tn_i_s(d, fail_members, suite)
results = s.hmget(fail_key, *fail_members)
mapping = dict(zip(fail_idxes, results))
d.hmset(dst_fail_key, mapping)
for tool in ['cobertura', 'codecover', 'jmockit', 'exec']:
for KN in ['fail']:
fail_key = ':'.join(
[PRE_SRC, KN, tool, project,
str(version), suite])
print fail_key
dst_fail_key = ':'.join(
[KN, tool, project,
str(version), suite])
fail_members = list(s.smembers(fail_key))
fail_idxes = tn_i_s(d, fail_members, suite)
if len(fail_members) > 0:
d.sadd(dst_fail_key, fail_idxes)
def main(options):
tool = options.tool
bunzip2 = local['bunzip2']
bzip2 = local['bzip2']
tar = local['tar']
for project, v in iter_versions(options.restrict_project, options.restrict_version, old=True, minimum=True):
tarname = get_name_of_tar(tool, project)
with tarfile.open(str(tarname), "r") as t:
def mapping(inv, v=v):
assert inv == v
_, _, result = d4()('match-commits', '-p', project, '-v', '{0}f'.format(inv), '-c', 'fse-commit-dbs').rstrip().partition('-> ')
return int(result)
for source, dest in get_extract_list(tool, project, v, mapping=mapping):
alt_source = source[:-len('.bak')]
try:
extract(t, source, OUT_DIR / dest)
# check for broken files
fixed_file = t.extractfile(alt_source)
with tarfile.open(fileobj=fixed_file) as t_fixed:
broken_files = [name[:-len('.broken')] for name in t_fixed.getnames() if name.endswith('.broken')]
fixed_file.close()
# now we have to remove the broken files from the archive
if broken_files:
plain_name = str(OUT_DIR / dest)[:-len('.bz2')]
bunzip2(str(OUT_DIR / dest))
# get number of .java currently in archive
with tarfile.open(plain_name) as t_current:
java_files = [name for name in t_current.getnames()
if name.endswith('.java') and not name.endswith('_scaffolding.java')]
if len(broken_files) == len(java_files):
# we are going to remove them all.
rm(plain_name)
touch(plain_name[:-len('.tar')] + '.empty')
else:
for broken_file in broken_files:
tar('--delete', '-f', plain_name, './' + broken_file)
bzip2(plain_name)
print "+ {source} -> {dest} ({broken} / {total} broken)".format(
source=source, dest=dest, broken=len(broken_files), total=len(java_files)
)
else:
print "+ {source} -> {dest} (none broken)".format(source=source, dest=dest)
except KeyError:
try:
# no .bak file was ever created, so we are good.
extract(t, alt_source, OUT_DIR / dest)
print "* {source} -> {dest} (as is)".format(source=alt_source, dest=dest)
except KeyError:
print "- {source} -> missing".format(source=alt_source)
touch(str(OUT_DIR / dest)[:-len('.tar.bz2')] + '.missing')
def mk_bundles():
s, d = StrictRedis.from_url(SRC), StrictRedis.from_url(DST)
for suite in ['evosuite-branch.{0}'.format(i) for i in xrange(0, 10)
] + ['randoop.{0}'.format(i + 1) for i in xrange(0, 10)]:
for project, version in iter_versions():
for tool in ['cobertura', 'codecover', 'jmockit']:
key = ':'.join([
PRE_SRC, 'test-methods-exec', tool, project,
str(version), suite, 'bundles'
])
result_key = ':'.join(
[PRE_SRC, 'cvg', tool, project,
str(version), suite])
print key
execed_bundles = s.hkeys(key)
if len(execed_bundles) == 0:
continue
idxes = tn_i_s(d, execed_bundles, suite)
results = s.hmget(result_key, *execed_bundles)
results_ = [
result if result is not None else json.dumps(None)
for result in results
]
dst_key = ':'.join(
['exec', tool, project,
str(version), suite])
mapping = dict(zip(idxes, results_)) #{idx: for idx in idxes}
if len(mapping) > 0:
d.hmset(dst_key, mapping)
def main():
r = StrictRedis.from_url(get_property('redis_url'))
parser = OptionParser()
parser.add_option("-p", "--project", dest="restrict_project", action="append")
parser.add_option("-v", "--version", dest="restrict_version", action="append")
parser.add_option("-x", "--commit", dest="action", action="store_true", default=False)
parser.add_option("-q", "--queue", dest="queue", action="store", default='default')
(options, args) = parser.parse_args(sys.argv)
#TOOLS = ['cobertura', 'codecover', 'jmockit', 'major']
SUITES = ['evosuite-strongmutation-fse.{i}'.format(i=i) for i in xrange(1,2)]
# ['randoop.{i}'.format(i=i) for i in xrange(1,11)] + \
# ['evosuite-branch.{i}'.format(i=i) for i in xrange(0,10)]
for suite in SUITES:
for project, v in iter_versions(options.restrict_project, options.restrict_version):
#for tool in TOOLS:
#result = r.get(mk_key('fetch', [project, v, suite]))
#if result == 'ok':
single_enqueue('cvgmeasure.cvg.test_lists_gen', json.dumps({
"project": project,
"version": v,
"suite": suite,
}), print_only=not options.action, timeout=1800, queue_name=options.queue)
def main(options):
r = redis.StrictRedis.from_url(get_property('redis_url'))
rr = redis.StrictRedis.from_url(REDIS_URL_TG)
rrr = redis.StrictRedis.from_url(REDIS_URL_OUT)
for qm in options.qms:
for gran in options.grans:
for experiment in options.experiments:
bases, _, pools = experiment.partition(',')
if options.print_only:
print '''./main.py qb cvgmeasure.select.m {project} {version} -j '{json}' {additional}'''.format(
project=''.join('-p {0}'.format(rp)
for rp in options.restrict_project),
version=''.join('-v {0}'.format(rv)
for rv in options.restrict_version),
json=json.dumps({
'granularity': gran,
'bases': bases,
'pools': pools,
"qm": qm
}),
additional=options.print_only)
else:
for project, v in iter_versions(
restrict_project=options.restrict_project,
restrict_version=options.restrict_version):
print "----( %s %d -- %s : %s)----" % (project, v, qm,
gran)
minimization(r, rr, rrr, qm, gran, project, v,
bases.split('.'), pools.split('.'))
print
def s3_files_for(r_from, r_to, projects=[], versions=[]):
for project, version in iter_versions(projects, versions):
key = 'results:test-methods:{project}:{version}:dev'.format(
project=project, version=version
)
test_methods = r_to.lrange(key, 0, -1)
print "{project}:{version} has {tms} methods".format(
project=project, version=version,
tms=len(test_methods)
)
for test_method in test_methods:
c_name, _, m_name = test_method.partition('::')
is_class_empty = []
for tool in ["codecover", "cobertura", "jmockit"]:
class_key = mk_key('test-classes-cvg', [tool, project, version])
class_r = json.loads(r_from.hget(class_key, c_name))
is_class_empty.append(is_empty(tool, class_r))
if all(is_class_empty):
continue
is_method_nonempty = []
for tool in ["codecover", "cobertura", "jmockit"]:
method_key = mk_key('test-methods-run-cvg', [tool, project, version])
try:
method_r = json.loads(r_from.hget(method_key, test_method))
if not is_empty(tool, method_r):
key = ':'.join([tool, project, str(version), test_method])
DIR = '/scratch/darioush/files'
p = LocalPath(DIR) / '{key}.tar.gz'.format(key=key)
assert p.exists()
is_method_nonempty.append(True)
with open(str(p)) as f:
s3_up = put_into_s3('cvg-files', [tool, project, version, 'dev'], test_method, f)
else:
is_method_nonempty.append(False)
except TypeError:
print "--> Missing result {tool}:{project}:{version}:dev:{test}".format(tool=tool,test=test_method, project=project, version=version)
except AssertionError:
print "--> Missing file {tool}:{project}:{version}:dev:{test}".format(tool=tool,test=test_method, project=project, version=version)
if any(is_method_nonempty):
mut_key = mk_key('test-methods-run-cvg', ['major', project, version])
try:
mut_r = json.loads(r_from.hget(mut_key, test_method))
if not is_empty('major', mut_r):
key = ':'.join(['major', project, str(version), test_method])
DIR = '/scratch/darioush/files'
p = LocalPath(DIR) / '{key}.tar.gz'.format(key=key)
assert p.exists()
with open(str(p)) as f:
s3_up = put_into_s3('cvg-files', ['major', project, version, 'dev'], test_method, f)
except TypeError:
tool = 'major'
print "--> Missing result {tool}:{project}:{version}:dev:{test}".format(tool=tool,test=test_method, project=project, version=version)
except AssertionError:
print "--> Missing file {tool}:{project}:{version}:dev:{test}".format(tool=tool,test=test_method, project=project, version=version)
def main(options):
r = StrictRedis.from_url(REDIS_URL_OUT)
for project, v in iter_versions(restrict_project=options.restrict_project,
restrict_version=options.restrict_version):
reasons = []
for qm in ['line', 'mutant', 'mutant-line']:
key = mk_key('out', [qm, 'file', '0', 'B.F', project, v]) + ':info'
info = json.loads(r.get(key))
reasons.append(info[1])
print '{project}:{v}'.format(**locals()), ' '.join(reasons)
def mk_index():
s, d = StrictRedis.from_url(SRC), StrictRedis.from_url(DST)
for suite in ['dev'
] + ['evosuite-branch.{0}'.format(i) for i in xrange(0, 10)]:
for project, version in iter_versions():
key = ':'.join(
[PRE_SRC, 'test-methods', project,
str(version), suite])
print key
tm_list = s.lrange(key, 0, -1)
idxes = tn_i_s(d, tm_list, suite)
assert (all(type(idx) is type(0) for idx in idxes))
def mk_fetch_result():
s, d = StrictRedis.from_url(SRC), StrictRedis.from_url(DST)
for suite in ['evosuite-branch.{0}'.format(i) for i in xrange(0, 10)
] + ['randoop.{0}'.format(i + 1) for i in xrange(0, 10)]:
for project, version in iter_versions():
key = ':'.join(
[PRE_SRC, 'fetch-result', project,
str(version), suite])
dst_key = ':'.join(['fetch', project, str(version)])
res = s.get(key)
if res is not None:
d.hset(dst_key, suite, res)
def migrate_test_classes(r_from, r_to, projects=[], versions=[]):
for project, version in iter_versions(projects, versions):
print project, version
tm_key = 'tms:{project}:{version}:dev'.format(project=project, version=version)
tm_is = r_to.lrange(tm_key, 0, -1)
tms = i_tn_s(r_to, tm_is, 'dev')
unique_set = set([])
def is_unique(tt):
retval = tt not in unique_set
unique_set.add(tt)
return retval
tc_tns = [tc for tc, _, _ in [tm.partition('::') for tm in tms] if is_unique(tc)]
tc_tis = tn_i_s(r_to, tc_tns, 'dev', allow_create=False)
# for tool in ['codecover', 'jmockit']:
# class_key = 'results:test-classes-cvg:{tool}:{project}:{version}'.format(
# tool=tool,project=project,version=version
# )
# class_cvgs = r_from.hmget(class_key, *tc_tns)
# assert all(lambda x: x is not None for x in class_cvgs)
# assert len(tc_tis) == len(class_cvgs)
# to_key_class = 'exec:{tool}:{project}:{version}:dev'.format(
# tool=tool, project=project, version=version
# )
# r_to.hmset(to_key_class, {ck: cv for (ck, cv) in zip(tc_tis, class_cvgs)})
for tool in ['cobertura', 'codecover', 'jmockit', 'major']:
method_key = 'results:test-methods-run-cvg:{tool}:{project}:{version}'.format(
tool=tool, project=project, version=version
)
res_dict = r_from.hgetall(method_key)
assert(type(res_dict) == dict)
res_list = res_dict.items()
res_idxs = tn_i_s(r_to, [k for (k, v) in res_list], 'dev')
res_vals = [v for (_, v) in res_list]
assert len(res_vals) == len(res_idxs)
res_map = {ki: v for (ki, v) in zip(res_idxs, res_vals)}
to_key = 'exec:{tool}:{project}:{version}:dev'.format(
tool=tool, project=project, version=version
)
if res_map:
r_to.hmset(to_key, res_map)
def mk_tms():
s, d = StrictRedis.from_url(SRC), StrictRedis.from_url(DST)
for suite in ['dev'] + [
'evosuite-branch.{0}'.format(i) for i in xrange(0, 10)
] + ['randoop.{0}'.format(i + 1) for i in xrange(0, 10)]:
for project, version in iter_versions():
key = ':'.join(
[PRE_SRC, 'test-methods', project,
str(version), suite])
print key
tm_list = s.lrange(key, 0, -1)
idxes = tn_i_s(d, tm_list, suite)
dst_key = ':'.join(['tms', project, str(version), suite])
assert (len(idxes) == len(tm_list))
for chunk in chunks(idxes, 100):
if len(chunk) == 0:
continue
d.rpush(dst_key, *chunk)
def main(r):
for p, v in iter_versions():
for tool in ['cobertura', 'codecover', 'jmockit', 'major']:
m = r.hgetall('exec:{t}:{p}:{v}:dev'.format(p=p, v=v, t=tool))
keys = m.keys()
tn_dict = {
ti: tn
for (ti, tn) in zip(keys, i_tn_s(r, keys, 'dev'))
}
mm = {
ti: 1
for ti, result in m.iteritems()
if (not is_empty(tool, json.loads(result)))
and tn_dict[ti].find('::') != -1
}
r.delete('nonempty:{t}:{p}:{v}:dev'.format(p=p, v=v, t=tool), mm)
print p, v, tool, len(mm)
if mm:
r.hmset('nonempty:{t}:{p}:{v}:dev'.format(p=p, v=v, t=tool),
mm)
def main():
parser = OptionParser()
parser.add_option("-p",
"--project",
dest="restrict_project",
action="append")
parser.add_option("-v",
"--version",
dest="restrict_version",
action="append")
(options, args) = parser.parse_args(sys.argv)
java = local['java']['-cp', JAR_PATH, 'edu.washington.cs.tgs.MapBuilder']
for p, v in iter_versions(options.restrict_project,
options.restrict_version):
print p, v
src_dir, f_list = get_modified_sources(p, v)
work_dir_path = '/tmp/work.{pid}'.format(pid=os.getpid())
with refresh_dir(work_dir_path, cleanup=True):
with checkout(p, v, work_dir_path):
with local.cwd(src_dir):
(java > '/tmp/results/{p}:{v}'.format(p=p, v=v))(*f_list)
def enqueue_bundles(fun_dotted,
json_str,
queue_name='default',
timeout=1800,
print_only=False,
restrict_project=None,
restrict_version=None,
tail_keys=[],
tail_key_descr=None,
at_front=False,
check_key=None,
**kwargs):
q = Queue(queue_name, connection=StrictRedis.from_url(REDIS_URL_RQ))
r = StrictRedis.from_url(get_property('redis_url'))
for project, i in iter_versions(restrict_project, restrict_version):
input = {'project': project, 'version': i}
additionals = json.loads(json_str)
input.update(additionals)
input.update({'timeout': timeout})
if tail_keys == []:
tail_keys_to_iterate = [
[]
] # run the forloop once, but don't add any tail_key
else:
tail_keys_to_iterate = [
[tk] for tk in tail_keys
] # each of the tk's now counts, but singly
for tail_key in tail_keys_to_iterate:
if check_key:
if r.hget(mk_key(check_key, [project, i]),
':'.join(tail_key)) is not None:
continue
input.update({} if tail_key_descr is None else
{tail_key_descr: ':'.join(tail_key)})
doQ(q, fun_dotted, json.dumps(input), timeout, print_only,
at_front)
def main(options):
ts, ts_ri = connect_db()
r = StrictRedis.from_url(REDIS_URL_OUT)
for qm in options.qms:
for granularity in options.grans:
for experiment in options.experiments:
bases, _, pools = experiment.partition(',')
for project, v in iter_versions(
restrict_project=options.restrict_project,
restrict_version=options.restrict_version):
bases_srtd = '.'.join(sorted(bases.split('.')))
pools_srtd = '.'.join(sorted(pools.split('.')))
fmt = '{qm}:{granularity}:{bases_srtd}:{pools_srtd}:{project}:{v}'.format(
**locals())
if options.skip:
print '{fmt}...'.format(fmt=fmt)
exists = sel(
ts_ri, {
'qm': qm,
'granularity': granularity,
'project': project,
'version': v,
'base': bases_srtd,
'select_from': pools_srtd
})
if exists > 0:
print "SKIP"
continue
session = Session()
runs = {
k: json.loads(v)
for (k, v) in r.hgetall('out:{fmt}'.format(
fmt=fmt)).iteritems()
}
keys = sorted([(alg, int(x)) for (
alg,
x) in [tuple(key.split(':')) for key in runs.keys()]])
rows = []
fixed_info = {
'qm': qm,
'granularity': granularity,
'project': project,
'version': v,
'base': bases_srtd,
'select_from': pools_srtd,
}
suite_schema = lambda x: [
'{0}_{1}'.format(x, i)
for i in ('triggers', 'count', 'tgs', 'time')
]
schema = [ 'relevant_tgs', 'determined_by',] + \
suite_schema('base') + suite_schema('base_relevant') + suite_schema('aug') + suite_schema('aug_relevant') \
+ suite_schema('aug_additional')
ri_val = dict(
zip(
schema,
json.loads(r.get(
'out:{fmt}:info'.format(fmt=fmt)))))
for algo, it in groupby(keys, key=lambda (a, b): a):
for algo, run_id in it:
uniq_key = '{fmt}:{algo}:{run_id}'.format(
**locals())
run_result = runs['{0}:{1}'.format(algo, run_id)]
row_info = {
'algorithm': algo,
'run_id': run_id,
'fault_detection': run_result[0],
'determined_by': run_result[1],
'selected_triggers': run_result[2],
'selected_count': run_result[3],
'selected_tgs': run_result[4],
'selected_time': run_result[5],
}
row_info.update(fixed_info)
rows.append(row_info)
save_rows(ts, rows)
ri_val.update(fixed_info)
save_rows(ts_ri, [ri_val])
print '{0}: {1} records'.format(fmt, len(rows))
session.commit()
def enqueue_bundles_sliced(fun_dotted,
json_str,
bundle_key,
source_key,
tail_keys=[],
tail_key_descr=None,
queue_name='default',
timeout=1800,
print_only=False,
restrict_project=None,
restrict_version=None,
bundle_size=10,
bundle_offset=0,
bundle_max=None,
alternates=None,
alternate_key=None,
check_key=None,
filter_function=None,
filter_arg=None,
map_function=None,
at_front=False,
**kwargs):
if bundle_key is None:
raise Exception("bundle key not provided [-k]")
if tail_keys == []:
tail_keys_to_iterate = [
[]
] # run the forloop once, but don't add any tail_key
else:
tail_keys_to_iterate = [[tk] for tk in tail_keys
] # each of the tk's now counts, but singly
q = Queue(queue_name, connection=StrictRedis.from_url(REDIS_URL_RQ))
r = StrictRedis.from_url(get_property('redis_url'))
rr = StrictRedis.from_url(REDIS_URL_TG)
R = defaultdict(lambda: r)
R['tgs'] = rr
key_type = None # such hack
if source_key.startswith('file:'):
key_type = 'file'
_, _, fn = source_key.partition(':')
source_key = 'file'
file_data = defaultdict(list)
with open(fn) as f:
for line in f:
line_data = json.loads(line)
file_data.update(line_data)
for tail_key in tail_keys_to_iterate:
for project, i in iter_versions(restrict_project, restrict_version):
key = mk_key(source_key, [project, i] + tail_key)
if key_type != 'file':
key_type = r.type(key)
if key_type == 'list':
size = r.llen(key)
elif key_type == 'hash':
size = r.hlen(key)
elif key_type == 'file':
size = len(file_data[key])
elif key_type == 'none':
size = 0
else:
raise Exception('-- Unexpected key type: {0}'.format(key_type))
if bundle_max is not None:
size = min(size, bundle_max)
mf = (lambda _1, x, _2: x
) if map_function is None else get_fun(map_function)
already_computed = {}
if alternate_key and check_key:
for alternate in alternates:
_key = mk_key(check_key,
[alternate, project, i] + tail_key)
already_computed[alternate] = set(
mf(r, R[check_key].hkeys(_key), tail_key))
if check_key and not alternate_key:
_key = mk_key(check_key, [project, i] + tail_key)
already_computed = set(
mf(r, R[check_key].hkeys(_key), tail_key))
else:
already_computed = set()
if key_type == 'hash':
all_items = r.hkeys(key)
elif key_type == 'file':
all_items = file_data[key]
elif key_type == 'list':
all_items = r.lrange(key, 0, -1)
elif key_type == 'none':
#print key
all_items = []
if filter_function is not None:
ff = get_fun(filter_function)
all_items = ff(r, project, i, tail_key, filter_arg, all_items)
all_items = mf(r, all_items, tail_key)
def bundle_it(l, more_dict={}):
if len(l) == 0:
return
for j in xrange(bundle_offset, size, bundle_size):
if key_type in ('hash', 'file', 'list'):
bundle = l[j:j + bundle_size]
elif key_type == 'none':
bundle = []
if len(bundle) == 0:
continue
input = {
'project': project,
'version': i,
bundle_key: bundle
}
tk_input = {} if tail_key_descr is None else {
tail_key_descr: ':'.join(tail_key)
}
additionals = json.loads(json_str)
input.update(tk_input)
input.update(more_dict)
input.update(additionals)
input.update({'timeout': timeout})
doQ(q, fun_dotted, json.dumps(input), timeout, print_only,
at_front)
if alternate_key:
for alternate in alternates:
if check_key:
all_items = [
item for item in all_items
if item not in already_computed[alternate]
]
bundle_it(all_items, {alternate_key: alternate})
else:
if check_key:
all_items = [
item for item in all_items
if item not in already_computed
]
bundle_it(all_items)
def main():
r = StrictRedis.from_url(get_property_defaults('redis_url'))
parser = OptionParser()
parser.add_option("-p",
"--project",
dest="restrict_project",
action="append")
parser.add_option("-v",
"--version",
dest="restrict_version",
action="append")
parser.add_option("-T",
"--tool",
dest="tools",
action="append",
default=[])
parser.add_option("-s",
"--suite",
dest="suite",
action="append",
default=[])
parser.add_option("-S",
"--suite-ids",
dest="suite_ids",
action="store",
default=None)
parser.add_option("-q",
"--queue",
dest="queue_name",
action="store",
type="string",
default="default")
parser.add_option("-t",
"--timeout",
dest="timeout",
action="store",
type="int",
default=1800)
parser.add_option("-b",
"--bundle-size",
dest="bundle_size",
action="store",
type="int",
default=10)
parser.add_option("-c",
"--commit",
dest="print_only",
action="store_false",
default=True)
(options, args) = parser.parse_args(sys.argv)
if options.suite_ids is None:
suites = options.suite
else:
rmin, _, rmax = options.suite_ids.partition('-')
rmin, rmax = int(rmin), int(rmax)
suites = [
'{name}.{id}'.format(name=suite, id=id)
for id in xrange(rmin, rmax + 1) for suite in options.suite
]
if options.tools == []:
tools = [[]] ## ??
else:
tools = [[tool] for tool in options.tools]
if suites == []:
suites = ['dev']
total_tms = 0
stragglers = False
for suite in suites:
suite_tms = 0
for project, v in iter_versions(options.restrict_project,
options.restrict_version):
v_idxs, v_tns = check_test_list(r, project, v, suite)
suite_tms += len(v_idxs)
total_tms += len(v_idxs)
if len(v_idxs) == 0:
print ".. passing"
continue
for tool_ in tools:
tool = ''.join(tool_)
print "- {tool}:{project}:{v}:{suite}".format(tool=tool,
project=project,
v=v,
suite=suite)
wasOK = False
with run_with_fixes():
fails = get_fails(r, tool, project, v, suite, v_idxs)
passing = [(v_idx, v_tn)
for (v_idx, v_tn) in zip(v_idxs, v_tns)
if v_idx not in fails]
print len(passing), len(fails)
if len(passing) > 0:
p_idxs, p_tns = zip(*passing)
cvg_info = check_cvg(r, tool, project, v, suite,
p_idxs, p_tns)
print cvg_info
else:
print "No passing tests."
wasOK = True
stragglers |= (not wasOK)
print 'suite {suite} tms: {tms}'.format(suite=suite, tms=suite_tms)
print 'total tms', total_tms
if not stragglers:
print Fore.GREEN + "NO STRAGGLERS"
