def _getty_append_invdiff(html_string, targets, go, prev_hash, curr_hash, iso):
global ignore_all_ws
ignore_all_ws = True
anchor = "<br>{{{__getty_invariant_diff__}}}<br>"
for target in sorted(targets, reverse=True):
if config.install_diffinv_only:
print ' -- processing inv diff for ' + target
tfs = fsformat(target)
osot_invf = go + "_getty_inv__" + tfs + "__" + prev_hash + "_.inv.out"
nsnt_invf = go + "_getty_inv__" + tfs + "__" + curr_hash + "_.inv.out"
invdiff_outft = go + "_getty_inv__" + tfs + "__" + ".inv.diff.html"
diff_settings = [("ni", osot_invf, nsnt_invf, invdiff_outft)]
if iso:
osnt_invf = go + "_getty_inv__" + tfs + "__" + prev_hash + "_" + curr_hash + "_.inv.out"
nsot_invf = go + "_getty_inv__" + tfs + "__" + curr_hash + "_" + prev_hash + "_.inv.out"
diff_settings += [
("si", osnt_invf, nsnt_invf, invdiff_outft[:-10]+".si.diff.html"),
("ti4o", osot_invf, osnt_invf, invdiff_outft[:-10]+".ti4o.diff.html"),
("ti4n", nsot_invf, nsnt_invf, invdiff_outft[:-10]+".ti4n.diff.html")]
for ds in diff_settings:
replacement = anchor + "\n" + \
__generate_append_diff(target, ds[0], ds[1], ds[2], ds[3])
html_string = html_string.replace(anchor, replacement)
html_string = html_string.replace(anchor, "")
if config.use_tmp_files:
remove_many_files(go, "*"+PRSV_TMP)
ignore_all_ws = False
return html_string
def _getty_append_invdiff(html_string, targets, go, prev_hash, curr_hash, iso):
global ignore_all_ws
ignore_all_ws = True
anchor = "<br>{{{__getty_invariant_diff__}}}<br>"
for target in sorted(targets, reverse=True):
if config.install_diffinv_only:
print ' -- processing inv diff for ' + target
tfs = fsformat_with_sigs(target)
osot_invf = go + "_getty_inv__" + tfs + "__" + prev_hash + "_.inv.out"
nsnt_invf = go + "_getty_inv__" + tfs + "__" + curr_hash + "_.inv.out"
invdiff_outft = go + "_getty_inv__" + tfs + "__" + ".inv.diff.html"
diff_settings = [("ni", osot_invf, nsnt_invf, invdiff_outft)]
if iso:
osnt_invf = go + "_getty_inv__" + tfs + "__" + prev_hash + "_" + curr_hash + "_.inv.out"
nsot_invf = go + "_getty_inv__" + tfs + "__" + curr_hash + "_" + prev_hash + "_.inv.out"
diff_settings += [("si", osnt_invf, nsnt_invf,
invdiff_outft[:-10] + ".si.diff.html"),
("ti4o", osot_invf, osnt_invf,
invdiff_outft[:-10] + ".ti4o.diff.html"),
("ti4n", nsot_invf, nsnt_invf,
invdiff_outft[:-10] + ".ti4n.diff.html")]
for ds in diff_settings:
replacement = anchor + "\n" + \
__generate_append_diff(target, ds[0], ds[1], ds[2], ds[3])
html_string = html_string.replace(anchor, replacement)
html_string = html_string.replace(anchor, "")
if config.use_tmp_files:
remove_many_files(go, "*" + PRSV_TMP)
ignore_all_ws = False
return html_string
def one_inv_pass(go,
cp,
junit_torun,
this_hash,
refined_target_set,
test_selection,
analysis_only=False):
if not analysis_only:
git_adapter.checkout(this_hash)
if SHOW_DEBUG_INFO:
print "\n===full classpath===\n" + cp + "\n"
java_cmd = " ".join([
"java",
"-cp",
cp,
# "-Xms"+config.min_heap,
"-Xmx" + config.max_heap,
"-XX:+UseConcMarkSweepGC",
# "-XX:-UseGCOverheadLimit",
# "-XX:-UseSplitVerifier", # FIXME: JDK 8- only!
])
maven_adapter.compile_tests(this_hash)
if SHOW_DEBUG_INFO:
print "\n===junit torun===\n" + junit_torun + "\n"
# v3.2, v4 execute with 4 core
num_primary_workers = config.num_master_workers
auto_parallel_targets = config.auto_fork
slave_load = config.classes_per_fork
target_map = daikon.target_s2m(refined_target_set)
all_classes = target_map.keys()
consider_expansion = (not analysis_only)
if len(refined_target_set) <= num_primary_workers or (
num_primary_workers == 1 and not auto_parallel_targets):
single_set_tuple = (refined_target_set, "0")
seq_get_invs(single_set_tuple, java_cmd, junit_torun, go, this_hash,
consider_expansion, test_selection)
elif num_primary_workers > 1: # FIXME: this distributation is buggy
target_set_inputs = []
all_target_set_list = list(refined_target_set)
each_bulk_size = int(len(refined_target_set) / num_primary_workers)
seq_func = partial(seq_get_invs,
java_cmd=java_cmd,
junit_torun=junit_torun,
go=go,
this_hash=this_hash,
consider_expansion=consider_expansion,
test_selection=test_selection)
for i in range(num_primary_workers):
if not (i == num_primary_workers - 1):
sub_list_tuple = (all_target_set_list[each_bulk_size *
i:each_bulk_size *
(i + 1)], str(i))
target_set_inputs.append(sub_list_tuple)
else:
sub_list_tuple = (all_target_set_list[each_bulk_size * i:],
str(i))
target_set_inputs.append(sub_list_tuple)
input_pool = Pool(num_primary_workers)
input_pool.map(seq_func, target_set_inputs)
input_pool.close()
input_pool.join()
elif num_primary_workers == 1 and auto_parallel_targets and slave_load >= 1:
# elastic automatic processing
target_set_inputs = []
num_processes = 0
# target_map has been calculated already
# target_map = daikon.target_s2m(refined_target_set)
# all_classes = target_map.keys()
num_keys = len(all_classes)
seq_func = partial(seq_get_invs,
java_cmd=java_cmd,
junit_torun=junit_torun,
go=go,
this_hash=this_hash,
consider_expansion=consider_expansion,
test_selection=test_selection)
for i in range(0, num_keys, slave_load):
# (inclusive) lower bound is i
# (exclusive) upper bound:
j = min(i + slave_load, num_keys)
sublist = []
for k in range(i, j):
the_key = all_classes[k]
sublist.append(
the_key) # so it won't miss class/object invariants
sublist += target_map[the_key]
sublist_tuple = (sublist, str(num_processes))
target_set_inputs.append(sublist_tuple)
num_processes += 1
max_parallel_processes = config.num_slave_workers
if not analysis_only:
profiler.log_csv([
"class_count", "process_count", "max_parallel_processes",
"slave_load"
], [[num_keys, num_processes, max_parallel_processes, slave_load]],
go + "_getty_y_elastic_count_" + this_hash +
"_.profile.readable")
input_pool = Pool(max_parallel_processes)
input_pool.map(seq_func, target_set_inputs)
input_pool.close()
input_pool.join()
else:
print "\nIncorrect option for one center pass:"******"\tnum_primary_workers:", str(num_primary_workers)
print "\tauto_parallel_targets:", str(auto_parallel_targets)
print "\tslave_load", str(slave_load)
sys.exit(1)
if config.compress_inv:
os.remove_many_files(go, "*.inv.gz")
else:
os.remove_many_files(go, "*.inv")
# include coverage report for compare
if config.analyze_test_coverage and not analysis_only:
try:
maven_adapter.generate_test_report(go, this_hash)
except:
pass
if not analysis_only:
git.clear_temp_checkout(this_hash)
if config.class_level_expansion:
extra_expansion = get_expansion_set(go)
os.remove_many_files(go, config.expansion_tmp_files + "*")
else:
extra_expansion = None
return all_classes, extra_expansion
def one_inv_pass(go, cp, junit_torun, this_hash, refined_target_set, analysis_only=False):
if not analysis_only:
os.sys_call("git checkout " + this_hash)
os.sys_call("mvn clean")
if SHOW_DEBUG_INFO:
print "\n===full classpath===\n" + cp + "\n"
java_cmd = " ".join(
[
"java",
"-cp",
cp,
# "-Xms"+config.min_heap,
"-Xmx" + config.max_heap,
"-XX:+UseConcMarkSweepGC",
# "-XX:-UseGCOverheadLimit",
"-XX:-UseSplitVerifier", # FIXME: JDK 8- only!
]
)
# os.sys_call("mvn test -DskipTests", ignore_bad_exit=True)
os.sys_call("mvn test-compile")
if SHOW_DEBUG_INFO:
print "\n===junit torun===\n" + junit_torun + "\n"
# v3.2, v4 execute with 4 core
num_primary_workers = config.num_master_workers
auto_parallel_targets = config.auto_fork
slave_load = config.classes_per_fork
target_map = daikon.target_s2m(refined_target_set)
all_classes = target_map.keys()
if len(refined_target_set) <= num_primary_workers or (num_primary_workers == 1 and not auto_parallel_targets):
single_set_tuple = (refined_target_set, "0")
seq_get_invs(single_set_tuple, java_cmd, junit_torun, go, this_hash)
elif num_primary_workers > 1: # FIXME: this distributation is buggy
target_set_inputs = []
all_target_set_list = list(refined_target_set)
each_bulk_size = int(len(refined_target_set) / num_primary_workers)
seq_func = partial(seq_get_invs, java_cmd=java_cmd, junit_torun=junit_torun, go=go, this_hash=this_hash)
for i in range(num_primary_workers):
if not (i == num_primary_workers - 1):
sub_list_tuple = (all_target_set_list[each_bulk_size * i : each_bulk_size * (i + 1)], str(i))
target_set_inputs.append(sub_list_tuple)
else:
sub_list_tuple = (all_target_set_list[each_bulk_size * i :], str(i))
target_set_inputs.append(sub_list_tuple)
input_pool = Pool(num_primary_workers)
input_pool.map(seq_func, target_set_inputs)
input_pool.close()
input_pool.join()
elif num_primary_workers == 1 and auto_parallel_targets and slave_load >= 1:
# elastic automatic processing
target_set_inputs = []
num_processes = 0
# target_map has been calculated already
# target_map = daikon.target_s2m(refined_target_set)
# all_classes = target_map.keys()
num_keys = len(all_classes)
seq_func = partial(seq_get_invs, java_cmd=java_cmd, junit_torun=junit_torun, go=go, this_hash=this_hash)
for i in range(0, num_keys, slave_load):
# (inclusive) lower bound is i
# (exclusive) upper bound:
j = min(i + slave_load, num_keys)
sublist = []
for k in range(i, j):
the_key = all_classes[k]
sublist.append(the_key) # so it won't miss class/object invariants
sublist += target_map[the_key]
sublist_tuple = (sublist, str(num_processes))
target_set_inputs.append(sublist_tuple)
num_processes += 1
max_parallel_processes = config.num_slave_workers
if not analysis_only:
profiler.log_csv(
["class_count", "process_count", "max_parallel_processes", "slave_load"],
[[num_keys, num_processes, max_parallel_processes, slave_load]],
go + "_getty_y_elastic_count_" + this_hash + "_.profile.readable",
)
input_pool = Pool(max_parallel_processes)
input_pool.map(seq_func, target_set_inputs)
input_pool.close()
input_pool.join()
else:
print "\nIncorrect option for one center pass:"******"\tnum_primary_workers:", str(num_primary_workers)
print "\tauto_parallel_targets:", str(auto_parallel_targets)
print "\tslave_load", str(slave_load)
sys.exit(1)
if config.compress_inv:
os.remove_many_files(go, "*.inv.gz")
else:
os.remove_many_files(go, "*.inv")
# include coverage report for compare
if config.analyze_test_coverage and not analysis_only:
try:
mvn.generate_coverage_report(go, this_hash)
except:
pass
if not analysis_only:
git.clear_temp_checkout(this_hash)
return all_classes
