def all_methods_expansion(candidates, target_set, go, this_hash, index, java_cmd, inv_gz):
exp_tmp = go + "expansion_temp." + this_hash + "." + str(index) + ".allinvs"
run_print_allinvs = " ".join([java_cmd, "daikon.PrintInvariants", "--output", exp_tmp, inv_gz])
os.sys_call(run_print_allinvs, ignore_bad_exit=True)
regex_header = "(.*):::(ENTER|EXIT|CLASS|OBJECT|THROW).*"
with open(exp_tmp, 'r') as rf:
alllines = rf.read().split("\n")
for line in alllines:
m = re.match(regex_header, line.strip())
if m:
full_method = m.group(1)
leftp_bound = full_method.find("(")
rightp_bound = full_method.find(")")
if leftp_bound != -1:
all_dots_mtdname = full_method[:leftp_bound]
last_dot_index = all_dots_mtdname.rfind(".")
if last_dot_index != -1:
raw_method_name = all_dots_mtdname[last_dot_index+1:]
further_last_dot_index = all_dots_mtdname[:last_dot_index].rfind(".")
if all_dots_mtdname[further_last_dot_index+1:last_dot_index] == raw_method_name:
raw_method_name = "<init>"
candidates.add(
all_dots_mtdname[:last_dot_index] + ":" + raw_method_name +
full_method[leftp_bound:rightp_bound+1].replace(" ", ""))
os.remove_file(exp_tmp)
ex.save_list_to(go + config.expansion_tmp_files + "." + this_hash +
"." + str(index) + "." + str(int(time.time())),
candidates)
def backup_and_stash_first():
sys_call("git reset HEAD .")
resp = from_sys_call("git stash save --keep-index --include-untracked").strip()
if resp == "No local changes to save":
return False
else:
return True
def backup_and_stash_first():
sys_call("git reset HEAD .")
resp = from_sys_call(
"git stash save --keep-index --include-untracked").strip()
if resp == "No local changes to save":
return False
else:
return True
def exam(iso, pwd, go, fe_path, common_package, all_classes_set,
targets, all_refined_target_set,
new_refined_target_set, old_refined_target_set,
new_modified_src, new_all_src,
new_caller_of, new_callee_of, new_pred_of, new_succ_of,
old_caller_of, old_callee_of, old_pred_of, old_succ_of,
all_changed_tests, old_changed_tests, new_changed_tests,
old_test_set, new_test_set,
prev_hash, post_hash, old_l2m, new_l2m, old_m2l, new_m2l):
refined_targets_parents_set = all_refined_target_set | all_classes_set
print 'generating full diff html ...'
# get extra diff file for full diff of each changed file
full_src_diff_in = go + "full.text.diff"
full_src_diff_out = go + "src.diff.html"
sys_call(
"git diff -U{3} {0} {1} > {2}".format(
prev_hash, post_hash, full_src_diff_in, config.max_context_line))
srcdiff2html(
full_src_diff_in, full_src_diff_out,
exclude_headers=None, old_l2m=old_l2m, new_l2m=new_l2m)
print 'generating main html from diff ...'
diff_in = go + "text.diff"
html_out = go + "sema.diff.html"
diff_to_html(diff_in, html_out,
exclude_headers=False, old_l2m=old_l2m, new_l2m=new_l2m)
print 'appending semainfo to the html ....'
commit_msgs = git_commit_msgs(prev_hash, post_hash)
github_link = github_info(prev_hash, post_hash)
getty_append_semainfo(html_out, refined_targets_parents_set, go, fe_path,
commit_msgs, github_link,
prev_hash, post_hash, old_l2m, new_l2m, iso)
print 'initialize csi report ...'
getty_csi_init(html_out, iso)
print 'setting csi target variables ...'
getty_csi_targets_prep(
html_out, go, prev_hash, post_hash, common_package,
all_changed_tests, old_changed_tests, new_changed_tests,
new_modified_src, new_all_src,
old_test_set, new_test_set,
old_caller_of, old_callee_of, old_pred_of, old_succ_of,
new_caller_of, new_callee_of, new_pred_of, new_succ_of,
old_refined_target_set, new_refined_target_set, all_refined_target_set,
all_classes_set, iso)
print 'csi report page is set.'
if config.review_auto_open:
print ' Opening rendered pages for review ...'
sys_call("open " + html_out)
else:
print ' CSI analysis completed - review at: ' + html_out
def mixed_passes(go, prev_hash, post_hash, refined_expansion_set,
refined_target_set, old_refined_target_set, new_refined_target_set,
old_cp, new_cp, old_junit_torun, new_junit_torun):
if config.class_level_expansion:
impact_set = refined_target_set | refined_expansion_set
else:
impact_set = refined_target_set
# checkout old commit, then checkout new tests
os.sys_call("git checkout " + prev_hash)
new_test_path = mvn.path_from_mvn_call("testSourceDirectory")
os.sys_call(" ".join(["git", "checkout", post_hash, new_test_path]))
# # may need to check whether it is compilable, return code?
# os.sys_call("mvn clean test-compile")
one_inv_pass(go, new_cp, new_junit_torun,
prev_hash + "_" + post_hash,
impact_set, analysis_only=True)
git.clear_temp_checkout(prev_hash)
# checkout old commit, then checkout new src
os.sys_call("git checkout " + prev_hash)
new_src_path = mvn.path_from_mvn_call("sourceDirectory")
os.sys_call(" ".join(["git", "checkout", post_hash, new_src_path]))
# # may need to check whether it is compilable, return code?
# os.sys_call("mvn clean test-compile")
one_inv_pass(go, new_cp, old_junit_torun,
post_hash + "_" + prev_hash,
impact_set, analysis_only=True)
git.clear_temp_checkout(prev_hash)
def mixed_passes(go, refined_target_set, prev_hash, post_hash, old_cp, new_cp, old_junit_torun, new_junit_torun):
# checkout old commit, then checkout new tests
os.sys_call("git checkout " + prev_hash)
new_test_path = mvn.path_from_mvn_call("testSourceDirectory")
os.sys_call(" ".join(["git", "checkout", post_hash, new_test_path]))
# # may need to check whether it is compilable, return code?
# os.sys_call("mvn clean test-compile")
one_inv_pass(go, new_cp, new_junit_torun, prev_hash + "_" + post_hash, refined_target_set, analysis_only=True)
git.clear_temp_checkout(prev_hash)
# checkout old commit, then checkout new src
os.sys_call("git checkout " + prev_hash)
new_src_path = mvn.path_from_mvn_call("sourceDirectory")
os.sys_call(" ".join(["git", "checkout", post_hash, new_src_path]))
# # may need to check whether it is compilable, return code?
# os.sys_call("mvn clean test-compile")
one_inv_pass(go, new_cp, old_junit_torun, post_hash + "_" + prev_hash, refined_target_set, analysis_only=True)
git.clear_temp_checkout(prev_hash)
def checkout_build(proj_dir, commit_hash):
os.sys_call("git checkout " + commit_hash)
os.sys_call("mvn clean")
bin_path = mvn.path_from_mvn_call("outputDirectory")
src_rel_path = mvn.path_from_mvn_call("sourceDirectory")
if src_rel_path.startswith(proj_dir):
src_rel_path = src_rel_path[len(proj_dir):]
else:
raise ValueError("proj_dir is not a prefix of src path")
print "current src path (relative): " + src_rel_path + "\n"
test_src_rel_path = mvn.path_from_mvn_call("testSourceDirectory")
if test_src_rel_path.startswith(proj_dir):
test_src_rel_path = test_src_rel_path[len(proj_dir):]
else:
raise ValueError("proj_dir is not a prefix of test src path")
print "current test src path (relative): " + test_src_rel_path + "\n"
os.sys_call("mvn test-compile")
return bin_path, src_rel_path, test_src_rel_path
def checkout_build(proj_dir, commit_hash):
os.sys_call("git checkout " + commit_hash)
os.sys_call("mvn clean")
bin_path = mvn.path_from_mvn_call("outputDirectory")
src_rel_path = mvn.path_from_mvn_call("sourceDirectory")
if src_rel_path.startswith(proj_dir):
src_rel_path = src_rel_path[len(proj_dir):]
else:
raise ValueError("proj_dir is not a prefix of src path")
print "current src path (relative): " + src_rel_path + "\n"
test_src_rel_path = mvn.path_from_mvn_call("testSourceDirectory")
if test_src_rel_path.startswith(proj_dir):
test_src_rel_path = test_src_rel_path[len(proj_dir):]
else:
raise ValueError("proj_dir is not a prefix of test src path")
print "current test src path (relative): " + test_src_rel_path + "\n"
os.sys_call("mvn test-compile")
return bin_path, src_rel_path, test_src_rel_path
def generate_coverage_report(go, curr_hash):
sys_call("mvn emma:emma", ignore_bad_exit=True)
emma_dir = path_from_mvn_call("directory") + "/site/emma"
target_dir = go + "_getty_emma_" + curr_hash + "_"
sys_call(" ".join(["mv", emma_dir, target_dir]), ignore_bad_exit=True)
def restore_and_pop_last(head_branch, should_further_recover):
sys_call("git checkout " + head_branch)
if should_further_recover:
sys_call("git stash pop", ignore_bad_exit=True)
def clear_temp_checkout(current_commit):
sys_call("git reset --hard " + current_commit)
resp = from_sys_call(
"git stash save --keep-index --include-untracked").strip()
if resp != "No local changes to save":
sys_call("git stash drop")
def restore_and_pop_last(head_branch, should_further_recover):
sys_call("git checkout " + head_branch)
if should_further_recover:
sys_call("git stash pop", ignore_bad_exit=True)
def visit(villa_path, pwd, proj_dir, go, prev_hash, post_hash, pkg_prefix="-"):
print("\n****************************************************************")
print(" Getty Villa: Semantiful Differential Analyzer ")
print("****************************************************************\n")
print "current working directory: " + pwd + "\n"
diff_out = go + "text.diff"
os.sys_call(" ".join(
["git diff",
str(config.git_diff_extra_ops),
"{0} {1} > {2}"]).format(prev_hash, post_hash, diff_out))
'''
1-st pass: checkout prev_commit as detached head, and get all sets and etc, in simple (bare) mode (-s)
remember to clear after this pass
'''
bin_path, src_rel_path, test_src_rel_path = checkout_build(prev_hash)
run_villa = "java -jar {0} -s {1} {2} {3} {4} {5} {6} -o {7}".format(
villa_path, diff_out, bin_path, test_src_rel_path, pkg_prefix,
prev_hash, post_hash, go)
run_villa_l4ms = "java -jar {0} -l {1} {2} {3} -o {4}".format(
villa_path, src_rel_path, test_src_rel_path, prev_hash, go)
print "\n\nstart to run Villa ... \n\n" + run_villa + "\n and \n" + run_villa_l4ms
chdir(proj_dir)
os.sys_call(run_villa)
os.sys_call(run_villa_l4ms)
chdir(pwd)
old_changed_methods = ex.read_str_from(
go + "_getty_chgmtd_src_old_{0}_.ex".format(prev_hash))
old_all_methods = ex.read_str_from(
go + "_getty_allmtd_src_{0}_.ex".format(prev_hash))
old_l2m = ex.read_str_from(go + "_getty_fl2m_{0}_.ex".format(prev_hash))
old_m2l = ex.read_str_from(go + "_getty_fm2l_{0}_.ex".format(prev_hash))
old_changed_tests = ex.read_str_from(
go + "_getty_chgmtd_test_old_{0}_.ex".format(prev_hash))
git.clear_temp_checkout(prev_hash)
'''
2-nd pass: checkout post_commit as detached head, and get all sets and etc, in complex mode (-c)
remember to clear after this pass
'''
bin_path, src_rel_path, test_src_rel_path = checkout_build(post_hash)
run_villa = "java -jar {0} -c {1} {2} {3} {4} {5} {6} -o {7}".format(
villa_path, diff_out, bin_path, test_src_rel_path, pkg_prefix,
prev_hash, post_hash, go)
run_villa_l4ms = "java -jar {0} -l {1} {2} {3} -o {4}".format(
villa_path, src_rel_path, test_src_rel_path, post_hash, go)
print "\n\nstart to run Villa ... \n\n" + run_villa + "\n and \n" + run_villa_l4ms
chdir(proj_dir)
os.sys_call(run_villa)
os.sys_call(run_villa_l4ms)
chdir(pwd)
new_changed_methods = ex.read_str_from(
go + "_getty_chgmtd_src_new_{0}_.ex".format(post_hash))
new_improved_changed_methods = ex.read_str_from(
go + "_getty_chgmtd_src_{0}_{1}_.ex".format(prev_hash, post_hash))
new_removed_changed_methods = ex.read_str_from(
go + "_getty_chgmtd_src_gone_{0}_{1}_.ex".format(prev_hash, post_hash))
# TODO or FIXME
# new_all_ccc_related = ex.read_str_from(go + "_getty_cccmtd_{0}_.ex".format(post_hash)) # not needed for now
# new_all_cccs = ex.read_str_from(go + "_getty_ccc_{0}_.ex".format(post_hash)) # not needed for now
new_all_methods = ex.read_str_from(
go + "_getty_allmtd_src_{0}_.ex".format(post_hash))
new_l2m = ex.read_str_from(go + "_getty_fl2m_{0}_.ex".format(post_hash))
new_m2l = ex.read_str_from(go + "_getty_fm2l_{0}_.ex".format(post_hash))
new_changed_tests = ex.read_str_from(
go + "_getty_chgmtd_test_new_{0}_.ex".format(post_hash))
git.clear_temp_checkout(post_hash)
'''
3-rd pass: checkout prev_commit as detached head, and get all sets and etc, in recovery mode (-r)
remember to clear after this pass
'''
bin_path, src_rel_path, test_src_rel_path = checkout_build(prev_hash)
run_villa = "java -jar {0} -r {1} {2} {3} {4} {5} {6} -o {7}".format(
villa_path, diff_out, bin_path, test_src_rel_path, pkg_prefix,
prev_hash, post_hash, go)
print "\n\nstart to run Villa ... \n\n" + run_villa
chdir(proj_dir)
os.sys_call(run_villa)
chdir(pwd)
old_improved_changed_methods = ex.read_str_from(
go + "_getty_chgmtd_src_{1}_{0}_.ex".format(prev_hash, post_hash))
old_added_changed_methods = ex.read_str_from(
go + "_getty_chgmtd_src_gain_{0}_{1}_.ex".format(prev_hash, post_hash))
# TODO or FIXME
# old_all_ccc_related = ex.read_str_from(go + "_getty_cccmtd_{0}_.ex".format(prev_hash)) # not needed for now
# old_all_cccs = ex.read_str_from(go + "_getty_ccc_{0}_.ex".format(prev_hash)) # not needed for now
git.clear_temp_checkout(prev_hash)
print 'Villa analysis is completed.'
return old_changed_methods, old_improved_changed_methods, old_added_changed_methods, \
old_all_methods, old_l2m, old_m2l, \
new_changed_methods, new_improved_changed_methods, new_removed_changed_methods, \
new_all_methods, new_l2m, new_m2l, \
old_changed_tests, new_changed_tests
def one_info_pass(junit_path, sys_classpath, agent_path, cust_mvn_repo,
dyng_go, go, this_hash, target_set, changed_methods,
changed_tests, json_filepath):
bin_path = maven_adapter.get_bin_path(this_hash)
test_bin_path = maven_adapter.get_test_bin_path(this_hash)
cp = maven_adapter.get_full_class_path(this_hash, junit_path,
sys_classpath, bin_path,
test_bin_path)
if SHOW_DEBUG_INFO:
print "\n===full classpath===\n" + cp + "\n"
print "\ncopying all code to specific directory ...\n"
all_code_dirs = [
maven_adapter.get_source_directory(this_hash),
maven_adapter.get_test_source_directory(this_hash)
]
getty_code_store = go + '_getty_allcode_' + this_hash + '_/'
print 'copy to ' + getty_code_store + '\n'
makedirs(getty_code_store)
for adir in all_code_dirs:
os.sys_call(" ".join(["cp -r", adir + "/*", getty_code_store]),
ignore_bad_exit=True)
if config.use_special_junit_for_dyn:
info_junit_path = os.rreplace(junit_path, config.default_junit_version,
config.special_junit_version, 1)
infocp = maven_adapter.get_full_class_path(this_hash, info_junit_path,
sys_classpath, bin_path,
test_bin_path)
else:
infocp = cp
maven_adapter.compile_tests(this_hash)
junit_torun = maven_adapter.get_junit_torun(cust_mvn_repo, this_hash)
if SHOW_DEBUG_INFO:
print "\n===junit torun===\n" + junit_torun + "\n"
#### dynamic run one round for all information
prefixes = daikon.common_prefixes(target_set)
common_package = ''
if len(prefixes) == 1:
last_period_index = prefixes[0].rindex('.')
if last_period_index > 0:
# the common package should be at least one period away from the rest
common_package = prefixes[0][:last_period_index]
prefix_regexes = []
for p in prefixes:
prefix_regexes.append(p + "*")
instrument_regex = "|".join(prefix_regexes)
if SHOW_DEBUG_INFO:
print "\n===instrumentation pattern===\n" + instrument_regex + "\n"
if not path.exists(dyng_go):
makedirs(dyng_go)
full_info_exfile = java.run_instrumented_tests(this_hash, go, infocp,
agent_path,
instrument_regex,
junit_torun)
full_method_info_map = {}
ext_start_index = len(config.method_info_line_prefix)
with open(full_info_exfile, 'r') as f:
contents = f.read().split("\n")
for line in contents:
line = line.strip()
if line.startswith(config.method_info_line_prefix):
rawdata = line[ext_start_index:]
k, v = rawdata.split(" : ")
full_method_info_map[k.strip()] = v.strip()
print "dyng_go=", dyng_go, " go=", go
os.merge_dyn_files(dyng_go, go, "_getty_dyncg_-hash-_.ex", this_hash)
os.merge_dyn_files(dyng_go, go, "_getty_dynfg_-hash-_.ex", this_hash)
caller_of, callee_of = agency.caller_callee(go, this_hash)
pred_of, succ_of = agency.pred_succ(go, this_hash)
if json_filepath != "":
junit_torun, target_set, test_set = get_tests_and_target_set(
go, json_filepath, junit_torun, this_hash)
else:
test_set = agency.get_test_set_dyn(callee_of, junit_torun)
# test_set is correct
# reset target set here
refined_target_set, changed_methods, changed_tests = \
agency.refine_targets(full_method_info_map, target_set, test_set,
caller_of, callee_of, pred_of, succ_of,
changed_methods, changed_tests, json_filepath)
profiler.log_csv(["method_count", "test_count", "refined_target_count"], [[
len(target_set),
len(test_set), len(refined_target_set)
]], go + "_getty_y_method_count_" + this_hash + "_.profile.readable")
git.clear_temp_checkout(this_hash)
return common_package, test_set, refined_target_set, changed_methods, changed_tests, \
cp, junit_torun, full_method_info_map
def one_info_pass(
junit_path, sys_classpath, agent_path, cust_mvn_repo, dyng_go, go, this_hash, target_set,
changed_methods, changed_tests, inner_dataflow_methods, outer_dataflow_methods):
os.sys_call("git checkout " + this_hash)
os.sys_call("mvn clean")
bin_path = mvn.path_from_mvn_call("outputDirectory")
test_bin_path = mvn.path_from_mvn_call("testOutputDirectory")
cp = mvn.full_classpath(junit_path, sys_classpath, bin_path, test_bin_path)
if SHOW_DEBUG_INFO:
print "\n===full classpath===\n" + cp + "\n"
print "\ncopying all code to specific directory ...\n"
all_code_dirs = [mvn.path_from_mvn_call("sourceDirectory"),
# mvn.path_from_mvn_call("scriptSourceDirectory"),
mvn.path_from_mvn_call("testSourceDirectory")]
getty_code_store = go + '_getty_allcode_' + this_hash + '_/'
print 'copy to ' + getty_code_store + '\n'
makedirs(getty_code_store)
for adir in all_code_dirs:
os.sys_call(" ".join(["cp -r", adir + "/*", getty_code_store]), ignore_bad_exit=True)
if config.use_special_junit_for_dyn:
info_junit_path = os.rreplace(junit_path, config.default_junit_version, config.special_junit_version, 1)
infocp = mvn.full_classpath(info_junit_path, sys_classpath, bin_path, test_bin_path)
else:
infocp = cp
java_cmd = " ".join(["java", "-cp", infocp,
# "-Xms"+config.min_heap,
"-Xmx"+config.max_heap,
"-XX:+UseConcMarkSweepGC",
# "-XX:-UseGCOverheadLimit",
"-XX:-UseSplitVerifier", # FIXME: JDK 8- only!
])
# os.sys_call("mvn test -DskipTests", ignore_bad_exit=True)
os.sys_call("mvn test-compile")
junit_torun = mvn.junit_torun_str(cust_mvn_repo)
if SHOW_DEBUG_INFO:
print "\n===junit torun===\n" + junit_torun + "\n"
#### dynamic run one round for all information
prefixes = daikon.common_prefixes(target_set)
common_package = ''
if len(prefixes) == 1:
last_period_index = prefixes[0].rindex('.')
if last_period_index > 0:
# the common package should be at least one period away from the rest
common_package = prefixes[0][:last_period_index]
prefix_regexes = []
for p in prefixes:
prefix_regexes.append(p + "*")
instrument_regex = "|".join(prefix_regexes)
if SHOW_DEBUG_INFO:
print "\n===instrumentation pattern===\n" + instrument_regex + "\n"
# run tests with instrumentation
run_instrumented_tests = \
" ".join([java_cmd, "-ea",
"-javaagent:" + agent_path + "=\"" + instrument_regex + "\"",
junit_torun])
if SHOW_DEBUG_INFO:
print "\n=== Instrumented testing command to run: \n" + run_instrumented_tests
if not path.exists(dyng_go):
makedirs(dyng_go)
full_info_exfile = go + "_getty_binary_info_" + this_hash + "_.ex"
os.sys_call(run_instrumented_tests +
" > " + full_info_exfile +
("" if config.show_stack_trace_info else " 2> /dev/null"),
ignore_bad_exit=True)
full_method_info_map = {}
ext_start_index = len(config.method_info_line_prefix)
with open(full_info_exfile, 'r') as f:
contents = f.read().split("\n")
for line in contents:
line = line.strip()
if line.startswith(config.method_info_line_prefix):
rawdata = line[ext_start_index:]
k, v = rawdata.split(" : ")
full_method_info_map[k.strip()] = v.strip()
os.merge_dyn_files(dyng_go, go, "_getty_dyncg_-hash-_.ex", this_hash)
os.merge_dyn_files(dyng_go, go, "_getty_dynfg_-hash-_.ex", this_hash)
caller_of, callee_of = agency.caller_callee(go, this_hash)
pred_of, succ_of = agency.pred_succ(go, this_hash)
# add test methods into target set
test_set = agency.get_test_set_dyn(target_set, callee_of, junit_torun)
# reset target set here
refined_target_set, changed_methods, changed_tests = \
agency.refine_targets(full_method_info_map, target_set, test_set,
caller_of, callee_of, pred_of, succ_of,
changed_methods, changed_tests,
inner_dataflow_methods, outer_dataflow_methods)
profiler.log_csv(["method_count", "test_count", "refined_target_count"],
[[len(target_set), len(test_set), len(refined_target_set)]],
go + "_getty_y_method_count_" + this_hash + "_.profile.readable")
git.clear_temp_checkout(this_hash)
return common_package, test_set, refined_target_set, changed_methods, changed_tests, \
cp, junit_torun, full_method_info_map
def exam(iso, pwd, go, fe_path, common_package, all_classes_set, targets,
all_refined_target_set, new_refined_target_set,
old_refined_target_set, all_changed_methods, new_changed_methods,
old_changed_methods, new_modified_src, new_all_src, new_caller_of,
new_callee_of, new_pred_of, new_succ_of, old_caller_of, old_callee_of,
old_pred_of, old_succ_of, all_changed_tests, old_changed_tests,
new_changed_tests, old_test_set, new_test_set, prev_hash, post_hash,
old_l2m, new_l2m, old_m2l, new_m2l):
# TODO: NEEDS BETTER REFINED TARGET SETS
refined_targets_parents_set = all_refined_target_set | all_classes_set
print 'generating full diff html ...'
# get extra diff file for full diff of each changed file
full_src_diff_in = go + "full.text.diff"
full_src_diff_out = go + "src.diff.html"
sys_call(" ".join([
"git diff", "--unified={3}",
str(config.git_diff_extra_ops), "{0} {1} > {2}"
]).format(prev_hash, post_hash, full_src_diff_in, config.max_context_line))
# correct old_l2m and new_l2m
old_l2m = _l2m_correction(old_l2m, old_refined_target_set)
new_l2m = _l2m_correction(new_l2m, new_refined_target_set)
srcdiff2html(full_src_diff_in,
full_src_diff_out,
exclude_headers=None,
old_l2m=old_l2m,
new_l2m=new_l2m)
print 'generating main html from diff ...'
diff_in = go + "text.diff"
html_out = go + "sema.diff.html"
diff_to_html(diff_in,
html_out,
exclude_headers=False,
old_l2m=old_l2m,
new_l2m=new_l2m)
print 'appending semainfo to the html ....'
commit_msgs = git_commit_msgs(prev_hash, post_hash)
github_link = github_info(prev_hash, post_hash)
getty_append_semainfo(html_out, refined_targets_parents_set, go, fe_path,
commit_msgs, github_link, prev_hash, post_hash,
old_l2m, new_l2m, iso)
print 'initialize csi report ...'
getty_csi_init(html_out, iso)
print 'setting csi target variables ...'
getty_csi_targets_prep(html_out,
go,
prev_hash,
post_hash,
common_package,
all_changed_tests,
old_changed_tests,
new_changed_tests,
all_changed_methods,
new_modified_src,
new_all_src,
old_test_set,
new_test_set,
old_caller_of,
old_callee_of,
old_pred_of,
old_succ_of,
new_caller_of,
new_callee_of,
new_pred_of,
new_succ_of,
old_refined_target_set,
new_refined_target_set,
all_refined_target_set,
all_classes_set,
iso,
expansion_set=refined_targets_parents_set
) # TODO: which expanded set to use?
print 'csi report page is set.'
if config.review_auto_open:
print ' Opening rendered pages for review ...'
sys_call("open " + html_out)
else:
print ' CSI analysis completed - review at: ' + html_out
def seq_get_invs(target_set_index_pair, java_cmd, junit_torun, go, this_hash):
index = target_set_index_pair[1]
target_set = target_set_index_pair[0]
print "\n\t****\n" + " forked: " + index + "\n\t****\n"
# select_pattern = daikon.select_full(target_set)
select_pattern = daikon.dfformat_full_ordered(target_set)
print "\n===select pattern===\n" + select_pattern + "\n"
inv_gz = go + "_getty_inv_" + this_hash + "_." + index
if config.compress_inv:
inv_gz += ".inv.gz"
else:
inv_gz += ".inv"
daikon_control_opt_list = []
if SHOW_MORE_DEBUG_INFO:
daikon_control_opt_list.append("--show_progress --no_text_output")
elif SHOW_DEBUG_INFO:
daikon_control_opt_list.append("--no_show_progress --no_text_output")
else:
daikon_control_opt_list.append("--no_text_output")
if config.disable_known_invs:
daikon_control_opt_list.append("--disable-all-invariants")
if config.omit_redundant_invs:
daikon_control_opt_list.append("--omit_from_output 0r")
if config.daikon_format_only:
daikon_control_opt_list.append("--format Daikon")
daikon_control_opt_list.append(config.blocked_daikon_invs_exp)
daikon_display_args = " ".join(daikon_control_opt_list)
# run Chicory + Daikon (online) for invariants without trace I/O
run_chicory_daikon = " ".join(
[
java_cmd,
"daikon.Chicory --daikon-online --exception-handling",
'--daikon-args="' + daikon_display_args,
"-o",
inv_gz + '"',
'--ppt-select-pattern="' + select_pattern + '"',
junit_torun,
]
)
if SHOW_DEBUG_INFO:
print "\n=== Daikon:Chicory+Daikon(online) command to run: \n" + run_chicory_daikon
os.sys_call(run_chicory_daikon, ignore_bad_exit=True)
if SHOW_DEBUG_INFO:
current_count = 0
total_count = len(target_set)
for tgt in target_set:
target_ff = daikon.fsformat(tgt)
out_file = go + "_getty_inv__" + target_ff + "__" + this_hash + "_.inv.out"
run_printinv = " ".join(
[
java_cmd,
"daikon.PrintInvariants",
"--format",
config.output_inv_format,
"--ppt-select-pattern='" + daikon.dpformat(tgt) + "'",
"--output",
out_file,
inv_gz,
]
)
if SHOW_DEBUG_INFO:
current_count += 1
os.print_progress(
current_count,
total_count,
prefix="Progress(" + index + "):",
suffix="(" + str(current_count) + "/" + str(total_count) + ": " + tgt + ")" + " " * 20,
bar_length=50,
)
elif SHOW_MORE_DEBUG_INFO:
print "\n=== Daikon:PrintInvs command to run: \n" + run_printinv
os.sys_call(run_printinv, ignore_bad_exit=True)
sort_txt_inv(out_file)
os.remove_file(inv_gz)
def clear_temp_checkout(current_commit):
sys_call("git reset --hard " + current_commit)
resp = from_sys_call("git stash save --keep-index --include-untracked").strip()
if resp != "No local changes to save":
sys_call("git stash drop")
def one_inv_pass(go, cp, junit_torun, this_hash, refined_target_set, analysis_only=False):
if not analysis_only:
os.sys_call("git checkout " + this_hash)
os.sys_call("mvn clean")
if SHOW_DEBUG_INFO:
print "\n===full classpath===\n" + cp + "\n"
java_cmd = " ".join(
[
"java",
"-cp",
cp,
# "-Xms"+config.min_heap,
"-Xmx" + config.max_heap,
"-XX:+UseConcMarkSweepGC",
# "-XX:-UseGCOverheadLimit",
"-XX:-UseSplitVerifier", # FIXME: JDK 8- only!
]
)
# os.sys_call("mvn test -DskipTests", ignore_bad_exit=True)
os.sys_call("mvn test-compile")
if SHOW_DEBUG_INFO:
print "\n===junit torun===\n" + junit_torun + "\n"
# v3.2, v4 execute with 4 core
num_primary_workers = config.num_master_workers
auto_parallel_targets = config.auto_fork
slave_load = config.classes_per_fork
target_map = daikon.target_s2m(refined_target_set)
all_classes = target_map.keys()
if len(refined_target_set) <= num_primary_workers or (num_primary_workers == 1 and not auto_parallel_targets):
single_set_tuple = (refined_target_set, "0")
seq_get_invs(single_set_tuple, java_cmd, junit_torun, go, this_hash)
elif num_primary_workers > 1: # FIXME: this distributation is buggy
target_set_inputs = []
all_target_set_list = list(refined_target_set)
each_bulk_size = int(len(refined_target_set) / num_primary_workers)
seq_func = partial(seq_get_invs, java_cmd=java_cmd, junit_torun=junit_torun, go=go, this_hash=this_hash)
for i in range(num_primary_workers):
if not (i == num_primary_workers - 1):
sub_list_tuple = (all_target_set_list[each_bulk_size * i : each_bulk_size * (i + 1)], str(i))
target_set_inputs.append(sub_list_tuple)
else:
sub_list_tuple = (all_target_set_list[each_bulk_size * i :], str(i))
target_set_inputs.append(sub_list_tuple)
input_pool = Pool(num_primary_workers)
input_pool.map(seq_func, target_set_inputs)
input_pool.close()
input_pool.join()
elif num_primary_workers == 1 and auto_parallel_targets and slave_load >= 1:
# elastic automatic processing
target_set_inputs = []
num_processes = 0
# target_map has been calculated already
# target_map = daikon.target_s2m(refined_target_set)
# all_classes = target_map.keys()
num_keys = len(all_classes)
seq_func = partial(seq_get_invs, java_cmd=java_cmd, junit_torun=junit_torun, go=go, this_hash=this_hash)
for i in range(0, num_keys, slave_load):
# (inclusive) lower bound is i
# (exclusive) upper bound:
j = min(i + slave_load, num_keys)
sublist = []
for k in range(i, j):
the_key = all_classes[k]
sublist.append(the_key) # so it won't miss class/object invariants
sublist += target_map[the_key]
sublist_tuple = (sublist, str(num_processes))
target_set_inputs.append(sublist_tuple)
num_processes += 1
max_parallel_processes = config.num_slave_workers
if not analysis_only:
profiler.log_csv(
["class_count", "process_count", "max_parallel_processes", "slave_load"],
[[num_keys, num_processes, max_parallel_processes, slave_load]],
go + "_getty_y_elastic_count_" + this_hash + "_.profile.readable",
)
input_pool = Pool(max_parallel_processes)
input_pool.map(seq_func, target_set_inputs)
input_pool.close()
input_pool.join()
else:
print "\nIncorrect option for one center pass:"******"\tnum_primary_workers:", str(num_primary_workers)
print "\tauto_parallel_targets:", str(auto_parallel_targets)
print "\tslave_load", str(slave_load)
sys.exit(1)
if config.compress_inv:
os.remove_many_files(go, "*.inv.gz")
else:
os.remove_many_files(go, "*.inv")
# include coverage report for compare
if config.analyze_test_coverage and not analysis_only:
try:
mvn.generate_coverage_report(go, this_hash)
except:
pass
if not analysis_only:
git.clear_temp_checkout(this_hash)
return all_classes
def one_info_pass(
junit_path,
sys_classpath,
agent_path,
cust_mvn_repo,
dyng_go,
go,
this_hash,
target_set,
changed_methods,
changed_tests,
inner_dataflow_methods,
outer_dataflow_methods,
):
os.sys_call("git checkout " + this_hash)
os.sys_call("mvn clean")
bin_path = mvn.path_from_mvn_call("outputDirectory")
test_bin_path = mvn.path_from_mvn_call("testOutputDirectory")
cp = mvn.full_classpath(junit_path, sys_classpath, bin_path, test_bin_path)
if SHOW_DEBUG_INFO:
print "\n===full classpath===\n" + cp + "\n"
print "\ncopying all code to specific directory ...\n"
all_code_dirs = [
mvn.path_from_mvn_call("sourceDirectory"),
# mvn.path_from_mvn_call("scriptSourceDirectory"),
mvn.path_from_mvn_call("testSourceDirectory"),
]
getty_code_store = go + "_getty_allcode_" + this_hash + "_/"
print "copy to " + getty_code_store + "\n"
makedirs(getty_code_store)
for adir in all_code_dirs:
os.sys_call(" ".join(["cp -r", adir + "/*", getty_code_store]), ignore_bad_exit=True)
if config.use_special_junit_for_dyn:
info_junit_path = os.rreplace(junit_path, config.default_junit_version, config.special_junit_version, 1)
infocp = mvn.full_classpath(info_junit_path, sys_classpath, bin_path, test_bin_path)
else:
infocp = cp
java_cmd = " ".join(
[
"java",
"-cp",
infocp,
# "-Xms"+config.min_heap,
"-Xmx" + config.max_heap,
"-XX:+UseConcMarkSweepGC",
# "-XX:-UseGCOverheadLimit",
"-XX:-UseSplitVerifier", # FIXME: JDK 8- only!
]
)
# os.sys_call("mvn test -DskipTests", ignore_bad_exit=True)
os.sys_call("mvn test-compile")
junit_torun = mvn.junit_torun_str(cust_mvn_repo)
if SHOW_DEBUG_INFO:
print "\n===junit torun===\n" + junit_torun + "\n"
#### dynamic run one round for all information
prefixes = daikon.common_prefixes(target_set)
common_package = ""
if len(prefixes) == 1:
last_period_index = prefixes[0].rindex(".")
if last_period_index > 0:
# the common package should be at least one period away from the rest
common_package = prefixes[0][:last_period_index]
prefix_regexes = []
for p in prefixes:
prefix_regexes.append(p + "*")
instrument_regex = "|".join(prefix_regexes)
if SHOW_DEBUG_INFO:
print "\n===instrumentation pattern===\n" + instrument_regex + "\n"
# run tests with instrumentation
run_instrumented_tests = " ".join(
[java_cmd, "-javaagent:" + agent_path + '="' + instrument_regex + '"', junit_torun]
)
if SHOW_DEBUG_INFO:
print "\n=== Instrumented testing command to run: \n" + run_instrumented_tests
if not path.exists(dyng_go):
makedirs(dyng_go)
os.sys_call(run_instrumented_tests, ignore_bad_exit=True)
os.merge_dyn_files(dyng_go, go, "_getty_dyncg_-hash-_.ex", this_hash)
os.merge_dyn_files(dyng_go, go, "_getty_dynfg_-hash-_.ex", this_hash)
caller_of, callee_of = agency.caller_callee(go, this_hash)
pred_of, succ_of = agency.pred_succ(go, this_hash)
# add test methods into target set
test_set = agency.get_test_set_dyn(target_set, callee_of, junit_torun)
# set target set here
refined_target_set = agency.refine_targets(
target_set,
test_set,
caller_of,
callee_of,
pred_of,
succ_of,
changed_methods,
changed_tests,
inner_dataflow_methods,
outer_dataflow_methods,
)
profiler.log_csv(
["method_count", "test_count", "refined_target_count"],
[[len(target_set), len(test_set), len(refined_target_set)]],
go + "_getty_y_method_count_" + this_hash + "_.profile.readable",
)
git.clear_temp_checkout(this_hash)
return common_package, test_set, refined_target_set, cp, junit_torun
def seq_get_invs(target_set_index_pair, java_cmd, junit_torun, go, this_hash, consider_expansion):
index = target_set_index_pair[1]
target_set = target_set_index_pair[0]
print "\n\t****\n" + " forked: " + index + "\n\t****\n"
# select_pattern = daikon.select_full(target_set)
select_pattern = daikon.dfformat_full_ordered(target_set)
print "\n=== select pattern ===\n" + select_pattern + "\n"
inv_gz = go + "_getty_inv_" + this_hash + "_." + index
if config.compress_inv:
inv_gz += ".inv.gz"
else:
inv_gz += ".inv"
daikon_control_opt_list = []
if SHOW_MORE_DEBUG_INFO:
daikon_control_opt_list.append("--show_progress --no_text_output")
elif SHOW_DEBUG_INFO:
daikon_control_opt_list.append("--no_show_progress --no_text_output")
else:
daikon_control_opt_list.append("--no_text_output")
if config.disable_known_invs:
daikon_control_opt_list.append("--disable-all-invariants")
if config.omit_redundant_invs:
daikon_control_opt_list.append("--omit_from_output 0r")
if config.daikon_format_only:
daikon_control_opt_list.append("--format Daikon")
daikon_control_opt_list.append(config.blocked_daikon_invs_exp)
daikon_display_args = " ".join(daikon_control_opt_list)
# run Chicory + Daikon (online) for invariants without trace I/O
run_chicory_daikon = \
" ".join([java_cmd, "daikon.Chicory --daikon-online --exception-handling",
"--daikon-args=\""+daikon_display_args,
"-o", inv_gz+"\"",
"--ppt-select-pattern=\""+select_pattern+"\"",
junit_torun])
if SHOW_DEBUG_INFO:
print "\n=== Daikon:Chicory+Daikon(online) command to run: \n" + run_chicory_daikon
os.sys_call(run_chicory_daikon, ignore_bad_exit=True)
expansion = set()
if consider_expansion and config.class_level_expansion:
try:
all_methods_expansion(expansion, target_set, go, this_hash, index, java_cmd, inv_gz)
except:
pass
if SHOW_DEBUG_INFO:
current_count = 0
total_count = len(target_set)
all_to_consider = set(target_set)
if config.class_level_expansion:
all_to_consider = (all_to_consider | expansion)
for tgt in all_to_consider:
target_ff = daikon.fsformat_with_sigs(tgt)
out_file = go+"_getty_inv__"+target_ff+"__"+this_hash+"_.inv.out"
run_printinv = \
" ".join([java_cmd, "daikon.PrintInvariants",
"--format", config.output_inv_format,
"--ppt-select-pattern=\'"+daikon.dpformat_with_sigs(tgt)+"\'",
"--output", out_file, inv_gz])
if SHOW_DEBUG_INFO:
current_count += 1
if config.show_regex_debug:
print "\n\tthe regex for: " + tgt + "\n\t\t" + daikon.dpformat_with_sigs(tgt) + "\n"
os.print_progress(current_count, total_count,
prefix='Progress('+index+'):',
suffix='('+str(current_count)+'/'+str(total_count)+': '+tgt+')'+' '*20,
bar_length=50)
elif SHOW_MORE_DEBUG_INFO:
print "\n=== Daikon:PrintInvs command to run: \n" + run_printinv
os.sys_call(run_printinv, ignore_bad_exit=True)
sort_txt_inv(out_file)
os.remove_file(inv_gz)
def visit(villa_path, pwd, proj_dir, go, prev_hash, post_hash, pkg_prefix="-"):
print("\n****************************************************************");
print(" Getty Villa: Semantiful Differential Analyzer ");
print("****************************************************************\n");
print "current working directory: " + pwd + "\n"
diff_out = go + "text.diff"
os.sys_call("git diff {0} {1} > {2}".format(prev_hash, post_hash, diff_out))
'''
1-st pass: checkout prev_commit as detached head, and get all sets and etc, in simple (bare) mode (-s)
remember to clear after this pass
'''
bin_path, src_rel_path, test_src_rel_path = checkout_build(proj_dir, prev_hash)
run_villa = "java -jar {0} -s {1} {2} {3} {4} {5} {6} -o {7}".format(
villa_path, diff_out, bin_path, test_src_rel_path, pkg_prefix, prev_hash, post_hash, go)
run_villa_l4ms = "java -jar {0} -l {1} {2} {3} -o {4}".format(
villa_path, src_rel_path, test_src_rel_path, prev_hash, go)
print "\n\nstart to run Villa ... \n\n" + run_villa + "\n and \n" + run_villa_l4ms
chdir(proj_dir)
os.sys_call(run_villa)
os.sys_call(run_villa_l4ms)
chdir(pwd)
old_changed_methods = ex.read_str_from(go + "_getty_chgmtd_src_old_{0}_.ex".format(prev_hash))
old_all_methods = ex.read_str_from(go + "_getty_allmtd_src_{0}_.ex".format(prev_hash))
old_l2m = ex.read_str_from(go + "_getty_fl2m_{0}_.ex".format(prev_hash))
old_m2l = ex.read_str_from(go + "_getty_fm2l_{0}_.ex".format(prev_hash))
old_changed_tests = ex.read_str_from(go + "_getty_chgmtd_test_old_{0}_.ex".format(prev_hash))
# # DEBUG ONLY
# print old_changed_methods
# print len(old_all_methods)
# print old_l2m
# print old_m2l
# print old_changed_tests
git.clear_temp_checkout(prev_hash)
'''
2-nd pass: checkout post_commit as detached head, and get all sets and etc, in complex mode (-c)
remember to clear after this pass
'''
bin_path, src_rel_path, test_src_rel_path = checkout_build(proj_dir, post_hash)
run_villa = "java -jar {0} -c {1} {2} {3} {4} {5} {6} -o {7}".format(
villa_path, diff_out, bin_path, test_src_rel_path, pkg_prefix, prev_hash, post_hash, go)
run_villa_l4ms = "java -jar {0} -l {1} {2} {3} -o {4}".format(
villa_path, src_rel_path, test_src_rel_path, post_hash, go)
print "\n\nstart to run Villa ... \n\n" + run_villa + "\n and \n" + run_villa_l4ms
chdir(proj_dir)
os.sys_call(run_villa)
os.sys_call(run_villa_l4ms)
chdir(pwd)
new_changed_methods = ex.read_str_from(go + "_getty_chgmtd_src_new_{0}_.ex".format(post_hash))
new_improved_changed_methods = ex.read_str_from(go + "_getty_chgmtd_src_{0}_{1}_.ex".format(prev_hash, post_hash))
new_removed_changed_methods = ex.read_str_from(go + "_getty_chgmtd_src_gone_{0}_{1}_.ex".format(prev_hash, post_hash))
# TODO or FIXME
# new_all_ccc_related = ex.read_str_from(go + "_getty_cccmtd_{0}_.ex".format(post_hash)) # not needed for now
# new_all_cccs = ex.read_str_from(go + "_getty_ccc_{0}_.ex".format(post_hash)) # not needed for now
new_all_methods = ex.read_str_from(go + "_getty_allmtd_src_{0}_.ex".format(post_hash))
new_l2m = ex.read_str_from(go + "_getty_fl2m_{0}_.ex".format(post_hash))
new_m2l = ex.read_str_from(go + "_getty_fm2l_{0}_.ex".format(post_hash))
new_inner_dataflow_methods = ex.read_str_from(go + "_getty_dfinner_{0}_.ex".format(post_hash))
new_outer_dataflow_methods = ex.read_str_from(go + "_getty_dfouter_{0}_.ex".format(post_hash))
new_changed_tests = ex.read_str_from(go + "_getty_chgmtd_test_new_{0}_.ex".format(post_hash))
# # DEBUG ONLY
# print new_changed_methods
# print new_improved_changed_methods
# print new_removed_changed_methods
# print new_all_ccc_related
# print new_all_cccs
# print len(new_all_methods)
# print new_l2m
# print new_m2l
# print new_inner_dataflow_methods
# print new_outer_dataflow_methods
# print new_changed_tests
git.clear_temp_checkout(post_hash)
'''
3-rd pass: checkout prev_commit as detached head, and get all sets and etc, in recovery mode (-r)
remember to clear after this pass
'''
bin_path, src_rel_path, test_src_rel_path = checkout_build(proj_dir, prev_hash)
run_villa = "java -jar {0} -r {1} {2} {3} {4} {5} {6} -o {7}".format(
villa_path, diff_out, bin_path, test_src_rel_path, pkg_prefix, prev_hash, post_hash, go)
print "\n\nstart to run Villa ... \n\n" + run_villa
chdir(proj_dir)
os.sys_call(run_villa)
chdir(pwd)
old_improved_changed_methods = ex.read_str_from(go + "_getty_chgmtd_src_{1}_{0}_.ex".format(prev_hash, post_hash))
old_added_changed_methods = ex.read_str_from(go + "_getty_chgmtd_src_gain_{0}_{1}_.ex".format(prev_hash, post_hash))
# TODO or FIXME
# old_all_ccc_related = ex.read_str_from(go + "_getty_cccmtd_{0}_.ex".format(prev_hash)) # not needed for now
# old_all_cccs = ex.read_str_from(go + "_getty_ccc_{0}_.ex".format(prev_hash)) # not needed for now
old_inner_dataflow_methods = ex.read_str_from(go + "_getty_dfinner_{0}_.ex".format(prev_hash))
old_outer_dataflow_methods = ex.read_str_from(go + "_getty_dfouter_{0}_.ex".format(prev_hash))
# # DEBUG ONLY
# print old_changed_methods
# print old_improved_changed_methods
# print old_added_changed_methods
# print old_all_ccc_related
# print old_all_cccs
# print len(old_all_methods)
# print old_inner_dataflow_methods
# print old_outer_dataflow_methods
git.clear_temp_checkout(prev_hash)
print 'Villa analysis is completed.'
return old_changed_methods, old_improved_changed_methods, old_added_changed_methods, \
old_all_methods, \
old_inner_dataflow_methods, old_outer_dataflow_methods, \
old_l2m, old_m2l, \
new_changed_methods, new_improved_changed_methods, new_removed_changed_methods, \
new_all_methods, \
new_inner_dataflow_methods, new_outer_dataflow_methods, \
new_l2m, new_m2l, \
old_changed_tests, new_changed_tests
def seq_get_invs(target_set_index_pair, java_cmd, junit_torun, go, this_hash,
consider_expansion, test_selection):
index = target_set_index_pair[1]
target_set = target_set_index_pair[0]
# if test selection remove class from target set
if test_selection:
ttarget_set = set(target_set)
for s in ttarget_set:
if not s.__contains__(":"):
target_set.remove(s)
# select_pattern = daikon.select_full(target_set)
select_pattern = daikon.dfformat_full_ordered(target_set, test_selection)
print "\n=== select pattern ===\n" + select_pattern + "\n"
inv_gz = go + "_getty_inv_" + this_hash + "_." + index
if config.compress_inv:
inv_gz += ".inv.gz"
else:
inv_gz += ".inv"
daikon_control_opt_list = []
if SHOW_MORE_DEBUG_INFO:
daikon_control_opt_list.append("--show_progress --no_text_output")
elif SHOW_DEBUG_INFO:
daikon_control_opt_list.append("--no_show_progress --no_text_output")
else:
daikon_control_opt_list.append("--no_text_output")
if config.disable_known_invs:
daikon_control_opt_list.append("--disable-all-invariants")
if config.omit_redundant_invs:
daikon_control_opt_list.append("--omit_from_output 0r")
if config.daikon_format_only:
daikon_control_opt_list.append("--format Daikon")
daikon_control_opt_list.append(config.blocked_daikon_invs_exp)
daikon_display_args = " ".join(daikon_control_opt_list)
# run Chicory + Daikon (online) for invariants without trace I/O
run_chicory_daikon = \
" ".join([java_cmd, "daikon.Chicory --daikon-online --exception-handling",
"--daikon-args=\"" + daikon_display_args,
"-o", inv_gz + "\"",
"--ppt-select-pattern=\"" + select_pattern + "\"",
junit_torun])
if SHOW_DEBUG_INFO:
print "\n=== Daikon:Chicory+Daikon(online) command to run: \n" + run_chicory_daikon
os.sys_call(run_chicory_daikon,
ignore_bad_exit=True,
cwd=ProjectUtils.get_version_path(this_hash))
expansion = set()
if consider_expansion and config.class_level_expansion:
try:
all_methods_expansion(expansion, go, this_hash, index, java_cmd,
inv_gz)
except:
pass
if SHOW_DEBUG_INFO:
current_count = 0
total_count = len(target_set)
all_to_consider = set(target_set)
if config.class_level_expansion:
all_to_consider = (all_to_consider | expansion)
classes_to_consider = set()
for tgt in all_to_consider:
class_ref = tgt.split(':')[0]
classes_to_consider.add(class_ref)
print "==== classes to consider: ", classes_to_consider, " hash: " + this_hash
for tgt in classes_to_consider:
print "============ target is: " + tgt + ", pattern is: " + daikon.dpformat_with_sigs(
tgt) + " ==============="
target_ff = daikon.fsformat_with_sigs(tgt)
out_file = go + "_getty_inv__" + target_ff + "__" + this_hash + "_.inv.out"
# TODO: For some reason adding this optimization leads to different results
# if py_os.path.isfile(out_file):
# f = open(out_file, "r")
# f_invs = f.read()
# f.close()
# if f_invs == "<NO INVARIANTS INFERRED>\n":
# print "no invariants found, running daikon.PrintInvariants again for class", tgt
# else:
# # don't run daikon.PrintInvariants twice for the same class
# print "not running daikon.PrintInvariants again for class", tgt, f_invs
# continue
run_printinv = \
" ".join([java_cmd, "daikon.PrintInvariants",
"--format", config.output_inv_format,
"--ppt-select-pattern=\'" + daikon.dpformat_with_sigs(tgt)[:-1] + "[.:]" + "\'",
"--output", out_file, inv_gz])
if SHOW_DEBUG_INFO:
current_count += 1
if config.show_regex_debug:
print "\n\tthe regex for: " + tgt + "\n\t\t" + daikon.dpformat_with_sigs(
tgt) + "\n"
os.print_progress(current_count,
total_count,
prefix='Progress(' + index + '):',
suffix='(' + str(current_count) + '/' +
str(total_count) + ': ' + tgt + ')' + ' ' * 20,
bar_length=50)
elif SHOW_MORE_DEBUG_INFO:
print "\n=== Daikon:PrintInvs command to run: \n" + run_printinv
os.sys_call(run_printinv,
ignore_bad_exit=True,
cwd=ProjectUtils.get_version_path(this_hash))
sort_txt_inv(out_file)
result = create_inv_out_file_per_method(out_file, all_to_consider,
this_hash, go)
if result is False:
print "create_inv_out_file_per_method returned False"
os.remove_file(inv_gz)
def one_inv_pass(go, cp, junit_torun, this_hash, refined_target_set, analysis_only=False):
if not analysis_only:
os.sys_call("git checkout " + this_hash)
os.sys_call("mvn clean")
if SHOW_DEBUG_INFO:
print "\n===full classpath===\n" + cp + "\n"
java_cmd = " ".join(["java", "-cp", cp,
# "-Xms"+config.min_heap,
"-Xmx"+config.max_heap,
"-XX:+UseConcMarkSweepGC",
# "-XX:-UseGCOverheadLimit",
"-XX:-UseSplitVerifier", # FIXME: JDK 8- only!
])
# os.sys_call("mvn test -DskipTests", ignore_bad_exit=True)
os.sys_call("mvn test-compile")
if SHOW_DEBUG_INFO:
print "\n===junit torun===\n" + junit_torun + "\n"
# v3.2, v4 execute with 4 core
num_primary_workers = config.num_master_workers
auto_parallel_targets = config.auto_fork
slave_load = config.classes_per_fork
target_map = daikon.target_s2m(refined_target_set)
all_classes = target_map.keys()
consider_expansion = (not analysis_only)
if len(refined_target_set) <= num_primary_workers or (num_primary_workers == 1 and not auto_parallel_targets):
single_set_tuple = (refined_target_set, "0")
seq_get_invs(single_set_tuple, java_cmd, junit_torun, go, this_hash, consider_expansion)
elif num_primary_workers > 1: # FIXME: this distributation is buggy
target_set_inputs = []
all_target_set_list = list(refined_target_set)
each_bulk_size = int(len(refined_target_set) / num_primary_workers)
seq_func = partial(seq_get_invs,
java_cmd=java_cmd, junit_torun=junit_torun, go=go, this_hash=this_hash,
consider_expansion=consider_expansion)
for i in range(num_primary_workers):
if not(i == num_primary_workers - 1):
sub_list_tuple = (all_target_set_list[each_bulk_size*i:each_bulk_size*(i+1)], str(i))
target_set_inputs.append(sub_list_tuple)
else:
sub_list_tuple = (all_target_set_list[each_bulk_size*i:], str(i))
target_set_inputs.append(sub_list_tuple)
input_pool = Pool(num_primary_workers)
input_pool.map(seq_func, target_set_inputs)
input_pool.close()
input_pool.join()
elif num_primary_workers == 1 and auto_parallel_targets and slave_load >= 1:
# elastic automatic processing
target_set_inputs = []
num_processes = 0
# target_map has been calculated already
# target_map = daikon.target_s2m(refined_target_set)
# all_classes = target_map.keys()
num_keys = len(all_classes)
seq_func = partial(seq_get_invs,
java_cmd=java_cmd, junit_torun=junit_torun, go=go, this_hash=this_hash,
consider_expansion=consider_expansion)
for i in range(0, num_keys, slave_load):
# (inclusive) lower bound is i
# (exclusive) upper bound:
j = min(i+slave_load, num_keys)
sublist = []
for k in range(i, j):
the_key = all_classes[k]
sublist.append(the_key) # so it won't miss class/object invariants
sublist += target_map[the_key]
sublist_tuple = (sublist, str(num_processes))
target_set_inputs.append(sublist_tuple)
num_processes += 1
max_parallel_processes = config.num_slave_workers
if not analysis_only:
profiler.log_csv(["class_count", "process_count", "max_parallel_processes", "slave_load"],
[[num_keys, num_processes, max_parallel_processes, slave_load]],
go + "_getty_y_elastic_count_" + this_hash + "_.profile.readable")
input_pool = Pool(max_parallel_processes)
input_pool.map(seq_func, target_set_inputs)
input_pool.close()
input_pool.join()
else:
print "\nIncorrect option for one center pass:"******"\tnum_primary_workers:", str(num_primary_workers)
print "\tauto_parallel_targets:", str(auto_parallel_targets)
print "\tslave_load", str(slave_load)
sys.exit(1)
if config.compress_inv:
os.remove_many_files(go, "*.inv.gz")
else:
os.remove_many_files(go, "*.inv")
# include coverage report for compare
if config.analyze_test_coverage and not analysis_only:
try:
mvn.generate_coverage_report(go, this_hash)
except:
pass
if not analysis_only:
git.clear_temp_checkout(this_hash)
if config.class_level_expansion:
extra_expansion = get_expansion_set(go)
os.remove_many_files(go, config.expansion_tmp_files + "*")
else:
extra_expansion = None
return all_classes, extra_expansion
