def remove_duplicate_patches_parallel(patch_list):
global pool, result_list
result_list = []
emitter.normal("\tremoving redundancy in patch pool")
mp_lock = mp.Lock()
for patch in patch_list:
index = list(patch_list).index(patch)
# emitter.emit_patch(patch, message="\tabstract patch " + str(index) + " :")
result_list.append(oracle.is_patch_duplicate(patch, index, mp_lock))
# mp_lock = mp.Lock()
#
# if values.CONF_OPERATION_MODE in ["sequential"]:
# for patch in patch_list:
# index = list(patch_list).index(patch)
# # emitter.emit_patch(patch, message="\tabstract patch " + str(index) + " :")
# result_list.append(oracle.is_patch_duplicate(patch, index, mp_lock))
# else:
# emitter.normal("\t\tstarting parallel computing")
# pool = mp.Pool(mp.cpu_count())
#
# for patch in patch_list:
# index = list(patch_list).index(patch)
# # emitter.emit_patch(patch, message="\tabstract patch " + str(index) + " :")
# pool.apply_async(oracle.is_patch_duplicate, args=(patch, index, mp_lock), callback=collect_result)
# pool.close()
# emitter.normal("\t\twaiting for thread completion")
# pool.join()
return result_list
def extract_byte_code(binary_path):
emitter.normal("\textracting bytecode")
directory_path = "/".join(binary_path.split("/")[:-1])
binary_name = binary_path.split("/")[-1]
extract_command = "cd " + directory_path + ";"
extract_command += "extract-bc " + binary_name
utilities.execute_command(extract_command)
def build_project(project_path, build_command=None):
emitter.normal("\tbuilding program")
dir_command = "cd " + project_path + ";"
if build_command is None:
build_command = "CC=" + CC + " CXX=" + CXX + " "
if values.CONF_BUILD_FLAGS == "disable":
build_command += "bear make -j`nproc` "
else:
build_command += "bear make CFLAGS=\"" + C_FLAGS + "\" "
build_command += "CXXFLAGS=\"" + CXX_FLAGS + " LDFLAGS=" + LD_FLAGS + "\" -j`nproc` > "
else:
if build_command == "skip":
emitter.warning("\t[warning] skipping build")
return
if not os.path.isfile(project_path + "/compile_commands.json"):
build_command = build_command.replace("make ", "bear make ")
if CC == "wllvm":
build_command = remove_fsanitize(build_command)
build_command = apply_flags(build_command)
if not build_command:
error_exit("[Not Found] Build Command")
build_command = dir_command + build_command
build_command = build_command + " > " + definitions.FILE_MAKE_LOG
ret_code = execute_command(build_command)
if int(ret_code) != 0:
emitter.error(build_command)
error_exit("BUILD FAILED!!\nExit Code: " + str(ret_code))
def rank_patches(patch_list):
filtered_list = []
# rank first based on coverage
emitter.normal("\tcomputing rank for each patch")
for patch in patch_list:
patch_formula = app.generator.generate_formula_from_patch(patch)
patch_constraint_str = patch_formula.serialize()
patch_code_str = ""
for (lid, prog) in patch.items():
patch_code_str = lid + ": " + (program_to_code(prog))
for comp_var, prog_var in values.MAP_PROG_VAR.items():
patch_code_str = patch_code_str.replace(comp_var, prog_var)
patch_index = utilities.get_hash(patch_constraint_str)
patch_score = values.LIST_PATCH_SCORE[patch_index]
over_approx_score = 10
if values.LIST_PATCH_OVERAPPROX_CHECK[patch_index]:
over_approx_score = 0
under_approx_score = 10
if values.LIST_PATCH_UNDERAPPROX_CHECK[patch_index]:
under_approx_score = 0
patch_len = 10000 - len(patch_constraint_str)
# if oracle.is_always_true(patch) or oracle.is_always_false(patch):
# patch_len = 10000 - 1
patch_count = 1000 - utilities.count_concrete_patches_per_template(
patch)
filtered_list.append((patch, under_approx_score, over_approx_score,
patch_score, patch_count, patch_len))
ranked_list = sorted(filtered_list, key=operator.itemgetter(3, 1, 2, 4, 5))
ranked_list.reverse()
patch_list = numpy.array(ranked_list)[:, 0]
return list(patch_list)
def collect_symbolic_path_prefix(log_path, project_path):
"""
This function will read the output log of a klee concolic execution and
extract the prefix of partial path condition that should be omitted in path generation
"""
emitter.normal("\textracting prefix of path condition")
prefix_ppc = ""
if os.path.exists(log_path):
source_path = ""
path_condition = ""
with open(log_path, 'r') as trace_file:
for line in trace_file:
if '[path:ppc]' in line:
if project_path in line:
break
else:
source_path = str(line.replace("[path:ppc]", '')).split(" : ")[0]
source_path = source_path.strip()
source_path = os.path.abspath(source_path)
path_condition = str(line.replace("[path:ppc]", '')).split(" : ")[1]
continue
if source_path:
if "(exit)" not in line:
path_condition = path_condition + line
else:
prefix_ppc = path_condition
source_path = ""
path_condition = ""
return prefix_ppc
def collect_symbolic_path(log_path, project_path):
"""
This function will read the output log of a klee concolic execution and
extract the partial path conditions
"""
emitter.normal("\textracting path conditions")
ppc_list = list()
last_sym_path = ""
if os.path.exists(log_path):
source_path = ""
path_condition = ""
with open(log_path, 'r') as trace_file:
for line in trace_file:
if '[path:ppc]' in line:
if project_path in line or definitions.DIRECTORY_LIB in line:
source_path = str(line.replace("[path:ppc]", '')).split(" : ")[0]
source_path = source_path.strip()
source_path = os.path.abspath(source_path)
path_condition = str(line.replace("[path:ppc]", '')).split(" : ")[1]
continue
if source_path:
if "(exit)" not in line:
path_condition = path_condition + line
else:
ppc_list.append((source_path, path_condition))
last_sym_path = path_condition
source_path = ""
path_condition = ""
# constraints['last-sym-path'] = last_sym_path
# print(constraints.keys())
parser = SmtLibParser()
script = parser.get_script(cStringIO(last_sym_path))
formula = script.get_last_formula()
return ppc_list, formula
def generate_assertion(assertion_temp, klee_dir):
emitter.normal("\tgenerating extended specification")
largest_path_condition = None
max_obs = 0
file_list = [f for f in os.listdir(klee_dir) if os.path.isfile(os.path.join(klee_dir, f))]
for file_name in file_list:
if ".smt2" in file_name:
file_path = os.path.join(klee_dir, file_name)
path_condition = extractor.extract_formula_from_file(file_path)
model = generate_model(path_condition)
var_list = list(model.keys())
count_obs = 0
declaration_line = assertion_temp[0]
specification_line = assertion_temp[1]
for var in var_list:
if "obs!" in var:
count_obs = count_obs + 1
if count_obs == 0:
continue
if max_obs < count_obs:
max_obs = count_obs
largest_path_condition = path_condition
declaration_line = assertion_temp[0]
specification_line = assertion_temp[1]
assertion_text = ""
for index in range(0, max_obs):
assertion_text = assertion_text + declaration_line.replace("obs!0", "obs!" + str(index))
assertion_text = assertion_text + specification_line.replace("obs!0", "obs!" + str(index))
specification_formula = generate_formula(assertion_text)
return specification_formula, max_obs
def validate_input_generation(patch_list, new_path):
global pool, result_list, found_one
result_list = []
if values.DEFAULT_OPERATION_MODE in ["sequential"]:
for patch in patch_list:
patch_formula = app.generator.generate_formula_from_patch(patch)
patch_formula_extended = generator.generate_extended_patch_formula(
patch_formula, new_path)
patch_space_constraint = patch_formula_extended
if values.DEFAULT_PATCH_TYPE == values.OPTIONS_PATCH_TYPE[1]:
patch_formula_str = str(patch_formula.serialize())
patch_index = utilities.get_hash(patch_formula_str)
patch_space = values.LIST_PATCH_SPACE[patch_index]
parameter_constraint = smt2.generate_constraint_for_patch_space(
patch_space)
if parameter_constraint:
patch_space_constraint = And(patch_formula_extended,
parameter_constraint)
index = list(patch_list).index(patch)
# emitter.emit_patch(patch, message="\tabstract patch " + str(index) + " :")
result_list.append(
oracle.check_input_feasibility(index, patch_space_constraint,
new_path))
else:
emitter.normal("\t\tstarting parallel computing")
pool = mp.Pool(mp.cpu_count(), initializer=mute)
lock = None
thread_list = []
interrupt_event = threading.Event()
for patch in patch_list:
try:
patch_formula = app.generator.generate_formula_from_patch(
patch)
patch_formula_extended = generator.generate_extended_patch_formula(
patch_formula, new_path)
patch_space_constraint = patch_formula
if values.DEFAULT_PATCH_TYPE == values.OPTIONS_PATCH_TYPE[1]:
patch_formula_str = str(patch_formula.serialize())
patch_index = utilities.get_hash(patch_formula_str)
patch_space = values.LIST_PATCH_SPACE[patch_index]
parameter_constraint = smt2.generate_constraint_for_patch_space(
patch_space)
if parameter_constraint:
patch_space_constraint = And(patch_formula_extended,
parameter_constraint)
index = list(patch_list).index(patch)
# emitter.emit_patch(patch, message="\tabstract patch " + str(index) + " :")
thread = pool.apply_async(oracle.check_input_feasibility,
args=(index, patch_space_constraint,
new_path),
callback=collect_result_one)
thread_list.append(thread)
except ValueError:
emitter.warning("\t\tvalue found before completing pool")
break
pool.close()
emitter.normal("\t\twaiting for thread completion")
pool.join()
return result_list
def collect_seed_list():
emitter.normal("reading seed information")
if values.CONF_SEED_SUITE_CONFIG:
for (seed_id, bin_path, test_input) in values.CONF_SEED_SUITE_CONFIG:
if values.CONF_SEED_SUITE_ID_LIST:
if str(seed_id) not in values.CONF_SEED_SUITE_ID_LIST:
continue
values.LIST_SEED_ID_LIST.add(str(seed_id))
bin_path = values.CONF_DIR_SRC + "/" + bin_path
values.LIST_SEED_BINARY.append(bin_path)
values.LIST_SEED_INPUT.append((seed_id, test_input))
else:
if values.CONF_SEED_BINARY_LIST:
for binary_path in values.CONF_SEED_BINARY_LIST:
binary_path = values.CONF_DIR_SRC + "/" + binary_path
values.LIST_SEED_BINARY.append(binary_path)
if values.CONF_SEED_LIST:
for seed_input in values.CONF_SEED_LIST:
values.LIST_SEED_INPUT.append(seed_input)
elif values.CONF_SEED_FILE:
with open(values.CONF_SEED_FILE, "r") as in_file:
content_lines = in_file.readlines()
for content in content_lines:
values.LIST_SEED_INPUT.append(content.strip().replace(
"\n", ""))
if values.CONF_SEED_DIR:
seed_dir = values.CONF_SEED_DIR
file_list = [
f for f in os.listdir(seed_dir)
if os.path.isfile(os.path.join(seed_dir, f))
]
for seed_file in file_list:
seed_file_index = seed_file
seed_abs_path = seed_dir + "/" + seed_file
values.LIST_SEED_FILES[seed_file_index] = seed_abs_path
if values.LIST_SEED_INPUT:
seed_index = 0
for (seed_id, seed_arg_list_str) in values.LIST_SEED_INPUT:
arg_list = extract_input_arg_list(seed_arg_list_str)
concretized_arg_list = []
for arg in arg_list:
if "$POC_" in arg:
file_index = "_".join(str(arg).split("_")[1:])
if file_index not in values.LIST_SEED_FILES:
continue
file_path = values.LIST_SEED_FILES[file_index]
concretized_arg_list.append(file_path)
elif "$POC" in arg:
file_index = list(values.LIST_TEST_FILES.keys())[0]
file_path = values.LIST_TEST_FILES[file_index]
concretized_arg_list.append(file_path)
else:
concretized_arg_list.append(arg)
concretized_arg_str = ",".join(concretized_arg_list)
values.LIST_TEST_INPUT.append(concretized_arg_str)
values.LIST_TEST_BINARY.append(values.LIST_SEED_BINARY[seed_index])
seed_index = seed_index + 1
def clean_project(project_path, binary_path):
emitter.normal("\tcleaning files")
binary_dir_path = "/".join(str(binary_path).split("/")[:-1])
if values.CONF_COMMAND_BUILD != "skip":
clean_command = "cd " + project_path
clean_command += "; make clean"
clean_command += "; rm compile_commands.json"
clean_command += "; rm CMakeCache.txt"
clean_command += "; rm -rf CMakeFiles"
execute_command(clean_command)
clean_residues = "cd " + binary_dir_path + ";" + "rm -rf ./patches/*;" + "rm -rf ./klee*"
execute_command(clean_residues)
def run_concrete_execution(program,
argument_list,
print_output=False,
output_dir=None):
"""
This function will execute the program in concrete mode using the concrete inputs
program: the absolute path of the bitcode of the program
argument_list : a list containing each argument in the order that should be fed to the program
second_var_list: a list of tuples where a tuple is (var identifier, var size, var value)
"""
logger.info("running concolic execution")
emitter.normal("\texecuting klee in concrete mode")
global File_Log_Path
current_dir = os.getcwd()
directory_path = "/".join(str(program).split("/")[:-1])
emitter.debug("changing directory:" + directory_path)
os.chdir(directory_path)
binary_name = str(program).split("/")[-1]
input_argument = ""
runtime_lib_path = definitions.DIRECTORY_LIB + "/libtrident_runtime.bca"
for argument in argument_list:
if "$POC" in argument:
argument = values.FILE_POC_GEN
# if "_" in argument:
# file_index = "_".join(str(argument).split("_")[1:])
# argument = values.LIST_TEST_FILES[file_index]
# else:
# argument = values.CONF_PATH_POC
# if values.FILE_POC_GEN:
# argument = values.FILE_POC_GEN
input_argument += " " + str(argument)
if output_dir:
klee_command = "klee --output-dir=" + str(output_dir) + " "
else:
klee_command = "klee "
klee_command += "--posix-runtime " \
"--libc=uclibc " \
"--search=dfs " \
"--write-smt2s " \
"--external-calls=all " \
"--max-forks {0} ".format(values.DEFAULT_MAX_FORK) \
+ values.CONF_KLEE_FLAGS + " " \
+ "--link-llvm-lib={0} ".format(runtime_lib_path) \
+ "{0} ".format(binary_name) \
+ input_argument
if not print_output:
klee_command += " > " + File_Log_Path + " 2>&1 "
return_code = utilities.execute_command(klee_command)
emitter.debug("changing directory:" + current_dir)
os.chdir(current_dir)
return return_code
def update_distance_map():
emitter.normal("\tupdating distance matrix")
latest_dist_map = generate_distance_map()
for loc in latest_dist_map:
if loc in values.MAP_LOC_DISTANCE:
if values.MAP_LOC_DISTANCE[loc] > latest_dist_map[loc]:
values.MAP_LOC_DISTANCE[loc] = latest_dist_map[loc]
else:
values.MAP_LOC_DISTANCE[loc] = latest_dist_map[loc]
values.MAP_LOC_DISTANCE = {
k: v
for k, v in sorted(values.MAP_LOC_DISTANCE.items(),
key=lambda item: item[1])
}
def collect_var_mapping():
emitter.normal("updating mapping for program variables")
patch_loc = values.CONF_LOC_PATCH
source_file_path, line_number = patch_loc.split(":")
with open(source_file_path, 'r') as source_file:
content = source_file.readlines()
patch_line = content[int(line_number) - 1]
trident_call_str = re.findall("trident_choice\((.+?)\)[\),\s,;]",
patch_line)[0]
patch_type = trident_call_str.split(",")[1]
if patch_type == "bool":
values.IS_PATCH_BOOL = True
prog_var_list = re.findall("{(.+?)}", trident_call_str)[0].split(",")
comp_name_list = re.findall("{(.+?)}", trident_call_str)[1].split(",")
for i in range(0, len(prog_var_list)):
values.MAP_PROG_VAR[comp_name_list[i].strip().replace(
"\"", "").replace("\'", "")] = prog_var_list[i]
def collect_symbolic_path_loc(log_path, project_path):
"""
This function will read the output log of a klee concolic execution and
extract the partial path condition insert locations (i.e. control location)
"""
emitter.normal("\textracting path conditions")
ppc_loc_list = list()
if os.path.exists(log_path):
with open(log_path, 'r') as trace_file:
for line in trace_file:
if '[path:ppc]' in line:
if project_path in line or definitions.DIRECTORY_LIB in line:
source_path = str(line.replace("[path:ppc]", '')).split(" : ")[0]
source_path = source_path.strip()
source_path = os.path.abspath(source_path)
ppc_loc_list.append(source_path)
return ppc_loc_list
def generate_model_cli(formula):
"""
This function will invoke the Z3 Cli interface to solve the provided formula and return the model byte list
Arguments:
formula: smtlib formatted formula
"""
emitter.normal("\textracting z3 model")
path_script = "/tmp/z3_script_model_cli"
path_result = "/tmp/z3_output_model_cli"
write_smtlib(formula, path_script)
with open(path_script, "a") as script_file:
script_file.writelines(["(get-model)\n", "(exit)\n"])
z3_command = "z3 " + path_script + " > " + path_result
utilities.execute_command(z3_command)
with open(path_result, "r") as result_file:
z3_output = result_file.readlines()
model_byte_list = parser.parse_z3_output(z3_output)
return model_byte_list
def generate_symbolic_paths(ppc_list, arg_list, poc_path):
"""
This function will analyse the partial path conditions collected at each branch location and isolate
the branch conditions added at each location, negate the constraint to create a new path
ppc_list : a dictionary containing the partial path condition at each branch location
returns a list of new partial path conditions
"""
emitter.normal("\tgenerating new paths")
path_list = []
if values.DEFAULT_GEN_SPECIAL_PATH:
path_list = generate_special_paths(ppc_list, arg_list, poc_path)
path_count = len(path_list)
result_list = generate_flipped_paths(ppc_list)
for result in result_list:
path_count = path_count + 1
path_list.append((result, arg_list, poc_path))
emitter.highlight("\t\tgenerated " + str(path_count) + " flipped path(s)")
return path_list
def check_infeasible_paths(patch_list):
global list_path_inprogress, list_path_infeasible, list_path_detected
emitter.sub_title("Evaluating Path Pool")
emitter.normal("\tcomputing infeasibility on remaining paths")
count = 0
for path in list_path_inprogress:
count = count + 1
emitter.sub_sub_title("Path #" + str(count))
control_loc, generated_path, ppc_len, reach_patch_loc, reach_obs_loc, _, _ = path
feasible_patch_list = select_patch_constraint_for_input(
patch_list, generated_path)
if not feasible_patch_list:
list_path_infeasible.append(path)
list_path_inprogress.remove(path)
emitter.highlight("\ttotal discovered: " + str(len(list_path_detected)) +
" path(s)")
emitter.highlight("\ttotal remaining: " + str(len(list_path_inprogress)) +
" path(s)")
emitter.highlight("\ttotal infeasible: " + str(len(list_path_infeasible)) +
" path(s)")
def extract_largest_path_condition(dir_path):
largest_path_condition = None
pc_formula_len = 0
emitter.normal("\textracting largest symbolic path")
file_list = [
f for f in os.listdir(dir_path)
if os.path.isfile(os.path.join(dir_path, f))
]
for file_name in file_list:
if ".smt2" in file_name:
file_path = os.path.join(dir_path, file_name)
path_condition = extract_formula_from_file(file_path)
if ".err" in file_name:
largest_path_condition = path_condition
break
pc_formula_str = str(path_condition.serialize())
if len(pc_formula_str) > pc_formula_len:
pc_formula_len = len(pc_formula_str)
largest_path_condition = path_condition
return largest_path_condition
def generate_ppc_from_formula(path_condition):
ppc_list = list()
emitter.normal("\textracting branches from path condition")
max_count = 2 * values.DEFAULT_MAX_FLIPPINGS
while path_condition.is_and():
constraint = path_condition.arg(1)
constraint_str = str(constraint.serialize())
if "!rvalue!" in constraint_str or "!obs!" in constraint_str:
path_condition = path_condition.arg(0)
continue
path_script = "/tmp/z3_script_ppc"
write_smtlib(path_condition, path_script)
with open(path_script, "r") as script_file:
script_lines = script_file.readlines()
script = "".join(script_lines)
ppc_list.append(("-no-info-", script))
path_condition = path_condition.arg(0)
if len(ppc_list) > max_count:
emitter.warning("\t[warning] maximum cap reach for branching")
break
return ppc_list
def generate_ktest(argument_list, second_var_list, print_output=False):
"""
This function will generate the ktest file provided the argument list and second order variable list
argument_list : a list containing each argument in the order that should be fed to the program
second_var_list: a list of tuples where a tuple is (var identifier, var size, var value)
"""
global File_Ktest_Path
emitter.normal("\tgenerating ktest file")
ktest_path = File_Ktest_Path
ktest_command = "gen-bout --out-file {0}".format(ktest_path)
for argument in argument_list:
index = list(argument_list).index(argument)
if "$POC" in argument:
binary_file_path = values.FILE_POC_GEN
# if "_" in argument:
# file_index = "_".join(str(argument).split("_")[1:])
# binary_file_path = values.LIST_TEST_FILES[file_index]
# else:
# binary_file_path = values.CONF_PATH_POC
# if values.FILE_POC_GEN:
# binary_file_path = values.FILE_POC_GEN
# elif values.FILE_POC_SEED:
# binary_file_path = values.FILE_POC_SEED
ktest_command += " --sym-file " + binary_file_path
elif str(index) in values.CONF_MASK_ARG:
continue
else:
if argument in ["''"]:
argument = ""
if "\"" in argument:
ktest_command += " --sym-arg '" + str(argument) + "'"
continue
ktest_command += " --sym-arg \"" + str(argument) + "\""
for var in second_var_list:
ktest_command += " --second-var \'{0}\' {1} {2}".format(
var['identifier'], var['size'], var['value'])
return_code = utilities.execute_command(ktest_command)
return ktest_path, return_code
def collect_trace(file_path, project_path):
"""
This function will read the output log of a klee concolic execution and
extract the instruction trace
"""
emitter.normal("\textracting instruction trace")
list_trace = list()
if os.path.exists(file_path):
with open(file_path, 'r') as trace_file:
for line in trace_file:
if '[klee:trace]' in line:
if project_path in line:
trace_line = str(line.replace("[klee:trace] ", ''))
trace_line = trace_line.strip()
source_path, line_number = trace_line.split(":")
source_path = os.path.abspath(source_path)
trace_line = source_path + ":" + str(line_number)
if (not list_trace) or (list_trace[-1] != trace_line):
list_trace.append(trace_line)
if values.CONF_LOC_PATCH:
if values.CONF_LOC_PATCH in list_trace:
emitter.note("\t\t[note] patch location detected in trace")
values.COUNT_HIT_PATCH_LOC = values.COUNT_HIT_PATCH_LOC + 1
if values.CONF_LOC_BUG:
if values.CONF_LOC_BUG in list_trace:
emitter.note("\t\t[note] fault location detected in trace")
values.COUNT_HIT_BUG_LOG = values.COUNT_HIT_BUG_LOG + 1
if values.CONF_LOC_LIST_CRASH:
if not set(values.CONF_LOC_LIST_CRASH).isdisjoint(list_trace):
emitter.note("\t\t[note] a crash location detected in trace")
values.COUNT_HIT_CRASH_LOC = values.COUNT_HIT_CRASH_LOC + 1
is_crash = collect_crash_point(values.FILE_MESSAGE_LOG)
if is_crash:
values.IS_CRASH = True
values.COUNT_HIT_CRASH = values.COUNT_HIT_CRASH + 1
emitter.note("\t\t[note] program crashed")
else:
values.IS_CRASH = False
emitter.note("\t\t[note] program did not crash")
return list_trace
def write_patch_set(patch_list, output_file_path):
emitter.normal("\twriting patch list to file")
template_count = 0
txt_lines = []
for patch in patch_list:
template_count = template_count + 1
txt_lines.append("Patch #" + str(template_count))
for (lid, prog) in patch.items():
code = lid + ": " + (program_to_code(prog))
for comp_var, prog_var in values.MAP_PROG_VAR.items():
code = code.replace(comp_var, prog_var)
txt_lines.append(code)
patch_formula = generator.generate_formula_from_patch(patch)
patch_formula_str = patch_formula.serialize()
patch_index = utilities.get_hash(patch_formula_str)
patch_score = values.LIST_PATCH_SCORE[patch_index]
concrete_patch_count = 1
if values.DEFAULT_PATCH_TYPE == values.OPTIONS_PATCH_TYPE[1]:
patch_space = values.LIST_PATCH_SPACE[patch_index]
partition_count = 0
for partition in patch_space:
partition_count = partition_count + 1
txt_lines.append("\t\tPartition: " + str(partition_count))
for constant_name in partition:
txt_lines.append("\t\t\tConstant: " + constant_name)
constant_info = partition[constant_name]
lower_bound = str(constant_info['lower-bound'])
upper_bound = str(constant_info['upper-bound'])
txt_lines.append("\t\t\tRange: " + lower_bound + " <= " + constant_name + " <= " + upper_bound)
dimension = len(range(int(lower_bound), int(upper_bound) + 1))
txt_lines.append("\t\t\tDimension: " + str(dimension))
concrete_patch_count = utilities.count_concrete_patches_per_template(patch)
txt_lines.append("\t\tPatch Count: " + str(concrete_patch_count))
txt_lines.append("\t\tPath Coverage: " + str(patch_score))
txt_lines.append("\t\tIs Under-approximating: " + str(values.LIST_PATCH_UNDERAPPROX_CHECK[patch_index]))
txt_lines.append("\t\tIs Over-approximating: " + str(values.LIST_PATCH_OVERAPPROX_CHECK[patch_index]))
with open(output_file_path, 'w') as out_file:
out_file.writelines(line + "\n" for line in txt_lines)
def load_component_list():
emitter.normal("loading custom/general components")
# base_list = ["equal.smt2", "not-equal.smt2", "less-than.smt2", "less-or-equal.smt2"]
base_list = []
if definitions.DIRECTORY_TESTS in values.CONF_PATH_PROJECT:
base_list = []
gen_comp_files = []
os.chdir(definitions.DIRECTORY_COMPONENTS)
if values.CONF_GENERAL_COMP_LIST and not values.CONF_ALL_COMPS:
comp_list = list(set(values.CONF_GENERAL_COMP_LIST + base_list))
for component_name in comp_list:
gen_comp_files.append(Path(component_name))
emitter.note("\tloading component: " + str(component_name))
else:
component_file_list = os.listdir(definitions.DIRECTORY_COMPONENTS)
for comp_file in component_file_list:
if ".smt2" in comp_file:
if any(x in comp_file for x in [
"logical-not", "post-decrement", "post-increment",
"minus", "constant", "assignment", "sequence",
"greater", "remainder"
]):
continue
gen_comp_files.append(Path(comp_file))
emitter.note("\tloading component: " + str(comp_file))
gen_comp_files = list(set(gen_comp_files))
general_components = synthesis.load_components(gen_comp_files)
proj_comp_files = []
os.chdir(values.CONF_PATH_PROJECT)
for component_name in values.CONF_CUSTOM_COMP_LIST:
proj_comp_files.append(Path(component_name))
emitter.note("\tloading component: " + str(component_name))
project_components = synthesis.load_components(proj_comp_files)
values.LIST_COMPONENTS = project_components + general_components
values.COUNT_COMPONENTS = len(values.LIST_COMPONENTS)
values.COUNT_COMPONENTS_CUS = len(project_components)
values.COUNT_COMPONENTS_GEN = len(general_components)
def collect_seed_list():
emitter.normal("reading seed information")
if values.CONF_SEED_LIST:
for seed_input in values.CONF_SEED_LIST:
values.LIST_SEED_INPUT.append(seed_input)
if values.CONF_SEED_FILE:
with open(values.CONF_SEED_FILE, "r") as in_file:
content_lines = in_file.readlines()
for content in content_lines:
values.LIST_SEED_INPUT.append(content.strip().replace(
"\n", ""))
if values.CONF_SEED_DIR:
seed_dir = values.CONF_SEED_DIR
file_list = [
f for f in os.listdir(seed_dir)
if os.path.isfile(os.path.join(seed_dir, f))
]
for seed_file in file_list:
seed_abs_path = seed_dir + "/" + seed_file
values.LIST_SEED_FILES.append(seed_abs_path)
if values.LIST_SEED_INPUT:
for seed_arg_list_str in values.LIST_SEED_INPUT:
arg_list = extract_input_arg_list(seed_arg_list_str)
concretized_arg_list = []
for arg in arg_list:
if "$POC_" in arg:
file_index = "_".join(str(arg).split("_")[1:])
file_path = values.LIST_TEST_FILES[file_index]
concretized_arg_list.append(file_path)
elif "$POC" in arg:
file_index = list(values.LIST_TEST_FILES.keys())[0]
file_path = values.LIST_TEST_FILES[file_index]
concretized_arg_list.append(file_path)
else:
concretized_arg_list.append(arg)
concretized_arg_str = ",".join(concretized_arg_list)
values.LIST_TEST_INPUT.append(concretized_arg_str)
def build_normal():
global CC, CXX, CXX_FLAGS, C_FLAGS, LD_FLAGS
emitter.sub_title("Building Program")
emitter.normal("\tsetting environment variables")
execute_command("export TRIDENT_CC=" + definitions.DIRECTORY_TOOLS +
"/trident-cc")
execute_command("export TRIDENT_CXX=" + definitions.DIRECTORY_TOOLS +
"/trident-cxx")
clean_project(values.CONF_DIR_SRC, values.CONF_PATH_PROGRAM)
CC = "$TRIDENT_CC"
CXX = "$TRIDENT_CXX"
C_FLAGS = "-g -O0"
CXX_FLAGS = "-g -O0"
config_project(values.CONF_DIR_SRC, False, values.CONF_COMMAND_CONFIG)
C_FLAGS = ""
LD_FLAGS = ""
CXX_FLAGS = C_FLAGS
if values.CONF_STATIC:
C_FLAGS += " -static"
CXX_FLAGS += " -static"
build_project(values.CONF_DIR_SRC, values.CONF_COMMAND_BUILD)
def generate_special_paths(ppc_list, arg_list, poc_path, bin_path):
global pool, result_list, expected_count
result_list = []
path_list = []
filtered_list = []
lock = None
count = 0
expected_count = len(ppc_list)
ppc_list.reverse()
if values.DEFAULT_OPERATION_MODE in ["sequential", "semi-parallel"]:
for con_loc, ppc_str in ppc_list[:values.DEFAULT_MAX_FLIPPINGS]:
if count == values.DEFAULT_GEN_SEARCH_LIMIT:
break
count = count + 1
result_list.append(
generator.generate_special_paths(con_loc, ppc_str))
else:
emitter.normal("\t\tstarting parallel computing")
pool = mp.Pool(mp.cpu_count(), initializer=mute)
for con_loc, ppc_str in ppc_list[:values.DEFAULT_MAX_FLIPPINGS]:
if count == values.DEFAULT_GEN_SEARCH_LIMIT:
break
count = count + 1
pool.apply_async(generator.generate_special_paths,
args=(con_loc, ppc_str),
callback=collect_result)
pool.close()
emitter.normal("\t\twaiting for thread completion")
pool.join()
# assert(len(result_list) == len(path_list))
for path_list in result_list:
for path in path_list:
con_loc, path_smt, path_str = path
filtered_list.append(
((con_loc, path_smt, path_str), arg_list, poc_path, bin_path))
return filtered_list
def build_verify(project_path):
global CC, CXX, CXX_FLAGS, C_FLAGS, LD_FLAGS
emitter.sub_sub_title("building projects")
CC = "clang-7"
CXX = "clang++-7"
CXX_FLAGS = "'-g -O0 -static -DNDEBUG'"
C_FLAGS = "'-g -O0 -static -DNDEBUG'"
emitter.normal("\t\t" + project_path)
clean_project(project_path)
if values.CONF_COMMAND_CONFIGURATATION:
config_project(project_path, False,
values.CONF_COMMAND_CONFIGURATATION)
else:
config_project(project_path, False)
if values.CONF_COMMAND_BUILD:
CXX_FLAGS = "'-g -O0 -static -DNDEBUG -fsanitize=" + values.CONF_FLAG_ASAN + "'"
C_FLAGS = "'-g -O0 -static -DNDEBUG -fsanitize=" + values.CONF_FLAG_ASAN + "'"
build_project(project_path, values.CONF_COMMAND_BUILD)
else:
CXX_FLAGS = "'-g -O0 -static -DNDEBUG -fsanitize=" + values.CONF_FLAG_ASAN + "'"
C_FLAGS = "'-g -O0 -static -DNDEBUG -fsanitize=" + values.CONF_FLAG_ASAN + "'"
build_project(project_path)
def generate_path_for_negation():
constraint_list = []
parser = SmtLibParser()
emitter.normal("\tgenerating path for negation of patch constraint")
for control_loc, sym_path in values.LIST_PPC:
if control_loc == values.CONF_LOC_PATCH:
script = parser.get_script(cStringIO(sym_path))
formula = script.get_last_formula()
patch_constraint = formula
if formula.is_and():
patch_constraint = formula.arg(1)
constraint_list.append(patch_constraint.serialize())
if not constraint_list:
return None
last_sym_path = values.LAST_PPC_FORMULA
# script = parser.get_script(cStringIO(last_sym_path))
# formula = script.get_last_formula()
negated_path = None
while constraint_list:
constraint = last_sym_path
if last_sym_path.is_and():
constraint = last_sym_path.arg(1)
constraint_str = constraint.serialize()
if constraint_str in constraint_list:
constraint_list.remove(constraint_str)
constraint = Not(constraint)
if negated_path is None:
negated_path = constraint
else:
negated_path = And(negated_path, constraint)
if last_sym_path.is_and():
last_sym_path = last_sym_path.arg(0)
else:
break
negated_path = And(negated_path, last_sym_path)
return negated_path
def partition_input_space(path_condition, assertion):
global pool, result_list
result_list = []
is_exist = And(path_condition, Not(assertion))
is_always = And(path_condition, assertion)
input_space = generator.generate_input_space(path_condition)
if oracle.is_loc_in_trace(values.CONF_LOC_BUG):
if is_sat(is_exist):
emitter.normal("\tpartitioning input space")
partition_model = generator.generate_model(is_exist)
partition_model, is_multi_dimension = extractor.extract_input_list(
partition_model)
partition_list = generator.generate_partition_for_input_space(
partition_model, input_space, is_multi_dimension)
if values.DEFAULT_OPERATION_MODE in ["sequential"]:
for partition in partition_list:
# emitter.emit_patch(patch, message="\tabstract patch: ")
result_list.append(
refine.refine_input_partition(path_condition,
assertion, partition,
is_multi_dimension))
else:
emitter.normal("\t\tstarting parallel computing")
pool = mp.Pool(mp.cpu_count(), initializer=mute)
for partition in partition_list:
pool.apply_async(refine.refine_input_partition,
args=(path_condition, assertion,
partition, is_multi_dimension),
callback=collect_result)
pool.close()
emitter.normal("\t\twaiting for thread completion")
pool.join()
filtered_list = list()
for partition in result_list:
if not partition:
continue
if isinstance(partition, list):
for sub_partition in partition:
filtered_list.append(sub_partition)
else:
filtered_list.append(partition)
result_list = filtered_list
return result_list
def refine_patch_space(patch_list,
path_condition,
assertion,
force_sequential=False):
global pool, result_list
result_list = []
emitter.normal("\tupdating patch pool")
if values.DEFAULT_OPERATION_MODE in ["sequential"] or force_sequential:
for patch in patch_list:
index = list(patch_list).index(patch)
patch_formula = app.generator.generate_formula_from_patch(patch)
patch_formula_extended = generator.generate_extended_patch_formula(
patch_formula, path_condition)
# emitter.emit_patch(patch, message="\tabstract patch " + str(index) + " :")
patch_formula_str = patch_formula.serialize()
patch_index = utilities.get_hash(patch_formula_str)
patch_space = values.LIST_PATCH_SPACE[patch_index]
result_list.append(
refine.refine_patch(assertion, patch_formula_extended,
path_condition, index, patch_space))
else:
emitter.normal("\t\tstarting parallel computing")
pool = mp.Pool(mp.cpu_count(), initializer=mute)
for patch in patch_list:
index = list(patch_list).index(patch)
patch_formula = app.generator.generate_formula_from_patch(patch)
patch_formula_extended = generator.generate_extended_patch_formula(
patch_formula, path_condition)
# emitter.emit_patch(patch, message="\tabstract patch " + str(index) + " :")
patch_formula_str = patch_formula.serialize()
patch_index = utilities.get_hash(patch_formula_str)
patch_space = values.LIST_PATCH_SPACE[patch_index]
pool.apply_async(refine.refine_patch,
args=(assertion, patch_formula_extended,
path_condition, index, patch_space),
callback=collect_result)
pool.close()
emitter.normal("\t\twaiting for thread completion")
pool.join()
return result_list
