def detect_native_arch():
check_isa_file = os.path.abspath(common.yarpgen_scripts + os.sep +
check_isa_file_name)
check_isa_binary = os.path.abspath(common.yarpgen_scripts + os.sep +
check_isa_file_name.replace(".cpp", ""))
sys_compiler = ""
for key in CompilerSpecs.all_comp_specs:
exec_name = CompilerSpecs.all_comp_specs[key].comp_cxx_name
if common.if_exec_exist(exec_name):
sys_compiler = exec_name
break
if sys_compiler == "":
common.print_and_exit("Can't find any compiler")
if not common.if_exec_exist(check_isa_binary):
if not os.path.exists(check_isa_file):
common.print_and_exit("Can't find " + check_isa_file)
ret_code, output, err_output, time_expired, elapsed_time = \
common.run_cmd([sys_compiler, check_isa_file, "-o", check_isa_binary], None)
if ret_code != 0:
common.print_and_exit("Can't compile " + check_isa_file + ": " +
str(err_output, "utf-8"))
ret_code, output, err_output, time_expired, elapsed_time = common.run_cmd(
[check_isa_binary], None)
if ret_code != 0:
common.print_and_exit("Error while executing " + check_isa_binary)
native_arch_str = str(output, "utf-8").split()[0]
for sde_target in SdeTarget.all_sde_targets:
if sde_target.name == native_arch_str:
return sde_target
common.print_and_exit("Can't detect system ISA")
def prepare_env_and_recheck(input_dir, out_dir, target, num_jobs, config_file):
if not common.check_if_dir_exists(input_dir):
common.print_and_exit("Can't use input directory")
common.check_dir_and_create(out_dir)
run_gen.gen_test_makefile_and_copy(out_dir, config_file)
run_gen.dump_testing_sets(target)
run_gen.print_compilers_version(target)
lock = multiprocessing.Lock()
task_queue = multiprocessing.JoinableQueue()
process_dir(input_dir, task_queue)
failed_queue = multiprocessing.SimpleQueue()
passed_queue = multiprocessing.SimpleQueue()
task_threads = [0] * num_jobs
for num in range(num_jobs):
task_threads[num] = \
multiprocessing.Process(target=recheck,
args=(num, lock, task_queue, failed_queue, passed_queue, target, out_dir))
task_threads[num].start()
task_queue.join()
task_queue.close()
for num in range(num_jobs):
task_threads[num].join()
def add_specs(spec_list):
spec_list = check_config_list(spec_list, spec_list_len, "Error in spec string, check it: ")
try:
CompilerSpecs(spec_list[0], spec_list[1], spec_list[2], spec_list[3], spec_list[4])
common.log_msg(logging.DEBUG, "Finished adding compiler spec")
except KeyError:
common.print_and_exit("Can't find key!")
def add_stats_options(stats_opt_list):
stats_opt_list = check_config_list(stats_opt_list, stats_capt_opt_list_len,
"Error in stats options string, check it: ")
try:
StatisticsOptions(CompilerSpecs.all_comp_specs[stats_opt_list[0]], stats_opt_list[1])
common.log_msg(logging.DEBUG, "Finished adding stats option string")
except KeyError:
common.print_and_exit("Can't find key!")
def add_sets(set_list):
set_list = check_config_list(set_list, set_list_len, "Error in set string, check it: ")
try:
CompilerTarget(set_list[0], CompilerSpecs.all_comp_specs[set_list[1]], set_list[2],
Arch(set_list[3], SdeArch[set_list[4]]))
common.log_msg(logging.DEBUG, "Finished adding testing set")
except KeyError:
common.print_and_exit("Can't find key!")
def parse_config(file_name):
# Before parsing, clean old data
CompilerSpecs.all_comp_specs = dict()
CompilerTarget.all_targets = []
# Parse
config_file = common.check_and_open_file(file_name, "r")
config = config_file.read().splitlines()
config_file.close()
if not any(s.startswith(comp_specs_line) for s in config) or not any(s.startswith(test_sets_line) for s in config):
common.print_and_exit("Invalid config file! Check it!")
config_iter = iter(config)
for config_line in config_iter:
if skip_line(config_line):
continue
if config_line.startswith(comp_specs_line):
read_compiler_specs(config_iter, add_specs, test_sets_line)
read_compiler_specs(config_iter, add_sets, stats_capt_opt_line)
read_compiler_specs(config_iter, add_stats_options)
def prepare_and_start(work_dir, config_file, timeout, num_jobs,
csmith_bin_path, csmith_runtime, csmith_args, optsets):
common.check_dir_and_create(work_dir)
# Check for binary of generator
csmith_home = os.environ.get("CSMITH_HOME")
if csmith_home is None:
common.print_and_exit("Please set CSMITH_HOME envirnoment variable")
csmith_bin = os.path.abspath(csmith_home + os.sep + csmith_bin_path +
os.sep + "csmith")
common.check_and_copy(csmith_bin, work_dir)
os.chdir(work_dir)
ret_code, output, err_output, time_expired, elapsed_time = \
common.run_cmd(["." + os.sep + "csmith", "-v"], csmith_timeout, 0)
csmith_version_str = str(output, "utf-8")
# TODO: need to add some check, but I hope that it is safe
common.log_msg(logging.DEBUG, "Csmith version: " + csmith_version_str)
gen_test_makefile.set_standard("c99")
gen_test_makefile.parse_config(config_file)
compiler_run_args = {}
optsets_list = optsets.split()
target_was_found = False
failed_targets = []
for i in optsets_list:
for target in gen_test_makefile.CompilerTarget.all_targets:
if target.name != i:
continue
target_was_found = True
common.log_msg(logging.DEBUG,
"Trying to form compiler args for " + str(i))
run_str = target.specs.comp_c_name + " "
run_str += target.args + " "
if target.arch.comp_name != "":
run_str += " " + target.specs.arch_prefix + target.arch.comp_name + " "
run_str += gen_test_makefile.StatisticsOptions.get_options(
target.specs) + " "
run_str += csmith_runtime + " "
common.log_msg(logging.DEBUG, "Formed compiler args: " + run_str)
compiler_run_args[i] = run_str
if not target_was_found:
failed_targets.append(i)
target_was_found = False
if len(failed_targets) > 0:
common.log_msg(logging.WARNING,
"Can't find relevant target: " + str(failed_targets))
for i in range(num_jobs):
common.check_dir_and_create(run_gen.process_dir + str(i))
manager_obj = run_gen.manager()
stat = manager_obj.Statistics()
start_time = time.time()
end_time = start_time + timeout * 60
if timeout == -1:
end_time = -1
task_threads = [0] * num_jobs
for num in range(num_jobs):
task_threads[num] = multiprocessing.Process(target=run_csmith,
args=(num, csmith_args,
compiler_run_args,
end_time, stat))
task_threads[num].start()
print_statistics(stat, task_threads, num_jobs)
sys.stdout.write("\n")
for i in range(num_jobs):
common.log_msg(logging.DEBUG,
"Removing " + run_gen.process_dir + str(i) + " dir")
shutil.rmtree(run_gen.process_dir + str(i))
stat_str, verbose_stat_str, prev_len = form_statistics(stat, 0)
sys.stdout.write(verbose_stat_str)
sys.stdout.flush()
def gen_makefile(out_file_name,
force,
config_file,
only_target=None,
inject_blame_opt=None,
creduce_file=None,
stat_targets=None):
# Somebody can prepare test specs and target, so we don't need to parse config file
check_if_std_defined()
if config_file is not None:
parse_config(config_file)
output = ""
if stat_targets is not None:
stat_targets = list(set(stat_targets))
# 1. License
license_file = common.check_and_open_file(
os.path.abspath(common.yarpgen_scripts + os.sep + ".." + os.sep +
license_file_name), "r")
for license_str in license_file:
output += "#" + license_str
license_file.close()
output += "###############################################################################\n"
output += "#This file was generated automatically.\n"
output += "#If you want to make a permanent changes, you should edit gen_test_makefile.py\n"
output += "###############################################################################\n\n"
# 2. Define common variables
for makefile_variable in Makefile_variable_list:
test_pwd = ""
if creduce_file and makefile_variable.name == "CXXFLAGS":
test_pwd = " -I$(TEST_PWD)"
output += makefile_variable.name + "=" + makefile_variable.value + test_pwd + "\n"
output += "\n"
# 3. Define build targets
for target in CompilerTarget.all_targets:
if only_target is not None and only_target.name != target.name:
continue
compiler_name = None
if selected_standard.is_c():
compiler_name = target.specs.comp_c_name
if selected_standard.is_cxx():
compiler_name = target.specs.comp_cxx_name
output += target.name + ": " + "COMPILER=" + compiler_name + "\n"
optflags_str = target.name + ": " + "OPTFLAGS=" + target.args
if target.arch.comp_name != "":
optflags_str += " " + target.specs.arch_prefix + target.arch.comp_name
optflags_str += "\n"
output += optflags_str
# For performance reasons driver should always be compiled with -O0
output += re.sub("-O\d", "-O0",
(optflags_str.replace("OPTFLAGS", "DRIVER_OPTFLAGS")))
if inject_blame_opt is not None:
output += target.name + ": " + "BLAMEOPTS=" + inject_blame_opt + "\n"
#TODO: one day we can decide to use gcc also.
if stat_targets is not None:
for stat_target in stat_targets:
if target.name == stat_target:
output += target.name + ": " + stat_options.name + "=" + \
StatisticsOptions.get_options(target.specs) + "\n"
stat_targets.remove(stat_target)
output += target.name + ": " + "EXECUTABLE=" + target.name + "_" + executable.value + "\n"
output += target.name + ": " + "$(addprefix " + target.name + "_,"
if common.selected_gen_std != common.GenStdID.ISPC:
output += "$(SOURCES:" + get_file_ext() + "=.o))\n"
else:
output += "$(patsubst %.ispc,%.o," + "$(SOURCES:" + get_file_ext(
) + "=.o))" + ")\n"
output += "\t" + "$(COMPILER) $(LDFLAGS) $(STDFLAGS) $(OPTFLAGS) -o $(EXECUTABLE) $^\n\n"
if stat_targets is not None and len(stat_targets) != 0:
common.log_msg(logging.WARNING,
"Can't find relevant stat_targets: " +
str(stat_targets),
forced_duplication=True)
# 4. Force make to rebuild everything
# TODO: replace with PHONY
output += "FORCE:\n\n"
for source in sources.value.split():
source_prefix = ""
force_str = " FORCE\n"
if creduce_file and creduce_file != source:
source_prefix = "$(TEST_PWD)/"
force_str = "\n"
source_name = source.split(".")[0]
output += "%" + source_name + ".o: " + source_prefix + source + force_str
# For performance reasons driver should always be compiled with -O0
optflags_name = "$(OPTFLAGS)" if source_name != "driver" else "$(DRIVER_OPTFLAGS)"
output += "\t" + "$(COMPILER) $(CXXFLAGS) $(STDFLAGS) " + optflags_name + " -o $@ -c $<"
if source_name == "func":
output += " $(STATFLAGS) "
if inject_blame_opt is not None:
output += " $(BLAMEOPTS) "
output += "\n\n"
output += "clean:\n"
output += "\trm *.o *_$(EXECUTABLE)\n\n"
# 5. Define run targets
native_arch = detect_native_arch()
for target in CompilerTarget.all_targets:
if only_target is not None and only_target.name != target.name:
continue
output += "run_" + target.name + ": " + target.name + "_" + executable.value + "\n"
output += "\t@"
required_sde_arch = define_sde_arch(native_arch, target.arch.sde_arch)
if required_sde_arch != "":
output += "sde -" + required_sde_arch + " -- "
output += "." + os.sep + target.name + "_" + executable.value + "\n\n"
out_file = None
if not os.path.isfile(out_file_name):
out_file = open(out_file_name, "w")
else:
if force:
out_file = open(out_file_name, "w")
else:
common.print_and_exit(
"File already exists. Use -f if you want to rewrite it.")
out_file.write(output)
out_file.close()
def check_config_list(config_list, fixed_len, message):
common.log_msg(logging.DEBUG, "Adding config list: " + str(config_list))
if len(config_list) < fixed_len:
common.print_and_exit(message + str(config_list))
config_list = [x.strip() for x in config_list]
return config_list
def get_options(spec):
try:
return StatisticsOptions.all_stats_options[spec.name].options
except KeyError:
common.print_and_exit("Can't find key!")
def check_if_std_defined():
if selected_standard is None or \
selected_standard == StdID.MAX_C_ID or \
selected_standard == StdID.MAX_CXX_ID:
common.print_and_exit("Language standard wasn't selected!")
