def detect_native_arch(): check_isa_file = os.path.abspath(common.yarpgen_scripts + os.sep + check_isa_file_name) check_isa_binary = os.path.abspath(common.yarpgen_scripts + os.sep + check_isa_file_name.replace(".cpp", "")) sys_compiler = "" for key in CompilerSpecs.all_comp_specs: exec_name = CompilerSpecs.all_comp_specs[key].comp_cxx_name if common.if_exec_exist(exec_name): sys_compiler = exec_name break if sys_compiler == "": common.print_and_exit("Can't find any compiler") if not common.if_exec_exist(check_isa_binary): if not os.path.exists(check_isa_file): common.print_and_exit("Can't find " + check_isa_file) ret_code, output, err_output, time_expired, elapsed_time = \ common.run_cmd([sys_compiler, check_isa_file, "-o", check_isa_binary], None) if ret_code != 0: common.print_and_exit("Can't compile " + check_isa_file + ": " + str(err_output, "utf-8")) ret_code, output, err_output, time_expired, elapsed_time = common.run_cmd( [check_isa_binary], None) if ret_code != 0: common.print_and_exit("Error while executing " + check_isa_binary) native_arch_str = str(output, "utf-8").split()[0] for sde_target in SdeTarget.all_sde_targets: if sde_target.name == native_arch_str: return sde_target common.print_and_exit("Can't detect system ISA")
def prepare_env_and_recheck(input_dir, out_dir, target, num_jobs, config_file): if not common.check_if_dir_exists(input_dir): common.print_and_exit("Can't use input directory") common.check_dir_and_create(out_dir) run_gen.gen_test_makefile_and_copy(out_dir, config_file) run_gen.dump_testing_sets(target) run_gen.print_compilers_version(target) lock = multiprocessing.Lock() task_queue = multiprocessing.JoinableQueue() process_dir(input_dir, task_queue) failed_queue = multiprocessing.SimpleQueue() passed_queue = multiprocessing.SimpleQueue() task_threads = [0] * num_jobs for num in range(num_jobs): task_threads[num] = \ multiprocessing.Process(target=recheck, args=(num, lock, task_queue, failed_queue, passed_queue, target, out_dir)) task_threads[num].start() task_queue.join() task_queue.close() for num in range(num_jobs): task_threads[num].join()
def add_specs(spec_list): spec_list = check_config_list(spec_list, spec_list_len, "Error in spec string, check it: ") try: CompilerSpecs(spec_list[0], spec_list[1], spec_list[2], spec_list[3], spec_list[4]) common.log_msg(logging.DEBUG, "Finished adding compiler spec") except KeyError: common.print_and_exit("Can't find key!")
def add_stats_options(stats_opt_list): stats_opt_list = check_config_list(stats_opt_list, stats_capt_opt_list_len, "Error in stats options string, check it: ") try: StatisticsOptions(CompilerSpecs.all_comp_specs[stats_opt_list[0]], stats_opt_list[1]) common.log_msg(logging.DEBUG, "Finished adding stats option string") except KeyError: common.print_and_exit("Can't find key!")
def add_sets(set_list): set_list = check_config_list(set_list, set_list_len, "Error in set string, check it: ") try: CompilerTarget(set_list[0], CompilerSpecs.all_comp_specs[set_list[1]], set_list[2], Arch(set_list[3], SdeArch[set_list[4]])) common.log_msg(logging.DEBUG, "Finished adding testing set") except KeyError: common.print_and_exit("Can't find key!")
def parse_config(file_name): # Before parsing, clean old data CompilerSpecs.all_comp_specs = dict() CompilerTarget.all_targets = [] # Parse config_file = common.check_and_open_file(file_name, "r") config = config_file.read().splitlines() config_file.close() if not any(s.startswith(comp_specs_line) for s in config) or not any(s.startswith(test_sets_line) for s in config): common.print_and_exit("Invalid config file! Check it!") config_iter = iter(config) for config_line in config_iter: if skip_line(config_line): continue if config_line.startswith(comp_specs_line): read_compiler_specs(config_iter, add_specs, test_sets_line) read_compiler_specs(config_iter, add_sets, stats_capt_opt_line) read_compiler_specs(config_iter, add_stats_options)
def prepare_and_start(work_dir, config_file, timeout, num_jobs, csmith_bin_path, csmith_runtime, csmith_args, optsets): common.check_dir_and_create(work_dir) # Check for binary of generator csmith_home = os.environ.get("CSMITH_HOME") if csmith_home is None: common.print_and_exit("Please set CSMITH_HOME envirnoment variable") csmith_bin = os.path.abspath(csmith_home + os.sep + csmith_bin_path + os.sep + "csmith") common.check_and_copy(csmith_bin, work_dir) os.chdir(work_dir) ret_code, output, err_output, time_expired, elapsed_time = \ common.run_cmd(["." + os.sep + "csmith", "-v"], csmith_timeout, 0) csmith_version_str = str(output, "utf-8") # TODO: need to add some check, but I hope that it is safe common.log_msg(logging.DEBUG, "Csmith version: " + csmith_version_str) gen_test_makefile.set_standard("c99") gen_test_makefile.parse_config(config_file) compiler_run_args = {} optsets_list = optsets.split() target_was_found = False failed_targets = [] for i in optsets_list: for target in gen_test_makefile.CompilerTarget.all_targets: if target.name != i: continue target_was_found = True common.log_msg(logging.DEBUG, "Trying to form compiler args for " + str(i)) run_str = target.specs.comp_c_name + " " run_str += target.args + " " if target.arch.comp_name != "": run_str += " " + target.specs.arch_prefix + target.arch.comp_name + " " run_str += gen_test_makefile.StatisticsOptions.get_options( target.specs) + " " run_str += csmith_runtime + " " common.log_msg(logging.DEBUG, "Formed compiler args: " + run_str) compiler_run_args[i] = run_str if not target_was_found: failed_targets.append(i) target_was_found = False if len(failed_targets) > 0: common.log_msg(logging.WARNING, "Can't find relevant target: " + str(failed_targets)) for i in range(num_jobs): common.check_dir_and_create(run_gen.process_dir + str(i)) manager_obj = run_gen.manager() stat = manager_obj.Statistics() start_time = time.time() end_time = start_time + timeout * 60 if timeout == -1: end_time = -1 task_threads = [0] * num_jobs for num in range(num_jobs): task_threads[num] = multiprocessing.Process(target=run_csmith, args=(num, csmith_args, compiler_run_args, end_time, stat)) task_threads[num].start() print_statistics(stat, task_threads, num_jobs) sys.stdout.write("\n") for i in range(num_jobs): common.log_msg(logging.DEBUG, "Removing " + run_gen.process_dir + str(i) + " dir") shutil.rmtree(run_gen.process_dir + str(i)) stat_str, verbose_stat_str, prev_len = form_statistics(stat, 0) sys.stdout.write(verbose_stat_str) sys.stdout.flush()
def gen_makefile(out_file_name, force, config_file, only_target=None, inject_blame_opt=None, creduce_file=None, stat_targets=None): # Somebody can prepare test specs and target, so we don't need to parse config file check_if_std_defined() if config_file is not None: parse_config(config_file) output = "" if stat_targets is not None: stat_targets = list(set(stat_targets)) # 1. License license_file = common.check_and_open_file( os.path.abspath(common.yarpgen_scripts + os.sep + ".." + os.sep + license_file_name), "r") for license_str in license_file: output += "#" + license_str license_file.close() output += "###############################################################################\n" output += "#This file was generated automatically.\n" output += "#If you want to make a permanent changes, you should edit gen_test_makefile.py\n" output += "###############################################################################\n\n" # 2. Define common variables for makefile_variable in Makefile_variable_list: test_pwd = "" if creduce_file and makefile_variable.name == "CXXFLAGS": test_pwd = " -I$(TEST_PWD)" output += makefile_variable.name + "=" + makefile_variable.value + test_pwd + "\n" output += "\n" # 3. Define build targets for target in CompilerTarget.all_targets: if only_target is not None and only_target.name != target.name: continue compiler_name = None if selected_standard.is_c(): compiler_name = target.specs.comp_c_name if selected_standard.is_cxx(): compiler_name = target.specs.comp_cxx_name output += target.name + ": " + "COMPILER=" + compiler_name + "\n" optflags_str = target.name + ": " + "OPTFLAGS=" + target.args if target.arch.comp_name != "": optflags_str += " " + target.specs.arch_prefix + target.arch.comp_name optflags_str += "\n" output += optflags_str # For performance reasons driver should always be compiled with -O0 output += re.sub("-O\d", "-O0", (optflags_str.replace("OPTFLAGS", "DRIVER_OPTFLAGS"))) if inject_blame_opt is not None: output += target.name + ": " + "BLAMEOPTS=" + inject_blame_opt + "\n" #TODO: one day we can decide to use gcc also. if stat_targets is not None: for stat_target in stat_targets: if target.name == stat_target: output += target.name + ": " + stat_options.name + "=" + \ StatisticsOptions.get_options(target.specs) + "\n" stat_targets.remove(stat_target) output += target.name + ": " + "EXECUTABLE=" + target.name + "_" + executable.value + "\n" output += target.name + ": " + "$(addprefix " + target.name + "_," if common.selected_gen_std != common.GenStdID.ISPC: output += "$(SOURCES:" + get_file_ext() + "=.o))\n" else: output += "$(patsubst %.ispc,%.o," + "$(SOURCES:" + get_file_ext( ) + "=.o))" + ")\n" output += "\t" + "$(COMPILER) $(LDFLAGS) $(STDFLAGS) $(OPTFLAGS) -o $(EXECUTABLE) $^\n\n" if stat_targets is not None and len(stat_targets) != 0: common.log_msg(logging.WARNING, "Can't find relevant stat_targets: " + str(stat_targets), forced_duplication=True) # 4. Force make to rebuild everything # TODO: replace with PHONY output += "FORCE:\n\n" for source in sources.value.split(): source_prefix = "" force_str = " FORCE\n" if creduce_file and creduce_file != source: source_prefix = "$(TEST_PWD)/" force_str = "\n" source_name = source.split(".")[0] output += "%" + source_name + ".o: " + source_prefix + source + force_str # For performance reasons driver should always be compiled with -O0 optflags_name = "$(OPTFLAGS)" if source_name != "driver" else "$(DRIVER_OPTFLAGS)" output += "\t" + "$(COMPILER) $(CXXFLAGS) $(STDFLAGS) " + optflags_name + " -o $@ -c $<" if source_name == "func": output += " $(STATFLAGS) " if inject_blame_opt is not None: output += " $(BLAMEOPTS) " output += "\n\n" output += "clean:\n" output += "\trm *.o *_$(EXECUTABLE)\n\n" # 5. Define run targets native_arch = detect_native_arch() for target in CompilerTarget.all_targets: if only_target is not None and only_target.name != target.name: continue output += "run_" + target.name + ": " + target.name + "_" + executable.value + "\n" output += "\t@" required_sde_arch = define_sde_arch(native_arch, target.arch.sde_arch) if required_sde_arch != "": output += "sde -" + required_sde_arch + " -- " output += "." + os.sep + target.name + "_" + executable.value + "\n\n" out_file = None if not os.path.isfile(out_file_name): out_file = open(out_file_name, "w") else: if force: out_file = open(out_file_name, "w") else: common.print_and_exit( "File already exists. Use -f if you want to rewrite it.") out_file.write(output) out_file.close()
def check_config_list(config_list, fixed_len, message): common.log_msg(logging.DEBUG, "Adding config list: " + str(config_list)) if len(config_list) < fixed_len: common.print_and_exit(message + str(config_list)) config_list = [x.strip() for x in config_list] return config_list
def get_options(spec): try: return StatisticsOptions.all_stats_options[spec.name].options except KeyError: common.print_and_exit("Can't find key!")
def check_if_std_defined(): if selected_standard is None or \ selected_standard == StdID.MAX_C_ID or \ selected_standard == StdID.MAX_CXX_ID: common.print_and_exit("Language standard wasn't selected!")