def get_process_output(self, file, repetition): # generate file names for output files. curr_stdout_file_name = replace_placeholders( self.config.stdout_file_name, file=file, repetition=repetition, config_name=self.config_name) curr_stderr_file_name = replace_placeholders( self.config.stderr_file_name, file=file, repetition=repetition, config_name=self.config_name) if self.config.print_output: stdout_output = None stderr_output = None else: stdout_output = subprocess.DEVNULL stderr_output = subprocess.DEVNULL try: # generate files / folders, if not existing if curr_stdout_file_name: if not os.path.exists(os.path.dirname(curr_stdout_file_name)): os.makedirs(os.path.dirname(curr_stdout_file_name)) stdout_output = open(curr_stdout_file_name, "w+") if curr_stderr_file_name: if not os.path.exists(os.path.dirname(curr_stderr_file_name)): os.makedirs(os.path.dirname(curr_stderr_file_name)) stderr_output = open(curr_stderr_file_name, "w+") except IOError as err: print("Unable to open output file. Aborting.") print(err) exit(-1) return stdout_output, stderr_output
def get_process_output(self, file, repetition): # generate file names for output files. curr_stdout_file_name = replace_placeholders(self.config.stdout_file_name, file=file, repetition=repetition, config_name=self.config_name) curr_stderr_file_name = replace_placeholders(self.config.stderr_file_name, file=file, repetition=repetition, config_name=self.config_name) if self.config.print_output: stdout_output = None stderr_output = None else: stdout_output = subprocess.DEVNULL stderr_output = subprocess.DEVNULL try: # generate files / folders, if not existing if curr_stdout_file_name: if not os.path.exists(os.path.dirname(curr_stdout_file_name)): os.makedirs(os.path.dirname(curr_stdout_file_name)) stdout_output = open(curr_stdout_file_name, "w+") if curr_stderr_file_name: if not os.path.exists(os.path.dirname(curr_stderr_file_name)): os.makedirs(os.path.dirname(curr_stderr_file_name)) stderr_output = open(curr_stderr_file_name, "w+") except IOError as err: print("Unable to open output file. Aborting.") print(err) exit(-1) return stdout_output, stderr_output
def run(self, n_run, total): results = [] for i in range(0, self.config.repetitions): # replace placeholders in command concrete_command = [ replace_placeholders(cmd, file=self.file, repetition=i, config_name=self.config_name) for cmd in self.command ] print(datetime.datetime.now().strftime("%d.%m.%Y, %H:%M:%S") + ": running job " + str(n_run + i) + " of " + str(total) + ", repetition " + str(i + 1) + " of " + str(self.config.repetitions) + "...") print("Command: '" + " ".join(concrete_command)) curr_result = SingleRunResult(self.config_name, self.file) self.run_process(concrete_command, self.file, i, curr_result) results.append(curr_result) print() print("Time elapsed: " + "{:.3f}".format(curr_result.time_elapsed) + " seconds") print() return results
def run(self, n_run, total): results = [] for i in range(0, self.config.repetitions): # replace placeholders in command concrete_command = [replace_placeholders(cmd, file=self.file, repetition=i, config_name=self.config_name) for cmd in self.command] print(datetime.datetime.now().strftime("%d.%m.%Y, %H:%M:%S") + ": running job " + str(n_run + i) + " of " + str(total) + ", repetition " + str(i + 1) + " of " + str(self.config.repetitions) + "...") print("Command: '" + " ".join(concrete_command)) curr_result = SingleRunResult(self.config_name, self.file) self.run_process(concrete_command, self.file, i, curr_result) results.append(curr_result) print() print("Time elapsed: " + "{:.3f}".format(curr_result.time_elapsed) + " seconds") print() return results
def run_as_benchmark(command, file, config_name, next_job, total_jobs, repetitions, timeout, stdout_fh, stderr_fh): run_results = [] for i in range(0, repetitions): # Replace placeholders in command concrete_command = \ [replace_placeholders(part, file=file, repetition=i, config_name=config_name) for part in command] # Print information about next repetition print(datetime.datetime.now().strftime("%d.%m.%Y, %H:%M:%S") + ": running job " + str(next_job + i) + " of " + str(total_jobs) + ", repetition " + str(i + 1) + " of " + str(repetitions) + "...") print("Command: '" + " ".join(concrete_command)) # Run command to benchmark process_result = ProcessRunner.run(concrete_command, timeout, stdout_fh, stderr_fh) # Create and initialize a run result, and append it to the list of recorded run results run_result = SingleRunResult(config_name, file) run_result.timeout_occurred = process_result.timeout_occurred run_result.return_code = process_result.return_code run_result.time_elapsed = process_result.time_elapsed run_results.append(run_result) print() print("Time elapsed: " + "{:.3f}".format(run_result.time_elapsed) + " seconds") print() return run_results
def read_config_file(self, config_file): """ Parses the configuration file. :return: None """ print("Parsing configuration file...") try: with open(config_file) as configFile: run_config_override = True for line in [l.strip(' \r\n') for l in configFile if l.strip(' \r\n') and not l.startswith('#')]: line_splits = line.split(" ", maxsplit=1) opt = line_splits.pop(0).strip() # skip invalid option without argument. if not line_splits: print("Ignoring option '" + opt + "' because it is missing the configuration argument.") continue # option parsing if opt == "test_folder": self.testFolder = line_splits.pop() elif opt == "run_configuration": # override default run configurations if run_config_override: run_config_override = False self.run_configurations = [] self.run_config_names = [] curr_conf = [os.path.normpath(line_splits.pop())] self.run_configurations.append(curr_conf) self.run_config_names.append("run_config_" + str(len(self.run_configurations))) elif opt == "ignore": # normalize path separators, so that filtering works correctly self.ignoreList.append(os.path.normpath(line_splits.pop())) elif opt == "repetitions": try: self.repetitions = int(line_splits[0]) except ValueError: print("Error: Unable to parse '" + line_splits[0] + "' as number of test repetitions.") elif opt == "timeout": try: self.timeout = int(line_splits[0]) if self.timeout <= 0: self.timeout = None except ValueError: print("Error: Unable to parse '" + line_splits[0] + "' as timeout [seconds].") elif opt == "list_files": self.list_files = parse_bool(line_splits.pop()) elif opt == "arg": # parse argument and add it to the last mentioned run_configuration. arg = line_splits.pop().split(" ", maxsplit=1) idx = len(self.run_configurations) self.run_configurations[idx - 1].extend(arg) elif opt == "config_name": # parse argument and add it to the last mentioned run_configuration. config_name = line_splits.pop() if config_name in self.run_config_names: print("Error: Configuration name '" + config_name + "' is not unique. Default name will be kept.") else: idx = len(self.run_configurations) self.run_config_names[idx - 1] = config_name elif opt == "timing_csv": self.timing_csv_file_name = replace_placeholders(line_splits.pop()) elif opt == "avg_per_config_timing_csv": self.avg_per_config_timing_csv_file_name = replace_placeholders(line_splits.pop()) elif opt == "per_config_timing_csv": self.per_config_timing_csv_file_name = replace_placeholders(line_splits.pop()) elif opt == "print_process_output": self.print_output = parse_bool(line_splits.pop()) elif opt == "stdout_file": self.stdout_file_name = line_splits.pop() elif opt == "stderr_file": self.stderr_file_name = line_splits.pop() else: print("Skipping unknown configuration option '" + opt + "'.") except IOError: print("Unable to read '" + config_file + "'. Resorting to default configuration.") print("Copying config to output folder...") # generate output folder if it does not yet exist output_dir = os.path.dirname(self.timing_csv_file_name) if not os.path.exists(output_dir): os.makedirs(output_dir) # create file in case it does not yet exist. config_copy_filename = os.path.join(output_dir, os.path.basename(config_file)) open(config_copy_filename, 'a').close() # copy content shutil.copyfile(config_file, config_copy_filename) print("Done.")