def get_process_output(self, file, repetition):
# generate file names for output files.
curr_stdout_file_name = replace_placeholders(
self.config.stdout_file_name,
file=file,
repetition=repetition,
config_name=self.config_name)
curr_stderr_file_name = replace_placeholders(
self.config.stderr_file_name,
file=file,
repetition=repetition,
config_name=self.config_name)
if self.config.print_output:
stdout_output = None
stderr_output = None
else:
stdout_output = subprocess.DEVNULL
stderr_output = subprocess.DEVNULL
try:
# generate files / folders, if not existing
if curr_stdout_file_name:
if not os.path.exists(os.path.dirname(curr_stdout_file_name)):
os.makedirs(os.path.dirname(curr_stdout_file_name))
stdout_output = open(curr_stdout_file_name, "w+")
if curr_stderr_file_name:
if not os.path.exists(os.path.dirname(curr_stderr_file_name)):
os.makedirs(os.path.dirname(curr_stderr_file_name))
stderr_output = open(curr_stderr_file_name, "w+")
except IOError as err:
print("Unable to open output file. Aborting.")
print(err)
exit(-1)
return stdout_output, stderr_output
def get_process_output(self, file, repetition):
# generate file names for output files.
curr_stdout_file_name = replace_placeholders(self.config.stdout_file_name,
file=file,
repetition=repetition,
config_name=self.config_name)
curr_stderr_file_name = replace_placeholders(self.config.stderr_file_name,
file=file,
repetition=repetition,
config_name=self.config_name)
if self.config.print_output:
stdout_output = None
stderr_output = None
else:
stdout_output = subprocess.DEVNULL
stderr_output = subprocess.DEVNULL
try:
# generate files / folders, if not existing
if curr_stdout_file_name:
if not os.path.exists(os.path.dirname(curr_stdout_file_name)):
os.makedirs(os.path.dirname(curr_stdout_file_name))
stdout_output = open(curr_stdout_file_name, "w+")
if curr_stderr_file_name:
if not os.path.exists(os.path.dirname(curr_stderr_file_name)):
os.makedirs(os.path.dirname(curr_stderr_file_name))
stderr_output = open(curr_stderr_file_name, "w+")
except IOError as err:
print("Unable to open output file. Aborting.")
print(err)
exit(-1)
return stdout_output, stderr_output
def run(self, n_run, total):
results = []
for i in range(0, self.config.repetitions):
# replace placeholders in command
concrete_command = [
replace_placeholders(cmd,
file=self.file,
repetition=i,
config_name=self.config_name)
for cmd in self.command
]
print(datetime.datetime.now().strftime("%d.%m.%Y, %H:%M:%S") +
": running job " + str(n_run + i) + " of " + str(total) +
", repetition " + str(i + 1) + " of " +
str(self.config.repetitions) + "...")
print("Command: '" + " ".join(concrete_command))
curr_result = SingleRunResult(self.config_name, self.file)
self.run_process(concrete_command, self.file, i, curr_result)
results.append(curr_result)
print()
print("Time elapsed: " +
"{:.3f}".format(curr_result.time_elapsed) + " seconds")
print()
return results
def run(self, n_run, total):
results = []
for i in range(0, self.config.repetitions):
# replace placeholders in command
concrete_command = [replace_placeholders(cmd, file=self.file, repetition=i, config_name=self.config_name)
for cmd in self.command]
print(datetime.datetime.now().strftime("%d.%m.%Y, %H:%M:%S") +
": running job " + str(n_run + i) +
" of " + str(total) + ", repetition " +
str(i + 1) + " of " + str(self.config.repetitions) + "...")
print("Command: '" + " ".join(concrete_command))
curr_result = SingleRunResult(self.config_name, self.file)
self.run_process(concrete_command, self.file, i, curr_result)
results.append(curr_result)
print()
print("Time elapsed: " + "{:.3f}".format(curr_result.time_elapsed) + " seconds")
print()
return results
def run_as_benchmark(command, file, config_name, next_job, total_jobs,
repetitions, timeout, stdout_fh, stderr_fh):
run_results = []
for i in range(0, repetitions):
# Replace placeholders in command
concrete_command = \
[replace_placeholders(part, file=file, repetition=i, config_name=config_name)
for part in command]
# Print information about next repetition
print(datetime.datetime.now().strftime("%d.%m.%Y, %H:%M:%S") +
": running job " + str(next_job + i) + " of " +
str(total_jobs) + ", repetition " + str(i + 1) + " of " +
str(repetitions) + "...")
print("Command: '" + " ".join(concrete_command))
# Run command to benchmark
process_result = ProcessRunner.run(concrete_command, timeout,
stdout_fh, stderr_fh)
# Create and initialize a run result, and append it to the list of recorded run results
run_result = SingleRunResult(config_name, file)
run_result.timeout_occurred = process_result.timeout_occurred
run_result.return_code = process_result.return_code
run_result.time_elapsed = process_result.time_elapsed
run_results.append(run_result)
print()
print("Time elapsed: " + "{:.3f}".format(run_result.time_elapsed) +
" seconds")
print()
return run_results
def read_config_file(self, config_file):
"""
Parses the configuration file.
:return: None
"""
print("Parsing configuration file...")
try:
with open(config_file) as configFile:
run_config_override = True
for line in [l.strip(' \r\n') for l in configFile if l.strip(' \r\n') and not l.startswith('#')]:
line_splits = line.split(" ", maxsplit=1)
opt = line_splits.pop(0).strip()
# skip invalid option without argument.
if not line_splits:
print("Ignoring option '" + opt + "' because it is missing the configuration argument.")
continue
# option parsing
if opt == "test_folder":
self.testFolder = line_splits.pop()
elif opt == "run_configuration":
# override default run configurations
if run_config_override:
run_config_override = False
self.run_configurations = []
self.run_config_names = []
curr_conf = [os.path.normpath(line_splits.pop())]
self.run_configurations.append(curr_conf)
self.run_config_names.append("run_config_" + str(len(self.run_configurations)))
elif opt == "ignore":
# normalize path separators, so that filtering works correctly
self.ignoreList.append(os.path.normpath(line_splits.pop()))
elif opt == "repetitions":
try:
self.repetitions = int(line_splits[0])
except ValueError:
print("Error: Unable to parse '" + line_splits[0] + "' as number of test repetitions.")
elif opt == "timeout":
try:
self.timeout = int(line_splits[0])
if self.timeout <= 0:
self.timeout = None
except ValueError:
print("Error: Unable to parse '" + line_splits[0] + "' as timeout [seconds].")
elif opt == "list_files":
self.list_files = parse_bool(line_splits.pop())
elif opt == "arg":
# parse argument and add it to the last mentioned run_configuration.
arg = line_splits.pop().split(" ", maxsplit=1)
idx = len(self.run_configurations)
self.run_configurations[idx - 1].extend(arg)
elif opt == "config_name":
# parse argument and add it to the last mentioned run_configuration.
config_name = line_splits.pop()
if config_name in self.run_config_names:
print("Error: Configuration name '" +
config_name + "' is not unique. Default name will be kept.")
else:
idx = len(self.run_configurations)
self.run_config_names[idx - 1] = config_name
elif opt == "timing_csv":
self.timing_csv_file_name = replace_placeholders(line_splits.pop())
elif opt == "avg_per_config_timing_csv":
self.avg_per_config_timing_csv_file_name = replace_placeholders(line_splits.pop())
elif opt == "per_config_timing_csv":
self.per_config_timing_csv_file_name = replace_placeholders(line_splits.pop())
elif opt == "print_process_output":
self.print_output = parse_bool(line_splits.pop())
elif opt == "stdout_file":
self.stdout_file_name = line_splits.pop()
elif opt == "stderr_file":
self.stderr_file_name = line_splits.pop()
else:
print("Skipping unknown configuration option '" + opt + "'.")
except IOError:
print("Unable to read '" + config_file + "'. Resorting to default configuration.")
print("Copying config to output folder...")
# generate output folder if it does not yet exist
output_dir = os.path.dirname(self.timing_csv_file_name)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# create file in case it does not yet exist.
config_copy_filename = os.path.join(output_dir, os.path.basename(config_file))
open(config_copy_filename, 'a').close()
# copy content
shutil.copyfile(config_file, config_copy_filename)
print("Done.")
