def test_starttime(self):
if not os.path.exists("/bin/echo"):
self.skipTest("missing /bin/echo")
before = util.read_local_time()
(result, _) = self.execute_run("/bin/echo")
after = util.read_local_time()
self.check_result_keys(result)
run_starttime = result["starttime"]
self.assertIsNotNone(run_starttime.tzinfo, "start time is not a local time")
self.assertLessEqual(before, run_starttime)
self.assertLessEqual(run_starttime, after)
def output_after_run_set(
self, runSet, cputime=None, walltime=None, energy={}, cache={}, end_time=None
):
"""
The method output_after_run_set() stores the times of a run set in XML.
@params cputime, walltime: accumulated times of the run set
"""
self.add_values_to_run_set_xml(runSet, cputime, walltime, energy, cache)
if end_time:
runSet.xml.set("endtime", end_time.isoformat())
elif not self.benchmark.config.start_time:
runSet.xml.set("endtime", util.read_local_time().isoformat())
# write results to files
self._write_pretty_result_xml_to_file(runSet.xml, runSet.xml_file_name)
if len(runSet.blocks) > 1:
for block in runSet.blocks:
blockFileName = self.get_filename(runSet.name, block.name + ".xml")
block_xml = self.runs_to_xml(runSet, block.runs, block.name)
block_xml.set("starttime", runSet.xml.get("starttime"))
if runSet.xml.get("endtime"):
block_xml.set("endtime", runSet.xml.get("endtime"))
self._write_pretty_result_xml_to_file(block_xml, blockFileName)
self.txt_file.append(
self.run_set_to_text(runSet, True, cputime, walltime, energy)
)
def test_read_local_time(self):
"""Test on Python 3.6+ that the fallback for older Pythons does the same."""
try:
time = datetime.datetime.now().astimezone() # desired code
except (ValueError, TypeError):
self.skipTest("datetime.datetime.now().astimezone() not supported")
time2 = util.read_local_time() # contains backwards-compatible code
self.assertLess(time2 - time, datetime.timedelta(seconds=1))
def parse_benchmark_definition(self, content):
with tempfile.NamedTemporaryFile(
prefix="BenchExec_test_benchmark_definition_", suffix=".xml", mode="w+"
) as temp:
temp.write(content)
temp.flush()
# Because we mocked everything that accesses the file system,
# we can parse the benchmark definition although task files do not exist.
return Benchmark(temp.name, DummyConfig, util.read_local_time())
def execute_benchmark(self, benchmark_file):
"""
Execute a single benchmark as defined in a file.
If called directly, ensure that config and executor attributes are set up.
@param benchmark_file: the name of a benchmark-definition XML file
@return: a result value from the executor module
"""
benchmark = Benchmark(
benchmark_file,
self.config,
self.config.start_time or util.read_local_time(),
)
self.check_existing_results(benchmark)
self.executor.init(self.config, benchmark)
output_handler = OutputHandler(benchmark,
self.executor.get_system_info(),
self.config.compress_results)
logging.debug(
"I'm benchmarking %r consisting of %s run sets using %s %s.",
benchmark_file,
len(benchmark.run_sets),
benchmark.tool_name,
benchmark.tool_version or "(unknown version)",
)
try:
result = self.executor.execute_benchmark(benchmark, output_handler)
finally:
benchmark.tool.close()
output_handler.close()
# remove useless log folder if it is empty
try:
os.rmdir(benchmark.log_folder)
except OSError:
pass
if self.config.commit and not self.stopped_by_interrupt:
try:
util.add_files_to_git_repository(
self.config.output_path,
output_handler.all_created_files,
self.config.commit_message + "\n\n" +
output_handler.description + "\n\n" +
str(output_handler.statistics),
)
except OSError as e:
logging.warning("Could not add files to git repository: %s", e)
return result
def output_before_run_set(self, runSet, start_time=None):
"""
The method output_before_run_set() calculates the length of the
first column for the output in terminal and stores information
about the runSet in XML.
@param runSet: current run set
"""
xml_file_name = self.get_filename(runSet.name, "xml")
identifier_names = [run.identifier for run in runSet.runs]
# common prefix of file names
runSet.common_prefix = util.common_base_dir(identifier_names)
if runSet.common_prefix:
runSet.common_prefix += os.path.sep
# length of the first column in terminal
runSet.max_length_of_filename = (max(
len(file)
for file in identifier_names) if identifier_names else 20)
runSet.max_length_of_filename = max(
20, runSet.max_length_of_filename - len(runSet.common_prefix))
# write run set name to terminal
numberOfFiles = len(runSet.runs)
numberOfFilesStr = (" (1 file)" if numberOfFiles == 1 else
f" ({numberOfFiles} files)")
util.printOut("\nexecuting run set" +
(" '" + runSet.name + "'" if runSet.name else "") +
numberOfFilesStr +
TERMINAL_TITLE.format(runSet.full_name))
# write information about the run set into txt_file
self.writeRunSetInfoToLog(runSet)
# prepare information for text output
for run in runSet.runs:
run.resultline = self.format_sourcefile_name(
run.identifier, runSet)
if run.sourcefiles:
adjusted_identifier = util.relative_path(
run.identifier, xml_file_name)
else:
# If no source files exist the task doesn't point to any file that could be downloaded.
# In this case, the name doesn't have to be adjusted because it's no path.
adjusted_identifier = run.identifier
# prepare XML structure for each run and runSet
run.xml = ElementTree.Element("run", name=adjusted_identifier)
if run.sourcefiles:
adjusted_sourcefiles = (util.relative_path(s, xml_file_name)
for s in run.sourcefiles)
run.xml.set("files",
"[" + ", ".join(adjusted_sourcefiles) + "]")
if run.specific_options:
run.xml.set("options", " ".join(run.specific_options))
if run.properties:
all_properties = (prop.name for prop in run.properties)
run.xml.set("properties", " ".join(sorted(all_properties)))
if len(run.properties) == 1:
prop = run.properties[0]
run.xml.set("propertyFile",
util.relative_path(prop.filename, xml_file_name))
expected_result = str(
run.expected_results.get(prop.filename, ""))
if expected_result:
run.xml.set("expectedVerdict", expected_result)
block_name = runSet.blocks[0].name if len(runSet.blocks) == 1 else None
runSet.xml = self.runs_to_xml(runSet, runSet.runs, block_name)
if start_time:
runSet.xml.set("starttime", start_time.isoformat())
elif not self.benchmark.config.start_time:
runSet.xml.set("starttime", util.read_local_time().isoformat())
# write (empty) results to XML
runSet.xml_file_name = xml_file_name
self._write_rough_result_xml_to_file(runSet.xml, runSet.xml_file_name)
runSet.xml_file_last_modified_time = time.monotonic()
self.all_created_files.add(runSet.xml_file_name)
self.xml_file_names.append(runSet.xml_file_name)
