def check_child_exit_code():
"""Check if the child process terminated cleanly
and raise an error otherwise."""
child_exitcode, unused_child_rusage = self._wait_for_process(
child_pid, args[0])
child_exitcode = util.ProcessExitCode.from_raw(child_exitcode)
logging.debug(
"Parent: child process of RunExecutor with PID %d"
" terminated with %s.",
child_pid,
child_exitcode,
)
if child_exitcode:
if child_exitcode.value:
if child_exitcode.value == CHILD_OSERROR:
# This was an OSError in the child,
# details were already logged
raise BenchExecException(
"execution in container failed, check log for details"
)
elif child_exitcode.value == CHILD_UNKNOWN_ERROR:
raise BenchExecException(
"unexpected error in container")
raise OSError(child_exitcode.value,
os.strerror(child_exitcode.value))
raise OSError(
0,
"Child process of RunExecutor terminated with " +
str(child_exitcode),
)
def expected_results_of_file(filename):
"""Create a dict of property->ExpectedResult from information encoded in a filename."""
results = {}
for (filename_part, (expected_result,
for_properties)) in _FILE_RESULTS.items():
if filename_part in filename:
expected_result_class = get_result_classification(expected_result)
assert expected_result_class in {
RESULT_CLASS_TRUE, RESULT_CLASS_FALSE
}
expected_result = expected_result_class == RESULT_CLASS_TRUE
subproperty = None
if len(for_properties) > 1:
assert for_properties == _MEMSAFETY_SUBPROPERTIES and expected_result
prop = _PROP_MEMSAFETY
else:
prop = next(iter(for_properties))
if prop in _MEMSAFETY_SUBPROPERTIES and not expected_result:
subproperty = prop
prop = _PROP_MEMSAFETY
if prop in results:
raise BenchExecException(
"Duplicate property {} in filename {}".format(
prop, filename))
results[prop] = ExpectedResult(expected_result, subproperty)
return results
def _init_container_and_load_tool(tool_module, *args, **kwargs):
"""Initialize container for the current process and load given tool-info module."""
try:
_init_container(*args, **kwargs)
except EnvironmentError as e:
raise BenchExecException("Failed to configure container: " + str(e))
return _load_tool(tool_module)
def get_java(self):
candidates = [
"java",
"/usr/bin/java",
"/opt/oracle-jdk-bin-1.8.0.202/bin/java",
"/usr/lib/jvm/java-8-openjdk-amd64/bin/java",
]
for c in candidates:
candidate = self.which(c)
if not candidate:
continue
try:
process = subprocess.Popen(
[candidate, "-version"],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
(stdout, stderr) = process.communicate()
except OSError:
continue
stdout = util.decode_to_string(stdout).strip()
if not stdout:
continue
if "1.8" in stdout:
return candidate
raise BenchExecException(
"Could not find a suitable Java version: Need Java 1.8")
def load_task_definition_file(task_def_file):
"""Open and parse a task-definition file in YAML format."""
try:
with open(task_def_file) as f:
task_def = yaml.safe_load(f)
except OSError as e:
raise BenchExecException("Cannot open task-definition file: " + str(e))
except yaml.YAMLError as e:
raise BenchExecException("Invalid task definition: " + str(e))
if str(task_def.get("format_version")) not in ["0.1", "1.0"]:
raise BenchExecException(
"Task-definition file {} specifies invalid format_version '{}'."
.format(task_def_file, task_def.get("format_version")))
return task_def
def extract_runs_from_xml(self, sourcefilesTagList, global_required_files_pattern):
'''
This function builds a list of SourcefileSets (containing filename with options).
The files and their options are taken from the list of sourcefilesTags.
'''
base_dir = self.benchmark.base_dir
# runs are structured as sourcefile sets, one set represents one sourcefiles tag
blocks = []
for index, sourcefilesTag in enumerate(sourcefilesTagList):
sourcefileSetName = sourcefilesTag.get("name")
matchName = sourcefileSetName or str(index)
if self.benchmark.config.selected_sourcefile_sets \
and not any(util.wildcard_match(matchName, sourcefile_set) for sourcefile_set in self.benchmark.config.selected_sourcefile_sets):
continue
required_files_pattern = global_required_files_pattern.union(
set(tag.text for tag in sourcefilesTag.findall('requiredfiles')))
# get lists of filenames
task_def_files = self.get_task_def_files_from_xml(sourcefilesTag, base_dir)
# get file-specific options for filenames
fileOptions = util.get_list_from_xml(sourcefilesTag)
propertyfile = util.text_or_none(util.get_single_child_from_xml(sourcefilesTag, PROPERTY_TAG))
# some runs need more than one sourcefile,
# the first sourcefile is a normal 'include'-file, we use its name as identifier
# for logfile and result-category all other files are 'append'ed.
appendFileTags = sourcefilesTag.findall("append")
currentRuns = []
for identifier in task_def_files:
if identifier.endswith('.yml'):
if appendFileTags:
raise BenchExecException(
"Cannot combine <append> and task-definition files in the same <tasks> tag.")
run = self.create_run_from_task_definition(
identifier, fileOptions, propertyfile, required_files_pattern)
else:
run = self.create_run_for_input_file(
identifier, fileOptions, propertyfile, required_files_pattern, appendFileTags)
if run:
currentRuns.append(run)
# add runs for cases without source files
for run in sourcefilesTag.findall("withoutfile"):
currentRuns.append(Run(run.text, [], fileOptions, self, propertyfile, required_files_pattern))
blocks.append(SourcefileSet(sourcefileSetName, index, currentRuns))
if self.benchmark.config.selected_sourcefile_sets:
for selected in self.benchmark.config.selected_sourcefile_sets:
if not any(util.wildcard_match(sourcefile_set.real_name, selected) for sourcefile_set in blocks):
logging.warning(
'The selected tasks "%s" are not present in the input file, '
'skipping them.',
selected)
return blocks
def create(cls, propertyfile, allow_unknown):
"""
Create a Property instance by attempting to parse the given property file.
@param propertyfile: A file name of a property file
@param allow_unknown: Whether to accept unknown properties
"""
with open(propertyfile) as f:
content = f.read().strip()
# parse content for known properties
is_svcomp = False
known_properties = []
only_known_svcomp_property = True
if content == "OBSERVER AUTOMATON" or content == "SATISFIABLE":
known_properties = [_PROPERTY_NAMES[content]]
elif content.startswith("CHECK"):
is_svcomp = True
for line in filter(None, content.splitlines()):
if content.startswith("CHECK"):
# SV-COMP property, either a well-known one or a new one
props_in_line = [
prop for (substring, prop) in _PROPERTY_NAMES.items()
if substring in line
]
if len(props_in_line) == 1:
known_properties.append(props_in_line[0])
else:
only_known_svcomp_property = False
else:
# not actually an SV-COMP property file
is_svcomp = False
known_properties = []
break
# check if some known property content was found
subproperties = None
if only_known_svcomp_property and len(known_properties) == 1:
is_well_known = True
name = known_properties[0]
elif (only_known_svcomp_property
and set(known_properties) == _MEMSAFETY_SUBPROPERTIES):
is_well_known = True
name = _PROP_MEMSAFETY
subproperties = list(known_properties)
else:
if not allow_unknown:
raise BenchExecException(
'File "{0}" does not contain a known property.'.format(
propertyfile))
is_well_known = False
name = os.path.splitext(os.path.basename(propertyfile))[0]
return cls(propertyfile, is_well_known, is_svcomp, name, subproperties)
def getAWSInput(benchmark):
(
requirements,
number_of_runs,
limits_and_num_runs,
run_definitions,
source_files,
) = getBenchmarkData(benchmark)
(working_dir, toolpaths) = getToolData(benchmark)
abs_working_dir = os.path.abspath(working_dir)
abs_tool_paths = list(map(os.path.abspath, toolpaths))
abs_source_files = list(map(os.path.abspath, source_files))
abs_base_dir = benchexec.util.common_base_dir(abs_source_files +
abs_tool_paths)
if abs_base_dir == "":
raise BenchExecException("No common base dir found.")
toolpaths = {
"absBaseDir": abs_base_dir,
"workingDir": working_dir,
"absWorkingDir": abs_working_dir,
"toolpaths": toolpaths,
"absToolpaths": abs_tool_paths,
"sourceFiles": source_files,
"absSourceFiles": abs_source_files,
}
aws_input = {
"requirements": requirements,
"workingDir": os.path.relpath(abs_working_dir, abs_base_dir),
}
if benchmark.result_files_patterns:
if len(benchmark.result_files_patterns) > 1:
raise BenchExecException(
"Multiple result-file patterns not supported in cloud mode.")
aws_input.update(
{"resultFilePatterns": benchmark.result_files_patterns[0]})
aws_input.update({"limitsAndNumRuns": limits_and_num_runs})
aws_input.update({"runDefinitions": run_definitions})
return (toolpaths, aws_input)
def getToolData(benchmark):
working_dir = benchmark.working_directory()
if not os.path.isdir(working_dir):
raise BenchExecException(
f"Missing working directory '{working_dir}', cannot run tool.")
logging.debug("Working dir: %s", working_dir)
toolpaths = benchmark.required_files()
valid_toolpaths = set()
for file in toolpaths:
if not os.path.exists(file):
raise BenchExecException(
f"Missing file '{os.path.normpath(file)}', "
f"not running benchmark without it.")
for glob in benchexec.util.expand_filename_pattern(file, working_dir):
valid_toolpaths.add(glob)
return (working_dir, valid_toolpaths)
def _call_tool_func(name, args, kwargs):
"""Call a method on the tool instance.
@param name: The method name to call.
@param args: List of arguments to be passed as positional arguments.
@param kwargs: Dict of arguments to be passed as keyword arguments.
"""
global tool
try:
return getattr(tool, name)(*args, **kwargs)
except SystemExit as e:
# SystemExit would terminate the worker process instead of being propagated.
raise BenchExecException(str(e.code))
def get_propertytag(parent):
tag = util.get_single_child_from_xml(parent, "propertyfile")
if tag is None:
return None
expected_verdict = tag.get("expectedverdict")
if (expected_verdict is not None and expected_verdict
not in _EXPECTED_RESULT_FILTER_VALUES.values()
and not re.match("false(.*)", expected_verdict)):
raise BenchExecException(
"Invalid value '{}' for expectedverdict of <propertyfile> in tag <{}>: "
"Only 'true', 'false', 'false(<subproperty>)' and 'unknown' "
"are allowed!".format(expected_verdict, parent.tag))
return tag
def expand_patterns_from_tag(tag):
result = []
patterns = task_def.get(tag, [])
if isinstance(patterns, str) or not isinstance(patterns, collections.Iterable):
# accept single string in addition to list of strings
patterns = [patterns]
for pattern in patterns:
expanded = util.expand_filename_pattern(
str(pattern), os.path.dirname(task_def_file))
if not expanded:
raise BenchExecException(
"Pattern '{}' in task-definition file {} did not match any paths."
.format(pattern, task_def_file))
expanded.sort()
result.extend(expanded)
return result
def _createArchiveFile(archive_path, abs_base_dir, abs_paths):
with zipfile.ZipFile(archive_path, "w", zipfile.ZIP_DEFLATED) as zipf:
for file in abs_paths:
if not os.path.exists(file):
zipf.close()
if os.path.isfile(archive_path):
os.remove(archive_path)
raise BenchExecException(
"Missing file '{0}', cannot run benchmark without it.".
format(os.path.normpath(file)))
if os.path.isdir(file):
_zipdir(file, zipf, abs_base_dir)
else:
zipf.write(file, os.path.relpath(file, abs_base_dir))
def _get_cgroup_version():
version = None
try:
with open("/proc/mounts") as mountsFile:
for mount in mountsFile:
mount = mount.split(" ")
if mount[2] == "cgroup":
version = CGROUPS_V1
# only set v2 if it's the only active mount
# we don't support crippled hybrid mode
elif mount[2] == "cgroup2" and version != CGROUPS_V1:
version = CGROUPS_V2
if version is None:
raise BenchExecException("Could not detect Cgroup Version")
except OSError:
logging.exception("Cannot read /proc/mounts")
return version
def _ultimate_version(self, executable):
data_dir = os.path.join(os.path.dirname(executable), "data")
launcher_jar = self._get_current_launcher_jar(executable)
cmds = [
# 2
[
self.get_java(),
"-Xss4m",
"-jar",
launcher_jar,
"-data",
"@noDefault",
"-ultimatedata",
data_dir,
"--version",
],
# 1
[
self.get_java(),
"-Xss4m",
"-jar",
launcher_jar,
"-data",
data_dir,
"--version",
],
]
self.api = len(cmds)
for cmd in cmds:
version = self._query_ultimate_version(cmd, self.api)
if version != "":
return version
self.api = self.api - 1
raise BenchExecException("Could not determine Ultimate version")
def _start_execution_in_container(self, args, stdin, stdout, stderr, env,
root_dir, cwd, temp_dir, cgroups,
output_dir, result_files_patterns,
parent_setup_fn, child_setup_fn,
parent_cleanup_fn):
"""Execute the given command and measure its resource usage similarly to super()._start_execution(),
but inside a container implemented using Linux namespaces.
The command has no network access (only loopback),
a fresh directory as /tmp and no write access outside of this,
and it does not see other processes except itself.
"""
assert self._use_namespaces
env.update(self._env_override)
args = self._build_cmdline(args, env=env)
# We have three processes involved:
# parent: the current Python process in which RunExecutor is executing
# child: child process in new namespace (PID 1 in inner namespace),
# configures inner namespace, serves as dummy init,
# collects result of grandchild and passes it to parent
# grandchild: child of child process (PID 2 in inner namespace), exec()s tool
# We need the following communication steps between these proceses:
# 1a) grandchild tells parent its PID (in outer namespace).
# 1b) grandchild tells parent that it is ready and measurement should begin.
# 2) parent tells grandchild that measurement has begun and tool should
# be exec()ed.
# 3) child tells parent about return value and resource consumption of grandchild.
# 1a and 1b are done together by sending the PID through a pipe.
# 2 is done by sending a null byte through a pipe.
# 3 is done by sending a pickled object through the same pipe as #2.
# We cannot use the same pipe for both directions, because otherwise a sender might
# read the bytes it has sent itself.
# Error codes from child to parent
CHILD_OSERROR = 128
CHILD_UNKNOWN_ERROR = 129
from_parent, to_grandchild = os.pipe(
) # "downstream" pipe parent->grandchild
from_grandchild, to_parent = os.pipe(
) # "upstream" pipe grandchild/child->parent
# If the current directory is within one of the bind mounts we create,
# we need to cd into this directory again, otherwise we would not see the bind mount,
# but the directory behind it. Thus we always set cwd to force a change of directory.
if root_dir is None:
cwd = os.path.abspath(cwd or os.curdir)
else:
root_dir = os.path.abspath(root_dir)
cwd = os.path.abspath(cwd)
def grandchild():
"""Setup everything inside the process that finally exec()s the tool."""
try:
# We know that this process has PID 2 in the inner namespace,
# but we actually need to know its PID in the outer namespace
# such that parent can put us into the correct cgroups.
# According to http://man7.org/linux/man-pages/man7/pid_namespaces.7.html,
# there are two ways to achieve this: sending a message with the PID
# via a socket (but Python < 3.3 lacks a convenient API for sendmsg),
# and reading /proc/self in the outer procfs instance (that's what we do).
my_outer_pid = container.get_my_pid_from_procfs()
container.mount_proc()
container.drop_capabilities()
container.reset_signal_handling()
child_setup_fn() # Do some other setup the caller wants.
# Signal readiness to parent by sending our PID and wait until parent is also ready
os.write(to_parent, str(my_outer_pid).encode())
received = os.read(from_parent, 1)
assert received == b'\0', received
finally:
# close remaining ends of pipe
os.close(from_parent)
os.close(to_parent)
# here Python will exec() the tool for us
def child():
"""Setup everything inside the container, start the tool, and wait for result."""
try:
logging.debug(
"Child: child process of RunExecutor with PID %d started",
container.get_my_pid_from_procfs())
# Put all received signals on hold until we handle them later.
container.block_all_signals()
# We want to avoid leaking file descriptors to the executed child.
# It is also nice if the child has only the minimal necessary file descriptors,
# to avoid keeping other pipes and files open, e.g., those that the parent
# uses to communicate with other containers (if containers are started in parallel).
# Thus we do not use the close_fds feature of subprocess.Popen,
# but do the same here manually.
# We keep the relevant ends of our pipes, and stdin/out/err of child and grandchild.
necessary_fds = {
sys.stdin, sys.stdout, sys.stderr, to_parent, from_parent,
stdin, stdout, stderr
} - {None}
container.close_open_fds(keep_files=necessary_fds)
try:
if not self._allow_network:
container.activate_network_interface("lo")
if root_dir is not None:
self._setup_root_filesystem(root_dir)
else:
self._setup_container_filesystem(temp_dir)
except EnvironmentError as e:
logging.critical("Failed to configure container: %s", e)
return CHILD_OSERROR
try:
os.chdir(cwd)
except EnvironmentError as e:
logging.critical(
"Cannot change into working directory inside container: %s",
e)
return CHILD_OSERROR
try:
grandchild_proc = subprocess.Popen(args,
stdin=stdin,
stdout=stdout,
stderr=stderr,
env=env,
close_fds=False,
preexec_fn=grandchild)
except (EnvironmentError, RuntimeError) as e:
logging.critical("Cannot start process: %s", e)
return CHILD_OSERROR
container.drop_capabilities()
# Close other fds that were still necessary above.
container.close_open_fds(
keep_files={sys.stdout, sys.stderr, to_parent})
# Set up signal handlers to forward signals to grandchild
# (because we are PID 1, there is a special signal handling otherwise).
# cf. dumb-init project: https://github.com/Yelp/dumb-init
# Also wait for grandchild and return its result.
if _HAS_SIGWAIT:
grandchild_result = container.wait_for_child_and_forward_all_signals(
grandchild_proc.pid, args[0])
else:
container.forward_all_signals_async(
grandchild_proc.pid, args[0])
grandchild_result = self._wait_for_process(
grandchild_proc.pid, args[0])
logging.debug(
"Child: process %s terminated with exit code %d.", args[0],
grandchild_result[0])
os.write(to_parent, pickle.dumps(grandchild_result))
os.close(to_parent)
return 0
except EnvironmentError as e:
logging.exception("Error in child process of RunExecutor")
return CHILD_OSERROR
except:
# Need to catch everything because this method always needs to return a int
# (we are inside a C callback that requires returning int).
logging.exception("Error in child process of RunExecutor")
return CHILD_UNKNOWN_ERROR
try: # parent
try:
child_pid = container.execute_in_namespace(
child, use_network_ns=not self._allow_network)
except OSError as e:
raise BenchExecException(
"Creating namespace for container mode failed: " +
os.strerror(e.errno))
logging.debug(
"Parent: child process of RunExecutor with PID %d started.",
child_pid)
def check_child_exit_code():
"""Check if the child process terminated cleanly and raise an error otherwise."""
child_exitcode, unused_child_rusage = self._wait_for_process(
child_pid, args[0])
child_exitcode = util.ProcessExitCode.from_raw(child_exitcode)
logging.debug(
"Parent: child process of RunExecutor with PID %d terminated with %s.",
child_pid, child_exitcode)
if child_exitcode:
if child_exitcode.value:
if child_exitcode.value == CHILD_OSERROR:
# This was an OSError in the child, details were already logged
raise BenchExecException(
"execution in container failed, check log for details"
)
elif child_exitcode.value == CHILD_UNKNOWN_ERROR:
raise BenchExecException(
"unexpected error in container")
raise OSError(child_exitcode.value,
os.strerror(child_exitcode.value))
raise OSError(
0, "Child process of RunExecutor terminated with " +
str(child_exitcode))
# Close unnecessary ends of pipes such that read() does not block forever
# if all other processes have terminated.
os.close(from_parent)
os.close(to_parent)
container.setup_user_mapping(child_pid,
uid=self._uid,
gid=self._gid)
try:
grandchild_pid = int(os.read(
from_grandchild, 10)) # 10 bytes is enough for 32bit int
except ValueError:
# probably empty read, i.e., pipe closed, i.e., child or grandchild failed
check_child_exit_code()
assert False, "Child process of RunExecutor terminated cleanly but did not send expected data."
logging.debug(
"Parent: executing %s in grand child with PID %d via child with PID %d.",
args[0], grandchild_pid, child_pid)
# start measurements
cgroups.add_task(grandchild_pid)
parent_setup = parent_setup_fn()
# Signal grandchild that setup is finished
os.write(to_grandchild, b'\0')
# Copy file descriptor, otherwise we could not close from_grandchild in finally block
# and would leak a file descriptor in case of exception.
from_grandchild_copy = os.dup(from_grandchild)
finally:
os.close(from_grandchild)
os.close(to_grandchild)
def wait_for_grandchild():
# 1024 bytes ought to be enough for everyone^Wour pickled result
try:
received = os.read(from_grandchild_copy, 1024)
except OSError as e:
if self.PROCESS_KILLED and e.errno == errno.EINTR:
# Read was interrupted because of Ctrl+C, we just try again
received = os.read(from_grandchild_copy, 1024)
else:
raise e
parent_cleanup = parent_cleanup_fn(parent_setup)
os.close(from_grandchild_copy)
check_child_exit_code()
if result_files_patterns:
self._transfer_output_files(temp_dir, cwd, output_dir,
result_files_patterns)
exitcode, ru_child = pickle.loads(received)
return exitcode, ru_child, parent_cleanup
return grandchild_pid, wait_for_grandchild
def create_run_from_task_definition(
self, task_def_file, options, propertyfile, required_files_pattern):
"""Create a Run from a task definition in yaml format"""
task_def = load_task_definition_file(task_def_file)
def expand_patterns_from_tag(tag):
result = []
patterns = task_def.get(tag, [])
if isinstance(patterns, str) or not isinstance(patterns, collections.Iterable):
# accept single string in addition to list of strings
patterns = [patterns]
for pattern in patterns:
expanded = util.expand_filename_pattern(
str(pattern), os.path.dirname(task_def_file))
if not expanded:
raise BenchExecException(
"Pattern '{}' in task-definition file {} did not match any paths."
.format(pattern, task_def_file))
expanded.sort()
result.extend(expanded)
return result
input_files = expand_patterns_from_tag("input_files")
if not input_files:
raise BenchExecException(
"Task-definition file {} does not define any input files.".format(task_def_file))
required_files = expand_patterns_from_tag("required_files")
run = Run(
task_def_file,
input_files,
options,
self,
propertyfile,
required_files_pattern,
required_files)
# run.propertyfile of Run is fully determined only after Run is created,
# thus we handle it and the expected results here.
if not run.propertyfile:
return run
# TODO: support "property_name" attribute in yaml
prop = result.Property.create(run.propertyfile, allow_unknown=True)
run.properties = [prop]
for prop_dict in task_def.get("properties", []):
if not isinstance(prop_dict, dict) or "property_file" not in prop_dict:
raise BenchExecException(
"Missing property file for property in task-definition file {}."
.format(task_def_file))
expanded = util.expand_filename_pattern(
prop_dict["property_file"], os.path.dirname(task_def_file))
if len(expanded) != 1:
raise BenchExecException(
"Property pattern '{}' in task-definition file {} does not refer to exactly one file."
.format(prop_dict["property_file"], task_def_file))
# TODO We could reduce I/O by checking absolute paths and using os.path.samestat
# with cached stat calls.
if prop.filename == expanded[0] or os.path.samefile(prop.filename, expanded[0]):
expected_result = prop_dict.get("expected_verdict")
if expected_result is not None and not isinstance(expected_result, bool):
raise BenchExecException(
"Invalid expected result '{}' for property {} in task-definition file {}."
.format(expected_result, prop_dict["property_file"], task_def_file))
run.expected_results[prop.filename] = \
result.ExpectedResult(expected_result, prop_dict.get("subproperty"))
if not run.expected_results:
logging.debug(
"Ignoring run '%s' because it does not have the property from %s.",
run.identifier, run.propertyfile)
return None
elif len(run.expected_results) > 1:
raise BenchExecException(
"Property '{}' specified multiple times in task-definition file {}."
.format(prop.filename, task_def_file))
else:
return run
def init(self, config, benchmark):
"""
This functions will set up the docker network to execute the test.
As a result, it needs root permission for the setup part.
"""
tool_locator = tooladapter.create_tool_locator(config)
benchmark.executable = benchmark.tool.executable(tool_locator)
benchmark.tool_version = benchmark.tool.version(benchmark.executable)
# Read test inputs paths
(
self.switch_source_path,
self.ptf_folder_path,
self.network_config_path,
) = self.read_folder_paths(benchmark)
if not os.path.isdir(self.switch_source_path):
logging.critical(
"Switch folder path not found: %s, {self.switch_source_path}"
)
raise BenchExecException(
"Switch folder path not found. Look over setup definition"
)
if not os.path.isdir(self.ptf_folder_path):
logging.critical(
"Ptf test folder path not found: %s, {self.ptf_folder_path}"
)
raise (
BenchExecException(
f"Ptf test folder path not found: {self.ptf_folder_path}"
)
)
if not self.switch_source_path or not self.ptf_folder_path:
raise BenchExecException(
"Switch or Ptf folder path not defined."
f"Switch path: {self.switch_source_path} Folder path: {self.ptf_folder_path}"
)
# Extract network config info
if not self.network_config_path:
logging.error("No network config file was defined")
raise BenchExecException("No network config file was defined")
with open(self.network_config_path) as json_file:
self.network_config = json.load(json_file)
setup_is_valid = self.network_file_isValid()
if not setup_is_valid:
raise BenchExecException("Network config file is not valid")
# Container setup
self.client = docker.from_env()
self.switch_target_path = "/app"
self.nrOfNodes = len(self.network_config["nodes"])
try:
# Create the ptf tester container
mount_ptf_tester = docker.types.Mount(
"/app", self.ptf_folder_path, type="bind"
)
try:
self.ptf_tester = self.client.containers.create(
PTF_IMAGE_NAME,
detach=True,
name="ptfTester",
mounts=[mount_ptf_tester],
tty=True,
)
except docker.errors.APIError:
self.ptf_tester = self.client.containers.get("ptfTester")
# Create node containers
self.nodes = []
for node_name in self.network_config["nodes"]:
try:
self.nodes.append(
self.client.containers.create(
NODE_IMAGE_NAME, detach=True, name=node_name
)
)
except docker.errors.APIError:
logging.error("Failed to setup node container.")
self.switches = []
# Each switch needs their own mount copy
for switch_info in self.network_config["switches"]:
mount_path = self.create_switch_mount_copy(switch_info)
mount_switch = docker.types.Mount(
self.switch_target_path, mount_path, type="bind"
)
try:
self.switches.append(
self.client.containers.create(
SWITCH_IMAGE_NAME,
detach=True,
name=switch_info,
mounts=[mount_switch],
)
)
except docker.errors.APIError:
self.switches.append(self.client.containers.get(switch_info))
logging.info("Setting up network")
self.setup_network()
self.connect_nodes_to_switch()
except docker.errors.APIError as e:
self.close()
raise BenchExecException(str(e))
def _start_execution_in_container(
self,
args,
stdin,
stdout,
stderr,
env,
root_dir,
cwd,
temp_dir,
memlimit,
memory_nodes,
cgroups,
output_dir,
result_files_patterns,
parent_setup_fn,
child_setup_fn,
parent_cleanup_fn,
):
"""Execute the given command and measure its resource usage similarly to
super()._start_execution(), but inside a container implemented using Linux
namespaces. The command has no network access (only loopback),
a fresh directory as /tmp and no write access outside of this,
and it does not see other processes except itself.
"""
assert self._use_namespaces
if root_dir is None:
env.update(self._env_override)
# We have three processes involved:
# parent: the current Python process in which RunExecutor is executing
# child: child process in new namespace (PID 1 in inner namespace),
# configures inner namespace, serves as dummy init,
# collects result of grandchild and passes it to parent
# grandchild: child of child process (PID 2 in inner namespace), exec()s tool
# We need the following communication steps between these proceses:
# 1a) grandchild tells parent its PID (in outer namespace).
# 1b) grandchild tells parent that it is ready and measurement should begin.
# 2) parent tells grandchild that measurement has begun and tool should
# be exec()ed.
# 3) child tells parent about return value and resource consumption of
# grandchild.
# 1a and 1b are done together by sending the PID through a pipe.
# 2 is done by sending a null byte through a pipe.
# 3 is done by sending a pickled object through the same pipe as #2.
# We cannot use the same pipe for both directions, because otherwise a sender
# might read the bytes it has sent itself.
# Error codes from child to parent
CHILD_OSERROR = 128 # noqa: N806 local constant
CHILD_UNKNOWN_ERROR = 129 # noqa: N806 local constant
# "downstream" pipe parent->grandchild
from_parent, to_grandchild = os.pipe()
# "upstream" pipe grandchild/child->parent
from_grandchild, to_parent = os.pipe()
# The protocol for these pipes is that first the parent sends the marker for
# user mappings, then the grand child sends its outer PID back,
# and finally the parent sends its completion marker.
# After the run, the child sends the result of the grand child and then waits
# for the post_run marker, before it terminates.
MARKER_USER_MAPPING_COMPLETED = b"A" # noqa: N806 local constant
MARKER_PARENT_COMPLETED = b"B" # noqa: N806 local constant
MARKER_PARENT_POST_RUN_COMPLETED = b"C" # noqa: N806 local constant
# If the current directory is within one of the bind mounts we create,
# we need to cd into this directory again, otherwise we would not see the
# bind mount, but the directory behind it.
# Thus we always set cwd to force a change of directory.
if root_dir is None:
cwd = os.path.abspath(cwd or os.curdir)
else:
root_dir = os.path.abspath(root_dir)
cwd = os.path.abspath(cwd)
def grandchild():
"""Setup everything inside the process that finally exec()s the tool."""
try:
# We know that this process has PID 2 in the inner namespace,
# but we actually need to know its PID in the outer namespace
# such that parent can put us into the correct cgroups. According to
# http://man7.org/linux/man-pages/man7/pid_namespaces.7.html,
# there are two ways to achieve this: sending a message with the PID
# via a socket (but Python 2 lacks a convenient API for sendmsg),
# and reading /proc/self in the outer procfs instance
# (that's what we do).
my_outer_pid = container.get_my_pid_from_procfs()
container.mount_proc(self._container_system_config)
container.drop_capabilities()
container.reset_signal_handling()
child_setup_fn() # Do some other setup the caller wants.
# Signal readiness to parent by sending our PID
# and wait until parent is also ready
os.write(to_parent, str(my_outer_pid).encode())
received = os.read(from_parent, 1)
assert received == MARKER_PARENT_COMPLETED, received
finally:
# close remaining ends of pipe
os.close(from_parent)
os.close(to_parent)
# here Python will exec() the tool for us
def child():
"""Setup everything inside the container,
start the tool, and wait for result."""
try:
logging.debug(
"Child: child process of RunExecutor with PID %d started",
container.get_my_pid_from_procfs(),
)
# Put all received signals on hold until we handle them later.
container.block_all_signals()
# We want to avoid leaking file descriptors to the executed child.
# It is also nice if the child has only the minimal necessary file
# descriptors, to avoid keeping other pipes and files open, e.g.,
# those that the parent uses to communicate with other containers
# (if containers are started in parallel).
# Thus we do not use the close_fds feature of subprocess.Popen,
# but do the same here manually. We keep the relevant ends of our pipes,
# and stdin/out/err of child and grandchild.
necessary_fds = {
sys.stdin,
sys.stdout,
sys.stderr,
to_parent,
from_parent,
stdin,
stdout,
stderr,
} - {None}
container.close_open_fds(keep_files=necessary_fds)
try:
if self._container_system_config:
# A standard hostname increases reproducibility.
socket.sethostname(container.CONTAINER_HOSTNAME)
if not self._allow_network:
container.activate_network_interface("lo")
# Wait until user mapping is finished,
# this is necessary for filesystem writes
received = os.read(from_parent,
len(MARKER_USER_MAPPING_COMPLETED))
assert received == MARKER_USER_MAPPING_COMPLETED, received
if root_dir is not None:
self._setup_root_filesystem(root_dir)
else:
self._setup_container_filesystem(
temp_dir,
output_dir if result_files_patterns else None,
memlimit,
memory_nodes,
)
# Marking this process as "non-dumpable" (no core dumps) also
# forbids several other ways how other processes can access and
# influence it:
# ptrace is forbidden and much of /proc/<child>/ is inaccessible.
# We set this to prevent the benchmarked tool from messing with this
# process or using it to escape from the container. More info:
# http://man7.org/linux/man-pages/man5/proc.5.html
# It needs to be done after MARKER_USER_MAPPING_COMPLETED.
libc.prctl(libc.PR_SET_DUMPABLE, libc.SUID_DUMP_DISABLE, 0,
0, 0)
except OSError as e:
logging.critical("Failed to configure container: %s", e)
return CHILD_OSERROR
try:
os.chdir(cwd)
except OSError as e:
logging.critical(
"Cannot change into working directory inside container: %s",
e)
return CHILD_OSERROR
container.setup_seccomp_filter()
try:
grandchild_proc = subprocess.Popen(
args,
stdin=stdin,
stdout=stdout,
stderr=stderr,
env=env,
close_fds=False,
preexec_fn=grandchild,
)
except (OSError, RuntimeError) as e:
logging.critical("Cannot start process: %s", e)
return CHILD_OSERROR
# keep capability for unmount if necessary later
necessary_capabilities = ([libc.CAP_SYS_ADMIN]
if result_files_patterns else [])
container.drop_capabilities(keep=necessary_capabilities)
# Close other fds that were still necessary above.
container.close_open_fds(keep_files={
sys.stdout, sys.stderr, to_parent, from_parent
})
# Set up signal handlers to forward signals to grandchild
# (because we are PID 1, there is a special signal handling otherwise).
# cf. dumb-init project: https://github.com/Yelp/dumb-init
# Also wait for grandchild and return its result.
grandchild_result = container.wait_for_child_and_forward_signals(
grandchild_proc.pid, args[0])
logging.debug(
"Child: process %s terminated with exit code %d.",
args[0],
grandchild_result[0],
)
if result_files_patterns:
# Remove the bind mount that _setup_container_filesystem added
# such that the parent can access the result files.
libc.umount(temp_dir.encode())
# Re-allow access to /proc/<child>/...,
# this is used by the parent for accessing output files
libc.prctl(libc.PR_SET_DUMPABLE, libc.SUID_DUMP_USER, 0, 0, 0)
os.write(to_parent, pickle.dumps(grandchild_result))
os.close(to_parent)
# Now the parent copies the output files, we need to wait until this is
# finished. If the child terminates, the container file system and its
# tmpfs go away.
assert os.read(from_parent,
1) == MARKER_PARENT_POST_RUN_COMPLETED
os.close(from_parent)
return 0
except OSError:
logging.exception("Error in child process of RunExecutor")
return CHILD_OSERROR
except BaseException:
# Need to catch everything because this method always needs to return an
# int (we are inside a C callback that requires returning int).
logging.exception("Error in child process of RunExecutor")
return CHILD_UNKNOWN_ERROR
try: # parent
try:
child_pid = container.execute_in_namespace(
child, use_network_ns=not self._allow_network)
except OSError as e:
if (e.errno == errno.EPERM and util.try_read_file(
"/proc/sys/kernel/unprivileged_userns_clone") == "0"):
raise BenchExecException(
"Unprivileged user namespaces forbidden on this system, please "
"enable them with 'sysctl -w kernel.unprivileged_userns_clone=1' "
"or disable container mode")
elif (e.errno in {errno.ENOSPC, errno.EINVAL} and
util.try_read_file("/proc/sys/user/max_user_namespaces")
== "0"):
# Ubuntu has ENOSPC, Centos seems to produce EINVAL in this case
raise BenchExecException(
"Unprivileged user namespaces forbidden on this system, please "
"enable by using 'sysctl -w user.max_user_namespaces=10000' "
"(or another value) or disable container mode")
else:
raise BenchExecException(
"Creating namespace for container mode failed: " +
os.strerror(e.errno))
logging.debug(
"Parent: child process of RunExecutor with PID %d started.",
child_pid)
def check_child_exit_code():
"""Check if the child process terminated cleanly
and raise an error otherwise."""
child_exitcode, unused_child_rusage = self._wait_for_process(
child_pid, args[0])
child_exitcode = util.ProcessExitCode.from_raw(child_exitcode)
logging.debug(
"Parent: child process of RunExecutor with PID %d"
" terminated with %s.",
child_pid,
child_exitcode,
)
if child_exitcode:
if child_exitcode.value:
if child_exitcode.value == CHILD_OSERROR:
# This was an OSError in the child,
# details were already logged
raise BenchExecException(
"execution in container failed, check log for details"
)
elif child_exitcode.value == CHILD_UNKNOWN_ERROR:
raise BenchExecException(
"unexpected error in container")
raise OSError(child_exitcode.value,
os.strerror(child_exitcode.value))
raise OSError(
0,
"Child process of RunExecutor terminated with " +
str(child_exitcode),
)
# Close unnecessary ends of pipes such that read() does not block forever
# if all other processes have terminated.
os.close(from_parent)
os.close(to_parent)
container.setup_user_mapping(child_pid,
uid=self._uid,
gid=self._gid)
# signal child to continue
os.write(to_grandchild, MARKER_USER_MAPPING_COMPLETED)
try:
# read at most 10 bytes because this is enough for 32bit int
grandchild_pid = int(os.read(from_grandchild, 10))
except ValueError:
# probably empty read, i.e., pipe closed,
# i.e., child or grandchild failed
check_child_exit_code()
assert False, (
"Child process of RunExecutor terminated cleanly"
" but did not send expected data.")
logging.debug(
"Parent: executing %s in grand child with PID %d"
" via child with PID %d.",
args[0],
grandchild_pid,
child_pid,
)
# start measurements
cgroups.add_task(grandchild_pid)
parent_setup = parent_setup_fn()
# Signal grandchild that setup is finished
os.write(to_grandchild, MARKER_PARENT_COMPLETED)
# Copy file descriptor, otherwise we could not close from_grandchild in
# finally block and would leak a file descriptor in case of exception.
from_grandchild_copy = os.dup(from_grandchild)
to_grandchild_copy = os.dup(to_grandchild)
finally:
os.close(from_grandchild)
os.close(to_grandchild)
def wait_for_grandchild():
# 1024 bytes ought to be enough for everyone^Wour pickled result
try:
received = os.read(from_grandchild_copy, 1024)
except OSError as e:
if self.PROCESS_KILLED and e.errno == errno.EINTR:
# Read was interrupted because of Ctrl+C, we just try again
received = os.read(from_grandchild_copy, 1024)
else:
raise e
if not received:
# Typically this means the child exited prematurely because an error
# occurred, and check_child_exitcode() will handle this.
# We close the pipe first, otherwise child could hang infinitely.
os.close(from_grandchild_copy)
os.close(to_grandchild_copy)
check_child_exit_code()
assert False, "Child process terminated cleanly without sending result"
exitcode, ru_child = pickle.loads(received)
base_path = "/proc/{}/root".format(child_pid)
parent_cleanup = parent_cleanup_fn(
parent_setup, util.ProcessExitCode.from_raw(exitcode),
base_path)
if result_files_patterns:
# As long as the child process exists
# we can access the container file system here
self._transfer_output_files(base_path + temp_dir, cwd,
output_dir, result_files_patterns)
os.close(from_grandchild_copy)
os.write(to_grandchild_copy, MARKER_PARENT_POST_RUN_COMPLETED)
os.close(to_grandchild_copy) # signal child that it can terminate
check_child_exit_code()
return exitcode, ru_child, parent_cleanup
return grandchild_pid, wait_for_grandchild
def getBenchmarkData(benchmark):
r = benchmark.requirements
# These values are currently not used internally, but the goal is
# to eventually integrate them in a later stage.
if r.cpu_model is None:
r.cpu_model = "AmazonAWS"
if r.cpu_cores is None or r.memory is None:
raise BenchExecException(
"The entry for either the amount of used cpu cores or memory "
"is missing from the benchmark definition")
requirements = {
"cpu_cores": r.cpu_cores,
"cpu_model": r.cpu_model,
"memory_in_mb": bytes_to_mb(r.memory),
}
# get limits and number of runs
time_limit = benchmark.rlimits.cputime_hard
mem_limit = bytes_to_mb(benchmark.rlimits.memory)
if time_limit is None or mem_limit is None:
raise BenchExecException(
"An entry for either the time- or memory-limit is missing "
"in the benchmark definition")
core_limit = benchmark.rlimits.cpu_cores
number_of_runs = sum(
len(run_set.runs) for run_set in benchmark.run_sets
if run_set.should_be_executed())
limits_and_num_runs = {
"number_of_runs": number_of_runs,
"time_limit_in_sec": time_limit,
"mem_limit_in_mb": mem_limit,
}
if core_limit is not None:
limits_and_num_runs.update({"core_limit": core_limit})
# get runs with args and source_files
source_files = set()
run_definitions = []
for run_set in benchmark.run_sets:
if not run_set.should_be_executed():
continue
if STOPPED_BY_INTERRUPT:
break
# get runs
for run in run_set.runs:
run_definition = {}
# wrap list-elements in quotations-marks if they contain whitespace
cmdline = [
"'{}'".format(x) if " " in x else x for x in run.cmdline()
]
cmdline = " ".join(cmdline)
log_file = os.path.relpath(run.log_file, benchmark.log_folder)
run_definition.update({
"cmdline": cmdline,
"log_file": log_file,
"sourcefile": run.sourcefiles,
"required_files": run.required_files,
})
run_definitions.append(run_definition)
source_files.update(run.sourcefiles)
source_files.update(run.required_files)
if not run_definitions:
raise BenchExecException("Benchmark has nothing to run.")
return (
requirements,
number_of_runs,
limits_and_num_runs,
run_definitions,
source_files,
)
def _init_container(
temp_dir,
network_access,
dir_modes,
container_system_config,
container_tmpfs, # ignored, tmpfs is always used
):
"""
Create a fork of this process in a container. This method only returns in the fork,
so calling it seems like moving the current process into a container.
"""
# Prepare for private home directory, some tools write there
if container_system_config:
dir_modes.setdefault(container.CONTAINER_HOME, container.DIR_HIDDEN)
os.environ["HOME"] = container.CONTAINER_HOME
# Preparations
temp_dir = temp_dir.encode()
dir_modes = collections.OrderedDict(
sorted(
((path.encode(), kind) for (path, kind) in dir_modes.items()),
key=lambda tupl: len(tupl[0]),
)
)
uid = container.CONTAINER_UID if container_system_config else os.getuid()
gid = container.CONTAINER_GID if container_system_config else os.getgid()
# Create container.
# Contrary to ContainerExecutor, which uses clone to start a new process in new
# namespaces, we use unshare, which puts the current process (the multiprocessing
# worker process) into new namespaces.
# The exception is the PID namespace, which will only apply to children processes.
flags = (
libc.CLONE_NEWNS
| libc.CLONE_NEWUTS
| libc.CLONE_NEWIPC
| libc.CLONE_NEWUSER
| libc.CLONE_NEWPID
)
if not network_access:
flags |= libc.CLONE_NEWNET
try:
libc.unshare(flags)
except OSError as e:
if (
e.errno == errno.EPERM
and util.try_read_file("/proc/sys/kernel/unprivileged_userns_clone") == "0"
):
raise BenchExecException(
"Unprivileged user namespaces forbidden on this system, please "
"enable them with 'sysctl kernel.unprivileged_userns_clone=1' "
"or disable container mode"
)
else:
raise BenchExecException(
"Creating namespace for container mode failed: " + os.strerror(e.errno)
)
# Container config
container.setup_user_mapping(os.getpid(), uid, gid)
_setup_container_filesystem(temp_dir, dir_modes, container_system_config)
if container_system_config:
libc.sethostname(container.CONTAINER_HOSTNAME)
if not network_access:
container.activate_network_interface("lo")
# Because this process is not actually in the new PID namespace, we fork.
# The child will be in the new PID namespace and will assume the role of the acting
# multiprocessing worker (which it can do because it inherits the file descriptors
# that multiprocessing uses for communication).
# The original multiprocessing worker (the parent of the fork) must do nothing in
# order to not confuse multiprocessing.
pid = os.fork()
if pid:
container.drop_capabilities()
# block parent such that it does nothing
os.waitpid(pid, 0)
os._exit(0)
# Finalize container setup in child
container.mount_proc(container_system_config) # only possible in child
container.drop_capabilities()
libc.prctl(libc.PR_SET_DUMPABLE, libc.SUID_DUMP_DISABLE, 0, 0, 0)
container.setup_seccomp_filter()
from p4.counter import Counter
from benchexec import tooladapter
from benchexec import util
from benchexec import BenchExecException
# File handling
from shutil import copyfile, rmtree
import json
from distutils.dir_util import copy_tree
try:
import docker
except ModuleNotFoundError:
raise BenchExecException(
"Python-docker package not found. Try reinstalling python docker module"
)
try:
from pyroute2 import IPRoute
from pyroute2 import NetNS
except ModuleNotFoundError:
raise BenchExecException(
"pyroute2 python package not found. Try reinstalling pyroute2"
)
STOPPED_BY_INTERRUPT = False
# Static Parameters
MGNT_NETWORK_SUBNET = "172.19" # Subnet 192.19.x.x/16
def __init__(self, benchmark_file, config, start_time):
"""
The constructor of Benchmark reads the source files, options, columns and the tool
from the XML in the benchmark_file..
"""
logging.debug("I'm loading the benchmark %s.", benchmark_file)
self.config = config
self.benchmark_file = benchmark_file
self.base_dir = os.path.dirname(self.benchmark_file)
# get benchmark-name
self.name = os.path.basename(
benchmark_file)[:-4] # remove ending ".xml"
if config.name:
self.name += "." + config.name
self.description = None
if config.description_file is not None:
try:
self.description = util.read_file(config.description_file)
except (OSError, UnicodeDecodeError) as e:
raise BenchExecException(
"File '{}' given for description could not be read: {}".
format(config.description_file, e))
self.start_time = start_time
self.instance = start_time.strftime(util.TIMESTAMP_FILENAME_FORMAT)
self.output_base_name = config.output_path + self.name + "." + self.instance
self.log_folder = self.output_base_name + ".logfiles" + os.path.sep
self.log_zip = self.output_base_name + ".logfiles.zip"
self.result_files_folder = self.output_base_name + ".files"
# parse XML
try:
rootTag = ElementTree.ElementTree().parse(benchmark_file)
except ElementTree.ParseError as e:
sys.exit("Benchmark file {} is invalid: {}".format(
benchmark_file, e))
if "benchmark" != rootTag.tag:
sys.exit("Benchmark file {} is invalid: "
"It's root element is not named 'benchmark'.".format(
benchmark_file))
# get tool
tool_name = rootTag.get("tool")
if not tool_name:
sys.exit(
"A tool needs to be specified in the benchmark definition file."
)
(self.tool_module, self.tool) = load_tool_info(tool_name, config)
self.tool_name = self.tool.name()
# will be set from the outside if necessary (may not be the case in SaaS environments)
self.tool_version = None
self.executable = None
self.display_name = rootTag.get("displayName")
def parse_memory_limit(value):
# In a future BenchExec version, we could treat unit-less limits as bytes
try:
value = int(value)
except ValueError:
return util.parse_memory_value(value)
else:
raise ValueError(
"Memory limit must have a unit suffix, e.g., '{} MB'".
format(value))
def handle_limit_value(name, key, cmdline_value, parse_fn):
value = rootTag.get(key, None)
# override limit from XML with values from command line
if cmdline_value is not None:
if cmdline_value.strip() == "-1": # infinity
value = None
else:
value = cmdline_value
if value is not None:
try:
self.rlimits[key] = parse_fn(value)
except ValueError as e:
sys.exit("Invalid value for {} limit: {}".format(
name.lower(), e))
if self.rlimits[key] <= 0:
sys.exit(
'{} limit "{}" is invalid, it needs to be a positive number '
"(or -1 on the command line for disabling it).".format(
name, value))
self.rlimits = {}
handle_limit_value("Time", TIMELIMIT, config.timelimit,
util.parse_timespan_value)
handle_limit_value("Hard time", HARDTIMELIMIT, config.timelimit,
util.parse_timespan_value)
handle_limit_value("Wall time", WALLTIMELIMIT, config.walltimelimit,
util.parse_timespan_value)
handle_limit_value("Memory", MEMLIMIT, config.memorylimit,
parse_memory_limit)
handle_limit_value("Core", CORELIMIT, config.corelimit, int)
if HARDTIMELIMIT in self.rlimits:
hardtimelimit = self.rlimits.pop(HARDTIMELIMIT)
if TIMELIMIT in self.rlimits:
if hardtimelimit < self.rlimits[TIMELIMIT]:
logging.warning(
"Hard timelimit %d is smaller than timelimit %d, ignoring the former.",
hardtimelimit,
self.rlimits[TIMELIMIT],
)
elif hardtimelimit > self.rlimits[TIMELIMIT]:
self.rlimits[SOFTTIMELIMIT] = self.rlimits[TIMELIMIT]
self.rlimits[TIMELIMIT] = hardtimelimit
else:
self.rlimits[TIMELIMIT] = hardtimelimit
self.num_of_threads = int(rootTag.get("threads", 1))
if config.num_of_threads is not None:
self.num_of_threads = config.num_of_threads
if self.num_of_threads < 1:
logging.error("At least ONE thread must be given!")
sys.exit()
# get global options and property file
self.options = util.get_list_from_xml(rootTag)
self.propertytag = get_propertytag(rootTag)
# get columns
self.columns = Benchmark.load_columns(rootTag.find("columns"))
# get global source files, they are used in all run sets
if rootTag.findall("sourcefiles"):
sys.exit(
"Benchmark file {} has unsupported old format. "
"Rename <sourcefiles> tags to <tasks>.".format(benchmark_file))
globalSourcefilesTags = rootTag.findall("tasks")
# get required files
self._required_files = set()
for required_files_tag in rootTag.findall("requiredfiles"):
required_files = util.expand_filename_pattern(
required_files_tag.text, self.base_dir)
if not required_files:
logging.warning(
"Pattern %s in requiredfiles tag did not match any file.",
required_files_tag.text,
)
self._required_files = self._required_files.union(required_files)
# get requirements
self.requirements = Requirements(rootTag.findall("require"),
self.rlimits, config)
result_files_tags = rootTag.findall("resultfiles")
if result_files_tags:
self.result_files_patterns = [
os.path.normpath(p.text) for p in result_files_tags if p.text
]
for pattern in self.result_files_patterns:
if pattern.startswith(".."):
sys.exit(
"Invalid relative result-files pattern '{}'.".format(
pattern))
else:
# default is "everything below current directory"
self.result_files_patterns = ["."]
# get benchmarks
self.run_sets = []
for (i,
rundefinitionTag) in enumerate(rootTag.findall("rundefinition")):
self.run_sets.append(
RunSet(rundefinitionTag, self, i + 1, globalSourcefilesTags))
if not self.run_sets:
logging.warning(
"Benchmark file %s specifies no runs to execute "
"(no <rundefinition> tags found).",
benchmark_file,
)
if not any(runSet.should_be_executed() for runSet in self.run_sets):
logging.warning(
"No <rundefinition> tag selected, nothing will be executed.")
if config.selected_run_definitions:
logging.warning(
"The selection %s does not match any run definitions of %s.",
config.selected_run_definitions,
[runSet.real_name for runSet in self.run_sets],
)
elif config.selected_run_definitions:
for selected in config.selected_run_definitions:
if not any(
util.wildcard_match(run_set.real_name, selected)
for run_set in self.run_sets):
logging.warning(
'The selected run definition "%s" is not present in the input file, '
"skipping it.",
selected,
)
