def prepare_job_arguments(logger, conf):
# RunExec arguments
runexec_bin = os.path.join(os.path.dirname(sys.executable), 'runexec')
args = [runexec_bin]
if "CPU cores" in conf["resource limits"] and conf["resource limits"][
"CPU cores"]:
args.append("--cores")
args.append(','.join(
list(map(str, conf["resource limits"]["CPU cores"]))))
if conf["resource limits"]["disk memory size"] and "runexec measure disk" in conf['client'] and \
conf['client']["runexec measure disk"]:
args.extend([
"--filesSizeLimit",
memory_units_converter(conf["resource limits"]["disk memory size"],
"MB")[1]
])
if 'memory size' in conf["resource limits"] and conf["resource limits"][
'memory size']:
args.extend([
'--memlimit',
memory_units_converter(conf["resource limits"]['memory size'],
"MB")[1]
])
# Check container mode
if "runexec container mode" in conf['client'] and conf['client'][
"runexec container mode"]:
args.append('--container')
if "runexec container mode options" in conf['client']:
args.extend(conf['client']["runexec container mode options"])
else:
args.append('--no-container')
# Determine Klever Core script path
if "Klever Core path" in conf["client"]:
cmd = conf["client"]["Klever Core path"]
else:
cmd = os.path.join(os.path.dirname(sys.executable), "klever-core")
# Add CIF path
pythonpaths = conf["client"].setdefault("addon python packages", [])
pythonpaths.append(os.path.join(os.path.dirname(cmd), os.path.pardir))
add_extra_paths(logger, conf)
# Check existence of the file
args.append(cmd)
return args
def __add_missing_restrictions(collection):
"""
If resource limits are incomplete the method adds to given json all necessary fields filled with zeroes.
:param collection: 'resource limits' dictionary from a task description or job configuration.
"""
if len(collection.keys()) == 0:
raise SchedulerException(
"Resource limitations are missing: upload correct tasks.json file and properly "
"set job resource limitiations")
for tag in ['memory size', 'number of CPU cores', 'disk memory size']:
if tag not in collection or collection[tag] is None:
collection[tag] = 0
if 'CPU model' not in collection:
collection['CPU model'] = None
# Make unit translation
try:
for tag in (m for m in ("memory size", "disk memory size")
if m in collection and collection[m] is not None):
collection[tag] = memory_units_converter(collection[tag])[0]
for tag in (t for t in ("wall time", "CPU time")
if t in collection and collection[t] is not None):
collection[tag] = time_units_converter(collection[tag])[0]
except Exception:
raise SchedulerException(
'Cannot interprete {} resource limitations: {!r}'.format(
tag, collection[tag]))
def __user_friendly_memory_resources(self, unit, value, available_value,
reserved_value):
"""
Obtain more user-friendly representation (GB instead of B) for memory resources.
:param unit: Resource unit.
:param value: Claimed/freed value.
:param available_value: Available value.
:param reserved_value: Reserved value.
:return: Converted values and resource unit.
"""
if unit == 'B':
value = memory_units_converter(value, 'GB')[0]
available_value = memory_units_converter(available_value, 'GB')[0]
reserved_value = memory_units_converter(reserved_value, 'GB')[0]
unit = 'GB'
return value, available_value, reserved_value, unit
def __make_limitation_error(self, resources, job_restrictions,
task_restrictions):
cpus, memory, disk = resources
error_block = {
"number of CPU cores":
f'available {cpus} of CPU cores but requested'
f' {job_restrictions["number of CPU cores"]} for the job '
f'and {task_restrictions["number of CPU cores"]} for a task',
"memory size":
"available {} of memory but requested {} for the"
" job and {} for a task".format(
memory_units_converter(memory, outunit='GB')[-1],
memory_units_converter(job_restrictions["memory size"],
outunit='GB')[-1],
memory_units_converter(task_restrictions["memory size"],
outunit='GB')[-1]),
"disk memory size":
"available {} of disk memory but requested {} for the"
" job and {} for a task".format(
memory_units_converter(disk, outunit='GB')[-1],
memory_units_converter(job_restrictions["disk memory size"],
outunit='GB')[-1],
memory_units_converter(task_restrictions["disk memory size"],
outunit='GB')[-1])
}
return error_block
def prepare_task_arguments(logger, conf):
"""
Prepare arguments for solution of a verification task with BenchExec.
:param conf: Configuration dictionary.
:return: List with options.
"""
# BenchExec arguments
benchexec_bin = os.path.join(os.path.dirname(sys.executable), 'benchexec')
args = [benchexec_bin]
if "CPU cores" in conf["resource limits"] and conf["resource limits"][
"CPU cores"]:
args.extend([
"--limitCores",
str(conf["resource limits"]["number of CPU cores"])
])
args.append("--allowedCores")
args.append(','.join(
list(map(str, conf["resource limits"]["CPU cores"]))))
if conf["resource limits"]["disk memory size"] and "benchexec measure disk" in conf['client'] and\
conf['client']["benchexec measure disk"]:
args.extend([
"--filesSizeLimit",
memory_units_converter(conf["resource limits"]["disk memory size"],
'MB')[1]
])
if 'memory size' in conf["resource limits"] and conf["resource limits"][
'memory size']:
numerical, string = memory_units_converter(
conf["resource limits"]['memory size'], 'MB')
# We do not need using precision more than one MB but the function can return float which can confuse BenchExec
args.extend(['--memorylimit', '{}MB'.format(int(numerical))])
if not conf["resource limits"].get('CPU time',
0) and not conf["resource limits"].get(
'soft CPU time', 0):
# Disable explicitly time limitations
args.extend(['--timelimit', '-1'])
if conf["resource limits"].get('wall time', 0):
args.extend([
'--walltimelimit',
time_units_converter(conf["resource limits"]["wall time"], "s")[1]
])
else:
# As we cannot just disable the limit set a very large value
args.extend(['--walltimelimit', str(60 * 60 * 24 * 365 * 100)])
# Check container mode
if "benchexec container mode" in conf['client'] and conf['client'][
"benchexec container mode"]:
args.append('--container')
if "benchexec container mode options" in conf['client']:
args.extend(conf['client']["benchexec container mode options"])
else:
args.append('--no-container')
args.extend(["--no-compress-results", "--outputpath", "./output/"])
args.append("benchmark.xml")
add_extra_paths(logger, conf)
return args
def prepare_node_info(node_info):
"""
Check that required values have been provided and add general node information.
:param node_info: Dictionary with "node configuration" part of the configuration.
:return: Updated dictionary.
"""
system_info = utils.extract_system_information()
result = node_info.copy()
result.update(system_info)
# Check required data
if "CPU number" not in result:
raise KeyError(
"Provide configuration property 'node configuration''CPU number'")
if "available RAM memory" not in result:
raise KeyError(
"Provide configuration property 'node configuration''available RAM memory'"
)
if "available disk memory" not in result:
raise KeyError(
"Provide configuration property 'node configuration''available disk memory'"
)
if "available for jobs" not in result:
raise KeyError(
"Provide configuration property 'node configuration''available for jobs'"
)
if "available for tasks" not in result:
raise KeyError(
"Provide configuration property 'node configuration''available for tasks'"
)
# TODO: extract this to the common library. Add debug printing in particular warn if specified values are out of bounds. Try to use some mathematical functions like min and max.
# Do magic transformations like in get_parallel_threads_num() from klever.core/utils.py to dynamically adjust available
# resources if they are specified as decimals.
if isinstance(result["available CPU number"], float):
result["available CPU number"] = int(result["CPU number"] *
result["available CPU number"])
elif result["available CPU number"] > result["CPU number"]:
result["available CPU number"] = result["CPU number"]
if isinstance(result["available RAM memory"], float):
result["available RAM memory"] = int(result["RAM memory"] *
result["available RAM memory"])
elif isinstance(result["available RAM memory"], str):
result["available RAM memory"] = utils.memory_units_converter(
result["available RAM memory"], '')[0]
if result["available RAM memory"] < 1000**3:
result["available RAM memory"] = 1000**3
elif result["available RAM memory"] > result["RAM memory"] - 1000**3:
result["available RAM memory"] = result["RAM memory"] - 1000**3
if isinstance(result["available disk memory"], float):
result["available disk memory"] = int(result["disk memory"] *
result["available disk memory"])
elif isinstance(result["available disk memory"], str):
result["available disk memory"] = utils.memory_units_converter(
result["available disk memory"], '')[0]
if result["available disk memory"] < 1000**3:
result["available disk memory"] = 1000**3
elif result["available disk memory"] > result["disk memory"] - 1000**3:
result["available disk memory"] = result["disk memory"] - 1000**3
# Check feasibility of limits
if result["available RAM memory"] > result["RAM memory"]:
raise ValueError(
"Node has {} bytes of RAM memory but {} is attempted to reserve".
format(result["RAM memory"], result["available RAM memory"]))
if result["available disk memory"] > result["disk memory"]:
raise ValueError(
"Node has {} bytes of disk memory but {} is attempted to reserve".
format(result["disk memory"], result["available disk memory"]))
if result["available CPU number"] > result["CPU number"]:
raise ValueError(
"Node has {} CPU cores but {} is attempted to reserve".format(
result["CPU number"], result["available CPU number"]))
return result