class Runner():
"""Base class to run runxec"""
def __init__(self):
self.executor = RunExecutor()
signal.signal(signal.SIGINT, self.stop_run)
def stop_run(self):
self.executor.stop()
def _runexec_args(self):
return []
def _run(self, output):
return self.executor.execute_run(args=self._runexec_args(),
output_filename=output)
# Should I spawn a process?
def run(self, output):
return self._run(output)
def validate(self):
raise NotImplementedError()
class _Worker(threading.Thread):
"""
A Worker is a deamonic thread, that takes jobs from the working_queue and runs them.
"""
working_queue = Queue()
def __init__(self, benchmark, my_cpus, my_memory_nodes, my_user, output_handler):
threading.Thread.__init__(self) # constuctor of superclass
self.benchmark = benchmark
self.my_cpus = my_cpus
self.my_memory_nodes = my_memory_nodes
self.output_handler = output_handler
self.run_executor = RunExecutor(user=my_user)
self.setDaemon(True)
self.start()
def run(self):
while not _Worker.working_queue.empty() and not STOPPED_BY_INTERRUPT:
currentRun = _Worker.working_queue.get_nowait()
try:
logging.debug('Executing run "%s"', currentRun.identifier)
self.execute(currentRun)
logging.debug('Finished run "%s"', currentRun.identifier)
except SystemExit as e:
logging.critical(e)
except BaseException as e:
logging.exception('Exception during run execution')
_Worker.working_queue.task_done()
def execute(self, run):
"""
This function executes the tool with a sourcefile with options.
It also calls functions for output before and after the run.
"""
self.output_handler.output_before_run(run)
benchmark = self.benchmark
memlimit = benchmark.rlimits.get(MEMLIMIT)
args = run.cmdline()
logging.debug('Command line of run is %s', args)
result = \
self.run_executor.execute_run(
args, run.log_file,
hardtimelimit=benchmark.rlimits.get(TIMELIMIT),
softtimelimit=benchmark.rlimits.get(SOFTTIMELIMIT),
cores=self.my_cpus,
memory_nodes=self.my_memory_nodes,
memlimit=memlimit,
environments=benchmark.environment(),
workingDir=benchmark.working_directory(),
maxLogfileSize=benchmark.config.maxLogfileSize)
for key, value in result.items():
if key == 'walltime':
run.walltime = value
elif key == 'cputime':
run.cputime = value
elif key == 'memory':
run.values['memUsage'] = result['memory']
elif key == 'energy':
for ekey, evalue in value.items():
run.values['energy-'+ekey] = evalue
else:
run.values['@' + key] = value
if self.my_cpus:
run.values['@cpuCores'] = self.my_cpus
if self.my_memory_nodes:
run.values['@memoryNodes'] = self.my_memory_nodes
if self.run_executor.PROCESS_KILLED:
# If the run was interrupted, we ignore the result and cleanup.
run.walltime = 0
run.cputime = 0
try:
if benchmark.config.debug:
os.rename(run.log_file, run.log_file + ".killed")
else:
os.remove(run.log_file)
except OSError:
pass
return 1
run.after_execution(result['exitcode'], termination_reason=result.get('terminationreason', None))
self.output_handler.output_after_run(run)
def stop(self):
# asynchronous call to runexecutor,
# the worker will stop asap, but not within this method.
self.run_executor.stop()
def cleanup(self):
self.run_executor.check_for_new_files_in_home()
class _Worker(threading.Thread):
"""
A Worker is a deamonic thread, that takes jobs from the working_queue and runs them.
"""
working_queue = Queue()
def __init__(self, benchmark, my_cpus, my_memory_nodes, my_user,
output_handler):
threading.Thread.__init__(self) # constuctor of superclass
self.benchmark = benchmark
self.my_cpus = my_cpus
self.my_memory_nodes = my_memory_nodes
self.output_handler = output_handler
self.run_executor = RunExecutor(user=my_user,
**benchmark.config.containerargs)
self.setDaemon(True)
self.start()
def run(self):
while not _Worker.working_queue.empty() and not STOPPED_BY_INTERRUPT:
currentRun = _Worker.working_queue.get_nowait()
try:
logging.debug('Executing run "%s"', currentRun.identifier)
self.execute(currentRun)
logging.debug('Finished run "%s"', currentRun.identifier)
except SystemExit as e:
logging.critical(e)
except BenchExecException as e:
logging.critical(e)
except BaseException as e:
logging.exception('Exception during run execution')
_Worker.working_queue.task_done()
def execute(self, run):
"""
This function executes the tool with a sourcefile with options.
It also calls functions for output before and after the run.
"""
self.output_handler.output_before_run(run)
benchmark = self.benchmark
memlimit = benchmark.rlimits.get(MEMLIMIT)
args = run.cmdline()
logging.debug('Command line of run is %s', args)
run_result = \
self.run_executor.execute_run(
args,
output_filename=run.log_file,
output_dir=run.result_files_folder,
result_files_patterns=benchmark.result_files_patterns,
hardtimelimit=benchmark.rlimits.get(TIMELIMIT),
softtimelimit=benchmark.rlimits.get(SOFTTIMELIMIT),
walltimelimit=benchmark.rlimits.get(WALLTIMELIMIT),
cores=self.my_cpus,
memory_nodes=self.my_memory_nodes,
memlimit=memlimit,
environments=benchmark.environment(),
workingDir=benchmark.working_directory(),
maxLogfileSize=benchmark.config.maxLogfileSize,
files_count_limit=benchmark.config.filesCountLimit,
files_size_limit=benchmark.config.filesSizeLimit)
if self.run_executor.PROCESS_KILLED:
# If the run was interrupted, we ignore the result and cleanup.
try:
if benchmark.config.debug:
os.rename(run.log_file, run.log_file + ".killed")
else:
os.remove(run.log_file)
except OSError:
pass
return 1
if self.my_cpus:
run_result['cpuCores'] = self.my_cpus
if self.my_memory_nodes:
run_result['memoryNodes'] = self.my_memory_nodes
run.set_result(run_result)
self.output_handler.output_after_run(run)
def stop(self):
# asynchronous call to runexecutor,
# the worker will stop asap, but not within this method.
self.run_executor.stop()
def cleanup(self):
self.run_executor.check_for_new_files_in_home()
class _Worker(threading.Thread):
"""
A Worker is a deamonic thread, that takes jobs from the working_queue and runs them.
"""
working_queue = queue.Queue()
def __init__(self, benchmark, my_cpus, my_memory_nodes, output_handler,
run_finished_callback):
threading.Thread.__init__(self) # constuctor of superclass
self.run_finished_callback = run_finished_callback
self.benchmark = benchmark
self.my_cpus = my_cpus
self.my_memory_nodes = my_memory_nodes
self.output_handler = output_handler
self.run_executor = RunExecutor(**benchmark.config.containerargs)
self.setDaemon(True)
self.start()
def run(self):
while not STOPPED_BY_INTERRUPT:
try:
currentRun = _Worker.working_queue.get_nowait()
except queue.Empty:
return
try:
logging.debug('Executing run "%s"', currentRun.identifier)
self.execute(currentRun)
logging.debug('Finished run "%s"', currentRun.identifier)
except SystemExit as e:
logging.critical(e)
except BenchExecException as e:
logging.critical(e)
except BaseException:
logging.exception("Exception during run execution")
self.run_finished_callback()
_Worker.working_queue.task_done()
def execute(self, run):
"""
This function executes the tool with a sourcefile with options.
It also calls functions for output before and after the run.
"""
self.output_handler.output_before_run(run)
benchmark = self.benchmark
args = run.cmdline()
logging.debug("Command line of run is %s", args)
pqos = Pqos()
if self.my_cpus:
pqos.start_monitoring([self.my_cpus])
run_result = self.run_executor.execute_run(
args,
output_filename=run.log_file,
output_dir=run.result_files_folder,
result_files_patterns=benchmark.result_files_patterns,
hardtimelimit=benchmark.rlimits.cputime_hard,
softtimelimit=benchmark.rlimits.cputime,
walltimelimit=benchmark.rlimits.walltime,
cores=self.my_cpus,
memory_nodes=self.my_memory_nodes,
memlimit=benchmark.rlimits.memory,
environments=benchmark.environment(),
workingDir=benchmark.working_directory(),
maxLogfileSize=benchmark.config.maxLogfileSize,
files_count_limit=benchmark.config.filesCountLimit,
files_size_limit=benchmark.config.filesSizeLimit,
)
mon_data = pqos.stop_monitoring()
run_result.update(mon_data)
if not mon_data:
logging.debug(
"Could not monitor cache and memory bandwidth events for run: %s",
run.identifier,
)
if self.run_executor.PROCESS_KILLED:
# If the run was interrupted, we ignore the result and cleanup.
try:
if benchmark.config.debug:
os.rename(run.log_file, run.log_file + ".killed")
else:
os.remove(run.log_file)
except OSError:
pass
return 1
if self.my_cpus:
run_result["cpuCores"] = self.my_cpus
if self.my_memory_nodes:
run_result["memoryNodes"] = self.my_memory_nodes
run.set_result(run_result)
self.output_handler.output_after_run(run)
return None
def stop(self):
# asynchronous call to runexecutor,
# the worker will stop asap, but not within this method.
self.run_executor.stop()
