class Runner(): """Base class to run runxec""" def __init__(self): self.executor = RunExecutor() signal.signal(signal.SIGINT, self.stop_run) def stop_run(self): self.executor.stop() def _runexec_args(self): return [] def _run(self, output): return self.executor.execute_run(args=self._runexec_args(), output_filename=output) # Should I spawn a process? def run(self, output): return self._run(output) def validate(self): raise NotImplementedError()
class _Worker(threading.Thread): """ A Worker is a deamonic thread, that takes jobs from the working_queue and runs them. """ working_queue = Queue() def __init__(self, benchmark, my_cpus, my_memory_nodes, my_user, output_handler): threading.Thread.__init__(self) # constuctor of superclass self.benchmark = benchmark self.my_cpus = my_cpus self.my_memory_nodes = my_memory_nodes self.output_handler = output_handler self.run_executor = RunExecutor(user=my_user) self.setDaemon(True) self.start() def run(self): while not _Worker.working_queue.empty() and not STOPPED_BY_INTERRUPT: currentRun = _Worker.working_queue.get_nowait() try: logging.debug('Executing run "%s"', currentRun.identifier) self.execute(currentRun) logging.debug('Finished run "%s"', currentRun.identifier) except SystemExit as e: logging.critical(e) except BaseException as e: logging.exception('Exception during run execution') _Worker.working_queue.task_done() def execute(self, run): """ This function executes the tool with a sourcefile with options. It also calls functions for output before and after the run. """ self.output_handler.output_before_run(run) benchmark = self.benchmark memlimit = benchmark.rlimits.get(MEMLIMIT) args = run.cmdline() logging.debug('Command line of run is %s', args) result = \ self.run_executor.execute_run( args, run.log_file, hardtimelimit=benchmark.rlimits.get(TIMELIMIT), softtimelimit=benchmark.rlimits.get(SOFTTIMELIMIT), cores=self.my_cpus, memory_nodes=self.my_memory_nodes, memlimit=memlimit, environments=benchmark.environment(), workingDir=benchmark.working_directory(), maxLogfileSize=benchmark.config.maxLogfileSize) for key, value in result.items(): if key == 'walltime': run.walltime = value elif key == 'cputime': run.cputime = value elif key == 'memory': run.values['memUsage'] = result['memory'] elif key == 'energy': for ekey, evalue in value.items(): run.values['energy-'+ekey] = evalue else: run.values['@' + key] = value if self.my_cpus: run.values['@cpuCores'] = self.my_cpus if self.my_memory_nodes: run.values['@memoryNodes'] = self.my_memory_nodes if self.run_executor.PROCESS_KILLED: # If the run was interrupted, we ignore the result and cleanup. run.walltime = 0 run.cputime = 0 try: if benchmark.config.debug: os.rename(run.log_file, run.log_file + ".killed") else: os.remove(run.log_file) except OSError: pass return 1 run.after_execution(result['exitcode'], termination_reason=result.get('terminationreason', None)) self.output_handler.output_after_run(run) def stop(self): # asynchronous call to runexecutor, # the worker will stop asap, but not within this method. self.run_executor.stop() def cleanup(self): self.run_executor.check_for_new_files_in_home()
class _Worker(threading.Thread): """ A Worker is a deamonic thread, that takes jobs from the working_queue and runs them. """ working_queue = Queue() def __init__(self, benchmark, my_cpus, my_memory_nodes, my_user, output_handler): threading.Thread.__init__(self) # constuctor of superclass self.benchmark = benchmark self.my_cpus = my_cpus self.my_memory_nodes = my_memory_nodes self.output_handler = output_handler self.run_executor = RunExecutor(user=my_user, **benchmark.config.containerargs) self.setDaemon(True) self.start() def run(self): while not _Worker.working_queue.empty() and not STOPPED_BY_INTERRUPT: currentRun = _Worker.working_queue.get_nowait() try: logging.debug('Executing run "%s"', currentRun.identifier) self.execute(currentRun) logging.debug('Finished run "%s"', currentRun.identifier) except SystemExit as e: logging.critical(e) except BenchExecException as e: logging.critical(e) except BaseException as e: logging.exception('Exception during run execution') _Worker.working_queue.task_done() def execute(self, run): """ This function executes the tool with a sourcefile with options. It also calls functions for output before and after the run. """ self.output_handler.output_before_run(run) benchmark = self.benchmark memlimit = benchmark.rlimits.get(MEMLIMIT) args = run.cmdline() logging.debug('Command line of run is %s', args) run_result = \ self.run_executor.execute_run( args, output_filename=run.log_file, output_dir=run.result_files_folder, result_files_patterns=benchmark.result_files_patterns, hardtimelimit=benchmark.rlimits.get(TIMELIMIT), softtimelimit=benchmark.rlimits.get(SOFTTIMELIMIT), walltimelimit=benchmark.rlimits.get(WALLTIMELIMIT), cores=self.my_cpus, memory_nodes=self.my_memory_nodes, memlimit=memlimit, environments=benchmark.environment(), workingDir=benchmark.working_directory(), maxLogfileSize=benchmark.config.maxLogfileSize, files_count_limit=benchmark.config.filesCountLimit, files_size_limit=benchmark.config.filesSizeLimit) if self.run_executor.PROCESS_KILLED: # If the run was interrupted, we ignore the result and cleanup. try: if benchmark.config.debug: os.rename(run.log_file, run.log_file + ".killed") else: os.remove(run.log_file) except OSError: pass return 1 if self.my_cpus: run_result['cpuCores'] = self.my_cpus if self.my_memory_nodes: run_result['memoryNodes'] = self.my_memory_nodes run.set_result(run_result) self.output_handler.output_after_run(run) def stop(self): # asynchronous call to runexecutor, # the worker will stop asap, but not within this method. self.run_executor.stop() def cleanup(self): self.run_executor.check_for_new_files_in_home()
class _Worker(threading.Thread): """ A Worker is a deamonic thread, that takes jobs from the working_queue and runs them. """ working_queue = queue.Queue() def __init__(self, benchmark, my_cpus, my_memory_nodes, output_handler, run_finished_callback): threading.Thread.__init__(self) # constuctor of superclass self.run_finished_callback = run_finished_callback self.benchmark = benchmark self.my_cpus = my_cpus self.my_memory_nodes = my_memory_nodes self.output_handler = output_handler self.run_executor = RunExecutor(**benchmark.config.containerargs) self.setDaemon(True) self.start() def run(self): while not STOPPED_BY_INTERRUPT: try: currentRun = _Worker.working_queue.get_nowait() except queue.Empty: return try: logging.debug('Executing run "%s"', currentRun.identifier) self.execute(currentRun) logging.debug('Finished run "%s"', currentRun.identifier) except SystemExit as e: logging.critical(e) except BenchExecException as e: logging.critical(e) except BaseException: logging.exception("Exception during run execution") self.run_finished_callback() _Worker.working_queue.task_done() def execute(self, run): """ This function executes the tool with a sourcefile with options. It also calls functions for output before and after the run. """ self.output_handler.output_before_run(run) benchmark = self.benchmark args = run.cmdline() logging.debug("Command line of run is %s", args) pqos = Pqos() if self.my_cpus: pqos.start_monitoring([self.my_cpus]) run_result = self.run_executor.execute_run( args, output_filename=run.log_file, output_dir=run.result_files_folder, result_files_patterns=benchmark.result_files_patterns, hardtimelimit=benchmark.rlimits.cputime_hard, softtimelimit=benchmark.rlimits.cputime, walltimelimit=benchmark.rlimits.walltime, cores=self.my_cpus, memory_nodes=self.my_memory_nodes, memlimit=benchmark.rlimits.memory, environments=benchmark.environment(), workingDir=benchmark.working_directory(), maxLogfileSize=benchmark.config.maxLogfileSize, files_count_limit=benchmark.config.filesCountLimit, files_size_limit=benchmark.config.filesSizeLimit, ) mon_data = pqos.stop_monitoring() run_result.update(mon_data) if not mon_data: logging.debug( "Could not monitor cache and memory bandwidth events for run: %s", run.identifier, ) if self.run_executor.PROCESS_KILLED: # If the run was interrupted, we ignore the result and cleanup. try: if benchmark.config.debug: os.rename(run.log_file, run.log_file + ".killed") else: os.remove(run.log_file) except OSError: pass return 1 if self.my_cpus: run_result["cpuCores"] = self.my_cpus if self.my_memory_nodes: run_result["memoryNodes"] = self.my_memory_nodes run.set_result(run_result) self.output_handler.output_after_run(run) return None def stop(self): # asynchronous call to runexecutor, # the worker will stop asap, but not within this method. self.run_executor.stop()