def on_complete(self, pypy=None): if pypy.executor.broken: return if self.log_file: IO.write(self.log_file, self.content) enabled = False if self.policy == self.POLICY_ALWAYS: enabled = True elif self.policy == self.POLICY_BATCH_OR_ERROR: enabled = pypy.with_error() or (not Printer.batched.is_muted()) elif self.policy == self.POLICY_ERROR_ONLY: enabled = pypy.with_error() if enabled: with Printer.all.with_level(): if self.pypy.full_output: Printer.all.sep() Printer.all.out( 'Output from file {self.pypy.full_output}'.format( **locals())) if not Printer.batched.is_muted(): Printer.batched.raw( format_n_lines(self.content, pypy.was_successful())) if not Printer.console.is_muted(): Printer.console.raw( format_n_lines(self.content, pypy.was_successful()))
def on_complete(self, pypy=None): if pypy.executor.broken: return if self.log_file: IO.write(self.log_file, self.content) enabled = False if self.policy == self.POLICY_ALWAYS: enabled = True elif self.policy == self.POLICY_BATCH_OR_ERROR: enabled = pypy.with_error() or (not Printer.batched.is_muted()) elif self.policy == self.POLICY_ERROR_ONLY: enabled = pypy.with_error() if enabled: with Printer.all.with_level(): if self.pypy.full_output: Printer.all.sep() Printer.all.out('Output from file {self.pypy.full_output}'.format(**locals())) if not Printer.batched.is_muted(): Printer.batched.raw(format_n_lines(self.content, pypy.was_successful())) if not Printer.console.is_muted(): Printer.console.raw(format_n_lines(self.content, pypy.was_successful()))
def prepare_pbs_files(self, pbs_module): """ :type pbs_module: scripts.pbs.modules.pbs_tarkil_cesnet_cz :rtype: list[(str, PBSModule)] """ jobs = list() for p in self.proc: case = ConfigCase(dict( proc=p, time_limit=self.time_limit, memory_limit=self.memory_limit, tmp='exec-parallel' ), None) pbs_run = pbs_module.Module(case) pbs_run.queue = self.arg_options.get('queue', True) pbs_run.ppn = self.arg_options.get('ppn', 1) pbs_content = self.create_pbs_job_content(pbs_module, case) IO.write(case.fs.pbs_script, pbs_content) qsub_command = pbs_run.get_pbs_command(case.fs.pbs_script) jobs.append((qsub_command, pbs_run)) return jobs
def on_complete(self, pypy=None): if self.log_file: IO.write(self.log_file, self.content) # finish after update if self.update_format and (not self.complete_format and not self.color_complete_format): printf.finish_rewrite() # print regular messages for fmt in ensure_iterable(self.complete_format): printf.out(fmt, monitor=self) # print messages if error if self.pypy.with_error(): for fmt in ensure_iterable(self.error_complete_format): printf.error(fmt, monitor=self) # print regular color messages based on result for fmt in ensure_iterable(self.color_complete_format): if self.pypy.returncode() == 0: printf.success(fmt, monitor=self) elif self.pypy.returncode() is None: printf.warning(fmt, monitor=self) else: printf.error(fmt, monitor=self) with printf: if printf.verbosity() is printf.OutputVerbosity.FULL: printf.sep() printf.out('Output from file {self.pypy.full_output}'.format( **locals())) printf.opt(raw=True).stream( format_n_lines(self.content, pypy.was_successful())) elif printf.verbosity( ) is printf.OutputVerbosity.SMART and pypy.with_error(): printf.sep() printf.out( 'Last 50 lines from file {self.pypy.full_output}'.format( **locals())) printf.opt(raw=True).stream( format_n_lines(self.content, success=False, n_lines=-50)) elif printf.verbosity( ) is printf.OutputVerbosity.MINIMAL and pypy.with_error(): printf.sep() printf.out( 'Last 50 lines from file {self.pypy.full_output}'.format( **locals())) printf.opt(raw=True).stream( format_n_lines(self.content, success=False, n_lines=-50))
def prepare_pbs_files(self, pbs_module): jobs = list() """ :type: list[(str, PBSModule)] """ for yaml_file, yaml_config in self.configs.files.items(): for case in yaml_config.get_one(yaml_file): pbs_run = pbs_module.Module(case) pbs_run.queue = self.arg_options.get('queue', True) pbs_run.ppn = self.arg_options.get('ppn', 1) pbs_content = self.create_pbs_job_content(pbs_module, case) IO.write(case.fs.pbs_script, pbs_content) qsub_command = pbs_run.get_pbs_command(case.fs.pbs_script) jobs.append((qsub_command, pbs_run)) return jobs
def generate_status_file(cls, target): """ Will generate status file if target has option turned on :type target: PyPy """ if target.status_file: IO.write(target.status_file, json.dumps(target.status(), indent=4)) output_dir = Paths.dirname(target.status_file) files = Paths.browse( output_dir, [PathFilters.filter_wildcards('*/profiler_info_*.log.json')]) # profiler json is missing? if not files: IO.write( Paths.join(output_dir, 'profiler_info_dummy.log.json'), '{}')
def run_local_mode_one(proc, time_limit, memory_limit): if proc == 0: command = arg_rest[1:] else: command = [arg_rest[0], '-np', proc] + arg_rest[1:] n_lines = 0 if arg_options.batch else 10 pypy = PyPy(BinExecutor(command)) # set limits pypy.limit_monitor.time_limit = time_limit pypy.limit_monitor.memory_limit = memory_limit pypy.progress = not arg_options.batch pypy.info_monitor.deactivate() pypy.error_monitor.deactivate() # catch output to variable # in batch mode we will keep the files # otherwise we will keep logs only on error log_file = Paths.temp_file('exec-parallel-{date}-{time}-{rnd}.log') pypy.executor.output = OutputMode.variable_output() pypy.full_output = log_file # start and wait for exit pypy.start() pypy.join() # add result to global json result GlobalResult.add(pypy) # in batch mode or on error if not pypy.with_success() or arg_options.batch: content = pypy.executor.output.read() IO.write(log_file, content) Printer.close() Printer.out(format_n_lines(content, indent=' ', n_lines=-n_lines)) Printer.open() return pypy
def run_pbs_mode(configs, debug=False): """ :type debug: bool :type configs: scripts.config.yaml_config.ConfigPool """ global arg_options, arg_others, arg_rest pbs_module = get_pbs_module(arg_options.host) Printer.dynamic_output = not arg_options.batch Printer.dyn('Parsing yaml files') jobs = list() """ :type: list[(str, PBSModule)] """ for yaml_file, yaml_config in configs.files.items(): for case in yaml_config.get_one(yaml_file): pbs_run = pbs_module.Module(case) pbs_run.queue = arg_options.get('queue', True) pbs_run.ppn = arg_options.get('ppn', 1) pbs_content = create_pbs_job_content(pbs_module, case) IO.write(case.fs.pbs_script, pbs_content) qsub_command = pbs_run.get_pbs_command(case.fs.pbs_script) jobs.append((qsub_command, pbs_run)) # start jobs Printer.dyn('Starting jobs') total = len(jobs) job_id = 0 multijob = MultiJob(pbs_module.ModuleJob) for qsub_command, pbs_run in jobs: job_id += 1 Printer.dyn('Starting jobs {:02d} of {:02d}', job_id, total) output = subprocess.check_output(qsub_command) job = pbs_module.ModuleJob.create(output, pbs_run.case) job.full_name = "Case {}".format(pbs_run.case) multijob.add(job) Printer.out() Printer.out('{} job/s inserted into queue', total) # # first update to get more info about multijob jobs Printer.out() Printer.separator() Printer.dyn('Updating job status') multijob.update() # print jobs statuses Printer.out() if not arg_options.batch: multijob.print_status() Printer.separator() Printer.dyn(multijob.get_status_line()) returncodes = dict() # wait for finish while multijob.is_running(): Printer.dyn('Updating job status') multijob.update() Printer.dyn(multijob.get_status_line()) # if some jobs changed status add new line to dynamic output remains jobs_changed = multijob.get_all(status=JobState.COMPLETED) if jobs_changed: Printer.out() Printer.separator() # get all jobs where was status update to COMPLETE state for job in jobs_changed: returncodes[job] = finish_pbs_job(job, arg_options.batch) if jobs_changed: Printer.separator() Printer.out() # after printing update status lets sleep for a bit if multijob.is_running(): time.sleep(5) Printer.out(multijob.get_status_line()) Printer.out('All jobs finished') # get max return code or number 2 if there are no returncodes returncode = max(returncodes.values()) if returncodes else 2 sys.exit(returncode)