def copy(self, context, sourcepath, targetpath): source = sourcepath / render(self.source, context) target = targetpath / render(self.target, context) target.parent.mkdir(parents=True, exist_ok=True) if not self.template: shutil.copyfile(source, target) return with open(source, 'r') as f: text = f.read() with open(target, 'w') as f: f.write(render(text, context))
def iter_paths(self, context, sourcepath, targetpath): if self.mode == 'simple': yield ( sourcepath / render(self.source, context), targetpath / render(self.target, context), ) elif self.mode == 'glob': target = targetpath / render(self.target, context) for path in sourcepath.glob(render(self.source, context)): path = path.relative_to(sourcepath) yield (sourcepath / path, target / path)
def copy(self, context, sourcepath, targetpath, sourcename='SRC', targetname='TGT', ignore_missing=False): for source, target in self.iter_paths(context, sourcepath, targetpath): logsrc = Path(sourcename) / source.relative_to(sourcepath) logtgt = Path(targetname) / target.relative_to(targetpath) if not sourcepath.exists(): level = log.warning if ignore_missing else log.error level(f"Missing file: {logsrc}") if not ignore_missing: return else: log.debug(logsrc, '->', logtgt) target.parent.mkdir(parents=True, exist_ok=True) if not self.template: shutil.copyfile(source, target) continue with open(source, 'r') as f: text = f.read() with open(target, 'w') as f: f.write(render(text, context))
def generate_legend(self, context: dict, yaxis: str) -> str: if self._legend is not None: return render(self._legend, {**context, 'yaxis': yaxis}) if any(self._parameters_of_kind('category')): name = ', '.join(f'{k}={repr(context[k])}' for k in self._parameters_of_kind('category')) return f'{name} ({yaxis})' return yaxis
def run(self, collector: 'ResultCollector', context: Dict, workpath: Path, logdir: Path) -> bool: kwargs = { 'cwd': workpath, 'capture_output': True, 'shell': False, } if isinstance(self._command, str): kwargs['shell'] = True command = render(self._command, context, mode='shell') else: command = [render(arg, context) for arg in self._command] with log.context(self.name): log.debug( command if isinstance(command, str) else ' '.join(command)) with time() as duration: result = subprocess.run(command, **kwargs) duration = duration() if logdir: stdout_path = logdir / f'{self.name}.stdout' with open(stdout_path, 'wb') as f: f.write(result.stdout) stderr_path = logdir / f'{self.name}.stderr' with open(stderr_path, 'wb') as f: f.write(result.stderr) stdout = result.stdout.decode() for capture in self._capture: capture.find_in(collector, stdout) if self._capture_walltime: collector.collect(f'walltime/{self.name}', duration) if result.returncode: log.error(f"Command returned exit status {result.returncode}") if logdir: log.error(f"stdout stored in {stdout_path}") log.error(f"stderr stored in {stderr_path}") return False else: log.info(f"Success ({duration:.3g}s)") return True
def run(self, collector, context, workpath, logdir): kwargs = { 'cwd': workpath, 'capture_output': True, } if isinstance(self._command, str): kwargs['shell'] = True command = render(self._command, context, mode='shell') else: command = [render(arg, context) for arg in self._command] with time() as duration: result = subprocess.run(command, **kwargs) duration = duration() if logdir and (result.returncode or self._capture_output): stdout_path = logdir / f'{self.name}.stdout' with open(stdout_path, 'wb') as f: f.write(result.stdout) stderr_path = logdir / f'{self.name}.stderr' with open(stderr_path, 'wb') as f: f.write(result.stderr) if result.returncode: log.error( f"command {self.name} returned exit status {result.returncode}" ) if logdir: log.error(f"stdout stored in {stdout_path}") log.error(f"stderr stored in {stderr_path}") return False stdout = result.stdout.decode() for capture in self._capture: capture.find_in(collector, stdout) if self._capture_walltime: collector.collect(self.name, duration) return True
def run_single(self, num, index, namespace): log.user(', '.join(f'{k}={repr(v)}' for k, v in namespace.items())) self.evaluate_context(namespace) namespace['_index'] = num collector = ResultCollector(self._types) for key, value in namespace.items(): collector.collect(key, value) namespace.update(self._constants) with TemporaryDirectory() as workpath: workpath = Path(workpath) if self._logdir: logdir = self.storagepath / render(self._logdir, namespace) logdir.mkdir(parents=True, exist_ok=True) else: logdir = None log.debug( f"Using SRC='{self.sourcepath}', WRK='{workpath}', LOG='{logdir}'" ) for filemap in self._pre_files: filemap.copy(namespace, self.sourcepath, workpath, sourcename='SRC', targetname='WRK') success = True for command in self._commands: if not command.run(collector, namespace, workpath, logdir): self.commit_result(index, collector) success = False break if logdir: for filemap in self._post_files: filemap.copy(namespace, workpath, logdir, sourcename='WRK', targetname='LOG', ignore_missing=not success) self.commit_result(index, collector) return success
def run_single(self, index, namespace): self.evaluate_context(namespace) collector = ResultCollector(self._types) for key, value in namespace.items(): collector.collect(key, value) with TemporaryDirectory() as workpath: workpath = Path(workpath) if self._logdir: logdir = self.storagepath / render(self._logdir, namespace) logdir.mkdir(parents=True, exist_ok=True) else: logdir = None for filemap in self._pre_files: filemap.copy(namespace, self.sourcepath, workpath) for command in self._commands: if not command.run(collector, namespace, workpath, logdir): return False self.commit_result(index, collector) return True
cat_name, xaxis, yaxes = self.generate_category( case, sub_context, sub_index) final_styles = self._styles.supplement(basestyle) for ax_name, data, style in zip(self._yaxis, yaxes, final_styles): legend = self.generate_legend(sub_context, ax_name) plotter(backends)(legend, xpoints=xaxis, ypoints=data, style=style) for attr in ['title', 'xlabel', 'ylabel']: template = getattr(self, f'_{attr}') if template is None: continue text = render(template, context) getattr(backends, f'set_{attr}')(text) backends.set_xmode(self._xmode) backends.set_ymode(self._ymode) backends.set_grid(self._grid) filename = case.storagepath / render(self._filename, context) backends.generate(filename) def generate_category(self, case: 'Case', context: dict, index): # Pick an arbitrary index for ignored parameters ignored = self._parameters_of_kind('ignore') index = case._parameters.make_index( _base=index, **{param: case._parameters[param][0] for param in ignored})