def _traceback(richoutput, postmortem, exit): try: yield except (KeyboardInterrupt, SystemExit, pdb.bdb.BdbQuit): treelog.error('killed by user') if exit: raise SystemExit(1) from None raise except: exc = traceback.TracebackException(*sys.exc_info()) prefix = '' while True: treelog.error(prefix + ''.join(exc.format_exception_only()).rstrip()) treelog.debug('Traceback (most recent call first):\n' + ''.join(reversed(exc.stack.format())).rstrip()) if exc.__cause__ is not None: exc = exc.__cause__ prefix = '.. caused by ' elif exc.__context__ is not None and not exc.__suppress_context__: exc = exc.__context__ prefix = '.. while handling ' else: break if postmortem: print(_mkbox( 'YOUR PROGRAM HAS DIED. The Python debugger', 'allows you to examine its post-mortem state', 'to figure out why this happened. Type "h"', 'for an overview of commands to get going.', richoutput=richoutput)) pdb.post_mortem() if exit: raise SystemExit(2) from None raise else: if exit: raise SystemExit(0)
def generate(self): treelog.user('my message') with treelog.infofile('test.dat', 'w') as f: f.write('test1') with treelog.context('my context'): with treelog.iter.plain('iter', 'abc') as items: for c in items: treelog.info(c) with treelog.context('empty'): pass treelog.error('multiple..\n ..lines') with treelog.userfile('test.dat', 'wb') as f: treelog.info('generating') f.write(b'test2') self.generate_test() with treelog.context('context step={}', 0) as format: treelog.info('foo') format(1) treelog.info('bar') with treelog.errorfile('same.dat', 'wb') as f: f.write(b'test3') with treelog.debugfile('dbg.dat', 'wb') as f: f.write(b'test4') treelog.debug('dbg') treelog.warning('warn')
def check(self, interactive=True) -> bool: if self._logdir is None: if self._post_files: log.error( "Error: logdir must be set for capture of stdout, stderr or files" ) return False prev_file = self.storagepath / 'badger.yaml' if prev_file.exists(): with open(self.yamlpath, 'r') as f: new_lines = f.readlines() with open(prev_file, 'r') as f: old_lines = f.readlines() diff = list(Differ().compare(old_lines, new_lines)) if not all(line.startswith(' ') for line in diff) and self.has_data(): if not self._check_decide_diff( diff, prev_file, interactive=interactive): return False shutil.copyfile(self.yamlpath, prev_file) if interactive: log.info("Derived types:") for key, value in self._types.items(): log.info(f" {key}: {_typename(value)}") return True
def _check_decide_diff(self, diff: List[str], prev_file: Path, interactive: bool = True) -> bool: decision = None decisions = ['exit', 'diff', 'new-delete', 'new-keep', 'old'] if interactive: readline.set_completer(util.completer(decisions)) readline.parse_and_bind('tab: complete') log.warning( "Warning: Badgerfile has changed and data have already been stored" ) log.warning("Pick an option:") log.warning(" exit - quit badger and fix the problem manually") log.warning(" diff - view a diff between old and new") log.warning( " new-delete - accept new version and delete existing data (significant changes made)" ) log.warning( " new-keep - accept new version and keep existing data (no significant changes made)" ) log.warning( " old - accept old version and exit (re-run badger to load the changed badgerfile)" ) while decision is None: decision = input('>>> ').strip().lower() if decision not in decisions: decision = None continue if decision == 'diff': pydoc.pager(''.join(diff)) decision = None if decision == 'exit': return False if decision == 'new-delete': self.clear_cache() break if decision == 'new-keep': break if decision == 'old': shutil.copyfile(prev_file, self.yamlpath) return False else: log.error( "Error: Badgerfile has changed and data have already been stored" ) log.error( "Try running 'badger check' for more information, or delete .badgerdata if you're sure" ) return False return True
def _step(self, lhs0, dt): arguments = lhs0.copy() arguments.update((old, lhs0[new]) for old, new in self.old_new) arguments[self.timetarget] = lhs0[self.timetarget] + dt try: return newton(self.target, residual=self.residuals, jacobian=self.jacobians, constrain=self.constrain, arguments=arguments, **self.newtonargs).solve(tol=self.newtontol) except (SolverError, matrix.MatrixError) as e: log.error('error: {}; retrying with timestep {}'.format(e, dt / 2)) return self._step(self._step(lhs0, dt / 2), dt / 2)
def fork(nprocs=None): '''continue as ``nprocs`` parallel processes by forking ``nprocs-1`` times If ``nprocs`` exceeds the configured ``maxprocs`` than it will silently be capped. It is up to the user to prepare shared memory and/or locks for inter-process communication. As a safety measure nested forks are blocked by limiting nprocs to 1; all secondary forks will be silently ignored. ''' if nprocs is None or nprocs > _maxprocs: nprocs = _maxprocs if nprocs == 1: yield 0 return if not hasattr(os, 'fork'): log.warning('fork is unavailable on this platform') yield 0 return child_pids = [] try: fail = 1 for procid in builtins.range(1, nprocs): pid = os.fork() if not pid: signal.signal( signal.SIGINT, signal.SIG_IGN) # disable sigint (ctrl+c) handler log.current = log.NullLog() break child_pids.append(pid) else: procid = 0 with maxprocs(1): yield procid fail = 0 finally: if procid: # before anything else can fail: os._exit(fail) # communicate exit status to main process nfails = fail + sum(os.waitpid(pid, 0)[1] != 0 for pid in child_pids) if fail: # failure in main process: exception has been reraised log.error( 'fork failed in {} out of {} processes; reraising exception for main process' .format(nfails, nprocs)) elif nfails: # failure in child process: raise exception raise Exception('fork failed in {} out of {} processes'.format( nfails, _maxprocs))
def call(func, kwargs, scriptname, funcname=None): '''set up compute environment and call function''' outdir = config.outdir or os.path.join(os.path.expanduser(config.outrootdir), scriptname) with contextlib.ExitStack() as stack: stack.enter_context(cache.enable(os.path.join(outdir, config.cachedir)) if config.cache else cache.disable()) stack.enter_context(matrix.backend(config.matrix)) stack.enter_context(log.set(log.FilterLog(log.RichOutputLog() if config.richoutput else log.StdoutLog(), minlevel=5-config.verbose))) if config.htmloutput: htmllog = stack.enter_context(log.HtmlLog(outdir, title=scriptname, htmltitle='<a href="http://www.nutils.org">{}</a> {}'.format(SVGLOGO, html.escape(scriptname)), favicon=FAVICON)) uri = (config.outrooturi.rstrip('/') + '/' + scriptname if config.outrooturi else pathlib.Path(outdir).resolve().as_uri()) + '/' + htmllog.filename if config.richoutput: t0 = time.perf_counter() bar = lambda running: '{0} [{1}] {2[0]}:{2[1]:02d}:{2[2]:02d}'.format(uri, 'RUNNING' if running else 'STOPPED', _hms(time.perf_counter()-t0)) stack.enter_context(stickybar.activate(bar, update=1)) else: log.info('opened log at', uri) htmllog.write('<ul style="list-style-position: inside; padding-left: 0px; margin-top: 0px;">{}</ul>'.format(''.join( '<li>{}={} <span style="color: gray;">{}</span></li>'.format(param.name, kwargs.get(param.name, param.default), param.annotation) for param in inspect.signature(func).parameters.values())), level=1, escape=False) stack.enter_context(log.add(htmllog)) stack.enter_context(warnings.via(lambda msg: log.warning(msg))) stack.callback(signal.signal, signal.SIGINT, signal.signal(signal.SIGINT, _sigint_handler)) log.info('nutils v{}'.format(_version())) log.info('start', time.ctime()) try: func(**kwargs) except (KeyboardInterrupt, SystemExit, pdb.bdb.BdbQuit): log.error('killed by user') return 1 except: log.error(traceback.format_exc()) if config.pdb: print(_mkbox( 'YOUR PROGRAM HAS DIED. The Python debugger', 'allows you to examine its post-mortem state', 'to figure out why this happened. Type "h"', 'for an overview of commands to get going.')) pdb.post_mortem() return 2 else: log.info('finish', time.ctime()) return 0
def load(cls, spec, parameters, types): # All parameters not mentioned are assumed to be ignored spec.setdefault('parameters', {}) for param in parameters: spec['parameters'].setdefault(param, 'ignore') # If there is exactly one variate, and the x-axis is not given, assume that is the x-axis variates = [ param for param, kind in spec['parameters'].items() if kind == 'variate' ] nvariate = len(variates) if nvariate == 1 and 'xaxis' not in spec: spec['xaxis'] = next(iter(variates)) elif 'xaxis' not in spec: spec['xaxis'] = None # Listify possible scalars for k in ('format', 'yaxis'): if isinstance(spec[k], str): spec[k] = [spec[k]] # Either all the axes are list type or none of them are list_type = util.is_list_type(types[spec['yaxis'][0]]) assert all( util.is_list_type(types[k]) == list_type for k in spec['yaxis'][1:]) if spec['xaxis']: assert util.is_list_type(types[spec['xaxis']]) == list_type # If the x-axis has list type, the effective number of variates is one higher eff_variates = nvariate + list_type # If there are more than one effective variate, the plot must be scatter if eff_variates > 1: if spec.get('type', 'scatter') != 'scatter': log.warning( "Line plots can have at most one variate dimension") spec['type'] = 'scatter' elif eff_variates == 0: log.error("Plot has no effective variate dimensions") return else: spec.setdefault('type', 'line') return call_yaml(cls, spec)
def _step(self, lhs0, t0, dt): try: return newton(self.target, residual=self._res, jacobian=self._jac, lhs0=lhs0, constrain=self.constrain, arguments={ self.target0: lhs0, self.timetarget0: t0, self.timetarget: t0 + dt, **self.arguments }, **self.newtonargs).solve(tol=self.newtontol) except (SolverError, matrix.MatrixError) as e: log.error('error: {}; retrying with timestep {}'.format(e, dt / 2)) return self._step(self._step(lhs0, t0, dt / 2), t0 + dt / 2, dt / 2)
def _step(self, lhs, t, timestep): res, jac = self._res_jac(timestep) try: return newton(self.target, residual=res, jacobian=jac, lhs0=lhs, constrain=self.constrain, arguments=collections.ChainMap( self.arguments, { self.target0: lhs, self.timetarget: t }), **self.newtonargs).solve(tol=self.newtontol) except (SolverError, matrix.MatrixError) as e: log.error('error: {}; retrying with timestep {}'.format( e, timestep / 2)) return self._step(self._step(lhs, t, timestep / 2), t + timestep / 2, timestep / 2)
def _wait(pid): '''wait for process to finish and return True upon success, False upon failure while logging the reason.''' pid_, status = os.waitpid(pid, 0) assert pid_ == pid if os.WIFEXITED(status): s = os.WEXITSTATUS(status) if not s: return True msg = 'exited with status {}'.format(s) elif os.WIFSIGNALED(status): s = os.WTERMSIG(status) msg = 'was killed with signal {} ({})'.format(s, signal.Signals(s).name) elif os.WIFSTOPPED(status): s = os.WSTOPSIG(status) msg = 'was stopped with signal {} ({})'.format(s, signal.Signals(s).name) else: msg = 'died of unnatural causes' treelog.error('process {} {}'.format(pid, msg)) return False
def run(self, collector: 'ResultCollector', context: Dict, workpath: Path, logdir: Path) -> bool: kwargs = { 'cwd': workpath, 'capture_output': True, 'shell': False, } if isinstance(self._command, str): kwargs['shell'] = True command = render(self._command, context, mode='shell') else: command = [render(arg, context) for arg in self._command] with log.context(self.name): log.debug( command if isinstance(command, str) else ' '.join(command)) with time() as duration: result = subprocess.run(command, **kwargs) duration = duration() if logdir: stdout_path = logdir / f'{self.name}.stdout' with open(stdout_path, 'wb') as f: f.write(result.stdout) stderr_path = logdir / f'{self.name}.stderr' with open(stderr_path, 'wb') as f: f.write(result.stderr) stdout = result.stdout.decode() for capture in self._capture: capture.find_in(collector, stdout) if self._capture_walltime: collector.collect(f'walltime/{self.name}', duration) if result.returncode: log.error(f"Command returned exit status {result.returncode}") if logdir: log.error(f"stdout stored in {stdout_path}") log.error(f"stderr stored in {stderr_path}") return False else: log.info(f"Success ({duration:.3g}s)") return True
def run(self, collector, context, workpath, logdir): kwargs = { 'cwd': workpath, 'capture_output': True, } if isinstance(self._command, str): kwargs['shell'] = True command = render(self._command, context, mode='shell') else: command = [render(arg, context) for arg in self._command] with time() as duration: result = subprocess.run(command, **kwargs) duration = duration() if logdir and (result.returncode or self._capture_output): stdout_path = logdir / f'{self.name}.stdout' with open(stdout_path, 'wb') as f: f.write(result.stdout) stderr_path = logdir / f'{self.name}.stderr' with open(stderr_path, 'wb') as f: f.write(result.stderr) if result.returncode: log.error( f"command {self.name} returned exit status {result.returncode}" ) if logdir: log.error(f"stdout stored in {stdout_path}") log.error(f"stderr stored in {stderr_path}") return False stdout = result.stdout.decode() for capture in self._capture: capture.find_in(collector, stdout) if self._capture_walltime: collector.collect(self.name, duration) return True
return iter(title, builtins.range(*args)) def enumerate(title, iterable): warnings.deprecation('log.enumerate is deprecated; use log.iter.percentage instead') return iter(title, builtins.enumerate(iterable), length=_len(iterable)) def zip(title, *iterables): warnings.deprecation('log.zip is deprecated; use log.iter.percentage instead') return iter(title, builtins.zip(*iterables), length=min(map(_len, iterables))) def count(title, start=0, step=1): warnings.deprecation('log.count is deprecated; use log.iter.percentage instead') return iter(title, itertools.count(start, step)) if distutils.version.StrictVersion(treelog.version) >= distutils.version.StrictVersion('1.0b5'): from treelog import debug, info, user, warning, error, debugfile, infofile, userfile, warningfile, errorfile, context else: debug = lambda *args, **kwargs: treelog.debug(*args, **kwargs) info = lambda *args, **kwargs: treelog.info(*args, **kwargs) user = lambda *args, **kwargs: treelog.user(*args, **kwargs) warning = lambda *args, **kwargs: treelog.warning(*args, **kwargs) error = lambda *args, **kwargs: treelog.error(*args, **kwargs) debugfile = lambda *args, **kwargs: treelog.debugfile(*args, **kwargs) infofile = lambda *args, **kwargs: treelog.infofile(*args, **kwargs) userfile = lambda *args, **kwargs: treelog.userfile(*args, **kwargs) warningfile = lambda *args, **kwargs: treelog.warningfile(*args, **kwargs) errorfile = lambda *args, **kwargs: treelog.errorfile(*args, **kwargs) context = lambda *args, **kwargs: treelog.context(title, *initargs, **initkwargs) # vim:sw=2:sts=2:et
def call(func, kwargs, scriptname, funcname=None): '''set up compute environment and call function''' outdir = config.outdir or os.path.join( os.path.expanduser(config.outrootdir), scriptname) with contextlib.ExitStack() as stack: stack.enter_context( cache.enable(os.path.join(outdir, config.cachedir)) if config. cache else cache.disable()) stack.enter_context(matrix.backend(config.matrix)) stack.enter_context( log.set( log.FilterLog(log.RichOutputLog() if config.richoutput else log.StdoutLog(), minlevel=5 - config.verbose))) if config.htmloutput: html = stack.enter_context( log.HtmlLog( outdir, title=scriptname, htmltitle='<a href="http://www.nutils.org">{}</a> {}'. format(SVGLOGO, scriptname), favicon=FAVICON)) uri = (config.outrooturi.rstrip('/') + '/' + scriptname if config.outrooturi else pathlib.Path( outdir).resolve().as_uri()) + '/' + html.filename if config.richoutput: t0 = time.perf_counter() bar = lambda running: '{0} [{1}] {2[0]}:{2[1]:02d}:{2[2]:02d}'.format( uri, 'RUNNING' if running else 'STOPPED', _hms(time.perf_counter() - t0)) stack.enter_context(stickybar.activate(bar, update=1)) else: log.info('opened log at', uri) html.write( '<ul style="list-style-position: inside; padding-left: 0px; margin-top: 0px;">{}</ul>' .format(''.join( '<li>{}={} <span style="color: gray;">{}</span></li>'. format(param.name, kwargs.get(param.name, param.default), param.annotation) for param in inspect.signature(func).parameters.values())), level=1, escape=False) stack.enter_context(log.add(html)) stack.enter_context(warnings.via(log.warning)) stack.callback(signal.signal, signal.SIGINT, signal.signal(signal.SIGINT, _sigint_handler)) log.info('nutils v{}'.format(_version())) log.info('start', time.ctime()) try: func(**kwargs) except (KeyboardInterrupt, SystemExit, pdb.bdb.BdbQuit): log.error('killed by user') return 1 except: log.error(traceback.format_exc()) if config.pdb: print( _mkbox('YOUR PROGRAM HAS DIED. The Python debugger', 'allows you to examine its post-mortem state', 'to figure out why this happened. Type "h"', 'for an overview of commands to get going.')) pdb.post_mortem() return 2 else: log.info('finish', time.ctime()) return 0
def load_and_validate(text: str, path: Path, schema: Validator): try: return load(text, schema, label=path).data except YAMLError as err: log.error(err) raise
def convert(ctx, verbosity, rich, infile, fmt, outfile, **kwargs): # Set up logging if rich: logger = RichOutputLog(sys.stdout) else: logger = log.TeeLog( log.FilterLog(log.StdoutLog(), maxlevel=log.proto.Level.user), log.FilterLog(log.StderrLog(), minlevel=log.proto.Level.warning), ) log.current = log.FilterLog(logger, minlevel=getattr(log.proto.Level, verbosity)) # Print potential warnings if '--global' in sys.argv: log.warning(f"--global is deprecated; use --coords geocentric instead") if '--local' in sys.argv: log.warning(f"--local is deprecated; use --coords local instead") if '--geometry' in sys.argv or '-g' in sys.argv: log.warning(f"--geometry is deprecated; use --coords instead") if '--endianness' in sys.argv: log.warning(f"--endianness is deprecated; use --in-endianness instead") # Convert to pathlib infile = Path(infile) outfile = Path(outfile) if outfile else None # Determine filename of output if outfile and not fmt: fmt = outfile.suffix[1:].lower() elif not outfile: fmt = fmt or 'pvd' outfile = Path(infile.name).with_suffix(f'.{fmt}') # Handle default values of multi-valued options that should be # distinguished from empty, as well as comma splitting and other # transformations explicit = { click.core.ParameterSource.COMMANDLINE, click.core.ParameterSource.ENVIRONMENT } kwargs['input_coords'] = dict(kwargs['input_coords']) for k in ['field_filter', 'only_bases']: kwargs[k] = tuple(split_commas(kwargs[k])) if kwargs['no_fields']: kwargs['field_filter'] = [] elif ctx.get_parameter_source('field_filter') not in explicit: kwargs['field_filter'] = None else: kwargs['field_filter'] = tuple(f.lower() for f in kwargs['field_filter']) if isinstance(kwargs['timestep_index'], int): n = kwargs['timestep_index'] kwargs['timestep_slice'] = f'{n}:{n+1}:1' config.require(multiple_timesteps=False, reason="--time is set") # Remove meta-options for k in ['timestep_index', 'no_fields']: kwargs.pop(k) try: # The config can influence the choice of readers or writers, # so apply it first. Since kwargs may include options that # are not explicity set by the user, we set the source to # Default, and later use the upgrade_source method. with config(source=ConfigSource.Default, **kwargs): for name in kwargs: if ctx.get_parameter_source(name) in explicit: config.upgrade_source(name, ConfigSource.User) if not infile.exists(): raise IOError(f"File or directory does not exist: {infile}") ReaderClass = Reader.find_applicable(infile) WriterClass = Writer.find_applicable(fmt) with ReaderClass(infile) as reader, WriterClass(outfile) as writer: reader.validate() writer.validate() pipeline(reader, writer) except Exception as e: if verbosity == 'debug': # In debug mode, allow exceptions to filter through in raw form traceback.print_exc() else: log.error(f"Error: {e}") log.error("More information may be available with --debug") sys.exit(1)
def show(self): log.error(str(self))