def _traceback(richoutput, postmortem, exit):
try:
yield
except (KeyboardInterrupt, SystemExit, pdb.bdb.BdbQuit):
treelog.error('killed by user')
if exit:
raise SystemExit(1) from None
raise
except:
exc = traceback.TracebackException(*sys.exc_info())
prefix = ''
while True:
treelog.error(prefix + ''.join(exc.format_exception_only()).rstrip())
treelog.debug('Traceback (most recent call first):\n' + ''.join(reversed(exc.stack.format())).rstrip())
if exc.__cause__ is not None:
exc = exc.__cause__
prefix = '.. caused by '
elif exc.__context__ is not None and not exc.__suppress_context__:
exc = exc.__context__
prefix = '.. while handling '
else:
break
if postmortem:
print(_mkbox(
'YOUR PROGRAM HAS DIED. The Python debugger',
'allows you to examine its post-mortem state',
'to figure out why this happened. Type "h"',
'for an overview of commands to get going.', richoutput=richoutput))
pdb.post_mortem()
if exit:
raise SystemExit(2) from None
raise
else:
if exit:
raise SystemExit(0)
def generate(self):
treelog.user('my message')
with treelog.infofile('test.dat', 'w') as f:
f.write('test1')
with treelog.context('my context'):
with treelog.iter.plain('iter', 'abc') as items:
for c in items:
treelog.info(c)
with treelog.context('empty'):
pass
treelog.error('multiple..\n ..lines')
with treelog.userfile('test.dat', 'wb') as f:
treelog.info('generating')
f.write(b'test2')
self.generate_test()
with treelog.context('context step={}', 0) as format:
treelog.info('foo')
format(1)
treelog.info('bar')
with treelog.errorfile('same.dat', 'wb') as f:
f.write(b'test3')
with treelog.debugfile('dbg.dat', 'wb') as f:
f.write(b'test4')
treelog.debug('dbg')
treelog.warning('warn')
def check(self, interactive=True) -> bool:
if self._logdir is None:
if self._post_files:
log.error(
"Error: logdir must be set for capture of stdout, stderr or files"
)
return False
prev_file = self.storagepath / 'badger.yaml'
if prev_file.exists():
with open(self.yamlpath, 'r') as f:
new_lines = f.readlines()
with open(prev_file, 'r') as f:
old_lines = f.readlines()
diff = list(Differ().compare(old_lines, new_lines))
if not all(line.startswith(' ')
for line in diff) and self.has_data():
if not self._check_decide_diff(
diff, prev_file, interactive=interactive):
return False
shutil.copyfile(self.yamlpath, prev_file)
if interactive:
log.info("Derived types:")
for key, value in self._types.items():
log.info(f" {key}: {_typename(value)}")
return True
def _check_decide_diff(self,
diff: List[str],
prev_file: Path,
interactive: bool = True) -> bool:
decision = None
decisions = ['exit', 'diff', 'new-delete', 'new-keep', 'old']
if interactive:
readline.set_completer(util.completer(decisions))
readline.parse_and_bind('tab: complete')
log.warning(
"Warning: Badgerfile has changed and data have already been stored"
)
log.warning("Pick an option:")
log.warning(" exit - quit badger and fix the problem manually")
log.warning(" diff - view a diff between old and new")
log.warning(
" new-delete - accept new version and delete existing data (significant changes made)"
)
log.warning(
" new-keep - accept new version and keep existing data (no significant changes made)"
)
log.warning(
" old - accept old version and exit (re-run badger to load the changed badgerfile)"
)
while decision is None:
decision = input('>>> ').strip().lower()
if decision not in decisions:
decision = None
continue
if decision == 'diff':
pydoc.pager(''.join(diff))
decision = None
if decision == 'exit':
return False
if decision == 'new-delete':
self.clear_cache()
break
if decision == 'new-keep':
break
if decision == 'old':
shutil.copyfile(prev_file, self.yamlpath)
return False
else:
log.error(
"Error: Badgerfile has changed and data have already been stored"
)
log.error(
"Try running 'badger check' for more information, or delete .badgerdata if you're sure"
)
return False
return True
def _step(self, lhs0, dt):
arguments = lhs0.copy()
arguments.update((old, lhs0[new]) for old, new in self.old_new)
arguments[self.timetarget] = lhs0[self.timetarget] + dt
try:
return newton(self.target,
residual=self.residuals,
jacobian=self.jacobians,
constrain=self.constrain,
arguments=arguments,
**self.newtonargs).solve(tol=self.newtontol)
except (SolverError, matrix.MatrixError) as e:
log.error('error: {}; retrying with timestep {}'.format(e, dt / 2))
return self._step(self._step(lhs0, dt / 2), dt / 2)
def fork(nprocs=None):
'''continue as ``nprocs`` parallel processes by forking ``nprocs-1`` times
If ``nprocs`` exceeds the configured ``maxprocs`` than it will silently be
capped. It is up to the user to prepare shared memory and/or locks for
inter-process communication. As a safety measure nested forks are blocked by
limiting nprocs to 1; all secondary forks will be silently ignored.
'''
if nprocs is None or nprocs > _maxprocs:
nprocs = _maxprocs
if nprocs == 1:
yield 0
return
if not hasattr(os, 'fork'):
log.warning('fork is unavailable on this platform')
yield 0
return
child_pids = []
try:
fail = 1
for procid in builtins.range(1, nprocs):
pid = os.fork()
if not pid:
signal.signal(
signal.SIGINT,
signal.SIG_IGN) # disable sigint (ctrl+c) handler
log.current = log.NullLog()
break
child_pids.append(pid)
else:
procid = 0
with maxprocs(1):
yield procid
fail = 0
finally:
if procid: # before anything else can fail:
os._exit(fail) # communicate exit status to main process
nfails = fail + sum(os.waitpid(pid, 0)[1] != 0 for pid in child_pids)
if fail: # failure in main process: exception has been reraised
log.error(
'fork failed in {} out of {} processes; reraising exception for main process'
.format(nfails, nprocs))
elif nfails: # failure in child process: raise exception
raise Exception('fork failed in {} out of {} processes'.format(
nfails, _maxprocs))
def call(func, kwargs, scriptname, funcname=None):
'''set up compute environment and call function'''
outdir = config.outdir or os.path.join(os.path.expanduser(config.outrootdir), scriptname)
with contextlib.ExitStack() as stack:
stack.enter_context(cache.enable(os.path.join(outdir, config.cachedir)) if config.cache else cache.disable())
stack.enter_context(matrix.backend(config.matrix))
stack.enter_context(log.set(log.FilterLog(log.RichOutputLog() if config.richoutput else log.StdoutLog(), minlevel=5-config.verbose)))
if config.htmloutput:
htmllog = stack.enter_context(log.HtmlLog(outdir, title=scriptname, htmltitle='<a href="http://www.nutils.org">{}</a> {}'.format(SVGLOGO, html.escape(scriptname)), favicon=FAVICON))
uri = (config.outrooturi.rstrip('/') + '/' + scriptname if config.outrooturi else pathlib.Path(outdir).resolve().as_uri()) + '/' + htmllog.filename
if config.richoutput:
t0 = time.perf_counter()
bar = lambda running: '{0} [{1}] {2[0]}:{2[1]:02d}:{2[2]:02d}'.format(uri, 'RUNNING' if running else 'STOPPED', _hms(time.perf_counter()-t0))
stack.enter_context(stickybar.activate(bar, update=1))
else:
log.info('opened log at', uri)
htmllog.write('<ul style="list-style-position: inside; padding-left: 0px; margin-top: 0px;">{}</ul>'.format(''.join(
'<li>{}={} <span style="color: gray;">{}</span></li>'.format(param.name, kwargs.get(param.name, param.default), param.annotation)
for param in inspect.signature(func).parameters.values())), level=1, escape=False)
stack.enter_context(log.add(htmllog))
stack.enter_context(warnings.via(lambda msg: log.warning(msg)))
stack.callback(signal.signal, signal.SIGINT, signal.signal(signal.SIGINT, _sigint_handler))
log.info('nutils v{}'.format(_version()))
log.info('start', time.ctime())
try:
func(**kwargs)
except (KeyboardInterrupt, SystemExit, pdb.bdb.BdbQuit):
log.error('killed by user')
return 1
except:
log.error(traceback.format_exc())
if config.pdb:
print(_mkbox(
'YOUR PROGRAM HAS DIED. The Python debugger',
'allows you to examine its post-mortem state',
'to figure out why this happened. Type "h"',
'for an overview of commands to get going.'))
pdb.post_mortem()
return 2
else:
log.info('finish', time.ctime())
return 0
def load(cls, spec, parameters, types):
# All parameters not mentioned are assumed to be ignored
spec.setdefault('parameters', {})
for param in parameters:
spec['parameters'].setdefault(param, 'ignore')
# If there is exactly one variate, and the x-axis is not given, assume that is the x-axis
variates = [
param for param, kind in spec['parameters'].items()
if kind == 'variate'
]
nvariate = len(variates)
if nvariate == 1 and 'xaxis' not in spec:
spec['xaxis'] = next(iter(variates))
elif 'xaxis' not in spec:
spec['xaxis'] = None
# Listify possible scalars
for k in ('format', 'yaxis'):
if isinstance(spec[k], str):
spec[k] = [spec[k]]
# Either all the axes are list type or none of them are
list_type = util.is_list_type(types[spec['yaxis'][0]])
assert all(
util.is_list_type(types[k]) == list_type
for k in spec['yaxis'][1:])
if spec['xaxis']:
assert util.is_list_type(types[spec['xaxis']]) == list_type
# If the x-axis has list type, the effective number of variates is one higher
eff_variates = nvariate + list_type
# If there are more than one effective variate, the plot must be scatter
if eff_variates > 1:
if spec.get('type', 'scatter') != 'scatter':
log.warning(
"Line plots can have at most one variate dimension")
spec['type'] = 'scatter'
elif eff_variates == 0:
log.error("Plot has no effective variate dimensions")
return
else:
spec.setdefault('type', 'line')
return call_yaml(cls, spec)
def _step(self, lhs0, t0, dt):
try:
return newton(self.target,
residual=self._res,
jacobian=self._jac,
lhs0=lhs0,
constrain=self.constrain,
arguments={
self.target0: lhs0,
self.timetarget0: t0,
self.timetarget: t0 + dt,
**self.arguments
},
**self.newtonargs).solve(tol=self.newtontol)
except (SolverError, matrix.MatrixError) as e:
log.error('error: {}; retrying with timestep {}'.format(e, dt / 2))
return self._step(self._step(lhs0, t0, dt / 2), t0 + dt / 2,
dt / 2)
def _step(self, lhs, t, timestep):
res, jac = self._res_jac(timestep)
try:
return newton(self.target,
residual=res,
jacobian=jac,
lhs0=lhs,
constrain=self.constrain,
arguments=collections.ChainMap(
self.arguments, {
self.target0: lhs,
self.timetarget: t
}),
**self.newtonargs).solve(tol=self.newtontol)
except (SolverError, matrix.MatrixError) as e:
log.error('error: {}; retrying with timestep {}'.format(
e, timestep / 2))
return self._step(self._step(lhs, t, timestep / 2),
t + timestep / 2, timestep / 2)
def _wait(pid):
'''wait for process to finish and return True upon success,
False upon failure while logging the reason.'''
pid_, status = os.waitpid(pid, 0)
assert pid_ == pid
if os.WIFEXITED(status):
s = os.WEXITSTATUS(status)
if not s:
return True
msg = 'exited with status {}'.format(s)
elif os.WIFSIGNALED(status):
s = os.WTERMSIG(status)
msg = 'was killed with signal {} ({})'.format(s,
signal.Signals(s).name)
elif os.WIFSTOPPED(status):
s = os.WSTOPSIG(status)
msg = 'was stopped with signal {} ({})'.format(s,
signal.Signals(s).name)
else:
msg = 'died of unnatural causes'
treelog.error('process {} {}'.format(pid, msg))
return False
def run(self, collector: 'ResultCollector', context: Dict, workpath: Path,
logdir: Path) -> bool:
kwargs = {
'cwd': workpath,
'capture_output': True,
'shell': False,
}
if isinstance(self._command, str):
kwargs['shell'] = True
command = render(self._command, context, mode='shell')
else:
command = [render(arg, context) for arg in self._command]
with log.context(self.name):
log.debug(
command if isinstance(command, str) else ' '.join(command))
with time() as duration:
result = subprocess.run(command, **kwargs)
duration = duration()
if logdir:
stdout_path = logdir / f'{self.name}.stdout'
with open(stdout_path, 'wb') as f:
f.write(result.stdout)
stderr_path = logdir / f'{self.name}.stderr'
with open(stderr_path, 'wb') as f:
f.write(result.stderr)
stdout = result.stdout.decode()
for capture in self._capture:
capture.find_in(collector, stdout)
if self._capture_walltime:
collector.collect(f'walltime/{self.name}', duration)
if result.returncode:
log.error(f"Command returned exit status {result.returncode}")
if logdir:
log.error(f"stdout stored in {stdout_path}")
log.error(f"stderr stored in {stderr_path}")
return False
else:
log.info(f"Success ({duration:.3g}s)")
return True
def run(self, collector, context, workpath, logdir):
kwargs = {
'cwd': workpath,
'capture_output': True,
}
if isinstance(self._command, str):
kwargs['shell'] = True
command = render(self._command, context, mode='shell')
else:
command = [render(arg, context) for arg in self._command]
with time() as duration:
result = subprocess.run(command, **kwargs)
duration = duration()
if logdir and (result.returncode or self._capture_output):
stdout_path = logdir / f'{self.name}.stdout'
with open(stdout_path, 'wb') as f:
f.write(result.stdout)
stderr_path = logdir / f'{self.name}.stderr'
with open(stderr_path, 'wb') as f:
f.write(result.stderr)
if result.returncode:
log.error(
f"command {self.name} returned exit status {result.returncode}"
)
if logdir:
log.error(f"stdout stored in {stdout_path}")
log.error(f"stderr stored in {stderr_path}")
return False
stdout = result.stdout.decode()
for capture in self._capture:
capture.find_in(collector, stdout)
if self._capture_walltime:
collector.collect(self.name, duration)
return True
return iter(title, builtins.range(*args))
def enumerate(title, iterable):
warnings.deprecation('log.enumerate is deprecated; use log.iter.percentage instead')
return iter(title, builtins.enumerate(iterable), length=_len(iterable))
def zip(title, *iterables):
warnings.deprecation('log.zip is deprecated; use log.iter.percentage instead')
return iter(title, builtins.zip(*iterables), length=min(map(_len, iterables)))
def count(title, start=0, step=1):
warnings.deprecation('log.count is deprecated; use log.iter.percentage instead')
return iter(title, itertools.count(start, step))
if distutils.version.StrictVersion(treelog.version) >= distutils.version.StrictVersion('1.0b5'):
from treelog import debug, info, user, warning, error, debugfile, infofile, userfile, warningfile, errorfile, context
else:
debug = lambda *args, **kwargs: treelog.debug(*args, **kwargs)
info = lambda *args, **kwargs: treelog.info(*args, **kwargs)
user = lambda *args, **kwargs: treelog.user(*args, **kwargs)
warning = lambda *args, **kwargs: treelog.warning(*args, **kwargs)
error = lambda *args, **kwargs: treelog.error(*args, **kwargs)
debugfile = lambda *args, **kwargs: treelog.debugfile(*args, **kwargs)
infofile = lambda *args, **kwargs: treelog.infofile(*args, **kwargs)
userfile = lambda *args, **kwargs: treelog.userfile(*args, **kwargs)
warningfile = lambda *args, **kwargs: treelog.warningfile(*args, **kwargs)
errorfile = lambda *args, **kwargs: treelog.errorfile(*args, **kwargs)
context = lambda *args, **kwargs: treelog.context(title, *initargs, **initkwargs)
# vim:sw=2:sts=2:et
def call(func, kwargs, scriptname, funcname=None):
'''set up compute environment and call function'''
outdir = config.outdir or os.path.join(
os.path.expanduser(config.outrootdir), scriptname)
with contextlib.ExitStack() as stack:
stack.enter_context(
cache.enable(os.path.join(outdir, config.cachedir)) if config.
cache else cache.disable())
stack.enter_context(matrix.backend(config.matrix))
stack.enter_context(
log.set(
log.FilterLog(log.RichOutputLog()
if config.richoutput else log.StdoutLog(),
minlevel=5 - config.verbose)))
if config.htmloutput:
html = stack.enter_context(
log.HtmlLog(
outdir,
title=scriptname,
htmltitle='<a href="http://www.nutils.org">{}</a> {}'.
format(SVGLOGO, scriptname),
favicon=FAVICON))
uri = (config.outrooturi.rstrip('/') + '/' +
scriptname if config.outrooturi else pathlib.Path(
outdir).resolve().as_uri()) + '/' + html.filename
if config.richoutput:
t0 = time.perf_counter()
bar = lambda running: '{0} [{1}] {2[0]}:{2[1]:02d}:{2[2]:02d}'.format(
uri, 'RUNNING'
if running else 'STOPPED', _hms(time.perf_counter() - t0))
stack.enter_context(stickybar.activate(bar, update=1))
else:
log.info('opened log at', uri)
html.write(
'<ul style="list-style-position: inside; padding-left: 0px; margin-top: 0px;">{}</ul>'
.format(''.join(
'<li>{}={} <span style="color: gray;">{}</span></li>'.
format(param.name, kwargs.get(param.name, param.default),
param.annotation)
for param in inspect.signature(func).parameters.values())),
level=1,
escape=False)
stack.enter_context(log.add(html))
stack.enter_context(warnings.via(log.warning))
stack.callback(signal.signal, signal.SIGINT,
signal.signal(signal.SIGINT, _sigint_handler))
log.info('nutils v{}'.format(_version()))
log.info('start', time.ctime())
try:
func(**kwargs)
except (KeyboardInterrupt, SystemExit, pdb.bdb.BdbQuit):
log.error('killed by user')
return 1
except:
log.error(traceback.format_exc())
if config.pdb:
print(
_mkbox('YOUR PROGRAM HAS DIED. The Python debugger',
'allows you to examine its post-mortem state',
'to figure out why this happened. Type "h"',
'for an overview of commands to get going.'))
pdb.post_mortem()
return 2
else:
log.info('finish', time.ctime())
return 0
def load_and_validate(text: str, path: Path, schema: Validator):
try:
return load(text, schema, label=path).data
except YAMLError as err:
log.error(err)
raise
def convert(ctx, verbosity, rich, infile, fmt, outfile, **kwargs):
# Set up logging
if rich:
logger = RichOutputLog(sys.stdout)
else:
logger = log.TeeLog(
log.FilterLog(log.StdoutLog(), maxlevel=log.proto.Level.user),
log.FilterLog(log.StderrLog(), minlevel=log.proto.Level.warning),
)
log.current = log.FilterLog(logger,
minlevel=getattr(log.proto.Level, verbosity))
# Print potential warnings
if '--global' in sys.argv:
log.warning(f"--global is deprecated; use --coords geocentric instead")
if '--local' in sys.argv:
log.warning(f"--local is deprecated; use --coords local instead")
if '--geometry' in sys.argv or '-g' in sys.argv:
log.warning(f"--geometry is deprecated; use --coords instead")
if '--endianness' in sys.argv:
log.warning(f"--endianness is deprecated; use --in-endianness instead")
# Convert to pathlib
infile = Path(infile)
outfile = Path(outfile) if outfile else None
# Determine filename of output
if outfile and not fmt:
fmt = outfile.suffix[1:].lower()
elif not outfile:
fmt = fmt or 'pvd'
outfile = Path(infile.name).with_suffix(f'.{fmt}')
# Handle default values of multi-valued options that should be
# distinguished from empty, as well as comma splitting and other
# transformations
explicit = {
click.core.ParameterSource.COMMANDLINE,
click.core.ParameterSource.ENVIRONMENT
}
kwargs['input_coords'] = dict(kwargs['input_coords'])
for k in ['field_filter', 'only_bases']:
kwargs[k] = tuple(split_commas(kwargs[k]))
if kwargs['no_fields']:
kwargs['field_filter'] = []
elif ctx.get_parameter_source('field_filter') not in explicit:
kwargs['field_filter'] = None
else:
kwargs['field_filter'] = tuple(f.lower()
for f in kwargs['field_filter'])
if isinstance(kwargs['timestep_index'], int):
n = kwargs['timestep_index']
kwargs['timestep_slice'] = f'{n}:{n+1}:1'
config.require(multiple_timesteps=False, reason="--time is set")
# Remove meta-options
for k in ['timestep_index', 'no_fields']:
kwargs.pop(k)
try:
# The config can influence the choice of readers or writers,
# so apply it first. Since kwargs may include options that
# are not explicity set by the user, we set the source to
# Default, and later use the upgrade_source method.
with config(source=ConfigSource.Default, **kwargs):
for name in kwargs:
if ctx.get_parameter_source(name) in explicit:
config.upgrade_source(name, ConfigSource.User)
if not infile.exists():
raise IOError(f"File or directory does not exist: {infile}")
ReaderClass = Reader.find_applicable(infile)
WriterClass = Writer.find_applicable(fmt)
with ReaderClass(infile) as reader, WriterClass(outfile) as writer:
reader.validate()
writer.validate()
pipeline(reader, writer)
except Exception as e:
if verbosity == 'debug':
# In debug mode, allow exceptions to filter through in raw form
traceback.print_exc()
else:
log.error(f"Error: {e}")
log.error("More information may be available with --debug")
sys.exit(1)
def show(self):
log.error(str(self))