def run_calc(job_id, oqparam, exports, log_level='info', log_file=None, **kw):
"""
Run a calculation.
:param job_id:
ID of the current job
:param oqparam:
:class:`openquake.commonlib.oqvalidation.OqParam` instance
:param exports:
A comma-separated string of export types.
"""
register_signals()
setproctitle('oq-job-%d' % job_id)
logs.init(job_id, getattr(logging, log_level.upper()))
with logs.handle(job_id, log_level, log_file):
calc = base.calculators(oqparam, calc_id=job_id)
logging.info('%s running %s [--hc=%s]', getpass.getuser(),
calc.oqparam.inputs['job_ini'],
calc.oqparam.hazard_calculation_id)
logging.info('Using engine version %s', __version__)
msg = check_obsolete_version(oqparam.calculation_mode)
if msg:
logging.warning(msg)
calc.from_engine = True
tb = 'None\n'
try:
if OQ_DISTRIBUTE.endswith('pool'):
logging.warning('Using %d cores on %s', parallel.CT // 2,
platform.node())
set_concurrent_tasks_default(calc)
t0 = time.time()
calc.run(exports=exports, **kw)
logging.info('Exposing the outputs to the database')
expose_outputs(calc.datastore)
path = calc.datastore.filename
size = general.humansize(os.path.getsize(path))
logging.info('Stored %s on %s in %d seconds', size, path,
time.time() - t0)
logs.dbcmd('finish', job_id, 'complete')
calc.datastore.close()
for line in logs.dbcmd('list_outputs', job_id, False):
general.safeprint(line)
except BaseException as exc:
if isinstance(exc, MasterKilled):
msg = 'aborted'
else:
msg = 'failed'
tb = traceback.format_exc()
try:
logging.critical(tb)
logs.dbcmd('finish', job_id, msg)
except BaseException: # an OperationalError may always happen
sys.stderr.write(tb)
raise
finally:
parallel.Starmap.shutdown()
# sanity check to make sure that the logging on file is working
if log_file and log_file != os.devnull and os.path.getsize(log_file) == 0:
logging.warning('The log file %s is empty!?' % log_file)
return calc
def build_report(job_ini, output_dir=None):
"""
Write a `report.csv` file with information about the calculation
without running it
:param job_ini:
full pathname of the job.ini file
:param output_dir:
the directory where the report is written (default the input directory)
"""
calc_id = logs.init()
oq = readinput.get_oqparam(job_ini)
if oq.calculation_mode == 'classical':
oq.calculation_mode = 'preclassical'
oq.ground_motion_fields = False
output_dir = output_dir or os.path.dirname(job_ini)
from openquake.calculators import base # ugly
calc = base.calculators(oq, calc_id)
calc.save_params() # needed to save oqparam
# some taken is care so that the real calculation is not run:
# the goal is to extract information about the source management only
calc.pre_execute()
if oq.calculation_mode == 'preclassical':
calc.execute()
rw = ReportWriter(calc.datastore)
rw.make_report()
report = (os.path.join(output_dir, 'report.rst')
if output_dir else calc.datastore.export_path('report.rst'))
try:
rw.save(report)
except IOError as exc: # permission error
sys.stderr.write(str(exc) + '\n')
readinput.exposure = None # ugly hack
return report
def run_job(job_ini, log_level='info', log_file=None, exports='',
username=getpass.getuser(), **kw):
"""
Run a job using the specified config file and other options.
:param str job_ini:
Path to calculation config (INI-style) files.
:param str log_level:
'debug', 'info', 'warn', 'error', or 'critical'
:param str log_file:
Path to log file.
:param exports:
A comma-separated string of export types requested by the user.
:param username:
Name of the user running the job
:param kw:
Extra parameters like hazard_calculation_id and calculation_mode
"""
job_id = logs.init('job', getattr(logging, log_level.upper()))
with logs.handle(job_id, log_level, log_file):
job_ini = os.path.abspath(job_ini)
oqparam = eng.job_from_file(job_ini, job_id, username, **kw)
kw['username'] = username
eng.run_calc(job_id, oqparam, exports, **kw)
for line in logs.dbcmd('list_outputs', job_id, False):
safeprint(line)
return job_id
def _run(job_ini, concurrent_tasks, pdb, reuse_input, loglevel, exports,
params):
global calc_path
if 'hazard_calculation_id' in params:
hc_id = int(params['hazard_calculation_id'])
if hc_id < 0: # interpret negative calculation ids
calc_ids = datastore.get_calc_ids()
try:
params['hazard_calculation_id'] = calc_ids[hc_id]
except IndexError:
raise SystemExit('There are %d old calculations, cannot '
'retrieve the %s' % (len(calc_ids), hc_id))
else:
params['hazard_calculation_id'] = hc_id
dic = readinput.get_params(job_ini, params)
# set the logs first of all
log = logs.init("job", dic, getattr(logging, loglevel.upper()))
# disable gzip_input
base.BaseCalculator.gzip_inputs = lambda self: None
with log, performance.Monitor('total runtime', measuremem=True) as monitor:
calc = base.calculators(log.get_oqparam(), log.calc_id)
if reuse_input: # enable caching
calc.oqparam.cachedir = datastore.get_datadir()
calc.run(concurrent_tasks=concurrent_tasks, pdb=pdb, exports=exports)
logging.info('Total time spent: %s s', monitor.duration)
logging.info('Memory allocated: %s', general.humansize(monitor.mem))
print('See the output with silx view %s' % calc.datastore.filename)
calc_path, _ = os.path.splitext(calc.datastore.filename) # used below
return calc
def main(calc_id: int, aggregate_by):
"""
Re-run the postprocessing after an event based risk calculation
"""
parent = util.read(calc_id)
oqp = parent['oqparam']
aggby = aggregate_by.split(',')
for tagname in aggby:
if tagname not in oqp.aggregate_by:
raise ValueError('%r not in %s' % (tagname, oqp.aggregate_by))
job_id = logs.init('job', level=logging.INFO)
dic = dict(
calculation_mode='reaggregate',
description=oqp.description + '[aggregate_by=%s]' % aggregate_by,
user_name=getpass.getuser(), is_running=1, status='executing',
pid=os.getpid(), hazard_calculation_id=job_id)
logs.dbcmd('update_job', job_id, dic)
if os.environ.get('OQ_DISTRIBUTE') not in ('no', 'processpool'):
os.environ['OQ_DISTRIBUTE'] = 'processpool'
with logs.handle(job_id, logging.INFO):
oqp.hazard_calculation_id = parent.calc_id
parallel.Starmap.init()
prc = PostRiskCalculator(oqp, job_id)
try:
prc.run(aggregate_by=aggby)
engine.expose_outputs(prc.datastore)
logs.dbcmd('finish', job_id, 'complete')
except Exception:
logs.dbcmd('finish', job_id, 'failed')
finally:
parallel.Starmap.shutdown()
def build_report(job_ini, output_dir=None):
"""
Write a `report.csv` file with information about the calculation
without running it
:param job_ini:
full pathname of the job.ini file
:param output_dir:
the directory where the report is written (default the input directory)
"""
calc_id = logs.init()
oq = readinput.get_oqparam(job_ini)
if oq.calculation_mode == 'classical':
oq.calculation_mode = 'preclassical'
oq.ground_motion_fields = False
output_dir = output_dir or os.path.dirname(job_ini)
from openquake.calculators import base # ugly
calc = base.calculators(oq, calc_id)
calc.save_params() # needed to save oqparam
# some taken is care so that the real calculation is not run:
# the goal is to extract information about the source management only
calc.pre_execute()
if oq.calculation_mode == 'preclassical':
calc.execute()
rw = ReportWriter(calc.datastore)
rw.make_report()
report = (os.path.join(output_dir, 'report.rst') if output_dir
else calc.datastore.export_path('report.rst'))
try:
rw.save(report)
except IOError as exc: # permission error
sys.stderr.write(str(exc) + '\n')
readinput.exposure = None # ugly hack
return report
def run_job(job_ini,
log_level='info',
log_file=None,
exports='',
username=getpass.getuser(),
**kw):
"""
Run a job using the specified config file and other options.
:param str job_ini:
Path to calculation config (INI-style) files.
:param str log_level:
'debug', 'info', 'warn', 'error', or 'critical'
:param str log_file:
Path to log file.
:param exports:
A comma-separated string of export types requested by the user.
:param username:
Name of the user running the job
:param kw:
Extra parameters like hazard_calculation_id and calculation_mode
"""
job_id = logs.init('job', getattr(logging, log_level.upper()))
with logs.handle(job_id, log_level, log_file):
job_ini = os.path.abspath(job_ini)
oqparam = eng.job_from_file(job_ini, job_id, username, **kw)
kw['username'] = username
eng.run_calc(job_id, oqparam, exports, **kw)
for line in logs.dbcmd('list_outputs', job_id, False):
safeprint(line)
return job_id
def main(calc_id: int, aggregate_by):
"""
Re-run the postprocessing after an event based risk calculation
"""
parent = datastore.read(calc_id)
oqp = parent['oqparam']
aggby = aggregate_by.split(',')
for tagname in aggby:
if tagname not in oqp.aggregate_by:
raise ValueError('%r not in %s' % (tagname, oqp.aggregate_by))
dic = dict(calculation_mode='reaggregate',
description=oqp.description +
'[aggregate_by=%s]' % aggregate_by,
user_name=getpass.getuser(),
is_running=1,
status='executing',
pid=os.getpid(),
hazard_calculation_id=calc_id)
log = logs.init('job', dic, logging.INFO)
if os.environ.get('OQ_DISTRIBUTE') not in ('no', 'processpool'):
os.environ['OQ_DISTRIBUTE'] = 'processpool'
with log:
oqp.hazard_calculation_id = parent.calc_id
parallel.Starmap.init()
prc = PostRiskCalculator(oqp, log.calc_id)
prc.run(aggregate_by=aggby)
engine.expose_outputs(prc.datastore)
def create_jobs(job_inis,
log_level=logging.INFO,
log_file=None,
user_name=None,
hc_id=None,
multi=False):
"""
Create job records on the database.
:returns: a list of LogContext objects
"""
if len(job_inis) > 1 and not hc_id and not multi: # first job as hc
job = logs.init("job", job_inis[0], log_level, log_file, user_name,
hc_id)
hc_id = job.calc_id
jobs = [job]
job_inis = job_inis[1:]
else:
jobs = []
for job_ini in job_inis:
if isinstance(job_ini, dict):
dic = job_ini
else:
# NB: `get_params` must NOT log, since the logging is not
# configured yet, otherwise the log will disappear :-(
dic = readinput.get_params(job_ini)
dic['hazard_calculation_id'] = hc_id
if 'sensitivity_analysis' in dic:
analysis = valid.dictionary(dic['sensitivity_analysis'])
for values in itertools.product(*analysis.values()):
new = logs.init('job', dic, log_level, None, user_name, hc_id)
pars = dict(zip(analysis, values))
for param, value in pars.items():
new.params[param] = str(value)
new.params['description'] = '%s %s' % (
new.params['description'], pars)
new.params['hazard_calculation_id'] = hc_id
logging.info('Job with %s', pars)
jobs.append(new)
else:
jobs.append(
logs.init('job', dic, log_level, None, user_name, hc_id))
if multi:
for job in jobs:
job.multi = True
return jobs
def main(job_ini_or_zip_or_nrmls):
"""
Check the validity of job.ini files, job.zip files and .xml files.
NB: `oq check_input job_haz.ini job_risk.ini` is special-cased so
that the risk files are checked before the hazard files.
"""
if os.environ.get('OQ_DISTRIBUTE') not in ('no', 'processpool'):
os.environ['OQ_DISTRIBUTE'] = 'processpool'
all_inis = all(f.endswith('.ini') for f in job_ini_or_zip_or_nrmls)
if all_inis: # the typical case is job_haz.ini + job_risk.ini
dic = {}
for ini in job_ini_or_zip_or_nrmls:
for key, val in readinput.get_params(ini).items():
if key == 'inputs' and key in dic:
dic[key].update(val)
else: # the last wins
dic[key] = val
with logs.init('job', dic) as log:
logging.info('Running oq check_input %s',
' '.join(job_ini_or_zip_or_nrmls))
calc = base.calculators(log.get_oqparam(), log.calc_id)
base.BaseCalculator.gzip_inputs = lambda self: None # disable
with mock.patch.dict(os.environ, {'OQ_CHECK_INPUT': '1'}):
calc.read_inputs()
return
for job_ini_or_zip_or_nrml in job_ini_or_zip_or_nrmls:
if job_ini_or_zip_or_nrml.endswith('.xml'):
try:
node = nrml.to_python(job_ini_or_zip_or_nrml)
if node.tag.endswith('exposureModel'):
err = Exposure.check(job_ini_or_zip_or_nrml)
if err:
logging.warning(err)
else:
logging.info('Checked %s', job_ini_or_zip_or_nrml)
except Exception as exc:
sys.exit(exc)
else: # .zip
with logs.init('job', job_ini_or_zip_or_nrml) as log:
path = os.path.abspath(job_ini_or_zip_or_nrml)
logging.info('Running oq check_input %s', path)
calc = base.calculators(log.get_oqparam(), log.calc_id)
base.BaseCalculator.gzip_inputs = lambda self: None # disable
with mock.patch.dict(os.environ, {'OQ_CHECK_INPUT': '1'}):
calc.read_inputs()
def test_recompute(self):
# test recomputing aggregate loss curves with post_risk
self.run_calc(recompute.__file__, 'job.ini')
parent = self.calc.datastore
# the parent has aggregate_by = NAME_1, NAME_2, taxonomy
oq = parent['oqparam']
oq.__dict__['aggregate_by'] = ['NAME_1']
job_id = logs.init('nojob', logging.INFO) # requires the DbServer
prc = PostRiskCalculator(oq, job_id)
oq.hazard_calculation_id = parent.calc_id
with mock.patch.dict(os.environ, {'OQ_DISTRIBUTE': 'no'}):
prc.run()
def submit_job(job_ini, username, **kw):
"""
Create a job object from the given job.ini file in the job directory
and run it in a new process. Returns a PID.
"""
# errors in validating oqparam are reported immediately
params = readinput.get_params(job_ini)
job_id = logs.init('job')
params['_job_id'] = job_id
proc = Process(target=engine.run_jobs,
args=([params], config.distribution.log_level, None,
'', username), kwargs=kw)
proc.start()
return job_id
def run2(job_haz, job_risk, calc_id, concurrent_tasks, pdb, loglevel,
exports, params):
"""
Run both hazard and risk, one after the other
"""
hcalc = base.calculators(readinput.get_oqparam(job_haz), calc_id)
hcalc.run(concurrent_tasks=concurrent_tasks, pdb=pdb,
exports=exports, **params)
hc_id = hcalc.datastore.calc_id
rcalc_id = logs.init(level=getattr(logging, loglevel.upper()))
oq = readinput.get_oqparam(job_risk, hc_id=hc_id)
rcalc = base.calculators(oq, rcalc_id)
rcalc.run(pdb=pdb, exports=exports, **params)
return rcalc
def get_calc(self, testfile, job_ini, **kw):
"""
Return the outputs of the calculation as a dictionary
"""
self.testdir = os.path.dirname(testfile) if os.path.isfile(testfile) \
else testfile
params = readinput.get_params(os.path.join(self.testdir, job_ini), kw)
oqvalidation.OqParam.calculation_mode.validator.choices = tuple(
base.calculators)
oq = oqvalidation.OqParam(**params)
oq.validate()
# change this when debugging the test
log = logs.init('calc', params)
return base.calculators(oq, log.calc_id)
def check_input(job_ini_or_zip_or_nrmls):
for job_ini_or_zip_or_nrml in job_ini_or_zip_or_nrmls:
if job_ini_or_zip_or_nrml.endswith('.xml'):
try:
node = nrml.to_python(job_ini_or_zip_or_nrml)
if node.tag.endswith('exposureModel'):
err = Exposure.check(job_ini_or_zip_or_nrml)
if err:
logging.warning(err)
else:
logging.info('Checked %s', job_ini_or_zip_or_nrml)
except Exception as exc:
sys.exit(exc)
else:
oq = readinput.get_oqparam(job_ini_or_zip_or_nrml)
base.calculators(oq, logs.init()).read_inputs()
def submit_job(job_ini, username, hazard_calculation_id=None):
"""
Create a job object from the given job.ini file in the job directory
and run it in a new process. Returns a PID.
"""
# errors in validating oqparam are reported immediately
params = vars(readinput.get_oqparam(job_ini))
job_id = logs.init('job')
params['_job_id'] = job_id
# errors in the calculation are not reported but are visible in the log
proc = Process(target=engine.run_jobs,
args=([params], config.distribution.log_level, None, '',
username),
kwargs={'hazard_calculation_id': hazard_calculation_id})
proc.start()
return job_id
def _run(job_inis, concurrent_tasks, calc_id, pdb, loglevel, hc, exports,
params):
global calc_path
assert len(job_inis) in (1, 2), job_inis
# set the logs first of all
calc_id = logs.init(calc_id, getattr(logging, loglevel.upper()))
# disable gzip_input
base.BaseCalculator.gzip_inputs = lambda self: None
with performance.Monitor('total runtime', measuremem=True) as monitor:
if os.environ.get('OQ_DISTRIBUTE') not in ('no', 'processpool'):
os.environ['OQ_DISTRIBUTE'] = 'processpool'
if len(job_inis) == 1: # run hazard or risk
if hc:
hc_id = hc[0]
rlz_ids = hc[1:]
else:
hc_id = None
rlz_ids = ()
oqparam = readinput.get_oqparam(job_inis[0], hc_id=hc_id)
if not oqparam.cachedir: # enable caching
oqparam.cachedir = datastore.get_datadir()
if hc_id and hc_id < 0: # interpret negative calculation ids
calc_ids = datastore.get_calc_ids()
try:
hc_id = calc_ids[hc_id]
except IndexError:
raise SystemExit('There are %d old calculations, cannot '
'retrieve the %s' %
(len(calc_ids), hc_id))
calc = base.calculators(oqparam, calc_id)
calc.run(concurrent_tasks=concurrent_tasks,
pdb=pdb,
exports=exports,
hazard_calculation_id=hc_id,
rlz_ids=rlz_ids,
**params)
else: # run hazard + risk
calc = run2(job_inis[0], job_inis[1], calc_id, concurrent_tasks,
pdb, loglevel, exports, params)
logging.info('Total time spent: %s s', monitor.duration)
logging.info('Memory allocated: %s', general.humansize(monitor.mem))
print('See the output with silx view %s' % calc.datastore.filename)
calc_path, _ = os.path.splitext(calc.datastore.filename) # used below
return calc
def run2(job_haz, job_risk, calc_id, concurrent_tasks, pdb, reuse_input,
loglevel, exports, params):
"""
Run both hazard and risk, one after the other
"""
oq = readinput.get_oqparam(job_haz, kw=params)
hcalc = base.calculators(oq, calc_id)
hcalc.run(concurrent_tasks=concurrent_tasks, pdb=pdb, exports=exports)
hcalc.datastore.close()
hc_id = hcalc.datastore.calc_id
rcalc_id = logs.init(level=getattr(logging, loglevel.upper()))
params['hazard_calculation_id'] = str(hc_id)
oq = readinput.get_oqparam(job_risk, kw=params)
rcalc = base.calculators(oq, rcalc_id)
if reuse_input: # enable caching
oq.cachedir = datastore.get_datadir()
rcalc.run(pdb=pdb, exports=exports)
return rcalc
def submit_job(job_ini, username, hazard_job_id=None):
"""
Create a job object from the given job.ini file in the job directory
and run it in a new process. Returns the job ID and PID.
"""
job_id = logs.init('job')
oq = engine.job_from_file(
job_ini, job_id, username, hazard_calculation_id=hazard_job_id)
pik = pickle.dumps(oq, protocol=0) # human readable protocol
code = RUNCALC % dict(job_id=job_id, hazard_job_id=hazard_job_id, pik=pik,
username=username)
tmp_py = gettemp(code, suffix='.py')
# print(code, tmp_py) # useful when debugging
devnull = subprocess.DEVNULL
popen = subprocess.Popen([sys.executable, tmp_py],
stdin=devnull, stdout=devnull, stderr=devnull)
threading.Thread(target=popen.wait).start()
logs.dbcmd('update_job', job_id, {'pid': popen.pid})
return job_id, popen.pid
def __init__(self):
try:
from matplotlib import pyplot
self.plt = pyplot
self.fig, self.ax = pyplot.subplots()
except Exception: # for instance, no Tkinter
pass
self.lookfor = partial(numpy.lookfor, module='openquake')
self.extract = extract
self.read = read
self.nrml = nrml
self.get__exposure = readinput.get_exposure
self.get_oqparam = readinput.get_oqparam
self.get_site_collection = readinput.get_site_collection
self.get_composite_source_model = readinput.get_composite_source_model
self.get_exposure = readinput.get_exposure
self.get_calc = lambda job_ini: get_calc(job_ini, logs.init())
self.make_hmap = calc.make_hmap
self.geodetic_distance = geodetic_distance
def recompute_losses(calc_id, aggregate_by):
"""Re-run the postprocessing after an event based risk calculation"""
parent = util.read(calc_id)
oqp = parent['oqparam']
aggby = aggregate_by.split(',')
for tagname in aggby:
if tagname not in oqp.aggregate_by:
raise ValueError('%r not in %s' % (tagname, oqp.aggregate_by))
job_id = logs.init('job', level=logging.INFO)
if os.environ.get('OQ_DISTRIBUTE') not in ('no', 'processpool'):
os.environ['OQ_DISTRIBUTE'] = 'processpool'
with logs.handle(job_id, logging.INFO):
oqp.hazard_calculation_id = calc_id
parallel.Starmap.init()
prc = PostRiskCalculator(oqp, job_id)
try:
prc.run(aggregate_by=aggby)
finally:
parallel.Starmap.shutdown()
def main(job_ini_or_zip_or_nrmls):
if os.environ.get('OQ_DISTRIBUTE') not in ('no', 'processpool'):
os.environ['OQ_DISTRIBUTE'] = 'processpool'
for job_ini_or_zip_or_nrml in job_ini_or_zip_or_nrmls:
if job_ini_or_zip_or_nrml.endswith('.xml'):
try:
node = nrml.to_python(job_ini_or_zip_or_nrml)
if node.tag.endswith('exposureModel'):
err = Exposure.check(job_ini_or_zip_or_nrml)
if err:
logging.warning(err)
else:
logging.info('Checked %s', job_ini_or_zip_or_nrml)
except Exception as exc:
sys.exit(exc)
else:
with logs.init('calc', job_ini_or_zip_or_nrml) as log:
calc = base.calculators(log.get_oqparam(), log.calc_id)
base.BaseCalculator.gzip_inputs = lambda self: None # disable
with mock.patch.dict(os.environ, {'OQ_CHECK_INPUT': '1'}):
calc.read_inputs()
def test_recompute(self):
# test recomputing aggregate loss curves with post_risk
# this is starting from a ruptures.csv file
out = self.run_calc(recompute.__file__, 'job.ini', exports='csv')
[fname] = out['agg_losses-rlzs', 'csv']
self.assertEqualFiles('expected/agg_losses.csv', fname, delta=1E-5)
[fname] = out['agg_curves-rlzs', 'csv']
self.assertEqualFiles('expected/agg_curves.csv', fname, delta=1E-5)
parent = self.calc.datastore
# the parent has aggregate_by = NAME_1, NAME_2, taxonomy
oq = parent['oqparam']
oq.__dict__['aggregate_by'] = ['NAME_1']
job_id = logs.init('nojob', logging.INFO) # requires the DbServer
prc = PostRiskCalculator(oq, job_id)
oq.hazard_calculation_id = parent.calc_id
with mock.patch.dict(os.environ, {'OQ_DISTRIBUTE': 'no'}):
prc.run()
[fname] = export(('agg_losses-rlzs', 'csv'), prc.datastore)
self.assertEqualFiles('expected/recomputed_losses.csv', fname,
delta=1E-5)
def _run(job_inis, concurrent_tasks, pdb, loglevel, hc, exports, params):
global calc_path
assert len(job_inis) in (1, 2), job_inis
# set the logs first of all
calc_id = logs.init(level=getattr(logging, loglevel.upper()))
with performance.Monitor('total runtime', measuremem=True) as monitor:
if len(job_inis) == 1: # run hazard or risk
if hc:
hc_id = hc[0]
rlz_ids = hc[1:]
else:
hc_id = None
rlz_ids = ()
oqparam = readinput.get_oqparam(job_inis[0], hc_id=hc_id)
vars(oqparam).update(params)
if hc_id and hc_id < 0: # interpret negative calculation ids
calc_ids = datastore.get_calc_ids()
try:
hc_id = calc_ids[hc_id]
except IndexError:
raise SystemExit('There are %d old calculations, cannot '
'retrieve the %s' %
(len(calc_ids), hc_id))
calc = base.calculators(oqparam, calc_id)
calc.run(concurrent_tasks=concurrent_tasks,
pdb=pdb,
exports=exports,
hazard_calculation_id=hc_id,
rlz_ids=rlz_ids)
else: # run hazard + risk
calc = run2(job_inis[0], job_inis[1], calc_id, concurrent_tasks,
pdb, loglevel, exports, params)
logging.info('Total time spent: %s s', monitor.duration)
logging.info('Memory allocated: %s', general.humansize(monitor.mem))
print('See the output with hdfview %s' % calc.datastore.hdf5path)
calc_path, _ = os.path.splitext(calc.datastore.hdf5path) # used below
return calc
def test_recompute(self):
# test recomputing aggregate loss curves with post_risk
# this is starting from a ruptures.csv file
out = self.run_calc(recompute.__file__, 'job.ini', exports='csv')
[fname] = out['aggrisk', 'csv']
self.assertEqualFiles('expected/agg_losses.csv', fname, delta=1E-5)
[fname] = out['aggcurves', 'csv']
self.assertEqualFiles('expected/aggcurves.csv', fname, delta=1E-5)
parent = self.calc.datastore
# the parent has aggregate_by = NAME_1, NAME_2, taxonomy
oq = parent['oqparam']
oq.__dict__['aggregate_by'] = ['NAME_1']
log = logs.init('job', {'calculation_mode': 'post_risk',
'description': 'test recompute'})
prc = PostRiskCalculator(oq, log.calc_id)
prc.assetcol = self.calc.assetcol
oq.hazard_calculation_id = parent.calc_id
with mock.patch.dict(os.environ, {'OQ_DISTRIBUTE': 'no'}), log:
prc.run()
[fname] = export(('aggrisk', 'csv'), prc.datastore)
self.assertEqualFiles('expected/recomputed_losses.csv', fname,
delta=1E-5)
def _run(job_inis, concurrent_tasks, pdb, loglevel, hc, exports, params):
global calc_path
assert len(job_inis) in (1, 2), job_inis
# set the logs first of all
calc_id = logs.init(level=getattr(logging, loglevel.upper()))
with performance.Monitor('total runtime', measuremem=True) as monitor:
if len(job_inis) == 1: # run hazard or risk
if hc:
hc_id = hc[0]
rlz_ids = hc[1:]
else:
hc_id = None
rlz_ids = ()
oqparam = readinput.get_oqparam(job_inis[0], hc_id=hc_id)
vars(oqparam).update(params)
if hc_id and hc_id < 0: # interpret negative calculation ids
calc_ids = datastore.get_calc_ids()
try:
hc_id = calc_ids[hc_id]
except IndexError:
raise SystemExit(
'There are %d old calculations, cannot '
'retrieve the %s' % (len(calc_ids), hc_id))
calc = base.calculators(oqparam, calc_id)
calc.run(concurrent_tasks=concurrent_tasks, pdb=pdb,
exports=exports, hazard_calculation_id=hc_id,
rlz_ids=rlz_ids)
else: # run hazard + risk
calc = run2(
job_inis[0], job_inis[1], calc_id, concurrent_tasks, pdb,
loglevel, exports, params)
logging.info('Total time spent: %s s', monitor.duration)
logging.info('Memory allocated: %s', general.humansize(monitor.mem))
print('See the output with silx view %s' % calc.datastore.filename)
calc_path, _ = os.path.splitext(calc.datastore.filename) # used below
return calc
def get_calc(self, job_ini):
log = logs.init("job", job_ini)
log.__enter__()
return calculators(log.get_oqparam(), log.calc_id)
def engine(log_file, no_distribute, yes, config_file, make_html_report,
upgrade_db, db_version, what_if_I_upgrade, run,
list_hazard_calculations, list_risk_calculations,
delete_calculation, delete_uncompleted_calculations,
hazard_calculation_id, list_outputs, show_log,
export_output, export_outputs, exports='',
log_level='info', reuse_hazard=False):
"""
Run a calculation using the traditional command line API
"""
if not run:
# configure a basic logging
logs.init()
if config_file:
config.read(os.path.abspath(os.path.expanduser(config_file)),
soft_mem_limit=int, hard_mem_limit=int, port=int,
multi_user=valid.boolean, multi_node=valid.boolean)
if no_distribute:
os.environ['OQ_DISTRIBUTE'] = 'no'
# check if the datadir exists
datadir = datastore.get_datadir()
if not os.path.exists(datadir):
os.makedirs(datadir)
dbserver.ensure_on()
# check if we are talking to the right server
err = dbserver.check_foreign()
if err:
sys.exit(err)
if upgrade_db:
msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
if msg.startswith('Your database is already updated'):
pass
elif yes or confirm('Proceed? (y/n) '):
logs.dbcmd('upgrade_db')
sys.exit(0)
if db_version:
safeprint(logs.dbcmd('db_version'))
sys.exit(0)
if what_if_I_upgrade:
safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
sys.exit(0)
# check if the db is outdated
outdated = logs.dbcmd('check_outdated')
if outdated:
sys.exit(outdated)
# hazard or hazard+risk
if hazard_calculation_id == -1:
# get the latest calculation of the current user
hc_id = get_job_id(hazard_calculation_id, getpass.getuser())
elif hazard_calculation_id:
# make it possible to use calculations made by another user
hc_id = get_job_id(hazard_calculation_id)
else:
hc_id = None
if run:
log_file = os.path.expanduser(log_file) \
if log_file is not None else None
job_inis = [os.path.expanduser(f) for f in run]
if len(job_inis) == 1 and not hc_id:
# init logs before calling get_oqparam
logs.init('nojob', getattr(logging, log_level.upper()))
# not using logs.handle that logs on the db
oq = readinput.get_oqparam(job_inis[0])
smart_run(job_inis[0], oq, log_level, log_file,
exports, reuse_hazard)
return
for i, job_ini in enumerate(job_inis):
open(job_ini, 'rb').read() # IOError if the file does not exist
job_id = run_job(job_ini, log_level, log_file,
exports, hazard_calculation_id=hc_id)
if not hc_id: # use the first calculation as base for the others
hc_id = job_id
# hazard
elif list_hazard_calculations:
for line in logs.dbcmd(
'list_calculations', 'hazard', getpass.getuser()):
safeprint(line)
elif delete_calculation is not None:
del_calculation(delete_calculation, yes)
# risk
elif list_risk_calculations:
for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
safeprint(line)
# export
elif make_html_report:
safeprint('Written %s' % make_report(make_html_report))
sys.exit(0)
elif list_outputs is not None:
hc_id = get_job_id(list_outputs)
for line in logs.dbcmd('list_outputs', hc_id):
safeprint(line)
elif show_log is not None:
hc_id = get_job_id(show_log)
for line in logs.dbcmd('get_log', hc_id):
safeprint(line)
elif export_output is not None:
output_id, target_dir = export_output
dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
for line in core.export_output(
dskey, calc_id, datadir, os.path.expanduser(target_dir),
exports or 'csv,xml'):
safeprint(line)
elif export_outputs is not None:
job_id, target_dir = export_outputs
hc_id = get_job_id(job_id)
for line in core.export_outputs(
hc_id, os.path.expanduser(target_dir), exports or 'csv,xml'):
safeprint(line)
elif delete_uncompleted_calculations:
logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())
else:
engine.parentparser.prog = 'oq engine'
engine.parentparser.print_usage()
def _init_logs(dic, lvl):
if '_job_id' in dic:
logs.init(dic['_job_id'], lvl)
else:
dic['_job_id'] = logs.init('job', lvl)
def post_execute(self, result):
oq = self.oqparam
if not oq.ground_motion_fields:
return
N = len(self.sitecol.complete)
L = len(oq.imtls.array)
if result and oq.hazard_curves_from_gmfs:
rlzs = self.rlzs_assoc.realizations
# compute and save statistics; this is done in process and can
# be very slow if there are thousands of realizations
weights = [rlz.weight for rlz in rlzs]
# NB: in the future we may want to save to individual hazard
# curves if oq.individual_curves is set; for the moment we
# save the statistical curves only
hstats = oq.hazard_stats()
S = len(hstats)
pmaps = list(result.values())
R = len(weights)
if len(pmaps) != R:
# this should never happen, unless I break the
# logic tree reduction mechanism during refactoring
raise AssertionError('Expected %d pmaps, got %d' %
(len(weights), len(pmaps)))
if oq.individual_curves:
logging.info('Saving individual hazard curves')
self.datastore.create_dset('hcurves-rlzs', F32, (N, R, L))
self.datastore.set_attrs('hcurves-rlzs', nbytes=N * R * L * 4)
if oq.poes:
P = len(oq.poes)
M = len(oq.imtls)
ds = self.datastore.create_dset(
'hmaps-rlzs', F32, (N, R, M, P))
self.datastore.set_attrs(
'hmaps-rlzs', nbytes=N * R * P * M * 4)
for r, pmap in enumerate(pmaps):
arr = numpy.zeros((N, L), F32)
for sid in pmap:
arr[sid] = pmap[sid].array[:, 0]
self.datastore['hcurves-rlzs'][:, r] = arr
if oq.poes:
hmap = calc.make_hmap(pmap, oq.imtls, oq.poes)
for sid in hmap:
ds[sid, r] = hmap[sid].array
if S:
logging.info('Computing statistical hazard curves')
self.datastore.create_dset('hcurves-stats', F32, (N, S, L))
self.datastore.set_attrs('hcurves-stats', nbytes=N * S * L * 4)
if oq.poes:
P = len(oq.poes)
M = len(oq.imtls)
ds = self.datastore.create_dset(
'hmaps-stats', F32, (N, S, M, P))
self.datastore.set_attrs(
'hmaps-stats', nbytes=N * S * P * M * 4)
for s, stat in enumerate(hstats):
pmap = compute_pmap_stats(
pmaps, [hstats[stat]], weights, oq.imtls)
arr = numpy.zeros((N, L), F32)
for sid in pmap:
arr[sid] = pmap[sid].array[:, 0]
self.datastore['hcurves-stats'][:, s] = arr
if oq.poes:
hmap = calc.make_hmap(pmap, oq.imtls, oq.poes)
for sid in hmap:
ds[sid, s] = hmap[sid].array
if self.datastore.parent:
self.datastore.parent.open('r')
if oq.compare_with_classical: # compute classical curves
export_dir = os.path.join(oq.export_dir, 'cl')
if not os.path.exists(export_dir):
os.makedirs(export_dir)
oq.export_dir = export_dir
job_id = logs.init('job')
self.cl = ClassicalCalculator(oq, job_id)
# TODO: perhaps it is possible to avoid reprocessing the source
# model, however usually this is quite fast and do not dominate
# the computation
self.cl.run(close=False)
engine.expose_outputs(self.cl.datastore)
cl_mean_curves = get_mean_curves(self.cl.datastore)
eb_mean_curves = get_mean_curves(self.datastore)
self.rdiff, index = util.max_rel_diff_index(
cl_mean_curves, eb_mean_curves)
logging.warning('Relative difference with the classical '
'mean curves: %d%% at site index %d',
self.rdiff * 100, index)
def main(no_distribute=False,
yes=False,
upgrade_db=False,
db_version=False,
what_if_I_upgrade=False,
list_hazard_calculations=False,
list_risk_calculations=False,
delete_uncompleted_calculations=False,
multi=False,
reuse_input=False,
*,
log_file=None,
make_html_report=None,
run=None,
delete_calculation: int = None,
hazard_calculation_id: int = None,
list_outputs: int = None,
show_log=None,
export_output=None,
export_outputs=None,
param='',
config_file=None,
exports='',
log_level='info'):
"""
Run a calculation using the traditional command line API
"""
if not run:
# configure a basic logging
logs.init()
if config_file:
config.read(os.path.abspath(os.path.expanduser(config_file)),
soft_mem_limit=int,
hard_mem_limit=int,
port=int,
multi_user=valid.boolean,
serialize_jobs=valid.boolean,
strict=valid.boolean,
code=exec)
if no_distribute:
os.environ['OQ_DISTRIBUTE'] = 'no'
# check if the datadir exists
datadir = datastore.get_datadir()
if not os.path.exists(datadir):
os.makedirs(datadir)
dbserver.ensure_on()
# check if we are talking to the right server
err = dbserver.check_foreign()
if err:
sys.exit(err)
if upgrade_db:
msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
if msg.startswith('Your database is already updated'):
pass
elif yes or confirm('Proceed? (y/n) '):
logs.dbcmd('upgrade_db')
sys.exit(0)
if db_version:
safeprint(logs.dbcmd('db_version'))
sys.exit(0)
if what_if_I_upgrade:
safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
sys.exit(0)
# check if the db is outdated
outdated = logs.dbcmd('check_outdated')
if outdated:
sys.exit(outdated)
# hazard or hazard+risk
if hazard_calculation_id == -1:
# get the latest calculation of the current user
hc_id = get_job_id(hazard_calculation_id, getpass.getuser())
elif hazard_calculation_id:
# make it possible to use calculations made by another user
hc_id = get_job_id(hazard_calculation_id)
else:
hc_id = None
if run:
pars = dict(p.split('=', 1) for p in param.split(',')) if param else {}
if reuse_input:
pars['cachedir'] = datadir
if hc_id:
pars['hazard_calculation_id'] = str(hc_id)
log_file = os.path.expanduser(log_file) \
if log_file is not None else None
job_inis = [os.path.expanduser(f) for f in run]
pars['multi'] = multi
run_jobs(job_inis, log_level, log_file, exports, **pars)
# hazard
elif list_hazard_calculations:
for line in logs.dbcmd('list_calculations', 'hazard',
getpass.getuser()):
safeprint(line)
elif delete_calculation is not None:
del_calculation(delete_calculation, yes)
# risk
elif list_risk_calculations:
for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
safeprint(line)
# export
elif make_html_report:
safeprint('Written %s' % make_report(make_html_report))
sys.exit(0)
elif list_outputs is not None:
hc_id = get_job_id(list_outputs)
for line in logs.dbcmd('list_outputs', hc_id):
safeprint(line)
elif show_log is not None:
hc_id = get_job_id(show_log)
for line in logs.dbcmd('get_log', hc_id):
safeprint(line)
elif export_output is not None:
output_id, target_dir = export_output
dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
for line in core.export_output(dskey, calc_id, datadir,
os.path.expanduser(target_dir), exports
or DEFAULT_EXPORTS):
safeprint(line)
elif export_outputs is not None:
job_id, target_dir = export_outputs
hc_id = get_job_id(job_id)
for line in core.export_outputs(hc_id, os.path.expanduser(target_dir),
exports or DEFAULT_EXPORTS):
safeprint(line)
elif delete_uncompleted_calculations:
logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())
else:
print("Please pass some option, see oq engine --help")
def _init_logs(dic, lvl):
if '_job_id' in dic: # reuse job_id
logs.init(dic['_job_id'], lvl)
else: # create a new job_id
dic['_job_id'] = logs.init('job', lvl)
def run_jobs(job_inis, log_level='info', log_file=None, exports='',
username=getpass.getuser(), **kw):
"""
Run jobs using the specified config file and other options.
:param str job_inis:
A list of paths to .ini files.
:param str log_level:
'debug', 'info', 'warn', 'error', or 'critical'
:param str log_file:
Path to log file.
:param exports:
A comma-separated string of export types requested by the user.
:param username:
Name of the user running the job
:param kw:
Extra parameters like hazard_calculation_id and calculation_mode
"""
dist = parallel.oq_distribute()
jobparams = []
for job_ini in job_inis:
# NB: the logs must be initialized BEFORE everything
job_id = logs.init('job', getattr(logging, log_level.upper()))
with logs.handle(job_id, log_level, log_file):
oqparam = eng.job_from_file(os.path.abspath(job_ini), job_id,
username, **kw)
if (not jobparams and 'csm_cache' not in kw
and 'hazard_calculation_id' not in kw):
kw['hazard_calculation_id'] = job_id
jobparams.append((job_id, oqparam))
jobarray = len(jobparams) > 1 and 'csm_cache' in kw
try:
eng.poll_queue(job_id, poll_time=15)
# wait for an empty slot or a CTRL-C
except BaseException:
# the job aborted even before starting
for job_id, oqparam in jobparams:
logs.dbcmd('finish', job_id, 'aborted')
return jobparams
else:
for job_id, oqparam in jobparams:
dic = {'status': 'executing', 'pid': eng._PID}
if jobarray:
dic['hazard_calculation_id'] = jobparams[0][0]
logs.dbcmd('update_job', job_id, dic)
try:
if dist == 'zmq' and config.zworkers['host_cores']:
logging.info('Asking the DbServer to start the workers')
logs.dbcmd('zmq_start') # start the zworkers
logs.dbcmd('zmq_wait') # wait for them to go up
allargs = [(job_id, oqparam, exports, log_level, log_file)
for job_id, oqparam in jobparams]
if jobarray:
with start_many(eng.run_calc, allargs):
pass
else:
for args in allargs:
eng.run_calc(*args)
finally:
if dist == 'zmq' and config.zworkers['host_cores']:
logging.info('Stopping the zworkers')
logs.dbcmd('zmq_stop')
elif dist.startswith('celery'):
eng.celery_cleanup(config.distribution.terminate_workers_on_revoke)
return jobparams
def post_execute(self, result):
oq = self.oqparam
if not oq.ground_motion_fields and not oq.hazard_curves_from_gmfs:
return
N = len(self.sitecol.complete)
M = len(oq.imtls)
L = len(oq.imtls.array)
L1 = L // M
if result and oq.hazard_curves_from_gmfs:
rlzs = self.datastore['full_lt'].get_realizations()
# compute and save statistics; this is done in process and can
# be very slow if there are thousands of realizations
weights = [rlz.weight for rlz in rlzs]
# NB: in the future we may want to save to individual hazard
# curves if oq.individual_curves is set; for the moment we
# save the statistical curves only
hstats = oq.hazard_stats()
S = len(hstats)
pmaps = list(result.values())
R = len(weights)
if len(pmaps) != R:
# this should never happen, unless I break the
# logic tree reduction mechanism during refactoring
raise AssertionError('Expected %d pmaps, got %d' %
(len(weights), len(pmaps)))
if oq.individual_curves:
logging.info('Saving individual hazard curves')
self.datastore.create_dset('hcurves-rlzs', F32, (N, R, M, L1))
self.datastore.set_shape_attrs('hcurves-rlzs',
site_id=N,
rlz_id=R,
imt=list(oq.imtls),
lvl=numpy.arange(L1))
if oq.poes:
P = len(oq.poes)
M = len(oq.imtls)
ds = self.datastore.create_dset('hmaps-rlzs', F32,
(N, R, M, P))
self.datastore.set_shape_attrs('hmaps-rlzs',
site_id=N,
rlz_id=R,
imt=list(oq.imtls),
poe=oq.poes)
for r, pmap in enumerate(pmaps):
arr = numpy.zeros((N, M, L1), F32)
for sid in pmap:
arr[sid] = pmap[sid].array.reshape(M, L1)
self.datastore['hcurves-rlzs'][:, r] = arr
if oq.poes:
hmap = calc.make_hmap(pmap, oq.imtls, oq.poes)
for sid in hmap:
ds[sid, r] = hmap[sid].array
if S:
logging.info('Computing statistical hazard curves')
self.datastore.create_dset('hcurves-stats', F32, (N, S, M, L1))
self.datastore.set_shape_attrs('hcurves-stats',
site_id=N,
stat=list(hstats),
imt=list(oq.imtls),
lvl=numpy.arange(L1))
if oq.poes:
P = len(oq.poes)
M = len(oq.imtls)
ds = self.datastore.create_dset('hmaps-stats', F32,
(N, S, M, P))
self.datastore.set_shape_attrs('hmaps-stats',
site_id=N,
stat=list(hstats),
imt=list(oq.imtls),
poes=oq.poes)
for s, stat in enumerate(hstats):
pmap = compute_pmap_stats(pmaps, [hstats[stat]], weights,
oq.imtls)
arr = numpy.zeros((N, M, L1), F32)
for sid in pmap:
arr[sid] = pmap[sid].array.reshape(M, L1)
self.datastore['hcurves-stats'][:, s] = arr
if oq.poes:
hmap = calc.make_hmap(pmap, oq.imtls, oq.poes)
for sid in hmap:
ds[sid, s] = hmap[sid].array
if self.datastore.parent:
self.datastore.parent.open('r')
if oq.compare_with_classical: # compute classical curves
export_dir = os.path.join(oq.export_dir, 'cl')
if not os.path.exists(export_dir):
os.makedirs(export_dir)
oq.export_dir = export_dir
job_id = logs.init('job')
oq.calculation_mode = 'classical'
self.cl = ClassicalCalculator(oq, job_id)
# TODO: perhaps it is possible to avoid reprocessing the source
# model, however usually this is quite fast and do not dominate
# the computation
self.cl.run()
engine.expose_outputs(self.datastore)
for imt in oq.imtls:
cl_mean_curves = get_mean_curves(self.datastore, imt)
eb_mean_curves = get_mean_curves(self.datastore, imt)
self.rdiff, index = util.max_rel_diff_index(
cl_mean_curves, eb_mean_curves)
logging.warning(
'Relative difference with the classical '
'mean curves: %d%% at site index %d, imt=%s',
self.rdiff * 100, index, imt)
def engine(log_file,
no_distribute,
yes,
config_file,
make_html_report,
upgrade_db,
db_version,
what_if_I_upgrade,
run,
list_hazard_calculations,
list_risk_calculations,
delete_calculation,
delete_uncompleted_calculations,
hazard_calculation_id,
list_outputs,
show_log,
export_output,
export_outputs,
exports='',
log_level='info',
reuse_hazard=False):
"""
Run a calculation using the traditional command line API
"""
if not run:
# configure a basic logging
logs.init()
if config_file:
config.read(os.path.abspath(os.path.expanduser(config_file)),
soft_mem_limit=int,
hard_mem_limit=int,
port=int,
multi_user=valid.boolean,
multi_node=valid.boolean)
if no_distribute:
os.environ['OQ_DISTRIBUTE'] = 'no'
# check if the datadir exists
datadir = datastore.get_datadir()
if not os.path.exists(datadir):
os.makedirs(datadir)
dbserver.ensure_on()
# check if we are talking to the right server
err = dbserver.check_foreign()
if err:
sys.exit(err)
if upgrade_db:
msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
if msg.startswith('Your database is already updated'):
pass
elif yes or confirm('Proceed? (y/n) '):
logs.dbcmd('upgrade_db')
sys.exit(0)
if db_version:
safeprint(logs.dbcmd('db_version'))
sys.exit(0)
if what_if_I_upgrade:
safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
sys.exit(0)
# check if the db is outdated
outdated = logs.dbcmd('check_outdated')
if outdated:
sys.exit(outdated)
# hazard or hazard+risk
if hazard_calculation_id == -1:
# get the latest calculation of the current user
hc_id = get_job_id(hazard_calculation_id, getpass.getuser())
elif hazard_calculation_id:
# make it possible to use calculations made by another user
hc_id = get_job_id(hazard_calculation_id)
else:
hc_id = None
if run:
log_file = os.path.expanduser(log_file) \
if log_file is not None else None
job_inis = [os.path.expanduser(f) for f in run]
if len(job_inis) == 1 and not hc_id:
# init logs before calling get_oqparam
logs.init('nojob', getattr(logging, log_level.upper()))
# not using logs.handle that logs on the db
oq = readinput.get_oqparam(job_inis[0])
smart_run(job_inis[0], oq, log_level, log_file, exports,
reuse_hazard)
return
for i, job_ini in enumerate(job_inis):
open(job_ini, 'rb').read() # IOError if the file does not exist
job_id = run_job(job_ini,
log_level,
log_file,
exports,
hazard_calculation_id=hc_id)
if not hc_id: # use the first calculation as base for the others
hc_id = job_id
# hazard
elif list_hazard_calculations:
for line in logs.dbcmd('list_calculations', 'hazard',
getpass.getuser()):
safeprint(line)
elif delete_calculation is not None:
del_calculation(delete_calculation, yes)
# risk
elif list_risk_calculations:
for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
safeprint(line)
# export
elif make_html_report:
safeprint('Written %s' % make_report(make_html_report))
sys.exit(0)
elif list_outputs is not None:
hc_id = get_job_id(list_outputs)
for line in logs.dbcmd('list_outputs', hc_id):
safeprint(line)
elif show_log is not None:
hc_id = get_job_id(show_log)
for line in logs.dbcmd('get_log', hc_id):
safeprint(line)
elif export_output is not None:
output_id, target_dir = export_output
dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
for line in core.export_output(dskey, calc_id, datadir,
os.path.expanduser(target_dir), exports
or 'csv,xml'):
safeprint(line)
elif export_outputs is not None:
job_id, target_dir = export_outputs
hc_id = get_job_id(job_id)
for line in core.export_outputs(hc_id, os.path.expanduser(target_dir),
exports or 'csv,xml'):
safeprint(line)
elif delete_uncompleted_calculations:
logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())
else:
engine.parentparser.prog = 'oq engine'
engine.parentparser.print_usage()
