def run_job(job_ini, log_level='info', log_file=None, exports='',
username=getpass.getuser(), **kw):
"""
Run a job using the specified config file and other options.
:param str job_ini:
Path to calculation config (INI-style) files.
:param str log_level:
'debug', 'info', 'warn', 'error', or 'critical'
:param str log_file:
Path to log file.
:param exports:
A comma-separated string of export types requested by the user.
:param username:
Name of the user running the job
:param kw:
Extra parameters like hazard_calculation_id and calculation_mode
"""
job_id = logs.init('job', getattr(logging, log_level.upper()))
with logs.handle(job_id, log_level, log_file):
job_ini = os.path.abspath(job_ini)
oqparam = eng.job_from_file(job_ini, job_id, username, **kw)
kw['username'] = username
eng.run_calc(job_id, oqparam, exports, **kw)
for line in logs.dbcmd('list_outputs', job_id, False):
safeprint(line)
return job_id
def restore(archive, oqdata):
"""
Build a new oqdata directory from the data contained in the zip archive
"""
t0 = time.time()
oqdata = os.path.abspath(oqdata)
assert archive.endswith('.zip'), archive
if os.path.exists(oqdata):
sys.exit('%s exists already' % oqdata)
os.mkdir(oqdata)
zipfile.ZipFile(archive).extractall(oqdata)
dbpath = os.path.join(oqdata, 'db.sqlite3')
db = Db(sqlite3.connect, dbpath, isolation_level=None,
detect_types=sqlite3.PARSE_DECLTYPES)
n = 0
for fname in os.listdir(oqdata):
mo = re.match('calc_(\d+)\.hdf5', fname)
if mo:
job_id = int(mo.group(1))
fullname = os.path.join(oqdata, fname)[:-5] # strip .hdf5
db("UPDATE job SET ds_calc_dir=?x WHERE id=?x", fullname, job_id)
safeprint('Restoring ' + fname)
n += 1
dt = time.time() - t0
safeprint('Extracted %d calculations into %s in %d seconds'
% (n, oqdata, dt))
def restore(archive, oqdata):
"""
Build a new oqdata directory from the data contained in the zip archive
"""
if os.path.exists(oqdata):
sys.exit('%s exists already' % oqdata)
if '://' in archive:
# get the zip archive from an URL
resp = requests.get(archive)
_, archive = archive.rsplit('/', 1)
with open(archive, 'wb') as f:
f.write(resp.content)
if not os.path.exists(archive):
sys.exit('%s does not exist' % archive)
t0 = time.time()
oqdata = os.path.abspath(oqdata)
assert archive.endswith('.zip'), archive
os.mkdir(oqdata)
zipfile.ZipFile(archive).extractall(oqdata)
dbpath = os.path.join(oqdata, 'db.sqlite3')
db = Db(sqlite3.connect, dbpath, isolation_level=None,
detect_types=sqlite3.PARSE_DECLTYPES)
n = 0
for fname in os.listdir(oqdata):
mo = re.match('calc_(\d+)\.hdf5', fname)
if mo:
job_id = int(mo.group(1))
fullname = os.path.join(oqdata, fname)[:-5] # strip .hdf5
db("UPDATE job SET user_name=?x, ds_calc_dir=?x WHERE id=?x",
getpass.getuser(), fullname, job_id)
safeprint('Restoring ' + fname)
n += 1
dt = time.time() - t0
safeprint('Extracted %d calculations into %s in %d seconds'
% (n, oqdata, dt))
def run_job(cfg_file, log_level='info', log_file=None, exports='',
hazard_calculation_id=None, **kw):
"""
Run a job using the specified config file and other options.
:param str cfg_file:
Path to calculation config (INI-style) files.
:param str log_level:
'debug', 'info', 'warn', 'error', or 'critical'
:param str log_file:
Path to log file.
:param exports:
A comma-separated string of export types requested by the user.
:param hazard_calculation_id:
ID of the previous calculation or None
"""
# if the master dies, automatically kill the workers
job_ini = os.path.abspath(cfg_file)
job_id, oqparam = eng.job_from_file(
job_ini, getpass.getuser(), hazard_calculation_id)
calc = eng.run_calc(job_id, oqparam, log_level, log_file, exports,
hazard_calculation_id=hazard_calculation_id, **kw)
calc._monitor.flush()
for line in logs.dbcmd('list_outputs', job_id, False):
safeprint(line)
return job_id
def run_calc(job_id, oqparam, exports, log_level='info', log_file=None, **kw):
"""
Run a calculation.
:param job_id:
ID of the current job
:param oqparam:
:class:`openquake.commonlib.oqvalidation.OqParam` instance
:param exports:
A comma-separated string of export types.
"""
register_signals()
setproctitle('oq-job-%d' % job_id)
logs.init(job_id, getattr(logging, log_level.upper()))
with logs.handle(job_id, log_level, log_file):
calc = base.calculators(oqparam, calc_id=job_id)
logging.info('%s running %s [--hc=%s]', getpass.getuser(),
calc.oqparam.inputs['job_ini'],
calc.oqparam.hazard_calculation_id)
logging.info('Using engine version %s', __version__)
msg = check_obsolete_version(oqparam.calculation_mode)
if msg:
logging.warning(msg)
calc.from_engine = True
tb = 'None\n'
try:
if OQ_DISTRIBUTE.endswith('pool'):
logging.warning('Using %d cores on %s', parallel.CT // 2,
platform.node())
set_concurrent_tasks_default(calc)
t0 = time.time()
calc.run(exports=exports, **kw)
logging.info('Exposing the outputs to the database')
expose_outputs(calc.datastore)
path = calc.datastore.filename
size = general.humansize(os.path.getsize(path))
logging.info('Stored %s on %s in %d seconds', size, path,
time.time() - t0)
logs.dbcmd('finish', job_id, 'complete')
calc.datastore.close()
for line in logs.dbcmd('list_outputs', job_id, False):
general.safeprint(line)
except BaseException as exc:
if isinstance(exc, MasterKilled):
msg = 'aborted'
else:
msg = 'failed'
tb = traceback.format_exc()
try:
logging.critical(tb)
logs.dbcmd('finish', job_id, msg)
except BaseException: # an OperationalError may always happen
sys.stderr.write(tb)
raise
finally:
parallel.Starmap.shutdown()
# sanity check to make sure that the logging on file is working
if log_file and log_file != os.devnull and os.path.getsize(log_file) == 0:
logging.warning('The log file %s is empty!?' % log_file)
return calc
def run_job(job_ini,
log_level='info',
log_file=None,
exports='',
username=getpass.getuser(),
**kw):
"""
Run a job using the specified config file and other options.
:param str job_ini:
Path to calculation config (INI-style) files.
:param str log_level:
'debug', 'info', 'warn', 'error', or 'critical'
:param str log_file:
Path to log file.
:param exports:
A comma-separated string of export types requested by the user.
:param username:
Name of the user running the job
:param kw:
Extra parameters like hazard_calculation_id and calculation_mode
"""
job_id = logs.init('job', getattr(logging, log_level.upper()))
with logs.handle(job_id, log_level, log_file):
job_ini = os.path.abspath(job_ini)
oqparam = eng.job_from_file(job_ini, job_id, username, **kw)
kw['username'] = username
eng.run_calc(job_id, oqparam, exports, **kw)
for line in logs.dbcmd('list_outputs', job_id, False):
safeprint(line)
return job_id
def del_calculation(job_id, confirmed=False):
"""
Delete a calculation and all associated outputs.
"""
if confirmed or confirm(
'Are you sure you want to delete this calculation and all '
'associated outputs?\nThis action cannot be undone. (y/n): '):
try:
logs.dbcmd('del_calc', job_id, getpass.getuser())
except RuntimeError as err:
safeprint(err)
def smart_save(dbpath, archive):
"""
Make a copy of the db, remove the incomplete jobs and add the copy
to the archive
"""
tmpdir = tempfile.mkdtemp()
newdb = os.path.join(tmpdir, os.path.basename(dbpath))
shutil.copy(dbpath, newdb)
try:
with sqlite3.connect(newdb) as conn:
conn.execute('DELETE FROM job WHERE status != "complete"')
except:
safeprint('Please check the copy of the db in %s' % newdb)
raise
zipfiles([newdb], archive, 'a', safeprint)
shutil.rmtree(tmpdir)
def del_calculation(job_id, confirmed=False):
"""
Delete a calculation and all associated outputs.
"""
if logs.dbcmd('get_job', job_id) is None:
print('There is no job %d' % job_id)
return
if confirmed or confirm(
'Are you sure you want to (abort and) delete this calculation and '
'all associated outputs?\nThis action cannot be undone. (y/n): '):
try:
abort(job_id)
logs.dbcmd('del_calc', job_id, getpass.getuser())
except RuntimeError as err:
safeprint(err)
else:
print('Removed %d' % job_id)
def smart_save(dbpath, archive, calc_id):
"""
Make a copy of the db, remove the incomplete jobs and add the copy
to the archive
"""
tmpdir = tempfile.mkdtemp()
newdb = os.path.join(tmpdir, os.path.basename(dbpath))
shutil.copy(dbpath, newdb)
try:
with sqlite3.connect(newdb) as conn:
conn.execute('DELETE FROM job WHERE status != "complete"')
if calc_id:
conn.execute('DELETE FROM job WHERE id != %d' % calc_id)
except:
safeprint('Please check the copy of the db in %s' % newdb)
raise
zipfiles([newdb], archive, 'a', safeprint)
shutil.rmtree(tmpdir)
def run_calc(log):
"""
Run a calculation.
:param log:
LogContext of the current job
"""
register_signals()
setproctitle('oq-job-%d' % log.calc_id)
with log:
oqparam = log.get_oqparam()
calc = base.calculators(oqparam, log.calc_id)
logging.info('%s running %s [--hc=%s]',
getpass.getuser(),
calc.oqparam.inputs['job_ini'],
calc.oqparam.hazard_calculation_id)
logging.info('Using engine version %s', __version__)
msg = check_obsolete_version(oqparam.calculation_mode)
# NB: disabling the warning should be done only for users with
# an updated LTS version, but we are doing it for all users
# if msg:
# logging.warning(msg)
calc.from_engine = True
if config.zworkers['host_cores']:
set_concurrent_tasks_default(calc)
else:
logging.warning('Assuming %d %s workers',
parallel.Starmap.num_cores, OQ_DISTRIBUTE)
t0 = time.time()
calc.run()
logging.info('Exposing the outputs to the database')
expose_outputs(calc.datastore)
path = calc.datastore.filename
size = general.humansize(getsize(path))
logging.info('Stored %s on %s in %d seconds',
size, path, time.time() - t0)
calc.datastore.close()
for line in logs.dbcmd('list_outputs', log.calc_id, False):
general.safeprint(line)
# sanity check to make sure that the logging on file is working
if (log.log_file and log.log_file != os.devnull and
getsize(log.log_file) == 0):
logging.warning('The log file %s is empty!?' % log.log_file)
return calc
def del_calculation(job_id, confirmed=False):
"""
Delete a calculation and all associated outputs.
"""
if logs.dbcmd('get_job', job_id) is None:
print('There is no job %d' % job_id)
return
if confirmed or confirm(
'Are you sure you want to (abort and) delete this calculation and '
'all associated outputs?\nThis action cannot be undone. (y/n): '):
try:
abort(job_id)
resp = logs.dbcmd('del_calc', job_id, getpass.getuser())
except RuntimeError as err:
safeprint(err)
else:
if 'success' in resp:
print('Removed %d' % job_id)
else:
print(resp['error'])
def dump(archive, calc_id=0, user=None):
"""
Dump the openquake database and all the complete calculations into a zip
file. In a multiuser installation must be run as administrator.
"""
t0 = time.time()
assert archive.endswith('.zip'), archive
getfnames = 'select ds_calc_dir || ".hdf5" from job where ?A'
param = dict(status='complete')
if calc_id:
param['id'] = calc_id
if user:
param['user_name'] = user
fnames = [f for f, in db(getfnames, param) if os.path.exists(f)]
zipfiles(fnames, archive, 'w', safeprint)
pending_jobs = db('select id, status, description from job '
'where status="executing"')
if pending_jobs:
safeprint('WARNING: there were calculations executing during the dump,'
' they have been not copied')
for job_id, status, descr in pending_jobs:
safeprint('%d %s %s' % (job_id, status, descr))
# this also checks that the copied db is not corrupted
smart_save(db.path, archive, calc_id)
dt = time.time() - t0
safeprint('Archived %d calculations into %s in %d seconds' %
(len(fnames), archive, dt))
def dump(archive, calc_id=0, user=None):
"""
Dump the openquake database and all the complete calculations into a zip
file. In a multiuser installation must be run as administrator.
"""
t0 = time.time()
assert archive.endswith('.zip'), archive
getfnames = 'select ds_calc_dir || ".hdf5" from job where ?A'
param = dict(status='complete')
if calc_id:
param['id'] = calc_id
if user:
param['user_name'] = user
fnames = [f for f, in db(getfnames, param) if os.path.exists(f)]
zipfiles(fnames, archive, 'w', safeprint)
pending_jobs = db('select id, status, description from job '
'where status="executing"')
if pending_jobs:
safeprint('WARNING: there were calculations executing during the dump,'
' they have been not copied')
for job_id, status, descr in pending_jobs:
safeprint('%d %s %s' % (job_id, status, descr))
# this also checks that the copied db is not corrupted
smart_save(db.path, archive, calc_id)
dt = time.time() - t0
safeprint('Archived %d calculations into %s in %d seconds'
% (len(fnames), archive, dt))
def main(no_distribute=False,
yes=False,
upgrade_db=False,
db_version=False,
what_if_I_upgrade=False,
list_hazard_calculations=False,
list_risk_calculations=False,
delete_uncompleted_calculations=False,
multi=False,
reuse_input=False,
*,
log_file=None,
make_html_report=None,
run=None,
delete_calculation: int = None,
hazard_calculation_id: int = None,
list_outputs: int = None,
show_log=None,
export_output=None,
export_outputs=None,
param='',
config_file=None,
exports='',
log_level='info'):
"""
Run a calculation using the traditional command line API
"""
if not run:
# configure a basic logging
logs.init()
if config_file:
config.read(os.path.abspath(os.path.expanduser(config_file)),
soft_mem_limit=int,
hard_mem_limit=int,
port=int,
multi_user=valid.boolean,
serialize_jobs=valid.boolean,
strict=valid.boolean,
code=exec)
if no_distribute:
os.environ['OQ_DISTRIBUTE'] = 'no'
# check if the datadir exists
datadir = datastore.get_datadir()
if not os.path.exists(datadir):
os.makedirs(datadir)
dbserver.ensure_on()
# check if we are talking to the right server
err = dbserver.check_foreign()
if err:
sys.exit(err)
if upgrade_db:
msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
if msg.startswith('Your database is already updated'):
pass
elif yes or confirm('Proceed? (y/n) '):
logs.dbcmd('upgrade_db')
sys.exit(0)
if db_version:
safeprint(logs.dbcmd('db_version'))
sys.exit(0)
if what_if_I_upgrade:
safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
sys.exit(0)
# check if the db is outdated
outdated = logs.dbcmd('check_outdated')
if outdated:
sys.exit(outdated)
# hazard or hazard+risk
if hazard_calculation_id == -1:
# get the latest calculation of the current user
hc_id = get_job_id(hazard_calculation_id, getpass.getuser())
elif hazard_calculation_id:
# make it possible to use calculations made by another user
hc_id = get_job_id(hazard_calculation_id)
else:
hc_id = None
if run:
pars = dict(p.split('=', 1) for p in param.split(',')) if param else {}
if reuse_input:
pars['cachedir'] = datadir
if hc_id:
pars['hazard_calculation_id'] = str(hc_id)
log_file = os.path.expanduser(log_file) \
if log_file is not None else None
job_inis = [os.path.expanduser(f) for f in run]
pars['multi'] = multi
run_jobs(job_inis, log_level, log_file, exports, **pars)
# hazard
elif list_hazard_calculations:
for line in logs.dbcmd('list_calculations', 'hazard',
getpass.getuser()):
safeprint(line)
elif delete_calculation is not None:
del_calculation(delete_calculation, yes)
# risk
elif list_risk_calculations:
for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
safeprint(line)
# export
elif make_html_report:
safeprint('Written %s' % make_report(make_html_report))
sys.exit(0)
elif list_outputs is not None:
hc_id = get_job_id(list_outputs)
for line in logs.dbcmd('list_outputs', hc_id):
safeprint(line)
elif show_log is not None:
hc_id = get_job_id(show_log)
for line in logs.dbcmd('get_log', hc_id):
safeprint(line)
elif export_output is not None:
output_id, target_dir = export_output
dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
for line in core.export_output(dskey, calc_id, datadir,
os.path.expanduser(target_dir), exports
or DEFAULT_EXPORTS):
safeprint(line)
elif export_outputs is not None:
job_id, target_dir = export_outputs
hc_id = get_job_id(job_id)
for line in core.export_outputs(hc_id, os.path.expanduser(target_dir),
exports or DEFAULT_EXPORTS):
safeprint(line)
elif delete_uncompleted_calculations:
logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())
else:
print("Please pass some option, see oq engine --help")
def engine(log_file, no_distribute, yes, config_file, make_html_report,
upgrade_db, db_version, what_if_I_upgrade,
run_hazard, run_risk, run,
list_hazard_calculations, list_risk_calculations,
delete_calculation, delete_uncompleted_calculations,
hazard_calculation_id, list_outputs, show_log,
export_output, export_outputs, exports='',
log_level='info'):
"""
Run a calculation using the traditional command line API
"""
if run or run_hazard or run_risk:
# the logging will be configured in engine.py
pass
else:
# configure a basic logging
logging.basicConfig(level=logging.INFO)
if config_file:
config.load(os.path.abspath(os.path.expanduser(config_file)))
if no_distribute:
os.environ['OQ_DISTRIBUTE'] = 'no'
# check if the datadir exists
datadir = datastore.get_datadir()
if not os.path.exists(datadir):
os.makedirs(datadir)
dbserver.ensure_on()
# check if we are talking to the right server
err = dbserver.check_foreign()
if err:
sys.exit(err)
if upgrade_db:
logs.set_level('info')
msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
if msg.startswith('Your database is already updated'):
pass
elif yes or confirm('Proceed? (y/n) '):
logs.dbcmd('upgrade_db')
sys.exit(0)
if db_version:
safeprint(logs.dbcmd('db_version'))
sys.exit(0)
if what_if_I_upgrade:
safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
sys.exit(0)
# check if the db is outdated
outdated = logs.dbcmd('check_outdated')
if outdated:
sys.exit(outdated)
# hazard or hazard+risk
if hazard_calculation_id:
hc_id = get_job_id(hazard_calculation_id)
else:
hc_id = None
if run:
job_ini = os.path.expanduser(run)
open(job_ini, 'rb').read() # IOError if the file does not exist
log_file = os.path.expanduser(log_file) \
if log_file is not None else None
run_job(os.path.expanduser(run), log_level, log_file,
exports, hazard_calculation_id=hc_id)
# hazard
elif list_hazard_calculations:
for line in logs.dbcmd(
'list_calculations', 'hazard', getpass.getuser()):
safeprint(line)
elif run_hazard is not None:
safeprint('WARN: --rh/--run-hazard are deprecated, use --run instead',
file=sys.stderr)
log_file = os.path.expanduser(log_file) \
if log_file is not None else None
run_job(os.path.expanduser(run_hazard), log_level,
log_file, exports)
elif delete_calculation is not None:
del_calculation(delete_calculation, yes)
# risk
elif list_risk_calculations:
for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
safeprint(line)
elif run_risk is not None:
safeprint('WARN: --rr/--run-risk are deprecated, use --run instead',
file=sys.stderr)
if hazard_calculation_id is None:
sys.exit(MISSING_HAZARD_MSG)
log_file = os.path.expanduser(log_file) \
if log_file is not None else None
run_job(
os.path.expanduser(run_risk),
log_level, log_file, exports,
hazard_calculation_id=hc_id)
# export
elif make_html_report:
safeprint('Written %s' % make_report(make_html_report))
sys.exit(0)
elif list_outputs is not None:
hc_id = get_job_id(list_outputs)
for line in logs.dbcmd('list_outputs', hc_id):
safeprint(line)
elif show_log is not None:
hc_id = get_job_id(show_log)
for line in logs.dbcmd('get_log', hc_id):
safeprint(line)
elif export_output is not None:
output_id, target_dir = export_output
dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
for line in core.export_output(
dskey, calc_id, datadir, os.path.expanduser(target_dir),
exports or 'csv,xml'):
safeprint(line)
elif export_outputs is not None:
job_id, target_dir = export_outputs
hc_id = get_job_id(job_id)
for line in core.export_outputs(
hc_id, os.path.expanduser(target_dir), exports or 'csv,xml'):
safeprint(line)
elif delete_uncompleted_calculations:
logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())
else:
engine.parentparser.prog = 'oq engine'
engine.parentparser.print_usage()
def engine(log_file,
no_distribute,
yes,
config_file,
make_html_report,
upgrade_db,
db_version,
what_if_I_upgrade,
run,
list_hazard_calculations,
list_risk_calculations,
delete_calculation,
delete_uncompleted_calculations,
hazard_calculation_id,
list_outputs,
show_log,
export_output,
export_outputs,
exports='',
log_level='info',
reuse_hazard=False):
"""
Run a calculation using the traditional command line API
"""
if not run:
# configure a basic logging
logs.init()
if config_file:
config.read(os.path.abspath(os.path.expanduser(config_file)),
soft_mem_limit=int,
hard_mem_limit=int,
port=int,
multi_user=valid.boolean,
multi_node=valid.boolean)
if no_distribute:
os.environ['OQ_DISTRIBUTE'] = 'no'
# check if the datadir exists
datadir = datastore.get_datadir()
if not os.path.exists(datadir):
os.makedirs(datadir)
dbserver.ensure_on()
# check if we are talking to the right server
err = dbserver.check_foreign()
if err:
sys.exit(err)
if upgrade_db:
msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
if msg.startswith('Your database is already updated'):
pass
elif yes or confirm('Proceed? (y/n) '):
logs.dbcmd('upgrade_db')
sys.exit(0)
if db_version:
safeprint(logs.dbcmd('db_version'))
sys.exit(0)
if what_if_I_upgrade:
safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
sys.exit(0)
# check if the db is outdated
outdated = logs.dbcmd('check_outdated')
if outdated:
sys.exit(outdated)
# hazard or hazard+risk
if hazard_calculation_id == -1:
# get the latest calculation of the current user
hc_id = get_job_id(hazard_calculation_id, getpass.getuser())
elif hazard_calculation_id:
# make it possible to use calculations made by another user
hc_id = get_job_id(hazard_calculation_id)
else:
hc_id = None
if run:
log_file = os.path.expanduser(log_file) \
if log_file is not None else None
job_inis = [os.path.expanduser(f) for f in run]
if len(job_inis) == 1 and not hc_id:
# init logs before calling get_oqparam
logs.init('nojob', getattr(logging, log_level.upper()))
# not using logs.handle that logs on the db
oq = readinput.get_oqparam(job_inis[0])
smart_run(job_inis[0], oq, log_level, log_file, exports,
reuse_hazard)
return
for i, job_ini in enumerate(job_inis):
open(job_ini, 'rb').read() # IOError if the file does not exist
job_id = run_job(job_ini,
log_level,
log_file,
exports,
hazard_calculation_id=hc_id)
if not hc_id: # use the first calculation as base for the others
hc_id = job_id
# hazard
elif list_hazard_calculations:
for line in logs.dbcmd('list_calculations', 'hazard',
getpass.getuser()):
safeprint(line)
elif delete_calculation is not None:
del_calculation(delete_calculation, yes)
# risk
elif list_risk_calculations:
for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
safeprint(line)
# export
elif make_html_report:
safeprint('Written %s' % make_report(make_html_report))
sys.exit(0)
elif list_outputs is not None:
hc_id = get_job_id(list_outputs)
for line in logs.dbcmd('list_outputs', hc_id):
safeprint(line)
elif show_log is not None:
hc_id = get_job_id(show_log)
for line in logs.dbcmd('get_log', hc_id):
safeprint(line)
elif export_output is not None:
output_id, target_dir = export_output
dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
for line in core.export_output(dskey, calc_id, datadir,
os.path.expanduser(target_dir), exports
or 'csv,xml'):
safeprint(line)
elif export_outputs is not None:
job_id, target_dir = export_outputs
hc_id = get_job_id(job_id)
for line in core.export_outputs(hc_id, os.path.expanduser(target_dir),
exports or 'csv,xml'):
safeprint(line)
elif delete_uncompleted_calculations:
logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())
else:
engine.parentparser.prog = 'oq engine'
engine.parentparser.print_usage()
def engine(log_file, no_distribute, yes, config_file, make_html_report,
upgrade_db, db_version, what_if_I_upgrade, run,
list_hazard_calculations, list_risk_calculations,
delete_calculation, delete_uncompleted_calculations,
hazard_calculation_id, list_outputs, show_log,
export_output, export_outputs, exports='',
log_level='info', reuse_hazard=False):
"""
Run a calculation using the traditional command line API
"""
if not run:
# configure a basic logging
logs.init()
if config_file:
config.read(os.path.abspath(os.path.expanduser(config_file)),
soft_mem_limit=int, hard_mem_limit=int, port=int,
multi_user=valid.boolean, multi_node=valid.boolean)
if no_distribute:
os.environ['OQ_DISTRIBUTE'] = 'no'
# check if the datadir exists
datadir = datastore.get_datadir()
if not os.path.exists(datadir):
os.makedirs(datadir)
dbserver.ensure_on()
# check if we are talking to the right server
err = dbserver.check_foreign()
if err:
sys.exit(err)
if upgrade_db:
msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
if msg.startswith('Your database is already updated'):
pass
elif yes or confirm('Proceed? (y/n) '):
logs.dbcmd('upgrade_db')
sys.exit(0)
if db_version:
safeprint(logs.dbcmd('db_version'))
sys.exit(0)
if what_if_I_upgrade:
safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
sys.exit(0)
# check if the db is outdated
outdated = logs.dbcmd('check_outdated')
if outdated:
sys.exit(outdated)
# hazard or hazard+risk
if hazard_calculation_id == -1:
# get the latest calculation of the current user
hc_id = get_job_id(hazard_calculation_id, getpass.getuser())
elif hazard_calculation_id:
# make it possible to use calculations made by another user
hc_id = get_job_id(hazard_calculation_id)
else:
hc_id = None
if run:
log_file = os.path.expanduser(log_file) \
if log_file is not None else None
job_inis = [os.path.expanduser(f) for f in run]
if len(job_inis) == 1 and not hc_id:
# init logs before calling get_oqparam
logs.init('nojob', getattr(logging, log_level.upper()))
# not using logs.handle that logs on the db
oq = readinput.get_oqparam(job_inis[0])
smart_run(job_inis[0], oq, log_level, log_file,
exports, reuse_hazard)
return
for i, job_ini in enumerate(job_inis):
open(job_ini, 'rb').read() # IOError if the file does not exist
job_id = run_job(job_ini, log_level, log_file,
exports, hazard_calculation_id=hc_id)
if not hc_id: # use the first calculation as base for the others
hc_id = job_id
# hazard
elif list_hazard_calculations:
for line in logs.dbcmd(
'list_calculations', 'hazard', getpass.getuser()):
safeprint(line)
elif delete_calculation is not None:
del_calculation(delete_calculation, yes)
# risk
elif list_risk_calculations:
for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
safeprint(line)
# export
elif make_html_report:
safeprint('Written %s' % make_report(make_html_report))
sys.exit(0)
elif list_outputs is not None:
hc_id = get_job_id(list_outputs)
for line in logs.dbcmd('list_outputs', hc_id):
safeprint(line)
elif show_log is not None:
hc_id = get_job_id(show_log)
for line in logs.dbcmd('get_log', hc_id):
safeprint(line)
elif export_output is not None:
output_id, target_dir = export_output
dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
for line in core.export_output(
dskey, calc_id, datadir, os.path.expanduser(target_dir),
exports or 'csv,xml'):
safeprint(line)
elif export_outputs is not None:
job_id, target_dir = export_outputs
hc_id = get_job_id(job_id)
for line in core.export_outputs(
hc_id, os.path.expanduser(target_dir), exports or 'csv,xml'):
safeprint(line)
elif delete_uncompleted_calculations:
logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())
else:
engine.parentparser.prog = 'oq engine'
engine.parentparser.print_usage()
