def run(job_ini,
slowest=False,
hc=None,
param='',
concurrent_tasks=None,
exports='',
loglevel='info',
calc_id='nojob',
pdb=None,
reuse_input=None):
"""
Run a calculation bypassing the database layer
"""
dbserver.ensure_on()
if param:
params = dict(p.split('=', 1) for p in param.split(','))
else:
params = {}
if hc:
params['hazard_calculation_id'] = str(hc)
if slowest:
prof = cProfile.Profile()
stmt = ('_run(job_ini, concurrent_tasks, calc_id, pdb, reuse_input, '
'loglevel, hc, exports, params)')
prof.runctx(stmt, globals(), locals())
pstat = calc_path + '.pstat'
prof.dump_stats(pstat)
print('Saved profiling info in %s' % pstat)
print(get_pstats(pstat, slowest))
return
try:
return _run(job_ini, concurrent_tasks, calc_id, pdb, reuse_input,
loglevel, exports, params)
finally:
parallel.Starmap.shutdown()
def importcalc(host, calc_id, username, password):
"""
Import a remote calculation into the local database
"""
if '/' in host.split('//', 1)[1]:
sys.exit('Wrong host ending with /%s' % host.rsplit('/', 1)[1])
calc_url = '/'.join([host, 'v1/calc', str(calc_id)])
dbserver.ensure_on()
job = logs.dbcmd('get_job', calc_id)
if job is not None:
sys.exit('There is already a job #%d in the local db' % calc_id)
datadir = datastore.get_datadir()
session = login(host, username, password)
status = session.get('%s/status' % calc_url)
if 'Log in to an existing account' in status.text:
sys.exit('Could not login')
json = status.json()
if json["parent_id"]:
sys.exit('The job has a parent (#%(parent_id)d) and cannot be '
'downloaded' % json)
resp = session.get('%s/datastore' % calc_url, stream=True)
assert resp.status_code == 200, resp.status_code
fname = '%s/calc_%d.hdf5' % (datadir, calc_id)
down = 0
with open(fname, 'wb') as f:
logging.info('%s -> %s', calc_url, fname)
for chunk in resp.iter_content(CHUNKSIZE):
f.write(chunk)
down += len(chunk)
general.println('Downloaded {:,} bytes'.format(down))
print()
with datastore.read(calc_id) as dstore:
engine.expose_outputs(dstore, json['owner'], json['status'])
logging.info('Imported calculation %d successfully', calc_id)
def importcalc(calc_id):
"""
Import a remote calculation into the local database. server, username
and password must be specified in an openquake.cfg file.
NB: calc_id can be a local pathname to a datastore not already
present in the database: in that case it is imported in the db.
"""
dbserver.ensure_on()
try:
calc_id = int(calc_id)
except ValueError: # assume calc_id is a pathname
remote = False
else:
remote = True
job = logs.dbcmd('get_job', calc_id)
if job is not None:
sys.exit('There is already a job #%d in the local db' % calc_id)
if remote:
datadir = datastore.get_datadir()
webex = WebExtractor(calc_id)
hc_id = webex.oqparam.hazard_calculation_id
if hc_id:
sys.exit('The job has a parent (#%d) and cannot be '
'downloaded' % hc_id)
webex.dump('%s/calc_%d.hdf5' % (datadir, calc_id))
webex.close()
with datastore.read(calc_id) as dstore:
engine.expose_outputs(dstore, status='complete')
logging.info('Imported calculation %s successfully', calc_id)
def run(job_ini,
slowest=False,
hc=None,
param='',
concurrent_tasks=None,
exports='',
loglevel='info',
pdb=None):
"""
Run a calculation bypassing the database layer
"""
dbserver.ensure_on()
if param:
params = oqvalidation.OqParam.check(
dict(p.split('=', 1) for p in param.split(',')))
else:
params = {}
if slowest:
prof = cProfile.Profile()
stmt = ('_run(job_ini, concurrent_tasks, pdb, loglevel, hc, '
'exports, params)')
prof.runctx(stmt, globals(), locals())
pstat = calc_path + '.pstat'
prof.dump_stats(pstat)
print('Saved profiling info in %s' % pstat)
print(get_pstats(pstat, slowest))
else:
return _run(job_ini, concurrent_tasks, pdb, loglevel, hc, exports,
params)
def importcalc(calc_id):
"""
Import a remote calculation into the local database. server, username
and password must be specified in an openquake.cfg file.
NB: calc_id can be a local pathname to a datastore not already
present in the database: in that case it is imported in the db.
"""
dbserver.ensure_on()
try:
calc_id = int(calc_id)
except ValueError: # assume calc_id is a pathname
calc_id, datadir = datastore.extract_calc_id_datadir(calc_id)
status = 'complete'
remote = False
else:
remote = True
job = logs.dbcmd('get_job', calc_id)
if job is not None:
sys.exit('There is already a job #%d in the local db' % calc_id)
if remote:
datadir = datastore.get_datadir()
webex = WebExtractor(calc_id)
status = webex.status['status']
hc_id = webex.oqparam.hazard_calculation_id
if hc_id:
sys.exit('The job has a parent (#%d) and cannot be '
'downloaded' % hc_id)
webex.dump('%s/calc_%d.hdf5' % (datadir, calc_id))
webex.close()
with datastore.read(calc_id) as dstore:
engine.expose_outputs(dstore, status=status)
logging.info('Imported calculation %d successfully', calc_id)
def main(yes=False):
"""
Remove all the datastores and the database of the current user
"""
ok = yes or confirm('Do you really want to destroy all your data? (y/n) ')
if not ok:
return
dbpath = os.path.realpath(os.path.expanduser(config.dbserver.file))
if not os.path.isfile(dbpath):
sys.exit('%s does not exist' % dbpath)
else:
dbserver.ensure_on() # start the dbserver in a subprocess
user = getpass.getuser()
for calc_id in logs.dbcmd('get_calc_ids', user):
purge_one(calc_id, user, force=True)
if os.access(dbpath, os.W_OK): # single user mode
purge_all(user) # calculations in oqdata not in the db
# stop the dbserver first
pid = logs.dbcmd('getpid')
os.kill(pid, signal.SIGTERM)
time.sleep(.5) # give time to stop
assert dbserver.get_status() == 'not-running'
print('dbserver stopped')
# remove the database
os.remove(dbpath)
print('Removed %s' % dbpath)
def webui(cmd, hostport="127.0.0.1:8800"):
"""
start the webui server in foreground or perform other operation on the
django application
"""
dbserver.ensure_on() # start the dbserver in a subproces
if cmd == "start":
rundjango("runserver", hostport)
elif cmd == "syncdb":
rundjango("syncdb")
def post_risk(ebr_id):
"""
Generate loss curves and maps from an event loss table
"""
dbserver.ensure_on()
dstore = util.read(ebr_id)
oq = dstore['oqparam']
prc = PostRiskCalculator(oq)
prc.datastore.parent = dstore
prc.run()
logging.info('Generated %s', prc.datastore.filename)
def webui(cmd, hostport='127.0.0.1:8800', skip_browser=False):
"""
start the webui server in foreground or perform other operation on the
django application
"""
dbpath = os.path.realpath(os.path.expanduser(config.dbserver.file))
if os.path.isfile(dbpath) and not os.access(dbpath, os.W_OK):
sys.exit('This command must be run by the proper user: '******'see the documentation for details')
if cmd == 'start':
dbserver.ensure_on() # start the dbserver in a subprocess
rundjango('runserver', hostport, skip_browser)
elif cmd in commands:
rundjango(cmd)
def test_oqdata(self):
# the that the environment variable OQ_DATADIR is honored
job_ini = os.path.join(os.path.dirname(case_2.__file__), 'job_2.ini')
tempdir = tempfile.mkdtemp()
dbserver.ensure_on()
with mock.patch.dict(os.environ, OQ_DATADIR=tempdir):
[(job_id, oq)] = run_jobs([job_ini], log_level='error')
job = commonlib.logs.dbcmd('get_job', job_id)
self.assertTrue(job.ds_calc_dir.startswith(tempdir),
job.ds_calc_dir)
with Print.patch() as p:
export('ruptures', job_id, 'csv', tempdir)
self.assertIn('Exported', str(p))
shutil.rmtree(tempdir)
def test_oqdata(self):
# the that the environment variable OQ_DATADIR is honored
job_ini = os.path.join(os.path.dirname(case_2.__file__), 'job_2.ini')
tempdir = tempfile.mkdtemp()
dbserver.ensure_on()
with mock.patch.dict(os.environ, OQ_DATADIR=tempdir):
job_id = run_job(job_ini, log_level='error')
job = commonlib.logs.dbcmd('get_job', job_id)
self.assertTrue(job.ds_calc_dir.startswith(tempdir),
job.ds_calc_dir)
with Print.patch() as p:
export('ruptures', job_id, 'csv', tempdir)
self.assertIn('Exported', str(p))
shutil.rmtree(tempdir)
def test_oqdata(self):
# the that the environment variable OQ_DATADIR is honored
job_ini = os.path.join(os.path.dirname(case_2.__file__), 'job_2.ini')
tempdir = tempfile.mkdtemp()
dbserver.ensure_on()
with mock.patch.dict(os.environ, OQ_DATADIR=tempdir):
[job] = run_jobs(create_jobs([job_ini], 'error'))
job = commonlib.logs.dbcmd('get_job', job.calc_id)
self.assertTrue(job.ds_calc_dir.startswith(tempdir),
job.ds_calc_dir)
with Print.patch() as p:
sap.runline(f'openquake.commands export ruptures {job.id} '
f'-e csv --export-dir={tempdir}')
self.assertIn('Exported', str(p))
shutil.rmtree(tempdir)
def main(cmd, args=()):
"""
Run a database command
"""
if cmd in commands and len(args) != len(commands[cmd]):
sys.exit('Wrong number of arguments, expected %s, got %s' %
(commands[cmd], args))
elif (cmd not in commands and not cmd.upper().startswith('SELECT')
and config.dbserver.multi_user and getpass.getuser() != 'openquake'):
sys.exit('You have no permission to run %s' % cmd)
dbserver.ensure_on()
res = logs.dbcmd(cmd, *convert(args))
if hasattr(res, '_fields') and res.__class__.__name__ != 'Row':
print(rst_table(res))
else:
print(res)
def webui(cmd, hostport='127.0.0.1:8800'):
"""
start the webui server in foreground or perform other operation on the
django application
"""
db_path = os.path.expanduser(config.get('dbserver', 'file'))
if os.path.isfile(db_path) and not os.access(db_path, os.W_OK):
sys.exit('This command must be run by the proper user: '******'see the documentation for details')
if cmd == 'start':
dbserver.ensure_on() # start the dbserver in a subproces
rundjango('runserver', hostport)
elif cmd == 'migrate':
rundjango('migrate')
# For backward compatibility with Django 1.6
elif cmd == 'syncdb':
rundjango('syncdb')
def db(cmd, args=()):
"""
Run a database command
"""
if cmd not in commands:
okcmds = '\n'.join(
'%s %s' % (name, repr(' '.join(args)) if args else '')
for name, args in sorted(commands.items()))
print('Invalid command "%s": choose one from\n%s' % (cmd, okcmds))
elif len(args) != len(commands[cmd]):
print('Wrong number of arguments, expected %s, got %s' % (
commands[cmd], args))
else:
dbserver.ensure_on()
res = logs.dbcmd(cmd, *convert(args))
if hasattr(res, '_fields') and res.__class__.__name__ != 'Row':
print(rst_table(res))
else:
print(res)
def run(job_ini, slowest=False, hc=None, param='', concurrent_tasks=None,
exports='', loglevel='info', pdb=None):
"""
Run a calculation bypassing the database layer
"""
dbserver.ensure_on()
if param:
params = oqvalidation.OqParam.check(
dict(p.split('=', 1) for p in param.split(',')))
else:
params = {}
if slowest:
prof = cProfile.Profile()
stmt = ('_run(job_ini, concurrent_tasks, pdb, loglevel, hc, '
'exports, params)')
prof.runctx(stmt, globals(), locals())
pstat = calc_path + '.pstat'
prof.dump_stats(pstat)
print('Saved profiling info in %s' % pstat)
print(get_pstats(pstat, slowest))
else:
_run(job_ini, concurrent_tasks, pdb, loglevel, hc, exports, params)
def main(cfg_file):
dbserver.ensure_on()
startTime = datetime.now()
cfg = configparser.ConfigParser()
cfg.read(cfg_file)
(gmf_file, gmf_file_gmpe_rate, sites, gsim_list, cinfo, oq_param,
mean_shift_inter_residuals, realizations_inter, realizations_intra,
intra_files_name, intra_files, csv_rate_gmf_file,
seed) = read_config_file(cfg)
gmfs_median = read_input_gmf(gmf_file, gmf_file_gmpe_rate)
imts = [PGA(), SA(0.3)]
vs30 = 180
(std_total, std_inter,
std_intra) = calculate_total_std(gsim_list, imts, vs30)
inter_residual, gmpe_imt = calc_inter_residuals(mean_shift_inter_residuals,
realizations_inter,
std_inter)
sp_correlation = cfg['input']['sp_correlation']
intra_residual, num_intra_matrices = calc_intra_residuals(
sp_correlation, realizations_intra, intra_files_name, intra_files,
sites, gmpe_imt, std_intra)
N = len(sites)
num_gmfs = (len(gmfs_median) * len(inter_residual['rates_inter']) *
len(intra_residual['rates_intra']))
f, calc_id = create_parent_hdf5(N, num_gmfs, sites, cinfo, oq_param)
zip_intra = create_zip_intra(gsim_list, imts, intra_residual)
save_hdf5_rate(num_gmfs, csv_rate_gmf_file, gmfs_median, gsim_list,
inter_residual, intra_residual, seed, num_intra_matrices,
realizations_intra, N, imts, zip_intra, f)
create_indices(N, f, num_gmfs)
f.close()
print('Saved', calc_id)
print(datetime.now() - startTime)
def main(job_ini,
pdb=False,
reuse_input=False,
*,
slowest: int = None,
hc: int = None,
param='',
concurrent_tasks: int = None,
exports: valid.export_formats = '',
loglevel='info'):
"""
Run a calculation
"""
dbserver.ensure_on()
if param:
params = dict(p.split('=', 1) for p in param.split(','))
else:
params = {}
if hc:
params['hazard_calculation_id'] = str(hc)
if slowest:
prof = cProfile.Profile()
prof.runctx(
'_run(job_ini[0], 0, pdb, reuse_input, loglevel, '
'exports, params)', globals(), locals())
pstat = calc_path + '.pstat'
prof.dump_stats(pstat)
print('Saved profiling info in %s' % pstat)
print(get_pstats(pstat, slowest))
return
if len(job_ini) == 1:
return _run(job_ini[0], concurrent_tasks, pdb, reuse_input, loglevel,
exports, params)
jobs = create_jobs(job_ini, loglevel, hc_id=hc)
for job in jobs:
job.params.update(params)
job.params['exports'] = ','.join(exports)
run_jobs(jobs)
def main(no_distribute=False,
yes=False,
upgrade_db=False,
db_version=False,
what_if_I_upgrade=False,
list_hazard_calculations=False,
list_risk_calculations=False,
delete_uncompleted_calculations=False,
multi=False,
reuse_input=False,
*,
log_file=None,
make_html_report=None,
run=None,
delete_calculation: int = None,
hazard_calculation_id: int = None,
list_outputs: int = None,
show_log=None,
export_output=None,
export_outputs=None,
param='',
config_file=None,
exports='',
log_level='info'):
"""
Run a calculation using the traditional command line API
"""
if not run:
# configure a basic logging
logs.init()
if config_file:
config.read(os.path.abspath(os.path.expanduser(config_file)),
soft_mem_limit=int,
hard_mem_limit=int,
port=int,
multi_user=valid.boolean,
serialize_jobs=valid.boolean,
strict=valid.boolean,
code=exec)
if no_distribute:
os.environ['OQ_DISTRIBUTE'] = 'no'
# check if the datadir exists
datadir = datastore.get_datadir()
if not os.path.exists(datadir):
os.makedirs(datadir)
dbserver.ensure_on()
# check if we are talking to the right server
err = dbserver.check_foreign()
if err:
sys.exit(err)
if upgrade_db:
msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
if msg.startswith('Your database is already updated'):
pass
elif yes or confirm('Proceed? (y/n) '):
logs.dbcmd('upgrade_db')
sys.exit(0)
if db_version:
safeprint(logs.dbcmd('db_version'))
sys.exit(0)
if what_if_I_upgrade:
safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
sys.exit(0)
# check if the db is outdated
outdated = logs.dbcmd('check_outdated')
if outdated:
sys.exit(outdated)
# hazard or hazard+risk
if hazard_calculation_id == -1:
# get the latest calculation of the current user
hc_id = get_job_id(hazard_calculation_id, getpass.getuser())
elif hazard_calculation_id:
# make it possible to use calculations made by another user
hc_id = get_job_id(hazard_calculation_id)
else:
hc_id = None
if run:
pars = dict(p.split('=', 1) for p in param.split(',')) if param else {}
if reuse_input:
pars['cachedir'] = datadir
if hc_id:
pars['hazard_calculation_id'] = str(hc_id)
log_file = os.path.expanduser(log_file) \
if log_file is not None else None
job_inis = [os.path.expanduser(f) for f in run]
pars['multi'] = multi
run_jobs(job_inis, log_level, log_file, exports, **pars)
# hazard
elif list_hazard_calculations:
for line in logs.dbcmd('list_calculations', 'hazard',
getpass.getuser()):
safeprint(line)
elif delete_calculation is not None:
del_calculation(delete_calculation, yes)
# risk
elif list_risk_calculations:
for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
safeprint(line)
# export
elif make_html_report:
safeprint('Written %s' % make_report(make_html_report))
sys.exit(0)
elif list_outputs is not None:
hc_id = get_job_id(list_outputs)
for line in logs.dbcmd('list_outputs', hc_id):
safeprint(line)
elif show_log is not None:
hc_id = get_job_id(show_log)
for line in logs.dbcmd('get_log', hc_id):
safeprint(line)
elif export_output is not None:
output_id, target_dir = export_output
dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
for line in core.export_output(dskey, calc_id, datadir,
os.path.expanduser(target_dir), exports
or DEFAULT_EXPORTS):
safeprint(line)
elif export_outputs is not None:
job_id, target_dir = export_outputs
hc_id = get_job_id(job_id)
for line in core.export_outputs(hc_id, os.path.expanduser(target_dir),
exports or DEFAULT_EXPORTS):
safeprint(line)
elif delete_uncompleted_calculations:
logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())
else:
print("Please pass some option, see oq engine --help")
headers = {'User-Agent': 'OpenQuake Engine %s;%s;%s;%s' %
(__version__, calculation_mode, platform.platform(),
config.distribution.oq_distribute)}
try:
req = Request(OQ_API + '/engine/latest', headers=headers)
# NB: a timeout < 1 does not work
data = urlopen(req, timeout=1).read() # bytes
tag_name = json.loads(decode(data))['tag_name']
current = version_triple(__version__)
latest = version_triple(tag_name)
except Exception: # page not available or wrong version tag
msg = ('An error occurred while calling %s/engine/latest to check'
' if the installed version of the engine is up to date.' %
OQ_API)
logging.warning(msg, exc_info=True)
return
if current < latest:
return ('Version %s of the engine is available, but you are '
'still using version %s' % (tag_name, __version__))
else:
return ''
if __name__ == '__main__':
from openquake.server import dbserver
# run a job object stored in a pickle file, called by job.yaml
with open(sys.argv[1], 'rb') as f:
job = pickle.load(f)
dbserver.ensure_on()
run_jobs([job])
def engine(log_file, no_distribute, yes, config_file, make_html_report,
upgrade_db, version_db, what_if_I_upgrade,
run_hazard, run_risk, run,
list_hazard_calculations, list_risk_calculations,
delete_calculation, delete_uncompleted_calculations,
hazard_calculation_id, list_outputs, show_log,
export_output, export_outputs, exports='',
log_level='info'):
"""
Run a calculation using the traditional command line API
"""
config.abort_if_no_config_available()
if run or run_hazard or run_risk:
# the logging will be configured in engine.py
pass
else:
# configure a basic logging
logging.basicConfig(level=logging.INFO)
if config_file:
os.environ[config.OQ_CONFIG_FILE_VAR] = os.path.abspath(
os.path.expanduser(config_file))
config.refresh()
if no_distribute:
os.environ['OQ_DISTRIBUTE'] = 'no'
# check if the datadir exists
if not os.path.exists(datastore.DATADIR):
os.makedirs(datastore.DATADIR)
dbserver.ensure_on()
if upgrade_db:
logs.set_level('info')
msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
if msg.startswith('Your database is already updated'):
pass
elif yes or confirm('Proceed? (y/n) '):
logs.dbcmd('upgrade_db')
sys.exit(0)
if version_db:
print(logs.dbcmd('version_db'))
sys.exit(0)
if what_if_I_upgrade:
print(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
sys.exit(0)
# check if the db is outdated
outdated = logs.dbcmd('check_outdated')
if outdated:
sys.exit(outdated)
# hazard or hazard+risk
if hazard_calculation_id:
hc_id = get_job_id(hazard_calculation_id)
else:
hc_id = None
if run:
job_inis = [os.path.expanduser(ini) for ini in run.split(',')]
if len(job_inis) not in (1, 2):
sys.exit('%s should be a .ini filename or a pair of filenames '
'separated by a comma' % run)
for job_ini in job_inis:
open(job_ini).read() # raise an IOError if the file does not exist
log_file = os.path.expanduser(log_file) \
if log_file is not None else None
if len(job_inis) == 2:
# run hazard
job_id = run_job(job_inis[0], log_level,
log_file, exports)
# run risk
run_job(job_inis[1], log_level, log_file,
exports, hazard_calculation_id=job_id)
else:
run_job(
os.path.expanduser(run), log_level, log_file,
exports, hazard_calculation_id=hc_id)
# hazard
elif list_hazard_calculations:
for line in logs.dbcmd(
'list_calculations', 'hazard', getpass.getuser()):
print(line)
elif run_hazard is not None:
print('WARN: --rh/--run-hazard are deprecated, use --run instead',
file=sys.stderr)
log_file = os.path.expanduser(log_file) \
if log_file is not None else None
run_job(os.path.expanduser(run_hazard), log_level,
log_file, exports)
elif delete_calculation is not None:
del_calculation(delete_calculation, yes)
# risk
elif list_risk_calculations:
for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
print(line)
elif run_risk is not None:
print('WARN: --rr/--run-risk are deprecated, use --run instead',
file=sys.stderr)
if hazard_calculation_id is None:
sys.exit(MISSING_HAZARD_MSG)
log_file = os.path.expanduser(log_file) \
if log_file is not None else None
run_job(
os.path.expanduser(run_risk),
log_level, log_file, exports,
hazard_calculation_id=hc_id)
# export
elif make_html_report:
print('Written %s' % make_report(make_html_report))
sys.exit(0)
elif list_outputs is not None:
hc_id = get_job_id(list_outputs)
for line in logs.dbcmd('list_outputs', hc_id):
print(line)
elif show_log is not None:
hc_id = get_job_id(show_log)
for line in logs.dbcmd('get_log', hc_id):
print(line)
elif export_output is not None:
output_id, target_dir = export_output
dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
for line in core.export_output(
dskey, calc_id, datadir, os.path.expanduser(target_dir),
exports or 'xml,csv'):
print(line)
elif export_outputs is not None:
job_id, target_dir = export_outputs
hc_id = get_job_id(job_id)
for line in core.export_outputs(
hc_id, os.path.expanduser(target_dir), exports or 'xml,csv'):
print(line)
elif delete_uncompleted_calculations:
logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())
else:
engine.parentparser.prog = 'oq engine'
engine.parentparser.print_usage()
def engine(log_file, no_distribute, yes, config_file, make_html_report,
upgrade_db, db_version, what_if_I_upgrade,
run_hazard, run_risk, run,
list_hazard_calculations, list_risk_calculations,
delete_calculation, delete_uncompleted_calculations,
hazard_calculation_id, list_outputs, show_log,
export_output, export_outputs, exports='',
log_level='info'):
"""
Run a calculation using the traditional command line API
"""
if run or run_hazard or run_risk:
# the logging will be configured in engine.py
pass
else:
# configure a basic logging
logging.basicConfig(level=logging.INFO)
if config_file:
config.load(os.path.abspath(os.path.expanduser(config_file)))
if no_distribute:
os.environ['OQ_DISTRIBUTE'] = 'no'
# check if the datadir exists
datadir = datastore.get_datadir()
if not os.path.exists(datadir):
os.makedirs(datadir)
dbserver.ensure_on()
# check if we are talking to the right server
err = dbserver.check_foreign()
if err:
sys.exit(err)
if upgrade_db:
logs.set_level('info')
msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
if msg.startswith('Your database is already updated'):
pass
elif yes or confirm('Proceed? (y/n) '):
logs.dbcmd('upgrade_db')
sys.exit(0)
if db_version:
safeprint(logs.dbcmd('db_version'))
sys.exit(0)
if what_if_I_upgrade:
safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
sys.exit(0)
# check if the db is outdated
outdated = logs.dbcmd('check_outdated')
if outdated:
sys.exit(outdated)
# hazard or hazard+risk
if hazard_calculation_id:
hc_id = get_job_id(hazard_calculation_id)
else:
hc_id = None
if run:
job_ini = os.path.expanduser(run)
open(job_ini, 'rb').read() # IOError if the file does not exist
log_file = os.path.expanduser(log_file) \
if log_file is not None else None
run_job(os.path.expanduser(run), log_level, log_file,
exports, hazard_calculation_id=hc_id)
# hazard
elif list_hazard_calculations:
for line in logs.dbcmd(
'list_calculations', 'hazard', getpass.getuser()):
safeprint(line)
elif run_hazard is not None:
safeprint('WARN: --rh/--run-hazard are deprecated, use --run instead',
file=sys.stderr)
log_file = os.path.expanduser(log_file) \
if log_file is not None else None
run_job(os.path.expanduser(run_hazard), log_level,
log_file, exports)
elif delete_calculation is not None:
del_calculation(delete_calculation, yes)
# risk
elif list_risk_calculations:
for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
safeprint(line)
elif run_risk is not None:
safeprint('WARN: --rr/--run-risk are deprecated, use --run instead',
file=sys.stderr)
if hazard_calculation_id is None:
sys.exit(MISSING_HAZARD_MSG)
log_file = os.path.expanduser(log_file) \
if log_file is not None else None
run_job(
os.path.expanduser(run_risk),
log_level, log_file, exports,
hazard_calculation_id=hc_id)
# export
elif make_html_report:
safeprint('Written %s' % make_report(make_html_report))
sys.exit(0)
elif list_outputs is not None:
hc_id = get_job_id(list_outputs)
for line in logs.dbcmd('list_outputs', hc_id):
safeprint(line)
elif show_log is not None:
hc_id = get_job_id(show_log)
for line in logs.dbcmd('get_log', hc_id):
safeprint(line)
elif export_output is not None:
output_id, target_dir = export_output
dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
for line in core.export_output(
dskey, calc_id, datadir, os.path.expanduser(target_dir),
exports or 'csv,xml'):
safeprint(line)
elif export_outputs is not None:
job_id, target_dir = export_outputs
hc_id = get_job_id(job_id)
for line in core.export_outputs(
hc_id, os.path.expanduser(target_dir), exports or 'csv,xml'):
safeprint(line)
elif delete_uncompleted_calculations:
logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())
else:
engine.parentparser.prog = 'oq engine'
engine.parentparser.print_usage()
def engine(log_file, no_distribute, yes, config_file, make_html_report,
upgrade_db, db_version, what_if_I_upgrade, run,
list_hazard_calculations, list_risk_calculations,
delete_calculation, delete_uncompleted_calculations,
hazard_calculation_id, list_outputs, show_log,
export_output, export_outputs, exports='',
log_level='info', reuse_hazard=False):
"""
Run a calculation using the traditional command line API
"""
if not run:
# configure a basic logging
logs.init()
if config_file:
config.read(os.path.abspath(os.path.expanduser(config_file)),
soft_mem_limit=int, hard_mem_limit=int, port=int,
multi_user=valid.boolean, multi_node=valid.boolean)
if no_distribute:
os.environ['OQ_DISTRIBUTE'] = 'no'
# check if the datadir exists
datadir = datastore.get_datadir()
if not os.path.exists(datadir):
os.makedirs(datadir)
dbserver.ensure_on()
# check if we are talking to the right server
err = dbserver.check_foreign()
if err:
sys.exit(err)
if upgrade_db:
msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
if msg.startswith('Your database is already updated'):
pass
elif yes or confirm('Proceed? (y/n) '):
logs.dbcmd('upgrade_db')
sys.exit(0)
if db_version:
safeprint(logs.dbcmd('db_version'))
sys.exit(0)
if what_if_I_upgrade:
safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
sys.exit(0)
# check if the db is outdated
outdated = logs.dbcmd('check_outdated')
if outdated:
sys.exit(outdated)
# hazard or hazard+risk
if hazard_calculation_id == -1:
# get the latest calculation of the current user
hc_id = get_job_id(hazard_calculation_id, getpass.getuser())
elif hazard_calculation_id:
# make it possible to use calculations made by another user
hc_id = get_job_id(hazard_calculation_id)
else:
hc_id = None
if run:
log_file = os.path.expanduser(log_file) \
if log_file is not None else None
job_inis = [os.path.expanduser(f) for f in run]
if len(job_inis) == 1 and not hc_id:
# init logs before calling get_oqparam
logs.init('nojob', getattr(logging, log_level.upper()))
# not using logs.handle that logs on the db
oq = readinput.get_oqparam(job_inis[0])
smart_run(job_inis[0], oq, log_level, log_file,
exports, reuse_hazard)
return
for i, job_ini in enumerate(job_inis):
open(job_ini, 'rb').read() # IOError if the file does not exist
job_id = run_job(job_ini, log_level, log_file,
exports, hazard_calculation_id=hc_id)
if not hc_id: # use the first calculation as base for the others
hc_id = job_id
# hazard
elif list_hazard_calculations:
for line in logs.dbcmd(
'list_calculations', 'hazard', getpass.getuser()):
safeprint(line)
elif delete_calculation is not None:
del_calculation(delete_calculation, yes)
# risk
elif list_risk_calculations:
for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
safeprint(line)
# export
elif make_html_report:
safeprint('Written %s' % make_report(make_html_report))
sys.exit(0)
elif list_outputs is not None:
hc_id = get_job_id(list_outputs)
for line in logs.dbcmd('list_outputs', hc_id):
safeprint(line)
elif show_log is not None:
hc_id = get_job_id(show_log)
for line in logs.dbcmd('get_log', hc_id):
safeprint(line)
elif export_output is not None:
output_id, target_dir = export_output
dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
for line in core.export_output(
dskey, calc_id, datadir, os.path.expanduser(target_dir),
exports or 'csv,xml'):
safeprint(line)
elif export_outputs is not None:
job_id, target_dir = export_outputs
hc_id = get_job_id(job_id)
for line in core.export_outputs(
hc_id, os.path.expanduser(target_dir), exports or 'csv,xml'):
safeprint(line)
elif delete_uncompleted_calculations:
logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())
else:
engine.parentparser.prog = 'oq engine'
engine.parentparser.print_usage()
def engine(log_file,
no_distribute,
yes,
config_file,
make_html_report,
upgrade_db,
db_version,
what_if_I_upgrade,
run,
list_hazard_calculations,
list_risk_calculations,
delete_calculation,
delete_uncompleted_calculations,
hazard_calculation_id,
list_outputs,
show_log,
export_output,
export_outputs,
exports='',
log_level='info',
reuse_hazard=False):
"""
Run a calculation using the traditional command line API
"""
if not run:
# configure a basic logging
logs.init()
if config_file:
config.read(os.path.abspath(os.path.expanduser(config_file)),
soft_mem_limit=int,
hard_mem_limit=int,
port=int,
multi_user=valid.boolean,
multi_node=valid.boolean)
if no_distribute:
os.environ['OQ_DISTRIBUTE'] = 'no'
# check if the datadir exists
datadir = datastore.get_datadir()
if not os.path.exists(datadir):
os.makedirs(datadir)
dbserver.ensure_on()
# check if we are talking to the right server
err = dbserver.check_foreign()
if err:
sys.exit(err)
if upgrade_db:
msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
if msg.startswith('Your database is already updated'):
pass
elif yes or confirm('Proceed? (y/n) '):
logs.dbcmd('upgrade_db')
sys.exit(0)
if db_version:
safeprint(logs.dbcmd('db_version'))
sys.exit(0)
if what_if_I_upgrade:
safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
sys.exit(0)
# check if the db is outdated
outdated = logs.dbcmd('check_outdated')
if outdated:
sys.exit(outdated)
# hazard or hazard+risk
if hazard_calculation_id == -1:
# get the latest calculation of the current user
hc_id = get_job_id(hazard_calculation_id, getpass.getuser())
elif hazard_calculation_id:
# make it possible to use calculations made by another user
hc_id = get_job_id(hazard_calculation_id)
else:
hc_id = None
if run:
log_file = os.path.expanduser(log_file) \
if log_file is not None else None
job_inis = [os.path.expanduser(f) for f in run]
if len(job_inis) == 1 and not hc_id:
# init logs before calling get_oqparam
logs.init('nojob', getattr(logging, log_level.upper()))
# not using logs.handle that logs on the db
oq = readinput.get_oqparam(job_inis[0])
smart_run(job_inis[0], oq, log_level, log_file, exports,
reuse_hazard)
return
for i, job_ini in enumerate(job_inis):
open(job_ini, 'rb').read() # IOError if the file does not exist
job_id = run_job(job_ini,
log_level,
log_file,
exports,
hazard_calculation_id=hc_id)
if not hc_id: # use the first calculation as base for the others
hc_id = job_id
# hazard
elif list_hazard_calculations:
for line in logs.dbcmd('list_calculations', 'hazard',
getpass.getuser()):
safeprint(line)
elif delete_calculation is not None:
del_calculation(delete_calculation, yes)
# risk
elif list_risk_calculations:
for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
safeprint(line)
# export
elif make_html_report:
safeprint('Written %s' % make_report(make_html_report))
sys.exit(0)
elif list_outputs is not None:
hc_id = get_job_id(list_outputs)
for line in logs.dbcmd('list_outputs', hc_id):
safeprint(line)
elif show_log is not None:
hc_id = get_job_id(show_log)
for line in logs.dbcmd('get_log', hc_id):
safeprint(line)
elif export_output is not None:
output_id, target_dir = export_output
dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
for line in core.export_output(dskey, calc_id, datadir,
os.path.expanduser(target_dir), exports
or 'csv,xml'):
safeprint(line)
elif export_outputs is not None:
job_id, target_dir = export_outputs
hc_id = get_job_id(job_id)
for line in core.export_outputs(hc_id, os.path.expanduser(target_dir),
exports or 'csv,xml'):
safeprint(line)
elif delete_uncompleted_calculations:
logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())
else:
engine.parentparser.prog = 'oq engine'
engine.parentparser.print_usage()