def reset(yes):
"""
Remove all the datastores and the database of the current user
"""
ok = yes or confirm('Do you really want to destroy all your data? (y/n) ')
if not ok:
return
dbpath = os.path.realpath(os.path.expanduser(config.dbserver.file))
# user must be able to access and write the databse file to remove it
if os.path.isfile(dbpath) and os.access(dbpath, os.W_OK):
if dbserver.get_status() == 'running':
if config.dbserver.multi_user:
sys.exit('The oq dbserver must be stopped '
'before proceeding')
else:
pid = logs.dbcmd('getpid')
os.kill(pid, signal.SIGTERM)
time.sleep(.5) # give time to stop
assert dbserver.get_status() == 'not-running'
print('dbserver stopped')
try:
os.remove(dbpath)
print('Removed %s' % dbpath)
except OSError as exc:
print(exc, file=sys.stderr)
# fast way of removing everything
purge_all(fast=True) # datastore of the current user
def reset(yes):
"""
Remove all the datastores and the database of the current user
"""
ok = yes or confirm('Do you really want to destroy all your data? (y/n) ')
if not ok:
return
status = dbserver.get_status()
dbpath = os.path.realpath(os.path.expanduser(config.dbserver.file))
if not os.path.isfile(dbpath):
sys.exit('%s does not exist' % dbpath)
elif status == 'running':
user = getpass.getuser()
for calc_id in logs.dbcmd('get_calc_ids', user):
purge_one(calc_id, user, force=True)
purge_all(user) # calculations in oqdata not in the db
if os.access(dbpath, os.W_OK): # single user mode
# stop the dbserver first
pid = logs.dbcmd('getpid')
os.kill(pid, signal.SIGTERM)
time.sleep(.5) # give time to stop
assert dbserver.get_status() == 'not-running'
print('dbserver stopped')
# remove the database
os.remove(dbpath)
print('Removed %s' % dbpath)
else:
sys.exit('The dbserver is not running')
def reset(yes):
"""
Remove all the datastores and the database of the current user
"""
ok = yes or confirm('Do you really want to destroy all your data? (y/n) ')
if not ok:
return
dbpath = os.path.realpath(os.path.expanduser(config.dbserver.file))
# user must be able to access and write the databse file to remove it
if os.path.isfile(dbpath) and os.access(dbpath, os.W_OK):
if dbserver.get_status() == 'running':
if config.dbserver.multi_user:
sys.exit('The oq dbserver must be stopped '
'before proceeding')
else:
pid = logs.dbcmd('getpid')
os.kill(pid, signal.SIGTERM)
time.sleep(.5) # give time to stop
assert dbserver.get_status() == 'not-running'
print('dbserver stopped')
try:
os.remove(dbpath)
print('Removed %s' % dbpath)
except OSError as exc:
print(exc, file=sys.stderr)
# fast way of removing everything
purge_all(fast=True) # datastore of the current user
def dbserver(cmd, dbhostport=None,
dbpath=os.path.expanduser(config.dbserver.file)):
"""
start/stop/restart the database server, or return its status
"""
if config.dbserver.multi_user and getpass.getuser() != 'openquake':
sys.exit('oq dbserver only works in single user mode')
status = dbs.get_status()
if cmd == 'status':
print('dbserver ' + status)
elif cmd == 'stop':
if status == 'running':
pid = logs.dbcmd('getpid')
os.kill(pid, signal.SIGINT) # this is trapped by the DbServer
else:
print('dbserver already stopped')
elif cmd == 'start':
if status == 'not-running':
dbs.run_server.func(dbpath, dbhostport)
else:
print('dbserver already running')
elif cmd == 'restart':
if status == 'running':
pid = logs.dbcmd('getpid')
os.kill(pid, signal.SIGINT)
dbs.run_server.func(dbpath, dbhostport)
def dbserver(cmd, dbhostport=None, dbpath=None):
"""
start/stop/restart the database server, or return its status
"""
if valid.boolean(config.get('dbserver', 'multi_user')):
sys.exit('oq dbserver only works in single user mode')
status = dbs.get_status()
if cmd == 'status':
print('dbserver ' + status)
elif cmd == 'stop':
if status == 'running':
logs.dbcmd('stop')
print('dbserver stopped')
else:
print('dbserver already stopped')
elif cmd == 'start':
if status == 'not-running':
dbs.run_server(dbhostport, dbpath)
else:
print('dbserver already running')
elif cmd == 'restart':
if status == 'running':
logs.dbcmd('stop')
print('dbserver stopped')
dbs.run_server(dbhostport, dbpath)
def main(cmd,
dbhostport=None,
foreground=False,
*,
loglevel='INFO'):
"""
start/stop/restart the database server, or return its status
"""
if config.dbserver.multi_user and getpass.getuser() != 'openquake':
sys.exit('oq dbserver only works in single user mode')
status = dbs.get_status()
if cmd == 'status':
print('dbserver ' + status)
elif cmd == 'stop':
if status == 'running':
pid = logs.dbcmd('getpid')
os.kill(pid, signal.SIGINT) # this is trapped by the DbServer
else:
print('dbserver already stopped')
elif cmd == 'start':
if status == 'not-running':
dbs.run_server(dbhostport, loglevel, foreground)
else:
print('dbserver already running')
def extract(what, calc_id=-1, hostport=None):
"""
Extract an output from the datastore and save it into an .hdf5 file.
"""
logging.basicConfig(level=logging.INFO)
if calc_id < 0:
calc_id = datastore.get_calc_ids()[calc_id]
hdf5path = None
if dbserver.get_status() == 'running':
job = dbcmd('get_job', calc_id)
if job is not None:
hdf5path = job.ds_calc_dir + '.hdf5'
dstore = datastore.read(hdf5path or calc_id)
parent_id = dstore['oqparam'].hazard_calculation_id
if parent_id:
dstore.parent = datastore.read(parent_id)
urlpath = '/v1/calc/%d/extract/%s' % (calc_id, quote(what))
with performance.Monitor('extract', measuremem=True) as mon, dstore:
if hostport:
data = urlopen('http://%s%s' % (hostport, urlpath)).read()
items = (item for item in numpy.load(io.BytesIO(data)).items())
else:
print('Emulating call to %s' % urlpath)
items = extract_(dstore, what)
if not inspect.isgenerator(items):
items = [(items.__class__.__name__, items)]
fname = '%s_%d.hdf5' % (what.replace('/', '-').replace(
'?', '-'), dstore.calc_id)
hdf5.save(fname, items)
print('Saved', fname)
if mon.duration > 1:
print(mon)
def reset(yes):
"""
Remove all the datastores and the database of the current user
"""
ok = yes or confirm('Do you really want to destroy all your data? (y/n) ')
if not ok:
return
dbpath = os.path.realpath(
os.path.expanduser(config.get('dbserver', 'file')))
# user must be able to access and write the databse file to remove it
if os.path.isfile(dbpath) and os.access(dbpath, os.W_OK):
if dbserver.get_status() == 'running':
if config.flag_set('dbserver', 'multi_user'):
sys.exit('The oq dbserver must be stopped '
'before proceeding')
else:
logs.dbcmd('stop')
print('dbserver stopped')
try:
os.remove(dbpath)
print('Removed %s' % dbpath)
except OSError as exc:
print(exc, file=sys.stderr)
# fast way of removing everything
purge_all(fast=True) # datastore of the current user
def dbserver(cmd):
"""
start/stop/restart the database server, or return its status
"""
if valid.boolean(config.get('dbserver', 'multi_user')):
sys.exit('oq dbserver only works in single user mode')
status = get_status()
if cmd == 'status':
print(status)
elif cmd == 'stop':
if status == 'running':
logs.dbcmd('stop')
print('stopped')
else:
print('already stopped')
elif cmd == 'start':
if status == 'not-running':
runserver()
print('started')
else:
print('already running')
elif cmd == 'restart':
if status == 'running':
logs.dbcmd('stop')
print('stopped')
runserver()
print('started')
def dbserver(cmd, dbhostport=None,
dbpath=os.path.expanduser(config.dbserver.file),
foreground=False):
"""
start/stop/restart the database server, or return its status
"""
if config.dbserver.multi_user and getpass.getuser() != 'openquake':
sys.exit('oq dbserver only works in single user mode')
status = dbs.get_status()
if cmd == 'status':
print('dbserver ' + status)
elif cmd == 'stop':
if status == 'running':
pid = logs.dbcmd('getpid')
os.kill(pid, signal.SIGINT) # this is trapped by the DbServer
else:
print('dbserver already stopped')
elif cmd == 'start':
if status == 'not-running':
dbs.run_server(dbpath, dbhostport)
else:
print('dbserver already running')
elif cmd == 'restart':
if status == 'running':
pid = logs.dbcmd('getpid')
os.kill(pid, signal.SIGINT)
dbs.run_server(dbpath, dbhostport)
def extract(what, calc_id=-1):
"""
Extract an output from the datastore and save it into an .hdf5 file.
"""
logging.basicConfig(level=logging.INFO)
if calc_id < 0:
calc_id = datastore.get_calc_ids()[calc_id]
if dbserver.get_status() == 'running':
job = dbcmd('get_job', calc_id)
if job is not None:
calc_id = job.ds_calc_dir + '.hdf5'
dstore = datastore.read(calc_id)
parent_id = dstore['oqparam'].hazard_calculation_id
if parent_id:
dstore.parent = datastore.read(parent_id)
print('Emulating call to /v1/calc/%s/extract/%s' % (calc_id, quote(what)))
with performance.Monitor('extract', measuremem=True) as mon, dstore:
items = extract_(dstore, what)
if not inspect.isgenerator(items):
items = [(items.__class__.__name__, items)]
fname = '%s_%d.hdf5' % (what.replace('/', '-').replace(
'?', '-'), dstore.calc_id)
hdf5.save(fname, items)
print('Saved', fname)
if mon.duration > 1:
print(mon)
def setUpClass(cls):
assert get_status() == 'running'
cls.job_ids = []
env = os.environ.copy()
env['OQ_DISTRIBUTE'] = 'no'
# let's impersonate the user openquake, the one running the WebUI:
# we need to set LOGNAME on Linux and USERNAME on Windows
env['LOGNAME'] = env['USERNAME'] = '******'
cls.c = Client()
def setUpClass(cls):
assert get_status() == 'running'
cls.job_ids = []
env = os.environ.copy()
env['OQ_DISTRIBUTE'] = 'no'
# let's impersonate the user openquake, the one running the WebUI:
# we need to set LOGNAME on Linux and USERNAME on Windows
env['LOGNAME'] = env['USERNAME'] = '******'
cls.c = Client()
def webui(cmd, hostport='127.0.0.1:8800'):
"""
start the webui server in foreground or perform other operation on the
django application
"""
dbstatus = get_status()
if dbstatus == 'not-running':
if valid.boolean(config.get('dbserver', 'multi_user')):
sys.exit('Please start the DbServer: '
'see the documentation for details')
dbserver.runserver()
if cmd == 'start':
rundjango('runserver', hostport)
elif cmd == 'syncdb':
rundjango('syncdb')
def extract(what, calc_id=-1, server_url='http://127.0.0.1:8800'):
"""
Extract an output from the datastore and save it into an .hdf5 file.
By default use the WebUI, unless server_url is set to 'local', in
which case the extraction is done directly bypassing tjhe WebUI and
the database.
"""
logging.basicConfig(level=logging.INFO)
if calc_id < 0:
calc_id = datastore.get_calc_ids()[calc_id]
filename = None
if dbserver.get_status() == 'running':
job = dbcmd('get_job', calc_id)
if job is not None:
filename = job.ds_calc_dir + '.hdf5'
dstore = util.read(filename or calc_id)
parent_id = dstore['oqparam'].hazard_calculation_id
if parent_id:
dstore.parent = util.read(parent_id)
urlpath = '/v1/calc/%d/extract/%s' % (calc_id, quote(what))
with performance.Monitor('extract', measuremem=True) as mon, dstore:
if server_url == 'local':
print('Emulating call to %s' % urlpath)
items = extract_(dstore, what)
else:
print('Calling %s%s' % (server_url, urlpath))
data = urlopen(server_url.rstrip('/') + urlpath).read()
items = (item for item in numpy.load(io.BytesIO(data)).items())
if not inspect.isgenerator(items):
items = [(items.__class__.__name__, items)]
fname = '%s_%d.hdf5' % (what.replace('/', '-').replace(
'?', '-'), dstore.calc_id)
hdf5.save(fname, items)
print('Saved', fname)
if mon.duration > 1:
print(mon)
def engine(log_file, no_distribute, yes, config_file, make_html_report,
upgrade_db, version_db, what_if_I_upgrade,
run_hazard, run_risk, run,
list_hazard_calculations, list_risk_calculations,
delete_calculation, delete_uncompleted_calculations,
hazard_calculation_id, list_outputs, show_log,
export_output, export_outputs, exports='',
log_level='info'):
"""
Run a calculation using the traditional command line API
"""
config.abort_if_no_config_available()
if run or run_hazard or run_risk:
# the logging will be configured in engine.py
pass
else:
# configure a basic logging
logging.basicConfig(level=logging.INFO)
if config_file:
os.environ[config.OQ_CONFIG_FILE_VAR] = os.path.abspath(
os.path.expanduser(config_file))
config.refresh()
if no_distribute:
os.environ['OQ_DISTRIBUTE'] = 'no'
# check if the datadir exists
if not os.path.exists(datastore.DATADIR):
os.makedirs(datastore.DATADIR)
# check if the DbServer is up
if dbserver.get_status() == 'not-running':
if valid.boolean(config.get('dbserver', 'multi_user')):
sys.exit('Please start the DbServer: '
'see the documentation for details')
# otherwise start the DbServer automatically
subprocess.Popen([sys.executable, '-m', 'openquake.server.dbserver',
'-l', 'INFO'])
if upgrade_db:
logs.set_level('info')
msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
if msg.startswith('Your database is already updated'):
pass
elif yes or confirm('Proceed? (y/n) '):
logs.dbcmd('upgrade_db')
sys.exit(0)
if version_db:
print(logs.dbcmd('version_db'))
sys.exit(0)
if what_if_I_upgrade:
print(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
sys.exit(0)
# check if the db is outdated
outdated = logs.dbcmd('check_outdated')
if outdated:
sys.exit(outdated)
# hazard or hazard+risk
if hazard_calculation_id:
hc_id = get_job_id(hazard_calculation_id)
else:
hc_id = None
if run:
job_inis = [os.path.expanduser(ini) for ini in run.split(',')]
if len(job_inis) not in (1, 2):
sys.exit('%s should be a .ini filename or a pair of filenames '
'separated by a comma' % run)
for job_ini in job_inis:
open(job_ini).read() # raise an IOError if the file does not exist
log_file = os.path.expanduser(log_file) \
if log_file is not None else None
if len(job_inis) == 2:
# run hazard
job_id = run_job(job_inis[0], log_level,
log_file, exports)
# run risk
run_job(job_inis[1], log_level, log_file,
exports, hazard_calculation_id=job_id)
else:
run_job(
os.path.expanduser(run), log_level, log_file,
exports, hazard_calculation_id=hc_id)
# hazard
elif list_hazard_calculations:
for line in logs.dbcmd(
'list_calculations', 'hazard', getpass.getuser()):
print(line)
elif run_hazard is not None:
print('WARN: --rh/--run-hazard are deprecated, use --run instead',
file=sys.stderr)
log_file = os.path.expanduser(log_file) \
if log_file is not None else None
run_job(os.path.expanduser(run_hazard), log_level,
log_file, exports)
elif delete_calculation is not None:
delete_calculation(delete_calculation, yes)
# risk
elif list_risk_calculations:
for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
print(line)
elif run_risk is not None:
print('WARN: --rr/--run-risk are deprecated, use --run instead',
file=sys.stderr)
if hazard_calculation_id is None:
sys.exit(MISSING_HAZARD_MSG)
log_file = os.path.expanduser(log_file) \
if log_file is not None else None
run_job(
os.path.expanduser(run_risk),
log_level, log_file, exports,
hazard_calculation_id=hc_id)
# export
elif make_html_report:
print('Written %s' % make_report(make_html_report))
sys.exit(0)
elif list_outputs is not None:
hc_id = get_job_id(list_outputs)
for line in logs.dbcmd('list_outputs', hc_id):
print(line)
elif show_log is not None:
hc_id = get_job_id(show_log)
for line in logs.dbcmd('get_log', hc_id):
print(line)
elif export_output is not None:
output_id, target_dir = export_output
dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
for line in core.export_output(
dskey, calc_id, datadir, os.path.expanduser(target_dir),
exports or 'xml,csv'):
print(line)
elif export_outputs is not None:
job_id, target_dir = export_outputs
hc_id = get_job_id(job_id)
for line in core.export_outputs(
hc_id, os.path.expanduser(target_dir), exports or 'xml,csv'):
print(line)
elif delete_uncompleted_calculations:
logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())
else:
parser.parentparser.prog = 'oq engine'
parser.parentparser.print_usage()
def __init__(self, job_ini, event_info, no_distribute=False):
"""
Initialize a calculation (reinvented from openquake.engine.engine)
:param job_ini:
Path to configuration file/archive or
dictionary of parameters with at least a key "calculation_mode"
"""
user_name = getpass.getuser()
if no_distribute:
os.environ['OQ_DISTRIBUTE'] = 'no'
# check if the datadir exists
datadir = datastore.get_datadir()
if not os.path.exists(datadir):
os.makedirs(datadir)
#dbserver.ensure_on()
if dbserver.get_status() == 'not-running':
if config.dbserver.multi_user:
sys.exit('Please start the DbServer: '
'see the documentation for details')
# otherwise start the DbServer automatically; NB: I tried to use
# multiprocessing.Process(target=run_server).start() and apparently
# it works, but then run-demos.sh hangs after the end of the first
# calculation, but only if the DbServer is started by oq engine (!?)
# Here is a trick to activate OpenQuake's dbserver
# We first cd to the openquake directory and invoke subprocess to open/hold on dbserver
# Then, we cd back to the original working directory
owd = os.getcwd()
os.chdir(os.path.dirname(os.path.realpath(__file__)))
subprocess.Popen([
sys.executable, '-m', 'openquake.commands', 'dbserver', 'start'
])
os.chdir(owd)
# wait for the dbserver to start
waiting_seconds = 30
while dbserver.get_status() == 'not-running':
if waiting_seconds == 0:
sys.exit(
'The DbServer cannot be started after 30 seconds. '
'Please check the configuration')
time.sleep(1)
waiting_seconds -= 1
# check if we are talking to the right server
err = dbserver.check_foreign()
if err:
sys.exit(err)
# Copy the event_info
self.event_info = event_info
# Create a job
#self.job = logs.init("job", job_ini, logging.INFO, None, None, None)
dic = readinput.get_params(job_ini)
#dic['hazard_calculation_id'] = self.job.calc_id
# Create the job log
self.log = logs.init('job', dic, logging.INFO, None, None, None)
# Get openquake parameters
self.oqparam = self.log.get_oqparam()
# Create the calculator
self.calculator = base.calculators(self.oqparam, self.log.calc_id)
self.calculator.from_engine = True
print('FetchOpenQuake: OpenQuake Hazard Calculator initiated.')
