def reset(yes):
"""
Remove all the datastores and the database of the current user
"""
ok = yes or confirm('Do you really want to destroy all your data? (y/n) ')
if not ok:
return
dbpath = os.path.realpath(
os.path.expanduser(config.get('dbserver', 'file')))
# user must be able to access and write the databse file to remove it
if os.path.isfile(dbpath) and os.access(dbpath, os.W_OK):
if dbserver.get_status() == 'running':
if config.flag_set('dbserver', 'multi_user'):
sys.exit('The oq dbserver must be stopped '
'before proceeding')
else:
logs.dbcmd('stop')
print('dbserver stopped')
try:
os.remove(dbpath)
print('Removed %s' % dbpath)
except OSError as exc:
print(exc, file=sys.stderr)
# fast way of removing everything
purge_all(fast=True) # datastore of the current user
def test_get_with_unknown_key(self):
# config.get() returns `None` if the `key` is not known
with patch('openquake.commonlib.config.get_section') as mock:
mock.return_value = dict(b=1)
self.assertTrue(config.get("arghh", "c") is None)
self.assertEqual(1, mock.call_count)
self.assertEqual([("arghh",), {}], mock.call_args)
def test_get_with_empty_section_data(self):
# config.get() returns `None` if the section data dict is empty
with patch('openquake.commonlib.config.get_section') as mock:
mock.return_value = dict()
self.assertTrue(config.get("whatever", "key") is None)
self.assertEqual(1, mock.call_count)
self.assertEqual([("whatever",), {}], mock.call_args)
def test_get_with_empty_section_data(self):
# config.get() returns `None` if the section data dict is empty
with patch('openquake.commonlib.config.get_section') as mock:
mock.return_value = dict()
self.assertTrue(config.get("whatever", "key") is None)
self.assertEqual(1, mock.call_count)
self.assertEqual([("whatever",), {}], mock.call_args)
def dbserver(cmd, dbhostport=None, dbpath=None):
"""
start/stop/restart the database server, or return its status
"""
if valid.boolean(config.get('dbserver', 'multi_user')):
sys.exit('oq dbserver only works in single user mode')
status = dbs.get_status()
if cmd == 'status':
print('dbserver ' + status)
elif cmd == 'stop':
if status == 'running':
logs.dbcmd('stop')
print('dbserver stopped')
else:
print('dbserver already stopped')
elif cmd == 'start':
if status == 'not-running':
dbs.run_server(dbhostport, dbpath)
else:
print('dbserver already running')
elif cmd == 'restart':
if status == 'running':
logs.dbcmd('stop')
print('dbserver stopped')
dbs.run_server(dbhostport, dbpath)
def dbserver(cmd):
"""
start/stop/restart the database server, or return its status
"""
if valid.boolean(config.get('dbserver', 'multi_user')):
sys.exit('oq dbserver only works in single user mode')
status = dbs.get_status()
if cmd == 'status':
print('dbserver ' + status)
elif cmd == 'stop':
if status == 'running':
logs.dbcmd('stop')
print('dbserver stopped')
else:
print('dbserver already stopped')
elif cmd == 'start':
if status == 'not-running':
dbs.run_server()
else:
print('dbserver already running')
elif cmd == 'restart':
if status == 'running':
logs.dbcmd('stop')
print('dbserver stopped')
dbs.run_server()
def test_get_with_unknown_key(self):
# config.get() returns `None` if the `key` is not known
with patch('openquake.commonlib.config.get_section') as mock:
mock.return_value = dict(b=1)
self.assertTrue(config.get("arghh", "c") is None)
self.assertEqual(1, mock.call_count)
self.assertEqual([("arghh",), {}], mock.call_args)
def test_get_with_nonempty_section_data_and_known_key(self):
# config.get() correctly returns the configuration datum for known
# sections/keys
with patch('openquake.commonlib.config.get_section') as mock:
mock.return_value = dict(a=11)
self.assertEqual(11, config.get("hmmm", "a"))
self.assertEqual(1, mock.call_count)
self.assertEqual([("hmmm",), {}], mock.call_args)
def test_get_with_nonempty_section_data_and_known_key(self):
# config.get() correctly returns the configuration datum for known
# sections/keys
with patch('openquake.commonlib.config.get_section') as mock:
mock.return_value = dict(a=11)
self.assertEqual(11, config.get("hmmm", "a"))
self.assertEqual(1, mock.call_count)
self.assertEqual([("hmmm",), {}], mock.call_args)
def webui(cmd, hostport='127.0.0.1:8800', skip_browser=False):
"""
start the webui server in foreground or perform other operation on the
django application
"""
db_path = os.path.expanduser(config.get('dbserver', 'file'))
if os.path.isfile(db_path) and not os.access(db_path, os.W_OK):
sys.exit('This command must be run by the proper user: '******'see the documentation for details')
if cmd == 'start':
dbserver.ensure_on() # start the dbserver in a subproces
rundjango('runserver', hostport, skip_browser)
elif cmd in commands:
rundjango(cmd)
def webui(cmd, hostport='127.0.0.1:8800'):
"""
start the webui server in foreground or perform other operation on the
django application
"""
db_path = os.path.expanduser(config.get('dbserver', 'file'))
if os.path.isfile(db_path) and not os.access(db_path, os.W_OK):
sys.exit('This command must be run by the proper user: '******'see the documentation for details')
if cmd == 'start':
dbserver.ensure_on() # start the dbserver in a subproces
rundjango('runserver', hostport)
elif cmd == 'migrate':
rundjango('migrate')
# For backward compatibility with Django 1.6
elif cmd == 'syncdb':
rundjango('syncdb')
def ensure_on():
"""
Start the DbServer if it is off
"""
if get_status() == 'not-running':
if valid.boolean(config.get('dbserver', 'multi_user')):
sys.exit('Please start the DbServer: '
'see the documentation for details')
# otherwise start the DbServer automatically
subprocess.Popen(
[sys.executable, '-m', 'openquake.server.dbserver', '-l', 'INFO'])
# wait for the dbserver to start
waiting_seconds = 10
while get_status() == 'not-running':
if waiting_seconds == 0:
sys.exit('The DbServer cannot be started after 10 seconds. '
'Please check the configuration')
time.sleep(1)
waiting_seconds -= 1
def ensure_on():
"""
Start the DbServer if it is off
"""
if get_status() == 'not-running':
if valid.boolean(config.get('dbserver', 'multi_user')):
sys.exit('Please start the DbServer: '
'see the documentation for details')
# otherwise start the DbServer automatically
subprocess.Popen([sys.executable, '-m', 'openquake.server.dbserver',
'-l', 'INFO'])
# wait for the dbserver to start
waiting_seconds = 10
while get_status() == 'not-running':
if waiting_seconds == 0:
sys.exit('The DbServer cannot be started after 10 seconds. '
'Please check the configuration')
time.sleep(1)
waiting_seconds -= 1
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
from openquake.baselib.performance import Monitor
from openquake.baselib import parallel
from openquake.commonlib import config
SOFT_MEM_LIMIT = int(config.get('memory', 'soft_mem_limit'))
HARD_MEM_LIMIT = int(config.get('memory', 'hard_mem_limit'))
parallel.check_mem_usage.__defaults__ = (Monitor(), SOFT_MEM_LIMIT,
HARD_MEM_LIMIT)
try:
raw_input
except NameError: # Python 3
raw_input = input
def confirm(prompt):
"""
Ask for confirmation, given a ``prompt`` and return a boolean value.
"""
import os
import sys
import signal
import traceback
from openquake.baselib.performance import Monitor
from openquake.hazardlib import valid
from openquake.baselib import parallel
from openquake.commonlib.oqvalidation import OqParam
from openquake.commonlib import datastore, config, readinput
from openquake.calculators import base, views, export
from openquake.commonlib import logs
TERMINATE = valid.boolean(
config.get('distribution', 'terminate_workers_on_revoke') or 'false')
USE_CELERY = config.get('distribution', 'oq_distribute') == 'celery'
if USE_CELERY:
import celery.task.control
def set_concurrent_tasks_default():
"""
Set the default for concurrent_tasks.
Returns the number of live celery nodes (i.e. the number of machines).
"""
stats = celery.task.control.inspect(timeout=1).stats()
if not stats:
sys.exit("No live compute nodes, aborting calculation")
num_cores = sum(stats[k]['pool']['max-concurrency'] for k in stats)
import os
import re
import getpass
import collections
import numpy
import h5py
from openquake.baselib.python3compat import pickle
from openquake.baselib import hdf5
from openquake.commonlib import config
from openquake.commonlib.writers import write_csv
DATADIR = os.environ.get('OQ_DATADIR')
if not DATADIR:
shared_dir = config.get('directory', 'shared_dir')
if shared_dir:
DATADIR = os.path.join(shared_dir, getpass.getuser(), 'oqdata')
else: # use the home of the user
DATADIR = os.path.join(os.path.expanduser('~'), 'oqdata')
def get_calc_ids(datadir=DATADIR):
"""
Extract the available calculation IDs from the datadir, in order.
"""
if not os.path.exists(datadir):
return []
calc_ids = []
for f in os.listdir(datadir):
mo = re.match(r'calc_(\d+)\.hdf5', f)
import os
import re
import getpass
import collections
import numpy
import h5py
from openquake.baselib.python3compat import pickle
from openquake.baselib import hdf5
from openquake.commonlib import config
from openquake.commonlib.writers import write_csv
DATADIR = os.environ.get('OQ_DATADIR')
if not DATADIR:
shared_dir = config.get('directory', 'shared_dir')
if shared_dir:
DATADIR = os.path.join(shared_dir, getpass.getuser(), 'oqdata')
else: # use the home of the user
DATADIR = os.path.join(os.path.expanduser('~'), 'oqdata')
def get_nbytes(dset):
"""
If the dataset has an attribute 'nbytes', return it. Otherwise get the size
of the underlying array. Returns None if the dataset is actually a group.
"""
if 'nbytes' in dset.attrs:
# look if the dataset has an attribute nbytes
return dset.attrs['nbytes']
elif hasattr(dset, 'value'):
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import os
import sys
import importlib
from openquake.baselib import sap
from openquake.commonlib import __version__
from openquake import commands
from openquake.commonlib import config
USE_CELERY = config.get('distribution', 'oq_distribute') == 'celery'
# the environment variable has the precedence over the configuration file
if 'OQ_DISTRIBUTE' not in os.environ and USE_CELERY:
os.environ['OQ_DISTRIBUTE'] = 'celery'
# force cluster users to use `oq engine` so that we have centralized logs
if USE_CELERY and 'run' in sys.argv:
sys.exit('You are on a cluster and you are using oq run?? '
'Use oq engine --run instead!')
def oq():
modnames = [
'openquake.commands.%s' % mod[:-3]
for mod in os.listdir(commands.__path__[0])
import os
import sys
import signal
import traceback
from openquake.baselib.performance import Monitor
from openquake.risklib import valid
from openquake.baselib import parallel
from openquake.commonlib.oqvalidation import OqParam
from openquake.commonlib import datastore, config, readinput
from openquake.calculators import base, views, export
from openquake.engine import logs
TERMINATE = valid.boolean(
config.get('celery', 'terminate_workers_on_revoke') or 'false')
USE_CELERY = valid.boolean(config.get('celery', 'use_celery') or 'false')
if USE_CELERY:
import celery.task.control
def set_concurrent_tasks_default():
"""
Set the default for concurrent_tasks.
Returns the number of live celery nodes (i.e. the number of machines).
"""
stats = celery.task.control.inspect(timeout=1).stats()
if not stats:
sys.exit("No live compute nodes, aborting calculation")
num_cores = sum(stats[k]['pool']['max-concurrency'] for k in stats)
import io
import os
import getpass
import logging
from openquake.hazardlib.calc.hazard_curve import zero_curves
from openquake.baselib import sap
from openquake.risklib import scientific, valid
from openquake.commonlib import datastore
from openquake.commonlib.writers import write_csv
from openquake.commonlib.util import rmsep
from openquake.commonlib import config
from openquake.engine import logs
from openquake.calculators.views import view
MULTI_USER = valid.boolean(config.get('dbserver', 'multi_user') or 'false')
if MULTI_USER:
# get the datastore of the user who ran the job
def read(calc_id):
job = logs.dbcmd('get_job', calc_id, getpass.getuser())
datadir = os.path.dirname(job.ds_calc_dir)
return datastore.read(job.id, datadir=datadir)
else: # get the datastore of the current user
read = datastore.read
def get_hcurves_and_means(dstore):
"""
Extract hcurves from the datastore and compute their means.
:returns: curves_by_rlz, mean_curves
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import os
import sys
import importlib
from openquake.baselib import sap
from openquake.commonlib import __version__
from openquake import commands
from openquake.risklib import valid
from openquake.commonlib import config
USE_CELERY = valid.boolean(config.get('celery', 'use_celery') or 'false')
# the environment variable has the precedence over the configuration file
if 'OQ_DISTRIBUTE' not in os.environ and USE_CELERY:
os.environ['OQ_DISTRIBUTE'] = 'celery'
# force cluster users to use `oq engine` so that we have centralized logs
if USE_CELERY and 'run' in sys.argv:
sys.exit('You are on a cluster and you are using oq run?? '
'Use oq engine --run instead!')
def oq():
modnames = ['openquake.commands.%s' % mod[:-3]
for mod in os.listdir(commands.__path__[0])
if mod.endswith('.py') and not mod.startswith('_')]
for modname in modnames:
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
from openquake.baselib.performance import Monitor
from openquake.baselib import parallel
from openquake.commonlib import config
SOFT_MEM_LIMIT = int(config.get('memory', 'soft_mem_limit'))
HARD_MEM_LIMIT = int(config.get('memory', 'hard_mem_limit'))
parallel.check_mem_usage.__defaults__ = (
Monitor(), SOFT_MEM_LIMIT, HARD_MEM_LIMIT)
def confirm(prompt):
"""
Ask for confirmation, given a ``prompt`` and return a boolean value.
"""
while True:
try:
answer = raw_input(prompt)
except KeyboardInterrupt:
# the user presses ctrl+c, just say 'no'
