def reset(yes): """ Remove all the datastores and the database of the current user """ ok = yes or confirm('Do you really want to destroy all your data? (y/n) ') if not ok: return dbpath = os.path.realpath( os.path.expanduser(config.get('dbserver', 'file'))) # user must be able to access and write the databse file to remove it if os.path.isfile(dbpath) and os.access(dbpath, os.W_OK): if dbserver.get_status() == 'running': if config.flag_set('dbserver', 'multi_user'): sys.exit('The oq dbserver must be stopped ' 'before proceeding') else: logs.dbcmd('stop') print('dbserver stopped') try: os.remove(dbpath) print('Removed %s' % dbpath) except OSError as exc: print(exc, file=sys.stderr) # fast way of removing everything purge_all(fast=True) # datastore of the current user
def test_get_with_unknown_key(self): # config.get() returns `None` if the `key` is not known with patch('openquake.commonlib.config.get_section') as mock: mock.return_value = dict(b=1) self.assertTrue(config.get("arghh", "c") is None) self.assertEqual(1, mock.call_count) self.assertEqual([("arghh",), {}], mock.call_args)
def test_get_with_empty_section_data(self): # config.get() returns `None` if the section data dict is empty with patch('openquake.commonlib.config.get_section') as mock: mock.return_value = dict() self.assertTrue(config.get("whatever", "key") is None) self.assertEqual(1, mock.call_count) self.assertEqual([("whatever",), {}], mock.call_args)
def dbserver(cmd, dbhostport=None, dbpath=None): """ start/stop/restart the database server, or return its status """ if valid.boolean(config.get('dbserver', 'multi_user')): sys.exit('oq dbserver only works in single user mode') status = dbs.get_status() if cmd == 'status': print('dbserver ' + status) elif cmd == 'stop': if status == 'running': logs.dbcmd('stop') print('dbserver stopped') else: print('dbserver already stopped') elif cmd == 'start': if status == 'not-running': dbs.run_server(dbhostport, dbpath) else: print('dbserver already running') elif cmd == 'restart': if status == 'running': logs.dbcmd('stop') print('dbserver stopped') dbs.run_server(dbhostport, dbpath)
def dbserver(cmd): """ start/stop/restart the database server, or return its status """ if valid.boolean(config.get('dbserver', 'multi_user')): sys.exit('oq dbserver only works in single user mode') status = dbs.get_status() if cmd == 'status': print('dbserver ' + status) elif cmd == 'stop': if status == 'running': logs.dbcmd('stop') print('dbserver stopped') else: print('dbserver already stopped') elif cmd == 'start': if status == 'not-running': dbs.run_server() else: print('dbserver already running') elif cmd == 'restart': if status == 'running': logs.dbcmd('stop') print('dbserver stopped') dbs.run_server()
def test_get_with_nonempty_section_data_and_known_key(self): # config.get() correctly returns the configuration datum for known # sections/keys with patch('openquake.commonlib.config.get_section') as mock: mock.return_value = dict(a=11) self.assertEqual(11, config.get("hmmm", "a")) self.assertEqual(1, mock.call_count) self.assertEqual([("hmmm",), {}], mock.call_args)
def webui(cmd, hostport='127.0.0.1:8800', skip_browser=False): """ start the webui server in foreground or perform other operation on the django application """ db_path = os.path.expanduser(config.get('dbserver', 'file')) if os.path.isfile(db_path) and not os.access(db_path, os.W_OK): sys.exit('This command must be run by the proper user: '******'see the documentation for details') if cmd == 'start': dbserver.ensure_on() # start the dbserver in a subproces rundjango('runserver', hostport, skip_browser) elif cmd in commands: rundjango(cmd)
def webui(cmd, hostport='127.0.0.1:8800'): """ start the webui server in foreground or perform other operation on the django application """ db_path = os.path.expanduser(config.get('dbserver', 'file')) if os.path.isfile(db_path) and not os.access(db_path, os.W_OK): sys.exit('This command must be run by the proper user: '******'see the documentation for details') if cmd == 'start': dbserver.ensure_on() # start the dbserver in a subproces rundjango('runserver', hostport) elif cmd == 'migrate': rundjango('migrate') # For backward compatibility with Django 1.6 elif cmd == 'syncdb': rundjango('syncdb')
def ensure_on(): """ Start the DbServer if it is off """ if get_status() == 'not-running': if valid.boolean(config.get('dbserver', 'multi_user')): sys.exit('Please start the DbServer: ' 'see the documentation for details') # otherwise start the DbServer automatically subprocess.Popen( [sys.executable, '-m', 'openquake.server.dbserver', '-l', 'INFO']) # wait for the dbserver to start waiting_seconds = 10 while get_status() == 'not-running': if waiting_seconds == 0: sys.exit('The DbServer cannot be started after 10 seconds. ' 'Please check the configuration') time.sleep(1) waiting_seconds -= 1
def ensure_on(): """ Start the DbServer if it is off """ if get_status() == 'not-running': if valid.boolean(config.get('dbserver', 'multi_user')): sys.exit('Please start the DbServer: ' 'see the documentation for details') # otherwise start the DbServer automatically subprocess.Popen([sys.executable, '-m', 'openquake.server.dbserver', '-l', 'INFO']) # wait for the dbserver to start waiting_seconds = 10 while get_status() == 'not-running': if waiting_seconds == 0: sys.exit('The DbServer cannot be started after 10 seconds. ' 'Please check the configuration') time.sleep(1) waiting_seconds -= 1
# by the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # OpenQuake is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with OpenQuake. If not, see <http://www.gnu.org/licenses/>. from openquake.baselib.performance import Monitor from openquake.baselib import parallel from openquake.commonlib import config SOFT_MEM_LIMIT = int(config.get('memory', 'soft_mem_limit')) HARD_MEM_LIMIT = int(config.get('memory', 'hard_mem_limit')) parallel.check_mem_usage.__defaults__ = (Monitor(), SOFT_MEM_LIMIT, HARD_MEM_LIMIT) try: raw_input except NameError: # Python 3 raw_input = input def confirm(prompt): """ Ask for confirmation, given a ``prompt`` and return a boolean value. """
import os import sys import signal import traceback from openquake.baselib.performance import Monitor from openquake.hazardlib import valid from openquake.baselib import parallel from openquake.commonlib.oqvalidation import OqParam from openquake.commonlib import datastore, config, readinput from openquake.calculators import base, views, export from openquake.commonlib import logs TERMINATE = valid.boolean( config.get('distribution', 'terminate_workers_on_revoke') or 'false') USE_CELERY = config.get('distribution', 'oq_distribute') == 'celery' if USE_CELERY: import celery.task.control def set_concurrent_tasks_default(): """ Set the default for concurrent_tasks. Returns the number of live celery nodes (i.e. the number of machines). """ stats = celery.task.control.inspect(timeout=1).stats() if not stats: sys.exit("No live compute nodes, aborting calculation") num_cores = sum(stats[k]['pool']['max-concurrency'] for k in stats)
import os import re import getpass import collections import numpy import h5py from openquake.baselib.python3compat import pickle from openquake.baselib import hdf5 from openquake.commonlib import config from openquake.commonlib.writers import write_csv DATADIR = os.environ.get('OQ_DATADIR') if not DATADIR: shared_dir = config.get('directory', 'shared_dir') if shared_dir: DATADIR = os.path.join(shared_dir, getpass.getuser(), 'oqdata') else: # use the home of the user DATADIR = os.path.join(os.path.expanduser('~'), 'oqdata') def get_calc_ids(datadir=DATADIR): """ Extract the available calculation IDs from the datadir, in order. """ if not os.path.exists(datadir): return [] calc_ids = [] for f in os.listdir(datadir): mo = re.match(r'calc_(\d+)\.hdf5', f)
import os import re import getpass import collections import numpy import h5py from openquake.baselib.python3compat import pickle from openquake.baselib import hdf5 from openquake.commonlib import config from openquake.commonlib.writers import write_csv DATADIR = os.environ.get('OQ_DATADIR') if not DATADIR: shared_dir = config.get('directory', 'shared_dir') if shared_dir: DATADIR = os.path.join(shared_dir, getpass.getuser(), 'oqdata') else: # use the home of the user DATADIR = os.path.join(os.path.expanduser('~'), 'oqdata') def get_nbytes(dset): """ If the dataset has an attribute 'nbytes', return it. Otherwise get the size of the underlying array. Returns None if the dataset is actually a group. """ if 'nbytes' in dset.attrs: # look if the dataset has an attribute nbytes return dset.attrs['nbytes'] elif hasattr(dset, 'value'):
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with OpenQuake. If not, see <http://www.gnu.org/licenses/>. import os import sys import importlib from openquake.baselib import sap from openquake.commonlib import __version__ from openquake import commands from openquake.commonlib import config USE_CELERY = config.get('distribution', 'oq_distribute') == 'celery' # the environment variable has the precedence over the configuration file if 'OQ_DISTRIBUTE' not in os.environ and USE_CELERY: os.environ['OQ_DISTRIBUTE'] = 'celery' # force cluster users to use `oq engine` so that we have centralized logs if USE_CELERY and 'run' in sys.argv: sys.exit('You are on a cluster and you are using oq run?? ' 'Use oq engine --run instead!') def oq(): modnames = [ 'openquake.commands.%s' % mod[:-3] for mod in os.listdir(commands.__path__[0])
import os import sys import signal import traceback from openquake.baselib.performance import Monitor from openquake.risklib import valid from openquake.baselib import parallel from openquake.commonlib.oqvalidation import OqParam from openquake.commonlib import datastore, config, readinput from openquake.calculators import base, views, export from openquake.engine import logs TERMINATE = valid.boolean( config.get('celery', 'terminate_workers_on_revoke') or 'false') USE_CELERY = valid.boolean(config.get('celery', 'use_celery') or 'false') if USE_CELERY: import celery.task.control def set_concurrent_tasks_default(): """ Set the default for concurrent_tasks. Returns the number of live celery nodes (i.e. the number of machines). """ stats = celery.task.control.inspect(timeout=1).stats() if not stats: sys.exit("No live compute nodes, aborting calculation") num_cores = sum(stats[k]['pool']['max-concurrency'] for k in stats)
import io import os import getpass import logging from openquake.hazardlib.calc.hazard_curve import zero_curves from openquake.baselib import sap from openquake.risklib import scientific, valid from openquake.commonlib import datastore from openquake.commonlib.writers import write_csv from openquake.commonlib.util import rmsep from openquake.commonlib import config from openquake.engine import logs from openquake.calculators.views import view MULTI_USER = valid.boolean(config.get('dbserver', 'multi_user') or 'false') if MULTI_USER: # get the datastore of the user who ran the job def read(calc_id): job = logs.dbcmd('get_job', calc_id, getpass.getuser()) datadir = os.path.dirname(job.ds_calc_dir) return datastore.read(job.id, datadir=datadir) else: # get the datastore of the current user read = datastore.read def get_hcurves_and_means(dstore): """ Extract hcurves from the datastore and compute their means. :returns: curves_by_rlz, mean_curves
# # You should have received a copy of the GNU Affero General Public License # along with OpenQuake. If not, see <http://www.gnu.org/licenses/>. import os import sys import importlib from openquake.baselib import sap from openquake.commonlib import __version__ from openquake import commands from openquake.risklib import valid from openquake.commonlib import config USE_CELERY = valid.boolean(config.get('celery', 'use_celery') or 'false') # the environment variable has the precedence over the configuration file if 'OQ_DISTRIBUTE' not in os.environ and USE_CELERY: os.environ['OQ_DISTRIBUTE'] = 'celery' # force cluster users to use `oq engine` so that we have centralized logs if USE_CELERY and 'run' in sys.argv: sys.exit('You are on a cluster and you are using oq run?? ' 'Use oq engine --run instead!') def oq(): modnames = ['openquake.commands.%s' % mod[:-3] for mod in os.listdir(commands.__path__[0]) if mod.endswith('.py') and not mod.startswith('_')] for modname in modnames:
# by the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # OpenQuake is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with OpenQuake. If not, see <http://www.gnu.org/licenses/>. from openquake.baselib.performance import Monitor from openquake.baselib import parallel from openquake.commonlib import config SOFT_MEM_LIMIT = int(config.get('memory', 'soft_mem_limit')) HARD_MEM_LIMIT = int(config.get('memory', 'hard_mem_limit')) parallel.check_mem_usage.__defaults__ = ( Monitor(), SOFT_MEM_LIMIT, HARD_MEM_LIMIT) def confirm(prompt): """ Ask for confirmation, given a ``prompt`` and return a boolean value. """ while True: try: answer = raw_input(prompt) except KeyboardInterrupt: # the user presses ctrl+c, just say 'no'