def reset(yes): """ Remove all the datastores and the database of the current user """ ok = yes or confirm('Do you really want to destroy all your data? (y/n) ') if not ok: return dbpath = os.path.realpath(os.path.expanduser(config.dbserver.file)) # user must be able to access and write the databse file to remove it if os.path.isfile(dbpath) and os.access(dbpath, os.W_OK): if dbserver.get_status() == 'running': if config.dbserver.multi_user: sys.exit('The oq dbserver must be stopped ' 'before proceeding') else: pid = logs.dbcmd('getpid') os.kill(pid, signal.SIGTERM) time.sleep(.5) # give time to stop assert dbserver.get_status() == 'not-running' print('dbserver stopped') try: os.remove(dbpath) print('Removed %s' % dbpath) except OSError as exc: print(exc, file=sys.stderr) # fast way of removing everything purge_all(fast=True) # datastore of the current user
def reset(yes): """ Remove all the datastores and the database of the current user """ ok = yes or confirm('Do you really want to destroy all your data? (y/n) ') if not ok: return status = dbserver.get_status() dbpath = os.path.realpath(os.path.expanduser(config.dbserver.file)) if not os.path.isfile(dbpath): sys.exit('%s does not exist' % dbpath) elif status == 'running': user = getpass.getuser() for calc_id in logs.dbcmd('get_calc_ids', user): purge_one(calc_id, user, force=True) purge_all(user) # calculations in oqdata not in the db if os.access(dbpath, os.W_OK): # single user mode # stop the dbserver first pid = logs.dbcmd('getpid') os.kill(pid, signal.SIGTERM) time.sleep(.5) # give time to stop assert dbserver.get_status() == 'not-running' print('dbserver stopped') # remove the database os.remove(dbpath) print('Removed %s' % dbpath) else: sys.exit('The dbserver is not running')
def reset(yes): """ Remove all the datastores and the database of the current user """ ok = yes or confirm('Do you really want to destroy all your data? (y/n) ') if not ok: return dbpath = os.path.realpath(os.path.expanduser(config.dbserver.file)) # user must be able to access and write the databse file to remove it if os.path.isfile(dbpath) and os.access(dbpath, os.W_OK): if dbserver.get_status() == 'running': if config.dbserver.multi_user: sys.exit('The oq dbserver must be stopped ' 'before proceeding') else: pid = logs.dbcmd('getpid') os.kill(pid, signal.SIGTERM) time.sleep(.5) # give time to stop assert dbserver.get_status() == 'not-running' print('dbserver stopped') try: os.remove(dbpath) print('Removed %s' % dbpath) except OSError as exc: print(exc, file=sys.stderr) # fast way of removing everything purge_all(fast=True) # datastore of the current user
def dbserver(cmd, dbhostport=None, dbpath=os.path.expanduser(config.dbserver.file)): """ start/stop/restart the database server, or return its status """ if config.dbserver.multi_user and getpass.getuser() != 'openquake': sys.exit('oq dbserver only works in single user mode') status = dbs.get_status() if cmd == 'status': print('dbserver ' + status) elif cmd == 'stop': if status == 'running': pid = logs.dbcmd('getpid') os.kill(pid, signal.SIGINT) # this is trapped by the DbServer else: print('dbserver already stopped') elif cmd == 'start': if status == 'not-running': dbs.run_server.func(dbpath, dbhostport) else: print('dbserver already running') elif cmd == 'restart': if status == 'running': pid = logs.dbcmd('getpid') os.kill(pid, signal.SIGINT) dbs.run_server.func(dbpath, dbhostport)
def dbserver(cmd, dbhostport=None, dbpath=None): """ start/stop/restart the database server, or return its status """ if valid.boolean(config.get('dbserver', 'multi_user')): sys.exit('oq dbserver only works in single user mode') status = dbs.get_status() if cmd == 'status': print('dbserver ' + status) elif cmd == 'stop': if status == 'running': logs.dbcmd('stop') print('dbserver stopped') else: print('dbserver already stopped') elif cmd == 'start': if status == 'not-running': dbs.run_server(dbhostport, dbpath) else: print('dbserver already running') elif cmd == 'restart': if status == 'running': logs.dbcmd('stop') print('dbserver stopped') dbs.run_server(dbhostport, dbpath)
def main(cmd, dbhostport=None, foreground=False, *, loglevel='INFO'): """ start/stop/restart the database server, or return its status """ if config.dbserver.multi_user and getpass.getuser() != 'openquake': sys.exit('oq dbserver only works in single user mode') status = dbs.get_status() if cmd == 'status': print('dbserver ' + status) elif cmd == 'stop': if status == 'running': pid = logs.dbcmd('getpid') os.kill(pid, signal.SIGINT) # this is trapped by the DbServer else: print('dbserver already stopped') elif cmd == 'start': if status == 'not-running': dbs.run_server(dbhostport, loglevel, foreground) else: print('dbserver already running')
def extract(what, calc_id=-1, hostport=None): """ Extract an output from the datastore and save it into an .hdf5 file. """ logging.basicConfig(level=logging.INFO) if calc_id < 0: calc_id = datastore.get_calc_ids()[calc_id] hdf5path = None if dbserver.get_status() == 'running': job = dbcmd('get_job', calc_id) if job is not None: hdf5path = job.ds_calc_dir + '.hdf5' dstore = datastore.read(hdf5path or calc_id) parent_id = dstore['oqparam'].hazard_calculation_id if parent_id: dstore.parent = datastore.read(parent_id) urlpath = '/v1/calc/%d/extract/%s' % (calc_id, quote(what)) with performance.Monitor('extract', measuremem=True) as mon, dstore: if hostport: data = urlopen('http://%s%s' % (hostport, urlpath)).read() items = (item for item in numpy.load(io.BytesIO(data)).items()) else: print('Emulating call to %s' % urlpath) items = extract_(dstore, what) if not inspect.isgenerator(items): items = [(items.__class__.__name__, items)] fname = '%s_%d.hdf5' % (what.replace('/', '-').replace( '?', '-'), dstore.calc_id) hdf5.save(fname, items) print('Saved', fname) if mon.duration > 1: print(mon)
def reset(yes): """ Remove all the datastores and the database of the current user """ ok = yes or confirm('Do you really want to destroy all your data? (y/n) ') if not ok: return dbpath = os.path.realpath( os.path.expanduser(config.get('dbserver', 'file'))) # user must be able to access and write the databse file to remove it if os.path.isfile(dbpath) and os.access(dbpath, os.W_OK): if dbserver.get_status() == 'running': if config.flag_set('dbserver', 'multi_user'): sys.exit('The oq dbserver must be stopped ' 'before proceeding') else: logs.dbcmd('stop') print('dbserver stopped') try: os.remove(dbpath) print('Removed %s' % dbpath) except OSError as exc: print(exc, file=sys.stderr) # fast way of removing everything purge_all(fast=True) # datastore of the current user
def dbserver(cmd): """ start/stop/restart the database server, or return its status """ if valid.boolean(config.get('dbserver', 'multi_user')): sys.exit('oq dbserver only works in single user mode') status = get_status() if cmd == 'status': print(status) elif cmd == 'stop': if status == 'running': logs.dbcmd('stop') print('stopped') else: print('already stopped') elif cmd == 'start': if status == 'not-running': runserver() print('started') else: print('already running') elif cmd == 'restart': if status == 'running': logs.dbcmd('stop') print('stopped') runserver() print('started')
def dbserver(cmd, dbhostport=None, dbpath=os.path.expanduser(config.dbserver.file), foreground=False): """ start/stop/restart the database server, or return its status """ if config.dbserver.multi_user and getpass.getuser() != 'openquake': sys.exit('oq dbserver only works in single user mode') status = dbs.get_status() if cmd == 'status': print('dbserver ' + status) elif cmd == 'stop': if status == 'running': pid = logs.dbcmd('getpid') os.kill(pid, signal.SIGINT) # this is trapped by the DbServer else: print('dbserver already stopped') elif cmd == 'start': if status == 'not-running': dbs.run_server(dbpath, dbhostport) else: print('dbserver already running') elif cmd == 'restart': if status == 'running': pid = logs.dbcmd('getpid') os.kill(pid, signal.SIGINT) dbs.run_server(dbpath, dbhostport)
def extract(what, calc_id=-1): """ Extract an output from the datastore and save it into an .hdf5 file. """ logging.basicConfig(level=logging.INFO) if calc_id < 0: calc_id = datastore.get_calc_ids()[calc_id] if dbserver.get_status() == 'running': job = dbcmd('get_job', calc_id) if job is not None: calc_id = job.ds_calc_dir + '.hdf5' dstore = datastore.read(calc_id) parent_id = dstore['oqparam'].hazard_calculation_id if parent_id: dstore.parent = datastore.read(parent_id) print('Emulating call to /v1/calc/%s/extract/%s' % (calc_id, quote(what))) with performance.Monitor('extract', measuremem=True) as mon, dstore: items = extract_(dstore, what) if not inspect.isgenerator(items): items = [(items.__class__.__name__, items)] fname = '%s_%d.hdf5' % (what.replace('/', '-').replace( '?', '-'), dstore.calc_id) hdf5.save(fname, items) print('Saved', fname) if mon.duration > 1: print(mon)
def setUpClass(cls): assert get_status() == 'running' cls.job_ids = [] env = os.environ.copy() env['OQ_DISTRIBUTE'] = 'no' # let's impersonate the user openquake, the one running the WebUI: # we need to set LOGNAME on Linux and USERNAME on Windows env['LOGNAME'] = env['USERNAME'] = '******' cls.c = Client()
def setUpClass(cls): assert get_status() == 'running' cls.job_ids = [] env = os.environ.copy() env['OQ_DISTRIBUTE'] = 'no' # let's impersonate the user openquake, the one running the WebUI: # we need to set LOGNAME on Linux and USERNAME on Windows env['LOGNAME'] = env['USERNAME'] = '******' cls.c = Client()
def webui(cmd, hostport='127.0.0.1:8800'): """ start the webui server in foreground or perform other operation on the django application """ dbstatus = get_status() if dbstatus == 'not-running': if valid.boolean(config.get('dbserver', 'multi_user')): sys.exit('Please start the DbServer: ' 'see the documentation for details') dbserver.runserver() if cmd == 'start': rundjango('runserver', hostport) elif cmd == 'syncdb': rundjango('syncdb')
def extract(what, calc_id=-1, server_url='http://127.0.0.1:8800'): """ Extract an output from the datastore and save it into an .hdf5 file. By default use the WebUI, unless server_url is set to 'local', in which case the extraction is done directly bypassing tjhe WebUI and the database. """ logging.basicConfig(level=logging.INFO) if calc_id < 0: calc_id = datastore.get_calc_ids()[calc_id] filename = None if dbserver.get_status() == 'running': job = dbcmd('get_job', calc_id) if job is not None: filename = job.ds_calc_dir + '.hdf5' dstore = util.read(filename or calc_id) parent_id = dstore['oqparam'].hazard_calculation_id if parent_id: dstore.parent = util.read(parent_id) urlpath = '/v1/calc/%d/extract/%s' % (calc_id, quote(what)) with performance.Monitor('extract', measuremem=True) as mon, dstore: if server_url == 'local': print('Emulating call to %s' % urlpath) items = extract_(dstore, what) else: print('Calling %s%s' % (server_url, urlpath)) data = urlopen(server_url.rstrip('/') + urlpath).read() items = (item for item in numpy.load(io.BytesIO(data)).items()) if not inspect.isgenerator(items): items = [(items.__class__.__name__, items)] fname = '%s_%d.hdf5' % (what.replace('/', '-').replace( '?', '-'), dstore.calc_id) hdf5.save(fname, items) print('Saved', fname) if mon.duration > 1: print(mon)
def engine(log_file, no_distribute, yes, config_file, make_html_report, upgrade_db, version_db, what_if_I_upgrade, run_hazard, run_risk, run, list_hazard_calculations, list_risk_calculations, delete_calculation, delete_uncompleted_calculations, hazard_calculation_id, list_outputs, show_log, export_output, export_outputs, exports='', log_level='info'): """ Run a calculation using the traditional command line API """ config.abort_if_no_config_available() if run or run_hazard or run_risk: # the logging will be configured in engine.py pass else: # configure a basic logging logging.basicConfig(level=logging.INFO) if config_file: os.environ[config.OQ_CONFIG_FILE_VAR] = os.path.abspath( os.path.expanduser(config_file)) config.refresh() if no_distribute: os.environ['OQ_DISTRIBUTE'] = 'no' # check if the datadir exists if not os.path.exists(datastore.DATADIR): os.makedirs(datastore.DATADIR) # check if the DbServer is up if dbserver.get_status() == 'not-running': if valid.boolean(config.get('dbserver', 'multi_user')): sys.exit('Please start the DbServer: ' 'see the documentation for details') # otherwise start the DbServer automatically subprocess.Popen([sys.executable, '-m', 'openquake.server.dbserver', '-l', 'INFO']) if upgrade_db: logs.set_level('info') msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts') if msg.startswith('Your database is already updated'): pass elif yes or confirm('Proceed? (y/n) '): logs.dbcmd('upgrade_db') sys.exit(0) if version_db: print(logs.dbcmd('version_db')) sys.exit(0) if what_if_I_upgrade: print(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts')) sys.exit(0) # check if the db is outdated outdated = logs.dbcmd('check_outdated') if outdated: sys.exit(outdated) # hazard or hazard+risk if hazard_calculation_id: hc_id = get_job_id(hazard_calculation_id) else: hc_id = None if run: job_inis = [os.path.expanduser(ini) for ini in run.split(',')] if len(job_inis) not in (1, 2): sys.exit('%s should be a .ini filename or a pair of filenames ' 'separated by a comma' % run) for job_ini in job_inis: open(job_ini).read() # raise an IOError if the file does not exist log_file = os.path.expanduser(log_file) \ if log_file is not None else None if len(job_inis) == 2: # run hazard job_id = run_job(job_inis[0], log_level, log_file, exports) # run risk run_job(job_inis[1], log_level, log_file, exports, hazard_calculation_id=job_id) else: run_job( os.path.expanduser(run), log_level, log_file, exports, hazard_calculation_id=hc_id) # hazard elif list_hazard_calculations: for line in logs.dbcmd( 'list_calculations', 'hazard', getpass.getuser()): print(line) elif run_hazard is not None: print('WARN: --rh/--run-hazard are deprecated, use --run instead', file=sys.stderr) log_file = os.path.expanduser(log_file) \ if log_file is not None else None run_job(os.path.expanduser(run_hazard), log_level, log_file, exports) elif delete_calculation is not None: delete_calculation(delete_calculation, yes) # risk elif list_risk_calculations: for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()): print(line) elif run_risk is not None: print('WARN: --rr/--run-risk are deprecated, use --run instead', file=sys.stderr) if hazard_calculation_id is None: sys.exit(MISSING_HAZARD_MSG) log_file = os.path.expanduser(log_file) \ if log_file is not None else None run_job( os.path.expanduser(run_risk), log_level, log_file, exports, hazard_calculation_id=hc_id) # export elif make_html_report: print('Written %s' % make_report(make_html_report)) sys.exit(0) elif list_outputs is not None: hc_id = get_job_id(list_outputs) for line in logs.dbcmd('list_outputs', hc_id): print(line) elif show_log is not None: hc_id = get_job_id(show_log) for line in logs.dbcmd('get_log', hc_id): print(line) elif export_output is not None: output_id, target_dir = export_output dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id)) for line in core.export_output( dskey, calc_id, datadir, os.path.expanduser(target_dir), exports or 'xml,csv'): print(line) elif export_outputs is not None: job_id, target_dir = export_outputs hc_id = get_job_id(job_id) for line in core.export_outputs( hc_id, os.path.expanduser(target_dir), exports or 'xml,csv'): print(line) elif delete_uncompleted_calculations: logs.dbcmd('delete_uncompleted_calculations', getpass.getuser()) else: parser.parentparser.prog = 'oq engine' parser.parentparser.print_usage()
def __init__(self, job_ini, event_info, no_distribute=False): """ Initialize a calculation (reinvented from openquake.engine.engine) :param job_ini: Path to configuration file/archive or dictionary of parameters with at least a key "calculation_mode" """ user_name = getpass.getuser() if no_distribute: os.environ['OQ_DISTRIBUTE'] = 'no' # check if the datadir exists datadir = datastore.get_datadir() if not os.path.exists(datadir): os.makedirs(datadir) #dbserver.ensure_on() if dbserver.get_status() == 'not-running': if config.dbserver.multi_user: sys.exit('Please start the DbServer: ' 'see the documentation for details') # otherwise start the DbServer automatically; NB: I tried to use # multiprocessing.Process(target=run_server).start() and apparently # it works, but then run-demos.sh hangs after the end of the first # calculation, but only if the DbServer is started by oq engine (!?) # Here is a trick to activate OpenQuake's dbserver # We first cd to the openquake directory and invoke subprocess to open/hold on dbserver # Then, we cd back to the original working directory owd = os.getcwd() os.chdir(os.path.dirname(os.path.realpath(__file__))) subprocess.Popen([ sys.executable, '-m', 'openquake.commands', 'dbserver', 'start' ]) os.chdir(owd) # wait for the dbserver to start waiting_seconds = 30 while dbserver.get_status() == 'not-running': if waiting_seconds == 0: sys.exit( 'The DbServer cannot be started after 30 seconds. ' 'Please check the configuration') time.sleep(1) waiting_seconds -= 1 # check if we are talking to the right server err = dbserver.check_foreign() if err: sys.exit(err) # Copy the event_info self.event_info = event_info # Create a job #self.job = logs.init("job", job_ini, logging.INFO, None, None, None) dic = readinput.get_params(job_ini) #dic['hazard_calculation_id'] = self.job.calc_id # Create the job log self.log = logs.init('job', dic, logging.INFO, None, None, None) # Get openquake parameters self.oqparam = self.log.get_oqparam() # Create the calculator self.calculator = base.calculators(self.oqparam, self.log.calc_id) self.calculator.from_engine = True print('FetchOpenQuake: OpenQuake Hazard Calculator initiated.')