def run_job(job_ini, log_level='info', log_file=None, exports='', username=getpass.getuser(), **kw): """ Run a job using the specified config file and other options. :param str job_ini: Path to calculation config (INI-style) files. :param str log_level: 'debug', 'info', 'warn', 'error', or 'critical' :param str log_file: Path to log file. :param exports: A comma-separated string of export types requested by the user. :param username: Name of the user running the job :param kw: Extra parameters like hazard_calculation_id and calculation_mode """ job_id = logs.init('job', getattr(logging, log_level.upper())) with logs.handle(job_id, log_level, log_file): job_ini = os.path.abspath(job_ini) oqparam = eng.job_from_file(job_ini, job_id, username, **kw) kw['username'] = username eng.run_calc(job_id, oqparam, exports, **kw) for line in logs.dbcmd('list_outputs', job_id, False): safeprint(line) return job_id
def restore(archive, oqdata): """ Build a new oqdata directory from the data contained in the zip archive """ t0 = time.time() oqdata = os.path.abspath(oqdata) assert archive.endswith('.zip'), archive if os.path.exists(oqdata): sys.exit('%s exists already' % oqdata) os.mkdir(oqdata) zipfile.ZipFile(archive).extractall(oqdata) dbpath = os.path.join(oqdata, 'db.sqlite3') db = Db(sqlite3.connect, dbpath, isolation_level=None, detect_types=sqlite3.PARSE_DECLTYPES) n = 0 for fname in os.listdir(oqdata): mo = re.match('calc_(\d+)\.hdf5', fname) if mo: job_id = int(mo.group(1)) fullname = os.path.join(oqdata, fname)[:-5] # strip .hdf5 db("UPDATE job SET ds_calc_dir=?x WHERE id=?x", fullname, job_id) safeprint('Restoring ' + fname) n += 1 dt = time.time() - t0 safeprint('Extracted %d calculations into %s in %d seconds' % (n, oqdata, dt))
def restore(archive, oqdata): """ Build a new oqdata directory from the data contained in the zip archive """ if os.path.exists(oqdata): sys.exit('%s exists already' % oqdata) if '://' in archive: # get the zip archive from an URL resp = requests.get(archive) _, archive = archive.rsplit('/', 1) with open(archive, 'wb') as f: f.write(resp.content) if not os.path.exists(archive): sys.exit('%s does not exist' % archive) t0 = time.time() oqdata = os.path.abspath(oqdata) assert archive.endswith('.zip'), archive os.mkdir(oqdata) zipfile.ZipFile(archive).extractall(oqdata) dbpath = os.path.join(oqdata, 'db.sqlite3') db = Db(sqlite3.connect, dbpath, isolation_level=None, detect_types=sqlite3.PARSE_DECLTYPES) n = 0 for fname in os.listdir(oqdata): mo = re.match('calc_(\d+)\.hdf5', fname) if mo: job_id = int(mo.group(1)) fullname = os.path.join(oqdata, fname)[:-5] # strip .hdf5 db("UPDATE job SET user_name=?x, ds_calc_dir=?x WHERE id=?x", getpass.getuser(), fullname, job_id) safeprint('Restoring ' + fname) n += 1 dt = time.time() - t0 safeprint('Extracted %d calculations into %s in %d seconds' % (n, oqdata, dt))
def run_job(cfg_file, log_level='info', log_file=None, exports='', hazard_calculation_id=None, **kw): """ Run a job using the specified config file and other options. :param str cfg_file: Path to calculation config (INI-style) files. :param str log_level: 'debug', 'info', 'warn', 'error', or 'critical' :param str log_file: Path to log file. :param exports: A comma-separated string of export types requested by the user. :param hazard_calculation_id: ID of the previous calculation or None """ # if the master dies, automatically kill the workers job_ini = os.path.abspath(cfg_file) job_id, oqparam = eng.job_from_file( job_ini, getpass.getuser(), hazard_calculation_id) calc = eng.run_calc(job_id, oqparam, log_level, log_file, exports, hazard_calculation_id=hazard_calculation_id, **kw) calc._monitor.flush() for line in logs.dbcmd('list_outputs', job_id, False): safeprint(line) return job_id
def run_calc(job_id, oqparam, exports, log_level='info', log_file=None, **kw): """ Run a calculation. :param job_id: ID of the current job :param oqparam: :class:`openquake.commonlib.oqvalidation.OqParam` instance :param exports: A comma-separated string of export types. """ register_signals() setproctitle('oq-job-%d' % job_id) logs.init(job_id, getattr(logging, log_level.upper())) with logs.handle(job_id, log_level, log_file): calc = base.calculators(oqparam, calc_id=job_id) logging.info('%s running %s [--hc=%s]', getpass.getuser(), calc.oqparam.inputs['job_ini'], calc.oqparam.hazard_calculation_id) logging.info('Using engine version %s', __version__) msg = check_obsolete_version(oqparam.calculation_mode) if msg: logging.warning(msg) calc.from_engine = True tb = 'None\n' try: if OQ_DISTRIBUTE.endswith('pool'): logging.warning('Using %d cores on %s', parallel.CT // 2, platform.node()) set_concurrent_tasks_default(calc) t0 = time.time() calc.run(exports=exports, **kw) logging.info('Exposing the outputs to the database') expose_outputs(calc.datastore) path = calc.datastore.filename size = general.humansize(os.path.getsize(path)) logging.info('Stored %s on %s in %d seconds', size, path, time.time() - t0) logs.dbcmd('finish', job_id, 'complete') calc.datastore.close() for line in logs.dbcmd('list_outputs', job_id, False): general.safeprint(line) except BaseException as exc: if isinstance(exc, MasterKilled): msg = 'aborted' else: msg = 'failed' tb = traceback.format_exc() try: logging.critical(tb) logs.dbcmd('finish', job_id, msg) except BaseException: # an OperationalError may always happen sys.stderr.write(tb) raise finally: parallel.Starmap.shutdown() # sanity check to make sure that the logging on file is working if log_file and log_file != os.devnull and os.path.getsize(log_file) == 0: logging.warning('The log file %s is empty!?' % log_file) return calc
def run_job(job_ini, log_level='info', log_file=None, exports='', username=getpass.getuser(), **kw): """ Run a job using the specified config file and other options. :param str job_ini: Path to calculation config (INI-style) files. :param str log_level: 'debug', 'info', 'warn', 'error', or 'critical' :param str log_file: Path to log file. :param exports: A comma-separated string of export types requested by the user. :param username: Name of the user running the job :param kw: Extra parameters like hazard_calculation_id and calculation_mode """ job_id = logs.init('job', getattr(logging, log_level.upper())) with logs.handle(job_id, log_level, log_file): job_ini = os.path.abspath(job_ini) oqparam = eng.job_from_file(job_ini, job_id, username, **kw) kw['username'] = username eng.run_calc(job_id, oqparam, exports, **kw) for line in logs.dbcmd('list_outputs', job_id, False): safeprint(line) return job_id
def del_calculation(job_id, confirmed=False): """ Delete a calculation and all associated outputs. """ if confirmed or confirm( 'Are you sure you want to delete this calculation and all ' 'associated outputs?\nThis action cannot be undone. (y/n): '): try: logs.dbcmd('del_calc', job_id, getpass.getuser()) except RuntimeError as err: safeprint(err)
def smart_save(dbpath, archive): """ Make a copy of the db, remove the incomplete jobs and add the copy to the archive """ tmpdir = tempfile.mkdtemp() newdb = os.path.join(tmpdir, os.path.basename(dbpath)) shutil.copy(dbpath, newdb) try: with sqlite3.connect(newdb) as conn: conn.execute('DELETE FROM job WHERE status != "complete"') except: safeprint('Please check the copy of the db in %s' % newdb) raise zipfiles([newdb], archive, 'a', safeprint) shutil.rmtree(tmpdir)
def del_calculation(job_id, confirmed=False): """ Delete a calculation and all associated outputs. """ if logs.dbcmd('get_job', job_id) is None: print('There is no job %d' % job_id) return if confirmed or confirm( 'Are you sure you want to (abort and) delete this calculation and ' 'all associated outputs?\nThis action cannot be undone. (y/n): '): try: abort(job_id) logs.dbcmd('del_calc', job_id, getpass.getuser()) except RuntimeError as err: safeprint(err) else: print('Removed %d' % job_id)
def smart_save(dbpath, archive, calc_id): """ Make a copy of the db, remove the incomplete jobs and add the copy to the archive """ tmpdir = tempfile.mkdtemp() newdb = os.path.join(tmpdir, os.path.basename(dbpath)) shutil.copy(dbpath, newdb) try: with sqlite3.connect(newdb) as conn: conn.execute('DELETE FROM job WHERE status != "complete"') if calc_id: conn.execute('DELETE FROM job WHERE id != %d' % calc_id) except: safeprint('Please check the copy of the db in %s' % newdb) raise zipfiles([newdb], archive, 'a', safeprint) shutil.rmtree(tmpdir)
def run_calc(log): """ Run a calculation. :param log: LogContext of the current job """ register_signals() setproctitle('oq-job-%d' % log.calc_id) with log: oqparam = log.get_oqparam() calc = base.calculators(oqparam, log.calc_id) logging.info('%s running %s [--hc=%s]', getpass.getuser(), calc.oqparam.inputs['job_ini'], calc.oqparam.hazard_calculation_id) logging.info('Using engine version %s', __version__) msg = check_obsolete_version(oqparam.calculation_mode) # NB: disabling the warning should be done only for users with # an updated LTS version, but we are doing it for all users # if msg: # logging.warning(msg) calc.from_engine = True if config.zworkers['host_cores']: set_concurrent_tasks_default(calc) else: logging.warning('Assuming %d %s workers', parallel.Starmap.num_cores, OQ_DISTRIBUTE) t0 = time.time() calc.run() logging.info('Exposing the outputs to the database') expose_outputs(calc.datastore) path = calc.datastore.filename size = general.humansize(getsize(path)) logging.info('Stored %s on %s in %d seconds', size, path, time.time() - t0) calc.datastore.close() for line in logs.dbcmd('list_outputs', log.calc_id, False): general.safeprint(line) # sanity check to make sure that the logging on file is working if (log.log_file and log.log_file != os.devnull and getsize(log.log_file) == 0): logging.warning('The log file %s is empty!?' % log.log_file) return calc
def del_calculation(job_id, confirmed=False): """ Delete a calculation and all associated outputs. """ if logs.dbcmd('get_job', job_id) is None: print('There is no job %d' % job_id) return if confirmed or confirm( 'Are you sure you want to (abort and) delete this calculation and ' 'all associated outputs?\nThis action cannot be undone. (y/n): '): try: abort(job_id) resp = logs.dbcmd('del_calc', job_id, getpass.getuser()) except RuntimeError as err: safeprint(err) else: if 'success' in resp: print('Removed %d' % job_id) else: print(resp['error'])
def dump(archive, calc_id=0, user=None): """ Dump the openquake database and all the complete calculations into a zip file. In a multiuser installation must be run as administrator. """ t0 = time.time() assert archive.endswith('.zip'), archive getfnames = 'select ds_calc_dir || ".hdf5" from job where ?A' param = dict(status='complete') if calc_id: param['id'] = calc_id if user: param['user_name'] = user fnames = [f for f, in db(getfnames, param) if os.path.exists(f)] zipfiles(fnames, archive, 'w', safeprint) pending_jobs = db('select id, status, description from job ' 'where status="executing"') if pending_jobs: safeprint('WARNING: there were calculations executing during the dump,' ' they have been not copied') for job_id, status, descr in pending_jobs: safeprint('%d %s %s' % (job_id, status, descr)) # this also checks that the copied db is not corrupted smart_save(db.path, archive, calc_id) dt = time.time() - t0 safeprint('Archived %d calculations into %s in %d seconds' % (len(fnames), archive, dt))
def dump(archive, calc_id=0, user=None): """ Dump the openquake database and all the complete calculations into a zip file. In a multiuser installation must be run as administrator. """ t0 = time.time() assert archive.endswith('.zip'), archive getfnames = 'select ds_calc_dir || ".hdf5" from job where ?A' param = dict(status='complete') if calc_id: param['id'] = calc_id if user: param['user_name'] = user fnames = [f for f, in db(getfnames, param) if os.path.exists(f)] zipfiles(fnames, archive, 'w', safeprint) pending_jobs = db('select id, status, description from job ' 'where status="executing"') if pending_jobs: safeprint('WARNING: there were calculations executing during the dump,' ' they have been not copied') for job_id, status, descr in pending_jobs: safeprint('%d %s %s' % (job_id, status, descr)) # this also checks that the copied db is not corrupted smart_save(db.path, archive, calc_id) dt = time.time() - t0 safeprint('Archived %d calculations into %s in %d seconds' % (len(fnames), archive, dt))
def main(no_distribute=False, yes=False, upgrade_db=False, db_version=False, what_if_I_upgrade=False, list_hazard_calculations=False, list_risk_calculations=False, delete_uncompleted_calculations=False, multi=False, reuse_input=False, *, log_file=None, make_html_report=None, run=None, delete_calculation: int = None, hazard_calculation_id: int = None, list_outputs: int = None, show_log=None, export_output=None, export_outputs=None, param='', config_file=None, exports='', log_level='info'): """ Run a calculation using the traditional command line API """ if not run: # configure a basic logging logs.init() if config_file: config.read(os.path.abspath(os.path.expanduser(config_file)), soft_mem_limit=int, hard_mem_limit=int, port=int, multi_user=valid.boolean, serialize_jobs=valid.boolean, strict=valid.boolean, code=exec) if no_distribute: os.environ['OQ_DISTRIBUTE'] = 'no' # check if the datadir exists datadir = datastore.get_datadir() if not os.path.exists(datadir): os.makedirs(datadir) dbserver.ensure_on() # check if we are talking to the right server err = dbserver.check_foreign() if err: sys.exit(err) if upgrade_db: msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts') if msg.startswith('Your database is already updated'): pass elif yes or confirm('Proceed? (y/n) '): logs.dbcmd('upgrade_db') sys.exit(0) if db_version: safeprint(logs.dbcmd('db_version')) sys.exit(0) if what_if_I_upgrade: safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts')) sys.exit(0) # check if the db is outdated outdated = logs.dbcmd('check_outdated') if outdated: sys.exit(outdated) # hazard or hazard+risk if hazard_calculation_id == -1: # get the latest calculation of the current user hc_id = get_job_id(hazard_calculation_id, getpass.getuser()) elif hazard_calculation_id: # make it possible to use calculations made by another user hc_id = get_job_id(hazard_calculation_id) else: hc_id = None if run: pars = dict(p.split('=', 1) for p in param.split(',')) if param else {} if reuse_input: pars['cachedir'] = datadir if hc_id: pars['hazard_calculation_id'] = str(hc_id) log_file = os.path.expanduser(log_file) \ if log_file is not None else None job_inis = [os.path.expanduser(f) for f in run] pars['multi'] = multi run_jobs(job_inis, log_level, log_file, exports, **pars) # hazard elif list_hazard_calculations: for line in logs.dbcmd('list_calculations', 'hazard', getpass.getuser()): safeprint(line) elif delete_calculation is not None: del_calculation(delete_calculation, yes) # risk elif list_risk_calculations: for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()): safeprint(line) # export elif make_html_report: safeprint('Written %s' % make_report(make_html_report)) sys.exit(0) elif list_outputs is not None: hc_id = get_job_id(list_outputs) for line in logs.dbcmd('list_outputs', hc_id): safeprint(line) elif show_log is not None: hc_id = get_job_id(show_log) for line in logs.dbcmd('get_log', hc_id): safeprint(line) elif export_output is not None: output_id, target_dir = export_output dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id)) for line in core.export_output(dskey, calc_id, datadir, os.path.expanduser(target_dir), exports or DEFAULT_EXPORTS): safeprint(line) elif export_outputs is not None: job_id, target_dir = export_outputs hc_id = get_job_id(job_id) for line in core.export_outputs(hc_id, os.path.expanduser(target_dir), exports or DEFAULT_EXPORTS): safeprint(line) elif delete_uncompleted_calculations: logs.dbcmd('delete_uncompleted_calculations', getpass.getuser()) else: print("Please pass some option, see oq engine --help")
def engine(log_file, no_distribute, yes, config_file, make_html_report, upgrade_db, db_version, what_if_I_upgrade, run_hazard, run_risk, run, list_hazard_calculations, list_risk_calculations, delete_calculation, delete_uncompleted_calculations, hazard_calculation_id, list_outputs, show_log, export_output, export_outputs, exports='', log_level='info'): """ Run a calculation using the traditional command line API """ if run or run_hazard or run_risk: # the logging will be configured in engine.py pass else: # configure a basic logging logging.basicConfig(level=logging.INFO) if config_file: config.load(os.path.abspath(os.path.expanduser(config_file))) if no_distribute: os.environ['OQ_DISTRIBUTE'] = 'no' # check if the datadir exists datadir = datastore.get_datadir() if not os.path.exists(datadir): os.makedirs(datadir) dbserver.ensure_on() # check if we are talking to the right server err = dbserver.check_foreign() if err: sys.exit(err) if upgrade_db: logs.set_level('info') msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts') if msg.startswith('Your database is already updated'): pass elif yes or confirm('Proceed? (y/n) '): logs.dbcmd('upgrade_db') sys.exit(0) if db_version: safeprint(logs.dbcmd('db_version')) sys.exit(0) if what_if_I_upgrade: safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts')) sys.exit(0) # check if the db is outdated outdated = logs.dbcmd('check_outdated') if outdated: sys.exit(outdated) # hazard or hazard+risk if hazard_calculation_id: hc_id = get_job_id(hazard_calculation_id) else: hc_id = None if run: job_ini = os.path.expanduser(run) open(job_ini, 'rb').read() # IOError if the file does not exist log_file = os.path.expanduser(log_file) \ if log_file is not None else None run_job(os.path.expanduser(run), log_level, log_file, exports, hazard_calculation_id=hc_id) # hazard elif list_hazard_calculations: for line in logs.dbcmd( 'list_calculations', 'hazard', getpass.getuser()): safeprint(line) elif run_hazard is not None: safeprint('WARN: --rh/--run-hazard are deprecated, use --run instead', file=sys.stderr) log_file = os.path.expanduser(log_file) \ if log_file is not None else None run_job(os.path.expanduser(run_hazard), log_level, log_file, exports) elif delete_calculation is not None: del_calculation(delete_calculation, yes) # risk elif list_risk_calculations: for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()): safeprint(line) elif run_risk is not None: safeprint('WARN: --rr/--run-risk are deprecated, use --run instead', file=sys.stderr) if hazard_calculation_id is None: sys.exit(MISSING_HAZARD_MSG) log_file = os.path.expanduser(log_file) \ if log_file is not None else None run_job( os.path.expanduser(run_risk), log_level, log_file, exports, hazard_calculation_id=hc_id) # export elif make_html_report: safeprint('Written %s' % make_report(make_html_report)) sys.exit(0) elif list_outputs is not None: hc_id = get_job_id(list_outputs) for line in logs.dbcmd('list_outputs', hc_id): safeprint(line) elif show_log is not None: hc_id = get_job_id(show_log) for line in logs.dbcmd('get_log', hc_id): safeprint(line) elif export_output is not None: output_id, target_dir = export_output dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id)) for line in core.export_output( dskey, calc_id, datadir, os.path.expanduser(target_dir), exports or 'csv,xml'): safeprint(line) elif export_outputs is not None: job_id, target_dir = export_outputs hc_id = get_job_id(job_id) for line in core.export_outputs( hc_id, os.path.expanduser(target_dir), exports or 'csv,xml'): safeprint(line) elif delete_uncompleted_calculations: logs.dbcmd('delete_uncompleted_calculations', getpass.getuser()) else: engine.parentparser.prog = 'oq engine' engine.parentparser.print_usage()
def engine(log_file, no_distribute, yes, config_file, make_html_report, upgrade_db, db_version, what_if_I_upgrade, run, list_hazard_calculations, list_risk_calculations, delete_calculation, delete_uncompleted_calculations, hazard_calculation_id, list_outputs, show_log, export_output, export_outputs, exports='', log_level='info', reuse_hazard=False): """ Run a calculation using the traditional command line API """ if not run: # configure a basic logging logs.init() if config_file: config.read(os.path.abspath(os.path.expanduser(config_file)), soft_mem_limit=int, hard_mem_limit=int, port=int, multi_user=valid.boolean, multi_node=valid.boolean) if no_distribute: os.environ['OQ_DISTRIBUTE'] = 'no' # check if the datadir exists datadir = datastore.get_datadir() if not os.path.exists(datadir): os.makedirs(datadir) dbserver.ensure_on() # check if we are talking to the right server err = dbserver.check_foreign() if err: sys.exit(err) if upgrade_db: msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts') if msg.startswith('Your database is already updated'): pass elif yes or confirm('Proceed? (y/n) '): logs.dbcmd('upgrade_db') sys.exit(0) if db_version: safeprint(logs.dbcmd('db_version')) sys.exit(0) if what_if_I_upgrade: safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts')) sys.exit(0) # check if the db is outdated outdated = logs.dbcmd('check_outdated') if outdated: sys.exit(outdated) # hazard or hazard+risk if hazard_calculation_id == -1: # get the latest calculation of the current user hc_id = get_job_id(hazard_calculation_id, getpass.getuser()) elif hazard_calculation_id: # make it possible to use calculations made by another user hc_id = get_job_id(hazard_calculation_id) else: hc_id = None if run: log_file = os.path.expanduser(log_file) \ if log_file is not None else None job_inis = [os.path.expanduser(f) for f in run] if len(job_inis) == 1 and not hc_id: # init logs before calling get_oqparam logs.init('nojob', getattr(logging, log_level.upper())) # not using logs.handle that logs on the db oq = readinput.get_oqparam(job_inis[0]) smart_run(job_inis[0], oq, log_level, log_file, exports, reuse_hazard) return for i, job_ini in enumerate(job_inis): open(job_ini, 'rb').read() # IOError if the file does not exist job_id = run_job(job_ini, log_level, log_file, exports, hazard_calculation_id=hc_id) if not hc_id: # use the first calculation as base for the others hc_id = job_id # hazard elif list_hazard_calculations: for line in logs.dbcmd('list_calculations', 'hazard', getpass.getuser()): safeprint(line) elif delete_calculation is not None: del_calculation(delete_calculation, yes) # risk elif list_risk_calculations: for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()): safeprint(line) # export elif make_html_report: safeprint('Written %s' % make_report(make_html_report)) sys.exit(0) elif list_outputs is not None: hc_id = get_job_id(list_outputs) for line in logs.dbcmd('list_outputs', hc_id): safeprint(line) elif show_log is not None: hc_id = get_job_id(show_log) for line in logs.dbcmd('get_log', hc_id): safeprint(line) elif export_output is not None: output_id, target_dir = export_output dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id)) for line in core.export_output(dskey, calc_id, datadir, os.path.expanduser(target_dir), exports or 'csv,xml'): safeprint(line) elif export_outputs is not None: job_id, target_dir = export_outputs hc_id = get_job_id(job_id) for line in core.export_outputs(hc_id, os.path.expanduser(target_dir), exports or 'csv,xml'): safeprint(line) elif delete_uncompleted_calculations: logs.dbcmd('delete_uncompleted_calculations', getpass.getuser()) else: engine.parentparser.prog = 'oq engine' engine.parentparser.print_usage()
def engine(log_file, no_distribute, yes, config_file, make_html_report, upgrade_db, db_version, what_if_I_upgrade, run, list_hazard_calculations, list_risk_calculations, delete_calculation, delete_uncompleted_calculations, hazard_calculation_id, list_outputs, show_log, export_output, export_outputs, exports='', log_level='info', reuse_hazard=False): """ Run a calculation using the traditional command line API """ if not run: # configure a basic logging logs.init() if config_file: config.read(os.path.abspath(os.path.expanduser(config_file)), soft_mem_limit=int, hard_mem_limit=int, port=int, multi_user=valid.boolean, multi_node=valid.boolean) if no_distribute: os.environ['OQ_DISTRIBUTE'] = 'no' # check if the datadir exists datadir = datastore.get_datadir() if not os.path.exists(datadir): os.makedirs(datadir) dbserver.ensure_on() # check if we are talking to the right server err = dbserver.check_foreign() if err: sys.exit(err) if upgrade_db: msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts') if msg.startswith('Your database is already updated'): pass elif yes or confirm('Proceed? (y/n) '): logs.dbcmd('upgrade_db') sys.exit(0) if db_version: safeprint(logs.dbcmd('db_version')) sys.exit(0) if what_if_I_upgrade: safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts')) sys.exit(0) # check if the db is outdated outdated = logs.dbcmd('check_outdated') if outdated: sys.exit(outdated) # hazard or hazard+risk if hazard_calculation_id == -1: # get the latest calculation of the current user hc_id = get_job_id(hazard_calculation_id, getpass.getuser()) elif hazard_calculation_id: # make it possible to use calculations made by another user hc_id = get_job_id(hazard_calculation_id) else: hc_id = None if run: log_file = os.path.expanduser(log_file) \ if log_file is not None else None job_inis = [os.path.expanduser(f) for f in run] if len(job_inis) == 1 and not hc_id: # init logs before calling get_oqparam logs.init('nojob', getattr(logging, log_level.upper())) # not using logs.handle that logs on the db oq = readinput.get_oqparam(job_inis[0]) smart_run(job_inis[0], oq, log_level, log_file, exports, reuse_hazard) return for i, job_ini in enumerate(job_inis): open(job_ini, 'rb').read() # IOError if the file does not exist job_id = run_job(job_ini, log_level, log_file, exports, hazard_calculation_id=hc_id) if not hc_id: # use the first calculation as base for the others hc_id = job_id # hazard elif list_hazard_calculations: for line in logs.dbcmd( 'list_calculations', 'hazard', getpass.getuser()): safeprint(line) elif delete_calculation is not None: del_calculation(delete_calculation, yes) # risk elif list_risk_calculations: for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()): safeprint(line) # export elif make_html_report: safeprint('Written %s' % make_report(make_html_report)) sys.exit(0) elif list_outputs is not None: hc_id = get_job_id(list_outputs) for line in logs.dbcmd('list_outputs', hc_id): safeprint(line) elif show_log is not None: hc_id = get_job_id(show_log) for line in logs.dbcmd('get_log', hc_id): safeprint(line) elif export_output is not None: output_id, target_dir = export_output dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id)) for line in core.export_output( dskey, calc_id, datadir, os.path.expanduser(target_dir), exports or 'csv,xml'): safeprint(line) elif export_outputs is not None: job_id, target_dir = export_outputs hc_id = get_job_id(job_id) for line in core.export_outputs( hc_id, os.path.expanduser(target_dir), exports or 'csv,xml'): safeprint(line) elif delete_uncompleted_calculations: logs.dbcmd('delete_uncompleted_calculations', getpass.getuser()) else: engine.parentparser.prog = 'oq engine' engine.parentparser.print_usage()