def test_sensitivity(self): job_ini = gettemp('''[general] description = sensitivity test calculation_mode = scenario sites = 0 0 sensitivity_analysis = { 'maximum_distance': [100, 200]}''') run_jobs([job_ini])
def test_multi_run(self): job_ini = os.path.join(os.path.dirname(case_4.__file__), 'job.ini') jobs = create_jobs([job_ini, job_ini], 'error', multi=True) run_jobs(jobs) with Print.patch(): [r1, r2] = commonlib.logs.dbcmd( 'select id, hazard_calculation_id from job ' 'where id in (?S) order by id', [job.calc_id for job in jobs]) self.assertEqual(r1.hazard_calculation_id, None) self.assertEqual(r2.hazard_calculation_id, None)
def test_multi_run(self): job_ini = os.path.join(os.path.dirname(case_4.__file__), 'job.ini') jobparams = run_jobs([job_ini, job_ini], log_level='error', multi=True) jobs, params = zip(*jobparams) with Print.patch(): [r1, r2] = commonlib.logs.dbcmd( 'select id, hazard_calculation_id from job ' 'where id in (?S) order by id', jobs) self.assertEqual(r1.hazard_calculation_id, r1.id) self.assertEqual(r2.hazard_calculation_id, r1.id)
def main(job_ini, pdb=False, reuse_input=False, *, slowest: int = None, hc: int = None, param='', concurrent_tasks: int = None, exports: valid.export_formats = '', loglevel='info'): """ Run a calculation """ dbserver.ensure_on() if param: params = dict(p.split('=', 1) for p in param.split(',')) else: params = {} if hc: params['hazard_calculation_id'] = str(hc) if slowest: prof = cProfile.Profile() prof.runctx( '_run(job_ini[0], 0, pdb, reuse_input, loglevel, ' 'exports, params)', globals(), locals()) pstat = calc_path + '.pstat' prof.dump_stats(pstat) print('Saved profiling info in %s' % pstat) print(get_pstats(pstat, slowest)) return if len(job_ini) == 1: return _run(job_ini[0], concurrent_tasks, pdb, reuse_input, loglevel, exports, params) jobs = create_jobs(job_ini, loglevel, hc_id=hc) for job in jobs: job.params.update(params) job.params['exports'] = ','.join(exports) run_jobs(jobs)
def test_oqdata(self): # the that the environment variable OQ_DATADIR is honored job_ini = os.path.join(os.path.dirname(case_2.__file__), 'job_2.ini') tempdir = tempfile.mkdtemp() dbserver.ensure_on() with mock.patch.dict(os.environ, OQ_DATADIR=tempdir): [(job_id, oq)] = run_jobs([job_ini], log_level='error') job = commonlib.logs.dbcmd('get_job', job_id) self.assertTrue(job.ds_calc_dir.startswith(tempdir), job.ds_calc_dir) with Print.patch() as p: export('ruptures', job_id, 'csv', tempdir) self.assertIn('Exported', str(p)) shutil.rmtree(tempdir)
def test_ebr(self): # test a single case of `run_jobs`, but it is the most complex one, # event based risk with post processing job_ini = os.path.join( os.path.dirname(case_master.__file__), 'job.ini') with Print.patch() as p: [(job_id, oqparam)] = run_jobs([job_ini], log_level='error') self.assertIn('id | name', str(p)) # sanity check on the performance views: make sure that the most # relevant information is stored (it can be lost due to a wrong # refactoring of the monitoring and it happened several times) with read(job_id) as dstore: perf = view('performance', dstore) self.assertIn('total event_based_risk', perf)
def test_oqdata(self): # the that the environment variable OQ_DATADIR is honored job_ini = os.path.join(os.path.dirname(case_2.__file__), 'job_2.ini') tempdir = tempfile.mkdtemp() dbserver.ensure_on() with mock.patch.dict(os.environ, OQ_DATADIR=tempdir): [job] = run_jobs(create_jobs([job_ini], 'error')) job = commonlib.logs.dbcmd('get_job', job.calc_id) self.assertTrue(job.ds_calc_dir.startswith(tempdir), job.ds_calc_dir) with Print.patch() as p: sap.runline(f'openquake.commands export ruptures {job.id} ' f'-e csv --export-dir={tempdir}') self.assertIn('Exported', str(p)) shutil.rmtree(tempdir)
def test_ebr(self): # test a single case of `run_jobs`, but it is the most complex one, # event based risk with post processing job_ini = os.path.join(os.path.dirname(case_master.__file__), 'job.ini') with Print.patch() as p: [log] = run_jobs(create_jobs([job_ini], 'error')) self.assertIn('id | name', str(p)) # check the exported outputs expected = set('''\ Aggregate Event Losses Aggregate Loss Curves Aggregate Loss Curves Statistics Aggregate Losses Aggregate Losses Statistics Average Asset Losses Average Asset Losses Statistics Average Ground Motion Field Earthquake Ruptures Events Full Report Ground Motion Fields Hazard Curves Hazard Maps Input Files Realizations Source Loss Table'''.splitlines()) with Print.patch() as p: sap.runline(f'openquake.commands engine --lo {log.calc_id}') got = set(re.findall(r'\| ([\w ]+)', str(p))) - {'name'} if got != expected: print('Missing output', expected - got, file=sys.stderr) # sanity check on the performance views: make sure that the most # relevant information is stored (it can be lost due to a wrong # refactoring of the monitoring and it happened several times) with read(log.calc_id) as dstore: perf = str(view('performance', dstore)) self.assertIn('total event_based_risk', perf)
def main(no_distribute=False, yes=False, upgrade_db=False, db_version=False, what_if_I_upgrade=False, list_hazard_calculations=False, list_risk_calculations=False, delete_uncompleted_calculations=False, multi=False, reuse_input=False, *, log_file=None, make_html_report=None, run=None, delete_calculation: int = None, hazard_calculation_id: int = None, list_outputs: int = None, show_log=None, export_output=None, export_outputs=None, param='', config_file=None, exports='', log_level='info'): """ Run a calculation using the traditional command line API """ if not run: # configure a basic logging logs.init() if config_file: config.read(os.path.abspath(os.path.expanduser(config_file)), soft_mem_limit=int, hard_mem_limit=int, port=int, multi_user=valid.boolean, serialize_jobs=valid.boolean, strict=valid.boolean, code=exec) if no_distribute: os.environ['OQ_DISTRIBUTE'] = 'no' # check if the datadir exists datadir = datastore.get_datadir() if not os.path.exists(datadir): os.makedirs(datadir) dbserver.ensure_on() # check if we are talking to the right server err = dbserver.check_foreign() if err: sys.exit(err) if upgrade_db: msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts') if msg.startswith('Your database is already updated'): pass elif yes or confirm('Proceed? (y/n) '): logs.dbcmd('upgrade_db') sys.exit(0) if db_version: safeprint(logs.dbcmd('db_version')) sys.exit(0) if what_if_I_upgrade: safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts')) sys.exit(0) # check if the db is outdated outdated = logs.dbcmd('check_outdated') if outdated: sys.exit(outdated) # hazard or hazard+risk if hazard_calculation_id == -1: # get the latest calculation of the current user hc_id = get_job_id(hazard_calculation_id, getpass.getuser()) elif hazard_calculation_id: # make it possible to use calculations made by another user hc_id = get_job_id(hazard_calculation_id) else: hc_id = None if run: pars = dict(p.split('=', 1) for p in param.split(',')) if param else {} if reuse_input: pars['cachedir'] = datadir if hc_id: pars['hazard_calculation_id'] = str(hc_id) log_file = os.path.expanduser(log_file) \ if log_file is not None else None job_inis = [os.path.expanduser(f) for f in run] pars['multi'] = multi run_jobs(job_inis, log_level, log_file, exports, **pars) # hazard elif list_hazard_calculations: for line in logs.dbcmd('list_calculations', 'hazard', getpass.getuser()): safeprint(line) elif delete_calculation is not None: del_calculation(delete_calculation, yes) # risk elif list_risk_calculations: for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()): safeprint(line) # export elif make_html_report: safeprint('Written %s' % make_report(make_html_report)) sys.exit(0) elif list_outputs is not None: hc_id = get_job_id(list_outputs) for line in logs.dbcmd('list_outputs', hc_id): safeprint(line) elif show_log is not None: hc_id = get_job_id(show_log) for line in logs.dbcmd('get_log', hc_id): safeprint(line) elif export_output is not None: output_id, target_dir = export_output dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id)) for line in core.export_output(dskey, calc_id, datadir, os.path.expanduser(target_dir), exports or DEFAULT_EXPORTS): safeprint(line) elif export_outputs is not None: job_id, target_dir = export_outputs hc_id = get_job_id(job_id) for line in core.export_outputs(hc_id, os.path.expanduser(target_dir), exports or DEFAULT_EXPORTS): safeprint(line) elif delete_uncompleted_calculations: logs.dbcmd('delete_uncompleted_calculations', getpass.getuser()) else: print("Please pass some option, see oq engine --help")
def test_OQ_REDUCE(self): with mock.patch.dict(os.environ, OQ_REDUCE='10'): job_ini = os.path.join(os.path.dirname(case_4.__file__), 'job.ini') run_jobs([job_ini])