def test_case_1(self): self.assert_curves_ok( ['hazard_curve-PGA.csv', 'hazard_curve-SA(0.1).csv'], case_1.__file__) if parallel.oq_distribute() != 'no': info = text_table(view('job_info', self.calc.datastore)) self.assertIn('task', info) self.assertIn('sent', info) self.assertIn('received', info) slow = view('task:classical:-1', self.calc.datastore) self.assertIn('taskno', slow) self.assertIn('duration', slow) self.assertIn('sources', slow) # there is a single source self.assertEqual(len(self.calc.datastore['source_info']), 1) # check npz export export(('hcurves', 'npz'), self.calc.datastore) # check extraction sitecol = extract(self.calc.datastore, 'sitecol') self.assertEqual(len(sitecol.array), 1) # check minimum_magnitude discards the source with self.assertRaises(RuntimeError) as ctx: self.run_calc(case_1.__file__, 'job.ini', minimum_magnitude='4.5') self.assertEqual(str(ctx.exception), 'All sources were discarded!?')
def sanity_check(self): """ Sanity check on the total number of assets """ if self.R == 1: arr = self.datastore.sel('damages-rlzs') # shape (A, 1, L, D) else: arr = self.datastore.sel('damages-stats', stat='mean') avg = arr.sum(axis=(0, 1)) # shape (L, D) if not len(self.datastore['risk_by_event/agg_id']): logging.warning('There is no damage at all!') elif 'avg_portfolio_damage' in self.datastore: df = views.portfolio_damage_error('avg_portfolio_damage', self.datastore) rst = views.text_table(df, ext='org') logging.info('Portfolio damage\n%s' % rst) num_assets = avg.sum(axis=1) # by loss_type expected = self.assetcol['value-number'].sum() nums = set(num_assets) | {expected} if len(nums) > 1: numdic = dict(expected=expected) for lt, num in zip(self.oqparam.loss_names, num_assets): numdic[lt] = num logging.info( 'Due to rounding errors inherent in floating-point arithmetic,' ' the total number of assets is not exact: %s', numdic)
def print_(aw): if hasattr(aw, 'json'): try: attrs = hdf5.get_shape_descr(aw.json) except KeyError: # no shape_descr, for instance for oqparam print(json.dumps(json.loads(aw.json), indent=2)) return vars(aw).update(attrs) if hasattr(aw, 'shape_descr'): print(text_table(aw.to_dframe(), ext='org')) elif hasattr(aw, 'array'): print(text_table(aw.array, ext='org')) elif isinstance(aw, numpy.ndarray): print(text_table(aw, ext='org')) else: print(aw)
def source_model_info(sm_nodes): """ Extract information about source models. Returns a table with TRTs as rows and source classes as columns. """ c = collections.Counter() for sm in sm_nodes: groups = [sm[0]] if sm['xmlns'].endswith('nrml/0.4') else sm[0] for group in groups: grp_trt = group.get('tectonicRegion') for src in group: trt = src.get('tectonicRegion', grp_trt) src_class = src.tag.split('}')[1] c[trt, src_class] += 1 trts, classes = zip(*c) trts = sorted(set(trts)) classes = sorted(set(classes)) dtlist = [('TRT', (bytes, 30))] + [(name, int) for name in classes] out = numpy.zeros(len(trts) + 1, dtlist) # +1 for the totals for i, trt in enumerate(trts): out[i]['TRT'] = trt for src_class in classes: out[i][src_class] = c[trt, src_class] out[-1]['TRT'] = 'Total' for name in out.dtype.names[1:]: out[-1][name] = out[name][:-1].sum() return text_table(out)
def get_pstats(pstatfile, n): """ Return profiling information as an RST table. :param pstatfile: path to a .pstat file :param n: the maximum number of stats to retrieve """ with tempfile.TemporaryFile(mode='w+') as stream: ps = pstats.Stats(pstatfile, stream=stream) ps.sort_stats('cumtime') ps.print_stats(n) stream.seek(0) lines = list(stream) for i, line in enumerate(lines): if line.startswith(' ncalls'): break data = [] for line in lines[i + 2:]: columns = line.split() if len(columns) == 6: data.append(PStatData(*columns)) rows = [(rec.ncalls, rec.cumtime, rec.path) for rec in data] # here is an example of the expected output table: # ====== ======= ======================================================== # ncalls cumtime path # ====== ======= ======================================================== # 1 33.502 commands/run.py:77(_run) # 1 33.483 calculators/base.py:110(run) # 1 25.166 calculators/classical.py:115(execute) # 1 25.104 baselib.parallel.py:249(apply_reduce) # 1 25.099 calculators/classical.py:41(classical) # 1 25.099 hazardlib/calc/hazard_curve.py:164(classical) return views.text_table(rows, header='ncalls cumtime path'.split())
def test_case_12(self): # test Modified GMPE self.assert_curves_ok(['hazard_curve-smltp_b1-gsimltp_b1_b2.csv'], case_12.__file__) # test disagg_by_grp df = self.calc.datastore.read_df('disagg_by_grp') fname = general.gettemp(text_table(df)) self.assertEqualFiles('expected/disagg_by_grp.rst', fname)
def compare(self, what, imt, files, samplesites, atol, rtol): sids = self.getsids(samplesites) if what == 'uhs': arrays = self.getuhs(what, imt, sids) elif what.startswith('avg_gmf'): arrays = self.getgmf(what, imt, sids) else: arrays = self.getdata(what, imt, sids) header = ['site_id'] if files else ['site_id', 'calc_id'] if what == 'hcurves': header += ['%.5f' % lvl for lvl in self.oq.imtls[imt]] elif what == 'hmaps': header += [str(poe) for poe in self.oq.poes] elif what == 'uhs': header += self.oq.imt_periods() else: # avg_gmf header += ['gmf'] rows = collections.defaultdict(list) diff_idxs = get_diff_idxs(arrays, rtol, atol) if len(diff_idxs) == 0: print('There are no differences within the tolerances ' 'atol=%s, rtol=%d%%, sids=%s' % (atol, rtol * 100, sids)) return [] arr = arrays.transpose(1, 0, 2) # shape (N, C, L) for sid, array in sorted(zip(sids[diff_idxs], arr[diff_idxs])): # each array has shape (C, L) for ex, cols in zip(self.extractors, array): # cols has shape L if files: rows[ex.calc_id].append([sid] + list(cols)) else: rows['all'].append([sid, ex.calc_id] + list(cols)) if files: fdict = {ex.calc_id: open('%s.txt' % ex.calc_id, 'w') for ex in self.extractors} for calc_id, f in fdict.items(): f.write(views.text_table(rows[calc_id], header)) print('Generated %s' % f.name) else: print(views.text_table(rows['all'], header)) return arrays
def test_case_1_eb(self): # this is a case with insured losses and tags self.run_calc(case_1.__file__, 'job_eb.ini', concurrent_tasks='4') [fname] = export(('avg_losses-stats', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/%s' % strip_calc_id(fname), fname, delta=1E-5) aw = extract(self.calc.datastore, 'agg_losses/structural') self.assertEqual(aw.stats, ['mean']) numpy.testing.assert_allclose(aw.array, [685.5015], atol=.001) [fname] = export(('aggrisk', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/%s' % strip_calc_id(fname), fname, delta=1E-5) fnames = export(('aggcurves', 'csv'), self.calc.datastore) for fname in fnames: self.assertEqualFiles('expected/eb_%s' % strip_calc_id(fname), fname, delta=1E-5) [fname] = export(('risk_by_event', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/%s' % strip_calc_id(fname), fname, delta=1E-5) # extract agg_curves with tags aw = extract(self.calc.datastore, 'agg_curves?kind=stats&' 'loss_type=structural&absolute=1&policy=A&taxonomy=RC') tmp = gettemp(text_table(aw.to_dframe())) self.assertEqualFiles('expected/agg_curves5.csv', tmp) aw = extract(self.calc.datastore, 'agg_curves?kind=stats&' 'loss_type=structural&absolute=0&policy=A&taxonomy=RC') tmp = gettemp(text_table(aw.to_dframe())) self.assertEqualFiles('expected/agg_curves7.csv', tmp) # test ct_independence loss4 = view('portfolio_losses', self.calc.datastore) self.run_calc(case_1.__file__, 'job_eb.ini', concurrent_tasks='0') loss0 = view('portfolio_losses', self.calc.datastore) self.assertEqual(loss0, loss4)
def __init__(self, dstore): self.dstore = dstore self.oq = oq = dstore['oqparam'] self.text = (decode(oq.description) + '\n' + '=' * len(oq.description)) try: num_rlzs = dstore['full_lt'].get_num_rlzs() except KeyError: num_rlzs = '?' versions = sorted(dstore['/'].attrs.items()) self.text += '\n\n' + views.text_table(versions) self.text += '\n\nnum_sites = %d, num_levels = %d, num_rlzs = %s' % ( len(dstore['sitecol']), oq.imtls.size, num_rlzs)
def test_case_2(self): self.run_calc(case_2.__file__, 'job.ini') # check view inputs lines = text_table(view('inputs', self.calc.datastore)).splitlines() self.assertEqual(len(lines), 13) # rst table with 13 rows [fname] = export(('hcurves', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/hcurve.csv', fname) # check disagg_by_src for a single realization check_disagg_by_src(self.calc.datastore)
def compare_uhs(calc_ids: int, files=False, *, poe_id: int = 0, samplesites='', rtol: float = 0, atol: float = 1E-3): """ Compare the uniform hazard spectra of two or more calculations. """ c = Comparator(calc_ids) arrays = c.compare('uhs', poe_id, files, samplesites, atol, rtol) if len(arrays) and len(calc_ids) == 2: # each array has shape (N, M) ms = numpy.mean((arrays[0] - arrays[1])**2) maxdiff = numpy.abs(arrays[0] - arrays[1]).max() row = ('%.5f' % c.oq.poes[poe_id], numpy.sqrt(ms), maxdiff) print(views.text_table([row], ['poe', 'rms-diff', 'max-diff']))
def compare_hmaps(imt, calc_ids: int, files=False, *, samplesites='', rtol: float = 0, atol: float = 1E-3): """ Compare the hazard maps of two or more calculations. """ c = Comparator(calc_ids) arrays = c.compare('hmaps', imt, files, samplesites, atol, rtol) if len(arrays) and len(calc_ids) == 2: ms = numpy.mean((arrays[0] - arrays[1])**2, axis=0) # P maxdiff = numpy.abs(arrays[0] - arrays[1]).max(axis=0) # P rows = [(str(poe), rms, md) for poe, rms, md in zip( c.oq.poes, numpy.sqrt(ms), maxdiff)] print(views.text_table(rows, ['poe', 'rms-diff', 'max-diff']))
def add(self, name, obj=None): """Add the view named `name` to the report text""" if obj: text = '\n::\n\n' + indent(str(obj)) else: res = views.view(name, self.dstore) if isinstance(res, (numpy.ndarray, pandas.DataFrame)): text = views.text_table(res) else: text = res if text: title = self.title[name] line = '-' * len(title) self.text += '\n'.join(['\n\n' + title, line, text])
def test_case_master(self): # this tests exercise the case of a complex logic tree self.run_calc(case_master.__file__, 'job.ini') fname = gettemp(text_table(view('mean_disagg', self.calc.datastore))) self.assertEqualFiles('expected/mean_disagg.rst', fname) os.remove(fname) fnames = export(('disagg', 'csv'), self.calc.datastore) self.assertEqual(len(fnames), 64) # 2 sid x 8 keys x 2 poe x 2 imt for fname in fnames: if 'Mag_Dist' in fname and 'Eps' not in fname: self.assertEqualFiles( 'expected_output/%s' % strip_calc_id(fname), fname) check_disagg_by_src(self.calc.datastore)
def main(cmd, args=()): """ Run a database command """ if cmd in commands and len(args) != len(commands[cmd]): sys.exit('Wrong number of arguments, expected %s, got %s' % (commands[cmd], args)) elif (cmd not in commands and not cmd.upper().startswith('SELECT') and config.dbserver.multi_user and getpass.getuser() != 'openquake'): sys.exit('You have no permission to run %s' % cmd) dbserver.ensure_on() res = logs.dbcmd(cmd, *convert(args)) if hasattr(res, '_fields') and res.__class__.__name__ != 'Row': print(text_table(res)) else: print(res)
def post_execute(self, result): """ Export the result in CSV format. :param result: a dictionary asset_ordinal -> array(R, L, D) """ D = len(self.crmodel.damage_states) damages = numpy.zeros((self.A, self.R, self.L, D), numpy.float32) for a in result: damages[a] = result[a] self.datastore['damages-rlzs'] = damages stats.set_rlzs_stats(self.datastore, 'damages', assets=self.assetcol['id'], loss_type=self.oqparam.loss_names, dmg_state=self.crmodel.damage_states) dmg = views.view('portfolio_damage', self.datastore) logging.info('\n' + views.text_table(dmg, ext='org'))
def main(datadir): lst = [] for fname in glob.glob(datadir + '/calc_*.hdf5'): try: dstore = read(fname) except OSError: # already open continue with dstore: try: descr = dstore['oqparam'].description except (KeyError, AttributeError): # not a calculation continue try: tot_ruptures = dstore['full_lt/sg_data']['totrup'].sum() except KeyError: tot_ruptures = 0 else: lst.append((descr, tot_ruptures)) print(text_table(lst, ['calculation', 'total number of ruptures']))
def export_ruptures_csv(ekey, dstore): """ :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ oq = dstore['oqparam'] if 'scenario' in oq.calculation_mode: return [] dest = dstore.export_path('ruptures.csv') arr = extract(dstore, 'rupture_info') if export.sanity_check: bad = view('bad_ruptures', dstore) if len(bad): # nonempty print(text_table(bad), file=sys.stderr) comment = dstore.metadata comment.update(investigation_time=oq.investigation_time, ses_per_logic_tree_path=oq.ses_per_logic_tree_path) arr.array.sort(order='rup_id') writers.write_csv(dest, arr, comment=comment) return [dest]
def test_case_11(self): # secondary perils without secondary simulations self.run_calc(case_11.__file__, 'job.ini', secondary_simulations="{}") calc1 = self.calc.datastore [fname] = export(('risk_by_event', 'csv'), calc1) self.assertEqualFiles('expected/risk_by_event_1.csv', fname) # secondary perils with secondary simulations self.run_calc(case_11.__file__, 'job.ini') calc2 = self.calc.datastore [fname] = export(('risk_by_event', 'csv'), calc2) self.assertEqualFiles('expected/risk_by_event_2.csv', fname) # check mean_perils fname = gettemp(text_table(view('mean_perils', self.calc.datastore))) self.assertEqualFiles('expected/mean_perils.rst', fname) # check damages-rlzs [fname] = export(('damages-rlzs', 'csv'), calc1) self.assertEqualFiles('expected/avg_damages1.csv', fname) [fname] = export(('damages-rlzs', 'csv'), calc2) self.assertEqualFiles('expected/avg_damages2.csv', fname)
def test_case_13(self): self.assert_curves_ok([ 'hazard_curve-mean_PGA.csv', 'hazard_curve-mean_SA(0.2).csv', 'hazard_map-mean.csv' ], case_13.__file__) # test recomputing the hazard maps self.run_calc(case_13.__file__, 'job.ini', exports='csv', hazard_calculation_id=str(self.calc.datastore.calc_id), gsim_logic_tree_file='', source_model_logic_tree_file='') [fname] = export(('hmaps', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/hazard_map-mean.csv', fname, delta=1E-5) csv = general.gettemp( text_table(view('extreme_sites', self.calc.datastore))) self.assertEqualFiles('expected/extreme_sites.rst', csv) # test extract/hcurves/rlz-0, used by the npz exports haz = vars(extract(self.calc.datastore, 'hcurves')) self.assertEqual(sorted(haz), ['_extra', 'all', 'investigation_time']) self.assertEqual(haz['all'].dtype.names, ('lon', 'lat', 'depth', 'mean')) array = haz['all']['mean'] self.assertEqual(array.dtype.names, ('PGA', 'SA(0.2)')) self.assertEqual( array['PGA'].dtype.names, ('0.005', '0.007', '0.0098', '0.0137', '0.0192', '0.0269', '0.0376', '0.0527', '0.0738', '0.103', '0.145', '0.203', '0.284')) # test disagg_by_src in a complex case with duplicated sources check_disagg_by_src(self.calc.datastore)
def test_case_17(self): # CSV exposure in latin1 self.run_calc(case_17.__file__, 'job.ini') tbl = text_table(self.calc.datastore['agg_keys'][:], ext='org') self.assertEqualFiles('agg_keys.org', gettemp(tbl))
def cumtime(self): data = [] for ex in self.extractors: time = ex.get('performance_data')['time_sec'].sum() data.append((ex.calc_id, time)) print(views.text_table(data, ['calc_id', 'time']))