def export_asset_risk_csv(ekey, dstore): """ :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) path = '%s.%s' % (sanitize(ekey[0]), ekey[1]) fname = dstore.export_path(path) md = extract(dstore, 'exposure_metadata') tostr = {'taxonomy': md.taxonomy} for tagname in md.tagnames: tostr[tagname] = getattr(md, tagname) tagnames = sorted(set(md.tagnames) - {'id'}) arr = extract(dstore, 'asset_risk').array rows = [] lossnames = sorted(name for name in arr.dtype.names if 'loss' in name) expnames = [name for name in arr.dtype.names if name not in md.tagnames and 'loss' not in name and name not in 'lon lat'] colnames = ['id'] + tagnames + ['lon', 'lat'] + expnames + lossnames # sanity check assert len(colnames) == len(arr.dtype.names) for rec in arr: row = [] for name in colnames: value = rec[name] try: row.append('"%s"' % tostr[name][value]) except KeyError: row.append(value) rows.append(row) writer.save(rows, fname, colnames) return [fname]
def test_case_1c(self): # this is a case with more hazard sites than exposure sites test_dir = os.path.dirname(case_1c.__file__) self.run_calc(test_dir, 'job.ini', exports='csv') total = extract(self.calc.datastore, 'agg_damages/structural') aae([[0.4906653, 0.3249882, 0.0708492, 0.0211334, 0.092364]], total) # shape (R, D) = (1, 5) # check extract gmf_data works with a filtered site collection gmf_data = dict(extract(self.calc.datastore, 'gmf_data')) self.assertEqual(gmf_data['rlz-000'].shape, (1,))
def test_case_7(self): self.assert_curves_ok( ['hazard_curve-mean.csv', 'hazard_curve-smltp_b1-gsimltp_b1.csv', 'hazard_curve-smltp_b2-gsimltp_b1.csv'], case_7.__file__) # exercising extract/mean_std_curves extract(self.calc.datastore, 'mean_std_curves') # exercise the warning for no output when mean_hazard_curves='false' self.run_calc( case_7.__file__, 'job.ini', mean_hazard_curves='false', calculation_mode='preclassical', poes='0.1')
def export_hmaps_xml(ekey, dstore): key, kind, fmt = get_kkf(ekey) oq = dstore['oqparam'] sitecol = dstore['sitecol'] sitemesh = get_mesh(sitecol) rlzs_assoc = dstore['csm_info'].get_rlzs_assoc() R = len(rlzs_assoc.realizations) fnames = [] writercls = hazard_writers.HazardMapXMLWriter for kind in oq.get_kinds(kind, R): # shape (N, M, P) hmaps = extract(dstore, 'hmaps?kind=' + kind)[kind] if kind.startswith('rlz-'): rlz = rlzs_assoc.realizations[int(kind[4:])] smlt_path = '_'.join(rlz.sm_lt_path) gsimlt_path = rlz.gsim_rlz.uid else: smlt_path = '' gsimlt_path = '' for m, imt in enumerate(oq.imtls): for p, poe in enumerate(oq.poes): suffix = '-%s-%s' % (poe, imt) fname = hazard_curve_name(dstore, ekey, kind + suffix) data = [HazardMap(site[0], site[1], hmap[m, p]) for site, hmap in zip(sitemesh, hmaps)] writer = writercls( fname, investigation_time=oq.investigation_time, imt=imt, poe=poe, smlt_path=smlt_path, gsimlt_path=gsimlt_path) writer.serialize(data) fnames.append(fname) return sorted(fnames)
def test_case_shakemap(self): self.run_calc(case_shakemap.__file__, 'pre-job.ini') self.run_calc(case_shakemap.__file__, 'job.ini', hazard_calculation_id=str(self.calc.datastore.calc_id)) sitecol = self.calc.datastore['sitecol'] self.assertEqual(len(sitecol), 9) gmfdict = dict(extract(self.calc.datastore, 'gmf_data')) gmfa = gmfdict['rlz-000'] self.assertEqual(gmfa.shape, (9,)) self.assertEqual(gmfa.dtype.names, ('lon', 'lat', 'PGA', 'SA(0.3)', 'SA(1.0)')) agglosses = extract(self.calc.datastore, 'agglosses') aac(agglosses['mean'], numpy.array([1848876.5], numpy.float32), atol=.1) aac(agglosses['stddev'], numpy.array([1902063.], numpy.float32), atol=.1)
def test_case_2d(self): # time_event not specified in job_h.ini but specified in job_r.ini out = self.run_calc(case_2d.__file__, 'job_h.ini,job_r.ini', exports='csv') # this is also a case with a single site but an exposure grid, # to test a corner case [fname] = out['losses_by_asset', 'csv'] self.assertEqualFiles('expected/losses_by_asset.csv', fname) # test agglosses tot = extract(self.calc.datastore, 'agg_losses/occupants') aac(tot.array, [0.031716], atol=1E-5) # test agglosses with * tbl = extract(self.calc.datastore, 'agg_losses/occupants?taxonomy=*') self.assertEqual(tbl.array.shape, (1, 1)) # 1 taxonomy, 1 rlz
def post_execute(self, arr): """ Compute aggregated risk """ md = extract(self.datastore, 'exposure_metadata') categories = [cat.replace('value-', 'loss-') for cat in md] + [ ds + '-structural' for ds in self.riskmodel.damage_states] multi_risk = list(md.array) multi_risk += sorted( set(arr.dtype.names) - set(self.datastore['assetcol/array'].dtype.names)) tot = {risk: arr[risk].sum() for risk in multi_risk} cats = [] values = [] for cat in categories: val = [tot.get(f, numpy.nan) for f in self.get_fields(cat)] if not numpy.isnan(val).all(): cats.append(cat) values.append(val) dt = [('peril', hdf5.vstr)] + [(c, float) for c in cats] agg_risk = numpy.zeros(len(self.all_perils), dt) for cat, val in zip(cats, values): agg_risk[cat] = val agg_risk['peril'] = self.all_perils self.datastore['agg_risk'] = agg_risk
def test_case_13(self): self.assert_curves_ok( ['hazard_curve-mean_PGA.csv', 'hazard_curve-mean_SA(0.2).csv', 'hazard_map-mean.csv'], case_13.__file__) # test recomputing the hazard maps self.run_calc( case_13.__file__, 'job.ini', exports='csv', hazard_calculation_id=str(self.calc.datastore.calc_id), gsim_logic_tree_file='', source_model_logic_tree_file='') [fname] = export(('hmaps', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/hazard_map-mean.csv', fname, delta=1E-5) # test extract/hcurves/rlz-0, used by the npz exports haz = vars(extract(self.calc.datastore, 'hcurves')) self.assertEqual(sorted(haz), ['all', 'investigation_time']) self.assertEqual( haz['all'].dtype.names, ('lon', 'lat', 'depth', 'mean')) array = haz['all']['mean'] self.assertEqual(array.dtype.names, ('PGA', 'SA(0.2)')) self.assertEqual(array['PGA'].dtype.names, ('0.005', '0.007', '0.0098', '0.0137', '0.0192', '0.0269', '0.0376', '0.0527', '0.0738', '0.103', '0.145', '0.203', '0.284'))
def test_case_1(self): self.assert_curves_ok( ['hazard_curve-PGA.csv', 'hazard_curve-SA(0.1).csv'], case_1.__file__) if parallel.oq_distribute() != 'no': info = view('job_info', self.calc.datastore) self.assertIn('task', info) self.assertIn('sent', info) self.assertIn('received', info) # there is a single source self.assertEqual(len(self.calc.datastore['source_info']), 1) # check npz export export(('hcurves', 'npz'), self.calc.datastore) # check extraction sitecol = extract(self.calc.datastore, 'sitecol') self.assertEqual(len(sitecol.array), 1) # check minimum_magnitude discards the source with self.assertRaises(RuntimeError) as ctx: self.run_calc(case_1.__file__, 'job.ini', minimum_magnitude='4.5') self.assertEqual(str(ctx.exception), 'All sources were filtered away!')
def test_case_1(self): # case with volcanic multiperil ASH, LAVA, LAHAR, PYRO self.run_calc(case_1.__file__, 'job.ini') [fname] = export(('asset_risk', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/asset_risk.csv', fname) [fname] = export(('agg_risk', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/agg_risk.csv', fname) # check extract md = extract(self.calc.datastore, 'exposure_metadata') ae(md.array, ['number', 'occupants_night', 'value-structural']) ae(md.multi_risk, ['collapse-structural-ASH_DRY', 'collapse-structural-ASH_WET', 'loss-structural-ASH_DRY', 'loss-structural-ASH_WET', 'loss-structural-LAHAR', 'loss-structural-LAVA', 'loss-structural-PYRO', 'no_damage-structural-ASH_DRY', 'no_damage-structural-ASH_WET', 'number-LAHAR', 'number-LAVA', 'number-PYRO', 'occupants_night-LAHAR', 'occupants_night-LAVA', 'occupants_night-PYRO'])
def export_hcurves_xml(ekey, dstore): key, kind, fmt = get_kkf(ekey) len_ext = len(fmt) + 1 oq = dstore['oqparam'] sitemesh = get_mesh(dstore['sitecol']) rlzs_assoc = dstore['csm_info'].get_rlzs_assoc() R = len(rlzs_assoc.realizations) fnames = [] writercls = hazard_writers.HazardCurveXMLWriter for kind in oq.get_kinds(kind, R): if kind.startswith('rlz-'): rlz = rlzs_assoc.realizations[int(kind[4:])] smlt_path = '_'.join(rlz.sm_lt_path) gsimlt_path = rlz.gsim_rlz.uid else: smlt_path = '' gsimlt_path = '' name = hazard_curve_name(dstore, ekey, kind) hcurves = extract(dstore, 'hcurves?kind=' + kind)[kind] for im in oq.imtls: slc = oq.imtls(im) imt = from_string(im) fname = name[:-len_ext] + '-' + im + '.' + fmt data = [HazardCurve(Location(site), poes[slc]) for site, poes in zip(sitemesh, hcurves)] writer = writercls(fname, investigation_time=oq.investigation_time, imls=oq.imtls[im], imt=imt.name, sa_period=getattr(imt, 'period', None) or None, sa_damping=getattr(imt, 'damping', None), smlt_path=smlt_path, gsimlt_path=gsimlt_path) writer.serialize(data) fnames.append(fname) return sorted(fnames)
def test_case_miriam(self): # this is a case with a grid and asset-hazard association self.run_calc(case_miriam.__file__, 'job.ini') # check minimum_magnitude >= 5.2 minmag = self.calc.datastore['ruptures']['mag'].min() self.assertGreaterEqual(minmag, 5.2) # check asset_loss_table tot = self.calc.datastore['asset_loss_table'].value.sum() self.assertEqual(tot, 15787827.0) [fname] = export(('agg_loss_table', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/agg_losses-rlz000-structural.csv', fname, delta=1E-5) fname = gettemp(view('portfolio_losses', self.calc.datastore)) self.assertEqualFiles( 'expected/portfolio_losses.txt', fname, delta=1E-5) # this is a case with exposure and region_grid_spacing=1 self.run_calc(case_miriam.__file__, 'job2.ini') hcurves = dict(extract(self.calc.datastore, 'hcurves'))['all'] sitecol = self.calc.datastore['sitecol'] # filtered sitecol self.assertEqual(len(hcurves), len(sitecol)) assetcol = self.calc.datastore['assetcol'] self.assertEqual(len(sitecol), 15) self.assertGreater(sitecol.vs30.sum(), 0) self.assertEqual(len(assetcol), 548)
def export_losses_by_asset_npz(ekey, dstore): """ :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ fname = dstore.export_path('%s.%s' % ekey) savez(fname, **dict(extract(dstore, 'losses_by_asset'))) return [fname]
def test_case_8(self): # a complex scenario_risk from GMFs where the hazard sites are # not in the asset locations self.run_calc(case_8.__file__, 'job.ini') agglosses = extract(self.calc.datastore, 'agg_losses/structural') aac(agglosses.array, [1159817.1]) # make sure the fullreport can be extracted view('fullreport', self.calc.datastore)
def plot_losses(calc_id, bins=7): """ losses_by_event plotter """ # read the hazard data dstore = util.read(calc_id) losses_by_rlzi = dict(extract(dstore, 'losses_by_event')) oq = dstore['oqparam'] plt = make_figure(losses_by_rlzi, oq.loss_dt().names, bins) plt.show()
def test_case_master(self): # a case with two GSIMs self.run_calc(case_master.__file__, 'job.ini', exports='npz') # check realizations [fname] = export(('realizations', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/realizations.csv', fname) # check losses by taxonomy agglosses = extract(self.calc.datastore, 'agg_losses/structural?' 'taxonomy=*').array # shape (T, R) = (3, 2) self.assertEqualFiles('expected/agglosses_taxo.txt', gettemp(str(agglosses))) # extract agglosses with a * and a selection obj = extract(self.calc.datastore, 'agg_losses/structural?' 'state=*&cresta=0.11') self.assertEqual(obj.selected, [b'state=*', b'cresta=0.11']) self.assertEqual(obj.tags, [b'state=01']) aac(obj.array, [[2493.7097, 2943.6640]])
def export_hcurves_csv(ekey, dstore): """ Exports the hazard curves into several .csv files :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ oq = dstore['oqparam'] info = get_info(dstore) rlzs_assoc = dstore['csm_info'].get_rlzs_assoc() R = len(rlzs_assoc.realizations) sitecol = dstore['sitecol'] sitemesh = get_mesh(sitecol) key, kind, fmt = get_kkf(ekey) fnames = [] checksum = dstore.get_attr('/', 'checksum32') hmap_dt = oq.hmap_dt() for kind in oq.get_kinds(kind, R): fname = hazard_curve_name(dstore, (key, fmt), kind, rlzs_assoc) comment = _comment(rlzs_assoc, kind, oq.investigation_time) if (key in ('hmaps', 'uhs') and oq.uniform_hazard_spectra or oq.hazard_maps): hmap = extract(dstore, 'hmaps?kind=' + kind)[kind] if key == 'uhs' and oq.poes and oq.uniform_hazard_spectra: uhs_curves = calc.make_uhs(hmap, info) writers.write_csv( fname, util.compose_arrays(sitemesh, uhs_curves), comment=comment + ', checksum=%d' % checksum) fnames.append(fname) elif key == 'hmaps' and oq.poes and oq.hazard_maps: fnames.extend( export_hmaps_csv(ekey, fname, sitemesh, hmap.flatten().view(hmap_dt), comment + ', checksum=%d' % checksum)) elif key == 'hcurves': hcurves = extract(dstore, 'hcurves?kind=' + kind)[kind] fnames.extend( export_hcurves_by_imt_csv( ekey, kind, rlzs_assoc, fname, sitecol, hcurves, oq, checksum)) return sorted(fnames)
def test_case_1(self): out = self.run_calc(case_1.__file__, 'job_risk.ini', exports='csv') [fname] = out['agglosses', 'csv'] self.assertEqualFiles('expected/agg.csv', fname) # check the exported GMFs [fname, _, sitefile] = export(('gmf_data', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/gmf-FromFile.csv', fname) self.assertEqualFiles('expected/sites.csv', sitefile) [fname] = out['losses_by_event', 'csv'] self.assertEqualFiles('expected/losses_by_event.csv', fname) # check the asset values by sid [val] = extract(self.calc.datastore, 'asset_values/0') self.assertEqual(val['aref'], b'a2') self.assertEqual(val['aid'], 0) self.assertEqual(val['structural'], 2000.) with self.assertRaises(IndexError): # non-existing site_id extract(self.calc.datastore, 'asset_values/1')
def test_case_1(self): # test with a single event self.assert_ok(case_1, 'job_risk.ini') got = view('num_units', self.calc.datastore) self.assertEqual('''\ ======== ========= taxonomy num_units ======== ========= RC 2,000 RM 4,000 *ALL* 6,000 ======== =========''', got) # test agg_damages, 1 realization x 3 damage states [dmg] = extract(self.calc.datastore, 'agg_damages/structural?' 'taxonomy=RC&CRESTA=01.1') numpy.testing.assert_almost_equal( [1498.0121, 472.96616, 29.021801], dmg, decimal=4) # test no intersection dmg = extract(self.calc.datastore, 'agg_damages/structural?' 'taxonomy=RM&CRESTA=01.1') self.assertEqual(dmg.shape, ())
def check_multi_tag(self, dstore): # multi-tag aggregations arr = extract(dstore, 'aggregate/avg_losses?' 'tag=taxonomy&tag=occupancy&kind=quantile-0.5') self.assertEqual(len(arr.to_table()), 1) # aggregate by all loss types fnames = export( ('aggregate_by/avg_losses?tag=taxonomy&tag=occupancy&kind=mean', 'csv'), dstore) for fname in fnames: self.assertEqualFiles('expected/%s' % strip_calc_id(fname), fname)
def export_aggregate_by_csv(ekey, dstore): """ :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ token, what = ekey[0].split('/', 1) aw = extract(dstore, 'aggregate/' + what) fnames = [] writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) path = '%s.%s' % (sanitize(ekey[0]), ekey[1]) fname = dstore.export_path(path) writer.save(aw.to_table(), fname) fnames.append(fname) return fnames
def export_by_tag_csv(ekey, dstore): """ :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ token, tag = ekey[0].split('/') data = extract(dstore, token + '/' + tag) fnames = [] writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) for stat, arr in data: tup = (ekey[0].replace('/', '-'), stat, ekey[1]) path = '%s-%s.%s' % tup fname = dstore.export_path(path) writer.save(arr, fname) fnames.append(fname) return fnames
def test_case_1(self): out = self.run_calc(case_1.__file__, 'job.ini', exports='csv,xml') [fname, _, _] = out['gmf_data', 'csv'] self.assertEqualFiles('expected/gmf-data.csv', fname) [fname] = export(('hcurves', 'csv'), self.calc.datastore) self.assertEqualFiles( 'expected/hazard_curve-smltp_b1-gsimltp_b1.csv', fname) [fname] = export(('gmf_scenario/rup-0', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/gmf-rlz-0-PGA.csv', fname) # test that the .npz export runs export(('gmf_data', 'npz'), self.calc.datastore) export(('hcurves', 'xml'), self.calc.datastore) [fname] = out['hcurves', 'xml'] self.assertEqualFiles( 'expected/hazard_curve-smltp_b1-gsimltp_b1-PGA.xml', fname) # test gsim_by_imt out = self.run_calc(case_1.__file__, 'job.ini', ses_per_logic_tree_path='20', gsim_logic_tree_file='gsim_by_imt_logic_tree.xml', exports='csv') # testing event_info einfo = dict(extract(self.calc.datastore, 'event_info/0')) self.assertEqual(einfo['trt'], 'active shallow crust') self.assertEqual(einfo['rupture_class'], 'ParametricProbabilisticRupture') self.assertEqual(einfo['surface_class'], 'PlanarSurface') self.assertEqual(einfo['serial'], 1066) self.assertEqual(str(einfo['gsim']), '[MultiGMPE."PGA".AkkarBommer2010]\n' '[MultiGMPE."SA(0.1)".SadighEtAl1997]') self.assertEqual(einfo['rlzi'], 0) self.assertEqual(einfo['grp_id'], 0) self.assertEqual(einfo['occurrence_rate'], 1.0) self.assertEqual(list(einfo['hypo']), [0., 0., 4.]) [fname, _, _] = out['gmf_data', 'csv'] self.assertEqualFiles('expected/gsim_by_imt.csv', fname)
def view_hmap(token, dstore): """ Display the highest 20 points of the mean hazard map. Called as $ oq show hmap:0.1 # 10% PoE """ try: poe = valid.probability(token.split(':')[1]) except IndexError: poe = 0.1 mean = dict(extract(dstore, 'hcurves?kind=mean'))['mean'] oq = dstore['oqparam'] hmap = calc.make_hmap_array(mean, oq.imtls, [poe], len(mean)) dt = numpy.dtype([('sid', U32)] + [(imt, F32) for imt in oq.imtls]) array = numpy.zeros(len(hmap), dt) for i, vals in enumerate(hmap): array[i] = (i, ) + tuple(vals) array.sort(order=list(oq.imtls)[0]) return rst_table(array[:20])
def test_case_1(self): self.run_calc(case_1.__file__, 'job.ini') ekeys = [('agg_curves-stats', 'csv')] for ekey in ekeys: for fname in export(ekey, self.calc.datastore): self.assertEqualFiles( 'expected/%s' % strip_calc_id(fname), fname) # make sure the agg_curves-stats has the right attrs self.check_attr('return_periods', [30, 60, 120, 240, 480, 960]) self.check_attr('units', [b'EUR', b'EUR']) self.check_attr('nbytes', 96) # test the loss curves exporter [f1] = export(('loss_curves/rlz-0', 'csv'), self.calc.datastore) [f2] = export(('loss_curves/rlz-1', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/loss_curves-rlz-000.csv', f1) self.assertEqualFiles('expected/loss_curves-rlz-001.csv', f2) [f] = export(('loss_curves/mean', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/loss_curves-mean.csv', f) # test the loss maps exporter fnames = export(('loss_maps-stats', 'csv'), self.calc.datastore) assert fnames for fname in fnames: self.assertEqualFiles('expected/' + strip_calc_id(fname), fname, delta=1E-5) # test portfolio loss tmp = gettemp(view('portfolio_loss', self.calc.datastore)) self.assertEqualFiles('expected/portfolio_loss.txt', tmp) # test the rup_loss_table exporter fnames = export(('rup_loss_table', 'xml'), self.calc.datastore) self.assertEqual(len(fnames), 2) for fname in fnames: self.assertEqualFiles('expected/' + strip_calc_id(fname), fname) # test the src_loss_table extractor arr = extract(self.calc.datastore, 'src_loss_table/structural') tmp = gettemp(rst_table(arr)) self.assertEqualFiles('expected/src_loss_table.txt', tmp)
def test_case_18(self): # GMPEtable self.assert_curves_ok( ['hazard_curve-mean_PGA.csv', 'hazard_curve-mean_SA(0.2).csv', 'hazard_curve-mean_SA(1.0).csv', 'hazard_map-mean.csv', 'hazard_uhs-mean.csv'], case_18.__file__, kind='stats', delta=1E-7) [fname] = export(('realizations', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/realizations.csv', fname) # check exporting a single realization in XML and CSV [fname] = export(('uhs/rlz-001', 'xml'), self.calc.datastore) if NOT_DARWIN: # broken on macOS self.assertEqualFiles('expected/uhs-rlz-1.xml', fname) [fname] = export(('uhs/rlz-001', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/uhs-rlz-1.csv', fname) # extracting hmaps hmaps = extract(self.calc.datastore, 'hmaps')['all']['mean'] self.assertEqual(hmaps.dtype.names, ('PGA', 'SA(0.2)', 'SA(1.0)'))
def test_case_2(self): # case with two damage states self.run_calc(case_1.__file__, 'job_2.ini') [fname] = export(('asset_risk', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/asset_risk_2.csv', fname) # check extract md = extract(self.calc.datastore, 'exposure_metadata') ae(md.array, ['number', 'occupants_night', 'value-structural']) ae(md.multi_risk, ['loss-structural-LAHAR', 'number-LAHAR', 'occupants_night-LAHAR']) # check invalid key structural_fragility_file with self.assertRaises(ValueError): self.run_calc(case_1.__file__, 'job.ini', structura_fragility_file='fragility_model.xml') # check invalid key structural_consequence_file with self.assertRaises(ValueError): self.run_calc(case_1.__file__, 'job.ini', structura_consequence_file='consequence_model.xml')
def export_uhs_xml(ekey, dstore): oq = dstore['oqparam'] rlzs_assoc = dstore['csm_info'].get_rlzs_assoc() R = len(rlzs_assoc.realizations) sitemesh = get_mesh(dstore['sitecol'].complete) key, kind, fmt = get_kkf(ekey) fnames = [] periods = [imt.period for imt in oq.imt_periods()] for kind in oq.get_kinds(kind, R): metadata = get_metadata(rlzs_assoc.realizations, kind) uhs = extract(dstore, 'uhs?kind=' + kind)[kind] for p, poe in enumerate(oq.poes): fname = hazard_curve_name(dstore, (key, fmt), kind + '-%s' % poe) writer = hazard_writers.UHSXMLWriter( fname, periods=periods, poe=poe, investigation_time=oq.investigation_time, **metadata) data = [] for site, curve in zip(sitemesh, uhs): data.append(UHS(curve[:, p], Location(site))) writer.serialize(data) fnames.append(fname) return sorted(fnames)
def export_hazard_npz(ekey, dstore): fname = dstore.export_path('%s.%s' % ekey) savez(fname, **dict(extract(dstore, ekey[0]))) return [fname]
def export_gmf_scenario_npz(ekey, dstore): fname = dstore.export_path('%s.%s' % ekey) savez(fname, **dict(extract(dstore, 'gmf_data'))) return [fname]
def get_mean_curves(dstore): """ Extract the mean hazard curves from the datastore, as a composite array of length nsites. """ return dict(extract.extract(dstore, 'hcurves?kind=mean'))['mean']
def test_case_5a(self): # this is a case with two gsims and one asset self.assert_ok(case_5a, 'job_haz.ini,job_risk.ini') dmg = extract(self.calc.datastore, 'agg_damages/structural?taxonomy=*') tmpname = write_csv(None, dmg) # shape (T, R, D) == (1, 2, 5) self.assertEqualFiles('expected/dmg_by_taxon.csv', tmpname)
def export_dmg_by_asset_npz(ekey, dstore): fname = dstore.export_path('%s.%s' % ekey) savez(fname, **dict(extract(dstore, 'dmg_by_asset'))) return [fname]
def export_hazard_npz(ekey, dstore): fname = dstore.export_path('%s.%s' % ekey) out = extract(dstore, ekey[0]) kw = {k: v for k, v in vars(out).items() if not k.startswith('_')} savez(fname, **kw) return [fname]
def export_disagg_csv_xml(ekey, dstore): oq = dstore['oqparam'] sitecol = dstore['sitecol'] hmap4 = dstore['hmap4'] N, M, P, Z = hmap4.shape imts = list(oq.imtls) rlzs = dstore['full_lt'].get_realizations() fnames = [] writercls = hazard_writers.DisaggXMLWriter bins = {name: dset[:] for name, dset in dstore['disagg-bins'].items()} ex = 'disagg?kind=%s&imt=%s&site_id=%s&poe_id=%d&z=%d' skip_keys = ('Mag', 'Dist', 'Lon', 'Lat', 'Eps', 'TRT') for s, m, p, z in iproduct(N, M, P, Z): dic = { k: dstore['disagg/' + k][s, m, p, ..., z] for k in oq.disagg_outputs } if sum(arr.sum() for arr in dic.values()) == 0: # no data continue imt = from_string(imts[m]) r = hmap4.rlzs[s, z] rlz = rlzs[r] iml = hmap4[s, m, p, z] poe_agg = dstore['poe4'][s, m, p, z] fname = dstore.export_path('rlz-%d-%s-sid-%d-poe-%d.xml' % (r, imt, s, p)) lon, lat = sitecol.lons[s], sitecol.lats[s] metadata = dstore.metadata metadata.update(investigation_time=oq.investigation_time, imt=imt.name, smlt_path='_'.join(rlz.sm_lt_path), gsimlt_path=rlz.gsim_rlz.pid, lon=lon, lat=lat, mag_bin_edges=bins['Mag'].tolist(), dist_bin_edges=bins['Dist'].tolist(), lon_bin_edges=bins['Lon'][s].tolist(), lat_bin_edges=bins['Lat'][s].tolist(), eps_bin_edges=bins['Eps'].tolist(), tectonic_region_types=decode(bins['TRT'].tolist())) if ekey[1] == 'xml': metadata['sa_period'] = getattr(imt, 'period', None) or None metadata['sa_damping'] = getattr(imt, 'damping', None) writer = writercls(fname, **metadata) data = [] for k in oq.disagg_outputs: data.append(DisaggMatrix(poe_agg, iml, k.split('_'), dic[k])) writer.serialize(data) fnames.append(fname) else: # csv metadata['poe'] = poe_agg for k in oq.disagg_outputs: header = k.lower().split('_') + ['poe'] com = { key: value for key, value in metadata.items() if value is not None and key not in skip_keys } com.update(metadata) fname = dstore.export_path('rlz-%d-%s-sid-%d-poe-%d_%s.csv' % (r, imt, s, p, k)) values = extract(dstore, ex % (k, imt, s, p, z)) writers.write_csv(fname, values, header=header, comment=com, fmt='%.5E') fnames.append(fname) return sorted(fnames)
def export_realizations(ekey, dstore): data = extract(dstore, 'realizations').array path = dstore.export_path('realizations.csv') writers.write_csv(path, data, fmt='%.7e') return [path]
def test_case_6(self): # this is a case with 5 assets on the same point self.assert_ok(case_6, 'job_h.ini,job_r.ini') dmg = extract(self.calc.datastore, 'agg_damages/structural?taxonomy=*') tmpname = write_csv(None, dmg) # shape (T, R, D) == (5, 1, 5) self.assertEqualFiles('expected/dmg_by_taxon.csv', tmpname)
def test_case_1_eb(self): # this is a case with insured losses self.run_calc(case_1.__file__, 'job_eb.ini') [fname] = export(('avg_losses-stats', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/%s' % strip_calc_id(fname), fname) aw = extract(self.calc.datastore, 'agg_losses/structural') self.assertEqual(aw.stats, ['mean']) self.assertEqual(aw.array, numpy.float32([767.82324])) fnames = export(('agg_curves-stats', 'csv'), self.calc.datastore) for fname in fnames: self.assertEqualFiles('expected/eb_%s' % strip_calc_id(fname), fname) fnames = export(('agg_losses-stats', 'csv'), self.calc.datastore) for fname in fnames: self.assertEqualFiles('expected/%s' % strip_calc_id(fname), fname, delta=1E-5) [fname] = export(('losses_by_event', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/%s' % strip_calc_id(fname), fname) # extract agg_curves, no tags aw = extract( self.calc.datastore, 'agg_curves?kind=stats&' 'loss_type=structural&absolute=1') tmp = gettemp(rst_table(aw.to_table())) self.assertEqualFiles('expected/agg_curves1.csv', tmp) aw = extract(self.calc.datastore, 'agg_curves?kind=rlzs&' 'loss_type=structural&absolute=1') tmp = gettemp(rst_table(aw.to_table())) self.assertEqualFiles('expected/agg_curves2.csv', tmp) aw = extract( self.calc.datastore, 'agg_curves?kind=stats&' 'loss_type=structural&absolute=0') tmp = gettemp(rst_table(aw.to_table())) self.assertEqualFiles('expected/agg_curves3.csv', tmp) aw = extract(self.calc.datastore, 'agg_curves?kind=rlzs&' 'loss_type=structural&absolute=0') tmp = gettemp(rst_table(aw.to_table())) self.assertEqualFiles('expected/agg_curves4.csv', tmp) # TODO: fix extract agg_curves for insured types # extract agg_curves with tags self.run_calc(case_1.__file__, 'job_eb.ini', aggregate_by='policy,taxonomy', hazard_calculation_id=str(self.calc.datastore.calc_id)) aw = extract( self.calc.datastore, 'agg_curves?kind=stats&' 'loss_type=structural&absolute=1&policy=A&taxonomy=RC') tmp = gettemp(rst_table(aw.to_table())) self.assertEqualFiles('expected/agg_curves5.csv', tmp) aw = extract( self.calc.datastore, 'agg_curves?kind=rlzs&' 'loss_type=structural&absolute=1&policy=A&taxonomy=RC') tmp = gettemp(rst_table(aw.to_table())) self.assertEqualFiles('expected/agg_curves6.csv', tmp) aw = extract( self.calc.datastore, 'agg_curves?kind=stats&' 'loss_type=structural&absolute=0&policy=A&taxonomy=RC') tmp = gettemp(rst_table(aw.to_table())) self.assertEqualFiles('expected/agg_curves7.csv', tmp) aw = extract( self.calc.datastore, 'agg_curves?kind=rlzs&' 'loss_type=structural&absolute=0&policy=A&taxonomy=RC') tmp = gettemp(rst_table(aw.to_table())) self.assertEqualFiles('expected/agg_curves8.csv', tmp)
def test_case_master(self): if sys.platform == 'darwin': raise unittest.SkipTest('MacOSX') self.run_calc(case_master.__file__, 'job.ini', exports='csv') fnames = export(('avg_losses-stats', 'csv'), self.calc.datastore) assert fnames, 'avg_losses-stats not exported?' for fname in fnames: self.assertEqualFiles('expected/' + strip_calc_id(fname), fname, delta=1E-5) # extract loss_curves/rlz-1 (with the first asset having zero losses) [fname] = export(('loss_curves/rlz-1', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/' + strip_calc_id(fname), fname, delta=1E-5) fnames = export(('loss_maps-rlzs', 'csv'), self.calc.datastore) assert fnames, 'loss_maps-rlzs not exported?' for fname in fnames: self.assertEqualFiles('expected/' + strip_calc_id(fname), fname, delta=1E-5) # extract curves by tag tags = 'taxonomy=tax1&state=01&cresta=0.11' a = extract(self.calc.datastore, 'agg_curves/structural?' + tags) self.assertEqual(a.array.shape, (4, 3)) # 4 stats, 3 return periods fname = gettemp(view('portfolio_losses', self.calc.datastore)) self.assertEqualFiles('expected/portfolio_losses.txt', fname, delta=1E-5) os.remove(fname) # check ruptures are stored correctly fname = gettemp(view('ruptures_events', self.calc.datastore)) self.assertEqualFiles('expected/ruptures_events.txt', fname) os.remove(fname) # check losses_by_tag fnames = export(('aggregate_by/occupancy/avg_losses', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/losses_by_occupancy.csv', fnames[0]) self.check_multi_tag(self.calc.datastore) # ------------------------- ebrisk calculator ---------------------- # self.run_calc(case_master.__file__, 'job.ini', hazard_calculation_id=str(self.calc.datastore.calc_id), calculation_mode='ebrisk', exports='', aggregate_by='taxonomy', insured_losses='false') # agg_losses-rlzs has shape (L=5, R=9) # agg_losses-stats has shape (L=5, S=4) fname = export(('agg_losses-stats', 'csv'), self.calc.datastore)[0] self.assertEqualFiles('expected/agglosses.csv', fname) fname = export(('avg_losses', 'csv'), self.calc.datastore)[0] self.assertEqualFiles('expected/avglosses.csv', fname, delta=1E-5)
def export_disagg_csv(ekey, dstore): oq = dstore['oqparam'] sitecol = dstore['sitecol'] hmap4 = dstore['hmap4'] rlzs = dstore['full_lt'].get_realizations() best_rlzs = dstore['best_rlzs'][:] N, M, P, Z = hmap4.shape imts = list(oq.imtls) fnames = [] bins = {name: dset[:] for name, dset in dstore['disagg-bins'].items()} ex = 'disagg?kind=%s&imt=%s&site_id=%s&poe_id=%d' if ekey[0] == 'disagg_traditional': ex += '&traditional=1' trad = '-traditional' else: trad = '' skip_keys = ('Mag', 'Dist', 'Lon', 'Lat', 'Eps', 'TRT') metadata = dstore.metadata poes_disagg = ['nan'] * P for p in range(P): try: poes_disagg[p] = str(oq.poes_disagg[p]) except IndexError: pass for s in range(N): lon, lat = sitecol.lons[s], sitecol.lats[s] weights = numpy.array([rlzs[r].weight['weight'] for r in best_rlzs[s]]) weights /= weights.sum() # normalize to 1 metadata.update(investigation_time=oq.investigation_time, mag_bin_edges=bins['Mag'].tolist(), dist_bin_edges=bins['Dist'].tolist(), lon_bin_edges=bins['Lon'][s].tolist(), lat_bin_edges=bins['Lat'][s].tolist(), eps_bin_edges=bins['Eps'].tolist(), tectonic_region_types=decode(bins['TRT'].tolist()), rlz_ids=best_rlzs[s].tolist(), weights=weights.tolist(), lon=lon, lat=lat) for k in oq.disagg_outputs: splits = k.lower().split('_') header = (['imt', 'poe'] + splits + ['poe%d' % z for z in range(Z)]) values = [] nonzeros = [] for m, p in iproduct(M, P): imt = imts[m] aw = extract(dstore, ex % (k, imt, s, p)) # for instance for Mag_Dist [(mag, dist, poe0, poe1), ...] poes = aw[:, len(splits):] if 'trt' in header: nonzeros.append(True) else: nonzeros.append(poes.any()) # nonzero poes for row in aw: values.append([imt, poes_disagg[p]] + list(row)) if any(nonzeros): com = { key: value for key, value in metadata.items() if value is not None and key not in skip_keys } com.update(metadata) fname = dstore.export_path('%s%s-%d.csv' % (k, trad, s)) writers.write_csv(fname, values, header=header, comment=com, fmt='%.5E') fnames.append(fname) return sorted(fnames)
def test_case_9(self): # using gmfs.xml self.run_calc(case_9.__file__, 'job.ini') agglosses = extract(self.calc.datastore, 'agglosses/structural') aac(agglosses.array, [7306.7124])