def export_gmf_spec(ekey, dstore, spec): """ :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object :param spec: a string specifying what to export exactly """ oq = dstore['oqparam'] eids = numpy.array([int(rid) for rid in spec.split(',')]) sitemesh = dstore['sitemesh'] writer = writers.CsvWriter(fmt='%.5f') etags = dstore['etags'] if 'scenario' in oq.calculation_mode: _, gmfs_by_trt_gsim = base.get_gmfs(dstore) gsims = sorted(gsim for trt, gsim in gmfs_by_trt_gsim) imts = gmfs_by_trt_gsim[0, gsims[0]].dtype.names gmf_dt = numpy.dtype([(str(gsim), F32) for gsim in gsims]) for eid in eids: etag = etags[eid] for imt in imts: gmfa = numpy.zeros(len(sitemesh), gmf_dt) for gsim in gsims: gmfa[str(gsim)] = gmfs_by_trt_gsim[0, gsim][imt][:, eid] dest = dstore.export_path('gmf-%s-%s.csv' % (etag, imt)) data = util.compose_arrays(sitemesh, gmfa) writer.save(data, dest) else: # event based for eid in eids: etag = etags[eid] for gmfa, imt in _get_gmfs(dstore, util.get_serial(etag), eid): dest = dstore.export_path('gmf-%s-%s.csv' % (etag, imt)) data = util.compose_arrays(sitemesh, gmfa) writer.save(data, dest) return writer.getsaved()
def export_hcurves_csv(ekey, dstore): """ Exports the hazard curves into several .csv files :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ oq = dstore['oqparam'] rlzs_assoc = dstore['csm_info'].get_rlzs_assoc() sitecol = dstore['sitecol'] sitemesh = dstore['sitemesh'] key, fmt = ekey fnames = [] items = dstore['hmaps' if key == 'uhs' else key].items() for kind, hcurves in sorted(items): fname = hazard_curve_name( dstore, ekey, kind, rlzs_assoc, oq.number_of_logic_tree_samples) if key == 'uhs': uhs_curves = calc.make_uhs(hcurves, oq.imtls, oq.poes) write_csv(fname, util.compose_arrays(sitemesh, uhs_curves)) elif key == 'hmaps': write_csv(fname, util.compose_arrays(sitemesh, hcurves)) else: export_hazard_curves_csv(ekey, fname, sitecol, hcurves, oq.imtls) fnames.append(fname) return sorted(fnames)
def export_hcurves_csv(ekey, dstore): """ Exports the hazard curves into several .csv files :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ oq = dstore['oqparam'] rlzs_assoc = dstore['csm_info'].get_rlzs_assoc() sitecol = dstore['sitecol'] sitemesh = get_mesh(sitecol) key, fmt = ekey fnames = [] items = dstore['hmaps' if key == 'uhs' else key].items() for kind, hcurves in sorted(items): fname = hazard_curve_name(dstore, ekey, kind, rlzs_assoc) if key == 'uhs': uhs_curves = calc.make_uhs(hcurves, oq.imtls, oq.poes) write_csv(fname, util.compose_arrays(sitemesh, uhs_curves)) fnames.append(fname) elif key == 'hmaps': write_csv(fname, util.compose_arrays(sitemesh, hcurves)) fnames.append(fname) else: if export.from_db: # called by export_from_db fnames.extend( export_hcurves_by_imt_csv( ekey, fname, sitecol, hcurves, oq.imtls)) else: # when exporting directly from the datastore fnames.extend( export_hazard_curves_csv( ekey, fname, sitecol, hcurves, oq.imtls)) return sorted(fnames)
def export_gmf_scenario_hdf5(ekey, dstore): # compute the GMFs on the fly from the stored rupture (if any) oq = dstore['oqparam'] if 'scenario' not in oq.calculation_mode: logging.warn('GMF export not implemented for %s', oq.calculation_mode) return [] sitemesh = get_mesh(dstore['sitecol'], complete=False) rlzs_assoc = dstore['csm_info'].get_rlzs_assoc() gsims = rlzs_assoc.gsims_by_grp_id[0] # there is a single grp_id E = oq.number_of_ground_motion_fields correl_model = oq.get_correl_model() computer = gmf.GmfComputer( dstore['rupture'], dstore['sitecol'], oq.imtls, gsims, oq.truncation_level, correl_model) fname = dstore.export_path('%s.%s' % ekey) gmf_dt = numpy.dtype([('%s-%03d' % (imt, eid), F32) for imt in oq.imtls for eid in range(E)]) imts = list(oq.imtls) with hdf5.File(fname, 'w') as f: for gsim in gsims: arr = computer.compute(gsim, E, oq.random_seed) I, S, E = arr.shape # #IMTs, #sites, #events gmfa = numpy.zeros(S, gmf_dt) for imti in range(I): for eid in range(E): field = '%s-%03d' % (imts[imti], eid) gmfa[field] = arr[imti, :, eid] f[str(gsim)] = util.compose_arrays(sitemesh, gmfa) return [fname]
def convert_to_array(pmap, sitemesh, imtls): """ Convert the probability map into a composite array with header of the form PGA-0.1, PGA-0.2 ... :param pmap: probability map :param sitemesh: mesh of N sites :param imtls: a DictArray with IMT and levels :returns: a composite array of lenght N """ nsites = len(sitemesh) lst = [] # build the export dtype, of the form PGA-0.1, PGA-0.2 ... for imt, imls in imtls.items(): for iml in imls: lst.append(('%s-%s' % (imt, iml), F64)) curves = numpy.zeros(nsites, numpy.dtype(lst)) for sid, pcurve in pmap.items(): curve = curves[sid] idx = 0 for imt, imls in imtls.items(): for iml in imls: curve['%s-%s' % (imt, iml)] = pcurve.array[idx] idx += 1 return util.compose_arrays(sitemesh, curves)
def view_task_info(token, dstore): """ Display statistical information about the tasks performance. It is possible to get full information about a specific task with a command like this one, for a classical calculation:: $ oq show task_info:classical """ args = token.split(':')[1:] # called as task_info:task_name if args: [task] = args array = dstore['task_info/' + task][()] rduration = array['duration'] / array['weight'] data = util.compose_arrays(rduration, array, 'rduration') data.sort(order='duration') return rst_table(data) data = ['operation-duration mean stddev min max outputs'.split()] for task in dstore['task_info']: val = dstore['task_info/' + task]['duration'] if len(val): data.append(stats(task, val)) if len(data) == 1: return 'Not available' return rst_table(data)
def export_gmf_data_csv(ekey, dstore): oq = dstore['oqparam'] if 'scenario' in oq.calculation_mode: imtls = dstore['oqparam'].imtls rlzs = dstore['csm_info'].get_rlzs_assoc().realizations gsims = [str(rlz.gsim_rlz) for rlz in rlzs] n_gmfs = oq.number_of_ground_motion_fields fields = ['%03d' % i for i in range(n_gmfs)] dt = numpy.dtype([(f, F32) for f in fields]) etags, gmfs_ = calc.get_gmfs(dstore) sitemesh = get_mesh(dstore['sitecol']) writer = writers.CsvWriter(fmt='%.5f') for gsim, gmfa in zip(gsims, gmfs_): # gmfa of shape (N, E) for imt in imtls: gmfs = numpy.zeros(len(gmfa), dt) for i in range(len(gmfa)): gmfs[i] = tuple(gmfa[imt][i]) dest = dstore.build_fname('gmf', '%s-%s' % (gsim, imt), 'csv') data = util.compose_arrays(sitemesh, gmfs) writer.save(data, dest) return writer.getsaved() else: # event based exporter = GmfExporter(dstore) sm_id, eid = get_sm_id_eid(ekey[0]) if eid is None: logging.info('Exporting only the first event') logging.info('Use the command `oq export gmf_data:*:* %d` ' 'to export everything', dstore.calc_id) return exporter.export_one(0, 0) elif eid == '*': return exporter.export_all() else: return exporter.export_one(int(sm_id), int(eid))
def export_hazard_hdf5(ekey, dstore): mesh = get_mesh(dstore['sitecol']) fname = dstore.export_path('%s.%s' % ekey) with hdf5.File(fname, 'w') as f: for dskey, ds in dstore[ekey[0]].items(): f['%s/%s' % (ekey[0], dskey)] = util.compose_arrays(mesh, ds.value) return [fname]
def export_gmf_scenario_npz(ekey, dstore): # compute the GMFs on the fly from the stored rupture (if any) oq = dstore['oqparam'] if 'scenario' not in oq.calculation_mode: logging.warn('GMF export not implemented for %s', oq.calculation_mode) return [] sitemesh = get_mesh(dstore['sitecol'], complete=False) rlzs_assoc = dstore['csm_info'].get_rlzs_assoc() gsims = rlzs_assoc.gsims_by_grp_id[0] # there is a single grp_id E = oq.number_of_ground_motion_fields correl_model = oq.get_correl_model() computer = gmf.GmfComputer( dstore['ruptures/grp-00/0'], dstore['sitecol'], oq.imtls, gsims, oq.truncation_level, correl_model) fname = dstore.export_path('%s.%s' % ekey) gmf_dt = numpy.dtype([(imt, (F32, E)) for imt in oq.imtls]) imts = list(oq.imtls) dic = {} for gsim in gsims: arr = computer.compute(gsim, E, oq.random_seed) I, S, E = arr.shape # #IMTs, #sites, #events gmfa = numpy.zeros(S, gmf_dt) for imti, imt in enumerate(imts): gmfa[imt] = arr[imti] dic[str(gsim)] = util.compose_arrays(sitemesh, gmfa) savez(fname, **dic) return [fname]
def export_csq_by_taxon_csv(ekey, dstore): taxonomies = add_quotes(dstore['assetcol/taxonomies'].value) rlzs = dstore['csm_info'].get_rlzs_assoc().realizations value = dstore[ekey[0]].value # matrix T x R writer = writers.CsvWriter(fmt=FIVEDIGITS) for rlz, values in zip(rlzs, value.T): fname = dstore.build_fname(ekey[0], rlz.gsim_rlz, ekey[1]) writer.save(compose_arrays(taxonomies, values, 'taxonomy'), fname) return writer.getsaved()
def export_loss_maps_csv(ekey, dstore): rlzs = dstore['csm_info'].get_rlzs_assoc().realizations assets = get_assets(dstore) value = get_loss_maps(dstore, 'rlzs') writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) for rlz, values in zip(rlzs, value.T): fname = dstore.build_fname('loss_maps', rlz, ekey[1]) writer.save(compose_arrays(assets, values), fname) return writer.getsaved()
def extract_dmg_by_asset_npz(dstore, what): damage_dt = build_damage_dt(dstore) rlzs = dstore['csm_info'].get_rlzs_assoc().realizations data = dstore['dmg_by_asset'] assets = util.get_assets(dstore) for rlz in rlzs: dmg_by_asset = build_damage_array(data[:, rlz.ordinal], damage_dt) yield 'rlz-%03d' % rlz.ordinal, util.compose_arrays( assets, dmg_by_asset)
def export_rlzs_by_asset_csv(ekey, dstore): rlzs = dstore['csm_info'].get_rlzs_assoc().realizations assets = get_assets(dstore) value = dstore[ekey[0]].value # matrix N x R or T x R writer = writers.CsvWriter(fmt=FIVEDIGITS) for rlz, values in zip(rlzs, value.T): fname = dstore.build_fname(ekey[0], rlz.gsim_rlz, ekey[1]) writer.save(compose_arrays(assets, values), fname) return writer.getsaved()
def export_uhs_hdf5(ekey, dstore): oq = dstore['oqparam'] mesh = get_mesh(dstore['sitecol']) fname = dstore.export_path('%s.%s' % ekey) with hdf5.File(fname, 'w') as f: for dskey in dstore['hcurves']: hcurves = dstore['hcurves/%s' % dskey] uhs_curves = calc.make_uhs(hcurves, oq.imtls, oq.poes, len(mesh)) f['uhs/%s' % dskey] = util.compose_arrays(mesh, uhs_curves) return [fname]
def view_slow_ruptures(token, dstore, maxrows=25): """ Show the slowest ruptures """ fields = ['code', 'n_occ', 'mag', 'trt_smr'] rups = dstore['ruptures'][()][fields] time = dstore['gmf_data/time_by_rup'][()] arr = util.compose_arrays(rups, time) arr = arr[arr['nsites'] > 0] arr.sort(order='time') return arr[-maxrows:]
def view_mean_avg_losses(token, dstore): try: array = dstore['avg_losses-stats'] # shape (N, S) data = array[:, 0] except KeyError: array = dstore['avg_losses-rlzs'] # shape (N, R) data = array[:, 0] assets = util.get_assets(dstore) losses = util.compose_arrays(assets, data) losses.sort() return rst_table(losses, fmt=FIVEDIGITS)
def export_dmg_by_asset_csv(ekey, dstore): damage_dt = build_damage_dt(dstore) rlzs = dstore['csm_info'].get_rlzs_assoc().realizations data = dstore[ekey[0]] writer = writers.CsvWriter(fmt='%.6E') assets = get_assets(dstore) for rlz in rlzs: dmg_by_asset = build_damage_array(data[:, rlz.ordinal], damage_dt) fname = dstore.build_fname(ekey[0], rlz, ekey[1]) writer.save(compose_arrays(assets, dmg_by_asset), fname) return writer.getsaved()
def extract_gmf_scenario_npz(dstore, what): oq = dstore['oqparam'] mesh = get_mesh(dstore['sitecol']) n = len(mesh) data = dstore['gmf_data/data'][()] rlz = dstore['events']['rlz_id'] for rlzi in sorted(set(rlz)): idx = rlz[data['eid']] == rlzi gmfa = _gmf_scenario(data[idx], n, oq.imtls) logging.info('Exporting array%s for rlz#%d', gmfa.shape, rlzi) yield 'rlz-%03d' % rlzi, util.compose_arrays(mesh, gmfa)
def view_mean_avg_losses(token, dstore): try: array = dstore['avg_losses-stats'] # shape (N, S) data = array[:, 0] except KeyError: array = dstore['avg_losses-rlzs'] # shape (N, R) data = array[:, 0] assets = util.get_assets(dstore) losses = util.compose_arrays(assets, data) losses.sort() return rst_table(losses, fmt='%8.6E')
def export_csq_by_taxon_csv(ekey, dstore): taxonomies = dstore['assetcol/taxonomies'].value rlzs = dstore['csm_info'].get_rlzs_assoc().realizations R = len(rlzs) value = dstore[ekey[0]].value # matrix T x R writer = writers.CsvWriter(fmt='%9.6E') for rlz, values in zip(rlzs, value.T): suffix = '.csv' if R == 1 else '-gsimltp_%s.csv' % rlz.uid fname = dstore.export_path(ekey[0] + suffix) writer.save(compose_arrays(taxonomies, values, 'taxonomy'), fname) return writer.getsaved()
def export_rlzs_by_asset_csv(ekey, dstore): rlzs = dstore['csm_info'].get_rlzs_assoc().realizations assets = get_assets(dstore) R = len(rlzs) value = dstore[ekey[0]].value # matrix N x R or T x R writer = writers.CsvWriter(fmt='%9.6E') for rlz, values in zip(rlzs, value.T): suffix = '.csv' if R == 1 else '-gsimltp_%s.csv' % rlz.uid fname = dstore.export_path(ekey[0] + suffix) writer.save(compose_arrays(assets, values), fname) return writer.getsaved()
def extract_gmf_scenario_npz(dstore, what): oq = dstore['oqparam'] mesh = get_mesh(dstore['sitecol']) n = len(mesh) data = dstore['gmf_data/data'][()] rlz = dstore['events']['rlz'] for rlzi in sorted(set(rlz)): idx = rlz[data['eid']] == rlzi gmfa = _gmf_scenario(data[idx], n, oq.imtls) logging.info('Exporting array%s for rlz#%d', gmfa.shape, rlzi) yield 'rlz-%03d' % rlzi, util.compose_arrays(mesh, gmfa)
def export_rcurves(ekey, dstore): rlzs = dstore['csm_info'].get_rlzs_assoc().realizations assets = get_assets(dstore) curves = compactify(dstore[ekey[0]].value) name = ekey[0].split('-')[0] writer = writers.CsvWriter(fmt=FIVEDIGITS) for rlz in rlzs: array = compose_arrays(assets, curves[:, rlz.ordinal]) path = dstore.build_fname(name, rlz, 'csv') writer.save(array, path) return writer.getsaved()
def export_rcurves(ekey, dstore): rlzs = dstore['csm_info'].get_rlzs_assoc().realizations assets = get_assets(dstore) curves = compactify(dstore[ekey[0]].value) name = ekey[0].split('-')[0] writer = writers.CsvWriter(fmt='%9.7E') for rlz in rlzs: array = compose_arrays(assets, curves[:, rlz.ordinal]) path = dstore.export_path('%s-%s.csv' % (name, rlz.uid)) writer.save(array, path) return writer.getsaved()
def view_mean_avg_losses(token, dstore): dt = dstore['oqparam'].multiloss_dt() try: array = dstore['avg_losses-stats'] # shape (N, S) except KeyError: array = dstore['avg_losses-rlzs'] # shape (N, R) data = numpy.array([tuple(row) for row in array[:, 0]], dt) assets = util.get_assets(dstore) losses = util.compose_arrays(assets, data) losses.sort() return rst_table(losses, fmt=FIVEDIGITS)
def export_rcurves(ekey, dstore): rlzs = dstore['csm_info'].get_rlzs_assoc().realizations assets = get_assets(dstore) curves = compactify(dstore[ekey[0]].value) name = ekey[0].split('-')[0] writer = writers.CsvWriter(fmt=FIVEDIGITS) for rlz in rlzs: array = compose_arrays(assets, curves[:, rlz.ordinal]) path = dstore.export_path('%s-%s.csv' % (name, rlz.uid)) writer.save(array, path) return writer.getsaved()
def export_hcurves_hdf5(ekey, dstore): mesh = get_mesh(dstore['sitecol']) imtls = dstore['oqparam'].imtls fname = dstore.export_path('%s.%s' % ekey) with hdf5.File(fname, 'w') as f: f['imtls'] = imtls for dskey in dstore[ekey[0]]: curves = dstore['%s/%s' % (ekey[0], dskey)].convert( imtls, len(mesh)) f['%s/%s' % (ekey[0], dskey)] = util.compose_arrays(mesh, curves) return [fname]
def export_dmg_by_asset_csv(ekey, dstore): damage_dt = build_damage_dt(dstore) rlzs = dstore['csm_info'].get_rlzs_assoc().realizations data = dstore[ekey[0]] writer = writers.CsvWriter(fmt='%.6E') assets = get_assets(dstore) for rlz in rlzs: dmg_by_asset = build_damage_array(data[:, rlz.ordinal], damage_dt) fname = dstore.build_fname(ekey[0], rlz.gsim_rlz, ekey[1]) writer.save(compose_arrays(assets, dmg_by_asset), fname) return writer.getsaved()
def export_uhs_npz(ekey, dstore): oq = dstore['oqparam'] mesh = get_mesh(dstore['sitecol']) fname = dstore.export_path('%s.%s' % ekey) dic = {} for dskey in dstore['hcurves']: hcurves = dstore['hcurves/%s' % dskey] uhs_curves = calc.make_uhs(hcurves, oq.imtls, oq.poes, len(mesh)) dic[dskey] = util.compose_arrays(mesh, uhs_curves) savez(fname, **dic) return [fname]
def view_portfolio_losses(token, dstore): """ The losses for the full portfolio, for each realization and loss type, extracted from the event loss table. """ oq = dstore['oqparam'] loss_dt = oq.loss_dt() data = portfolio_loss(dstore).view(loss_dt)[:, 0] rlzids = [str(r) for r in range(len(data))] array = util.compose_arrays(numpy.array(rlzids), data, 'rlz') # this is very sensitive to rounding errors, so I am using a low precision return rst_table(array, fmt='%.5E')
def export_dmg_by_taxon_csv(ekey, dstore): damage_dt = build_damage_dt(dstore) taxonomies = add_quotes(dstore['assetcol/taxonomies'].value) rlzs = dstore['csm_info'].get_rlzs_assoc().realizations data = dstore[ekey[0]] writer = writers.CsvWriter(fmt='%.6E') for rlz in rlzs: dmg_by_taxon = build_damage_array(data[:, rlz.ordinal], damage_dt) fname = dstore.build_fname(ekey[0], rlz.gsim_rlz, ekey[1]) array = compose_arrays(taxonomies, dmg_by_taxon, 'taxonomy') writer.save(array, fname) return writer.getsaved()
def prepare_site_model(exposure_xml, vs30_csv, grid_spacing=0, site_param_distance=5, output='sites.csv'): """ Prepare a site_model.csv file from an exposure xml file, a vs30 csv file and a grid spacing which can be 0 (meaning no grid). Sites far away from the vs30 records are discarded and you can see them with the command `oq plot_assets`. It is up to you decide if you need to fix your exposure or if it is right to ignore the discarded sites. """ logging.basicConfig(level=logging.INFO) hdf5 = datastore.hdf5new() with performance.Monitor(hdf5.path, hdf5, measuremem=True) as mon: mesh, assets_by_site = Exposure.read( exposure_xml, check_dupl=False).get_mesh_assets_by_site() mon.hdf5['assetcol'] = assetcol = site.SiteCollection.from_points( mesh.lons, mesh.lats, req_site_params={'vs30'}) if grid_spacing: grid = mesh.get_convex_hull().dilate(grid_spacing).discretize( grid_spacing) haz_sitecol = site.SiteCollection.from_points( grid.lons, grid.lats, req_site_params={'vs30'}) logging.info( 'Reducing exposure grid with %d locations to %d sites' ' with assets', len(haz_sitecol), len(assets_by_site)) haz_sitecol, assets_by, _discarded = assoc(assets_by_site, haz_sitecol, grid_spacing * SQRT2, 'filter') haz_sitecol.make_complete() else: haz_sitecol = assetcol vs30orig = read_vs30(vs30_csv.split(',')) logging.info('Associating %d hazard sites to %d site parameters', len(haz_sitecol), len(vs30orig)) sitecol, vs30, discarded = assoc( vs30orig, haz_sitecol, grid_spacing * SQRT2 or site_param_distance, 'filter') sitecol.array['vs30'] = vs30['vs30'] mon.hdf5['sitecol'] = sitecol if discarded: mon.hdf5['discarded'] = numpy.array(discarded) sids = numpy.arange(len(vs30), dtype=numpy.uint32) sites = compose_arrays(sids, vs30, 'site_id') write_csv(output, sites) if discarded: logging.info('Discarded %d sites with assets [use oq plot_assets]', len(discarded)) logging.info('Saved %d rows in %s' % (len(sitecol), output)) logging.info(mon) return sitecol
def export_dmg_by_asset_csv(ekey, dstore): damage_dt = build_damage_dt(dstore) rlzs = dstore['rlzs_assoc'].realizations data = dstore[ekey[0]] writer = writers.CsvWriter(fmt='%.6E') assets = get_assets(dstore) for rlz in rlzs: gsim, = rlz.value dmg_by_asset = build_damage_array(data[:, rlz.ordinal], damage_dt) fname = dstore.export_path('%s-%s.%s' % (ekey[0], gsim, ekey[1])) writer.save(compose_arrays(assets, dmg_by_asset), fname) return writer.getsaved()
def export_dmg_by_asset_npz(ekey, dstore): damage_dt = build_damage_dt(dstore) rlzs = dstore['csm_info'].get_rlzs_assoc().realizations data = dstore[ekey[0]] assets = get_assets(dstore) dic = {} for rlz in rlzs: dmg_by_asset = build_damage_array(data[:, rlz.ordinal], damage_dt) dic['rlz-%03d' % rlz.ordinal] = compose_arrays(assets, dmg_by_asset) fname = dstore.export_path('%s.%s' % ekey) savez(fname, **dic) return [fname]
def view_portfolio_losses(token, dstore): """ The losses for the full portfolio, for each realization and loss type, extracted from the event loss table. """ oq = dstore['oqparam'] loss_dt = oq.loss_dt() data = _portfolio_loss(dstore).view(loss_dt)[:, 0] # shape R rlzids = [str(r) for r in range(len(data))] array = util.compose_arrays(numpy.array(rlzids), data, 'rlz_id') # this is very sensitive to rounding errors, so I am using a low precision return text_table(array, fmt='%.5E')
def export_dmg_by_taxon_csv(ekey, dstore): damage_dt = build_damage_dt(dstore) taxonomies = add_quotes(dstore['assetcol/taxonomies'].value) rlzs = dstore['csm_info'].get_rlzs_assoc().realizations data = dstore[ekey[0]] writer = writers.CsvWriter(fmt='%.6E') for rlz in rlzs: dmg_by_taxon = build_damage_array(data[:, rlz.ordinal], damage_dt) fname = dstore.build_fname(ekey[0], rlz, ekey[1]) array = compose_arrays(taxonomies, dmg_by_taxon, 'taxonomy') writer.save(array, fname) return writer.getsaved()
def view_mean_avg_losses(token, dstore): oq = dstore['oqparam'] R = dstore['csm_info'].get_num_rlzs() if R == 1: # one realization mean = dstore['avg_losses-rlzs'][:, 0] else: mean = dstore['avg_losses-stats'][:, 0] data = numpy.array([tuple(row) for row in mean], oq.loss_dt()) assets = util.get_assets(dstore) losses = util.compose_arrays(assets, data) losses.sort() return rst_table(losses, fmt=FIVEDIGITS)
def export_all_losses_npz(ekey, dstore): rlzs = dstore['csm_info'].get_rlzs_assoc().realizations assets = get_assets(dstore) losses = dstore['all_losses-rlzs'] dic = {} for rlz in rlzs: rlz_losses = _compact(losses[:, :, rlz.ordinal]) data = compose_arrays(assets, rlz_losses) dic['all_losses-%03d' % rlz.ordinal] = data fname = dstore.build_fname('all_losses', 'rlzs', 'npz') savez(fname, **dic) return [fname]
def save_events(self, rup_array): """ :param rup_array: an array of ruptures with fields et_id :returns: a list of RuptureGetters """ from openquake.calculators.getters import (get_eid_rlz, gen_rupture_getters) # this is very fast compared to saving the ruptures E = rup_array['n_occ'].sum() self.check_overflow(E) # check the number of events events = numpy.zeros(E, rupture.events_dt) # when computing the events all ruptures must be considered, # including the ones far away that will be discarded later on rgetters = gen_rupture_getters(self.datastore, self.oqparam.concurrent_tasks) # build the associations eid -> rlz sequentially or in parallel # this is very fast: I saw 30 million events associated in 1 minute! logging.info('Associating event_id -> rlz_id for {:_d} events ' 'and {:_d} ruptures'.format(len(events), len(rup_array))) iterargs = ((rg.proxies, rg.rlzs_by_gsim) for rg in rgetters) if len(events) < 1E5: it = itertools.starmap(get_eid_rlz, iterargs) else: it = parallel.Starmap(get_eid_rlz, iterargs, progress=logging.debug, h5=self.datastore.hdf5) i = 0 for eid_rlz in it: for er in eid_rlz: events[i] = er i += 1 if i >= TWO32: raise ValueError('There are more than %d events!' % i) events.sort(order='rup_id') # fast too # sanity check n_unique_events = len(numpy.unique(events[['id', 'rup_id']])) assert n_unique_events == len(events), (n_unique_events, len(events)) events['id'] = numpy.arange(len(events)) # set event year and event ses starting from 1 nses = self.oqparam.ses_per_logic_tree_path extra = numpy.zeros(len(events), [('year', U32), ('ses_id', U32)]) numpy.random.seed(self.oqparam.ses_seed) if self.oqparam.investigation_time: itime = int(self.oqparam.investigation_time) extra['year'] = numpy.random.choice(itime, len(events)) + 1 extra['ses_id'] = numpy.random.choice(nses, len(events)) + 1 self.datastore['events'] = util.compose_arrays(events, extra) eindices = get_indices(events['rup_id']) arr = numpy.array(list(eindices.values()))[:, 0, :] self.datastore['ruptures']['e0'] = arr[:, 0] self.datastore['ruptures']['e1'] = arr[:, 1]
def export_loss_curves(ekey, dstore): rlzs = dstore['csm_info'].get_rlzs_assoc().realizations loss_types = dstore.get_attr('composite_risk_model', 'loss_types') assets = get_assets(dstore) curves = dstore[ekey[0]] name = ekey[0].split('-')[0] writer = writers.CsvWriter(fmt='%9.6E') for rlz in rlzs: for ltype in loss_types: array = compose_arrays(assets, curves[ltype][:, rlz.ordinal]) path = dstore.export_path('%s-%s-%s.csv' % (name, ltype, rlz.uid)) writer.save(array, path) return writer.getsaved()
def export_hmaps_csv(key, dest, sitemesh, array, comment): """ Export the hazard maps of the given realization into CSV. :param key: output_type and export_type :param dest: name of the exported file :param sitemesh: site collection :param array: a composite array of dtype hmap_dt :param comment: comment to use as header of the exported CSV file """ curves = util.compose_arrays(sitemesh, array) writers.write_csv(dest, curves, comment=comment) return [dest]
def export_dmg_by_taxon_csv(ekey, dstore): damage_dt = build_damage_dt(dstore) taxonomies = dstore['assetcol/taxonomies'].value rlzs = dstore['rlzs_assoc'].realizations data = dstore[ekey[0]] writer = writers.CsvWriter(fmt='%.6E') for rlz in rlzs: gsim, = rlz.value dmg_by_taxon = build_damage_array(data[:, rlz.ordinal], damage_dt) fname = dstore.export_path('%s-%s.%s' % (ekey[0], gsim, ekey[1])) array = compose_arrays(taxonomies, dmg_by_taxon, 'taxonomy') writer.save(array, fname) return writer.getsaved()
def export_hcurves_npz(ekey, dstore): mesh = get_mesh(dstore['sitecol']) imtls = dstore['oqparam'].imtls fname = dstore.export_path('%s.%s' % ekey) arr = numpy.zeros(1, imtls.dt) for imt in imtls: arr[imt] = imtls[imt] dic = dict(imtls=arr[0]) for kind, hcurves in calc.PmapGetter(dstore).items(): curves = hcurves.convert(imtls, len(mesh)) dic[kind] = util.compose_arrays(mesh, curves) savez(fname, **dic) return [fname]
def extract_losses_by_asset(dstore, what): loss_dt = dstore['oqparam'].loss_dt() rlzs = dstore['full_lt'].get_realizations() assets = util.get_assets(dstore) if 'losses_by_asset' in dstore: losses_by_asset = dstore['losses_by_asset'][()] for rlz in rlzs: # I am exporting the 'mean' and ignoring the 'stddev' losses = cast(losses_by_asset[:, rlz.ordinal]['mean'], loss_dt) data = util.compose_arrays(assets, losses) yield 'rlz-%03d' % rlz.ordinal, data elif 'avg_losses-stats' in dstore: aw = hdf5.ArrayWrapper.from_(dstore['avg_losses-stats']) for s, stat in enumerate(aw.stat): losses = cast(aw[:, s], loss_dt) data = util.compose_arrays(assets, losses) yield stat, data elif 'avg_losses-rlzs' in dstore: # there is only one realization avg_losses = dstore['avg_losses-rlzs'][()] losses = cast(avg_losses, loss_dt) data = util.compose_arrays(assets, losses) yield 'rlz-000', data
def export_loss_curves(ekey, dstore): rlzs = dstore['rlzs_assoc'].realizations loss_types = dstore.get_attr('composite_risk_model', 'loss_types') assets = get_assets(dstore) curves = dstore[ekey[0]] name = ekey[0].split('-')[0] writer = writers.CsvWriter(fmt='%9.6E') for rlz in rlzs: for ltype in loss_types: array = compose_arrays(assets, curves[ltype][:, rlz.ordinal]) path = dstore.export_path('%s-%s-%s.csv' % (name, ltype, rlz.uid)) writer.save(array, path) return writer.getsaved()
def view_mean_avg_losses(token, dstore): dt = dstore['oqparam'].loss_dt() weights = dstore['realizations']['weight'] array = dstore['avg_losses-rlzs'].value # shape (N, R) if len(weights) == 1: # one realization mean = array[:, 0] else: mean = hstats.compute_stats2(array, [hstats.mean_curve], weights)[:, 0] data = numpy.array([tuple(row) for row in mean], dt) assets = util.get_assets(dstore) losses = util.compose_arrays(assets, data) losses.sort() return rst_table(losses, fmt=FIVEDIGITS)
def extract_gmf_npz(dstore, what): oq = dstore['oqparam'] qdict = parse(what) [eid] = qdict.get('event_id', [None]) mesh = get_mesh(dstore['sitecol']) n = len(mesh) data = dstore['gmf_data/data'] if eid is None: # get all events rlz = dstore['events']['rlz_id'] for rlzi in sorted(set(rlz)): idx = rlz[data['eid']] == rlzi gmfa = _gmf(data[idx], n, oq.imtls) logging.info('Exporting array%s for rlz#%d', gmfa.shape, rlzi) yield 'rlz-%03d' % rlzi, util.compose_arrays(mesh, gmfa) else: # get a single event rlzi = dstore['events'][eid]['rlz_id'] idx = data['eid'] == eid if idx.any(): gmfa = _gmf(data[idx], n, oq.imtls) yield 'rlz-%03d' % rlzi, util.compose_arrays(mesh, gmfa) else: # zero GMF yield 'rlz-%03d' % rlzi, []
def export_gmf_scenario_csv(ekey, dstore): what = ekey[0].split('/') if len(what) == 1: raise ValueError('Missing "/rup-\d+"') oq = dstore['oqparam'] csm_info = dstore['csm_info'] rlzs_assoc = csm_info.get_rlzs_assoc() samples = csm_info.get_samples_by_grp() imts = list(oq.imtls) mo = re.match('rup-(\d+)$', what[1]) if mo is None: raise ValueError("Invalid format: %r does not match 'rup-(\d+)$'" % what[1]) rup_id = int(mo.group(1)) grp_ids = sorted(int(grp[4:]) for grp in dstore['ruptures']) events = dstore['events'] ruptures = list(calc._get_ruptures(dstore, events, grp_ids, rup_id)) if not ruptures: logging.warn('There is no rupture %d', rup_id) return [] [ebr] = ruptures rlzs_by_gsim = rlzs_assoc.get_rlzs_by_gsim(ebr.grp_id) samples = samples[ebr.grp_id] min_iml = calc.fix_minimum_intensity(oq.minimum_intensity, imts) correl_model = oq.get_correl_model() sitecol = dstore['sitecol'].complete getter = GmfGetter(rlzs_by_gsim, ruptures, sitecol, imts, min_iml, oq.truncation_level, correl_model, samples) getter.init() hazardr = getter.get_hazard() rlzs = rlzs_assoc.realizations fields = ['eid-%03d' % eid for eid in getter.eids] dt = numpy.dtype([(f, F32) for f in fields]) mesh = numpy.zeros(len(ebr.sids), [('lon', F64), ('lat', F64)]) mesh['lon'] = sitecol.lons[ebr.sids] mesh['lat'] = sitecol.lats[ebr.sids] writer = writers.CsvWriter(fmt='%.5f') for rlzi in range(len(rlzs)): hazard = hazardr[rlzi] for imti, imt in enumerate(imts): gmfs = numpy.zeros(len(ebr.sids), dt) for s, sid in enumerate(ebr.sids): for rec in hazard[sid]: event = 'eid-%03d' % rec['eid'] gmfs[s][event] = rec['gmv'][imti] dest = dstore.build_fname('gmf', 'rup-%s-rlz-%s-%s' % (rup_id, rlzi, imt), 'csv') data = util.compose_arrays(mesh, gmfs) writer.save(data, dest) return writer.getsaved()
def extract_losses_by_asset(dstore, what): loss_dt = dstore['oqparam'].loss_dt() rlzs = dstore['csm_info'].get_rlzs_assoc().realizations assets = util.get_assets(dstore) if 'losses_by_asset' in dstore: losses_by_asset = dstore['losses_by_asset'][()] for rlz in rlzs: # I am exporting the 'mean' and ignoring the 'stddev' losses = cast(losses_by_asset[:, rlz.ordinal]['mean'], loss_dt) data = util.compose_arrays(assets, losses) yield 'rlz-%03d' % rlz.ordinal, data elif 'avg_losses-stats' in dstore: avg_losses = dstore['avg_losses-stats'][()] stats = decode(dstore['avg_losses-stats'].attrs['stats']) for s, stat in enumerate(stats): losses = cast(avg_losses[:, s], loss_dt) data = util.compose_arrays(assets, losses) yield stat, data elif 'avg_losses-rlzs' in dstore: # there is only one realization avg_losses = dstore['avg_losses-rlzs'][()] losses = cast(avg_losses, loss_dt) data = util.compose_arrays(assets, losses) yield 'rlz-000', data
def export_loss_maps_csv(ekey, dstore): kind = ekey[0].split('-')[1] # rlzs or stats assets = get_assets(dstore) value = get_loss_maps(dstore, kind) if kind == 'rlzs': tags = dstore['csm_info'].get_rlzs_assoc().realizations else: oq = dstore['oqparam'] tags = ['mean'] + ['quantile-%s' % q for q in oq.quantile_loss_curves] writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) for tag, values in zip(tags, value.T): fname = dstore.build_fname('loss_maps', tag, ekey[1]) writer.save(compose_arrays(assets, values), fname) return writer.getsaved()
def export_damages_csv(ekey, dstore): rlzs = dstore['csm_info'].get_rlzs_assoc().realizations oq = dstore['oqparam'] assets = get_assets(dstore) value = dstore[ekey[0]].value # matrix N x R or T x R writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) if ekey[0].endswith('stats'): tags = ['mean'] + ['quantile-%s' % q for q in oq.quantile_loss_curves] else: tags = ['rlz-%03d' % r for r in range(len(rlzs))] for tag, values in zip(tags, value.T): fname = dstore.build_fname('damages', tag, ekey[1]) writer.save(compose_arrays(assets, values), fname) return writer.getsaved()
def export_gmf_data_csv(ekey, dstore): oq = dstore['oqparam'] rlzs = dstore['full_lt'].get_realizations() imts = list(oq.imtls) sc = dstore['sitecol'].array arr = sc[['lon', 'lat']] eid = int(ekey[0].split('/')[1]) if '/' in ekey[0] else None gmfa = numpy.zeros(len(dstore['gmf_data/eid']), oq.gmf_data_dt()) df = dstore.read_df('gmf_data', 'sid') gmfa['eid'] = df.eid.to_numpy() gmfa['sid'] = df.index.to_numpy() for m in range(len(imts)): gmfa['gmv'][:, m] = df[f'gmv_{m}'].to_numpy() event_id = dstore['events']['id'] gmfa['eid'] = event_id[gmfa['eid']] if eid is None: # we cannot use extract here f = dstore.build_fname('sitemesh', '', 'csv') sids = numpy.arange(len(arr), dtype=U32) sites = util.compose_arrays(sids, arr, 'site_id') writers.write_csv(f, sites) fname = dstore.build_fname('gmf', 'data', 'csv') gmfa.sort(order=['eid', 'sid']) writers.write_csv(fname, _expand_gmv(gmfa, imts), renamedict={ 'sid': 'site_id', 'eid': 'event_id' }) if 'sigma_epsilon' in dstore['gmf_data']: sig_eps_csv = dstore.build_fname('sigma_epsilon', '', 'csv') sig_eps = dstore['gmf_data/sigma_epsilon'][()] sig_eps['eid'] = event_id[sig_eps['eid']] sig_eps.sort(order='eid') header = list(sig_eps.dtype.names) header[0] = 'event_id' writers.write_csv(sig_eps_csv, sig_eps, header=header) return [fname, sig_eps_csv, f] else: return [fname, f] # old format for single eid # TODO: is this still used? gmfa = gmfa[gmfa['eid'] == eid] eid2rlz = dict(dstore['events']) rlzi = eid2rlz[eid] rlz = rlzs[rlzi] data, comment = _build_csv_data(gmfa, rlz, dstore['sitecol'], imts, oq.investigation_time) fname = dstore.build_fname('gmf', '%d-rlz-%03d' % (eid, rlzi), 'csv') return writers.write_csv(fname, data, comment=comment)
def export_agg_losses(ekey, dstore): """ :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ agg_losses = compactify(dstore[ekey[0]].value) rlzs = dstore['rlzs_assoc'].realizations etags = dstore['etags'].value writer = writers.CsvWriter(fmt='%10.6E') for rlz in rlzs: losses = agg_losses[:, rlz.ordinal] dest = dstore.export_path('agg_losses-rlz%03d.csv' % rlz.ordinal) data = compose_arrays(etags, losses) writer.save(data, dest) return writer.getsaved()
def export_hazard_csv(key, dest, sitemesh, pmap, imtls, comment): """ Export the curves of the given realization into CSV. :param key: output_type and export_type :param dest: name of the exported file :param sitemesh: site collection :param pmap: a ProbabilityMap :param dict imtls: intensity measure types and levels :param comment: comment to use as header of the exported CSV file """ curves = util.compose_arrays( sitemesh, calc.convert_to_array(pmap, len(sitemesh), imtls)) writers.write_csv(dest, curves, comment=comment) return [dest]
def export_agg_losses(ekey, dstore): """ :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ agg_losses = dstore[ekey[0]].value rlzs = dstore['csm_info'].get_rlzs_assoc().realizations eids = calc.build_eids(dstore['events'], 0) writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) for rlz in rlzs: losses = agg_losses[:, rlz.ordinal] dest = dstore.build_fname('agg_losses', rlz, 'csv') data = compose_arrays(eids, losses) writer.save(data, dest) return writer.getsaved()