def export_gmf_data_csv(ekey, dstore): oq = dstore['oqparam'] imts = list(oq.imtls) df = dstore.read_df('gmf_data').sort_values(['eid', 'sid']) ren = {'sid': 'site_id', 'eid': 'event_id'} for m, imt in enumerate(imts): ren[f'gmv_{m}'] = 'gmv_' + imt for imt in oq.get_sec_imts(): ren[imt] = f'sep_{imt}' df.rename(columns=ren, inplace=True) event_id = dstore['events']['id'] f = dstore.build_fname('sitemesh', '', 'csv') arr = dstore['sitecol'][['lon', 'lat']] sids = numpy.arange(len(arr), dtype=U32) sites = util.compose_arrays(sids, arr, 'site_id') writers.write_csv(f, sites, comment=dstore.metadata) fname = dstore.build_fname('gmf', 'data', 'csv') writers.CsvWriter(fmt=writers.FIVEDIGITS).save(df, fname, comment=dstore.metadata) if 'sigma_epsilon' in dstore['gmf_data']: sig_eps_csv = dstore.build_fname('sigma_epsilon', '', 'csv') sig_eps = dstore['gmf_data/sigma_epsilon'][()] sig_eps['eid'] = event_id[sig_eps['eid']] sig_eps.sort(order='eid') header = list(sig_eps.dtype.names) header[0] = 'event_id' writers.write_csv(sig_eps_csv, sig_eps, header=header, comment=dstore.metadata) return [fname, sig_eps_csv, f] else: return [fname, f]
def export_events(ekey, dstore): events = dstore['events'][()] path = dstore.export_path('events.csv') writers.write_csv(path, events, fmt='%s', renamedict=dict(id='event_id'), comment=dstore.metadata) return [path]
def test_case_23(self): # case with implicit grid and site model on a larger grid out = self.run_calc(case_23.__file__, 'job.ini', exports='csv') [fname] = out['ruptures', 'csv'] self.assertEqualFiles('expected/%s' % strip_calc_id(fname), fname, delta=1E-4) sio = io.StringIO() write_csv(sio, self.calc.datastore['sitecol'].array) tmp = gettemp(sio.getvalue()) self.assertEqualFiles('expected/sitecol.csv', tmp)
def main(fname, out=None): """ Convert logic tree source model file from XML into CSV """ smlt = SourceModelLogicTree(fname) array, attrs = smlt.__toh5__() if out is None: out = fname[:-4] + '.csv' write_csv(out, array, comment=attrs) logging.info('Saved %s', out)
def export_hmaps_csv(key, dest, sitemesh, array, comment): """ Export the hazard maps of the given realization into CSV. :param key: output_type and export_type :param dest: name of the exported file :param sitemesh: site collection :param array: a composite array of dtype hmap_dt :param comment: comment to use as header of the exported CSV file """ curves = util.compose_arrays(sitemesh, array) writers.write_csv(dest, curves, comment=comment) return [dest]
def export_hcurves_csv(ekey, dstore): """ Exports the hazard curves into several .csv files :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ oq = dstore['oqparam'] info = get_info(dstore) R = dstore['full_lt'].get_num_rlzs() sitecol = dstore['sitecol'] sitemesh = get_sites(sitecol) key, kind, fmt = get_kkf(ekey) fnames = [] comment = dstore.metadata hmap_dt = oq.hmap_dt() for kind in oq.get_kinds(kind, R): fname = hazard_curve_name(dstore, (key, fmt), kind) comment.update(kind=kind, investigation_time=oq.investigation_time) if (key in ('hmaps', 'uhs') and oq.uniform_hazard_spectra or oq.hazard_maps): hmap = extract(dstore, 'hmaps?kind=' + kind)[kind] if key == 'uhs' and oq.poes and oq.uniform_hazard_spectra: uhs_curves = calc.make_uhs(hmap, info) writers.write_csv( fname, util.compose_arrays(sitemesh, uhs_curves), comment=comment) fnames.append(fname) elif key == 'hmaps' and oq.poes and oq.hazard_maps: fnames.extend( export_hmaps_csv(ekey, fname, sitemesh, hmap.flatten().view(hmap_dt), comment)) elif key == 'hcurves': # shape (N, R|S, M, L1) if ('amplification' in oq.inputs and oq.amplification_method == 'convolution'): imtls = DictArray( {imt: oq.soil_intensities for imt in oq.imtls}) else: imtls = oq.imtls for imt, imls in imtls.items(): hcurves = extract( dstore, 'hcurves?kind=%s&imt=%s' % (kind, imt))[kind] fnames.append( export_hcurves_by_imt_csv( ekey, kind, fname, sitecol, hcurves, imt, imls, comment)) return sorted(fnames)
def test_case_6(self): # this is a case with 5 assets on the same point self.assert_ok(case_6, 'job_h.ini,job_r.ini') dmg = extract(self.calc.datastore, 'agg_damages/structural?taxonomy=*') tmpname = write_csv(None, dmg, fmt='%.5E') # (T, R, D) == (5, 1, 5) self.assertEqualFiles('expected/dmg_by_taxon.csv', tmpname, delta=1E-5)
def convert_to(fmt, fnames, chatty=False, *, outdir='.', geometry=''): """ Convert source models into CSV files (or geopackages, if fiona is installed). """ t0 = time.time() if geometry: fnames = [geometry] + fnames sections = {} for fname in fnames: logging.info('Reading %s', fname) converter.fname = fname name, _ext = os.path.splitext(os.path.basename(fname)) root = nrml.read(fname) srcs = collections.defaultdict(list) # geom_index -> rows if fname == geometry: for srcnode in root.geometryModel: sec = converter.convert_node(srcnode) sections[sec.sec_id] = sec sections = {sid: sections[sid] for sid in sections} elif 'nrml/0.4' in root['xmlns']: for srcnode in root.sourceModel: appendrow(converter.convert_node(srcnode), srcs, chatty, sections) else: # nrml/0.5 for srcgroup in root.sourceModel: trt = srcgroup['tectonicRegion'] for srcnode in srcgroup: srcnode['tectonicRegion'] = trt appendrow(converter.convert_node(srcnode), srcs, chatty, sections) if fmt == 'csv': for kind, rows in srcs.items(): dest = os.path.join(outdir, '%s_%s.csv' % (name, kind)) logging.info('Saving %d sources on %s', len(rows), dest) header = [ a for a in rows[0].__class__.__annotations__ if a not in 'geom coords' ] write_csv(dest, map(to_tuple, rows), header=header) else: # gpkg gpkg = GeoPackager(name + '.gpkg') for kind, rows in srcs.items(): logging.info('Saving %d sources on layer %s', len(rows), kind) gpkg.save_layer(kind, rows) logging.info('Finished in %d seconds', time.time() - t0)
def export_agglosses(ekey, dstore): oq = dstore['oqparam'] loss_dt = oq.loss_dt() cc = dstore['cost_calculator'] unit_by_lt = cc.units unit_by_lt['occupants'] = 'people' agglosses = dstore[ekey[0]] losses = [] header = ['rlz_id', 'loss_type', 'unit', 'mean', 'stddev'] for r in range(len(agglosses)): for li, lt in enumerate(loss_dt.names): unit = unit_by_lt[lt] mean = agglosses[r, li]['mean'] stddev = agglosses[r, li]['stddev'] losses.append((r, lt, unit, mean, stddev)) dest = dstore.build_fname('agglosses', '', 'csv') writers.write_csv(dest, losses, header=header, comment=dstore.metadata) return [dest]
def export_ruptures_csv(ekey, dstore): """ :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ oq = dstore['oqparam'] if 'scenario' in oq.calculation_mode: return [] dest = dstore.export_path('ruptures.csv') arr = extract(dstore, 'rupture_info') if export.sanity_check: bad = view('bad_ruptures', dstore) if len(bad): # nonempty print(text_table(bad), file=sys.stderr) comment = dstore.metadata comment.update(investigation_time=oq.investigation_time, ses_per_logic_tree_path=oq.ses_per_logic_tree_path) arr.array.sort(order='rup_id') writers.write_csv(dest, arr, comment=comment) return [dest]
def export_csv(ekey, dstore): """ Default csv exporter for arrays stored in the output.hdf5 file :param ekey: export key :param dstore: datastore object :returns: a list with the path of the exported file """ name = ekey[0] + '.csv' try: array = dstore[ekey[0]][()] except AttributeError: # this happens if the key correspond to a HDF5 group return [] # write a custom exporter in this case if len(array.shape) == 1: # vector array = array.reshape((len(array), 1)) return [write_csv(dstore.export_path(name), array)]
def export_hcurves_by_imt_csv(key, kind, fname, sitecol, array, imt, imls, comment): """ Export the curves of the given realization into CSV. :param key: output_type and export_type :param kind: a string with the kind of output (realization or statistics) :param fname: name of the exported file :param sitecol: site collection :param array: an array of shape (N, 1, L1) and dtype numpy.float32 :param imt: intensity measure type :param imls: intensity measure levels :param comment: comment dictionary """ nsites = len(sitecol) dest = add_imt(fname, imt) lst = [('lon', F32), ('lat', F32), ('depth', F32)] for iml in imls: lst.append(('poe-%.7f' % iml, F32)) custom = 'custom_site_id' in sitecol.array.dtype.names if custom: lst.insert(0, ('custom_site_id', U32)) hcurves = numpy.zeros(nsites, lst) if custom: for sid, csi, lon, lat, dep in zip(range(nsites), sitecol.custom_site_id, sitecol.lons, sitecol.lats, sitecol.depths): hcurves[sid] = (csi, lon, lat, dep) + tuple(array[sid, 0, :]) else: hcurves = numpy.zeros(nsites, lst) for sid, lon, lat, dep in zip(range(nsites), sitecol.lons, sitecol.lats, sitecol.depths): hcurves[sid] = (lon, lat, dep) + tuple(array[sid, 0, :]) comment.update(imt=imt) return writers.write_csv(dest, hcurves, comment=comment, header=[name for (name, dt) in lst])
def test_case_5a(self): # this is a case with two gsims and one asset self.assert_ok(case_5a, 'job_haz.ini,job_risk.ini') dmg = extract(self.calc.datastore, 'agg_damages/structural?taxonomy=*') tmpname = write_csv(None, dmg) # shape (T, R, D) == (1, 2, 5) self.assertEqualFiles('expected/dmg_by_taxon.csv', tmpname)
def export_realizations(ekey, dstore): data = extract(dstore, 'realizations').array path = dstore.export_path('realizations.csv') writers.write_csv(path, data, fmt='%.7e', comment=dstore.metadata) return [path]
def export_disagg_csv_xml(ekey, dstore): oq = dstore['oqparam'] sitecol = dstore['sitecol'] hmap4 = dstore['hmap4'] N, M, P, Z = hmap4.shape imts = list(oq.imtls) rlzs = dstore['full_lt'].get_realizations() fnames = [] writercls = hazard_writers.DisaggXMLWriter bins = {name: dset[:] for name, dset in dstore['disagg-bins'].items()} ex = 'disagg?kind=%s&imt=%s&site_id=%s&poe_id=%d&z=%d' if ekey[0] == 'disagg_traditional': ex += '&traditional=1' trad = '-traditional' else: trad = '' skip_keys = ('Mag', 'Dist', 'Lon', 'Lat', 'Eps', 'TRT') for s, m, p, z in iproduct(N, M, P, Z): dic = { k: dstore['disagg/' + k][s, m, p, ..., z] for k in oq.disagg_outputs } if sum(arr.sum() for arr in dic.values()) == 0: # no data continue imt = from_string(imts[m]) r = hmap4.rlzs[s, z] rlz = rlzs[r] iml = hmap4[s, m, p, z] poe_agg = dstore['poe4'][s, m, p, z] fname = dstore.export_path('rlz-%d-%s-sid-%d-poe-%d.xml' % (r, imt, s, p)) lon, lat = sitecol.lons[s], sitecol.lats[s] metadata = dstore.metadata metadata.update(investigation_time=oq.investigation_time, imt=imt.name, smlt_path='_'.join(rlz.sm_lt_path), gsimlt_path=rlz.gsim_rlz.pid, lon=lon, lat=lat, mag_bin_edges=bins['Mag'].tolist(), dist_bin_edges=bins['Dist'].tolist(), lon_bin_edges=bins['Lon'][s].tolist(), lat_bin_edges=bins['Lat'][s].tolist(), eps_bin_edges=bins['Eps'].tolist(), tectonic_region_types=decode(bins['TRT'].tolist())) if ekey[1] == 'xml': metadata['sa_period'] = getattr(imt, 'period', None) or None metadata['sa_damping'] = getattr(imt, 'damping', None) writer = writercls(fname, **metadata) data = [] for k in oq.disagg_outputs: data.append(DisaggMatrix(poe_agg, iml, k.split('_'), dic[k])) writer.serialize(data) fnames.append(fname) else: # csv metadata['poe'] = poe_agg for k in oq.disagg_outputs: header = k.lower().split('_') + ['poe'] com = { key: value for key, value in metadata.items() if value is not None and key not in skip_keys } com.update(metadata) fname = dstore.export_path('rlz-%d-%s-sid-%d-poe-%d%s_%s.csv' % (r, imt, s, p, trad, k)) values = extract(dstore, ex % (k, imt, s, p, z)) writers.write_csv(fname, values, header=header, comment=com, fmt='%.5E') fnames.append(fname) return sorted(fnames)
def assert_export(self, array, expected, header=()): fname = tempfile.NamedTemporaryFile().name write_csv(fname, array, header=header) with open(fname) as f: txt = f.read() self.assertEqual(txt, expected)
def main(vs30_csv, z1pt0=False, z2pt5=False, vs30measured=False, *, exposure_xml=None, sites_csv=None, grid_spacing: float = 0, assoc_distance: float = 5, output='site_model.csv'): """ Prepare a site_model.csv file from exposure xml files/site csv files, vs30 csv files and a grid spacing which can be 0 (meaning no grid). For each site the closest vs30 parameter is used. The command can also generate (on demand) the additional fields z1pt0, z2pt5 and vs30measured which may be needed by your hazard model, depending on the required GSIMs. """ hdf5 = datastore.hdf5new() req_site_params = {'vs30'} fields = ['lon', 'lat', 'vs30'] if z1pt0: req_site_params.add('z1pt0') fields.append('z1pt0') if z2pt5: req_site_params.add('z2pt5') fields.append('z2pt5') if vs30measured: req_site_params.add('vs30measured') fields.append('vs30measured') with performance.Monitor(measuremem=True) as mon: if exposure_xml: mesh, assets_by_site = Exposure.read( exposure_xml, check_dupl=False).get_mesh_assets_by_site() hdf5['assetcol'] = assetcol = site.SiteCollection.from_points( mesh.lons, mesh.lats, req_site_params=req_site_params) if grid_spacing: grid = mesh.get_convex_hull().dilate(grid_spacing).discretize( grid_spacing) haz_sitecol = site.SiteCollection.from_points( grid.lons, grid.lats, req_site_params=req_site_params) logging.info( 'Associating exposure grid with %d locations to %d ' 'exposure sites', len(haz_sitecol), len(assets_by_site)) haz_sitecol, assets_by, discarded = assoc( assets_by_site, haz_sitecol, grid_spacing * SQRT2, 'filter') if len(discarded): logging.info( 'Discarded %d sites with assets ' '[use oq plot_assets]', len(discarded)) hdf5['discarded'] = numpy.array(discarded) haz_sitecol.make_complete() else: haz_sitecol = assetcol discarded = [] elif sites_csv: lons, lats = [], [] for fname in sites_csv: check_fname(fname, 'sites_csv', output) with read(fname) as csv: for line in csv: if line.startswith('lon,lat'): # possible header continue lon, lat = line.split(',')[:2] lons.append(valid.longitude(lon)) lats.append(valid.latitude(lat)) haz_sitecol = site.SiteCollection.from_points( lons, lats, req_site_params=req_site_params) if grid_spacing: grid = haz_sitecol.mesh.get_convex_hull().dilate( grid_spacing).discretize(grid_spacing) haz_sitecol = site.SiteCollection.from_points( grid.lons, grid.lats, req_site_params=req_site_params) else: raise RuntimeError('Missing exposures or missing sites') vs30 = associate(haz_sitecol, vs30_csv, assoc_distance) if z1pt0: haz_sitecol.array['z1pt0'] = calculate_z1pt0(vs30['vs30']) if z2pt5: haz_sitecol.array['z2pt5'] = calculate_z2pt5_ngaw2(vs30['vs30']) hdf5['sitecol'] = haz_sitecol writers.write_csv(output, haz_sitecol.array[fields]) logging.info('Saved %d rows in %s' % (len(haz_sitecol), output)) logging.info(mon) return haz_sitecol
def export_disagg_csv(ekey, dstore): oq = dstore['oqparam'] sitecol = dstore['sitecol'] hmap4 = dstore['hmap4'] rlzs = dstore['full_lt'].get_realizations() best_rlzs = dstore['best_rlzs'][:] N, M, P, Z = hmap4.shape imts = list(oq.imtls) fnames = [] bins = {name: dset[:] for name, dset in dstore['disagg-bins'].items()} ex = 'disagg?kind=%s&imt=%s&site_id=%s&poe_id=%d' if ekey[0] == 'disagg_traditional': ex += '&traditional=1' trad = '-traditional' else: trad = '' skip_keys = ('Mag', 'Dist', 'Lon', 'Lat', 'Eps', 'TRT') metadata = dstore.metadata poes_disagg = ['nan'] * P for p in range(P): try: poes_disagg[p] = str(oq.poes_disagg[p]) except IndexError: pass for s in range(N): rlzcols = ['rlz%d' % r for r in best_rlzs[s]] lon, lat = sitecol.lons[s], sitecol.lats[s] weights = numpy.array([rlzs[r].weight['weight'] for r in best_rlzs[s]]) weights /= weights.sum() # normalize to 1 metadata.update(investigation_time=oq.investigation_time, mag_bin_edges=bins['Mag'].tolist(), dist_bin_edges=bins['Dist'].tolist(), lon_bin_edges=bins['Lon'][s].tolist(), lat_bin_edges=bins['Lat'][s].tolist(), eps_bin_edges=bins['Eps'].tolist(), tectonic_region_types=decode(bins['TRT'].tolist()), rlz_ids=best_rlzs[s].tolist(), weights=weights.tolist(), lon=lon, lat=lat) for k in oq.disagg_outputs: splits = k.lower().split('_') header = ['imt', 'poe'] + splits + rlzcols values = [] nonzeros = [] for m, p in iproduct(M, P): imt = imts[m] aw = extract(dstore, ex % (k, imt, s, p)) # for instance for Mag_Dist [(mag, dist, poe0, poe1), ...] poes = aw[:, len(splits):] if 'trt' in header: nonzeros.append(True) else: nonzeros.append(poes.any()) # nonzero poes for row in aw: values.append([imt, poes_disagg[p]] + list(row)) if any(nonzeros): com = {key: value for key, value in metadata.items() if value is not None and key not in skip_keys} com.update(metadata) fname = dstore.export_path('%s%s-%d.csv' % (k, trad, s)) writers.write_csv(fname, values, header=header, comment=com, fmt='%.5E') fnames.append(fname) return sorted(fnames)