def export_gmf_data_csv(ekey, dstore):
oq = dstore['oqparam']
imts = list(oq.imtls)
df = dstore.read_df('gmf_data').sort_values(['eid', 'sid'])
ren = {'sid': 'site_id', 'eid': 'event_id'}
for m, imt in enumerate(imts):
ren[f'gmv_{m}'] = 'gmv_' + imt
for imt in oq.get_sec_imts():
ren[imt] = f'sep_{imt}'
df.rename(columns=ren, inplace=True)
event_id = dstore['events']['id']
f = dstore.build_fname('sitemesh', '', 'csv')
arr = dstore['sitecol'][['lon', 'lat']]
sids = numpy.arange(len(arr), dtype=U32)
sites = util.compose_arrays(sids, arr, 'site_id')
writers.write_csv(f, sites, comment=dstore.metadata)
fname = dstore.build_fname('gmf', 'data', 'csv')
writers.CsvWriter(fmt=writers.FIVEDIGITS).save(df,
fname,
comment=dstore.metadata)
if 'sigma_epsilon' in dstore['gmf_data']:
sig_eps_csv = dstore.build_fname('sigma_epsilon', '', 'csv')
sig_eps = dstore['gmf_data/sigma_epsilon'][()]
sig_eps['eid'] = event_id[sig_eps['eid']]
sig_eps.sort(order='eid')
header = list(sig_eps.dtype.names)
header[0] = 'event_id'
writers.write_csv(sig_eps_csv,
sig_eps,
header=header,
comment=dstore.metadata)
return [fname, sig_eps_csv, f]
else:
return [fname, f]
def export_events(ekey, dstore):
events = dstore['events'][()]
path = dstore.export_path('events.csv')
writers.write_csv(path,
events,
fmt='%s',
renamedict=dict(id='event_id'),
comment=dstore.metadata)
return [path]
def test_case_23(self):
# case with implicit grid and site model on a larger grid
out = self.run_calc(case_23.__file__, 'job.ini', exports='csv')
[fname] = out['ruptures', 'csv']
self.assertEqualFiles('expected/%s' % strip_calc_id(fname), fname,
delta=1E-4)
sio = io.StringIO()
write_csv(sio, self.calc.datastore['sitecol'].array)
tmp = gettemp(sio.getvalue())
self.assertEqualFiles('expected/sitecol.csv', tmp)
def main(fname, out=None):
"""
Convert logic tree source model file from XML into CSV
"""
smlt = SourceModelLogicTree(fname)
array, attrs = smlt.__toh5__()
if out is None:
out = fname[:-4] + '.csv'
write_csv(out, array, comment=attrs)
logging.info('Saved %s', out)
def export_hmaps_csv(key, dest, sitemesh, array, comment):
"""
Export the hazard maps of the given realization into CSV.
:param key: output_type and export_type
:param dest: name of the exported file
:param sitemesh: site collection
:param array: a composite array of dtype hmap_dt
:param comment: comment to use as header of the exported CSV file
"""
curves = util.compose_arrays(sitemesh, array)
writers.write_csv(dest, curves, comment=comment)
return [dest]
def export_hcurves_csv(ekey, dstore):
"""
Exports the hazard curves into several .csv files
:param ekey: export key, i.e. a pair (datastore key, fmt)
:param dstore: datastore object
"""
oq = dstore['oqparam']
info = get_info(dstore)
R = dstore['full_lt'].get_num_rlzs()
sitecol = dstore['sitecol']
sitemesh = get_sites(sitecol)
key, kind, fmt = get_kkf(ekey)
fnames = []
comment = dstore.metadata
hmap_dt = oq.hmap_dt()
for kind in oq.get_kinds(kind, R):
fname = hazard_curve_name(dstore, (key, fmt), kind)
comment.update(kind=kind, investigation_time=oq.investigation_time)
if (key in ('hmaps', 'uhs') and oq.uniform_hazard_spectra or
oq.hazard_maps):
hmap = extract(dstore, 'hmaps?kind=' + kind)[kind]
if key == 'uhs' and oq.poes and oq.uniform_hazard_spectra:
uhs_curves = calc.make_uhs(hmap, info)
writers.write_csv(
fname, util.compose_arrays(sitemesh, uhs_curves),
comment=comment)
fnames.append(fname)
elif key == 'hmaps' and oq.poes and oq.hazard_maps:
fnames.extend(
export_hmaps_csv(ekey, fname, sitemesh,
hmap.flatten().view(hmap_dt), comment))
elif key == 'hcurves':
# shape (N, R|S, M, L1)
if ('amplification' in oq.inputs and
oq.amplification_method == 'convolution'):
imtls = DictArray(
{imt: oq.soil_intensities for imt in oq.imtls})
else:
imtls = oq.imtls
for imt, imls in imtls.items():
hcurves = extract(
dstore, 'hcurves?kind=%s&imt=%s' % (kind, imt))[kind]
fnames.append(
export_hcurves_by_imt_csv(
ekey, kind, fname, sitecol, hcurves, imt, imls,
comment))
return sorted(fnames)
def test_case_6(self):
# this is a case with 5 assets on the same point
self.assert_ok(case_6, 'job_h.ini,job_r.ini')
dmg = extract(self.calc.datastore, 'agg_damages/structural?taxonomy=*')
tmpname = write_csv(None, dmg, fmt='%.5E') # (T, R, D) == (5, 1, 5)
self.assertEqualFiles('expected/dmg_by_taxon.csv', tmpname,
delta=1E-5)
def convert_to(fmt, fnames, chatty=False, *, outdir='.', geometry=''):
"""
Convert source models into CSV files (or geopackages, if fiona is
installed).
"""
t0 = time.time()
if geometry:
fnames = [geometry] + fnames
sections = {}
for fname in fnames:
logging.info('Reading %s', fname)
converter.fname = fname
name, _ext = os.path.splitext(os.path.basename(fname))
root = nrml.read(fname)
srcs = collections.defaultdict(list) # geom_index -> rows
if fname == geometry:
for srcnode in root.geometryModel:
sec = converter.convert_node(srcnode)
sections[sec.sec_id] = sec
sections = {sid: sections[sid] for sid in sections}
elif 'nrml/0.4' in root['xmlns']:
for srcnode in root.sourceModel:
appendrow(converter.convert_node(srcnode), srcs, chatty,
sections)
else: # nrml/0.5
for srcgroup in root.sourceModel:
trt = srcgroup['tectonicRegion']
for srcnode in srcgroup:
srcnode['tectonicRegion'] = trt
appendrow(converter.convert_node(srcnode), srcs, chatty,
sections)
if fmt == 'csv':
for kind, rows in srcs.items():
dest = os.path.join(outdir, '%s_%s.csv' % (name, kind))
logging.info('Saving %d sources on %s', len(rows), dest)
header = [
a for a in rows[0].__class__.__annotations__
if a not in 'geom coords'
]
write_csv(dest, map(to_tuple, rows), header=header)
else: # gpkg
gpkg = GeoPackager(name + '.gpkg')
for kind, rows in srcs.items():
logging.info('Saving %d sources on layer %s', len(rows), kind)
gpkg.save_layer(kind, rows)
logging.info('Finished in %d seconds', time.time() - t0)
def export_agglosses(ekey, dstore):
oq = dstore['oqparam']
loss_dt = oq.loss_dt()
cc = dstore['cost_calculator']
unit_by_lt = cc.units
unit_by_lt['occupants'] = 'people'
agglosses = dstore[ekey[0]]
losses = []
header = ['rlz_id', 'loss_type', 'unit', 'mean', 'stddev']
for r in range(len(agglosses)):
for li, lt in enumerate(loss_dt.names):
unit = unit_by_lt[lt]
mean = agglosses[r, li]['mean']
stddev = agglosses[r, li]['stddev']
losses.append((r, lt, unit, mean, stddev))
dest = dstore.build_fname('agglosses', '', 'csv')
writers.write_csv(dest, losses, header=header, comment=dstore.metadata)
return [dest]
def export_ruptures_csv(ekey, dstore):
"""
:param ekey: export key, i.e. a pair (datastore key, fmt)
:param dstore: datastore object
"""
oq = dstore['oqparam']
if 'scenario' in oq.calculation_mode:
return []
dest = dstore.export_path('ruptures.csv')
arr = extract(dstore, 'rupture_info')
if export.sanity_check:
bad = view('bad_ruptures', dstore)
if len(bad): # nonempty
print(text_table(bad), file=sys.stderr)
comment = dstore.metadata
comment.update(investigation_time=oq.investigation_time,
ses_per_logic_tree_path=oq.ses_per_logic_tree_path)
arr.array.sort(order='rup_id')
writers.write_csv(dest, arr, comment=comment)
return [dest]
def export_csv(ekey, dstore):
"""
Default csv exporter for arrays stored in the output.hdf5 file
:param ekey: export key
:param dstore: datastore object
:returns: a list with the path of the exported file
"""
name = ekey[0] + '.csv'
try:
array = dstore[ekey[0]][()]
except AttributeError:
# this happens if the key correspond to a HDF5 group
return [] # write a custom exporter in this case
if len(array.shape) == 1: # vector
array = array.reshape((len(array), 1))
return [write_csv(dstore.export_path(name), array)]
def export_hcurves_by_imt_csv(key, kind, fname, sitecol, array, imt, imls,
comment):
"""
Export the curves of the given realization into CSV.
:param key: output_type and export_type
:param kind: a string with the kind of output (realization or statistics)
:param fname: name of the exported file
:param sitecol: site collection
:param array: an array of shape (N, 1, L1) and dtype numpy.float32
:param imt: intensity measure type
:param imls: intensity measure levels
:param comment: comment dictionary
"""
nsites = len(sitecol)
dest = add_imt(fname, imt)
lst = [('lon', F32), ('lat', F32), ('depth', F32)]
for iml in imls:
lst.append(('poe-%.7f' % iml, F32))
custom = 'custom_site_id' in sitecol.array.dtype.names
if custom:
lst.insert(0, ('custom_site_id', U32))
hcurves = numpy.zeros(nsites, lst)
if custom:
for sid, csi, lon, lat, dep in zip(range(nsites),
sitecol.custom_site_id,
sitecol.lons, sitecol.lats,
sitecol.depths):
hcurves[sid] = (csi, lon, lat, dep) + tuple(array[sid, 0, :])
else:
hcurves = numpy.zeros(nsites, lst)
for sid, lon, lat, dep in zip(range(nsites), sitecol.lons,
sitecol.lats, sitecol.depths):
hcurves[sid] = (lon, lat, dep) + tuple(array[sid, 0, :])
comment.update(imt=imt)
return writers.write_csv(dest,
hcurves,
comment=comment,
header=[name for (name, dt) in lst])
def test_case_5a(self):
# this is a case with two gsims and one asset
self.assert_ok(case_5a, 'job_haz.ini,job_risk.ini')
dmg = extract(self.calc.datastore, 'agg_damages/structural?taxonomy=*')
tmpname = write_csv(None, dmg) # shape (T, R, D) == (1, 2, 5)
self.assertEqualFiles('expected/dmg_by_taxon.csv', tmpname)
def export_realizations(ekey, dstore):
data = extract(dstore, 'realizations').array
path = dstore.export_path('realizations.csv')
writers.write_csv(path, data, fmt='%.7e', comment=dstore.metadata)
return [path]
def export_disagg_csv_xml(ekey, dstore):
oq = dstore['oqparam']
sitecol = dstore['sitecol']
hmap4 = dstore['hmap4']
N, M, P, Z = hmap4.shape
imts = list(oq.imtls)
rlzs = dstore['full_lt'].get_realizations()
fnames = []
writercls = hazard_writers.DisaggXMLWriter
bins = {name: dset[:] for name, dset in dstore['disagg-bins'].items()}
ex = 'disagg?kind=%s&imt=%s&site_id=%s&poe_id=%d&z=%d'
if ekey[0] == 'disagg_traditional':
ex += '&traditional=1'
trad = '-traditional'
else:
trad = ''
skip_keys = ('Mag', 'Dist', 'Lon', 'Lat', 'Eps', 'TRT')
for s, m, p, z in iproduct(N, M, P, Z):
dic = {
k: dstore['disagg/' + k][s, m, p, ..., z]
for k in oq.disagg_outputs
}
if sum(arr.sum() for arr in dic.values()) == 0: # no data
continue
imt = from_string(imts[m])
r = hmap4.rlzs[s, z]
rlz = rlzs[r]
iml = hmap4[s, m, p, z]
poe_agg = dstore['poe4'][s, m, p, z]
fname = dstore.export_path('rlz-%d-%s-sid-%d-poe-%d.xml' %
(r, imt, s, p))
lon, lat = sitecol.lons[s], sitecol.lats[s]
metadata = dstore.metadata
metadata.update(investigation_time=oq.investigation_time,
imt=imt.name,
smlt_path='_'.join(rlz.sm_lt_path),
gsimlt_path=rlz.gsim_rlz.pid,
lon=lon,
lat=lat,
mag_bin_edges=bins['Mag'].tolist(),
dist_bin_edges=bins['Dist'].tolist(),
lon_bin_edges=bins['Lon'][s].tolist(),
lat_bin_edges=bins['Lat'][s].tolist(),
eps_bin_edges=bins['Eps'].tolist(),
tectonic_region_types=decode(bins['TRT'].tolist()))
if ekey[1] == 'xml':
metadata['sa_period'] = getattr(imt, 'period', None) or None
metadata['sa_damping'] = getattr(imt, 'damping', None)
writer = writercls(fname, **metadata)
data = []
for k in oq.disagg_outputs:
data.append(DisaggMatrix(poe_agg, iml, k.split('_'), dic[k]))
writer.serialize(data)
fnames.append(fname)
else: # csv
metadata['poe'] = poe_agg
for k in oq.disagg_outputs:
header = k.lower().split('_') + ['poe']
com = {
key: value
for key, value in metadata.items()
if value is not None and key not in skip_keys
}
com.update(metadata)
fname = dstore.export_path('rlz-%d-%s-sid-%d-poe-%d%s_%s.csv' %
(r, imt, s, p, trad, k))
values = extract(dstore, ex % (k, imt, s, p, z))
writers.write_csv(fname,
values,
header=header,
comment=com,
fmt='%.5E')
fnames.append(fname)
return sorted(fnames)
def assert_export(self, array, expected, header=()):
fname = tempfile.NamedTemporaryFile().name
write_csv(fname, array, header=header)
with open(fname) as f:
txt = f.read()
self.assertEqual(txt, expected)
def main(vs30_csv,
z1pt0=False,
z2pt5=False,
vs30measured=False,
*,
exposure_xml=None,
sites_csv=None,
grid_spacing: float = 0,
assoc_distance: float = 5,
output='site_model.csv'):
"""
Prepare a site_model.csv file from exposure xml files/site csv files,
vs30 csv files and a grid spacing which can be 0 (meaning no grid).
For each site the closest vs30 parameter is used. The command can also
generate (on demand) the additional fields z1pt0, z2pt5 and vs30measured
which may be needed by your hazard model, depending on the required GSIMs.
"""
hdf5 = datastore.hdf5new()
req_site_params = {'vs30'}
fields = ['lon', 'lat', 'vs30']
if z1pt0:
req_site_params.add('z1pt0')
fields.append('z1pt0')
if z2pt5:
req_site_params.add('z2pt5')
fields.append('z2pt5')
if vs30measured:
req_site_params.add('vs30measured')
fields.append('vs30measured')
with performance.Monitor(measuremem=True) as mon:
if exposure_xml:
mesh, assets_by_site = Exposure.read(
exposure_xml, check_dupl=False).get_mesh_assets_by_site()
hdf5['assetcol'] = assetcol = site.SiteCollection.from_points(
mesh.lons, mesh.lats, req_site_params=req_site_params)
if grid_spacing:
grid = mesh.get_convex_hull().dilate(grid_spacing).discretize(
grid_spacing)
haz_sitecol = site.SiteCollection.from_points(
grid.lons, grid.lats, req_site_params=req_site_params)
logging.info(
'Associating exposure grid with %d locations to %d '
'exposure sites', len(haz_sitecol), len(assets_by_site))
haz_sitecol, assets_by, discarded = assoc(
assets_by_site, haz_sitecol, grid_spacing * SQRT2,
'filter')
if len(discarded):
logging.info(
'Discarded %d sites with assets '
'[use oq plot_assets]', len(discarded))
hdf5['discarded'] = numpy.array(discarded)
haz_sitecol.make_complete()
else:
haz_sitecol = assetcol
discarded = []
elif sites_csv:
lons, lats = [], []
for fname in sites_csv:
check_fname(fname, 'sites_csv', output)
with read(fname) as csv:
for line in csv:
if line.startswith('lon,lat'): # possible header
continue
lon, lat = line.split(',')[:2]
lons.append(valid.longitude(lon))
lats.append(valid.latitude(lat))
haz_sitecol = site.SiteCollection.from_points(
lons, lats, req_site_params=req_site_params)
if grid_spacing:
grid = haz_sitecol.mesh.get_convex_hull().dilate(
grid_spacing).discretize(grid_spacing)
haz_sitecol = site.SiteCollection.from_points(
grid.lons, grid.lats, req_site_params=req_site_params)
else:
raise RuntimeError('Missing exposures or missing sites')
vs30 = associate(haz_sitecol, vs30_csv, assoc_distance)
if z1pt0:
haz_sitecol.array['z1pt0'] = calculate_z1pt0(vs30['vs30'])
if z2pt5:
haz_sitecol.array['z2pt5'] = calculate_z2pt5_ngaw2(vs30['vs30'])
hdf5['sitecol'] = haz_sitecol
writers.write_csv(output, haz_sitecol.array[fields])
logging.info('Saved %d rows in %s' % (len(haz_sitecol), output))
logging.info(mon)
return haz_sitecol
def export_disagg_csv(ekey, dstore):
oq = dstore['oqparam']
sitecol = dstore['sitecol']
hmap4 = dstore['hmap4']
rlzs = dstore['full_lt'].get_realizations()
best_rlzs = dstore['best_rlzs'][:]
N, M, P, Z = hmap4.shape
imts = list(oq.imtls)
fnames = []
bins = {name: dset[:] for name, dset in dstore['disagg-bins'].items()}
ex = 'disagg?kind=%s&imt=%s&site_id=%s&poe_id=%d'
if ekey[0] == 'disagg_traditional':
ex += '&traditional=1'
trad = '-traditional'
else:
trad = ''
skip_keys = ('Mag', 'Dist', 'Lon', 'Lat', 'Eps', 'TRT')
metadata = dstore.metadata
poes_disagg = ['nan'] * P
for p in range(P):
try:
poes_disagg[p] = str(oq.poes_disagg[p])
except IndexError:
pass
for s in range(N):
rlzcols = ['rlz%d' % r for r in best_rlzs[s]]
lon, lat = sitecol.lons[s], sitecol.lats[s]
weights = numpy.array([rlzs[r].weight['weight'] for r in best_rlzs[s]])
weights /= weights.sum() # normalize to 1
metadata.update(investigation_time=oq.investigation_time,
mag_bin_edges=bins['Mag'].tolist(),
dist_bin_edges=bins['Dist'].tolist(),
lon_bin_edges=bins['Lon'][s].tolist(),
lat_bin_edges=bins['Lat'][s].tolist(),
eps_bin_edges=bins['Eps'].tolist(),
tectonic_region_types=decode(bins['TRT'].tolist()),
rlz_ids=best_rlzs[s].tolist(),
weights=weights.tolist(),
lon=lon, lat=lat)
for k in oq.disagg_outputs:
splits = k.lower().split('_')
header = ['imt', 'poe'] + splits + rlzcols
values = []
nonzeros = []
for m, p in iproduct(M, P):
imt = imts[m]
aw = extract(dstore, ex % (k, imt, s, p))
# for instance for Mag_Dist [(mag, dist, poe0, poe1), ...]
poes = aw[:, len(splits):]
if 'trt' in header:
nonzeros.append(True)
else:
nonzeros.append(poes.any()) # nonzero poes
for row in aw:
values.append([imt, poes_disagg[p]] + list(row))
if any(nonzeros):
com = {key: value for key, value in metadata.items()
if value is not None and key not in skip_keys}
com.update(metadata)
fname = dstore.export_path('%s%s-%d.csv' % (k, trad, s))
writers.write_csv(fname, values, header=header,
comment=com, fmt='%.5E')
fnames.append(fname)
return sorted(fnames)
