def execute(self):
"""
Build statistical hazard curves from the stored PoEs
"""
if 'poes' not in self.datastore: # for short report
return
oq = self.oqparam
rlzs = self.rlzs_assoc.realizations
if len(rlzs) == 1: # no stats to compute
return {}
elif not oq.hazard_stats():
if oq.hazard_maps or oq.uniform_hazard_spectra:
raise ValueError('The job.ini says that no statistics should '
'be computed, but then there is no output!')
else:
return {}
# initialize datasets
N = len(self.sitecol)
L = len(oq.imtls.array)
attrs = dict(
__pyclass__='openquake.hazardlib.probability_map.ProbabilityMap',
sids=numpy.arange(N, dtype=numpy.uint32))
nbytes = N * L * 4 # bytes per realization (32 bit floats)
totbytes = 0
if len(rlzs) > 1:
for name, stat in oq.hazard_stats():
self.datastore.create_dset('hcurves/' + name,
F32, (N, L, 1),
attrs=attrs)
totbytes += nbytes
if 'hcurves' in self.datastore:
self.datastore.set_attrs('hcurves', nbytes=totbytes)
self.datastore.flush()
with self.monitor('sending pmaps', autoflush=True, measuremem=True):
if self.datastore.parent != ():
# workers read from the parent datastore
pgetter = calc.PmapGetter(self.datastore.parent, lazy=True)
allargs = list(self.gen_args(pgetter))
self.datastore.parent.close()
else:
# workers read from the cache
pgetter = calc.PmapGetter(self.datastore)
allargs = self.gen_args(pgetter)
ires = parallel.Starmap(self.core_task.__func__,
allargs).submit_all()
if self.datastore.parent != ():
self.datastore.parent.open() # if closed
nbytes = ires.reduce(self.save_hcurves)
return nbytes
def get_pmaps(dstore, indices):
getter = calc.PmapGetter(dstore)
getter.init()
pmaps = getter.get_pmaps(indices)
weights = dstore['csm_info'].rlzs['weight']
mean = compute_pmap_stats(pmaps, [mean_curve], weights)
return mean, pmaps
def export_hcurves_xml_json(ekey, dstore):
export_type = ekey[1]
len_ext = len(export_type) + 1
oq = dstore['oqparam']
sitemesh = get_mesh(dstore['sitecol'])
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
fnames = []
writercls = (hazard_writers.HazardCurveGeoJSONWriter
if export_type == 'geojson' else
hazard_writers.HazardCurveXMLWriter)
for kind, hcurves in calc.PmapGetter(dstore).items():
if kind.startswith('rlz-'):
rlz = rlzs_assoc.realizations[int(kind[4:])]
smlt_path = '_'.join(rlz.sm_lt_path)
gsimlt_path = rlz.gsim_rlz.uid
else:
smlt_path = ''
gsimlt_path = ''
curves = hcurves.convert(oq.imtls, len(sitemesh))
name = hazard_curve_name(dstore, ekey, kind, rlzs_assoc)
for imt in oq.imtls:
imtype, sa_period, sa_damping = from_string(imt)
fname = name[:-len_ext] + '-' + imt + '.' + export_type
data = [HazardCurve(Location(site), poes[imt])
for site, poes in zip(sitemesh, curves)]
writer = writercls(fname,
investigation_time=oq.investigation_time,
imls=oq.imtls[imt], imt=imtype,
sa_period=sa_period, sa_damping=sa_damping,
smlt_path=smlt_path, gsimlt_path=gsimlt_path)
writer.serialize(data)
fnames.append(fname)
return sorted(fnames)
def export_hcurves_csv(ekey, dstore):
"""
Exports the hazard curves into several .csv files
:param ekey: export key, i.e. a pair (datastore key, fmt)
:param dstore: datastore object
"""
oq = dstore['oqparam']
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
sitecol = dstore['sitecol']
sitemesh = get_mesh(sitecol)
key, kind, fmt = get_kkf(ekey)
fnames = []
if oq.poes:
pdic = DictArray({imt: oq.poes for imt in oq.imtls})
for kind, hcurves in calc.PmapGetter(dstore).items(kind):
fname = hazard_curve_name(dstore, (key, fmt), kind, rlzs_assoc)
comment = _comment(rlzs_assoc, kind, oq.investigation_time)
if key == 'uhs' and oq.poes and oq.uniform_hazard_spectra:
uhs_curves = calc.make_uhs(hcurves, oq.imtls, oq.poes,
len(sitemesh))
writers.write_csv(fname,
util.compose_arrays(sitemesh, uhs_curves),
comment=comment)
fnames.append(fname)
elif key == 'hmaps' and oq.poes and oq.hazard_maps:
hmap = calc.make_hmap(hcurves, oq.imtls, oq.poes)
fnames.extend(
export_hazard_csv(ekey, fname, sitemesh, hmap, pdic, comment))
elif key == 'hcurves':
fnames.extend(
export_hcurves_by_imt_csv(ekey, kind, rlzs_assoc, fname,
sitecol, hcurves, oq))
return sorted(fnames)
def extract_hazard(dstore, what):
"""
Extracts hazard curves and possibly hazard maps and/or uniform hazard
spectra. Use it as /extract/hazard/mean or /extract/hazard/rlz-0, etc
"""
oq = dstore['oqparam']
sitecol = dstore['sitecol']
yield 'sitecol', sitecol
yield 'oqparam', oq
yield 'imtls', oq.imtls
yield 'realizations', dstore['realizations'].value
yield 'checksum32', dstore['/'].attrs['checksum32']
nsites = len(sitecol)
M = len(oq.imtls)
P = len(oq.poes)
for kind, pmap in calc.PmapGetter(dstore).items(what):
for imt in oq.imtls:
key = 'hcurves/%s/%s' % (imt, kind)
arr = numpy.zeros((nsites, len(oq.imtls[imt])))
for sid in pmap:
arr[sid] = pmap[sid].array[oq.imtls.slicedic[imt], 0]
logging.info('extracting %s', key)
yield key, arr
if oq.poes:
hmap = calc.make_hmap(pmap, oq.imtls, oq.poes)
for p, poe in enumerate(oq.poes):
key = 'hmaps/poe-%s/%s' % (poe, kind)
arr = numpy.zeros((nsites, M))
idx = [m * P + p for m in range(M)]
for sid in pmap:
arr[sid] = hmap[sid].array[idx, 0]
logging.info('extracting %s', key)
yield key, arr
def test_case_1(self):
self.assert_curves_ok([
'poe-0.02-rlz-0-PGA-10.1-40.1_Mag.csv',
'poe-0.02-rlz-0-PGA-10.1-40.1_Mag_Dist.csv',
'poe-0.02-rlz-0-PGA-10.1-40.1_Lon_Lat.csv',
'poe-0.02-rlz-0-SA(0.025)-10.1-40.1_Mag.csv',
'poe-0.02-rlz-0-SA(0.025)-10.1-40.1_Mag_Dist.csv',
'poe-0.02-rlz-0-SA(0.025)-10.1-40.1_Lon_Lat.csv',
'poe-0.1-rlz-0-PGA-10.1-40.1_Mag.csv',
'poe-0.1-rlz-0-PGA-10.1-40.1_Mag_Dist.csv',
'poe-0.1-rlz-0-PGA-10.1-40.1_Lon_Lat.csv',
'poe-0.1-rlz-0-SA(0.025)-10.1-40.1_Mag.csv',
'poe-0.1-rlz-0-SA(0.025)-10.1-40.1_Mag_Dist.csv',
'poe-0.1-rlz-0-SA(0.025)-10.1-40.1_Lon_Lat.csv'
],
case_1.__file__,
fmt='csv')
# disaggregation by source group
pgetter = calc.PmapGetter(self.calc.datastore)
pgetter.init()
pmaps = []
for grp in sorted(pgetter.dstore['poes']):
pmaps.append(pgetter.get_mean(grp))
# make sure that the combination of the contributions is okay
pmap = pgetter.get_mean() # total mean map
cmap = combine(pmaps) # combination of the mean maps per source group
for sid in pmap:
numpy.testing.assert_almost_equal(pmap[sid].array, cmap[sid].array)
def export_uhs_xml(ekey, dstore):
oq = dstore['oqparam']
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
pgetter = calc.PmapGetter(dstore)
sitemesh = get_mesh(dstore['sitecol'].complete)
key, kind, fmt = get_kkf(ekey)
fnames = []
periods = [imt for imt in oq.imtls if imt.startswith('SA') or imt == 'PGA']
for kind, hcurves in pgetter.items(kind):
metadata = get_metadata(rlzs_assoc.realizations, kind)
_, periods = calc.get_imts_periods(oq.imtls)
uhs = calc.make_uhs(hcurves, oq.imtls, oq.poes, len(sitemesh))
for poe in oq.poes:
fname = hazard_curve_name(dstore, (key, fmt), kind + '-%s' % poe,
rlzs_assoc)
writer = hazard_writers.UHSXMLWriter(
fname,
periods=periods,
poe=poe,
investigation_time=oq.investigation_time,
**metadata)
data = []
for site, curve in zip(sitemesh, uhs[str(poe)]):
data.append(UHS(curve, Location(site)))
writer.serialize(data)
fnames.append(fname)
return sorted(fnames)
def get_curves(self, sid):
"""
Get all the relevant hazard curves for the given site ordinal.
Returns a dictionary rlz_id -> curve_by_imt.
"""
dic = {}
imtls = self.oqparam.imtls
pgetter = calc.PmapGetter(self.datastore, sids=numpy.array([sid]))
for rlz in self.rlzs_assoc.realizations:
try:
pmap = pgetter.get(rlz.ordinal)
except ValueError: # empty pmaps
logging.info(
'hazard curve contains all zero probabilities; '
'skipping site %d, rlz=%d', sid, rlz.ordinal)
continue
if sid not in pmap:
continue
poes = pmap[sid].convert(imtls)
for imt_str in imtls:
if all(x == 0.0 for x in poes[imt_str]):
logging.info(
'hazard curve contains all zero probabilities; '
'skipping site %d, rlz=%d, IMT=%s', sid, rlz.ordinal,
imt_str)
continue
dic[rlz.ordinal] = poes
return dic
def show(what='contents', calc_id=-1, extra=()):
"""
Show the content of a datastore (by default the last one).
"""
datadir = datastore.get_datadir()
if what == 'all': # show all
if not os.path.exists(datadir):
return
rows = []
for calc_id in datastore.get_calc_ids(datadir):
try:
ds = read(calc_id)
oq = ds['oqparam']
cmode, descr = oq.calculation_mode, oq.description
except:
# invalid datastore file, or missing calculation_mode
# and description attributes, perhaps due to a manual kill
f = os.path.join(datadir, 'calc_%s.hdf5' % calc_id)
logging.warn('Unreadable datastore %s', f)
continue
else:
rows.append((calc_id, cmode, descr.encode('utf-8')))
for row in sorted(rows, key=lambda row: row[0]): # by calc_id
print('#%d %s: %s' % row)
return
ds = read(calc_id)
# this part is experimental
if what == 'rlzs' and 'poes' in ds:
min_value = 0.01 # used in rmsep
getter = calc.PmapGetter(ds)
sitecol = ds['sitecol']
pmaps = getter.get_pmaps(sitecol.sids)
weights = [rlz.weight for rlz in getter.rlzs]
mean = stats.compute_pmap_stats(pmaps, [numpy.mean], weights)
dists = []
for rlz, pmap in zip(getter.rlzs, pmaps):
dist = rmsep(mean.array, pmap.array, min_value)
dists.append((dist, rlz))
print('Realizations in order of distance from the mean curves')
for dist, rlz in sorted(dists):
print('%s: rmsep=%s' % (rlz, dist))
elif view.keyfunc(what) in view:
print(view(what, ds))
elif what.split('/', 1)[0] in extract:
print(extract(ds, what, *extra))
elif what in ds:
obj = ds[what]
if hasattr(obj, 'value'): # an array
print(write_csv(io.BytesIO(), obj.value).decode('utf8'))
else:
print(obj)
else:
print('%s not found' % what)
ds.close()
def export_uhs_np(ekey, dstore):
oq = dstore['oqparam']
mesh = get_mesh(dstore['sitecol'])
fname = dstore.export_path('%s.%s' % ekey)
dic = {}
for kind, hcurves in calc.PmapGetter(dstore).items():
dic[kind] = calc.make_uhs(hcurves, oq.imtls, oq.poes, len(mesh))
save_np(fname, dic, mesh, investigation_time=oq.investigation_time)
return [fname]
def build_riskinputs(self, kind, eps=None, eids=None):
"""
:param kind:
kind of hazard getter, can be 'poe' or 'gmf'
:param eps:
a matrix of epsilons (or None)
:param eids:
an array of event IDs (or None)
:returns:
a list of RiskInputs objects, sorted by IMT.
"""
imtls = self.oqparam.imtls
if not set(self.oqparam.risk_imtls) & set(imtls):
rsk = ', '.join(self.oqparam.risk_imtls)
haz = ', '.join(imtls)
raise ValueError('The IMTs in the risk models (%s) are disjoint '
"from the IMTs in the hazard (%s)" % (rsk, haz))
num_tasks = self.oqparam.concurrent_tasks or 1
assets_by_site = self.assetcol.assets_by_site()
self.tagmask = self.assetcol.tagmask()
with self.monitor('building riskinputs', autoflush=True):
riskinputs = []
sid_weight_pairs = [(sid, len(assets))
for sid, assets in enumerate(assets_by_site)]
blocks = general.split_in_blocks(sid_weight_pairs,
num_tasks,
weight=operator.itemgetter(1))
dstore = self.can_read_parent()
for block in blocks:
sids = numpy.array([sid for sid, _weight in block])
reduced_assets = assets_by_site[sids]
# dictionary of epsilons for the reduced assets
reduced_eps = {}
for assets in reduced_assets:
for ass in assets:
ass.tagmask = self.tagmask[ass.ordinal]
if eps is not None and len(eps):
reduced_eps[ass.ordinal] = eps[ass.ordinal]
# build the riskinputs
if dstore is None:
dstore = self.datastore
if kind == 'poe': # hcurves, shape (R, N)
getter = calc.PmapGetter(dstore, sids)
getter.num_rlzs = self.R
else: # gmf
getter = riskinput.GmfDataGetter(dstore, sids, self.R,
eids)
if dstore is self.datastore:
# read the hazard data in the controller node
logging.info('Reading hazard')
getter.init()
ri = riskinput.RiskInput(getter, reduced_assets, reduced_eps)
if ri.weight > 0:
riskinputs.append(ri)
assert riskinputs
logging.info('Built %d risk inputs', len(riskinputs))
return riskinputs
def export_uhs_npz(ekey, dstore):
oq = dstore['oqparam']
mesh = get_mesh(dstore['sitecol'])
fname = dstore.export_path('%s.%s' % ekey)
dic = {}
for kind, hcurves in calc.PmapGetter(dstore).items():
uhs_curves = calc.make_uhs(hcurves, oq.imtls, oq.poes, len(mesh))
dic[kind] = util.compose_arrays(mesh, uhs_curves)
savez(fname, **dic)
return [fname]
def get_hcurves_and_means(dstore):
"""
Extract hcurves from the datastore and compute their means.
:returns: curves_by_rlz, mean_curves
"""
getter = calc.PmapGetter(dstore)
sitecol = dstore['sitecol']
pmaps = getter.get_pmaps(sitecol.sids)
return dict(zip(getter.rlzs, pmaps)), dstore['hcurves/mean']
def export_hmaps_npz(ekey, dstore):
oq = dstore['oqparam']
mesh = get_mesh(dstore['sitecol'])
pdic = DictArray({imt: oq.poes for imt in oq.imtls})
fname = dstore.export_path('%s.%s' % ekey)
dic = {}
for kind, hcurves in calc.PmapGetter(dstore).items():
hmap = calc.make_hmap(hcurves, oq.imtls, oq.poes)
dic[kind] = convert_to_array(hmap, mesh, pdic)
savez(fname, **dic)
return [fname]
def gen_args(self):
"""
:yields: pgetter, hstats, monitor
"""
monitor = self.monitor('build_hcurves_and_stats')
hstats = self.oqparam.hazard_stats()
for t in self.sitecol.split_in_tiles(self.oqparam.concurrent_tasks):
pgetter = calc.PmapGetter(self.datastore, t.sids, self.rlzs_assoc)
if not self.get_parent(): # read now, not in the workers
pgetter.init()
yield pgetter, hstats, monitor
def plot_hmaps(calc_id):
"""
Mean hazard maps plotter.
"""
dstore = datastore.read(calc_id)
oq = dstore['oqparam']
mean = calc.PmapGetter(dstore).get_mean()
hmaps = calc.make_hmap(mean, oq.imtls, oq.poes)
M, P = len(oq.imtls), len(oq.poes)
array = hmaps.array.reshape(len(hmaps.array), M, P)
plt = make_figure(dstore['sitecol'], oq.imtls, oq.poes, array)
plt.show()
def export_hcurves_npz(ekey, dstore):
mesh = get_mesh(dstore['sitecol'])
imtls = dstore['oqparam'].imtls
fname = dstore.export_path('%s.%s' % ekey)
arr = numpy.zeros(1, imtls.dt)
for imt in imtls:
arr[imt] = imtls[imt]
dic = dict(imtls=arr[0])
for kind, hcurves in calc.PmapGetter(dstore).items():
curves = hcurves.convert(imtls, len(mesh))
dic[kind] = util.compose_arrays(mesh, curves)
savez(fname, **dic)
return [fname]
def view_flat_hcurves(token, dstore):
"""
Display the flat hazard curves for the calculation. They are
used for debugging purposes when comparing the results of two
calculations. They are the mean over the sites of the mean hazard
curves.
"""
oq = dstore['oqparam']
nsites = len(dstore['sitecol'])
mean = calc.PmapGetter(dstore).get_mean()
array = calc.convert_to_array(mean, nsites, oq.imtls)
res = numpy.zeros(1, array.dtype)
for name in array.dtype.names:
res[name] = array[name].mean()
return rst_table(res)
def gen_args(self):
"""
:yields: pgetter, hstats, monitor
"""
monitor = self.monitor('build_hcurves_and_stats')
hstats = self.oqparam.hazard_stats()
parent = self.can_read_parent()
if parent is None:
parent = self.datastore
for t in self.sitecol.split_in_tiles(self.oqparam.concurrent_tasks):
pgetter = calc.PmapGetter(parent, t.sids, self.rlzs_assoc)
if parent is self.datastore: # read now, not in the workers
logging.info('Reading PoEs on %d sites', len(t))
pgetter.init()
yield pgetter, hstats, monitor
def export_hmaps_np(ekey, dstore):
oq = dstore['oqparam']
sitecol = dstore['sitecol']
mesh = get_mesh(sitecol)
pdic = DictArray({imt: oq.poes for imt in oq.imtls})
fname = dstore.export_path('%s.%s' % ekey)
dic = {}
for kind, hcurves in calc.PmapGetter(dstore).items():
hmap = calc.make_hmap(hcurves, oq.imtls, oq.poes)
dic[kind] = calc.convert_to_array(hmap, len(mesh), pdic)
save_np(fname,
dic,
mesh, ('vs30', F32, sitecol.vs30),
investigation_time=oq.investigation_time)
return [fname]
def execute(self):
"""
Build statistical hazard curves from the stored PoEs
"""
if 'poes' not in self.datastore: # for short report
return
oq = self.oqparam
num_rlzs = len(self.datastore['realizations'])
if num_rlzs == 1: # no stats to compute
return {}
elif not oq.hazard_stats():
if oq.hazard_maps or oq.uniform_hazard_spectra:
raise ValueError('The job.ini says that no statistics should '
'be computed, but then there is no output!')
else:
return {}
# initialize datasets
N = len(self.sitecol)
L = len(oq.imtls.array)
attrs = dict(
__pyclass__='openquake.hazardlib.probability_map.ProbabilityMap',
sids=numpy.arange(N, dtype=numpy.uint32))
nbytes = N * L * 4 # bytes per realization (32 bit floats)
totbytes = 0
if num_rlzs > 1:
for name, stat in oq.hazard_stats():
self.datastore.create_dset('hcurves/' + name,
F32, (N, L, 1),
attrs=attrs)
totbytes += nbytes
if 'hcurves' in self.datastore:
self.datastore.set_attrs('hcurves', nbytes=totbytes)
self.datastore.flush()
with self.monitor('sending pmaps', autoflush=True, measuremem=True):
monitor = self.monitor('build_hcurves_and_stats')
hstats = oq.hazard_stats()
allargs = (
(calc.PmapGetter(self.datastore, tile.sids,
self.rlzs_assoc), hstats, monitor)
for tile in self.sitecol.split_in_tiles(oq.concurrent_tasks))
ires = parallel.Starmap(self.core_task.__func__,
allargs).submit_all()
nbytes = ires.reduce(self.save_hcurves)
return nbytes
def plot_uhs(calc_id, sites='0'):
"""
UHS plotter.
"""
# read the hazard data
dstore = datastore.read(calc_id)
getter = calc.PmapGetter(dstore)
oq = dstore['oqparam']
indices = list(map(int, sites.split(',')))
n_sites = len(dstore['sitecol'])
if not set(indices) <= set(range(n_sites)):
invalid = sorted(set(indices) - set(range(n_sites)))
print('The indices %s are invalid: no graph for them' % invalid)
valid = sorted(set(range(n_sites)) & set(indices))
print('Found %d site(s); plotting %d of them' % (n_sites, len(valid)))
pmaps = getter.get_pmaps(numpy.array(indices))
plt = make_figure(valid, n_sites, oq.imtls, oq.poes, pmaps)
plt.show()
def view_flat_hmaps(token, dstore):
"""
Display the flat hazard maps for the calculation. They are
used for debugging purposes when comparing the results of two
calculations. They are the mean over the sites of the mean hazard
maps.
"""
oq = dstore['oqparam']
assert oq.poes
nsites = len(dstore['sitecol'])
pdic = DictArray({imt: oq.poes for imt in oq.imtls})
mean = calc.PmapGetter(dstore).get_mean()
hmaps = calc.make_hmap(mean, oq.imtls, oq.poes)
array = calc.convert_to_array(hmaps, nsites, pdic)
res = numpy.zeros(1, array.dtype)
for name in array.dtype.names:
res[name] = array[name].mean()
return rst_table(res)
def pre_execute(self):
"""
Associate the assets to the sites and build the riskinputs.
"""
oq = self.oqparam
if oq.insured_losses:
raise ValueError(
'insured_losses are not supported for classical_risk')
if 'hazard_curves' in oq.inputs: # read hazard from file
haz_sitecol, haz_curves = readinput.get_hcurves(oq)
self.save_params()
self.read_exposure() # define .assets_by_site
self.load_riskmodel()
self.sitecol, self.assetcol = self.assoc_assets_sites(haz_sitecol)
self.datastore['csm_info'] = fake = source.CompositionInfo.fake()
self.rlzs_assoc = fake.get_rlzs_assoc()
rlzs = self.rlzs_assoc.realizations
curves = {rlzs[0]: haz_curves} # there is one realization
else: # compute hazard or read it from the datastore
super(ClassicalRiskCalculator, self).pre_execute()
if 'poes' not in self.datastore: # when building short report
return
logging.info('Combining the hazard curves')
pgetter = calc.PmapGetter(self.datastore)
sids = self.sitecol.complete.sids
with self.monitor('combining hcurves',
measuremem=True,
autoflush=True):
pmaps = pgetter.get_pmaps(sids)
rlzs = self.rlzs_assoc.realizations
curves = {
rlz: pmap.convert(oq.imtls, len(sids))
for rlz, pmap in zip(rlzs, pmaps)
}
with self.monitor('build riskinputs', measuremem=True, autoflush=True):
self.riskinputs = self.build_riskinputs('poe', curves)
self.param = dict(insured_losses=oq.insured_losses,
stats=oq.risk_stats())
self.N = len(self.assetcol)
self.L = len(self.riskmodel.loss_types)
self.R = len(self.rlzs_assoc.realizations)
self.I = oq.insured_losses
self.S = len(oq.risk_stats())
def export_hmaps_xml_json(ekey, dstore):
key, kind, fmt = get_kkf(ekey)
if fmt == 'geojson':
logging.warn('The geojson exporters will be removed soon')
oq = dstore['oqparam']
sitecol = dstore['sitecol']
sitemesh = get_mesh(sitecol)
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
fnames = []
writercls = (hazard_writers.HazardMapGeoJSONWriter
if fmt == 'geojson' else hazard_writers.HazardMapXMLWriter)
pdic = DictArray({imt: oq.poes for imt in oq.imtls})
nsites = len(sitemesh)
for kind, hcurves in calc.PmapGetter(dstore).items():
hmaps = calc.make_hmap(hcurves, oq.imtls,
oq.poes).convert(pdic, nsites)
if kind.startswith('rlz-'):
rlz = rlzs_assoc.realizations[int(kind[4:])]
smlt_path = '_'.join(rlz.sm_lt_path)
gsimlt_path = rlz.gsim_rlz.uid
else:
smlt_path = ''
gsimlt_path = ''
for imt in oq.imtls:
for j, poe in enumerate(oq.poes):
suffix = '-%s-%s' % (poe, imt)
fname = hazard_curve_name(dstore, ekey, kind + suffix,
rlzs_assoc)
data = [
HazardMap(site[0], site[1], _extract(hmap, imt, j))
for site, hmap in zip(sitemesh, hmaps)
]
writer = writercls(fname,
investigation_time=oq.investigation_time,
imt=imt,
poe=poe,
smlt_path=smlt_path,
gsimlt_path=gsimlt_path)
writer.serialize(data)
fnames.append(fname)
return sorted(fnames)
def extract_hazard_for_qgis(dstore, what):
"""
Extracts hazard curves and possibly hazard maps and/or uniform hazard
spectra. Use it as /extract/qgis-hazard/rlz-0, etc
"""
oq = dstore['oqparam']
sitecol = dstore['sitecol']
yield 'sitecol', sitecol
yield 'oqparam', oq
yield 'realizations', dstore['realizations'].value
yield 'checksum32', dstore['/'].attrs['checksum32']
N = len(sitecol)
if oq.poes:
pdic = {imt: oq.poes for imt in oq.imtls}
for kind, hcurves in calc.PmapGetter(dstore).items(what):
logging.info('extracting hazard/%s', kind)
yield 'hcurves-' + kind, calc.convert_to_array(hcurves, N, oq.imtls)
if oq.poes and oq.uniform_hazard_spectra:
yield 'uhs-' + kind, calc.make_uhs(hcurves, oq.imtls, oq.poes, N)
if oq.poes and oq.hazard_maps:
hmaps = calc.make_hmap(hcurves, oq.imtls, oq.poes)
yield 'hmaps-' + kind, calc.convert_to_array(hmaps, N, pdic)
def export_hcurves_rlzs(ekey, dstore):
"""
Export all hazard curves in a single .hdf5 file. This is not
recommended, even if this exporter is parallel and very efficient.
I was able to export 6 GB of curves per minute. However for large
calculations it is then impossible to view the .hdf5 file with the
hdfviewer because you will run out of memory. Also, compression is not
enabled, otherwise all the time will be spent in the compression phase
in the controller node with the workers doing nothing.
The recommended way to postprocess large computations is to instantiate
the PmapGetter and to work one block of sites at the time,
discarding what it is not needed. The exporter here is meant for
small/medium calculation and as an example of what you should
implement yourself if you need to postprocess the hazard curves.
"""
oq = dstore['oqparam']
imtls = oq.imtls
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
sitecol = dstore['sitecol']
pgetter = calc.PmapGetter(dstore, rlzs_assoc)
N = len(sitecol)
R = len(rlzs_assoc.realizations)
fname = dstore.export_path('%s.%s' % ekey)
monitor = performance.Monitor(ekey[0], fname)
size = humansize(dstore.get_attr('poes', 'nbytes'))
logging.info('Reading %s of probability maps', size)
allargs = [(pgetter.new(tile.sids), imtls, monitor)
for tile in sitecol.split_in_tiles(R)]
with hdf5.File(fname, 'w') as f:
f['imtls'] = imtls
dset = f.create_dataset('hcurves-rlzs', (N, R), imtls.dt)
dset.attrs['investigation_time'] = oq.investigation_time
logging.info('Building the hazard curves for %d sites, %d rlzs', N, R)
for sids, allcurves in parallel.Processmap(build_hcurves, allargs):
for sid, curves in zip(sids, allcurves):
dset[sid] = curves
return [fname]
def get_pmaps(dstore, indices):
getter = calc.PmapGetter(dstore)
pmaps = getter.get_pmaps(indices)
weights = dstore['realizations']['weight']
mean = compute_pmap_stats(pmaps, [mean_curve], weights)
return mean, pmaps
def get_pmaps(dstore, indices):
getter = calc.PmapGetter(dstore)
pmaps = getter.get_pmaps(indices)
weights = [rlz.weight for rlz in getter.rlzs]
mean = compute_pmap_stats(pmaps, [mean_curve], weights)
return mean, pmaps
