def extract_uhs(dstore, what):
"""
Extracts uniform hazard spectra. Use it as /extract/uhs/mean or
/extract/uhs/rlz-0, etc
"""
oq = dstore['oqparam']
mesh = get_mesh(dstore['sitecol'])
dic = {}
for kind, hcurves in getters.PmapGetter(dstore).items(what):
dic[kind] = calc.make_uhs(hcurves, oq.imtls, oq.poes, len(mesh))
return hazard_items(dic, mesh, investigation_time=oq.investigation_time)
def extract_mean_std_curves(dstore, what):
"""
Yield imls/IMT and poes/IMT containg mean and stddev for all sites
"""
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
w = [rlz.weight for rlz in rlzs_assoc.realizations]
getter = getters.PmapGetter(dstore, w)
arr = getter.get_mean().array
for imt in getter.imtls:
yield 'imls/' + imt, getter.imtls[imt]
yield 'poes/' + imt, arr[:, getter.imtls(imt)]
def view_pmap(token, dstore):
"""
Display the mean ProbabilityMap associated to a given source group name
"""
grp = token.split(':')[1] # called as pmap:grp
pmap = {}
rlzs = dstore['full_lt'].get_realizations()
weights = [rlz.weight for rlz in rlzs]
pgetter = getters.PmapGetter(dstore, weights)
pmap = pgetter.get_mean(grp)
return str(pmap)
def extract_hcurves(dstore, what):
"""
Extracts hazard curves. Use it as /extract/hcurves/mean or
/extract/hcurves/rlz-0, /extract/hcurves/stats, /extract/hcurves/rlzs etc
"""
oq = dstore['oqparam']
sitecol = dstore['sitecol']
mesh = get_mesh(sitecol, complete=False)
dic = {}
for kind, hcurves in getters.PmapGetter(dstore).items(what):
dic[kind] = hcurves.convert_npy(oq.imtls, sitecol.sids)
return hazard_items(dic, mesh, investigation_time=oq.investigation_time)
def view_pmap(token, dstore):
"""
Display the mean ProbabilityMap associated to a given source group name
"""
name = token.split(':')[1] # called as pmap:name
pmap = {}
pgetter = getters.PmapGetter(dstore)
for grp, dset in dstore['poes'].items():
if dset.attrs['name'] == name:
pmap = pgetter.get_mean(grp)
break
return str(pmap)
def gen_getters(self, parent):
"""
:yields: pgetter, hstats, monitor
"""
monitor = self.monitor('build_hazard_stats')
hstats = self.oqparam.hazard_stats()
for t in self.sitecol.split_in_tiles(self.oqparam.concurrent_tasks):
pgetter = getters.PmapGetter(parent, self.rlzs_assoc, t.sids)
if parent is self.datastore: # read now, not in the workers
logging.info('Reading PoEs on %d sites', len(t))
pgetter.init()
yield pgetter, hstats, monitor
def plot_hmaps(calc_id):
"""
Mean hazard maps plotter.
"""
dstore = datastore.read(calc_id)
oq = dstore['oqparam']
mean = getters.PmapGetter(dstore).get_mean()
hmaps = calc.make_hmap(mean, oq.imtls, oq.poes)
M, P = len(oq.imtls), len(oq.poes)
array = hmaps.array.reshape(len(hmaps.array), M, P)
plt = make_figure(dstore['sitecol'], oq.imtls, oq.poes, array)
plt.show()
def plot_hmaps(calc_id):
"""
Mean hazard maps plotter.
"""
dstore = engine.read(calc_id)
oq = dstore['oqparam']
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
mean = getters.PmapGetter(dstore, rlzs_assoc).get_mean()
hmaps = calc.make_hmap(mean, oq.imtls, oq.poes)
M, P = len(oq.imtls), len(oq.poes)
array = hmaps.array.reshape(len(hmaps.array), M, P)
plt = make_figure(dstore['sitecol'], oq.imtls, oq.poes, array)
plt.show()
def execute(self):
"""
Run in parallel `core_task(sources, sitecol, monitor)`, by
parallelizing on the sources according to their weight and
tectonic region type.
"""
oq = self.oqparam
if oq.hazard_calculation_id and not oq.compare_with_classical:
with datastore.read(self.oqparam.hazard_calculation_id) as parent:
self.full_lt = parent['full_lt']
self.calc_stats() # post-processing
return {}
assert oq.max_sites_per_tile > oq.max_sites_disagg, (
oq.max_sites_per_tile, oq.max_sites_disagg)
psd = self.set_psd() # must go before to set the pointsource_distance
run_preclassical(self.csm, oq, self.datastore)
# exit early if we want to perform only a preclassical
if oq.calculation_mode == 'preclassical':
recs = [tuple(row) for row in self.csm.source_info.values()]
self.datastore['source_info'] = numpy.array(
recs, readinput.source_info_dt)
self.datastore['full_lt'] = self.csm.full_lt
self.datastore.swmr_on() # fixes HDF5 error in build_hazard
return
self.create_dsets() # create the rup/ datasets BEFORE swmr_on()
grp_ids = numpy.arange(len(self.csm.src_groups))
self.calc_times = AccumDict(accum=numpy.zeros(3, F32))
weights = [rlz.weight for rlz in self.realizations]
pgetter = getters.PmapGetter(self.datastore, weights,
self.sitecol.sids, oq.imtls)
srcidx = {
rec[0]: i
for i, rec in enumerate(self.csm.source_info.values())
}
self.haz = Hazard(self.datastore, self.full_lt, pgetter, srcidx)
args = self.get_args(grp_ids, self.haz)
logging.info('Sending %d tasks', len(args))
smap = parallel.Starmap(classical, args, h5=self.datastore.hdf5)
smap.monitor.save('srcfilter', self.src_filter())
self.datastore.swmr_on()
smap.h5 = self.datastore.hdf5
pmaps = smap.reduce(self.agg_dicts)
logging.debug("busy time: %s", smap.busytime)
self.haz.store_disagg(pmaps)
if not oq.hazard_calculation_id:
self.haz.store_disagg()
self.store_info(psd)
return True
def _gen_riskinputs_poe(self, dstore):
assets_by_site = self.assetcol.assets_by_site()
for sid, assets in enumerate(assets_by_site):
if len(assets) == 0:
continue
# hcurves, shape (R, N)
ws = [rlz.weight for rlz in self.realizations]
getter = getters.PmapGetter(dstore, ws, [sid], self.oqparam.imtls)
for block in general.block_splitter(
assets, self.oqparam.assets_per_site_limit):
yield riskinput.RiskInput(sid, getter, numpy.array(block))
if len(block) >= TWO16:
logging.error('There are %d assets on site #%d!', len(block),
sid)
def extract_hmaps(dstore, what):
"""
Extracts hazard maps. Use it as /extract/hmaps/mean or
/extract/hmaps/rlz-0, etc
"""
oq = dstore['oqparam']
sitecol = dstore['sitecol']
mesh = get_mesh(sitecol)
pdic = DictArray({imt: oq.poes for imt in oq.imtls})
dic = {}
for kind, hcurves in getters.PmapGetter(dstore).items(what):
hmap = calc.make_hmap(hcurves, oq.imtls, oq.poes)
dic[kind] = calc.convert_to_array(hmap, len(mesh), pdic)
return hazard_items(dic, mesh, investigation_time=oq.investigation_time)
def view_flat_hcurves(token, dstore):
"""
Display the flat hazard curves for the calculation. They are
used for debugging purposes when comparing the results of two
calculations. They are the mean over the sites of the mean hazard
curves.
"""
oq = dstore['oqparam']
nsites = len(dstore['sitecol'])
mean = getters.PmapGetter(dstore).get_mean()
array = calc.convert_to_array(mean, nsites, oq.imtls)
res = numpy.zeros(1, array.dtype)
for name in array.dtype.names:
res[name] = array[name].mean()
return rst_table(res)
def post_execute(self, pmap_by_key):
"""
Collect the hazard curves by realization and export them.
:param pmap_by_key:
a dictionary key -> hazard curves
"""
nr = {
name: len(dset['mag'])
for name, dset in self.datastore.items() if name.startswith('rup_')
}
if nr: # few sites, log the number of ruptures per magnitude
logging.info('%s', nr)
oq = self.oqparam
et_ids = self.datastore['et_ids'][:]
rlzs_by_gsim_list = self.full_lt.get_rlzs_by_gsim_list(et_ids)
slice_by_g = getters.get_slice_by_g(rlzs_by_gsim_list)
data = []
weights = [rlz.weight for rlz in self.realizations]
pgetter = getters.PmapGetter(self.datastore, weights,
self.sitecol.sids, oq.imtls)
logging.info('Saving _poes')
enum = enumerate(self.datastore['source_info']['source_id'])
srcid = {source_id: i for i, source_id in enum}
with self.monitor('saving probability maps'):
for key, pmap in pmap_by_key.items():
if isinstance(key, str): # disagg_by_src
rlzs_by_gsim = rlzs_by_gsim_list[pmap.grp_id]
self.datastore['disagg_by_src'][..., srcid[key]] = (
pgetter.get_hcurves(pmap, rlzs_by_gsim))
elif pmap: # pmap can be missing if the group is filtered away
# key is the group ID
trt = self.full_lt.trt_by_et[et_ids[key][0]]
# avoid saving PoEs == 1
base.fix_ones(pmap)
sids = sorted(pmap)
arr = numpy.array([pmap[sid].array for sid in sids])
self.datastore['_poes'][sids, :, slice_by_g[key]] = arr
extreme = max(
get_extreme_poe(pmap[sid].array, oq.imtls)
for sid in pmap)
data.append((key, trt, extreme))
if oq.hazard_calculation_id is None and '_poes' in self.datastore:
self.datastore['disagg_by_grp'] = numpy.array(
sorted(data), grp_extreme_dt)
self.datastore.swmr_on() # needed
self.calc_stats()
def post_execute(self, pmap_by_key):
"""
Collect the hazard curves by realization and export them.
:param pmap_by_key:
a dictionary key -> hazard curves
"""
nr = {
name: len(dset['mag'])
for name, dset in self.datastore.items() if name.startswith('rup_')
}
if nr: # few sites, log the number of ruptures per magnitude
logging.info('%s', nr)
oq = self.oqparam
if oq.calculation_mode.endswith(('risk', 'damage', 'bcr')):
with hdf5.File(self.datastore.tempname, 'a') as cache:
cache['oqparam'] = oq
cache['rlzs_by_grp'] = self.full_lt.get_rlzs_by_grp()
data = []
weights = [rlz.weight for rlz in self.realizations]
pgetter = getters.PmapGetter(self.datastore, weights,
self.sitecol.sids, oq.imtls)
with self.monitor('saving probability maps'):
for key, pmap in pmap_by_key.items():
if isinstance(key, str): # disagg_by_src
serial = self.csm.source_info[key][readinput.SERIAL]
self.datastore['disagg_by_src'][..., serial] = (
pgetter.get_hcurves(
{'grp-%02d' % gid: pmap[gid]
for gid in pmap}))
elif pmap: # pmap can be missing if the group is filtered away
# key is the group ID
base.fix_ones(pmap) # avoid saving PoEs == 1
trt = self.full_lt.trt_by_grp[key]
name = 'poes/grp-%02d' % key
self.datastore[name] = pmap
if oq.calculation_mode.endswith(('risk', 'damage', 'bcr')):
with hdf5.File(self.datastore.tempname, 'a') as cache:
cache[name] = pmap
extreme = max(
get_extreme_poe(pmap[sid].array, oq.imtls)
for sid in pmap)
data.append((key, trt, extreme))
if oq.hazard_calculation_id is None and 'poes' in self.datastore:
self.datastore['disagg_by_grp'] = numpy.array(
sorted(data), grp_extreme_dt)
self.calc_stats()
def view_global_hcurves(token, dstore):
"""
Display the global hazard curves for the calculation. They are
used for debugging purposes when comparing the results of two
calculations. They are the mean over the sites of the mean hazard
curves.
"""
oq = dstore['oqparam']
nsites = len(dstore['sitecol'])
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
weights = [rlz.weight for rlz in rlzs_assoc.realizations]
mean = getters.PmapGetter(dstore, rlzs_assoc, weights).get_mean()
array = calc.convert_to_array(mean, nsites, oq.imtls)
res = numpy.zeros(1, array.dtype)
for name in array.dtype.names:
res[name] = array[name].mean()
return rst_table(res)
def view_flat_hmaps(token, dstore):
"""
Display the flat hazard maps for the calculation. They are
used for debugging purposes when comparing the results of two
calculations. They are the mean over the sites of the mean hazard
maps.
"""
oq = dstore['oqparam']
assert oq.poes
nsites = len(dstore['sitecol'])
pdic = DictArray({imt: oq.poes for imt in oq.imtls})
mean = getters.PmapGetter(dstore).get_mean()
hmaps = calc.make_hmap(mean, oq.imtls, oq.poes)
array = calc.convert_to_array(hmaps, nsites, pdic)
res = numpy.zeros(1, array.dtype)
for name in array.dtype.names:
res[name] = array[name].mean()
return rst_table(res)
def plot_uhs(calc_id, sites='0'):
"""
UHS plotter.
"""
# read the hazard data
dstore = datastore.read(calc_id)
getter = getters.PmapGetter(dstore)
getter.init()
oq = dstore['oqparam']
indices = list(map(int, sites.split(',')))
n_sites = len(dstore['sitecol'])
if not set(indices) <= set(range(n_sites)):
invalid = sorted(set(indices) - set(range(n_sites)))
print('The indices %s are invalid: no graph for them' % invalid)
valid = sorted(set(range(n_sites)) & set(indices))
print('Found %d site(s); plotting %d of them' % (n_sites, len(valid)))
pmaps = getter.get_pmaps(numpy.array(indices))
plt = make_figure(valid, n_sites, oq.imtls, oq.poes, pmaps)
plt.show()
def plot_uhs(calc_id, sites='0'):
"""
UHS plotter.
"""
# read the hazard data
dstore = util.read(calc_id)
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
indices = list(map(int, sites.split(',')))
getter = getters.PmapGetter(dstore, rlzs_assoc, indices)
getter.init()
oq = dstore['oqparam']
n_sites = len(dstore['sitecol'])
if not set(indices) <= set(range(n_sites)):
invalid = sorted(set(indices) - set(range(n_sites)))
print('The indices %s are invalid: no graph for them' % invalid)
valid = sorted(set(range(n_sites)) & set(indices))
print('Found %d site(s); plotting %d of them' % (n_sites, len(valid)))
plt = make_figure(valid, n_sites, oq, dstore['hmaps'])
plt.show()
def extract_uhs(dstore, what):
"""
Extracts uniform hazard spectra. Use it as /extract/uhs/mean or
/extract/uhs/rlz-0, etc
"""
oq = dstore['oqparam']
imts = oq.imt_periods()
mesh = get_mesh(dstore['sitecol'])
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
dic = {}
imts_dt = numpy.dtype([(str(imt), F32) for imt in imts])
uhs_dt = numpy.dtype([(str(poe), imts_dt) for poe in oq.poes])
for name, hcurves in getters.PmapGetter(dstore, rlzs_assoc).items(what):
hmap = calc.make_hmap_array(hcurves, oq.imtls, oq.poes, len(mesh))
uhs = numpy.zeros(len(hmap), uhs_dt)
for field in hmap.dtype.names:
imt, poe = field.split('-')
if imt in imts_dt.names:
uhs[poe][imt] = hmap[field]
dic[name] = uhs
return hazard_items(dic, mesh, investigation_time=oq.investigation_time)
def extract_hazard_for_qgis(dstore, what):
"""
Extracts hazard curves and possibly hazard maps and/or uniform hazard
spectra. Use it as /extract/qgis-hazard/rlz-0, etc
"""
oq = dstore['oqparam']
sitecol = dstore['sitecol']
yield 'sitecol', sitecol
yield 'oqparam', oq
yield 'realizations', dstore['csm_info'].rlzs
yield 'checksum32', dstore['/'].attrs['checksum32']
N = len(sitecol)
if oq.poes:
pdic = {imt: oq.poes for imt in oq.imtls}
for kind, hcurves in getters.PmapGetter(dstore).items(what):
logging.info('extracting hazard/%s', kind)
yield 'hcurves-' + kind, calc.convert_to_array(hcurves, N, oq.imtls)
if oq.poes and oq.uniform_hazard_spectra:
yield 'uhs-' + kind, calc.make_uhs(hcurves, oq.imtls, oq.poes, N)
if oq.poes and oq.hazard_maps:
hmaps = calc.make_hmap(hcurves, oq.imtls, oq.poes)
yield 'hmaps-' + kind, calc.convert_to_array(hmaps, N, pdic)
def build_disagg_by_src(self, iml2s):
"""
:param dstore: a datastore
:param iml2s: N arrays of IMLs with shape (M, P)
"""
logging.warning('Disaggregation by source is experimental')
oq = self.oqparam
poes_disagg = oq.poes_disagg or (None,)
by_grp = self.rlzs_assoc.by_grp()
ws = [rlz.weight for rlz in self.rlzs_assoc.realizations]
pmap_by_grp = getters.PmapGetter(
self.datastore, by_grp, ws, self.sitecol.sids).pmap_by_grp
grp_ids = numpy.array(sorted(int(grp[4:]) for grp in pmap_by_grp))
G = len(pmap_by_grp)
P = len(poes_disagg)
for rec in self.sitecol.array:
sid = rec['sids']
iml2 = iml2s[sid]
for imti, imt in enumerate(oq.imtls):
xs = oq.imtls[imt]
poes = numpy.zeros((G, P))
for g, grp_id in enumerate(grp_ids):
pmap = pmap_by_grp['grp-%02d' % grp_id]
if sid in pmap:
ys = pmap[sid].array[oq.imtls(imt), 0]
poes[g] = numpy.interp(iml2[imti, :], xs, ys)
for p, poe in enumerate(poes_disagg):
prefix = ('iml-%s' % oq.iml_disagg[imt] if poe is None
else 'poe-%s' % poe)
name = 'disagg_by_src/%s-%s-%s-%s' % (
prefix, imt, rec['lon'], rec['lat'])
if poes[:, p].sum(): # nonzero contribution
poe_agg = 1 - numpy.prod(1 - poes[:, p])
if poe and abs(1 - poe_agg / poe) > .1:
logging.warning(
'poe_agg=%s is quite different from '
'the expected poe=%s', poe_agg, poe)
self.datastore[name] = poes[:, p]
self.datastore.set_attrs(name, poe_agg=poe_agg)
def calc_stats(self):
oq = self.oqparam
hstats = oq.hazard_stats()
# initialize datasets
N = len(self.sitecol.complete)
P = len(oq.poes)
M = len(oq.imtls)
if oq.soil_intensities is not None:
L = M * len(oq.soil_intensities)
else:
L = len(oq.imtls.array)
R = len(self.rlzs_assoc.realizations)
S = len(hstats)
if R > 1 and oq.individual_curves or not hstats:
self.datastore.create_dset('hcurves-rlzs', F32, (N, R, L))
if oq.poes:
self.datastore.create_dset('hmaps-rlzs', F32, (N, R, M, P))
if hstats:
self.datastore.create_dset('hcurves-stats', F32, (N, S, L))
if oq.poes:
self.datastore.create_dset('hmaps-stats', F32, (N, S, M, P))
ct = oq.concurrent_tasks
logging.info('Building hazard statistics with %d concurrent_tasks', ct)
weights = [rlz.weight for rlz in self.rlzs_assoc.realizations]
if 'amplification' in oq.inputs:
amplifier = Amplifier(oq.imtls, self.datastore['amplification'],
oq.soil_intensities)
amplifier.check(self.sitecol.vs30, oq.vs30_tolerance)
else:
amplifier = None
allargs = [ # this list is very fast to generate
(getters.PmapGetter(self.datastore, weights, t.sids, oq.poes), N,
hstats, oq.individual_curves, oq.max_sites_disagg, amplifier)
for t in self.sitecol.split_in_tiles(ct)
]
self.datastore.swmr_on()
parallel.Starmap(build_hazard, allargs,
h5=self.datastore.hdf5).reduce(self.save_hazard)
def test_case_1(self):
out = self.assert_curves_ok([
'rlz-0-PGA-sid-0-poe-0_Lon_Lat.csv',
'rlz-0-PGA-sid-0-poe-0_Mag.csv',
'rlz-0-PGA-sid-0-poe-0_Mag_Dist.csv',
'rlz-0-PGA-sid-0-poe-1_Lon_Lat.csv',
'rlz-0-PGA-sid-0-poe-1_Mag.csv',
'rlz-0-PGA-sid-0-poe-1_Mag_Dist.csv',
'rlz-0-SA(0.025)-sid-0-poe-0_Lon_Lat.csv',
'rlz-0-SA(0.025)-sid-0-poe-0_Mag.csv',
'rlz-0-SA(0.025)-sid-0-poe-0_Mag_Dist.csv',
'rlz-0-SA(0.025)-sid-0-poe-1_Lon_Lat.csv',
'rlz-0-SA(0.025)-sid-0-poe-1_Mag.csv',
'rlz-0-SA(0.025)-sid-0-poe-1_Mag_Dist.csv'
],
case_1.__file__,
fmt='csv')
# check disagg_by_src, poe=0.02, 0.1, imt=PGA, SA(0.025)
#self.assertEqual(len(out['disagg_by_src', 'csv']), 4)
#for fname in out['disagg_by_src', 'csv']:
# self.assertEqualFiles('expected_output/%s' % strip_calc_id(fname),
# fname)
# disaggregation by source group
rlzs_assoc = self.calc.datastore['csm_info'].get_rlzs_assoc()
ws = [rlz.weight for rlz in rlzs_assoc.realizations]
pgetter = getters.PmapGetter(self.calc.datastore, ws)
pgetter.init()
pmaps = []
for grp in sorted(pgetter.dstore['poes']):
pmaps.append(pgetter.get_mean(grp))
# make sure that the combination of the contributions is okay
pmap = pgetter.get_mean() # total mean map
cmap = combine(pmaps) # combination of the mean maps per source group
for sid in pmap:
numpy.testing.assert_almost_equal(pmap[sid].array, cmap[sid].array)
def calc_stats(self):
oq = self.oqparam
hstats = oq.hazard_stats()
# initialize datasets
N = len(self.sitecol.complete)
P = len(oq.poes)
M = len(oq.imtls)
if oq.soil_intensities is not None:
L = M * len(oq.soil_intensities)
else:
L = len(oq.imtls.array)
R = len(self.realizations)
S = len(hstats)
if R > 1 and oq.individual_curves or not hstats:
self.datastore.create_dset('hcurves-rlzs', F32, (N, R, L))
if oq.poes:
self.datastore.create_dset('hmaps-rlzs', F32, (N, R, M, P))
if hstats:
self.datastore.create_dset('hcurves-stats', F32, (N, S, L))
if oq.poes:
self.datastore.create_dset('hmaps-stats', F32, (N, S, M, P))
ct = oq.concurrent_tasks or 1
logging.info('Building hazard statistics')
weights = [rlz.weight for rlz in self.realizations]
allargs = [ # this list is very fast to generate
(getters.PmapGetter(self.datastore, weights, t.sids, oq.poes), N,
hstats, oq.individual_curves, oq.max_sites_disagg, self.amplifier)
for t in self.sitecol.split_in_tiles(ct)
]
if N <= oq.max_sites_disagg: # few sites
dist = 'no'
else:
dist = None # parallelize as usual
self.datastore.swmr_on()
parallel.Starmap(build_hazard,
allargs,
distribute=dist,
h5=self.datastore.hdf5).reduce(self.save_hazard)
def get_getter(self, kind, sid):
"""
:param kind: 'poe' or 'gmf'
:param sid: a site ID
:returns: a PmapGetter or GmfDataGetter
"""
hdf5cache = getattr(self, 'hdf5cache', None)
if hdf5cache:
dstore = hdf5cache
elif (self.oqparam.hazard_calculation_id and
'gmf_data' not in self.datastore):
# 'gmf_data' in self.datastore happens for ShakeMap calculations
self.datastore.parent.close() # make sure it is closed
dstore = self.datastore.parent
else:
dstore = self.datastore
if kind == 'poe': # hcurves, shape (R, N)
getter = getters.PmapGetter(dstore, self.rlzs_assoc, [sid])
else: # gmf
getter = getters.GmfDataGetter(dstore, [sid], self.R)
if dstore is self.datastore:
getter.init()
return getter
def build_disagg_by_src(self):
"""
:param dstore: a datastore
:param iml2s: N arrays of IMLs with shape (M, P)
"""
logging.warning('Disaggregation by source is experimental')
oq = self.oqparam
ws = [rlz.weight for rlz in self.rlzs_assoc.realizations]
pgetter = getters.PmapGetter(self.datastore, ws, self.sitecol.sids)
groups = list(self.datastore['rlzs_by_grp'])
M = len(oq.imtls)
P = len(self.poes_disagg)
for sid in self.sitecol.sids:
poes = numpy.zeros((M, P, len(groups)))
iml2 = self.iml2s[sid]
for g, grp_id in enumerate(groups):
pcurve = pgetter.get_pcurve(sid, iml2.rlzi, int(grp_id[4:]))
if pcurve is None:
continue
for m, imt in enumerate(oq.imtls):
xs = oq.imtls[imt]
ys = pcurve.array[oq.imtls(imt), 0]
poes[m, :, g] = numpy.interp(iml2[m], xs, ys)
for m, imt in enumerate(oq.imtls):
for p, poe in enumerate(self.poes_disagg):
pref = ('iml-%s' %
oq.iml_disagg[imt] if poe is None else 'poe-%s' %
poe)
name = 'disagg_by_src/%s-%s-sid-%s' % (pref, imt, sid)
if poes[m, p].sum(): # nonzero contribution
poe_agg = 1 - numpy.prod(1 - poes[m, p])
if poe and abs(1 - poe_agg / poe) > .1:
logging.warning(
'poe_agg=%s is quite different from '
'the expected poe=%s', poe_agg, poe)
self.datastore[name] = poes[m, p]
self.datastore.set_attrs(name, poe_agg=poe_agg)
def test_case_1(self):
self.assert_curves_ok([
'rlz-0-PGA-sid-0-poe-0_Lon_Lat.csv',
'rlz-0-PGA-sid-0-poe-0_Mag.csv',
'rlz-0-PGA-sid-0-poe-0_Mag_Dist.csv',
'rlz-0-PGA-sid-0-poe-1_Lon_Lat.csv',
'rlz-0-PGA-sid-0-poe-1_Mag.csv',
'rlz-0-PGA-sid-0-poe-1_Mag_Dist.csv',
'rlz-0-SA(0.025)-sid-0-poe-0_Lon_Lat.csv',
'rlz-0-SA(0.025)-sid-0-poe-0_Mag.csv',
'rlz-0-SA(0.025)-sid-0-poe-0_Mag_Dist.csv',
'rlz-0-SA(0.025)-sid-0-poe-1_Lon_Lat.csv',
'rlz-0-SA(0.025)-sid-0-poe-1_Mag.csv',
'rlz-0-SA(0.025)-sid-0-poe-1_Mag_Dist.csv'
],
case_1.__file__,
fmt='csv')
# disaggregation by source group
rlzs = self.calc.datastore['full_lt'].get_realizations()
ws = [rlz.weight for rlz in rlzs]
sids = self.calc.sitecol.sids
oq = self.calc.oqparam
pgetter = getters.PmapGetter(self.calc.datastore, ws, sids, oq.imtls,
oq.poes)
pgetter.init()
pmaps = []
for grp in sorted(pgetter.dstore['poes']):
pmaps.append(pgetter.get_mean(grp))
# make sure that the combination of the contributions is okay
pmap = pgetter.get_mean() # total mean map
cmap = combine(pmaps) # combination of the mean maps per source group
for sid in pmap:
numpy.testing.assert_almost_equal(pmap[sid].array, cmap[sid].array)
check_disagg_by_src(self.calc.datastore)
def extract_hazard(dstore, what):
"""
Extracts hazard curves and possibly hazard maps and/or uniform hazard
spectra. Use it as /extract/hazard/mean or /extract/hazard/rlz-0, etc
"""
oq = dstore['oqparam']
sitecol = dstore['sitecol']
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
yield 'sitecol', sitecol
yield 'oqparam', oq
yield 'imtls', oq.imtls
yield 'realizations', dstore['csm_info'].rlzs
yield 'checksum32', dstore['/'].attrs['checksum32']
nsites = len(sitecol)
M = len(oq.imtls)
P = len(oq.poes)
for kind, pmap in getters.PmapGetter(dstore, rlzs_assoc).items(what):
for imt in oq.imtls:
key = 'hcurves/%s/%s' % (imt, kind)
arr = numpy.zeros((nsites, len(oq.imtls[imt])))
for sid in pmap:
arr[sid] = pmap[sid].array[oq.imtls.slicedic[imt], 0]
logging.info('extracting %s', key)
yield key, arr
try:
hmap = dstore['hmaps/' + kind]
except KeyError: # for kind=rlz-XXX
hmap = calc.make_hmap(pmap, oq.imtls, oq.poes)
for p, poe in enumerate(oq.poes):
key = 'hmaps/poe-%s/%s' % (poe, kind)
arr = numpy.zeros((nsites, M))
idx = [m * P + p for m in range(M)]
for sid in pmap:
arr[sid] = hmap[sid].array[idx, 0]
logging.info('extracting %s', key)
yield key, arr
def full_disaggregation(self):
"""
Run the disaggregation phase.
"""
oq = self.oqparam
mags_by_trt = self.datastore['source_mags']
all_edges, shapedic = disagg.get_edges_shapedic(
oq, self.sitecol, mags_by_trt)
*self.bin_edges, self.trts = all_edges
src_filter = self.src_filter()
if hasattr(self, 'csm'):
for sg in self.csm.src_groups:
if sg.atomic:
raise NotImplementedError(
'Atomic groups are not supported yet')
self.full_lt = self.datastore['full_lt']
self.poes_disagg = oq.poes_disagg or (None,)
self.imts = list(oq.imtls)
self.ws = [rlz.weight for rlz in self.full_lt.get_realizations()]
self.pgetter = getters.PmapGetter(
self.datastore, self.ws, self.sitecol.sids)
# build array rlzs (N, Z)
if oq.rlz_index is None:
Z = oq.num_rlzs_disagg
rlzs = numpy.zeros((self.N, Z), int)
if self.R > 1:
for sid in self.sitecol.sids:
curves = numpy.array(
[pc.array for pc in self.pgetter.get_pcurves(sid)])
mean = getters.build_stat_curve(
curves, oq.imtls, stats.mean_curve, self.ws)
rlzs[sid] = util.closest_to_ref(curves, mean.array)[:Z]
self.datastore['best_rlzs'] = rlzs
else:
Z = len(oq.rlz_index)
rlzs = numpy.zeros((self.N, Z), int)
for z in range(Z):
rlzs[:, z] = oq.rlz_index[z]
assert Z <= self.R, (Z, self.R)
self.Z = Z
self.rlzs = rlzs
if oq.iml_disagg:
# no hazard curves are needed
self.poe_id = {None: 0}
curves = [[None for z in range(Z)] for s in range(self.N)]
self.ok_sites = set(self.sitecol.sids)
else:
self.poe_id = {poe: i for i, poe in enumerate(oq.poes_disagg)}
curves = [self.get_curve(sid, rlzs[sid])
for sid in self.sitecol.sids]
self.ok_sites = set(self.check_poes_disagg(curves, rlzs))
self.iml3 = _iml3(rlzs, oq.iml_disagg, oq.imtls,
self.poes_disagg, curves)
if oq.disagg_by_src:
self.build_disagg_by_src(rlzs)
self.save_bin_edges()
sd = shapedic.copy()
sd.pop('trt')
nbytes, msg = get_array_nbytes(sd)
if nbytes > oq.max_data_transfer:
raise ValueError(
'Estimated data transfer too big\n%s > max_data_transfer=%s' %
(msg, humansize(oq.max_data_transfer)))
logging.info('Estimated data transfer:\n%s', msg)
tot = get_outputs_size(shapedic, oq.disagg_outputs or disagg.pmf_map)
logging.info('Total output size: %s', humansize(sum(tot.values())))
self.imldic = {} # sid, rlz, poe, imt -> iml
for s in self.sitecol.sids:
for z, rlz in enumerate(rlzs[s]):
for p, poe in enumerate(self.poes_disagg):
for imt in oq.imtls:
self.imldic[s, rlz, poe, imt] = self.iml3[imt][s, p, z]
# submit #groups disaggregation tasks
dstore = (self.datastore.parent if self.datastore.parent
else self.datastore)
M = len(oq.imtls)
tasks_per_imt = numpy.ceil(oq.concurrent_tasks / M) or 1
rups_per_task = len(dstore['rup/mag']) / tasks_per_imt
logging.info('Considering ~%d ruptures per task', rups_per_task)
indices = get_indices(dstore, tasks_per_imt)
self.datastore.swmr_on()
smap = parallel.Starmap(compute_disagg, h5=self.datastore.hdf5)
trt_num = {trt: i for i, trt in enumerate(self.trts)}
for grp_id, trt in self.full_lt.trt_by_grp.items():
logging.info('Group #%d, sending rup_data for %s', grp_id, trt)
trti = trt_num[trt]
cmaker = ContextMaker(
trt, self.full_lt.get_rlzs_by_gsim(grp_id),
{'truncation_level': oq.truncation_level,
'maximum_distance': src_filter.integration_distance,
'imtls': oq.imtls})
for idxs in indices[grp_id]:
for imt in oq.imtls:
smap.submit((dstore, idxs, cmaker, self.iml3[imt], trti,
self.bin_edges, oq))
results = smap.reduce(self.agg_result, AccumDict(accum={}))
return results # sid -> trti-> 8D array
def calc_stats(self):
oq = self.oqparam
hstats = oq.hazard_stats()
# initialize datasets
N = len(self.sitecol.complete)
P = len(oq.poes)
M = self.M = len(oq.imtls)
imts = list(oq.imtls)
if oq.soil_intensities is not None:
L = M * len(oq.soil_intensities)
else:
L = oq.imtls.size
L1 = self.L1 = L // M
R = len(self.realizations)
S = len(hstats)
if R > 1 and oq.individual_curves or not hstats:
self.datastore.create_dset('hcurves-rlzs', F32, (N, R, M, L1))
self.datastore.set_shape_descr(
'hcurves-rlzs', site_id=N, rlz_id=R, imt=imts, lvl=L1)
if oq.poes:
self.datastore.create_dset('hmaps-rlzs', F32, (N, R, M, P))
self.datastore.set_shape_descr(
'hmaps-rlzs', site_id=N, rlz_id=R,
imt=list(oq.imtls), poe=oq.poes)
if hstats:
self.datastore.create_dset('hcurves-stats', F32, (N, S, M, L1))
self.datastore.set_shape_descr(
'hcurves-stats', site_id=N, stat=list(hstats),
imt=imts, lvl=numpy.arange(L1))
if oq.poes:
self.datastore.create_dset('hmaps-stats', F32, (N, S, M, P))
self.datastore.set_shape_descr(
'hmaps-stats', site_id=N, stat=list(hstats),
imt=list(oq.imtls), poe=oq.poes)
ct = oq.concurrent_tasks or 1
logging.info('Building hazard statistics')
self.weights = [rlz.weight for rlz in self.realizations]
dstore = (self.datastore.parent if oq.hazard_calculation_id
else self.datastore)
allargs = [ # this list is very fast to generate
(getters.PmapGetter(
dstore, self.weights, t.sids, oq.imtls, oq.poes),
N, hstats, oq.individual_curves, oq.max_sites_disagg,
self.amplifier)
for t in self.sitecol.split_in_tiles(ct)]
if self.few_sites:
dist = 'no'
else:
dist = None # parallelize as usual
if oq.hazard_calculation_id is None: # essential before Starmap
self.datastore.swmr_on()
self.hazard = {} # kind -> array
parallel.Starmap(
build_hazard, allargs, distribute=dist, h5=self.datastore.hdf5
).reduce(self.save_hazard)
for kind in sorted(self.hazard):
logging.info('Saving %s', kind)
self.datastore[kind][:] = self.hazard.pop(kind)
task_info = self.datastore.read_df('task_info', 'taskname')
try:
dur = task_info.loc[b'classical'].duration
except KeyError: # no data
pass
else:
slow_tasks = len(dur[dur > 3 * dur.mean()])
if slow_tasks:
logging.info('There were %d slow tasks', slow_tasks)
if 'hmaps-stats' in self.datastore:
hmaps = self.datastore.sel('hmaps-stats', stat='mean') # NSMP
maxhaz = hmaps.max(axis=(0, 1, 3))
mh = dict(zip(self.oqparam.imtls, maxhaz))
logging.info('The maximum hazard map values are %s', mh)
if Image is None or not self.from_engine: # missing PIL
return
M, P = hmaps.shape[2:]
logging.info('Saving %dx%d mean hazard maps', M, P)
inv_time = oq.investigation_time
allargs = []
for m, imt in enumerate(self.oqparam.imtls):
for p, poe in enumerate(self.oqparam.poes):
dic = dict(m=m, p=p, imt=imt, poe=poe, inv_time=inv_time,
calc_id=self.datastore.calc_id,
array=hmaps[:, 0, m, p])
allargs.append((dic, self.sitecol.lons, self.sitecol.lats))
smap = parallel.Starmap(make_hmap_png, allargs)
for dic in smap:
self.datastore['png/hmap_%(m)d_%(p)d' % dic] = dic['img']
