def compute_store_stats(self, rlzs, builder):
"""
Compute and store the statistical outputs.
:param rlzs: list of realizations
"""
oq = self.oqparam
ltypes = self.riskmodel.loss_types
all_stats = map(builder.build, self._collect_all_data())
if not all_stats:
return
loss_curves = numpy.zeros((self.N, self.Q1), self.loss_curve_dt)
if oq.conditional_loss_poes:
loss_maps = numpy.zeros((self.N, self.Q1), self.loss_maps_dt)
for stats in all_stats:
# there is one stat for each loss_type
cb = self.riskmodel.curve_builders[ltypes.index(stats.loss_type)]
if not cb.user_provided:
continue
sb = scientific.StatsBuilder(oq.quantile_loss_curves,
oq.conditional_loss_poes, [],
len(cb.ratios),
scientific.normalize_curves_eb,
oq.insured_losses)
curves, maps = sb.get_curves_maps(stats) # matrices (Q1, N)
loss_curves[cb.loss_type] = curves.T
if oq.conditional_loss_poes:
loss_maps[cb.loss_type] = maps.T
self.datastore['loss_curves-stats'] = loss_curves
if oq.conditional_loss_poes:
self.datastore['loss_maps-stats'] = loss_maps
def compute_store_stats(self, rlzs, kind):
"""
Compute and store the statistical outputs
"""
oq = self.oqparam
builder = scientific.StatsBuilder(oq.quantile_loss_curves,
oq.conditional_loss_poes, [],
scientific.normalize_curves_eb)
if kind == '_specific':
all_stats = [
builder.build(data, prefix='specific-')
for data in self._collect_specific_data()
]
else:
all_stats = map(builder.build, self._collect_all_data())
for stat in all_stats:
# there is one stat for each loss_type
curves, ins_curves, maps = scientific.get_stat_curves(stat)
for i, path in enumerate(stat.paths):
# there are paths like
# %s-stats/structural/mean
# %s-stats/structural/quantile-0.1
# ...
self.datastore[path % 'loss_curves'] = curves[i]
if oq.insured_losses:
self.datastore[path % 'ins_curves'] = ins_curves[i]
if oq.conditional_loss_poes:
self.datastore[path % 'loss_maps'] = maps[i]
stats = scientific.SimpleStats(rlzs, oq.quantile_loss_curves)
nbytes = stats.compute('avg_losses-rlzs', self.datastore)
self.datastore['avg_losses-stats'].attrs['nbytes'] = nbytes
self.datastore.hdf5.flush()
def build_stats(self, loss_curve_key):
"""
Compute all statistics for the specified assets starting from the
stored loss curves. Yield a statistical output object for each
loss type.
"""
oq = self.oqparam
rlzs = self.rlzs_assoc.realizations
stats = scientific.StatsBuilder(
oq.quantile_loss_curves, oq.conditional_loss_poes, [],
scientific.normalize_curves_eb)
# NB: should we encounter memory issues in the future, the easy
# solution is to split the specific assets in blocks and perform
# the computation one block at the time
for loss_type in self.riskmodel.get_loss_types():
outputs = []
for rlz in rlzs:
key = '%s-rlzs/%s' % (loss_curve_key, rlz.uid)
lcs = self.datastore[key][loss_type]
assets = [None] if key.startswith('agg') else self.assets
losses_poes = numpy.array( # -> shape (N, 2, C)
[lcs['losses'], lcs['poes']]).transpose(1, 0, 2)
out = scientific.Output(
assets, loss_type, rlz.ordinal, rlz.weight,
loss_curves=losses_poes, insured_curves=None)
outputs.append(out)
yield stats.build(outputs)
def setUpClass(cls):
assets = [
asset('a1', 101),
asset('a2', 151),
asset('a3', 91),
asset('a4', 81)
]
asset_refs = [a.id for a in assets]
outputs = []
weights = [0.3, 0.7]
baselosses = numpy.array([.10, .14, .17, .20, .21])
for i, w in enumerate(weights):
lc = loss_curves(assets, baselosses, i)
out = scientific.Output(asset_refs,
'structural',
weight=w,
loss_curves=lc,
insured_curves=None,
average_losses=[.1, .12, .13, .9],
average_insured_losses=None)
outputs.append(out)
cls.builder = scientific.StatsBuilder(
quantiles=[0.1, 0.9],
conditional_loss_poes=[0.35, 0.24, 0.13],
poes_disagg=[],
curve_resolution=len(baselosses))
cls.stats = cls.builder.build(outputs)
def classical_risk(riskinputs, riskmodel, rlzs_assoc, monitor):
"""
Compute and return the average losses for each asset.
:param riskinputs:
a list of :class:`openquake.risklib.riskinput.RiskInput` objects
:param riskmodel:
a :class:`openquake.risklib.riskinput.RiskModel` instance
:param rlzs_assoc:
associations (trt_id, gsim) -> realizations
:param monitor:
:class:`openquake.baselib.performance.PerformanceMonitor` instance
"""
lti = riskmodel.lti
oq = monitor.oqparam
ins = oq.insured_losses
result = dict(loss_curves=[], loss_maps=[], stat_curves=[], stat_maps=[])
for out_by_rlz in riskmodel.gen_outputs(riskinputs, rlzs_assoc, monitor):
l = lti[out_by_rlz.loss_type]
values = workflows.get_values(out_by_rlz.loss_type, out_by_rlz.assets)
for out in out_by_rlz:
r = out.hid
for i, asset in enumerate(out.assets):
aid = asset.idx
val = values[i]
avg = out.average_losses[i] * val
avg_ins = (out.average_insured_losses[i] *
val if ins else numpy.nan)
lcurve = (out.loss_curves[i, 0] * val, out.loss_curves[i,
1], avg)
if ins:
lcurve += (out.insured_curves[i, 0] * val,
out.insured_curves[i, 1], avg_ins)
else:
lcurve += (None, None, None)
result['loss_curves'].append((l, r, aid, lcurve))
# no insured, shape (P, N)
result['loss_maps'].append(
(l, r, aid, out.loss_maps[:, i] * val))
# compute statistics
if len(out_by_rlz) > 1:
cb = riskmodel.curve_builders[l]
statsbuilder = scientific.StatsBuilder(
oq.quantile_loss_curves,
oq.conditional_loss_poes,
oq.poes_disagg,
cb.curve_resolution,
insured_losses=oq.insured_losses)
stats = statsbuilder.build(out_by_rlz)
stat_curves, stat_maps = statsbuilder.get_curves_maps(stats)
for asset, stat_curve, stat_map in zip(out_by_rlz.assets,
stat_curves, stat_maps):
result['stat_curves'].append((l, asset.idx, stat_curve))
result['stat_maps'].append((l, asset.idx, stat_map))
return result
def classical_risk(riskinput, riskmodel, rlzs_assoc, monitor):
"""
Compute and return the average losses for each asset.
:param riskinput:
a :class:`openquake.risklib.riskinput.RiskInput` object
:param riskmodel:
a :class:`openquake.risklib.riskinput.CompositeRiskModel` instance
:param rlzs_assoc:
associations (trt_id, gsim) -> realizations
:param monitor:
:class:`openquake.baselib.performance.Monitor` instance
"""
oq = monitor.oqparam
ins = oq.insured_losses
R = len(rlzs_assoc.realizations)
result = dict(
loss_curves=[], loss_maps=[], stat_curves=[], stat_maps=[])
for out_by_lr in riskmodel.gen_outputs(riskinput, rlzs_assoc, monitor):
for (l, r), out in sorted(out_by_lr.items()):
for i, asset in enumerate(out.assets):
aid = asset.ordinal
avg = out.average_losses[i]
avg_ins = (out.average_insured_losses[i]
if ins else numpy.nan)
lcurve = (
out.loss_curves[i, 0],
out.loss_curves[i, 1], avg)
if ins:
lcurve += (
out.insured_curves[i, 0],
out.insured_curves[i, 1], avg_ins)
else:
lcurve += (None, None, None)
result['loss_curves'].append((l, r, aid, lcurve))
# no insured, shape (P, N)
result['loss_maps'].append(
(l, r, aid, out.loss_maps[:, i]))
# compute statistics
if R > 1:
for l, lrs in groupby(out_by_lr, operator.itemgetter(0)).items():
outs = [out_by_lr[lr] for lr in lrs]
curve_resolution = outs[0].loss_curves.shape[-1]
statsbuilder = scientific.StatsBuilder(
oq.quantile_loss_curves,
oq.conditional_loss_poes, oq.poes_disagg,
curve_resolution, insured_losses=oq.insured_losses)
stats = statsbuilder.build(outs)
stat_curves, stat_maps = statsbuilder.get_curves_maps(stats)
for i, asset in enumerate(out_by_lr.assets):
result['stat_curves'].append(
(l, asset.ordinal, stat_curves[:, i]))
if len(stat_maps):
result['stat_maps'].append(
(l, asset.ordinal, stat_maps[:, i]))
return result
def statistics(self, all_outputs, quantiles=()):
"""
:param quantiles:
quantile levels used to compute quantile outputs
:returns:
a :class:`openquake.risklib.scientific.Output`
instance holding statistical outputs (e.g. mean loss curves).
"""
if len(all_outputs) == 1: # single realization
return
stats = scientific.StatsBuilder(quantiles, self.conditional_loss_poes,
self.poes_disagg)
return stats.build(all_outputs)
def statistics(self, all_outputs, quantiles):
"""
:returns:
a :class:`openquake.risklib.scientific.Output`
instance holding statistical outputs (e.g. mean loss curves).
:param quantiles:
quantile levels used to compute quantile outputs
"""
if len(all_outputs) == 1: # single realization
return
stats = scientific.StatsBuilder(quantiles, self.conditional_loss_poes,
[], scientific.normalize_curves_eb)
out = stats.build(all_outputs)
out.event_loss_table = sum((out.event_loss_table
for out in all_outputs),
collections.Counter())
return out
def compute_store_stats(self, rlzs, kind):
"""
Compute and store the statistical outputs
"""
oq = self.oqparam
N = (len(self.oqparam.specific_assets)
if kind == '_specific' else len(self.assetcol))
Q = 1 + len(oq.quantile_loss_curves)
C = oq.loss_curve_resolution # TODO: could be loss_type-dependent
loss_curve_dt = numpy.dtype([('losses', (float, C)),
('poes', (float, C)), ('avg', float)])
if oq.conditional_loss_poes:
lm_names = _loss_map_names(oq.conditional_loss_poes)
loss_map_dt = numpy.dtype([(f, float) for f in lm_names])
loss_curve_stats = numpy.zeros((Q, N), loss_curve_dt)
ins_curve_stats = numpy.zeros((Q, N), loss_curve_dt)
if oq.conditional_loss_poes:
loss_map_stats = numpy.zeros((Q, N), loss_map_dt)
builder = scientific.StatsBuilder(oq.quantile_loss_curves,
oq.conditional_loss_poes, [],
scientific.normalize_curves_eb)
build_stats = getattr(self, 'build%s_stats' % kind)
all_stats = build_stats(builder)
for stat in all_stats:
# there is one stat for each loss_type
curves, ins_curves, maps = scientific.get_stat_curves(stat)
loss_curve_stats[:] = curves
if oq.insured_losses:
ins_curve_stats[:] = ins_curves
if oq.conditional_loss_poes:
loss_map_stats[:] = maps
for i, path in enumerate(stat.paths):
self._store(path % 'loss_curves', loss_curve_stats[i])
self._store(path % 'ins_curves', ins_curve_stats[i])
if oq.conditional_loss_poes:
self._store(path % 'loss_maps', loss_map_stats[i])
stats = scientific.SimpleStats(rlzs, oq.quantile_loss_curves)
stats.compute_and_store('avg_losses', self.datastore)
def post_execute(self, result):
"""
Save the event loss table in the datastore.
:param result:
the dictionary returned by the .execute method
"""
logging.info('Generated %s of GMFs', humansize(self.gmfbytes))
self.datastore.save('job_info', {'gmfbytes': self.gmfbytes})
if self.oqparam.asset_loss_table:
asslt = self.datastore['ass_loss_table']
asslt.attrs['nbytes'] = self.ass_bytes
for rlz, dset in asslt.items():
for ds in dset.values():
ds.attrs['nonzero_fraction'] = len(ds) / (self.N * self.E)
agglt = self.datastore['agg_loss_table']
agglt.attrs['nbytes'] = self.agg_bytes
for rlz, dset in agglt.items():
for ds in dset.values():
ds.attrs['nonzero_fraction'] = len(ds) / self.E
insured_losses = self.oqparam.insured_losses
ses_ratio = self.oqparam.ses_ratio
saved = self.saved
self.N = N = len(self.assetcol)
self.R = R = len(self.rlzs_assoc.realizations)
ltypes = self.riskmodel.loss_types
self.loss_curve_dt, self.loss_maps_dt = (
self.riskmodel.build_loss_dtypes(
self.oqparam.conditional_loss_poes, self.I))
self.vals = {} # asset values by loss_type
for ltype in ltypes:
asset_values = []
for assets in self.assets_by_site:
for asset in assets:
asset_values.append(
asset.value(ltype, self.oqparam.time_event))
self.vals[ltype] = numpy.array(asset_values)
# loss curves
multi_lr_dt = numpy.dtype([
(ltype, (F32, cbuilder.curve_resolution))
for ltype, cbuilder in zip(ltypes, self.riskmodel.curve_builders)
])
rcurves = numpy.zeros((N, R, 2), multi_lr_dt)
# AVGLOSS
if self.oqparam.avg_losses:
with self.monitor('building avg_losses-rlzs'):
for (l, r), avgloss in numpy.ndenumerate(result['AVGLOSS']):
lt = self.riskmodel.loss_types[l]
avg_losses_lt = self.avg_losses[lt]
for i, avalue in enumerate(self.vals[lt]):
avg_losses_lt[i, r] = avgloss[i, 0] * avalue
if self.oqparam.insured_losses:
self.avg_losses[lt + '_ins'][i,
r] = (avgloss[i, 1] *
avalue)
self.datastore['avg_losses-rlzs'] = self.avg_losses
saved['avg_losses-rlzs'] = self.avg_losses.nbytes
# RC, IC
if self.oqparam.loss_ratios:
with self.monitor('building rcurves-rlzs'):
for (l, r), data in numpy.ndenumerate(result['RC']):
cb = self.riskmodel.curve_builders[l]
if data and cb.user_provided:
# data is a dict asset idx -> counts
lt = self.riskmodel.loss_types[l]
poes = cb.build_poes(N, [data], ses_ratio)
rcurves[lt][:, r, 0] = poes
saved['rcurves-rlzs'] += poes.nbytes
for (l, r), data in numpy.ndenumerate(result['IC']):
cb = self.riskmodel.curve_builders[l]
if data and cb.user_provided and insured_losses:
# data is a dict asset idx -> counts
lt = self.riskmodel.loss_types[l]
poes = cb.build_poes(N, [data], ses_ratio)
rcurves[lt][:, r, 1] = poes
saved['rcurves-rlzs'] += poes.nbytes
self.datastore['rcurves-rlzs'] = rcurves
oq = self.oqparam
builder = scientific.StatsBuilder(oq.quantile_loss_curves,
oq.conditional_loss_poes, [],
oq.loss_curve_resolution,
scientific.normalize_curves_eb,
oq.insured_losses)
# build an aggregate loss curve per realization plus statistics
with self.monitor('building agg_curve'):
self.build_agg_curve_and_stats(builder)
self.datastore.hdf5.flush()
for out in sorted(saved):
nbytes = saved[out]
if nbytes:
self.datastore[out].attrs['nbytes'] = nbytes
logging.info('Saved %s in %s', humansize(nbytes), out)
if self.oqparam.asset_loss_table:
pass # TODO: build specific loss curves
rlzs = self.rlzs_assoc.realizations
if self.loss_maps_dt:
with self.monitor('building loss_maps-rlzs'):
if (self.oqparam.conditional_loss_poes
and 'rcurves-rlzs' in self.datastore):
loss_maps = numpy.zeros((N, R), self.loss_maps_dt)
rcurves = self.datastore['rcurves-rlzs']
for cb in self.riskmodel.curve_builders:
if cb.user_provided:
lm = loss_maps[cb.loss_type]
for r, lmaps in cb.build_loss_maps(
self.assetcol.array, rcurves):
lm[:, r] = lmaps
self.datastore['loss_maps-rlzs'] = loss_maps
if len(rlzs) > 1:
self.Q1 = len(self.oqparam.quantile_loss_curves) + 1
with self.monitor('computing stats'):
if 'rcurves-rlzs' in self.datastore:
self.compute_store_stats(rlzs, builder)
if oq.avg_losses: # stats for avg_losses
stats = scientific.SimpleStats(rlzs,
oq.quantile_loss_curves)
stats.compute_and_store('avg_losses', self.datastore)
self.datastore.hdf5.flush()
