def post_execute(self, dummy):
"""
Compute and store average losses from the risk_by_event dataset,
and then loss curves and maps.
"""
oq = self.oqparam
# sanity check on the risk_by_event
alt = self.datastore.read_df('risk_by_event')
K = self.datastore['risk_by_event'].attrs.get('K', 0)
upper_limit = self.E * self.L * (K + 1)
size = len(alt)
assert size <= upper_limit, (size, upper_limit)
# sanity check on uniqueness by (agg_id, loss_id, event_id)
arr = alt[['agg_id', 'loss_id', 'event_id']].to_numpy()
uni = numpy.unique(arr, axis=0)
if len(uni) < len(arr):
raise RuntimeError('risk_by_event contains %d duplicates!' %
(len(arr) - len(uni)))
if oq.avg_losses:
for r in range(self.R):
self.avg_losses[:, r] *= self.avg_ratio[r]
self.datastore['avg_losses-rlzs'] = self.avg_losses
stats.set_rlzs_stats(self.datastore,
'avg_losses',
asset_id=self.assetcol['id'],
loss_type=oq.loss_types)
self.build_aggcurves()
if oq.reaggregate_by:
post_aggregate(self.datastore.calc_id, ','.join(oq.reaggregate_by))
def post_execute(self, dummy):
"""
Compute and store average losses from the risk_by_event dataset,
and then loss curves and maps.
"""
oq = self.oqparam
# sanity check on the risk_by_event
alt = self.datastore.read_df('risk_by_event', 'event_id')
K = self.datastore['risk_by_event'].attrs.get('K', 0)
upper_limit = self.E * self.L * (K + 1)
size = len(alt)
assert size <= upper_limit, (size, upper_limit)
if oq.avg_losses:
for r in range(self.R):
self.avg_losses[:, r] *= self.avg_ratio[r]
self.datastore['avg_losses-rlzs'] = self.avg_losses
stats.set_rlzs_stats(self.datastore,
'avg_losses',
asset_id=self.assetcol['id'],
loss_type=oq.loss_names)
# save agg_losses
units = self.datastore['cost_calculator'].get_units(oq.loss_names)
if oq.calculation_mode == 'scenario_risk': # compute agg_losses
alt['rlz_id'] = self.rlzs[alt.index.to_numpy()]
agglosses = numpy.zeros((K + 1, self.R, self.L), F32)
for (agg_id, rlz_id,
loss_id), df in alt.groupby(['agg_id', 'rlz_id', 'loss_id']):
agglosses[agg_id, rlz_id,
loss_id] = (df.loss.sum() * self.avg_ratio[rlz_id])
self.datastore['agg_losses-rlzs'] = agglosses
stats.set_rlzs_stats(self.datastore,
'agg_losses',
agg_id=K,
loss_types=oq.loss_names,
units=units)
logging.info('Total portfolio loss\n' +
views.view('portfolio_loss', self.datastore))
else: # event_based_risk, run post_risk
prc = PostRiskCalculator(oq, self.datastore.calc_id)
if hasattr(self, 'exported'):
prc.exported = self.exported
with prc.datastore:
prc.run(exports='')
if (oq.investigation_time or not oq.avg_losses
or 'agg_losses-rlzs' not in self.datastore):
return
# sanity check on the agg_losses and sum_losses
sumlosses = self.avg_losses.sum(axis=0)
if not numpy.allclose(agglosses[K], sumlosses, rtol=1E-6):
url = ('https://docs.openquake.org/oq-engine/advanced/'
'addition-is-non-associative.html')
logging.warning(
'Due to rounding errors inherent in floating-point arithmetic,'
' agg_losses != sum(avg_losses): %s != %s\nsee %s',
agglosses[K].mean(), sumlosses.mean(), url)
def post_execute(self, result):
# NB: defined only for loss_type = 'structural'
bcr_data = numpy.zeros((self.A, self.R), bcr_dt)
for aid, data in result['bcr_data'].items():
bcr_data[aid]['annual_loss_orig'] = data[:, 0]
bcr_data[aid]['annual_loss_retro'] = data[:, 1]
bcr_data[aid]['bcr'] = data[:, 2]
stats.set_rlzs_stats(self.datastore, 'bcr', bcr_data)
def post_execute(self, result):
# NB: defined only for loss_type = 'structural'
bcr_data = numpy.zeros((self.A, self.R), bcr_dt)
for aid, data in result['bcr_data'].items():
bcr_data[aid]['annual_loss_orig'] = data[:, 0]
bcr_data[aid]['annual_loss_retro'] = data[:, 1]
bcr_data[aid]['bcr'] = data[:, 2]
stats.set_rlzs_stats(self.datastore, 'bcr', bcr_data)
def execute(self):
oq = self.oqparam
if oq.return_periods != [0]:
# setting return_periods = 0 disable loss curves
eff_time = oq.investigation_time * oq.ses_per_logic_tree_path
if eff_time < 2:
logging.warning(
'eff_time=%s is too small to compute loss curves',
eff_time)
return
logging.info('Building loss tables')
build_loss_tables(self.datastore)
shp = self.get_shape(self.L) # (L, T...)
text = ' x '.join('%d(%s)' % (n, t)
for t, n in zip(oq.aggregate_by, shp[1:]))
logging.info('Producing %d(loss_types) x %s loss curves', self.L, text)
builder = get_loss_builder(self.datastore)
self.build_datasets(builder, oq.aggregate_by, 'agg_')
self.build_datasets(builder, [], 'app_')
self.build_datasets(builder, [], 'tot_')
ds = self.datastore
if oq.aggregate_by:
pr = post_ebrisk
ds.swmr_on()
smap = parallel.Starmap(pr, [(self.datastore, rlzi)
for rlzi in range(self.R)],
h5=self.datastore.hdf5)
else:
# do everything in process since it is really fast
elt = ds.read_df('losses_by_event', ['event_id', 'rlzi'])
smap = []
for rlzi, losses_df in elt.groupby('rlzi'):
losses = numpy.array(losses_df)
smap.append({
'tot_curves': builder.build_curves(losses, rlzi),
'tot_losses': losses.sum(axis=0) * oq.ses_ratio,
'rlzi': rlzi
})
for dic in smap:
if not dic:
continue
r = dic['rlzi']
ds['tot_curves-rlzs'][:, r] = dic['tot_curves'] # PL
ds['tot_losses-rlzs'][:, r] = dic['tot_losses'] # L
if oq.aggregate_by:
ds['agg_curves-rlzs'][:, r] = dic['agg_curves'] # PLT..
ds['agg_losses-rlzs'][:, r] = dic['agg_losses'] # LT...
if 'app_curves' in dic:
ds['app_curves-rlzs'][:, r] = dic['app_curves'] # PL
if self.R > 1:
logging.info('Computing aggregate statistics')
set_rlzs_stats(self.datastore, 'app_curves')
set_rlzs_stats(self.datastore, 'tot_curves')
set_rlzs_stats(self.datastore, 'tot_losses')
if oq.aggregate_by:
set_rlzs_stats(self.datastore, 'agg_curves')
set_rlzs_stats(self.datastore, 'agg_losses')
def execute(self):
oq = self.oqparam
if oq.return_periods != [0]:
# setting return_periods = 0 disable loss curves
eff_time = oq.investigation_time * oq.ses_per_logic_tree_path
if eff_time < 2:
logging.warning(
'eff_time=%s is too small to compute loss curves',
eff_time)
return
logging.info('Building loss tables')
build_loss_tables(self.datastore)
shp = self.get_shape(self.L) # (L, T...)
text = ' x '.join('%d(%s)' % (n, t)
for t, n in zip(oq.aggregate_by, shp[1:]))
logging.info('Producing %d(loss_types) x %s loss curves', self.L, text)
builder = get_loss_builder(self.datastore)
if oq.aggregate_by:
self.build_datasets(builder, oq.aggregate_by, 'agg_')
self.build_datasets(builder, [], 'app_')
self.build_datasets(builder, [], 'tot_')
ds = self.datastore
if oq.aggregate_by:
aggkeys = list(ds['event_loss_table'])
ds.swmr_on()
smap = parallel.Starmap(post_ebrisk, [(self.datastore, aggkey)
for aggkey in aggkeys],
h5=self.datastore.hdf5)
else:
smap = ()
# do everything in process since it is really fast
elt = ds.read_df('losses_by_event', ['event_id', 'rlzi'])
for r, curves, losses in builder.gen_curves_by_rlz(elt, oq.ses_ratio):
ds['tot_curves-rlzs'][:, r] = curves # PL
ds['tot_losses-rlzs'][:, r] = losses # L
for res in smap:
if not res:
continue
for r, dic in res.items():
if oq.aggregate_by:
ds['agg_curves-rlzs'][(slice(None), r, slice(None)) +
dic['idx'] # PRLT..
] = dic['agg_curves']
ds['agg_losses-rlzs'][(slice(None), r) +
dic['idx'] # LRT...
] = dic['agg_losses']
ds['app_curves-rlzs'][:, r] += dic['agg_curves'] # PL
if self.R > 1:
logging.info('Computing aggregate statistics')
set_rlzs_stats(self.datastore, 'app_curves')
set_rlzs_stats(self.datastore, 'tot_curves')
set_rlzs_stats(self.datastore, 'tot_losses')
if oq.aggregate_by:
set_rlzs_stats(self.datastore, 'agg_curves')
set_rlzs_stats(self.datastore, 'agg_losses')
return oq.aggregate_by
def postproc(self):
"""
Build aggregate loss curves and run EbrPostCalculator
"""
dstore = self.datastore
self.before_export() # set 'realizations'
oq = self.oqparam
eff_time = oq.investigation_time * oq.ses_per_logic_tree_path
if eff_time < 2:
logging.warn('eff_time=%s is too small to compute agg_curves',
eff_time)
return
stats = oq.risk_stats()
# store avg_losses-stats
if oq.avg_losses:
set_rlzs_stats(self.datastore, 'avg_losses')
b = get_loss_builder(dstore)
if 'ruptures' in dstore:
logging.info('Building loss tables')
with self.monitor('building loss tables', measuremem=True):
rlt, lbr = build_loss_tables(dstore)
dstore['rup_loss_table'] = rlt
dstore['losses_by_rlzi'] = lbr
ridx = [rlt[:, lti].argmax() for lti in range(self.L)]
dstore.set_attrs('rup_loss_table', ridx=ridx)
logging.info('Building aggregate loss curves')
with self.monitor('building agg_curves', measuremem=True):
array, arr_stats = b.build(dstore['losses_by_event'].value, stats)
self.datastore['agg_curves-rlzs'] = array
units = self.assetcol.units(loss_types=array.dtype.names)
self.datastore.set_attrs('agg_curves-rlzs',
return_periods=b.return_periods,
units=units)
if arr_stats is not None:
self.datastore['agg_curves-stats'] = arr_stats
self.datastore.set_attrs(
'agg_curves-stats',
return_periods=b.return_periods,
stats=[encode(name) for (name, func) in stats],
units=units)
if 'all_loss_ratios' in self.datastore:
self.datastore.save_vlen('all_loss_ratios/indices', [
numpy.array(self.indices[aid], riskinput.indices_dt)
for aid in range(self.A)
])
self.datastore.set_attrs('all_loss_ratios',
loss_types=' '.join(
self.riskmodel.loss_types))
dset = self.datastore['all_loss_ratios/data']
nbytes = dset.size * dset.dtype.itemsize
self.datastore.set_attrs('all_loss_ratios/data',
nbytes=nbytes,
bytes_per_asset=nbytes / self.A)
EbrPostCalculator(self).run(close=False)
def post_execute(self, dummy):
oq = self.oqparam
time_ratio = oq.time_ratio / len(self.datastore['weights'])
A, L, Dc = self.dmgcsq.shape
D = len(self.crmodel.damage_states)
# fix no_damage distribution for events with zero damage
number = self.assetcol['number']
for li in range(L):
self.dmgcsq[:, li, 0] = (number * self.E -
self.dmgcsq[:, li, 1:D].sum(axis=1))
self.dmgcsq /= self.E
self.datastore['damages-rlzs'] = self.dmgcsq.reshape((A, 1, L, Dc))
set_rlzs_stats(self.datastore,
'damages',
asset_id=self.assetcol['id'],
rlz=[0],
loss_type=oq.loss_names,
dmg_state=['no_damage'] + self.crmodel.get_dmg_csq())
size = self.datastore.getsize('risk_by_event')
logging.info('Building aggregated curves from %s of risk_by_event',
general.humansize(size))
alt_df = self.datastore.read_df('risk_by_event')
del alt_df['event_id']
dic = general.AccumDict(accum=[])
columns = sorted(
set(alt_df.columns) - {'agg_id', 'loss_id', 'variance'})
periods = [0] + list(self.builder.return_periods)
wdd = {'agg_id': [], 'loss_id': [], 'event_id': []}
for col in columns[:D - 1]:
wdd[col] = []
for (agg_id,
loss_id), df in alt_df.groupby([alt_df.agg_id, alt_df.loss_id]):
worst_dmgdist(df, agg_id, loss_id, wdd)
tots = [df[col].sum() * time_ratio for col in columns]
curves = [
self.builder.build_curve(df[col].to_numpy()) for col in columns
]
for p, period in enumerate(periods):
dic['agg_id'].append(agg_id)
dic['loss_id'].append(loss_id)
dic['return_period'].append(period)
if p == 0:
for col, tot in zip(columns, tots):
dic[col].append(tot)
else:
for col, curve in zip(columns, curves):
dic[col].append(curve[p - 1])
fix_dic(dic, columns)
fix_dic(wdd, columns[:D - 1])
ls = ' '.join(self.crmodel.damage_states[1:])
self.datastore.create_df('worst_dmgdist', wdd.items(), limit_states=ls)
self.datastore.create_df('aggcurves', dic.items(), limit_states=ls)
self.sanity_check(time_ratio)
def post_execute(self, result):
"""
Compute stats for the aggregated distributions and save
the results on the datastore.
"""
loss_dt = self.oqparam.loss_dt()
L = len(loss_dt.names)
dtlist = [('event_id', U32), ('loss', (F32, (L,)))]
R = self.R
with self.monitor('saving outputs'):
A = len(self.assetcol)
# agg losses
res = result['agg']
E, L = res.shape
agglosses = numpy.zeros((R, L), stat_dt)
for r in range(R):
mean, std = scientific.mean_std(res[self.rlzs == r])
agglosses[r]['mean'] = mean
agglosses[r]['stddev'] = std
# avg losses
losses_by_asset = numpy.zeros((A, R, L), F64)
for (l, r, aid, avg) in result['avg']:
losses_by_asset[aid, r, l] = avg
self.datastore['avg_losses-rlzs'] = losses_by_asset
set_rlzs_stats(self.datastore, 'avg_losses',
asset_id=self.assetcol['id'],
loss_type=self.oqparam.loss_names)
self.datastore['agglosses'] = agglosses
# losses by event
lbe = numpy.zeros(E, dtlist)
lbe['event_id'] = range(E)
lbe['loss'] = res
self.datastore['event_loss_table/,'] = lbe
loss_types = self.oqparam.loss_dt().names
self.datastore.set_attrs(
'event_loss_table/,', loss_types=loss_types)
# sanity check
totlosses = losses_by_asset.sum(axis=0)
msg = ('%s, rlz=%d: the total loss %s is different from the sum '
'of the average losses %s')
for r in range(R):
for l, name in enumerate(loss_dt.names):
totloss = totlosses[r, l]
aggloss = agglosses[r, l]['mean']
if not numpy.allclose(totloss, aggloss, rtol=1E-6):
logging.warning(msg, name, r, totloss, aggloss)
logging.info('Mean portfolio loss\n' +
views.view('portfolio_loss', self.datastore))
def post_execute(self, result):
"""
Export the result in CSV format.
:param result:
a dictionary (l, r) -> asset_ordinal -> fractions per damage state
"""
damages_dt = numpy.dtype([(ds, numpy.float32)
for ds in self.riskmodel.damage_states])
damages = numpy.zeros((self.A, self.R, self.L), damages_dt)
for l, r in result:
for aid, fractions in result[l, r].items():
damages[aid, r, l] = tuple(fractions)
stats.set_rlzs_stats(self.datastore, 'damages', damages)
def execute(self):
oq = self.oqparam
if oq.return_periods != [0]:
# setting return_periods = 0 disable loss curves
eff_time = oq.investigation_time * oq.ses_per_logic_tree_path
if eff_time < 2:
logging.warning(
'eff_time=%s is too small to compute loss curves',
eff_time)
return
logging.info('Building loss tables')
build_loss_tables(self.datastore)
shp = self.get_shape(self.L) # (L, T...)
text = ' x '.join('%d(%s)' % (n, t)
for t, n in zip(oq.aggregate_by, shp[1:]))
logging.info('Producing %d(loss_types) x %s loss curves', self.L, text)
builder = get_loss_builder(self.datastore)
self.build_datasets(builder, oq.aggregate_by, 'agg_')
self.build_datasets(builder, [], 'app_')
self.build_datasets(builder, [], 'tot_')
if oq.aggregate_by:
pr = post_ebrisk
else:
pr = post_risk
self.datastore.swmr_on()
smap = parallel.Starmap(pr, [(self.datastore, rlzi)
for rlzi in range(self.R)],
h5=self.datastore.hdf5)
ds = self.datastore
for dic in smap:
if not dic:
continue
r = dic['rlzi']
ds['tot_curves-rlzs'][:, r] = dic['tot_curves'] # PL
ds['tot_losses-rlzs'][:, r] = dic['tot_losses'] # L
if oq.aggregate_by:
ds['agg_curves-rlzs'][:, r] = dic['agg_curves'] # PLT..
ds['agg_losses-rlzs'][:, r] = dic['agg_losses'] # LT...
if 'app_curves' in dic:
ds['app_curves-rlzs'][:, r] = dic['app_curves'] # PL
if self.R > 1:
logging.info('Computing aggregate statistics')
set_rlzs_stats(self.datastore, 'app_curves')
set_rlzs_stats(self.datastore, 'tot_curves')
set_rlzs_stats(self.datastore, 'tot_losses')
if oq.aggregate_by:
set_rlzs_stats(self.datastore, 'agg_curves')
set_rlzs_stats(self.datastore, 'agg_losses')
def post_execute(self, result):
"""
Export the result in CSV format.
:param result:
a dictionary asset_ordinal -> array(R, L, D)
"""
D = len(self.crmodel.damage_states)
damages = numpy.zeros((self.A, self.R, self.L, D), numpy.float32)
for a in result:
damages[a] = result[a]
self.datastore['damages-rlzs'] = damages
stats.set_rlzs_stats(self.datastore, 'damages',
assets=self.assetcol['id'],
loss_types=self.oqparam.loss_names,
dmg_state=self.crmodel.damage_states)
def post_execute(self, result):
"""
Export the result in CSV format.
:param result:
a dictionary (l, r) -> asset_ordinal -> fractions per damage state
"""
D = len(self.crmodel.damage_states)
damages = numpy.zeros((self.A, self.R, self.L, D), numpy.float32)
for l, r in result:
for aid, fractions in result[l, r].items():
damages[aid, r, l] = fractions
self.datastore['damages-rlzs'] = damages
stats.set_rlzs_stats(self.datastore, 'damages',
assets=self.assetcol['id'],
loss_types=self.oqparam.loss_names,
dmg_state=self.crmodel.damage_states)
def postproc(self):
"""
Build aggregate loss curves in process
"""
dstore = self.datastore
self.before_export() # set 'realizations'
oq = self.oqparam
stats = self.param['stats']
# store avg_losses-stats
if oq.avg_losses:
set_rlzs_stats(self.datastore, 'avg_losses')
try:
b = self.param['builder']
except KeyError: # don't build auxiliary tables
return
if dstore.parent:
dstore.parent.open('r') # to read the ruptures
if 'ruptures' in self.datastore and len(self.datastore['ruptures']):
logging.info('Building loss tables')
with self.monitor('building loss tables', measuremem=True):
rlt, lbr = build_loss_tables(dstore)
dstore['rup_loss_table'] = rlt
dstore['losses_by_rlzi'] = lbr
ridx = [rlt[:, lti].argmax() for lti in range(self.L)]
dstore.set_attrs('rup_loss_table', ridx=ridx)
logging.info('Building aggregate loss curves')
with self.monitor('building agg_curves', measuremem=True):
lbr = group_array(dstore['losses_by_event'][()], 'rlzi')
dic = {r: arr['loss'] for r, arr in lbr.items()}
array, arr_stats = b.build(dic, stats)
loss_types = ' '.join(oq.loss_dt().names)
units = self.datastore['cost_calculator'].get_units(loss_types.split())
if oq.individual_curves or self.R == 1:
self.datastore['agg_curves-rlzs'] = array # shape (P, R, L)
self.datastore.set_attrs('agg_curves-rlzs',
return_periods=b.return_periods,
loss_types=loss_types,
units=units)
if arr_stats is not None:
self.datastore['agg_curves-stats'] = arr_stats # shape (P, S, L)
self.datastore.set_attrs(
'agg_curves-stats',
return_periods=b.return_periods,
stats=[encode(name) for (name, func) in stats],
loss_types=loss_types,
units=units)
def post_execute(self, result):
"""
Compute stats for the aggregated distributions and save
the results on the datastore.
"""
loss_dt = self.oqparam.loss_dt()
L = len(loss_dt.names)
dtlist = [('event_id', U32), ('loss', (F32, (L,)))]
R = self.R
with self.monitor('saving outputs'):
A = len(self.assetcol)
# agg losses
res = result['agg']
E, L = res.shape
agglosses = numpy.zeros((R, L), stat_dt)
for r in range(R):
mean, std = scientific.mean_std(res[self.rlzs == r])
agglosses[r]['mean'] = F32(mean)
agglosses[r]['stddev'] = F32(std)
# avg losses
losses_by_asset = numpy.zeros((A, R, L), F32)
for (l, r, aid, avg) in result['avg']:
losses_by_asset[aid, r, l] = avg
self.datastore['avg_losses-rlzs'] = losses_by_asset
set_rlzs_stats(self.datastore, 'avg_losses',
asset_id=self.assetcol['id'],
loss_type=self.oqparam.loss_names)
self.datastore['agglosses'] = agglosses
# losses by event
lbe = numpy.zeros(E, dtlist)
lbe['event_id'] = range(E)
lbe['loss'] = res
self.datastore['losses_by_event'] = lbe
loss_types = self.oqparam.loss_dt().names
self.datastore.set_attrs('losses_by_event', loss_types=loss_types)
# sanity check
numpy.testing.assert_allclose(
losses_by_asset.sum(axis=0), agglosses['mean'], rtol=1E-4)
logging.info('Mean portfolio loss\n' +
views.view('portfolio_loss', self.datastore))
def post_execute(self, result):
"""
Export the result in CSV format.
:param result:
a dictionary asset_ordinal -> array(R, L, D)
"""
D = len(self.crmodel.damage_states)
damages = numpy.zeros((self.A, self.R, self.L, D), numpy.float32)
for a in result:
damages[a] = result[a]
self.datastore['damages-rlzs'] = damages
stats.set_rlzs_stats(self.datastore, 'damages',
assets=self.assetcol['id'],
loss_type=self.oqparam.loss_names,
dmg_state=self.crmodel.damage_states)
dmg = views.view('portfolio_damage', self.datastore)
logging.info('\n' + views.text_table(dmg, ext='org'))
def postproc(self):
"""
Build aggregate loss curves in process
"""
dstore = self.datastore
self.before_export() # set 'realizations'
oq = self.oqparam
stats = self.param['stats']
# store avg_losses-stats
if oq.avg_losses:
set_rlzs_stats(self.datastore, 'avg_losses')
try:
b = self.param['builder']
except KeyError: # don't build auxiliary tables
return
if dstore.parent:
dstore.parent.open('r') # to read the ruptures
if 'ruptures' in self.datastore and len(self.datastore['ruptures']):
logging.info('Building loss tables')
with self.monitor('building loss tables', measuremem=True):
rlt, lbr = build_loss_tables(dstore)
dstore['rup_loss_table'] = rlt
dstore['losses_by_rlzi'] = lbr
ridx = [rlt[:, lti].argmax() for lti in range(self.L)]
dstore.set_attrs('rup_loss_table', ridx=ridx)
logging.info('Building aggregate loss curves')
with self.monitor('building agg_curves', measuremem=True):
lbr = group_array(dstore['losses_by_event'][()], 'rlzi')
dic = {r: arr['loss'] for r, arr in lbr.items()}
array, arr_stats = b.build(dic, stats)
loss_types = ' '.join(oq.loss_dt().names)
units = self.datastore['cost_calculator'].get_units(loss_types.split())
if oq.individual_curves or self.R == 1:
self.datastore['agg_curves-rlzs'] = array
self.datastore.set_attrs(
'agg_curves-rlzs',
return_periods=b.return_periods,
loss_types=loss_types, units=units)
if arr_stats is not None:
self.datastore['agg_curves-stats'] = arr_stats
self.datastore.set_attrs(
'agg_curves-stats', return_periods=b.return_periods,
stats=[encode(name) for (name, func) in stats],
loss_types=loss_types, units=units)
def postproc(self):
"""
Build aggregate loss curves in process
"""
dstore = self.datastore
oq = self.oqparam
stats = self.param['stats']
# store avg_losses-stats
if oq.avg_losses:
set_rlzs_stats(self.datastore, 'avg_losses')
try:
b = self.param['builder']
except KeyError: # don't build auxiliary tables
return
if dstore.parent:
dstore.parent.open('r') # to read the ruptures
logging.info('Building loss tables')
build_loss_tables(dstore)
logging.info('Building aggregate loss curves')
with self.monitor('building agg_curves', measuremem=True):
lbr = group_array(dstore['losses_by_event'][()], 'rlzi')
dic = {r: arr['loss'] for r, arr in lbr.items()}
array, arr_stats = b.build(dic, stats)
loss_types = oq.loss_dt().names
units = self.datastore['cost_calculator'].get_units(loss_types)
if oq.individual_curves or self.R == 1:
self.datastore['agg_curves-rlzs'] = array # shape (P, R, L)
self.datastore.set_attrs(
'agg_curves-rlzs',
shape_descr=['return_periods', 'rlzs', 'loss_types'],
return_periods=b.return_periods,
rlzs=numpy.arange(self.R),
loss_types=loss_types,
units=units)
if arr_stats is not None:
self.datastore['agg_curves-stats'] = arr_stats # shape (P, S, L)
self.datastore.set_attrs(
'agg_curves-stats',
shape_descr=['return_periods', 'stats', 'loss_types'],
return_periods=b.return_periods,
stats=[encode(name) for (name, func) in stats],
loss_types=loss_types,
units=units)
def post_execute(self, result):
"""
Save risk data and build the aggregate loss curves
"""
oq = self.oqparam
loss_types = oq.loss_dt().names
logging.info('Saving event loss table')
elt_dt = numpy.dtype([('event_id', U32), ('rlzi', U16),
('loss', (F32, (self.L, )))])
with self.monitor('saving event loss table', measuremem=True):
agglosses = numpy.fromiter(
((eid, rlz, losses)
for (eid, rlz), losses in zip(self.events, self.agglosses)
if losses.any()), elt_dt)
self.datastore['losses_by_event'] = agglosses
self.datastore.set_attrs('losses_by_event', loss_types=loss_types)
if oq.avg_losses:
set_rlzs_stats(self.datastore, 'avg_losses')
post_risk.PostRiskCalculator(oq, self.datastore.calc_id).run()
def post_execute(self, dummy):
oq = self.oqparam
A, L, Dc = self.dmgcsq.shape
dmgcsq = self.dmgcsq * oq.time_ratio
self.datastore['damages-rlzs'] = dmgcsq.reshape((A, 1, L, Dc))
set_rlzs_stats(self.datastore,
'damages',
asset_id=self.assetcol['id'],
loss_type=oq.loss_names,
dmg_state=['no_damage'] + self.crmodel.get_dmg_csq())
size = self.datastore.getsize('risk_by_event')
logging.info('Building aggregated curves from %s of risk_by_event',
general.humansize(size))
alt_df = self.datastore.read_df('risk_by_event')
del alt_df['event_id']
dic = general.AccumDict(accum=[])
columns = sorted(
set(alt_df.columns) - {'agg_id', 'loss_id', 'variance'})
periods = [0] + list(self.builder.return_periods)
for (agg_id,
loss_id), df in alt_df.groupby([alt_df.agg_id, alt_df.loss_id]):
tots = [df[col].sum() * oq.time_ratio for col in columns]
curves = [
self.builder.build_curve(df[col].to_numpy()) for col in columns
]
for p, period in enumerate(periods):
dic['agg_id'].append(agg_id)
dic['loss_id'].append(loss_id)
dic['return_period'].append(period)
if p == 0:
for col, tot in zip(columns, tots):
dic[col].append(tot)
else:
for col, curve in zip(columns, curves):
dic[col].append(curve[p - 1])
fix_dtype(dic, U16, ['agg_id'])
fix_dtype(dic, U8, ['loss_id'])
fix_dtype(dic, U32, ['return_period'])
fix_dtype(dic, F32, columns)
ls = ' '.join(self.crmodel.damage_states[1:])
self.datastore.create_df('aggcurves', dic.items(), limit_states=ls)
self.sanity_check(dmgcsq)
def postproc(self):
"""
Build aggregate loss curves in process
"""
dstore = self.datastore
self.before_export() # set 'realizations'
oq = self.oqparam
stats = oq.risk_stats()
# store avg_losses-stats
if oq.avg_losses:
set_rlzs_stats(self.datastore, 'avg_losses')
try:
b = self.param['builder']
except KeyError: # don't build auxiliary tables
return
if dstore.parent:
dstore.parent.open('r') # to read the ruptures
if 'ruptures' in self.datastore and len(self.datastore['ruptures']):
logging.info('Building loss tables')
with self.monitor('building loss tables', measuremem=True):
rlt, lbr = build_loss_tables(dstore)
dstore['rup_loss_table'] = rlt
dstore['losses_by_rlzi'] = lbr
ridx = [rlt[:, lti].argmax() for lti in range(self.L)]
dstore.set_attrs('rup_loss_table', ridx=ridx)
logging.info('Building aggregate loss curves')
with self.monitor('building agg_curves', measuremem=True):
array, arr_stats = b.build(dstore['losses_by_event'].value, stats)
units = self.assetcol.cost_calculator.get_units(
loss_types=array.dtype.names)
if oq.individual_curves:
self.datastore['agg_curves-rlzs'] = array
self.datastore.set_attrs('agg_curves-rlzs',
return_periods=b.return_periods,
units=units)
if arr_stats is not None:
self.datastore['agg_curves-stats'] = arr_stats
self.datastore.set_attrs(
'agg_curves-stats',
return_periods=b.return_periods,
stats=[encode(name) for (name, func) in stats],
units=units)
def post_execute(self, result):
"""
Save risk data and build the aggregate loss curves
"""
oq = self.oqparam
logging.info('Saving event loss table')
with self.monitor('saving event loss table', measuremem=True):
out = []
for eid, losses in zip(self.events, self.agglosses):
if losses.sum():
out.append((eid, 0) + tuple(losses))
arr = numpy.array(out, oq.alt_dt())
self.datastore.create_dframe('agg_loss_table',
[(n, arr[n])
for n in arr.dtype.names])
if oq.avg_losses:
set_rlzs_stats(self.datastore,
'avg_losses',
asset_id=self.assetcol['id'],
loss_type=oq.loss_names)
post_risk.PostRiskCalculator(oq, self.datastore.calc_id).run()
def post_execute(self, result):
"""
Compute stats for the aggregated distributions and save
the results on the datastore.
"""
loss_dt = self.oqparam.loss_dt()
L = len(loss_dt.names)
dtlist = [('event_id', U32), ('rlzi', U16), ('loss', (F32, (L,)))]
R = self.R
with self.monitor('saving outputs'):
A = len(self.assetcol)
# agg losses
res = result['agg']
E, L = res.shape
agglosses = numpy.zeros((R, L), stat_dt)
for r in range(R):
mean, std = scientific.mean_std(res[self.event_slice(r)])
agglosses[r]['mean'] = F32(mean)
agglosses[r]['stddev'] = F32(std)
# losses by asset
losses_by_asset = numpy.zeros((A, R, L), F32)
for (l, r, aid, avg) in result['avg']:
losses_by_asset[aid, r, l] = avg
self.datastore['avg_losses-rlzs'] = losses_by_asset
set_rlzs_stats(self.datastore, 'avg_losses',
asset_id=self.assetcol['id'],
loss_type=self.oqparam.loss_names)
self.datastore['agglosses'] = agglosses
# losses by event
lbe = numpy.zeros(E, dtlist)
lbe['event_id'] = range(E)
lbe['rlzi'] = (lbe['event_id'] //
self.oqparam.number_of_ground_motion_fields)
lbe['loss'] = res
self.datastore['losses_by_event'] = lbe
loss_types = self.oqparam.loss_dt().names
self.datastore.set_attrs('losses_by_event', loss_types=loss_types)
def post_execute(self, dummy):
"""
Compute and store average losses from the losses_by_event dataset,
and then loss curves and maps.
"""
oq = self.oqparam
if oq.avg_losses:
self.datastore['avg_losses-stats'].attrs['stats'] = [b'mean']
logging.info('Building loss tables')
build_loss_tables(self.datastore)
self.datastore.flush() # just to be sure
shp = self.get_shape(self.L) # (L, T...)
text = ' x '.join('%d(%s)' % (n, t)
for t, n in zip(oq.aggregate_by, shp[1:]))
logging.info('Producing %d(loss_types) x %s loss curves', self.L, text)
builder = get_loss_builder(self.datastore)
self.build_datasets(builder)
self.datastore.swmr_on()
args = [(self.datastore.filename, builder, oq.ses_ratio, rlzi)
for rlzi in range(self.R)]
acc = list(parallel.Starmap(postprocess, args, h5=self.datastore.hdf5))
for r, (curves, maps), agg_losses in acc:
if len(curves): # some realization can give zero contribution
self.datastore['agg_curves-rlzs'][:, r] = curves
if len(maps): # conditional_loss_poes can be empty
self.datastore['agg_maps-rlzs'][:, r] = maps
self.datastore['agg_losses-rlzs'][:, r] = agg_losses
if self.R > 1:
logging.info('Computing aggregate statistics')
set_rlzs_stats(self.datastore, 'agg_curves')
set_rlzs_stats(self.datastore, 'agg_losses')
if oq.conditional_loss_poes:
set_rlzs_stats(self.datastore, 'agg_maps')
# sanity check with the asset_loss_table
if oq.asset_loss_table and len(oq.aggregate_by) == 1:
alt = self.datastore['asset_loss_table'][()]
if alt.sum() == 0: # nothing was saved
return
logging.info('Checking the loss curves')
tags = getattr(self.assetcol.tagcol, oq.aggregate_by[0])[1:]
T = len(tags)
P = len(builder.return_periods)
# sanity check on the loss curves for simple tag aggregation
arr = self.assetcol.aggregate_by(oq.aggregate_by, alt)
# shape (T, E, L)
rlzs = self.datastore['events']['rlz_id']
curves = numpy.zeros((P, self.R, self.L, T))
for t in range(T):
for r in range(self.R):
for l in range(self.L):
curves[:, r, l,
t] = losses_by_period(arr[t, rlzs == r, l],
builder.return_periods,
builder.num_events[r],
builder.eff_time)
numpy.testing.assert_allclose(
curves, self.datastore['agg_curves-rlzs'][()])
def postproc(self):
"""
Build aggregate loss curves
"""
dstore = self.datastore
self.before_export() # set 'realizations'
oq = self.oqparam
eff_time = oq.investigation_time * oq.ses_per_logic_tree_path
if eff_time < 2:
logging.warn('eff_time=%s is too small to compute agg_curves',
eff_time)
return
stats = oq. risk_stats()
# store avg_losses-stats
if oq.avg_losses:
set_rlzs_stats(self.datastore, 'avg_losses')
b = get_loss_builder(self.datastore)
if 'ruptures' in dstore:
logging.info('Building loss tables')
with self.monitor('building loss tables', measuremem=True):
rlt, lbr = build_loss_tables(dstore)
dstore['rup_loss_table'] = rlt
dstore['losses_by_rlzi'] = lbr
ridx = [rlt[:, lti].argmax() for lti in range(self.L)]
dstore.set_attrs('rup_loss_table', ridx=ridx)
logging.info('Building aggregate loss curves')
with self.monitor('building agg_curves', measuremem=True):
array, arr_stats = b.build(dstore['losses_by_event'].value, stats)
self.datastore['agg_curves-rlzs'] = array
units = self.assetcol.units(loss_types=array.dtype.names)
self.datastore.set_attrs(
'agg_curves-rlzs', return_periods=b.return_periods, units=units)
if arr_stats is not None:
self.datastore['agg_curves-stats'] = arr_stats
self.datastore.set_attrs(
'agg_curves-stats', return_periods=b.return_periods,
stats=[encode(name) for (name, func) in stats], units=units)
def execute(self):
oq = self.oqparam
if oq.return_periods != [0]:
# setting return_periods = 0 disable loss curves
eff_time = oq.investigation_time * oq.ses_per_logic_tree_path
if eff_time < 2:
logging.warning(
'eff_time=%s is too small to compute loss curves',
eff_time)
return
logging.info('Building loss tables')
build_loss_tables(self.datastore)
shp = self.get_shape(self.L) # (L, T...)
text = ' x '.join('%d(%s)' % (n, t)
for t, n in zip(oq.aggregate_by, shp[1:]))
logging.info('Producing %d(loss_types) x %s loss curves', self.L, text)
builder = get_loss_builder(self.datastore)
self.build_datasets(builder, oq.aggregate_by, 'agg_')
if oq.aggregate_by:
self.build_datasets(builder, [], 'tot_')
self.datastore.swmr_on()
args = [(self.datastore, builder, oq.ses_ratio, rlzi)
for rlzi in range(self.R)]
acc = list(parallel.Starmap(post_risk, args, h5=self.datastore.hdf5))
for dic in acc:
r = dic['rlzi']
curves = dic['agg_curves']
agg_losses = dic['agg_losses']
if len(curves): # some realization can give zero contribution
self.datastore['agg_curves-rlzs'][:, r] = curves
self.datastore['agg_losses-rlzs'][:, r] = agg_losses
if oq.aggregate_by:
curves = dic['tot_curves']
tot_losses = dic['tot_losses']
if len(curves): # some realization can give zero contribution
self.datastore['tot_curves-rlzs'][:, r] = curves
self.datastore['tot_losses-rlzs'][:, r] = tot_losses
if self.R > 1:
logging.info('Computing aggregate statistics')
set_rlzs_stats(self.datastore, 'agg_curves')
set_rlzs_stats(self.datastore, 'agg_losses')
if oq.aggregate_by:
set_rlzs_stats(self.datastore, 'tot_curves')
set_rlzs_stats(self.datastore, 'tot_losses')
def post_execute(self, result):
"""
Compute stats for the aggregated distributions and save
the results on the datastore.
"""
if not result:
self.collapsed()
return
dstates = self.crmodel.damage_states
ltypes = self.crmodel.loss_types
L = self.L = len(ltypes)
R = self.R
D = len(dstates)
A = len(self.assetcol)
E = len(self.datastore['events'])
# reduction factor
_, msg1 = get_nbytes_msg(dict(A=A, E=E, L=L))
_, msg2 = get_nbytes_msg(
dict(nrows=len(self.datastore['dd_data/eid']), ncols=D + 2))
logging.info('Using %s\ninstead of %s', msg2, msg1)
# avg_ratio = ratio used when computing the averages
oq = self.oqparam
if oq.investigation_time: # event_based_damage
avg_ratio = numpy.array([oq.ses_ratio] * R)
else: # scenario_damage
avg_ratio = 1. / self.param['num_events']
# damage by asset
d_asset = numpy.zeros((A, R, L, D), F32)
for (l, r, a, tot) in result['d_asset']:
d_asset[a, r, l] = tot * avg_ratio[r]
self.datastore['damages-rlzs'] = d_asset
set_rlzs_stats(self.datastore,
'damages',
asset_id=self.assetcol['id'],
loss_type=oq.loss_names,
dmg_state=dstates)
self.sanity_check()
# damage by event: make sure the sum of the buildings is consistent
tot = self.assetcol['number'].sum()
dt = F32 if self.param['approx_ddd'] else U32
dbe = numpy.zeros((self.E, L, D), dt) # shape E, L, D
dbe[:, :, 0] = tot
for e, dmg_by_lt in result['d_event'].items():
for l, dmg in enumerate(dmg_by_lt):
dbe[e, l, 0] = tot - dmg.sum()
dbe[e, l, 1:] = dmg
self.datastore['dmg_by_event'] = dbe
# consequence distributions
del result['d_asset']
del result['d_event']
dtlist = [('event_id', U32), ('rlz_id', U16), ('loss', (F32, (L, )))]
rlz = self.datastore['events']['rlz_id']
for name, csq in result.items():
if name.startswith('avg_'):
c_asset = numpy.zeros((A, R, L), F32)
for (l, r, a, stat) in result[name]:
c_asset[a, r, l] = stat * avg_ratio[r]
self.datastore[name + '-rlzs'] = c_asset
set_rlzs_stats(self.datastore,
name,
asset_id=self.assetcol['id'],
loss_type=oq.loss_names)
elif name.endswith('_by_event'):
arr = numpy.zeros(len(csq), dtlist)
for i, (eid, loss) in enumerate(csq.items()):
arr[i] = (eid, rlz[eid], loss)
self.datastore[name] = arr
def post_execute(self, result):
"""
Compute stats for the aggregated distributions and save
the results on the datastore.
"""
if not result:
self.collapsed()
return
dstates = self.crmodel.damage_states
ltypes = self.crmodel.loss_types
L = len(ltypes)
R = self.R
D = len(dstates)
A = len(self.assetcol)
indices = self.datastore['dd_data/indices'][()]
if not len(self.datastore['dd_data/data']):
logging.warning('There is no damage at all!')
events_per_asset = (indices[:, 1] - indices[:, 0]).mean()
logging.info('Found ~%d dmg distributions per asset', events_per_asset)
# avg_ratio = ratio used when computing the averages
oq = self.oqparam
if oq.investigation_time: # event_based_damage
avg_ratio = oq.ses_ratio
else: # scenario_damage
avg_ratio = 1. / oq.number_of_ground_motion_fields
# damage by asset
d_asset = numpy.zeros((A, R, L, D), F32)
for (l, r, a, tot) in result['d_asset']:
d_asset[a, r, l] = tot
self.datastore['avg_damages-rlzs'] = d_asset * avg_ratio
set_rlzs_stats(self.datastore,
'avg_damages',
asset_id=self.assetcol['id'],
loss_type=oq.loss_names,
dmg_state=dstates)
# damage by event: make sure the sum of the buildings is consistent
tot = self.assetcol['number'].sum()
dt = F32 if self.param['approx_ddd'] else U32
dbe = numpy.zeros((self.E, L, D), dt) # shape E, L, D
dbe[:, :, 0] = tot
for e, dmg_by_lt in result['d_event'].items():
for l, dmg in enumerate(dmg_by_lt):
dbe[e, l, 0] = tot - dmg.sum()
dbe[e, l, 1:] = dmg
self.datastore['dmg_by_event'] = dbe
# consequence distributions
del result['d_asset']
del result['d_event']
dtlist = [('event_id', U32), ('rlz_id', U16), ('loss', (F32, (L, )))]
rlz = self.datastore['events']['rlz_id']
for name, csq in result.items():
if name.startswith('avg_'):
c_asset = numpy.zeros((A, R, L), F32)
for (l, r, a, stat) in result[name]:
c_asset[a, r, l] = stat
self.datastore[name + '-rlzs'] = c_asset * avg_ratio
set_rlzs_stats(self.datastore,
name,
asset_id=self.assetcol['id'],
loss_type=oq.loss_names)
elif name.endswith('_by_event'):
arr = numpy.zeros(len(csq), dtlist)
for i, (eid, loss) in enumerate(csq.items()):
arr[i] = (eid, rlz[eid], loss)
self.datastore[name] = arr
def execute(self):
oq = self.oqparam
if oq.return_periods != [0]:
# setting return_periods = 0 disable loss curves
eff_time = oq.investigation_time * oq.ses_per_logic_tree_path
if eff_time < 2:
logging.warning(
'eff_time=%s is too small to compute loss curves',
eff_time)
return
if 'source_info' in self.datastore: # missing for gmf_ebrisk
logging.info('Building src_loss_table')
source_ids, losses = get_src_loss_table(self.datastore, self.L)
self.datastore['src_loss_table'] = losses
self.datastore.set_shape_attrs('src_loss_table',
source=source_ids,
loss_type=oq.loss_names)
shp = self.get_shape(self.L) # (L, T...)
text = ' x '.join('%d(%s)' % (n, t)
for t, n in zip(oq.aggregate_by, shp[1:]))
logging.info('Producing %d(loss_types) x %s loss curves', self.L, text)
builder = get_loss_builder(self.datastore)
if oq.aggregate_by:
self.build_datasets(builder, oq.aggregate_by, 'agg_')
self.build_datasets(builder, [], 'app_')
self.build_datasets(builder, [], 'tot_')
ds = self.datastore
if oq.aggregate_by:
aggkeys = list(ds['event_loss_table'])
ds.swmr_on()
smap = parallel.Starmap(post_ebrisk, [(self.datastore, aggkey)
for aggkey in aggkeys],
h5=self.datastore.hdf5)
else:
smap = ()
# do everything in process since it is really fast
elt = ds.read_df('losses_by_event', ['event_id', 'rlzi'])
for r, curves, losses in builder.gen_curves_by_rlz(elt, oq.ses_ratio):
ds['tot_curves-rlzs'][:, r] = curves # PL
ds['tot_losses-rlzs'][:, r] = losses # L
for res in smap:
if not res:
continue
for r, dic in res.items():
if oq.aggregate_by:
ds['agg_curves-rlzs'][(slice(None), r, slice(None)) +
dic['idx'] # PRLT..
] = dic['agg_curves']
ds['agg_losses-rlzs'][(slice(None), r) +
dic['idx'] # LRT...
] = dic['agg_losses']
ds['app_curves-rlzs'][:, r] += dic['agg_curves'] # PL
units = self.datastore['cost_calculator'].get_units(oq.loss_names)
aggby = {
tagname: encode(getattr(self.tagcol, tagname)[1:])
for tagname in oq.aggregate_by
}
set_rlzs_stats(self.datastore,
'app_curves',
return_periods=builder.return_periods,
loss_types=oq.loss_names,
**aggby,
units=units)
set_rlzs_stats(self.datastore,
'tot_curves',
return_periods=builder.return_periods,
loss_types=oq.loss_names,
**aggby,
units=units)
set_rlzs_stats(self.datastore,
'tot_losses',
loss_types=oq.loss_names,
**aggby,
units=units)
if oq.aggregate_by:
set_rlzs_stats(self.datastore,
'agg_curves',
return_periods=builder.return_periods,
loss_types=oq.loss_names,
**aggby,
units=units)
set_rlzs_stats(self.datastore,
'agg_losses',
loss_types=oq.loss_names,
**aggby,
units=units)
return oq.aggregate_by
def execute(self):
oq = self.oqparam
if oq.return_periods != [0]:
# setting return_periods = 0 disable loss curves
eff_time = oq.investigation_time * oq.ses_per_logic_tree_path
if eff_time < 2:
logging.warning(
'eff_time=%s is too small to compute loss curves',
eff_time)
return
if 'source_info' in self.datastore: # missing for gmf_ebrisk
logging.info('Building the src_loss_table')
with self.monitor('src_loss_table', measuremem=True):
source_ids, losses = get_src_loss_table(self.datastore, self.L)
self.datastore['src_loss_table'] = losses
self.datastore.set_shape_descr('src_loss_table',
source=source_ids,
loss_type=oq.loss_names)
builder = get_loss_builder(self.datastore)
K = len(self.aggkey) if oq.aggregate_by else 0
P = len(builder.return_periods)
# do everything in process since it is really fast
rlz_id = self.datastore['events']['rlz_id']
if oq.collect_rlzs:
rlz_id = numpy.zeros_like(rlz_id)
alt_df = self.datastore.read_df('agg_loss_table')
if self.reaggreate:
idxs = numpy.concatenate([
reagg_idxs(self.num_tags, oq.aggregate_by),
numpy.array([K], int)
])
alt_df['agg_id'] = idxs[alt_df['agg_id'].to_numpy()]
alt_df = alt_df.groupby(['event_id', 'agg_id']).sum().reset_index()
alt_df['rlz_id'] = rlz_id[alt_df.event_id.to_numpy()]
units = self.datastore['cost_calculator'].get_units(oq.loss_names)
dist = (
'no' if os.environ.get('OQ_DISTRIBUTE') == 'no' else 'processpool'
) # use only the local cores
smap = parallel.Starmap(post_risk,
h5=self.datastore.hdf5,
distribute=dist)
# producing concurrent_tasks/2 = num_cores tasks
blocksize = int(
numpy.ceil((K + 1) * self.R / (oq.concurrent_tasks // 2 or 1)))
krl_losses = []
agg_losses = numpy.zeros((K + 1, self.R, self.L), F32)
agg_curves = numpy.zeros((K + 1, self.R, self.L, P), F32)
gb = alt_df.groupby([alt_df.agg_id, alt_df.rlz_id, alt_df.loss_id])
# NB: in the future we may use multiprocessing.shared_memory
for (k, r, lni), df in gb:
agg_losses[k, r, lni] = df.loss.sum()
krl_losses.append((k, r, lni, df.loss.to_numpy()))
if len(krl_losses) >= blocksize:
size = len(krl_losses) * 8
logging.info('Sending %s of losses', general.humansize(size))
smap.submit((builder, krl_losses))
krl_losses[:] = []
if krl_losses:
smap.submit((builder, krl_losses))
for krl, curve in smap.reduce().items():
agg_curves[krl] = curve
R = len(self.datastore['weights'])
time_ratio = oq.time_ratio / R if oq.collect_rlzs else oq.time_ratio
self.datastore['agg_losses-rlzs'] = agg_losses * time_ratio
set_rlzs_stats(self.datastore,
'agg_losses',
agg_id=K + 1,
loss_types=oq.loss_names,
units=units)
self.datastore['agg_curves-rlzs'] = agg_curves
set_rlzs_stats(self.datastore,
'agg_curves',
agg_id=K + 1,
lti=self.L,
return_period=builder.return_periods,
units=units)
return 1
def post_execute(self, times):
"""
Compute and store average losses from the losses_by_event dataset,
and then loss curves and maps.
"""
if len(times):
self.datastore.set_attrs(
'task_info/start_ebrisk', times=times,
events_per_sid=numpy.mean(self.events_per_sid))
oq = self.oqparam
shp = self.get_shape(self.L) # (L, T...)
text = ' x '.join(
'%d(%s)' % (n, t) for t, n in zip(oq.aggregate_by, shp[1:]))
logging.info('Producing %d(loss_types) x %s loss curves', self.L, text)
builder = get_loss_builder(self.datastore)
self.build_datasets(builder)
self.datastore.close()
if 'losses_by_event' in self.datastore.parent:
dstore = self.datastore.parent
else:
dstore = self.datastore
allargs = [(dstore.filename, builder, rlzi) for rlzi in range(self.R)]
mon = performance.Monitor(hdf5=hdf5.File(self.datastore.hdf5cache()))
acc = list(parallel.Starmap(compute_loss_curves_maps, allargs, mon))
# copy performance information from the cache to the datastore
pd = mon.hdf5['performance_data'][()]
hdf5.extend3(self.datastore.filename, 'performance_data', pd)
self.datastore.open('r+') # reopen
self.datastore['task_info/compute_loss_curves_and_maps'] = (
mon.hdf5['task_info/compute_loss_curves_maps'][()])
self.datastore.open('r+')
with self.monitor('saving loss_curves and maps', autoflush=True):
for r, (curves, maps) in acc:
if len(curves): # some realization can give zero contribution
self.datastore['agg_curves-rlzs'][:, r] = curves
if len(maps): # conditional_loss_poes can be empty
self.datastore['agg_maps-rlzs'][:, r] = maps
if self.R > 1:
logging.info('Computing aggregate loss curves statistics')
set_rlzs_stats(self.datastore, 'agg_curves')
self.datastore.set_attrs(
'agg_curves-stats', return_periods=builder.return_periods,
loss_types=' '.join(self.riskmodel.loss_types))
if oq.conditional_loss_poes:
logging.info('Computing aggregate loss maps statistics')
set_rlzs_stats(self.datastore, 'agg_maps')
# sanity check with the asset_loss_table
if oq.asset_loss_table and len(oq.aggregate_by) == 1:
alt = self.datastore['asset_loss_table'][()]
if alt.sum() == 0: # nothing was saved
return
logging.info('Checking the loss curves')
tags = getattr(self.assetcol.tagcol, oq.aggregate_by[0])[1:]
T = len(tags)
P = len(builder.return_periods)
# sanity check on the loss curves for simple tag aggregation
arr = self.assetcol.aggregate_by(oq.aggregate_by, alt)
# shape (T, E, L)
rlzs = self.datastore['events']['rlz']
curves = numpy.zeros((P, self.R, self.L, T))
for t in range(T):
for r in range(self.R):
for l in range(self.L):
curves[:, r, l, t] = losses_by_period(
arr[t, rlzs == r, l],
builder.return_periods,
builder.num_events[r],
builder.eff_time)
numpy.testing.assert_allclose(
curves, self.datastore['agg_curves-rlzs'][()])
def post_execute(self, times):
"""
Compute and store average losses from the losses_by_event dataset,
and then loss curves and maps.
"""
if len(times):
self.datastore.set_attrs('task_info/start_ebrisk',
times=times,
events_per_sid=numpy.mean(
self.events_per_sid))
oq = self.oqparam
shp = self.get_shape(self.L) # (L, T...)
text = ' x '.join('%d(%s)' % (n, t)
for t, n in zip(oq.aggregate_by, shp[1:]))
logging.info('Producing %d(loss_types) x %s loss curves', self.L, text)
builder = get_loss_builder(self.datastore)
self.build_datasets(builder)
self.datastore.close()
if 'losses_by_event' in self.datastore.parent:
dstore = self.datastore.parent
else:
dstore = self.datastore
allargs = [(dstore.filename, builder, rlzi) for rlzi in range(self.R)]
mon = performance.Monitor(hdf5=hdf5.File(self.datastore.hdf5cache()))
acc = list(parallel.Starmap(compute_loss_curves_maps, allargs, mon))
# copy performance information from the cache to the datastore
pd = mon.hdf5['performance_data'][()]
hdf5.extend3(self.datastore.filename, 'performance_data', pd)
self.datastore.open('r+') # reopen
self.datastore['task_info/compute_loss_curves_and_maps'] = (
mon.hdf5['task_info/compute_loss_curves_maps'][()])
self.datastore.open('r+')
with self.monitor('saving loss_curves and maps', autoflush=True):
for r, (curves, maps) in acc:
if len(curves): # some realization can give zero contribution
self.datastore['agg_curves-rlzs'][:, r] = curves
if len(maps): # conditional_loss_poes can be empty
self.datastore['agg_maps-rlzs'][:, r] = maps
if self.R > 1:
logging.info('Computing aggregate loss curves statistics')
set_rlzs_stats(self.datastore, 'agg_curves')
self.datastore.set_attrs('agg_curves-stats',
return_periods=builder.return_periods,
loss_types=' '.join(
self.riskmodel.loss_types))
if oq.conditional_loss_poes:
logging.info('Computing aggregate loss maps statistics')
set_rlzs_stats(self.datastore, 'agg_maps')
# sanity check with the asset_loss_table
if oq.asset_loss_table and len(oq.aggregate_by) == 1:
alt = self.datastore['asset_loss_table'][()]
if alt.sum() == 0: # nothing was saved
return
logging.info('Checking the loss curves')
tags = getattr(self.assetcol.tagcol, oq.aggregate_by[0])[1:]
T = len(tags)
P = len(builder.return_periods)
# sanity check on the loss curves for simple tag aggregation
arr = self.assetcol.aggregate_by(oq.aggregate_by, alt)
# shape (T, E, L)
rlzs = self.datastore['events']['rlz']
curves = numpy.zeros((P, self.R, self.L, T))
for t in range(T):
for r in range(self.R):
for l in range(self.L):
curves[:, r, l,
t] = losses_by_period(arr[t, rlzs == r, l],
builder.return_periods,
builder.num_events[r],
builder.eff_time)
numpy.testing.assert_allclose(
curves, self.datastore['agg_curves-rlzs'][()])
def post_execute(self, result):
"""
Compute stats for the aggregated distributions and save
the results on the datastore.
"""
if not result:
self.collapsed()
return
dstates = self.crmodel.damage_states
ltypes = self.crmodel.loss_types
L = self.L = len(ltypes)
R = self.R
D = len(dstates)
A = len(self.assetcol)
E = len(self.datastore['events'])
# reduction factor
matrixsize = A * E * L * 4
realsize = self.datastore.getsize('risk_by_event')
logging.info('Saving %s in risk_by_event (instead of %s)',
humansize(realsize), humansize(matrixsize))
# avg_ratio = ratio used when computing the averages
oq = self.oqparam
avg_ratio = 1. / self.param['num_events']
# damage by asset
d_asset = numpy.zeros((A, R, L, D), F32)
for (l, r, a, tot) in result['d_asset']:
d_asset[a, r, l] = tot * avg_ratio[r]
self.datastore['damages-rlzs'] = d_asset
set_rlzs_stats(self.datastore,
'damages',
asset_id=self.assetcol['id'],
loss_type=oq.loss_names,
dmg_state=dstates)
tot = self.assetcol['value-number'].sum()
dt = F32 if self.param['float_dmg_dist'] else U32
dbe = numpy.zeros((self.E, L, D), dt) # shape E, L, D
dbe[:, :, 0] = tot
alt = self.datastore.read_df('risk_by_event')
df = alt.groupby(['event_id', 'loss_id']).sum().reset_index()
df['agg_id'] = A
for col in df.columns:
hdf5.extend(self.datastore['risk_by_event/' + col], df[col])
self.datastore.set_attrs('risk_by_event', K=A)
self.sanity_check()
# consequence distributions
del result['d_asset']
del result['d_event']
for name, csq in result.items():
# name is something like avg_losses
c_asset = numpy.zeros((A, R, L), F32)
for (l, r, a, stat) in result[name]:
c_asset[a, r, l] = stat * avg_ratio[r]
self.datastore[name + '-rlzs'] = c_asset
set_rlzs_stats(self.datastore,
name,
asset_id=self.assetcol['id'],
loss_type=oq.loss_names)
