def view_portfolio_damage(token, dstore):
"""
The mean full portfolio damage for each loss type,
extracted from the average damages
"""
# dimensions assets, stat, loss_types, dmg_state
if 'damages-stats' in dstore:
attrs = get_shape_descr(dstore['damages-stats'].attrs['json'])
arr = dstore.sel('damages-stats', stat='mean').sum(axis=(0, 1))
else:
attrs = get_shape_descr(dstore['damages-rlzs'].attrs['json'])
arr = dstore.sel('damages-rlzs', rlz=0).sum(axis=(0, 1))
rows = [[lt] + list(row) for lt, row in zip(attrs['loss_type'], arr)]
return rst_table(rows, ['loss_type'] + list(attrs['dmg_state']))
def export_agg_curve_rlzs(ekey, dstore):
oq = dstore['oqparam']
lnames = numpy.array(oq.loss_names)
agg_tags = get_agg_tags(dstore, oq.aggregate_by)
aggvalue = dstore['agg_values'][()] # shape (K+1, L)
md = dstore.metadata
md['risk_investigation_time'] = (
oq.risk_investigation_time or oq.investigation_time)
writer = writers.CsvWriter(fmt=writers.FIVEDIGITS)
descr = hdf5.get_shape_descr(dstore[ekey[0]].attrs['json'])
name, suffix = ekey[0].split('-')
rlzs_or_stats = descr[suffix[:-1]]
aw = hdf5.ArrayWrapper(dstore[ekey[0]], descr, ('loss_value',))
dataf = aw.to_dframe().set_index(suffix[:-1])
for r, ros in enumerate(rlzs_or_stats):
md['kind'] = f'{name}-' + (
ros if isinstance(ros, str) else 'rlz-%03d' % ros)
try:
df = dataf[dataf.index == ros]
except KeyError:
logging.warning('No data for %s', md['kind'])
continue
dic = {col: df[col].to_numpy() for col in dataf.columns}
dic['loss_type'] = lnames[dic['lti']]
for tagname in oq.aggregate_by:
dic[tagname] = agg_tags[tagname][dic['agg_id']]
dic['loss_ratio'] = dic['loss_value'] / aggvalue[
dic['agg_id'], dic.pop('lti')]
dic['annual_frequency_of_exceedence'] = 1 / dic['return_period']
del dic['agg_id']
dest = dstore.build_fname(md['kind'], '', 'csv')
writer.save(pandas.DataFrame(dic), dest, comment=md)
return writer.getsaved()
def dset2df(dset, indexfield, filterdict):
"""
Converts an HDF5 dataset with an attribute shape_descr into a Pandas
dataframe. NB: this is very slow for large datasets.
"""
arr = sel(dset, filterdict)
dic = hdf5.get_shape_descr(dset.attrs['json'])
tags = []
idxs = []
for dim in dic['shape_descr']:
values = dic[dim]
if dim in filterdict:
val = filterdict[dim]
idx = values.index(val)
idxs.append([idx])
values = [val]
elif hasattr(values, 'stop'): # a range object already
idxs.append(values)
else:
idxs.append(range(len(values)))
tags.append(values)
acc = general.AccumDict(accum=[])
index = []
for idx, vals in zip(itertools.product(*idxs), itertools.product(*tags)):
for field, val in zip(dic['shape_descr'], vals):
if field == indexfield:
index.append(val)
else:
acc[field].append(val)
acc['value'].append(arr[idx])
return pandas.DataFrame(acc, index or None)
def portfolio_damage_error(dstore, avg=None):
"""
The damages and errors for the full portfolio, extracted from
the asset damage table.
"""
df = dstore.read_df('dd_data', 'eid')
dset = dstore.getitem('damages-rlzs')
dic = get_shape_descr(dset.attrs['json'])
A, R, L, D = dset.shape
dmg_states = dic['dmg_state']
loss_types = dic['loss_type']
sums = numpy.zeros((10, L, D - 1))
for i in range(10):
section = df[df.index % 10 == i]
for l, grp in section.groupby('lid'):
ser = grp.sum() # aid, lid, ds...
sums[i, l, :] = numpy.array(ser)[2:]
if avg is None:
if 'damages-stats' in dstore:
arr = dstore.sel('damages-stats', stat='mean')
else:
arr = dstore.sel('damages-rlzs', rlz=0) # shape (A, 1, L, D)
avg = arr.sum(axis=(0, 1))[:, 1:] # shape (L, D)
errors = avg * numpy.std(sums, axis=0) / numpy.mean(sums, axis=0)
dic = dict(dmg_state=[], loss_type=[], mean=[], error=[])
for l, lt in enumerate(loss_types):
for d, dmg in enumerate(dmg_states[1:]):
dic['dmg_state'].append(dmg)
dic['loss_type'].append(lt)
dic['mean'].append(avg[l, d])
dic['error'].append(errors[l, d])
return pandas.DataFrame(dic)
def extract_tot_curves(dstore, what):
"""
Aggregate loss curves from the ebrisk calculator:
/extract/tot_curves?
kind=stats&absolute=1&loss_type=occupants
Returns an array of shape (P, S) or (P, R)
"""
info = get_info(dstore)
qdic = parse(what, info)
k = qdic['k'] # rlz or stat index
[l] = qdic['loss_type'] # loss type index
if qdic['rlzs']:
kinds = ['rlz-%d' % r for r in k]
name = 'agg_curves-rlzs'
else:
kinds = list(info['stats'])
name = 'agg_curves-stats'
shape_descr = hdf5.get_shape_descr(dstore.get_attr(name, 'json'))
units = dstore.get_attr(name, 'units')
rps = shape_descr['return_period']
K = shape_descr.get('K', 0)
arr = dstore[name][K, k, l].T # shape P, R
if qdic['absolute'] == [1]:
pass
elif qdic['absolute'] == [0]: # relative
arr /= dstore['agg_values'][K, l]
else:
raise ValueError('"absolute" must be 0 or 1 in %s' % what)
attrs = dict(shape_descr=['return_period', 'kind'])
attrs['return_period'] = rps
attrs['kind'] = kinds
attrs['units'] = list(units) # used by the QGIS plugin
return ArrayWrapper(arr, dict(json=hdf5.dumps(attrs)))
def extract_agg_curves(dstore, what):
"""
Aggregate loss curves from the ebrisk calculator:
/extract/agg_curves?
kind=stats&absolute=1&loss_type=occupants&occupancy=RES
Returns an array of shape (P, S, 1...) or (P, R, 1...)
"""
info = get_info(dstore)
qdic = parse(what, info)
tagdict = qdic.copy()
for a in ('k', 'rlzs', 'kind', 'loss_type', 'absolute'):
del tagdict[a]
k = qdic['k'] # rlz or stat index
[l] = qdic['loss_type'] # loss type index
tagnames = sorted(tagdict)
if set(tagnames) != set(info['tagnames']):
raise ValueError('Expected tagnames=%s, got %s' %
(info['tagnames'], tagnames))
tagvalues = [tagdict[t][0] for t in tagnames]
idx = -1
if tagnames:
for i, tags in enumerate(dstore['agg_keys'][:][tagnames]):
if list(tags) == tagvalues:
idx = i
break
if qdic['rlzs']:
kinds = ['rlz-%d' % r for r in k]
name = 'agg_curves-rlzs'
else:
kinds = list(info['stats'])
name = 'agg_curves-stats'
units = dstore.get_attr(name, 'units')
shape_descr = hdf5.get_shape_descr(dstore.get_attr(name, 'json'))
units = dstore.get_attr(name, 'units')
rps = shape_descr['return_period']
tup = (idx, k, l)
arr = dstore[name][tup].T # shape P, R
if qdic['absolute'] == [1]:
pass
elif qdic['absolute'] == [0]:
evalue = dstore['agg_values'][idx, l] # shape K, L
arr /= evalue
else:
raise ValueError('"absolute" must be 0 or 1 in %s' % what)
attrs = dict(shape_descr=['return_period', 'kind'] + tagnames)
attrs['return_period'] = list(rps)
attrs['kind'] = kinds
attrs['units'] = list(units) # used by the QGIS plugin
for tagname, tagvalue in zip(tagnames, tagvalues):
attrs[tagname] = [tagvalue]
if tagnames:
arr = arr.reshape(arr.shape + (1,) * len(tagnames))
return ArrayWrapper(arr, dict(json=hdf5.dumps(attrs)))
def view_portfolio_damage(token, dstore):
"""
The mean full portfolio damage for each loss type,
extracted from the average damages
"""
if 'aggcurves' in dstore: # event_based_damage
K = dstore.get_attr('risk_by_event', 'K')
df = dstore.read_df('aggcurves', sel=dict(agg_id=K, return_period=0))
lnames = numpy.array(dstore['oqparam'].loss_names)
df['loss_type'] = lnames[df.loss_id.to_numpy()]
del df['loss_id']
del df['agg_id']
del df['return_period']
return df.set_index('loss_type')
# dimensions assets, stat, loss_types, dmg_state
if 'damages-stats' in dstore:
attrs = get_shape_descr(dstore['damages-stats'].attrs['json'])
arr = dstore.sel('damages-stats', stat='mean').sum(axis=(0, 1))
else:
attrs = get_shape_descr(dstore['damages-rlzs'].attrs['json'])
arr = dstore.sel('damages-rlzs', rlz=0).sum(axis=(0, 1))
rows = [(lt, ) + tuple(row) for lt, row in zip(attrs['loss_type'], arr)]
return numpy.array(rows, dt(['loss_type'] + list(attrs['dmg_state'])))
def sel(dset, filterdict):
"""
Select a dataset with shape_descr. For instance
dstore.sel('hcurves', imt='PGA', sid=2)
"""
dic = hdf5.get_shape_descr(dset.attrs['json'])
lst = []
for dim in dic['shape_descr']:
if dim in filterdict:
val = filterdict[dim]
values = dic[dim]
idx = values.index(val)
lst.append(slice(idx, idx + 1))
else:
lst.append(slice(None))
return dset[tuple(lst)]
def print_(aw):
if hasattr(aw, 'json'):
try:
attrs = hdf5.get_shape_descr(aw.json)
except KeyError: # no shape_descr, for instance for oqparam
print(json.dumps(json.loads(aw.json), indent=2))
return
vars(aw).update(attrs)
if hasattr(aw, 'shape_descr'):
print(text_table(aw.to_dframe(), ext='org'))
elif hasattr(aw, 'array'):
print(text_table(aw.array, ext='org'))
elif isinstance(aw, numpy.ndarray):
print(text_table(aw, ext='org'))
else:
print(aw)
def extract_disagg_by_src(dstore, what):
"""
Extract the disagg_by_src information Example:
http://127.0.0.1:8800/v1/calc/30/extract/disagg_by_src?site_id=0&imt_id=0&rlz_id=0&lvl_id=-1
"""
qdict = parse(what)
dic = hdf5.get_shape_descr(dstore['disagg_by_src'].attrs['json'])
src_id = dic['src_id']
f = norm(qdict, 'site_id rlz_id lvl_id imt_id'.split())
poe = dstore['disagg_by_src'][
f['site_id'], f['rlz_id'], f['imt_id'], f['lvl_id']]
arr = numpy.zeros(len(src_id), [('src_id', '<S16'), ('poe', '<f8')])
arr['src_id'] = src_id
arr['poe'] = poe
arr.sort(order='poe')
return ArrayWrapper(arr[::-1], dict(json=hdf5.dumps(f)))
def print_(aw):
if hasattr(aw, 'json'):
try:
attrs = hdf5.get_shape_descr(aw.json)
except KeyError: # no shape_descr, for instance for oqparam
print(json.dumps(json.loads(aw.json), indent=2))
return
vars(aw).update(attrs)
if hasattr(aw, 'shape_descr'):
print(rst_table(aw.to_dframe()))
elif hasattr(aw, 'array') and aw.dtype.names:
sio = io.StringIO()
write_csv(sio, aw.array)
print(sio.getvalue())
elif hasattr(aw, 'array'):
print(aw.array)
else:
print(aw)
