def view_portfolio_damage(token, dstore): """ The mean full portfolio damage for each loss type, extracted from the average damages """ # dimensions assets, stat, loss_types, dmg_state if 'damages-stats' in dstore: attrs = get_shape_descr(dstore['damages-stats'].attrs['json']) arr = dstore.sel('damages-stats', stat='mean').sum(axis=(0, 1)) else: attrs = get_shape_descr(dstore['damages-rlzs'].attrs['json']) arr = dstore.sel('damages-rlzs', rlz=0).sum(axis=(0, 1)) rows = [[lt] + list(row) for lt, row in zip(attrs['loss_type'], arr)] return rst_table(rows, ['loss_type'] + list(attrs['dmg_state']))
def export_agg_curve_rlzs(ekey, dstore): oq = dstore['oqparam'] lnames = numpy.array(oq.loss_names) agg_tags = get_agg_tags(dstore, oq.aggregate_by) aggvalue = dstore['agg_values'][()] # shape (K+1, L) md = dstore.metadata md['risk_investigation_time'] = ( oq.risk_investigation_time or oq.investigation_time) writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) descr = hdf5.get_shape_descr(dstore[ekey[0]].attrs['json']) name, suffix = ekey[0].split('-') rlzs_or_stats = descr[suffix[:-1]] aw = hdf5.ArrayWrapper(dstore[ekey[0]], descr, ('loss_value',)) dataf = aw.to_dframe().set_index(suffix[:-1]) for r, ros in enumerate(rlzs_or_stats): md['kind'] = f'{name}-' + ( ros if isinstance(ros, str) else 'rlz-%03d' % ros) try: df = dataf[dataf.index == ros] except KeyError: logging.warning('No data for %s', md['kind']) continue dic = {col: df[col].to_numpy() for col in dataf.columns} dic['loss_type'] = lnames[dic['lti']] for tagname in oq.aggregate_by: dic[tagname] = agg_tags[tagname][dic['agg_id']] dic['loss_ratio'] = dic['loss_value'] / aggvalue[ dic['agg_id'], dic.pop('lti')] dic['annual_frequency_of_exceedence'] = 1 / dic['return_period'] del dic['agg_id'] dest = dstore.build_fname(md['kind'], '', 'csv') writer.save(pandas.DataFrame(dic), dest, comment=md) return writer.getsaved()
def dset2df(dset, indexfield, filterdict): """ Converts an HDF5 dataset with an attribute shape_descr into a Pandas dataframe. NB: this is very slow for large datasets. """ arr = sel(dset, filterdict) dic = hdf5.get_shape_descr(dset.attrs['json']) tags = [] idxs = [] for dim in dic['shape_descr']: values = dic[dim] if dim in filterdict: val = filterdict[dim] idx = values.index(val) idxs.append([idx]) values = [val] elif hasattr(values, 'stop'): # a range object already idxs.append(values) else: idxs.append(range(len(values))) tags.append(values) acc = general.AccumDict(accum=[]) index = [] for idx, vals in zip(itertools.product(*idxs), itertools.product(*tags)): for field, val in zip(dic['shape_descr'], vals): if field == indexfield: index.append(val) else: acc[field].append(val) acc['value'].append(arr[idx]) return pandas.DataFrame(acc, index or None)
def portfolio_damage_error(dstore, avg=None): """ The damages and errors for the full portfolio, extracted from the asset damage table. """ df = dstore.read_df('dd_data', 'eid') dset = dstore.getitem('damages-rlzs') dic = get_shape_descr(dset.attrs['json']) A, R, L, D = dset.shape dmg_states = dic['dmg_state'] loss_types = dic['loss_type'] sums = numpy.zeros((10, L, D - 1)) for i in range(10): section = df[df.index % 10 == i] for l, grp in section.groupby('lid'): ser = grp.sum() # aid, lid, ds... sums[i, l, :] = numpy.array(ser)[2:] if avg is None: if 'damages-stats' in dstore: arr = dstore.sel('damages-stats', stat='mean') else: arr = dstore.sel('damages-rlzs', rlz=0) # shape (A, 1, L, D) avg = arr.sum(axis=(0, 1))[:, 1:] # shape (L, D) errors = avg * numpy.std(sums, axis=0) / numpy.mean(sums, axis=0) dic = dict(dmg_state=[], loss_type=[], mean=[], error=[]) for l, lt in enumerate(loss_types): for d, dmg in enumerate(dmg_states[1:]): dic['dmg_state'].append(dmg) dic['loss_type'].append(lt) dic['mean'].append(avg[l, d]) dic['error'].append(errors[l, d]) return pandas.DataFrame(dic)
def extract_tot_curves(dstore, what): """ Aggregate loss curves from the ebrisk calculator: /extract/tot_curves? kind=stats&absolute=1&loss_type=occupants Returns an array of shape (P, S) or (P, R) """ info = get_info(dstore) qdic = parse(what, info) k = qdic['k'] # rlz or stat index [l] = qdic['loss_type'] # loss type index if qdic['rlzs']: kinds = ['rlz-%d' % r for r in k] name = 'agg_curves-rlzs' else: kinds = list(info['stats']) name = 'agg_curves-stats' shape_descr = hdf5.get_shape_descr(dstore.get_attr(name, 'json')) units = dstore.get_attr(name, 'units') rps = shape_descr['return_period'] K = shape_descr.get('K', 0) arr = dstore[name][K, k, l].T # shape P, R if qdic['absolute'] == [1]: pass elif qdic['absolute'] == [0]: # relative arr /= dstore['agg_values'][K, l] else: raise ValueError('"absolute" must be 0 or 1 in %s' % what) attrs = dict(shape_descr=['return_period', 'kind']) attrs['return_period'] = rps attrs['kind'] = kinds attrs['units'] = list(units) # used by the QGIS plugin return ArrayWrapper(arr, dict(json=hdf5.dumps(attrs)))
def extract_agg_curves(dstore, what): """ Aggregate loss curves from the ebrisk calculator: /extract/agg_curves? kind=stats&absolute=1&loss_type=occupants&occupancy=RES Returns an array of shape (P, S, 1...) or (P, R, 1...) """ info = get_info(dstore) qdic = parse(what, info) tagdict = qdic.copy() for a in ('k', 'rlzs', 'kind', 'loss_type', 'absolute'): del tagdict[a] k = qdic['k'] # rlz or stat index [l] = qdic['loss_type'] # loss type index tagnames = sorted(tagdict) if set(tagnames) != set(info['tagnames']): raise ValueError('Expected tagnames=%s, got %s' % (info['tagnames'], tagnames)) tagvalues = [tagdict[t][0] for t in tagnames] idx = -1 if tagnames: for i, tags in enumerate(dstore['agg_keys'][:][tagnames]): if list(tags) == tagvalues: idx = i break if qdic['rlzs']: kinds = ['rlz-%d' % r for r in k] name = 'agg_curves-rlzs' else: kinds = list(info['stats']) name = 'agg_curves-stats' units = dstore.get_attr(name, 'units') shape_descr = hdf5.get_shape_descr(dstore.get_attr(name, 'json')) units = dstore.get_attr(name, 'units') rps = shape_descr['return_period'] tup = (idx, k, l) arr = dstore[name][tup].T # shape P, R if qdic['absolute'] == [1]: pass elif qdic['absolute'] == [0]: evalue = dstore['agg_values'][idx, l] # shape K, L arr /= evalue else: raise ValueError('"absolute" must be 0 or 1 in %s' % what) attrs = dict(shape_descr=['return_period', 'kind'] + tagnames) attrs['return_period'] = list(rps) attrs['kind'] = kinds attrs['units'] = list(units) # used by the QGIS plugin for tagname, tagvalue in zip(tagnames, tagvalues): attrs[tagname] = [tagvalue] if tagnames: arr = arr.reshape(arr.shape + (1,) * len(tagnames)) return ArrayWrapper(arr, dict(json=hdf5.dumps(attrs)))
def view_portfolio_damage(token, dstore): """ The mean full portfolio damage for each loss type, extracted from the average damages """ if 'aggcurves' in dstore: # event_based_damage K = dstore.get_attr('risk_by_event', 'K') df = dstore.read_df('aggcurves', sel=dict(agg_id=K, return_period=0)) lnames = numpy.array(dstore['oqparam'].loss_names) df['loss_type'] = lnames[df.loss_id.to_numpy()] del df['loss_id'] del df['agg_id'] del df['return_period'] return df.set_index('loss_type') # dimensions assets, stat, loss_types, dmg_state if 'damages-stats' in dstore: attrs = get_shape_descr(dstore['damages-stats'].attrs['json']) arr = dstore.sel('damages-stats', stat='mean').sum(axis=(0, 1)) else: attrs = get_shape_descr(dstore['damages-rlzs'].attrs['json']) arr = dstore.sel('damages-rlzs', rlz=0).sum(axis=(0, 1)) rows = [(lt, ) + tuple(row) for lt, row in zip(attrs['loss_type'], arr)] return numpy.array(rows, dt(['loss_type'] + list(attrs['dmg_state'])))
def sel(dset, filterdict): """ Select a dataset with shape_descr. For instance dstore.sel('hcurves', imt='PGA', sid=2) """ dic = hdf5.get_shape_descr(dset.attrs['json']) lst = [] for dim in dic['shape_descr']: if dim in filterdict: val = filterdict[dim] values = dic[dim] idx = values.index(val) lst.append(slice(idx, idx + 1)) else: lst.append(slice(None)) return dset[tuple(lst)]
def print_(aw): if hasattr(aw, 'json'): try: attrs = hdf5.get_shape_descr(aw.json) except KeyError: # no shape_descr, for instance for oqparam print(json.dumps(json.loads(aw.json), indent=2)) return vars(aw).update(attrs) if hasattr(aw, 'shape_descr'): print(text_table(aw.to_dframe(), ext='org')) elif hasattr(aw, 'array'): print(text_table(aw.array, ext='org')) elif isinstance(aw, numpy.ndarray): print(text_table(aw, ext='org')) else: print(aw)
def extract_disagg_by_src(dstore, what): """ Extract the disagg_by_src information Example: http://127.0.0.1:8800/v1/calc/30/extract/disagg_by_src?site_id=0&imt_id=0&rlz_id=0&lvl_id=-1 """ qdict = parse(what) dic = hdf5.get_shape_descr(dstore['disagg_by_src'].attrs['json']) src_id = dic['src_id'] f = norm(qdict, 'site_id rlz_id lvl_id imt_id'.split()) poe = dstore['disagg_by_src'][ f['site_id'], f['rlz_id'], f['imt_id'], f['lvl_id']] arr = numpy.zeros(len(src_id), [('src_id', '<S16'), ('poe', '<f8')]) arr['src_id'] = src_id arr['poe'] = poe arr.sort(order='poe') return ArrayWrapper(arr[::-1], dict(json=hdf5.dumps(f)))
def print_(aw): if hasattr(aw, 'json'): try: attrs = hdf5.get_shape_descr(aw.json) except KeyError: # no shape_descr, for instance for oqparam print(json.dumps(json.loads(aw.json), indent=2)) return vars(aw).update(attrs) if hasattr(aw, 'shape_descr'): print(rst_table(aw.to_dframe())) elif hasattr(aw, 'array') and aw.dtype.names: sio = io.StringIO() write_csv(sio, aw.array) print(sio.getvalue()) elif hasattr(aw, 'array'): print(aw.array) else: print(aw)