def export_aggcurves_csv(ekey, dstore): """ :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ oq = dstore['oqparam'] E = len(dstore['events']) R = len(dstore['weights']) lossnames = numpy.array(oq.loss_names) aggtags = get_agg_tags(dstore, oq.aggregate_by) df = dstore.read_df('aggcurves') for tagname, tags in aggtags.items(): df[tagname] = tags[df.agg_id] df['loss_type'] = lossnames[df.loss_id.to_numpy()] del df['loss_id'] dest1 = dstore.export_path('%s.%s' % ekey) dest2 = dstore.export_path('dmgcsq.%s' % ekey[1]) writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) md = dstore.metadata md['risk_investigation_time'] = (oq.risk_investigation_time or oq.investigation_time) md['num_events'] = E md['effective_time'] = ( oq.investigation_time * oq.ses_per_logic_tree_path * R) md['limit_states'] = dstore.get_attr('aggcurves', 'limit_states') dmg_states = ['nodamage'] + md['limit_states'].split() # aggregate damages/consequences dmgcsq = df[df.return_period == 0].set_index('agg_id') # length K+1 agg_number = dstore['agg_number'][dmgcsq.index.to_numpy()] dmgs = [col for col in df.columns if col.startswith('dmg_')] dmg0 = agg_number * E * oq.time_ratio - dmgcsq[dmgs].to_numpy().sum(axis=1) dmgcsq.insert(0, 'dmg_0', dmg0) dmgcsq.insert(0, 'number', agg_number) del dmgcsq['return_period'] writer.save(rename(dmgcsq, dmg_states), dest2, comment=md) # aggcurves del df['agg_id'] writer.save(rename(df[df.return_period > 0], dmg_states), dest1, comment=md) return [dest1, dest2]
def export_agg_maps_csv(ekey, dstore): name, kind = ekey[0].split('-') oq = dstore['oqparam'] tagcol = dstore['assetcol/tagcol'] agg_maps = dstore[ekey[0]][()] # shape (C, R, L, T...) R = agg_maps.shape[1] kinds = (['rlz-%03d' % r for r in range(R)] if ekey[0].endswith('-rlzs') else list(oq.hazard_stats())) clp = [str(p) for p in oq.conditional_loss_poes] dic = dict(tagnames=['clp', 'kind', 'loss_type'] + oq.aggregate_by, clp=['?'] + clp, kind=['?'] + kinds, loss_type=('?', ) + oq.loss_dt().names) for tagname in oq.aggregate_by: dic[tagname] = getattr(tagcol, tagname) aw = hdf5.ArrayWrapper(agg_maps, dic) writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) fname = dstore.export_path('%s.%s' % ekey) writer.save(aw.to_table(), fname) return [fname]
def export_loss_maps_csv(ekey, dstore): kind = ekey[0].split('-')[1] # rlzs or stats assets = get_assets(dstore) value = get_loss_maps(dstore, kind) oq = dstore['oqparam'] if kind == 'rlzs': rlzs_or_stats = dstore['full_lt'].get_realizations() else: rlzs_or_stats = oq.hazard_stats() writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) md = dstore.metadata for i, ros in enumerate(rlzs_or_stats): if hasattr(ros, 'ordinal'): # is a realization ros = 'rlz-%d' % ros.ordinal fname = dstore.build_fname('loss_maps', ros, ekey[1]) md.update( dict(kind=ros, risk_investigation_time=oq.risk_investigation_time)) writer.save(compose_arrays(assets, value[:, i]), fname, comment=md, renamedict=dict(id='asset_id')) return writer.getsaved()
def export_losses_by_asset(ekey, dstore): """ :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ oq = dstore['oqparam'] loss_dt = oq.loss_dt(stat_dt) losses_by_asset = dstore[ekey[0]][()] rlzs = dstore['csm_info'].get_rlzs_assoc().realizations assets = get_assets(dstore) writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) md = dstore.metadata md.update(dict(investigation_time=oq.investigation_time, risk_investigation_time=oq.risk_investigation_time)) for rlz in rlzs: losses = losses_by_asset[:, rlz.ordinal] dest = dstore.build_fname('losses_by_asset', rlz, 'csv') data = compose_arrays(assets, losses.copy().view(loss_dt)[:, 0]) writer.save(data, dest, comment=md, renamedict=dict(id='asset_id')) return writer.getsaved()
def export_gmf_data_csv(ekey, dstore): oq = dstore['oqparam'] rlzs_assoc = dstore['csm_info'].get_rlzs_assoc() imts = list(oq.imtls) if 'scenario' in oq.calculation_mode: imtls = oq.imtls gsims = [str(rlz.gsim_rlz) for rlz in rlzs_assoc.realizations] n_gmfs = oq.number_of_ground_motion_fields fields = ['%03d' % i for i in range(n_gmfs)] dt = numpy.dtype([(f, F32) for f in fields]) eids, gmfs_ = calc.get_gmfs(dstore) sitemesh = get_mesh(dstore['sitecol']) writer = writers.CsvWriter(fmt='%.5f') for gsim, gmfa in zip(gsims, gmfs_): # gmfa of shape (N, E, I) for imti, imt in enumerate(imtls): gmfs = numpy.zeros(len(gmfa), dt) for e, event in enumerate(dt.names): gmfs[event] = gmfa[:, e, imti] dest = dstore.build_fname('gmf', '%s-%s' % (gsim, imt), 'csv') data = util.compose_arrays(sitemesh, gmfs) writer.save(data, dest) return writer.getsaved() else: # event based eid = int(ekey[0].split('/')[1]) if '/' in ekey[0] else None gmfa = GmfDataGetter.gen_gmfs(dstore['gmf_data'], rlzs_assoc, eid) if eid is None: # new format fname = dstore.build_fname('gmf', 'data', 'csv') gmfa.sort(order=['rlzi', 'sid', 'eid']) writers.write_csv(fname, _expand_gmv(gmfa, imts)) return [fname] # old format for single eid fnames = [] for rlzi, array in group_array(gmfa, 'rlzi').items(): rlz = rlzs_assoc.realizations[rlzi] data, comment = _build_csv_data( array, rlz, dstore['sitecol'], imts, oq.investigation_time) fname = dstore.build_fname( 'gmf', '%d-rlz-%03d' % (eid, rlzi), 'csv') writers.write_csv(fname, data, comment=comment) fnames.append(fname) return fnames
def export_gmf_scenario(ekey, dstore): oq = dstore['oqparam'] if 'scenario' in oq.calculation_mode: fields = ['%03d' % i for i in range(len(dstore['etags']))] dt = numpy.dtype([(f, F32) for f in fields]) etags, gmfs_by_trt_gsim = calc.get_gmfs(dstore) sitemesh = dstore['sitemesh'] writer = writers.CsvWriter(fmt='%.5f') for (trt, gsim), gmfs_ in gmfs_by_trt_gsim.items(): for imt in gmfs_.dtype.names: gmfs = numpy.zeros(len(gmfs_), dt) for i in range(len(gmfs)): gmfs[i] = tuple(gmfs_[imt][i]) dest = dstore.export_path('gmf-%s-%s.csv' % (gsim, imt)) data = util.compose_arrays(sitemesh, gmfs) writer.save(data, dest) else: # event based logging.warn('Not exporting the full GMFs for event_based, but you can' ' specify the rupture ordinals with gmfs:R1,...,Rn') return [] return writer.getsaved()
def export_loss_maps_csv(ekey, dstore): kind = ekey[0].split('-')[1] # rlzs or stats assets = get_assets(dstore) value = get_loss_maps(dstore, kind) oq = dstore['oqparam'] if kind == 'rlzs': tags = dstore['csm_info'].get_rlzs_assoc().realizations else: tags = oq.hazard_stats() writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) md = dstore.metadata for i, tag in enumerate(tags): uid = getattr(tag, 'uid', tag) fname = dstore.build_fname('loss_maps', tag, ekey[1]) md.update( dict(kind=uid, risk_investigation_time=oq.risk_investigation_time)) writer.save(compose_arrays(assets, value[:, i]), fname, comment=md, renamedict=dict(id='asset_id')) return writer.getsaved()
def export_avg_losses(ekey, dstore): """ :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ dskey = ekey[0] oq = dstore['oqparam'] dt = [(ln, F32) for ln in oq.loss_names] name, value, tags = _get_data(dstore, dskey, oq.hazard_stats()) writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) assets = get_assets(dstore) md = dstore.metadata md.update( dict(investigation_time=oq.investigation_time, risk_investigation_time=oq.risk_investigation_time)) for tag, values in zip(tags, value.transpose(1, 0, 2)): dest = dstore.build_fname(name, tag, 'csv') array = numpy.zeros(len(values), dt) for li, ln in enumerate(oq.loss_names): array[ln] = values[:, li] writer.save(compose_arrays(assets, array), dest, comment=md) return writer.getsaved()
def export_losses_by_event(ekey, dstore): """ :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ oq = dstore['oqparam'] writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) dest = dstore.build_fname('losses_by_event', '', 'csv') if (oq.calculation_mode.startswith('scenario') or oq.calculation_mode == 'ebrisk'): tagcol = dstore['assetcol/tagcol'] lbe = dstore['losses_by_event'][()] lbe.sort(order='event_id') dic = dict(tagnames=['event_id'] + oq.aggregate_by) for tagname in oq.aggregate_by: dic[tagname] = getattr(tagcol, tagname) dic['event_id'] = ['?'] + list(lbe['event_id']) # example (0, 1, 2, 3) -> (0, 2, 3, 1) axis = [0] + list(range(2, len(lbe['loss'].shape))) + [1] data = lbe['loss'].transpose(axis) # shape (E, T..., L) aw = hdf5.ArrayWrapper(data, dic, oq.loss_names) writer.save(aw.to_table(), dest) else: dtlist = [('event_id', U64), ('rlz_id', U16), ('rup_id', U32), ('year', U32)] + oq.loss_dt_list() eids = dstore['losses_by_event']['event_id'] events = dstore['events'] year_of = year_dict(events['id'], oq.investigation_time, oq.ses_seed) arr = numpy.zeros(len(dstore['losses_by_event']), dtlist) arr['event_id'] = eids arr['rup_id'] = arr['event_id'] / TWO32 arr['rlz_id'] = get_rlz_ids(events, eids) arr['year'] = [year_of[eid] for eid in eids] loss = dstore['losses_by_event']['loss'].T # shape (L, E) for losses, loss_type in zip(loss, oq.loss_names): arr[loss_type] = losses writer.save(arr, dest) return writer.getsaved()
def export_losses_by_taxon_csv(ekey, dstore): oq = dstore['oqparam'] taxonomies = add_quotes(dstore['assetcol/taxonomies'].value) rlzs = dstore['csm_info'].get_rlzs_assoc().realizations loss_types = oq.loss_dt().names key, kind = ekey[0].split('-') value = dstore[key + '-rlzs'].value if kind == 'stats': weights = dstore['realizations']['weight'] tags, stats = zip(*oq.risk_stats()) value = compute_stats2(value, stats, weights) else: # rlzs tags = rlzs writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) dt = numpy.dtype([('taxonomy', taxonomies.dtype)] + oq.loss_dt_list()) for tag, values in zip(tags, value.transpose(1, 0, 2)): fname = dstore.build_fname(key, tag, ekey[1]) array = numpy.zeros(len(values), dt) array['taxonomy'] = taxonomies for l, lt in enumerate(loss_types): array[lt] = values[:, l] writer.save(array, fname) return writer.getsaved()
def export_avg_losses(ekey, dstore): """ :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ oq = dstore['oqparam'] dt = oq.loss_dt() assets = get_assets(dstore) writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) name, kind = ekey[0].split('-') value = dstore[name + '-rlzs'].value # shape (A, R, L') if kind == 'stats': weights = dstore['realizations']['weight'] tags, stats = zip(*oq.risk_stats()) value = compute_stats2(value, stats, weights) else: # rlzs tags = ['rlz-%03d' % r for r in range(len(dstore['realizations']))] for tag, values in zip(tags, value.transpose(1, 0, 2)): dest = dstore.build_fname(name, tag, 'csv') array = numpy.zeros(len(values), dt) for l, lt in enumerate(dt.names): array[lt] = values[:, l] writer.save(compose_arrays(assets, array), dest) return writer.getsaved()
def export_agg_curve_rlzs(ekey, dstore): oq = dstore['oqparam'] assetcol = dstore['assetcol'] if ekey[0].startswith('agg_'): aggregate_by = oq.aggregate_by else: # tot_curves aggregate_by = [] name = '_'.join(['agg'] + oq.aggregate_by) aggvalue = dstore['exposed_values/' + name][()] lti = tag2idx(oq.loss_names) tagi = { tagname: tag2idx(getattr(assetcol.tagcol, tagname)) for tagname in aggregate_by } def get_loss_ratio(rec): idxs = tuple(tagi[tagname][getattr(rec, tagname)] - 1 for tagname in aggregate_by) + (lti[rec.loss_types], ) return rec.loss_value / aggvalue[idxs] # shape (T1, T2, ..., L) md = dstore.metadata md.update( dict(kind=ekey[0], risk_investigation_time=oq.risk_investigation_time)) fname = dstore.export_path('%s.%s' % ekey) writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) aw = hdf5.ArrayWrapper.from_(dstore[ekey[0]], 'loss_value') table = add_columns( aw.to_table(), loss_ratio=get_loss_ratio, annual_frequency_of_exceedence=lambda rec: 1 / rec.return_periods) table[0] = [c[:-1] if c.endswith('s') else c for c in table[0]] writer.save(table, fname, comment=md) return writer.getsaved()
def export_agg_curve_rlzs(ekey, dstore): oq = dstore['oqparam'] lnames = numpy.array(oq.loss_names) if oq.aggregate_by: agg_keys = dstore['agg_keys'][:] agg_tags = {} for tagname in oq.aggregate_by: agg_tags[tagname] = numpy.concatenate([agg_keys[tagname], ['*total*']]) aggvalue = dstore['agg_values'][()] # shape (K+1, L) md = dstore.metadata md['risk_investigation_time'] = oq.risk_investigation_time writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) descr = hdf5.get_shape_descr(dstore[ekey[0]].attrs['json']) name, suffix = ekey[0].split('-') rlzs_or_stats = descr[suffix[:-1]] aw = hdf5.ArrayWrapper(dstore[ekey[0]], descr, ('loss_value', )) dataf = aw.to_dframe().set_index(suffix[:-1]) for r, ros in enumerate(rlzs_or_stats): md['kind'] = f'{name}-' + (ros if isinstance(ros, str) else 'rlz-%03d' % ros) try: df = dataf[dataf.index == ros] except KeyError: logging.warning('No data for %s', md['kind']) continue dic = {col: df[col].to_numpy() for col in dataf.columns} dic['loss_type'] = lnames[dic['lti']] for tagname in oq.aggregate_by: dic[tagname] = agg_tags[tagname][dic['agg_id']] dic['loss_ratio'] = dic['loss_value'] / aggvalue[dic['agg_id'], dic.pop('lti')] dic['annual_frequency_of_exceedence'] = 1 / dic['return_period'] del dic['agg_id'] dest = dstore.build_fname(md['kind'], '', 'csv') writer.save(pandas.DataFrame(dic), dest, comment=md) return writer.getsaved()
def export_agg_losses_ebr(ekey, dstore): """ :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ agg_losses = dstore[ekey[0]] rlzs = dstore['rlzs_assoc'].realizations loss_types = dstore['riskmodel'].loss_types tags = dstore['tags'].value ext_loss_types = loss_types + [lt + '_ins' for lt in loss_types] ext_dt = numpy.dtype([('tag', (bytes, 100))] + [(elt, numpy.float32) for elt in ext_loss_types]) writer = writers.CsvWriter(fmt='%10.6E') for rlz in rlzs: rows = agg_losses[rlz.uid] data = [] for row in rows: loss = row['loss'] # matrix L x 2 data.append((tags[row['rup_id']], ) + tuple(loss[:, 0]) + tuple(loss[:, 1])) data.sort() dest = dstore.export_path('agg_losses-rlz%03d.csv' % rlz.ordinal) writer.save(numpy.array(data, ext_dt), dest) return writer.getsaved()
def export_bcr_map(ekey, dstore): assetcol = dstore['assetcol/array'].value aref = dstore['asset_refs'].value bcr_data = dstore['bcr-rlzs'] N, R = bcr_data.shape realizations = dstore['csm_info'].get_rlzs_assoc().realizations loss_types = dstore.get_attr('composite_risk_model', 'loss_types') fnames = [] writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) for rlz in realizations: for l, loss_type in enumerate(loss_types): rlz_data = bcr_data[loss_type][:, rlz.ordinal] path = dstore.build_fname('bcr-%s' % loss_type, rlz, 'csv') data = [[ 'lon', 'lat', 'asset_ref', 'average_annual_loss_original', 'average_annual_loss_retrofitted', 'bcr' ]] for ass, value in zip(assetcol, rlz_data): data.append((ass['lon'], ass['lat'], decode(aref[ass['idx']]), value['annual_loss_orig'], value['annual_loss_retro'], value['bcr'])) writer.save(data, path) fnames.append(path) return writer.getsaved()
def export_src_loss_table(ekey, dstore): """ :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ oq = dstore['oqparam'] trts = dstore['full_lt'].trts trt_by_source_id = {} for rec in dstore['source_info']: trt_by_source_id[rec['source_id'][:16]] = trts[rec['trti']] def get_trt(row): return trt_by_source_id[row.source] md = dstore.metadata md.update( dict(investigation_time=oq.investigation_time, risk_investigation_time=oq.risk_investigation_time)) aw = hdf5.ArrayWrapper.from_(dstore['src_loss_table'], 'loss_value') dest = dstore.build_fname('src_loss_table', '', 'csv') writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) rows = add_columns(aw.to_table(), trt=get_trt) writer.save(rows, dest, comment=md) return writer.getsaved()
def export_dmg_by_event(ekey, dstore): """ :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ damage_dt = build_damage_dt(dstore) dt_list = [('event_id', U32), ('rlz_id', U16)] + [ (f, damage_dt.fields[f][0]) for f in damage_dt.names] dmg_by_event = dstore[ekey[0]][()] # shape E, L, D events = dstore['events'][()] writer = writers.CsvWriter(fmt='%g') fname = dstore.build_fname('dmg_by_event', '', 'csv') writer.save(numpy.zeros(0, dt_list), fname) with open(fname, 'a') as dest: for rlz_id in numpy.unique(events['rlz_id']): ok, = numpy.where(events['rlz_id'] == rlz_id) arr = numpy.zeros(len(ok), dt_list) arr['event_id'] = events['id'][ok] arr['rlz_id'] = rlz_id for li, loss_type in enumerate(damage_dt.names): for d, dmg_state in enumerate(damage_dt[loss_type].names): arr[loss_type][dmg_state] = dmg_by_event[ok, li, d] writer.save_block(arr, dest) return [fname]
def export_dmg_by_event(ekey, dstore): """ :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ damage_dt = build_damage_dt(dstore, mean_std=False) dt_list = [('event_id', numpy.uint64), ('rlzi', numpy.uint16)] + [ (f, damage_dt.fields[f][0]) for f in damage_dt.names] all_losses = dstore[ekey[0]].value # shape (E, R, LI) eids = dstore['events']['eid'] rlzs = dstore['csm_info'].get_rlzs_assoc().realizations writer = writers.CsvWriter(fmt=writers.FIVEDIGITS) fname = dstore.build_fname('dmg_by_event', '', 'csv') writer.save(numpy.zeros(0, dt_list), fname) with open(fname, 'ab') as dest: for rlz in rlzs: data = all_losses[:, rlz.ordinal].copy().view(damage_dt) # shape E arr = numpy.zeros(len(data), dt_list) arr['event_id'] = eids arr['rlzi'] = rlz.ordinal for field in damage_dt.names: arr[field] = data[field].squeeze() writer.save_block(arr, dest) return [fname]