def export_curves_rlzs(self, aids, key): """ :returns: a dictionary key -> record of dtype loss_curve_dt """ if 'loss_curves-stats' in self.dstore: # classical_risk if self.R == 1: data = self.dstore['loss_curves-stats'][aids] # shape (A, R) else: data = self.dstore['loss_curves-rlzs'][aids] # shape (A, R) if key.startswith('rlz-'): rlzi = int(key[4:]) return {key: data[:, rlzi]} # else key == 'rlzs', returns all data return {'rlz-%03d' % rlzi: data[:, rlzi] for rlzi in range(self.R)} # otherwise event_based avalues = self.assetcol.values(aids) lrgetter = getters.LossRatiosGetter(self.dstore, aids) build = self.builder.build_rlz if key.startswith('rlz-'): rlzi = int(key[4:]) ratios = lrgetter.get(rlzi) return {'rlz-%03d' % rlzi: build(avalues, ratios, rlzi)} else: # key is 'rlzs', return a dictionary will all realizations # this may be disabled in the future unless an asset is specified dic = {} for rlzi in range(self.R): ratios = lrgetter.get(rlzi) dic['rlz-%03d' % rlzi] = build(avalues, ratios, rlzi) return dic
def execute(self): oq = self.oqparam R = len(self.loss_builder.weights) # build loss maps if 'all_loss_ratios' in self.datastore and oq.conditional_loss_poes: assetcol = self.datastore['assetcol'] stats = oq.risk_stats() builder = self.loss_builder A = len(assetcol) S = len(stats) P = len(builder.return_periods) # create loss_maps datasets self.datastore.create_dset('loss_maps-rlzs', self.loss_maps_dt, (A, R), fillvalue=None) if R > 1: self.datastore.create_dset('loss_maps-stats', self.loss_maps_dt, (A, S), fillvalue=None) self.datastore.set_attrs( 'loss_maps-stats', stats=[encode(name) for (name, func) in stats]) self.datastore.create_dset('curves-stats', oq.loss_dt(), (A, S, P), fillvalue=None) self.datastore.set_attrs( 'curves-stats', return_periods=builder.return_periods, stats=[encode(name) for (name, func) in stats]) mon = self.monitor('loss maps') lazy = ('all_loss_ratios' in self.datastore.parent and self.can_read_parent()) logging.info('Instantiating LossRatiosGetters') with self.monitor('building lrgetters', measuremem=True, autoflush=True): allargs = [] for aids in split_in_blocks(range(A), oq.concurrent_tasks): dstore = self.datastore.parent if lazy else self.datastore getter = getters.LossRatiosGetter(dstore, aids, lazy) # a lazy getter will read the loss_ratios from the workers # an eager getter reads the loss_ratios upfront allargs.append((assetcol.values(aids), builder, getter, stats, oq.conditional_loss_poes, mon)) if lazy: # avoid OSError: Can't read data (Wrong b-tree signature) self.datastore.parent.close() parallel.Starmap(build_curves_maps, allargs).reduce(self.save_curves_maps) if lazy: # the parent was closed, reopen it self.datastore.parent.open()
def export_asset_loss_table(ekey, dstore): """ Export in parallel the asset loss table from the datastore. NB1: for large calculation this may run out of memory NB2: due to an heisenbug in the parallel reading of .hdf5 files this works reliably only if the datastore has been created by a different process The recommendation is: *do not use this exporter*: rather, study its source code and write what you need. Every postprocessing is different. """ key, fmt = ekey oq = dstore['oqparam'] assetcol = dstore['assetcol'] arefs = assetcol.asset_refs avals = assetcol.values() loss_types = dstore.get_attr('all_loss_ratios', 'loss_types').split() dtlist = [(lt, F32) for lt in loss_types] if oq.insured_losses: for lt in loss_types: dtlist.append((lt + '_ins', F32)) lrs_dt = numpy.dtype([('rlzi', U16), ('losses', dtlist)]) fname = dstore.export_path('%s.%s' % ekey) monitor = performance.Monitor(key, fname) aids = range(len(assetcol)) allargs = [(getters.LossRatiosGetter(dstore, block), monitor) for block in split_in_blocks(aids, oq.concurrent_tasks)] dstore.close() # avoid OSError: Can't read data (Wrong b-tree signature) L = len(loss_types) with hdf5.File(fname, 'w') as f: nbytes = 0 total = numpy.zeros(len(dtlist), F32) for pairs in parallel.Starmap(get_loss_ratios, allargs): for aid, data in pairs: asset = assetcol[aid] avalue = avals[aid] for l, lt in enumerate(loss_types): aval = avalue[lt] for i in range(oq.insured_losses + 1): data['ratios'][:, l + L * i] *= aval aref = arefs[asset.ordinal] f[b'asset_loss_table/' + aref] = data.view(lrs_dt) total += data['ratios'].sum(axis=0) nbytes += data.nbytes f['asset_loss_table'].attrs['loss_types'] = ' '.join(loss_types) f['asset_loss_table'].attrs['total'] = total f['asset_loss_table'].attrs['nbytes'] = nbytes return [fname]