def compute_store_stats(self, rlzs, builder): """ Compute and store the statistical outputs. :param rlzs: list of realizations """ oq = self.oqparam ltypes = self.riskmodel.loss_types all_stats = map(builder.build, self._collect_all_data()) if not all_stats: return loss_curves = numpy.zeros((self.N, self.Q1), self.loss_curve_dt) if oq.conditional_loss_poes: loss_maps = numpy.zeros((self.N, self.Q1), self.loss_maps_dt) for stats in all_stats: # there is one stat for each loss_type cb = self.riskmodel.curve_builders[ltypes.index(stats.loss_type)] if not cb.user_provided: continue sb = scientific.StatsBuilder(oq.quantile_loss_curves, oq.conditional_loss_poes, [], len(cb.ratios), scientific.normalize_curves_eb, oq.insured_losses) curves, maps = sb.get_curves_maps(stats) # matrices (Q1, N) loss_curves[cb.loss_type] = curves.T if oq.conditional_loss_poes: loss_maps[cb.loss_type] = maps.T self.datastore['loss_curves-stats'] = loss_curves if oq.conditional_loss_poes: self.datastore['loss_maps-stats'] = loss_maps
def compute_store_stats(self, rlzs, kind): """ Compute and store the statistical outputs """ oq = self.oqparam builder = scientific.StatsBuilder(oq.quantile_loss_curves, oq.conditional_loss_poes, [], scientific.normalize_curves_eb) if kind == '_specific': all_stats = [ builder.build(data, prefix='specific-') for data in self._collect_specific_data() ] else: all_stats = map(builder.build, self._collect_all_data()) for stat in all_stats: # there is one stat for each loss_type curves, ins_curves, maps = scientific.get_stat_curves(stat) for i, path in enumerate(stat.paths): # there are paths like # %s-stats/structural/mean # %s-stats/structural/quantile-0.1 # ... self.datastore[path % 'loss_curves'] = curves[i] if oq.insured_losses: self.datastore[path % 'ins_curves'] = ins_curves[i] if oq.conditional_loss_poes: self.datastore[path % 'loss_maps'] = maps[i] stats = scientific.SimpleStats(rlzs, oq.quantile_loss_curves) nbytes = stats.compute('avg_losses-rlzs', self.datastore) self.datastore['avg_losses-stats'].attrs['nbytes'] = nbytes self.datastore.hdf5.flush()
def build_stats(self, loss_curve_key): """ Compute all statistics for the specified assets starting from the stored loss curves. Yield a statistical output object for each loss type. """ oq = self.oqparam rlzs = self.rlzs_assoc.realizations stats = scientific.StatsBuilder( oq.quantile_loss_curves, oq.conditional_loss_poes, [], scientific.normalize_curves_eb) # NB: should we encounter memory issues in the future, the easy # solution is to split the specific assets in blocks and perform # the computation one block at the time for loss_type in self.riskmodel.get_loss_types(): outputs = [] for rlz in rlzs: key = '%s-rlzs/%s' % (loss_curve_key, rlz.uid) lcs = self.datastore[key][loss_type] assets = [None] if key.startswith('agg') else self.assets losses_poes = numpy.array( # -> shape (N, 2, C) [lcs['losses'], lcs['poes']]).transpose(1, 0, 2) out = scientific.Output( assets, loss_type, rlz.ordinal, rlz.weight, loss_curves=losses_poes, insured_curves=None) outputs.append(out) yield stats.build(outputs)
def setUpClass(cls): assets = [ asset('a1', 101), asset('a2', 151), asset('a3', 91), asset('a4', 81) ] asset_refs = [a.id for a in assets] outputs = [] weights = [0.3, 0.7] baselosses = numpy.array([.10, .14, .17, .20, .21]) for i, w in enumerate(weights): lc = loss_curves(assets, baselosses, i) out = scientific.Output(asset_refs, 'structural', weight=w, loss_curves=lc, insured_curves=None, average_losses=[.1, .12, .13, .9], average_insured_losses=None) outputs.append(out) cls.builder = scientific.StatsBuilder( quantiles=[0.1, 0.9], conditional_loss_poes=[0.35, 0.24, 0.13], poes_disagg=[], curve_resolution=len(baselosses)) cls.stats = cls.builder.build(outputs)
def classical_risk(riskinputs, riskmodel, rlzs_assoc, monitor): """ Compute and return the average losses for each asset. :param riskinputs: a list of :class:`openquake.risklib.riskinput.RiskInput` objects :param riskmodel: a :class:`openquake.risklib.riskinput.RiskModel` instance :param rlzs_assoc: associations (trt_id, gsim) -> realizations :param monitor: :class:`openquake.baselib.performance.PerformanceMonitor` instance """ lti = riskmodel.lti oq = monitor.oqparam ins = oq.insured_losses result = dict(loss_curves=[], loss_maps=[], stat_curves=[], stat_maps=[]) for out_by_rlz in riskmodel.gen_outputs(riskinputs, rlzs_assoc, monitor): l = lti[out_by_rlz.loss_type] values = workflows.get_values(out_by_rlz.loss_type, out_by_rlz.assets) for out in out_by_rlz: r = out.hid for i, asset in enumerate(out.assets): aid = asset.idx val = values[i] avg = out.average_losses[i] * val avg_ins = (out.average_insured_losses[i] * val if ins else numpy.nan) lcurve = (out.loss_curves[i, 0] * val, out.loss_curves[i, 1], avg) if ins: lcurve += (out.insured_curves[i, 0] * val, out.insured_curves[i, 1], avg_ins) else: lcurve += (None, None, None) result['loss_curves'].append((l, r, aid, lcurve)) # no insured, shape (P, N) result['loss_maps'].append( (l, r, aid, out.loss_maps[:, i] * val)) # compute statistics if len(out_by_rlz) > 1: cb = riskmodel.curve_builders[l] statsbuilder = scientific.StatsBuilder( oq.quantile_loss_curves, oq.conditional_loss_poes, oq.poes_disagg, cb.curve_resolution, insured_losses=oq.insured_losses) stats = statsbuilder.build(out_by_rlz) stat_curves, stat_maps = statsbuilder.get_curves_maps(stats) for asset, stat_curve, stat_map in zip(out_by_rlz.assets, stat_curves, stat_maps): result['stat_curves'].append((l, asset.idx, stat_curve)) result['stat_maps'].append((l, asset.idx, stat_map)) return result
def classical_risk(riskinput, riskmodel, rlzs_assoc, monitor): """ Compute and return the average losses for each asset. :param riskinput: a :class:`openquake.risklib.riskinput.RiskInput` object :param riskmodel: a :class:`openquake.risklib.riskinput.CompositeRiskModel` instance :param rlzs_assoc: associations (trt_id, gsim) -> realizations :param monitor: :class:`openquake.baselib.performance.Monitor` instance """ oq = monitor.oqparam ins = oq.insured_losses R = len(rlzs_assoc.realizations) result = dict( loss_curves=[], loss_maps=[], stat_curves=[], stat_maps=[]) for out_by_lr in riskmodel.gen_outputs(riskinput, rlzs_assoc, monitor): for (l, r), out in sorted(out_by_lr.items()): for i, asset in enumerate(out.assets): aid = asset.ordinal avg = out.average_losses[i] avg_ins = (out.average_insured_losses[i] if ins else numpy.nan) lcurve = ( out.loss_curves[i, 0], out.loss_curves[i, 1], avg) if ins: lcurve += ( out.insured_curves[i, 0], out.insured_curves[i, 1], avg_ins) else: lcurve += (None, None, None) result['loss_curves'].append((l, r, aid, lcurve)) # no insured, shape (P, N) result['loss_maps'].append( (l, r, aid, out.loss_maps[:, i])) # compute statistics if R > 1: for l, lrs in groupby(out_by_lr, operator.itemgetter(0)).items(): outs = [out_by_lr[lr] for lr in lrs] curve_resolution = outs[0].loss_curves.shape[-1] statsbuilder = scientific.StatsBuilder( oq.quantile_loss_curves, oq.conditional_loss_poes, oq.poes_disagg, curve_resolution, insured_losses=oq.insured_losses) stats = statsbuilder.build(outs) stat_curves, stat_maps = statsbuilder.get_curves_maps(stats) for i, asset in enumerate(out_by_lr.assets): result['stat_curves'].append( (l, asset.ordinal, stat_curves[:, i])) if len(stat_maps): result['stat_maps'].append( (l, asset.ordinal, stat_maps[:, i])) return result
def statistics(self, all_outputs, quantiles=()): """ :param quantiles: quantile levels used to compute quantile outputs :returns: a :class:`openquake.risklib.scientific.Output` instance holding statistical outputs (e.g. mean loss curves). """ if len(all_outputs) == 1: # single realization return stats = scientific.StatsBuilder(quantiles, self.conditional_loss_poes, self.poes_disagg) return stats.build(all_outputs)
def statistics(self, all_outputs, quantiles): """ :returns: a :class:`openquake.risklib.scientific.Output` instance holding statistical outputs (e.g. mean loss curves). :param quantiles: quantile levels used to compute quantile outputs """ if len(all_outputs) == 1: # single realization return stats = scientific.StatsBuilder(quantiles, self.conditional_loss_poes, [], scientific.normalize_curves_eb) out = stats.build(all_outputs) out.event_loss_table = sum((out.event_loss_table for out in all_outputs), collections.Counter()) return out
def compute_store_stats(self, rlzs, kind): """ Compute and store the statistical outputs """ oq = self.oqparam N = (len(self.oqparam.specific_assets) if kind == '_specific' else len(self.assetcol)) Q = 1 + len(oq.quantile_loss_curves) C = oq.loss_curve_resolution # TODO: could be loss_type-dependent loss_curve_dt = numpy.dtype([('losses', (float, C)), ('poes', (float, C)), ('avg', float)]) if oq.conditional_loss_poes: lm_names = _loss_map_names(oq.conditional_loss_poes) loss_map_dt = numpy.dtype([(f, float) for f in lm_names]) loss_curve_stats = numpy.zeros((Q, N), loss_curve_dt) ins_curve_stats = numpy.zeros((Q, N), loss_curve_dt) if oq.conditional_loss_poes: loss_map_stats = numpy.zeros((Q, N), loss_map_dt) builder = scientific.StatsBuilder(oq.quantile_loss_curves, oq.conditional_loss_poes, [], scientific.normalize_curves_eb) build_stats = getattr(self, 'build%s_stats' % kind) all_stats = build_stats(builder) for stat in all_stats: # there is one stat for each loss_type curves, ins_curves, maps = scientific.get_stat_curves(stat) loss_curve_stats[:] = curves if oq.insured_losses: ins_curve_stats[:] = ins_curves if oq.conditional_loss_poes: loss_map_stats[:] = maps for i, path in enumerate(stat.paths): self._store(path % 'loss_curves', loss_curve_stats[i]) self._store(path % 'ins_curves', ins_curve_stats[i]) if oq.conditional_loss_poes: self._store(path % 'loss_maps', loss_map_stats[i]) stats = scientific.SimpleStats(rlzs, oq.quantile_loss_curves) stats.compute_and_store('avg_losses', self.datastore)
def post_execute(self, result): """ Save the event loss table in the datastore. :param result: the dictionary returned by the .execute method """ logging.info('Generated %s of GMFs', humansize(self.gmfbytes)) self.datastore.save('job_info', {'gmfbytes': self.gmfbytes}) if self.oqparam.asset_loss_table: asslt = self.datastore['ass_loss_table'] asslt.attrs['nbytes'] = self.ass_bytes for rlz, dset in asslt.items(): for ds in dset.values(): ds.attrs['nonzero_fraction'] = len(ds) / (self.N * self.E) agglt = self.datastore['agg_loss_table'] agglt.attrs['nbytes'] = self.agg_bytes for rlz, dset in agglt.items(): for ds in dset.values(): ds.attrs['nonzero_fraction'] = len(ds) / self.E insured_losses = self.oqparam.insured_losses ses_ratio = self.oqparam.ses_ratio saved = self.saved self.N = N = len(self.assetcol) self.R = R = len(self.rlzs_assoc.realizations) ltypes = self.riskmodel.loss_types self.loss_curve_dt, self.loss_maps_dt = ( self.riskmodel.build_loss_dtypes( self.oqparam.conditional_loss_poes, self.I)) self.vals = {} # asset values by loss_type for ltype in ltypes: asset_values = [] for assets in self.assets_by_site: for asset in assets: asset_values.append( asset.value(ltype, self.oqparam.time_event)) self.vals[ltype] = numpy.array(asset_values) # loss curves multi_lr_dt = numpy.dtype([ (ltype, (F32, cbuilder.curve_resolution)) for ltype, cbuilder in zip(ltypes, self.riskmodel.curve_builders) ]) rcurves = numpy.zeros((N, R, 2), multi_lr_dt) # AVGLOSS if self.oqparam.avg_losses: with self.monitor('building avg_losses-rlzs'): for (l, r), avgloss in numpy.ndenumerate(result['AVGLOSS']): lt = self.riskmodel.loss_types[l] avg_losses_lt = self.avg_losses[lt] for i, avalue in enumerate(self.vals[lt]): avg_losses_lt[i, r] = avgloss[i, 0] * avalue if self.oqparam.insured_losses: self.avg_losses[lt + '_ins'][i, r] = (avgloss[i, 1] * avalue) self.datastore['avg_losses-rlzs'] = self.avg_losses saved['avg_losses-rlzs'] = self.avg_losses.nbytes # RC, IC if self.oqparam.loss_ratios: with self.monitor('building rcurves-rlzs'): for (l, r), data in numpy.ndenumerate(result['RC']): cb = self.riskmodel.curve_builders[l] if data and cb.user_provided: # data is a dict asset idx -> counts lt = self.riskmodel.loss_types[l] poes = cb.build_poes(N, [data], ses_ratio) rcurves[lt][:, r, 0] = poes saved['rcurves-rlzs'] += poes.nbytes for (l, r), data in numpy.ndenumerate(result['IC']): cb = self.riskmodel.curve_builders[l] if data and cb.user_provided and insured_losses: # data is a dict asset idx -> counts lt = self.riskmodel.loss_types[l] poes = cb.build_poes(N, [data], ses_ratio) rcurves[lt][:, r, 1] = poes saved['rcurves-rlzs'] += poes.nbytes self.datastore['rcurves-rlzs'] = rcurves oq = self.oqparam builder = scientific.StatsBuilder(oq.quantile_loss_curves, oq.conditional_loss_poes, [], oq.loss_curve_resolution, scientific.normalize_curves_eb, oq.insured_losses) # build an aggregate loss curve per realization plus statistics with self.monitor('building agg_curve'): self.build_agg_curve_and_stats(builder) self.datastore.hdf5.flush() for out in sorted(saved): nbytes = saved[out] if nbytes: self.datastore[out].attrs['nbytes'] = nbytes logging.info('Saved %s in %s', humansize(nbytes), out) if self.oqparam.asset_loss_table: pass # TODO: build specific loss curves rlzs = self.rlzs_assoc.realizations if self.loss_maps_dt: with self.monitor('building loss_maps-rlzs'): if (self.oqparam.conditional_loss_poes and 'rcurves-rlzs' in self.datastore): loss_maps = numpy.zeros((N, R), self.loss_maps_dt) rcurves = self.datastore['rcurves-rlzs'] for cb in self.riskmodel.curve_builders: if cb.user_provided: lm = loss_maps[cb.loss_type] for r, lmaps in cb.build_loss_maps( self.assetcol.array, rcurves): lm[:, r] = lmaps self.datastore['loss_maps-rlzs'] = loss_maps if len(rlzs) > 1: self.Q1 = len(self.oqparam.quantile_loss_curves) + 1 with self.monitor('computing stats'): if 'rcurves-rlzs' in self.datastore: self.compute_store_stats(rlzs, builder) if oq.avg_losses: # stats for avg_losses stats = scientific.SimpleStats(rlzs, oq.quantile_loss_curves) stats.compute_and_store('avg_losses', self.datastore) self.datastore.hdf5.flush()