def _get_gmfs(dstore, serial, eid): oq = dstore['oqparam'] min_iml = event_based.fix_minimum_intensity(oq.minimum_intensity, oq.imtls) rlzs_assoc = dstore['csm_info'].get_rlzs_assoc() sitecol = dstore['sitecol'].complete N = len(sitecol.complete) rup = dstore['sescollection/' + serial] correl_model = readinput.get_correl_model(oq) gsims = rlzs_assoc.gsims_by_trt_id[rup.trt_id] rlzs = [rlz for gsim in map(str, gsims) for rlz in rlzs_assoc[rup.trt_id, gsim]] gmf_dt = numpy.dtype([('%03d' % rlz.ordinal, F32) for rlz in rlzs]) gmfadict = create(event_based.GmfColl, [rup], sitecol, oq.imtls, rlzs_assoc, oq.truncation_level, correl_model, min_iml).by_rlzi() for imti, imt in enumerate(oq.imtls): gmfa = numpy.zeros(N, gmf_dt) for rlzname in gmf_dt.names: rlzi = int(rlzname) gmvs = get_array(gmfadict[rlzi], eid=eid, imti=imti)['gmv'] gmfa[rlzname][rup.indices] = gmvs yield gmfa, imt
def _get_gmfs(dstore, serial, eid): oq = dstore['oqparam'] min_iml = event_based.fix_minimum_intensity(oq.minimum_intensity, oq.imtls) rlzs_assoc = dstore['csm_info'].get_rlzs_assoc() sitecol = dstore['sitecol'].complete N = len(sitecol.complete) rup = dstore['sescollection/' + serial] correl_model = readinput.get_correl_model(oq) gsims = rlzs_assoc.gsims_by_trt_id[rup.trt_id] rlzs = [ rlz for gsim in map(str, gsims) for rlz in rlzs_assoc[rup.trt_id, gsim] ] gmf_dt = numpy.dtype([('%03d' % rlz.ordinal, F32) for rlz in rlzs]) gmfadict = create(event_based.GmfColl, [rup], sitecol, oq.imtls, rlzs_assoc, oq.truncation_level, correl_model, min_iml).by_rlzi() for imti, imt in enumerate(oq.imtls): gmfa = numpy.zeros(N, gmf_dt) for rlzname in gmf_dt.names: rlzi = int(rlzname) gmvs = get_array(gmfadict[rlzi], eid=eid, imti=imti)['gmv'] gmfa[rlzname][rup.indices] = gmvs yield gmfa, imt
def execute(self): """ Run the event_based_risk calculator and aggregate the results """ oq = self.oqparam correl_model = readinput.get_correl_model(oq) self.N = len(self.assetcol) self.E = len(self.etags) logging.info('Populating the risk inputs') rlzs_by_tr_id = self.rlzs_assoc.get_rlzs_by_trt_id() num_rlzs = {t: len(rlzs) for t, rlzs in rlzs_by_tr_id.items()} num_assets = { sid: len(self.assets_by_site[sid]) for sid in self.sitecol.sids } all_ruptures = [] for serial in self.datastore['sescollection']: rup = self.datastore['sescollection/' + serial] rup.set_weight(num_rlzs, num_assets) all_ruptures.append(rup) all_ruptures.sort(key=operator.attrgetter('serial')) if not self.riskmodel.covs: # do not generate epsilons eps = None else: eps = riskinput.make_eps(self.assets_by_site, self.E, oq.master_seed, oq.asset_correlation) logging.info('Generated %s epsilons', eps.shape) # preparing empty datasets loss_types = self.riskmodel.loss_types self.C = self.oqparam.loss_curve_resolution self.L = L = len(loss_types) self.R = R = len(self.rlzs_assoc.realizations) self.I = self.oqparam.insured_losses # ugly: attaching attributes needed in the task function mon = self.monitor mon.num_assets = self.count_assets() mon.avg_losses = self.oqparam.avg_losses mon.asset_loss_table = self.oqparam.asset_loss_table mon.insured_losses = self.I mon.ses_ratio = (oq.risk_investigation_time or oq.investigation_time) / (oq.investigation_time * oq.ses_per_logic_tree_path) self.N = N = len(self.assetcol) self.E = len(self.datastore['etags']) # average losses, stored in a composite array of shape N, R multi_avg_dt = self.riskmodel.loss_type_dt(insured=self.I) self.avg_losses = numpy.zeros((N, R), multi_avg_dt) self.ass_loss_table = square(L, R, lambda: None) self.agg_loss_table = square(L, R, lambda: None) self.ela_dt, self.elt_dt = mon.ela_dt, mon.elt_dt = build_el_dtypes( self.I) for (l, r) in itertools.product(range(L), range(R)): lt = loss_types[l] if self.oqparam.asset_loss_table: self.ass_loss_table[l, r] = self.datastore.create_dset( 'ass_loss_table/rlz-%03d/%s' % (r, lt), self.ela_dt) self.agg_loss_table[l, r] = self.datastore.create_dset( 'agg_loss_table/rlz-%03d/%s' % (r, lt), self.elt_dt) self.saved = collections.Counter() # nbytes per HDF5 key self.ass_bytes = 0 self.agg_bytes = 0 self.gmfbytes = 0 rlz_ids = getattr(self.oqparam, 'rlz_ids', ()) if rlz_ids: self.rlzs_assoc = self.rlzs_assoc.extract(rlz_ids) if not oq.minimum_intensity: # infer it from the risk models if not directly set in job.ini oq.minimum_intensity = self.riskmodel.get_min_iml() min_iml = event_based.fix_minimum_intensity(oq.minimum_intensity, oq.imtls) if min_iml.sum() == 0: logging.warn('The GMFs are not filtered: ' 'you may want to set a minimum_intensity') else: logging.info('minimum_intensity=%s', oq.minimum_intensity) with self.monitor('building riskinputs', autoflush=True): riskinputs = self.riskmodel.build_inputs_from_ruptures( self.sitecol.complete, all_ruptures, oq.truncation_level, correl_model, min_iml, eps, oq.concurrent_tasks or 1) # NB: I am using generators so that the tasks are submitted one at # the time, without keeping all of the arguments in memory tm = starmap(self.core_task.__func__, ((riskinput, self.riskmodel, self.rlzs_assoc, self.assetcol, self.monitor.new('task')) for riskinput in riskinputs)) return tm.reduce(agg=self.agg, posthook=self.save_data_transfer)
def execute(self): """ Run the event_based_risk calculator and aggregate the results """ oq = self.oqparam correl_model = readinput.get_correl_model(oq) self.N = len(self.assetcol) self.E = len(self.etags) logging.info('Populating the risk inputs') rlzs_by_tr_id = self.rlzs_assoc.get_rlzs_by_trt_id() num_rlzs = {t: len(rlzs) for t, rlzs in rlzs_by_tr_id.items()} num_assets = {sid: len(self.assets_by_site[sid]) for sid in self.sitecol.sids} all_ruptures = [] for serial in self.datastore['sescollection']: rup = self.datastore['sescollection/' + serial] rup.set_weight(num_rlzs, num_assets) all_ruptures.append(rup) all_ruptures.sort(key=operator.attrgetter('serial')) if not self.riskmodel.covs: # do not generate epsilons eps = None else: eps = riskinput.make_eps( self.assets_by_site, self.E, oq.master_seed, oq.asset_correlation) logging.info('Generated %s epsilons', eps.shape) # preparing empty datasets loss_types = self.riskmodel.loss_types self.C = self.oqparam.loss_curve_resolution self.L = L = len(loss_types) self.R = R = len(self.rlzs_assoc.realizations) self.I = self.oqparam.insured_losses # ugly: attaching attributes needed in the task function mon = self.monitor mon.num_assets = self.count_assets() mon.avg_losses = self.oqparam.avg_losses mon.asset_loss_table = self.oqparam.asset_loss_table mon.insured_losses = self.I mon.ses_ratio = ( oq.risk_investigation_time or oq.investigation_time) / ( oq.investigation_time * oq.ses_per_logic_tree_path) self.N = N = len(self.assetcol) self.E = len(self.datastore['etags']) # average losses, stored in a composite array of shape N, R multi_avg_dt = self.riskmodel.loss_type_dt(insured=self.I) self.avg_losses = numpy.zeros((N, R), multi_avg_dt) self.ass_loss_table = square(L, R, lambda: None) self.agg_loss_table = square(L, R, lambda: None) self.ela_dt, self.elt_dt = mon.ela_dt, mon.elt_dt = build_el_dtypes( self.I) for (l, r) in itertools.product(range(L), range(R)): lt = loss_types[l] if self.oqparam.asset_loss_table: self.ass_loss_table[l, r] = self.datastore.create_dset( 'ass_loss_table/rlz-%03d/%s' % (r, lt), self.ela_dt) self.agg_loss_table[l, r] = self.datastore.create_dset( 'agg_loss_table/rlz-%03d/%s' % (r, lt), self.elt_dt) self.saved = collections.Counter() # nbytes per HDF5 key self.ass_bytes = 0 self.agg_bytes = 0 self.gmfbytes = 0 rlz_ids = getattr(self.oqparam, 'rlz_ids', ()) if rlz_ids: self.rlzs_assoc = self.rlzs_assoc.extract(rlz_ids) if not oq.minimum_intensity: # infer it from the risk models if not directly set in job.ini oq.minimum_intensity = self.riskmodel.get_min_iml() min_iml = event_based.fix_minimum_intensity( oq.minimum_intensity, oq.imtls) if min_iml.sum() == 0: logging.warn('The GMFs are not filtered: ' 'you may want to set a minimum_intensity') else: logging.info('minimum_intensity=%s', oq.minimum_intensity) with self.monitor('building riskinputs', autoflush=True): riskinputs = self.riskmodel.build_inputs_from_ruptures( self.sitecol.complete, all_ruptures, oq.truncation_level, correl_model, min_iml, eps, oq.concurrent_tasks or 1) # NB: I am using generators so that the tasks are submitted one at # the time, without keeping all of the arguments in memory tm = starmap( self.core_task.__func__, ((riskinput, self.riskmodel, self.rlzs_assoc, self.assetcol, self.monitor.new('task')) for riskinput in riskinputs)) return tm.reduce(agg=self.agg, posthook=self.save_data_transfer)