def _get_gmfs(dstore, serial, eid):
oq = dstore['oqparam']
min_iml = event_based.fix_minimum_intensity(oq.minimum_intensity, oq.imtls)
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
sitecol = dstore['sitecol'].complete
N = len(sitecol.complete)
rup = dstore['sescollection/' + serial]
correl_model = readinput.get_correl_model(oq)
gsims = rlzs_assoc.gsims_by_trt_id[rup.trt_id]
rlzs = [rlz for gsim in map(str, gsims)
for rlz in rlzs_assoc[rup.trt_id, gsim]]
gmf_dt = numpy.dtype([('%03d' % rlz.ordinal, F32) for rlz in rlzs])
gmfadict = create(event_based.GmfColl,
[rup], sitecol, oq.imtls, rlzs_assoc,
oq.truncation_level, correl_model, min_iml).by_rlzi()
for imti, imt in enumerate(oq.imtls):
gmfa = numpy.zeros(N, gmf_dt)
for rlzname in gmf_dt.names:
rlzi = int(rlzname)
gmvs = get_array(gmfadict[rlzi], eid=eid, imti=imti)['gmv']
gmfa[rlzname][rup.indices] = gmvs
yield gmfa, imt
def _get_gmfs(dstore, serial, eid):
oq = dstore['oqparam']
min_iml = event_based.fix_minimum_intensity(oq.minimum_intensity, oq.imtls)
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
sitecol = dstore['sitecol'].complete
N = len(sitecol.complete)
rup = dstore['sescollection/' + serial]
correl_model = readinput.get_correl_model(oq)
gsims = rlzs_assoc.gsims_by_trt_id[rup.trt_id]
rlzs = [
rlz for gsim in map(str, gsims) for rlz in rlzs_assoc[rup.trt_id, gsim]
]
gmf_dt = numpy.dtype([('%03d' % rlz.ordinal, F32) for rlz in rlzs])
gmfadict = create(event_based.GmfColl, [rup], sitecol, oq.imtls,
rlzs_assoc, oq.truncation_level, correl_model,
min_iml).by_rlzi()
for imti, imt in enumerate(oq.imtls):
gmfa = numpy.zeros(N, gmf_dt)
for rlzname in gmf_dt.names:
rlzi = int(rlzname)
gmvs = get_array(gmfadict[rlzi], eid=eid, imti=imti)['gmv']
gmfa[rlzname][rup.indices] = gmvs
yield gmfa, imt
def execute(self):
"""
Run the event_based_risk calculator and aggregate the results
"""
oq = self.oqparam
correl_model = readinput.get_correl_model(oq)
self.N = len(self.assetcol)
self.E = len(self.etags)
logging.info('Populating the risk inputs')
rlzs_by_tr_id = self.rlzs_assoc.get_rlzs_by_trt_id()
num_rlzs = {t: len(rlzs) for t, rlzs in rlzs_by_tr_id.items()}
num_assets = {
sid: len(self.assets_by_site[sid])
for sid in self.sitecol.sids
}
all_ruptures = []
for serial in self.datastore['sescollection']:
rup = self.datastore['sescollection/' + serial]
rup.set_weight(num_rlzs, num_assets)
all_ruptures.append(rup)
all_ruptures.sort(key=operator.attrgetter('serial'))
if not self.riskmodel.covs:
# do not generate epsilons
eps = None
else:
eps = riskinput.make_eps(self.assets_by_site, self.E,
oq.master_seed, oq.asset_correlation)
logging.info('Generated %s epsilons', eps.shape)
# preparing empty datasets
loss_types = self.riskmodel.loss_types
self.C = self.oqparam.loss_curve_resolution
self.L = L = len(loss_types)
self.R = R = len(self.rlzs_assoc.realizations)
self.I = self.oqparam.insured_losses
# ugly: attaching attributes needed in the task function
mon = self.monitor
mon.num_assets = self.count_assets()
mon.avg_losses = self.oqparam.avg_losses
mon.asset_loss_table = self.oqparam.asset_loss_table
mon.insured_losses = self.I
mon.ses_ratio = (oq.risk_investigation_time or
oq.investigation_time) / (oq.investigation_time *
oq.ses_per_logic_tree_path)
self.N = N = len(self.assetcol)
self.E = len(self.datastore['etags'])
# average losses, stored in a composite array of shape N, R
multi_avg_dt = self.riskmodel.loss_type_dt(insured=self.I)
self.avg_losses = numpy.zeros((N, R), multi_avg_dt)
self.ass_loss_table = square(L, R, lambda: None)
self.agg_loss_table = square(L, R, lambda: None)
self.ela_dt, self.elt_dt = mon.ela_dt, mon.elt_dt = build_el_dtypes(
self.I)
for (l, r) in itertools.product(range(L), range(R)):
lt = loss_types[l]
if self.oqparam.asset_loss_table:
self.ass_loss_table[l, r] = self.datastore.create_dset(
'ass_loss_table/rlz-%03d/%s' % (r, lt), self.ela_dt)
self.agg_loss_table[l, r] = self.datastore.create_dset(
'agg_loss_table/rlz-%03d/%s' % (r, lt), self.elt_dt)
self.saved = collections.Counter() # nbytes per HDF5 key
self.ass_bytes = 0
self.agg_bytes = 0
self.gmfbytes = 0
rlz_ids = getattr(self.oqparam, 'rlz_ids', ())
if rlz_ids:
self.rlzs_assoc = self.rlzs_assoc.extract(rlz_ids)
if not oq.minimum_intensity:
# infer it from the risk models if not directly set in job.ini
oq.minimum_intensity = self.riskmodel.get_min_iml()
min_iml = event_based.fix_minimum_intensity(oq.minimum_intensity,
oq.imtls)
if min_iml.sum() == 0:
logging.warn('The GMFs are not filtered: '
'you may want to set a minimum_intensity')
else:
logging.info('minimum_intensity=%s', oq.minimum_intensity)
with self.monitor('building riskinputs', autoflush=True):
riskinputs = self.riskmodel.build_inputs_from_ruptures(
self.sitecol.complete, all_ruptures, oq.truncation_level,
correl_model, min_iml, eps, oq.concurrent_tasks or 1)
# NB: I am using generators so that the tasks are submitted one at
# the time, without keeping all of the arguments in memory
tm = starmap(self.core_task.__func__,
((riskinput, self.riskmodel, self.rlzs_assoc,
self.assetcol, self.monitor.new('task'))
for riskinput in riskinputs))
return tm.reduce(agg=self.agg, posthook=self.save_data_transfer)
def execute(self):
"""
Run the event_based_risk calculator and aggregate the results
"""
oq = self.oqparam
correl_model = readinput.get_correl_model(oq)
self.N = len(self.assetcol)
self.E = len(self.etags)
logging.info('Populating the risk inputs')
rlzs_by_tr_id = self.rlzs_assoc.get_rlzs_by_trt_id()
num_rlzs = {t: len(rlzs) for t, rlzs in rlzs_by_tr_id.items()}
num_assets = {sid: len(self.assets_by_site[sid])
for sid in self.sitecol.sids}
all_ruptures = []
for serial in self.datastore['sescollection']:
rup = self.datastore['sescollection/' + serial]
rup.set_weight(num_rlzs, num_assets)
all_ruptures.append(rup)
all_ruptures.sort(key=operator.attrgetter('serial'))
if not self.riskmodel.covs:
# do not generate epsilons
eps = None
else:
eps = riskinput.make_eps(
self.assets_by_site, self.E, oq.master_seed,
oq.asset_correlation)
logging.info('Generated %s epsilons', eps.shape)
# preparing empty datasets
loss_types = self.riskmodel.loss_types
self.C = self.oqparam.loss_curve_resolution
self.L = L = len(loss_types)
self.R = R = len(self.rlzs_assoc.realizations)
self.I = self.oqparam.insured_losses
# ugly: attaching attributes needed in the task function
mon = self.monitor
mon.num_assets = self.count_assets()
mon.avg_losses = self.oqparam.avg_losses
mon.asset_loss_table = self.oqparam.asset_loss_table
mon.insured_losses = self.I
mon.ses_ratio = (
oq.risk_investigation_time or oq.investigation_time) / (
oq.investigation_time * oq.ses_per_logic_tree_path)
self.N = N = len(self.assetcol)
self.E = len(self.datastore['etags'])
# average losses, stored in a composite array of shape N, R
multi_avg_dt = self.riskmodel.loss_type_dt(insured=self.I)
self.avg_losses = numpy.zeros((N, R), multi_avg_dt)
self.ass_loss_table = square(L, R, lambda: None)
self.agg_loss_table = square(L, R, lambda: None)
self.ela_dt, self.elt_dt = mon.ela_dt, mon.elt_dt = build_el_dtypes(
self.I)
for (l, r) in itertools.product(range(L), range(R)):
lt = loss_types[l]
if self.oqparam.asset_loss_table:
self.ass_loss_table[l, r] = self.datastore.create_dset(
'ass_loss_table/rlz-%03d/%s' % (r, lt), self.ela_dt)
self.agg_loss_table[l, r] = self.datastore.create_dset(
'agg_loss_table/rlz-%03d/%s' % (r, lt), self.elt_dt)
self.saved = collections.Counter() # nbytes per HDF5 key
self.ass_bytes = 0
self.agg_bytes = 0
self.gmfbytes = 0
rlz_ids = getattr(self.oqparam, 'rlz_ids', ())
if rlz_ids:
self.rlzs_assoc = self.rlzs_assoc.extract(rlz_ids)
if not oq.minimum_intensity:
# infer it from the risk models if not directly set in job.ini
oq.minimum_intensity = self.riskmodel.get_min_iml()
min_iml = event_based.fix_minimum_intensity(
oq.minimum_intensity, oq.imtls)
if min_iml.sum() == 0:
logging.warn('The GMFs are not filtered: '
'you may want to set a minimum_intensity')
else:
logging.info('minimum_intensity=%s', oq.minimum_intensity)
with self.monitor('building riskinputs', autoflush=True):
riskinputs = self.riskmodel.build_inputs_from_ruptures(
self.sitecol.complete, all_ruptures, oq.truncation_level,
correl_model, min_iml, eps, oq.concurrent_tasks or 1)
# NB: I am using generators so that the tasks are submitted one at
# the time, without keeping all of the arguments in memory
tm = starmap(
self.core_task.__func__,
((riskinput, self.riskmodel, self.rlzs_assoc,
self.assetcol, self.monitor.new('task'))
for riskinput in riskinputs))
return tm.reduce(agg=self.agg, posthook=self.save_data_transfer)