def execute(self):
if self.oqparam.hazard_calculation_id:
def saving_sources_by_task(allargs, dstore):
return allargs
else:
from openquake.calculators.classical import saving_sources_by_task
self.gmdata = {}
self.offset = 0
self.gmf_size = 0
self.indices = collections.defaultdict(list) # sid, idx -> indices
acc = self.zerodict()
with self.monitor('managing sources', autoflush=True):
allargs = self.gen_args(self.monitor('classical'))
iterargs = saving_sources_by_task(allargs, self.datastore)
if isinstance(allargs, list):
# there is a trick here: if the arguments are known
# (a list, not an iterator), keep them as a list
# then the Starmap will understand the case of a single
# argument tuple and it will run in core the task
iterargs = list(iterargs)
if self.oqparam.ground_motion_fields is False:
logging.info('Generating ruptures only')
ires = parallel.Starmap(self.core_task.__func__,
iterargs).submit_all()
acc = ires.reduce(self.agg_dicts, acc)
if self.oqparam.hazard_calculation_id is None:
with self.monitor('store source_info', autoflush=True):
self.store_source_info(self.csm.infos, acc)
self.check_overflow() # check the number of events
base.save_gmdata(self, self.R)
if self.indices:
N = len(self.sitecol.complete)
logging.info('Saving gmf_data/indices')
with self.monitor('saving gmf_data/indices',
measuremem=True,
autoflush=True):
dset = self.datastore.create_dset('gmf_data/indices',
hdf5.vuint32,
shape=(N, 2),
fillvalue=None)
for sid in self.sitecol.complete.sids:
dset[sid, 0] = self.indices[sid, 0]
dset[sid, 1] = self.indices[sid, 1]
elif (self.oqparam.ground_motion_fields
and 'ucerf' not in self.oqparam.calculation_mode):
raise RuntimeError('No GMFs were generated, perhaps they were '
'all below the minimum_intensity threshold')
return acc
def post_execute(self, result):
"""
Compute stats for the aggregated distributions and save
the results on the datastore.
"""
dstates = self.riskmodel.damage_states
ltypes = self.riskmodel.loss_types
L = len(ltypes)
R = len(self.rlzs_assoc.realizations)
D = len(dstates)
N = len(self.assetcol)
E = self.oqparam.number_of_ground_motion_fields
# damage distributions
dt_list = []
for ltype in ltypes:
dt_list.append(
(ltype, numpy.dtype([('mean', (F32, D)),
('stddev', (F32, D))])))
multi_stat_dt = numpy.dtype(dt_list)
d_asset = numpy.zeros((N, R, L, 2, D), F32)
for (l, r, a, stat) in result['d_asset']:
d_asset[a, r, l] = stat
self.datastore['dmg_by_asset'] = dist_by_asset(
d_asset, multi_stat_dt, self.assetcol.array['number'])
dmg_dt = [(ds, F32) for ds in self.riskmodel.damage_states]
d_event = numpy.zeros((E, R, L), dmg_dt)
for d, ds in enumerate(self.riskmodel.damage_states):
d_event[ds] = result['d_event'][:, :, :, d]
self.datastore['dmg_by_event'] = d_event
# consequence distributions
if result['c_asset']:
dtlist = [('eid', U64), ('rlzi', U16), ('loss', (F32, L))]
stat_dt = numpy.dtype([('mean', F32), ('stddev', F32)])
c_asset = numpy.zeros((N, R, L), stat_dt)
for (l, r, a, stat) in result['c_asset']:
c_asset[a, r, l] = stat
multi_stat_dt = self.oqparam.loss_dt(stat_dt)
self.datastore['losses_by_asset'] = c_asset
self.datastore['losses_by_event'] = numpy.fromiter(
((eid, rlzi, F32(result['c_event'][eid, rlzi]))
for rlzi in range(R) for eid in range(E)), dtlist)
# save gmdata
self.gmdata = result['gmdata']
for arr in self.gmdata.values():
arr[-2] = self.oqparam.number_of_ground_motion_fields # events
base.save_gmdata(self, R)
def save_results(self, allres, num_rlzs):
"""
:param allres: an iterable of result iterators
:param num_rlzs: the total number of realizations
:returns: the total number of events
"""
oq = self.oqparam
self.A = len(self.assetcol)
if oq.asset_loss_table:
# save all_loss_ratios
self.alr_nbytes = 0
self.indices = collections.defaultdict(list) # sid -> pairs
if oq.avg_losses:
self.dset = self.datastore.create_dset(
'avg_losses-rlzs', F32, (self.A, num_rlzs, self.L * self.I))
num_events = collections.Counter()
self.gmdata = AccumDict(accum=numpy.zeros(len(oq.imtls) + 2, F32))
self.taskno = 0
self.start = 0
self.num_losses = numpy.zeros((self.A, num_rlzs), U32)
for res in allres:
start, stop = res.rlz_slice.start, res.rlz_slice.stop
for dic in res:
for r, arr in dic.pop('gmdata').items():
self.gmdata[start + r] += arr
self.save_losses(dic, start)
logging.debug(
'Saving results for source model #%d, realizations %d:%d',
res.sm_id + 1, start, stop)
if hasattr(res, 'eff_ruptures'): # for UCERF
self.eff_ruptures += res.eff_ruptures
if hasattr(res, 'ruptures_by_grp'): # for UCERF
save_ruptures(self, res.ruptures_by_grp)
elif hasattr(res, 'events_by_grp'): # for UCERF
for grp_id in res.events_by_grp:
events = res.events_by_grp[grp_id]
self.datastore.extend('events', events)
num_events[res.sm_id] += res.num_events
if 'all_loss_ratios' in self.datastore:
self.datastore['all_loss_ratios/num_losses'] = self.num_losses
self.datastore.set_attrs(
'all_loss_ratios/num_losses', nbytes=self.num_losses.nbytes)
del self.num_losses
base.save_gmdata(self, num_rlzs)
return num_events
def execute(self):
oq = self.oqparam
self.gmdata = {}
self.offset = 0
self.indices = collections.defaultdict(list) # sid, idx -> indices
self.min_iml = self.get_min_iml(oq)
param = dict(oqparam=oq,
min_iml=self.min_iml,
save_ruptures=oq.save_ruptures,
gmf=oq.ground_motion_fields,
truncation_level=oq.truncation_level,
imtls=oq.imtls,
filter_distance=oq.filter_distance,
ses_per_logic_tree_path=oq.ses_per_logic_tree_path)
if oq.hazard_calculation_id: # from ruptures
assert oq.ground_motion_fields, 'must be True!'
self.datastore.parent = datastore.read(oq.hazard_calculation_id)
iterargs = self.from_ruptures(param)
else: # from sources
iterargs = self.from_sources(param)
if oq.ground_motion_fields is False:
for args in iterargs: # store the ruptures/events
pass
return {}
acc = parallel.Starmap(self.core_task.__func__, iterargs,
self.monitor()).reduce(self.agg_dicts,
self.zerodict())
self.check_overflow() # check the number of events
base.save_gmdata(self, self.R)
if self.indices:
N = len(self.sitecol.complete)
logging.info('Saving gmf_data/indices')
with self.monitor('saving gmf_data/indices',
measuremem=True,
autoflush=True):
self.datastore['gmf_data/imts'] = ' '.join(oq.imtls)
dset = self.datastore.create_dset('gmf_data/indices',
hdf5.vuint32,
shape=(N, 2),
fillvalue=None)
for sid in self.sitecol.complete.sids:
dset[sid, 0] = self.indices[sid, 0]
dset[sid, 1] = self.indices[sid, 1]
elif (oq.ground_motion_fields and 'ucerf' not in oq.calculation_mode):
raise RuntimeError('No GMFs were generated, perhaps they were '
'all below the minimum_intensity threshold')
return acc
def execute(self):
"""
Run in parallel `core_task(sources, sitecol, monitor)`, by
parallelizing on the ruptures according to their weight and
tectonic region type.
"""
oq = self.oqparam
calc.check_overflow(self)
self.csm_info = self.datastore['csm_info']
self.sm_id = {
tuple(sm.path): sm.ordinal
for sm in self.csm_info.source_models
}
L = len(oq.imtls.array)
R = self.datastore['csm_info'].get_num_rlzs()
self.gmdata = {}
self.offset = 0
self.indices = collections.defaultdict(list) # sid -> indices
ires = parallel.Starmap(self.core_task.__func__,
self.gen_args()).submit_all()
if self.precalc and self.precalc.result:
# remove the ruptures in memory to save memory
self.precalc.result.clear()
acc = ires.reduce(self.combine_pmaps_and_save_gmfs,
{r: ProbabilityMap(L)
for r in range(R)})
base.save_gmdata(self, R)
if self.indices:
logging.info('Saving gmf_data/indices')
with self.monitor('saving gmf_data/indices',
measuremem=True,
autoflush=True):
self.datastore.save_vlen('gmf_data/indices', [
numpy.array(self.indices[sid], indices_dt)
for sid in self.sitecol.complete.sids
])
else:
raise RuntimeError('No GMFs were generated, perhaps they were '
'all below the minimum_intensity threshold')
return acc
def save_results(self, allres, num_rlzs):
"""
:param allres: an iterable of result iterators
:param num_rlzs: the total number of realizations
:returns: the total number of events
"""
oq = self.oqparam
self.A = len(self.assetcol)
if oq.avg_losses:
self.dset = self.datastore.create_dset(
'avg_losses-rlzs', F32, (self.A, num_rlzs, self.L * self.I))
num_events = collections.Counter()
self.gmdata = AccumDict(accum=numpy.zeros(len(oq.imtls) + 1, F32))
self.taskno = 0
self.start = 0
for res in allres:
start, stop = res.rlz_slice.start, res.rlz_slice.stop
for dic in res:
for r, arr in dic.pop('gmdata').items():
self.gmdata[start + r] += arr
self.save_losses(dic, start)
logging.debug(
'Saving results for source model #%d, realizations %d:%d',
res.sm_id + 1, start, stop)
if hasattr(res, 'eff_ruptures'):
self.eff_ruptures += res.eff_ruptures
if hasattr(res, 'ruptures_by_grp'):
for ruptures in res.ruptures_by_grp.values():
save_ruptures(self, ruptures)
elif hasattr(res, 'events_by_grp'):
for grp_id in res.events_by_grp:
events = res.events_by_grp[grp_id]
self.datastore.extend('events', events)
num_events[res.sm_id] += res.num_events
base.save_gmdata(self, num_rlzs)
return num_events
