def execute(self):
"""
Run in parallel `core_task(sources, sitecol, monitor)`, by
parallelizing on the sources according to their weight and
tectonic region type.
"""
oq = self.oqparam
monitor = self.monitor.new(
self.core_task.__name__,
truncation_level=oq.truncation_level,
imtls=oq.imtls,
maximum_distance=oq.maximum_distance,
disagg=oq.poes_disagg or oq.iml_disagg,
ses_per_logic_tree_path=oq.ses_per_logic_tree_path,
seed=oq.random_seed)
with self.monitor('managing sources', autoflush=True):
src_groups = list(self.csm.src_groups)
iterargs = saving_sources_by_task(
self.gen_args(src_groups, oq, monitor), self.datastore)
res = parallel.starmap(
self.core_task.__func__, iterargs).submit_all()
acc = reduce(self.agg_dicts, res, self.zerodict())
self.save_data_transfer(res)
with self.monitor('store source_info', autoflush=True):
self.store_source_info(self.infos)
self.rlzs_assoc = self.csm.info.get_rlzs_assoc(
partial(self.count_eff_ruptures, acc))
self.datastore['csm_info'] = self.csm.info
return acc
def execute(self):
num_rlzs = len(self.rlzs_assoc.realizations)
self.grp_trt = self.csm.info.grp_trt()
allres = parallel.starmap(compute_losses, self.gen_args()).submit_all()
num_events = self.save_results(allres, num_rlzs)
self.save_data_transfer(allres)
return num_events
def build_starmap(self, sm_id, ruptures_by_grp, sitecol,
assetcol, riskmodel, imts, trunc_level, correl_model,
min_iml, monitor):
"""
:param sm_id: source model ordinal
:param ruptures_by_grp: dictionary of ruptures by src_group_id
:param sitecol: a SiteCollection instance
:param assetcol: an AssetCollection instance
:param riskmodel: a RiskModel instance
:param imts: a list of Intensity Measure Types
:param trunc_level: truncation level
:param correl_model: correlation model
:param min_iml: vector of minimum intensities, one per IMT
:param monitor: a Monitor instance
:returns: a pair (starmap, dictionary of attributes)
"""
csm_info = self.csm_info.get_info(sm_id)
grp_ids = sorted(csm_info.get_sm_by_grp())
rlzs_assoc = csm_info.get_rlzs_assoc(
count_ruptures=lambda grp: len(ruptures_by_grp.get(grp.id, [])))
num_events = sum(ebr.multiplicity for grp in ruptures_by_grp
for ebr in ruptures_by_grp[grp])
seeds = self.oqparam.random_seed + numpy.arange(num_events)
allargs = []
# prepare the risk inputs
ruptures_per_block = self.oqparam.ruptures_per_block
start = 0
grp_trt = csm_info.grp_trt()
ignore_covs = self.oqparam.ignore_covs
for grp_id in grp_ids:
for rupts in block_splitter(
ruptures_by_grp.get(grp_id, []), ruptures_per_block):
if ignore_covs or not self.riskmodel.covs:
eps = None
elif self.oqparam.asset_correlation:
eps = EpsilonMatrix1(num_events, self.oqparam.master_seed)
else:
n_events = sum(ebr.multiplicity for ebr in rupts)
eps = EpsilonMatrix0(
len(self.assetcol), seeds[start: start + n_events])
start += n_events
ri = riskinput.RiskInputFromRuptures(
grp_trt[grp_id], rlzs_assoc, imts, sitecol,
rupts, trunc_level, correl_model, min_iml, eps)
allargs.append((ri, riskmodel, assetcol, monitor))
self.vals = self.assetcol.values()
taskname = '%s#%d' % (event_based_risk.__name__, sm_id + 1)
smap = starmap(event_based_risk, allargs, name=taskname)
attrs = dict(num_ruptures={
sg_id: len(rupts) for sg_id, rupts in ruptures_by_grp.items()},
num_events=num_events,
num_rlzs=len(rlzs_assoc.realizations),
sm_id=sm_id)
return smap, attrs
def test_spawn(self):
all_data = [
('a', list(range(10))), ('b', list(range(20))),
('c', list(range(15)))]
res = {key: parallel.starmap(get_length, [(data,)])
for key, data in all_data}
for key, val in res.items():
res[key] = val.reduce()
parallel.TaskManager.restart()
self.assertEqual(res, {'a': {'n': 10}, 'c': {'n': 15}, 'b': {'n': 20}})
def execute(self):
"""
Run the ucerf calculation
"""
res = parallel.starmap(compute_events, self.gen_args()).submit_all()
acc = self.zerodict()
num_ruptures = {}
for ruptures_by_grp in res:
[(grp_id, ruptures)] = ruptures_by_grp.items()
num_ruptures[grp_id] = len(ruptures)
acc.calc_times.extend(ruptures_by_grp.calc_times[grp_id])
self.save_events(ruptures_by_grp)
self.save_data_transfer(res)
with self.monitor('store source_info', autoflush=True):
self.store_source_info(self.infos)
self.datastore['csm_info'] = self.csm.info
return num_ruptures
def execute(self):
"""
Builds hcurves and stats from the stored PoEs
"""
if 'poes' not in self.datastore: # for short report
return
oq = self.oqparam
rlzs = self.rlzs_assoc.realizations
# initialize datasets
N = len(self.sitecol)
L = len(oq.imtls.array)
attrs = dict(
__pyclass__='openquake.hazardlib.probability_map.ProbabilityMap',
sids=numpy.arange(N, dtype=numpy.uint32))
if oq.individual_curves:
for rlz in rlzs:
self.datastore.create_dset(
'hcurves/rlz-%03d' % rlz.ordinal, F32,
(N, L, 1), attrs=attrs)
if oq.mean_hazard_curves:
self.datastore.create_dset(
'hcurves/mean', F32, (N, L, 1), attrs=attrs)
for q in oq.quantile_hazard_curves:
self.datastore.create_dset(
'hcurves/quantile-%s' % q, F32, (N, L, 1), attrs=attrs)
self.datastore.flush()
logging.info('Building hazard curves')
with self.monitor('submitting poes', autoflush=True):
pmap_by_grp = {
int(group_id): self.datastore['poes/' + group_id]
for group_id in self.datastore['poes']}
res = parallel.starmap(
build_hcurves_and_stats,
list(self.gen_args(pmap_by_grp))).submit_all()
nbytes = reduce(self.save_hcurves, res, AccumDict())
self.save_data_transfer(res)
return nbytes
def execute(self):
"""
Run in parallel `core_task(sources, sitecol, monitor)`, by
parallelizing on the ruptures according to their weight and
tectonic region type.
"""
oq = self.oqparam
if not oq.hazard_curves_from_gmfs and not oq.ground_motion_fields:
return
ruptures_by_grp = (self.precalc.result if self.precalc
else get_ruptures_by_grp(self.datastore.parent))
if self.oqparam.ground_motion_fields:
calc.check_overflow(self)
self.sm_id = {sm.path: sm.ordinal
for sm in self.csm.info.source_models}
L = len(oq.imtls.array)
res = parallel.starmap(
self.core_task.__func__, self.gen_args(ruptures_by_grp)
).submit_all()
acc = functools.reduce(self.combine_pmaps_and_save_gmfs, res, {
rlz.ordinal: ProbabilityMap(L, 1)
for rlz in self.rlzs_assoc.realizations})
self.save_data_transfer(res)
return acc
def execute(self):
"""
Run in parallel `core_task(sources, sitecol, monitor)`, by
parallelizing on the sources according to their weight and
tectonic region type.
"""
monitor = self.monitor.new(self.core_task.__name__)
monitor.oqparam = oq = self.oqparam
ucerf_source = self.src_group.sources[0]
self.src_filter = SourceFilter(self.sitecol, oq.maximum_distance)
max_dist = oq.maximum_distance[DEFAULT_TRT]
acc = AccumDict({
grp_id: ProbabilityMap(len(oq.imtls.array), len(gsims))
for grp_id, gsims in self.rlzs_assoc.gsims_by_grp_id.items()})
acc.calc_times = []
acc.eff_ruptures = AccumDict() # grp_id -> eff_ruptures
acc.bb_dict = {}
if len(self.csm) > 1:
# when multiple branches, parallelise by branch
branches = [br.value for br in self.smlt.branches.values()]
rup_res = parallel.starmap(
ucerf_classical_hazard_by_branch,
self.gen_args(branches, ucerf_source, monitor)).submit_all()
else:
# single branch
gsims = self.rlzs_assoc.gsims_by_grp_id[0]
[(branch_id, branch)] = self.smlt.branches.items()
branchname = branch.value
ucerf_source.src_group_id = 0
ucerf_source.weight = 1
ucerf_source.nsites = len(self.sitecol)
self.infos[0, ucerf_source.source_id] = source.SourceInfo(
ucerf_source)
logging.info('Getting the background point sources')
with self.monitor('getting background sources', autoflush=True):
ucerf_source.build_idx_set()
background_sids = ucerf_source.get_background_sids(
self.sitecol, max_dist)
bckgnd_sources = ucerf_source.get_background_sources(
background_sids)
# parallelize on the background sources, small tasks
args = (bckgnd_sources, self.src_filter, oq.imtls,
gsims, self.oqparam.truncation_level, (), monitor)
bg_res = parallel.apply(
pmap_from_grp, args,
concurrent_tasks=self.oqparam.concurrent_tasks).submit_all()
# parallelize by rupture subsets
tasks = self.oqparam.concurrent_tasks * 2 # they are big tasks
rup_sets = ucerf_source.get_rupture_indices(branchname)
rup_res = parallel.apply(
ucerf_classical_hazard_by_rupture_set,
(rup_sets, branchname, ucerf_source, self.src_group.id,
self.src_filter, gsims, monitor),
concurrent_tasks=tasks).submit_all()
# compose probabilities from background sources
for pmap in bg_res:
acc[0] |= pmap
self.save_data_transfer(bg_res)
pmap_by_grp_id = functools.reduce(self.agg_dicts, rup_res, acc)
with self.monitor('store source_info', autoflush=True):
self.store_source_info(self.infos)
self.save_data_transfer(rup_res)
self.datastore['csm_info'] = self.csm.info
self.rlzs_assoc = self.csm.info.get_rlzs_assoc(
functools.partial(self.count_eff_ruptures, pmap_by_grp_id))
self.datastore['csm_info'] = self.csm.info
return pmap_by_grp_id
def full_disaggregation(self):
"""
Run the disaggregation phase after hazard curve finalization.
"""
oq = self.oqparam
tl = self.oqparam.truncation_level
bb_dict = self.datastore["bb_dict"]
sitecol = self.sitecol
mag_bin_width = self.oqparam.mag_bin_width
eps_edges = numpy.linspace(-tl, tl, self.oqparam.num_epsilon_bins + 1)
logging.info("%d epsilon bins from %s to %s", len(eps_edges) - 1, min(eps_edges), max(eps_edges))
self.bin_edges = {}
curves_dict = {sid: self.get_curves(sid) for sid in sitecol.sids}
all_args = []
num_trts = sum(len(sm.src_groups) for sm in self.csm.source_models)
nblocks = math.ceil(oq.concurrent_tasks / num_trts)
for smodel in self.csm.source_models:
sm_id = smodel.ordinal
trt_names = tuple(mod.trt for mod in smodel.src_groups)
max_mag = max(mod.max_mag for mod in smodel.src_groups)
min_mag = min(mod.min_mag for mod in smodel.src_groups)
mag_edges = mag_bin_width * numpy.arange(
int(numpy.floor(min_mag / mag_bin_width)), int(numpy.ceil(max_mag / mag_bin_width) + 1)
)
logging.info("%d mag bins from %s to %s", len(mag_edges) - 1, min_mag, max_mag)
for src_group in smodel.src_groups:
if src_group.id not in self.rlzs_assoc.gsims_by_grp_id:
continue # the group has been filtered away
for sid, site in zip(sitecol.sids, sitecol):
curves = curves_dict[sid]
if not curves:
continue # skip zero-valued hazard curves
bb = bb_dict[sm_id, sid]
if not bb:
logging.info("location %s was too far, skipping disaggregation", site.location)
continue
dist_edges, lon_edges, lat_edges = bb.bins_edges(oq.distance_bin_width, oq.coordinate_bin_width)
logging.info("%d dist bins from %s to %s", len(dist_edges) - 1, min(dist_edges), max(dist_edges))
logging.info("%d lon bins from %s to %s", len(lon_edges) - 1, bb.west, bb.east)
logging.info("%d lat bins from %s to %s", len(lon_edges) - 1, bb.south, bb.north)
self.bin_edges[sm_id, sid] = (mag_edges, dist_edges, lon_edges, lat_edges, eps_edges)
bin_edges = {}
for sid, site in zip(sitecol.sids, sitecol):
if (sm_id, sid) in self.bin_edges:
bin_edges[sid] = self.bin_edges[sm_id, sid]
src_filter = SourceFilter(sitecol, oq.maximum_distance)
split_sources = []
for src in src_group:
for split, _sites in src_filter(sourceconverter.split_source(src), sitecol):
split_sources.append(split)
for srcs in split_in_blocks(split_sources, nblocks):
all_args.append(
(
src_filter,
srcs,
src_group.id,
self.rlzs_assoc,
trt_names,
curves_dict,
bin_edges,
oq,
self.monitor,
)
)
results = parallel.starmap(compute_disagg, all_args).reduce(self.agg_result)
self.save_disagg_results(results)