def filter_and_split(src, sourceprocessor):
"""
Filter and split the source by using the source processor.
Also, sets the sub sources `.weight` attribute.
:param src: a hazardlib source object
:param sourceprocessor: a SourceFilterSplitter object
:returns: a named tuple of type SourceInfo
"""
if sourceprocessor.sitecol: # filter
info = sourceprocessor.filter(src)
if not info.sources:
return info # filtered away
filter_time = info.filter_time
else: # only split
filter_time = 0
t1 = time.time()
out = []
weight_time = 0
weight = 0
for ss in sourceconverter.split_source(src, sourceprocessor.asd):
if sourceprocessor.weight:
t = time.time()
ss.weight = get_weight(ss)
weight_time += time.time() - t
weight += ss.weight
out.append(ss)
src.weight = weight
split_time = time.time() - t1 - weight_time
return SourceInfo(src.trt_model_id, src.source_id, src.__class__.__name__,
weight, out, filter_time, weight_time, split_time)
def filter_and_split(src, sourceprocessor):
"""
Filter and split the source by using the source processor.
Also, sets the sub sources `.weight` attribute.
:param src: a hazardlib source object
:param sourceprocessor: a SourceFilterSplitter object
:returns: a named tuple of type SourceInfo
"""
if sourceprocessor.sitecol: # filter
info = sourceprocessor.filter(src)
if not info.sources:
return info # filtered away
filter_time = info.filter_time
else: # only split
filter_time = 0
t1 = time.time()
out = []
weight_time = 0
weight = 0
for ss in sourceconverter.split_source(src):
if sourceprocessor.weight:
t = time.time()
ss.num_ruptures = ss.count_ruptures()
weight += ss.weight
weight_time += time.time() - t
out.append(ss)
split_time = time.time() - t1 - weight_time
return SourceInfo(src.trt_model_id, src.source_id, src.__class__.__name__,
weight, out, filter_time, weight_time, split_time)
def get_sources(self, kind, sitecol):
"""
:param kind: a string 'light', 'heavy' or 'all'
:param sitecol: a SiteCollection instance
:returns: the sources of the given kind affecting the given sitecol
"""
filter_mon = self.monitor('filtering sources')
split_mon = self.monitor('splitting sources')
for src in self.csm.get_sources(kind):
filter_time = split_time = 0
if self.filter_sources:
try:
max_dist = self.maximum_distance[src.tectonic_region_type]
except KeyError:
max_dist = self.maximum_distance['default']
with filter_mon:
try:
sites = src.filter_sites_by_distance_to_source(
max_dist, sitecol)
except:
etype, err, tb = sys.exc_info()
msg = 'An error occurred with source id=%s: %s'
msg %= (src.source_id, unicode(err))
raise_(etype, msg, tb)
filter_time = filter_mon.dt
if sites is None:
continue
if kind == 'heavy':
if (src.trt_model_id, src.id) not in self.split_map:
logging.info('splitting %s of weight %s',
src, src.weight)
with split_mon:
sources = list(sourceconverter.split_source(src))
self.split_map[src.trt_model_id, src.id] = sources
split_time = split_mon.dt
self.set_serial(src, sources)
for ss in self.split_map[src.trt_model_id, src.id]:
ss.id = src.id
yield ss
else:
self.set_serial(src)
yield src
split_sources = self.split_map.get(
(src.trt_model_id, src.id), [src])
info = SourceInfo(src.trt_model_id, src.source_id,
src.__class__.__name__,
src.weight, len(split_sources),
filter_time, split_time, 0)
key = (src.trt_model_id, src.source_id)
if key in self.infos:
self.infos[key] += info
else:
self.infos[key] = info
filter_mon.flush()
split_mon.flush()
def get_sources(self, kind, sitecol):
"""
:param kind: a string 'light', 'heavy' or 'all'
:param sitecol: a SiteCollection instance
:returns: the sources of the given kind affecting the given sitecol
"""
filter_mon = self.monitor('filtering sources')
split_mon = self.monitor('splitting sources')
for src in self.csm.get_sources(kind):
filter_time = split_time = 0
if self.filter_sources:
try:
max_dist = self.maximum_distance[src.tectonic_region_type]
except KeyError:
max_dist = self.maximum_distance['default']
with filter_mon:
try:
sites = src.filter_sites_by_distance_to_source(
max_dist, sitecol)
except:
etype, err, tb = sys.exc_info()
msg = 'An error occurred with source id=%s: %s'
msg %= (src.source_id, unicode(err))
raise_(etype, msg, tb)
filter_time = filter_mon.dt
if sites is None:
continue
if kind == 'heavy':
if (src.trt_model_id, src.id) not in self.split_map:
logging.info('splitting %s of weight %s', src, src.weight)
with split_mon:
sources = list(sourceconverter.split_source(src))
self.split_map[src.trt_model_id, src.id] = sources
split_time = split_mon.dt
self.set_serial(src, sources)
for ss in self.split_map[src.trt_model_id, src.id]:
ss.id = src.id
yield ss
else:
self.set_serial(src)
yield src
split_sources = self.split_map.get((src.trt_model_id, src.id),
[src])
info = SourceInfo(src.trt_model_id, src.source_id,
src.__class__.__name__, src.weight,
len(split_sources), filter_time, split_time, 0)
key = (src.trt_model_id, src.source_id)
if key in self.infos:
self.infos[key] += info
else:
self.infos[key] = info
filter_mon.flush()
split_mon.flush()
def get_sources(self, kind, tile):
"""
:param kind: a string 'light', 'heavy' or 'all'
:param tile: a :class:`openquake.hazardlib.site.Tile` instance
:returns: the sources of the given kind affecting the given tile
"""
filter_mon = self.monitor('filtering sources')
split_mon = self.monitor('splitting sources')
for src in self.csm.get_sources(kind):
filter_time = split_time = 0
if self.filter_sources:
with filter_mon:
try:
if src not in tile:
continue
except:
etype, err, tb = sys.exc_info()
msg = 'An error occurred with source id=%s: %s'
msg %= (src.source_id, err)
raise_(etype, msg, tb)
filter_time = filter_mon.dt
if kind == 'heavy':
if (src.trt_model_id, src.id) not in self.split_map:
logging.info('splitting %s of weight %s',
src, src.weight)
with split_mon:
sources = list(sourceconverter.split_source(src))
self.split_map[src.trt_model_id, src.id] = sources
split_time = split_mon.dt
self.set_serial(src, sources)
for ss in self.split_map[src.trt_model_id, src.id]:
ss.id = src.id
yield ss
else:
self.set_serial(src)
yield src
split_sources = self.split_map.get(
(src.trt_model_id, src.id), [src])
info = SourceInfo(src.trt_model_id, src.source_id,
src.__class__.__name__,
src.weight, len(split_sources),
filter_time, split_time, 0)
key = (src.trt_model_id, src.source_id)
if key in self.infos:
self.infos[key] += info
else:
self.infos[key] = info
filter_mon.flush()
split_mon.flush()
def get_sources(self, kind, tile):
"""
:param kind: a string 'light', 'heavy' or 'all'
:param tile: a :class:`openquake.hazardlib.site.Tile` instance
:returns: the sources of the given kind affecting the given tile
"""
filter_mon = self.monitor('filtering sources')
split_mon = self.monitor('splitting sources')
for src in self.csm.get_sources(kind):
filter_time = split_time = 0
if self.filter_sources:
with filter_mon:
try:
if src not in tile:
continue
except:
etype, err, tb = sys.exc_info()
msg = 'An error occurred with source id=%s: %s'
msg %= (src.source_id, err)
raise_(etype, msg, tb)
filter_time = filter_mon.dt
if kind == 'heavy':
if (src.trt_model_id, src.id) not in self.split_map:
logging.info('splitting %s of weight %s', src, src.weight)
with split_mon:
sources = list(sourceconverter.split_source(src))
self.split_map[src.trt_model_id, src.id] = sources
split_time = split_mon.dt
self.set_serial(src, sources)
for ss in self.split_map[src.trt_model_id, src.id]:
ss.id = src.id
yield ss
else:
self.set_serial(src)
yield src
split_sources = self.split_map.get((src.trt_model_id, src.id),
[src])
info = SourceInfo(src.trt_model_id, src.source_id,
src.__class__.__name__, src.weight,
len(split_sources), filter_time, split_time, 0)
key = (src.trt_model_id, src.source_id)
if key in self.infos:
self.infos[key] += info
else:
self.infos[key] = info
filter_mon.flush()
split_mon.flush()
def split_filter_source(src, sites, ss_filter, random_seed):
"""
:param src: an heavy source
:param sites: sites affected by the source
:param ss_filter: a SourceSitesFilter instance
:random_seed: used only for event based calculations
:returns: a list of split sources
"""
split_sources = []
start = 0
for split in sourceconverter.split_source(src):
if random_seed:
nr = split.num_ruptures
split.serial = src.serial[start:start + nr]
start += nr
if ss_filter.affected(split) is not None:
split_sources.append(split)
return split_sources
def split_sources(sources, maxweight, splitmap):
"""
Split the sources with weight greater than maxweight. `splitmap`
is a cache to avoid splitting twice the same source.
"""
ss = []
for src in sources:
if src.weight > maxweight:
key = (src.trt_model_id, src.source_id)
try:
srcs = splitmap[key]
except KeyError:
logging.info('Splitting %s of weight %d > %d',
src, src.weight, maxweight)
srcs = splitmap[key] = list(sourceconverter.split_source(src))
ss.extend(srcs)
else:
ss.append(src)
return ss
def full_disaggregation(self):
"""
Run the disaggregation phase after hazard curve finalization.
"""
oq = self.oqparam
tl = self.oqparam.truncation_level
bb_dict = self.datastore["bb_dict"]
sitecol = self.sitecol
mag_bin_width = self.oqparam.mag_bin_width
eps_edges = numpy.linspace(-tl, tl, self.oqparam.num_epsilon_bins + 1)
logging.info("%d epsilon bins from %s to %s", len(eps_edges) - 1, min(eps_edges), max(eps_edges))
self.bin_edges = {}
curves_dict = {sid: self.get_curves(sid) for sid in sitecol.sids}
all_args = []
num_trts = sum(len(sm.src_groups) for sm in self.csm.source_models)
nblocks = math.ceil(oq.concurrent_tasks / num_trts)
for smodel in self.csm.source_models:
sm_id = smodel.ordinal
trt_names = tuple(mod.trt for mod in smodel.src_groups)
max_mag = max(mod.max_mag for mod in smodel.src_groups)
min_mag = min(mod.min_mag for mod in smodel.src_groups)
mag_edges = mag_bin_width * numpy.arange(
int(numpy.floor(min_mag / mag_bin_width)), int(numpy.ceil(max_mag / mag_bin_width) + 1)
)
logging.info("%d mag bins from %s to %s", len(mag_edges) - 1, min_mag, max_mag)
for src_group in smodel.src_groups:
if src_group.id not in self.rlzs_assoc.gsims_by_grp_id:
continue # the group has been filtered away
for sid, site in zip(sitecol.sids, sitecol):
curves = curves_dict[sid]
if not curves:
continue # skip zero-valued hazard curves
bb = bb_dict[sm_id, sid]
if not bb:
logging.info("location %s was too far, skipping disaggregation", site.location)
continue
dist_edges, lon_edges, lat_edges = bb.bins_edges(oq.distance_bin_width, oq.coordinate_bin_width)
logging.info("%d dist bins from %s to %s", len(dist_edges) - 1, min(dist_edges), max(dist_edges))
logging.info("%d lon bins from %s to %s", len(lon_edges) - 1, bb.west, bb.east)
logging.info("%d lat bins from %s to %s", len(lon_edges) - 1, bb.south, bb.north)
self.bin_edges[sm_id, sid] = (mag_edges, dist_edges, lon_edges, lat_edges, eps_edges)
bin_edges = {}
for sid, site in zip(sitecol.sids, sitecol):
if (sm_id, sid) in self.bin_edges:
bin_edges[sid] = self.bin_edges[sm_id, sid]
ss_filter = SourceSitesFilter(oq.maximum_distance)
split_sources = []
for src in src_group:
for split, _sites in ss_filter(sourceconverter.split_source(src), sitecol):
split_sources.append(split)
for srcs in split_in_blocks(split_sources, nblocks):
all_args.append(
(
sitecol,
srcs,
src_group.id,
self.rlzs_assoc,
trt_names,
curves_dict,
bin_edges,
oq,
self.monitor,
)
)
results = parallel.starmap(compute_disagg, all_args).reduce(self.agg_result)
self.save_disagg_results(results)