def classical_split_filter(srcs, srcfilter, gsims, params, monitor):
"""
Split the given sources, filter the subsources and the compute the
PoEs. Yield back subtasks if the split sources contain more than
maxweight ruptures.
"""
# first check if we are sampling the sources
ss = int(os.environ.get('OQ_SAMPLE_SOURCES', 0))
if ss:
splits, stime = split_sources(srcs)
srcs = random_filtered_sources(splits, srcfilter, ss)
yield classical(srcs, srcfilter, gsims, params, monitor)
return
# NB: splitting all the sources improves the distribution significantly,
# compared to splitting only the big source
sources = []
with monitor("filtering/splitting sources"):
for src, _sites in srcfilter(srcs):
splits, _stime = split_sources([src])
sources.extend(srcfilter.filter(splits))
if sources:
sources.sort(key=weight)
totsites = len(srcfilter.sitecol)
mw = 1000 if totsites <= params['max_sites_disagg'] else 50000
mweight = max(mw, sum(src.weight for src in sources) /
params['task_multiplier'])
blocks = list(block_splitter(sources, mweight, weight))
for block in blocks[:-1]:
yield classical, block, srcfilter, gsims, params
yield classical(blocks[-1], srcfilter, gsims, params, monitor)
def execute(self):
"""
Run in parallel `core_task(sources, sitecol, monitor)`, by
parallelizing on the sources according to their weight and
tectonic region type.
"""
monitor = self.monitor(self.core_task.__name__)
oq = self.oqparam
acc = self.acc0()
self.nsites = [] # used in agg_dicts
self.maxdists = []
param = dict(imtls=oq.imtls, truncation_level=oq.truncation_level,
filter_distance=oq.filter_distance,
max_weight=oq.max_weight,
max_sites_disagg=oq.max_sites_disagg)
self.calc_times = general.AccumDict(accum=np.zeros(3, np.float32))
[gsims] = self.csm.info.get_gsims_by_trt().values()
sample = .001 if os.environ.get('OQ_SAMPLE_SOURCES') else None
srcfilter = self.src_filter()
for sm in self.csm.source_models: # one branch at the time
[grp] = sm.src_groups
[src] = grp
srcs = list(src)
if sample:
srcs = random_filtered_sources(srcs, srcfilter, 1)
acc = parallel.Starmap.apply(
classical_split_filter,
(srcs, srcfilter, gsims, param, monitor),
weight=operator.attrgetter('weight'),
concurrent_tasks=oq.concurrent_tasks,
h5=self.datastore.hdf5
).reduce(self.agg_dicts, acc)
ucerf = grp.sources[0].orig
logging.info('Getting background sources from %s', ucerf.source_id)
srcs = ucerf.get_background_sources(srcfilter, sample)
acc = parallel.Starmap.apply(
classical, (srcs, srcfilter, gsims, param, monitor),
weight=operator.attrgetter('weight'),
concurrent_tasks=oq.concurrent_tasks,
h5=self.datastore.hdf5
).reduce(self.agg_dicts, acc)
self.store_rlz_info(acc.eff_ruptures)
self.store_source_info(self.calc_times)
return acc # {grp_id: pmap}
def classical_split_filter(srcs, gsims, params, monitor):
"""
Split the given sources, filter the subsources and the compute the
PoEs. Yield back subtasks if the split sources contain more than
maxweight ruptures.
"""
srcfilter = monitor.read('srcfilter')
# first check if we are sampling the sources
ss = int(os.environ.get('OQ_SAMPLE_SOURCES', 0))
if ss:
splits, stime = split_sources(srcs)
srcs = random_filtered_sources(splits, srcfilter, ss)
yield classical(srcs, srcfilter, gsims, params, monitor)
return
# NB: splitting all the sources improves the distribution significantly,
# compared to splitting only the big sources
with monitor("splitting/filtering sources"):
splits, _stime = split_sources(srcs)
sources = list(srcfilter.filter(splits))
if not sources:
yield {'pmap': {}}
return
maxw = params['max_weight']
N = len(srcfilter.sitecol.complete)
def weight(src):
n = 10 * numpy.sqrt(len(src.indices) / N)
return src.weight * params['rescale_weight'] * n
blocks = list(block_splitter(sources, maxw, weight))
subtasks = len(blocks) - 1
for block in blocks[:-1]:
yield classical_, block, gsims, params
if monitor.calc_id and subtasks:
msg = 'produced %d subtask(s) with mean weight %d' % (
subtasks, numpy.mean([b.weight for b in blocks[:-1]]))
try:
logs.dbcmd('log', monitor.calc_id, datetime.utcnow(), 'DEBUG',
'classical_split_filter#%d' % monitor.task_no, msg)
except Exception:
# a foreign key error in case of `oq run` is expected
print(msg)
yield classical(blocks[-1], srcfilter, gsims, params, monitor)
def classical_split_filter(srcs, srcfilter, gsims, params, monitor):
"""
Split the given sources, filter the subsources and the compute the
PoEs. Yield back subtasks if the split sources contain more than
maxweight ruptures.
"""
# first check if we are sampling the sources
ss = int(os.environ.get('OQ_SAMPLE_SOURCES', 0))
if ss:
splits, stime = split_sources(srcs)
srcs = random_filtered_sources(splits, srcfilter, ss)
yield classical(srcs, srcfilter, gsims, params, monitor)
return
# NB: splitting all the sources improves the distribution significantly,
# compared to splitting only the big sources
with monitor("splitting/filtering sources"):
splits, _stime = split_sources(srcs)
sources = list(srcfilter.filter(splits))
if not sources:
yield {'pmap': {}}
return
maxw = min(sum(src.weight for src in sources)/5, params['max_weight'])
if maxw < MINWEIGHT*5: # task too small to be resubmitted
yield classical(sources, srcfilter, gsims, params, monitor)
return
blocks = list(block_splitter(sources, maxw, weight))
subtasks = len(blocks) - 1
for block in blocks[:-1]:
yield classical, block, srcfilter, gsims, params
if monitor.calc_id and subtasks:
msg = 'produced %d subtask(s) with max weight=%d' % (
subtasks, max(b.weight for b in blocks))
try:
logs.dbcmd('log', monitor.calc_id, datetime.utcnow(), 'DEBUG',
'classical_split_filter#%d' % monitor.task_no, msg)
except Exception:
# a foreign key error in case of `oq run` is expected
print(msg)
yield classical(blocks[-1], srcfilter, gsims, params, monitor)
def classical_split_filter(srcs, srcfilter, gsims, params, monitor):
"""
Split the given sources, filter the subsources and the compute the
PoEs. Yield back subtasks if the split sources contain more than
maxweight ruptures.
"""
# first check if we are sampling the sources
ss = int(os.environ.get('OQ_SAMPLE_SOURCES', 0))
if ss:
splits, stime = split_sources(srcs)
srcs = random_filtered_sources(splits, srcfilter, ss)
yield classical(srcs, srcfilter, gsims, params, monitor)
return
# NB: splitting all the sources improves the distribution significantly,
# compared to splitting only the big source
sources = []
with monitor("filtering/splitting sources"):
for src, _sites in srcfilter(srcs):
splits, _stime = split_sources([src])
sources.extend(srcfilter.filter(splits))
if sources:
yield from parallel.split_task(
classical, sources, srcfilter, gsims, params, monitor,
duration=params['task_duration'])