def classical_split_filter(srcs, srcfilter, gsims, params, monitor):
"""
Split the given sources, filter the subsources and the compute the
PoEs. Yield back subtasks if the split sources contain more than
maxweight ruptures.
"""
# first check if we are sampling the sources
ss = int(os.environ.get('OQ_SAMPLE_SOURCES', 0))
if ss:
splits, stime = split_sources(srcs)
srcs = random_filtered_sources(splits, srcfilter, ss)
yield classical(srcs, srcfilter, gsims, params, monitor)
return
# NB: splitting all the sources improves the distribution significantly,
# compared to splitting only the big source
sources = []
with monitor("filtering/splitting sources"):
for src, _sites in srcfilter(srcs):
splits, _stime = split_sources([src])
sources.extend(srcfilter.filter(splits))
if sources:
sources.sort(key=weight)
totsites = len(srcfilter.sitecol)
mw = 1000 if totsites <= params['max_sites_disagg'] else 50000
mweight = max(mw, sum(src.weight for src in sources) /
params['task_multiplier'])
blocks = list(block_splitter(sources, mweight, weight))
for block in blocks[:-1]:
yield classical, block, srcfilter, gsims, params
yield classical(blocks[-1], srcfilter, gsims, params, monitor)
def classical_split_filter(srcs, srcfilter, gsims, params, monitor):
"""
Split the given sources, filter the subsources and the compute the
PoEs. Yield back subtasks if the split sources contain more than
maxweight ruptures.
"""
# first check if we are sampling the sources
ss = int(os.environ.get('OQ_SAMPLE_SOURCES', 0))
if ss:
splits, stime = split_sources(srcs)
srcs = readinput.random_filtered_sources(splits, srcfilter, ss)
yield classical(srcs, srcfilter, gsims, params, monitor)
return
sources = []
with monitor("filtering/splitting sources"):
for src, _sites in srcfilter(srcs):
if src.num_ruptures >= params['maxweight']:
splits, stime = split_sources([src])
sources.extend(srcfilter.filter(splits))
else:
sources.append(src)
blocks = list(block_splitter(sources, params['maxweight'],
operator.attrgetter('num_ruptures')))
if blocks:
# yield the first blocks (if any) and compute the last block in core
# NB: the last block is usually the smallest one
for block in blocks[:-1]:
yield classical, block, srcfilter, gsims, params
yield classical(blocks[-1], srcfilter, gsims, params, monitor)
def classical_split_filter(srcs, srcfilter, gsims, params, monitor):
"""
Split the given sources, filter the subsources and the compute the
PoEs. Yield back subtasks if the split sources contain more than
maxweight ruptures.
"""
# first check if we are sampling the sources
ss = int(os.environ.get('OQ_SAMPLE_SOURCES', 0))
if ss:
splits, stime = split_sources(srcs)
srcs = readinput.random_filtered_sources(splits, srcfilter, ss)
yield classical(srcs, srcfilter, gsims, params, monitor)
return
sources = []
with monitor("filtering/splitting sources"):
for src, _sites in srcfilter(srcs):
if src.num_ruptures >= params['maxweight']:
splits, stime = split_sources([src])
sources.extend(srcfilter.filter(splits))
else:
sources.append(src)
blocks = list(block_splitter(sources, params['maxweight'],
operator.attrgetter('num_ruptures')))
if blocks:
# yield the first blocks (if any) and compute the last block in core
# NB: the last block is usually the smallest one
for block in blocks[:-1]:
yield classical, block, srcfilter, gsims, params
yield classical(blocks[-1], srcfilter, gsims, params, monitor)
def full_enum(self):
# compute the mean curve with full enumeration
srcs = []
weights = []
grp_id = 0
for weight, branches in self.bs0.enumerate_paths():
path = tuple(br.branch_id for br in branches)
bset_values = self.bs0.get_bset_values(path)
# first path: [(<b01 b02>, (4.6, 1.1)), (<b11 b12>, 7.0)]
sg = lt.apply_uncertainties(bset_values, self.sg)
for src in sg:
src.grp_id = grp_id
grp_id += 1
srcs.extend(sg)
weights.append(weight)
for i, src in enumerate(srcs):
src.id = i
res = classical(
srcs, self.srcfilter, self.gsims,
dict(imtls=self.imtls, truncation_level2=2, collapse_ctxs=True))
pmap = res['pmap']
effrups = sum(nr for nr, ns, dt in res['calc_times'].values())
curves = [pmap[grp_id].array[0, :, 0] for grp_id in sorted(pmap)]
mean = numpy.average(curves, axis=0, weights=weights)
return mean, srcs, effrups, weights
def full_enum(self):
# compute the mean curve with full enumeration
srcs = []
weights = []
grp_id = trt_smr = 0
for weight, branches in self.bs0.enumerate_paths():
path = tuple(br.branch_id for br in branches)
bset_values = self.bs0.get_bset_values(path)
# first path: [(<b01 b02>, (4.6, 1.1)), (<b11 b12>, 7.0)]
sg = lt.apply_uncertainties(bset_values, self.sg)
for src in sg:
src.grp_id = grp_id
src.trt_smr = trt_smr
grp_id += 1
trt_smr += 1
srcs.extend(sg)
weights.append(weight)
for i, src in enumerate(srcs):
src.id = i
N = len(self.sitecol.complete)
time_span = srcs[0].temporal_occurrence_model.time_span
idist = calc.filters.IntegrationDistance.new('200')
params = dict(imtls=self.imtls,
truncation_level2=2,
collapse_level=2,
investigation_time=time_span,
maximum_distance=idist('default'))
cmaker = contexts.ContextMaker(srcs[0].tectonic_region_type,
self.gsims, params)
res = classical(srcs, self.sitecol, cmaker)
pmap = res['pmap']
effrups = sum(nr for nr, ns, dt in res['calc_times'].values())
curve = pmap.array(N)[0, :, 0]
return curve, srcs, effrups, weights
def test_mutually_exclusive_ruptures(self):
# Test the calculation of hazard curves using mutually exclusive
# ruptures for a single source
gsim_by_trt = [SadighEtAl1997()]
rupture = _create_rupture(10., 6.)
data = [(rupture, PMF([(0.7, 0), (0.3, 1)])),
(rupture, PMF([(0.6, 0), (0.4, 1)]))]
data[0][0].weight = 0.5
data[1][0].weight = 0.5
src = NonParametricSeismicSource('0', 'test', "Active Shallow Crust",
data)
src.id = 0
src.grp_id = 0
src.trt_smr = 0
src.mutex_weight = 1
group = SourceGroup(src.tectonic_region_type, [src], 'test', 'mutex',
'mutex')
param = dict(imtls=self.imtls,
src_interdep=group.src_interdep,
rup_interdep=group.rup_interdep,
grp_probability=group.grp_probability)
cmaker = ContextMaker(src.tectonic_region_type, gsim_by_trt, param)
crv = classical(group, self.sites, cmaker)['pmap'][0]
npt.assert_almost_equal(numpy.array([0.35000, 0.32497, 0.10398]),
crv.array[:, 0],
decimal=4)
def classical_split_filter(srcs, srcfilter, gsims, params, monitor):
"""
Split the given sources, filter the subsources and the compute the
PoEs. Yield back subtasks if the split sources contain more than
maxweight ruptures.
"""
# first check if we are sampling the sources
ss = int(os.environ.get('OQ_SAMPLE_SOURCES', 0))
if ss:
splits, stime = split_sources(srcs)
srcs = random_filtered_sources(splits, srcfilter, ss)
yield classical(srcs, srcfilter, gsims, params, monitor)
return
# NB: splitting all the sources improves the distribution significantly,
# compared to splitting only the big source
sources = []
with monitor("filtering/splitting sources"):
for src, _sites in srcfilter(srcs):
splits, _stime = split_sources([src])
sources.extend(srcfilter.filter(splits))
if sources:
yield from parallel.split_task(classical,
sources,
srcfilter,
gsims,
params,
monitor,
duration=params['task_duration'])
def test_mutually_exclusive_ruptures(self):
# Test the calculation of hazard curves using mutually exclusive
# ruptures for a single source
gsim_by_trt = [SadighEtAl1997()]
rupture = _create_rupture(10., 6.)
data = [(rupture, PMF([(0.7, 0), (0.3, 1)])),
(rupture, PMF([(0.6, 0), (0.4, 1)]))]
print(data[0][0])
data[0][0].weight = 0.5
data[1][0].weight = 0.5
print(data[0][0].weight)
src = NonParametricSeismicSource('0', 'test', TRT.ACTIVE_SHALLOW_CRUST,
data)
src.src_group_id = 0
src.mutex_weight = 1
group = SourceGroup(src.tectonic_region_type, [src], 'test', 'mutex',
'mutex')
param = dict(imtls=self.imtls,
filter_distance='rjb',
src_interdep=group.src_interdep,
rup_interdep=group.rup_interdep,
grp_probability=group.grp_probability)
crv = classical(group, self.sites, gsim_by_trt, param)['pmap'][0]
npt.assert_almost_equal(numpy.array([0.35000, 0.32497, 0.10398]),
crv[0].array[:, 0],
decimal=4)
def full_enum(self):
# compute the mean curve with full enumeration
srcs = []
weights = []
grp_id = et_id = 0
for weight, branches in self.bs0.enumerate_paths():
path = tuple(br.branch_id for br in branches)
bset_values = self.bs0.get_bset_values(path)
# first path: [(<b01 b02>, (4.6, 1.1)), (<b11 b12>, 7.0)]
sg = lt.apply_uncertainties(bset_values, self.sg)
for src in sg:
src.grp_id = grp_id
src.et_id = et_id
grp_id += 1
et_id += 1
srcs.extend(sg)
weights.append(weight)
for i, src in enumerate(srcs):
src.id = i
N = len(self.srcfilter.sitecol.complete)
res = classical(
srcs, self.srcfilter, self.gsims,
dict(imtls=self.imtls, truncation_level2=2, collapse_level=2))
pmap = res['pmap']
effrups = sum(nr for nr, ns, dt in res['calc_times'].values())
curve = pmap.array(N)[0, :, 0]
return curve, srcs, effrups, weights
def classical_split_filter(srcs, gsims, params, monitor):
"""
Split the given sources, filter the subsources and the compute the
PoEs. Yield back subtasks if the split sources contain more than
maxweight ruptures.
"""
srcfilter = monitor.read('srcfilter')
# first check if we are sampling the sources
ss = int(os.environ.get('OQ_SAMPLE_SOURCES', 0))
if ss:
splits, stime = split_sources(srcs)
srcs = random_filtered_sources(splits, srcfilter, ss)
yield classical(srcs, srcfilter, gsims, params, monitor)
return
# NB: splitting all the sources improves the distribution significantly,
# compared to splitting only the big sources
with monitor("splitting/filtering sources"):
splits, _stime = split_sources(srcs)
sources = list(srcfilter.filter(splits))
if not sources:
yield {'pmap': {}}
return
maxw = params['max_weight']
N = len(srcfilter.sitecol.complete)
def weight(src):
n = 10 * numpy.sqrt(len(src.indices) / N)
return src.weight * params['rescale_weight'] * n
blocks = list(block_splitter(sources, maxw, weight))
subtasks = len(blocks) - 1
for block in blocks[:-1]:
yield classical_, block, gsims, params
if monitor.calc_id and subtasks:
msg = 'produced %d subtask(s) with mean weight %d' % (
subtasks, numpy.mean([b.weight for b in blocks[:-1]]))
try:
logs.dbcmd('log', monitor.calc_id, datetime.utcnow(), 'DEBUG',
'classical_split_filter#%d' % monitor.task_no, msg)
except Exception:
# a foreign key error in case of `oq run` is expected
print(msg)
yield classical(blocks[-1], srcfilter, gsims, params, monitor)
def classical1(srcs, gsims, params, slc, monitor=None):
"""
Read the SourceFilter, get the current slice of it (if tiling is
enabled) and then call the classical calculator in hazardlib
"""
if monitor is None: # fix mispassed parameters (for disagg_by_src)
monitor = slc
slc = slice(None)
srcfilter = monitor.read('srcfilter')[slc]
return classical(srcs, srcfilter, gsims, params, monitor)
def classical_split_filter(srcs, srcfilter, gsims, params, monitor):
"""
Split the given sources, filter the subsources and the compute the
PoEs. Yield back subtasks if the split sources contain more than
maxweight ruptures.
"""
# first check if we are sampling the sources
ss = int(os.environ.get('OQ_SAMPLE_SOURCES', 0))
if ss:
splits, stime = split_sources(srcs)
srcs = random_filtered_sources(splits, srcfilter, ss)
yield classical(srcs, srcfilter, gsims, params, monitor)
return
# NB: splitting all the sources improves the distribution significantly,
# compared to splitting only the big sources
with monitor("splitting/filtering sources"):
splits, _stime = split_sources(srcs)
sources = list(srcfilter.filter(splits))
if not sources:
yield {'pmap': {}}
return
maxw = min(sum(src.weight for src in sources)/5, params['max_weight'])
if maxw < MINWEIGHT*5: # task too small to be resubmitted
yield classical(sources, srcfilter, gsims, params, monitor)
return
blocks = list(block_splitter(sources, maxw, weight))
subtasks = len(blocks) - 1
for block in blocks[:-1]:
yield classical, block, srcfilter, gsims, params
if monitor.calc_id and subtasks:
msg = 'produced %d subtask(s) with max weight=%d' % (
subtasks, max(b.weight for b in blocks))
try:
logs.dbcmd('log', monitor.calc_id, datetime.utcnow(), 'DEBUG',
'classical_split_filter#%d' % monitor.task_no, msg)
except Exception:
# a foreign key error in case of `oq run` is expected
print(msg)
yield classical(blocks[-1], srcfilter, gsims, params, monitor)
def test_mutually_exclusive_ruptures(self):
# Test the calculation of hazard curves using mutually exclusive
# ruptures for a single source
gsim_by_trt = [SadighEtAl1997()]
rupture = _create_rupture(10., 6.)
data = [(rupture, PMF([(0.7, 0), (0.3, 1)])),
(rupture, PMF([(0.6, 0), (0.4, 1)]))]
src = NonParametricSeismicSource('0', 'test', TRT.ACTIVE_SHALLOW_CRUST,
data)
src.src_group_id = [0]
group = SourceGroup(
src.tectonic_region_type, [src], 'test', 'mutex', 'mutex')
param = dict(imtls=self.imtls)
crv = classical(group, self.sites, gsim_by_trt, param)[0]
npt.assert_almost_equal(numpy.array([0.35000, 0.32497, 0.10398]),
crv[0].array[:, 0], decimal=4)
def _run_calculator(self, gdem, imtls, sites):
"""
Executes the classical calculator
"""
param = {
"imtls": DictArray(imtls),
"truncation_level": 3.,
"filter_distance": "rrup"
}
curves = classical(self.source,
SourceFilter(sites, valid.floatdict("200")), [gdem],
param)
pmap = ProbabilityMap(param["imtls"].array, 1)
for res in [curves]:
for grp_id in res:
pmap |= res[grp_id]
return pmap.convert(param["imtls"], len(sites))
def classical_split_filter(srcs, srcfilter, gsims, params, monitor):
"""
Split the given sources, filter the subsources and the compute the
PoEs. Yield back subtasks if the split sources contain more than
maxweight ruptures.
"""
# first check if we are sampling the sources
ss = int(os.environ.get('OQ_SAMPLE_SOURCES', 0))
if ss:
splits, stime = split_sources(srcs)
srcs = readinput.random_filtered_sources(splits, srcfilter, ss)
yield classical(srcs, srcfilter, gsims, params, monitor)
return
# NB: splitting all the sources improves the distribution significantly,
# compared to splitting only the big source
sources = []
with monitor("filtering/splitting sources"):
for src, _sites in srcfilter(srcs):
splits, _stime = split_sources([src])
sources.extend(srcfilter.filter(splits))
if sources:
tot = 0
sd = AccumDict(accum=numpy.zeros(3)) # nsites, nrupts, weight
for src in sources:
arr = numpy.array([src.nsites, src.num_ruptures, src.weight])
sd[src.id] += arr
tot += 1
source_data = numpy.array([(monitor.task_no, src_id, s / tot, r, w)
for src_id, (s, r, w) in sd.items()],
source_data_dt)
first = True
for out in parallel.split_task(classical,
sources,
srcfilter,
gsims,
params,
monitor,
duration=params['task_duration'],
weight=nrup):
if first:
out['source_data'] = source_data
first = False
yield out
def test_mutually_exclusive_ruptures(self):
# Test the calculation of hazard curves using mutually exclusive
# ruptures for a single source
gsim_by_trt = [SadighEtAl1997()]
rupture = _create_rupture(10., 6.)
data = [(rupture, PMF([(0.7, 0), (0.3, 1)])),
(rupture, PMF([(0.6, 0), (0.4, 1)]))]
print(data[0][0])
data[0][0].weight = 0.5
data[1][0].weight = 0.5
print(data[0][0].weight)
src = NonParametricSeismicSource('0', 'test', TRT.ACTIVE_SHALLOW_CRUST,
data)
src.src_group_id = 0
src.mutex_weight = 1
group = SourceGroup(
src.tectonic_region_type, [src], 'test', 'mutex', 'mutex')
param = dict(imtls=self.imtls, filter_distance='rjb',
src_interdep=group.src_interdep,
rup_interdep=group.rup_interdep,
grp_probability=group.grp_probability)
crv = classical(group, self.sites, gsim_by_trt, param)['pmap'][0]
npt.assert_almost_equal(numpy.array([0.35000, 0.32497, 0.10398]),
crv[0].array[:, 0], decimal=4)
def classical_(srcs, gsims, params, monitor):
srcfilter = monitor.read('srcfilter')
return classical(srcs, srcfilter, gsims, params, monitor)
