def pmap_from_grp(
sources, sites, imtls, gsims, truncation_level=None,
source_site_filter='SourceSitesFilter', maximum_distance=None, bbs=(),
monitor=Monitor()):
"""
Compute the hazard curves for a set of sources belonging to the same
tectonic region type for all the GSIMs associated to that TRT.
The arguments are the same as in :func:`calc_hazard_curves`, except
for ``gsims``, which is a list of GSIM instances.
:returns: a ProbabilityMap instance
"""
if source_site_filter == 'SourceSitesFilter': # default
source_site_filter = (
filters.SourceSitesFilter(maximum_distance)
if maximum_distance else filters.source_site_noop_filter)
with GroundShakingIntensityModel.forbid_instantiation():
imtls = DictArray(imtls)
cmaker = ContextMaker(gsims, maximum_distance)
ctx_mon = monitor('making contexts', measuremem=False)
pne_mon = monitor('computing poes', measuremem=False)
disagg_mon = monitor('get closest points', measuremem=False)
monitor.calc_times = [] # pairs (src_id, delta_t)
pmap = ProbabilityMap(len(imtls.array), len(gsims))
for src, s_sites in source_site_filter(sources, sites):
t0 = time.time()
pmap |= poe_map(src, s_sites, imtls, cmaker, truncation_level, bbs,
ctx_mon, pne_mon, disagg_mon)
# we are attaching the calculation times to the monitor
# so that oq-lite (and the engine) can store them
monitor.calc_times.append((src.id, time.time() - t0))
# NB: source.id is an integer; it should not be confused
# with source.source_id, which is a string
monitor.eff_ruptures = pne_mon.counts # contributing ruptures
return pmap
def wrapped(*args):
"""
Initialize logs, make sure the job is still running, and run the task
code surrounded by a try-except. If any error occurs, log it as a
critical failure.
"""
# the last argument is assumed to be a monitor
monitor = args[-1]
job = models.OqJob.objects.get(id=monitor.job_id)
if job.is_running is False:
# the job was killed, it is useless to run the task
raise JobNotRunning(monitor.job_id)
# it is important to save the task id soon, so that
# the revoke functionality can work
with monitor("storing task id", task=tsk, autoflush=True):
pass
with logs.handle(job):
check_mem_usage() # warn if too much memory is used
# run the task
try:
total = "total " + task_func.__name__
with monitor(total, task=tsk):
with GroundShakingIntensityModel.forbid_instantiation():
return task_func(*args)
finally:
# save on the db
CacheInserter.flushall()
# the task finished, we can remove from the performance
# table the associated row 'storing task id'
models.Performance.objects.filter(
oq_job=job, operation="storing task id", task_id=tsk.request.id
).delete()
def classical(group, src_filter, gsims, param, monitor=Monitor()):
"""
Compute the hazard curves for a set of sources belonging to the same
tectonic region type for all the GSIMs associated to that TRT.
The arguments are the same as in :func:`calc_hazard_curves`, except
for ``gsims``, which is a list of GSIM instances.
:returns:
a dictionary {grp_id: pmap} with attributes .grp_ids, .calc_times,
.eff_ruptures
"""
if getattr(group, 'src_interdep', None) == 'mutex':
mutex_weight = {
src.source_id: weight
for src, weight in zip(group.sources, group.srcs_weights)
}
else:
mutex_weight = None
grp_ids = set()
for src in group:
grp_ids.update(src.src_group_ids)
maxdist = src_filter.integration_distance
with GroundShakingIntensityModel.forbid_instantiation():
imtls = param['imtls']
trunclevel = param.get('truncation_level')
cmaker = ContextMaker(gsims, maxdist)
ctx_mon = monitor('make_contexts', measuremem=False)
poe_mon = monitor('get_poes', measuremem=False)
pmap = AccumDict({
grp_id: ProbabilityMap(len(imtls.array), len(gsims))
for grp_id in grp_ids
})
# AccumDict of arrays with 4 elements weight, nsites, calc_time, split
pmap.calc_times = AccumDict(accum=numpy.zeros(4))
pmap.eff_ruptures = AccumDict() # grp_id -> num_ruptures
for src, s_sites in src_filter(group): # filter now
t0 = time.time()
indep = group.rup_interdep == 'indep' if mutex_weight else True
poemap = cmaker.poe_map(src, s_sites, imtls, trunclevel, ctx_mon,
poe_mon, indep)
if mutex_weight: # mutex sources
weight = mutex_weight[src.source_id]
for sid in poemap:
pcurve = pmap[group.id].setdefault(sid, 0)
pcurve += poemap[sid] * weight
elif poemap:
for grp_id in src.src_group_ids:
pmap[grp_id] |= poemap
src_id = src.source_id.split(':', 1)[0]
pmap.calc_times[src_id] += numpy.array(
[src.weight, len(s_sites),
time.time() - t0, 1])
# storing the number of contributing ruptures too
pmap.eff_ruptures += {
grp_id: getattr(poemap, 'eff_ruptures', 0)
for grp_id in src.src_group_ids
}
if mutex_weight and group.grp_probability is not None:
pmap[group.id] *= group.grp_probability
return pmap
def wrapper(*args):
check_mem_usage() # check if too much memory is used
monitor = args[-1]
monitor.flush = NoFlush(monitor, func.__name__)
with monitor('total ' + func.__name__, measuremem=True), \
GroundShakingIntensityModel.forbid_instantiation():
result = func(*args)
rec_delattr(monitor, 'flush')
return result
def wrapper(*args):
monitor = args[-1]
check_mem_usage(monitor) # check if too much memory is used
monitor.flush = NoFlush(monitor, func.__name__)
with monitor('total ' + func.__name__, measuremem=True), \
GroundShakingIntensityModel.forbid_instantiation():
result = func(*args)
rec_delattr(monitor, 'flush')
return result
def wrapper(*args):
monitor = args[-1]
check_mem_usage(monitor) # check if too much memory is used
monitor.flush = NoFlush(monitor, func.__name__)
with monitor('total ' + func.__name__, measuremem=True), \
GroundShakingIntensityModel.forbid_instantiation():
result = func(*args)
# NB: flush must not be called in the workers - they must not
# have access to the datastore - so we remove it
rec_delattr(monitor, 'flush')
return result
def pmap_from_grp(sources,
source_site_filter,
imtls,
gsims,
truncation_level=None,
bbs=(),
monitor=Monitor()):
"""
Compute the hazard curves for a set of sources belonging to the same
tectonic region type for all the GSIMs associated to that TRT.
The arguments are the same as in :func:`calc_hazard_curves`, except
for ``gsims``, which is a list of GSIM instances.
:returns: a ProbabilityMap instance
"""
if isinstance(sources, SourceGroup):
group = sources
sources = group.sources
trt = sources[0].tectonic_region_type
else: # list of sources
trt = sources[0].tectonic_region_type
group = SourceGroup(trt, sources, 'src_group', 'indep', 'indep')
try:
maxdist = source_site_filter.integration_distance[trt]
except:
maxdist = source_site_filter.integration_distance
if hasattr(gsims, 'keys'): # dictionary trt -> gsim
gsims = [gsims[trt]]
with GroundShakingIntensityModel.forbid_instantiation():
imtls = DictArray(imtls)
cmaker = ContextMaker(gsims, maxdist)
ctx_mon = monitor('making contexts', measuremem=False)
pne_mon = monitor('computing poes', measuremem=False)
disagg_mon = monitor('get closest points', measuremem=False)
src_indep = group.src_interdep == 'indep'
pmap = ProbabilityMap(len(imtls.array), len(gsims))
pmap.calc_times = [] # pairs (src_id, delta_t)
pmap.grp_id = sources[0].src_group_id
for src, s_sites in source_site_filter(sources):
t0 = time.time()
poemap = poe_map(src, s_sites, imtls, cmaker, truncation_level,
bbs, group.rup_interdep == 'indep', ctx_mon,
pne_mon, disagg_mon)
if src_indep: # usual composition of probabilities
pmap |= poemap
else: # mutually exclusive probabilities
weight = float(group.srcs_weights[src.source_id])
for sid in poemap:
pmap[sid] += poemap[sid] * weight
pmap.calc_times.append(
(src.source_id, len(s_sites), time.time() - t0))
# storing the number of contributing ruptures too
pmap.eff_ruptures = {pmap.grp_id: pne_mon.counts}
return pmap
def pmap_from_grp(
sources, source_site_filter, imtls, gsims, truncation_level=None,
bbs=(), monitor=Monitor()):
"""
Compute the hazard curves for a set of sources belonging to the same
tectonic region type for all the GSIMs associated to that TRT.
The arguments are the same as in :func:`calc_hazard_curves`, except
for ``gsims``, which is a list of GSIM instances.
:returns: a ProbabilityMap instance
"""
if isinstance(sources, SourceGroup):
group = sources
sources = group.src_list
else:
group = SourceGroup(sources, 'src_group', 'indep', 'indep')
sources = group.src_list
trt = sources[0].tectonic_region_type
try:
maxdist = source_site_filter.integration_distance[trt]
except:
maxdist = source_site_filter.integration_distance
if hasattr(gsims, 'keys'):
gsims = [gsims[trt]]
# check all the sources belong to the same tectonic region
trts = set(src.tectonic_region_type for src in sources)
assert len(trts) == 1, 'Multiple TRTs: %s' % ', '.join(trts)
with GroundShakingIntensityModel.forbid_instantiation():
imtls = DictArray(imtls)
cmaker = ContextMaker(gsims, maxdist)
ctx_mon = monitor('making contexts', measuremem=False)
pne_mon = monitor('computing poes', measuremem=False)
disagg_mon = monitor('get closest points', measuremem=False)
src_indep = group.src_interdep == 'indep'
pmap = ProbabilityMap(len(imtls.array), len(gsims))
pmap.calc_times = [] # pairs (src_id, delta_t)
pmap.grp_id = sources[0].src_group_id
for src, s_sites in source_site_filter(sources):
t0 = time.time()
poemap = poe_map(
src, s_sites, imtls, cmaker, truncation_level, bbs,
group.rup_interdep == 'indep', ctx_mon, pne_mon, disagg_mon)
if src_indep: # usual composition of probabilities
pmap |= poemap
else: # mutually exclusive probabilities
weight = float(group.srcs_weights[src.source_id])
for sid in poemap:
pmap[sid] += poemap[sid] * weight
pmap.calc_times.append(
(src.source_id, len(s_sites), time.time() - t0))
# storing the number of contributing ruptures too
pmap.eff_ruptures = {pmap.grp_id: pne_mon.counts}
return pmap
def wrapper(*args):
monitor = args[-1]
check_mem_usage(monitor) # check if too much memory is used
monitor.flush = NoFlush(monitor, func.__name__)
with monitor('total ' + func.__name__, measuremem=True), \
GroundShakingIntensityModel.forbid_instantiation():
result = func(*args)
# NB: flush must not be called in the workers - they must not
# have access to the datastore - so we remove it
rec_delattr(monitor, 'flush')
return result
def pmap_from_trt(sources, src_filter, gsims, param, monitor=Monitor()):
"""
Compute the hazard curves for a set of sources belonging to the same
tectonic region type for all the GSIMs associated to that TRT.
:returns:
a dictionary {grp_id: pmap} with attributes .grp_ids, .calc_times,
.eff_ruptures
"""
maxdist = src_filter.integration_distance
srcs = []
grp_ids = set()
for src in sources:
if hasattr(src, '__iter__'): # MultiPointSource
srcs.extend(src)
else:
srcs.append(src)
grp_ids.update(src.src_group_ids)
del sources
with GroundShakingIntensityModel.forbid_instantiation():
imtls = param['imtls']
trunclevel = param.get('truncation_level')
cmaker = ContextMaker(gsims, maxdist)
ctx_mon = monitor('making contexts', measuremem=False)
pne_mons = [
monitor('%s.get_poes' % gsim, measuremem=False) for gsim in gsims
]
pmap = AccumDict({
grp_id: ProbabilityMap(len(imtls.array), len(gsims))
for grp_id in grp_ids
})
pmap.calc_times = [] # pairs (src_id, delta_t)
pmap.eff_ruptures = AccumDict() # grp_id -> num_ruptures
for src, s_sites in src_filter(srcs):
t0 = time.time()
poe = poe_map(src, s_sites, imtls, cmaker, trunclevel, ctx_mon,
pne_mons)
for grp_id in src.src_group_ids:
pmap[grp_id] |= poe
pmap.calc_times.append(
(src.source_id, src.weight, len(s_sites), time.time() - t0))
# storing the number of contributing ruptures too
pmap.eff_ruptures += {
grp_id: poe.eff_ruptures
for grp_id in src.src_group_ids
}
return pmap
def pmap_from_trt(sources, src_filter, gsims, param, monitor=Monitor()):
"""
Compute the hazard curves for a set of sources belonging to the same
tectonic region type for all the GSIMs associated to that TRT.
:returns:
a dictionary {grp_id: pmap} with attributes .grp_ids, .calc_times,
.eff_ruptures
"""
grp_ids = set()
for src in sources:
grp_ids.update(src.src_group_ids)
maxdist = src_filter.integration_distance
srcs = sum([split_source(src) for src in sources], []) # split first
with GroundShakingIntensityModel.forbid_instantiation():
imtls = param['imtls']
trunclevel = param.get('truncation_level')
cmaker = ContextMaker(gsims, maxdist)
ctx_mon = monitor('make_contexts', measuremem=False)
poe_mon = monitor('get_poes', measuremem=False)
pmap = AccumDict({
grp_id: ProbabilityMap(len(imtls.array), len(gsims))
for grp_id in grp_ids
})
pmap.calc_times = [] # pairs (src_id, delta_t)
pmap.eff_ruptures = AccumDict() # grp_id -> num_ruptures
for src, s_sites in src_filter(srcs): # filter now
t0 = time.time()
poemap = cmaker.poe_map(src, s_sites, imtls, trunclevel, ctx_mon,
poe_mon)
if poemap:
for grp_id in src.src_group_ids:
pmap[grp_id] |= poemap
pmap.calc_times.append(
(src.source_id, src.weight, len(s_sites), time.time() - t0))
# storing the number of contributing ruptures too
pmap.eff_ruptures += {
grp_id: getattr(poemap, 'eff_ruptures', 0)
for grp_id in src.src_group_ids
}
return pmap
def pmap_from_grp(sources,
sites,
imtls,
gsims,
truncation_level=None,
source_site_filter='SourceSitesFilter',
maximum_distance=None,
bbs=(),
monitor=Monitor()):
"""
Compute the hazard curves for a set of sources belonging to the same
tectonic region type for all the GSIMs associated to that TRT.
The arguments are the same as in :func:`calc_hazard_curves`, except
for ``gsims``, which is a list of GSIM instances.
:returns: a ProbabilityMap instance
"""
if source_site_filter == 'SourceSitesFilter': # default
source_site_filter = (filters.SourceSitesFilter(maximum_distance)
if maximum_distance else
filters.source_site_noop_filter)
with GroundShakingIntensityModel.forbid_instantiation():
imtls = DictArray(imtls)
cmaker = ContextMaker(gsims, maximum_distance)
ctx_mon = monitor('making contexts', measuremem=False)
pne_mon = monitor('computing poes', measuremem=False)
disagg_mon = monitor('get closest points', measuremem=False)
monitor.calc_times = [] # pairs (src_id, delta_t)
pmap = ProbabilityMap(len(imtls.array), len(gsims))
for src, s_sites in source_site_filter(sources, sites):
t0 = time.time()
pmap |= poe_map(src, s_sites, imtls, cmaker, truncation_level, bbs,
ctx_mon, pne_mon, disagg_mon)
# we are attaching the calculation times to the monitor
# so that oq-lite (and the engine) can store them
monitor.calc_times.append(
(src.source_id, len(s_sites), time.time() - t0))
# NB: source.id is an integer; it should not be confused
# with source.source_id, which is a string
monitor.eff_ruptures = pne_mon.counts # contributing ruptures
return pmap
def pmap_from_grp(group, src_filter, gsims, param, monitor=Monitor()):
"""
Compute the hazard curves for a set of sources belonging to the same
tectonic region type for all the GSIMs associated to that TRT.
The arguments are the same as in :func:`calc_hazard_curves`, except
for ``gsims``, which is a list of GSIM instances.
:returns: a dictionary {grp_id: ProbabilityMap instance}
"""
mutex_weight = {
src.source_id: weight
for src, weight in zip(group.sources, group.srcs_weights)
}
maxdist = src_filter.integration_distance
srcs = sum([split_source(src) for src in group.sources], [])
with GroundShakingIntensityModel.forbid_instantiation():
imtls = param['imtls']
trunclevel = param.get('truncation_level')
cmaker = ContextMaker(gsims, maxdist)
ctx_mon = monitor('make_contexts', measuremem=False)
poe_mon = monitor('get_poes', measuremem=False)
pmap = ProbabilityMap(len(imtls.array), len(gsims))
calc_times = [] # pairs (src_id, delta_t)
for src, s_sites in src_filter(srcs):
t0 = time.time()
poemap = cmaker.poe_map(src, s_sites, imtls, trunclevel, ctx_mon,
poe_mon, group.rup_interdep == 'indep')
weight = mutex_weight[src.source_id]
for sid in poemap:
pcurve = pmap.setdefault(sid, 0)
pcurve += poemap[sid] * weight
calc_times.append(
(src.source_id, src.weight, len(s_sites), time.time() - t0))
if group.grp_probability is not None:
pmap *= group.grp_probability
acc = AccumDict({group.id: pmap})
# adding the number of contributing ruptures too
acc.eff_ruptures = {group.id: ctx_mon.counts}
acc.calc_times = calc_times
return acc
def pmap_from_grp(sources, src_filter, gsims, param, monitor=Monitor()):
"""
Compute the hazard curves for a set of sources belonging to the same
tectonic region type for all the GSIMs associated to that TRT.
The arguments are the same as in :func:`calc_hazard_curves`, except
for ``gsims``, which is a list of GSIM instances.
:returns: a ProbabilityMap instance
"""
if isinstance(sources, SourceGroup):
group = sources
sources = group.sources
trt = sources[0].tectonic_region_type
mutex_weight = {
src.source_id: weight
for src, weight in zip(group.sources, group.srcs_weights)
}
else: # list of sources
trt = sources[0].tectonic_region_type
group = SourceGroup(trt, sources, 'src_group', 'indep', 'indep')
grp_id = sources[0].src_group_id
maxdist = src_filter.integration_distance
if hasattr(gsims, 'keys'): # dictionary trt -> gsim
gsims = [gsims[trt]]
srcs = []
for src in sources:
if hasattr(src, '__iter__'): # MultiPointSource
srcs.extend(src)
else:
srcs.append(src)
del sources
with GroundShakingIntensityModel.forbid_instantiation():
imtls = param['imtls']
trunclevel = param.get('truncation_level')
cmaker = ContextMaker(gsims, maxdist)
ctx_mon = monitor('making contexts', measuremem=False)
pne_mons = [
monitor('%s.get_poes' % gsim, measuremem=False) for gsim in gsims
]
src_indep = group.src_interdep == 'indep'
pmap = ProbabilityMap(len(imtls.array), len(gsims))
pmap.calc_times = [] # pairs (src_id, delta_t)
pmap.grp_id = grp_id
for src, s_sites in src_filter(srcs):
t0 = time.time()
poemap = poe_map(src, s_sites, imtls, cmaker, trunclevel, ctx_mon,
pne_mons, group.rup_interdep == 'indep')
if src_indep: # usual composition of probabilities
pmap |= poemap
else: # mutually exclusive probabilities
weight = mutex_weight[src.source_id]
for sid in poemap:
pcurve = pmap.setdefault(sid, 0)
pcurve += poemap[sid] * weight
pmap.calc_times.append(
(src.source_id, src.weight, len(s_sites), time.time() - t0))
# storing the number of contributing ruptures too
pmap.eff_ruptures = {pmap.grp_id: pne_mons[0].counts}
if group.grp_probability is not None:
return pmap * group.grp_probability
return pmap
