def test(self):
mags = numpy.array([5, 5], float)
dists = numpy.array([6, 6], float)
lons = numpy.array([19, 19], float)
lats = numpy.array([41.5, 41.5], float)
trts = numpy.array([0, 0], int)
trt_bins = ['trt1', 'trt2']
probs_one_or_more = numpy.array([0.1] * len(mags)).reshape(2, 1)
probs_exceed_given_rup = numpy.ones((len(mags), 2)) * 0.1
probs_no_exceed = (1 - probs_one_or_more)**probs_exceed_given_rup
bins_data = (mags, dists, lons, lats, trts, trt_bins, probs_no_exceed)
mag_bins = numpy.array([4, 6, 7], float)
dist_bins = numpy.array([0, 4, 8], float)
lon_bins = numpy.array([18, 20, 21], float)
lat_bins = numpy.array([40, 41, 42], float)
eps_bins = numpy.array([-2, 0, 2], float)
bin_edges = mag_bins, dist_bins, lon_bins, lat_bins, eps_bins, trt_bins
diss_matrix = disagg._arrange_data_in_bins(bins_data, bin_edges)
self.assertEqual(diss_matrix.shape, (2, 2, 2, 2, 2, 2))
for idx, value in [((0, 1, 0, 1, 0, 0), 0.02085163763902309),
((0, 1, 0, 1, 1, 0), 0.02085163763902309)]:
self.assertAlmostEqual(diss_matrix[idx], value)
diss_matrix[idx] = 0
self.assertEqual(diss_matrix.sum(), 0)
def test(self):
mags = numpy.array([5, 5], float)
dists = numpy.array([6, 6], float)
lons = numpy.array([19, 19], float)
lats = numpy.array([41.5, 41.5], float)
trts = numpy.array([0, 0], int)
trt_bins = ['trt1', 'trt2']
probs_one_or_more = numpy.array([0.1] * len(mags)).reshape(2, 1)
probs_exceed_given_rup = numpy.ones((len(mags), 2)) * 0.1
probs_no_exceed = (1 - probs_one_or_more) ** probs_exceed_given_rup
bins_data = (mags, dists, lons, lats, trts, trt_bins,
probs_no_exceed)
mag_bins = numpy.array([4, 6, 7], float)
dist_bins = numpy.array([0, 4, 8], float)
lon_bins = numpy.array([18, 20, 21], float)
lat_bins = numpy.array([40, 41, 42], float)
eps_bins = numpy.array([-2, 0, 2], float)
bin_edges = mag_bins, dist_bins, lon_bins, lat_bins, eps_bins, trt_bins
diss_matrix = disagg._arrange_data_in_bins(bins_data, bin_edges)
self.assertEqual(diss_matrix.shape, (2, 2, 2, 2, 2, 2))
for idx, value in [((0, 1, 0, 1, 0, 0), 0.02085163763902309),
((0, 1, 0, 1, 1, 0), 0.02085163763902309)]:
self.assertAlmostEqual(diss_matrix[idx], value)
diss_matrix[idx] = 0
self.assertEqual(diss_matrix.sum(), 0)
def compute_disagg(job_id, sitecol, sources, lt_model, gsim_by_rlz,
trt_num, curves_dict, bin_edges):
# see https://bugs.launchpad.net/oq-engine/+bug/1279247 for an explanation
# of the algorithm used
"""
:param int job_id:
ID of the currently running :class:`openquake.engine.db.models.OqJob`
:param sitecol:
a :class:`openquake.hazardlib.site.SiteCollection` instance
:param list sources:
list of hazardlib source objects
:param lt_model:
an instance of :class:`openquake.engine.db.models.LtSourceModel`
:param dict gsim_by_rlz:
a dictionary of gsims, one for each realization
:param dict trt_num:
a dictionary Tectonic Region Type -> incremental number
:param curves_dict:
a dictionary with the hazard curves for sites, realizations and IMTs
:param bin_egdes:
a dictionary (lt_model_id, site_id) -> edges
:returns:
a dictionary of probability arrays, with composite key
(site.id, rlz.id, poe, imt, iml, trt_names).
"""
mon = LightMonitor('disagg', job_id, compute_disagg)
hc = models.OqJob.objects.get(id=job_id).hazard_calculation
trt_names = tuple(lt_model.get_tectonic_region_types())
result = {} # site.id, rlz.id, poe, imt, iml, trt_names -> array
for site in sitecol:
# edges as wanted by disagg._arrange_data_in_bins
try:
edges = bin_edges[lt_model.id, site.id]
except KeyError:
# bin_edges for a given site are missing if the site is far away
continue
# generate source, rupture, sites once per site
source_ruptures = list(hc.gen_ruptures_for_site(site, sources, mon))
if not source_ruptures:
continue
logs.LOG.info('Collecting bins from %d ruptures close to %s',
sum(len(rupts) for src, rupts in source_ruptures),
site.location)
with EnginePerformanceMonitor(
'collecting bins', job_id, compute_disagg):
bdata = _collect_bins_data(
mon, trt_num, source_ruptures, site, curves_dict[site.id],
gsim_by_rlz, hc.intensity_measure_types_and_levels,
hc.poes_disagg, hc.truncation_level,
hc.num_epsilon_bins)
if not bdata.pnes: # no contributions for this site
continue
for poe in hc.poes_disagg:
for imt in hc.intensity_measure_types_and_levels:
for rlz in gsim_by_rlz:
# extract the probabilities of non-exceedance for the
# given realization, disaggregation PoE, and IMT
iml_pne_pairs = [pne[rlz.id, poe, imt]
for pne in bdata.pnes]
iml = iml_pne_pairs[0][0]
probs = numpy.array([p for (i, p) in iml_pne_pairs], float)
# bins in a format handy for hazardlib
bins = [bdata.mags, bdata.dists, bdata.lons, bdata.lats,
bdata.trts, None, probs]
# call disagg._arrange_data_in_bins and populate the result
with EnginePerformanceMonitor(
'arranging bins', job_id, compute_disagg):
key = (site.id, rlz.id, poe, imt, iml, trt_names)
matrix = disagg._arrange_data_in_bins(
bins, edges + (trt_names,))
result[key] = numpy.array(
[fn(matrix) for fn in disagg.pmf_map.values()])
return result
def compute_disagg(sitecol, sources, src_group_id, rlzs_assoc, trt_names, curves_dict, bin_edges, oqparam, monitor):
# see https://bugs.launchpad.net/oq-engine/+bug/1279247 for an explanation
# of the algorithm used
"""
:param sitecol:
a :class:`openquake.hazardlib.site.SiteCollection` instance
:param sources:
list of hazardlib source objects
:param src_group_id:
numeric ID of a SourceGroup instance
:param rlzs_assoc:
a :class:`openquake.commonlib.source.RlzsAssoc` instance
:param dict trt_names:
a tuple of names for the given tectonic region type
:param curves_dict:
a dictionary with the hazard curves for sites, realizations and IMTs
:param bin_egdes:
a dictionary site_id -> edges
:param oqparam:
the parameters in the job.ini file
:param monitor:
monitor of the currently running job
:returns:
a dictionary of probability arrays, with composite key
(sid, rlz.id, poe, imt, iml, trt_names).
"""
trt = sources[0].tectonic_region_type
try:
max_dist = oqparam.maximum_distance[trt]
except KeyError:
max_dist = oqparam.maximum_distance["default"]
trt_num = dict((trt, i) for i, trt in enumerate(trt_names))
gsims = rlzs_assoc.gsims_by_grp_id[src_group_id]
result = {} # sid, rlz.id, poe, imt, iml, trt_names -> array
collecting_mon = monitor("collecting bins")
arranging_mon = monitor("arranging bins")
for site, sid in zip(sitecol, sitecol.sids):
# edges as wanted by disagg._arrange_data_in_bins
try:
edges = bin_edges[sid]
except KeyError:
# bin_edges for a given site are missing if the site is far away
continue
# generate source, rupture, sites once per site
sitecol = SiteCollection([site])
source_ruptures = [
(src, src.iter_ruptures()) for src in sources if src.filter_sites_by_distance_to_source(max_dist, sitecol)
]
if not source_ruptures:
continue
with collecting_mon:
bdata = disagg._collect_bins_data(
trt_num,
source_ruptures,
site,
curves_dict[sid],
src_group_id,
rlzs_assoc,
gsims,
oqparam.imtls,
oqparam.poes_disagg,
oqparam.truncation_level,
oqparam.num_epsilon_bins,
monitor,
)
if not bdata.pnes: # no contributions for this site
continue
for poe in oqparam.poes_disagg:
for imt in oqparam.imtls:
for gsim in gsims:
for rlz in rlzs_assoc[src_group_id, gsim]:
rlzi = rlz.ordinal
# extract the probabilities of non-exceedance for the
# given realization, disaggregation PoE, and IMT
iml_pne_pairs = [pne[rlzi, poe, imt] for pne in bdata.pnes]
iml = iml_pne_pairs[0][0]
probs = numpy.array([p for (i, p) in iml_pne_pairs], float)
# bins in a format handy for hazardlib
bins = [bdata.mags, bdata.dists, bdata.lons, bdata.lats, bdata.trts, None, probs]
# call disagg._arrange_data_in_bins
with arranging_mon:
key = (sid, rlzi, poe, imt, iml, trt_names)
matrix = disagg._arrange_data_in_bins(bins, edges + (trt_names,))
result[key] = numpy.array([fn(matrix) for fn in disagg.pmf_map.values()])
return result
def compute_disagg(sitecol, sources, trt_model_id, trt_num, curves_dict,
bin_edges, monitor):
# see https://bugs.launchpad.net/oq-engine/+bug/1279247 for an explanation
# of the algorithm used
"""
:param sitecol:
a :class:`openquake.hazardlib.site.SiteCollection` instance
:param list sources:
list of hazardlib source objects
:param lt_model:
an instance of :class:`openquake.engine.db.models.LtSourceModel`
:param dict trt_num:
a dictionary Tectonic Region Type -> incremental number
:param curves_dict:
a dictionary with the hazard curves for sites, realizations and IMTs
:param bin_egdes:
a dictionary (lt_model_id, site_id) -> edges
:param monitor:
monitor of the currently running job
:returns:
a dictionary of probability arrays, with composite key
(site.id, rlz.id, poe, imt, iml, trt_names).
"""
hc = models.oqparam(monitor.job_id)
trt_model = models.TrtModel.objects.get(pk=trt_model_id)
gsims = trt_model.get_gsim_instances()
lt_model_id = trt_model.lt_model.id
rlzs = trt_model.get_rlzs_by_gsim()
trt_names = tuple(trt_model.lt_model.get_tectonic_region_types())
result = {} # site.id, rlz.id, poe, imt, iml, trt_names -> array
collecting_mon = monitor('collecting bins')
arranging_mon = monitor('arranging bins')
for site in sitecol:
# edges as wanted by disagg._arrange_data_in_bins
try:
edges = bin_edges[lt_model_id, site.id]
except KeyError:
# bin_edges for a given site are missing if the site is far away
continue
# generate source, rupture, sites once per site
source_ruptures = list(
gen_ruptures_for_site(site, sources, hc.maximum_distance, monitor))
if not source_ruptures:
continue
logs.LOG.info('Collecting bins from %d ruptures close to %s',
sum(len(rupts) for src, rupts in source_ruptures),
site.location)
with collecting_mon:
bdata = _collect_bins_data(trt_num, source_ruptures, site,
curves_dict[site.id], trt_model_id,
gsims, hc.imtls, hc.poes_disagg,
hc.truncation_level,
hc.num_epsilon_bins, monitor)
if not bdata.pnes: # no contributions for this site
continue
for poe in hc.poes_disagg:
for imt in hc.imtls:
for gsim in gsims:
for rlz in rlzs[gsim.__class__.__name__]:
# extract the probabilities of non-exceedance for the
# given realization, disaggregation PoE, and IMT
iml_pne_pairs = [
pne[rlz.id, poe, imt] for pne in bdata.pnes
]
iml = iml_pne_pairs[0][0]
probs = numpy.array([p for (i, p) in iml_pne_pairs],
float)
# bins in a format handy for hazardlib
bins = [
bdata.mags, bdata.dists, bdata.lons, bdata.lats,
bdata.trts, None, probs
]
# call disagg._arrange_data_in_bins
with arranging_mon:
key = (site.id, rlz.id, poe, imt, iml, trt_names)
matrix = disagg._arrange_data_in_bins(
bins, edges + (trt_names, ))
result[key] = numpy.array(
[fn(matrix) for fn in disagg.pmf_map.values()])
collecting_mon.flush()
arranging_mon.flush()
return result
def compute_disagg(sitecol, sources, trt_model_id, rlzs_assoc, trt_names,
curves_dict, bin_edges, oqparam, monitor):
# see https://bugs.launchpad.net/oq-engine/+bug/1279247 for an explanation
# of the algorithm used
"""
:param sitecol:
a :class:`openquake.hazardlib.site.SiteCollection` instance
:param sources:
list of hazardlib source objects
:param trt_model_id:
numeric ID of a TrtModel instance
:param rlzs_assoc:
a :class:`openquake.commonlib.source.RlzsAssoc` instance
:param dict trt_names:
a tuple of names for the given tectonic region type
:param curves_dict:
a dictionary with the hazard curves for sites, realizations and IMTs
:param bin_egdes:
a dictionary site_id -> edges
:param oqparam:
the parameters in the job.ini file
:param monitor:
monitor of the currently running job
:returns:
a dictionary of probability arrays, with composite key
(site.id, rlz.id, poe, imt, iml, trt_names).
"""
trt_num = dict((trt, i) for i, trt in enumerate(trt_names))
gsims = rlzs_assoc.gsims_by_trt_id[trt_model_id]
result = {} # site.id, rlz.id, poe, imt, iml, trt_names -> array
collecting_mon = monitor('collecting bins')
arranging_mon = monitor('arranging bins')
for site in sitecol:
# edges as wanted by disagg._arrange_data_in_bins
try:
edges = bin_edges[site.id]
except KeyError:
# bin_edges for a given site are missing if the site is far away
continue
# generate source, rupture, sites once per site
source_ruptures = list(
gen_ruptures_for_site(site, sources, oqparam.maximum_distance,
monitor))
if not source_ruptures:
continue
with collecting_mon:
bdata = _collect_bins_data(trt_num, source_ruptures, site,
curves_dict[site.id], trt_model_id,
rlzs_assoc, gsims, oqparam.imtls,
oqparam.poes_disagg,
oqparam.truncation_level,
oqparam.num_epsilon_bins, monitor)
if not bdata.pnes: # no contributions for this site
continue
for poe in oqparam.poes_disagg:
for imt in oqparam.imtls:
for gsim in gsims:
for rlz in rlzs_assoc[trt_model_id,
gsim.__class__.__name__]:
rlzi = rlz.ordinal
# extract the probabilities of non-exceedance for the
# given realization, disaggregation PoE, and IMT
iml_pne_pairs = [
pne[rlzi, poe, imt] for pne in bdata.pnes
]
iml = iml_pne_pairs[0][0]
probs = numpy.array([p for (i, p) in iml_pne_pairs],
float)
# bins in a format handy for hazardlib
bins = [
bdata.mags, bdata.dists, bdata.lons, bdata.lats,
bdata.trts, None, probs
]
# call disagg._arrange_data_in_bins
with arranging_mon:
key = (site.id, rlzi, poe, imt, iml, trt_names)
matrix = disagg._arrange_data_in_bins(
bins, edges + (trt_names, ))
result[key] = numpy.array(
[fn(matrix) for fn in disagg.pmf_map.values()])
return result
def test(self):
mags = numpy.array([5, 9, 5, 5, 9, 7, 5, 5, 6, 6, 9.2, 8, 7], float)
dists = numpy.array([3, 1, 5, 13, 14, 6, 12, 10, 7, 4, 11, 13.4, 5], float)
lons = numpy.array([22, 21, 20, 21, 21, 22, 21, 21, 20.3, 21, 20.5, 21.5, 22], float)
lats = numpy.array([44, 44, 45, 45, 44, 44, 45, 45, 44, 44, 45, 45, 43.3], float)
joint_probs = numpy.array(
[
[0.0, 0.0, 0.0],
[0.02, 0.04, 0.02],
[0.0, 0.0, 0.003],
[0.0, 0.0165, 0.00033],
[0.0, 0.0, 0.0],
[0.0, 0.0, 0.001],
[0.0212, 0.053, 0.0212],
[0.0132, 0.0198, 0.0132],
[0.03, 0.04, 0.03],
[0.0, 0.0, 0.01],
[0.0, 0.0, 0.0],
[0.0, 0.004, 0.0016],
[0.003, 0.015, 0.003],
]
)
trts = numpy.array([0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1], int)
trt_bins = ["trt1", "trt2"]
bins_data = mags, dists, lons, lats, joint_probs, trts, trt_bins
mag_bins = numpy.array([4, 6, 8, 10], float)
dist_bins = numpy.array([0, 4, 8, 12, 16], float)
lon_bins = numpy.array([19.2, 21, 22.8], float)
lat_bins = numpy.array([43.2, 44.4, 45.6], float)
eps_bins = numpy.array([-1.2, -0.4, 0.4, 1.2], float)
bin_edges = mag_bins, dist_bins, lon_bins, lat_bins, eps_bins, trt_bins
diss_matrix = disagg._arrange_data_in_bins(bins_data, bin_edges)
self.assertEqual(diss_matrix.shape, (3, 4, 2, 2, 3, 2))
self.assertAlmostEqual(diss_matrix.sum(), 1)
for idx, value in [
((0, 0, 0, 0, 2, 0), 0.0263831),
((0, 1, 0, 0, 0, 0), 0.0791494),
((0, 1, 0, 0, 1, 0), 0.1055325),
((0, 1, 0, 0, 2, 0), 0.0791494),
((0, 1, 0, 1, 2, 1), 0.0079149),
((0, 2, 0, 1, 0, 0), 0.0559322),
((0, 2, 0, 1, 0, 1), 0.0348257),
((0, 2, 0, 1, 1, 0), 0.1398306),
((0, 2, 0, 1, 1, 1), 0.0522386),
((0, 2, 0, 1, 2, 0), 0.0559322),
((0, 2, 0, 1, 2, 1), 0.0348257),
((0, 3, 0, 1, 1, 1), 0.0435322),
((0, 3, 0, 1, 2, 1), 0.0008706),
((1, 1, 1, 0, 0, 1), 0.0079149),
((1, 1, 1, 0, 1, 1), 0.0395747),
((1, 1, 1, 0, 2, 1), 0.0105533),
((1, 3, 1, 1, 1, 1), 0.0105533),
((1, 3, 1, 1, 2, 1), 0.0042213),
((2, 0, 0, 0, 0, 0), 0.0527663),
((2, 0, 0, 0, 1, 0), 0.1055325),
((2, 0, 0, 0, 2, 0), 0.0527663),
]:
self.assertAlmostEqual(diss_matrix[idx], value)
diss_matrix[idx] = 0
self.assertEqual(diss_matrix.sum(), 0)
def compute_disagg(src_filter, sources, src_group_id, rlzs_assoc, trt_names,
curves_dict, bin_edges, oqparam, monitor):
# see https://bugs.launchpad.net/oq-engine/+bug/1279247 for an explanation
# of the algorithm used
"""
:param src_filter:
a :class:`openquake.hazardlib.calc.filter.SourceFilter` instance
:param sources:
list of hazardlib source objects
:param src_group_id:
numeric ID of a SourceGroup instance
:param rlzs_assoc:
a :class:`openquake.commonlib.source.RlzsAssoc` instance
:param dict trt_names:
a tuple of names for the given tectonic region type
:param curves_dict:
a dictionary with the hazard curves for sites, realizations and IMTs
:param bin_egdes:
a dictionary site_id -> edges
:param oqparam:
the parameters in the job.ini file
:param monitor:
monitor of the currently running job
:returns:
a dictionary of probability arrays, with composite key
(sid, rlz.id, poe, imt, iml, trt_names).
"""
sitecol = src_filter.sitecol
trt_num = dict((trt, i) for i, trt in enumerate(trt_names))
gsims = rlzs_assoc.gsims_by_grp_id[src_group_id]
result = {} # sid, rlz.id, poe, imt, iml, trt_names -> array
collecting_mon = monitor('collecting bins')
arranging_mon = monitor('arranging bins')
for site, sid in zip(sitecol, sitecol.sids):
# edges as wanted by disagg._arrange_data_in_bins
try:
edges = bin_edges[sid]
except KeyError:
# bin_edges for a given site are missing if the site is far away
continue
# generate source, rupture, sites once per site
with collecting_mon:
bdata = disagg._collect_bins_data(
trt_num, sources, site, curves_dict[sid], src_group_id,
rlzs_assoc, gsims, oqparam.imtls, oqparam.poes_disagg,
oqparam.truncation_level, oqparam.num_epsilon_bins,
oqparam.iml_disagg, monitor)
for (rlzi, poe, imt), iml_pne_pairs in bdata.pnes.items():
# extract the probabilities of non-exceedance for the
# given realization, disaggregation PoE, and IMT
iml = iml_pne_pairs[0][0]
probs = numpy.array([p for (i, p) in iml_pne_pairs], float)
# bins in a format handy for hazardlib
bins = [
bdata.mags, bdata.dists, bdata.lons, bdata.lats, bdata.trts,
None, probs
]
# call disagg._arrange_data_in_bins
with arranging_mon:
key = (sid, rlzi, poe, imt, iml, trt_names)
matrix = disagg._arrange_data_in_bins(bins,
edges + (trt_names, ))
result[key] = numpy.array(
[fn(matrix) for fn in disagg.pmf_map.values()])
return result