def test(self): mags = numpy.array([5, 5], float) dists = numpy.array([6, 6], float) lons = numpy.array([19, 19], float) lats = numpy.array([41.5, 41.5], float) trts = numpy.array([0, 0], int) trt_bins = ['trt1', 'trt2'] probs_one_or_more = numpy.array([0.1] * len(mags)).reshape(2, 1) probs_exceed_given_rup = numpy.ones((len(mags), 2)) * 0.1 probs_no_exceed = (1 - probs_one_or_more)**probs_exceed_given_rup bins_data = (mags, dists, lons, lats, trts, trt_bins, probs_no_exceed) mag_bins = numpy.array([4, 6, 7], float) dist_bins = numpy.array([0, 4, 8], float) lon_bins = numpy.array([18, 20, 21], float) lat_bins = numpy.array([40, 41, 42], float) eps_bins = numpy.array([-2, 0, 2], float) bin_edges = mag_bins, dist_bins, lon_bins, lat_bins, eps_bins, trt_bins diss_matrix = disagg._arrange_data_in_bins(bins_data, bin_edges) self.assertEqual(diss_matrix.shape, (2, 2, 2, 2, 2, 2)) for idx, value in [((0, 1, 0, 1, 0, 0), 0.02085163763902309), ((0, 1, 0, 1, 1, 0), 0.02085163763902309)]: self.assertAlmostEqual(diss_matrix[idx], value) diss_matrix[idx] = 0 self.assertEqual(diss_matrix.sum(), 0)
def test(self): mags = numpy.array([5, 5], float) dists = numpy.array([6, 6], float) lons = numpy.array([19, 19], float) lats = numpy.array([41.5, 41.5], float) trts = numpy.array([0, 0], int) trt_bins = ['trt1', 'trt2'] probs_one_or_more = numpy.array([0.1] * len(mags)).reshape(2, 1) probs_exceed_given_rup = numpy.ones((len(mags), 2)) * 0.1 probs_no_exceed = (1 - probs_one_or_more) ** probs_exceed_given_rup bins_data = (mags, dists, lons, lats, trts, trt_bins, probs_no_exceed) mag_bins = numpy.array([4, 6, 7], float) dist_bins = numpy.array([0, 4, 8], float) lon_bins = numpy.array([18, 20, 21], float) lat_bins = numpy.array([40, 41, 42], float) eps_bins = numpy.array([-2, 0, 2], float) bin_edges = mag_bins, dist_bins, lon_bins, lat_bins, eps_bins, trt_bins diss_matrix = disagg._arrange_data_in_bins(bins_data, bin_edges) self.assertEqual(diss_matrix.shape, (2, 2, 2, 2, 2, 2)) for idx, value in [((0, 1, 0, 1, 0, 0), 0.02085163763902309), ((0, 1, 0, 1, 1, 0), 0.02085163763902309)]: self.assertAlmostEqual(diss_matrix[idx], value) diss_matrix[idx] = 0 self.assertEqual(diss_matrix.sum(), 0)
def compute_disagg(job_id, sitecol, sources, lt_model, gsim_by_rlz, trt_num, curves_dict, bin_edges): # see https://bugs.launchpad.net/oq-engine/+bug/1279247 for an explanation # of the algorithm used """ :param int job_id: ID of the currently running :class:`openquake.engine.db.models.OqJob` :param sitecol: a :class:`openquake.hazardlib.site.SiteCollection` instance :param list sources: list of hazardlib source objects :param lt_model: an instance of :class:`openquake.engine.db.models.LtSourceModel` :param dict gsim_by_rlz: a dictionary of gsims, one for each realization :param dict trt_num: a dictionary Tectonic Region Type -> incremental number :param curves_dict: a dictionary with the hazard curves for sites, realizations and IMTs :param bin_egdes: a dictionary (lt_model_id, site_id) -> edges :returns: a dictionary of probability arrays, with composite key (site.id, rlz.id, poe, imt, iml, trt_names). """ mon = LightMonitor('disagg', job_id, compute_disagg) hc = models.OqJob.objects.get(id=job_id).hazard_calculation trt_names = tuple(lt_model.get_tectonic_region_types()) result = {} # site.id, rlz.id, poe, imt, iml, trt_names -> array for site in sitecol: # edges as wanted by disagg._arrange_data_in_bins try: edges = bin_edges[lt_model.id, site.id] except KeyError: # bin_edges for a given site are missing if the site is far away continue # generate source, rupture, sites once per site source_ruptures = list(hc.gen_ruptures_for_site(site, sources, mon)) if not source_ruptures: continue logs.LOG.info('Collecting bins from %d ruptures close to %s', sum(len(rupts) for src, rupts in source_ruptures), site.location) with EnginePerformanceMonitor( 'collecting bins', job_id, compute_disagg): bdata = _collect_bins_data( mon, trt_num, source_ruptures, site, curves_dict[site.id], gsim_by_rlz, hc.intensity_measure_types_and_levels, hc.poes_disagg, hc.truncation_level, hc.num_epsilon_bins) if not bdata.pnes: # no contributions for this site continue for poe in hc.poes_disagg: for imt in hc.intensity_measure_types_and_levels: for rlz in gsim_by_rlz: # extract the probabilities of non-exceedance for the # given realization, disaggregation PoE, and IMT iml_pne_pairs = [pne[rlz.id, poe, imt] for pne in bdata.pnes] iml = iml_pne_pairs[0][0] probs = numpy.array([p for (i, p) in iml_pne_pairs], float) # bins in a format handy for hazardlib bins = [bdata.mags, bdata.dists, bdata.lons, bdata.lats, bdata.trts, None, probs] # call disagg._arrange_data_in_bins and populate the result with EnginePerformanceMonitor( 'arranging bins', job_id, compute_disagg): key = (site.id, rlz.id, poe, imt, iml, trt_names) matrix = disagg._arrange_data_in_bins( bins, edges + (trt_names,)) result[key] = numpy.array( [fn(matrix) for fn in disagg.pmf_map.values()]) return result
def compute_disagg(sitecol, sources, src_group_id, rlzs_assoc, trt_names, curves_dict, bin_edges, oqparam, monitor): # see https://bugs.launchpad.net/oq-engine/+bug/1279247 for an explanation # of the algorithm used """ :param sitecol: a :class:`openquake.hazardlib.site.SiteCollection` instance :param sources: list of hazardlib source objects :param src_group_id: numeric ID of a SourceGroup instance :param rlzs_assoc: a :class:`openquake.commonlib.source.RlzsAssoc` instance :param dict trt_names: a tuple of names for the given tectonic region type :param curves_dict: a dictionary with the hazard curves for sites, realizations and IMTs :param bin_egdes: a dictionary site_id -> edges :param oqparam: the parameters in the job.ini file :param monitor: monitor of the currently running job :returns: a dictionary of probability arrays, with composite key (sid, rlz.id, poe, imt, iml, trt_names). """ trt = sources[0].tectonic_region_type try: max_dist = oqparam.maximum_distance[trt] except KeyError: max_dist = oqparam.maximum_distance["default"] trt_num = dict((trt, i) for i, trt in enumerate(trt_names)) gsims = rlzs_assoc.gsims_by_grp_id[src_group_id] result = {} # sid, rlz.id, poe, imt, iml, trt_names -> array collecting_mon = monitor("collecting bins") arranging_mon = monitor("arranging bins") for site, sid in zip(sitecol, sitecol.sids): # edges as wanted by disagg._arrange_data_in_bins try: edges = bin_edges[sid] except KeyError: # bin_edges for a given site are missing if the site is far away continue # generate source, rupture, sites once per site sitecol = SiteCollection([site]) source_ruptures = [ (src, src.iter_ruptures()) for src in sources if src.filter_sites_by_distance_to_source(max_dist, sitecol) ] if not source_ruptures: continue with collecting_mon: bdata = disagg._collect_bins_data( trt_num, source_ruptures, site, curves_dict[sid], src_group_id, rlzs_assoc, gsims, oqparam.imtls, oqparam.poes_disagg, oqparam.truncation_level, oqparam.num_epsilon_bins, monitor, ) if not bdata.pnes: # no contributions for this site continue for poe in oqparam.poes_disagg: for imt in oqparam.imtls: for gsim in gsims: for rlz in rlzs_assoc[src_group_id, gsim]: rlzi = rlz.ordinal # extract the probabilities of non-exceedance for the # given realization, disaggregation PoE, and IMT iml_pne_pairs = [pne[rlzi, poe, imt] for pne in bdata.pnes] iml = iml_pne_pairs[0][0] probs = numpy.array([p for (i, p) in iml_pne_pairs], float) # bins in a format handy for hazardlib bins = [bdata.mags, bdata.dists, bdata.lons, bdata.lats, bdata.trts, None, probs] # call disagg._arrange_data_in_bins with arranging_mon: key = (sid, rlzi, poe, imt, iml, trt_names) matrix = disagg._arrange_data_in_bins(bins, edges + (trt_names,)) result[key] = numpy.array([fn(matrix) for fn in disagg.pmf_map.values()]) return result
def compute_disagg(sitecol, sources, trt_model_id, trt_num, curves_dict, bin_edges, monitor): # see https://bugs.launchpad.net/oq-engine/+bug/1279247 for an explanation # of the algorithm used """ :param sitecol: a :class:`openquake.hazardlib.site.SiteCollection` instance :param list sources: list of hazardlib source objects :param lt_model: an instance of :class:`openquake.engine.db.models.LtSourceModel` :param dict trt_num: a dictionary Tectonic Region Type -> incremental number :param curves_dict: a dictionary with the hazard curves for sites, realizations and IMTs :param bin_egdes: a dictionary (lt_model_id, site_id) -> edges :param monitor: monitor of the currently running job :returns: a dictionary of probability arrays, with composite key (site.id, rlz.id, poe, imt, iml, trt_names). """ hc = models.oqparam(monitor.job_id) trt_model = models.TrtModel.objects.get(pk=trt_model_id) gsims = trt_model.get_gsim_instances() lt_model_id = trt_model.lt_model.id rlzs = trt_model.get_rlzs_by_gsim() trt_names = tuple(trt_model.lt_model.get_tectonic_region_types()) result = {} # site.id, rlz.id, poe, imt, iml, trt_names -> array collecting_mon = monitor('collecting bins') arranging_mon = monitor('arranging bins') for site in sitecol: # edges as wanted by disagg._arrange_data_in_bins try: edges = bin_edges[lt_model_id, site.id] except KeyError: # bin_edges for a given site are missing if the site is far away continue # generate source, rupture, sites once per site source_ruptures = list( gen_ruptures_for_site(site, sources, hc.maximum_distance, monitor)) if not source_ruptures: continue logs.LOG.info('Collecting bins from %d ruptures close to %s', sum(len(rupts) for src, rupts in source_ruptures), site.location) with collecting_mon: bdata = _collect_bins_data(trt_num, source_ruptures, site, curves_dict[site.id], trt_model_id, gsims, hc.imtls, hc.poes_disagg, hc.truncation_level, hc.num_epsilon_bins, monitor) if not bdata.pnes: # no contributions for this site continue for poe in hc.poes_disagg: for imt in hc.imtls: for gsim in gsims: for rlz in rlzs[gsim.__class__.__name__]: # extract the probabilities of non-exceedance for the # given realization, disaggregation PoE, and IMT iml_pne_pairs = [ pne[rlz.id, poe, imt] for pne in bdata.pnes ] iml = iml_pne_pairs[0][0] probs = numpy.array([p for (i, p) in iml_pne_pairs], float) # bins in a format handy for hazardlib bins = [ bdata.mags, bdata.dists, bdata.lons, bdata.lats, bdata.trts, None, probs ] # call disagg._arrange_data_in_bins with arranging_mon: key = (site.id, rlz.id, poe, imt, iml, trt_names) matrix = disagg._arrange_data_in_bins( bins, edges + (trt_names, )) result[key] = numpy.array( [fn(matrix) for fn in disagg.pmf_map.values()]) collecting_mon.flush() arranging_mon.flush() return result
def compute_disagg(sitecol, sources, trt_model_id, rlzs_assoc, trt_names, curves_dict, bin_edges, oqparam, monitor): # see https://bugs.launchpad.net/oq-engine/+bug/1279247 for an explanation # of the algorithm used """ :param sitecol: a :class:`openquake.hazardlib.site.SiteCollection` instance :param sources: list of hazardlib source objects :param trt_model_id: numeric ID of a TrtModel instance :param rlzs_assoc: a :class:`openquake.commonlib.source.RlzsAssoc` instance :param dict trt_names: a tuple of names for the given tectonic region type :param curves_dict: a dictionary with the hazard curves for sites, realizations and IMTs :param bin_egdes: a dictionary site_id -> edges :param oqparam: the parameters in the job.ini file :param monitor: monitor of the currently running job :returns: a dictionary of probability arrays, with composite key (site.id, rlz.id, poe, imt, iml, trt_names). """ trt_num = dict((trt, i) for i, trt in enumerate(trt_names)) gsims = rlzs_assoc.gsims_by_trt_id[trt_model_id] result = {} # site.id, rlz.id, poe, imt, iml, trt_names -> array collecting_mon = monitor('collecting bins') arranging_mon = monitor('arranging bins') for site in sitecol: # edges as wanted by disagg._arrange_data_in_bins try: edges = bin_edges[site.id] except KeyError: # bin_edges for a given site are missing if the site is far away continue # generate source, rupture, sites once per site source_ruptures = list( gen_ruptures_for_site(site, sources, oqparam.maximum_distance, monitor)) if not source_ruptures: continue with collecting_mon: bdata = _collect_bins_data(trt_num, source_ruptures, site, curves_dict[site.id], trt_model_id, rlzs_assoc, gsims, oqparam.imtls, oqparam.poes_disagg, oqparam.truncation_level, oqparam.num_epsilon_bins, monitor) if not bdata.pnes: # no contributions for this site continue for poe in oqparam.poes_disagg: for imt in oqparam.imtls: for gsim in gsims: for rlz in rlzs_assoc[trt_model_id, gsim.__class__.__name__]: rlzi = rlz.ordinal # extract the probabilities of non-exceedance for the # given realization, disaggregation PoE, and IMT iml_pne_pairs = [ pne[rlzi, poe, imt] for pne in bdata.pnes ] iml = iml_pne_pairs[0][0] probs = numpy.array([p for (i, p) in iml_pne_pairs], float) # bins in a format handy for hazardlib bins = [ bdata.mags, bdata.dists, bdata.lons, bdata.lats, bdata.trts, None, probs ] # call disagg._arrange_data_in_bins with arranging_mon: key = (site.id, rlzi, poe, imt, iml, trt_names) matrix = disagg._arrange_data_in_bins( bins, edges + (trt_names, )) result[key] = numpy.array( [fn(matrix) for fn in disagg.pmf_map.values()]) return result
def test(self): mags = numpy.array([5, 9, 5, 5, 9, 7, 5, 5, 6, 6, 9.2, 8, 7], float) dists = numpy.array([3, 1, 5, 13, 14, 6, 12, 10, 7, 4, 11, 13.4, 5], float) lons = numpy.array([22, 21, 20, 21, 21, 22, 21, 21, 20.3, 21, 20.5, 21.5, 22], float) lats = numpy.array([44, 44, 45, 45, 44, 44, 45, 45, 44, 44, 45, 45, 43.3], float) joint_probs = numpy.array( [ [0.0, 0.0, 0.0], [0.02, 0.04, 0.02], [0.0, 0.0, 0.003], [0.0, 0.0165, 0.00033], [0.0, 0.0, 0.0], [0.0, 0.0, 0.001], [0.0212, 0.053, 0.0212], [0.0132, 0.0198, 0.0132], [0.03, 0.04, 0.03], [0.0, 0.0, 0.01], [0.0, 0.0, 0.0], [0.0, 0.004, 0.0016], [0.003, 0.015, 0.003], ] ) trts = numpy.array([0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1], int) trt_bins = ["trt1", "trt2"] bins_data = mags, dists, lons, lats, joint_probs, trts, trt_bins mag_bins = numpy.array([4, 6, 8, 10], float) dist_bins = numpy.array([0, 4, 8, 12, 16], float) lon_bins = numpy.array([19.2, 21, 22.8], float) lat_bins = numpy.array([43.2, 44.4, 45.6], float) eps_bins = numpy.array([-1.2, -0.4, 0.4, 1.2], float) bin_edges = mag_bins, dist_bins, lon_bins, lat_bins, eps_bins, trt_bins diss_matrix = disagg._arrange_data_in_bins(bins_data, bin_edges) self.assertEqual(diss_matrix.shape, (3, 4, 2, 2, 3, 2)) self.assertAlmostEqual(diss_matrix.sum(), 1) for idx, value in [ ((0, 0, 0, 0, 2, 0), 0.0263831), ((0, 1, 0, 0, 0, 0), 0.0791494), ((0, 1, 0, 0, 1, 0), 0.1055325), ((0, 1, 0, 0, 2, 0), 0.0791494), ((0, 1, 0, 1, 2, 1), 0.0079149), ((0, 2, 0, 1, 0, 0), 0.0559322), ((0, 2, 0, 1, 0, 1), 0.0348257), ((0, 2, 0, 1, 1, 0), 0.1398306), ((0, 2, 0, 1, 1, 1), 0.0522386), ((0, 2, 0, 1, 2, 0), 0.0559322), ((0, 2, 0, 1, 2, 1), 0.0348257), ((0, 3, 0, 1, 1, 1), 0.0435322), ((0, 3, 0, 1, 2, 1), 0.0008706), ((1, 1, 1, 0, 0, 1), 0.0079149), ((1, 1, 1, 0, 1, 1), 0.0395747), ((1, 1, 1, 0, 2, 1), 0.0105533), ((1, 3, 1, 1, 1, 1), 0.0105533), ((1, 3, 1, 1, 2, 1), 0.0042213), ((2, 0, 0, 0, 0, 0), 0.0527663), ((2, 0, 0, 0, 1, 0), 0.1055325), ((2, 0, 0, 0, 2, 0), 0.0527663), ]: self.assertAlmostEqual(diss_matrix[idx], value) diss_matrix[idx] = 0 self.assertEqual(diss_matrix.sum(), 0)
def compute_disagg(src_filter, sources, src_group_id, rlzs_assoc, trt_names, curves_dict, bin_edges, oqparam, monitor): # see https://bugs.launchpad.net/oq-engine/+bug/1279247 for an explanation # of the algorithm used """ :param src_filter: a :class:`openquake.hazardlib.calc.filter.SourceFilter` instance :param sources: list of hazardlib source objects :param src_group_id: numeric ID of a SourceGroup instance :param rlzs_assoc: a :class:`openquake.commonlib.source.RlzsAssoc` instance :param dict trt_names: a tuple of names for the given tectonic region type :param curves_dict: a dictionary with the hazard curves for sites, realizations and IMTs :param bin_egdes: a dictionary site_id -> edges :param oqparam: the parameters in the job.ini file :param monitor: monitor of the currently running job :returns: a dictionary of probability arrays, with composite key (sid, rlz.id, poe, imt, iml, trt_names). """ sitecol = src_filter.sitecol trt_num = dict((trt, i) for i, trt in enumerate(trt_names)) gsims = rlzs_assoc.gsims_by_grp_id[src_group_id] result = {} # sid, rlz.id, poe, imt, iml, trt_names -> array collecting_mon = monitor('collecting bins') arranging_mon = monitor('arranging bins') for site, sid in zip(sitecol, sitecol.sids): # edges as wanted by disagg._arrange_data_in_bins try: edges = bin_edges[sid] except KeyError: # bin_edges for a given site are missing if the site is far away continue # generate source, rupture, sites once per site with collecting_mon: bdata = disagg._collect_bins_data( trt_num, sources, site, curves_dict[sid], src_group_id, rlzs_assoc, gsims, oqparam.imtls, oqparam.poes_disagg, oqparam.truncation_level, oqparam.num_epsilon_bins, oqparam.iml_disagg, monitor) for (rlzi, poe, imt), iml_pne_pairs in bdata.pnes.items(): # extract the probabilities of non-exceedance for the # given realization, disaggregation PoE, and IMT iml = iml_pne_pairs[0][0] probs = numpy.array([p for (i, p) in iml_pne_pairs], float) # bins in a format handy for hazardlib bins = [ bdata.mags, bdata.dists, bdata.lons, bdata.lats, bdata.trts, None, probs ] # call disagg._arrange_data_in_bins with arranging_mon: key = (sid, rlzi, poe, imt, iml, trt_names) matrix = disagg._arrange_data_in_bins(bins, edges + (trt_names, )) result[key] = numpy.array( [fn(matrix) for fn in disagg.pmf_map.values()]) return result