Пример #1
0
def _collect_bins_data(trt_num, sources, site, curves, src_group_id,
                       rlzs_assoc, gsims, imtls, poes, truncation_level,
                       n_epsilons, iml_disagg, mon):
    # returns a BinData instance
    sitecol = SiteCollection([site])
    mags = []
    dists = []
    lons = []
    lats = []
    trts = []
    pnes = collections.defaultdict(list)
    sitemesh = sitecol.mesh
    make_ctxt = mon('making contexts', measuremem=False)
    disagg_poe = mon('disaggregate_poe', measuremem=False)
    cmaker = ContextMaker(gsims)
    for source in sources:
        try:
            tect_reg = trt_num[source.tectonic_region_type]
            for rupture in source.iter_ruptures():
                with make_ctxt:
                    try:
                        sctx, rctx, dctx = cmaker.make_contexts(
                            sitecol, rupture)
                    except filters.FarAwayRupture:
                        continue
                # extract rupture parameters of interest
                mags.append(rupture.mag)
                dists.append(dctx.rjb[0])  # single site => single distance
                [closest_point] = rupture.surface.get_closest_points(sitemesh)
                lons.append(closest_point.longitude)
                lats.append(closest_point.latitude)
                trts.append(tect_reg)
                # a dictionary rlz.id, poe, imt_str -> (iml, prob_no_exceed)
                for gsim in gsims:
                    gs = str(gsim)
                    for imt_str, imls in imtls.items():
                        imt = from_string(imt_str)
                        imls = numpy.array(imls[::-1])
                        for rlz in rlzs_assoc[src_group_id, gs]:
                            rlzi = rlz.ordinal
                            iml = iml_disagg.get(imt_str)
                            curve_poes = curves[rlzi, imt_str][::-1]
                            for k, v in _disagg(
                                    iml, poes, curve_poes, imls, gsim, rupture,
                                    rlzi, imt, imt_str, sctx, rctx, dctx,
                                    truncation_level, n_epsilons, disagg_poe):
                                pnes[k].append(v)
        except Exception as err:
            etype, err, tb = sys.exc_info()
            msg = 'An error occurred with source id=%s. Error: %s'
            msg %= (source.source_id, err)
            raise_(etype, msg, tb)

    return BinData(numpy.array(mags, float),
                   numpy.array(dists, float),
                   numpy.array(lons, float),
                   numpy.array(lats, float),
                   numpy.array(trts, int),
                   pnes)
Пример #2
0
def get_hazard_curve_source(input_set):
    """
    From a dictionary input set returns hazard curves
    """
    try:
        cmaker = ContextMaker(
            [input_set["gsims"][key] for key in input_set["gsims"]],
            None)
        for rupture, r_sites in input_set["ruptures_sites"]:
            gsim = input_set["gsims"][rupture.tectonic_region_type]
            sctx, rctx, dctx = cmaker.make_contexts(r_sites, rupture)
            for iimt in input_set["imts"]:
                poes = gsim.get_poes(sctx, rctx, dctx, imt.from_string(iimt),
                                     input_set["imts"][iimt],
                                     input_set["truncation_level"])
                pno = rupture.get_probability_no_exceedance(poes)
                input_set["curves"][iimt] *= r_sites.expand(pno, placeholder=1)
    except Exception, err:
        pass
Пример #3
0
 def __init__(self, trt, gsims):
     self.trt = trt
     self.cmaker = ContextMaker(gsims)
     self.params = sorted(self.cmaker.REQUIRES_RUPTURE_PARAMETERS -
                          set('mag strike dip rake hypo_depth'.split()))
     self.dt = numpy.dtype([
         ('rupserial', U32), ('multiplicity', U16),
         ('numsites', U32), ('occurrence_rate', F64),
         ('mag', F64), ('lon', F32), ('lat', F32), ('depth', F32),
         ('strike', F64), ('dip', F64), ('rake', F64),
         ('boundary', hdf5.vstr)] + [(param, F64) for param in self.params])
Пример #4
0
class RuptureData(object):
    """
    Container for information about the ruptures of a given
    tectonic region type.
    """
    def __init__(self, trt, gsims):
        self.trt = trt
        self.cmaker = ContextMaker(trt, gsims)
        self.params = sorted(self.cmaker.REQUIRES_RUPTURE_PARAMETERS -
                             set('mag strike dip rake hypo_depth'.split()))
        self.dt = numpy.dtype([
            ('rup_id', U32), ('srcidx', U32), ('multiplicity', U16),
            ('occurrence_rate', F64),
            ('mag', F32), ('lon', F32), ('lat', F32), ('depth', F32),
            ('strike', F32), ('dip', F32), ('rake', F32),
            ('boundary', hdf5.vstr)] + [(param, F32) for param in self.params])

    def to_array(self, ebruptures):
        """
        Convert a list of ebruptures into an array of dtype RuptureRata.dt
        """
        data = []
        for ebr in ebruptures:
            rup = ebr.rupture
            self.cmaker.add_rup_params(rup)
            ruptparams = tuple(getattr(rup, param) for param in self.params)
            point = rup.surface.get_middle_point()
            multi_lons, multi_lats = rup.surface.get_surface_boundaries()
            bounds = ','.join('((%s))' % ','.join(
                '%.5f %.5f' % (lon, lat) for lon, lat in zip(lons, lats))
                              for lons, lats in zip(multi_lons, multi_lats))
            try:
                rate = ebr.rupture.occurrence_rate
            except AttributeError:  # for nonparametric sources
                rate = numpy.nan
            data.append(
                (ebr.serial, ebr.srcidx, ebr.n_occ, rate,
                 rup.mag, point.x, point.y, point.z, rup.surface.get_strike(),
                 rup.surface.get_dip(), rup.rake,
                 'MULTIPOLYGON(%s)' % decode(bounds)) + ruptparams)
        return numpy.array(data, self.dt)
Пример #5
0
 def get_ctx(self, rupture, site_collection):
     return ContextMaker('*', [self.gsim_class],
                         dict(imtls={})).get_ctxs([rupture],
                                                  site_collection,
                                                  'src_id')[0]
Пример #6
0
def event_based(proxies, full_lt, oqparam, dstore, monitor):
    """
    Compute GMFs and optionally hazard curves
    """
    alldata = AccumDict(accum=[])
    sig_eps = []
    times = []  # rup_id, nsites, dt
    hcurves = {}  # key -> poes
    trt_smr = proxies[0]['trt_smr']
    fmon = monitor('filtering ruptures', measuremem=False)
    cmon = monitor('computing gmfs', measuremem=False)
    with dstore:
        trt = full_lt.trts[trt_smr // len(full_lt.sm_rlzs)]
        srcfilter = SourceFilter(dstore['sitecol'],
                                 oqparam.maximum_distance(trt))
        rupgeoms = dstore['rupgeoms']
        rlzs_by_gsim = full_lt._rlzs_by_gsim(trt_smr)
        param = vars(oqparam).copy()
        param['imtls'] = oqparam.imtls
        param['min_iml'] = oqparam.min_iml
        param['maximum_distance'] = oqparam.maximum_distance(trt)
        cmaker = ContextMaker(trt, rlzs_by_gsim, param)
        min_mag = getdefault(oqparam.minimum_magnitude, trt)
        for proxy in proxies:
            t0 = time.time()
            with fmon:
                if proxy['mag'] < min_mag:
                    continue
                sids = srcfilter.close_sids(proxy, trt)
                if len(sids) == 0:  # filtered away
                    continue
                proxy.geom = rupgeoms[proxy['geom_id']]
                ebr = proxy.to_ebr(cmaker.trt)  # after the geometry is set
                try:
                    computer = GmfComputer(ebr,
                                           srcfilter.sitecol.filtered(sids),
                                           cmaker, oqparam.correl_model,
                                           oqparam.cross_correl,
                                           oqparam._amplifier,
                                           oqparam._sec_perils)
                except FarAwayRupture:
                    continue
            with cmon:
                data = computer.compute_all(sig_eps)
            dt = time.time() - t0
            times.append((computer.ebrupture.id, len(computer.ctx.sids), dt))
            for key in data:
                alldata[key].extend(data[key])
    for key, val in sorted(alldata.items()):
        if key in 'eid sid rlz':
            alldata[key] = U32(alldata[key])
        else:
            alldata[key] = F32(alldata[key])
    gmfdata = strip_zeros(pandas.DataFrame(alldata))
    if len(gmfdata) and oqparam.hazard_curves_from_gmfs:
        hc_mon = monitor('building hazard curves', measuremem=False)
        for (sid, rlz), df in gmfdata.groupby(['sid', 'rlz']):
            with hc_mon:
                poes = calc.gmvs_to_poes(df, oqparam.imtls,
                                         oqparam.ses_per_logic_tree_path)
                for m, imt in enumerate(oqparam.imtls):
                    hcurves[rsi2str(rlz, sid, imt)] = poes[m]
    times = numpy.array([tup + (monitor.task_no, ) for tup in times], time_dt)
    times.sort(order='rup_id')
    if not oqparam.ground_motion_fields:
        gmfdata = ()
    return dict(gmfdata=gmfdata,
                hcurves=hcurves,
                times=times,
                sig_eps=numpy.array(sig_eps, sig_eps_dt(oqparam.imtls)))
Пример #7
0
    def build_events_from_sources(self):
        """
        Prefilter the composite source model and store the source_info
        """
        oq = self.oqparam
        params = dict(imtls=oq.imtls,
                      ses_per_logic_tree_path=oq.ses_per_logic_tree_path,
                      ses_seed=oq.ses_seed)
        gsims_by_trt = self.csm.full_lt.get_gsims_by_trt()
        sources = self.csm.get_sources()
        # weighting the heavy sources
        nrups = parallel.Starmap(count_ruptures,
                                 [(src, )
                                  for src in sources if src.code in b'AMC'],
                                 progress=logging.debug).reduce()
        for src in sources:
            try:
                src.num_ruptures = nrups[src.source_id]
            except KeyError:
                src.num_ruptures = src.count_ruptures()
            src.weight = src.num_ruptures
        maxweight = sum(sg.weight for sg in self.csm.src_groups) / (
            self.oqparam.concurrent_tasks or 1)
        eff_ruptures = AccumDict(accum=0)  # grp_id => potential ruptures
        calc_times = AccumDict(accum=numpy.zeros(3, F32))  # nr, ns, dt
        allargs = []
        if self.oqparam.is_ucerf():
            # manage the filtering in a special way
            for sg in self.csm.src_groups:
                for src in sg:
                    src.src_filter = self.srcfilter
            srcfilter = nofilter  # otherwise it would be ultra-slow
        else:
            srcfilter = self.srcfilter
        logging.info('Building ruptures')
        for sg in self.csm.src_groups:
            if not sg.sources:
                continue
            logging.info('Sending %s', sg)
            params['maximum_distance'] = oq.maximum_distance(sg.trt)
            cmaker = ContextMaker(sg.trt, gsims_by_trt[sg.trt], params)
            for src_group in sg.split(maxweight):
                allargs.append((src_group, cmaker, srcfilter.sitecol))
        smap = parallel.Starmap(sample_ruptures,
                                allargs,
                                h5=self.datastore.hdf5)
        mon = self.monitor('saving ruptures')
        self.nruptures = 0  # estimated classical ruptures within maxdist
        for dic in smap:
            # NB: dic should be a dictionary, but when the calculation dies
            # for an OOM it can become None, thus giving a very confusing error
            if dic is None:
                raise MemoryError('You ran out of memory!')
            rup_array = dic['rup_array']
            if len(rup_array) == 0:
                continue
            if dic['calc_times']:
                calc_times += dic['calc_times']
            if dic['eff_ruptures']:
                eff_ruptures += dic['eff_ruptures']
            with mon:
                n = len(rup_array)
                rup_array['id'] = numpy.arange(self.nruptures,
                                               self.nruptures + n)
                self.nruptures += n
                hdf5.extend(self.datastore['ruptures'], rup_array)
                hdf5.extend(self.datastore['rupgeoms'], rup_array.geom)
        if len(self.datastore['ruptures']) == 0:
            raise RuntimeError('No ruptures were generated, perhaps the '
                               'investigation time is too short')

        # don't change the order of the 3 things below!
        self.store_source_info(calc_times)
        self.store_rlz_info(eff_ruptures)
        imp = calc.RuptureImporter(self.datastore)
        with self.monitor('saving ruptures and events'):
            imp.import_rups_events(
                self.datastore.getitem('ruptures')[()], get_rupture_getters)
Пример #8
0
def compute_ruptures(sources, sitecol, siteidx, rlzs_assoc, monitor):
    """
    :param sources:
        List of commonlib.source.Source tuples
    :param sitecol:
        a :class:`openquake.hazardlib.site.SiteCollection` instance
    :param siteidx:
        always equal to 0
    :param rlzs_assoc:
        a :class:`openquake.commonlib.source.RlzsAssoc` instance
    :param monitor:
        monitor instance
    :returns:
        a dictionary trt_model_id -> [Rupture instances]
    """
    assert siteidx == 0, (
        'siteidx can be nonzero only for the classical_tiling calculations: '
        'tiling with the EventBasedRuptureCalculator is an error')
    # NB: by construction each block is a non-empty list with
    # sources of the same trt_model_id
    trt_model_id = sources[0].trt_model_id
    oq = monitor.oqparam
    trt = sources[0].tectonic_region_type
    max_dist = oq.maximum_distance[trt]
    cmaker = ContextMaker(rlzs_assoc.gsims_by_trt_id[trt_model_id])
    params = cmaker.REQUIRES_RUPTURE_PARAMETERS
    rup_data_dt = numpy.dtype([('rupserial', U32), ('multiplicity',
                                                    U16), ('numsites', U32)] +
                              [(param, F32) for param in params])
    eb_ruptures = []
    rup_data = []
    calc_times = []
    rup_mon = monitor('filtering ruptures', measuremem=False)
    num_samples = rlzs_assoc.samples[trt_model_id]

    # Compute and save stochastic event sets
    for src in sources:
        t0 = time.time()
        s_sites = src.filter_sites_by_distance_to_source(max_dist, sitecol)
        if s_sites is None:
            continue
        rupture_filter = functools.partial(filter_sites_by_distance_to_rupture,
                                           integration_distance=max_dist,
                                           sites=s_sites)
        num_occ_by_rup = sample_ruptures(src, oq.ses_per_logic_tree_path,
                                         num_samples, rlzs_assoc.seed)
        # NB: the number of occurrences is very low, << 1, so it is
        # more efficient to filter only the ruptures that occur, i.e.
        # to call sample_ruptures *before* the filtering
        for ebr in build_eb_ruptures(src, num_occ_by_rup, rupture_filter,
                                     oq.random_seed, rup_mon):
            nsites = len(ebr.indices)
            rc = cmaker.make_rupture_context(ebr.rupture)
            ruptparams = tuple(getattr(rc, param) for param in params)
            rup_data.append((ebr.serial, len(ebr.etags), nsites) + ruptparams)
            eb_ruptures.append(ebr)
        dt = time.time() - t0
        calc_times.append((src.id, dt))
    res = AccumDict({trt_model_id: eb_ruptures})
    res.calc_times = calc_times
    res.rup_data = numpy.array(rup_data, rup_data_dt)
    res.trt = trt
    return res
Пример #9
0
def disaggregation(sources,
                   site,
                   imt,
                   iml,
                   gsim_by_trt,
                   truncation_level,
                   n_epsilons,
                   mag_bin_width,
                   dist_bin_width,
                   coord_bin_width,
                   source_filter=filters.source_site_noop_filter,
                   filter_distance='rjb'):
    """
    Compute "Disaggregation" matrix representing conditional probability of an
    intensity mesaure type ``imt`` exceeding, at least once, an intensity
    measure level ``iml`` at a geographical location ``site``, given rupture
    scenarios classified in terms of:

    - rupture magnitude
    - Joyner-Boore distance from rupture surface to site
    - longitude and latitude of the surface projection of a rupture's point
      closest to ``site``
    - epsilon: number of standard deviations by which an intensity measure
      level deviates from the median value predicted by a GSIM, given the
      rupture parameters
    - rupture tectonic region type

    In other words, the disaggregation matrix allows to compute the probability
    of each scenario with the specified properties (e.g., magnitude, or the
    magnitude and distance) to cause one or more exceedences of a given hazard
    level.

    For more detailed information about the disaggregation, see for instance
    "Disaggregation of Seismic Hazard", Paolo Bazzurro, C. Allin Cornell,
    Bulletin of the Seismological Society of America, Vol. 89, pp. 501-520,
    April 1999.

    :param sources:
        Seismic source model, as for
        :mod:`PSHA <openquake.hazardlib.calc.hazard_curve>` calculator it
        should be an iterator of seismic sources.
    :param site:
        :class:`~openquake.hazardlib.site.Site` of interest to calculate
        disaggregation matrix for.
    :param imt:
        Instance of :mod:`intensity measure type <openquake.hazardlib.imt>`
        class.
    :param iml:
        Intensity measure level. A float value in units of ``imt``.
    :param gsim_by_trt:
        Tectonic region type to GSIM objects mapping.
    :param truncation_level:
        Float, number of standard deviations for truncation of the intensity
        distribution.
    :param n_epsilons:
        Integer number of epsilon histogram bins in the result matrix.
    :param mag_bin_width:
        Magnitude discretization step, width of one magnitude histogram bin.
    :param dist_bin_width:
        Distance histogram discretization step, in km.
    :param coord_bin_width:
        Longitude and latitude histograms discretization step,
        in decimal degrees.
    :param source_filter:
        Optional source-site filter function. See
        :mod:`openquake.hazardlib.calc.filters`.

    :returns:
        A tuple of two items. First is itself a tuple of bin edges information
        for (in specified order) magnitude, distance, longitude, latitude,
        epsilon and tectonic region types.

        Second item is 6d-array representing the full disaggregation matrix.
        Dimensions are in the same order as bin edges in the first item
        of the result tuple. The matrix can be used directly by pmf-extractor
        functions.
    """
    trts = sorted(set(src.tectonic_region_type for src in sources))
    trt_num = dict((trt, i) for i, trt in enumerate(trts))
    rlzs_by_gsim = {gsim_by_trt[trt]: [0] for trt in trts}
    cmaker = ContextMaker(rlzs_by_gsim, source_filter.integration_distance,
                          filter_distance)
    iml4 = make_iml4(1, {str(imt): iml})
    by_trt = groupby(sources, operator.attrgetter('tectonic_region_type'))
    bdata = {}
    sitecol = SiteCollection([site])
    for trt, srcs in by_trt.items():
        bdata[trt] = collect_bin_data(srcs, sitecol, cmaker, iml4,
                                      truncation_level, n_epsilons)
    if sum(len(bd.mags) for bd in bdata.values()) == 0:
        warnings.warn(
            'No ruptures have contributed to the hazard at site %s' % site,
            RuntimeWarning)
        return None, None

    min_mag = min(bd.mags.min() for bd in bdata.values())
    max_mag = max(bd.mags.max() for bd in bdata.values())
    mag_bins = mag_bin_width * numpy.arange(
        int(numpy.floor(min_mag / mag_bin_width)),
        int(numpy.ceil(max_mag / mag_bin_width) + 1))

    min_dist = min(bd.dists.min() for bd in bdata.values())
    max_dist = max(bd.dists.max() for bd in bdata.values())
    dist_bins = dist_bin_width * numpy.arange(
        int(numpy.floor(min_dist / dist_bin_width)),
        int(numpy.ceil(max_dist / dist_bin_width) + 1))

    bb = (min(bd.lons.min() for bd in bdata.values()),
          min(bd.lats.min() for bd in bdata.values()),
          max(bd.lons.max() for bd in bdata.values()),
          max(bd.lats.max() for bd in bdata.values()))
    lon_bins, lat_bins = lon_lat_bins(bb, coord_bin_width)

    eps_bins = numpy.linspace(-truncation_level, truncation_level,
                              n_epsilons + 1)

    bin_edges = (mag_bins, dist_bins, [lon_bins], [lat_bins], eps_bins)
    matrix = numpy.zeros(
        (len(mag_bins) - 1, len(dist_bins) - 1, len(lon_bins) - 1,
         len(lat_bins) - 1, len(eps_bins) - 1, len(trts)))
    for trt in bdata:
        dic = build_disagg_matrix(bdata[trt], bin_edges, sid=0)
        if dic:  # (poe, imt, rlzi) -> matrix
            [mat] = dic.values()
            matrix[..., trt_num[trt]] = mat
    return bin_edges + (trts, ), matrix
Пример #10
0
def classical(group, src_filter, gsims, param, monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns:
        a dictionary {grp_id: pmap} with attributes .grp_ids, .calc_times,
        .eff_ruptures
    """
    if getattr(group, 'src_interdep', None) == 'mutex':
        mutex_weight = {
            src.source_id: weight
            for src, weight in zip(group.sources, group.srcs_weights)
        }
    else:
        mutex_weight = None
    grp_ids = set()
    for src in group:
        if not src.num_ruptures:
            # src.num_ruptures is set when parsing the XML, but not when
            # the source is instantiated manually, so it is set here
            src.num_ruptures = src.count_ruptures()
        grp_ids.update(src.src_group_ids)
    maxdist = src_filter.integration_distance
    imtls = param['imtls']
    trunclevel = param.get('truncation_level')
    cmaker = ContextMaker(gsims, maxdist, param, monitor)
    pmap = AccumDict({
        grp_id: ProbabilityMap(len(imtls.array), len(gsims))
        for grp_id in grp_ids
    })
    # AccumDict of arrays with 4 elements weight, nsites, calc_time, split
    pmap.calc_times = AccumDict(accum=numpy.zeros(4))
    pmap.eff_ruptures = AccumDict()  # grp_id -> num_ruptures
    for src, s_sites in src_filter(group):  # filter now
        t0 = time.time()
        indep = group.rup_interdep == 'indep' if mutex_weight else True
        try:
            poemap = cmaker.poe_map(src, s_sites, imtls, trunclevel, indep)
        except Exception as err:
            etype, err, tb = sys.exc_info()
            msg = '%s (source id=%s)' % (str(err), src.source_id)
            raise etype(msg).with_traceback(tb)
        if mutex_weight:  # mutex sources
            weight = mutex_weight[src.source_id]
            for sid in poemap:
                pcurve = pmap[group.id].setdefault(sid, 0)
                pcurve += poemap[sid] * weight
        elif poemap:
            for grp_id in src.src_group_ids:
                pmap[grp_id] |= poemap
        src_id = src.source_id.split(':', 1)[0]
        pmap.calc_times[src_id] += numpy.array(
            [src.weight, len(s_sites),
             time.time() - t0, 1])
        # storing the number of contributing ruptures too
        pmap.eff_ruptures += {
            grp_id: getattr(poemap, 'eff_ruptures', 0)
            for grp_id in src.src_group_ids
        }
    if mutex_weight and group.grp_probability is not None:
        pmap[group.id] *= group.grp_probability
    return pmap
Пример #11
0
def disaggregation(sources,
                   site,
                   imt,
                   iml,
                   gsim_by_trt,
                   truncation_level,
                   n_epsilons,
                   mag_bin_width,
                   dist_bin_width,
                   coord_bin_width,
                   source_filter=filters.nofilter,
                   **kwargs):
    """
    Compute "Disaggregation" matrix representing conditional probability of an
    intensity mesaure type ``imt`` exceeding, at least once, an intensity
    measure level ``iml`` at a geographical location ``site``, given rupture
    scenarios classified in terms of:

    - rupture magnitude
    - Joyner-Boore distance from rupture surface to site
    - longitude and latitude of the surface projection of a rupture's point
      closest to ``site``
    - epsilon: number of standard deviations by which an intensity measure
      level deviates from the median value predicted by a GSIM, given the
      rupture parameters
    - rupture tectonic region type

    In other words, the disaggregation matrix allows to compute the probability
    of each scenario with the specified properties (e.g., magnitude, or the
    magnitude and distance) to cause one or more exceedences of a given hazard
    level.

    For more detailed information about the disaggregation, see for instance
    "Disaggregation of Seismic Hazard", Paolo Bazzurro, C. Allin Cornell,
    Bulletin of the Seismological Society of America, Vol. 89, pp. 501-520,
    April 1999.

    :param sources:
        Seismic source model, as for
        :mod:`PSHA <openquake.hazardlib.calc.hazard_curve>` calculator it
        should be an iterator of seismic sources.
    :param site:
        :class:`~openquake.hazardlib.site.Site` of interest to calculate
        disaggregation matrix for.
    :param imt:
        Instance of :mod:`intensity measure type <openquake.hazardlib.imt>`
        class.
    :param iml:
        Intensity measure level. A float value in units of ``imt``.
    :param gsim_by_trt:
        Tectonic region type to GSIM objects mapping.
    :param truncation_level:
        Float, number of standard deviations for truncation of the intensity
        distribution.
    :param n_epsilons:
        Integer number of epsilon histogram bins in the result matrix.
    :param mag_bin_width:
        Magnitude discretization step, width of one magnitude histogram bin.
    :param dist_bin_width:
        Distance histogram discretization step, in km.
    :param coord_bin_width:
        Longitude and latitude histograms discretization step,
        in decimal degrees.
    :param source_filter:
        Optional source-site filter function. See
        :mod:`openquake.hazardlib.calc.filters`.

    :returns:
        A tuple of two items. First is itself a tuple of bin edges information
        for (in specified order) magnitude, distance, longitude, latitude,
        epsilon and tectonic region types.

        Second item is 6d-array representing the full disaggregation matrix.
        Dimensions are in the same order as bin edges in the first item
        of the result tuple. The matrix can be used directly by pmf-extractor
        functions.
    """
    trts = sorted(set(src.tectonic_region_type for src in sources))
    trt_num = dict((trt, i) for i, trt in enumerate(trts))
    rlzs_by_gsim = {gsim_by_trt[trt]: [0] for trt in trts}
    by_trt = groupby(sources, operator.attrgetter('tectonic_region_type'))
    bdata = {}  # by trt, magi
    sitecol = SiteCollection([site])
    iml2 = numpy.array([[iml]])
    eps3 = _eps3(truncation_level, n_epsilons)

    rups = AccumDict(accum=[])
    cmaker = {}  # trt -> cmaker
    for trt, srcs in by_trt.items():
        contexts.RuptureContext.temporal_occurrence_model = (
            srcs[0].temporal_occurrence_model)
        cmaker[trt] = ContextMaker(
            trt, rlzs_by_gsim, {
                'truncation_level': truncation_level,
                'maximum_distance': source_filter.integration_distance,
                'imtls': {
                    str(imt): [iml]
                }
            })
        rups[trt].extend(cmaker[trt].from_srcs(srcs, sitecol))
    min_mag = min(r.mag for rs in rups.values() for r in rs)
    max_mag = max(r.mag for rs in rups.values() for r in rs)
    mag_bins = mag_bin_width * numpy.arange(
        int(numpy.floor(min_mag / mag_bin_width)),
        int(numpy.ceil(max_mag / mag_bin_width) + 1))

    for trt in cmaker:
        gsim = gsim_by_trt[trt]
        for magi, ctxs in enumerate(_magbin_groups(rups[trt], mag_bins)):
            set_mean_std(ctxs, [imt], [gsim])
            bdata[trt, magi] = disaggregate(ctxs, [0], {imt: iml2}, eps3)

    if sum(len(bd.dists) for bd in bdata.values()) == 0:
        warnings.warn(
            'No ruptures have contributed to the hazard at site %s' % site,
            RuntimeWarning)
        return None, None

    min_dist = min(bd.dists.min() for bd in bdata.values())
    max_dist = max(bd.dists.max() for bd in bdata.values())
    dist_bins = dist_bin_width * numpy.arange(
        int(numpy.floor(min_dist / dist_bin_width)),
        int(numpy.ceil(max_dist / dist_bin_width) + 1))
    lon_bins, lat_bins = lon_lat_bins(site.location.x, site.location.y,
                                      max_dist, coord_bin_width)
    eps_bins = numpy.linspace(-truncation_level, truncation_level,
                              n_epsilons + 1)
    bin_edges = (mag_bins, dist_bins, lon_bins, lat_bins, eps_bins)
    matrix = numpy.zeros(
        (len(mag_bins) - 1, len(dist_bins) - 1, len(lon_bins) - 1,
         len(lat_bins) - 1, len(eps_bins) - 1, len(trts)))  # 6D
    for trt, magi in bdata:
        mat7 = _build_disagg_matrix(bdata[trt, magi], bin_edges[1:])
        matrix[magi, ..., trt_num[trt]] = mat7[..., 0, 0, 0]
    return bin_edges + (trts, ), matrix
Пример #12
0
 def make_contexts(self, site_collection, rupture):
     param = dict(imtls={})
     return ContextMaker('faketrt', [self.gsim_class],
                         param).make_contexts(site_collection, rupture)
Пример #13
0
def classical(group, src_filter, gsims, param, monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns:
        a dictionary {grp_id: pmap} with attributes .grp_ids, .calc_times,
        .eff_ruptures
    """
    if not hasattr(src_filter, 'sitecol'):  # a sitecol was passed
        src_filter = SourceFilter(src_filter, {})

    # Get the parameters assigned to the group
    src_mutex = getattr(group, 'src_interdep', None) == 'mutex'
    rup_mutex = getattr(group, 'rup_interdep', None) == 'mutex'
    cluster = getattr(group, 'cluster', None)
    # Compute the number of ruptures
    grp_ids = set()
    for src in group:
        if not src.num_ruptures:
            # src.num_ruptures is set when parsing the XML, but not when
            # the source is instantiated manually, so it is set here
            src.num_ruptures = src.count_ruptures()
        # This sets the proper TOM in case of a cluster
        if cluster:
            src.temporal_occurrence_model = FatedTOM(time_span=1)
        # Updating IDs
        grp_ids.update(src.src_group_ids)
    # Now preparing context
    maxdist = src_filter.integration_distance
    imtls = param['imtls']
    trunclevel = param.get('truncation_level')
    cmaker = ContextMaker(
        src.tectonic_region_type, gsims, maxdist, param, monitor)
    # Prepare the accumulator for the probability maps
    pmap = AccumDict({grp_id: ProbabilityMap(len(imtls.array), len(gsims))
                      for grp_id in grp_ids})
    rupdata = {grp_id: [] for grp_id in grp_ids}
    # AccumDict of arrays with 2 elements weight, calc_time
    calc_times = AccumDict(accum=numpy.zeros(2, numpy.float32))
    eff_ruptures = AccumDict(accum=0)  # grp_id -> num_ruptures
    nsites = {}  # src.id -> num_sites
    # Computing hazard
    for src, s_sites in src_filter(group):  # filter now
        nsites[src.id] = src.nsites
        t0 = time.time()
        try:
            poemap = cmaker.poe_map(src, s_sites, imtls, trunclevel,
                                    rup_indep=not rup_mutex)
        except Exception as err:
            etype, err, tb = sys.exc_info()
            msg = '%s (source id=%s)' % (str(err), src.source_id)
            raise etype(msg).with_traceback(tb)
        if src_mutex:  # mutex sources, there is a single group
            for sid in poemap:
                pcurve = pmap[src.src_group_id].setdefault(sid, 0)
                pcurve += poemap[sid] * src.mutex_weight
        elif poemap:
            for gid in src.src_group_ids:
                pmap[gid] |= poemap
        if len(cmaker.rupdata):
            for gid in src.src_group_ids:
                rupdata[gid].append(cmaker.rupdata)
        calc_times[src.id] += numpy.array([src.weight, time.time() - t0])
        # storing the number of contributing ruptures too
        eff_ruptures += {gid: getattr(poemap, 'eff_ruptures', 0)
                         for gid in src.src_group_ids}
    # Updating the probability map in the case of mutually exclusive
    # sources
    group_probability = getattr(group, 'grp_probability', None)
    if src_mutex and group_probability:
        pmap[src.src_group_id] *= group_probability
    # Processing cluster
    if cluster:
        tom = getattr(group, 'temporal_occurrence_model')
        pmap = _cluster(param, tom, imtls, gsims, grp_ids, pmap)
    # Return results
    for gid, data in rupdata.items():
        if len(data):
            rupdata[gid] = numpy.concatenate(data)
    return dict(pmap=pmap, calc_times=calc_times, eff_ruptures=eff_ruptures,
                rup_data=rupdata, nsites=nsites)
Пример #14
0
    def full_disaggregation(self):
        """
        Run the disaggregation phase after hazard curve finalization.
        """
        oq = self.oqparam
        tl = self.oqparam.truncation_level
        sitecol = self.sitecol
        eps_edges = numpy.linspace(-tl, tl, self.oqparam.num_epsilon_bins + 1)

        self.bin_edges = {}
        curves = [self.get_curves(sid) for sid in sitecol.sids]
        # determine the number of effective source groups
        sg_data = self.datastore['csm_info/sg_data']
        num_grps = sum(1 for effrup in sg_data['effrup'] if effrup > 0)
        nblocks = math.ceil(oq.concurrent_tasks / num_grps)
        src_filter = SourceFilter(sitecol, oq.maximum_distance)
        R = len(self.rlzs_assoc.realizations)
        max_poe = numpy.zeros(R, oq.imt_dt())

        # build trt_edges
        trts = tuple(
            sorted(
                set(sg.trt for smodel in self.csm.source_models
                    for sg in smodel.src_groups)))

        # build mag_edges
        min_mag = min(sg.min_mag for smodel in self.csm.source_models
                      for sg in smodel.src_groups)
        max_mag = max(sg.max_mag for smodel in self.csm.source_models
                      for sg in smodel.src_groups)
        mag_edges = oq.mag_bin_width * numpy.arange(
            int(numpy.floor(min_mag / oq.mag_bin_width)),
            int(numpy.ceil(max_mag / oq.mag_bin_width) + 1))

        # build dist_edges
        maxdist = max(oq.maximum_distance(trt, max_mag) for trt in trts)
        dist_edges = oq.distance_bin_width * numpy.arange(
            0, int(numpy.ceil(maxdist / oq.distance_bin_width) + 1))
        logging.info('dist = %s...%s', min(dist_edges), max(dist_edges))

        # build eps_edges
        eps_edges = numpy.linspace(-tl, tl, oq.num_epsilon_bins + 1)

        # build lon_edges, lat_edges per sid
        bbs = src_filter.get_bounding_boxes(mag=max_mag)
        for sid, bb in zip(self.sitecol.sids, bbs):
            lon_edges, lat_edges = disagg.lon_lat_bins(bb,
                                                       oq.coordinate_bin_width)
            logging.info('site %d, lon = %s...%s', sid, min(lon_edges),
                         max(lon_edges))
            logging.info('site %d, lat = %s...%s', sid, min(lat_edges),
                         max(lat_edges))
            self.bin_edges[sid] = bs = (mag_edges, dist_edges, lon_edges,
                                        lat_edges, eps_edges)
            shape = [len(edges) - 1 for edges in bs] + [len(trts)]
            logging.info('%s for sid %d', shape, sid)

        # check poes
        for smodel in self.csm.source_models:
            sm_id = smodel.ordinal
            for i, site in enumerate(sitecol):
                sid = sitecol.sids[i]
                curve = curves[i]
                # populate max_poe array
                for rlzi, poes in curve.items():
                    for imt in oq.imtls:
                        max_poe[rlzi][imt] = max(max_poe[rlzi][imt],
                                                 poes[imt].max())
                if not curve:
                    continue  # skip zero-valued hazard curves

            # check for too big poes_disagg
            for poe in oq.poes_disagg:
                for rlz in self.rlzs_assoc.rlzs_by_smodel[sm_id]:
                    rlzi = rlz.ordinal
                    for imt in oq.imtls:
                        min_poe = max_poe[rlzi][imt]
                        if poe > min_poe:
                            raise ValueError(
                                self.POE_TOO_BIG %
                                (poe, sm_id, smodel.name, min_poe, rlzi, imt))

        # build all_args
        all_args = []
        for smodel in self.csm.source_models:
            for sg in smodel.src_groups:
                split_sources = []
                for src in sg:
                    for split, _sites in src_filter(
                            sourceconverter.split_source(src), sitecol):
                        split_sources.append(split)
                if not split_sources:
                    continue
                mon = self.monitor('disaggregation')
                rlzs_by_gsim = self.rlzs_assoc.get_rlzs_by_gsim(
                    sg.trt, smodel.ordinal)
                cmaker = ContextMaker(rlzs_by_gsim,
                                      src_filter.integration_distance)
                imls = [
                    disagg.make_imldict(rlzs_by_gsim, oq.imtls, oq.iml_disagg,
                                        oq.poes_disagg, curve)
                    for curve in curves
                ]
                for srcs in split_in_blocks(split_sources, nblocks):
                    all_args.append((src_filter, srcs, cmaker, imls, trts,
                                     self.bin_edges, oq, mon))

        self.cache_info = numpy.zeros(2)  # operations, cache_hits
        results = parallel.Starmap(compute_disagg,
                                   all_args).reduce(self.agg_result)
        ops, hits = self.cache_info
        logging.info('Cache speedup %s', ops / (ops - hits))
        self.save_disagg_results(results)
Пример #15
0
def compute_ruptures(sources, sitecol, gsims, monitor):
    """
    :param sources:
        List of commonlib.source.Source tuples
    :param sitecol:
        a :class:`openquake.hazardlib.site.SiteCollection` instance
    :param gsims:
        a list of GSIMs for the current tectonic region model
    :param monitor:
        monitor instance
    :returns:
        a dictionary src_group_id -> [Rupture instances]
    """
    # NB: by construction each block is a non-empty list with
    # sources of the same src_group_id
    src_group_id = sources[0].src_group_id
    trt = sources[0].tectonic_region_type
    max_dist = monitor.maximum_distance[trt]
    cmaker = ContextMaker(gsims)
    params = sorted(cmaker.REQUIRES_RUPTURE_PARAMETERS)
    rup_data_dt = numpy.dtype(
        [('rupserial', U32), ('multiplicity', U16),
         ('numsites', U32), ('occurrence_rate', F64)] + [
            (param, F64) for param in params])
    eb_ruptures = []
    rup_data = []
    calc_times = []
    rup_mon = monitor('filtering ruptures', measuremem=False)
    num_samples = monitor.samples
    num_events = 0

    # Compute and save stochastic event sets
    for src in sources:
        t0 = time.time()
        s_sites = src.filter_sites_by_distance_to_source(max_dist, sitecol)
        if s_sites is None:
            continue
        rupture_filter = functools.partial(
            filter_sites_by_distance_to_rupture,
            integration_distance=max_dist, sites=s_sites)
        num_occ_by_rup = sample_ruptures(
            src, monitor.ses_per_logic_tree_path, num_samples,
            monitor.seed)
        # NB: the number of occurrences is very low, << 1, so it is
        # more efficient to filter only the ruptures that occur, i.e.
        # to call sample_ruptures *before* the filtering
        for ebr in build_eb_ruptures(
                src, num_occ_by_rup, rupture_filter, monitor.seed, rup_mon):
            nsites = len(ebr.indices)
            try:
                rate = ebr.rupture.occurrence_rate
            except AttributeError:  # for nonparametric sources
                rate = numpy.nan
            rc = cmaker.make_rupture_context(ebr.rupture)
            ruptparams = tuple(getattr(rc, param) for param in params)
            rup_data.append((ebr.serial, ebr.multiplicity, nsites, rate) +
                            ruptparams)
            eb_ruptures.append(ebr)
            num_events += ebr.multiplicity
        dt = time.time() - t0
        calc_times.append((src.id, dt))
    res = AccumDict({src_group_id: eb_ruptures})
    res.num_events = num_events
    res.calc_times = calc_times
    res.rup_data = numpy.array(rup_data, rup_data_dt)
    res.trt = trt
    return res
Пример #16
0
    def full_disaggregation(self):
        """
        Run the disaggregation phase.
        """
        oq = self.oqparam
        tl = oq.truncation_level
        src_filter = self.src_filter()
        if hasattr(self, 'csm'):
            for sg in self.csm.src_groups:
                if sg.atomic:
                    raise NotImplementedError(
                        'Atomic groups are not supported yet')

        self.full_lt = self.datastore['full_lt']
        self.poes_disagg = oq.poes_disagg or (None,)
        self.imts = list(oq.imtls)

        self.ws = [rlz.weight for rlz in self.full_lt.get_realizations()]
        self.pgetter = getters.PmapGetter(
            self.datastore, self.ws, self.sitecol.sids)

        # build array rlzs (N, Z)
        if oq.rlz_index is None:
            Z = oq.num_rlzs_disagg
            rlzs = numpy.zeros((self.N, Z), int)
            if self.R > 1:
                for sid in self.sitecol.sids:
                    curves = numpy.array(
                        [pc.array for pc in self.pgetter.get_pcurves(sid)])
                    mean = getters.build_stat_curve(
                        curves, oq.imtls, stats.mean_curve, self.ws)
                    rlzs[sid] = util.closest_to_ref(curves, mean.array)[:Z]
                self.datastore['best_rlzs'] = rlzs
        else:
            Z = len(oq.rlz_index)
            rlzs = numpy.zeros((self.N, Z), int)
            for z in range(Z):
                rlzs[:, z] = oq.rlz_index[z]
        assert Z <= self.R, (Z, self.R)
        self.Z = Z
        self.rlzs = rlzs

        if oq.iml_disagg:
            # no hazard curves are needed
            self.poe_id = {None: 0}
            curves = [[None for z in range(Z)] for s in range(self.N)]
            self.ok_sites = set(self.sitecol.sids)
        else:
            self.poe_id = {poe: i for i, poe in enumerate(oq.poes_disagg)}
            curves = [self.get_curve(sid, rlzs[sid])
                      for sid in self.sitecol.sids]
            self.ok_sites = set(self.check_poes_disagg(curves, rlzs))
        self.iml4 = _iml4(rlzs, oq.iml_disagg, oq.imtls,
                          self.poes_disagg, curves)
        if oq.disagg_by_src:
            self.build_disagg_by_src(rlzs)

        eps_edges = numpy.linspace(-tl, tl, oq.num_epsilon_bins + 1)

        # build trt_edges
        trts = tuple(self.full_lt.trts)
        trt_num = {trt: i for i, trt in enumerate(trts)}
        self.trts = trts

        # build mag_edges
        mags = [float(mag) for mag in self.datastore['source_mags']]
        mag_edges = oq.mag_bin_width * numpy.arange(
            int(numpy.floor(min(mags) / oq.mag_bin_width)),
            int(numpy.ceil(max(mags) / oq.mag_bin_width) + 1))

        # build dist_edges
        maxdist = max(oq.maximum_distance(trt) for trt in trts)
        dist_edges = oq.distance_bin_width * numpy.arange(
            0, int(numpy.ceil(maxdist / oq.distance_bin_width) + 1))

        # build eps_edges
        eps_edges = numpy.linspace(-tl, tl, oq.num_epsilon_bins + 1)

        # build lon_edges, lat_edges per sid
        bbs = src_filter.get_bounding_boxes(mag=max(mags))
        lon_edges, lat_edges = {}, {}  # by sid
        for sid, bb in zip(self.sitecol.sids, bbs):
            lon_edges[sid], lat_edges[sid] = disagg.lon_lat_bins(
                bb, oq.coordinate_bin_width)
        self.bin_edges = mag_edges, dist_edges, lon_edges, lat_edges, eps_edges
        shapedic = self.save_bin_edges()
        del shapedic['trt']
        shapedic['N'] = self.N
        shapedic['M'] = len(oq.imtls)
        shapedic['P'] = len(oq.poes_disagg)
        shapedic['Z'] = Z
        shapedic['concurrent_tasks'] = oq.concurrent_tasks
        nbytes, msg = get_array_nbytes(shapedic)
        if nbytes > oq.max_data_transfer:
            raise ValueError('Estimated data transfer too big\n%s' % msg)
        logging.info('Estimated data transfer: %s', msg)
        self.imldict = {}  # sid, rlz, poe, imt -> iml
        for s in self.sitecol.sids:
            for z, rlz in enumerate(rlzs[s]):
                for p, poe in enumerate(self.poes_disagg):
                    for m, imt in enumerate(oq.imtls):
                        self.imldict[s, rlz, poe, imt] = self.iml4[s, m, p, z]

        # submit #groups disaggregation tasks
        dstore = (self.datastore.parent if self.datastore.parent
                  else self.datastore)
        indices = get_indices(dstore, oq.concurrent_tasks or 1)
        self.datastore.swmr_on()
        smap = parallel.Starmap(compute_disagg, h5=self.datastore.hdf5)
        for grp_id, trt in self.full_lt.trt_by_grp.items():
            logging.info('Group #%d, sending rup_data for %s', grp_id, trt)
            trti = trt_num[trt]
            cmaker = ContextMaker(
                trt, self.full_lt.get_rlzs_by_gsim(grp_id),
                {'truncation_level': oq.truncation_level,
                 'maximum_distance': src_filter.integration_distance,
                 'filter_distance': oq.filter_distance, 'imtls': oq.imtls})
            for idxs in indices[grp_id]:
                smap.submit((dstore, idxs, cmaker, self.iml4, trti,
                             self.bin_edges))
        results = smap.reduce(self.agg_result, AccumDict(accum={}))
        return results  # sid -> trti-> 8D array
Пример #17
0
    def full_disaggregation(self, curves):
        """
        Run the disaggregation phase.

        :param curves: a list of hazard curves, one per site

        The curves can be all None if iml_disagg is set in the job.ini
        """
        oq = self.oqparam
        tl = oq.truncation_level
        src_filter = SourceFilter(self.sitecol,
                                  oq.maximum_distance,
                                  use_rtree=False)
        csm = self.csm.filter(src_filter)  # fine filtering
        self.datastore['csm_info'] = csm.info
        eps_edges = numpy.linspace(-tl, tl, oq.num_epsilon_bins + 1)
        self.bin_edges = {}

        # build trt_edges
        trts = tuple(
            sorted(
                set(sg.trt for smodel in csm.source_models
                    for sg in smodel.src_groups)))
        trt_num = {trt: i for i, trt in enumerate(trts)}
        self.trts = trts

        # build mag_edges
        min_mag = min(sg.min_mag for smodel in csm.source_models
                      for sg in smodel.src_groups)
        max_mag = max(sg.max_mag for smodel in csm.source_models
                      for sg in smodel.src_groups)
        mag_edges = oq.mag_bin_width * numpy.arange(
            int(numpy.floor(min_mag / oq.mag_bin_width)),
            int(numpy.ceil(max_mag / oq.mag_bin_width) + 1))

        # build dist_edges
        maxdist = max(oq.maximum_distance(trt, max_mag) for trt in trts)
        dist_edges = oq.distance_bin_width * numpy.arange(
            0, int(numpy.ceil(maxdist / oq.distance_bin_width) + 1))

        # build eps_edges
        eps_edges = numpy.linspace(-tl, tl, oq.num_epsilon_bins + 1)

        # build lon_edges, lat_edges per sid
        bbs = src_filter.get_bounding_boxes(mag=max_mag)
        lon_edges, lat_edges = {}, {}  # by sid
        for sid, bb in zip(self.sitecol.sids, bbs):
            lon_edges[sid], lat_edges[sid] = disagg.lon_lat_bins(
                bb, oq.coordinate_bin_width)
        self.bin_edges = mag_edges, dist_edges, lon_edges, lat_edges, eps_edges
        self.save_bin_edges()

        # build all_args
        all_args = []
        maxweight = csm.get_maxweight(oq.concurrent_tasks)
        mon = self.monitor('disaggregation')
        R = len(self.rlzs_assoc.realizations)
        iml4 = disagg.make_iml4(R, oq.imtls, oq.iml_disagg, oq.poes_disagg
                                or (None, ), curves)
        self.imldict = {}  # sid, rlzi, poe, imt -> iml
        for s in self.sitecol.sids:
            for r in range(R):
                for p, poe in enumerate(oq.poes_disagg or [None]):
                    for m, imt in enumerate(oq.imtls):
                        self.imldict[s, r, poe, imt] = iml4[s, r, m, p]

        for smodel in csm.source_models:
            sm_id = smodel.ordinal
            for trt, groups in groupby(smodel.src_groups,
                                       operator.attrgetter('trt')).items():
                trti = trt_num[trt]
                sources = sum([grp.sources for grp in groups], [])
                rlzs_by_gsim = self.rlzs_assoc.get_rlzs_by_gsim(trt, sm_id)
                cmaker = ContextMaker(rlzs_by_gsim,
                                      src_filter.integration_distance)
                for block in csm.split_in_blocks(maxweight, sources):
                    all_args.append((src_filter, block, cmaker, iml4, trti,
                                     self.bin_edges, oq, mon))

        self.num_ruptures = [0] * len(self.trts)
        self.cache_info = numpy.zeros(3)  # operations, cache_hits, num_zeros
        results = parallel.Starmap(compute_disagg,
                                   all_args).reduce(self.agg_result,
                                                    AccumDict(accum={}))
        ops, hits, num_zeros = self.cache_info
        logging.info('Cache speedup %s', ops / (ops - hits))
        logging.info('Discarded zero matrices: %d', num_zeros)
        return results
Пример #18
0
def _collect_bins_data(sources, site, imt, iml, gsims,
                       truncation_level, n_epsilons,
                       source_site_filter, rupture_site_filter):
    """
    Extract values of magnitude, distance, closest point, tectonic region
    types and PoE distribution.

    This method processes the source model (generates ruptures) and collects
    all needed parameters to arrays. It also defines tectonic region type
    bins sequence.
    """
    mags = []
    dists = []
    lons = []
    lats = []
    tect_reg_types = []
    probs_no_exceed = []
    sitecol = SiteCollection([site])
    sitemesh = sitecol.mesh

    _next_trt_num = 0
    trt_nums = {}
    # here we ignore filtered site collection because either it is the same
    # as the original one (with one site), or the source/rupture is filtered
    # out and doesn't show up in the filter's output
    for src_idx, (source, s_sites) in \
            enumerate(source_site_filter(sources, sitecol)):
        try:
            tect_reg = source.tectonic_region_type
            gsim = gsims[tect_reg]
            cmaker = ContextMaker([gsim])
            if tect_reg not in trt_nums:
                trt_nums[tect_reg] = _next_trt_num
                _next_trt_num += 1
            tect_reg = trt_nums[tect_reg]

            for rupture, r_sites in rupture_site_filter(
                    source.iter_ruptures(), s_sites):
                # extract rupture parameters of interest
                mags.append(rupture.mag)
                [jb_dist] = rupture.surface.get_joyner_boore_distance(sitemesh)
                dists.append(jb_dist)
                [closest_point] = rupture.surface.get_closest_points(sitemesh)
                lons.append(closest_point.longitude)
                lats.append(closest_point.latitude)
                tect_reg_types.append(tect_reg)

                # compute conditional probability of exceeding iml given
                # the current rupture, and different epsilon level, that is
                # ``P(IMT >= iml | rup, epsilon_bin)`` for each of epsilon bins
                sctx, rctx, dctx = cmaker.make_contexts(sitecol, rupture)
                [poes_given_rup_eps] = gsim.disaggregate_poe(
                    sctx, rctx, dctx, imt, iml, truncation_level, n_epsilons
                )

                # collect probability of a rupture causing no exceedances
                probs_no_exceed.append(
                    rupture.get_probability_no_exceedance(poes_given_rup_eps)
                )
        except Exception as err:
            etype, err, tb = sys.exc_info()
            msg = 'An error occurred with source id=%s. Error: %s'
            msg %= (source.source_id, str(err))
            raise_(etype, msg, tb)

    mags = numpy.array(mags, float)
    dists = numpy.array(dists, float)
    lons = numpy.array(lons, float)
    lats = numpy.array(lats, float)
    tect_reg_types = numpy.array(tect_reg_types, int)
    probs_no_exceed = numpy.array(probs_no_exceed, float)

    trt_bins = [
        trt for (num, trt) in sorted((num, trt)
                                     for (trt, num) in trt_nums.items())
    ]

    return (mags, dists, lons, lats, tect_reg_types, trt_bins, probs_no_exceed)
def get_conditional_gmfs(database,
                         rupture,
                         sites,
                         gsims,
                         imts,
                         number_simulations,
                         truncation_level,
                         correlation_model=DEFAULT_CORRELATION):
    """
    Get a set of random fields conditioned on a set of observations
    :param database:
        Ground motion records for the event as instance of :class:
        smtk.sm_database.GroundMotionDatabase
    :param rupture:
        Event rupture as instance of :class:
        openquake.hazardlib.source.rupture.Rupture
    :param sites:
        Target sites as instance of :class:
        openquake.hazardlib.site.SiteCollection
    :param list gsims:
        List of GMPEs required
    :param list imts:
        List of intensity measures required
    :param int number_simulations:
        Number of simulated fields required
    :param float truncation_level:
        Ground motion truncation level
    """

    # Get known sites mesh
    known_sites = database.get_site_collection()

    # Get Observed Residuals
    residuals = Residuals(gsims, imts)
    residuals.get_residuals(database)
    imt_dict = OrderedDict([(imtx,
                             np.zeros([len(sites.lons), number_simulations]))
                            for imtx in imts])
    gmfs = OrderedDict([(gmpe, imt_dict) for gmpe in gsims])
    gmpe_list = [GSIM_LIST[gmpe]() for gmpe in gsims]
    cmaker = ContextMaker(rupture.tectonic_region_type, gmpe_list)
    ctxs = cmaker.make_contexts(sites, rupture)
    if len(ctxs) == 3:  # engine version >= 3.10
        _rctx, sctx, dctx = ctxs
    else:  # old version, 2 contexts
        sctx, dctx = ctxs
    for gsim in gmpe_list:
        gmpe = gsim.__class__.__name__
        for imtx in imts:
            if truncation_level == 0:
                gmfs[gmpe][imtx], _ = gsim.get_mean_and_stddevs(
                    sctx, rupture, dctx, from_string(imtx), stddev_types=[])
                continue
            if "Intra event" in gsim.DEFINED_FOR_STANDARD_DEVIATION_TYPES:
                epsilon = conditional_simulation(
                    known_sites,
                    residuals.residuals[gmpe][imtx]["Intra event"], sites,
                    imtx, number_simulations, correlation_model)
                tau = np.unique(residuals.residuals[gmpe][imtx]["Inter event"])
                mean, [stddev_inter, stddev_intra] = gsim.get_mean_and_stddevs(
                    sctx, rupture, dctx, from_string(imtx),
                    ["Inter event", "Intra event"])
                for iloc in range(0, number_simulations):
                    gmfs[gmpe][imtx][:, iloc] = np.exp(mean +
                                                       (tau * stddev_inter) +
                                                       (epsilon[:, iloc] *
                                                        stddev_intra))
            else:
                epsilon = conditional_simulation(
                    known_sites, residuals.residuals[gmpe][imtx]["Total"],
                    sites, imtx, number_simulations, correlation_model)
                tau = None
                mean, [stddev_total
                       ] = gsim.get_mean_and_stddevs(sctx, rupture, dctx,
                                                     from_string(imtx),
                                                     ["Total"])
                for iloc in range(0, number_simulations):
                    gmfs[gmpe][imtx][:,
                                     iloc] = np.exp(mean + epsilon[:, iloc] *
                                                    stddev_total.flatten())
    return gmfs
Пример #20
0
    def full_disaggregation(self):
        """
        Run the disaggregation phase.
        """
        oq = self.oqparam
        mags_by_trt = self.datastore['source_mags']
        all_edges, self.shapedic = disagg.get_edges_shapedic(
            oq, self.sitecol, mags_by_trt)
        *self.bin_edges, self.trts = all_edges
        if hasattr(self, 'csm'):
            for sg in self.csm.src_groups:
                if sg.atomic:
                    raise NotImplementedError(
                        'Atomic groups are not supported yet')
        elif self.datastore['source_info'].attrs['atomic']:
            raise NotImplementedError(
                'Atomic groups are not supported yet')

        self.full_lt = self.datastore['full_lt']
        self.poes_disagg = oq.poes_disagg or (None,)
        self.imts = list(oq.imtls)
        self.M = len(self.imts)
        ws = [rlz.weight for rlz in self.full_lt.get_realizations()]
        self.pgetter = getters.PmapGetter(
            self.datastore, ws, self.sitecol.sids)

        # build array rlzs (N, Z)
        if oq.rlz_index is None:
            Z = oq.num_rlzs_disagg or 1
            rlzs = numpy.zeros((self.N, Z), int)
            if self.R > 1:
                for sid in self.sitecol.sids:
                    curves = numpy.array(
                        [pc.array for pc in self.pgetter.get_pcurves(sid)])
                    mean = getters.build_stat_curve(
                        curves, oq.imtls, stats.mean_curve, ws)
                    rlzs[sid] = util.closest_to_ref(curves, mean.array)[:Z]
            self.datastore['best_rlzs'] = rlzs
        else:
            Z = len(oq.rlz_index)
            rlzs = numpy.zeros((self.N, Z), int)
            for z in range(Z):
                rlzs[:, z] = oq.rlz_index[z]
            self.datastore['best_rlzs'] = rlzs
        assert Z <= self.R, (Z, self.R)
        self.Z = Z
        self.rlzs = rlzs

        if oq.iml_disagg:
            # no hazard curves are needed
            self.poe_id = {None: 0}
            curves = [[None for z in range(Z)] for s in range(self.N)]
            self.ok_sites = set(self.sitecol.sids)
        else:
            self.poe_id = {poe: i for i, poe in enumerate(oq.poes_disagg)}
            curves = [self.get_curve(sid, rlzs[sid])
                      for sid in self.sitecol.sids]
            self.ok_sites = set(self.check_poes_disagg(curves, rlzs))
        self.iml4 = _iml4(rlzs, oq.iml_disagg, oq.imtls,
                          self.poes_disagg, curves)
        self.datastore['iml4'] = self.iml4
        self.datastore['poe4'] = numpy.zeros_like(self.iml4.array)

        self.save_bin_edges()
        tot = get_outputs_size(self.shapedic, oq.disagg_outputs)
        logging.info('Total output size: %s', humansize(sum(tot.values())))
        self.imldic = {}  # sid, rlz, poe, imt -> iml
        for s in self.sitecol.sids:
            iml3 = self.iml4[s]
            for z, rlz in enumerate(rlzs[s]):
                for p, poe in enumerate(self.poes_disagg):
                    for m, imt in enumerate(oq.imtls):
                        self.imldic[s, rlz, poe, imt] = iml3[m, p, z]

        # submit disaggregation tasks
        dstore = (self.datastore.parent if self.datastore.parent
                  else self.datastore)
        mag_edges = self.bin_edges[0]
        indices = get_indices_by_gidx_mag(dstore, mag_edges)
        allargs = []
        totweight = sum(sum(ri.weight for ri in indices[gm])
                        for gm in indices)
        maxweight = int(numpy.ceil(totweight / (oq.concurrent_tasks or 1)))
        grp_ids = dstore['grp_ids'][:]
        rlzs_by_gsim = self.full_lt.get_rlzs_by_gsim_list(grp_ids)
        num_eff_rlzs = len(self.full_lt.sm_rlzs)
        task_inputs = []
        G, U = 0, 0
        for gidx, magi in indices:
            trti = grp_ids[gidx][0] // num_eff_rlzs
            trt = self.trts[trti]
            cmaker = ContextMaker(
                trt, rlzs_by_gsim[gidx],
                {'truncation_level': oq.truncation_level,
                 'maximum_distance': oq.maximum_distance,
                 'collapse_level': oq.collapse_level,
                 'imtls': oq.imtls})
            G = max(G, len(cmaker.gsims))
            for rupidxs in block_splitter(
                    indices[gidx, magi], maxweight, weight):
                idxs = numpy.array([ri.index for ri in rupidxs])
                U = max(U, len(idxs))
                allargs.append((dstore, idxs, cmaker, self.iml4,
                                trti, magi, self.bin_edges[1:], oq))
                task_inputs.append((trti, magi, len(idxs)))

        nbytes, msg = get_array_nbytes(dict(N=self.N, M=self.M, G=G, U=U))
        logging.info('Maximum mean_std per task:\n%s', msg)
        sd = self.shapedic.copy()
        sd.pop('trt')
        sd.pop('mag')
        sd['tasks'] = numpy.ceil(len(allargs))
        nbytes, msg = get_array_nbytes(sd)
        if nbytes > oq.max_data_transfer:
            raise ValueError(
                'Estimated data transfer too big\n%s > max_data_transfer=%s' %
                (msg, humansize(oq.max_data_transfer)))
        logging.info('Estimated data transfer:\n%s', msg)
        dt = numpy.dtype([('trti', U8), ('magi', U8), ('nrups', U32)])
        self.datastore['disagg_task'] = numpy.array(task_inputs, dt)
        self.datastore.swmr_on()
        smap = parallel.Starmap(
            compute_disagg, allargs, h5=self.datastore.hdf5)
        results = smap.reduce(self.agg_result, AccumDict(accum={}))
        return results  # imti, sid -> trti, magi -> 6D array
Пример #21
0
    def full_disaggregation(self):
        """
        Run the disaggregation phase.
        """
        oq = self.oqparam
        tl = oq.truncation_level
        src_filter = SourceFilter(self.sitecol, oq.maximum_distance)
        if hasattr(self, 'csm'):
            for sg in self.csm.src_groups:
                if sg.atomic:
                    raise NotImplementedError(
                        'Atomic groups are not supported yet')
            if not self.csm.get_sources():
                raise RuntimeError('All sources were filtered away!')

        csm_info = self.datastore['csm_info']
        self.poes_disagg = oq.poes_disagg or (None, )
        self.imts = list(oq.imtls)
        if oq.rlz_index is None:
            try:
                rlzs = self.datastore['best_rlz'][()]
            except KeyError:
                rlzs = numpy.zeros(self.N, int)
        else:
            rlzs = [oq.rlz_index] * self.N

        if oq.iml_disagg:
            self.poe_id = {None: 0}
            curves = [None] * len(self.sitecol)  # no hazard curves are needed
            self.ok_sites = set(self.sitecol.sids)
        else:
            self.poe_id = {poe: i for i, poe in enumerate(oq.poes_disagg)}
            curves = [self.get_curve(sid, rlzs) for sid in self.sitecol.sids]
            self.ok_sites = set(self.check_poes_disagg(curves, rlzs))
        self.iml2s = _iml2s(rlzs, oq.iml_disagg, oq.imtls, self.poes_disagg,
                            curves)
        if oq.disagg_by_src:
            self.build_disagg_by_src()

        eps_edges = numpy.linspace(-tl, tl, oq.num_epsilon_bins + 1)

        # build trt_edges
        trts = tuple(csm_info.trts)
        trt_num = {trt: i for i, trt in enumerate(trts)}
        self.trts = trts

        # build mag_edges
        min_mag = csm_info.min_mag
        max_mag = csm_info.max_mag
        mag_edges = oq.mag_bin_width * numpy.arange(
            int(numpy.floor(min_mag / oq.mag_bin_width)),
            int(numpy.ceil(max_mag / oq.mag_bin_width) + 1))

        # build dist_edges
        maxdist = max(oq.maximum_distance(trt, max_mag) for trt in trts)
        dist_edges = oq.distance_bin_width * numpy.arange(
            0, int(numpy.ceil(maxdist / oq.distance_bin_width) + 1))

        # build eps_edges
        eps_edges = numpy.linspace(-tl, tl, oq.num_epsilon_bins + 1)

        # build lon_edges, lat_edges per sid
        bbs = src_filter.get_bounding_boxes(mag=max_mag)
        lon_edges, lat_edges = {}, {}  # by sid
        for sid, bb in zip(self.sitecol.sids, bbs):
            lon_edges[sid], lat_edges[sid] = disagg.lon_lat_bins(
                bb, oq.coordinate_bin_width)
        self.bin_edges = mag_edges, dist_edges, lon_edges, lat_edges, eps_edges
        self.save_bin_edges()

        self.imldict = {}  # sid, rlzi, poe, imt -> iml
        for s in self.sitecol.sids:
            iml2 = self.iml2s[s]
            r = rlzs[s]
            logging.info('Site #%d, disaggregating for rlz=#%d', s, r)
            for p, poe in enumerate(self.poes_disagg):
                for m, imt in enumerate(oq.imtls):
                    self.imldict[s, r, poe, imt] = iml2[m, p]

        # submit disagg tasks
        gid = self.datastore['rup/grp_id'][()]
        indices_by_grp = get_indices(gid)  # grp_id -> [(start, stop),...]
        blocksize = len(gid) // (oq.concurrent_tasks or 1) + 1
        allargs = []
        for grp_id, trt in csm_info.trt_by_grp.items():
            trti = trt_num[trt]
            rlzs_by_gsim = self.rlzs_assoc.get_rlzs_by_gsim(grp_id)
            cmaker = ContextMaker(
                trt, rlzs_by_gsim, {
                    'truncation_level': oq.truncation_level,
                    'maximum_distance': src_filter.integration_distance,
                    'filter_distance': oq.filter_distance,
                    'imtls': oq.imtls
                })
            for start, stop in indices_by_grp[grp_id]:
                for slc in gen_slices(start, stop, blocksize):
                    allargs.append((self.datastore, slc, cmaker, self.iml2s,
                                    trti, self.bin_edges))
        self.datastore.close()
        results = parallel.Starmap(compute_disagg,
                                   allargs,
                                   hdf5path=self.datastore.filename).reduce(
                                       self.agg_result, AccumDict(accum={}))
        return results  # sid -> trti-> 7D array
Пример #22
0
 def make_contexts(self, site_collection, rupture):
     return ContextMaker([self.gsim_class]).make_contexts(
         site_collection, rupture)
Пример #23
0
    def full_disaggregation(self):
        """
        Run the disaggregation phase.
        """
        oq = self.oqparam
        tl = oq.truncation_level
        src_filter = SourceFilter(self.sitecol, oq.maximum_distance)
        if hasattr(self, 'csm'):
            for sg in self.csm.src_groups:
                if sg.atomic:
                    raise NotImplemented('Atomic groups are not supported yet')
            if not self.csm.get_sources():
                raise RuntimeError('All sources were filtered away!')

        csm_info = self.datastore['csm_info']
        poes_disagg = oq.poes_disagg or (None, )
        R = len(self.rlzs_assoc.realizations)
        rlzs = extract.disagg_key(self.datastore).rlzs
        if oq.iml_disagg:
            self.poe_id = {None: 0}
            curves = [None] * len(self.sitecol)  # no hazard curves are needed
        else:
            self.poe_id = {poe: i for i, poe in enumerate(oq.poes_disagg)}
            curves = [self.get_curve(sid, rlzs) for sid in self.sitecol.sids]
            self.check_poes_disagg(curves, rlzs)
        iml2s = _iml2s(rlzs, oq.iml_disagg, oq.imtls, poes_disagg, curves)
        if oq.disagg_by_src:
            if R == 1:
                self.build_disagg_by_src(iml2s)
            else:
                logging.warning(
                    'disagg_by_src works only with 1 realization, '
                    'you have %d', R)

        eps_edges = numpy.linspace(-tl, tl, oq.num_epsilon_bins + 1)

        # build trt_edges
        trts = tuple(csm_info.trts)
        trt_num = {trt: i for i, trt in enumerate(trts)}
        self.trts = trts

        # build mag_edges
        min_mag = csm_info.min_mag
        max_mag = csm_info.max_mag
        mag_edges = oq.mag_bin_width * numpy.arange(
            int(numpy.floor(min_mag / oq.mag_bin_width)),
            int(numpy.ceil(max_mag / oq.mag_bin_width) + 1))

        # build dist_edges
        maxdist = max(oq.maximum_distance(trt, max_mag) for trt in trts)
        dist_edges = oq.distance_bin_width * numpy.arange(
            0, int(numpy.ceil(maxdist / oq.distance_bin_width) + 1))

        # build eps_edges
        eps_edges = numpy.linspace(-tl, tl, oq.num_epsilon_bins + 1)

        # build lon_edges, lat_edges per sid
        bbs = src_filter.get_bounding_boxes(mag=max_mag)
        lon_edges, lat_edges = {}, {}  # by sid
        for sid, bb in zip(self.sitecol.sids, bbs):
            lon_edges[sid], lat_edges[sid] = disagg.lon_lat_bins(
                bb, oq.coordinate_bin_width)
        self.bin_edges = mag_edges, dist_edges, lon_edges, lat_edges, eps_edges
        self.save_bin_edges()

        # build all_args
        all_args = []
        self.imldict = {}  # sid, rlzi, poe, imt -> iml
        for s in self.sitecol.sids:
            iml2 = iml2s[s]
            r = rlzs[s]
            logging.info('Site #%d, disaggregating for rlz=#%d', s, r)
            for p, poe in enumerate(oq.poes_disagg or [None]):
                for m, imt in enumerate(oq.imtls):
                    self.imldict[s, r, poe, imt] = iml2[m, p]

        for grp, dset in self.datastore['rup'].items():
            grp_id = int(grp[4:])
            trt = csm_info.trt_by_grp[grp_id]
            trti = trt_num[trt]
            rlzs_by_gsim = self.rlzs_assoc.get_rlzs_by_gsim(grp_id)
            cmaker = ContextMaker(trt, rlzs_by_gsim,
                                  src_filter.integration_distance,
                                  {'filter_distance': oq.filter_distance})
            for block in block_splitter(dset[()], 1000):
                all_args.append((src_filter.sitecol, numpy.array(block),
                                 cmaker, iml2s, trti, self.bin_edges, oq))

        self.num_ruptures = [0] * len(self.trts)
        mon = self.monitor()
        results = parallel.Starmap(compute_disagg, all_args,
                                   mon).reduce(self.agg_result,
                                               AccumDict(accum={}))
        return results
Пример #24
0
    def full_disaggregation(self, curves):
        """
        Run the disaggregation phase.

        :param curves: a list of hazard curves, one per site

        The curves can be all None if iml_disagg is set in the job.ini
        """
        oq = self.oqparam
        tl = oq.truncation_level
        src_filter = SourceFilter(self.sitecol, oq.maximum_distance)
        csm = self.csm
        if not csm.get_sources():
            raise RuntimeError('All sources were filtered away!')

        R = len(self.rlzs_assoc.realizations)
        I = len(oq.imtls)
        P = len(oq.poes_disagg) or 1
        if R * I * P > 10:
            logging.warn(
                'You have %d realizations, %d IMTs and %d poes_disagg: the '
                'disaggregation will be heavy and memory consuming', R, I, P)
        iml4 = disagg.make_iml4(R, oq.iml_disagg, oq.imtls, oq.poes_disagg
                                or (None, ), curves)
        if oq.disagg_by_src:
            if R == 1:
                self.build_disagg_by_src(iml4)
            else:
                logging.warn(
                    'disagg_by_src works only with 1 realization, '
                    'you have %d', R)

        eps_edges = numpy.linspace(-tl, tl, oq.num_epsilon_bins + 1)
        self.bin_edges = {}

        # build trt_edges
        trts = tuple(
            sorted(
                set(sg.trt for smodel in csm.source_models
                    for sg in smodel.src_groups)))
        trt_num = {trt: i for i, trt in enumerate(trts)}
        self.trts = trts

        # build mag_edges
        min_mag = min(sg.min_mag for smodel in csm.source_models
                      for sg in smodel.src_groups)
        max_mag = max(sg.max_mag for smodel in csm.source_models
                      for sg in smodel.src_groups)
        mag_edges = oq.mag_bin_width * numpy.arange(
            int(numpy.floor(min_mag / oq.mag_bin_width)),
            int(numpy.ceil(max_mag / oq.mag_bin_width) + 1))

        # build dist_edges
        maxdist = max(oq.maximum_distance(trt, max_mag) for trt in trts)
        dist_edges = oq.distance_bin_width * numpy.arange(
            0, int(numpy.ceil(maxdist / oq.distance_bin_width) + 1))

        # build eps_edges
        eps_edges = numpy.linspace(-tl, tl, oq.num_epsilon_bins + 1)

        # build lon_edges, lat_edges per sid
        bbs = src_filter.get_bounding_boxes(mag=max_mag)
        lon_edges, lat_edges = {}, {}  # by sid
        for sid, bb in zip(self.sitecol.sids, bbs):
            lon_edges[sid], lat_edges[sid] = disagg.lon_lat_bins(
                bb, oq.coordinate_bin_width)
        self.bin_edges = mag_edges, dist_edges, lon_edges, lat_edges, eps_edges
        self.save_bin_edges()

        # build all_args
        all_args = []
        maxweight = csm.get_maxweight(weight, oq.concurrent_tasks)
        mon = self.monitor('disaggregation')
        R = iml4.shape[1]
        self.imldict = {}  # sid, rlzi, poe, imt -> iml
        for s in self.sitecol.sids:
            for r in range(R):
                for p, poe in enumerate(oq.poes_disagg or [None]):
                    for m, imt in enumerate(oq.imtls):
                        self.imldict[s, r, poe, imt] = iml4[s, r, m, p]

        for smodel in csm.source_models:
            sm_id = smodel.ordinal
            for trt, groups in groupby(smodel.src_groups,
                                       operator.attrgetter('trt')).items():
                trti = trt_num[trt]
                sources = sum([grp.sources for grp in groups], [])
                rlzs_by_gsim = self.rlzs_assoc.get_rlzs_by_gsim(trt, sm_id)
                cmaker = ContextMaker(rlzs_by_gsim,
                                      src_filter.integration_distance,
                                      {'filter_distance': oq.filter_distance})
                for block in block_splitter(sources, maxweight, weight):
                    all_args.append((src_filter, block, cmaker, iml4, trti,
                                     self.bin_edges, oq, mon))

        self.num_ruptures = [0] * len(self.trts)
        self.cache_info = numpy.zeros(3)  # operations, cache_hits, num_zeros
        results = parallel.Starmap(compute_disagg,
                                   all_args).reduce(self.agg_result,
                                                    AccumDict(accum={}))

        # set eff_ruptures
        trti = csm.info.trt2i()
        for smodel in csm.info.source_models:
            for sg in smodel.src_groups:
                sg.eff_ruptures = self.num_ruptures[trti[sg.trt]]
        self.datastore['csm_info'] = csm.info

        ops, hits, num_zeros = self.cache_info
        logging.info('Cache speedup %s', ops / (ops - hits))
        logging.info('Discarded zero matrices: %d', num_zeros)
        return results
Пример #25
0
def classical(group, src_filter, gsims, param, monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns:
        a dictionary {grp_id: pmap} with attributes .grp_ids, .calc_times,
        .eff_ruptures
    """
    if getattr(group, 'src_interdep', None) == 'mutex':
        mutex_weight = {
            src.source_id: weight
            for src, weight in zip(group.sources, group.srcs_weights)
        }
        srcs = group.sources
    else:
        mutex_weight = None
        srcs = sum([split_source(src) for src in group], [])
    grp_ids = set()
    for src in group:
        grp_ids.update(src.src_group_ids)
    maxdist = src_filter.integration_distance
    with GroundShakingIntensityModel.forbid_instantiation():
        imtls = param['imtls']
        trunclevel = param.get('truncation_level')
        cmaker = ContextMaker(gsims, maxdist)
        ctx_mon = monitor('make_contexts', measuremem=False)
        poe_mon = monitor('get_poes', measuremem=False)
        pmap = AccumDict({
            grp_id: ProbabilityMap(len(imtls.array), len(gsims))
            for grp_id in grp_ids
        })
        # AccumDict of arrays with 4 elements weight, nsites, calc_time, split
        pmap.calc_times = AccumDict(accum=numpy.zeros(4))
        pmap.eff_ruptures = AccumDict()  # grp_id -> num_ruptures
        for src, s_sites in src_filter(srcs):  # filter now
            t0 = time.time()
            indep = group.rup_interdep == 'indep' if mutex_weight else True
            poemap = cmaker.poe_map(src, s_sites, imtls, trunclevel, ctx_mon,
                                    poe_mon, indep)
            if mutex_weight:  # mutex sources
                weight = mutex_weight[src.source_id]
                for sid in poemap:
                    pcurve = pmap[group.id].setdefault(sid, 0)
                    pcurve += poemap[sid] * weight
            elif poemap:
                for grp_id in src.src_group_ids:
                    pmap[grp_id] |= poemap
            src_id = src.source_id.split(':', 1)[0]
            pmap.calc_times[src_id] += numpy.array(
                [src.weight, len(s_sites),
                 time.time() - t0, 1])
            # storing the number of contributing ruptures too
            pmap.eff_ruptures += {
                grp_id: getattr(poemap, 'eff_ruptures', 0)
                for grp_id in src.src_group_ids
            }
        if mutex_weight and group.grp_probability is not None:
            pmap[group.id] *= group.grp_probability
        return pmap
Пример #26
0
def hazard_curves_per_trt(
        sources, sites, imtls, gsims, truncation_level=None,
        source_site_filter=filters.source_site_noop_filter,
        rupture_site_filter=filters.rupture_site_noop_filter,
        maximum_distance=None, bbs=(), monitor=DummyMonitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns:
        A list of G arrays of size N, where N is the number of sites and
        G the number of gsims. Each array contains records with fields given
        by the intensity measure types; the size of each field is given by the
        number of levels in ``imtls``.
    """
    cmaker = ContextMaker(gsims, maximum_distance)
    gnames = list(map(str, gsims))
    imt_dt = numpy.dtype([(imt, float, len(imtls[imt]))
                          for imt in sorted(imtls)])
    imts = {from_string(imt): imls for imt, imls in imtls.items()}
    curves = [numpy.ones(len(sites), imt_dt) for gname in gnames]
    sources_sites = ((source, sites) for source in sources)
    ctx_mon = monitor('making contexts', measuremem=False)
    rup_mon = monitor('getting ruptures', measuremem=False)
    pne_mon = monitor('computing poes', measuremem=False)
    monitor.calc_times = []  # pairs (src_id, delta_t)
    for source, s_sites in source_site_filter(sources_sites):
        t0 = time.time()
        try:
            with rup_mon:
                rupture_sites = list(rupture_site_filter(
                    (rupture, s_sites) for rupture in source.iter_ruptures()))
            for rupture, r_sites in rupture_sites:
                with ctx_mon:
                    try:
                        sctx, rctx, dctx = cmaker.make_contexts(
                            r_sites, rupture)
                    except FarAwayRupture:
                        continue

                    # add optional disaggregation information (bounding boxes)
                    if bbs:
                        sids = set(sctx.sites.sids)
                        jb_dists = dctx.rjb
                        closest_points = rupture.surface.get_closest_points(
                            sctx.sites.mesh)
                        bs = [bb for bb in bbs if bb.site_id in sids]
                        # NB: the assert below is always true; we are
                        # protecting against possible refactoring errors
                        assert len(bs) == len(jb_dists) == len(closest_points)
                        for bb, dist, p in zip(bs, jb_dists, closest_points):
                            if dist < maximum_distance:
                                # ruptures too far away are ignored
                                bb.update([dist], [p.longitude], [p.latitude])

                for i, gsim in enumerate(gsims):
                    with pne_mon:
                        for imt in imts:
                            poes = gsim.get_poes(
                                sctx, rctx, dctx, imt, imts[imt],
                                truncation_level)
                            pno = rupture.get_probability_no_exceedance(poes)
                            expanded_pno = sctx.sites.expand(pno, 1.0)
                            curves[i][str(imt)] *= expanded_pno
        except Exception as err:
            etype, err, tb = sys.exc_info()
            msg = 'An error occurred with source id=%s. Error: %s'
            msg %= (source.source_id, str(err))
            raise_(etype, msg, tb)

        # we are attaching the calculation times to the monitor
        # so that oq-lite (and the engine) can store them
        monitor.calc_times.append((source.id, time.time() - t0))
        # NB: source.id is an integer; it should not be confused
        # with source.source_id, which is a string
    for i in range(len(gnames)):
        for imt in imtls:
            curves[i][imt] = 1. - curves[i][imt]
    return curves
Пример #27
0
def pmap_from_grp(sources, src_filter, gsims, param, monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns: a ProbabilityMap instance
    """
    if isinstance(sources, SourceGroup):
        group = sources
        sources = group.sources
        trt = sources[0].tectonic_region_type
        mutex_weight = {
            src.source_id: weight
            for src, weight in zip(group.sources, group.srcs_weights)
        }
    else:  # list of sources
        trt = sources[0].tectonic_region_type
        group = SourceGroup(trt, sources, 'src_group', 'indep', 'indep')
    grp_id = sources[0].src_group_id
    maxdist = src_filter.integration_distance
    if hasattr(gsims, 'keys'):  # dictionary trt -> gsim
        gsims = [gsims[trt]]
    srcs = []
    for src in sources:
        if hasattr(src, '__iter__'):  # MultiPointSource
            srcs.extend(src)
        else:
            srcs.append(src)
    del sources
    with GroundShakingIntensityModel.forbid_instantiation():
        imtls = param['imtls']
        trunclevel = param.get('truncation_level')
        cmaker = ContextMaker(gsims, maxdist)
        ctx_mon = monitor('making contexts', measuremem=False)
        pne_mons = [
            monitor('%s.get_poes' % gsim, measuremem=False) for gsim in gsims
        ]
        src_indep = group.src_interdep == 'indep'
        pmap = ProbabilityMap(len(imtls.array), len(gsims))
        pmap.calc_times = []  # pairs (src_id, delta_t)
        pmap.grp_id = grp_id
        for src, s_sites in src_filter(srcs):
            t0 = time.time()
            poemap = poe_map(src, s_sites, imtls, cmaker, trunclevel, ctx_mon,
                             pne_mons, group.rup_interdep == 'indep')
            if src_indep:  # usual composition of probabilities
                pmap |= poemap
            else:  # mutually exclusive probabilities
                weight = mutex_weight[src.source_id]
                for sid in poemap:
                    pcurve = pmap.setdefault(sid, 0)
                    pcurve += poemap[sid] * weight
            pmap.calc_times.append(
                (src.source_id, src.weight, len(s_sites), time.time() - t0))
        # storing the number of contributing ruptures too
        pmap.eff_ruptures = {pmap.grp_id: pne_mons[0].counts}
        if group.grp_probability is not None:
            return pmap * group.grp_probability
        return pmap
Пример #28
0
def hazard_curves_per_trt(
        sources, sites, imtls, gsims, truncation_level=None,
        source_site_filter=filters.source_site_noop_filter,
        rupture_site_filter=filters.rupture_site_noop_filter,
        maximum_distance=None, bbs=(), monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns:
        A list of G arrays of size N, where N is the number of sites and
        G the number of gsims. Each array contains records with fields given
        by the intensity measure types; the size of each field is given by the
        number of levels in ``imtls``.
    """
    cmaker = ContextMaker(gsims, maximum_distance)
    gnames = list(map(str, gsims))
    imt_dt = numpy.dtype([(imt, float, len(imtls[imt]))
                          for imt in sorted(imtls)])
    imts = {from_string(imt): imls for imt, imls in imtls.items()}
    curves = [numpy.ones(len(sites), imt_dt) for gname in gnames]
    sources_sites = ((source, sites) for source in sources)
    ctx_mon = monitor('making contexts', measuremem=False)
    pne_mon = monitor('computing poes', measuremem=False)
    monitor.calc_times = []  # pairs (src_id, delta_t)
    monitor.eff_ruptures = 0  # effective number of contributing ruptures
    for source, s_sites in source_site_filter(sources_sites):
        t0 = time.time()
        try:
            rupture_sites = rupture_site_filter(
                (rupture, s_sites) for rupture in source.iter_ruptures())
            for rupture, r_sites in rupture_sites:
                with ctx_mon:
                    try:
                        sctx, rctx, dctx = cmaker.make_contexts(
                            r_sites, rupture)
                    except FarAwayRupture:
                        continue

                    monitor.eff_ruptures += 1

                    # add optional disaggregation information (bounding boxes)
                    if bbs:
                        sids = set(sctx.sites.sids)
                        jb_dists = dctx.rjb
                        closest_points = rupture.surface.get_closest_points(
                            sctx.sites.mesh)
                        bs = [bb for bb in bbs if bb.site_id in sids]
                        # NB: the assert below is always true; we are
                        # protecting against possible refactoring errors
                        assert len(bs) == len(jb_dists) == len(closest_points)
                        for bb, dist, p in zip(bs, jb_dists, closest_points):
                            if dist < maximum_distance:
                                # ruptures too far away are ignored
                                bb.update([dist], [p.longitude], [p.latitude])

                for i, gsim in enumerate(gsims):
                    with pne_mon:
                        for imt in imts:
                            poes = gsim.get_poes(
                                sctx, rctx, dctx, imt, imts[imt],
                                truncation_level)
                            pno = rupture.get_probability_no_exceedance(poes)
                            expanded_pno = sctx.sites.expand(pno, 1.0)
                            curves[i][str(imt)] *= expanded_pno
        except Exception as err:
            etype, err, tb = sys.exc_info()
            msg = 'An error occurred with source id=%s. Error: %s'
            msg %= (source.source_id, str(err))
            raise_(etype, msg, tb)

        # we are attaching the calculation times to the monitor
        # so that oq-lite (and the engine) can store them
        monitor.calc_times.append((source.id, time.time() - t0))
        # NB: source.id is an integer; it should not be confused
        # with source.source_id, which is a string
    for i in range(len(gnames)):
        for imt in imtls:
            curves[i][imt] = 1. - curves[i][imt]
    return curves
Пример #29
0
def _collect_bins_data(sources, site, imt, iml, gsims,
                       truncation_level, n_epsilons,
                       source_site_filter, rupture_site_filter):
    """
    Extract values of magnitude, distance, closest point, tectonic region
    types and PoE distribution.

    This method processes the source model (generates ruptures) and collects
    all needed parameters to arrays. It also defines tectonic region type
    bins sequence.
    """
    mags = []
    dists = []
    lons = []
    lats = []
    tect_reg_types = []
    probs_no_exceed = []
    sitecol = SiteCollection([site])
    sitemesh = sitecol.mesh

    _next_trt_num = 0
    trt_nums = {}

    sources_sites = ((source, sitecol) for source in sources)
    # here we ignore filtered site collection because either it is the same
    # as the original one (with one site), or the source/rupture is filtered
    # out and doesn't show up in the filter's output
    for src_idx, (source, s_sites) in \
            enumerate(source_site_filter(sources_sites)):
        try:
            tect_reg = source.tectonic_region_type
            gsim = gsims[tect_reg]
            cmaker = ContextMaker([gsim])
            if tect_reg not in trt_nums:
                trt_nums[tect_reg] = _next_trt_num
                _next_trt_num += 1
            tect_reg = trt_nums[tect_reg]

            ruptures_sites = ((rupture, s_sites)
                              for rupture in source.iter_ruptures())
            for rupture, r_sites in rupture_site_filter(ruptures_sites):
                # extract rupture parameters of interest
                mags.append(rupture.mag)
                [jb_dist] = rupture.surface.get_joyner_boore_distance(sitemesh)
                dists.append(jb_dist)
                [closest_point] = rupture.surface.get_closest_points(sitemesh)
                lons.append(closest_point.longitude)
                lats.append(closest_point.latitude)
                tect_reg_types.append(tect_reg)

                # compute conditional probability of exceeding iml given
                # the current rupture, and different epsilon level, that is
                # ``P(IMT >= iml | rup, epsilon_bin)`` for each of epsilon bins
                sctx, rctx, dctx = cmaker.make_contexts(sitecol, rupture)
                [poes_given_rup_eps] = gsim.disaggregate_poe(
                    sctx, rctx, dctx, imt, iml, truncation_level, n_epsilons
                )

                # collect probability of a rupture causing no exceedances
                probs_no_exceed.append(
                    rupture.get_probability_no_exceedance(poes_given_rup_eps)
                )
        except Exception as err:
            etype, err, tb = sys.exc_info()
            msg = 'An error occurred with source id=%s. Error: %s'
            msg %= (source.source_id, str(err))
            raise_(etype, msg, tb)

    mags = numpy.array(mags, float)
    dists = numpy.array(dists, float)
    lons = numpy.array(lons, float)
    lats = numpy.array(lats, float)
    tect_reg_types = numpy.array(tect_reg_types, int)
    probs_no_exceed = numpy.array(probs_no_exceed, float)

    trt_bins = [
        trt for (num, trt) in sorted((num, trt)
                                     for (trt, num) in trt_nums.items())
    ]

    return (mags, dists, lons, lats, tect_reg_types, trt_bins, probs_no_exceed)
Пример #30
0
    def _read_scenario_ruptures(self):
        oq = self.oqparam
        gsim_lt = readinput.get_gsim_lt(self.oqparam)
        G = gsim_lt.get_num_paths()
        if oq.calculation_mode.startswith('scenario'):
            ngmfs = oq.number_of_ground_motion_fields
        if oq.inputs['rupture_model'].endswith('.xml'):
            # check the number of branchsets
            bsets = len(gsim_lt._ltnode)
            if bsets > 1:
                raise InvalidFile(
                    '%s for a scenario calculation must contain a single '
                    'branchset, found %d!' % (oq.inputs['job_ini'], bsets))
            [(trt, rlzs_by_gsim)] = gsim_lt.get_rlzs_by_gsim_trt().items()
            self.cmaker = ContextMaker(
                trt, rlzs_by_gsim, {
                    'maximum_distance': oq.maximum_distance(trt),
                    'minimum_distance': oq.minimum_distance,
                    'truncation_level': oq.truncation_level,
                    'imtls': oq.imtls
                })
            rup = readinput.get_rupture(oq)
            if self.N > oq.max_sites_disagg:  # many sites, split rupture
                ebrs = [
                    EBRupture(copyobj(rup, rup_id=rup.rup_id + i),
                              'NA',
                              0,
                              G,
                              e0=i * G,
                              scenario=True) for i in range(ngmfs)
                ]
            else:  # keep a single rupture with a big occupation number
                ebrs = [
                    EBRupture(rup,
                              'NA',
                              0,
                              G * ngmfs,
                              rup.rup_id,
                              scenario=True)
                ]
            srcfilter = SourceFilter(self.sitecol, oq.maximum_distance(trt))
            aw = get_rup_array(ebrs, srcfilter)
            if len(aw) == 0:
                raise RuntimeError(
                    'The rupture is too far from the sites! Please check the '
                    'maximum_distance and the position of the rupture')
        elif oq.inputs['rupture_model'].endswith('.csv'):
            aw = get_ruptures(oq.inputs['rupture_model'])
            if len(gsim_lt.values) == 1:  # fix for scenario_damage/case_12
                aw['trt_smr'] = 0  # a single TRT
            if oq.calculation_mode.startswith('scenario'):
                # rescale n_occ by ngmfs and nrlzs
                aw['n_occ'] *= ngmfs * gsim_lt.get_num_paths()
        else:
            raise InvalidFile("Something wrong in %s" % oq.inputs['job_ini'])
        rup_array = aw.array
        hdf5.extend(self.datastore['rupgeoms'], aw.geom)

        if len(rup_array) == 0:
            raise RuntimeError(
                'There are no sites within the maximum_distance'
                ' of %s km from the rupture' %
                oq.maximum_distance(rup.tectonic_region_type)(rup.mag))

        fake = logictree.FullLogicTree.fake(gsim_lt)
        self.realizations = fake.get_realizations()
        self.datastore['full_lt'] = fake
        self.store_rlz_info({})  # store weights
        self.save_params()
        imp = calc.RuptureImporter(self.datastore)
        imp.import_rups_events(rup_array, get_rupture_getters)
Пример #31
0
    def compute(self):
        """
        Submit disaggregation tasks and return the results
        """
        oq = self.oqparam
        dstore = (self.datastore.parent
                  if self.datastore.parent else self.datastore)
        magi = numpy.searchsorted(self.bin_edges[0], dstore['rup/mag'][:]) - 1
        magi[magi == -1] = 0  # when the magnitude is on the edge
        totrups = len(magi)
        logging.info('Reading {:_d} ruptures'.format(totrups))
        rdt = [('grp_id', U16), ('magi', U8), ('nsites', U16), ('idx', U32)]
        rdata = numpy.zeros(totrups, rdt)
        rdata['magi'] = magi
        rdata['idx'] = numpy.arange(totrups)
        rdata['grp_id'] = dstore['rup/grp_id'][:]
        rdata['nsites'] = dstore['rup/nsites'][:]
        totweight = rdata['nsites'].sum()
        et_ids = dstore['et_ids'][:]
        rlzs_by_gsim = self.full_lt.get_rlzs_by_gsim_list(et_ids)
        G = max(len(rbg) for rbg in rlzs_by_gsim)
        maxw = 2 * 1024**3 / (16 * G * self.M)  # at max 2 GB
        maxweight = min(numpy.ceil(totweight / (oq.concurrent_tasks or 1)),
                        maxw)
        num_eff_rlzs = len(self.full_lt.sm_rlzs)
        task_inputs = []
        U = 0
        self.datastore.swmr_on()
        smap = parallel.Starmap(compute_disagg, h5=self.datastore.hdf5)
        # ABSURDLY IMPORTANT!! we rely on the fact that the classical part
        # of the calculation stores the ruptures in chunks of constant
        # grp_id, therefore it is possible to build (start, stop) slices;
        # we are NOT grouping by operator.itemgetter('grp_id', 'magi'):
        # that would break the ordering of the indices causing an incredibly
        # worse performance, but visible only in extra-large calculations!
        for block in block_splitter(rdata, maxweight,
                                    operator.itemgetter('nsites'),
                                    operator.itemgetter('grp_id')):
            grp_id = block[0]['grp_id']
            trti = et_ids[grp_id][0] // num_eff_rlzs
            trt = self.trts[trti]
            cmaker = ContextMaker(
                trt, rlzs_by_gsim[grp_id], {
                    'truncation_level': oq.truncation_level,
                    'maximum_distance': oq.maximum_distance,
                    'collapse_level': oq.collapse_level,
                    'num_epsilon_bins': oq.num_epsilon_bins,
                    'investigation_time': oq.investigation_time,
                    'imtls': oq.imtls
                })
            U = max(U, block.weight)
            slc = slice(block[0]['idx'], block[-1]['idx'] + 1)
            smap.submit((dstore, slc, cmaker, self.hmap4, trti, magi[slc],
                         self.bin_edges))
            task_inputs.append((trti, slc.stop - slc.start))

        nbytes, msg = get_nbytes_msg(dict(M=self.M, G=G, U=U, F=2))
        logging.info('Maximum mean_std per task:\n%s', msg)

        s = self.shapedic
        sd = dict(N=s['N'],
                  M=s['M'],
                  P=s['P'],
                  Z=s['Z'],
                  D=s['dist'],
                  E=s['eps'],
                  Lo=s['lon'],
                  La=s['lat'])
        sd['tasks'] = numpy.ceil(len(task_inputs))
        nbytes, msg = get_nbytes_msg(sd)
        if nbytes > oq.max_data_transfer:
            raise ValueError(
                'Estimated data transfer too big\n%s > max_data_transfer=%s' %
                (msg, humansize(oq.max_data_transfer)))
        logging.info('Estimated data transfer:\n%s', msg)

        sd.pop('tasks')
        dt = numpy.dtype([('trti', U8), ('nrups', U32)])
        self.datastore['disagg_task'] = numpy.array(task_inputs, dt)
        results = smap.reduce(self.agg_result, AccumDict(accum={}))
        return results  # imti, sid -> trti, magi -> 6D array
Пример #32
0
    def full_disaggregation(self):
        """
        Run the disaggregation phase.
        """
        oq = self.oqparam
        tl = oq.truncation_level
        src_filter = self.src_filter()
        if hasattr(self, 'csm'):
            for sg in self.csm.src_groups:
                if sg.atomic:
                    raise NotImplementedError(
                        'Atomic groups are not supported yet')
            if not self.csm.get_sources():
                raise RuntimeError('All sources were filtered away!')

        csm_info = self.datastore['csm_info']
        self.poes_disagg = oq.poes_disagg or (None,)
        self.imts = list(oq.imtls)

        self.ws = [rlz.weight for rlz in self.rlzs_assoc.realizations]
        self.pgetter = getters.PmapGetter(
            self.datastore, self.ws, self.sitecol.sids)

        # build array rlzs (N, Z)
        if oq.rlz_index is None:
            Z = oq.num_rlzs_disagg
            rlzs = numpy.zeros((self.N, Z), int)
            if self.R > 1:
                for sid in self.sitecol.sids:
                    curves = numpy.array(
                        [pc.array for pc in self.pgetter.get_pcurves(sid)])
                    mean = getters.build_stat_curve(
                        curves, oq.imtls, stats.mean_curve, self.ws)
                    rlzs[sid] = util.closest_to_ref(curves, mean.array)[:Z]
                self.datastore['best_rlzs'] = rlzs
        else:
            Z = len(oq.rlz_index)
            rlzs = numpy.zeros((self.N, Z), int)
            for z in range(Z):
                rlzs[:, z] = oq.rlz_index[z]
        assert Z <= self.R, (Z, self.R)
        self.Z = Z
        self.rlzs = rlzs

        if oq.iml_disagg:
            # no hazard curves are needed
            self.poe_id = {None: 0}
            curves = [[None for z in range(Z)] for s in range(self.N)]
            self.ok_sites = set(self.sitecol.sids)
        else:
            self.poe_id = {poe: i for i, poe in enumerate(oq.poes_disagg)}
            curves = [self.get_curve(sid, rlzs[sid])
                      for sid in self.sitecol.sids]
            self.ok_sites = set(self.check_poes_disagg(curves, rlzs))
        self.iml4 = _iml4(rlzs, oq.iml_disagg, oq.imtls,
                          self.poes_disagg, curves)
        if oq.disagg_by_src:
            self.build_disagg_by_src(rlzs)

        eps_edges = numpy.linspace(-tl, tl, oq.num_epsilon_bins + 1)

        # build trt_edges
        trts = tuple(csm_info.trts)
        trt_num = {trt: i for i, trt in enumerate(trts)}
        self.trts = trts

        # build mag_edges
        mags = [float(mag) for mag in self.datastore['source_mags']]
        mag_edges = oq.mag_bin_width * numpy.arange(
            int(numpy.floor(min(mags) / oq.mag_bin_width)),
            int(numpy.ceil(max(mags) / oq.mag_bin_width) + 1))

        # build dist_edges
        maxdist = max(oq.maximum_distance(trt) for trt in trts)
        dist_edges = oq.distance_bin_width * numpy.arange(
            0, int(numpy.ceil(maxdist / oq.distance_bin_width) + 1))

        # build eps_edges
        eps_edges = numpy.linspace(-tl, tl, oq.num_epsilon_bins + 1)

        # build lon_edges, lat_edges per sid
        bbs = src_filter.get_bounding_boxes(mag=max(mags))
        lon_edges, lat_edges = {}, {}  # by sid
        for sid, bb in zip(self.sitecol.sids, bbs):
            lon_edges[sid], lat_edges[sid] = disagg.lon_lat_bins(
                bb, oq.coordinate_bin_width)
        self.bin_edges = mag_edges, dist_edges, lon_edges, lat_edges, eps_edges
        self.save_bin_edges()

        self.imldict = {}  # sid, rlz, poe, imt -> iml
        for s in self.sitecol.sids:
            for z, rlz in enumerate(rlzs[s]):
                logging.info('Site #%d, disaggregating for rlz=#%d', s, rlz)
                for p, poe in enumerate(self.poes_disagg):
                    for m, imt in enumerate(oq.imtls):
                        self.imldict[s, rlz, poe, imt] = self.iml4[s, m, p, z]

        # submit disagg tasks
        gid = self.datastore['rup/grp_id'][()]
        indices_by_grp = get_indices(gid)  # grp_id -> [(start, stop),...]
        blocksize = len(gid) // (oq.concurrent_tasks or 1) + 1
        # NB: removing the blocksize causes slow disaggregation tasks
        dstore = (self.datastore.parent if self.datastore.parent
                  else self.datastore)
        smap = parallel.Starmap(compute_disagg, h5=self.datastore.hdf5)
        for grp_id, trt in csm_info.trt_by_grp.items():
            logging.info('Group #%d, sending rup_data for %s', grp_id, trt)
            trti = trt_num[trt]
            rlzs_by_gsim = self.rlzs_assoc.get_rlzs_by_gsim(grp_id)
            cmaker = ContextMaker(
                trt, rlzs_by_gsim,
                {'truncation_level': oq.truncation_level,
                 'maximum_distance': src_filter.integration_distance,
                 'filter_distance': oq.filter_distance, 'imtls': oq.imtls})
            for start, stop in indices_by_grp[grp_id]:
                for slc in gen_slices(start, stop, blocksize):
                    rupdata = {k: dstore['rup/' + k][slc]
                               for k in self.datastore['rup']}
                    smap.submit((rupdata, self.sitecol, oq, cmaker,
                                 self.iml4, trti, self.bin_edges))
        results = smap.reduce(self.agg_result, AccumDict(accum={}))
        return results  # sid -> trti-> 8D array
Пример #33
0
    def compute(self):
        """
        Submit disaggregation tasks and return the results
        """
        logging.info('Reading ruptures')
        oq = self.oqparam
        dstore = (self.datastore.parent
                  if self.datastore.parent else self.datastore)
        mags = set()
        for trt, dset in self.datastore['source_mags'].items():
            mags.update(dset[:])
        mags = sorted(mags)
        allargs = []
        totweight = sum(d['rctx']['nsites'].sum() for n, d in dstore.items()
                        if n.startswith('mag_') and len(d['rctx']))
        et_ids = dstore['et_ids'][:]
        rlzs_by_gsim = self.full_lt.get_rlzs_by_gsim_list(et_ids)
        G = max(len(rbg) for rbg in rlzs_by_gsim)
        maxw = 2 * 1024**3 / (16 * G * self.M)  # at max 2 GB
        maxweight = min(numpy.ceil(totweight / (oq.concurrent_tasks or 1)),
                        maxw)
        num_eff_rlzs = len(self.full_lt.sm_rlzs)
        task_inputs = []
        U = 0
        totrups = 0
        for mag in mags:
            rctx = dstore['mag_%s/rctx' % mag][:]
            totrups += len(rctx)
            for grp_id, gids in enumerate(et_ids):
                idxs, = numpy.where(rctx['grp_id'] == grp_id)
                if len(idxs) == 0:
                    continue
                trti = gids[0] // num_eff_rlzs
                trt = self.trts[trti]
                cmaker = ContextMaker(
                    trt, rlzs_by_gsim[grp_id], {
                        'truncation_level': oq.truncation_level,
                        'maximum_distance': oq.maximum_distance,
                        'collapse_level': oq.collapse_level,
                        'imtls': oq.imtls
                    })
                for blk in block_splitter(rctx[idxs], maxweight, nsites):
                    nr = len(blk)
                    U = max(U, blk.weight)
                    allargs.append((dstore, numpy.array(blk), cmaker,
                                    self.hmap4, trti, self.bin_edges, oq))
                    task_inputs.append((trti, mag, nr))
        logging.info('Found {:_d} ruptures'.format(totrups))
        nbytes, msg = get_array_nbytes(dict(M=self.M, G=G, U=U, F=2))
        logging.info('Maximum mean_std per task:\n%s', msg)

        s = self.shapedic
        size = s['dist'] * s['eps'] + s['lon'] * s['lat']
        sd = dict(N=s['N'], M=s['M'], P=s['P'], Z=s['Z'], size=size)
        sd['tasks'] = numpy.ceil(len(allargs))
        nbytes, msg = get_array_nbytes(sd)
        if nbytes > oq.max_data_transfer:
            raise ValueError(
                'Estimated data transfer too big\n%s > max_data_transfer=%s' %
                (msg, humansize(oq.max_data_transfer)))
        logging.info('Estimated data transfer:\n%s', msg)

        sd.pop('tasks')
        sd['mags_trt'] = sum(
            len(mags) for mags in self.datastore['source_mags'].values())
        nbytes, msg = get_array_nbytes(sd)
        logging.info('Estimated memory on the master:\n%s', msg)

        dt = numpy.dtype([('trti', U8), ('mag', '|S4'), ('nrups', U32)])
        self.datastore['disagg_task'] = numpy.array(task_inputs, dt)
        self.datastore.swmr_on()
        smap = parallel.Starmap(compute_disagg,
                                allargs,
                                h5=self.datastore.hdf5)
        results = smap.reduce(self.agg_result, AccumDict(accum={}))
        return results  # imti, sid -> trti, magi -> 6D array
Пример #34
0
def compute_ruptures(sources, sitecol, siteidx, rlzs_assoc, monitor):
    """
    :param sources:
        List of commonlib.source.Source tuples
    :param sitecol:
        a :class:`openquake.hazardlib.site.SiteCollection` instance
    :param siteidx:
        always equal to 0
    :param rlzs_assoc:
        a :class:`openquake.commonlib.source.RlzsAssoc` instance
    :param monitor:
        monitor instance
    :returns:
        a dictionary src_group_id -> [Rupture instances]
    """
    assert siteidx == 0, (
        'siteidx can be nonzero only for the classical_tiling calculations: '
        'tiling with the EventBasedRuptureCalculator is an error')
    # NB: by construction each block is a non-empty list with
    # sources of the same src_group_id
    src_group_id = sources[0].src_group_id
    oq = monitor.oqparam
    trt = sources[0].tectonic_region_type
    max_dist = oq.maximum_distance[trt]
    cmaker = ContextMaker(rlzs_assoc.gsims_by_grp_id[src_group_id])
    params = sorted(cmaker.REQUIRES_RUPTURE_PARAMETERS)
    rup_data_dt = numpy.dtype(
        [('rupserial', U32), ('multiplicity', U16),
         ('numsites', U32), ('occurrence_rate', F32)] + [
            (param, F32) for param in params])
    eb_ruptures = []
    rup_data = []
    calc_times = []
    rup_mon = monitor('filtering ruptures', measuremem=False)
    num_samples = rlzs_assoc.samples[src_group_id]

    # Compute and save stochastic event sets
    for src in sources:
        t0 = time.time()
        s_sites = src.filter_sites_by_distance_to_source(max_dist, sitecol)
        if s_sites is None:
            continue
        rupture_filter = functools.partial(
            filter_sites_by_distance_to_rupture,
            integration_distance=max_dist, sites=s_sites)
        num_occ_by_rup = sample_ruptures(
            src, oq.ses_per_logic_tree_path, num_samples,
            rlzs_assoc.seed)
        # NB: the number of occurrences is very low, << 1, so it is
        # more efficient to filter only the ruptures that occur, i.e.
        # to call sample_ruptures *before* the filtering
        for ebr in build_eb_ruptures(
                src, num_occ_by_rup, rupture_filter, oq.random_seed, rup_mon):
            nsites = len(ebr.indices)
            try:
                rate = ebr.rupture.occurrence_rate
            except AttributeError:  # for nonparametric sources
                rate = numpy.nan
            rc = cmaker.make_rupture_context(ebr.rupture)
            ruptparams = tuple(getattr(rc, param) for param in params)
            rup_data.append((ebr.serial, len(ebr.etags), nsites, rate) +
                            ruptparams)
            eb_ruptures.append(ebr)
        dt = time.time() - t0
        calc_times.append((src.id, dt))
    res = AccumDict({src_group_id: eb_ruptures})
    res.calc_times = calc_times
    res.rup_data = numpy.array(rup_data, rup_data_dt)
    res.trt = trt
    return res
Пример #35
0
def _collect_bins_data(trt_num, source_ruptures, site, curves, trt_model_id,
                       rlzs_assoc, gsims, imtls, poes, truncation_level,
                       n_epsilons, mon):
    # returns a BinData instance
    sitecol = SiteCollection([site])
    mags = []
    dists = []
    lons = []
    lats = []
    trts = []
    pnes = []
    sitemesh = sitecol.mesh
    make_ctxt = mon('making contexts', measuremem=False)
    disagg_poe = mon('disaggregate_poe', measuremem=False)
    cmaker = ContextMaker(gsims)
    for source, ruptures in source_ruptures:
        try:
            tect_reg = trt_num[source.tectonic_region_type]
            for rupture in ruptures:
                with make_ctxt:
                    sctx, rctx, dctx = cmaker.make_contexts(sitecol, rupture)
                # extract rupture parameters of interest
                mags.append(rupture.mag)
                dists.append(dctx.rjb[0])  # single site => single distance
                [closest_point] = rupture.surface.get_closest_points(sitemesh)
                lons.append(closest_point.longitude)
                lats.append(closest_point.latitude)
                trts.append(tect_reg)

                pne_dict = {}
                # a dictionary rlz.id, poe, imt_str -> prob_no_exceed
                for gsim in gsims:
                    for imt_str, imls in imtls.iteritems():
                        imt = from_string(imt_str)
                        imls = numpy.array(imls[::-1])
                        for rlz in rlzs_assoc[trt_model_id,
                                              gsim.__class__.__name__]:
                            rlzi = rlz.ordinal
                            curve_poes = curves[rlzi, imt_str][::-1]
                            for poe in poes:
                                iml = numpy.interp(poe, curve_poes, imls)
                                # compute probability of exceeding iml given
                                # the current rupture and epsilon_bin, that is
                                # ``P(IMT >= iml | rup, epsilon_bin)``
                                # for each of the epsilon bins
                                with disagg_poe:
                                    [poes_given_rup_eps] = \
                                        gsim.disaggregate_poe(
                                            sctx, rctx, dctx, imt, iml,
                                            truncation_level, n_epsilons)
                                pne = rupture.get_probability_no_exceedance(
                                    poes_given_rup_eps)
                                pne_dict[rlzi, poe, imt_str] = (iml, pne)

                pnes.append(pne_dict)
        except Exception as err:
            etype, err, tb = sys.exc_info()
            msg = 'An error occurred with source id=%s. Error: %s'
            msg %= (source.source_id, err)
            raise etype, msg, tb

    return BinData(numpy.array(mags, float), numpy.array(dists, float),
                   numpy.array(lons, float), numpy.array(lats, float),
                   numpy.array(trts, int), pnes)
Пример #36
0
def _collect_bins_data(trt_num, source_ruptures, site, curves, src_group_id,
                       rlzs_assoc, gsims, imtls, poes, truncation_level,
                       n_epsilons, mon):
    # returns a BinData instance
    sitecol = SiteCollection([site])
    mags = []
    dists = []
    lons = []
    lats = []
    trts = []
    pnes = []
    sitemesh = sitecol.mesh
    make_ctxt = mon('making contexts', measuremem=False)
    disagg_poe = mon('disaggregate_poe', measuremem=False)
    cmaker = ContextMaker(gsims)
    for source, ruptures in source_ruptures:
        try:
            tect_reg = trt_num[source.tectonic_region_type]
            for rupture in ruptures:
                with make_ctxt:
                    sctx, rctx, dctx = cmaker.make_contexts(sitecol, rupture)
                # extract rupture parameters of interest
                mags.append(rupture.mag)
                dists.append(dctx.rjb[0])  # single site => single distance
                [closest_point] = rupture.surface.get_closest_points(sitemesh)
                lons.append(closest_point.longitude)
                lats.append(closest_point.latitude)
                trts.append(tect_reg)

                pne_dict = {}
                # a dictionary rlz.id, poe, imt_str -> prob_no_exceed
                for gsim in gsims:
                    gs = str(gsim)
                    for imt_str, imls in imtls.items():
                        imt = from_string(imt_str)
                        imls = numpy.array(imls[::-1])
                        for rlz in rlzs_assoc[src_group_id, gs]:
                            rlzi = rlz.ordinal
                            curve_poes = curves[rlzi, imt_str][::-1]
                            for poe in poes:
                                iml = numpy.interp(poe, curve_poes, imls)
                                # compute probability of exceeding iml given
                                # the current rupture and epsilon_bin, that is
                                # ``P(IMT >= iml | rup, epsilon_bin)``
                                # for each of the epsilon bins
                                with disagg_poe:
                                    [poes_given_rup_eps] = \
                                        gsim.disaggregate_poe(
                                            sctx, rctx, dctx, imt, iml,
                                            truncation_level, n_epsilons)
                                pne = rupture.get_probability_no_exceedance(
                                    poes_given_rup_eps)
                                pne_dict[rlzi, poe, imt_str] = (iml, pne)

                pnes.append(pne_dict)
        except Exception as err:
            etype, err, tb = sys.exc_info()
            msg = 'An error occurred with source id=%s. Error: %s'
            msg %= (source.source_id, err)
            raise_(etype, msg, tb)

    return BinData(numpy.array(mags, float),
                   numpy.array(dists, float),
                   numpy.array(lons, float),
                   numpy.array(lats, float),
                   numpy.array(trts, int),
                   pnes)
Пример #37
0
def classical(group, src_filter, gsims, param, monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns:
        a dictionary {grp_id: pmap} with attributes .grp_ids, .calc_times,
        .eff_ruptures
    """
    if not hasattr(src_filter, 'sitecol'):  # a sitecol was passed
        src_filter = SourceFilter(src_filter, {})

    # Get the parameters assigned to the group
    src_mutex = getattr(group, 'src_interdep', None) == 'mutex'
    rup_mutex = getattr(group, 'rup_interdep', None) == 'mutex'
    cluster = getattr(group, 'cluster', None)
    # Compute the number of ruptures
    grp_ids = set()
    trts = set()
    for src in group:
        if not src.num_ruptures:
            # src.num_ruptures is set when parsing the XML, but not when
            # the source is instantiated manually, so it is set here
            src.num_ruptures = src.count_ruptures()
        # This sets the proper TOM in case of a cluster
        if cluster:
            src.temporal_occurrence_model = FatedTOM(time_span=1)
        # Updating IDs
        grp_ids.update(src.src_group_ids)
        trts.add(src.tectonic_region_type)
    # Now preparing context
    maxdist = src_filter.integration_distance
    imtls = param['imtls']
    trunclevel = param.get('truncation_level')
    [trt] = trts  # there must be a single tectonic region type
    cmaker = ContextMaker(trt, gsims, maxdist, param, monitor)
    # Prepare the accumulator for the probability maps
    pmap = AccumDict({grp_id: ProbabilityMap(len(imtls.array), len(gsims))
                      for grp_id in grp_ids})
    pmap.trt = trt
    rupdata = {grp_id: [] for grp_id in grp_ids}
    # AccumDict of arrays with 2 elements weight, calc_time
    calc_times = AccumDict(accum=numpy.zeros(2, numpy.float32))
    eff_ruptures = AccumDict(accum=0)  # grp_id -> num_ruptures
    nsites = {}  # src.id -> num_sites
    # Computing hazard
    for src, s_sites in src_filter(group):  # filter now
        nsites[src.id] = src.nsites
        t0 = time.time()
        try:
            poemap = cmaker.poe_map(src, s_sites, imtls, trunclevel,
                                    rup_indep=not rup_mutex)
        except Exception as err:
            etype, err, tb = sys.exc_info()
            msg = '%s (source id=%s)' % (str(err), src.source_id)
            raise etype(msg).with_traceback(tb)
        if src_mutex:  # mutex sources, there is a single group
            for sid in poemap:
                pcurve = pmap[src.src_group_id].setdefault(sid, 0)
                pcurve += poemap[sid] * src.mutex_weight
        elif poemap:
            for gid in src.src_group_ids:
                pmap[gid] |= poemap
        if len(cmaker.rupdata):
            for gid in src.src_group_ids:
                rupdata[gid].append(cmaker.rupdata)
        calc_times[src.id] += numpy.array([src.weight, time.time() - t0])
        # storing the number of contributing ruptures too
        eff_ruptures += {gid: getattr(poemap, 'eff_ruptures', 0)
                         for gid in src.src_group_ids}
    # Updating the probability map in the case of mutually exclusive
    # sources
    group_probability = getattr(group, 'grp_probability', None)
    if src_mutex and group_probability:
        pmap[src.src_group_id] *= group_probability
    # Processing cluster
    if cluster:
        tom = getattr(group, 'temporal_occurrence_model')
        pmap = _cluster(param, tom, imtls, gsims, grp_ids, pmap)
    # Return results
    for gid, data in rupdata.items():
        if len(data):
            rupdata[gid] = numpy.concatenate(data)
    return dict(pmap=pmap, calc_times=calc_times, eff_ruptures=eff_ruptures,
                rup_data=rupdata, nsites=nsites)
def get_conditional_gmfs(database, rupture, sites, gsims, imts,
        number_simulations, truncation_level,
        correlation_model=DEFAULT_CORRELATION):
    """
    Get a set of random fields conditioned on a set of observations
    :param database:
        Ground motion records for the event as instance of :class:
        smtk.sm_database.GroundMotionDatabase
    :param rupture:
        Event rupture as instance of :class:
        openquake.hazardlib.source.rupture.Rupture
    :param sites:
        Target sites as instance of :class:
        openquake.hazardlib.site.SiteCollection
    :param list gsims:
        List of GMPEs required
    :param list imts:
        List of intensity measures required
    :param int number_simulations:
        Number of simulated fields required
    :param float truncation_level:
        Ground motion truncation level
    """

    # Get known sites mesh
    known_sites = database.get_site_collection()

    # Get Observed Residuals
    residuals = Residuals(gsims, imts)
    residuals.get_residuals(database)
    imt_dict = OrderedDict([
        (imtx,  np.zeros([len(sites.lons), number_simulations]))
        for imtx in imts])
    gmfs = OrderedDict([(gmpe, imt_dict) for gmpe in gsims])
    gmpe_list = [GSIM_LIST[gmpe]() for gmpe in gsims]
    cmaker = ContextMaker(gmpe_list)
    sctx, rctx, dctx = cmaker.make_contexts(sites, rupture)
    for gsim in gmpe_list:
        gmpe = gsim.__class__.__name__
        #gsim = GSIM_LIST[gmpe]()
        #sctx, rctx, dctx = gsim.make_contexts(sites, rupture)
        for imtx in imts:
            if truncation_level == 0:
                gmfs[gmpe][imtx], _ = gsim.get_mean_and_stddevs(sctx, rctx,
                    dctx, from_string(imtx), stddev_types=[])
                continue
            if "Intra event" in gsim.DEFINED_FOR_STANDARD_DEVIATION_TYPES:
                epsilon = conditional_simulation(
                    known_sites,
                    residuals.residuals[gmpe][imtx]["Intra event"],
                    sites,
                    imtx,
                    number_simulations,
                    correlation_model)
                tau = np.unique(residuals.residuals[gmpe][imtx]["Inter event"])
                mean, [stddev_inter, stddev_intra] = gsim.get_mean_and_stddevs(
                    sctx,
                    rctx,
                    dctx, 
                    from_string(imtx), 
                    ["Inter event", "Intra event"])
                for iloc in range(0, number_simulations):
                    gmfs[gmpe][imtx][:, iloc] = np.exp(
                        mean +
                        (tau * stddev_inter) +
                        (epsilon[:, iloc].A1 * stddev_intra))
                        
            else:
                epsilon = conditional_simulation(
                    known_sites,
                    residuals.residuals[gmpe][imtx]["Total"],
                    sites,
                    imtx,
                    number_simulations,
                    correlation_model)
                tau = None
                mean, [stddev_total] = gsim.get_mean_and_stddevs(
                    sctx,
                    rctx,
                    dctx,
                    from_string(imtx),
                    ["Total"])
                for iloc in range(0, number_simulations):
                    gmfs[gmpe][imtx][:, iloc] = np.exp(
                        mean +
                        (epsilon[:, iloc].A1 * stddev_total.flatten()))
    return gmfs
Пример #39
0
def calc_hazard_curves(groups,
                       srcfilter,
                       imtls,
                       gsim_by_trt,
                       truncation_level=None,
                       apply=sequential_apply,
                       reqv=None,
                       **kwargs):
    """
    Compute hazard curves on a list of sites, given a set of seismic source
    groups and a dictionary of ground shaking intensity models (one per
    tectonic region type).

    Probability of ground motion exceedance is computed in different ways
    depending if the sources are independent or mutually exclusive.

    :param groups:
        A sequence of groups of seismic sources objects (instances of
        of :class:`~openquake.hazardlib.source.base.BaseSeismicSource`).
    :param srcfilter:
        A source filter over the site collection or the site collection itself
    :param imtls:
        Dictionary mapping intensity measure type strings
        to lists of intensity measure levels.
    :param gsim_by_trt:
        Dictionary mapping tectonic region types (members
        of :class:`openquake.hazardlib.const.TRT`) to
        :class:`~openquake.hazardlib.gsim.base.GMPE` or
        :class:`~openquake.hazardlib.gsim.base.IPE` objects.
    :param truncation_level:
        Float, number of standard deviations for truncation of the intensity
        distribution.
    :param apply:
        apply function to use (default sequential_apply)
    :param reqv:
        If not None, an instance of RjbEquivalent
    :returns:
        An array of size N, where N is the number of sites, which elements
        are records with fields given by the intensity measure types; the
        size of each field is given by the number of levels in ``imtls``.
    """
    # This is ensuring backward compatibility i.e. processing a list of
    # sources
    if not isinstance(groups[0], SourceGroup):  # sent a list of sources
        odic = groupby(groups, operator.attrgetter('tectonic_region_type'))
        groups = [
            SourceGroup(trt, odic[trt], 'src_group', 'indep', 'indep')
            for trt in odic
        ]
    idx = 0
    span = None
    for i, grp in enumerate(groups):
        for src in grp:
            tom = getattr(src, 'temporal_occurrence_model', None)
            if tom:
                span = tom.time_span
            src.weight = src.count_ruptures()
            src.grp_id = i
            src.id = idx
            idx += 1
    imtls = DictArray(imtls)
    shift_hypo = kwargs['shift_hypo'] if 'shift_hypo' in kwargs else False
    param = dict(imtls=imtls,
                 truncation_level=truncation_level,
                 reqv=reqv,
                 cluster=grp.cluster,
                 shift_hypo=shift_hypo,
                 investigation_time=kwargs.get('investigation_time', span))
    pmap = ProbabilityMap(imtls.size, 1)
    # Processing groups with homogeneous tectonic region
    mon = Monitor()
    sitecol = getattr(srcfilter, 'sitecol', srcfilter)
    for group in groups:
        trt = group.trt
        if sitecol is not srcfilter:
            param['maximum_distance'] = srcfilter.integration_distance(trt)
        cmaker = ContextMaker(trt, [gsim_by_trt[trt]], param, mon)
        if group.atomic:  # do not split
            it = [classical(group, sitecol, cmaker)]
        else:  # split the group and apply `classical` in parallel
            it = apply(classical, (group.sources, sitecol, cmaker),
                       weight=operator.attrgetter('weight'))
        for dic in it:
            pmap |= dic['pmap']
    return pmap.convert(imtls, len(sitecol.complete))