Exemplo n.º 1
0
def filter_and_split(src, sourceprocessor):
    """
    Filter and split the source by using the source processor.
    Also, sets the sub sources `.weight` attribute.

    :param src: a hazardlib source object
    :param sourceprocessor: a SourceFilterSplitter object
    :returns: a named tuple of type SourceInfo
    """
    if sourceprocessor.sitecol:  # filter
        info = sourceprocessor.filter(src)
        if not info.sources:
            return info  # filtered away
        filter_time = info.filter_time
    else:  # only split
        filter_time = 0
    t1 = time.time()
    out = []
    weight_time = 0
    weight = 0
    for ss in sourceconverter.split_source(src, sourceprocessor.asd):
        if sourceprocessor.weight:
            t = time.time()
            ss.weight = get_weight(ss)
            weight_time += time.time() - t
            weight += ss.weight
        out.append(ss)
    src.weight = weight
    split_time = time.time() - t1 - weight_time
    return SourceInfo(src.trt_model_id, src.source_id, src.__class__.__name__,
                      weight, out, filter_time, weight_time, split_time)
Exemplo n.º 2
0
def filter_and_split(src, sourceprocessor):
    """
    Filter and split the source by using the source processor.
    Also, sets the sub sources `.weight` attribute.

    :param src: a hazardlib source object
    :param sourceprocessor: a SourceFilterSplitter object
    :returns: a named tuple of type SourceInfo
    """
    if sourceprocessor.sitecol:  # filter
        info = sourceprocessor.filter(src)
        if not info.sources:
            return info  # filtered away
        filter_time = info.filter_time
    else:  # only split
        filter_time = 0
    t1 = time.time()
    out = []
    weight_time = 0
    weight = 0
    for ss in sourceconverter.split_source(src):
        if sourceprocessor.weight:
            t = time.time()
            ss.num_ruptures = ss.count_ruptures()
            weight += ss.weight
            weight_time += time.time() - t
        out.append(ss)
    split_time = time.time() - t1 - weight_time
    return SourceInfo(src.trt_model_id, src.source_id, src.__class__.__name__,
                      weight, out, filter_time, weight_time, split_time)
Exemplo n.º 3
0
    def get_sources(self, kind, sitecol):
        """
        :param kind: a string 'light', 'heavy' or 'all'
        :param sitecol: a SiteCollection instance
        :returns: the sources of the given kind affecting the given sitecol
        """
        filter_mon = self.monitor('filtering sources')
        split_mon = self.monitor('splitting sources')
        for src in self.csm.get_sources(kind):
            filter_time = split_time = 0
            if self.filter_sources:
                try:
                    max_dist = self.maximum_distance[src.tectonic_region_type]
                except KeyError:
                    max_dist = self.maximum_distance['default']
                with filter_mon:
                    try:
                        sites = src.filter_sites_by_distance_to_source(
                            max_dist, sitecol)
                    except:
                        etype, err, tb = sys.exc_info()
                        msg = 'An error occurred with source id=%s: %s'
                        msg %= (src.source_id, unicode(err))
                        raise_(etype, msg, tb)
                filter_time = filter_mon.dt
                if sites is None:
                    continue
            if kind == 'heavy':
                if (src.trt_model_id, src.id) not in self.split_map:
                    logging.info('splitting %s of weight %s',
                                 src, src.weight)
                    with split_mon:
                        sources = list(sourceconverter.split_source(src))
                        self.split_map[src.trt_model_id, src.id] = sources
                    split_time = split_mon.dt
                    self.set_serial(src, sources)
                for ss in self.split_map[src.trt_model_id, src.id]:
                    ss.id = src.id
                    yield ss
            else:
                self.set_serial(src)
                yield src
            split_sources = self.split_map.get(
                (src.trt_model_id, src.id), [src])
            info = SourceInfo(src.trt_model_id, src.source_id,
                              src.__class__.__name__,
                              src.weight, len(split_sources),
                              filter_time, split_time, 0)
            key = (src.trt_model_id, src.source_id)
            if key in self.infos:
                self.infos[key] += info
            else:
                self.infos[key] = info

        filter_mon.flush()
        split_mon.flush()
Exemplo n.º 4
0
    def get_sources(self, kind, sitecol):
        """
        :param kind: a string 'light', 'heavy' or 'all'
        :param sitecol: a SiteCollection instance
        :returns: the sources of the given kind affecting the given sitecol
        """
        filter_mon = self.monitor('filtering sources')
        split_mon = self.monitor('splitting sources')
        for src in self.csm.get_sources(kind):
            filter_time = split_time = 0
            if self.filter_sources:
                try:
                    max_dist = self.maximum_distance[src.tectonic_region_type]
                except KeyError:
                    max_dist = self.maximum_distance['default']
                with filter_mon:
                    try:
                        sites = src.filter_sites_by_distance_to_source(
                            max_dist, sitecol)
                    except:
                        etype, err, tb = sys.exc_info()
                        msg = 'An error occurred with source id=%s: %s'
                        msg %= (src.source_id, unicode(err))
                        raise_(etype, msg, tb)
                filter_time = filter_mon.dt
                if sites is None:
                    continue
            if kind == 'heavy':
                if (src.trt_model_id, src.id) not in self.split_map:
                    logging.info('splitting %s of weight %s', src, src.weight)
                    with split_mon:
                        sources = list(sourceconverter.split_source(src))
                        self.split_map[src.trt_model_id, src.id] = sources
                    split_time = split_mon.dt
                    self.set_serial(src, sources)
                for ss in self.split_map[src.trt_model_id, src.id]:
                    ss.id = src.id
                    yield ss
            else:
                self.set_serial(src)
                yield src
            split_sources = self.split_map.get((src.trt_model_id, src.id),
                                               [src])
            info = SourceInfo(src.trt_model_id, src.source_id,
                              src.__class__.__name__, src.weight,
                              len(split_sources), filter_time, split_time, 0)
            key = (src.trt_model_id, src.source_id)
            if key in self.infos:
                self.infos[key] += info
            else:
                self.infos[key] = info

        filter_mon.flush()
        split_mon.flush()
Exemplo n.º 5
0
    def get_sources(self, kind, tile):
        """
        :param kind: a string 'light', 'heavy' or 'all'
        :param tile: a :class:`openquake.hazardlib.site.Tile` instance
        :returns: the sources of the given kind affecting the given tile
        """
        filter_mon = self.monitor('filtering sources')
        split_mon = self.monitor('splitting sources')
        for src in self.csm.get_sources(kind):
            filter_time = split_time = 0
            if self.filter_sources:
                with filter_mon:
                    try:
                        if src not in tile:
                            continue
                    except:
                        etype, err, tb = sys.exc_info()
                        msg = 'An error occurred with source id=%s: %s'
                        msg %= (src.source_id, err)
                        raise_(etype, msg, tb)
                filter_time = filter_mon.dt
            if kind == 'heavy':
                if (src.trt_model_id, src.id) not in self.split_map:
                    logging.info('splitting %s of weight %s',
                                 src, src.weight)
                    with split_mon:
                        sources = list(sourceconverter.split_source(src))
                        self.split_map[src.trt_model_id, src.id] = sources
                    split_time = split_mon.dt
                    self.set_serial(src, sources)
                for ss in self.split_map[src.trt_model_id, src.id]:
                    ss.id = src.id
                    yield ss
            else:
                self.set_serial(src)
                yield src
            split_sources = self.split_map.get(
                (src.trt_model_id, src.id), [src])
            info = SourceInfo(src.trt_model_id, src.source_id,
                              src.__class__.__name__,
                              src.weight, len(split_sources),
                              filter_time, split_time, 0)
            key = (src.trt_model_id, src.source_id)
            if key in self.infos:
                self.infos[key] += info
            else:
                self.infos[key] = info

        filter_mon.flush()
        split_mon.flush()
Exemplo n.º 6
0
    def get_sources(self, kind, tile):
        """
        :param kind: a string 'light', 'heavy' or 'all'
        :param tile: a :class:`openquake.hazardlib.site.Tile` instance
        :returns: the sources of the given kind affecting the given tile
        """
        filter_mon = self.monitor('filtering sources')
        split_mon = self.monitor('splitting sources')
        for src in self.csm.get_sources(kind):
            filter_time = split_time = 0
            if self.filter_sources:
                with filter_mon:
                    try:
                        if src not in tile:
                            continue
                    except:
                        etype, err, tb = sys.exc_info()
                        msg = 'An error occurred with source id=%s: %s'
                        msg %= (src.source_id, err)
                        raise_(etype, msg, tb)
                filter_time = filter_mon.dt
            if kind == 'heavy':
                if (src.trt_model_id, src.id) not in self.split_map:
                    logging.info('splitting %s of weight %s', src, src.weight)
                    with split_mon:
                        sources = list(sourceconverter.split_source(src))
                        self.split_map[src.trt_model_id, src.id] = sources
                    split_time = split_mon.dt
                    self.set_serial(src, sources)
                for ss in self.split_map[src.trt_model_id, src.id]:
                    ss.id = src.id
                    yield ss
            else:
                self.set_serial(src)
                yield src
            split_sources = self.split_map.get((src.trt_model_id, src.id),
                                               [src])
            info = SourceInfo(src.trt_model_id, src.source_id,
                              src.__class__.__name__, src.weight,
                              len(split_sources), filter_time, split_time, 0)
            key = (src.trt_model_id, src.source_id)
            if key in self.infos:
                self.infos[key] += info
            else:
                self.infos[key] = info

        filter_mon.flush()
        split_mon.flush()
Exemplo n.º 7
0
def split_filter_source(src, sites, ss_filter, random_seed):
    """
    :param src: an heavy source
    :param sites: sites affected by the source
    :param ss_filter: a SourceSitesFilter instance
    :random_seed: used only for event based calculations
    :returns: a list of split sources
    """
    split_sources = []
    start = 0
    for split in sourceconverter.split_source(src):
        if random_seed:
            nr = split.num_ruptures
            split.serial = src.serial[start:start + nr]
            start += nr
        if ss_filter.affected(split) is not None:
            split_sources.append(split)
    return split_sources
Exemplo n.º 8
0
def split_sources(sources, maxweight, splitmap):
    """
    Split the sources with weight greater than maxweight. `splitmap`
    is a cache to avoid splitting twice the same source.
    """
    ss = []
    for src in sources:
        if src.weight > maxweight:
            key = (src.trt_model_id, src.source_id)
            try:
                srcs = splitmap[key]
            except KeyError:
                logging.info('Splitting %s of weight %d > %d',
                             src, src.weight, maxweight)
                srcs = splitmap[key] = list(sourceconverter.split_source(src))
            ss.extend(srcs)
        else:
            ss.append(src)
    return ss
Exemplo n.º 9
0
    def full_disaggregation(self):
        """
        Run the disaggregation phase after hazard curve finalization.
        """
        oq = self.oqparam
        tl = self.oqparam.truncation_level
        bb_dict = self.datastore["bb_dict"]
        sitecol = self.sitecol
        mag_bin_width = self.oqparam.mag_bin_width
        eps_edges = numpy.linspace(-tl, tl, self.oqparam.num_epsilon_bins + 1)
        logging.info("%d epsilon bins from %s to %s", len(eps_edges) - 1, min(eps_edges), max(eps_edges))

        self.bin_edges = {}
        curves_dict = {sid: self.get_curves(sid) for sid in sitecol.sids}
        all_args = []
        num_trts = sum(len(sm.src_groups) for sm in self.csm.source_models)
        nblocks = math.ceil(oq.concurrent_tasks / num_trts)
        for smodel in self.csm.source_models:
            sm_id = smodel.ordinal
            trt_names = tuple(mod.trt for mod in smodel.src_groups)
            max_mag = max(mod.max_mag for mod in smodel.src_groups)
            min_mag = min(mod.min_mag for mod in smodel.src_groups)
            mag_edges = mag_bin_width * numpy.arange(
                int(numpy.floor(min_mag / mag_bin_width)), int(numpy.ceil(max_mag / mag_bin_width) + 1)
            )
            logging.info("%d mag bins from %s to %s", len(mag_edges) - 1, min_mag, max_mag)
            for src_group in smodel.src_groups:
                if src_group.id not in self.rlzs_assoc.gsims_by_grp_id:
                    continue  # the group has been filtered away
                for sid, site in zip(sitecol.sids, sitecol):
                    curves = curves_dict[sid]
                    if not curves:
                        continue  # skip zero-valued hazard curves
                    bb = bb_dict[sm_id, sid]
                    if not bb:
                        logging.info("location %s was too far, skipping disaggregation", site.location)
                        continue

                    dist_edges, lon_edges, lat_edges = bb.bins_edges(oq.distance_bin_width, oq.coordinate_bin_width)
                    logging.info("%d dist bins from %s to %s", len(dist_edges) - 1, min(dist_edges), max(dist_edges))
                    logging.info("%d lon bins from %s to %s", len(lon_edges) - 1, bb.west, bb.east)
                    logging.info("%d lat bins from %s to %s", len(lon_edges) - 1, bb.south, bb.north)

                    self.bin_edges[sm_id, sid] = (mag_edges, dist_edges, lon_edges, lat_edges, eps_edges)

                bin_edges = {}
                for sid, site in zip(sitecol.sids, sitecol):
                    if (sm_id, sid) in self.bin_edges:
                        bin_edges[sid] = self.bin_edges[sm_id, sid]

                ss_filter = SourceSitesFilter(oq.maximum_distance)
                split_sources = []
                for src in src_group:
                    for split, _sites in ss_filter(sourceconverter.split_source(src), sitecol):
                        split_sources.append(split)
                for srcs in split_in_blocks(split_sources, nblocks):
                    all_args.append(
                        (
                            sitecol,
                            srcs,
                            src_group.id,
                            self.rlzs_assoc,
                            trt_names,
                            curves_dict,
                            bin_edges,
                            oq,
                            self.monitor,
                        )
                    )

        results = parallel.starmap(compute_disagg, all_args).reduce(self.agg_result)
        self.save_disagg_results(results)