def filter_and_split(src, sourceprocessor): """ Filter and split the source by using the source processor. Also, sets the sub sources `.weight` attribute. :param src: a hazardlib source object :param sourceprocessor: a SourceFilterSplitter object :returns: a named tuple of type SourceInfo """ if sourceprocessor.sitecol: # filter info = sourceprocessor.filter(src) if not info.sources: return info # filtered away filter_time = info.filter_time else: # only split filter_time = 0 t1 = time.time() out = [] weight_time = 0 weight = 0 for ss in sourceconverter.split_source(src, sourceprocessor.asd): if sourceprocessor.weight: t = time.time() ss.weight = get_weight(ss) weight_time += time.time() - t weight += ss.weight out.append(ss) src.weight = weight split_time = time.time() - t1 - weight_time return SourceInfo(src.trt_model_id, src.source_id, src.__class__.__name__, weight, out, filter_time, weight_time, split_time)
def filter_and_split(src, sourceprocessor): """ Filter and split the source by using the source processor. Also, sets the sub sources `.weight` attribute. :param src: a hazardlib source object :param sourceprocessor: a SourceFilterSplitter object :returns: a named tuple of type SourceInfo """ if sourceprocessor.sitecol: # filter info = sourceprocessor.filter(src) if not info.sources: return info # filtered away filter_time = info.filter_time else: # only split filter_time = 0 t1 = time.time() out = [] weight_time = 0 weight = 0 for ss in sourceconverter.split_source(src): if sourceprocessor.weight: t = time.time() ss.num_ruptures = ss.count_ruptures() weight += ss.weight weight_time += time.time() - t out.append(ss) split_time = time.time() - t1 - weight_time return SourceInfo(src.trt_model_id, src.source_id, src.__class__.__name__, weight, out, filter_time, weight_time, split_time)
def get_sources(self, kind, sitecol): """ :param kind: a string 'light', 'heavy' or 'all' :param sitecol: a SiteCollection instance :returns: the sources of the given kind affecting the given sitecol """ filter_mon = self.monitor('filtering sources') split_mon = self.monitor('splitting sources') for src in self.csm.get_sources(kind): filter_time = split_time = 0 if self.filter_sources: try: max_dist = self.maximum_distance[src.tectonic_region_type] except KeyError: max_dist = self.maximum_distance['default'] with filter_mon: try: sites = src.filter_sites_by_distance_to_source( max_dist, sitecol) except: etype, err, tb = sys.exc_info() msg = 'An error occurred with source id=%s: %s' msg %= (src.source_id, unicode(err)) raise_(etype, msg, tb) filter_time = filter_mon.dt if sites is None: continue if kind == 'heavy': if (src.trt_model_id, src.id) not in self.split_map: logging.info('splitting %s of weight %s', src, src.weight) with split_mon: sources = list(sourceconverter.split_source(src)) self.split_map[src.trt_model_id, src.id] = sources split_time = split_mon.dt self.set_serial(src, sources) for ss in self.split_map[src.trt_model_id, src.id]: ss.id = src.id yield ss else: self.set_serial(src) yield src split_sources = self.split_map.get( (src.trt_model_id, src.id), [src]) info = SourceInfo(src.trt_model_id, src.source_id, src.__class__.__name__, src.weight, len(split_sources), filter_time, split_time, 0) key = (src.trt_model_id, src.source_id) if key in self.infos: self.infos[key] += info else: self.infos[key] = info filter_mon.flush() split_mon.flush()
def get_sources(self, kind, sitecol): """ :param kind: a string 'light', 'heavy' or 'all' :param sitecol: a SiteCollection instance :returns: the sources of the given kind affecting the given sitecol """ filter_mon = self.monitor('filtering sources') split_mon = self.monitor('splitting sources') for src in self.csm.get_sources(kind): filter_time = split_time = 0 if self.filter_sources: try: max_dist = self.maximum_distance[src.tectonic_region_type] except KeyError: max_dist = self.maximum_distance['default'] with filter_mon: try: sites = src.filter_sites_by_distance_to_source( max_dist, sitecol) except: etype, err, tb = sys.exc_info() msg = 'An error occurred with source id=%s: %s' msg %= (src.source_id, unicode(err)) raise_(etype, msg, tb) filter_time = filter_mon.dt if sites is None: continue if kind == 'heavy': if (src.trt_model_id, src.id) not in self.split_map: logging.info('splitting %s of weight %s', src, src.weight) with split_mon: sources = list(sourceconverter.split_source(src)) self.split_map[src.trt_model_id, src.id] = sources split_time = split_mon.dt self.set_serial(src, sources) for ss in self.split_map[src.trt_model_id, src.id]: ss.id = src.id yield ss else: self.set_serial(src) yield src split_sources = self.split_map.get((src.trt_model_id, src.id), [src]) info = SourceInfo(src.trt_model_id, src.source_id, src.__class__.__name__, src.weight, len(split_sources), filter_time, split_time, 0) key = (src.trt_model_id, src.source_id) if key in self.infos: self.infos[key] += info else: self.infos[key] = info filter_mon.flush() split_mon.flush()
def get_sources(self, kind, tile): """ :param kind: a string 'light', 'heavy' or 'all' :param tile: a :class:`openquake.hazardlib.site.Tile` instance :returns: the sources of the given kind affecting the given tile """ filter_mon = self.monitor('filtering sources') split_mon = self.monitor('splitting sources') for src in self.csm.get_sources(kind): filter_time = split_time = 0 if self.filter_sources: with filter_mon: try: if src not in tile: continue except: etype, err, tb = sys.exc_info() msg = 'An error occurred with source id=%s: %s' msg %= (src.source_id, err) raise_(etype, msg, tb) filter_time = filter_mon.dt if kind == 'heavy': if (src.trt_model_id, src.id) not in self.split_map: logging.info('splitting %s of weight %s', src, src.weight) with split_mon: sources = list(sourceconverter.split_source(src)) self.split_map[src.trt_model_id, src.id] = sources split_time = split_mon.dt self.set_serial(src, sources) for ss in self.split_map[src.trt_model_id, src.id]: ss.id = src.id yield ss else: self.set_serial(src) yield src split_sources = self.split_map.get( (src.trt_model_id, src.id), [src]) info = SourceInfo(src.trt_model_id, src.source_id, src.__class__.__name__, src.weight, len(split_sources), filter_time, split_time, 0) key = (src.trt_model_id, src.source_id) if key in self.infos: self.infos[key] += info else: self.infos[key] = info filter_mon.flush() split_mon.flush()
def get_sources(self, kind, tile): """ :param kind: a string 'light', 'heavy' or 'all' :param tile: a :class:`openquake.hazardlib.site.Tile` instance :returns: the sources of the given kind affecting the given tile """ filter_mon = self.monitor('filtering sources') split_mon = self.monitor('splitting sources') for src in self.csm.get_sources(kind): filter_time = split_time = 0 if self.filter_sources: with filter_mon: try: if src not in tile: continue except: etype, err, tb = sys.exc_info() msg = 'An error occurred with source id=%s: %s' msg %= (src.source_id, err) raise_(etype, msg, tb) filter_time = filter_mon.dt if kind == 'heavy': if (src.trt_model_id, src.id) not in self.split_map: logging.info('splitting %s of weight %s', src, src.weight) with split_mon: sources = list(sourceconverter.split_source(src)) self.split_map[src.trt_model_id, src.id] = sources split_time = split_mon.dt self.set_serial(src, sources) for ss in self.split_map[src.trt_model_id, src.id]: ss.id = src.id yield ss else: self.set_serial(src) yield src split_sources = self.split_map.get((src.trt_model_id, src.id), [src]) info = SourceInfo(src.trt_model_id, src.source_id, src.__class__.__name__, src.weight, len(split_sources), filter_time, split_time, 0) key = (src.trt_model_id, src.source_id) if key in self.infos: self.infos[key] += info else: self.infos[key] = info filter_mon.flush() split_mon.flush()
def split_filter_source(src, sites, ss_filter, random_seed): """ :param src: an heavy source :param sites: sites affected by the source :param ss_filter: a SourceSitesFilter instance :random_seed: used only for event based calculations :returns: a list of split sources """ split_sources = [] start = 0 for split in sourceconverter.split_source(src): if random_seed: nr = split.num_ruptures split.serial = src.serial[start:start + nr] start += nr if ss_filter.affected(split) is not None: split_sources.append(split) return split_sources
def split_sources(sources, maxweight, splitmap): """ Split the sources with weight greater than maxweight. `splitmap` is a cache to avoid splitting twice the same source. """ ss = [] for src in sources: if src.weight > maxweight: key = (src.trt_model_id, src.source_id) try: srcs = splitmap[key] except KeyError: logging.info('Splitting %s of weight %d > %d', src, src.weight, maxweight) srcs = splitmap[key] = list(sourceconverter.split_source(src)) ss.extend(srcs) else: ss.append(src) return ss
def full_disaggregation(self): """ Run the disaggregation phase after hazard curve finalization. """ oq = self.oqparam tl = self.oqparam.truncation_level bb_dict = self.datastore["bb_dict"] sitecol = self.sitecol mag_bin_width = self.oqparam.mag_bin_width eps_edges = numpy.linspace(-tl, tl, self.oqparam.num_epsilon_bins + 1) logging.info("%d epsilon bins from %s to %s", len(eps_edges) - 1, min(eps_edges), max(eps_edges)) self.bin_edges = {} curves_dict = {sid: self.get_curves(sid) for sid in sitecol.sids} all_args = [] num_trts = sum(len(sm.src_groups) for sm in self.csm.source_models) nblocks = math.ceil(oq.concurrent_tasks / num_trts) for smodel in self.csm.source_models: sm_id = smodel.ordinal trt_names = tuple(mod.trt for mod in smodel.src_groups) max_mag = max(mod.max_mag for mod in smodel.src_groups) min_mag = min(mod.min_mag for mod in smodel.src_groups) mag_edges = mag_bin_width * numpy.arange( int(numpy.floor(min_mag / mag_bin_width)), int(numpy.ceil(max_mag / mag_bin_width) + 1) ) logging.info("%d mag bins from %s to %s", len(mag_edges) - 1, min_mag, max_mag) for src_group in smodel.src_groups: if src_group.id not in self.rlzs_assoc.gsims_by_grp_id: continue # the group has been filtered away for sid, site in zip(sitecol.sids, sitecol): curves = curves_dict[sid] if not curves: continue # skip zero-valued hazard curves bb = bb_dict[sm_id, sid] if not bb: logging.info("location %s was too far, skipping disaggregation", site.location) continue dist_edges, lon_edges, lat_edges = bb.bins_edges(oq.distance_bin_width, oq.coordinate_bin_width) logging.info("%d dist bins from %s to %s", len(dist_edges) - 1, min(dist_edges), max(dist_edges)) logging.info("%d lon bins from %s to %s", len(lon_edges) - 1, bb.west, bb.east) logging.info("%d lat bins from %s to %s", len(lon_edges) - 1, bb.south, bb.north) self.bin_edges[sm_id, sid] = (mag_edges, dist_edges, lon_edges, lat_edges, eps_edges) bin_edges = {} for sid, site in zip(sitecol.sids, sitecol): if (sm_id, sid) in self.bin_edges: bin_edges[sid] = self.bin_edges[sm_id, sid] ss_filter = SourceSitesFilter(oq.maximum_distance) split_sources = [] for src in src_group: for split, _sites in ss_filter(sourceconverter.split_source(src), sitecol): split_sources.append(split) for srcs in split_in_blocks(split_sources, nblocks): all_args.append( ( sitecol, srcs, src_group.id, self.rlzs_assoc, trt_names, curves_dict, bin_edges, oq, self.monitor, ) ) results = parallel.starmap(compute_disagg, all_args).reduce(self.agg_result) self.save_disagg_results(results)