def classical_split_filter(srcs, srcfilter, gsims, params, monitor): """ Split the given sources, filter the subsources and the compute the PoEs. Yield back subtasks if the split sources contain more than maxweight ruptures. """ # first check if we are sampling the sources ss = int(os.environ.get('OQ_SAMPLE_SOURCES', 0)) if ss: splits, stime = split_sources(srcs) srcs = random_filtered_sources(splits, srcfilter, ss) yield classical(srcs, srcfilter, gsims, params, monitor) return # NB: splitting all the sources improves the distribution significantly, # compared to splitting only the big source sources = [] with monitor("filtering/splitting sources"): for src, _sites in srcfilter(srcs): splits, _stime = split_sources([src]) sources.extend(srcfilter.filter(splits)) if sources: sources.sort(key=weight) totsites = len(srcfilter.sitecol) mw = 1000 if totsites <= params['max_sites_disagg'] else 50000 mweight = max(mw, sum(src.weight for src in sources) / params['task_multiplier']) blocks = list(block_splitter(sources, mweight, weight)) for block in blocks[:-1]: yield classical, block, srcfilter, gsims, params yield classical(blocks[-1], srcfilter, gsims, params, monitor)
def classical_split_filter(srcs, srcfilter, gsims, params, monitor): """ Split the given sources, filter the subsources and the compute the PoEs. Yield back subtasks if the split sources contain more than maxweight ruptures. """ # first check if we are sampling the sources ss = int(os.environ.get('OQ_SAMPLE_SOURCES', 0)) if ss: splits, stime = split_sources(srcs) srcs = readinput.random_filtered_sources(splits, srcfilter, ss) yield classical(srcs, srcfilter, gsims, params, monitor) return sources = [] with monitor("filtering/splitting sources"): for src, _sites in srcfilter(srcs): if src.num_ruptures >= params['maxweight']: splits, stime = split_sources([src]) sources.extend(srcfilter.filter(splits)) else: sources.append(src) blocks = list(block_splitter(sources, params['maxweight'], operator.attrgetter('num_ruptures'))) if blocks: # yield the first blocks (if any) and compute the last block in core # NB: the last block is usually the smallest one for block in blocks[:-1]: yield classical, block, srcfilter, gsims, params yield classical(blocks[-1], srcfilter, gsims, params, monitor)
def full_enum(self): # compute the mean curve with full enumeration srcs = [] weights = [] grp_id = 0 for weight, branches in self.bs0.enumerate_paths(): path = tuple(br.branch_id for br in branches) bset_values = self.bs0.get_bset_values(path) # first path: [(<b01 b02>, (4.6, 1.1)), (<b11 b12>, 7.0)] sg = lt.apply_uncertainties(bset_values, self.sg) for src in sg: src.grp_id = grp_id grp_id += 1 srcs.extend(sg) weights.append(weight) for i, src in enumerate(srcs): src.id = i res = classical( srcs, self.srcfilter, self.gsims, dict(imtls=self.imtls, truncation_level2=2, collapse_ctxs=True)) pmap = res['pmap'] effrups = sum(nr for nr, ns, dt in res['calc_times'].values()) curves = [pmap[grp_id].array[0, :, 0] for grp_id in sorted(pmap)] mean = numpy.average(curves, axis=0, weights=weights) return mean, srcs, effrups, weights
def full_enum(self): # compute the mean curve with full enumeration srcs = [] weights = [] grp_id = trt_smr = 0 for weight, branches in self.bs0.enumerate_paths(): path = tuple(br.branch_id for br in branches) bset_values = self.bs0.get_bset_values(path) # first path: [(<b01 b02>, (4.6, 1.1)), (<b11 b12>, 7.0)] sg = lt.apply_uncertainties(bset_values, self.sg) for src in sg: src.grp_id = grp_id src.trt_smr = trt_smr grp_id += 1 trt_smr += 1 srcs.extend(sg) weights.append(weight) for i, src in enumerate(srcs): src.id = i N = len(self.sitecol.complete) time_span = srcs[0].temporal_occurrence_model.time_span idist = calc.filters.IntegrationDistance.new('200') params = dict(imtls=self.imtls, truncation_level2=2, collapse_level=2, investigation_time=time_span, maximum_distance=idist('default')) cmaker = contexts.ContextMaker(srcs[0].tectonic_region_type, self.gsims, params) res = classical(srcs, self.sitecol, cmaker) pmap = res['pmap'] effrups = sum(nr for nr, ns, dt in res['calc_times'].values()) curve = pmap.array(N)[0, :, 0] return curve, srcs, effrups, weights
def test_mutually_exclusive_ruptures(self): # Test the calculation of hazard curves using mutually exclusive # ruptures for a single source gsim_by_trt = [SadighEtAl1997()] rupture = _create_rupture(10., 6.) data = [(rupture, PMF([(0.7, 0), (0.3, 1)])), (rupture, PMF([(0.6, 0), (0.4, 1)]))] data[0][0].weight = 0.5 data[1][0].weight = 0.5 src = NonParametricSeismicSource('0', 'test', "Active Shallow Crust", data) src.id = 0 src.grp_id = 0 src.trt_smr = 0 src.mutex_weight = 1 group = SourceGroup(src.tectonic_region_type, [src], 'test', 'mutex', 'mutex') param = dict(imtls=self.imtls, src_interdep=group.src_interdep, rup_interdep=group.rup_interdep, grp_probability=group.grp_probability) cmaker = ContextMaker(src.tectonic_region_type, gsim_by_trt, param) crv = classical(group, self.sites, cmaker)['pmap'][0] npt.assert_almost_equal(numpy.array([0.35000, 0.32497, 0.10398]), crv.array[:, 0], decimal=4)
def classical_split_filter(srcs, srcfilter, gsims, params, monitor): """ Split the given sources, filter the subsources and the compute the PoEs. Yield back subtasks if the split sources contain more than maxweight ruptures. """ # first check if we are sampling the sources ss = int(os.environ.get('OQ_SAMPLE_SOURCES', 0)) if ss: splits, stime = split_sources(srcs) srcs = random_filtered_sources(splits, srcfilter, ss) yield classical(srcs, srcfilter, gsims, params, monitor) return # NB: splitting all the sources improves the distribution significantly, # compared to splitting only the big source sources = [] with monitor("filtering/splitting sources"): for src, _sites in srcfilter(srcs): splits, _stime = split_sources([src]) sources.extend(srcfilter.filter(splits)) if sources: yield from parallel.split_task(classical, sources, srcfilter, gsims, params, monitor, duration=params['task_duration'])
def test_mutually_exclusive_ruptures(self): # Test the calculation of hazard curves using mutually exclusive # ruptures for a single source gsim_by_trt = [SadighEtAl1997()] rupture = _create_rupture(10., 6.) data = [(rupture, PMF([(0.7, 0), (0.3, 1)])), (rupture, PMF([(0.6, 0), (0.4, 1)]))] print(data[0][0]) data[0][0].weight = 0.5 data[1][0].weight = 0.5 print(data[0][0].weight) src = NonParametricSeismicSource('0', 'test', TRT.ACTIVE_SHALLOW_CRUST, data) src.src_group_id = 0 src.mutex_weight = 1 group = SourceGroup(src.tectonic_region_type, [src], 'test', 'mutex', 'mutex') param = dict(imtls=self.imtls, filter_distance='rjb', src_interdep=group.src_interdep, rup_interdep=group.rup_interdep, grp_probability=group.grp_probability) crv = classical(group, self.sites, gsim_by_trt, param)['pmap'][0] npt.assert_almost_equal(numpy.array([0.35000, 0.32497, 0.10398]), crv[0].array[:, 0], decimal=4)
def full_enum(self): # compute the mean curve with full enumeration srcs = [] weights = [] grp_id = et_id = 0 for weight, branches in self.bs0.enumerate_paths(): path = tuple(br.branch_id for br in branches) bset_values = self.bs0.get_bset_values(path) # first path: [(<b01 b02>, (4.6, 1.1)), (<b11 b12>, 7.0)] sg = lt.apply_uncertainties(bset_values, self.sg) for src in sg: src.grp_id = grp_id src.et_id = et_id grp_id += 1 et_id += 1 srcs.extend(sg) weights.append(weight) for i, src in enumerate(srcs): src.id = i N = len(self.srcfilter.sitecol.complete) res = classical( srcs, self.srcfilter, self.gsims, dict(imtls=self.imtls, truncation_level2=2, collapse_level=2)) pmap = res['pmap'] effrups = sum(nr for nr, ns, dt in res['calc_times'].values()) curve = pmap.array(N)[0, :, 0] return curve, srcs, effrups, weights
def classical_split_filter(srcs, gsims, params, monitor): """ Split the given sources, filter the subsources and the compute the PoEs. Yield back subtasks if the split sources contain more than maxweight ruptures. """ srcfilter = monitor.read('srcfilter') # first check if we are sampling the sources ss = int(os.environ.get('OQ_SAMPLE_SOURCES', 0)) if ss: splits, stime = split_sources(srcs) srcs = random_filtered_sources(splits, srcfilter, ss) yield classical(srcs, srcfilter, gsims, params, monitor) return # NB: splitting all the sources improves the distribution significantly, # compared to splitting only the big sources with monitor("splitting/filtering sources"): splits, _stime = split_sources(srcs) sources = list(srcfilter.filter(splits)) if not sources: yield {'pmap': {}} return maxw = params['max_weight'] N = len(srcfilter.sitecol.complete) def weight(src): n = 10 * numpy.sqrt(len(src.indices) / N) return src.weight * params['rescale_weight'] * n blocks = list(block_splitter(sources, maxw, weight)) subtasks = len(blocks) - 1 for block in blocks[:-1]: yield classical_, block, gsims, params if monitor.calc_id and subtasks: msg = 'produced %d subtask(s) with mean weight %d' % ( subtasks, numpy.mean([b.weight for b in blocks[:-1]])) try: logs.dbcmd('log', monitor.calc_id, datetime.utcnow(), 'DEBUG', 'classical_split_filter#%d' % monitor.task_no, msg) except Exception: # a foreign key error in case of `oq run` is expected print(msg) yield classical(blocks[-1], srcfilter, gsims, params, monitor)
def classical1(srcs, gsims, params, slc, monitor=None): """ Read the SourceFilter, get the current slice of it (if tiling is enabled) and then call the classical calculator in hazardlib """ if monitor is None: # fix mispassed parameters (for disagg_by_src) monitor = slc slc = slice(None) srcfilter = monitor.read('srcfilter')[slc] return classical(srcs, srcfilter, gsims, params, monitor)
def classical_split_filter(srcs, srcfilter, gsims, params, monitor): """ Split the given sources, filter the subsources and the compute the PoEs. Yield back subtasks if the split sources contain more than maxweight ruptures. """ # first check if we are sampling the sources ss = int(os.environ.get('OQ_SAMPLE_SOURCES', 0)) if ss: splits, stime = split_sources(srcs) srcs = random_filtered_sources(splits, srcfilter, ss) yield classical(srcs, srcfilter, gsims, params, monitor) return # NB: splitting all the sources improves the distribution significantly, # compared to splitting only the big sources with monitor("splitting/filtering sources"): splits, _stime = split_sources(srcs) sources = list(srcfilter.filter(splits)) if not sources: yield {'pmap': {}} return maxw = min(sum(src.weight for src in sources)/5, params['max_weight']) if maxw < MINWEIGHT*5: # task too small to be resubmitted yield classical(sources, srcfilter, gsims, params, monitor) return blocks = list(block_splitter(sources, maxw, weight)) subtasks = len(blocks) - 1 for block in blocks[:-1]: yield classical, block, srcfilter, gsims, params if monitor.calc_id and subtasks: msg = 'produced %d subtask(s) with max weight=%d' % ( subtasks, max(b.weight for b in blocks)) try: logs.dbcmd('log', monitor.calc_id, datetime.utcnow(), 'DEBUG', 'classical_split_filter#%d' % monitor.task_no, msg) except Exception: # a foreign key error in case of `oq run` is expected print(msg) yield classical(blocks[-1], srcfilter, gsims, params, monitor)
def test_mutually_exclusive_ruptures(self): # Test the calculation of hazard curves using mutually exclusive # ruptures for a single source gsim_by_trt = [SadighEtAl1997()] rupture = _create_rupture(10., 6.) data = [(rupture, PMF([(0.7, 0), (0.3, 1)])), (rupture, PMF([(0.6, 0), (0.4, 1)]))] src = NonParametricSeismicSource('0', 'test', TRT.ACTIVE_SHALLOW_CRUST, data) src.src_group_id = [0] group = SourceGroup( src.tectonic_region_type, [src], 'test', 'mutex', 'mutex') param = dict(imtls=self.imtls) crv = classical(group, self.sites, gsim_by_trt, param)[0] npt.assert_almost_equal(numpy.array([0.35000, 0.32497, 0.10398]), crv[0].array[:, 0], decimal=4)
def _run_calculator(self, gdem, imtls, sites): """ Executes the classical calculator """ param = { "imtls": DictArray(imtls), "truncation_level": 3., "filter_distance": "rrup" } curves = classical(self.source, SourceFilter(sites, valid.floatdict("200")), [gdem], param) pmap = ProbabilityMap(param["imtls"].array, 1) for res in [curves]: for grp_id in res: pmap |= res[grp_id] return pmap.convert(param["imtls"], len(sites))
def classical_split_filter(srcs, srcfilter, gsims, params, monitor): """ Split the given sources, filter the subsources and the compute the PoEs. Yield back subtasks if the split sources contain more than maxweight ruptures. """ # first check if we are sampling the sources ss = int(os.environ.get('OQ_SAMPLE_SOURCES', 0)) if ss: splits, stime = split_sources(srcs) srcs = readinput.random_filtered_sources(splits, srcfilter, ss) yield classical(srcs, srcfilter, gsims, params, monitor) return # NB: splitting all the sources improves the distribution significantly, # compared to splitting only the big source sources = [] with monitor("filtering/splitting sources"): for src, _sites in srcfilter(srcs): splits, _stime = split_sources([src]) sources.extend(srcfilter.filter(splits)) if sources: tot = 0 sd = AccumDict(accum=numpy.zeros(3)) # nsites, nrupts, weight for src in sources: arr = numpy.array([src.nsites, src.num_ruptures, src.weight]) sd[src.id] += arr tot += 1 source_data = numpy.array([(monitor.task_no, src_id, s / tot, r, w) for src_id, (s, r, w) in sd.items()], source_data_dt) first = True for out in parallel.split_task(classical, sources, srcfilter, gsims, params, monitor, duration=params['task_duration'], weight=nrup): if first: out['source_data'] = source_data first = False yield out
def test_mutually_exclusive_ruptures(self): # Test the calculation of hazard curves using mutually exclusive # ruptures for a single source gsim_by_trt = [SadighEtAl1997()] rupture = _create_rupture(10., 6.) data = [(rupture, PMF([(0.7, 0), (0.3, 1)])), (rupture, PMF([(0.6, 0), (0.4, 1)]))] print(data[0][0]) data[0][0].weight = 0.5 data[1][0].weight = 0.5 print(data[0][0].weight) src = NonParametricSeismicSource('0', 'test', TRT.ACTIVE_SHALLOW_CRUST, data) src.src_group_id = 0 src.mutex_weight = 1 group = SourceGroup( src.tectonic_region_type, [src], 'test', 'mutex', 'mutex') param = dict(imtls=self.imtls, filter_distance='rjb', src_interdep=group.src_interdep, rup_interdep=group.rup_interdep, grp_probability=group.grp_probability) crv = classical(group, self.sites, gsim_by_trt, param)['pmap'][0] npt.assert_almost_equal(numpy.array([0.35000, 0.32497, 0.10398]), crv[0].array[:, 0], decimal=4)
def classical_(srcs, gsims, params, monitor): srcfilter = monitor.read('srcfilter') return classical(srcs, srcfilter, gsims, params, monitor)