def test_case_7(self): # test with 7+2 ruptures of two source models, 1 GSIM, 1 site self.run_calc(case_7.__file__, 'job.ini') cmakers = read_cmakers(self.calc.datastore) ctxs0 = cmakers[0].read_ctxs(self.calc.datastore) ctxs1 = cmakers[1].read_ctxs(self.calc.datastore) self.assertEqual(len(ctxs0), 7) # rlz-0, the closest to the mean self.assertEqual(len(ctxs1), 2) # rlz-1, the one to discard # checking that the wrong realization is indeed discarded pd = self.calc.datastore['performance_data'][:] pd = pd[pd['operation'] == b'disaggregate'] self.assertEqual(pd['counts'], 1) # because g_by_z is empty haz = self.calc.datastore['hmap4'][0, 0, :, 0] # shape NMPZ self.assertEqual(haz[0], 0) # shortest return period => 0 hazard self.assertEqual(haz[1], 0.18757115242025785) # test normal disaggregation [fname] = export(('disagg', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/rlz-0-PGA-sid-0-poe-1_TRT.csv', fname) # test conditional disaggregation [fname] = export(('disagg_traditional', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/rlz-0-PGA-sid-0-poe-1-cond_TRT.csv', fname)
def __init__(self, dstore, full_lt, pgetter, srcidx): self.datastore = dstore self.full_lt = full_lt self.cmakers = read_cmakers(dstore, full_lt) self.get_hcurves = pgetter.get_hcurves self.imtls = pgetter.imtls self.sids = pgetter.sids self.srcidx = srcidx self.data = []
def __init__(self, dstore, full_lt, srcidx): self.datastore = dstore self.full_lt = full_lt self.cmakers = read_cmakers(dstore, full_lt) self.imtls = imtls = dstore['oqparam'].imtls self.level_weights = imtls.array / imtls.array.sum() self.sids = dstore['sitecol/sids'][:] self.srcidx = srcidx self.N = len(dstore['sitecol/sids']) self.R = full_lt.get_num_paths() self.acc = AccumDict(accum={})
def test_case_71(self): # test with oversampling # there are 6 potential paths 1A 1B 1C 2A 2B 2C # 10 rlzs are being sampled: 1C 1A 1B 1A 1C 1A 2B 2A 2B 2A # rlzs_by_g is 135 2 4, 79 68 i.e. 1A*3 1B*1 1C*1, 2A*2 2B*2 self.run_calc(case_71.__file__, 'job.ini', concurrent_tasks='0') [fname] = export(('hcurves/mean', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/hcurves.csv', fname) cmakers = contexts.read_cmakers(self.calc.datastore) ae(list(cmakers[0].gsims.values()), [[1, 3, 5], [2], [0, 4]]) ae(list(cmakers[1].gsims.values()), [[7, 9], [6, 8]]) # there are two slices 0:3 and 3:5 with length 3 and 2 respectively self.assertEqual(cmakers[1].start, 3)
def execute(self): """ Compute the conditional spectrum """ oq = self.oqparam self.full_lt = self.datastore['full_lt'] self.trts = list(self.full_lt.gsim_lt.values) [self.poe] = oq.poes_disagg self.imts = list(oq.imtls) self.M = len(self.imts) dstore = (self.datastore.parent if self.datastore.parent else self.datastore) totrups = len(dstore['rup/mag']) logging.info('Reading {:_d} ruptures'.format(totrups)) rdt = [('grp_id', U16), ('nsites', U16), ('idx', U32)] rdata = numpy.zeros(totrups, rdt) rdata['idx'] = numpy.arange(totrups) rdata['grp_id'] = dstore['rup/grp_id'][:] rdata['nsites'] = dstore['rup/nsites'][:] totweight = rdata['nsites'].sum() et_ids = dstore['et_ids'][:] rlzs_by_gsim = self.full_lt.get_rlzs_by_gsim_list(et_ids) self.slice_by_g = getters.get_slice_by_g(rlzs_by_gsim) L = oq.imtls.size poes_shape = (sum(len(rbg) for rbg in rlzs_by_gsim), self.N, L) self.datastore.create_dset('poes', float, poes_shape) G = max(len(rbg) for rbg in rlzs_by_gsim) maxw = 2 * 1024**3 / (16 * G * self.M) # at max 2 GB maxweight = min( numpy.ceil(totweight / (oq.concurrent_tasks or 1)), maxw) U = 0 Ta = 0 cmakers = read_cmakers(self.datastore) self.datastore.swmr_on() smap = parallel.Starmap(conditional_spectrum, h5=self.datastore.hdf5) # IMPORTANT!! we rely on the fact that the classical part # of the calculation stores the ruptures in chunks of constant # grp_id, therefore it is possible to build (start, stop) slices for block in general.block_splitter(rdata, maxweight, operator.itemgetter('nsites'), operator.itemgetter('grp_id')): Ta += 1 grp_id = block[0]['grp_id'] G = len(rlzs_by_gsim[grp_id]) cmaker = cmakers[grp_id] U = max(U, block.weight) slc = slice(block[0]['idx'], block[-1]['idx'] + 1) smap.submit((dstore, slc, cmaker)) results = smap.reduce(self.agg_result) return results
def __init__(self, dstore, full_lt, pgetter, srcidx, mon): self.datastore = dstore self.full_lt = full_lt self.cmakers = read_cmakers(dstore, full_lt) self.get_hcurves = pgetter.get_hcurves self.imtls = pgetter.imtls self.sids = pgetter.sids self.srcidx = srcidx self.mon = mon self.N = len(dstore['sitecol/sids']) extreme = [] n = len(full_lt.sm_rlzs) for grp_id, indices in enumerate(dstore['trt_smrs']): trti, smrs = numpy.divmod(indices, n) trt = full_lt.trts[trti[0]] extreme.append((0, trt, 0, smrs)) self.extreme = numpy.array(extreme, grp_extreme_dt)
from openquake.hazardlib.contexts import read_cmakers, get_pmap from openquake.commonlib.datastore import read dstore = read(-1) # first run case_1 cmakers = read_cmakers(dstore) for grp_id, cmaker in enumerate(cmakers): ctxs = cmaker.read_ctxs(dstore) print(grp_id, len(ctxs), get_pmap(ctxs, cmaker))
def compute(self): """ Submit disaggregation tasks and return the results """ oq = self.oqparam dstore = (self.datastore.parent if self.datastore.parent else self.datastore) magi = numpy.searchsorted(self.bin_edges[0], dstore['rup/mag'][:]) - 1 magi[magi == -1] = 0 # when the magnitude is on the edge totrups = len(magi) logging.info('Reading {:_d} ruptures'.format(totrups)) rdt = [('grp_id', U16), ('magi', U8), ('nsites', U16), ('idx', U32)] rdata = numpy.zeros(totrups, rdt) rdata['magi'] = magi rdata['idx'] = numpy.arange(totrups) rdata['grp_id'] = dstore['rup/grp_id'][:] rdata['nsites'] = [len(sids) for sids in dstore['rup/sids_']] totweight = rdata['nsites'].sum() trt_smrs = dstore['trt_smrs'][:] rlzs_by_gsim = self.full_lt.get_rlzs_by_gsim_list(trt_smrs) G = max(len(rbg) for rbg in rlzs_by_gsim) maxw = 2 * 1024**3 / (16 * G * self.M) # at max 2 GB maxweight = min( numpy.ceil(totweight / (oq.concurrent_tasks or 1)), maxw) task_inputs = [] U = 0 self.datastore.swmr_on() smap = parallel.Starmap(compute_disagg, h5=self.datastore.hdf5) # IMPORTANT!! we rely on the fact that the classical part # of the calculation stores the ruptures in chunks of constant # grp_id, therefore it is possible to build (start, stop) slices; # we are NOT grouping by operator.itemgetter('grp_id', 'magi'): # that would break the ordering of the indices causing an incredibly # worse performance, but visible only in extra-large calculations! cmakers = read_cmakers(self.datastore) for block in block_splitter(rdata, maxweight, operator.itemgetter('nsites'), operator.itemgetter('grp_id')): grp_id = block[0]['grp_id'] cmaker = cmakers[grp_id] U = max(U, block.weight) slc = slice(block[0]['idx'], block[-1]['idx'] + 1) smap.submit((dstore, slc, cmaker, self.hmap4, magi[slc], self.bin_edges)) task_inputs.append((cmaker.trti, slc.stop-slc.start)) nbytes, msg = get_nbytes_msg(dict(M=self.M, G=G, U=U, F=2)) logging.info('Maximum mean_std per task:\n%s', msg) s = self.shapedic Ta = len(task_inputs) nbytes = s['N'] * s['M'] * s['P'] * s['Z'] * Ta * 8 data_transfer = (s['dist'] * s['eps'] + s['lon'] * s['lat']) * nbytes if data_transfer > oq.max_data_transfer: raise ValueError( 'Estimated data transfer too big\n%s > max_data_transfer=%s' % (humansize(data_transfer), humansize(oq.max_data_transfer))) logging.info('Estimated data transfer: %s', humansize(data_transfer)) dt = numpy.dtype([('trti', U8), ('nrups', U32)]) self.datastore['disagg_task'] = numpy.array(task_inputs, dt) results = smap.reduce(self.agg_result, AccumDict(accum={})) return results # imti, sid -> trti, magi -> 6D array
def run_preclassical(calc): """ :param csm: a CompositeSourceModel :param oqparam: the parameters in job.ini file :param h5: a DataStore instance """ csm = calc.csm calc.datastore['trt_smrs'] = csm.get_trt_smrs() calc.datastore['toms'] = numpy.array( [sg.tom_name for sg in csm.src_groups], hdf5.vstr) cmakers = read_cmakers(calc.datastore, csm.full_lt) h5 = calc.datastore.hdf5 calc.sitecol = sites = csm.sitecol if csm.sitecol else None # do nothing for atomic sources except counting the ruptures atomic_sources = [] normal_sources = [] for sg in csm.src_groups: grp_id = sg.sources[0].grp_id if sg.atomic: cmakers[grp_id].set_weight(sg, sites) atomic_sources.extend(sg) else: normal_sources.extend(sg) # run preclassical for non-atomic sources sources_by_grp = groupby(normal_sources, lambda src: (src.grp_id, msr_name(src))) if csm.sitecol: logging.info('Sending %s', sites) smap = parallel.Starmap(preclassical, h5=h5) for (grp_id, msr), srcs in sources_by_grp.items(): pointsources, pointlike, others = [], [], [] for src in srcs: if hasattr(src, 'location'): pointsources.append(src) elif hasattr(src, 'nodal_plane_distribution'): pointlike.append(src) else: others.append(src) if calc.oqparam.ps_grid_spacing: if pointsources or pointlike: smap.submit((pointsources + pointlike, sites, cmakers[grp_id])) else: smap.submit_split((pointsources, sites, cmakers[grp_id]), 10, 100) for src in pointlike: # area, multipoint smap.submit(([src], sites, cmakers[grp_id])) smap.submit_split((others, sites, cmakers[grp_id]), 10, 100) normal = smap.reduce() if atomic_sources: # case_35 n = len(atomic_sources) atomic = AccumDict({'before': n, 'after': n}) for grp_id, srcs in groupby(atomic_sources, lambda src: src.grp_id).items(): atomic[grp_id] = srcs else: atomic = AccumDict() res = normal + atomic if res['before'] != res['after']: logging.info( 'Reduced the number of point sources from {:_d} -> {:_d}'.format( res['before'], res['after'])) acc = AccumDict(accum=0) code2cls = get_code2cls() for grp_id, srcs in res.items(): # srcs can be empty if the minimum_magnitude filter is on if srcs and not isinstance(grp_id, str) and grp_id not in atomic: # check if OQ_SAMPLE_SOURCES is set ss = os.environ.get('OQ_SAMPLE_SOURCES') if ss: logging.info('Sampled sources for group #%d', grp_id) srcs = general.random_filter(srcs, float(ss)) or [srcs[0]] newsg = SourceGroup(srcs[0].tectonic_region_type) newsg.sources = srcs csm.src_groups[grp_id] = newsg for src in srcs: assert src.weight assert src.num_ruptures acc[src.code] += int(src.num_ruptures) for val, key in sorted((val, key) for key, val in acc.items()): cls = code2cls[key].__name__ logging.info('{} ruptures: {:_d}'.format(cls, val)) source_data = zero_times(csm.get_sources()) calc.store_source_info(source_data) # store ps_grid data, if any for key, sources in res.items(): if isinstance(key, str) and key.startswith('ps_grid/'): arrays = [] for ps in sources: if hasattr(ps, 'location'): lonlats = [ps.location.x, ps.location.y] for src in getattr(ps, 'pointsources', []): lonlats.extend([src.location.x, src.location.y]) arrays.append(F32(lonlats)) h5[key] = arrays h5['full_lt'] = csm.full_lt return res
def execute(self): """ Compute the conditional spectrum """ oq = self.oqparam self.full_lt = self.datastore['full_lt'] self.trts = list(self.full_lt.gsim_lt.values) self.imts = list(oq.imtls) imti = self.imts.index(oq.imt_ref) self.M = len(self.imts) dstore = (self.datastore.parent if self.datastore.parent else self.datastore) totrups = len(dstore['rup/mag']) logging.info('Reading {:_d} ruptures'.format(totrups)) rdt = [('grp_id', U16), ('nsites', U16), ('idx', U32)] rdata = numpy.zeros(totrups, rdt) rdata['idx'] = numpy.arange(totrups) rdata['grp_id'] = dstore['rup/grp_id'][:] rdata['nsites'] = [len(sids) for sids in dstore['rup/sids_']] totweight = rdata['nsites'].sum() trt_smrs = dstore['trt_smrs'][:] rlzs_by_gsim = self.full_lt.get_rlzs_by_gsim_list(trt_smrs) G_ = sum(len(rbg) for rbg in rlzs_by_gsim) self.periods = [from_string(imt).period for imt in self.imts] if oq.imls_ref: self.imls = oq.imls_ref else: # extract imls from the "mean" hazard map curve = self.datastore.sel('hcurves-stats', stat='mean')[0, 0, imti] [self.imls] = compute_hazard_maps(curve, oq.imtls[oq.imt_ref], oq.poes) # there is 1 site self.P = P = len(self.imls) self.datastore.create_dset('cs-rlzs', float, (P, self.R, 2, self.M)) self.datastore.set_shape_descr('cs-rlzs', poe_id=P, rlz_id=self.R, cs=2, m=self.M) self.datastore.create_dset('cond-spectra', float, (P, 2, self.M)) self.datastore.set_shape_descr('cond-spectra', poe_id=P, cs=['spec', 'std'], period=self.periods) self.datastore.create_dset('_c', float, (G_, P, 2, self.M)) self.datastore.create_dset('_s', float, ( G_, P, )) G = max(len(rbg) for rbg in rlzs_by_gsim) maxw = 2 * 1024**3 / (16 * G * self.M) # at max 2 GB maxweight = min(numpy.ceil(totweight / (oq.concurrent_tasks or 1)), maxw) U = 0 Ta = 0 self.cmakers = read_cmakers(self.datastore) self.datastore.swmr_on() smap = parallel.Starmap(conditional_spectrum, h5=self.datastore.hdf5) # IMPORTANT!! we rely on the fact that the classical part # of the calculation stores the ruptures in chunks of constant # grp_id, therefore it is possible to build (start, stop) slices for block in general.block_splitter(rdata, maxweight, operator.itemgetter('nsites'), operator.itemgetter('grp_id')): Ta += 1 grp_id = block[0]['grp_id'] G = len(rlzs_by_gsim[grp_id]) cmaker = self.cmakers[grp_id] U = max(U, block.weight) slc = slice(block[0]['idx'], block[-1]['idx'] + 1) smap.submit((dstore, slc, cmaker, imti, self.imls)) return smap.reduce()