def test_case_7(self):
# test with 7+2 ruptures of two source models, 1 GSIM, 1 site
self.run_calc(case_7.__file__, 'job.ini')
cmakers = read_cmakers(self.calc.datastore)
ctxs0 = cmakers[0].read_ctxs(self.calc.datastore)
ctxs1 = cmakers[1].read_ctxs(self.calc.datastore)
self.assertEqual(len(ctxs0), 7) # rlz-0, the closest to the mean
self.assertEqual(len(ctxs1), 2) # rlz-1, the one to discard
# checking that the wrong realization is indeed discarded
pd = self.calc.datastore['performance_data'][:]
pd = pd[pd['operation'] == b'disaggregate']
self.assertEqual(pd['counts'], 1) # because g_by_z is empty
haz = self.calc.datastore['hmap4'][0, 0, :, 0] # shape NMPZ
self.assertEqual(haz[0], 0) # shortest return period => 0 hazard
self.assertEqual(haz[1], 0.18757115242025785)
# test normal disaggregation
[fname] = export(('disagg', 'csv'), self.calc.datastore)
self.assertEqualFiles('expected/rlz-0-PGA-sid-0-poe-1_TRT.csv', fname)
# test conditional disaggregation
[fname] = export(('disagg_traditional', 'csv'), self.calc.datastore)
self.assertEqualFiles('expected/rlz-0-PGA-sid-0-poe-1-cond_TRT.csv',
fname)
def __init__(self, dstore, full_lt, pgetter, srcidx):
self.datastore = dstore
self.full_lt = full_lt
self.cmakers = read_cmakers(dstore, full_lt)
self.get_hcurves = pgetter.get_hcurves
self.imtls = pgetter.imtls
self.sids = pgetter.sids
self.srcidx = srcidx
self.data = []
def __init__(self, dstore, full_lt, srcidx):
self.datastore = dstore
self.full_lt = full_lt
self.cmakers = read_cmakers(dstore, full_lt)
self.imtls = imtls = dstore['oqparam'].imtls
self.level_weights = imtls.array / imtls.array.sum()
self.sids = dstore['sitecol/sids'][:]
self.srcidx = srcidx
self.N = len(dstore['sitecol/sids'])
self.R = full_lt.get_num_paths()
self.acc = AccumDict(accum={})
def test_case_71(self):
# test with oversampling
# there are 6 potential paths 1A 1B 1C 2A 2B 2C
# 10 rlzs are being sampled: 1C 1A 1B 1A 1C 1A 2B 2A 2B 2A
# rlzs_by_g is 135 2 4, 79 68 i.e. 1A*3 1B*1 1C*1, 2A*2 2B*2
self.run_calc(case_71.__file__, 'job.ini', concurrent_tasks='0')
[fname] = export(('hcurves/mean', 'csv'), self.calc.datastore)
self.assertEqualFiles('expected/hcurves.csv', fname)
cmakers = contexts.read_cmakers(self.calc.datastore)
ae(list(cmakers[0].gsims.values()), [[1, 3, 5], [2], [0, 4]])
ae(list(cmakers[1].gsims.values()), [[7, 9], [6, 8]])
# there are two slices 0:3 and 3:5 with length 3 and 2 respectively
self.assertEqual(cmakers[1].start, 3)
def execute(self):
"""
Compute the conditional spectrum
"""
oq = self.oqparam
self.full_lt = self.datastore['full_lt']
self.trts = list(self.full_lt.gsim_lt.values)
[self.poe] = oq.poes_disagg
self.imts = list(oq.imtls)
self.M = len(self.imts)
dstore = (self.datastore.parent if self.datastore.parent
else self.datastore)
totrups = len(dstore['rup/mag'])
logging.info('Reading {:_d} ruptures'.format(totrups))
rdt = [('grp_id', U16), ('nsites', U16), ('idx', U32)]
rdata = numpy.zeros(totrups, rdt)
rdata['idx'] = numpy.arange(totrups)
rdata['grp_id'] = dstore['rup/grp_id'][:]
rdata['nsites'] = dstore['rup/nsites'][:]
totweight = rdata['nsites'].sum()
et_ids = dstore['et_ids'][:]
rlzs_by_gsim = self.full_lt.get_rlzs_by_gsim_list(et_ids)
self.slice_by_g = getters.get_slice_by_g(rlzs_by_gsim)
L = oq.imtls.size
poes_shape = (sum(len(rbg) for rbg in rlzs_by_gsim), self.N, L)
self.datastore.create_dset('poes', float, poes_shape)
G = max(len(rbg) for rbg in rlzs_by_gsim)
maxw = 2 * 1024**3 / (16 * G * self.M) # at max 2 GB
maxweight = min(
numpy.ceil(totweight / (oq.concurrent_tasks or 1)), maxw)
U = 0
Ta = 0
cmakers = read_cmakers(self.datastore)
self.datastore.swmr_on()
smap = parallel.Starmap(conditional_spectrum, h5=self.datastore.hdf5)
# IMPORTANT!! we rely on the fact that the classical part
# of the calculation stores the ruptures in chunks of constant
# grp_id, therefore it is possible to build (start, stop) slices
for block in general.block_splitter(rdata, maxweight,
operator.itemgetter('nsites'),
operator.itemgetter('grp_id')):
Ta += 1
grp_id = block[0]['grp_id']
G = len(rlzs_by_gsim[grp_id])
cmaker = cmakers[grp_id]
U = max(U, block.weight)
slc = slice(block[0]['idx'], block[-1]['idx'] + 1)
smap.submit((dstore, slc, cmaker))
results = smap.reduce(self.agg_result)
return results
def __init__(self, dstore, full_lt, pgetter, srcidx, mon):
self.datastore = dstore
self.full_lt = full_lt
self.cmakers = read_cmakers(dstore, full_lt)
self.get_hcurves = pgetter.get_hcurves
self.imtls = pgetter.imtls
self.sids = pgetter.sids
self.srcidx = srcidx
self.mon = mon
self.N = len(dstore['sitecol/sids'])
extreme = []
n = len(full_lt.sm_rlzs)
for grp_id, indices in enumerate(dstore['trt_smrs']):
trti, smrs = numpy.divmod(indices, n)
trt = full_lt.trts[trti[0]]
extreme.append((0, trt, 0, smrs))
self.extreme = numpy.array(extreme, grp_extreme_dt)
from openquake.hazardlib.contexts import read_cmakers, get_pmap
from openquake.commonlib.datastore import read
dstore = read(-1) # first run case_1
cmakers = read_cmakers(dstore)
for grp_id, cmaker in enumerate(cmakers):
ctxs = cmaker.read_ctxs(dstore)
print(grp_id, len(ctxs), get_pmap(ctxs, cmaker))
def compute(self):
"""
Submit disaggregation tasks and return the results
"""
oq = self.oqparam
dstore = (self.datastore.parent if self.datastore.parent
else self.datastore)
magi = numpy.searchsorted(self.bin_edges[0], dstore['rup/mag'][:]) - 1
magi[magi == -1] = 0 # when the magnitude is on the edge
totrups = len(magi)
logging.info('Reading {:_d} ruptures'.format(totrups))
rdt = [('grp_id', U16), ('magi', U8), ('nsites', U16), ('idx', U32)]
rdata = numpy.zeros(totrups, rdt)
rdata['magi'] = magi
rdata['idx'] = numpy.arange(totrups)
rdata['grp_id'] = dstore['rup/grp_id'][:]
rdata['nsites'] = [len(sids) for sids in dstore['rup/sids_']]
totweight = rdata['nsites'].sum()
trt_smrs = dstore['trt_smrs'][:]
rlzs_by_gsim = self.full_lt.get_rlzs_by_gsim_list(trt_smrs)
G = max(len(rbg) for rbg in rlzs_by_gsim)
maxw = 2 * 1024**3 / (16 * G * self.M) # at max 2 GB
maxweight = min(
numpy.ceil(totweight / (oq.concurrent_tasks or 1)), maxw)
task_inputs = []
U = 0
self.datastore.swmr_on()
smap = parallel.Starmap(compute_disagg, h5=self.datastore.hdf5)
# IMPORTANT!! we rely on the fact that the classical part
# of the calculation stores the ruptures in chunks of constant
# grp_id, therefore it is possible to build (start, stop) slices;
# we are NOT grouping by operator.itemgetter('grp_id', 'magi'):
# that would break the ordering of the indices causing an incredibly
# worse performance, but visible only in extra-large calculations!
cmakers = read_cmakers(self.datastore)
for block in block_splitter(rdata, maxweight,
operator.itemgetter('nsites'),
operator.itemgetter('grp_id')):
grp_id = block[0]['grp_id']
cmaker = cmakers[grp_id]
U = max(U, block.weight)
slc = slice(block[0]['idx'], block[-1]['idx'] + 1)
smap.submit((dstore, slc, cmaker, self.hmap4,
magi[slc], self.bin_edges))
task_inputs.append((cmaker.trti, slc.stop-slc.start))
nbytes, msg = get_nbytes_msg(dict(M=self.M, G=G, U=U, F=2))
logging.info('Maximum mean_std per task:\n%s', msg)
s = self.shapedic
Ta = len(task_inputs)
nbytes = s['N'] * s['M'] * s['P'] * s['Z'] * Ta * 8
data_transfer = (s['dist'] * s['eps'] + s['lon'] * s['lat']) * nbytes
if data_transfer > oq.max_data_transfer:
raise ValueError(
'Estimated data transfer too big\n%s > max_data_transfer=%s' %
(humansize(data_transfer), humansize(oq.max_data_transfer)))
logging.info('Estimated data transfer: %s', humansize(data_transfer))
dt = numpy.dtype([('trti', U8), ('nrups', U32)])
self.datastore['disagg_task'] = numpy.array(task_inputs, dt)
results = smap.reduce(self.agg_result, AccumDict(accum={}))
return results # imti, sid -> trti, magi -> 6D array
def run_preclassical(calc):
"""
:param csm: a CompositeSourceModel
:param oqparam: the parameters in job.ini file
:param h5: a DataStore instance
"""
csm = calc.csm
calc.datastore['trt_smrs'] = csm.get_trt_smrs()
calc.datastore['toms'] = numpy.array(
[sg.tom_name for sg in csm.src_groups], hdf5.vstr)
cmakers = read_cmakers(calc.datastore, csm.full_lt)
h5 = calc.datastore.hdf5
calc.sitecol = sites = csm.sitecol if csm.sitecol else None
# do nothing for atomic sources except counting the ruptures
atomic_sources = []
normal_sources = []
for sg in csm.src_groups:
grp_id = sg.sources[0].grp_id
if sg.atomic:
cmakers[grp_id].set_weight(sg, sites)
atomic_sources.extend(sg)
else:
normal_sources.extend(sg)
# run preclassical for non-atomic sources
sources_by_grp = groupby(normal_sources, lambda src:
(src.grp_id, msr_name(src)))
if csm.sitecol:
logging.info('Sending %s', sites)
smap = parallel.Starmap(preclassical, h5=h5)
for (grp_id, msr), srcs in sources_by_grp.items():
pointsources, pointlike, others = [], [], []
for src in srcs:
if hasattr(src, 'location'):
pointsources.append(src)
elif hasattr(src, 'nodal_plane_distribution'):
pointlike.append(src)
else:
others.append(src)
if calc.oqparam.ps_grid_spacing:
if pointsources or pointlike:
smap.submit((pointsources + pointlike, sites, cmakers[grp_id]))
else:
smap.submit_split((pointsources, sites, cmakers[grp_id]), 10, 100)
for src in pointlike: # area, multipoint
smap.submit(([src], sites, cmakers[grp_id]))
smap.submit_split((others, sites, cmakers[grp_id]), 10, 100)
normal = smap.reduce()
if atomic_sources: # case_35
n = len(atomic_sources)
atomic = AccumDict({'before': n, 'after': n})
for grp_id, srcs in groupby(atomic_sources,
lambda src: src.grp_id).items():
atomic[grp_id] = srcs
else:
atomic = AccumDict()
res = normal + atomic
if res['before'] != res['after']:
logging.info(
'Reduced the number of point sources from {:_d} -> {:_d}'.format(
res['before'], res['after']))
acc = AccumDict(accum=0)
code2cls = get_code2cls()
for grp_id, srcs in res.items():
# srcs can be empty if the minimum_magnitude filter is on
if srcs and not isinstance(grp_id, str) and grp_id not in atomic:
# check if OQ_SAMPLE_SOURCES is set
ss = os.environ.get('OQ_SAMPLE_SOURCES')
if ss:
logging.info('Sampled sources for group #%d', grp_id)
srcs = general.random_filter(srcs, float(ss)) or [srcs[0]]
newsg = SourceGroup(srcs[0].tectonic_region_type)
newsg.sources = srcs
csm.src_groups[grp_id] = newsg
for src in srcs:
assert src.weight
assert src.num_ruptures
acc[src.code] += int(src.num_ruptures)
for val, key in sorted((val, key) for key, val in acc.items()):
cls = code2cls[key].__name__
logging.info('{} ruptures: {:_d}'.format(cls, val))
source_data = zero_times(csm.get_sources())
calc.store_source_info(source_data)
# store ps_grid data, if any
for key, sources in res.items():
if isinstance(key, str) and key.startswith('ps_grid/'):
arrays = []
for ps in sources:
if hasattr(ps, 'location'):
lonlats = [ps.location.x, ps.location.y]
for src in getattr(ps, 'pointsources', []):
lonlats.extend([src.location.x, src.location.y])
arrays.append(F32(lonlats))
h5[key] = arrays
h5['full_lt'] = csm.full_lt
return res
def execute(self):
"""
Compute the conditional spectrum
"""
oq = self.oqparam
self.full_lt = self.datastore['full_lt']
self.trts = list(self.full_lt.gsim_lt.values)
self.imts = list(oq.imtls)
imti = self.imts.index(oq.imt_ref)
self.M = len(self.imts)
dstore = (self.datastore.parent
if self.datastore.parent else self.datastore)
totrups = len(dstore['rup/mag'])
logging.info('Reading {:_d} ruptures'.format(totrups))
rdt = [('grp_id', U16), ('nsites', U16), ('idx', U32)]
rdata = numpy.zeros(totrups, rdt)
rdata['idx'] = numpy.arange(totrups)
rdata['grp_id'] = dstore['rup/grp_id'][:]
rdata['nsites'] = [len(sids) for sids in dstore['rup/sids_']]
totweight = rdata['nsites'].sum()
trt_smrs = dstore['trt_smrs'][:]
rlzs_by_gsim = self.full_lt.get_rlzs_by_gsim_list(trt_smrs)
G_ = sum(len(rbg) for rbg in rlzs_by_gsim)
self.periods = [from_string(imt).period for imt in self.imts]
if oq.imls_ref:
self.imls = oq.imls_ref
else: # extract imls from the "mean" hazard map
curve = self.datastore.sel('hcurves-stats', stat='mean')[0, 0,
imti]
[self.imls] = compute_hazard_maps(curve, oq.imtls[oq.imt_ref],
oq.poes) # there is 1 site
self.P = P = len(self.imls)
self.datastore.create_dset('cs-rlzs', float, (P, self.R, 2, self.M))
self.datastore.set_shape_descr('cs-rlzs',
poe_id=P,
rlz_id=self.R,
cs=2,
m=self.M)
self.datastore.create_dset('cond-spectra', float, (P, 2, self.M))
self.datastore.set_shape_descr('cond-spectra',
poe_id=P,
cs=['spec', 'std'],
period=self.periods)
self.datastore.create_dset('_c', float, (G_, P, 2, self.M))
self.datastore.create_dset('_s', float, (
G_,
P,
))
G = max(len(rbg) for rbg in rlzs_by_gsim)
maxw = 2 * 1024**3 / (16 * G * self.M) # at max 2 GB
maxweight = min(numpy.ceil(totweight / (oq.concurrent_tasks or 1)),
maxw)
U = 0
Ta = 0
self.cmakers = read_cmakers(self.datastore)
self.datastore.swmr_on()
smap = parallel.Starmap(conditional_spectrum, h5=self.datastore.hdf5)
# IMPORTANT!! we rely on the fact that the classical part
# of the calculation stores the ruptures in chunks of constant
# grp_id, therefore it is possible to build (start, stop) slices
for block in general.block_splitter(rdata, maxweight,
operator.itemgetter('nsites'),
operator.itemgetter('grp_id')):
Ta += 1
grp_id = block[0]['grp_id']
G = len(rlzs_by_gsim[grp_id])
cmaker = self.cmakers[grp_id]
U = max(U, block.weight)
slc = slice(block[0]['idx'], block[-1]['idx'] + 1)
smap.submit((dstore, slc, cmaker, imti, self.imls))
return smap.reduce()
