def build_events_from_sources(self):
"""
Prefilter the composite source model and store the source_info
"""
gsims_by_trt = self.csm.full_lt.get_gsims_by_trt()
logging.info('Building ruptures')
for src in self.csm.get_sources():
src.nsites = 1 # avoid 0 weight
maxweight = sum(sg.weight for sg in self.csm.src_groups) / (
self.oqparam.concurrent_tasks or 1)
eff_ruptures = AccumDict(accum=0) # trt => potential ruptures
calc_times = AccumDict(accum=numpy.zeros(3, F32)) # nr, ns, dt
allargs = []
if self.oqparam.is_ucerf():
# manage the filtering in a special way
for sg in self.csm.src_groups:
for src in sg:
src.src_filter = self.srcfilter
srcfilter = nofilter # otherwise it would be ultra-slow
else:
srcfilter = self.srcfilter
for sg in self.csm.src_groups:
if not sg.sources:
continue
logging.info('Sending %s', sg)
par = self.param.copy()
par['gsims'] = gsims_by_trt[sg.trt]
for src_group in sg.split(maxweight):
allargs.append((src_group, srcfilter, par))
smap = parallel.Starmap(sample_ruptures,
allargs,
h5=self.datastore.hdf5)
mon = self.monitor('saving ruptures')
self.nruptures = 0
for dic in smap:
rup_array = dic['rup_array']
if len(rup_array) == 0:
continue
if dic['calc_times']:
calc_times += dic['calc_times']
if dic['eff_ruptures']:
eff_ruptures += dic['eff_ruptures']
with mon:
n = len(rup_array)
rup_array['id'] = numpy.arange(self.nruptures,
self.nruptures + n)
self.nruptures += n
hdf5.extend(self.datastore['ruptures'], rup_array)
hdf5.extend(self.datastore['rupgeoms'], rup_array.geom)
if len(self.datastore['ruptures']) == 0:
raise RuntimeError('No ruptures were generated, perhaps the '
'investigation time is too short')
# must be called before storing the events
self.store_rlz_info(eff_ruptures) # store full_lt
with self.monitor('store source_info'):
self.store_source_info(calc_times)
imp = calc.RuptureImporter(self.datastore)
with self.monitor('saving ruptures and events'):
imp.import_rups(self.datastore.getitem('ruptures')[()])
def _read_scenario_ruptures(self):
oq = self.oqparam
gsim_lt = readinput.get_gsim_lt(self.oqparam)
G = gsim_lt.get_num_paths()
if oq.calculation_mode.startswith('scenario'):
ngmfs = oq.number_of_ground_motion_fields
if oq.inputs['rupture_model'].endswith('.xml'):
self.gsims = [gsim_rlz.value[0] for gsim_rlz in gsim_lt]
self.cmaker = ContextMaker('*', self.gsims, {
'maximum_distance': oq.maximum_distance,
'imtls': oq.imtls
})
rup = readinput.get_rupture(oq)
if self.N > oq.max_sites_disagg: # many sites, split rupture
ebrs = [
EBRupture(copyobj(rup, rup_id=rup.rup_id + i),
0,
0,
G,
e0=i * G) for i in range(ngmfs)
]
else: # keep a single rupture with a big occupation number
ebrs = [EBRupture(rup, 0, 0, G * ngmfs, rup.rup_id)]
aw = get_rup_array(ebrs, self.srcfilter)
if len(aw) == 0:
raise RuntimeError(
'The rupture is too far from the sites! Please check the '
'maximum_distance and the position of the rupture')
elif oq.inputs['rupture_model'].endswith('.csv'):
aw = readinput.get_ruptures(oq.inputs['rupture_model'])
num_gsims = numpy.array(
[len(gsim_lt.values[trt]) for trt in gsim_lt.values], U32)
if oq.calculation_mode.startswith('scenario'):
# rescale n_occ
aw['n_occ'] *= ngmfs * num_gsims[aw['trt_smr']]
rup_array = aw.array
hdf5.extend(self.datastore['rupgeoms'], aw.geom)
if len(rup_array) == 0:
raise RuntimeError(
'There are no sites within the maximum_distance'
' of %s km from the rupture' %
oq.maximum_distance(rup.tectonic_region_type, rup.mag))
# check the number of branchsets
branchsets = len(gsim_lt._ltnode)
if len(rup_array) == 1 and branchsets > 1:
raise InvalidFile(
'%s for a scenario calculation must contain a single '
'branchset, found %d!' % (oq.inputs['job_ini'], branchsets))
fake = logictree.FullLogicTree.fake(gsim_lt)
self.realizations = fake.get_realizations()
self.datastore['full_lt'] = fake
self.store_rlz_info({}) # store weights
self.save_params()
imp = calc.RuptureImporter(self.datastore)
imp.import_rups_events(rup_array, get_rupture_getters)
def _read_scenario_ruptures(self):
oq = self.oqparam
gsim_lt = readinput.get_gsim_lt(self.oqparam)
G = gsim_lt.get_num_paths()
if oq.inputs['rupture_model'].endswith('.xml'):
ngmfs = oq.number_of_ground_motion_fields
self.gsims = readinput.get_gsims(oq)
self.cmaker = ContextMaker('*', self.gsims, {
'maximum_distance': oq.maximum_distance,
'imtls': oq.imtls
})
rup = readinput.get_rupture(oq)
mesh = surface_to_array(rup.surface).transpose(1, 2, 0).flatten()
if self.N > oq.max_sites_disagg: # many sites, split rupture
ebrs = [
EBRupture(copyobj(rup, rup_id=rup.rup_id + i),
0,
0,
G,
e0=i * G) for i in range(ngmfs)
]
meshes = numpy.array([mesh] * ngmfs, object)
else: # keep a single rupture with a big occupation number
ebrs = [EBRupture(rup, 0, 0, G * ngmfs, rup.rup_id)]
meshes = numpy.array([mesh] * ngmfs, object)
rup_array = get_rup_array(ebrs, self.srcfilter).array
hdf5.extend(self.datastore['rupgeoms'], meshes)
elif oq.inputs['rupture_model'].endswith('.csv'):
aw = readinput.get_ruptures(oq.inputs['rupture_model'])
aw.array['n_occ'] = G
rup_array = aw.array
hdf5.extend(self.datastore['rupgeoms'], aw.geom)
if len(rup_array) == 0:
raise RuntimeError(
'There are no sites within the maximum_distance'
' of %s km from the rupture' %
oq.maximum_distance(rup.tectonic_region_type, rup.mag))
# check the number of branchsets
branchsets = len(gsim_lt._ltnode)
if len(rup_array) == 1 and branchsets > 1:
raise InvalidFile(
'%s for a scenario calculation must contain a single '
'branchset, found %d!' % (oq.inputs['job_ini'], branchsets))
fake = logictree.FullLogicTree.fake(gsim_lt)
self.realizations = fake.get_realizations()
self.datastore['full_lt'] = fake
self.store_rlz_info({}) # store weights
self.save_params()
calc.RuptureImporter(self.datastore).import_rups(rup_array)
def _read_scenario_ruptures(self):
oq = self.oqparam
if oq.inputs['rupture_model'].endswith(('.xml', '.toml', '.txt')):
self.gsims = readinput.get_gsims(oq)
self.cmaker = ContextMaker(
'*', self.gsims,
{'maximum_distance': oq.maximum_distance,
'filter_distance': oq.filter_distance})
n_occ = numpy.array([oq.number_of_ground_motion_fields])
rup = readinput.get_rupture(oq)
ebr = EBRupture(rup, 0, 0, n_occ)
ebr.e0 = 0
rup_array = get_rup_array([ebr], self.srcfilter).array
mesh = surface_to_array(rup.surface).transpose(1, 2, 0).flatten()
hdf5.extend(self.datastore['rupgeoms'],
numpy.array([mesh], object))
elif oq.inputs['rupture_model'].endswith('.csv'):
aw = readinput.get_ruptures(oq.inputs['rupture_model'])
rup_array = aw.array
hdf5.extend(self.datastore['rupgeoms'], aw.geom)
if len(rup_array) == 0:
raise RuntimeError(
'There are no sites within the maximum_distance'
' of %s km from the rupture' % oq.maximum_distance(
rup.tectonic_region_type, rup.mag))
gsim_lt = readinput.get_gsim_lt(self.oqparam)
# check the number of branchsets
branchsets = len(gsim_lt._ltnode)
if len(rup_array) == 1 and branchsets > 1:
raise InvalidFile(
'%s for a scenario calculation must contain a single '
'branchset, found %d!' % (oq.inputs['job_ini'], branchsets))
fake = logictree.FullLogicTree.fake(gsim_lt)
self.realizations = fake.get_realizations()
self.datastore['full_lt'] = fake
self.store_rlz_info({}) # store weights
self.save_params()
calc.RuptureImporter(self.datastore).import_rups(rup_array)
def build_events_from_sources(self):
"""
Prefilter the composite source model and store the source_info
"""
gsims_by_trt = self.csm.full_lt.get_gsims_by_trt()
sources = self.csm.get_sources()
# weighting the heavy sources
nrups = parallel.Starmap(
count_ruptures, [(src,) for src in sources if src.code in b'AMC'],
progress=logging.debug
).reduce()
for src in sources:
src.nsites = 1 # avoid 0 weight
try:
src.num_ruptures = nrups[src.source_id]
except KeyError:
src.num_ruptures = src.count_ruptures()
maxweight = sum(sg.weight for sg in self.csm.src_groups) / (
self.oqparam.concurrent_tasks or 1)
eff_ruptures = AccumDict(accum=0) # trt => potential ruptures
calc_times = AccumDict(accum=numpy.zeros(3, F32)) # nr, ns, dt
allargs = []
if self.oqparam.is_ucerf():
# manage the filtering in a special way
for sg in self.csm.src_groups:
for src in sg:
src.src_filter = self.srcfilter
srcfilter = nofilter # otherwise it would be ultra-slow
else:
srcfilter = self.srcfilter
logging.info('Building ruptures')
for sg in self.csm.src_groups:
if not sg.sources:
continue
logging.info('Sending %s', sg)
par = self.param.copy()
par['gsims'] = gsims_by_trt[sg.trt]
for src_group in sg.split(maxweight):
allargs.append((src_group, srcfilter, par))
smap = parallel.Starmap(
sample_ruptures, allargs, h5=self.datastore.hdf5)
mon = self.monitor('saving ruptures')
self.nruptures = 0
for dic in smap:
# NB: dic should be a dictionary, but when the calculation dies
# for an OOM it can become None, thus giving a very confusing error
if dic is None:
raise MemoryError('You ran out of memory!')
rup_array = dic['rup_array']
if len(rup_array) == 0:
continue
if dic['calc_times']:
calc_times += dic['calc_times']
if dic['eff_ruptures']:
eff_ruptures += dic['eff_ruptures']
with mon:
n = len(rup_array)
rup_array['id'] = numpy.arange(
self.nruptures, self.nruptures + n)
self.nruptures += n
hdf5.extend(self.datastore['ruptures'], rup_array)
hdf5.extend(self.datastore['rupgeoms'], rup_array.geom)
if len(self.datastore['ruptures']) == 0:
raise RuntimeError('No ruptures were generated, perhaps the '
'investigation time is too short')
# must be called before storing the events
self.store_rlz_info(eff_ruptures) # store full_lt
self.store_source_info(calc_times)
imp = calc.RuptureImporter(self.datastore)
with self.monitor('saving ruptures and events'):
imp.import_rups_events(
self.datastore.getitem('ruptures')[()], get_rupture_getters)
def _read_scenario_ruptures(self):
oq = self.oqparam
gsim_lt = readinput.get_gsim_lt(self.oqparam)
G = gsim_lt.get_num_paths()
if oq.calculation_mode.startswith('scenario'):
ngmfs = oq.number_of_ground_motion_fields
if oq.inputs['rupture_model'].endswith('.xml'):
# check the number of branchsets
bsets = len(gsim_lt._ltnode)
if bsets > 1:
raise InvalidFile(
'%s for a scenario calculation must contain a single '
'branchset, found %d!' % (oq.inputs['job_ini'], bsets))
[(trt, rlzs_by_gsim)] = gsim_lt.get_rlzs_by_gsim_trt().items()
self.cmaker = ContextMaker(
trt, rlzs_by_gsim, {
'maximum_distance': oq.maximum_distance(trt),
'minimum_distance': oq.minimum_distance,
'truncation_level': oq.truncation_level,
'imtls': oq.imtls
})
rup = readinput.get_rupture(oq)
if self.N > oq.max_sites_disagg: # many sites, split rupture
ebrs = [
EBRupture(copyobj(rup, rup_id=rup.rup_id + i),
'NA',
0,
G,
e0=i * G,
scenario=True) for i in range(ngmfs)
]
else: # keep a single rupture with a big occupation number
ebrs = [
EBRupture(rup,
'NA',
0,
G * ngmfs,
rup.rup_id,
scenario=True)
]
srcfilter = SourceFilter(self.sitecol, oq.maximum_distance(trt))
aw = get_rup_array(ebrs, srcfilter)
if len(aw) == 0:
raise RuntimeError(
'The rupture is too far from the sites! Please check the '
'maximum_distance and the position of the rupture')
elif oq.inputs['rupture_model'].endswith('.csv'):
aw = get_ruptures(oq.inputs['rupture_model'])
if len(gsim_lt.values) == 1: # fix for scenario_damage/case_12
aw['trt_smr'] = 0 # a single TRT
if oq.calculation_mode.startswith('scenario'):
# rescale n_occ by ngmfs and nrlzs
aw['n_occ'] *= ngmfs * gsim_lt.get_num_paths()
else:
raise InvalidFile("Something wrong in %s" % oq.inputs['job_ini'])
rup_array = aw.array
hdf5.extend(self.datastore['rupgeoms'], aw.geom)
if len(rup_array) == 0:
raise RuntimeError(
'There are no sites within the maximum_distance'
' of %s km from the rupture' %
oq.maximum_distance(rup.tectonic_region_type)(rup.mag))
fake = logictree.FullLogicTree.fake(gsim_lt)
self.realizations = fake.get_realizations()
self.datastore['full_lt'] = fake
self.store_rlz_info({}) # store weights
self.save_params()
imp = calc.RuptureImporter(self.datastore)
imp.import_rups_events(rup_array, get_rupture_getters)
def build_events_from_sources(self):
"""
Prefilter the composite source model and store the source_info
"""
oq = self.oqparam
params = dict(imtls=oq.imtls,
ses_per_logic_tree_path=oq.ses_per_logic_tree_path,
ses_seed=oq.ses_seed)
gsims_by_trt = self.csm.full_lt.get_gsims_by_trt()
sources = self.csm.get_sources()
# weighting the heavy sources
nrups = parallel.Starmap(count_ruptures,
[(src, )
for src in sources if src.code in b'AMC'],
progress=logging.debug).reduce()
for src in sources:
try:
src.num_ruptures = nrups[src.source_id]
except KeyError:
src.num_ruptures = src.count_ruptures()
src.weight = src.num_ruptures
maxweight = sum(sg.weight for sg in self.csm.src_groups) / (
self.oqparam.concurrent_tasks or 1)
eff_ruptures = AccumDict(accum=0) # grp_id => potential ruptures
source_data = AccumDict(accum=[])
allargs = []
if self.oqparam.is_ucerf():
# manage the filtering in a special way
for sg in self.csm.src_groups:
for src in sg:
src.src_filter = self.srcfilter
srcfilter = nofilter # otherwise it would be ultra-slow
else:
srcfilter = self.srcfilter
logging.info('Building ruptures')
for sg in self.csm.src_groups:
if not sg.sources:
continue
logging.info('Sending %s', sg)
cmaker = ContextMaker(sg.trt, gsims_by_trt[sg.trt], oq)
for src_group in sg.split(maxweight):
allargs.append((src_group, cmaker, srcfilter.sitecol))
smap = parallel.Starmap(sample_ruptures,
allargs,
h5=self.datastore.hdf5)
mon = self.monitor('saving ruptures')
self.nruptures = 0 # estimated classical ruptures within maxdist
for dic in smap:
# NB: dic should be a dictionary, but when the calculation dies
# for an OOM it can become None, thus giving a very confusing error
if dic is None:
raise MemoryError('You ran out of memory!')
rup_array = dic['rup_array']
if len(rup_array) == 0:
continue
if dic['source_data']:
source_data += dic['source_data']
if dic['eff_ruptures']:
eff_ruptures += dic['eff_ruptures']
with mon:
n = len(rup_array)
rup_array['id'] = numpy.arange(self.nruptures,
self.nruptures + n)
self.nruptures += n
hdf5.extend(self.datastore['ruptures'], rup_array)
hdf5.extend(self.datastore['rupgeoms'], rup_array.geom)
if len(self.datastore['ruptures']) == 0:
raise RuntimeError('No ruptures were generated, perhaps the '
'investigation time is too short')
# don't change the order of the 3 things below!
self.store_source_info(source_data)
self.store_rlz_info(eff_ruptures)
imp = calc.RuptureImporter(self.datastore)
with self.monitor('saving ruptures and events'):
imp.import_rups_events(
self.datastore.getitem('ruptures')[()], get_rupture_getters)
