def get_composite_source_model(oqparam, h5=None):
"""
Parse the XML and build a complete composite source model in memory.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param h5:
an open hdf5.File where to store the source info
"""
ucerf = oqparam.calculation_mode.startswith('ucerf')
source_model_lt = get_source_model_lt(oqparam, validate=not ucerf)
trts = source_model_lt.tectonic_region_types
trts_lower = {trt.lower() for trt in trts}
reqv = oqparam.inputs.get('reqv', {})
for trt in reqv:
if trt.lower() not in trts_lower:
raise ValueError('Unknown TRT=%s in %s [reqv]' %
(trt, oqparam.inputs['job_ini']))
gsim_lt = get_gsim_lt(oqparam, trts or ['*'])
p = source_model_lt.num_paths * gsim_lt.get_num_paths()
if oqparam.number_of_logic_tree_samples:
logging.info('Considering {:_d} logic tree paths out of {:_d}'.format(
oqparam.number_of_logic_tree_samples, p))
else: # full enumeration
if (oqparam.is_event_based() and
(oqparam.ground_motion_fields or oqparam.hazard_curves_from_gmfs)
and p > oqparam.max_potential_paths):
raise ValueError(
'There are too many potential logic tree paths (%d):'
'use sampling instead of full enumeration or reduce the '
'source model with oq reduce_sm' % p)
logging.info('Potential number of logic tree paths = {:_d}'.format(p))
if source_model_lt.on_each_source:
logging.info('There is a logic tree on each source')
ltmodels = get_ltmodels(oqparam, gsim_lt, source_model_lt, h5)
csm = source.CompositeSourceModel(gsim_lt, source_model_lt, ltmodels)
key = operator.attrgetter('source_id', 'checksum')
srcidx = 0
if h5:
info = hdf5.create(h5, 'source_info', source_info_dt)
data = []
for k, srcs in groupby(csm.get_sources(), key).items():
for src in srcs:
src.id = srcidx
data.append((0, src.src_group_ids[0], src.source_id, src.code,
src.num_ruptures, 0, 0, 0, src.checksum, src._wkt))
srcidx += 1
if h5:
hdf5.extend(info, numpy.array(data, source_info_dt))
if oqparam.is_event_based():
# initialize the rupture rup_id numbers before splitting/filtering; in
# this way the serials are independent from the site collection
csm.init_serials(oqparam.ses_seed)
if oqparam.disagg_by_src:
csm = csm.grp_by_src() # one group per source
csm.info.gsim_lt.check_imts(oqparam.imtls)
return csm
def pre_execute(self):
"""
parse the logic tree and source model input
"""
self.sitecol = readinput.get_site_collection(self.oqparam)
self.save_mesh()
self.gsim_lt = readinput.get_gsim_lt(self.oqparam, [DEFAULT_TRT])
self.smlt = readinput.get_source_model_lt(self.oqparam)
parser = source.SourceModelParser(
UCERFSourceConverter(self.oqparam.investigation_time,
self.oqparam.rupture_mesh_spacing))
[self.source
] = parser.parse_sources(self.oqparam.inputs["source_model"])
branches = sorted(self.smlt.branches.items())
min_mag, max_mag = self.source.min_mag, None
source_models = []
num_gsim_paths = self.gsim_lt.get_num_paths()
for ordinal, (name, branch) in enumerate(branches):
tm = source.TrtModel(DEFAULT_TRT, [],
min_mag,
max_mag,
ordinal,
eff_ruptures=-1)
sm = source.SourceModel(name, branch.weight, [name], [tm],
num_gsim_paths, ordinal, 1)
source_models.append(sm)
self.csm = source.CompositeSourceModel(self.gsim_lt,
self.smlt,
source_models,
set_weight=False)
self.rup_data = {}
self.infos = []
def get_composite_source_model(oqparam, in_memory=True):
"""
Parse the XML and build a complete composite source model in memory.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param in_memory:
if False, just parse the XML without instantiating the sources
"""
smodels = []
grp_id = 0
idx = 0
gsim_lt = get_gsim_lt(oqparam)
source_model_lt = get_source_model_lt(oqparam)
if source_model_lt.on_each_source():
logging.info('There is a logic tree on each source')
for source_model in get_source_models(oqparam,
gsim_lt,
source_model_lt,
in_memory=in_memory):
for src_group in source_model.src_groups:
src_group.sources = sorted(src_group, key=getid)
src_group.id = grp_id
for src in src_group:
# there are two cases depending on the flag in_memory:
# 1) src is a hazardlib source and has a src_group_id
# attribute; in that case the source has to be numbered
# 2) src is a Node object, then nothing must be done
if isinstance(src, Node):
continue
src.src_group_id = grp_id
src.id = idx
idx += 1
grp_id += 1
if grp_id >= TWO16:
# the limit is really needed only for event based calculations
raise ValueError('There is a limit of %d src groups!' % TWO16)
smodels.append(source_model)
csm = source.CompositeSourceModel(gsim_lt, source_model_lt, smodels,
oqparam.optimize_same_id_sources)
for sm in csm.source_models:
counter = collections.Counter()
for sg in sm.src_groups:
for srcid in map(getid, sg):
counter[srcid] += 1
dupl = [srcid for srcid in counter if counter[srcid] > 1]
if dupl:
raise nrml.DuplicatedID('Found duplicated source IDs in %s: %s' %
(sm, dupl))
return csm
def get_composite_source_model(oqparam, h5=None):
"""
Parse the XML and build a complete composite source model in memory.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param h5:
an open hdf5.File where to store the source info
"""
ucerf = oqparam.calculation_mode.startswith('ucerf')
source_model_lt = get_source_model_lt(oqparam, validate=not ucerf)
trts = source_model_lt.tectonic_region_types
trts_lower = {trt.lower() for trt in trts}
reqv = oqparam.inputs.get('reqv', {})
for trt in reqv: # these are lowercase because they come from the job.ini
if trt not in trts_lower:
raise ValueError('Unknown TRT=%s in %s [reqv]' %
(trt, oqparam.inputs['job_ini']))
gsim_lt = get_gsim_lt(oqparam, trts or ['*'])
p = source_model_lt.num_paths * gsim_lt.get_num_paths()
if oqparam.number_of_logic_tree_samples:
logging.info('Considering {:,d} logic tree paths out of {:,d}'.format(
oqparam.number_of_logic_tree_samples, p))
else: # full enumeration
if (oqparam.is_event_based() and
(oqparam.ground_motion_fields or oqparam.hazard_curves_from_gmfs)
and p > oqparam.max_potential_paths):
raise ValueError(
'There are too many potential logic tree paths (%d):'
'use sampling instead of full enumeration or reduce the '
'source model with oq reduce_sm' % p)
logging.info('Potential number of logic tree paths = {:,d}'.format(p))
if source_model_lt.on_each_source:
logging.info('There is a logic tree on each source')
ltmodels = get_ltmodels(oqparam, gsim_lt, source_model_lt, h5)
csm = source.CompositeSourceModel(gsim_lt, source_model_lt, ltmodels)
if oqparam.is_event_based():
# initialize the rupture rup_id numbers before splitting/filtering; in
# this way the serials are independent from the site collection
csm.init_serials(oqparam.ses_seed)
if oqparam.disagg_by_src:
csm = csm.grp_by_src() # one group per source
csm.info.gsim_lt.check_imts(oqparam.imtls)
return csm
def get_composite_source_model(oqparam, in_memory=True):
"""
Parse the XML and build a complete composite source model in memory.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param in_memory:
if False, just parse the XML without instantiating the sources
"""
smodels = []
grp_id = 0
idx = 0
def getid(src):
try:
return src.source_id
except:
return src['id']
gsim_lt = get_gsim_lt(oqparam)
source_model_lt = get_source_model_lt(oqparam)
for source_model in get_source_models(oqparam,
gsim_lt,
source_model_lt,
in_memory=in_memory):
for src_group in source_model.src_groups:
src_group.sources = sorted(src_group, key=getid)
src_group.id = grp_id
for src in src_group:
# there are two cases depending on the flag in_memory:
# 1) src is a hazardlib source and has a src_group_id
# attribute; in that case the source has to be numbered
# 2) src is a Node object, then nothing must be done
if isinstance(src, Node):
continue
src.src_group_id = grp_id
src.id = idx
idx += 1
grp_id += 1
if grp_id >= TWO16:
# the limit is really needed only for event based calculations
raise ValueError('There is a limit of %d src groups!' % TWO16)
smodels.append(source_model)
csm = source.CompositeSourceModel(gsim_lt, source_model_lt, smodels,
in_memory)
return csm
def get_composite_source_model(oqparam, in_memory=True):
"""
Parse the XML and build a complete composite source model in memory.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param in_memory:
if False, just parse the XML without instantiating the sources
"""
source_model_lt = get_source_model_lt(oqparam)
smodels = []
trt_id = 0
idx = 0
def getid(src):
try:
return src.source_id
except:
return src['id']
gsim_lt = get_gsim_lt(oqparam)
for source_model in get_source_models(oqparam,
gsim_lt,
source_model_lt,
in_memory=in_memory):
for trt_model in source_model.trt_models:
trt_model.sources = sorted(trt_model, key=getid)
trt_model.id = trt_id
for src in trt_model:
# there are two cases depending on the flag in_memory:
# 1) src is a hazardlib source and has a trt_model_id
# attribute; in that case the source has to be numbered
# 2) src is a Node object, then nothing must be done
if hasattr(src, 'trt_model_id'):
# .trt_model_id is missing for source nodes
src.trt_model_id = trt_id
src.id = idx
idx += 1
trt_id += 1
smodels.append(source_model)
csm = source.CompositeSourceModel(gsim_lt, source_model_lt, smodels,
in_memory)
if hasattr(csm, 'weight'):
csm.maxweight = math.ceil(csm.weight / (oqparam.concurrent_tasks or 1))
return csm
def get_composite_source_model(oqparam,
sitecol=None,
SourceProcessor=source.SourceFilterSplitter,
monitor=DummyMonitor(),
no_distribute=parallel.no_distribute()):
"""
Build the source models by splitting the sources. If prefiltering is
enabled, also reduce the GSIM logic trees in the underlying source models.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param sitecol:
a :class:`openquake.hazardlib.site.SiteCollection` instance
:param SourceProcessor:
the class used to process the sources
:param monitor:
a monitor instance
:param no_distribute:
used to disable parallel splitting of the sources
:returns:
an iterator over :class:`openquake.commonlib.source.SourceModel`
"""
processor = SourceProcessor(sitecol, oqparam.maximum_distance,
oqparam.area_source_discretization)
source_model_lt = get_source_model_lt(oqparam)
smodels = []
trt_id = 0
for source_model in get_source_models(oqparam,
source_model_lt,
processor.sitecol,
in_memory=hasattr(
processor, 'process')):
for trt_model in source_model.trt_models:
trt_model.id = trt_id
trt_id += 1
smodels.append(source_model)
csm = source.CompositeSourceModel(source_model_lt, smodels)
if sitecol is not None and hasattr(processor, 'process'):
seqtime, partime = processor.process(csm, no_distribute)
monitor.write(['fast sources filtering/splitting', str(seqtime), '0'])
monitor.write(['slow sources filtering/splitting', str(partime), '0'])
if not csm.get_sources():
raise RuntimeError('All sources were filtered away')
csm.count_ruptures()
return csm
def get_composite_source_model(oq):
"""
:param oq: :class:`openquake.commonlib.oqvalidation.OqParam` instance
:returns: a `class:`openquake.commonlib.source.CompositeSourceModel`
"""
[src_group] = nrml.to_python(
oq.inputs["source_model"],
SourceConverter(oq.investigation_time, oq.rupture_mesh_spacing))
source_models = []
gsim_lt = readinput.get_gsim_lt(oq, [DEFAULT_TRT])
smlt = readinput.get_source_model_lt(oq)
for sm in smlt.gen_source_models(gsim_lt):
sg = copy.copy(src_group)
sg.id = sm.ordinal
sm.src_groups = [sg]
sg.sources = [sg[0].new(sm.ordinal, sm.names)]
source_models.append(sm)
return source.CompositeSourceModel(gsim_lt, smlt, source_models)
def get_composite_source_model(oqparam,
monitor=None,
in_memory=True,
srcfilter=SourceFilter(None, {})):
"""
Parse the XML and build a complete composite source model in memory.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param monitor:
a `openquake.baselib.performance.Monitor` instance
:param in_memory:
if False, just parse the XML without instantiating the sources
:param srcfilter:
if not None, use it to prefilter the sources
"""
ucerf = oqparam.calculation_mode.startswith('ucerf')
source_model_lt = get_source_model_lt(oqparam, validate=not ucerf)
trts = source_model_lt.tectonic_region_types
trts_lower = {trt.lower() for trt in trts}
reqv = oqparam.inputs.get('reqv', {})
for trt in reqv: # these are lowercase because they come from the job.ini
if trt not in trts_lower:
raise ValueError('Unknown TRT=%s in %s [reqv]' %
(trt, oqparam.inputs['job_ini']))
gsim_lt = get_gsim_lt(oqparam, trts or ['*'])
p = source_model_lt.num_paths * gsim_lt.get_num_paths()
if oqparam.number_of_logic_tree_samples:
logging.info('Considering {:,d} logic tree paths out of {:,d}'.format(
oqparam.number_of_logic_tree_samples, p))
else: # full enumeration
if oqparam.is_event_based() and p > oqparam.max_potential_paths:
raise ValueError(
'There are too many potential logic tree paths (%d) '
'use sampling instead of full enumeration' % p)
logging.info('Potential number of logic tree paths = {:,d}'.format(p))
if source_model_lt.on_each_source:
logging.info('There is a logic tree on each source')
if monitor is None:
monitor = performance.Monitor()
smodels = []
for source_model in get_source_models(oqparam, gsim_lt, source_model_lt,
monitor, in_memory, srcfilter):
for src_group in source_model.src_groups:
src_group.sources = sorted(src_group, key=getid)
for src in src_group:
# there are two cases depending on the flag in_memory:
# 1) src is a hazardlib source and has a src_group_id
# attribute; in that case the source has to be numbered
# 2) src is a Node object, then nothing must be done
if isinstance(src, Node):
continue
smodels.append(source_model)
csm = source.CompositeSourceModel(gsim_lt, source_model_lt, smodels,
oqparam.optimize_same_id_sources)
for sm in csm.source_models:
counter = collections.Counter()
for sg in sm.src_groups:
for srcid in map(getid, sg):
counter[srcid] += 1
dupl = [srcid for srcid in counter if counter[srcid] > 1]
if dupl:
raise nrml.DuplicatedID('Found duplicated source IDs in %s: %s' %
(sm, dupl))
if not in_memory:
return csm
if oqparam.is_event_based():
# initialize the rupture serial numbers before splitting/filtering; in
# this way the serials are independent from the site collection
csm.init_serials(oqparam.ses_seed)
if oqparam.disagg_by_src:
csm = csm.grp_by_src() # one group per source
csm.info.gsim_lt.check_imts(oqparam.imtls)
return csm
def get_composite_source_model(oqparam,
monitor=None,
in_memory=True,
split_all=True,
srcfilter=None):
"""
Parse the XML and build a complete composite source model in memory.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param monitor:
a `openquake.baselib.performance.Monitor` instance
:param in_memory:
if False, just parse the XML without instantiating the sources
:param split_all:
if True, split all the sources in the models
:param srcfilter:
if not None, use it to prefilter the sources
"""
smodels = []
gsim_lt = get_gsim_lt(oqparam)
source_model_lt = get_source_model_lt(oqparam)
if oqparam.number_of_logic_tree_samples == 0:
logging.info('Potential number of logic tree paths = {:,d}'.format(
source_model_lt.num_paths * gsim_lt.get_num_paths()))
if source_model_lt.on_each_source:
logging.info('There is a logic tree on each source')
if monitor is None:
monitor = performance.Monitor()
for source_model in get_source_models(oqparam,
gsim_lt,
source_model_lt,
monitor,
in_memory=in_memory):
for src_group in source_model.src_groups:
src_group.sources = sorted(src_group, key=getid)
for src in src_group:
# there are two cases depending on the flag in_memory:
# 1) src is a hazardlib source and has a src_group_id
# attribute; in that case the source has to be numbered
# 2) src is a Node object, then nothing must be done
if isinstance(src, Node):
continue
smodels.append(source_model)
csm = source.CompositeSourceModel(gsim_lt, source_model_lt, smodels,
oqparam.optimize_same_id_sources)
for sm in csm.source_models:
counter = collections.Counter()
for sg in sm.src_groups:
for srcid in map(getid, sg):
counter[srcid] += 1
dupl = [srcid for srcid in counter if counter[srcid] > 1]
if dupl:
raise nrml.DuplicatedID('Found duplicated source IDs in %s: %s' %
(sm, dupl))
if not in_memory:
return csm
if 'event_based' in oqparam.calculation_mode:
if oqparam.pointsource_distance == 0:
# remove splitting/floating ruptures
for src in csm.get_sources():
if hasattr(src, 'hypocenter_distribution'):
src.hypocenter_distribution.reduce()
src.nodal_plane_distribution.reduce()
src.num_ruptures = src.count_ruptures()
# initialize the rupture serial numbers before splitting/filtering; in
# this way the serials are independent from the site collection
csm.init_serials(oqparam.ses_seed)
# TODO: check why the seeds still depend on the minimun_magnitude
set_min_mag(csm.get_sources(), oqparam.minimum_magnitude)
if oqparam.disagg_by_src:
csm = csm.grp_by_src() # one group per source
csm.info.gsim_lt.check_imts(oqparam.imtls)
if monitor.hdf5:
csm.info.gsim_lt.store_gmpe_tables(monitor.hdf5)
# splitting assumes that the serials have been initialized already
if split_all and 'ucerf' not in oqparam.calculation_mode:
csm = parallel_split_filter(csm, srcfilter, oqparam.random_seed,
monitor('prefilter'))
return csm