def test_magnitude_bins(self):
""" Testing build disaggregation matrix """
fname = os.path.join(DATA_PATH, 'data', 'ssm.xml')
converter = sourceconverter.SourceConverter(50., 1., 10, 0.1, 10)
groups = to_python(fname, converter)
sources = []
for g in groups:
sources += g.sources
site = Site(Point(172.63, -43.53),
vs30=250,
vs30measured=False,
z1pt0=330)
imt = SA(3.0)
iml = 0.25612220
gsim_by_trt = {TRT.ACTIVE_SHALLOW_CRUST: Bradley2013()}
truncation_level = 3.0
n_epsilons = 1
mag_bin_width = 0.1
dist_bin_width = 100.
coord_bin_width = 100.
# Compute the disaggregation matrix
edges, mtx = disagg.disaggregation(sources, site, imt, iml,
gsim_by_trt, truncation_level,
n_epsilons, mag_bin_width,
dist_bin_width, coord_bin_width)
tm = disagg.mag_pmf(mtx[:, :, :, :, :, 0])
numpy.testing.assert_array_less(numpy.zeros_like(tm[2:]), tm[2:])
def get(xml,
investigation_time=50.,
rupture_mesh_spacing=5.,
width_of_mfd_bin=1.0,
area_source_discretization=10):
"""
:param xml: the XML representation of a source
:param investigation_time: investigation time
:param rupture_mesh_spacing: rupture mesh spacing
:param width_of_mfd_bin: width of MFD bin
:param area_source_discretization: area source discretization
:returns: a python source object
"""
text = '''<?xml version='1.0' encoding='UTF-8'?>
<nrml xmlns="http://openquake.org/xmlns/nrml/0.4"
xmlns:gml="http://www.opengis.net/gml">
%s
</nrml>''' % xml
[node] = read(gettemp(text))
conv = sourceconverter.SourceConverter(
investigation_time,
rupture_mesh_spacing,
width_of_mfd_bin=width_of_mfd_bin,
area_source_discretization=area_source_discretization)
src = conv.convert_node(node)
src.grp_id = src.id = 0
return src
def get_ltmodels(oq, gsim_lt, source_model_lt, h5=None):
"""
Build source models from the logic tree and to store
them inside the `source_info` dataset.
"""
if oq.pointsource_distance['default'] == {}:
spinning_off = False
else:
spinning_off = sum(oq.pointsource_distance.values()) == 0
if spinning_off:
logging.info('Removing nodal plane and hypocenter distributions')
# NB: the source models file are often NOT in the shared directory
# (for instance in oq-engine/demos) so the processpool must be used
dist = ('no' if os.environ.get('OQ_DISTRIBUTE') == 'no'
else 'processpool')
smlt_dir = os.path.dirname(source_model_lt.filename)
converter = sourceconverter.SourceConverter(
oq.investigation_time, oq.rupture_mesh_spacing,
oq.complex_fault_mesh_spacing, oq.width_of_mfd_bin,
oq.area_source_discretization, oq.minimum_magnitude,
not spinning_off, oq.source_id)
rlzs = source_model_lt.get_eff_rlzs()
lt_models = []
for rlz in rlzs:
ltm = LtSourceModel(
rlz['value'], rlz['weight'], rlz['lt_path'], [],
rlz['ordinal'], rlz['samples'], rlz['offset'])
lt_models.append(ltm)
if oq.calculation_mode.startswith('ucerf'):
idx = 0
[grp] = nrml.to_python(oq.inputs["source_model"], converter)
for grp_id, ltm in enumerate(lt_models):
sg = copy.copy(grp)
sg.id = grp_id
ltm.src_groups = [sg]
src = sg[0].new(ltm.ordinal, ltm.names) # one source
src.src_group_id = grp_id
idx += 1
src.samples = ltm.samples
sg.sources = [src]
data = [((grp_id, grp_id, src.source_id, src.code,
0, 0, -1, src.num_ruptures, 0, ''))]
sg.info = numpy.array(data, source_info_dt)
return lt_models
logging.info('Reading the source model(s) in parallel')
allargs = []
fileno = 0
for rlz in rlzs:
for name in rlz['value'].split():
fname = os.path.abspath(os.path.join(smlt_dir, name))
allargs.append((rlz['ordinal'], rlz['lt_path'],
source_model_lt.apply_uncertainties, fname,
fileno))
fileno += 1
smap = parallel.Starmap(
SourceReader(converter, smlt_dir, h5),
allargs, distribute=dist, h5=h5 if h5 else None)
# NB: h5 is None in logictree_test.py
return _store_results(smap, lt_models, source_model_lt, gsim_lt, oq, h5)
def test_alternative_mfds(self):
converter = s.SourceConverter(
investigation_time=1.,
rupture_mesh_spacing=1, # km
complex_fault_mesh_spacing=5, # km
width_of_mfd_bin=0.1, # for Truncated GR MFDs
area_source_discretization=1.)
grp_nodes = nrml.read(ALT_MFDS_SRC_MODEL).sourceModel.nodes
[[sflt1, sflt2], [cplx1]] = map(converter.convert_node, grp_nodes)
# Check the values
# Arbitrary MFD
assert_close(cplx1.mfd.magnitudes, [8.6, 8.8, 9.0])
assert_close(cplx1.mfd.occurrence_rates, [0.0006, 0.0008, 0.0004])
# Youngs & Coppersmith from characteristic rate
self.assertAlmostEqual(sflt1.mfd.b_val, 1.0)
self.assertAlmostEqual(sflt1.mfd.a_val, 3.3877843113)
self.assertAlmostEqual(sflt1.mfd.char_mag, 7.0)
self.assertAlmostEqual(sflt1.mfd.char_rate, 0.005)
self.assertAlmostEqual(sflt1.mfd.min_mag, 5.0)
# Youngs & Coppersmith from total moment rate
self.assertAlmostEqual(sflt2.mfd.b_val, 1.0)
self.assertAlmostEqual(sflt2.mfd.a_val, 5.0800, 3)
self.assertAlmostEqual(sflt2.mfd.char_mag, 7.0)
self.assertAlmostEqual(sflt2.mfd.char_rate, 0.24615, 5)
self.assertAlmostEqual(sflt2.mfd.min_mag, 5.0)
def test_nonparametric_source_ok(self):
converter = s.SourceConverter(
investigation_time=50.,
rupture_mesh_spacing=1, # km
complex_fault_mesh_spacing=1, # km
width_of_mfd_bin=1., # for Truncated GR MFDs
area_source_discretization=1.)
[np] = nrml.read(NONPARAMETRIC_SOURCE).sourceModel
converter.convert_node(np)
def test_duplicate_id(self):
parser = nrml.SourceModelParser(s.SourceConverter(
investigation_time=50.,
rupture_mesh_spacing=1,
complex_fault_mesh_spacing=1,
width_of_mfd_bin=0.1,
area_source_discretization=10,
))
with self.assertRaises(nrml.DuplicatedID):
parser.parse_groups(DUPLICATE_ID_SRC_MODEL)
def test_duplicate_id(self):
conv = s.SourceConverter(
investigation_time=50.,
rupture_mesh_spacing=1,
complex_fault_mesh_spacing=1,
width_of_mfd_bin=0.1,
area_source_discretization=10,
)
with self.assertRaises(nrml.DuplicatedID):
nrml.to_python(DUPLICATE_ID_SRC_MODEL, conv)
def setUpClass(cls):
cls.parser = nrml.SourceModelParser(s.SourceConverter(
investigation_time=50.,
rupture_mesh_spacing=1, # km
complex_fault_mesh_spacing=1, # km
width_of_mfd_bin=1., # for Truncated GR MFDs
area_source_discretization=1.))
cls.source_collector = {
sc.trt: sc for sc in cls.parser.parse_src_groups(MIXED_SRC_MODEL)}
cls.sitecol = site.SiteCollection(cls.SITES)
def setUpClass(cls):
conv = s.SourceConverter(
investigation_time=50.,
rupture_mesh_spacing=1, # km
complex_fault_mesh_spacing=1, # km
width_of_mfd_bin=1., # for Truncated GR MFDs
area_source_discretization=1.)
cls.source_collector = {
sc.trt: sc for sc in nrml.to_python(MIXED_SRC_MODEL, conv)}
cls.sitecol = site.SiteCollection(cls.SITES)
def get_ltmodels(oq, gsim_lt, source_model_lt, h5=None):
"""
Build source models from the logic tree and to store
them inside the `source_info` dataset.
"""
spinning_off = oq.collapse_factor == 0 or oq.pointsource_distance == 0
if spinning_off:
logging.info('Removing nodal plane and hypocenter distributions')
# NB: the source models file are often NOT in the shared directory
# (for instance in oq-engine/demos) so the processpool must be used
dist = ('no' if os.environ.get('OQ_DISTRIBUTE') == 'no'
else 'processpool')
smlt_dir = os.path.dirname(source_model_lt.filename)
converter = sourceconverter.SourceConverter(
oq.investigation_time, oq.rupture_mesh_spacing,
oq.complex_fault_mesh_spacing, oq.width_of_mfd_bin,
oq.area_source_discretization, oq.minimum_magnitude,
not spinning_off, oq.source_id)
if h5:
sources = hdf5.create(h5, 'source_info', source_info_dt)
lt_models = list(source_model_lt.gen_source_models(gsim_lt))
if oq.calculation_mode.startswith('ucerf'):
idx = 0
[grp] = nrml.to_python(oq.inputs["source_model"], converter)
for grp_id, ltm in enumerate(lt_models):
sg = copy.copy(grp)
sg.id = grp_id
ltm.src_groups = [sg]
src = sg[0].new(ltm.ordinal, ltm.names) # one source
src.src_group_id = grp_id
src.id = idx
idx += 1
if oq.number_of_logic_tree_samples:
src.samples = ltm.samples
sg.sources = [src]
data = [((grp_id, grp_id, src.source_id, src.code,
0, 0, -1, src.num_ruptures, 0, '', ''))]
hdf5.extend(sources, numpy.array(data, source_info_dt))
return lt_models
logging.info('Reading the source model(s) in parallel')
allargs = []
fileno = 0
for ltm in lt_models:
apply_unc = functools.partial(
source_model_lt.apply_uncertainties, ltm.path)
for name in ltm.names.split():
fname = os.path.abspath(os.path.join(smlt_dir, name))
allargs.append((ltm, apply_unc, fname, fileno))
fileno += 1
smap = parallel.Starmap(
SourceReader(converter, smlt_dir, h5),
allargs, distribute=dist, h5=h5 if h5 else None)
# NB: h5 is None in logictree_test.py
return _store_results(smap, lt_models, source_model_lt, gsim_lt, oq, h5)
def test(self):
fname = os.path.join(os.path.dirname(__file__),
'faults_backg_source_model.xml')
converter = sourceconverter.SourceConverter()
# check that the input file requires the fix indeed
with self.assertRaises(InvalidFile):
nrml.SourceModelParser(converter).parse_groups(fname)
fd, tmpname = tempfile.mkstemp(suffix='.xml')
os.close(fd)
fix(fname, tmpname) # invoke the fix
print('meld %s %s' % (fname, tmpname))
def setUpClass(cls):
cls.parser = nrml.SourceModelParser(s.SourceConverter(
investigation_time=50.,
rupture_mesh_spacing=1, # km
complex_fault_mesh_spacing=1, # km
width_of_mfd_bin=1., # for Truncated GR MFDs
area_source_discretization=1., # km
))
groups = cls.parser.parse_groups(MIXED_SRC_MODEL)
([cls.point], [cls.cmplx], [cls.area, cls.simple],
[cls.char_simple, cls.char_complex, cls.char_multi]) = groups
# the parameters here would typically be specified in the job .ini
cls.investigation_time = 50.
cls.rupture_mesh_spacing = 1 # km
cls.complex_fault_mesh_spacing = 1 # km
cls.width_of_mfd_bin = 1. # for Truncated GR MFDs
cls.area_source_discretization = 1. # km
def test(self):
mod = mock.Mock(reference_vs30_value=760,
reference_vs30_type='measured',
reference_depth_to_1pt0km_per_sec=100.,
reference_depth_to_2pt5km_per_sec=5.0,
reference_backarc=False)
sitecol = site.SiteCollection.from_points([102.32], [-2.9107], [0],
mod)
parser = nrml.SourceModelParser(
s.SourceConverter(
investigation_time=50.,
rupture_mesh_spacing=1, # km
complex_fault_mesh_spacing=1, # km
width_of_mfd_bin=1., # for Truncated GR MFDs
area_source_discretization=1., # km
))
[[src]] = parser.parse_groups(self.bad_source)
with self.assertRaises(AttributeError) as ctx, context(src):
max_dist = 250
# NB: with a distance of 200 km the error does not happen
src.filter_sites_by_distance_to_source(max_dist, sitecol)
self.assertIn('An error occurred with source id=61',
str(ctx.exception))
calling nrml.read() and node_to_obj() in sequence.
"""
[node] = read(fname)
return node_to_obj(node, fname, *args)
node_to_obj = CallableDict(keyfunc=get_tag_version, keymissing=lambda n, f: n)
# dictionary of functions with at least two arguments, node and fname
@node_to_obj.add(('ruptureCollection', 'nrml/0.5'))
def get_rupture_collection(node, fname, converter):
return converter.convert_node(node)
default = sourceconverter.SourceConverter() # rupture_mesh_spacing=10
@node_to_obj.add(('sourceModel', 'nrml/0.4'))
def get_source_model_04(node, fname, converter=default):
sources = []
source_ids = set()
converter.fname = fname
for src_node in node:
src = converter.convert_node(src_node)
if src.source_id in source_ids:
raise DuplicatedID('The source ID %s is duplicated!' %
src.source_id)
sources.append(src)
source_ids.add(src.source_id)
groups = groupby(sources, operator.attrgetter('tectonic_region_type'))
def get_source_models(oqparam,
gsim_lt,
source_model_lt,
monitor,
in_memory=True):
"""
Build all the source models generated by the logic tree.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param gsim_lt:
a :class:`openquake.commonlib.logictree.GsimLogicTree` instance
:param source_model_lt:
a :class:`openquake.commonlib.logictree.SourceModelLogicTree` instance
:param monitor:
a `openquake.baselib.performance.Monitor` instance
:param in_memory:
if True, keep in memory the sources, else just collect the TRTs
:returns:
an iterator over :class:`openquake.commonlib.logictree.LtSourceModel`
tuples
"""
make_sm = SourceModelFactory()
converter = sourceconverter.SourceConverter(
oqparam.investigation_time, oqparam.rupture_mesh_spacing,
oqparam.complex_fault_mesh_spacing, oqparam.width_of_mfd_bin,
oqparam.area_source_discretization, oqparam.source_id)
if oqparam.calculation_mode.startswith('ucerf'):
[grp] = nrml.to_python(oqparam.inputs["source_model"], converter)
elif in_memory:
logging.info('Reading the source model(s)')
dic = logictree.parallel_read_source_models(gsim_lt, source_model_lt,
converter, monitor)
# consider only the effective realizations
smlt_dir = os.path.dirname(source_model_lt.filename)
idx = 0
grp_id = 0
if monitor.hdf5:
sources = hdf5.create(monitor.hdf5, 'source_info', source_info_dt)
hdf5.create(monitor.hdf5, 'source_geom', point3d)
for sm in source_model_lt.gen_source_models(gsim_lt):
src_groups = []
for name in sm.names.split():
fname = os.path.abspath(os.path.join(smlt_dir, name))
if oqparam.calculation_mode.startswith('ucerf'):
sg = copy.copy(grp)
sg.id = grp_id
src = sg[0].new(sm.ordinal, sm.names) # one source
src.id = idx
sg.sources = [src]
src_groups.append(sg)
idx += 1
grp_id += 1
data = [((sg.id, src.source_id, src.code, 0, 0,
src.num_ruptures, 0, 0, 0, 0, 0))]
hdf5.extend(sources, numpy.array(data, source_info_dt))
elif in_memory:
apply_unc = source_model_lt.make_apply_uncertainties(sm.path)
newsm = make_sm(fname, dic[fname], apply_unc,
oqparam.investigation_time)
for sg in newsm:
for src in sg:
src.src_group_id = grp_id
src.id = idx
idx += 1
sg.id = grp_id
grp_id += 1
if monitor.hdf5:
store_sm(newsm, monitor.hdf5)
src_groups.extend(newsm.src_groups)
else: # just collect the TRT models
src_groups.extend(logictree.read_source_groups(fname))
if grp_id >= TWO16:
# the limit is really needed only for event based calculations
raise ValueError('There is a limit of %d src groups!' % TWO16)
num_sources = sum(len(sg.sources) for sg in src_groups)
sm.src_groups = src_groups
trts = [mod.trt for mod in src_groups]
source_model_lt.tectonic_region_types.update(trts)
logging.info(
'Processed source model %d with %d potential gsim path(s) and %d '
'sources', sm.ordinal + 1, sm.num_gsim_paths, num_sources)
gsim_file = oqparam.inputs.get('gsim_logic_tree')
if gsim_file: # check TRTs
for src_group in src_groups:
if src_group.trt not in gsim_lt.values:
raise ValueError(
"Found in %r a tectonic region type %r inconsistent "
"with the ones in %r" % (sm, src_group.trt, gsim_file))
yield sm
# log if some source file is being used more than once
dupl = 0
for fname, hits in make_sm.fname_hits.items():
if hits > 1:
logging.info('%s has been considered %d times', fname, hits)
if not make_sm.changed_sources:
dupl += hits
if (dupl and not oqparam.optimize_same_id_sources
and 'event_based' not in oqparam.calculation_mode):
logging.warn('You are doing redundant calculations: please make sure '
'that different sources have different IDs and set '
'optimize_same_id_sources=true in your .ini file')
if make_sm.changed_sources:
logging.info('Modified %d sources in the composite source model',
make_sm.changed_sources)
def get_csm(oq, full_lt, h5=None):
"""
Build source models from the logic tree and to store
them inside the `source_full_lt` dataset.
"""
converter = sourceconverter.SourceConverter(
oq.investigation_time, oq.rupture_mesh_spacing,
oq.complex_fault_mesh_spacing, oq.width_of_mfd_bin,
oq.area_source_discretization, oq.minimum_magnitude,
oq.source_id, discard_trts=oq.discard_trts,
floating_x_step=oq.floating_x_step,
floating_y_step=oq.floating_y_step)
classical = not oq.is_event_based()
full_lt.ses_seed = oq.ses_seed
if oq.is_ucerf():
[grp] = nrml.to_python(oq.inputs["source_model"], converter)
src_groups = []
for grp_id, sm_rlz in enumerate(full_lt.sm_rlzs):
sg = copy.copy(grp)
src_groups.append(sg)
src = sg[0].new(sm_rlz.ordinal, sm_rlz.value[0]) # one source
src.checksum = src.grp_id = src.trt_smr = grp_id
src.samples = sm_rlz.samples
logging.info('Reading sections and rupture planes for %s', src)
planes = src.get_planes()
if classical:
src.ruptures_per_block = oq.ruptures_per_block
sg.sources = list(src)
for s in sg:
s.planes = planes
s.sections = s.get_sections()
# add background point sources
sg = copy.copy(grp)
src_groups.append(sg)
sg.sources = src.get_background_sources()
else: # event_based, use one source
sg.sources = [src]
src.planes = planes
src.sections = src.get_sections()
return CompositeSourceModel(full_lt, src_groups)
logging.info('Reading the source model(s) in parallel')
# NB: the source models file are often NOT in the shared directory
# (for instance in oq-engine/demos) so the processpool must be used
dist = ('no' if os.environ.get('OQ_DISTRIBUTE') == 'no'
else 'processpool')
# NB: h5 is None in logictree_test.py
allargs = []
for fname in full_lt.source_model_lt.info.smpaths:
allargs.append((fname, converter))
smdict = parallel.Starmap(read_source_model, allargs, distribute=dist,
h5=h5 if h5 else None).reduce()
if len(smdict) > 1: # really parallel
parallel.Starmap.shutdown() # save memory
fix_geometry_sections(smdict)
groups = _build_groups(full_lt, smdict)
# checking the changes
changes = sum(sg.changes for sg in groups)
if changes:
logging.info('Applied {:_d} changes to the composite source model'.
format(changes))
return _get_csm(full_lt, groups)
def get_models(self):
"""
:yields: :class:`openquake.commonlib.logictree.LtSourceModel` tuples
"""
oq = self.oqparam
spinning_off = self.oqparam.pointsource_distance == {'default': 0.0}
if spinning_off:
logging.info('Removing nodal plane and hypocenter distributions')
dist = ('no'
if os.environ.get('OQ_DISTRIBUTE') == 'no' else 'processpool')
smlt_dir = os.path.dirname(self.source_model_lt.filename)
converter = sourceconverter.SourceConverter(
oq.investigation_time, oq.rupture_mesh_spacing,
oq.complex_fault_mesh_spacing, oq.width_of_mfd_bin,
oq.area_source_discretization, oq.minimum_magnitude,
not spinning_off, oq.source_id)
if oq.calculation_mode.startswith('ucerf'):
[grp] = nrml.to_python(oq.inputs["source_model"], converter)
dic = {'ucerf': grp}
elif self.in_memory:
logging.info('Reading the source model(s) in parallel')
smap = parallel.Starmap(
nrml.read_source_models,
distribute=dist,
hdf5path=self.hdf5.filename if self.hdf5 else None)
for sm in self.source_model_lt.gen_source_models(self.gsim_lt):
for name in sm.names.split():
fname = os.path.abspath(os.path.join(smlt_dir, name))
smap.submit([fname], converter)
dic = {sm.fname: sm for sm in smap}
else:
dic = {}
# consider only the effective realizations
idx = 0
if self.hdf5:
sources = hdf5.create(self.hdf5, 'source_info', source_info_dt)
hdf5.create(self.hdf5, 'source_geom', point3d)
hdf5.create(self.hdf5, 'source_mfds', hdf5.vstr)
grp_id = 0
for sm in self.source_model_lt.gen_source_models(self.gsim_lt):
if 'ucerf' in dic:
sg = copy.copy(dic['ucerf'])
sm.src_groups = [sg]
sg.id = grp_id
src = sg[0].new(sm.ordinal, sm.names) # one source
src.src_group_id = grp_id
src.id = idx
if oq.number_of_logic_tree_samples:
src.samples = sm.samples
sg.sources = [src]
idx += 1
grp_id += 1
data = [((sg.id, src.source_id, src.code, 0, 0, -1,
src.num_ruptures, 0, 0, 0, idx))]
hdf5.extend(sources, numpy.array(data, source_info_dt))
else:
self.apply_uncertainties(sm, idx, dic)
yield sm
if self.hdf5:
hdf5.extend(self.hdf5['source_mfds'],
numpy.array(list(self.mfds), hdf5.vstr))
# log if some source file is being used more than once
dupl = 0
for fname, hits in self.fname_hits.items():
if hits > 1:
logging.info('%s has been considered %d times', fname, hits)
if not self.changes:
dupl += hits
if self.changes:
logging.info('Applied %d changes to the composite source model',
self.changes)
return tag, version
def to_python(fname, *args):
"""
Parse a NRML file and return an associated Python object. It works by
calling nrml.read() and node_to_obj() in sequence.
"""
[node] = read(fname)
return node_to_obj(node, fname, *args)
node_to_obj = CallableDict(keyfunc=get_tag_version, keymissing=lambda n, f: n)
# dictionary of functions with at least two arguments, node and fname
default = sourceconverter.SourceConverter(area_source_discretization=10,
rupture_mesh_spacing=10)
@node_to_obj.add(('ruptureCollection', 'nrml/0.5'))
def get_rupture_collection(node, fname, converter):
return converter.convert_node(node)
@node_to_obj.add(('geometryModel', 'nrml/0.5'))
def get_geometry_model(node, fname, converter):
return GeometryModel(converter.convert_node(node))
@node_to_obj.add(('sourceModel', 'nrml/0.4'))
def get_source_model_04(node, fname, converter=default):
sources = []
def get_source_models(oqparam,
gsim_lt,
source_model_lt,
monitor,
in_memory=True,
srcfilter=None):
"""
Build all the source models generated by the logic tree.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param gsim_lt:
a :class:`openquake.commonlib.logictree.GsimLogicTree` instance
:param source_model_lt:
a :class:`openquake.commonlib.logictree.SourceModelLogicTree` instance
:param monitor:
a `openquake.baselib.performance.Monitor` instance
:param in_memory:
if True, keep in memory the sources, else just collect the TRTs
:param srcfilter:
a SourceFilter instance with an .filename pointing to the cache file
:returns:
an iterator over :class:`openquake.commonlib.logictree.LtSourceModel`
tuples
"""
make_sm = SourceModelFactory()
spinning_off = oqparam.pointsource_distance == {'default': 0.0}
if spinning_off:
logging.info('Removing nodal plane and hypocenter distributions')
dist = 'no' if os.environ.get('OQ_DISTRIBUTE') == 'no' else 'processpool'
smlt_dir = os.path.dirname(source_model_lt.filename)
converter = sourceconverter.SourceConverter(
oqparam.investigation_time, oqparam.rupture_mesh_spacing,
oqparam.complex_fault_mesh_spacing, oqparam.width_of_mfd_bin,
oqparam.area_source_discretization, oqparam.minimum_magnitude,
not spinning_off, oqparam.source_id)
if oqparam.calculation_mode.startswith('ucerf'):
[grp] = nrml.to_python(oqparam.inputs["source_model"], converter)
elif in_memory:
logging.info('Reading the source model(s) in parallel')
smap = parallel.Starmap(nrml.read_source_models,
monitor=monitor,
distribute=dist)
for sm in source_model_lt.gen_source_models(gsim_lt):
for name in sm.names.split():
fname = os.path.abspath(os.path.join(smlt_dir, name))
smap.submit([fname], converter)
dic = {sm.fname: sm for sm in smap}
# consider only the effective realizations
nr = 0
idx = 0
grp_id = 0
if monitor.hdf5:
sources = hdf5.create(monitor.hdf5, 'source_info', source_info_dt)
hdf5.create(monitor.hdf5, 'source_geom', point3d)
filename = None
source_ids = set()
for sm in source_model_lt.gen_source_models(gsim_lt):
apply_unc = functools.partial(source_model_lt.apply_uncertainties,
sm.path)
src_groups = []
for name in sm.names.split():
fname = os.path.abspath(os.path.join(smlt_dir, name))
if oqparam.calculation_mode.startswith('ucerf'):
sg = copy.copy(grp)
sg.id = grp_id
src = sg[0].new(sm.ordinal, sm.names) # one source
source_ids.add(src.source_id)
src.src_group_id = grp_id
src.id = idx
if oqparam.number_of_logic_tree_samples:
src.samples = sm.samples
sg.sources = [src]
src_groups.append(sg)
idx += 1
grp_id += 1
data = [((sg.id, src.source_id, src.code, 0, 0,
src.num_ruptures, 0, 0, 0))]
hdf5.extend(sources, numpy.array(data, source_info_dt))
elif in_memory:
newsm = make_sm(fname, dic[fname], apply_unc,
oqparam.investigation_time)
for sg in newsm:
nr += sum(src.num_ruptures for src in sg)
# sample a source for each group
if os.environ.get('OQ_SAMPLE_SOURCES'):
sg.sources = random_filtered_sources(
sg.sources, srcfilter, sg.id + oqparam.random_seed)
for src in sg:
source_ids.add(src.source_id)
src.src_group_id = grp_id
src.id = idx
idx += 1
sg.id = grp_id
grp_id += 1
src_groups.append(sg)
if monitor.hdf5:
store_sm(newsm, filename, monitor)
else: # just collect the TRT models
groups = logictree.read_source_groups(fname)
for group in groups:
source_ids.update(src['id'] for src in group)
src_groups.extend(groups)
if grp_id >= TWO16:
# the limit is really needed only for event based calculations
raise ValueError('There is a limit of %d src groups!' % TWO16)
for brid, srcids in source_model_lt.info.applytosources.items():
for srcid in srcids:
if srcid not in source_ids:
raise ValueError(
'The source %s is not in the source model, please fix '
'applyToSources in %s or the source model' %
(srcid, source_model_lt.filename))
num_sources = sum(len(sg.sources) for sg in src_groups)
sm.src_groups = src_groups
trts = [mod.trt for mod in src_groups]
source_model_lt.tectonic_region_types.update(trts)
logging.info(
'Processed source model %d with %d gsim path(s) and %d '
'sources', sm.ordinal + 1, sm.num_gsim_paths, num_sources)
gsim_file = oqparam.inputs.get('gsim_logic_tree')
if gsim_file: # check TRTs
for src_group in src_groups:
if src_group.trt not in gsim_lt.values:
raise ValueError(
"Found in %r a tectonic region type %r inconsistent "
"with the ones in %r" % (sm, src_group.trt, gsim_file))
yield sm
logging.info('The composite source model has {:,d} ruptures'.format(nr))
# log if some source file is being used more than once
dupl = 0
for fname, hits in make_sm.fname_hits.items():
if hits > 1:
logging.info('%s has been considered %d times', fname, hits)
if not make_sm.changes:
dupl += hits
if (dupl and not oqparam.optimize_same_id_sources
and not oqparam.is_event_based()):
logging.warning(
'You are doing redundant calculations: please make sure '
'that different sources have different IDs and set '
'optimize_same_id_sources=true in your .ini file')
if make_sm.changes:
logging.info('Applied %d changes to the composite source model',
make_sm.changes)
def get_csm(oq, source_model_lt, gsim_lt, h5=None):
"""
Build source models from the logic tree and to store
them inside the `source_full_lt` dataset.
"""
if oq.pointsource_distance['default'] == {}:
spinning_off = False
else:
spinning_off = sum(oq.pointsource_distance.values()) == 0
if spinning_off:
logging.info('Removing nodal plane and hypocenter distributions')
smlt_dir = os.path.dirname(source_model_lt.filename)
converter = sourceconverter.SourceConverter(
oq.investigation_time, oq.rupture_mesh_spacing,
oq.complex_fault_mesh_spacing, oq.width_of_mfd_bin,
oq.area_source_discretization, oq.minimum_magnitude,
not spinning_off, oq.source_id, discard_trts=oq.discard_trts)
full_lt = FullLogicTree(source_model_lt, gsim_lt)
classical = not oq.is_event_based()
if oq.is_ucerf():
sample = .001 if os.environ.get('OQ_SAMPLE_SOURCES') else None
[grp] = nrml.to_python(oq.inputs["source_model"], converter)
checksum = 0
src_groups = []
for grp_id, sm_rlz in enumerate(full_lt.sm_rlzs):
sg = copy.copy(grp)
src_groups.append(sg)
src = sg[0].new(sm_rlz.ordinal, sm_rlz.value) # one source
src.checksum = src.grp_id = src.id = grp_id
src.samples = sm_rlz.samples
if classical:
# split the sources upfront to improve the task distribution
sg.sources = src.get_background_sources(sample)
if not sample:
for s in src:
sg.sources.append(s)
s.checksum = checksum
checksum += 1
else: # event_based, use one source
sg.sources = [src]
return CompositeSourceModel(full_lt, src_groups)
logging.info('Reading the source model(s) in parallel')
groups = [[] for sm_rlz in full_lt.sm_rlzs]
allargs = []
fileno = 0
for rlz in full_lt.sm_rlzs:
for name in rlz.value.split():
fname = os.path.abspath(os.path.join(smlt_dir, name))
allargs.append((rlz.ordinal, rlz.lt_path,
source_model_lt.apply_uncertainties, fname,
fileno))
fileno += 1
# NB: the source models file are often NOT in the shared directory
# (for instance in oq-engine/demos) so the processpool must be used
dist = ('no' if os.environ.get('OQ_DISTRIBUTE') == 'no'
else 'processpool')
smap = parallel.Starmap(
SourceReader(converter, smlt_dir, h5),
allargs, distribute=dist, h5=h5 if h5 else None)
# NB: h5 is None in logictree_test.py
# various checks
changes = 0
for dic in sorted(smap, key=operator.itemgetter('fileno')):
eri = dic['ordinal']
groups[eri].extend(dic['src_groups'])
for sg in dic['src_groups']:
changes += sg.changes
gsim_file = oq.inputs.get('gsim_logic_tree')
if gsim_file: # check TRTs
for src_group in dic['src_groups']:
if src_group.trt not in gsim_lt.values:
raise ValueError(
"Found in the source models a tectonic region type %r "
"inconsistent with the ones in %r" %
(src_group.trt, gsim_file))
for sm_rlz in full_lt.sm_rlzs:
# check applyToSources
source_ids = set(src.source_id for grp in groups[sm_rlz.ordinal]
for src in grp)
for brid, srcids in source_model_lt.info.applytosources.items():
if brid in sm_rlz.lt_path:
for srcid in srcids:
if srcid not in source_ids:
raise ValueError(
"The source %s is not in the source model,"
" please fix applyToSources in %s or the "
"source model" % (srcid, source_model_lt.filename))
if changes:
logging.info('Applied %d changes to the composite source model',
changes)
return _get_csm(full_lt, groups)
def get_source_models(oqparam, gsim_lt, source_model_lt, in_memory=True):
"""
Build all the source models generated by the logic tree.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param gsim_lt:
a :class:`openquake.commonlib.logictree.GsimLogicTree` instance
:param source_model_lt:
a :class:`openquake.commonlib.logictree.SourceModelLogicTree` instance
:param in_memory:
if True, keep in memory the sources, else just collect the TRTs
:returns:
an iterator over :class:`openquake.commonlib.logictree.LtSourceModel`
tuples
"""
converter = sourceconverter.SourceConverter(
oqparam.investigation_time, oqparam.rupture_mesh_spacing,
oqparam.complex_fault_mesh_spacing, oqparam.width_of_mfd_bin,
oqparam.area_source_discretization)
psr = nrml.SourceModelParser(converter)
if oqparam.calculation_mode.startswith('ucerf'):
[grp] = nrml.to_python(oqparam.inputs["source_model"], converter)
# consider only the effective realizations
smlt_dir = os.path.dirname(source_model_lt.filename)
for sm in source_model_lt.gen_source_models(gsim_lt):
src_groups = []
for name in sm.names.split():
fname = os.path.abspath(os.path.join(smlt_dir, name))
if oqparam.calculation_mode.startswith('ucerf'):
sg = copy.copy(grp)
sg.id = sm.ordinal
sg.sources = [sg[0].new(sm.ordinal, sm.names)] # one source
src_groups.append(sg)
elif in_memory:
apply_unc = source_model_lt.make_apply_uncertainties(sm.path)
logging.info('Reading %s', fname)
src_groups.extend(psr.parse_src_groups(fname, apply_unc))
else: # just collect the TRT models
src_groups.extend(read_source_groups(fname))
num_sources = sum(len(sg.sources) for sg in src_groups)
sm.src_groups = src_groups
trts = [mod.trt for mod in src_groups]
source_model_lt.tectonic_region_types.update(trts)
logging.info(
'Processed source model %d with %d potential gsim path(s) and %d '
'sources', sm.ordinal + 1, sm.num_gsim_paths, num_sources)
gsim_file = oqparam.inputs.get('gsim_logic_tree')
if gsim_file: # check TRTs
for src_group in src_groups:
if src_group.trt not in gsim_lt.values:
raise ValueError(
"Found in %r a tectonic region type %r inconsistent "
"with the ones in %r" % (sm, src_group.trt, gsim_file))
yield sm
# check investigation_time
psr.check_nonparametric_sources(oqparam.investigation_time)
# log if some source file is being used more than once
dupl = 0
for fname, hits in psr.fname_hits.items():
if hits > 1:
logging.info('%s has been considered %d times', fname, hits)
if not psr.changed_sources:
dupl += hits
if dupl and not oqparam.optimize_same_id_sources:
logging.warn('You are doing redundant calculations: please make sure '
'that different sources have different IDs and set '
'optimize_same_id_sources=true in your .ini file')
def get_csm(oq, full_lt, h5=None):
"""
Build source models from the logic tree and to store
them inside the `source_full_lt` dataset.
"""
if oq.pointsource_distance is None:
spinning_off = False
else:
spinning_off = sum(oq.pointsource_distance.max().values()) == 0
if spinning_off:
logging.info('Removing nodal plane and hypocenter distributions')
converter = sourceconverter.SourceConverter(oq.investigation_time,
oq.rupture_mesh_spacing,
oq.complex_fault_mesh_spacing,
oq.width_of_mfd_bin,
oq.area_source_discretization,
oq.minimum_magnitude,
not spinning_off,
oq.source_id,
discard_trts=oq.discard_trts)
logging.info('%d effective smlt realization(s)', len(full_lt.sm_rlzs))
classical = not oq.is_event_based()
if oq.is_ucerf():
sample = .001 if os.environ.get('OQ_SAMPLE_SOURCES') else None
[grp] = nrml.to_python(oq.inputs["source_model"], converter)
src_groups = []
for grp_id, sm_rlz in enumerate(full_lt.sm_rlzs):
sg = copy.copy(grp)
src_groups.append(sg)
src = sg[0].new(sm_rlz.ordinal, sm_rlz.value) # one source
sg.mags = numpy.unique(numpy.round(src.mags, 2))
del src.__dict__['mags'] # remove cache
src.checksum = src.grp_id = src.id = grp_id
src.samples = sm_rlz.samples
if classical:
src.ruptures_per_block = oq.ruptures_per_block
if sample:
sg.sources = [list(src)[0]] # take the first source
else:
sg.sources = list(src)
# add background point sources
sg.sources.extend(src.get_background_sources(sample))
else: # event_based, use one source
sg.sources = [src]
return CompositeSourceModel(full_lt, src_groups)
logging.info('Reading the source model(s) in parallel')
if 'OQ_SAMPLE_SOURCES' in os.environ and h5:
srcfilter = calc.filters.SourceFilter(h5['sitecol'],
h5['oqparam'].maximum_distance)
else:
srcfilter = None
# NB: the source models file are often NOT in the shared directory
# (for instance in oq-engine/demos) so the processpool must be used
dist = ('no' if os.environ.get('OQ_DISTRIBUTE') == 'no' else 'processpool')
# NB: h5 is None in logictree_test.py
allargs = []
for fname in full_lt.source_model_lt.info.smpaths:
allargs.append((fname, converter, srcfilter))
smdict = parallel.Starmap(read_source_model,
allargs,
distribute=dist,
h5=h5 if h5 else None).reduce()
if len(smdict) > 1: # really parallel
parallel.Starmap.shutdown() # save memory
check_dupl_ids(smdict)
groups = _build_groups(full_lt, smdict)
# checking the changes
changes = sum(sg.changes for sg in groups)
if changes:
logging.info('Applied %d changes to the composite source model',
changes)
return _get_csm(full_lt, groups)
[node] = read(fname)
return node_to_obj(node, fname, *args)
parse = deprecated('Use nrml.to_python instead')(to_python)
node_to_obj = CallableDict(keyfunc=get_tag_version, keymissing=lambda n, f: n)
# dictionary of functions with at least two arguments, node and fname
@node_to_obj.add(('ruptureCollection', 'nrml/0.5'))
def get_rupture_collection(node, fname, converter):
return converter.convert_node(node)
default = sourceconverter.SourceConverter()
@node_to_obj.add(('sourceModel', 'nrml/0.4'))
def get_source_model_04(node, fname, converter=default):
sources = []
source_ids = set()
converter.fname = fname
for no, src_node in enumerate(node, 1):
src = converter.convert_node(src_node)
if src.source_id in source_ids:
raise DuplicatedID('The source ID %s is duplicated!' %
src.source_id)
sources.append(src)
source_ids.add(src.source_id)
if no % 10000 == 0: # log every 10,000 sources parsed
def read_input(hparams, **extra):
"""
:param hparams: a dictionary of hazard parameters
:returns: an Input namedtuple (groups, sitecol, gsim_lt, cmakerdict)
The dictionary must contain the keys
- "maximum_distance"
- "imtls"
- "source_model_file" or "rupture_model_file"
- "sites" or "site_model_file"
- "gsim" or "gsim_logic_tree_file"
Moreover:
- if "source_model_file" is given, then "investigation_time" is mandatory
- if "rupture_model_file" is given, the "number_of_ground_motion_fields"
and "ses_seed" are mandatory
- if there is an area source, then "area_source_discretization" is needed
- if "site_model_file" is missing, then global site parameters are needed
The optional keys include
- "rupture_mesh_spacing" (default 5.)
- "complex_fault_mesh_spacing" (default rupture_mesh_spacing)
- "width_of_mfd_bin" (default 1.)
- "minimum_magnitude"
- "discard_trts" (default "")
- "number_of_logic_tree_samples" (default 0)
- "ses_per_logic_tree_path" (default 1)
"""
if isinstance(hparams, str):
hparams = read_hparams(hparams)
if extra:
hparams = hparams.copy()
hparams.update(extra)
assert 'imts' in hparams or 'imtls' in hparams
assert isinstance(hparams['maximum_distance'], IntegrationDistance)
smfname = hparams.get('source_model_file')
if smfname: # nonscenario
itime = hparams['investigation_time']
else:
itime = 50. # ignored in scenario
rmfname = hparams.get('rupture_model_file')
if rmfname:
ngmfs = hparams["number_of_ground_motion_fields"]
ses_seed = hparams["ses_seed"]
converter = sourceconverter.SourceConverter(
itime,
hparams.get('rupture_mesh_spacing', 5.),
hparams.get('complex_fault_mesh_spacing'),
hparams.get('width_of_mfd_bin', 1.0),
hparams.get('area_source_discretization'),
hparams.get('minimum_magnitude', {'default': 0}),
hparams.get('source_id'),
discard_trts=hparams.get('discard_trts', ''))
if smfname:
[sm] = nrml.read_source_models([smfname], converter)
groups = sm.src_groups
elif rmfname:
ebrs = _get_ebruptures(rmfname, converter, ses_seed)
groups = _rupture_groups(ebrs)
else:
raise KeyError('Missing source_model_file or rupture_file')
trts = set(grp.trt for grp in groups)
if 'gsim' in hparams:
gslt = gsim_lt.GsimLogicTree.from_(hparams['gsim'])
else:
gslt = gsim_lt.GsimLogicTree(hparams['gsim_logic_tree_file'], trts)
# fix source attributes
idx = 0
num_rlzs = gslt.get_num_paths()
for grp_id, sg in enumerate(groups):
assert len(sg) # sanity check
for src in sg:
src.id = idx
src.grp_id = grp_id
src.trt_smr = grp_id
src.samples = num_rlzs
idx += 1
cmakerdict = {} # trt => cmaker
start = 0
n = hparams.get('number_of_logic_tree_samples', 0)
s = hparams.get('random_seed', 42)
for trt, rlzs_by_gsim in gslt.get_rlzs_by_gsim_trt(n, s).items():
cmakerdict[trt] = contexts.ContextMaker(trt, rlzs_by_gsim, hparams)
cmakerdict[trt].start = start
start += len(rlzs_by_gsim)
if rmfname:
# for instance for 2 TRTs with 5x2 GSIMs and ngmfs=10, then the
# number of occupation is 100 for each rupture, for a total
# of 200 events, see scenario/case_13
nrlzs = gslt.get_num_paths()
for grp in groups:
for ebr in grp:
ebr.n_occ = ngmfs * nrlzs
sitecol = _get_sitecol(hparams, gslt.req_site_params)
return Input(groups, sitecol, gslt, cmakerdict)
def get_source_models(oqparam, gsim_lt, source_model_lt, in_memory=True):
"""
Build all the source models generated by the logic tree.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param gsim_lt:
a :class:`openquake.commonlib.logictree.GsimLogicTree` instance
:param source_model_lt:
a :class:`openquake.commonlib.logictree.SourceModelLogicTree` instance
:param in_memory:
if True, keep in memory the sources, else just collect the TRTs
:returns:
an iterator over :class:`openquake.commonlib.logictree.SourceModel`
tuples
"""
converter = sourceconverter.SourceConverter(
oqparam.investigation_time, oqparam.rupture_mesh_spacing,
oqparam.complex_fault_mesh_spacing, oqparam.width_of_mfd_bin,
oqparam.area_source_discretization)
psr = nrml.SourceModelParser(converter)
# consider only the effective realizations
for sm in source_model_lt.gen_source_models(gsim_lt):
src_groups = []
for name in sm.name.split():
fname = possibly_gunzip(
os.path.abspath(os.path.join(oqparam.base_path, name)))
if in_memory:
apply_unc = source_model_lt.make_apply_uncertainties(sm.path)
try:
logging.info('Parsing %s', fname)
src_groups.extend(psr.parse_src_groups(fname, apply_unc))
except ValueError as e:
if str(e) in ('Surface does not conform with Aki & '
'Richards convention',
'Edges points are not in the right order'):
raise InvalidFile('''\
%s: %s. Probably you are using an obsolete model.
In that case you can fix the file with the command
python -m openquake.engine.tools.correct_complex_sources %s
''' % (fname, e, fname))
else:
raise
else: # just collect the TRT models
smodel = nrml.read(fname).sourceModel
if smodel[0].tag.endswith('sourceGroup'): # NRML 0.5 format
for sg_node in smodel:
sg = sourceconverter.SourceGroup(
sg_node['tectonicRegion'])
sg.sources = sg_node.nodes
src_groups.append(sg)
else: # NRML 0.4 format: smodel is a list of source nodes
src_groups.extend(
sourceconverter.SourceGroup.collect(smodel))
num_sources = sum(len(sg.sources) for sg in src_groups)
sm.src_groups = src_groups
trts = [mod.trt for mod in src_groups]
source_model_lt.tectonic_region_types.update(trts)
logging.info(
'Processed source model %d with %d potential gsim path(s) and %d '
'sources', sm.ordinal + 1, sm.num_gsim_paths, num_sources)
gsim_file = oqparam.inputs.get('gsim_logic_tree')
if gsim_file: # check TRTs
for src_group in src_groups:
if src_group.trt not in gsim_lt.values:
raise ValueError(
"Found in %r a tectonic region type %r inconsistent "
"with the ones in %r" % (sm, src_group.trt, gsim_file))
yield sm
# log if some source file is being used more than once
for fname, hits in psr.fname_hits.items():
if hits > 1:
logging.info('%s has been considered %d times', fname, hits)
