def test_applyToSources(self):
oq = readinput.get_oqparam('job.ini', case_21)
with mock.patch('logging.info') as info:
readinput.get_composite_source_model(oq)
self.assertEqual(
info.call_args[0],
('Applied %d changes to the composite source model', 81))
def test_wrong_trts(self):
# invalid TRT in job.ini [reqv]
oq = readinput.get_oqparam('job.ini', case_2)
fname = oq.inputs['reqv'].pop('active shallow crust')
oq.inputs['reqv']['act shallow crust'] = fname
with self.assertRaises(ValueError) as ctx:
readinput.get_composite_source_model(oq, in_memory=False)
self.assertIn('Unknown TRT=act shallow crust', str(ctx.exception))
def test_one_rlz(self):
oqparam = tests.get_oqparam('classical_job.ini')
# the example has number_of_logic_tree_samples = 1
sitecol = readinput.get_site_collection(oqparam)
csm = readinput.get_composite_source_model(oqparam, sitecol)
self.assertEqual(str(csm[0].gsim_lt), '''\
<GsimLogicTree
Active Shallow Crust,b1,SadighEtAl1997,w=0.5
Active Shallow Crust,b2,ChiouYoungs2008,w=0.5
Subduction Interface,b3,SadighEtAl1997,w=1.0>''')
assoc = csm.get_rlzs_assoc(
lambda trtmod: sum(src.count_ruptures() for src in trtmod.sources))
[rlz] = assoc.realizations
self.assertEqual(assoc.gsim_by_trt[rlz.ordinal],
{'Subduction Interface': 'SadighEtAl1997',
'Active Shallow Crust': 'ChiouYoungs2008'})
# ignoring the end of the tuple, with the uid field
self.assertEqual(rlz.ordinal, 0)
self.assertEqual(rlz.sm_lt_path, ('b1', 'b5', 'b8'))
self.assertEqual(rlz.gsim_lt_path, ('b2', 'b3'))
self.assertEqual(rlz.weight, 1.)
self.assertEqual(
str(assoc),
"<RlzsAssoc(2)\n0,SadighEtAl1997: ['<0,b1_b5_b8,b2_b3,w=1.0>']\n"
"1,ChiouYoungs2008: ['<0,b1_b5_b8,b2_b3,w=1.0>']>")
def initialize_sources(self):
"""
Parse source models, apply uncertainties and validate source logic
trees. Save in the database LtSourceModel and TrtModel objects.
"""
logs.LOG.progress("initializing sources")
parallel_source_splitting = valid.boolean(
config.get('hazard', 'parallel_source_splitting') or 'false')
self.composite_model = readinput.get_composite_source_model(
self.oqparam, self.site_collection,
no_distribute=not parallel_source_splitting)
for sm in self.composite_model:
# create an LtSourceModel for each distinct source model
lt_model = models.LtSourceModel.objects.create(
hazard_calculation=self.job,
sm_lt_path=self.tilepath + sm.path,
ordinal=sm.ordinal, sm_name=sm.name, weight=sm.weight,
samples=sm.samples)
self._source_models.append(lt_model)
gsims_by_trt = sm.gsim_lt.values
# save TrtModels for each tectonic region type
# and stored the db ID in the in-memory models
for trt_mod in sm.trt_models:
trt_mod.id = models.TrtModel.objects.create(
lt_model=lt_model,
tectonic_region_type=trt_mod.trt,
num_sources=len(trt_mod),
num_ruptures=trt_mod.num_ruptures,
min_mag=trt_mod.min_mag,
max_mag=trt_mod.max_mag,
gsims=gsims_by_trt[trt_mod.trt]).id
# rebuild the info object with the trt_ids coming from the db
self.composite_model.info = source.CompositionInfo(
self.composite_model.source_model_lt,
self.composite_model.source_models)
def read_sources(self):
"""
Read the composite source model (if any).
This method must be called after read_exposure_sitecol, to be able
to filter to sources according to the site collection.
"""
if 'source' in self.oqparam.inputs:
logging.info('Reading the composite source model')
with self.monitor(
'reading composite source model', autoflush=True):
self.csm = readinput.get_composite_source_model(
self.oqparam, self.sitecol, self.SourceProcessor,
self.monitor)
# we could manage limits here
self.source_pre_info = self.csm.source_info
self.job_info = readinput.get_job_info(
self.oqparam, self.csm, self.sitecol)
self.csm.count_ruptures()
self.rlzs_assoc = self.csm.get_rlzs_assoc()
logging.info(
'Total weight of the sources=%s',
self.job_info['input_weight'])
logging.info(
'Expected output size=%s',
self.job_info['output_weight'])
def test_one_rlz(self):
oqparam = tests.get_oqparam('classical_job.ini')
# the example has number_of_logic_tree_samples = 1
csm = readinput.get_composite_source_model(oqparam)
# check the attributes of the groups are set
[grp0, grp1] = csm.src_groups
for grp in csm.src_groups:
self.assertEqual(grp.src_interdep, 'indep')
self.assertEqual(grp.rup_interdep, 'indep')
self.assertEqual(repr(csm.gsim_lt), '''\
<GsimLogicTree
Active Shallow Crust,b1,[SadighEtAl1997],w=0.5
Active Shallow Crust,b2,[ChiouYoungs2008],w=0.5
Subduction Interface,b3,[SadighEtAl1997],w=1.0>''')
assoc = csm.info.get_rlzs_assoc()
[rlz] = assoc.realizations
self.assertEqual(assoc.gsim_by_trt[rlz.ordinal],
{'Subduction Interface': '[SadighEtAl1997]',
'Active Shallow Crust': '[ChiouYoungs2008]'})
# ignoring the end of the tuple, with the uid field
self.assertEqual(rlz.ordinal, 0)
self.assertEqual(rlz.sm_lt_path, ('b1', 'b4', 'b7'))
self.assertEqual(rlz.gsim_lt_path, ('b2', 'b3'))
self.assertEqual(rlz.weight['default'], 1.)
self.assertEqual(
str(assoc),
"<RlzsAssoc(size=2, rlzs=1)\n0,'[SadighEtAl1997]': "
"[0]\n1,'[ChiouYoungs2008]': [0]>")
def pre_execute(self):
"""
Check if there is a pre_calculator or a previous calculation ID.
If yes, read the inputs by invoking the precalculator or by retrieving
the previous calculation; if not, read the inputs directly.
"""
job_info = hdf5.LiteralAttrs()
if self.pre_calculator is not None:
# the parameter hazard_calculation_id is only meaningful if
# there is a precalculator
precalc_id = self.oqparam.hazard_calculation_id
if precalc_id is None: # recompute everything
precalc = calculators[self.pre_calculator](
self.oqparam, self.monitor("precalculator"), self.datastore.calc_id
)
precalc.run()
if "scenario" not in self.oqparam.calculation_mode:
self.csm = precalc.csm
pre_attrs = vars(precalc)
for name in ("riskmodel", "assets_by_site"):
if name in pre_attrs:
setattr(self, name, getattr(precalc, name))
else: # read previously computed data
parent = datastore.read(precalc_id)
self.datastore.set_parent(parent)
# copy missing parameters from the parent
params = {
name: value for name, value in vars(parent["oqparam"]).items() if name not in vars(self.oqparam)
}
self.save_params(**params)
self.read_risk_data()
self.init()
else: # we are in a basic calculator
self.read_risk_data()
if "source" in self.oqparam.inputs:
with self.monitor("reading composite source model", autoflush=True):
self.csm = readinput.get_composite_source_model(self.oqparam)
self.datastore["csm_info"] = self.csm.info
self.rup_data = {}
# we could manage limits here
vars(job_info).update(readinput.get_job_info(self.oqparam, self.csm, self.sitecol))
logging.info("Expected output size=%s", job_info.hazard["output_weight"])
logging.info("Total weight of the sources=%s", job_info.hazard["input_weight"])
self.init()
if "source" in self.oqparam.inputs:
with self.monitor("managing sources", autoflush=True):
self.send_sources()
self.manager.store_source_info(self.datastore)
attrs = self.datastore.hdf5["composite_source_model"].attrs
attrs["weight"] = self.csm.weight
attrs["filtered_weight"] = self.csm.filtered_weight
attrs["maxweight"] = self.csm.maxweight
job_info.hostname = socket.gethostname()
if hasattr(self, "riskmodel"):
job_info.require_epsilons = bool(self.riskmodel.covs)
self.job_info = job_info
self.datastore.flush()
def test_many_rlzs(self):
oqparam = tests.get_oqparam('classical_job.ini')
oqparam.number_of_logic_tree_samples = 0
sitecol = readinput.get_site_collection(oqparam)
csm = readinput.get_composite_source_model(oqparam, sitecol)
self.assertEqual(len(csm), 9) # the smlt example has 1 x 3 x 3 paths;
# there are 2 distinct tectonic region types, so 18 trt_models
rlzs_assoc = csm.get_rlzs_assoc()
rlzs = rlzs_assoc.realizations
self.assertEqual(len(rlzs), 18) # the gsimlt has 1 x 2 paths
self.assertEqual([1, 584, 1, 584, 1, 584, 1, 582, 1, 582,
1, 582, 1, 582, 1, 582, 1, 582],
map(len, csm.trt_models))
# test the method csm_info.get_col_ids
col_ids_first = rlzs_assoc.csm_info.get_col_ids(rlzs[0])
self.assertEqual(col_ids_first, set([0, 1]))
col_ids_last = rlzs_assoc.csm_info.get_col_ids(rlzs[-1])
self.assertEqual(col_ids_last, set([16, 17]))
# removing 9 trt_models out of 18
for trt_model in csm.trt_models:
if trt_model.trt == 'Active Shallow Crust': # no ruptures
trt_model.num_ruptures = 0
assoc = csm.get_rlzs_assoc()
expected_assoc = """\
<RlzsAssoc(18)
0,SadighEtAl1997: ['<0,b1_b3_b6,@_b3,w=0.04>']
1,SadighEtAl1997: ['<0,b1_b3_b6,@_b3,w=0.04>']
2,SadighEtAl1997: ['<1,b1_b3_b7,@_b3,w=0.12>']
3,SadighEtAl1997: ['<1,b1_b3_b7,@_b3,w=0.12>']
4,SadighEtAl1997: ['<2,b1_b3_b8,@_b3,w=0.04>']
5,SadighEtAl1997: ['<2,b1_b3_b8,@_b3,w=0.04>']
6,SadighEtAl1997: ['<3,b1_b4_b6,@_b3,w=0.12>']
7,SadighEtAl1997: ['<3,b1_b4_b6,@_b3,w=0.12>']
8,SadighEtAl1997: ['<4,b1_b4_b7,@_b3,w=0.36>']
9,SadighEtAl1997: ['<4,b1_b4_b7,@_b3,w=0.36>']
10,SadighEtAl1997: ['<5,b1_b4_b8,@_b3,w=0.12>']
11,SadighEtAl1997: ['<5,b1_b4_b8,@_b3,w=0.12>']
12,SadighEtAl1997: ['<6,b1_b5_b6,@_b3,w=0.04>']
13,SadighEtAl1997: ['<6,b1_b5_b6,@_b3,w=0.04>']
14,SadighEtAl1997: ['<7,b1_b5_b7,@_b3,w=0.12>']
15,SadighEtAl1997: ['<7,b1_b5_b7,@_b3,w=0.12>']
16,SadighEtAl1997: ['<8,b1_b5_b8,@_b3,w=0.04>']
17,SadighEtAl1997: ['<8,b1_b5_b8,@_b3,w=0.04>']>"""
self.assertEqual(str(assoc), expected_assoc)
self.assertEqual(len(assoc.realizations), 9)
# removing all trt_models
for trt_model in csm.trt_models:
if trt_model.trt == 'Subduction Interface': # no ruptures
trt_model.num_ruptures = 0
self.assertEqual(csm.get_rlzs_assoc().realizations, [])
def print_csm_info(fname):
"""
Parse the composite source model without instantiating the sources and
prints information about its composition and the full logic tree
"""
oqparam = readinput.get_oqparam(fname)
csm = readinput.get_composite_source_model(oqparam, in_memory=False)
print(csm.info)
print('See https://github.com/gem/oq-risklib/blob/master/doc/'
'effective-realizations.rst for an explanation')
print(csm.info.get_rlzs_assoc())
def read_inputs(self):
"""
Read risk data and sources if any
"""
oq = self.oqparam
self._read_risk_data()
self.check_overflow() # check if self.sitecol is too large
if ('source_model_logic_tree' in oq.inputs and
oq.hazard_calculation_id is None):
self.csm = readinput.get_composite_source_model(
oq, self.monitor(), srcfilter=self.src_filter)
self.init() # do this at the end of pre-execute
def print_csm_info(fname):
"""
Parse the composite source model without instantiating the sources and
prints information about its composition and the full logic tree
"""
oqparam = readinput.get_oqparam(fname)
csm = readinput.get_composite_source_model(oqparam, in_memory=False)
print(csm.info)
print('See https://github.com/gem/oq-risklib/blob/master/doc/'
'effective-realizations.rst for an explanation')
rlzs_assoc = csm.info.get_rlzs_assoc()
print(rlzs_assoc)
tot, pairs = get_pickled_sizes(rlzs_assoc)
print(views.rst_table(pairs, ['attribute', 'nbytes']))
def test_many_rlzs(self):
oqparam = tests.get_oqparam('classical_job.ini')
oqparam.number_of_logic_tree_samples = 0
sitecol = readinput.get_site_collection(oqparam)
csm = readinput.get_composite_source_model(oqparam, sitecol)
self.assertEqual(len(csm), 9) # the smlt example has 1 x 3 x 3 paths;
# there are 2 distinct tectonic region types, so 18 trt_models
self.assertEqual(sum(1 for tm in csm.trt_models), 18)
rlzs_assoc = csm.info.get_rlzs_assoc()
rlzs = rlzs_assoc.realizations
self.assertEqual(len(rlzs), 18) # the gsimlt has 1 x 2 paths
# counting the sources in each TRT model (unsplit)
self.assertEqual(
[1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2],
list(map(len, csm.trt_models)))
# test the method extract
assoc = rlzs_assoc.extract([1, 5])
self.assertEqual(str(assoc), """\
<RlzsAssoc(size=4, rlzs=2)
0,SadighEtAl1997: ['<1,b1_b3_b6,b2_b3,w=0.5>']
1,ChiouYoungs2008: ['<1,b1_b3_b6,b2_b3,w=0.5>']
4,SadighEtAl1997: ['<5,b1_b3_b8,b2_b3,w=0.5>']
5,ChiouYoungs2008: ['<5,b1_b3_b8,b2_b3,w=0.5>']>""")
# removing 9 trt_models out of 18
def count_ruptures(trt_model):
if trt_model.trt == 'Active Shallow Crust': # no ruptures
return 0
else:
return 1
assoc = csm.info.get_rlzs_assoc(count_ruptures)
expected_assoc = """\
<RlzsAssoc(size=9, rlzs=9)
0,SadighEtAl1997: ['<0,b1_b3_b6,@_b3,w=0.04>']
2,SadighEtAl1997: ['<1,b1_b3_b7,@_b3,w=0.12>']
4,SadighEtAl1997: ['<2,b1_b3_b8,@_b3,w=0.04>']
6,SadighEtAl1997: ['<3,b1_b4_b6,@_b3,w=0.12>']
8,SadighEtAl1997: ['<4,b1_b4_b7,@_b3,w=0.36>']
10,SadighEtAl1997: ['<5,b1_b4_b8,@_b3,w=0.12>']
12,SadighEtAl1997: ['<6,b1_b5_b6,@_b3,w=0.04>']
14,SadighEtAl1997: ['<7,b1_b5_b7,@_b3,w=0.12>']
16,SadighEtAl1997: ['<8,b1_b5_b8,@_b3,w=0.04>']>"""
self.assertEqual(str(assoc), expected_assoc)
self.assertEqual(len(assoc.realizations), 9)
# removing all trt_models
self.assertEqual(csm.info.get_rlzs_assoc(lambda t: 0).realizations, [])
def basic_pre_execute(self):
self.read_risk_data()
if 'source' in self.oqparam.inputs:
with self.monitor(
'reading composite source model', autoflush=True):
self.csm = readinput.get_composite_source_model(self.oqparam)
self.datastore['csm_info'] = self.csm.info
self.rup_data = {}
self.init()
if 'source' in self.oqparam.inputs:
with self.monitor('managing sources', autoflush=True):
self.taskman = self.send_sources()
attrs = self.datastore.hdf5['composite_source_model'].attrs
attrs['weight'] = self.csm.weight
attrs['filtered_weight'] = self.csm.filtered_weight
def print_csm_info(fname):
"""
Parse the composite source model without instantiating the sources and
prints information about its composition and the full logic tree
"""
oqparam = readinput.get_oqparam(fname)
csm = readinput.get_composite_source_model(oqparam, in_memory=False)
print(csm.info)
print('See http://docs.openquake.org/oq-engine/stable/'
'effective-realizations.html for an explanation')
rlzs_assoc = csm.info.get_rlzs_assoc()
print(rlzs_assoc)
dupl = [(srcs[0]['id'], len(srcs)) for srcs in csm.check_dupl_sources()]
if dupl:
print(rst_table(dupl, ['source_id', 'multiplicity']))
tot, pairs = get_pickled_sizes(rlzs_assoc)
print(rst_table(pairs, ['attribute', 'nbytes']))
def test_oversampling(self):
from openquake.qa_tests_data.classical import case_17
oq = readinput.get_oqparam(
os.path.join(os.path.dirname(case_17.__file__), 'job.ini'))
sitecol = readinput.get_site_collection(oq)
csm = readinput.get_composite_source_model(oq, sitecol)
assoc = csm.info.get_rlzs_assoc()
self.assertEqual(
str(assoc),
"<RlzsAssoc(size=2, rlzs=5)\n"
"0,SadighEtAl1997: ['<0,b1,b1,w=0.2>']\n"
"1,SadighEtAl1997: ['<1,b2,b1,w=0.2>', '<2,b2,b1,w=0.2>', '<3,b2,b1,w=0.2>', '<4,b2,b1,w=0.2>']>")
# check CompositionInfo serialization
array, attrs = assoc.csm_info.__toh5__()
new = object.__new__(CompositionInfo)
new.__fromh5__(array, attrs)
self.assertEqual(repr(new), repr(assoc.csm_info))
def test_oversampling(self):
from openquake.qa_tests_data.classical import case_17
oq = readinput.get_oqparam(
os.path.join(os.path.dirname(case_17.__file__), 'job.ini'))
csm = readinput.get_composite_source_model(oq)
csm.info.update_eff_ruptures(lambda tm: 1)
assoc = csm.info.get_rlzs_assoc()
self.assertEqual(
str(assoc),
"<RlzsAssoc(size=2, rlzs=5)\n"
"0,'[SadighEtAl1997]': [0 1 2]\n"
"1,'[SadighEtAl1997]': [3 4]>")
# check CompositionInfo serialization
dic, attrs = csm.info.__toh5__()
new = object.__new__(CompositionInfo)
new.__fromh5__(dic, attrs)
self.assertEqual(repr(new), repr(csm.info).
replace('0.20000000000000004', '0.2'))
def _info(name, filtersources, weightsources):
if name in base.calculators:
print(textwrap.dedent(base.calculators[name].__doc__.strip()))
elif name == 'gsims':
for gs in gsim.get_available_gsims():
print(gs)
elif name.endswith('.xml'):
print(nrml.read(name).to_str())
elif name.endswith(('.ini', '.zip')):
oqparam = readinput.get_oqparam(name)
if 'exposure' in oqparam.inputs:
expo = readinput.get_exposure(oqparam)
sitecol, assets_by_site = readinput.get_sitecol_assets(
oqparam, expo)
elif filtersources or weightsources:
sitecol, assets_by_site = readinput.get_site_collection(
oqparam), []
else:
sitecol, assets_by_site = None, []
if 'source_model_logic_tree' in oqparam.inputs:
print('Reading the source model...')
if weightsources:
sp = source.SourceFilterWeighter
elif filtersources:
sp = source.SourceFilter
else:
sp = source.BaseSourceProcessor # do nothing
csm = readinput.get_composite_source_model(oqparam, sitecol, sp)
assoc = csm.get_rlzs_assoc()
_print_info(
dict(rlzs_assoc=assoc, oqparam=oqparam,
composite_source_model=csm, sitecol=sitecol),
filtersources, weightsources)
if len(assets_by_site):
assetcol = riskinput.build_asset_collection(assets_by_site)
dic = groupby(assetcol, operator.attrgetter('taxonomy'))
for taxo, num in dic.items():
print('taxonomy #%d, %d assets' % (taxo, num))
print('total assets = %d' % len(assetcol))
else:
print("No info for '%s'" % name)
def test_many_rlzs(self):
oqparam = tests.get_oqparam('classical_job.ini')
oqparam.number_of_logic_tree_samples = 0
csm = readinput.get_composite_source_model(oqparam)
self.assertEqual(len(csm), 9) # the smlt example has 1 x 3 x 3 paths;
# there are 2 distinct tectonic region types, so 18 src_groups
self.assertEqual(sum(1 for tm in csm.src_groups), 18)
rlzs_assoc = csm.info.get_rlzs_assoc()
rlzs = rlzs_assoc.realizations
self.assertEqual(len(rlzs), 18) # the gsimlt has 1 x 2 paths
# counting the sources in each TRT model (after splitting)
self.assertEqual(
[1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2],
list(map(len, csm.src_groups)))
# removing 9 src_groups out of 18
def count_ruptures(src_group_id):
if src_group_id % 2 == 1: # Active Shallow Crust
return 0
else:
return 1
csm.info.update_eff_ruptures(count_ruptures)
assoc = csm.info.get_rlzs_assoc()
expected_assoc = """\
<RlzsAssoc(size=9, rlzs=9)
0,'[SadighEtAl1997]': [0]
2,'[SadighEtAl1997]': [1]
4,'[SadighEtAl1997]': [2]
6,'[SadighEtAl1997]': [3]
8,'[SadighEtAl1997]': [4]
10,'[SadighEtAl1997]': [5]
12,'[SadighEtAl1997]': [6]
14,'[SadighEtAl1997]': [7]
16,'[SadighEtAl1997]': [8]>"""
self.assertEqual(str(assoc), expected_assoc)
self.assertEqual(len(assoc.realizations), 9)
# removing all src_groups
csm.info.update_eff_ruptures(lambda t: 0)
self.assertEqual(csm.info.get_rlzs_assoc().realizations, [])
def test_oversampling(self):
from openquake.qa_tests_data.classical import case_17
oq = readinput.get_oqparam(
os.path.join(os.path.dirname(case_17.__file__), 'job.ini'))
sitecol = readinput.get_site_collection(oq)
with mock.patch('logging.warn') as warn:
csm = readinput.get_composite_source_model(oq, sitecol)
messages = [args[0][0] % args[0][1:] for args in warn.call_args_list]
self.assertEqual(
messages, ["The source path ('b2',) was sampled 4 times"])
assoc = csm.get_rlzs_assoc()
self.assertEqual(
str(assoc),
"<RlzsAssoc(2)\n"
"0,SadighEtAl1997: ['<0,b1,b1,w=0.2>']\n"
"1,SadighEtAl1997: ['<1,b2,b1,w=0.2,col=1>', '<2,b2,b1,w=0.2,col=2>', '<3,b2,b1,w=0.2,col=3>', '<4,b2,b1,w=0.2,col=4>']>")
# test the method csm_info.get_col_ids
col_ids_first = assoc.csm_info.get_col_ids(assoc.realizations[0])
self.assertEqual(col_ids_first, set([0]))
col_ids_last = assoc.csm_info.get_col_ids(assoc.realizations[-1])
self.assertEqual(col_ids_last, set([4]))
def read_inputs(self):
"""
Read risk data and sources if any
"""
oq = self.oqparam
self._read_risk_data()
self.check_overflow() # check if self.sitecol is too large
if getattr(self, 'sitecol', None):
# can be None for the ruptures-only calculator
with hdf5.File(self.datastore.tempname, 'w') as tmp:
tmp['sitecol'] = self.sitecol
elif (oq.calculation_mode == 'disaggregation'
and oq.max_sites_disagg < len(self.sitecol)):
raise ValueError('Please set max_sites_disagg=%d in %s' %
(len(self.sitecol), oq.inputs['job_ini']))
elif oq.disagg_by_src and len(self.sitecol) > oq.max_sites_disagg:
raise ValueError(
'There are too many sites to use disagg_by_src=true')
if ('source_model_logic_tree' in oq.inputs
and oq.hazard_calculation_id is None):
full_lt = readinput.get_full_lt(oq)
with self.monitor('composite source model', measuremem=True):
self.csm = csm = readinput.get_composite_source_model(
oq, full_lt, self.datastore.hdf5)
srcs = [src for sg in csm.src_groups for src in sg]
if not srcs:
raise RuntimeError('All sources were discarded!?')
logging.info('Checking the sources bounding box')
sids = self.src_filter().within_bbox(srcs)
if len(sids) == 0:
raise RuntimeError('All sources were discarded!?')
self.full_lt = csm.full_lt
self.init() # do this at the end of pre-execute
if (not oq.hazard_calculation_id
and oq.calculation_mode != 'preclassical'
and not oq.save_disk_space):
self.gzip_inputs()
def test_one_rlz(self):
oqparam = tests.get_oqparam('classical_job.ini')
# the example has number_of_logic_tree_samples = 1
csm = readinput.get_composite_source_model(oqparam)
# check the attributes of the groups are set
[grp0, grp1] = csm.src_groups
for grp in [grp0, grp1]:
self.assertEqual(grp.src_interdep, 'indep')
self.assertEqual(grp.rup_interdep, 'indep')
self.assertEqual(repr(csm.gsim_lt), '''\
<GsimLogicTree
Active Shallow Crust,b1,[SadighEtAl1997],w=0.5
Active Shallow Crust,b2,[ChiouYoungs2008],w=0.5
Subduction Interface,b3,[SadighEtAl1997],w=1.0>''')
[rlz] = csm.full_lt.get_realizations()
self.assertEqual(csm.full_lt.gsim_by_trt(rlz),
{'Subduction Interface': '[SadighEtAl1997]',
'Active Shallow Crust': '[ChiouYoungs2008]'})
self.assertEqual(rlz.ordinal, 0)
self.assertEqual(rlz.sm_lt_path, ('b1', 'b5', 'b7'))
self.assertEqual(rlz.gsim_lt_path, ('b2', 'b3'))
self.assertEqual(rlz.weight['default'], 1.)
def _info(name, filtersources, weightsources):
if name in base.calculators:
print(textwrap.dedent(base.calculators[name].__doc__.strip()))
elif name == "gsims":
for gs in gsim.get_available_gsims():
print(gs)
elif name.endswith(".xml"):
print(nrml.read(name).to_str())
elif name.endswith((".ini", ".zip")):
oqparam = readinput.get_oqparam(name)
if "exposure" in oqparam.inputs:
expo = readinput.get_exposure(oqparam)
sitecol, assets_by_site = readinput.get_sitecol_assets(oqparam, expo)
elif filtersources or weightsources:
sitecol, assets_by_site = readinput.get_site_collection(oqparam), []
else:
sitecol, assets_by_site = None, []
if "source_model_logic_tree" in oqparam.inputs:
print("Reading the source model...")
if weightsources:
sp = source.SourceFilterWeighter
elif filtersources:
sp = source.SourceFilter
else:
sp = source.BaseSourceProcessor # do nothing
csm = readinput.get_composite_source_model(oqparam, sitecol, sp)
assoc = csm.get_rlzs_assoc()
dstore = datastore.Fake(vars(oqparam), rlzs_assoc=assoc, composite_source_model=csm, sitecol=sitecol)
_print_info(dstore, filtersources, weightsources)
if len(assets_by_site):
assetcol = riskinput.build_asset_collection(assets_by_site)
dic = groupby(assetcol, operator.attrgetter("taxonomy"))
for taxo, num in dic.items():
print("taxonomy #%d, %d assets" % (taxo, num))
print("total assets = %d" % len(assetcol))
else:
print("No info for '%s'" % name)
def read_sources(self):
"""
Read the composite source model (if any).
This method must be called after read_exposure_sitecol, to be able
to filter to sources according to the site collection.
"""
if 'source' in self.oqparam.inputs:
logging.info('Reading the composite source model')
with self.monitor('reading composite source model',
autoflush=True):
self.csm = readinput.get_composite_source_model(
self.oqparam, self.sitecol, self.SourceProcessor,
self.monitor)
# we could manage limits here
self.source_pre_info = self.csm.source_info
self.job_info = readinput.get_job_info(self.oqparam, self.csm,
self.sitecol)
self.csm.count_ruptures()
self.rlzs_assoc = self.csm.get_rlzs_assoc()
logging.info('Total weight of the sources=%s',
self.job_info['input_weight'])
logging.info('Expected output size=%s',
self.job_info['output_weight'])
def read_inputs(self):
"""
Read risk data and sources if any
"""
oq = self.oqparam
self._read_risk_data()
self.check_overflow() # check if self.sitecol is too large
if getattr(self, 'sitecol', None):
# can be None for the ruptures-only calculator
with hdf5.File(self.datastore.tempname, 'w') as tmp:
tmp['sitecol'] = self.sitecol
if ('source_model_logic_tree' in oq.inputs
and oq.hazard_calculation_id is None):
with self.monitor('composite source model', measuremem=True):
self.csm = csm = readinput.get_composite_source_model(
oq, self.datastore.hdf5)
self.csm_info = csm.info
self.datastore['source_model_lt'] = csm.source_model_lt
res = views.view('dupl_sources', self.datastore)
logging.info(f'The composite source model has {res.val:,d} '
'ruptures')
if res:
logging.info(res)
self.init() # do this at the end of pre-execute
def _info(name, filtersources, weightsources):
if name in base.calculators:
print(textwrap.dedent(base.calculators[name].__doc__.strip()))
elif name == 'gsims':
for gs in gsim.get_available_gsims():
print(gs)
elif name.endswith('.xml'):
print(nrml.read(name).to_str())
elif name.endswith(('.ini', '.zip')):
oqparam = readinput.get_oqparam(name)
if 'exposure' in oqparam.inputs:
expo = readinput.get_exposure(oqparam)
sitecol, assets_by_site = readinput.get_sitecol_assets(
oqparam, expo)
elif filtersources or weightsources:
sitecol, assets_by_site = readinput.get_site_collection(
oqparam), []
else:
sitecol, assets_by_site = None, []
if 'source_model_logic_tree' in oqparam.inputs:
print('Reading the source model...')
if weightsources:
sp = source.SourceFilterWeighter
elif filtersources:
sp = source.SourceFilter
else:
sp = source.BaseSourceProcessor # do nothing
csm = readinput.get_composite_source_model(oqparam, sitecol, sp)
assoc = csm.get_rlzs_assoc()
_print_info(assoc, oqparam, csm, sitecol,
filtersources, weightsources)
if len(assets_by_site):
print('assets = %d' %
sum(len(assets) for assets in assets_by_site))
else:
print("No info for '%s'" % name)
def test_one_rlz(self):
oqparam = tests.get_oqparam('classical_job.ini')
# the example has number_of_logic_tree_samples = 1
sitecol = readinput.get_site_collection(oqparam)
csm = readinput.get_composite_source_model(oqparam, sitecol)
self.assertEqual(str(csm[0].gsim_lt), '''\
<GsimLogicTree
Active Shallow Crust,b1,SadighEtAl1997,w=0.5
Active Shallow Crust,b2,ChiouYoungs2008,w=0.5
Subduction Interface,b3,SadighEtAl1997,w=1.0>''')
assoc = csm.info.get_rlzs_assoc()
[rlz] = assoc.realizations
self.assertEqual(assoc.gsim_by_trt[rlz.ordinal],
{'Subduction Interface': 'SadighEtAl1997',
'Active Shallow Crust': 'ChiouYoungs2008'})
# ignoring the end of the tuple, with the uid field
self.assertEqual(rlz.ordinal, 0)
self.assertEqual(rlz.sm_lt_path, ('b1', 'b5', 'b8'))
self.assertEqual(rlz.gsim_lt_path, ('b2', 'b3'))
self.assertEqual(rlz.weight, 1.)
self.assertEqual(
str(assoc),
"<RlzsAssoc(size=2, rlzs=1)\n0,SadighEtAl1997: ['<0,b1_b5_b8,b2_b3,w=1.0>']\n"
"1,ChiouYoungs2008: ['<0,b1_b5_b8,b2_b3,w=1.0>']>")
def test_nrml04(self):
oq = readinput.get_oqparam('job.ini', case_2)
csm = readinput.get_composite_source_model(oq, in_memory=False)
srcs = csm.get_sources() # a single PointSource
self.assertEqual(len(srcs), 1)
def pre_execute(self):
"""
Check if there is a pre_calculator or a previous calculation ID.
If yes, read the inputs by invoking the precalculator or by retrieving
the previous calculation; if not, read the inputs directly.
"""
job_info = hdf5.LiteralAttrs()
if self.pre_calculator is not None:
# the parameter hazard_calculation_id is only meaningful if
# there is a precalculator
precalc_id = self.oqparam.hazard_calculation_id
if precalc_id is None: # recompute everything
precalc = calculators[self.pre_calculator](
self.oqparam, self.monitor('precalculator'),
self.datastore.calc_id)
precalc.run()
if 'scenario' not in self.oqparam.calculation_mode:
self.csm = precalc.csm
pre_attrs = vars(precalc)
for name in ('riskmodel', 'assets_by_site'):
if name in pre_attrs:
setattr(self, name, getattr(precalc, name))
else: # read previously computed data
parent = datastore.read(precalc_id)
self.datastore.set_parent(parent)
# copy missing parameters from the parent
params = {name: value for name, value in
vars(parent['oqparam']).items()
if name not in vars(self.oqparam)}
self.save_params(**params)
self.read_risk_data()
else: # we are in a basic calculator
self.read_risk_data()
if 'source' in self.oqparam.inputs:
with self.monitor(
'reading composite source model', autoflush=True):
self.csm = readinput.get_composite_source_model(
self.oqparam)
self.rlzs_assoc = self.csm.info.get_rlzs_assoc()
self.datastore['csm_info'] = self.rlzs_assoc.csm_info
self.rup_data = {}
# we could manage limits here
vars(job_info).update(readinput.get_job_info(
self.oqparam, self.csm, self.sitecol))
logging.info('Expected output size=%s',
job_info.hazard['output_weight'])
logging.info('Total weight of the sources=%s',
job_info.hazard['input_weight'])
self.init()
with self.monitor('managing sources', autoflush=True):
self.send_sources()
self.manager.store_source_info(self.datastore)
attrs = self.datastore.hdf5['composite_source_model'].attrs
attrs['weight'] = self.csm.weight
attrs['filtered_weight'] = self.csm.filtered_weight
attrs['maxweight'] = self.csm.maxweight
job_info.hostname = socket.gethostname()
if hasattr(self, 'riskmodel'):
job_info.require_epsilons = bool(self.riskmodel.covs)
self.job_info = job_info
self.datastore.flush()
def test_nrml05(self):
oq = readinput.get_oqparam('job.ini', case_2)
csm = readinput.get_composite_source_model(oq, in_memory=False)
srcs = csm.get_sources() # two PointSources
self.assertEqual(len(srcs), 2)
