def test_applyToSources(self): oq = readinput.get_oqparam('job.ini', case_21) with mock.patch('logging.info') as info: readinput.get_composite_source_model(oq) self.assertEqual( info.call_args[0], ('Applied %d changes to the composite source model', 81))
def test_wrong_trts(self): # invalid TRT in job.ini [reqv] oq = readinput.get_oqparam('job.ini', case_2) fname = oq.inputs['reqv'].pop('active shallow crust') oq.inputs['reqv']['act shallow crust'] = fname with self.assertRaises(ValueError) as ctx: readinput.get_composite_source_model(oq, in_memory=False) self.assertIn('Unknown TRT=act shallow crust', str(ctx.exception))
def test_one_rlz(self): oqparam = tests.get_oqparam('classical_job.ini') # the example has number_of_logic_tree_samples = 1 sitecol = readinput.get_site_collection(oqparam) csm = readinput.get_composite_source_model(oqparam, sitecol) self.assertEqual(str(csm[0].gsim_lt), '''\ <GsimLogicTree Active Shallow Crust,b1,SadighEtAl1997,w=0.5 Active Shallow Crust,b2,ChiouYoungs2008,w=0.5 Subduction Interface,b3,SadighEtAl1997,w=1.0>''') assoc = csm.get_rlzs_assoc( lambda trtmod: sum(src.count_ruptures() for src in trtmod.sources)) [rlz] = assoc.realizations self.assertEqual(assoc.gsim_by_trt[rlz.ordinal], {'Subduction Interface': 'SadighEtAl1997', 'Active Shallow Crust': 'ChiouYoungs2008'}) # ignoring the end of the tuple, with the uid field self.assertEqual(rlz.ordinal, 0) self.assertEqual(rlz.sm_lt_path, ('b1', 'b5', 'b8')) self.assertEqual(rlz.gsim_lt_path, ('b2', 'b3')) self.assertEqual(rlz.weight, 1.) self.assertEqual( str(assoc), "<RlzsAssoc(2)\n0,SadighEtAl1997: ['<0,b1_b5_b8,b2_b3,w=1.0>']\n" "1,ChiouYoungs2008: ['<0,b1_b5_b8,b2_b3,w=1.0>']>")
def initialize_sources(self): """ Parse source models, apply uncertainties and validate source logic trees. Save in the database LtSourceModel and TrtModel objects. """ logs.LOG.progress("initializing sources") parallel_source_splitting = valid.boolean( config.get('hazard', 'parallel_source_splitting') or 'false') self.composite_model = readinput.get_composite_source_model( self.oqparam, self.site_collection, no_distribute=not parallel_source_splitting) for sm in self.composite_model: # create an LtSourceModel for each distinct source model lt_model = models.LtSourceModel.objects.create( hazard_calculation=self.job, sm_lt_path=self.tilepath + sm.path, ordinal=sm.ordinal, sm_name=sm.name, weight=sm.weight, samples=sm.samples) self._source_models.append(lt_model) gsims_by_trt = sm.gsim_lt.values # save TrtModels for each tectonic region type # and stored the db ID in the in-memory models for trt_mod in sm.trt_models: trt_mod.id = models.TrtModel.objects.create( lt_model=lt_model, tectonic_region_type=trt_mod.trt, num_sources=len(trt_mod), num_ruptures=trt_mod.num_ruptures, min_mag=trt_mod.min_mag, max_mag=trt_mod.max_mag, gsims=gsims_by_trt[trt_mod.trt]).id # rebuild the info object with the trt_ids coming from the db self.composite_model.info = source.CompositionInfo( self.composite_model.source_model_lt, self.composite_model.source_models)
def read_sources(self): """ Read the composite source model (if any). This method must be called after read_exposure_sitecol, to be able to filter to sources according to the site collection. """ if 'source' in self.oqparam.inputs: logging.info('Reading the composite source model') with self.monitor( 'reading composite source model', autoflush=True): self.csm = readinput.get_composite_source_model( self.oqparam, self.sitecol, self.SourceProcessor, self.monitor) # we could manage limits here self.source_pre_info = self.csm.source_info self.job_info = readinput.get_job_info( self.oqparam, self.csm, self.sitecol) self.csm.count_ruptures() self.rlzs_assoc = self.csm.get_rlzs_assoc() logging.info( 'Total weight of the sources=%s', self.job_info['input_weight']) logging.info( 'Expected output size=%s', self.job_info['output_weight'])
def test_one_rlz(self): oqparam = tests.get_oqparam('classical_job.ini') # the example has number_of_logic_tree_samples = 1 csm = readinput.get_composite_source_model(oqparam) # check the attributes of the groups are set [grp0, grp1] = csm.src_groups for grp in csm.src_groups: self.assertEqual(grp.src_interdep, 'indep') self.assertEqual(grp.rup_interdep, 'indep') self.assertEqual(repr(csm.gsim_lt), '''\ <GsimLogicTree Active Shallow Crust,b1,[SadighEtAl1997],w=0.5 Active Shallow Crust,b2,[ChiouYoungs2008],w=0.5 Subduction Interface,b3,[SadighEtAl1997],w=1.0>''') assoc = csm.info.get_rlzs_assoc() [rlz] = assoc.realizations self.assertEqual(assoc.gsim_by_trt[rlz.ordinal], {'Subduction Interface': '[SadighEtAl1997]', 'Active Shallow Crust': '[ChiouYoungs2008]'}) # ignoring the end of the tuple, with the uid field self.assertEqual(rlz.ordinal, 0) self.assertEqual(rlz.sm_lt_path, ('b1', 'b4', 'b7')) self.assertEqual(rlz.gsim_lt_path, ('b2', 'b3')) self.assertEqual(rlz.weight['default'], 1.) self.assertEqual( str(assoc), "<RlzsAssoc(size=2, rlzs=1)\n0,'[SadighEtAl1997]': " "[0]\n1,'[ChiouYoungs2008]': [0]>")
def pre_execute(self): """ Check if there is a pre_calculator or a previous calculation ID. If yes, read the inputs by invoking the precalculator or by retrieving the previous calculation; if not, read the inputs directly. """ job_info = hdf5.LiteralAttrs() if self.pre_calculator is not None: # the parameter hazard_calculation_id is only meaningful if # there is a precalculator precalc_id = self.oqparam.hazard_calculation_id if precalc_id is None: # recompute everything precalc = calculators[self.pre_calculator]( self.oqparam, self.monitor("precalculator"), self.datastore.calc_id ) precalc.run() if "scenario" not in self.oqparam.calculation_mode: self.csm = precalc.csm pre_attrs = vars(precalc) for name in ("riskmodel", "assets_by_site"): if name in pre_attrs: setattr(self, name, getattr(precalc, name)) else: # read previously computed data parent = datastore.read(precalc_id) self.datastore.set_parent(parent) # copy missing parameters from the parent params = { name: value for name, value in vars(parent["oqparam"]).items() if name not in vars(self.oqparam) } self.save_params(**params) self.read_risk_data() self.init() else: # we are in a basic calculator self.read_risk_data() if "source" in self.oqparam.inputs: with self.monitor("reading composite source model", autoflush=True): self.csm = readinput.get_composite_source_model(self.oqparam) self.datastore["csm_info"] = self.csm.info self.rup_data = {} # we could manage limits here vars(job_info).update(readinput.get_job_info(self.oqparam, self.csm, self.sitecol)) logging.info("Expected output size=%s", job_info.hazard["output_weight"]) logging.info("Total weight of the sources=%s", job_info.hazard["input_weight"]) self.init() if "source" in self.oqparam.inputs: with self.monitor("managing sources", autoflush=True): self.send_sources() self.manager.store_source_info(self.datastore) attrs = self.datastore.hdf5["composite_source_model"].attrs attrs["weight"] = self.csm.weight attrs["filtered_weight"] = self.csm.filtered_weight attrs["maxweight"] = self.csm.maxweight job_info.hostname = socket.gethostname() if hasattr(self, "riskmodel"): job_info.require_epsilons = bool(self.riskmodel.covs) self.job_info = job_info self.datastore.flush()
def test_many_rlzs(self): oqparam = tests.get_oqparam('classical_job.ini') oqparam.number_of_logic_tree_samples = 0 sitecol = readinput.get_site_collection(oqparam) csm = readinput.get_composite_source_model(oqparam, sitecol) self.assertEqual(len(csm), 9) # the smlt example has 1 x 3 x 3 paths; # there are 2 distinct tectonic region types, so 18 trt_models rlzs_assoc = csm.get_rlzs_assoc() rlzs = rlzs_assoc.realizations self.assertEqual(len(rlzs), 18) # the gsimlt has 1 x 2 paths self.assertEqual([1, 584, 1, 584, 1, 584, 1, 582, 1, 582, 1, 582, 1, 582, 1, 582, 1, 582], map(len, csm.trt_models)) # test the method csm_info.get_col_ids col_ids_first = rlzs_assoc.csm_info.get_col_ids(rlzs[0]) self.assertEqual(col_ids_first, set([0, 1])) col_ids_last = rlzs_assoc.csm_info.get_col_ids(rlzs[-1]) self.assertEqual(col_ids_last, set([16, 17])) # removing 9 trt_models out of 18 for trt_model in csm.trt_models: if trt_model.trt == 'Active Shallow Crust': # no ruptures trt_model.num_ruptures = 0 assoc = csm.get_rlzs_assoc() expected_assoc = """\ <RlzsAssoc(18) 0,SadighEtAl1997: ['<0,b1_b3_b6,@_b3,w=0.04>'] 1,SadighEtAl1997: ['<0,b1_b3_b6,@_b3,w=0.04>'] 2,SadighEtAl1997: ['<1,b1_b3_b7,@_b3,w=0.12>'] 3,SadighEtAl1997: ['<1,b1_b3_b7,@_b3,w=0.12>'] 4,SadighEtAl1997: ['<2,b1_b3_b8,@_b3,w=0.04>'] 5,SadighEtAl1997: ['<2,b1_b3_b8,@_b3,w=0.04>'] 6,SadighEtAl1997: ['<3,b1_b4_b6,@_b3,w=0.12>'] 7,SadighEtAl1997: ['<3,b1_b4_b6,@_b3,w=0.12>'] 8,SadighEtAl1997: ['<4,b1_b4_b7,@_b3,w=0.36>'] 9,SadighEtAl1997: ['<4,b1_b4_b7,@_b3,w=0.36>'] 10,SadighEtAl1997: ['<5,b1_b4_b8,@_b3,w=0.12>'] 11,SadighEtAl1997: ['<5,b1_b4_b8,@_b3,w=0.12>'] 12,SadighEtAl1997: ['<6,b1_b5_b6,@_b3,w=0.04>'] 13,SadighEtAl1997: ['<6,b1_b5_b6,@_b3,w=0.04>'] 14,SadighEtAl1997: ['<7,b1_b5_b7,@_b3,w=0.12>'] 15,SadighEtAl1997: ['<7,b1_b5_b7,@_b3,w=0.12>'] 16,SadighEtAl1997: ['<8,b1_b5_b8,@_b3,w=0.04>'] 17,SadighEtAl1997: ['<8,b1_b5_b8,@_b3,w=0.04>']>""" self.assertEqual(str(assoc), expected_assoc) self.assertEqual(len(assoc.realizations), 9) # removing all trt_models for trt_model in csm.trt_models: if trt_model.trt == 'Subduction Interface': # no ruptures trt_model.num_ruptures = 0 self.assertEqual(csm.get_rlzs_assoc().realizations, [])
def print_csm_info(fname): """ Parse the composite source model without instantiating the sources and prints information about its composition and the full logic tree """ oqparam = readinput.get_oqparam(fname) csm = readinput.get_composite_source_model(oqparam, in_memory=False) print(csm.info) print('See https://github.com/gem/oq-risklib/blob/master/doc/' 'effective-realizations.rst for an explanation') print(csm.info.get_rlzs_assoc())
def read_inputs(self): """ Read risk data and sources if any """ oq = self.oqparam self._read_risk_data() self.check_overflow() # check if self.sitecol is too large if ('source_model_logic_tree' in oq.inputs and oq.hazard_calculation_id is None): self.csm = readinput.get_composite_source_model( oq, self.monitor(), srcfilter=self.src_filter) self.init() # do this at the end of pre-execute
def print_csm_info(fname): """ Parse the composite source model without instantiating the sources and prints information about its composition and the full logic tree """ oqparam = readinput.get_oqparam(fname) csm = readinput.get_composite_source_model(oqparam, in_memory=False) print(csm.info) print('See https://github.com/gem/oq-risklib/blob/master/doc/' 'effective-realizations.rst for an explanation') rlzs_assoc = csm.info.get_rlzs_assoc() print(rlzs_assoc) tot, pairs = get_pickled_sizes(rlzs_assoc) print(views.rst_table(pairs, ['attribute', 'nbytes']))
def test_many_rlzs(self): oqparam = tests.get_oqparam('classical_job.ini') oqparam.number_of_logic_tree_samples = 0 sitecol = readinput.get_site_collection(oqparam) csm = readinput.get_composite_source_model(oqparam, sitecol) self.assertEqual(len(csm), 9) # the smlt example has 1 x 3 x 3 paths; # there are 2 distinct tectonic region types, so 18 trt_models self.assertEqual(sum(1 for tm in csm.trt_models), 18) rlzs_assoc = csm.info.get_rlzs_assoc() rlzs = rlzs_assoc.realizations self.assertEqual(len(rlzs), 18) # the gsimlt has 1 x 2 paths # counting the sources in each TRT model (unsplit) self.assertEqual( [1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2], list(map(len, csm.trt_models))) # test the method extract assoc = rlzs_assoc.extract([1, 5]) self.assertEqual(str(assoc), """\ <RlzsAssoc(size=4, rlzs=2) 0,SadighEtAl1997: ['<1,b1_b3_b6,b2_b3,w=0.5>'] 1,ChiouYoungs2008: ['<1,b1_b3_b6,b2_b3,w=0.5>'] 4,SadighEtAl1997: ['<5,b1_b3_b8,b2_b3,w=0.5>'] 5,ChiouYoungs2008: ['<5,b1_b3_b8,b2_b3,w=0.5>']>""") # removing 9 trt_models out of 18 def count_ruptures(trt_model): if trt_model.trt == 'Active Shallow Crust': # no ruptures return 0 else: return 1 assoc = csm.info.get_rlzs_assoc(count_ruptures) expected_assoc = """\ <RlzsAssoc(size=9, rlzs=9) 0,SadighEtAl1997: ['<0,b1_b3_b6,@_b3,w=0.04>'] 2,SadighEtAl1997: ['<1,b1_b3_b7,@_b3,w=0.12>'] 4,SadighEtAl1997: ['<2,b1_b3_b8,@_b3,w=0.04>'] 6,SadighEtAl1997: ['<3,b1_b4_b6,@_b3,w=0.12>'] 8,SadighEtAl1997: ['<4,b1_b4_b7,@_b3,w=0.36>'] 10,SadighEtAl1997: ['<5,b1_b4_b8,@_b3,w=0.12>'] 12,SadighEtAl1997: ['<6,b1_b5_b6,@_b3,w=0.04>'] 14,SadighEtAl1997: ['<7,b1_b5_b7,@_b3,w=0.12>'] 16,SadighEtAl1997: ['<8,b1_b5_b8,@_b3,w=0.04>']>""" self.assertEqual(str(assoc), expected_assoc) self.assertEqual(len(assoc.realizations), 9) # removing all trt_models self.assertEqual(csm.info.get_rlzs_assoc(lambda t: 0).realizations, [])
def basic_pre_execute(self): self.read_risk_data() if 'source' in self.oqparam.inputs: with self.monitor( 'reading composite source model', autoflush=True): self.csm = readinput.get_composite_source_model(self.oqparam) self.datastore['csm_info'] = self.csm.info self.rup_data = {} self.init() if 'source' in self.oqparam.inputs: with self.monitor('managing sources', autoflush=True): self.taskman = self.send_sources() attrs = self.datastore.hdf5['composite_source_model'].attrs attrs['weight'] = self.csm.weight attrs['filtered_weight'] = self.csm.filtered_weight
def print_csm_info(fname): """ Parse the composite source model without instantiating the sources and prints information about its composition and the full logic tree """ oqparam = readinput.get_oqparam(fname) csm = readinput.get_composite_source_model(oqparam, in_memory=False) print(csm.info) print('See http://docs.openquake.org/oq-engine/stable/' 'effective-realizations.html for an explanation') rlzs_assoc = csm.info.get_rlzs_assoc() print(rlzs_assoc) dupl = [(srcs[0]['id'], len(srcs)) for srcs in csm.check_dupl_sources()] if dupl: print(rst_table(dupl, ['source_id', 'multiplicity'])) tot, pairs = get_pickled_sizes(rlzs_assoc) print(rst_table(pairs, ['attribute', 'nbytes']))
def test_oversampling(self): from openquake.qa_tests_data.classical import case_17 oq = readinput.get_oqparam( os.path.join(os.path.dirname(case_17.__file__), 'job.ini')) sitecol = readinput.get_site_collection(oq) csm = readinput.get_composite_source_model(oq, sitecol) assoc = csm.info.get_rlzs_assoc() self.assertEqual( str(assoc), "<RlzsAssoc(size=2, rlzs=5)\n" "0,SadighEtAl1997: ['<0,b1,b1,w=0.2>']\n" "1,SadighEtAl1997: ['<1,b2,b1,w=0.2>', '<2,b2,b1,w=0.2>', '<3,b2,b1,w=0.2>', '<4,b2,b1,w=0.2>']>") # check CompositionInfo serialization array, attrs = assoc.csm_info.__toh5__() new = object.__new__(CompositionInfo) new.__fromh5__(array, attrs) self.assertEqual(repr(new), repr(assoc.csm_info))
def test_oversampling(self): from openquake.qa_tests_data.classical import case_17 oq = readinput.get_oqparam( os.path.join(os.path.dirname(case_17.__file__), 'job.ini')) csm = readinput.get_composite_source_model(oq) csm.info.update_eff_ruptures(lambda tm: 1) assoc = csm.info.get_rlzs_assoc() self.assertEqual( str(assoc), "<RlzsAssoc(size=2, rlzs=5)\n" "0,'[SadighEtAl1997]': [0 1 2]\n" "1,'[SadighEtAl1997]': [3 4]>") # check CompositionInfo serialization dic, attrs = csm.info.__toh5__() new = object.__new__(CompositionInfo) new.__fromh5__(dic, attrs) self.assertEqual(repr(new), repr(csm.info). replace('0.20000000000000004', '0.2'))
def _info(name, filtersources, weightsources): if name in base.calculators: print(textwrap.dedent(base.calculators[name].__doc__.strip())) elif name == 'gsims': for gs in gsim.get_available_gsims(): print(gs) elif name.endswith('.xml'): print(nrml.read(name).to_str()) elif name.endswith(('.ini', '.zip')): oqparam = readinput.get_oqparam(name) if 'exposure' in oqparam.inputs: expo = readinput.get_exposure(oqparam) sitecol, assets_by_site = readinput.get_sitecol_assets( oqparam, expo) elif filtersources or weightsources: sitecol, assets_by_site = readinput.get_site_collection( oqparam), [] else: sitecol, assets_by_site = None, [] if 'source_model_logic_tree' in oqparam.inputs: print('Reading the source model...') if weightsources: sp = source.SourceFilterWeighter elif filtersources: sp = source.SourceFilter else: sp = source.BaseSourceProcessor # do nothing csm = readinput.get_composite_source_model(oqparam, sitecol, sp) assoc = csm.get_rlzs_assoc() _print_info( dict(rlzs_assoc=assoc, oqparam=oqparam, composite_source_model=csm, sitecol=sitecol), filtersources, weightsources) if len(assets_by_site): assetcol = riskinput.build_asset_collection(assets_by_site) dic = groupby(assetcol, operator.attrgetter('taxonomy')) for taxo, num in dic.items(): print('taxonomy #%d, %d assets' % (taxo, num)) print('total assets = %d' % len(assetcol)) else: print("No info for '%s'" % name)
def test_many_rlzs(self): oqparam = tests.get_oqparam('classical_job.ini') oqparam.number_of_logic_tree_samples = 0 csm = readinput.get_composite_source_model(oqparam) self.assertEqual(len(csm), 9) # the smlt example has 1 x 3 x 3 paths; # there are 2 distinct tectonic region types, so 18 src_groups self.assertEqual(sum(1 for tm in csm.src_groups), 18) rlzs_assoc = csm.info.get_rlzs_assoc() rlzs = rlzs_assoc.realizations self.assertEqual(len(rlzs), 18) # the gsimlt has 1 x 2 paths # counting the sources in each TRT model (after splitting) self.assertEqual( [1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2], list(map(len, csm.src_groups))) # removing 9 src_groups out of 18 def count_ruptures(src_group_id): if src_group_id % 2 == 1: # Active Shallow Crust return 0 else: return 1 csm.info.update_eff_ruptures(count_ruptures) assoc = csm.info.get_rlzs_assoc() expected_assoc = """\ <RlzsAssoc(size=9, rlzs=9) 0,'[SadighEtAl1997]': [0] 2,'[SadighEtAl1997]': [1] 4,'[SadighEtAl1997]': [2] 6,'[SadighEtAl1997]': [3] 8,'[SadighEtAl1997]': [4] 10,'[SadighEtAl1997]': [5] 12,'[SadighEtAl1997]': [6] 14,'[SadighEtAl1997]': [7] 16,'[SadighEtAl1997]': [8]>""" self.assertEqual(str(assoc), expected_assoc) self.assertEqual(len(assoc.realizations), 9) # removing all src_groups csm.info.update_eff_ruptures(lambda t: 0) self.assertEqual(csm.info.get_rlzs_assoc().realizations, [])
def test_oversampling(self): from openquake.qa_tests_data.classical import case_17 oq = readinput.get_oqparam( os.path.join(os.path.dirname(case_17.__file__), 'job.ini')) sitecol = readinput.get_site_collection(oq) with mock.patch('logging.warn') as warn: csm = readinput.get_composite_source_model(oq, sitecol) messages = [args[0][0] % args[0][1:] for args in warn.call_args_list] self.assertEqual( messages, ["The source path ('b2',) was sampled 4 times"]) assoc = csm.get_rlzs_assoc() self.assertEqual( str(assoc), "<RlzsAssoc(2)\n" "0,SadighEtAl1997: ['<0,b1,b1,w=0.2>']\n" "1,SadighEtAl1997: ['<1,b2,b1,w=0.2,col=1>', '<2,b2,b1,w=0.2,col=2>', '<3,b2,b1,w=0.2,col=3>', '<4,b2,b1,w=0.2,col=4>']>") # test the method csm_info.get_col_ids col_ids_first = assoc.csm_info.get_col_ids(assoc.realizations[0]) self.assertEqual(col_ids_first, set([0])) col_ids_last = assoc.csm_info.get_col_ids(assoc.realizations[-1]) self.assertEqual(col_ids_last, set([4]))
def read_inputs(self): """ Read risk data and sources if any """ oq = self.oqparam self._read_risk_data() self.check_overflow() # check if self.sitecol is too large if getattr(self, 'sitecol', None): # can be None for the ruptures-only calculator with hdf5.File(self.datastore.tempname, 'w') as tmp: tmp['sitecol'] = self.sitecol elif (oq.calculation_mode == 'disaggregation' and oq.max_sites_disagg < len(self.sitecol)): raise ValueError('Please set max_sites_disagg=%d in %s' % (len(self.sitecol), oq.inputs['job_ini'])) elif oq.disagg_by_src and len(self.sitecol) > oq.max_sites_disagg: raise ValueError( 'There are too many sites to use disagg_by_src=true') if ('source_model_logic_tree' in oq.inputs and oq.hazard_calculation_id is None): full_lt = readinput.get_full_lt(oq) with self.monitor('composite source model', measuremem=True): self.csm = csm = readinput.get_composite_source_model( oq, full_lt, self.datastore.hdf5) srcs = [src for sg in csm.src_groups for src in sg] if not srcs: raise RuntimeError('All sources were discarded!?') logging.info('Checking the sources bounding box') sids = self.src_filter().within_bbox(srcs) if len(sids) == 0: raise RuntimeError('All sources were discarded!?') self.full_lt = csm.full_lt self.init() # do this at the end of pre-execute if (not oq.hazard_calculation_id and oq.calculation_mode != 'preclassical' and not oq.save_disk_space): self.gzip_inputs()
def test_one_rlz(self): oqparam = tests.get_oqparam('classical_job.ini') # the example has number_of_logic_tree_samples = 1 csm = readinput.get_composite_source_model(oqparam) # check the attributes of the groups are set [grp0, grp1] = csm.src_groups for grp in [grp0, grp1]: self.assertEqual(grp.src_interdep, 'indep') self.assertEqual(grp.rup_interdep, 'indep') self.assertEqual(repr(csm.gsim_lt), '''\ <GsimLogicTree Active Shallow Crust,b1,[SadighEtAl1997],w=0.5 Active Shallow Crust,b2,[ChiouYoungs2008],w=0.5 Subduction Interface,b3,[SadighEtAl1997],w=1.0>''') [rlz] = csm.full_lt.get_realizations() self.assertEqual(csm.full_lt.gsim_by_trt(rlz), {'Subduction Interface': '[SadighEtAl1997]', 'Active Shallow Crust': '[ChiouYoungs2008]'}) self.assertEqual(rlz.ordinal, 0) self.assertEqual(rlz.sm_lt_path, ('b1', 'b5', 'b7')) self.assertEqual(rlz.gsim_lt_path, ('b2', 'b3')) self.assertEqual(rlz.weight['default'], 1.)
def _info(name, filtersources, weightsources): if name in base.calculators: print(textwrap.dedent(base.calculators[name].__doc__.strip())) elif name == "gsims": for gs in gsim.get_available_gsims(): print(gs) elif name.endswith(".xml"): print(nrml.read(name).to_str()) elif name.endswith((".ini", ".zip")): oqparam = readinput.get_oqparam(name) if "exposure" in oqparam.inputs: expo = readinput.get_exposure(oqparam) sitecol, assets_by_site = readinput.get_sitecol_assets(oqparam, expo) elif filtersources or weightsources: sitecol, assets_by_site = readinput.get_site_collection(oqparam), [] else: sitecol, assets_by_site = None, [] if "source_model_logic_tree" in oqparam.inputs: print("Reading the source model...") if weightsources: sp = source.SourceFilterWeighter elif filtersources: sp = source.SourceFilter else: sp = source.BaseSourceProcessor # do nothing csm = readinput.get_composite_source_model(oqparam, sitecol, sp) assoc = csm.get_rlzs_assoc() dstore = datastore.Fake(vars(oqparam), rlzs_assoc=assoc, composite_source_model=csm, sitecol=sitecol) _print_info(dstore, filtersources, weightsources) if len(assets_by_site): assetcol = riskinput.build_asset_collection(assets_by_site) dic = groupby(assetcol, operator.attrgetter("taxonomy")) for taxo, num in dic.items(): print("taxonomy #%d, %d assets" % (taxo, num)) print("total assets = %d" % len(assetcol)) else: print("No info for '%s'" % name)
def read_sources(self): """ Read the composite source model (if any). This method must be called after read_exposure_sitecol, to be able to filter to sources according to the site collection. """ if 'source' in self.oqparam.inputs: logging.info('Reading the composite source model') with self.monitor('reading composite source model', autoflush=True): self.csm = readinput.get_composite_source_model( self.oqparam, self.sitecol, self.SourceProcessor, self.monitor) # we could manage limits here self.source_pre_info = self.csm.source_info self.job_info = readinput.get_job_info(self.oqparam, self.csm, self.sitecol) self.csm.count_ruptures() self.rlzs_assoc = self.csm.get_rlzs_assoc() logging.info('Total weight of the sources=%s', self.job_info['input_weight']) logging.info('Expected output size=%s', self.job_info['output_weight'])
def read_inputs(self): """ Read risk data and sources if any """ oq = self.oqparam self._read_risk_data() self.check_overflow() # check if self.sitecol is too large if getattr(self, 'sitecol', None): # can be None for the ruptures-only calculator with hdf5.File(self.datastore.tempname, 'w') as tmp: tmp['sitecol'] = self.sitecol if ('source_model_logic_tree' in oq.inputs and oq.hazard_calculation_id is None): with self.monitor('composite source model', measuremem=True): self.csm = csm = readinput.get_composite_source_model( oq, self.datastore.hdf5) self.csm_info = csm.info self.datastore['source_model_lt'] = csm.source_model_lt res = views.view('dupl_sources', self.datastore) logging.info(f'The composite source model has {res.val:,d} ' 'ruptures') if res: logging.info(res) self.init() # do this at the end of pre-execute
def _info(name, filtersources, weightsources): if name in base.calculators: print(textwrap.dedent(base.calculators[name].__doc__.strip())) elif name == 'gsims': for gs in gsim.get_available_gsims(): print(gs) elif name.endswith('.xml'): print(nrml.read(name).to_str()) elif name.endswith(('.ini', '.zip')): oqparam = readinput.get_oqparam(name) if 'exposure' in oqparam.inputs: expo = readinput.get_exposure(oqparam) sitecol, assets_by_site = readinput.get_sitecol_assets( oqparam, expo) elif filtersources or weightsources: sitecol, assets_by_site = readinput.get_site_collection( oqparam), [] else: sitecol, assets_by_site = None, [] if 'source_model_logic_tree' in oqparam.inputs: print('Reading the source model...') if weightsources: sp = source.SourceFilterWeighter elif filtersources: sp = source.SourceFilter else: sp = source.BaseSourceProcessor # do nothing csm = readinput.get_composite_source_model(oqparam, sitecol, sp) assoc = csm.get_rlzs_assoc() _print_info(assoc, oqparam, csm, sitecol, filtersources, weightsources) if len(assets_by_site): print('assets = %d' % sum(len(assets) for assets in assets_by_site)) else: print("No info for '%s'" % name)
def test_one_rlz(self): oqparam = tests.get_oqparam('classical_job.ini') # the example has number_of_logic_tree_samples = 1 sitecol = readinput.get_site_collection(oqparam) csm = readinput.get_composite_source_model(oqparam, sitecol) self.assertEqual(str(csm[0].gsim_lt), '''\ <GsimLogicTree Active Shallow Crust,b1,SadighEtAl1997,w=0.5 Active Shallow Crust,b2,ChiouYoungs2008,w=0.5 Subduction Interface,b3,SadighEtAl1997,w=1.0>''') assoc = csm.info.get_rlzs_assoc() [rlz] = assoc.realizations self.assertEqual(assoc.gsim_by_trt[rlz.ordinal], {'Subduction Interface': 'SadighEtAl1997', 'Active Shallow Crust': 'ChiouYoungs2008'}) # ignoring the end of the tuple, with the uid field self.assertEqual(rlz.ordinal, 0) self.assertEqual(rlz.sm_lt_path, ('b1', 'b5', 'b8')) self.assertEqual(rlz.gsim_lt_path, ('b2', 'b3')) self.assertEqual(rlz.weight, 1.) self.assertEqual( str(assoc), "<RlzsAssoc(size=2, rlzs=1)\n0,SadighEtAl1997: ['<0,b1_b5_b8,b2_b3,w=1.0>']\n" "1,ChiouYoungs2008: ['<0,b1_b5_b8,b2_b3,w=1.0>']>")
def test_nrml04(self): oq = readinput.get_oqparam('job.ini', case_2) csm = readinput.get_composite_source_model(oq, in_memory=False) srcs = csm.get_sources() # a single PointSource self.assertEqual(len(srcs), 1)
def pre_execute(self): """ Check if there is a pre_calculator or a previous calculation ID. If yes, read the inputs by invoking the precalculator or by retrieving the previous calculation; if not, read the inputs directly. """ job_info = hdf5.LiteralAttrs() if self.pre_calculator is not None: # the parameter hazard_calculation_id is only meaningful if # there is a precalculator precalc_id = self.oqparam.hazard_calculation_id if precalc_id is None: # recompute everything precalc = calculators[self.pre_calculator]( self.oqparam, self.monitor('precalculator'), self.datastore.calc_id) precalc.run() if 'scenario' not in self.oqparam.calculation_mode: self.csm = precalc.csm pre_attrs = vars(precalc) for name in ('riskmodel', 'assets_by_site'): if name in pre_attrs: setattr(self, name, getattr(precalc, name)) else: # read previously computed data parent = datastore.read(precalc_id) self.datastore.set_parent(parent) # copy missing parameters from the parent params = {name: value for name, value in vars(parent['oqparam']).items() if name not in vars(self.oqparam)} self.save_params(**params) self.read_risk_data() else: # we are in a basic calculator self.read_risk_data() if 'source' in self.oqparam.inputs: with self.monitor( 'reading composite source model', autoflush=True): self.csm = readinput.get_composite_source_model( self.oqparam) self.rlzs_assoc = self.csm.info.get_rlzs_assoc() self.datastore['csm_info'] = self.rlzs_assoc.csm_info self.rup_data = {} # we could manage limits here vars(job_info).update(readinput.get_job_info( self.oqparam, self.csm, self.sitecol)) logging.info('Expected output size=%s', job_info.hazard['output_weight']) logging.info('Total weight of the sources=%s', job_info.hazard['input_weight']) self.init() with self.monitor('managing sources', autoflush=True): self.send_sources() self.manager.store_source_info(self.datastore) attrs = self.datastore.hdf5['composite_source_model'].attrs attrs['weight'] = self.csm.weight attrs['filtered_weight'] = self.csm.filtered_weight attrs['maxweight'] = self.csm.maxweight job_info.hostname = socket.gethostname() if hasattr(self, 'riskmodel'): job_info.require_epsilons = bool(self.riskmodel.covs) self.job_info = job_info self.datastore.flush()
def test_nrml05(self): oq = readinput.get_oqparam('job.ini', case_2) csm = readinput.get_composite_source_model(oq, in_memory=False) srcs = csm.get_sources() # two PointSources self.assertEqual(len(srcs), 2)