def test_magnitude_bins(self): """ Testing build disaggregation matrix """ fname = os.path.join(DATA_PATH, 'data', 'ssm.xml') converter = sourceconverter.SourceConverter(50., 1., 10, 0.1, 10) groups = to_python(fname, converter) sources = [] for g in groups: sources += g.sources site = Site(Point(172.63, -43.53), vs30=250, vs30measured=False, z1pt0=330) imt = SA(3.0) iml = 0.25612220 gsim_by_trt = {TRT.ACTIVE_SHALLOW_CRUST: Bradley2013()} truncation_level = 3.0 n_epsilons = 1 mag_bin_width = 0.1 dist_bin_width = 100. coord_bin_width = 100. # Compute the disaggregation matrix edges, mtx = disagg.disaggregation(sources, site, imt, iml, gsim_by_trt, truncation_level, n_epsilons, mag_bin_width, dist_bin_width, coord_bin_width) tm = disagg.mag_pmf(mtx[:, :, :, :, :, 0]) numpy.testing.assert_array_less(numpy.zeros_like(tm[2:]), tm[2:])
def get(xml, investigation_time=50., rupture_mesh_spacing=5., width_of_mfd_bin=1.0, area_source_discretization=10): """ :param xml: the XML representation of a source :param investigation_time: investigation time :param rupture_mesh_spacing: rupture mesh spacing :param width_of_mfd_bin: width of MFD bin :param area_source_discretization: area source discretization :returns: a python source object """ text = '''<?xml version='1.0' encoding='UTF-8'?> <nrml xmlns="http://openquake.org/xmlns/nrml/0.4" xmlns:gml="http://www.opengis.net/gml"> %s </nrml>''' % xml [node] = read(gettemp(text)) conv = sourceconverter.SourceConverter( investigation_time, rupture_mesh_spacing, width_of_mfd_bin=width_of_mfd_bin, area_source_discretization=area_source_discretization) src = conv.convert_node(node) src.grp_id = src.id = 0 return src
def get_ltmodels(oq, gsim_lt, source_model_lt, h5=None): """ Build source models from the logic tree and to store them inside the `source_info` dataset. """ if oq.pointsource_distance['default'] == {}: spinning_off = False else: spinning_off = sum(oq.pointsource_distance.values()) == 0 if spinning_off: logging.info('Removing nodal plane and hypocenter distributions') # NB: the source models file are often NOT in the shared directory # (for instance in oq-engine/demos) so the processpool must be used dist = ('no' if os.environ.get('OQ_DISTRIBUTE') == 'no' else 'processpool') smlt_dir = os.path.dirname(source_model_lt.filename) converter = sourceconverter.SourceConverter( oq.investigation_time, oq.rupture_mesh_spacing, oq.complex_fault_mesh_spacing, oq.width_of_mfd_bin, oq.area_source_discretization, oq.minimum_magnitude, not spinning_off, oq.source_id) rlzs = source_model_lt.get_eff_rlzs() lt_models = [] for rlz in rlzs: ltm = LtSourceModel( rlz['value'], rlz['weight'], rlz['lt_path'], [], rlz['ordinal'], rlz['samples'], rlz['offset']) lt_models.append(ltm) if oq.calculation_mode.startswith('ucerf'): idx = 0 [grp] = nrml.to_python(oq.inputs["source_model"], converter) for grp_id, ltm in enumerate(lt_models): sg = copy.copy(grp) sg.id = grp_id ltm.src_groups = [sg] src = sg[0].new(ltm.ordinal, ltm.names) # one source src.src_group_id = grp_id idx += 1 src.samples = ltm.samples sg.sources = [src] data = [((grp_id, grp_id, src.source_id, src.code, 0, 0, -1, src.num_ruptures, 0, ''))] sg.info = numpy.array(data, source_info_dt) return lt_models logging.info('Reading the source model(s) in parallel') allargs = [] fileno = 0 for rlz in rlzs: for name in rlz['value'].split(): fname = os.path.abspath(os.path.join(smlt_dir, name)) allargs.append((rlz['ordinal'], rlz['lt_path'], source_model_lt.apply_uncertainties, fname, fileno)) fileno += 1 smap = parallel.Starmap( SourceReader(converter, smlt_dir, h5), allargs, distribute=dist, h5=h5 if h5 else None) # NB: h5 is None in logictree_test.py return _store_results(smap, lt_models, source_model_lt, gsim_lt, oq, h5)
def test_alternative_mfds(self): converter = s.SourceConverter( investigation_time=1., rupture_mesh_spacing=1, # km complex_fault_mesh_spacing=5, # km width_of_mfd_bin=0.1, # for Truncated GR MFDs area_source_discretization=1.) grp_nodes = nrml.read(ALT_MFDS_SRC_MODEL).sourceModel.nodes [[sflt1, sflt2], [cplx1]] = map(converter.convert_node, grp_nodes) # Check the values # Arbitrary MFD assert_close(cplx1.mfd.magnitudes, [8.6, 8.8, 9.0]) assert_close(cplx1.mfd.occurrence_rates, [0.0006, 0.0008, 0.0004]) # Youngs & Coppersmith from characteristic rate self.assertAlmostEqual(sflt1.mfd.b_val, 1.0) self.assertAlmostEqual(sflt1.mfd.a_val, 3.3877843113) self.assertAlmostEqual(sflt1.mfd.char_mag, 7.0) self.assertAlmostEqual(sflt1.mfd.char_rate, 0.005) self.assertAlmostEqual(sflt1.mfd.min_mag, 5.0) # Youngs & Coppersmith from total moment rate self.assertAlmostEqual(sflt2.mfd.b_val, 1.0) self.assertAlmostEqual(sflt2.mfd.a_val, 5.0800, 3) self.assertAlmostEqual(sflt2.mfd.char_mag, 7.0) self.assertAlmostEqual(sflt2.mfd.char_rate, 0.24615, 5) self.assertAlmostEqual(sflt2.mfd.min_mag, 5.0)
def test_nonparametric_source_ok(self): converter = s.SourceConverter( investigation_time=50., rupture_mesh_spacing=1, # km complex_fault_mesh_spacing=1, # km width_of_mfd_bin=1., # for Truncated GR MFDs area_source_discretization=1.) [np] = nrml.read(NONPARAMETRIC_SOURCE).sourceModel converter.convert_node(np)
def test_duplicate_id(self): parser = nrml.SourceModelParser(s.SourceConverter( investigation_time=50., rupture_mesh_spacing=1, complex_fault_mesh_spacing=1, width_of_mfd_bin=0.1, area_source_discretization=10, )) with self.assertRaises(nrml.DuplicatedID): parser.parse_groups(DUPLICATE_ID_SRC_MODEL)
def test_duplicate_id(self): conv = s.SourceConverter( investigation_time=50., rupture_mesh_spacing=1, complex_fault_mesh_spacing=1, width_of_mfd_bin=0.1, area_source_discretization=10, ) with self.assertRaises(nrml.DuplicatedID): nrml.to_python(DUPLICATE_ID_SRC_MODEL, conv)
def setUpClass(cls): cls.parser = nrml.SourceModelParser(s.SourceConverter( investigation_time=50., rupture_mesh_spacing=1, # km complex_fault_mesh_spacing=1, # km width_of_mfd_bin=1., # for Truncated GR MFDs area_source_discretization=1.)) cls.source_collector = { sc.trt: sc for sc in cls.parser.parse_src_groups(MIXED_SRC_MODEL)} cls.sitecol = site.SiteCollection(cls.SITES)
def setUpClass(cls): conv = s.SourceConverter( investigation_time=50., rupture_mesh_spacing=1, # km complex_fault_mesh_spacing=1, # km width_of_mfd_bin=1., # for Truncated GR MFDs area_source_discretization=1.) cls.source_collector = { sc.trt: sc for sc in nrml.to_python(MIXED_SRC_MODEL, conv)} cls.sitecol = site.SiteCollection(cls.SITES)
def get_ltmodels(oq, gsim_lt, source_model_lt, h5=None): """ Build source models from the logic tree and to store them inside the `source_info` dataset. """ spinning_off = oq.collapse_factor == 0 or oq.pointsource_distance == 0 if spinning_off: logging.info('Removing nodal plane and hypocenter distributions') # NB: the source models file are often NOT in the shared directory # (for instance in oq-engine/demos) so the processpool must be used dist = ('no' if os.environ.get('OQ_DISTRIBUTE') == 'no' else 'processpool') smlt_dir = os.path.dirname(source_model_lt.filename) converter = sourceconverter.SourceConverter( oq.investigation_time, oq.rupture_mesh_spacing, oq.complex_fault_mesh_spacing, oq.width_of_mfd_bin, oq.area_source_discretization, oq.minimum_magnitude, not spinning_off, oq.source_id) if h5: sources = hdf5.create(h5, 'source_info', source_info_dt) lt_models = list(source_model_lt.gen_source_models(gsim_lt)) if oq.calculation_mode.startswith('ucerf'): idx = 0 [grp] = nrml.to_python(oq.inputs["source_model"], converter) for grp_id, ltm in enumerate(lt_models): sg = copy.copy(grp) sg.id = grp_id ltm.src_groups = [sg] src = sg[0].new(ltm.ordinal, ltm.names) # one source src.src_group_id = grp_id src.id = idx idx += 1 if oq.number_of_logic_tree_samples: src.samples = ltm.samples sg.sources = [src] data = [((grp_id, grp_id, src.source_id, src.code, 0, 0, -1, src.num_ruptures, 0, '', ''))] hdf5.extend(sources, numpy.array(data, source_info_dt)) return lt_models logging.info('Reading the source model(s) in parallel') allargs = [] fileno = 0 for ltm in lt_models: apply_unc = functools.partial( source_model_lt.apply_uncertainties, ltm.path) for name in ltm.names.split(): fname = os.path.abspath(os.path.join(smlt_dir, name)) allargs.append((ltm, apply_unc, fname, fileno)) fileno += 1 smap = parallel.Starmap( SourceReader(converter, smlt_dir, h5), allargs, distribute=dist, h5=h5 if h5 else None) # NB: h5 is None in logictree_test.py return _store_results(smap, lt_models, source_model_lt, gsim_lt, oq, h5)
def test(self): fname = os.path.join(os.path.dirname(__file__), 'faults_backg_source_model.xml') converter = sourceconverter.SourceConverter() # check that the input file requires the fix indeed with self.assertRaises(InvalidFile): nrml.SourceModelParser(converter).parse_groups(fname) fd, tmpname = tempfile.mkstemp(suffix='.xml') os.close(fd) fix(fname, tmpname) # invoke the fix print('meld %s %s' % (fname, tmpname))
def setUpClass(cls): cls.parser = nrml.SourceModelParser(s.SourceConverter( investigation_time=50., rupture_mesh_spacing=1, # km complex_fault_mesh_spacing=1, # km width_of_mfd_bin=1., # for Truncated GR MFDs area_source_discretization=1., # km )) groups = cls.parser.parse_groups(MIXED_SRC_MODEL) ([cls.point], [cls.cmplx], [cls.area, cls.simple], [cls.char_simple, cls.char_complex, cls.char_multi]) = groups # the parameters here would typically be specified in the job .ini cls.investigation_time = 50. cls.rupture_mesh_spacing = 1 # km cls.complex_fault_mesh_spacing = 1 # km cls.width_of_mfd_bin = 1. # for Truncated GR MFDs cls.area_source_discretization = 1. # km
def test(self): mod = mock.Mock(reference_vs30_value=760, reference_vs30_type='measured', reference_depth_to_1pt0km_per_sec=100., reference_depth_to_2pt5km_per_sec=5.0, reference_backarc=False) sitecol = site.SiteCollection.from_points([102.32], [-2.9107], [0], mod) parser = nrml.SourceModelParser( s.SourceConverter( investigation_time=50., rupture_mesh_spacing=1, # km complex_fault_mesh_spacing=1, # km width_of_mfd_bin=1., # for Truncated GR MFDs area_source_discretization=1., # km )) [[src]] = parser.parse_groups(self.bad_source) with self.assertRaises(AttributeError) as ctx, context(src): max_dist = 250 # NB: with a distance of 200 km the error does not happen src.filter_sites_by_distance_to_source(max_dist, sitecol) self.assertIn('An error occurred with source id=61', str(ctx.exception))
calling nrml.read() and node_to_obj() in sequence. """ [node] = read(fname) return node_to_obj(node, fname, *args) node_to_obj = CallableDict(keyfunc=get_tag_version, keymissing=lambda n, f: n) # dictionary of functions with at least two arguments, node and fname @node_to_obj.add(('ruptureCollection', 'nrml/0.5')) def get_rupture_collection(node, fname, converter): return converter.convert_node(node) default = sourceconverter.SourceConverter() # rupture_mesh_spacing=10 @node_to_obj.add(('sourceModel', 'nrml/0.4')) def get_source_model_04(node, fname, converter=default): sources = [] source_ids = set() converter.fname = fname for src_node in node: src = converter.convert_node(src_node) if src.source_id in source_ids: raise DuplicatedID('The source ID %s is duplicated!' % src.source_id) sources.append(src) source_ids.add(src.source_id) groups = groupby(sources, operator.attrgetter('tectonic_region_type'))
def get_source_models(oqparam, gsim_lt, source_model_lt, monitor, in_memory=True): """ Build all the source models generated by the logic tree. :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :param gsim_lt: a :class:`openquake.commonlib.logictree.GsimLogicTree` instance :param source_model_lt: a :class:`openquake.commonlib.logictree.SourceModelLogicTree` instance :param monitor: a `openquake.baselib.performance.Monitor` instance :param in_memory: if True, keep in memory the sources, else just collect the TRTs :returns: an iterator over :class:`openquake.commonlib.logictree.LtSourceModel` tuples """ make_sm = SourceModelFactory() converter = sourceconverter.SourceConverter( oqparam.investigation_time, oqparam.rupture_mesh_spacing, oqparam.complex_fault_mesh_spacing, oqparam.width_of_mfd_bin, oqparam.area_source_discretization, oqparam.source_id) if oqparam.calculation_mode.startswith('ucerf'): [grp] = nrml.to_python(oqparam.inputs["source_model"], converter) elif in_memory: logging.info('Reading the source model(s)') dic = logictree.parallel_read_source_models(gsim_lt, source_model_lt, converter, monitor) # consider only the effective realizations smlt_dir = os.path.dirname(source_model_lt.filename) idx = 0 grp_id = 0 if monitor.hdf5: sources = hdf5.create(monitor.hdf5, 'source_info', source_info_dt) hdf5.create(monitor.hdf5, 'source_geom', point3d) for sm in source_model_lt.gen_source_models(gsim_lt): src_groups = [] for name in sm.names.split(): fname = os.path.abspath(os.path.join(smlt_dir, name)) if oqparam.calculation_mode.startswith('ucerf'): sg = copy.copy(grp) sg.id = grp_id src = sg[0].new(sm.ordinal, sm.names) # one source src.id = idx sg.sources = [src] src_groups.append(sg) idx += 1 grp_id += 1 data = [((sg.id, src.source_id, src.code, 0, 0, src.num_ruptures, 0, 0, 0, 0, 0))] hdf5.extend(sources, numpy.array(data, source_info_dt)) elif in_memory: apply_unc = source_model_lt.make_apply_uncertainties(sm.path) newsm = make_sm(fname, dic[fname], apply_unc, oqparam.investigation_time) for sg in newsm: for src in sg: src.src_group_id = grp_id src.id = idx idx += 1 sg.id = grp_id grp_id += 1 if monitor.hdf5: store_sm(newsm, monitor.hdf5) src_groups.extend(newsm.src_groups) else: # just collect the TRT models src_groups.extend(logictree.read_source_groups(fname)) if grp_id >= TWO16: # the limit is really needed only for event based calculations raise ValueError('There is a limit of %d src groups!' % TWO16) num_sources = sum(len(sg.sources) for sg in src_groups) sm.src_groups = src_groups trts = [mod.trt for mod in src_groups] source_model_lt.tectonic_region_types.update(trts) logging.info( 'Processed source model %d with %d potential gsim path(s) and %d ' 'sources', sm.ordinal + 1, sm.num_gsim_paths, num_sources) gsim_file = oqparam.inputs.get('gsim_logic_tree') if gsim_file: # check TRTs for src_group in src_groups: if src_group.trt not in gsim_lt.values: raise ValueError( "Found in %r a tectonic region type %r inconsistent " "with the ones in %r" % (sm, src_group.trt, gsim_file)) yield sm # log if some source file is being used more than once dupl = 0 for fname, hits in make_sm.fname_hits.items(): if hits > 1: logging.info('%s has been considered %d times', fname, hits) if not make_sm.changed_sources: dupl += hits if (dupl and not oqparam.optimize_same_id_sources and 'event_based' not in oqparam.calculation_mode): logging.warn('You are doing redundant calculations: please make sure ' 'that different sources have different IDs and set ' 'optimize_same_id_sources=true in your .ini file') if make_sm.changed_sources: logging.info('Modified %d sources in the composite source model', make_sm.changed_sources)
def get_csm(oq, full_lt, h5=None): """ Build source models from the logic tree and to store them inside the `source_full_lt` dataset. """ converter = sourceconverter.SourceConverter( oq.investigation_time, oq.rupture_mesh_spacing, oq.complex_fault_mesh_spacing, oq.width_of_mfd_bin, oq.area_source_discretization, oq.minimum_magnitude, oq.source_id, discard_trts=oq.discard_trts, floating_x_step=oq.floating_x_step, floating_y_step=oq.floating_y_step) classical = not oq.is_event_based() full_lt.ses_seed = oq.ses_seed if oq.is_ucerf(): [grp] = nrml.to_python(oq.inputs["source_model"], converter) src_groups = [] for grp_id, sm_rlz in enumerate(full_lt.sm_rlzs): sg = copy.copy(grp) src_groups.append(sg) src = sg[0].new(sm_rlz.ordinal, sm_rlz.value[0]) # one source src.checksum = src.grp_id = src.trt_smr = grp_id src.samples = sm_rlz.samples logging.info('Reading sections and rupture planes for %s', src) planes = src.get_planes() if classical: src.ruptures_per_block = oq.ruptures_per_block sg.sources = list(src) for s in sg: s.planes = planes s.sections = s.get_sections() # add background point sources sg = copy.copy(grp) src_groups.append(sg) sg.sources = src.get_background_sources() else: # event_based, use one source sg.sources = [src] src.planes = planes src.sections = src.get_sections() return CompositeSourceModel(full_lt, src_groups) logging.info('Reading the source model(s) in parallel') # NB: the source models file are often NOT in the shared directory # (for instance in oq-engine/demos) so the processpool must be used dist = ('no' if os.environ.get('OQ_DISTRIBUTE') == 'no' else 'processpool') # NB: h5 is None in logictree_test.py allargs = [] for fname in full_lt.source_model_lt.info.smpaths: allargs.append((fname, converter)) smdict = parallel.Starmap(read_source_model, allargs, distribute=dist, h5=h5 if h5 else None).reduce() if len(smdict) > 1: # really parallel parallel.Starmap.shutdown() # save memory fix_geometry_sections(smdict) groups = _build_groups(full_lt, smdict) # checking the changes changes = sum(sg.changes for sg in groups) if changes: logging.info('Applied {:_d} changes to the composite source model'. format(changes)) return _get_csm(full_lt, groups)
def get_models(self): """ :yields: :class:`openquake.commonlib.logictree.LtSourceModel` tuples """ oq = self.oqparam spinning_off = self.oqparam.pointsource_distance == {'default': 0.0} if spinning_off: logging.info('Removing nodal plane and hypocenter distributions') dist = ('no' if os.environ.get('OQ_DISTRIBUTE') == 'no' else 'processpool') smlt_dir = os.path.dirname(self.source_model_lt.filename) converter = sourceconverter.SourceConverter( oq.investigation_time, oq.rupture_mesh_spacing, oq.complex_fault_mesh_spacing, oq.width_of_mfd_bin, oq.area_source_discretization, oq.minimum_magnitude, not spinning_off, oq.source_id) if oq.calculation_mode.startswith('ucerf'): [grp] = nrml.to_python(oq.inputs["source_model"], converter) dic = {'ucerf': grp} elif self.in_memory: logging.info('Reading the source model(s) in parallel') smap = parallel.Starmap( nrml.read_source_models, distribute=dist, hdf5path=self.hdf5.filename if self.hdf5 else None) for sm in self.source_model_lt.gen_source_models(self.gsim_lt): for name in sm.names.split(): fname = os.path.abspath(os.path.join(smlt_dir, name)) smap.submit([fname], converter) dic = {sm.fname: sm for sm in smap} else: dic = {} # consider only the effective realizations idx = 0 if self.hdf5: sources = hdf5.create(self.hdf5, 'source_info', source_info_dt) hdf5.create(self.hdf5, 'source_geom', point3d) hdf5.create(self.hdf5, 'source_mfds', hdf5.vstr) grp_id = 0 for sm in self.source_model_lt.gen_source_models(self.gsim_lt): if 'ucerf' in dic: sg = copy.copy(dic['ucerf']) sm.src_groups = [sg] sg.id = grp_id src = sg[0].new(sm.ordinal, sm.names) # one source src.src_group_id = grp_id src.id = idx if oq.number_of_logic_tree_samples: src.samples = sm.samples sg.sources = [src] idx += 1 grp_id += 1 data = [((sg.id, src.source_id, src.code, 0, 0, -1, src.num_ruptures, 0, 0, 0, idx))] hdf5.extend(sources, numpy.array(data, source_info_dt)) else: self.apply_uncertainties(sm, idx, dic) yield sm if self.hdf5: hdf5.extend(self.hdf5['source_mfds'], numpy.array(list(self.mfds), hdf5.vstr)) # log if some source file is being used more than once dupl = 0 for fname, hits in self.fname_hits.items(): if hits > 1: logging.info('%s has been considered %d times', fname, hits) if not self.changes: dupl += hits if self.changes: logging.info('Applied %d changes to the composite source model', self.changes)
return tag, version def to_python(fname, *args): """ Parse a NRML file and return an associated Python object. It works by calling nrml.read() and node_to_obj() in sequence. """ [node] = read(fname) return node_to_obj(node, fname, *args) node_to_obj = CallableDict(keyfunc=get_tag_version, keymissing=lambda n, f: n) # dictionary of functions with at least two arguments, node and fname default = sourceconverter.SourceConverter(area_source_discretization=10, rupture_mesh_spacing=10) @node_to_obj.add(('ruptureCollection', 'nrml/0.5')) def get_rupture_collection(node, fname, converter): return converter.convert_node(node) @node_to_obj.add(('geometryModel', 'nrml/0.5')) def get_geometry_model(node, fname, converter): return GeometryModel(converter.convert_node(node)) @node_to_obj.add(('sourceModel', 'nrml/0.4')) def get_source_model_04(node, fname, converter=default): sources = []
def get_source_models(oqparam, gsim_lt, source_model_lt, monitor, in_memory=True, srcfilter=None): """ Build all the source models generated by the logic tree. :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :param gsim_lt: a :class:`openquake.commonlib.logictree.GsimLogicTree` instance :param source_model_lt: a :class:`openquake.commonlib.logictree.SourceModelLogicTree` instance :param monitor: a `openquake.baselib.performance.Monitor` instance :param in_memory: if True, keep in memory the sources, else just collect the TRTs :param srcfilter: a SourceFilter instance with an .filename pointing to the cache file :returns: an iterator over :class:`openquake.commonlib.logictree.LtSourceModel` tuples """ make_sm = SourceModelFactory() spinning_off = oqparam.pointsource_distance == {'default': 0.0} if spinning_off: logging.info('Removing nodal plane and hypocenter distributions') dist = 'no' if os.environ.get('OQ_DISTRIBUTE') == 'no' else 'processpool' smlt_dir = os.path.dirname(source_model_lt.filename) converter = sourceconverter.SourceConverter( oqparam.investigation_time, oqparam.rupture_mesh_spacing, oqparam.complex_fault_mesh_spacing, oqparam.width_of_mfd_bin, oqparam.area_source_discretization, oqparam.minimum_magnitude, not spinning_off, oqparam.source_id) if oqparam.calculation_mode.startswith('ucerf'): [grp] = nrml.to_python(oqparam.inputs["source_model"], converter) elif in_memory: logging.info('Reading the source model(s) in parallel') smap = parallel.Starmap(nrml.read_source_models, monitor=monitor, distribute=dist) for sm in source_model_lt.gen_source_models(gsim_lt): for name in sm.names.split(): fname = os.path.abspath(os.path.join(smlt_dir, name)) smap.submit([fname], converter) dic = {sm.fname: sm for sm in smap} # consider only the effective realizations nr = 0 idx = 0 grp_id = 0 if monitor.hdf5: sources = hdf5.create(monitor.hdf5, 'source_info', source_info_dt) hdf5.create(monitor.hdf5, 'source_geom', point3d) filename = None source_ids = set() for sm in source_model_lt.gen_source_models(gsim_lt): apply_unc = functools.partial(source_model_lt.apply_uncertainties, sm.path) src_groups = [] for name in sm.names.split(): fname = os.path.abspath(os.path.join(smlt_dir, name)) if oqparam.calculation_mode.startswith('ucerf'): sg = copy.copy(grp) sg.id = grp_id src = sg[0].new(sm.ordinal, sm.names) # one source source_ids.add(src.source_id) src.src_group_id = grp_id src.id = idx if oqparam.number_of_logic_tree_samples: src.samples = sm.samples sg.sources = [src] src_groups.append(sg) idx += 1 grp_id += 1 data = [((sg.id, src.source_id, src.code, 0, 0, src.num_ruptures, 0, 0, 0))] hdf5.extend(sources, numpy.array(data, source_info_dt)) elif in_memory: newsm = make_sm(fname, dic[fname], apply_unc, oqparam.investigation_time) for sg in newsm: nr += sum(src.num_ruptures for src in sg) # sample a source for each group if os.environ.get('OQ_SAMPLE_SOURCES'): sg.sources = random_filtered_sources( sg.sources, srcfilter, sg.id + oqparam.random_seed) for src in sg: source_ids.add(src.source_id) src.src_group_id = grp_id src.id = idx idx += 1 sg.id = grp_id grp_id += 1 src_groups.append(sg) if monitor.hdf5: store_sm(newsm, filename, monitor) else: # just collect the TRT models groups = logictree.read_source_groups(fname) for group in groups: source_ids.update(src['id'] for src in group) src_groups.extend(groups) if grp_id >= TWO16: # the limit is really needed only for event based calculations raise ValueError('There is a limit of %d src groups!' % TWO16) for brid, srcids in source_model_lt.info.applytosources.items(): for srcid in srcids: if srcid not in source_ids: raise ValueError( 'The source %s is not in the source model, please fix ' 'applyToSources in %s or the source model' % (srcid, source_model_lt.filename)) num_sources = sum(len(sg.sources) for sg in src_groups) sm.src_groups = src_groups trts = [mod.trt for mod in src_groups] source_model_lt.tectonic_region_types.update(trts) logging.info( 'Processed source model %d with %d gsim path(s) and %d ' 'sources', sm.ordinal + 1, sm.num_gsim_paths, num_sources) gsim_file = oqparam.inputs.get('gsim_logic_tree') if gsim_file: # check TRTs for src_group in src_groups: if src_group.trt not in gsim_lt.values: raise ValueError( "Found in %r a tectonic region type %r inconsistent " "with the ones in %r" % (sm, src_group.trt, gsim_file)) yield sm logging.info('The composite source model has {:,d} ruptures'.format(nr)) # log if some source file is being used more than once dupl = 0 for fname, hits in make_sm.fname_hits.items(): if hits > 1: logging.info('%s has been considered %d times', fname, hits) if not make_sm.changes: dupl += hits if (dupl and not oqparam.optimize_same_id_sources and not oqparam.is_event_based()): logging.warning( 'You are doing redundant calculations: please make sure ' 'that different sources have different IDs and set ' 'optimize_same_id_sources=true in your .ini file') if make_sm.changes: logging.info('Applied %d changes to the composite source model', make_sm.changes)
def get_csm(oq, source_model_lt, gsim_lt, h5=None): """ Build source models from the logic tree and to store them inside the `source_full_lt` dataset. """ if oq.pointsource_distance['default'] == {}: spinning_off = False else: spinning_off = sum(oq.pointsource_distance.values()) == 0 if spinning_off: logging.info('Removing nodal plane and hypocenter distributions') smlt_dir = os.path.dirname(source_model_lt.filename) converter = sourceconverter.SourceConverter( oq.investigation_time, oq.rupture_mesh_spacing, oq.complex_fault_mesh_spacing, oq.width_of_mfd_bin, oq.area_source_discretization, oq.minimum_magnitude, not spinning_off, oq.source_id, discard_trts=oq.discard_trts) full_lt = FullLogicTree(source_model_lt, gsim_lt) classical = not oq.is_event_based() if oq.is_ucerf(): sample = .001 if os.environ.get('OQ_SAMPLE_SOURCES') else None [grp] = nrml.to_python(oq.inputs["source_model"], converter) checksum = 0 src_groups = [] for grp_id, sm_rlz in enumerate(full_lt.sm_rlzs): sg = copy.copy(grp) src_groups.append(sg) src = sg[0].new(sm_rlz.ordinal, sm_rlz.value) # one source src.checksum = src.grp_id = src.id = grp_id src.samples = sm_rlz.samples if classical: # split the sources upfront to improve the task distribution sg.sources = src.get_background_sources(sample) if not sample: for s in src: sg.sources.append(s) s.checksum = checksum checksum += 1 else: # event_based, use one source sg.sources = [src] return CompositeSourceModel(full_lt, src_groups) logging.info('Reading the source model(s) in parallel') groups = [[] for sm_rlz in full_lt.sm_rlzs] allargs = [] fileno = 0 for rlz in full_lt.sm_rlzs: for name in rlz.value.split(): fname = os.path.abspath(os.path.join(smlt_dir, name)) allargs.append((rlz.ordinal, rlz.lt_path, source_model_lt.apply_uncertainties, fname, fileno)) fileno += 1 # NB: the source models file are often NOT in the shared directory # (for instance in oq-engine/demos) so the processpool must be used dist = ('no' if os.environ.get('OQ_DISTRIBUTE') == 'no' else 'processpool') smap = parallel.Starmap( SourceReader(converter, smlt_dir, h5), allargs, distribute=dist, h5=h5 if h5 else None) # NB: h5 is None in logictree_test.py # various checks changes = 0 for dic in sorted(smap, key=operator.itemgetter('fileno')): eri = dic['ordinal'] groups[eri].extend(dic['src_groups']) for sg in dic['src_groups']: changes += sg.changes gsim_file = oq.inputs.get('gsim_logic_tree') if gsim_file: # check TRTs for src_group in dic['src_groups']: if src_group.trt not in gsim_lt.values: raise ValueError( "Found in the source models a tectonic region type %r " "inconsistent with the ones in %r" % (src_group.trt, gsim_file)) for sm_rlz in full_lt.sm_rlzs: # check applyToSources source_ids = set(src.source_id for grp in groups[sm_rlz.ordinal] for src in grp) for brid, srcids in source_model_lt.info.applytosources.items(): if brid in sm_rlz.lt_path: for srcid in srcids: if srcid not in source_ids: raise ValueError( "The source %s is not in the source model," " please fix applyToSources in %s or the " "source model" % (srcid, source_model_lt.filename)) if changes: logging.info('Applied %d changes to the composite source model', changes) return _get_csm(full_lt, groups)
def get_source_models(oqparam, gsim_lt, source_model_lt, in_memory=True): """ Build all the source models generated by the logic tree. :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :param gsim_lt: a :class:`openquake.commonlib.logictree.GsimLogicTree` instance :param source_model_lt: a :class:`openquake.commonlib.logictree.SourceModelLogicTree` instance :param in_memory: if True, keep in memory the sources, else just collect the TRTs :returns: an iterator over :class:`openquake.commonlib.logictree.LtSourceModel` tuples """ converter = sourceconverter.SourceConverter( oqparam.investigation_time, oqparam.rupture_mesh_spacing, oqparam.complex_fault_mesh_spacing, oqparam.width_of_mfd_bin, oqparam.area_source_discretization) psr = nrml.SourceModelParser(converter) if oqparam.calculation_mode.startswith('ucerf'): [grp] = nrml.to_python(oqparam.inputs["source_model"], converter) # consider only the effective realizations smlt_dir = os.path.dirname(source_model_lt.filename) for sm in source_model_lt.gen_source_models(gsim_lt): src_groups = [] for name in sm.names.split(): fname = os.path.abspath(os.path.join(smlt_dir, name)) if oqparam.calculation_mode.startswith('ucerf'): sg = copy.copy(grp) sg.id = sm.ordinal sg.sources = [sg[0].new(sm.ordinal, sm.names)] # one source src_groups.append(sg) elif in_memory: apply_unc = source_model_lt.make_apply_uncertainties(sm.path) logging.info('Reading %s', fname) src_groups.extend(psr.parse_src_groups(fname, apply_unc)) else: # just collect the TRT models src_groups.extend(read_source_groups(fname)) num_sources = sum(len(sg.sources) for sg in src_groups) sm.src_groups = src_groups trts = [mod.trt for mod in src_groups] source_model_lt.tectonic_region_types.update(trts) logging.info( 'Processed source model %d with %d potential gsim path(s) and %d ' 'sources', sm.ordinal + 1, sm.num_gsim_paths, num_sources) gsim_file = oqparam.inputs.get('gsim_logic_tree') if gsim_file: # check TRTs for src_group in src_groups: if src_group.trt not in gsim_lt.values: raise ValueError( "Found in %r a tectonic region type %r inconsistent " "with the ones in %r" % (sm, src_group.trt, gsim_file)) yield sm # check investigation_time psr.check_nonparametric_sources(oqparam.investigation_time) # log if some source file is being used more than once dupl = 0 for fname, hits in psr.fname_hits.items(): if hits > 1: logging.info('%s has been considered %d times', fname, hits) if not psr.changed_sources: dupl += hits if dupl and not oqparam.optimize_same_id_sources: logging.warn('You are doing redundant calculations: please make sure ' 'that different sources have different IDs and set ' 'optimize_same_id_sources=true in your .ini file')
def get_csm(oq, full_lt, h5=None): """ Build source models from the logic tree and to store them inside the `source_full_lt` dataset. """ if oq.pointsource_distance is None: spinning_off = False else: spinning_off = sum(oq.pointsource_distance.max().values()) == 0 if spinning_off: logging.info('Removing nodal plane and hypocenter distributions') converter = sourceconverter.SourceConverter(oq.investigation_time, oq.rupture_mesh_spacing, oq.complex_fault_mesh_spacing, oq.width_of_mfd_bin, oq.area_source_discretization, oq.minimum_magnitude, not spinning_off, oq.source_id, discard_trts=oq.discard_trts) logging.info('%d effective smlt realization(s)', len(full_lt.sm_rlzs)) classical = not oq.is_event_based() if oq.is_ucerf(): sample = .001 if os.environ.get('OQ_SAMPLE_SOURCES') else None [grp] = nrml.to_python(oq.inputs["source_model"], converter) src_groups = [] for grp_id, sm_rlz in enumerate(full_lt.sm_rlzs): sg = copy.copy(grp) src_groups.append(sg) src = sg[0].new(sm_rlz.ordinal, sm_rlz.value) # one source sg.mags = numpy.unique(numpy.round(src.mags, 2)) del src.__dict__['mags'] # remove cache src.checksum = src.grp_id = src.id = grp_id src.samples = sm_rlz.samples if classical: src.ruptures_per_block = oq.ruptures_per_block if sample: sg.sources = [list(src)[0]] # take the first source else: sg.sources = list(src) # add background point sources sg.sources.extend(src.get_background_sources(sample)) else: # event_based, use one source sg.sources = [src] return CompositeSourceModel(full_lt, src_groups) logging.info('Reading the source model(s) in parallel') if 'OQ_SAMPLE_SOURCES' in os.environ and h5: srcfilter = calc.filters.SourceFilter(h5['sitecol'], h5['oqparam'].maximum_distance) else: srcfilter = None # NB: the source models file are often NOT in the shared directory # (for instance in oq-engine/demos) so the processpool must be used dist = ('no' if os.environ.get('OQ_DISTRIBUTE') == 'no' else 'processpool') # NB: h5 is None in logictree_test.py allargs = [] for fname in full_lt.source_model_lt.info.smpaths: allargs.append((fname, converter, srcfilter)) smdict = parallel.Starmap(read_source_model, allargs, distribute=dist, h5=h5 if h5 else None).reduce() if len(smdict) > 1: # really parallel parallel.Starmap.shutdown() # save memory check_dupl_ids(smdict) groups = _build_groups(full_lt, smdict) # checking the changes changes = sum(sg.changes for sg in groups) if changes: logging.info('Applied %d changes to the composite source model', changes) return _get_csm(full_lt, groups)
[node] = read(fname) return node_to_obj(node, fname, *args) parse = deprecated('Use nrml.to_python instead')(to_python) node_to_obj = CallableDict(keyfunc=get_tag_version, keymissing=lambda n, f: n) # dictionary of functions with at least two arguments, node and fname @node_to_obj.add(('ruptureCollection', 'nrml/0.5')) def get_rupture_collection(node, fname, converter): return converter.convert_node(node) default = sourceconverter.SourceConverter() @node_to_obj.add(('sourceModel', 'nrml/0.4')) def get_source_model_04(node, fname, converter=default): sources = [] source_ids = set() converter.fname = fname for no, src_node in enumerate(node, 1): src = converter.convert_node(src_node) if src.source_id in source_ids: raise DuplicatedID('The source ID %s is duplicated!' % src.source_id) sources.append(src) source_ids.add(src.source_id) if no % 10000 == 0: # log every 10,000 sources parsed
def read_input(hparams, **extra): """ :param hparams: a dictionary of hazard parameters :returns: an Input namedtuple (groups, sitecol, gsim_lt, cmakerdict) The dictionary must contain the keys - "maximum_distance" - "imtls" - "source_model_file" or "rupture_model_file" - "sites" or "site_model_file" - "gsim" or "gsim_logic_tree_file" Moreover: - if "source_model_file" is given, then "investigation_time" is mandatory - if "rupture_model_file" is given, the "number_of_ground_motion_fields" and "ses_seed" are mandatory - if there is an area source, then "area_source_discretization" is needed - if "site_model_file" is missing, then global site parameters are needed The optional keys include - "rupture_mesh_spacing" (default 5.) - "complex_fault_mesh_spacing" (default rupture_mesh_spacing) - "width_of_mfd_bin" (default 1.) - "minimum_magnitude" - "discard_trts" (default "") - "number_of_logic_tree_samples" (default 0) - "ses_per_logic_tree_path" (default 1) """ if isinstance(hparams, str): hparams = read_hparams(hparams) if extra: hparams = hparams.copy() hparams.update(extra) assert 'imts' in hparams or 'imtls' in hparams assert isinstance(hparams['maximum_distance'], IntegrationDistance) smfname = hparams.get('source_model_file') if smfname: # nonscenario itime = hparams['investigation_time'] else: itime = 50. # ignored in scenario rmfname = hparams.get('rupture_model_file') if rmfname: ngmfs = hparams["number_of_ground_motion_fields"] ses_seed = hparams["ses_seed"] converter = sourceconverter.SourceConverter( itime, hparams.get('rupture_mesh_spacing', 5.), hparams.get('complex_fault_mesh_spacing'), hparams.get('width_of_mfd_bin', 1.0), hparams.get('area_source_discretization'), hparams.get('minimum_magnitude', {'default': 0}), hparams.get('source_id'), discard_trts=hparams.get('discard_trts', '')) if smfname: [sm] = nrml.read_source_models([smfname], converter) groups = sm.src_groups elif rmfname: ebrs = _get_ebruptures(rmfname, converter, ses_seed) groups = _rupture_groups(ebrs) else: raise KeyError('Missing source_model_file or rupture_file') trts = set(grp.trt for grp in groups) if 'gsim' in hparams: gslt = gsim_lt.GsimLogicTree.from_(hparams['gsim']) else: gslt = gsim_lt.GsimLogicTree(hparams['gsim_logic_tree_file'], trts) # fix source attributes idx = 0 num_rlzs = gslt.get_num_paths() for grp_id, sg in enumerate(groups): assert len(sg) # sanity check for src in sg: src.id = idx src.grp_id = grp_id src.trt_smr = grp_id src.samples = num_rlzs idx += 1 cmakerdict = {} # trt => cmaker start = 0 n = hparams.get('number_of_logic_tree_samples', 0) s = hparams.get('random_seed', 42) for trt, rlzs_by_gsim in gslt.get_rlzs_by_gsim_trt(n, s).items(): cmakerdict[trt] = contexts.ContextMaker(trt, rlzs_by_gsim, hparams) cmakerdict[trt].start = start start += len(rlzs_by_gsim) if rmfname: # for instance for 2 TRTs with 5x2 GSIMs and ngmfs=10, then the # number of occupation is 100 for each rupture, for a total # of 200 events, see scenario/case_13 nrlzs = gslt.get_num_paths() for grp in groups: for ebr in grp: ebr.n_occ = ngmfs * nrlzs sitecol = _get_sitecol(hparams, gslt.req_site_params) return Input(groups, sitecol, gslt, cmakerdict)
def get_source_models(oqparam, gsim_lt, source_model_lt, in_memory=True): """ Build all the source models generated by the logic tree. :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :param gsim_lt: a :class:`openquake.commonlib.logictree.GsimLogicTree` instance :param source_model_lt: a :class:`openquake.commonlib.logictree.SourceModelLogicTree` instance :param in_memory: if True, keep in memory the sources, else just collect the TRTs :returns: an iterator over :class:`openquake.commonlib.logictree.SourceModel` tuples """ converter = sourceconverter.SourceConverter( oqparam.investigation_time, oqparam.rupture_mesh_spacing, oqparam.complex_fault_mesh_spacing, oqparam.width_of_mfd_bin, oqparam.area_source_discretization) psr = nrml.SourceModelParser(converter) # consider only the effective realizations for sm in source_model_lt.gen_source_models(gsim_lt): src_groups = [] for name in sm.name.split(): fname = possibly_gunzip( os.path.abspath(os.path.join(oqparam.base_path, name))) if in_memory: apply_unc = source_model_lt.make_apply_uncertainties(sm.path) try: logging.info('Parsing %s', fname) src_groups.extend(psr.parse_src_groups(fname, apply_unc)) except ValueError as e: if str(e) in ('Surface does not conform with Aki & ' 'Richards convention', 'Edges points are not in the right order'): raise InvalidFile('''\ %s: %s. Probably you are using an obsolete model. In that case you can fix the file with the command python -m openquake.engine.tools.correct_complex_sources %s ''' % (fname, e, fname)) else: raise else: # just collect the TRT models smodel = nrml.read(fname).sourceModel if smodel[0].tag.endswith('sourceGroup'): # NRML 0.5 format for sg_node in smodel: sg = sourceconverter.SourceGroup( sg_node['tectonicRegion']) sg.sources = sg_node.nodes src_groups.append(sg) else: # NRML 0.4 format: smodel is a list of source nodes src_groups.extend( sourceconverter.SourceGroup.collect(smodel)) num_sources = sum(len(sg.sources) for sg in src_groups) sm.src_groups = src_groups trts = [mod.trt for mod in src_groups] source_model_lt.tectonic_region_types.update(trts) logging.info( 'Processed source model %d with %d potential gsim path(s) and %d ' 'sources', sm.ordinal + 1, sm.num_gsim_paths, num_sources) gsim_file = oqparam.inputs.get('gsim_logic_tree') if gsim_file: # check TRTs for src_group in src_groups: if src_group.trt not in gsim_lt.values: raise ValueError( "Found in %r a tectonic region type %r inconsistent " "with the ones in %r" % (sm, src_group.trt, gsim_file)) yield sm # log if some source file is being used more than once for fname, hits in psr.fname_hits.items(): if hits > 1: logging.info('%s has been considered %d times', fname, hits)