def _get_csm(full_lt, groups): # 1. extract a single source from multiple sources with the same ID # 2. regroup the sources in non-atomic groups by TRT # 3. reorder the sources by source_id atomic = [] acc = general.AccumDict(accum=[]) for grp in groups: if grp and grp.atomic: atomic.append(grp) elif grp: acc[grp.trt].extend(grp) key = operator.attrgetter('source_id', 'code') src_groups = [] for trt in acc: lst = [] for srcs in general.groupby(acc[trt], key).values(): if len(srcs) > 1: srcs = reduce_sources(srcs) lst.extend(srcs) for sources in general.groupby(lst, trt_smrs).values(): # set ._wkt attribute (for later storage in the source_wkt dataset) for src in sources: src._wkt = src.wkt() src_groups.append(sourceconverter.SourceGroup(trt, sources)) for ag in atomic: for src in ag: src._wkt = src.wkt() src_groups.extend(atomic) _check_dupl_ids(src_groups) for sg in src_groups: sg.sources.sort(key=operator.attrgetter('source_id')) return CompositeSourceModel(full_lt, src_groups)
def _get_csm(full_lt, groups): # extract a single source from multiple sources with the same ID # and regroup the sources in non-atomic groups by TRT atomic = [] acc = general.AccumDict(accum=[]) for grp in groups: if grp and grp.atomic: atomic.append(grp) elif grp: acc[grp.trt].extend(grp) key = operator.attrgetter('source_id', 'code') src_groups = [] serial = full_lt.ses_seed for trt in acc: lst = [] for srcs in general.groupby(acc[trt], key).values(): if len(srcs) > 1: srcs = reduce_sources(srcs) for src in srcs: src._wkt = src.wkt() lst.append(src) serial = init_serials(lst, serial) for grp in general.groupby(lst, et_ids).values(): src_groups.append(sourceconverter.SourceGroup(trt, grp)) for ag in atomic: serial = init_serials(ag.sources, serial) for src in ag: src._wkt = src.wkt() src_groups.extend(atomic) _check_dupl_ids(src_groups) return CompositeSourceModel(full_lt, src_groups)
def write_source_model(dest, sources_or_groups, name=None): """ Writes a source model to XML. :param str dest: Destination path :param list sources_or_groups: Source model as list of sources or a list of SourceGroups :param str name: Name of the source model (if missing, extracted from the filename) """ if isinstance(sources_or_groups[0], sourceconverter.SourceGroup): groups = sources_or_groups else: # passed a list of sources srcs_by_trt = groupby(sources_or_groups, operator.attrgetter('tectonic_region_type')) groups = [ sourceconverter.SourceGroup(trt, srcs_by_trt[trt]) for trt in srcs_by_trt ] name = name or os.path.splitext(os.path.basename(dest))[0] nodes = list(map(obj_to_node, sorted(groups))) source_model = Node("sourceModel", {"name": name}, nodes=nodes) with open(dest, 'wb') as f: nrml.write([source_model], f, '%s') return dest
def __fromh5__(self, dic, attrs): # TODO: this is called more times than needed, maybe we should cache it sg_data = group_array(dic['sg_data'], 'sm_id') sm_data = dic['sm_data'] vars(self).update(attrs) self.gsim_fname = decode(self.gsim_fname) if self.gsim_fname.endswith('.xml'): # otherwise it would look in the current directory GMPETable.GMPE_DIR = os.path.dirname(self.gsim_fname) trts = sorted(self.trts) tmp = gettemp(self.gsim_lt_xml, suffix='.xml') self.gsim_lt = logictree.GsimLogicTree(tmp, trts) else: # fake file with the name of the GSIM self.gsim_lt = logictree.GsimLogicTree.from_(self.gsim_fname) self.source_models = [] for sm_id, rec in enumerate(sm_data): tdata = sg_data[sm_id] srcgroups = [ sourceconverter.SourceGroup(self.trts[data['trti']], id=data['grp_id'], eff_ruptures=data['effrup'], tot_ruptures=get_totrup(data)) for data in tdata if data['effrup'] ] path = tuple(str(decode(rec['path'])).split('_')) trts = set(sg.trt for sg in srcgroups) sm = logictree.LtSourceModel(rec['name'], rec['weight'], path, srcgroups, rec['num_rlzs'], sm_id, rec['samples']) self.source_models.append(sm) self.init() try: os.remove(tmp) # gsim_lt file except NameError: # tmp is defined only in the regular case, see above pass
def _get_csm(full_lt, groups): # extract a single source from multiple sources with the same ID # and regroup the sources in non-atomic groups by TRT atomic = [] acc = general.AccumDict(accum=[]) for grp in groups: if grp and grp.atomic: atomic.append(grp) elif grp: acc[grp.trt].extend(grp) key = operator.attrgetter('source_id', 'code') idx = 0 src_groups = [] for trt in acc: lst = [] for srcs in general.groupby(acc[trt], key).values(): if len(srcs) > 1: srcs = reduce_sources(srcs) for src in srcs: src.id = idx src._wkt = src.wkt() idx += 1 lst.append(src) src_groups.append(sourceconverter.SourceGroup(trt, lst)) for ag in atomic: for src in ag: src.id = idx src._wkt = src.wkt() idx += 1 src_groups.extend(atomic) return CompositeSourceModel(full_lt, src_groups)
def setUp(self): # simple logic tree with 3 realizations # ___/ b11 (w=.2) # _/ \ b12 (w=.2) # \____ b02 (w=.6) self.bs0 = bs0 = lt.BranchSet('abGRAbsolute') bs0.branches = [ lt.Branch('bs0', 'b01', .4, (4.6, 1.1)), lt.Branch('bs0', 'b02', .6, (4.4, 0.9)) ] self.bs1 = bs1 = lt.BranchSet('maxMagGRAbsolute') bs1.branches = [ lt.Branch('bs1', 'b11', .5, 7.0), lt.Branch('bs1', 'b12', .5, 7.6) ] bs0.branches[0].bset = bs1 # setup sitecol, srcfilter, gsims, imtls sitecol = site.SiteCollection( [site.Site(Point(0, 0), numpy.array([760.]))]) self.srcfilter = calc.filters.SourceFilter(sitecol, {'default': 200}) self.gsims = [valid.gsim('ToroEtAl2002')] self.imtls = DictArray({'PGA': valid.logscale(.01, 1, 5)}) self.sg = sourceconverter.SourceGroup(ps.tectonic_region_type, [ps])
def _rupture_groups(ebruptures): ebruptures.sort(key=bytrt) rup_groups = [] for trt, ebrs in itertools.groupby(ebruptures, bytrt): grp = sourceconverter.SourceGroup(trt) grp.sources = list(ebrs) rup_groups.append(grp) return rup_groups
def write_source_model(dest, sources_or_groups, name=None, investigation_time=None): """ Writes a source model to XML. :param dest: Destination path :param sources_or_groups: Source model in different formats :param name: Name of the source model (if missing, extracted from the filename) """ if isinstance(sources_or_groups, nrml.SourceModel): groups = sources_or_groups.src_groups attrs = dict(name=sources_or_groups.name, investigation_time=sources_or_groups.investigation_time) elif isinstance(sources_or_groups[0], sourceconverter.SourceGroup): groups = sources_or_groups attrs = dict(investigation_time=investigation_time) else: # passed a list of sources srcs_by_trt = groupby( sources_or_groups, operator.attrgetter('tectonic_region_type')) groups = [sourceconverter.SourceGroup(trt, srcs_by_trt[trt]) for trt in srcs_by_trt] attrs = dict(investigation_time=investigation_time) if name or 'name' not in attrs: attrs['name'] = name or os.path.splitext(os.path.basename(dest))[0] if attrs['investigation_time'] is None: del attrs['investigation_time'] nodes = list(map(obj_to_node, groups)) ddict = extract_ddict(groups) if ddict: # remove duplicate content from nodes for grp_node in nodes: for src_node in grp_node: src_node.nodes = [] # save HDF5 file dest5 = os.path.splitext(dest)[0] + '.hdf5' with hdf5.File(dest5, 'w') as h: for src_id, dic in ddict.items(): for k, v in dic.items(): key = '%s/%s' % (src_id, k) if isinstance(v, numpy.ndarray): h.create_dataset(key, v.shape, v.dtype, compression='gzip', compression_opts=9) h[key][:] = v else: h[key] = v source_model = Node("sourceModel", attrs, nodes=nodes) with open(dest, 'wb') as f: nrml.write([source_model], f, '%s') if ddict: return [dest, dest5] else: return [dest]
def __fromh5__(self, dic, attrs): vars(self).update(attrs) self.src_groups = [] for grp_name, grp in dic.items(): trt = grp.attrs['trt'] srcs = [] for src_id in sorted(grp): src = grp[src_id] src.num_ruptures = src.count_ruptures() srcs.append(src) grp = sourceconverter.SourceGroup(trt, srcs, grp_name) self.src_groups.append(grp)
def fake(cls, gsimlt=None): """ :returns: a fake `CompositionInfo` instance with the given gsim logic tree object; if None, builds automatically a fake gsim logic tree """ weight = 1 gsim_lt = gsimlt or logictree.GsimLogicTree.from_('FromFile') fakeSM = logictree.SourceModel( 'fake', weight, 'b1', [sourceconverter.SourceGroup('*', eff_ruptures=1)], gsim_lt.get_num_paths(), ordinal=0, samples=1) return cls(gsim_lt, seed=0, num_samples=0, source_models=[fakeSM], tot_weight=0)
def get_source_model_04(node, fname, converter=default): sources = [] source_ids = set() converter.fname = fname for src_node in node: src = converter.convert_node(src_node) if src.source_id in source_ids: raise DuplicatedID('The source ID %s is duplicated!' % src.source_id) sources.append(src) source_ids.add(src.source_id) groups = groupby(sources, operator.attrgetter('tectonic_region_type')) src_groups = sorted( sourceconverter.SourceGroup(trt, srcs) for trt, srcs in groups.items()) return SourceModel(src_groups, node.get('name', ''))
def read_source_groups(fname): """ :param fname: a path to a source model XML file :return: a list of SourceGroup objects containing source nodes """ smodel = nrml.read(fname).sourceModel src_groups = [] if smodel[0].tag.endswith('sourceGroup'): # NRML 0.5 format for sg_node in smodel: sg = sourceconverter.SourceGroup(sg_node['tectonicRegion']) sg.sources = sg_node.nodes src_groups.append(sg) else: # NRML 0.4 format: smodel is a list of source nodes src_groups.extend(sourceconverter.SourceGroup.collect(smodel)) return src_groups
def fake(cls, gsimlt=None): """ :returns: a fake `CompositionInfo` instance with the given gsim logic tree object; if None, builds automatically a fake gsim logic tree """ weight = 1 gsim_lt = gsimlt or logictree.GsimLogicTree.from_('[FromFile]') fakeSM = source_reader.LtSourceModel( 'scenario', weight, 'b1', [sourceconverter.SourceGroup('*', eff_ruptures=1)], ordinal=0, samples=1, offset=0) return cls(gsim_lt, seed=0, num_samples=0, source_models=[fakeSM])
def get_source_model_04(node, fname, converter): sources = [] source_ids = set() converter.fname = fname for no, src_node in enumerate(node, 1): src = converter.convert_node(src_node) if src.source_id in source_ids: raise DuplicatedID('The source ID %s is duplicated!' % src.source_id) sources.append(src) source_ids.add(src.source_id) if no % 10000 == 0: # log every 10,000 sources parsed logging.info('Instantiated %d sources from %s', no, fname) groups = groupby(sources, operator.attrgetter('tectonic_region_type')) return sorted( sourceconverter.SourceGroup(trt, srcs) for trt, srcs in groups.items())
def __init__(self, info, groups, ses_seed=0, event_based=False): self.gsim_lt = info.gsim_lt self.source_model_lt = info.source_model_lt self.sm_rlzs = info.sm_rlzs self.info = info # extract a single source from multiple sources with the same ID # and regroup the sources in non-atomic groups by TRT atomic = [] acc = AccumDict(accum=[]) get_grp_id = info.source_model_lt.get_grp_id(info.gsim_lt.values) for sm in self.sm_rlzs: for grp in groups[sm.ordinal]: if grp and grp.atomic: atomic.append(grp) elif grp: acc[grp.trt].extend(grp) grp_id = get_grp_id(grp.trt, sm.ordinal) for src in grp: src.grp_id = grp_id if sm.samples > 1: src.samples = sm.samples dic = {} key = operator.attrgetter('source_id', 'checksum') idx = 0 for trt in acc: lst = [] for srcs in groupby(acc[trt], key).values(): for src in srcs: src.id = idx idx += 1 if len(srcs) > 1: # happens in classical/case_20 src.grp_id = [s.grp_id for s in srcs] lst.append(src) dic[trt] = sourceconverter.SourceGroup(trt, lst) for ag in atomic: for src in ag: src.id = idx idx += 1 self.src_groups = list(dic.values()) + atomic if event_based: # init serials serial = ses_seed for sg in self.src_groups: for src in sg: src.serial = serial serial += src.num_ruptures * len(src.grp_ids)
def __fromh5__(self, dic, attrs): # TODO: this is called more times than needed, maybe we should cache it sg_data = group_array(dic['sg_data'], 'sm_id') sm_data = dic['sm_data'] vars(self).update(attrs) self.gsim_fname = decode(self.gsim_fname) if self.gsim_fname.endswith('.xml'): trts = sorted(self.trts) if 'gmpe_table' in self.gsim_lt_xml: # the canadian gsims depends on external files which are not # in the datastore; I am storing the path to the original # file so that the external files can be found; unfortunately, # this means that copying the datastore on a different machine # and exporting from there works only if the gsim_fname and all # the external files are copied in the exact same place self.gsim_lt = logictree.GsimLogicTree(self.gsim_fname, trts) else: # regular case: read the logic tree from self.gsim_lt_xml, # so that you do not need to copy anything except the datastore tmp = writetmp(self.gsim_lt_xml, suffix='.xml') self.gsim_lt = logictree.GsimLogicTree(tmp, trts) else: # fake file with the name of the GSIM self.gsim_lt = logictree.GsimLogicTree.from_(self.gsim_fname) self.source_models = [] for sm_id, rec in enumerate(sm_data): tdata = sg_data[sm_id] srcgroups = [ sourceconverter.SourceGroup(self.trts[trti], id=grp_id, eff_ruptures=effrup, tot_ruptures=totrup) for grp_id, trti, effrup, totrup, sm_id in tdata if effrup ] path = tuple(str(decode(rec['path'])).split('_')) trts = set(sg.trt for sg in srcgroups) num_gsim_paths = self.gsim_lt.reduce(trts).get_num_paths() sm = logictree.SourceModel(rec['name'], rec['weight'], path, srcgroups, num_gsim_paths, sm_id, rec['samples']) self.source_models.append(sm) self.init() try: os.remove(tmp) # gsim_lt file except NameError: # tmp is defined only in the regular case, see above pass
def grp_by_src(self): """ :returns: a new CompositeSourceModel with one group per source """ smodels = [] for sm in self.source_models: src_groups = [] smodel = sm.__class__(sm.names, sm.weight, sm.path, src_groups, sm.ordinal, sm.samples, sm.offset) for sg in sm.src_groups: for src in sg.sources: src.src_group_id = sg.id src_groups.append( sourceconverter.SourceGroup(sg.trt, [src], name=src.source_id, id=sg.id)) smodels.append(smodel) return self.__class__(self.gsim_lt, self.source_model_lt, smodels)
def _get_csm(full_lt, groups): # 1. extract a single source from multiple sources with the same ID # 2. regroup the sources in non-atomic groups by TRT # 3. reorder the sources by source_id atomic = [] acc = general.AccumDict(accum=[]) for grp in groups: if grp and grp.atomic: atomic.append(grp) elif grp: acc[grp.trt].extend(grp) key = operator.attrgetter('source_id', 'code') src_groups = [] for trt in acc: lst = [] for srcs in general.groupby(acc[trt], key).values(): if len(srcs) > 1: srcs = reduce_sources(srcs) lst.extend(srcs) for sources in general.groupby(lst, trt_smrs).values(): # check if OQ_SAMPLE_SOURCES is set ss = os.environ.get('OQ_SAMPLE_SOURCES') if ss: logging.info('Reducing the number of sources for %s', trt) split = [] for src in sources: for s in src: s.trt_smr = src.trt_smr split.append(s) sources = general.random_filter(split, float(ss)) or split[0] # set ._wkt attribute (for later storage in the source_wkt dataset) for src in sources: src._wkt = src.wkt() src_groups.append(sourceconverter.SourceGroup(trt, sources)) for ag in atomic: for src in ag: src._wkt = src.wkt() src_groups.extend(atomic) _check_dupl_ids(src_groups) for sg in src_groups: sg.sources.sort(key=operator.attrgetter('source_id')) return CompositeSourceModel(full_lt, src_groups)
def merge_groups(groups): """ :param groups: a list of :class:`openquake.hazardlib.sourceconverter.SourceGroup`s :returns: a reduced list of SourceGroups where groups with the same TRT are merged together (unless they are atomic) """ lst = [] acc = {} # trt -> SourceGroup for grp in groups: if grp.atomic: lst.append(grp) else: try: g = acc[grp.trt] except KeyError: g = acc[grp.trt] = sourceconverter.SourceGroup(grp.trt) lst.append(g) g.sources.extend(grp.sources) return lst
def _get_csm(full_lt, groups): # 1. extract a single source from multiple sources with the same ID # 2. regroup the sources in non-atomic groups by TRT # 3. reorder the sources by source_id atomic = [] acc = general.AccumDict(accum=[]) for grp in groups: if grp and grp.atomic: atomic.append(grp) elif grp: acc[grp.trt].extend(grp) key = operator.attrgetter('source_id', 'code') src_groups = [] for trt in acc: lst = [] for srcs in general.groupby(acc[trt], key).values(): if len(srcs) > 1: srcs = reduce_sources(srcs) lst.extend(srcs) for sources in general.groupby(lst, trt_smrs).values(): # set ._wkt attribute (for later storage in the source_wkt dataset) for src in sources: # check on MultiFaultSources and NonParametricSources mesh_size = getattr(src, 'mesh_size', 0) if mesh_size > 1E6: msg = ('src "{}" has {:_d} underlying meshes with a total ' 'of {:_d} points!').format( src.source_id, src.count_ruptures(), mesh_size) logging.warning(msg) src._wkt = src.wkt() src_groups.append(sourceconverter.SourceGroup(trt, sources)) for ag in atomic: for src in ag: src._wkt = src.wkt() src_groups.extend(atomic) _check_dupl_ids(src_groups) for sg in src_groups: sg.sources.sort(key=operator.attrgetter('source_id')) return CompositeSourceModel(full_lt, src_groups)
def __fromh5__(self, dic, attrs): # TODO: this is called more times than needed, maybe we should cache it sg_data = group_array(dic['sg_data'], 'sm_id') sm_data = dic['sm_data'] vars(self).update(attrs) self.gsim_lt = dic['gsim_lt'] self.source_models = [] for sm_id, rec in enumerate(sm_data): tdata = sg_data[sm_id] srcgroups = [ sourceconverter.SourceGroup( self.trts[data['trti']], id=data['grp_id'], name=get_field(data, 'name', ''), eff_ruptures=data['effrup'], tot_ruptures=get_field(data, 'totrup', 0)) for data in tdata] path = tuple(str(decode(rec['path'])).split('_')) sm = logictree.LtSourceModel( rec['name'], rec['weight'], path, srcgroups, rec['num_rlzs'], sm_id, rec['samples']) self.source_models.append(sm) self.init()
def _get_csm(full_lt, groups): # extract a single source from multiple sources with the same ID # and regroup the sources in non-atomic groups by TRT atomic = [] acc = general.AccumDict(accum=[]) get_grp_id = full_lt.source_model_lt.get_grp_id(full_lt.gsim_lt.values) for sm in full_lt.sm_rlzs: for grp in groups[sm.ordinal]: if grp and grp.atomic: atomic.append(grp) elif grp: acc[grp.trt].extend(grp) grp_id = get_grp_id(grp.trt, sm.ordinal) for src in grp: src.grp_id = grp_id if sm.samples > 1: src.samples = sm.samples dic = {} key = operator.attrgetter('source_id', 'checksum') idx = 0 for trt in acc: lst = [] for srcs in general.groupby(acc[trt], key).values(): for src in srcs: src.id = idx idx += 1 if len(srcs) > 1: # happens in classical/case_20 src.grp_id = [s.grp_id for s in srcs] lst.append(src) dic[trt] = sourceconverter.SourceGroup(trt, lst) for ag in atomic: for src in ag: src.id = idx idx += 1 src_groups = list(dic.values()) + atomic return CompositeSourceModel(full_lt, src_groups)
def get_source_models(oqparam, gsim_lt, source_model_lt, in_memory=True): """ Build all the source models generated by the logic tree. :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :param gsim_lt: a :class:`openquake.commonlib.logictree.GsimLogicTree` instance :param source_model_lt: a :class:`openquake.commonlib.logictree.SourceModelLogicTree` instance :param in_memory: if True, keep in memory the sources, else just collect the TRTs :returns: an iterator over :class:`openquake.commonlib.logictree.LtSourceModel` tuples """ converter = sourceconverter.SourceConverter( oqparam.investigation_time, oqparam.rupture_mesh_spacing, oqparam.complex_fault_mesh_spacing, oqparam.width_of_mfd_bin, oqparam.area_source_discretization) psr = nrml.SourceModelParser(converter) # consider only the effective realizations smlt_dir = os.path.dirname(source_model_lt.filename) for sm in source_model_lt.gen_source_models(gsim_lt): src_groups = [] for name in sm.names.split(): fname = os.path.abspath(os.path.join(smlt_dir, name)) if in_memory: apply_unc = source_model_lt.make_apply_uncertainties(sm.path) logging.info('Reading %s', fname) src_groups.extend(psr.parse_src_groups(fname, apply_unc)) else: # just collect the TRT models smodel = nrml.read(fname).sourceModel if smodel[0].tag.endswith('sourceGroup'): # NRML 0.5 format for sg_node in smodel: sg = sourceconverter.SourceGroup( sg_node['tectonicRegion']) sg.sources = sg_node.nodes src_groups.append(sg) else: # NRML 0.4 format: smodel is a list of source nodes src_groups.extend( sourceconverter.SourceGroup.collect(smodel)) num_sources = sum(len(sg.sources) for sg in src_groups) sm.src_groups = src_groups trts = [mod.trt for mod in src_groups] source_model_lt.tectonic_region_types.update(trts) logging.info( 'Processed source model %d with %d potential gsim path(s) and %d ' 'sources', sm.ordinal + 1, sm.num_gsim_paths, num_sources) gsim_file = oqparam.inputs.get('gsim_logic_tree') if gsim_file: # check TRTs for src_group in src_groups: if src_group.trt not in gsim_lt.values: raise ValueError( "Found in %r a tectonic region type %r inconsistent " "with the ones in %r" % (sm, src_group.trt, gsim_file)) yield sm # check investigation_time psr.check_nonparametric_sources(oqparam.investigation_time) # log if some source file is being used more than once dupl = 0 for fname, hits in psr.fname_hits.items(): if hits > 1: logging.info('%s has been considered %d times', fname, hits) if not psr.changed_sources: dupl += hits if dupl and not oqparam.optimize_same_id_sources: logging.warn('You are doing redundant calculations: please make sure ' 'that different sources have different IDs and set ' 'optimize_same_id_sources=true in your .ini file')
def write_source_model(dest, sources_or_groups, name=None, investigation_time=None): """ Writes a source model to XML. :param dest: Destination path :param sources_or_groups: Source model in different formats :param name: Name of the source model (if missing, extracted from the filename) :returns: the list of generated filenames """ if isinstance(sources_or_groups, nrml.SourceModel): groups = sources_or_groups.src_groups attrs = dict(name=sources_or_groups.name, investigation_time=sources_or_groups.investigation_time) elif isinstance(sources_or_groups[0], sourceconverter.SourceGroup): groups = sources_or_groups attrs = dict(investigation_time=investigation_time) else: # passed a list of sources srcs_by_trt = groupby( sources_or_groups, operator.attrgetter('tectonic_region_type')) groups = [sourceconverter.SourceGroup(trt, srcs_by_trt[trt]) for trt in srcs_by_trt] attrs = dict(investigation_time=investigation_time) if name or 'name' not in attrs: attrs['name'] = name or os.path.splitext(os.path.basename(dest))[0] if attrs['investigation_time'] is None: del attrs['investigation_time'] nodes = list(map(obj_to_node, groups)) ddict = extract_ddict(groups) out = [dest] if ddict: # remove duplicate content from nodes for grp_node in nodes: for src_node in grp_node: if src_node["id"] in ddict: src_node.nodes = [] # save HDF5 file dest5 = os.path.splitext(dest)[0] + '.hdf5' with hdf5.File(dest5, 'w') as h: for src_id, dic in ddict.items(): for k, v in dic.items(): key = '%s/%s' % (src_id, k) if isinstance(v, numpy.ndarray): h.create_dataset(key, v.shape, v.dtype, compression='gzip', compression_opts=9) h[key][:] = v else: h[key] = v out.append(dest5) # produce a geometryModel if there are MultiFaultSources sections = {} for group in groups: for src in group: if hasattr(src, 'sections'): sections.update(src.sections) sections = {sid: sections[sid] for sid in sections} smodel = Node("sourceModel", attrs, nodes=nodes) with open(dest, 'wb') as f: nrml.write([smodel], f, '%s') if sections: secnodes = [obj_to_node(sec) for sec in sections.values()] gmodel = Node("geometryModel", attrs, nodes=secnodes) with open(dest[:-4] + '_sections.xml', 'wb') as f: nrml.write([gmodel], f, '%s') out.append(f.name) return out
def get_source_models(oqparam, gsim_lt, source_model_lt, in_memory=True): """ Build all the source models generated by the logic tree. :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :param gsim_lt: a :class:`openquake.commonlib.logictree.GsimLogicTree` instance :param source_model_lt: a :class:`openquake.commonlib.logictree.SourceModelLogicTree` instance :param in_memory: if True, keep in memory the sources, else just collect the TRTs :returns: an iterator over :class:`openquake.commonlib.logictree.SourceModel` tuples """ converter = sourceconverter.SourceConverter( oqparam.investigation_time, oqparam.rupture_mesh_spacing, oqparam.complex_fault_mesh_spacing, oqparam.width_of_mfd_bin, oqparam.area_source_discretization) psr = nrml.SourceModelParser(converter) # consider only the effective realizations for sm in source_model_lt.gen_source_models(gsim_lt): src_groups = [] for name in sm.name.split(): fname = possibly_gunzip( os.path.abspath(os.path.join(oqparam.base_path, name))) if in_memory: apply_unc = source_model_lt.make_apply_uncertainties(sm.path) try: logging.info('Parsing %s', fname) src_groups.extend(psr.parse_src_groups(fname, apply_unc)) except ValueError as e: if str(e) in ('Surface does not conform with Aki & ' 'Richards convention', 'Edges points are not in the right order'): raise InvalidFile('''\ %s: %s. Probably you are using an obsolete model. In that case you can fix the file with the command python -m openquake.engine.tools.correct_complex_sources %s ''' % (fname, e, fname)) else: raise else: # just collect the TRT models smodel = nrml.read(fname).sourceModel if smodel[0].tag.endswith('sourceGroup'): # NRML 0.5 format for sg_node in smodel: sg = sourceconverter.SourceGroup( sg_node['tectonicRegion']) sg.sources = sg_node.nodes src_groups.append(sg) else: # NRML 0.4 format: smodel is a list of source nodes src_groups.extend( sourceconverter.SourceGroup.collect(smodel)) num_sources = sum(len(sg.sources) for sg in src_groups) sm.src_groups = src_groups trts = [mod.trt for mod in src_groups] source_model_lt.tectonic_region_types.update(trts) logging.info( 'Processed source model %d with %d potential gsim path(s) and %d ' 'sources', sm.ordinal + 1, sm.num_gsim_paths, num_sources) gsim_file = oqparam.inputs.get('gsim_logic_tree') if gsim_file: # check TRTs for src_group in src_groups: if src_group.trt not in gsim_lt.values: raise ValueError( "Found in %r a tectonic region type %r inconsistent " "with the ones in %r" % (sm, src_group.trt, gsim_file)) yield sm # log if some source file is being used more than once for fname, hits in psr.fname_hits.items(): if hits > 1: logging.info('%s has been considered %d times', fname, hits)