def _populate_from(self, asset_nodes, param): asset_refs = set() for idx, asset_node in enumerate(asset_nodes): asset_id = asset_node['id'] if asset_id in asset_refs: raise nrml.DuplicatedID(asset_id) asset_refs.add(asset_id) self._add_asset(idx, asset_node, param)
def _populate_from(self, asset_array, param, check_dupl): asset_refs = set() for idx, asset in enumerate(asset_array): asset_id = asset['id'] # check_dupl is False only in oq prepare_site_model since # in that case we are only interested in the asset locations if check_dupl and asset_id in asset_refs: raise nrml.DuplicatedID(asset_id) asset_refs.add(param['asset_prefix'] + asset_id) self._add_asset(idx, asset, param)
def parse_branchset(self, branchset_node, depth): """ :param branchset_ node: ``etree.Element`` object with tag "logicTreeBranchSet". :param depth: The sequential number of this branching level, based on 0. Enumerates children branchsets and call :meth:`parse_branchset`, :meth:`validate_branchset`, :meth:`parse_branches` and finally :meth:`apply_branchset` for each. Keeps track of "open ends" -- the set of branches that don't have any child branchset on this step of execution. After processing of every branchset only those branches that are listed in it can have child branchsets (if there is one on the next level). """ attrs = branchset_node.attrib.copy() uncertainty_type = branchset_node.attrib.get('uncertaintyType') dic = dict((filtername, branchset_node.attrib.get(filtername)) for filtername in self.FILTERS if filtername in branchset_node.attrib) self.validate_filters(branchset_node, uncertainty_type, dic) filters = self.parse_filters(branchset_node, uncertainty_type, dic) ordinal = len(self.bsetdict) branchset = BranchSet(uncertainty_type, ordinal, filters) branchset.id = bsid = attrs.pop('branchSetID') if bsid in self.bsetdict: raise nrml.DuplicatedID('%s in %s' % (bsid, self.filename)) self.bsetdict[bsid] = attrs self.validate_branchset(branchset_node, depth, branchset) self.parse_branches(branchset_node, branchset) dummies = [] # dummy branches in case of applyToBranches if self.root_branchset is None: # not set yet self.num_paths = 1 self.root_branchset = branchset else: prev_ids = ' '.join(pb.branch_id for pb in self.previous_branches) app2brs = branchset_node.attrib.get('applyToBranches') or prev_ids if app2brs != prev_ids: branchset.applied = app2brs self.apply_branchset( app2brs, branchset_node.lineno, branchset) for brid in set(prev_ids.split()) - set(app2brs.split()): self.branches[brid].bset = dummy = dummy_branchset() [dummybranch] = dummy.branches self.branches[dummybranch.branch_id] = dummybranch dummies.append(dummybranch) else: # apply to all previous branches for branch in self.previous_branches: branch.bset = branchset self.previous_branches = branchset.branches + dummies self.num_paths *= len(branchset) self.branchsets.append(branchset)
def get_composite_source_model(oqparam, in_memory=True): """ Parse the XML and build a complete composite source model in memory. :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :param in_memory: if False, just parse the XML without instantiating the sources """ smodels = [] grp_id = 0 idx = 0 gsim_lt = get_gsim_lt(oqparam) source_model_lt = get_source_model_lt(oqparam) if source_model_lt.on_each_source(): logging.info('There is a logic tree on each source') for source_model in get_source_models(oqparam, gsim_lt, source_model_lt, in_memory=in_memory): for src_group in source_model.src_groups: src_group.sources = sorted(src_group, key=getid) src_group.id = grp_id for src in src_group: # there are two cases depending on the flag in_memory: # 1) src is a hazardlib source and has a src_group_id # attribute; in that case the source has to be numbered # 2) src is a Node object, then nothing must be done if isinstance(src, Node): continue src.src_group_id = grp_id src.id = idx idx += 1 grp_id += 1 if grp_id >= TWO16: # the limit is really needed only for event based calculations raise ValueError('There is a limit of %d src groups!' % TWO16) smodels.append(source_model) csm = source.CompositeSourceModel(gsim_lt, source_model_lt, smodels, oqparam.optimize_same_id_sources) for sm in csm.source_models: counter = collections.Counter() for sg in sm.src_groups: for srcid in map(getid, sg): counter[srcid] += 1 dupl = [srcid for srcid in counter if counter[srcid] > 1] if dupl: raise nrml.DuplicatedID('Found duplicated source IDs in %s: %s' % (sm, dupl)) return csm
def parse_branchset(self, branchset_node, depth): """ :param branchset_ node: ``etree.Element`` object with tag "logicTreeBranchSet". :param depth: The sequential number of this branching level, based on 0. Enumerates children branchsets and call :meth:`parse_branchset`, :meth:`validate_branchset`, :meth:`parse_branches` and finally :meth:`apply_branchset` for each. Keeps track of "open ends" -- the set of branches that don't have any child branchset on this step of execution. After processing of every branchset only those branches that are listed in it can have child branchsets (if there is one on the next level). """ attrs = branchset_node.attrib.copy() uncertainty_type = branchset_node.attrib.get('uncertaintyType') filters = dict((filtername, branchset_node.attrib.get(filtername)) for filtername in self.FILTERS if filtername in branchset_node.attrib) self.validate_filters(branchset_node, uncertainty_type, filters) filters = self.parse_filters(branchset_node, uncertainty_type, filters) branchset = BranchSet(uncertainty_type, len(self.bsetdict), filters) bsid = attrs.pop('branchSetID') if bsid in self.bsetdict: raise nrml.DuplicatedID('%s in %s' % (bsid, self.filename)) self.bsetdict[bsid] = attrs self.validate_branchset(branchset_node, depth, branchset) self.parse_branches(branchset_node, branchset) if self.root_branchset is None: # not set yet self.num_paths = 1 self.root_branchset = branchset else: apply_to_branches = branchset_node.attrib.get('applyToBranches') if apply_to_branches: self.apply_branchset(apply_to_branches, branchset_node.lineno, branchset) else: for branch in self.previous_branches: branch.bset = branchset self.previous_branches = branchset.branches self.num_paths *= len(branchset) self.branchsets.append(branchset)
def get_composite_source_model(oqparam, monitor=None, in_memory=True, srcfilter=SourceFilter(None, {})): """ Parse the XML and build a complete composite source model in memory. :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :param monitor: a `openquake.baselib.performance.Monitor` instance :param in_memory: if False, just parse the XML without instantiating the sources :param srcfilter: if not None, use it to prefilter the sources """ ucerf = oqparam.calculation_mode.startswith('ucerf') source_model_lt = get_source_model_lt(oqparam, validate=not ucerf) trts = source_model_lt.tectonic_region_types trts_lower = {trt.lower() for trt in trts} reqv = oqparam.inputs.get('reqv', {}) for trt in reqv: # these are lowercase because they come from the job.ini if trt not in trts_lower: raise ValueError('Unknown TRT=%s in %s [reqv]' % (trt, oqparam.inputs['job_ini'])) gsim_lt = get_gsim_lt(oqparam, trts or ['*']) p = source_model_lt.num_paths * gsim_lt.get_num_paths() if oqparam.number_of_logic_tree_samples: logging.info('Considering {:,d} logic tree paths out of {:,d}'.format( oqparam.number_of_logic_tree_samples, p)) else: # full enumeration if oqparam.is_event_based() and p > oqparam.max_potential_paths: raise ValueError( 'There are too many potential logic tree paths (%d) ' 'use sampling instead of full enumeration' % p) logging.info('Potential number of logic tree paths = {:,d}'.format(p)) if source_model_lt.on_each_source: logging.info('There is a logic tree on each source') if monitor is None: monitor = performance.Monitor() smodels = [] for source_model in get_source_models(oqparam, gsim_lt, source_model_lt, monitor, in_memory, srcfilter): for src_group in source_model.src_groups: src_group.sources = sorted(src_group, key=getid) for src in src_group: # there are two cases depending on the flag in_memory: # 1) src is a hazardlib source and has a src_group_id # attribute; in that case the source has to be numbered # 2) src is a Node object, then nothing must be done if isinstance(src, Node): continue smodels.append(source_model) csm = source.CompositeSourceModel(gsim_lt, source_model_lt, smodels, oqparam.optimize_same_id_sources) for sm in csm.source_models: counter = collections.Counter() for sg in sm.src_groups: for srcid in map(getid, sg): counter[srcid] += 1 dupl = [srcid for srcid in counter if counter[srcid] > 1] if dupl: raise nrml.DuplicatedID('Found duplicated source IDs in %s: %s' % (sm, dupl)) if not in_memory: return csm if oqparam.is_event_based(): # initialize the rupture serial numbers before splitting/filtering; in # this way the serials are independent from the site collection csm.init_serials(oqparam.ses_seed) if oqparam.disagg_by_src: csm = csm.grp_by_src() # one group per source csm.info.gsim_lt.check_imts(oqparam.imtls) return csm
def get_composite_source_model(oqparam, monitor=None, in_memory=True, split_all=True, srcfilter=None): """ Parse the XML and build a complete composite source model in memory. :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :param monitor: a `openquake.baselib.performance.Monitor` instance :param in_memory: if False, just parse the XML without instantiating the sources :param split_all: if True, split all the sources in the models :param srcfilter: if not None, use it to prefilter the sources """ smodels = [] gsim_lt = get_gsim_lt(oqparam) source_model_lt = get_source_model_lt(oqparam) if oqparam.number_of_logic_tree_samples == 0: logging.info('Potential number of logic tree paths = {:,d}'.format( source_model_lt.num_paths * gsim_lt.get_num_paths())) if source_model_lt.on_each_source: logging.info('There is a logic tree on each source') if monitor is None: monitor = performance.Monitor() for source_model in get_source_models(oqparam, gsim_lt, source_model_lt, monitor, in_memory=in_memory): for src_group in source_model.src_groups: src_group.sources = sorted(src_group, key=getid) for src in src_group: # there are two cases depending on the flag in_memory: # 1) src is a hazardlib source and has a src_group_id # attribute; in that case the source has to be numbered # 2) src is a Node object, then nothing must be done if isinstance(src, Node): continue smodels.append(source_model) csm = source.CompositeSourceModel(gsim_lt, source_model_lt, smodels, oqparam.optimize_same_id_sources) for sm in csm.source_models: counter = collections.Counter() for sg in sm.src_groups: for srcid in map(getid, sg): counter[srcid] += 1 dupl = [srcid for srcid in counter if counter[srcid] > 1] if dupl: raise nrml.DuplicatedID('Found duplicated source IDs in %s: %s' % (sm, dupl)) if not in_memory: return csm if 'event_based' in oqparam.calculation_mode: if oqparam.pointsource_distance == 0: # remove splitting/floating ruptures for src in csm.get_sources(): if hasattr(src, 'hypocenter_distribution'): src.hypocenter_distribution.reduce() src.nodal_plane_distribution.reduce() src.num_ruptures = src.count_ruptures() # initialize the rupture serial numbers before splitting/filtering; in # this way the serials are independent from the site collection csm.init_serials(oqparam.ses_seed) # TODO: check why the seeds still depend on the minimun_magnitude set_min_mag(csm.get_sources(), oqparam.minimum_magnitude) if oqparam.disagg_by_src: csm = csm.grp_by_src() # one group per source csm.info.gsim_lt.check_imts(oqparam.imtls) if monitor.hdf5: csm.info.gsim_lt.store_gmpe_tables(monitor.hdf5) # splitting assumes that the serials have been initialized already if split_all and 'ucerf' not in oqparam.calculation_mode: csm = parallel_split_filter(csm, srcfilter, oqparam.random_seed, monitor('prefilter')) return csm