def get_composite_source_model(oqparam, full_lt=None, h5=None): """ Parse the XML and build a complete composite source model in memory. :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :param full_lt: a :class:`openquake.commonlib.logictree.FullLogicTree` or None :param h5: an open hdf5.File where to store the source info """ if full_lt is None: full_lt = get_full_lt(oqparam) csm = get_csm(oqparam, full_lt, h5) grp_ids = csm.get_grp_ids() gidx = {tuple(arr): i for i, arr in enumerate(grp_ids)} if oqparam.is_event_based(): csm.init_serials(oqparam.ses_seed) data = {} # src_id -> row mags = AccumDict(accum=set()) # trt -> mags wkts = [] ns = 0 for sg in csm.src_groups: if hasattr(sg, 'mags'): # UCERF mags[sg.trt].update('%.2f' % mag for mag in sg.mags) for src in sg: ns += 1 if src.source_id in data: num_sources = data[src.source_id][3] + 1 else: num_sources = 1 row = [ src.source_id, gidx[tuple(src.grp_ids)], src.code, num_sources, 0, 0, 0, src.checksum, src.serial ] wkts.append(src._wkt) # this is a bit slow but okay data[src.source_id] = row if hasattr(src, 'mags'): # UCERF continue # already accounted for in sg.mags elif hasattr(src, 'data'): # nonparametric srcmags = ['%.2f' % item[0].mag for item in src.data] else: srcmags = [ '%.2f' % item[0] for item in src.get_annual_occurrence_rates() ] mags[sg.trt].update(srcmags) logging.info('There are %d sources with %d unique IDs', ns, len(data)) if h5: hdf5.create(h5, 'source_info', source_info_dt) # avoid hdf5 damned bug h5['source_wkt'] = numpy.array(wkts, hdf5.vstr) for trt in mags: h5['source_mags/' + trt] = numpy.array(sorted(mags[trt])) h5['grp_ids'] = grp_ids csm.gsim_lt.check_imts(oqparam.imtls) csm.source_info = data if os.environ.get('OQ_CHECK_INPUT'): source.check_complex_faults(csm.get_sources()) return csm
def get_composite_source_model(oqparam, h5=None): """ Parse the XML and build a complete composite source model in memory. :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :param h5: an open hdf5.File where to store the source info """ full_lt = get_full_lt(oqparam) if oqparam.csm_cache and not oqparam.is_ucerf(): csm = _get_csm_cached(oqparam, full_lt, h5) else: csm = get_csm(oqparam, full_lt, h5) grp_ids = csm.get_grp_ids() gidx = {tuple(arr): i for i, arr in enumerate(grp_ids)} if oqparam.is_event_based(): csm.init_serials(oqparam.ses_seed) data = {} # src_id -> row mags = AccumDict(accum=set()) # trt -> mags wkts = [] ns = -1 for sg in csm.src_groups: if hasattr(sg, 'mags'): # UCERF mags[sg.trt].update('%.2f' % mag for mag in sg.mags) for src in sg: if src.source_id in data: multiplicity = data[src.source_id][MULTIPLICITY] + 1 else: multiplicity = 1 ns += 1 src.gidx = gidx[tuple(src.grp_ids)] row = [src.source_id, src.gidx, src.code, multiplicity, 0, 0, 0, src.checksum, src.serial or ns, full_lt.trti[src.tectonic_region_type]] wkts.append(src._wkt) # this is a bit slow but okay data[src.source_id] = row if hasattr(src, 'mags'): # UCERF continue # already accounted for in sg.mags elif hasattr(src, 'data'): # nonparametric srcmags = ['%.2f' % item[0].mag for item in src.data] else: srcmags = ['%.2f' % item[0] for item in src.get_annual_occurrence_rates()] mags[sg.trt].update(srcmags) logging.info('There are %d sources', ns + 1) if h5: attrs = dict(atomic=any(grp.atomic for grp in csm.src_groups)) # avoid hdf5 damned bug by creating source_info in advance hdf5.create(h5, 'source_info', source_info_dt, attrs=attrs) h5['source_wkt'] = numpy.array(wkts, hdf5.vstr) for trt in mags: h5['source_mags/' + trt] = numpy.array(sorted(mags[trt])) h5['grp_ids'] = grp_ids csm.gsim_lt.check_imts(oqparam.imtls) csm.source_info = data # src_id -> row if os.environ.get('OQ_CHECK_INPUT'): source.check_complex_faults(csm.get_sources()) return csm
def get_composite_source_model(oqparam, h5=None): """ Parse the XML and build a complete composite source model in memory. :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :param h5: an open hdf5.File where to store the source info """ logging.info('Reading the CompositeSourceModel') full_lt = get_full_lt(oqparam) if oqparam.cachedir and not oqparam.is_ucerf(): csm = _get_cachedir(oqparam, full_lt, h5) else: csm = get_csm(oqparam, full_lt, h5) et_ids = csm.get_et_ids() logging.info('%d effective smlt realization(s)', len(full_lt.sm_rlzs)) data = {} # src_id -> row mags_by_trt = csm.get_mags_by_trt() wkts = [] lens = [] for sg in csm.src_groups: for src in sg: lens.append(len(src.et_ids)) row = [ src.source_id, src.grp_id, src.code, 0, 0, 0, full_lt.trti[src.tectonic_region_type], 0 ] wkts.append(src._wkt) data[src.id] = row logging.info('There are %d groups and %d sources with len(et_ids)=%.2f', len(csm.src_groups), sum(len(sg) for sg in csm.src_groups), numpy.mean(lens)) if h5: attrs = dict(atomic=any(grp.atomic for grp in csm.src_groups)) # avoid hdf5 damned bug by creating source_info in advance hdf5.create(h5, 'source_info', source_info_dt, attrs=attrs) h5['source_wkt'] = numpy.array(wkts, hdf5.vstr) h5['et_ids'] = et_ids for trt in mags_by_trt: h5['source_mags/' + trt] = numpy.array(mags_by_trt[trt]) oqparam.maximum_distance.interp(mags_by_trt) csm.gsim_lt.check_imts(oqparam.imtls) csm.source_info = data # src_id -> row if os.environ.get('OQ_CHECK_INPUT'): source.check_complex_faults(csm.get_sources()) return csm
def _get_cachedir(oq, full_lt, h5=None): # read the composite source model from the cache if not os.path.exists(oq.cachedir): os.makedirs(oq.cachedir) checksum = get_checksum32(oq, h5) fname = os.path.join(oq.cachedir, 'csm_%s.pik' % checksum) if os.path.exists(fname): logging.info('Reading %s', fname) with open(fname, 'rb') as f: csm = pickle.load(f) csm.full_lt = full_lt return csm csm = get_csm(oq, full_lt, h5) if not csm.src_groups: # everything was filtered away return csm logging.info('Saving %s', fname) with open(fname, 'wb') as f: pickle.dump(csm, f) return csm
def _get_csm_cached(oq, full_lt, h5=None): # read the composite source model from the cache if not os.path.exists(oq.csm_cache): os.makedirs(oq.csm_cache) checksum = get_checksum32(oq) if h5: h5.attrs['checksum32'] = checksum fname = os.path.join(oq.csm_cache, '%s.pik' % checksum) if os.path.exists(fname): with open(fname, 'rb') as f: return pickle.load(f) csm = get_csm(oq, full_lt, h5) logging.info('Weighting the sources') for sg in csm.src_groups: for src in sg: src.weight # cache .num_ruptures with open(fname, 'wb') as f: pickle.dump(csm, f) return csm
def get_composite_source_model(oqparam, h5=None): """ Parse the XML and build a complete composite source model in memory. :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :param h5: an open hdf5.File where to store the source info """ # first read the logic tree full_lt = get_full_lt(oqparam) # then read the composite source model from the cache if possible if oqparam.cachedir and not os.path.exists(oqparam.cachedir): os.makedirs(oqparam.cachedir) if oqparam.cachedir and not oqparam.is_ucerf(): # for UCERF pickling the csm is slower checksum = get_checksum32(oqparam, h5) fname = os.path.join(oqparam.cachedir, 'csm_%s.pik' % checksum) if os.path.exists(fname): logging.info('Reading %s', fname) with open(fname, 'rb') as f: csm = pickle.load(f) csm.full_lt = full_lt if h5: # avoid errors with --reuse_hazard h5['et_ids'] = csm.get_et_ids() hdf5.create(h5, 'source_info', source_info_dt) _check_csm(csm, oqparam, h5) return csm # read and process the composite source model from the input files csm = get_csm(oqparam, full_lt, h5) save_source_info(csm, h5) if oqparam.cachedir and not oqparam.is_ucerf(): logging.info('Saving %s', fname) with open(fname, 'wb') as f: pickle.dump(csm, f) _check_csm(csm, oqparam, h5) return csm
def _get_cachedir(oq, full_lt, h5=None): # read the composite source model from the cache if not os.path.exists(oq.cachedir): os.makedirs(oq.cachedir) checksum = get_checksum32(oq, h5) fname = os.path.join(oq.cachedir, 'csm_%s.pik' % checksum) if os.path.exists(fname): logging.info('Reading %s', fname) with open(fname, 'rb') as f: csm = pickle.load(f) csm.full_lt = full_lt return csm csm = get_csm(oq, full_lt, h5) logging.info('Weighting the sources') for sg in csm.src_groups: for src in sg: src.weight # cache .num_ruptures logging.info('Saving %s', fname) with open(fname, 'wb') as f: pickle.dump(csm, f) return csm
def get_composite_source_model(oqparam, h5=None): """ Parse the XML and build a complete composite source model in memory. :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :param h5: an open hdf5.File where to store the source info """ source_model_lt = get_source_model_lt(oqparam) trts = source_model_lt.tectonic_region_types trts_lower = {trt.lower() for trt in trts} reqv = oqparam.inputs.get('reqv', {}) for trt in reqv: if trt in oqparam.discard_trts: continue elif trt.lower() not in trts_lower: raise ValueError('Unknown TRT=%s in %s [reqv]' % (trt, oqparam.inputs['job_ini'])) gsim_lt = get_gsim_lt(oqparam, trts or ['*']) p = source_model_lt.num_paths * gsim_lt.get_num_paths() if oqparam.number_of_logic_tree_samples: logging.info('Considering {:_d} logic tree paths out of {:_d}'.format( oqparam.number_of_logic_tree_samples, p)) else: # full enumeration if (oqparam.is_event_based() and (oqparam.ground_motion_fields or oqparam.hazard_curves_from_gmfs) and p > oqparam.max_potential_paths): raise ValueError( 'There are too many potential logic tree paths (%d):' 'use sampling instead of full enumeration or reduce the ' 'source model with oq reduce_sm' % p) logging.info('Potential number of logic tree paths = {:_d}'.format(p)) if source_model_lt.on_each_source: logging.info('There is a logic tree on each source') csm = get_csm(oqparam, source_model_lt, gsim_lt, h5) if oqparam.is_event_based(): csm.init_serials(oqparam.ses_seed) if h5: info = hdf5.create(h5, 'source_info', source_info_dt) data = [] mags = set() n = len(csm.full_lt.sm_rlzs) for sg in csm.src_groups: for src in sg: eri = src.grp_ids[0] % n data.append((eri, src.grp_ids[0], src.source_id, src.code, src.num_ruptures, 0, 0, 0, src.checksum, src._wkt)) if hasattr(src, 'mags'): # UCERF srcmags = ['%.2f' % mag for mag in src.mags] elif hasattr(src, 'data'): # nonparametric srcmags = ['%.2f' % item[0].mag for item in src.data] else: srcmags = ['%.2f' % item[0] for item in src.get_annual_occurrence_rates()] mags.update(srcmags) if h5: hdf5.extend(info, numpy.array(data, source_info_dt)) h5['source_mags'] = numpy.array(sorted(mags)) csm.gsim_lt.check_imts(oqparam.imtls) return csm
def get_composite_source_model(oqparam, h5=None): """ Parse the XML and build a complete composite source model in memory. :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :param h5: an open hdf5.File where to store the source info """ full_lt = get_full_lt(oqparam) if oqparam.cachedir and not oqparam.is_ucerf(): csm = _get_cachedir(oqparam, full_lt, h5) else: csm = get_csm(oqparam, full_lt, h5) et_ids = csm.get_et_ids() logging.info('%d effective smlt realization(s)', len(full_lt.sm_rlzs)) grp_id = {tuple(arr): i for i, arr in enumerate(et_ids)} data = {} # src_id -> row mags = AccumDict(accum=set()) # trt -> mags wkts = [] lens = [] for sg in csm.src_groups: if hasattr(sg, 'mags'): # UCERF mags[sg.trt].update('%.2f' % mag for mag in sg.mags) for src in sg: lens.append(len(src.et_ids)) src.grp_id = grp_id[tuple(src.et_ids)] row = [ src.source_id, src.grp_id, src.code, 0, 0, 0, src.id, full_lt.trti[src.tectonic_region_type] ] wkts.append(src._wkt) # this is a bit slow but okay data[src.source_id] = row if hasattr(src, 'mags'): # UCERF continue # already accounted for in sg.mags elif hasattr(src, 'data'): # nonparametric srcmags = ['%.2f' % item[0].mag for item in src.data] else: srcmags = [ '%.2f' % item[0] for item in src.get_annual_occurrence_rates() ] mags[sg.trt].update(srcmags) logging.info('There are %d groups and %d sources with len(et_ids)=%.1f', len(csm.src_groups), sum(len(sg) for sg in csm.src_groups), numpy.mean(lens)) if h5: attrs = dict(atomic=any(grp.atomic for grp in csm.src_groups)) # avoid hdf5 damned bug by creating source_info in advance hdf5.create(h5, 'source_info', source_info_dt, attrs=attrs) h5['source_wkt'] = numpy.array(wkts, hdf5.vstr) h5['et_ids'] = et_ids mags_by_trt = {} for trt in mags: mags_by_trt[trt] = arr = numpy.array(sorted(mags[trt])) h5['source_mags/' + trt] = arr oqparam.maximum_distance.interp(mags_by_trt) csm.gsim_lt.check_imts(oqparam.imtls) csm.source_info = data # src_id -> row if os.environ.get('OQ_CHECK_INPUT'): source.check_complex_faults(csm.get_sources()) return csm