def get_composite_source_model(oqparam, full_lt=None, h5=None):
"""
Parse the XML and build a complete composite source model in memory.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param full_lt:
a :class:`openquake.commonlib.logictree.FullLogicTree` or None
:param h5:
an open hdf5.File where to store the source info
"""
if full_lt is None:
full_lt = get_full_lt(oqparam)
csm = get_csm(oqparam, full_lt, h5)
grp_ids = csm.get_grp_ids()
gidx = {tuple(arr): i for i, arr in enumerate(grp_ids)}
if oqparam.is_event_based():
csm.init_serials(oqparam.ses_seed)
data = {} # src_id -> row
mags = AccumDict(accum=set()) # trt -> mags
wkts = []
ns = 0
for sg in csm.src_groups:
if hasattr(sg, 'mags'): # UCERF
mags[sg.trt].update('%.2f' % mag for mag in sg.mags)
for src in sg:
ns += 1
if src.source_id in data:
num_sources = data[src.source_id][3] + 1
else:
num_sources = 1
row = [
src.source_id, gidx[tuple(src.grp_ids)], src.code, num_sources,
0, 0, 0, src.checksum, src.serial
]
wkts.append(src._wkt) # this is a bit slow but okay
data[src.source_id] = row
if hasattr(src, 'mags'): # UCERF
continue # already accounted for in sg.mags
elif hasattr(src, 'data'): # nonparametric
srcmags = ['%.2f' % item[0].mag for item in src.data]
else:
srcmags = [
'%.2f' % item[0]
for item in src.get_annual_occurrence_rates()
]
mags[sg.trt].update(srcmags)
logging.info('There are %d sources with %d unique IDs', ns, len(data))
if h5:
hdf5.create(h5, 'source_info', source_info_dt) # avoid hdf5 damned bug
h5['source_wkt'] = numpy.array(wkts, hdf5.vstr)
for trt in mags:
h5['source_mags/' + trt] = numpy.array(sorted(mags[trt]))
h5['grp_ids'] = grp_ids
csm.gsim_lt.check_imts(oqparam.imtls)
csm.source_info = data
if os.environ.get('OQ_CHECK_INPUT'):
source.check_complex_faults(csm.get_sources())
return csm
def get_composite_source_model(oqparam, h5=None):
"""
Parse the XML and build a complete composite source model in memory.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param h5:
an open hdf5.File where to store the source info
"""
full_lt = get_full_lt(oqparam)
if oqparam.csm_cache and not oqparam.is_ucerf():
csm = _get_csm_cached(oqparam, full_lt, h5)
else:
csm = get_csm(oqparam, full_lt, h5)
grp_ids = csm.get_grp_ids()
gidx = {tuple(arr): i for i, arr in enumerate(grp_ids)}
if oqparam.is_event_based():
csm.init_serials(oqparam.ses_seed)
data = {} # src_id -> row
mags = AccumDict(accum=set()) # trt -> mags
wkts = []
ns = -1
for sg in csm.src_groups:
if hasattr(sg, 'mags'): # UCERF
mags[sg.trt].update('%.2f' % mag for mag in sg.mags)
for src in sg:
if src.source_id in data:
multiplicity = data[src.source_id][MULTIPLICITY] + 1
else:
multiplicity = 1
ns += 1
src.gidx = gidx[tuple(src.grp_ids)]
row = [src.source_id, src.gidx, src.code,
multiplicity, 0, 0, 0, src.checksum, src.serial or ns,
full_lt.trti[src.tectonic_region_type]]
wkts.append(src._wkt) # this is a bit slow but okay
data[src.source_id] = row
if hasattr(src, 'mags'): # UCERF
continue # already accounted for in sg.mags
elif hasattr(src, 'data'): # nonparametric
srcmags = ['%.2f' % item[0].mag for item in src.data]
else:
srcmags = ['%.2f' % item[0] for item in
src.get_annual_occurrence_rates()]
mags[sg.trt].update(srcmags)
logging.info('There are %d sources', ns + 1)
if h5:
attrs = dict(atomic=any(grp.atomic for grp in csm.src_groups))
# avoid hdf5 damned bug by creating source_info in advance
hdf5.create(h5, 'source_info', source_info_dt, attrs=attrs)
h5['source_wkt'] = numpy.array(wkts, hdf5.vstr)
for trt in mags:
h5['source_mags/' + trt] = numpy.array(sorted(mags[trt]))
h5['grp_ids'] = grp_ids
csm.gsim_lt.check_imts(oqparam.imtls)
csm.source_info = data # src_id -> row
if os.environ.get('OQ_CHECK_INPUT'):
source.check_complex_faults(csm.get_sources())
return csm
def get_composite_source_model(oqparam, h5=None):
"""
Parse the XML and build a complete composite source model in memory.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param h5:
an open hdf5.File where to store the source info
"""
logging.info('Reading the CompositeSourceModel')
full_lt = get_full_lt(oqparam)
if oqparam.cachedir and not oqparam.is_ucerf():
csm = _get_cachedir(oqparam, full_lt, h5)
else:
csm = get_csm(oqparam, full_lt, h5)
et_ids = csm.get_et_ids()
logging.info('%d effective smlt realization(s)', len(full_lt.sm_rlzs))
data = {} # src_id -> row
mags_by_trt = csm.get_mags_by_trt()
wkts = []
lens = []
for sg in csm.src_groups:
for src in sg:
lens.append(len(src.et_ids))
row = [
src.source_id, src.grp_id, src.code, 0, 0, 0,
full_lt.trti[src.tectonic_region_type], 0
]
wkts.append(src._wkt)
data[src.id] = row
logging.info('There are %d groups and %d sources with len(et_ids)=%.2f',
len(csm.src_groups), sum(len(sg) for sg in csm.src_groups),
numpy.mean(lens))
if h5:
attrs = dict(atomic=any(grp.atomic for grp in csm.src_groups))
# avoid hdf5 damned bug by creating source_info in advance
hdf5.create(h5, 'source_info', source_info_dt, attrs=attrs)
h5['source_wkt'] = numpy.array(wkts, hdf5.vstr)
h5['et_ids'] = et_ids
for trt in mags_by_trt:
h5['source_mags/' + trt] = numpy.array(mags_by_trt[trt])
oqparam.maximum_distance.interp(mags_by_trt)
csm.gsim_lt.check_imts(oqparam.imtls)
csm.source_info = data # src_id -> row
if os.environ.get('OQ_CHECK_INPUT'):
source.check_complex_faults(csm.get_sources())
return csm
def _get_cachedir(oq, full_lt, h5=None):
# read the composite source model from the cache
if not os.path.exists(oq.cachedir):
os.makedirs(oq.cachedir)
checksum = get_checksum32(oq, h5)
fname = os.path.join(oq.cachedir, 'csm_%s.pik' % checksum)
if os.path.exists(fname):
logging.info('Reading %s', fname)
with open(fname, 'rb') as f:
csm = pickle.load(f)
csm.full_lt = full_lt
return csm
csm = get_csm(oq, full_lt, h5)
if not csm.src_groups: # everything was filtered away
return csm
logging.info('Saving %s', fname)
with open(fname, 'wb') as f:
pickle.dump(csm, f)
return csm
def _get_csm_cached(oq, full_lt, h5=None):
# read the composite source model from the cache
if not os.path.exists(oq.csm_cache):
os.makedirs(oq.csm_cache)
checksum = get_checksum32(oq)
if h5:
h5.attrs['checksum32'] = checksum
fname = os.path.join(oq.csm_cache, '%s.pik' % checksum)
if os.path.exists(fname):
with open(fname, 'rb') as f:
return pickle.load(f)
csm = get_csm(oq, full_lt, h5)
logging.info('Weighting the sources')
for sg in csm.src_groups:
for src in sg:
src.weight # cache .num_ruptures
with open(fname, 'wb') as f:
pickle.dump(csm, f)
return csm
def get_composite_source_model(oqparam, h5=None):
"""
Parse the XML and build a complete composite source model in memory.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param h5:
an open hdf5.File where to store the source info
"""
# first read the logic tree
full_lt = get_full_lt(oqparam)
# then read the composite source model from the cache if possible
if oqparam.cachedir and not os.path.exists(oqparam.cachedir):
os.makedirs(oqparam.cachedir)
if oqparam.cachedir and not oqparam.is_ucerf():
# for UCERF pickling the csm is slower
checksum = get_checksum32(oqparam, h5)
fname = os.path.join(oqparam.cachedir, 'csm_%s.pik' % checksum)
if os.path.exists(fname):
logging.info('Reading %s', fname)
with open(fname, 'rb') as f:
csm = pickle.load(f)
csm.full_lt = full_lt
if h5:
# avoid errors with --reuse_hazard
h5['et_ids'] = csm.get_et_ids()
hdf5.create(h5, 'source_info', source_info_dt)
_check_csm(csm, oqparam, h5)
return csm
# read and process the composite source model from the input files
csm = get_csm(oqparam, full_lt, h5)
save_source_info(csm, h5)
if oqparam.cachedir and not oqparam.is_ucerf():
logging.info('Saving %s', fname)
with open(fname, 'wb') as f:
pickle.dump(csm, f)
_check_csm(csm, oqparam, h5)
return csm
def _get_cachedir(oq, full_lt, h5=None):
# read the composite source model from the cache
if not os.path.exists(oq.cachedir):
os.makedirs(oq.cachedir)
checksum = get_checksum32(oq, h5)
fname = os.path.join(oq.cachedir, 'csm_%s.pik' % checksum)
if os.path.exists(fname):
logging.info('Reading %s', fname)
with open(fname, 'rb') as f:
csm = pickle.load(f)
csm.full_lt = full_lt
return csm
csm = get_csm(oq, full_lt, h5)
logging.info('Weighting the sources')
for sg in csm.src_groups:
for src in sg:
src.weight # cache .num_ruptures
logging.info('Saving %s', fname)
with open(fname, 'wb') as f:
pickle.dump(csm, f)
return csm
def get_composite_source_model(oqparam, h5=None):
"""
Parse the XML and build a complete composite source model in memory.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param h5:
an open hdf5.File where to store the source info
"""
source_model_lt = get_source_model_lt(oqparam)
trts = source_model_lt.tectonic_region_types
trts_lower = {trt.lower() for trt in trts}
reqv = oqparam.inputs.get('reqv', {})
for trt in reqv:
if trt in oqparam.discard_trts:
continue
elif trt.lower() not in trts_lower:
raise ValueError('Unknown TRT=%s in %s [reqv]' %
(trt, oqparam.inputs['job_ini']))
gsim_lt = get_gsim_lt(oqparam, trts or ['*'])
p = source_model_lt.num_paths * gsim_lt.get_num_paths()
if oqparam.number_of_logic_tree_samples:
logging.info('Considering {:_d} logic tree paths out of {:_d}'.format(
oqparam.number_of_logic_tree_samples, p))
else: # full enumeration
if (oqparam.is_event_based() and
(oqparam.ground_motion_fields or oqparam.hazard_curves_from_gmfs)
and p > oqparam.max_potential_paths):
raise ValueError(
'There are too many potential logic tree paths (%d):'
'use sampling instead of full enumeration or reduce the '
'source model with oq reduce_sm' % p)
logging.info('Potential number of logic tree paths = {:_d}'.format(p))
if source_model_lt.on_each_source:
logging.info('There is a logic tree on each source')
csm = get_csm(oqparam, source_model_lt, gsim_lt, h5)
if oqparam.is_event_based():
csm.init_serials(oqparam.ses_seed)
if h5:
info = hdf5.create(h5, 'source_info', source_info_dt)
data = []
mags = set()
n = len(csm.full_lt.sm_rlzs)
for sg in csm.src_groups:
for src in sg:
eri = src.grp_ids[0] % n
data.append((eri, src.grp_ids[0], src.source_id, src.code,
src.num_ruptures, 0, 0, 0, src.checksum, src._wkt))
if hasattr(src, 'mags'): # UCERF
srcmags = ['%.2f' % mag for mag in src.mags]
elif hasattr(src, 'data'): # nonparametric
srcmags = ['%.2f' % item[0].mag for item in src.data]
else:
srcmags = ['%.2f' % item[0] for item in
src.get_annual_occurrence_rates()]
mags.update(srcmags)
if h5:
hdf5.extend(info, numpy.array(data, source_info_dt))
h5['source_mags'] = numpy.array(sorted(mags))
csm.gsim_lt.check_imts(oqparam.imtls)
return csm
def get_composite_source_model(oqparam, h5=None):
"""
Parse the XML and build a complete composite source model in memory.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param h5:
an open hdf5.File where to store the source info
"""
full_lt = get_full_lt(oqparam)
if oqparam.cachedir and not oqparam.is_ucerf():
csm = _get_cachedir(oqparam, full_lt, h5)
else:
csm = get_csm(oqparam, full_lt, h5)
et_ids = csm.get_et_ids()
logging.info('%d effective smlt realization(s)', len(full_lt.sm_rlzs))
grp_id = {tuple(arr): i for i, arr in enumerate(et_ids)}
data = {} # src_id -> row
mags = AccumDict(accum=set()) # trt -> mags
wkts = []
lens = []
for sg in csm.src_groups:
if hasattr(sg, 'mags'): # UCERF
mags[sg.trt].update('%.2f' % mag for mag in sg.mags)
for src in sg:
lens.append(len(src.et_ids))
src.grp_id = grp_id[tuple(src.et_ids)]
row = [
src.source_id, src.grp_id, src.code, 0, 0, 0, src.id,
full_lt.trti[src.tectonic_region_type]
]
wkts.append(src._wkt) # this is a bit slow but okay
data[src.source_id] = row
if hasattr(src, 'mags'): # UCERF
continue # already accounted for in sg.mags
elif hasattr(src, 'data'): # nonparametric
srcmags = ['%.2f' % item[0].mag for item in src.data]
else:
srcmags = [
'%.2f' % item[0]
for item in src.get_annual_occurrence_rates()
]
mags[sg.trt].update(srcmags)
logging.info('There are %d groups and %d sources with len(et_ids)=%.1f',
len(csm.src_groups), sum(len(sg) for sg in csm.src_groups),
numpy.mean(lens))
if h5:
attrs = dict(atomic=any(grp.atomic for grp in csm.src_groups))
# avoid hdf5 damned bug by creating source_info in advance
hdf5.create(h5, 'source_info', source_info_dt, attrs=attrs)
h5['source_wkt'] = numpy.array(wkts, hdf5.vstr)
h5['et_ids'] = et_ids
mags_by_trt = {}
for trt in mags:
mags_by_trt[trt] = arr = numpy.array(sorted(mags[trt]))
h5['source_mags/' + trt] = arr
oqparam.maximum_distance.interp(mags_by_trt)
csm.gsim_lt.check_imts(oqparam.imtls)
csm.source_info = data # src_id -> row
if os.environ.get('OQ_CHECK_INPUT'):
source.check_complex_faults(csm.get_sources())
return csm
