def run_preclassical(csm, oqparam, h5):
"""
:param csm: a CompositeSourceModel with attribute .srcfilter
:param oqparam: the parameters in job.ini file
:param h5: a DataStore instance
"""
logging.info('Sending %s', csm.sitecol)
# do nothing for atomic sources except counting the ruptures
for src in csm.get_sources(atomic=True):
src.num_ruptures = src.count_ruptures()
src.nsites = len(csm.sitecol)
# run preclassical for non-atomic sources
sources_by_grp = groupby(
csm.get_sources(atomic=False),
lambda src: (src.grp_id, msr_name(src)))
param = dict(maximum_distance=oqparam.maximum_distance,
pointsource_distance=oqparam.pointsource_distance,
ps_grid_spacing=oqparam.ps_grid_spacing,
split_sources=oqparam.split_sources)
srcfilter = SourceFilter(
csm.sitecol.reduce(10000) if csm.sitecol else None,
oqparam.maximum_distance)
res = parallel.Starmap(
preclassical,
((srcs, srcfilter, param) for srcs in sources_by_grp.values()),
h5=h5, distribute=None if len(sources_by_grp) > 1 else 'no').reduce()
if res and res['before'] != res['after']:
logging.info('Reduced the number of sources from {:_d} -> {:_d}'.
format(res['before'], res['after']))
if res and h5:
csm.update_source_info(res['calc_times'], nsites=True)
for grp_id, srcs in res.items():
# srcs can be empty if the minimum_magnitude filter is on
if srcs and not isinstance(grp_id, str):
newsg = SourceGroup(srcs[0].tectonic_region_type)
newsg.sources = srcs
csm.src_groups[grp_id] = newsg
# sanity check
for sg in csm.src_groups:
for src in sg:
assert src.num_ruptures
assert src.nsites
# store ps_grid data, if any
for key, sources in res.items():
if isinstance(key, str) and key.startswith('ps_grid/'):
arrays = []
for ps in sources:
if hasattr(ps, 'location'):
lonlats = [ps.location.x, ps.location.y]
for src in getattr(ps, 'pointsources', []):
lonlats.extend([src.location.x, src.location.y])
arrays.append(F32(lonlats))
h5[key] = arrays
def run_preclassical(calc):
"""
:param csm: a CompositeSourceModel
:param oqparam: the parameters in job.ini file
:param h5: a DataStore instance
"""
csm = calc.csm
calc.datastore['trt_smrs'] = csm.get_trt_smrs()
calc.datastore['toms'] = numpy.array(
[sg.tom_name for sg in csm.src_groups], hdf5.vstr)
cmakers = read_cmakers(calc.datastore, csm.full_lt)
h5 = calc.datastore.hdf5
calc.sitecol = sites = csm.sitecol if csm.sitecol else None
# do nothing for atomic sources except counting the ruptures
atomic_sources = []
normal_sources = []
for sg in csm.src_groups:
grp_id = sg.sources[0].grp_id
if sg.atomic:
cmakers[grp_id].set_weight(sg, sites)
atomic_sources.extend(sg)
else:
normal_sources.extend(sg)
# run preclassical for non-atomic sources
sources_by_grp = groupby(normal_sources, lambda src:
(src.grp_id, msr_name(src)))
if csm.sitecol:
logging.info('Sending %s', sites)
smap = parallel.Starmap(preclassical, h5=h5)
for (grp_id, msr), srcs in sources_by_grp.items():
pointsources, pointlike, others = [], [], []
for src in srcs:
if hasattr(src, 'location'):
pointsources.append(src)
elif hasattr(src, 'nodal_plane_distribution'):
pointlike.append(src)
else:
others.append(src)
if calc.oqparam.ps_grid_spacing:
if pointsources or pointlike:
smap.submit((pointsources + pointlike, sites, cmakers[grp_id]))
else:
smap.submit_split((pointsources, sites, cmakers[grp_id]), 10, 100)
for src in pointlike: # area, multipoint
smap.submit(([src], sites, cmakers[grp_id]))
smap.submit_split((others, sites, cmakers[grp_id]), 10, 100)
normal = smap.reduce()
if atomic_sources: # case_35
n = len(atomic_sources)
atomic = AccumDict({'before': n, 'after': n})
for grp_id, srcs in groupby(atomic_sources,
lambda src: src.grp_id).items():
atomic[grp_id] = srcs
else:
atomic = AccumDict()
res = normal + atomic
if res['before'] != res['after']:
logging.info(
'Reduced the number of point sources from {:_d} -> {:_d}'.format(
res['before'], res['after']))
acc = AccumDict(accum=0)
code2cls = get_code2cls()
for grp_id, srcs in res.items():
# srcs can be empty if the minimum_magnitude filter is on
if srcs and not isinstance(grp_id, str) and grp_id not in atomic:
# check if OQ_SAMPLE_SOURCES is set
ss = os.environ.get('OQ_SAMPLE_SOURCES')
if ss:
logging.info('Sampled sources for group #%d', grp_id)
srcs = general.random_filter(srcs, float(ss)) or [srcs[0]]
newsg = SourceGroup(srcs[0].tectonic_region_type)
newsg.sources = srcs
csm.src_groups[grp_id] = newsg
for src in srcs:
assert src.weight
assert src.num_ruptures
acc[src.code] += int(src.num_ruptures)
for val, key in sorted((val, key) for key, val in acc.items()):
cls = code2cls[key].__name__
logging.info('{} ruptures: {:_d}'.format(cls, val))
source_data = zero_times(csm.get_sources())
calc.store_source_info(source_data)
# store ps_grid data, if any
for key, sources in res.items():
if isinstance(key, str) and key.startswith('ps_grid/'):
arrays = []
for ps in sources:
if hasattr(ps, 'location'):
lonlats = [ps.location.x, ps.location.y]
for src in getattr(ps, 'pointsources', []):
lonlats.extend([src.location.x, src.location.y])
arrays.append(F32(lonlats))
h5[key] = arrays
h5['full_lt'] = csm.full_lt
return res
def run_preclassical(csm, oqparam, h5):
"""
:param csm: a CompositeSourceModel with attribute .srcfilter
:param oqparam: the parameters in job.ini file
:param h5: a DataStore instance
"""
# do nothing for atomic sources except counting the ruptures
for src in csm.get_sources(atomic=True):
src.num_ruptures = src.count_ruptures()
src.nsites = len(csm.sitecol) if csm.sitecol else 1
# run preclassical for non-atomic sources
sources_by_grp = groupby(csm.get_sources(atomic=False), lambda src:
(src.grp_id, msr_name(src)))
param = dict(maximum_distance=oqparam.maximum_distance,
pointsource_distance=oqparam.pointsource_distance,
ps_grid_spacing=oqparam.ps_grid_spacing,
split_sources=oqparam.split_sources)
srcfilter = SourceFilter(
csm.sitecol.reduce(10000) if csm.sitecol else None,
oqparam.maximum_distance)
if csm.sitecol:
logging.info('Sending %s', srcfilter.sitecol)
if oqparam.ps_grid_spacing:
# produce a preclassical task for each group
allargs = ((srcs, srcfilter, param)
for srcs in sources_by_grp.values())
else:
# produce many preclassical task
maxw = sum(len(srcs) for srcs in sources_by_grp.values()) / (
oqparam.concurrent_tasks or 1)
allargs = ((blk, srcfilter, param) for srcs in sources_by_grp.values()
for blk in block_splitter(srcs, maxw))
res = parallel.Starmap(
preclassical,
allargs,
h5=h5,
distribute=None if len(sources_by_grp) > 1 else 'no').reduce()
if res and res['before'] != res['after']:
logging.info(
'Reduced the number of sources from {:_d} -> {:_d}'.format(
res['before'], res['after']))
if res and h5:
csm.update_source_info(res['calc_times'], nsites=True)
acc = AccumDict(accum=0)
code2cls = get_code2cls()
for grp_id, srcs in res.items():
# srcs can be empty if the minimum_magnitude filter is on
if srcs and not isinstance(grp_id, str):
newsg = SourceGroup(srcs[0].tectonic_region_type)
newsg.sources = srcs
csm.src_groups[grp_id] = newsg
for src in srcs:
acc[src.code] += int(src.num_ruptures)
for val, key in sorted((val, key) for key, val in acc.items()):
cls = code2cls[key].__name__
logging.info('{} ruptures: {:_d}'.format(cls, val))
# sanity check
for sg in csm.src_groups:
for src in sg:
assert src.num_ruptures
assert src.nsites
# store ps_grid data, if any
for key, sources in res.items():
if isinstance(key, str) and key.startswith('ps_grid/'):
arrays = []
for ps in sources:
if hasattr(ps, 'location'):
lonlats = [ps.location.x, ps.location.y]
for src in getattr(ps, 'pointsources', []):
lonlats.extend([src.location.x, src.location.y])
arrays.append(F32(lonlats))
h5[key] = arrays