def reduce(fname, reduction_factor):
"""
Produce a submodel from `fname` by sampling the nodes randomly.
Supports source models, site models and exposure models. As a special
case, it is also able to reduce .csv files by sampling the lines.
This is a debugging utility to reduce large computations to small ones.
"""
if fname.endswith('.csv'):
with open(fname) as f:
line = f.readline() # read the first line
if csv.Sniffer().has_header(line):
header = line
all_lines = f.readlines()
else:
header = None
f.seek(0)
all_lines = f.readlines()
lines = general.random_filter(all_lines, reduction_factor)
shutil.copy(fname, fname + '.bak')
print('Copied the original file in %s.bak' % fname)
_save_csv(fname, lines, header)
print('Extracted %d lines out of %d' % (len(lines), len(all_lines)))
return
elif fname.endswith('.npy'):
array = numpy.load(fname)
shutil.copy(fname, fname + '.bak')
print('Copied the original file in %s.bak' % fname)
arr = numpy.array(general.random_filter(array, reduction_factor))
numpy.save(fname, arr)
print('Extracted %d rows out of %d' % (len(arr), len(array)))
return
node = nrml.read(fname)
model = node[0]
if model.tag.endswith('exposureModel'):
total = len(model.assets)
model.assets.nodes = general.random_filter(
model.assets, reduction_factor)
num_nodes = len(model.assets)
elif model.tag.endswith('siteModel'):
total = len(model)
model.nodes = general.random_filter(model, reduction_factor)
num_nodes = len(model)
elif model.tag.endswith('sourceModel'):
if node['xmlns'] != 'http://openquake.org/xmlns/nrml/0.5':
raise InvalidFile('%s: not NRML0.5' % fname)
total = sum(len(sg) for sg in model)
num_nodes = 0
for sg in model:
sg.nodes = general.random_filter(sg, reduction_factor)
num_nodes += len(sg)
else:
raise RuntimeError('Unknown model tag: %s' % model.tag)
shutil.copy(fname, fname + '.bak')
print('Copied the original file in %s.bak' % fname)
with open(fname, 'wb') as f:
nrml.write([model], f, xmlns=node['xmlns'])
print('Extracted %d nodes out of %d' % (num_nodes, total))
def reduce(fname, reduction_factor):
"""
Produce a submodel from `fname` by sampling the nodes randomly.
Supports source models, site models and exposure models. As a special
case, it is also able to reduce .csv files by sampling the lines.
This is a debugging utility to reduce large computations to small ones.
"""
if fname.endswith('.csv'):
with open(fname) as f:
line = f.readline() # read the first line
if csv.Sniffer().has_header(line):
header = line
all_lines = f.readlines()
else:
header = None
f.seek(0)
all_lines = f.readlines()
lines = general.random_filter(all_lines, reduction_factor)
shutil.copy(fname, fname + '.bak')
print('Copied the original file in %s.bak' % fname)
_save_csv(fname, lines, header)
print('Extracted %d lines out of %d' % (len(lines), len(all_lines)))
return
elif fname.endswith('.npy'):
array = numpy.load(fname)
shutil.copy(fname, fname + '.bak')
print('Copied the original file in %s.bak' % fname)
arr = numpy.array(general.random_filter(array, reduction_factor))
numpy.save(fname, arr)
print('Extracted %d rows out of %d' % (len(arr), len(array)))
return
node = nrml.read(fname)
model = node[0]
if model.tag.endswith('exposureModel'):
total = len(model.assets)
model.assets.nodes = general.random_filter(model.assets,
reduction_factor)
num_nodes = len(model.assets)
elif model.tag.endswith('siteModel'):
total = len(model)
model.nodes = general.random_filter(model, reduction_factor)
num_nodes = len(model)
elif model.tag.endswith('sourceModel'):
reduce_source_model(fname, reduction_factor)
return
elif model.tag.endswith('logicTree'):
for smpath in logictree.collect_info(fname).smpaths:
reduce_source_model(smpath, reduction_factor)
return
else:
raise RuntimeError('Unknown model tag: %s' % model.tag)
save_bak(fname, node, num_nodes, total)
def reduce(fname, reduction_factor):
"""
Produce a submodel from `fname` by sampling the nodes randomly.
Supports source models, site models and exposure models. As a special
case, it is also able to reduce .csv files by sampling the lines.
This is a debugging utility to reduce large computations to small ones.
"""
if fname.endswith('.csv'):
with open(fname) as f:
line = f.readline() # read the first line
if csv.Sniffer().has_header(line):
header = line
all_lines = f.readlines()
else:
header = None
f.seek(0)
all_lines = f.readlines()
lines = general.random_filter(all_lines, reduction_factor)
shutil.copy(fname, fname + '.bak')
print('Copied the original file in %s.bak' % fname)
_save_csv(fname, lines, header)
print('Extracted %d lines out of %d' % (len(lines), len(all_lines)))
return
elif fname.endswith('.npy'):
array = numpy.load(fname)
shutil.copy(fname, fname + '.bak')
print('Copied the original file in %s.bak' % fname)
arr = numpy.array(general.random_filter(array, reduction_factor))
numpy.save(fname, arr)
print('Extracted %d rows out of %d' % (len(arr), len(array)))
return
node = nrml.read(fname)
model = node[0]
if model.tag.endswith('exposureModel'):
total = len(model.assets)
model.assets.nodes = general.random_filter(
model.assets, reduction_factor)
num_nodes = len(model.assets)
elif model.tag.endswith('siteModel'):
total = len(model)
model.nodes = general.random_filter(model, reduction_factor)
num_nodes = len(model)
elif model.tag.endswith('sourceModel'):
reduce_source_model(fname, reduction_factor)
return
elif model.tag.endswith('logicTree'):
for smpath in logictree.collect_info(fname).smpaths:
reduce_source_model(smpath, reduction_factor)
return
else:
raise RuntimeError('Unknown model tag: %s' % model.tag)
save_bak(fname, node, num_nodes, total)
def reduce_source_model(fname, reduction_factor):
node = nrml.read(fname)
if node['xmlns'] == 'http://openquake.org/xmlns/nrml/0.5':
total = sum(len(sg) for sg in node[0])
num_nodes = 0
for sg in node[0]:
sg.nodes = general.random_filter(sg, reduction_factor)
num_nodes += len(sg)
else: # nrml/0.4
total = len(node[0].nodes)
node[0].nodes = general.random_filter(node[0], reduction_factor)
num_nodes = len(node[0].nodes)
save_bak(fname, node, num_nodes, total)
def reduce_source_model(fname, reduction_factor):
node = nrml.read(fname)
if node['xmlns'] == 'http://openquake.org/xmlns/nrml/0.5':
total = sum(len(sg) for sg in node[0])
num_nodes = 0
for sg in node[0]:
sg.nodes = general.random_filter(sg, reduction_factor)
num_nodes += len(sg)
else: # nrml/0.4
total = len(node[0].nodes)
node[0].nodes = general.random_filter(node[0], reduction_factor)
num_nodes = len(node[0].nodes)
save_bak(fname, node, num_nodes, total)
def split_all(self):
"""
Split all sources in the composite source model.
:param samples_factor: if given, sample the sources
:returns: a dictionary source_id -> split_time
"""
sample_factor = os.environ.get('OQ_SAMPLE_SOURCES')
ngsims = {trt: len(gs) for trt, gs in self.gsim_lt.values.items()}
split_time = AccumDict()
for sm in self.source_models:
for src_group in sm.src_groups:
self.add_infos(src_group)
for src in src_group:
split_time[src.source_id] = 0
src.ngsims = ngsims[src.tectonic_region_type]
if getattr(src_group, 'src_interdep', None) != 'mutex':
# mutex sources cannot be split
srcs, stime = split_sources(src_group)
for src in src_group:
s = src.source_id
self.infos[s].split_time = stime[s]
if sample_factor:
# debugging tip to reduce the size of a calculation
# OQ_SAMPLE_SOURCES=.01 oq engine --run job.ini
# will run a computation 100 times smaller
srcs = random_filter(srcs, float(sample_factor))
src_group.sources = srcs
split_time += stime
return split_time
def get_site_collection(oqparam):
"""
Returns a SiteCollection instance by looking at the points and the
site model defined by the configuration parameters.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
"""
mesh = get_mesh(oqparam)
if mesh is None and oqparam.ground_motion_fields:
raise InvalidFile('You are missing sites.csv or site_model.csv in %s' %
oqparam.inputs['job_ini'])
elif mesh is None:
# a None sitecol is okay when computing the ruptures only
return
else: # use the default site params
req_site_params = get_gsim_lt(oqparam).req_site_params
sitecol = site.SiteCollection.from_points(mesh.lons, mesh.lats,
mesh.depths, oqparam,
req_site_params)
ss = os.environ.get('OQ_SAMPLE_SITES')
if ss:
# debugging tip to reduce the size of a calculation
# OQ_SAMPLE_SITES=.1 oq engine --run job.ini
# will run a computation with 10 times less sites
sitecol.array = numpy.array(random_filter(sitecol.array, float(ss)))
sitecol.make_complete()
return sitecol
def get_site_collection(oqparam):
"""
Returns a SiteCollection instance by looking at the points and the
site model defined by the configuration parameters.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
"""
mesh = get_mesh(oqparam)
req_site_params = get_gsim_lt(oqparam).req_site_params
grid_spacing = oqparam.region_grid_spacing
if oqparam.inputs.get('site_model'):
sm = get_site_model(oqparam)
try:
# in the future we could have elevation in the site model
depth = sm['depth']
except ValueError:
# this is the normal case
depth = None
if grid_spacing:
grid = mesh.get_convex_hull().dilate(grid_spacing).discretize(
grid_spacing)
grid_sites = site.SiteCollection.from_points(
grid.lons, grid.lats, req_site_params=req_site_params)
sitecol, params, _ = geo.utils.assoc(
sm, grid_sites, oqparam.region_grid_spacing * 1.414, 'filter')
logging.info('Associating %d site model sites to %d grid sites',
len(sm), len(sitecol))
sitecol.make_complete()
else:
sitecol = site.SiteCollection.from_points(sm['lon'], sm['lat'],
depth, sm,
req_site_params)
params = sm
for name in req_site_params:
if name in ('vs30measured', 'backarc') \
and name not in params.dtype.names:
sitecol._set(name, 0) # the default
else:
sitecol._set(name, params[name])
elif mesh is None and oqparam.ground_motion_fields:
raise InvalidFile('You are missing sites.csv or site_model.csv in %s' %
oqparam.inputs['job_ini'])
elif mesh is None:
# a None sitecol is okay when computing the ruptures only
return
else: # use the default site params
sitecol = site.SiteCollection.from_points(mesh.lons, mesh.lats,
mesh.depths, oqparam,
req_site_params)
ss = os.environ.get('OQ_SAMPLE_SITES')
if ss:
# debugging tip to reduce the size of a calculation
# OQ_SAMPLE_SITES=.1 oq engine --run job.ini
# will run a computation with 10 times less sites
sitecol.array = numpy.array(random_filter(sitecol.array, float(ss)))
sitecol.make_complete()
return sitecol
def get_site_collection(oqparam):
"""
Returns a SiteCollection instance by looking at the points and the
site model defined by the configuration parameters.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
"""
mesh = get_mesh(oqparam)
req_site_params = get_gsim_lt(oqparam).req_site_params
if oqparam.inputs.get('site_model'):
sm = get_site_model(oqparam, req_site_params)
try:
# in the future we could have elevation in the site model
depth = sm['depth']
except ValueError:
# this is the normal case
depth = None
if mesh is None:
# extract the site collection directly from the site model
sitecol = site.SiteCollection.from_points(sm['lon'], sm['lat'],
depth, sm,
req_site_params)
else:
sitecol = site.SiteCollection.from_points(mesh.lons, mesh.lats,
mesh.depths, None,
req_site_params)
if oqparam.region_grid_spacing:
# associate the site parameters to the grid assuming they
# have been prepared correctly, i.e. they are on the location
# of the assets; discard empty sites silently
sitecol, params, discarded = geo.utils.assoc(
sm, sitecol, oqparam.region_grid_spacing * 1.414, 'filter')
sitecol.make_complete()
else:
# associate the site parameters to the sites without
# discarding any site but warning for far away parameters
sc, params, discarded = geo.utils.assoc(
sm, sitecol, oqparam.max_site_model_distance, 'warn')
for name in req_site_params:
sitecol._set(name, params[name])
else: # use the default site params
sitecol = site.SiteCollection.from_points(mesh.lons, mesh.lats,
mesh.depths, oqparam,
req_site_params)
ss = os.environ.get('OQ_SAMPLE_SITES')
if ss:
# debugging tip to reduce the size of a calculation
# OQ_SAMPLE_SITES=.1 oq engine --run job.ini
# will run a computation with 10 times less sites
sitecol.array = numpy.array(random_filter(sitecol.array, float(ss)))
sitecol.make_complete()
return sitecol
def get_site_collection(oqparam):
"""
Returns a SiteCollection instance by looking at the points and the
site model defined by the configuration parameters.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
"""
mesh = get_mesh(oqparam)
req_site_params = get_gsim_lt(oqparam).req_site_params
if oqparam.inputs.get('site_model'):
sm = get_site_model(oqparam)
try:
# in the future we could have elevation in the site model
depth = sm['depth']
except ValueError:
# this is the normal case
depth = None
sitecol = site.SiteCollection.from_points(
sm['lon'], sm['lat'], depth, sm, req_site_params)
if oqparam.region_grid_spacing:
logging.info('Reducing the grid sites to the site '
'parameters within the grid spacing')
sitecol, params, _ = geo.utils.assoc(
sm, sitecol, oqparam.region_grid_spacing * 1.414, 'filter')
sitecol.make_complete()
else:
params = sm
for name in req_site_params:
if name in ('vs30measured', 'backarc') \
and name not in params.dtype.names:
sitecol._set(name, 0) # the default
else:
sitecol._set(name, params[name])
elif mesh is None and oqparam.ground_motion_fields:
raise InvalidFile('You are missing sites.csv or site_model.csv in %s'
% oqparam.inputs['job_ini'])
elif mesh is None:
# a None sitecol is okay when computing the ruptures only
return
else: # use the default site params
sitecol = site.SiteCollection.from_points(
mesh.lons, mesh.lats, mesh.depths, oqparam, req_site_params)
ss = os.environ.get('OQ_SAMPLE_SITES')
if ss:
# debugging tip to reduce the size of a calculation
# OQ_SAMPLE_SITES=.1 oq engine --run job.ini
# will run a computation with 10 times less sites
sitecol.array = numpy.array(random_filter(sitecol.array, float(ss)))
sitecol.make_complete()
return sitecol
def get_site_collection(oqparam, mesh=None):
"""
Returns a SiteCollection instance by looking at the points and the
site model defined by the configuration parameters.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param mesh:
the mesh to use; if None, it is extracted from the job.ini
"""
mesh = mesh or get_mesh(oqparam)
req_site_params = get_gsim_lt(oqparam).req_site_params
if oqparam.inputs.get('site_model'):
sm = get_site_model(oqparam, req_site_params)
try:
# in the future we could have elevation in the site model
depth = sm['depth']
except ValueError:
# this is the normal case
depth = None
if mesh is None:
# extract the site collection directly from the site model
sitecol = site.SiteCollection.from_points(sm['lon'], sm['lat'],
depth, sm,
req_site_params)
else:
# associate the site parameters to the mesh
sitecol = site.SiteCollection.from_points(mesh.lons, mesh.lats,
mesh.depths, None,
req_site_params)
sc, params = geo.utils.assoc(sm, sitecol,
oqparam.max_site_model_distance,
'warn')
for name in req_site_params:
sitecol._set(name, params[name])
else: # use the default site params
sitecol = site.SiteCollection.from_points(mesh.lons, mesh.lats,
mesh.depths, oqparam,
req_site_params)
ss = os.environ.get('OQ_SAMPLE_SITES')
if ss:
# debugging tip to reduce the size of a calculation
# OQ_SAMPLE_SITES=.1 oq engine --run job.ini
# will run a computation with 10 times less sites
sitecol.array = numpy.array(random_filter(sitecol.array, float(ss)))
sitecol.make_complete()
return sitecol
def get_site_collection(oqparam, h5=None):
"""
Returns a SiteCollection instance by looking at the points and the
site model defined by the configuration parameters.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
"""
if h5 and 'sitecol' in h5:
return h5['sitecol']
mesh = get_mesh(oqparam, h5)
if mesh is None and oqparam.ground_motion_fields:
raise InvalidFile('You are missing sites.csv or site_model.csv in %s'
% oqparam.inputs['job_ini'])
elif mesh is None:
# a None sitecol is okay when computing the ruptures only
return
else: # use the default site params
req_site_params = get_gsim_lt(oqparam).req_site_params
if 'amplification' in oqparam.inputs:
req_site_params.add('ampcode')
if h5 and 'site_model' in h5: # comes from a site_model.csv
sm = h5['site_model'][:]
else:
sm = oqparam
sitecol = site.SiteCollection.from_points(
mesh.lons, mesh.lats, mesh.depths, sm, req_site_params)
ss = oqparam.sites_slice # can be None or (start, stop)
if ss:
if 'custom_site_id' not in sitecol.array.dtype.names:
gh = sitecol.geohash(6)
assert len(numpy.unique(gh)) == len(gh), 'geohashes are not unique'
sitecol.add_col('custom_site_id', 'S6', gh)
mask = (sitecol.sids >= ss[0]) & (sitecol.sids < ss[1])
sitecol = sitecol.filter(mask)
sitecol.make_complete()
ss = os.environ.get('OQ_SAMPLE_SITES')
if ss:
# debugging tip to reduce the size of a calculation
# OQ_SAMPLE_SITES=.1 oq engine --run job.ini
# will run a computation with 10 times less sites
sitecol.array = numpy.array(random_filter(sitecol.array, float(ss)))
sitecol.make_complete()
if h5:
h5['sitecol'] = sitecol
return sitecol
def split_filter(srcs, srcfilter, seed, sample_factor, monitor):
"""
Split the given source and filter the subsources by distance and by
magnitude. Perform sampling if a nontrivial sample_factor is passed.
Yields a pair (split_sources, split_time) if split_sources is non-empty.
"""
splits, stime = split_sources(srcs)
if splits and sample_factor:
# debugging tip to reduce the size of a calculation
# OQ_SAMPLE_SOURCES=.01 oq engine --run job.ini
# will run a computation 100 times smaller
splits = random_filter(splits, sample_factor, seed)
# NB: for performance, sample before splitting
if splits and srcfilter:
splits = list(srcfilter.filter(splits))
if splits:
yield splits, stime
def _get_csm(full_lt, groups):
# 1. extract a single source from multiple sources with the same ID
# 2. regroup the sources in non-atomic groups by TRT
# 3. reorder the sources by source_id
atomic = []
acc = general.AccumDict(accum=[])
for grp in groups:
if grp and grp.atomic:
atomic.append(grp)
elif grp:
acc[grp.trt].extend(grp)
key = operator.attrgetter('source_id', 'code')
src_groups = []
for trt in acc:
lst = []
for srcs in general.groupby(acc[trt], key).values():
if len(srcs) > 1:
srcs = reduce_sources(srcs)
lst.extend(srcs)
for sources in general.groupby(lst, trt_smrs).values():
# check if OQ_SAMPLE_SOURCES is set
ss = os.environ.get('OQ_SAMPLE_SOURCES')
if ss:
logging.info('Reducing the number of sources for %s', trt)
split = []
for src in sources:
for s in src:
s.trt_smr = src.trt_smr
split.append(s)
sources = general.random_filter(split, float(ss)) or split[0]
# set ._wkt attribute (for later storage in the source_wkt dataset)
for src in sources:
src._wkt = src.wkt()
src_groups.append(sourceconverter.SourceGroup(trt, sources))
for ag in atomic:
for src in ag:
src._wkt = src.wkt()
src_groups.extend(atomic)
_check_dupl_ids(src_groups)
for sg in src_groups:
sg.sources.sort(key=operator.attrgetter('source_id'))
return CompositeSourceModel(full_lt, src_groups)
def get_background_sources(self, src_filter, sample_factor=None):
"""
Turn the background model of a given branch into a set of point sources
:param src_filter:
SourceFilter instance
:param sample_factor:
Used to reduce the sources if OQ_SAMPLE_SOURCES is set
"""
background_sids = self.get_background_sids(src_filter)
if sample_factor is not None: # hack for use in the mosaic
background_sids = random_filter(
background_sids, sample_factor, seed=42)
with h5py.File(self.source_file, "r") as hdf5:
grid_loc = "/".join(["Grid", self.idx_set["grid_key"]])
# for instance Grid/FM0_0_MEANFS_MEANMSR_MeanRates
mags = hdf5[grid_loc + "/Magnitude"].value
mmax = hdf5[grid_loc + "/MMax"][background_sids]
rates = hdf5[grid_loc + "/RateArray"][background_sids, :]
locations = hdf5["Grid/Locations"][background_sids, :]
sources = []
for i, bg_idx in enumerate(background_sids):
src_id = "_".join([self.idx_set["grid_key"], str(bg_idx)])
src_name = "|".join([self.idx_set["total_key"], str(bg_idx)])
mag_idx = (self.min_mag <= mags) & (mags < mmax[i])
src_mags = mags[mag_idx]
src_mfd = EvenlyDiscretizedMFD(
src_mags[0],
src_mags[1] - src_mags[0],
rates[i, mag_idx].tolist())
ps = PointSource(
src_id, src_name, self.tectonic_region_type, src_mfd,
self.mesh_spacing, self.msr, self.aspect, self.tom,
self.usd, self.lsd,
Point(locations[i, 0], locations[i, 1]),
self.npd, self.hdd)
ps.id = self.id
ps.src_group_id = self.src_group_id
ps.num_ruptures = ps.count_ruptures()
sources.append(ps)
return sources
def get_background_sources(self, sample_factor=None):
"""
Turn the background model of a given branch into a set of point sources
:param sample_factor:
Used to reduce the sources if OQ_SAMPLE_SOURCES is set
"""
background_sids = self.get_background_sids()
if sample_factor is not None: # hack for use in the mosaic
background_sids = random_filter(
background_sids, sample_factor, seed=42)
with h5py.File(self.source_file, "r") as hdf5:
grid_loc = "/".join(["Grid", self.idx_set["grid_key"]])
# for instance Grid/FM0_0_MEANFS_MEANMSR_MeanRates
mags = hdf5[grid_loc + "/Magnitude"][()]
mmax = hdf5[grid_loc + "/MMax"][background_sids]
rates = hdf5[grid_loc + "/RateArray"][background_sids, :]
locations = hdf5["Grid/Locations"][background_sids, :]
sources = []
for i, bg_idx in enumerate(background_sids):
src_id = "_".join([self.idx_set["grid_key"], str(bg_idx)])
src_name = "|".join([self.idx_set["total_key"], str(bg_idx)])
mag_idx = (self.min_mag <= mags) & (mags < mmax[i])
src_mags = mags[mag_idx]
src_mfd = EvenlyDiscretizedMFD(
src_mags[0],
src_mags[1] - src_mags[0],
rates[i, mag_idx].tolist())
ps = PointSource(
src_id, src_name, self.tectonic_region_type, src_mfd,
self.mesh_spacing, self.msr, self.aspect, self.tom,
self.usd, self.lsd,
Point(locations[i, 0], locations[i, 1]),
self.npd, self.hdd)
ps.checksum = zlib.adler32(pickle.dumps(vars(ps), protocol=4))
ps._wkt = ps.wkt()
ps.id = self.id
ps.grp_id = self.grp_id
ps.num_ruptures = ps.count_ruptures()
sources.append(ps)
return sources
def run_preclassical(calc):
"""
:param csm: a CompositeSourceModel
:param oqparam: the parameters in job.ini file
:param h5: a DataStore instance
"""
csm = calc.csm
calc.datastore['trt_smrs'] = csm.get_trt_smrs()
calc.datastore['toms'] = numpy.array(
[sg.tom_name for sg in csm.src_groups], hdf5.vstr)
cmakers = read_cmakers(calc.datastore, csm.full_lt)
h5 = calc.datastore.hdf5
calc.sitecol = sites = csm.sitecol if csm.sitecol else None
# do nothing for atomic sources except counting the ruptures
atomic_sources = []
normal_sources = []
for sg in csm.src_groups:
grp_id = sg.sources[0].grp_id
if sg.atomic:
cmakers[grp_id].set_weight(sg, sites)
atomic_sources.extend(sg)
else:
normal_sources.extend(sg)
# run preclassical for non-atomic sources
sources_by_grp = groupby(normal_sources, lambda src:
(src.grp_id, msr_name(src)))
if csm.sitecol:
logging.info('Sending %s', sites)
smap = parallel.Starmap(preclassical, h5=h5)
for (grp_id, msr), srcs in sources_by_grp.items():
pointsources, pointlike, others = [], [], []
for src in srcs:
if hasattr(src, 'location'):
pointsources.append(src)
elif hasattr(src, 'nodal_plane_distribution'):
pointlike.append(src)
else:
others.append(src)
if calc.oqparam.ps_grid_spacing:
if pointsources or pointlike:
smap.submit((pointsources + pointlike, sites, cmakers[grp_id]))
else:
smap.submit_split((pointsources, sites, cmakers[grp_id]), 10, 100)
for src in pointlike: # area, multipoint
smap.submit(([src], sites, cmakers[grp_id]))
smap.submit_split((others, sites, cmakers[grp_id]), 10, 100)
normal = smap.reduce()
if atomic_sources: # case_35
n = len(atomic_sources)
atomic = AccumDict({'before': n, 'after': n})
for grp_id, srcs in groupby(atomic_sources,
lambda src: src.grp_id).items():
atomic[grp_id] = srcs
else:
atomic = AccumDict()
res = normal + atomic
if res['before'] != res['after']:
logging.info(
'Reduced the number of point sources from {:_d} -> {:_d}'.format(
res['before'], res['after']))
acc = AccumDict(accum=0)
code2cls = get_code2cls()
for grp_id, srcs in res.items():
# srcs can be empty if the minimum_magnitude filter is on
if srcs and not isinstance(grp_id, str) and grp_id not in atomic:
# check if OQ_SAMPLE_SOURCES is set
ss = os.environ.get('OQ_SAMPLE_SOURCES')
if ss:
logging.info('Sampled sources for group #%d', grp_id)
srcs = general.random_filter(srcs, float(ss)) or [srcs[0]]
newsg = SourceGroup(srcs[0].tectonic_region_type)
newsg.sources = srcs
csm.src_groups[grp_id] = newsg
for src in srcs:
assert src.weight
assert src.num_ruptures
acc[src.code] += int(src.num_ruptures)
for val, key in sorted((val, key) for key, val in acc.items()):
cls = code2cls[key].__name__
logging.info('{} ruptures: {:_d}'.format(cls, val))
source_data = zero_times(csm.get_sources())
calc.store_source_info(source_data)
# store ps_grid data, if any
for key, sources in res.items():
if isinstance(key, str) and key.startswith('ps_grid/'):
arrays = []
for ps in sources:
if hasattr(ps, 'location'):
lonlats = [ps.location.x, ps.location.y]
for src in getattr(ps, 'pointsources', []):
lonlats.extend([src.location.x, src.location.y])
arrays.append(F32(lonlats))
h5[key] = arrays
h5['full_lt'] = csm.full_lt
return res