def execute(self):
"""
Run in parallel `core_func(sources, sitecol, monitor)`, by
parallelizing on the sources according to their weight and
tectonic region type.
"""
monitor = self.monitor.new(self.core_func.__name__)
monitor.oqparam = self.oqparam
sources = self.csm.get_sources()
zc = zero_curves(len(self.sitecol.complete), self.oqparam.imtls)
zerodict = AccumDict((key, zc) for key in self.rlzs_assoc)
zerodict.calc_times = []
zerodict.bb_dict = {
(smodel.ordinal, site.id): BoundingBox(smodel.ordinal, site.id)
for site in self.sitecol
for smodel in self.csm.source_models
} if self.oqparam.poes_disagg else {}
curves_by_trt_gsim = parallel.apply_reduce(
self.core_func.__func__,
(sources, self.sitecol, 0, self.rlzs_assoc, monitor),
agg=self.agg_dicts, acc=zerodict,
concurrent_tasks=self.oqparam.concurrent_tasks,
weight=operator.attrgetter('weight'),
key=operator.attrgetter('trt_model_id'))
store_source_chunks(self.datastore)
return curves_by_trt_gsim
def execute(self):
"""
Split the computation by tiles which are run in parallel.
"""
monitor = self.monitor(self.core_func.__name__)
monitor.oqparam = oq = self.oqparam
self.tiles = split_in_blocks(
self.sitecol, self.oqparam.concurrent_tasks or 1)
oq.concurrent_tasks = 0
calculator = ClassicalCalculator(
self.oqparam, monitor, persistent=False)
calculator.csm = self.csm
rlzs_assoc = self.csm.get_rlzs_assoc()
self.rlzs_assoc = calculator.rlzs_assoc = rlzs_assoc
# parallelization
all_args = []
position = 0
for (i, tile) in enumerate(self.tiles):
all_args.append((calculator, SiteCollection(tile),
position, i, monitor))
position += len(tile)
acc = {trt_gsim: zero_curves(len(self.sitecol), oq.imtls)
for trt_gsim in calculator.rlzs_assoc}
acc['calc_times'] = []
return parallel.starmap(classical_tiling, all_args).reduce(
agg_curves_by_trt_gsim, acc)
def get_hcurves_from_nrml(oqparam, fname):
"""
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param fname:
an XML file containing hazard curves
:returns:
sitecol, curve array
"""
hcurves_by_imt = {}
oqparam.hazard_imtls = imtls = collections.OrderedDict()
for hcurves in nrml.read(fname):
imt = hcurves['IMT']
oqparam.investigation_time = hcurves['investigationTime']
if imt == 'SA':
imt += '(%s)' % hcurves['saPeriod']
imtls[imt] = ~hcurves.IMLs
data = sorted((~node.Point.pos, ~node.poEs) for node in hcurves[1:])
hcurves_by_imt[imt] = numpy.array([d[1] for d in data])
n = len(hcurves_by_imt[imt])
curves = zero_curves(n, imtls)
for imt in imtls:
curves[imt] = hcurves_by_imt[imt]
lons, lats = [], []
for xy, poes in data:
lons.append(xy[0])
lats.append(xy[1])
mesh = geo.Mesh(numpy.array(lons), numpy.array(lats))
sitecol = get_site_collection(oqparam, mesh)
return sitecol, curves
def execute(self):
"""
Run in parallel `core_func(sources, sitecol, monitor)`, by
parallelizing on the ruptures according to their weight and
tectonic region type.
"""
oq = self.oqparam
if not oq.hazard_curves_from_gmfs and not oq.ground_motion_fields:
return
monitor = self.monitor(self.core_func.__name__)
monitor.oqparam = oq
zc = zero_curves(len(self.sitecol.complete), self.oqparam.imtls)
zerodict = AccumDict((key, zc) for key in self.rlzs_assoc)
self.nbytes = 0
curves_by_trt_gsim = parallel.apply_reduce(
self.core_func.__func__,
(self.sesruptures, self.sitecol, self.rlzs_assoc, monitor),
concurrent_tasks=self.oqparam.concurrent_tasks,
acc=zerodict,
agg=self.combine_curves_and_save_gmfs,
key=operator.attrgetter('col_id'))
if oq.ground_motion_fields:
# sanity check on the saved gmfs size
expected_nbytes = self.datastore['counts_per_rlz'].attrs[
'gmfs_nbytes']
self.datastore['gmfs'].attrs['nbytes'] = self.nbytes
assert self.nbytes == expected_nbytes, (self.nbytes,
expected_nbytes)
return curves_by_trt_gsim
def execute(self):
"""
Run in parallel `core_func(sources, sitecol, monitor)`, by
parallelizing on the ruptures according to their weight and
tectonic region type.
"""
oq = self.oqparam
if not oq.hazard_curves_from_gmfs and not oq.ground_motion_fields:
return
monitor = self.monitor(self.core_func.__name__)
monitor.oqparam = oq
zc = zero_curves(len(self.sitecol.complete), self.oqparam.imtls)
zerodict = AccumDict((key, zc) for key in self.rlzs_assoc)
self.nbytes = 0
curves_by_trt_gsim = parallel.apply_reduce(
self.core_func.__func__,
(self.sesruptures, self.sitecol, self.rlzs_assoc, monitor),
concurrent_tasks=self.oqparam.concurrent_tasks,
acc=zerodict, agg=self.combine_curves_and_save_gmfs,
key=operator.attrgetter('col_id'))
if oq.ground_motion_fields:
# sanity check on the saved gmfs size
expected_nbytes = self.datastore[
'counts_per_rlz'].attrs['gmfs_nbytes']
self.datastore['gmfs'].attrs['nbytes'] = self.nbytes
assert self.nbytes == expected_nbytes, (
self.nbytes, expected_nbytes)
return curves_by_trt_gsim
def get_hcurves_from_nrml(oqparam, fname):
"""
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param fname:
an XML file containing hazard curves
:returns:
sitecol, curve array
"""
hcurves_by_imt = {}
oqparam.hazard_imtls = imtls = {}
for hcurves in nrml.read(fname):
imt = hcurves['IMT']
oqparam.investigation_time = hcurves['investigationTime']
if imt == 'SA':
imt += '(%s)' % hcurves['saPeriod']
imtls[imt] = ~hcurves.IMLs
data = []
for node in hcurves[1:]:
xy = ~node.Point.pos
poes = ~node.poEs
data.append((xy, poes))
data.sort()
hcurves_by_imt[imt] = numpy.array([d[1] for d in data])
n = len(hcurves_by_imt[imt])
curves = zero_curves(n, imtls)
for imt in imtls:
curves[imt] = hcurves_by_imt[imt]
lons, lats = [], []
for xy, poes in data:
lons.append(xy[0])
lats.append(xy[1])
mesh = geo.Mesh(numpy.array(lons), numpy.array(lats))
sitecol = get_site_collection(oqparam, mesh)
return sitecol, curves
def zerodict(self):
"""
Initial accumulator, a dictionary (trt_id, gsim) -> curves
"""
zc = zero_curves(len(self.sitecol.complete), self.oqparam.imtls)
zd = AccumDict((key, zc) for key in self.rlzs_assoc)
zd.calc_times = []
zd.eff_ruptures = AccumDict() # trt_id -> eff_ruptures
zd.bb_dict = {
(smodel.ordinal, site.id): BoundingBox(smodel.ordinal, site.id)
for site in self.sitecol for smodel in self.csm.source_models
} if self.oqparam.poes_disagg else {}
return zd
def execute(self):
"""
Split the computation by tiles which are run in parallel.
"""
acc = AccumDict(
{trt_gsim: zero_curves(len(self.sitecol), self.oqparam.imtls)
for trt_gsim in self.rlzs_assoc})
acc.calc_times = []
acc.n = len(self.sitecol)
hint = math.ceil(acc.n / self.oqparam.sites_per_tile)
tiles = self.sitecol.split_in_tiles(hint)
logging.info('Generating %d tiles of %d sites each',
len(tiles), len(tiles[0]))
sources = self.csm.get_sources()
rlzs_assoc = self.csm.get_rlzs_assoc()
ctasks = self.oqparam.concurrent_tasks or 1
maxweight = math.ceil(self.csm.weight / ctasks)
siteidx = 0
tmanagers = []
maximum_distance = self.oqparam.maximum_distance
# try to produce more tasks than self.oqparam.concurrent_tasks
num_blocks = math.ceil(self.MORE_TASKS * ctasks / len(tiles))
splitmap = {}
for i, tile in enumerate(tiles, 1):
monitor = self.monitor.new()
monitor.oqparam = self.oqparam
with self.monitor('filtering sources per tile', autoflush=True):
filtered_sources = [
src for src in sources
if src.filter_sites_by_distance_to_source(
maximum_distance, tile) is not None]
if not filtered_sources:
continue
blocks = split_in_blocks(
split_sources(sources, maxweight, splitmap), num_blocks,
weight=operator.attrgetter('weight'),
key=operator.attrgetter('trt_model_id'))
tm = parallel.starmap(
classical,
((blk, tile, siteidx, rlzs_assoc, monitor) for blk in blocks),
name='tile_%d/%d' % (i, len(tiles)))
tmanagers.append(tm)
siteidx += len(tile)
logging.info('Total number of tasks submitted: %d',
sum(len(tm.results) for tm in tmanagers))
for tm in tmanagers:
tm.reduce(self.agg_dicts, acc)
self.rlzs_assoc = self.csm.get_rlzs_assoc(
partial(is_effective_trt_model, acc))
return acc
def zerodict(self):
"""
Initial accumulator, a dictionary (trt_id, gsim) -> curves
"""
zc = zero_curves(len(self.sitecol.complete), self.oqparam.imtls)
zd = AccumDict((key, zc) for key in self.rlzs_assoc)
zd.calc_times = []
zd.eff_ruptures = AccumDict() # trt_id -> eff_ruptures
zd.bb_dict = {
(smodel.ordinal, site.id): BoundingBox(smodel.ordinal, site.id)
for site in self.sitecol
for smodel in self.csm.source_models
} if self.oqparam.poes_disagg else {}
return zd
def to_haz_curves(num_sites, gmvs_by_sid, gs, imtls,
investigation_time, duration):
"""
:param num_sites: length of the full site collection
:param gmvs_by_sid: dictionary site_id -> gmvs
:param gs: GSIM string
:param imtls: ordered dictionary {IMT: intensity measure levels}
:param investigation_time: investigation time
:param duration: investigation_time * number of Stochastic Event Sets
"""
curves = zero_curves(num_sites, imtls)
for imt in imtls:
curves[imt] = numpy.array([
gmvs_to_haz_curve(
[gmv[gs][imt] for gmv in gmvs_by_sid[sid]],
imtls[imt], investigation_time, duration)
for sid in range(num_sites)])
return curves
def to_haz_curves(num_sites, gmvs_by_sid, gs, imtls, investigation_time,
duration):
"""
:param num_sites: length of the full site collection
:param gmvs_by_sid: dictionary site_id -> gmvs
:param gs: GSIM string
:param imtls: ordered dictionary {IMT: intensity measure levels}
:param investigation_time: investigation time
:param duration: investigation_time * number of Stochastic Event Sets
"""
curves = zero_curves(num_sites, imtls)
for imt in imtls:
curves[imt] = numpy.array([
gmvs_to_haz_curve([gmv[gs][imt] for gmv in gmvs_by_sid[sid]],
imtls[imt], investigation_time, duration)
for sid in range(num_sites)
])
return curves
def get_hcurves_and_means(dstore):
"""
Extract hcurves from the datastore and compute their means.
:returns: curves_by_rlz, mean_curves
"""
oq = dstore['oqparam']
hcurves = dstore['hcurves']
realizations = dstore['csm_info'].get_rlzs_assoc().realizations
weights = [rlz.weight for rlz in realizations]
curves_by_rlz = {rlz: hcurves['rlz-%03d' % rlz.ordinal]
for rlz in realizations}
N = len(dstore['sitecol'])
mean_curves = zero_curves(N, oq.imtls)
for imt in oq.imtls:
mean_curves[imt] = scientific.mean_curve(
[curves_by_rlz[rlz][imt] for rlz in sorted(curves_by_rlz)],
weights)
return curves_by_rlz, mean_curves
def compute_gmfs_and_curves(ses_ruptures, sitecol, rlzs_assoc, monitor):
"""
:param ses_ruptures:
a list of blocks of SESRuptures of the same SESCollection
:param sitecol:
a :class:`openquake.hazardlib.site.SiteCollection` instance
:param rlzs_assoc:
a RlzsAssoc instance
:param monitor:
a Monitor instance
:returns:
a dictionary trt_model_id -> curves_by_gsim
where the list of bounding boxes is empty
"""
oq = monitor.oqparam
# NB: by construction each block is a non-empty list with
# ruptures of the same col_id and therefore trt_model_id
col_id = ses_ruptures[0].col_id
trt_id = rlzs_assoc.csm_info.get_trt_id(col_id)
gsims = rlzs_assoc.get_gsims_by_col()[col_id]
trunc_level = getattr(oq, 'truncation_level', None)
correl_model = readinput.get_correl_model(oq)
num_sites = len(sitecol)
dic = make_gmf_by_tag(
ses_ruptures, sitecol.complete, oq.imtls, gsims,
trunc_level, correl_model, monitor)
zero = zero_curves(num_sites, oq.imtls)
result = AccumDict({(trt_id, str(gsim)): [dic, zero] for gsim in gsims})
gmfs = [dic[tag] for tag in sorted(dic)]
if oq.hazard_curves_from_gmfs:
duration = oq.investigation_time * oq.ses_per_logic_tree_path * (
oq.number_of_logic_tree_samples or 1)
for gsim in gsims:
gs = str(gsim)
result[trt_id, gs][1] = to_haz_curves(
num_sites, gs, gmfs, oq.imtls,
oq.investigation_time, duration)
if not oq.ground_motion_fields:
# reset the gmf_by_tag dictionary to avoid
# transferring a lot of unused data
for key in result:
result[key][0].clear()
return result
def post_execute(self, curves_by_trt_gsim):
"""
Collect the hazard curves by realization and export them.
:param curves_by_trt_gsim:
a dictionary (trt_id, gsim) -> hazard curves
"""
self.curves_by_trt_gsim = curves_by_trt_gsim
oq = self.oqparam
zc = zero_curves(len(self.sitecol), oq.imtls)
curves_by_rlz = self.rlzs_assoc.combine_curves(
curves_by_trt_gsim, agg_curves, zc)
rlzs = self.rlzs_assoc.realizations
nsites = len(self.sitecol)
if oq.individual_curves:
for rlz, curves in curves_by_rlz.iteritems():
self.store_curves('rlz-%d' % rlz.ordinal, curves)
if len(rlzs) == 1: # cannot compute statistics
[self.mean_curves] = curves_by_rlz.values()
return
weights = (None if oq.number_of_logic_tree_samples
else [rlz.weight for rlz in rlzs])
mean = oq.mean_hazard_curves
if mean:
self.mean_curves = numpy.array(zc)
for imt in oq.imtls:
self.mean_curves[imt] = scientific.mean_curve(
[curves_by_rlz[rlz][imt] for rlz in rlzs], weights)
self.quantile = {}
for q in oq.quantile_hazard_curves:
self.quantile[q] = qc = numpy.array(zc)
for imt in oq.imtls:
curves = [curves_by_rlz[rlz][imt] for rlz in rlzs]
qc[imt] = scientific.quantile_curve(
curves, q, weights).reshape((nsites, -1))
if mean:
self.store_curves('mean', self.mean_curves)
for q in self.quantile:
self.store_curves('quantile-%s' % q, self.quantile[q])
def get_hcurves_and_means(dstore):
"""
Extract hcurves from the datastore and compute their means.
:returns: curves_by_rlz, mean_curves
"""
oq = dstore['oqparam']
hcurves = dstore['hcurves']
realizations = dstore['csm_info'].get_rlzs_assoc().realizations
weights = [rlz.weight for rlz in realizations]
curves_by_rlz = {rlz: hcurves['rlz-%03d' % rlz.ordinal]
for rlz in realizations}
N = len(dstore['sitemesh'])
mean_curves = zero_curves(N, oq.imtls)
for imt in oq.imtls:
mean_curves[imt] = scientific.mean_curve(
[curves_by_rlz[rlz][imt] for rlz in sorted(curves_by_rlz)],
weights)
return curves_by_rlz, mean_curves
def to_haz_curves(num_sites, gs, gmfs, imtls, investigation_time, duration):
"""
:param num_sites: length of the full site collection
:param gs: a GSIM string
:param gmfs: gmf arrays
:param imtls: ordered dictionary {IMT: intensity measure levels}
:param investigation_time: investigation time
:param duration: investigation_time * number of Stochastic Event Sets
"""
# group gmvs by site index
data = [[] for idx in range(num_sites)]
for gmf in gmfs:
for idx, gmv in zip(gmf['idx'], gmf[gs]):
data[idx].append(gmv)
curves = zero_curves(num_sites, imtls)
for imt in imtls:
curves[imt] = numpy.array([
gmvs_to_haz_curve([gmv[imt] for gmv in gmvs], imtls[imt],
investigation_time, duration) for gmvs in data])
return curves
def execute(self):
"""
Run in parallel `core_func(sources, sitecol, monitor)`, by
parallelizing on the sources according to their weight and
tectonic region type.
"""
monitor = self.monitor(self.core_func.__name__)
monitor.oqparam = self.oqparam
sources = self.composite_source_model.get_sources()
zc = zero_curves(len(self.sitecol), self.oqparam.imtls)
zerodict = AccumDict((key, zc) for key in self.rlzs_assoc)
gsims_assoc = self.rlzs_assoc.get_gsims_by_trt_id()
curves_by_trt_gsim = parallel.apply_reduce(
self.core_func.__func__,
(sources, self.sitecol, gsims_assoc, monitor),
agg=agg_dicts, acc=zerodict,
concurrent_tasks=self.oqparam.concurrent_tasks,
weight=operator.attrgetter('weight'),
key=operator.attrgetter('trt_model_id'))
return curves_by_trt_gsim
def execute(self):
"""
Run in parallel `core_func(sources, sitecol, monitor)`, by
parallelizing on the ruptures according to their weight and
tectonic region type.
"""
oq = self.oqparam
if not oq.hazard_curves_from_gmfs and not oq.ground_motion_fields:
return
monitor = self.monitor(self.core_func.__name__)
monitor.oqparam = oq
zc = zero_curves(len(self.sitecol), self.oqparam.imtls)
zerodict = AccumDict((key, zc) for key in self.rlzs_assoc)
self.gmf_dict = collections.defaultdict(AccumDict)
curves_by_trt_gsim = parallel.apply_reduce(
self.core_func.__func__,
(self.sesruptures, self.sitecol, self.rlzs_assoc, monitor),
concurrent_tasks=self.oqparam.concurrent_tasks,
acc=zerodict, agg=self.combine_curves_and_save_gmfs,
key=operator.attrgetter('col_id'))
return curves_by_trt_gsim
def save_curves(self, curves_by_rlz):
"""
Save the dictionary curves_by_rlz
"""
oq = self.oqparam
rlzs = self.rlzs_assoc.realizations
nsites = len(self.sitecol)
if oq.individual_curves:
with self.monitor('save curves_by_rlz', autoflush=True):
for rlz, curves in curves_by_rlz.items():
self.store_curves('rlz-%03d' % rlz.ordinal, curves, rlz)
if len(rlzs) == 1: # cannot compute statistics
[self.mean_curves] = curves_by_rlz.values()
return
with self.monitor('compute and save statistics', autoflush=True):
weights = (None if oq.number_of_logic_tree_samples else
[rlz.weight for rlz in rlzs])
# mean curves are always computed but stored only on request
zc = zero_curves(nsites, oq.imtls)
self.mean_curves = numpy.array(zc)
for imt in oq.imtls:
self.mean_curves[imt] = scientific.mean_curve(
[curves_by_rlz.get(rlz, zc)[imt] for rlz in rlzs], weights)
self.quantile = {}
for q in oq.quantile_hazard_curves:
self.quantile[q] = qc = numpy.array(zc)
for imt in oq.imtls:
curves = [curves_by_rlz[rlz][imt] for rlz in rlzs]
qc[imt] = scientific.quantile_curve(curves, q,
weights).reshape(
(nsites, -1))
if oq.mean_hazard_curves:
self.store_curves('mean', self.mean_curves)
for q in self.quantile:
self.store_curves('quantile-%s' % q, self.quantile[q])
def save_curves(self, curves_by_rlz):
"""
Save the dictionary curves_by_rlz
"""
oq = self.oqparam
rlzs = self.rlzs_assoc.realizations
nsites = len(self.sitecol)
if oq.individual_curves:
with self.monitor('save curves_by_rlz', autoflush=True):
for rlz, curves in curves_by_rlz.items():
self.store_curves('rlz-%03d' % rlz.ordinal, curves, rlz)
if len(rlzs) == 1: # cannot compute statistics
[self.mean_curves] = curves_by_rlz.values()
return
with self.monitor('compute and save statistics', autoflush=True):
weights = (None if oq.number_of_logic_tree_samples
else [rlz.weight for rlz in rlzs])
# mean curves are always computed but stored only on request
zc = zero_curves(nsites, oq.imtls)
self.mean_curves = numpy.array(zc)
for imt in oq.imtls:
self.mean_curves[imt] = scientific.mean_curve(
[curves_by_rlz.get(rlz, zc)[imt] for rlz in rlzs], weights)
self.quantile = {}
for q in oq.quantile_hazard_curves:
self.quantile[q] = qc = numpy.array(zc)
for imt in oq.imtls:
curves = [curves_by_rlz[rlz][imt] for rlz in rlzs]
qc[imt] = scientific.quantile_curve(
curves, q, weights).reshape((nsites, -1))
if oq.mean_hazard_curves:
self.store_curves('mean', self.mean_curves)
for q in self.quantile:
self.store_curves('quantile-%s' % q, self.quantile[q])
def execute(self):
"""
Run in parallel `core_func(sources, sitecol, monitor)`, by
parallelizing on the sources according to their weight and
tectonic region type.
"""
monitor = self.monitor(self.core_func.__name__)
monitor.oqparam = self.oqparam
sources = self.csm.get_sources()
zc = zero_curves(len(self.sitecol.complete), self.oqparam.imtls)
zerodict = AccumDict((key, zc) for key in self.rlzs_assoc)
zerodict['calc_times'] = []
gsims_assoc = self.rlzs_assoc.gsims_by_trt_id
curves_by_trt_gsim = parallel.apply_reduce(
self.core_func.__func__,
(sources, self.sitecol, gsims_assoc, monitor),
agg=agg_dicts, acc=zerodict,
concurrent_tasks=self.oqparam.concurrent_tasks,
weight=operator.attrgetter('weight'),
key=operator.attrgetter('trt_model_id'))
if self.persistent:
store_source_chunks(self.datastore)
return curves_by_trt_gsim
def execute(self):
"""
Run in parallel `core_task(sources, sitecol, monitor)`, by
parallelizing on the ruptures according to their weight and
tectonic region type.
"""
oq = self.oqparam
if not oq.hazard_curves_from_gmfs and not oq.ground_motion_fields:
return
monitor = self.monitor(self.core_task.__name__)
monitor.oqparam = oq
zc = zero_curves(len(self.sitecol.complete), self.oqparam.imtls)
zerodict = AccumDict((key, zc) for key in self.rlzs_assoc)
curves_by_trt_gsim = parallel.apply_reduce(
self.core_task.__func__,
(self.sesruptures, self.sitecol, self.rlzs_assoc, monitor),
concurrent_tasks=self.oqparam.concurrent_tasks,
acc=zerodict, agg=self.combine_curves_and_save_gmfs,
key=operator.attrgetter('trt_id'),
weight=operator.attrgetter('multiplicity'))
if oq.ground_motion_fields:
self.datastore.set_nbytes('gmf_data')
self.datastore.set_nbytes('sid_data')
return curves_by_trt_gsim
def get_hcurves_from_csv(oqparam, fname):
"""
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param fname:
a .txt file with format `IMT lon lat poe1 ... poeN`
:returns:
the site collection and the hazard curves read by the .txt file
"""
if not oqparam.imtls:
oqparam.set_risk_imtls(get_risk_models(oqparam))
if not oqparam.imtls:
raise ValueError('Missing intensity_measure_types_and_levels in %s'
% oqparam.inputs['job_ini'])
num_values = list(map(len, list(oqparam.imtls.values())))
with open(oqparam.inputs['hazard_curves']) as csvfile:
mesh, hcurves_by_imt = get_mesh_csvdata(
csvfile, list(oqparam.imtls), num_values,
valid.decreasing_probabilities)
sitecol = get_site_collection(oqparam, mesh)
curves = zero_curves(len(mesh), oqparam.imtls)
for imt_ in oqparam.imtls:
curves[imt_] = hcurves_by_imt[imt_]
return sitecol, curves
def post_execute(self, curves_by_trt_gsim):
"""
Collect the hazard curves by realization and export them.
:param curves_by_trt_gsim:
a dictionary (trt_id, gsim) -> hazard curves
"""
# save calculation time per source
calc_times = getattr(curves_by_trt_gsim, 'calc_times', [])
sources = self.csm.get_sources()
infodict = collections.defaultdict(float)
weight = {}
for src_idx, dt in calc_times:
src = sources[src_idx]
weight[src.trt_model_id, src.source_id] = src.weight
infodict[src.trt_model_id, src.source_id] += dt
infolist = [key + (dt, weight[key]) for key, dt in infodict.items()]
infolist.sort(key=operator.itemgetter(1), reverse=True)
if infolist:
self.source_info = numpy.array(infolist, source_info_dt)
with self.monitor('save curves_by_trt_gsim', autoflush=True):
for sm in self.rlzs_assoc.csm_info.source_models:
group = self.datastore.hdf5.create_group(
'curves_by_sm/' + '_'.join(sm.path))
group.attrs['source_model'] = sm.name
for tm in sm.trt_models:
for i, gsim in enumerate(tm.gsims):
try:
curves = curves_by_trt_gsim[tm.id, gsim]
except KeyError: # no data for the trt_model
pass
else:
ts = '%03d-%d' % (tm.id, i)
if nonzero(curves):
group[ts] = curves
group[ts].attrs['trt'] = tm.trt
group[ts].attrs['nbytes'] = curves.nbytes
group[ts].attrs['gsim'] = str(gsim)
self.datastore.set_nbytes(group.name)
self.datastore.set_nbytes('curves_by_sm')
oq = self.oqparam
with self.monitor('combine and save curves_by_rlz', autoflush=True):
zc = zero_curves(len(self.sitecol.complete), oq.imtls)
curves_by_rlz = self.rlzs_assoc.combine_curves(
curves_by_trt_gsim, agg_curves, zc)
rlzs = self.rlzs_assoc.realizations
nsites = len(self.sitecol)
if oq.individual_curves:
for rlz, curves in curves_by_rlz.items():
self.store_curves('rlz-%03d' % rlz.ordinal, curves, rlz)
if len(rlzs) == 1: # cannot compute statistics
[self.mean_curves] = curves_by_rlz.values()
return
with self.monitor('compute and save statistics', autoflush=True):
weights = (None if oq.number_of_logic_tree_samples
else [rlz.weight for rlz in rlzs])
mean = oq.mean_hazard_curves
if mean:
self.mean_curves = numpy.array(zc)
for imt in oq.imtls:
self.mean_curves[imt] = scientific.mean_curve(
[curves_by_rlz[rlz][imt] for rlz in rlzs], weights)
self.quantile = {}
for q in oq.quantile_hazard_curves:
self.quantile[q] = qc = numpy.array(zc)
for imt in oq.imtls:
curves = [curves_by_rlz[rlz][imt] for rlz in rlzs]
qc[imt] = scientific.quantile_curve(
curves, q, weights).reshape((nsites, -1))
if mean:
self.store_curves('mean', self.mean_curves)
for q in self.quantile:
self.store_curves('quantile-%s' % q, self.quantile[q])
def post_execute(self, curves_by_trt_gsim):
"""
Collect the hazard curves by realization and export them.
:param curves_by_trt_gsim:
a dictionary (trt_id, gsim) -> hazard curves
"""
with self.monitor('save curves_by_trt_gsim', autoflush=True):
for sm in self.rlzs_assoc.csm_info.source_models:
group = self.datastore.hdf5.create_group('curves_by_sm/' +
'_'.join(sm.path))
group.attrs['source_model'] = sm.name
for tm in sm.trt_models:
for i, gsim in enumerate(tm.gsims):
try:
curves = curves_by_trt_gsim[tm.id, gsim]
except KeyError: # no data for the trt_model
pass
else:
ts = '%03d-%d' % (tm.id, i)
if nonzero(curves):
group[ts] = curves
group[ts].attrs['trt'] = tm.trt
group[ts].attrs['nbytes'] = curves.nbytes
group[ts].attrs['gsim'] = str(gsim)
self.datastore.set_nbytes(group.name)
self.datastore.set_nbytes('curves_by_sm')
oq = self.oqparam
with self.monitor('combine and save curves_by_rlz', autoflush=True):
zc = zero_curves(len(self.sitecol.complete), oq.imtls)
curves_by_rlz = self.rlzs_assoc.combine_curves(
curves_by_trt_gsim, agg_curves, zc)
rlzs = self.rlzs_assoc.realizations
nsites = len(self.sitecol)
if oq.individual_curves:
for rlz, curves in curves_by_rlz.items():
self.store_curves('rlz-%03d' % rlz.ordinal, curves, rlz)
if len(rlzs) == 1: # cannot compute statistics
[self.mean_curves] = curves_by_rlz.values()
return
with self.monitor('compute and save statistics', autoflush=True):
weights = (None if oq.number_of_logic_tree_samples else
[rlz.weight for rlz in rlzs])
# mean curves are always computed but stored only on request
self.mean_curves = numpy.array(zc)
for imt in oq.imtls:
self.mean_curves[imt] = scientific.mean_curve(
[curves_by_rlz[rlz][imt] for rlz in rlzs], weights)
self.quantile = {}
for q in oq.quantile_hazard_curves:
self.quantile[q] = qc = numpy.array(zc)
for imt in oq.imtls:
curves = [curves_by_rlz[rlz][imt] for rlz in rlzs]
qc[imt] = scientific.quantile_curve(curves, q,
weights).reshape(
(nsites, -1))
if oq.mean_hazard_curves:
self.store_curves('mean', self.mean_curves)
for q in self.quantile:
self.store_curves('quantile-%s' % q, self.quantile[q])
def post_execute(self, curves_by_trt_gsim):
"""
Collect the hazard curves by realization and export them.
:param curves_by_trt_gsim:
a dictionary (trt_id, gsim) -> hazard curves
"""
with self.monitor('save curves_by_trt_gsim', autoflush=True):
for sm in self.rlzs_assoc.csm_info.source_models:
group = self.datastore.hdf5.create_group(
'curves_by_sm/' + '_'.join(sm.path))
group.attrs['source_model'] = sm.name
for tm in sm.trt_models:
for i, gsim in enumerate(tm.gsims):
try:
curves = curves_by_trt_gsim[tm.id, gsim]
except KeyError: # no data for the trt_model
pass
else:
ts = '%03d-%d' % (tm.id, i)
if nonzero(curves):
group[ts] = curves
group[ts].attrs['trt'] = tm.trt
group[ts].attrs['nbytes'] = curves.nbytes
group[ts].attrs['gsim'] = str(gsim)
self.datastore.set_nbytes(group.name)
self.datastore.set_nbytes('curves_by_sm')
oq = self.oqparam
with self.monitor('combine and save curves_by_rlz', autoflush=True):
zc = zero_curves(len(self.sitecol.complete), oq.imtls)
curves_by_rlz = self.rlzs_assoc.combine_curves(
curves_by_trt_gsim, agg_curves, zc)
rlzs = self.rlzs_assoc.realizations
nsites = len(self.sitecol)
if oq.individual_curves:
for rlz, curves in curves_by_rlz.items():
self.store_curves('rlz-%03d' % rlz.ordinal, curves, rlz)
if len(rlzs) == 1: # cannot compute statistics
[self.mean_curves] = curves_by_rlz.values()
return
with self.monitor('compute and save statistics', autoflush=True):
weights = (None if oq.number_of_logic_tree_samples
else [rlz.weight for rlz in rlzs])
# mean curves are always computed but stored only on request
self.mean_curves = numpy.array(zc)
for imt in oq.imtls:
self.mean_curves[imt] = scientific.mean_curve(
[curves_by_rlz[rlz][imt] for rlz in rlzs], weights)
self.quantile = {}
for q in oq.quantile_hazard_curves:
self.quantile[q] = qc = numpy.array(zc)
for imt in oq.imtls:
curves = [curves_by_rlz[rlz][imt] for rlz in rlzs]
qc[imt] = scientific.quantile_curve(
curves, q, weights).reshape((nsites, -1))
if oq.mean_hazard_curves:
self.store_curves('mean', self.mean_curves)
for q in self.quantile:
self.store_curves('quantile-%s' % q, self.quantile[q])
def post_execute(self, curves_by_trt_gsim):
"""
Collect the hazard curves by realization and export them.
:param curves_by_trt_gsim:
a dictionary (trt_id, gsim) -> hazard curves
"""
# save calculation time per source
try:
calc_times = curves_by_trt_gsim.pop('calc_times')
except KeyError:
pass
else:
sources = self.csm.get_sources()
info = []
for i, dt in calc_times:
src = sources[i]
info.append((src.trt_model_id, src.source_id, dt))
info.sort(key=operator.itemgetter(2), reverse=True)
self.source_info = numpy.array(info, source_info_dt)
# save curves_by_trt_gsim
for sm in self.rlzs_assoc.csm_info.source_models:
group = self.datastore.hdf5.create_group(
'curves_by_sm/' + '_'.join(sm.path))
group.attrs['source_model'] = sm.name
for tm in sm.trt_models:
for gsim in tm.gsims:
try:
curves = curves_by_trt_gsim[tm.id, gsim]
except KeyError: # no data for the trt_model
pass
else:
ts = '%03d-%s' % (tm.id, gsim)
group[ts] = curves
group[ts].attrs['trt'] = tm.trt
oq = self.oqparam
zc = zero_curves(len(self.sitecol.complete), oq.imtls)
curves_by_rlz = self.rlzs_assoc.combine_curves(
curves_by_trt_gsim, agg_curves, zc)
rlzs = self.rlzs_assoc.realizations
nsites = len(self.sitecol)
if oq.individual_curves:
for rlz, curves in curves_by_rlz.items():
self.store_curves('rlz-%03d' % rlz.ordinal, curves, rlz)
if len(rlzs) == 1: # cannot compute statistics
[self.mean_curves] = curves_by_rlz.values()
return
weights = (None if oq.number_of_logic_tree_samples
else [rlz.weight for rlz in rlzs])
mean = oq.mean_hazard_curves
if mean:
self.mean_curves = numpy.array(zc)
for imt in oq.imtls:
self.mean_curves[imt] = scientific.mean_curve(
[curves_by_rlz[rlz][imt] for rlz in rlzs], weights)
self.quantile = {}
for q in oq.quantile_hazard_curves:
self.quantile[q] = qc = numpy.array(zc)
for imt in oq.imtls:
curves = [curves_by_rlz[rlz][imt] for rlz in rlzs]
qc[imt] = scientific.quantile_curve(
curves, q, weights).reshape((nsites, -1))
if mean:
self.store_curves('mean', self.mean_curves)
for q in self.quantile:
self.store_curves('quantile-%s' % q, self.quantile[q])