def save_gmf(gmfcoll_id, gmf_dict, sites):
"""
Helper method to save computed GMF data to the database.
:param int gmfcoll_id:
the id of a :class:`openquake.engine.db.models.Gmf` record
:param dict gmf_dict:
The GMF results during the calculation
:param sites:
An :class:`openquake.engine.models.SiteCollection`
object
"""
inserter = writer.CacheInserter(models.GmfData, 100)
# NB: GmfData may contain large arrays and the cache may become large
for imt, gmfs_ in gmf_dict.iteritems():
# ``gmfs`` comes in as a numpy.matrix
# we want it is an array; it handles subscripting
# in the way that we want
gmfarray = numpy.array(gmfs_)
sa_period = None
sa_damping = None
if isinstance(imt, openquake.hazardlib.imt.SA):
sa_period = imt.period
sa_damping = imt.damping
imt_name = imt.__class__.__name__
for i, site in enumerate(sites):
inserter.add(models.GmfData(
gmf_id=gmfcoll_id,
ses_id=None,
imt=imt_name,
sa_period=sa_period,
sa_damping=sa_damping,
site_id=site.id,
rupture_ids=None,
gmvs=gmfarray[i].tolist()))
inserter.flush()
def post_execute(self, result=None):
"""
Saving the GMFs in the database
"""
gmf_id = self.gmf.id
inserter = writer.CacheInserter(models.GmfData, max_cache_size=1000)
gsim = self.oqparam.gsim
for imt in self.imts:
gmfs = numpy.array([
self.acc[tag][gsim][str(imt)] for tag in self.tags
]).transpose()
for site_id, gmvs in zip(self.sites.sids, gmfs):
inserter.add(
models.GmfData(gmf_id=gmf_id,
task_no=0,
imt=imt[0],
sa_period=imt[1],
sa_damping=imt[2],
site_id=site_id,
rupture_ids=self.rupids,
gmvs=list(gmvs)))
inserter.flush()
def create_db_ruptures(rupture, ses_coll, tags, seed):
"""
Insert the SESRuptures associated to the given rupture and
SESCollection.
:param rupture: hazardlib rupture
:param ses_coll: SESCollection instance
:param tags: tags of the ruptures to insert
:seed: a random seed
:returns: the IDs of the inserted ProbabilisticRupture and SESRuptures
"""
prob_rup = models.ProbabilisticRupture.create(rupture, ses_coll)
inserter = writer.CacheInserter(models.SESRupture, max_cache_size=100000)
rnd = random.Random()
rnd.seed(seed)
seeds = []
sesrupts = []
for tag in tags:
s = rnd.randint(0, models.MAX_SINT_32)
seeds.append(s)
sesrupts.append(
models.SESRupture(ses_id=1, rupture=prob_rup, tag=tag, seed=s))
return prob_rup.id, inserter.saveall(sesrupts), seeds
def post_process(self):
"""
Compute aggregate loss curves and event loss tables
"""
with EnginePerformanceMonitor('post processing', self.job.id):
time_span, tses = self.hazard_times()
for loss_type, event_loss_table in self.event_loss_tables.items():
for hazard_output in self.rc.hazard_outputs():
event_loss = models.EventLoss.objects.create(
output=models.Output.objects.create_output(
self.job, "Event Loss Table. type=%s, hazard=%s" %
(loss_type, hazard_output.id), "event_loss"),
loss_type=loss_type,
hazard_output=hazard_output)
inserter = writer.CacheInserter(models.EventLossData, 9999)
rupture_ids = models.SESRupture.objects.filter(
ses__ses_collection__lt_realization=hazard_output.
output_container.lt_realization).values_list('id',
flat=True)
for rupture_id in rupture_ids:
if rupture_id in event_loss_table:
inserter.add(
models.EventLossData(
event_loss_id=event_loss.id,
rupture_id=rupture_id,
aggregate_loss=event_loss_table[rupture_id]
))
inserter.flush()
aggregate_losses = [
event_loss_table[rupture_id]
for rupture_id in rupture_ids
if rupture_id in event_loss_table
]
if aggregate_losses:
aggregate_loss_losses, aggregate_loss_poes = (
scientific.event_based(
aggregate_losses,
tses=tses,
time_span=time_span,
curve_resolution=self.rc.loss_curve_resolution)
)
models.AggregateLossCurveData.objects.create(
loss_curve=models.LossCurve.objects.create(
aggregate=True,
insured=False,
hazard_output=hazard_output,
loss_type=loss_type,
output=models.Output.objects.create_output(
self.job, "aggregate loss curves. "
"loss_type=%s hazard=%s" %
(loss_type, hazard_output),
"agg_loss_curve")),
losses=aggregate_loss_losses,
poes=aggregate_loss_poes,
average_loss=scientific.average_loss(
aggregate_loss_losses, aggregate_loss_poes),
stddev_loss=numpy.std(aggregate_losses))
def do_aggregate_post_proc(self):
"""
Grab hazard data for all realizations and sites from the database and
compute mean and/or quantile aggregates (depending on which options are
enabled in the calculation).
Post-processing results will be stored directly into the database.
"""
num_rlzs = models.LtRealization.objects.filter(
hazard_calculation=self.hc).count()
num_site_blocks_per_incr = int(CURVE_CACHE_SIZE) / int(num_rlzs)
if num_site_blocks_per_incr == 0:
# This means we have `num_rlzs` >= `CURVE_CACHE_SIZE`.
# The minimum number of sites should be 1.
num_site_blocks_per_incr = 1
slice_incr = num_site_blocks_per_incr * num_rlzs # unit: num records
if self.hc.mean_hazard_curves:
# create a new `HazardCurve` 'container' record for mean
# curves (virtual container for multiple imts)
models.HazardCurve.objects.create(
output=models.Output.objects.create_output(
self.job, "mean-curves-multi-imt", "hazard_curve_multi"),
statistics="mean",
imt=None,
investigation_time=self.hc.investigation_time)
if self.hc.quantile_hazard_curves:
for quantile in self.hc.quantile_hazard_curves:
# create a new `HazardCurve` 'container' record for quantile
# curves (virtual container for multiple imts)
models.HazardCurve.objects.create(
output=models.Output.objects.create_output(
self.job, 'quantile(%s)-curves' % quantile,
"hazard_curve_multi"),
statistics="quantile",
imt=None,
quantile=quantile,
investigation_time=self.hc.investigation_time)
for imt, imls in self.hc.intensity_measure_types_and_levels.items():
im_type, sa_period, sa_damping = models.parse_imt(imt)
# prepare `output` and `hazard_curve` containers in the DB:
container_ids = dict()
if self.hc.mean_hazard_curves:
mean_output = models.Output.objects.create_output(
job=self.job,
display_name='mean-curves-%s' % imt,
output_type='hazard_curve')
mean_hc = models.HazardCurve.objects.create(
output=mean_output,
investigation_time=self.hc.investigation_time,
imt=im_type,
imls=imls,
sa_period=sa_period,
sa_damping=sa_damping,
statistics='mean')
container_ids['mean'] = mean_hc.id
if self.hc.quantile_hazard_curves:
for quantile in self.hc.quantile_hazard_curves:
q_output = models.Output.objects.create_output(
job=self.job,
display_name=('quantile(%s)-curves-%s' %
(quantile, imt)),
output_type='hazard_curve')
q_hc = models.HazardCurve.objects.create(
output=q_output,
investigation_time=self.hc.investigation_time,
imt=im_type,
imls=imls,
sa_period=sa_period,
sa_damping=sa_damping,
statistics='quantile',
quantile=quantile)
container_ids['q%s' % quantile] = q_hc.id
all_curves_for_imt = models.order_by_location(
models.HazardCurveData.objects.all_curves_for_imt(
self.job.id, im_type, sa_period, sa_damping))
with transaction.commit_on_success(using='reslt_writer'):
inserter = writer.CacheInserter(models.HazardCurveData,
CURVE_CACHE_SIZE)
for chunk in models.queryset_iter(all_curves_for_imt,
slice_incr):
# slice each chunk by `num_rlzs` into `site_chunk`
# and compute the aggregate
for site_chunk in block_splitter(chunk, num_rlzs):
site = site_chunk[0].location
curves_poes = [x.poes for x in site_chunk]
curves_weights = [x.weight for x in site_chunk]
# do means and quantiles
# quantiles first:
if self.hc.quantile_hazard_curves:
for quantile in self.hc.quantile_hazard_curves:
if self.hc.number_of_logic_tree_samples == 0:
# explicitly weighted quantiles
q_curve = weighted_quantile_curve(
curves_poes, curves_weights, quantile)
else:
# implicitly weighted quantiles
q_curve = quantile_curve(
curves_poes, quantile)
inserter.add(
models.HazardCurveData(
hazard_curve_id=(
container_ids['q%s' % quantile]),
poes=q_curve.tolist(),
location=site.wkt))
# then means
if self.hc.mean_hazard_curves:
m_curve = mean_curve(curves_poes,
weights=curves_weights)
inserter.add(
models.HazardCurveData(
hazard_curve_id=container_ids['mean'],
poes=m_curve.tolist(),
location=site.wkt))
inserter.flush()
post_processing as cls_post_proc)
from openquake.engine.calculators.hazard.event_based import post_processing
from openquake.engine.db import models
from openquake.engine.input import logictree
from openquake.engine.utils import tasks
from openquake.engine.performance import EnginePerformanceMonitor
#: Always 1 for the computation of ground motion fields in the event-based
#: hazard calculator.
DEFAULT_GMF_REALIZATIONS = 1
BLOCK_SIZE = 5000 # TODO: put this in openquake.cfg
# NB: beware of large caches
inserter = writer.CacheInserter(models.GmfData, 1000)
# Disabling pylint for 'Too many local variables'
# pylint: disable=R0914
@tasks.oqtask
def compute_ses(job_id, src_ids, ses, src_seeds, ltp):
"""
Celery task for the stochastic event set calculator.
Samples logic trees and calls the stochastic event set calculator.
Once stochastic event sets are calculated, results will be saved to the
database. See :class:`openquake.engine.db.models.SESCollection`.
Optionally (specified in the job configuration using the
def event_based(workflow, getter, outputdict, params, monitor):
"""
Celery task for the event based risk calculator.
:param job_id: the id of the current
:class:`openquake.engine.db.models.OqJob`
:param workflow:
A :class:`openquake.risklib.workflows.Workflow` instance
:param getter:
A :class:`HazardGetter` instance
:param outputdict:
An instance of :class:`..writers.OutputDict` containing
output container instances (e.g. a LossCurve)
:param params:
An instance of :class:`..base.CalcParams` used to compute
derived outputs
:param monitor:
A monitor instance
:returns:
A dictionary {loss_type: event_loss_table}
"""
# NB: event_loss_table is a dictionary (loss_type, out_id) -> loss,
# out_id can be None, and it that case it stores the statistics
event_loss_table = {}
# num_loss is a dictionary asset_ref -> array([not_zeros, total])
num_losses = collections.defaultdict(lambda: numpy.zeros(2, dtype=int))
specific_assets = set(params.specific_assets)
statistics = params.statistics # enabled by default
# keep in memory the loss_matrix only when specific_assets are set
workflow.return_loss_matrix = bool(specific_assets)
# the insert here will work only if specific_assets is set
inserter = writer.CacheInserter(
models.EventLossAsset, max_cache_size=10000)
for loss_type in workflow.loss_types:
with monitor('computing individual risk', autoflush=True):
outputs = workflow.compute_all_outputs(getter, loss_type)
if statistics:
outputs = list(outputs) # expand the generator
# this is needed, otherwise the call to workflow.statistics
# below will find an empty iterable; notice that by disabling
# the statistics we can save memory by keeping only one
# hazard realization at the time
for out in outputs:
event_loss_table[loss_type, out.hid] = out.event_loss_table
disagg_outputs = None # changed if params.sites_disagg is set
if specific_assets:
loss_matrix, assets = _filter_loss_matrix_assets(
out.loss_matrix, out.assets, specific_assets)
if len(assets):
# compute the loss per rupture per asset
event_loss = models.EventLoss.objects.get(
output__oq_job=monitor.job_id,
output__output_type='event_loss_asset',
loss_type=loss_type, hazard_output=out.hid)
# losses is E x n matrix, where E is the number of ruptures
# and n the number of assets in the specific_assets set
losses = (loss_matrix.transpose() *
numpy_map(lambda a: a.value(loss_type), assets))
# save an EventLossAsset record for each specific asset
for rup_id, losses_per_rup in zip(
getter.rupture_ids, losses):
for asset, loss_per_rup in zip(assets, losses_per_rup):
if loss_per_rup: # save only non-zero losses
ela = models.EventLossAsset(
event_loss=event_loss, rupture_id=rup_id,
asset=asset, loss=loss_per_rup)
inserter.add(ela)
# update the counters: not_zeros is incremented
# only if loss_per_rup is nonzero, total always
num_losses[asset.asset_ref] += numpy.array(
[bool(loss_per_rup), 1])
if params.sites_disagg:
with monitor('disaggregating results', autoflush=True):
ruptures = [models.SESRupture.objects.get(pk=rid)
for rid in getter.rupture_ids]
disagg_outputs = disaggregate(
out, [r.rupture for r in ruptures], params)
with monitor('saving individual risk', autoflush=True):
save_individual_outputs(
outputdict.with_args(hazard_output_id=out.hid,
loss_type=loss_type),
out, disagg_outputs, params)
if statistics and len(outputs) > 1:
stats = workflow.statistics(
outputs, params.quantile_loss_curves)
with monitor('saving risk statistics', autoflush=True):
save_statistical_output(
outputdict.with_args(
hazard_output_id=None, loss_type=loss_type),
stats, params)
event_loss_table[loss_type, None] = stats.event_loss_table
inserter.flush()
# log info about the rows entered in the event_loss_asset table
for asset_ref in sorted(num_losses):
not_zeros, total = num_losses[asset_ref]
logs.LOG.info('Saved %d/%d losses for asset %s',
not_zeros, total, asset_ref)
return event_loss_table
def post_process(self):
"""
Compute aggregate loss curves and event loss tables
"""
oq = self.oqparam
tses = oq.investigation_time * oq.ses_per_logic_tree_path
with self.monitor('post processing', autoflush=True):
inserter = writer.CacheInserter(models.EventLossData,
max_cache_size=10000)
for (loss_type, out_id), event_loss_table in self.acc.items():
if out_id: # values for individual realizations
hazard_output = models.Output.objects.get(pk=out_id)
event_loss = models.EventLoss.objects.get(
output__oq_job=self.job,
output__output_type='event_loss',
loss_type=loss_type, hazard_output=hazard_output)
if isinstance(hazard_output.output_container,
models.SESCollection):
ses_coll = hazard_output.output_container
rupture_ids = ses_coll.get_ruptures().values_list(
'id', flat=True)
else: # extract the SES collection from the Gmf
rupture_ids = models.SESRupture.objects.filter(
rupture__ses_collection__trt_model__lt_model=
hazard_output.output_container.
lt_realization.lt_model).values_list(
'id', flat=True)
for rupture_id in rupture_ids:
if rupture_id in event_loss_table:
inserter.add(
models.EventLossData(
event_loss_id=event_loss.id,
rupture_id=rupture_id,
aggregate_loss=event_loss_table[
rupture_id]))
inserter.flush()
aggregate_losses = [
event_loss_table[rupture_id]
for rupture_id in rupture_ids
if rupture_id in event_loss_table]
if aggregate_losses:
aggregate_loss = scientific.event_based(
aggregate_losses, tses=tses,
time_span=oq.investigation_time,
curve_resolution=oq.loss_curve_resolution)
models.AggregateLossCurveData.objects.create(
loss_curve=models.LossCurve.objects.create(
aggregate=True, insured=False,
hazard_output=hazard_output,
loss_type=loss_type,
output=models.Output.objects.create_output(
self.job,
"aggregate loss curves. "
"loss_type=%s hazard=%s" % (
loss_type, hazard_output),
"agg_loss_curve")),
losses=aggregate_loss[0],
poes=aggregate_loss[1],
average_loss=scientific.average_loss(
aggregate_loss),
stddev_loss=numpy.std(aggregate_losses))
def do_aggregate_post_proc(self):
"""
Grab hazard data for all realizations and sites from the database and
compute mean and/or quantile aggregates (depending on which options are
enabled in the calculation).
Post-processing results will be stored directly into the database.
"""
num_rlzs = len(self._realizations)
if not num_rlzs:
logs.LOG.warn('No realizations for hazard_calculation_id=%d',
self.job.id)
return
elif num_rlzs == 1 and self.quantile_hazard_curves:
logs.LOG.warn(
'There is only one realization, the configuration parameter '
'quantile_hazard_curves should not be set')
return
weights = (None if self.oqparam.number_of_logic_tree_samples else
[rlz.weight for rlz in self._realizations])
if self.oqparam.mean_hazard_curves:
# create a new `HazardCurve` 'container' record for mean
# curves (virtual container for multiple imts)
models.HazardCurve.objects.create(
output=models.Output.objects.create_output(
self.job, "mean-curves-multi-imt", "hazard_curve_multi"),
statistics="mean",
imt=None,
investigation_time=self.oqparam.investigation_time)
for quantile in self.quantile_hazard_curves:
# create a new `HazardCurve` 'container' record for quantile
# curves (virtual container for multiple imts)
models.HazardCurve.objects.create(
output=models.Output.objects.create_output(
self.job, 'quantile(%s)-curves' % quantile,
"hazard_curve_multi"),
statistics="quantile",
imt=None,
quantile=quantile,
investigation_time=self.oqparam.investigation_time)
for imt, imls in self.oqparam.imtls.items():
im_type, sa_period, sa_damping = from_string(imt)
# prepare `output` and `hazard_curve` containers in the DB:
container_ids = dict()
if self.oqparam.mean_hazard_curves:
mean_output = self.job.get_or_create_output(
display_name='Mean Hazard Curves %s' % imt,
output_type='hazard_curve')
mean_hc = models.HazardCurve.objects.create(
output=mean_output,
investigation_time=self.oqparam.investigation_time,
imt=im_type,
imls=imls,
sa_period=sa_period,
sa_damping=sa_damping,
statistics='mean')
self._hazard_curves.append(mean_hc)
container_ids['mean'] = mean_hc.id
for quantile in self.quantile_hazard_curves:
q_output = self.job.get_or_create_output(
display_name=('%s quantile Hazard Curves %s' %
(quantile, imt)),
output_type='hazard_curve')
q_hc = models.HazardCurve.objects.create(
output=q_output,
investigation_time=self.oqparam.investigation_time,
imt=im_type,
imls=imls,
sa_period=sa_period,
sa_damping=sa_damping,
statistics='quantile',
quantile=quantile)
self._hazard_curves.append(q_hc)
container_ids['q%s' % quantile] = q_hc.id
# num_rlzs * num_sites * num_levels
# NB: different IMTs can have different num_levels
all_curves_for_imt = numpy.array(self.curves_by_imt[imt])
del self.curves_by_imt[imt] # save memory
inserter = writer.CacheInserter(models.HazardCurveData,
max_cache_size=10000)
# curve_poes below is an array num_rlzs * num_levels
for i, site in enumerate(self.site_collection):
wkt = site.location.wkt2d
curve_poes = numpy.array(
[c_by_rlz[i] for c_by_rlz in all_curves_for_imt])
# calc quantiles first
for quantile in self.quantile_hazard_curves:
q_curve = scientific.quantile_curve(
curve_poes, quantile, weights)
inserter.add(
models.HazardCurveData(
hazard_curve_id=(container_ids['q%s' % quantile]),
poes=q_curve.tolist(),
location=wkt))
# then means
if self.mean_hazard_curves:
m_curve = scientific.mean_curve(curve_poes, weights)
inserter.add(
models.HazardCurveData(
hazard_curve_id=container_ids['mean'],
poes=m_curve.tolist(),
location=wkt))
inserter.flush()
import numpy
from openquake.hazardlib.imt import from_string
from openquake.hazardlib.geo.utils import get_spherical_bounding_box
from openquake.hazardlib.geo.utils import get_longitudinal_extent
from openquake.hazardlib.geo.geodetic import npoints_between
from openquake.commonlib.calculators.calc import gen_ruptures
from openquake.engine import writer
from openquake.engine.calculators import calculators
from openquake.engine.calculators.hazard import general
from openquake.engine.db import models
from openquake.engine.utils import tasks
inserter = writer.CacheInserter(models.SourceInfo, 100)
class BoundingBox(object):
"""
A class to store the bounding box in distances, longitudes and magnitudes,
given a source model and a site. This is used for disaggregation
calculations. The goal is to determine the minimum and maximum
distances of the ruptures generated from the model from the site;
moreover the maximum and minimum longitudes and magnitudes are stored, by
taking in account the international date line.
"""
def __init__(self, lt_model_id, site_id):
self.lt_model_id = lt_model_id
self.site_id = site_id
self.min_dist = self.max_dist = None
