def _update_curves(hc, matrices, lt_rlz, src_ids):
"""
Helper function for updating source, hazard curve, and realization progress
records in the database.
This is intended to be used by :func:`compute_hazard_curves`.
:param hc:
:class:`openquake.engine.db.models.HazardCalculation` instance.
:param lt_rlz:
:class:`openquake.engine.db.models.LtRealization` record for the
current realization.
:param src_ids:
List of source IDs considered for this calculation task.
"""
with logs.tracing('_update_curves for all IMTs'):
for imt in hc.intensity_measure_types_and_levels.keys():
with transaction.commit_on_success():
logs.LOG.debug('> updating hazard for IMT=%s' % imt)
hazardlib_imt = haz_general.imt_to_hazardlib(imt)
query = """
SELECT * FROM htemp.hazard_curve_progress
WHERE lt_realization_id = %s
AND imt = %s
FOR UPDATE"""
[hc_progress] = models.HazardCurveProgress.objects.raw(
query, [lt_rlz.id, imt])
hc_progress.result_matrix = update_result_matrix(
hc_progress.result_matrix, matrices[hazardlib_imt])
hc_progress.save()
logs.LOG.debug('< done updating hazard for IMT=%s' % imt)
with transaction.commit_on_success():
# Check here if any of records in source progress model
# with parsed_source_id from src_ids are marked as complete,
# and rollback and abort if there is at least one
src_prog = models.SourceProgress.objects.filter(
lt_realization=lt_rlz, parsed_source__in=src_ids)
if any(x.is_complete for x in src_prog):
msg = (
'One or more `source_progress` records were marked as '
'complete. This was unexpected and probably means that the'
' calculation workload was not distributed properly.'
)
logs.LOG.critical(msg)
transaction.rollback()
raise RuntimeError(msg)
# Mark source_progress records as complete
src_prog.update(is_complete=True)
# Update realiation progress,
# mark realization as complete if it is done
haz_general.update_realization(lt_rlz.id, len(src_ids))
def compute_disagg(job_id, sites, lt_rlz_id, ltp):
"""
Calculate disaggregation histograms and saving the results to the database.
Here is the basic calculation workflow:
1. Get all sources
2. Get IMTs
3. Get the hazard curve for each point, IMT, and realization
4. For each `poes_disagg`, interpolate the IML for each curve.
5. Get GSIMs, TOM (Temporal Occurence Model), and truncation level.
6. Get histogram bin edges.
7. Prepare calculation args.
8. Call the hazardlib calculator
(see :func:`openquake.hazardlib.calc.disagg.disaggregation`
for more info).
:param int job_id:
ID of the currently running :class:`openquake.engine.db.models.OqJob`
:param list sites:
`list` of :class:`openquake.hazardlib.site.Site` objects, which
indicate the locations (and associated soil parameters) for which we
need to compute disaggregation histograms.
:param int lt_rlz_id:
ID of the :class:`openquake.engine.db.models.LtRealization` for which
we want to compute disaggregation histograms. This realization will
determine which hazard curve results to use as a basis for the
calculation.
:param ltp:
a :class:`openquake.engine.input.LogicTreeProcessor` instance
"""
# Silencing 'Too many local variables'
# pylint: disable=R0914
logs.LOG.debug(
'> computing disaggregation for %(np)s sites for realization %(rlz)s'
% dict(np=len(sites), rlz=lt_rlz_id))
job = models.OqJob.objects.get(id=job_id)
hc = job.hazard_calculation
lt_rlz = models.LtRealization.objects.get(id=lt_rlz_id)
apply_uncertainties = ltp.parse_source_model_logictree_path(
lt_rlz.sm_lt_path)
gsims = ltp.parse_gmpe_logictree_path(lt_rlz.gsim_lt_path)
src_ids = models.SourceProgress.objects.filter(lt_realization=lt_rlz)\
.order_by('id').values_list('parsed_source_id', flat=True)
sources = [apply_uncertainties(s.nrml)
for s in models.ParsedSource.objects.filter(pk__in=src_ids)]
# Make filters for distance to source and distance to rupture:
# a better approach would be to filter the sources on distance
# before, see the comment in the classical calculator
src_site_filter = openquake.hazardlib.calc.filters.\
source_site_distance_filter(hc.maximum_distance)
rup_site_filter = openquake.hazardlib.calc.filters.\
rupture_site_distance_filter(hc.maximum_distance)
for imt, imls in hc.intensity_measure_types_and_levels.iteritems():
hazardlib_imt = haz_general.imt_to_hazardlib(imt)
hc_im_type, sa_period, sa_damping = models.parse_imt(imt)
imls = numpy.array(imls[::-1])
# loop over sites
for site in sites:
# get curve for this point/IMT/realization
[curve] = models.HazardCurveData.objects.filter(
location=site.location.wkt2d,
hazard_curve__lt_realization=lt_rlz_id,
hazard_curve__imt=hc_im_type,
hazard_curve__sa_period=sa_period,
hazard_curve__sa_damping=sa_damping,
)
# If the hazard curve is all zeros, don't even do the
# disagg calculation.
if all(x == 0.0 for x in curve.poes):
logs.LOG.debug(
'* hazard curve contained all 0 probability values; '
'skipping')
continue
for poe in hc.poes_disagg:
iml = numpy.interp(poe, curve.poes[::-1], imls)
calc_kwargs = {
'sources': sources,
'site': site,
'imt': hazardlib_imt,
'iml': iml,
'gsims': gsims,
'time_span': hc.investigation_time,
'truncation_level': hc.truncation_level,
'n_epsilons': hc.num_epsilon_bins,
'mag_bin_width': hc.mag_bin_width,
'dist_bin_width': hc.distance_bin_width,
'coord_bin_width': hc.coordinate_bin_width,
'source_site_filter': src_site_filter,
'rupture_site_filter': rup_site_filter,
}
with EnginePerformanceMonitor(
'computing disaggregation', job_id, disagg_task):
bin_edges, diss_matrix = openquake.hazardlib.calc.\
disagg.disaggregation_poissonian(**calc_kwargs)
if not bin_edges: # no ruptures generated
continue
with EnginePerformanceMonitor(
'saving disaggregation', job_id, disagg_task):
_save_disagg_matrix(
job, site, bin_edges, diss_matrix, lt_rlz,
hc.investigation_time, hc_im_type, iml, poe, sa_period,
sa_damping
)
with transaction.commit_on_success():
# Update realiation progress,
# mark realization as complete if it is done
haz_general.update_realization(lt_rlz_id, len(sites))
logs.LOG.debug('< done computing disaggregation')
def compute_hazard_curves(job_id, src_ids, lt_rlz_id):
"""
Celery task for hazard curve calculator.
Samples logic trees, gathers site parameters, and calls the hazard curve
calculator.
Once hazard curve data is computed, result progress updated (within a
transaction, to prevent race conditions) in the
`htemp.hazard_curve_progress` table.
Once all of this work is complete, a signal will be sent via AMQP to let
the control node know that the work is complete. (If there is any work left
to be dispatched, this signal will indicate to the control node that more
work can be enqueued.)
:param int job_id:
ID of the currently running job.
:param src_ids:
List of ids of parsed source models to take into account.
:param lt_rlz_id:
Id of logic tree realization model to calculate for.
"""
hc = models.HazardCalculation.objects.get(oqjob=job_id)
lt_rlz = models.LtRealization.objects.get(id=lt_rlz_id)
ltp = logictree.LogicTreeProcessor(hc.id)
apply_uncertainties = ltp.parse_source_model_logictree_path(
lt_rlz.sm_lt_path)
gsims = ltp.parse_gmpe_logictree_path(lt_rlz.gsim_lt_path)
sources = haz_general.gen_sources(
src_ids, apply_uncertainties, hc.rupture_mesh_spacing,
hc.width_of_mfd_bin, hc.area_source_discretization)
imts = haz_general.im_dict_to_hazardlib(
hc.intensity_measure_types_and_levels)
# Prepare args for the calculator.
calc_kwargs = {'gsims': gsims,
'truncation_level': hc.truncation_level,
'time_span': hc.investigation_time,
'sources': sources,
'imts': imts,
'sites': hc.site_collection}
if hc.maximum_distance:
dist = hc.maximum_distance
calc_kwargs['source_site_filter'] = (
openquake.hazardlib.calc.filters.source_site_distance_filter(dist))
calc_kwargs['rupture_site_filter'] = (
openquake.hazardlib.calc.filters.rupture_site_distance_filter(
dist))
# mapping "imt" to 2d array of hazard curves: first dimension -- sites,
# second -- IMLs
logs.LOG.debug('> computing hazard matrices')
matrices = openquake.hazardlib.calc.hazard_curve.hazard_curves_poissonian(
**calc_kwargs)
logs.LOG.debug('< done computing hazard matrices')
logs.LOG.debug('> starting transaction')
with transaction.commit_on_success():
logs.LOG.debug('looping over IMTs')
for imt in hc.intensity_measure_types_and_levels.keys():
logs.LOG.debug('> updating hazard for IMT=%s' % imt)
hazardlib_imt = haz_general.imt_to_hazardlib(imt)
query = """
SELECT * FROM htemp.hazard_curve_progress
WHERE lt_realization_id = %s
AND imt = %s
FOR UPDATE"""
[hc_progress] = models.HazardCurveProgress.objects.raw(
query, [lt_rlz.id, imt])
hc_progress.result_matrix = update_result_matrix(
hc_progress.result_matrix, matrices[hazardlib_imt])
hc_progress.save()
logs.LOG.debug('< done updating hazard for IMT=%s' % imt)
# Before the transaction completes:
# Check here if any of records in source progress model
# with parsed_source_id from src_ids are marked as complete,
# and rollback and abort if there is at least one
src_prog = models.SourceProgress.objects.filter(
lt_realization=lt_rlz, parsed_source__in=src_ids)
if any(x.is_complete for x in src_prog):
msg = (
'One or more `source_progress` records were marked as '
'complete. This was unexpected and probably means that the'
' calculation workload was not distributed properly.'
)
logs.LOG.critical(msg)
transaction.rollback()
raise RuntimeError(msg)
# Mark source_progress records as complete
src_prog.update(is_complete=True)
# Update realiation progress,
# mark realization as complete if it is done
haz_general.update_realization(lt_rlz.id, len(src_ids))
logs.LOG.debug('< transaction complete')
def _update_curves(hc, matrices, lt_rlz, src_ids):
"""
Helper function for updating source, hazard curve, and realization progress
records in the database.
This is intended to be used by :func:`compute_hazard_curves`.
:param hc:
:class:`openquake.engine.db.models.HazardCalculation` instance.
:param lt_rlz:
:class:`openquake.engine.db.models.LtRealization` record for the
current realization.
:param src_ids:
List of source IDs considered for this calculation task.
"""
with logs.tracing('_update_curves for all IMTs'):
for imt in hc.intensity_measure_types_and_levels.keys():
hazardlib_imt = haz_general.imt_to_hazardlib(imt)
matrix = matrices[hazardlib_imt]
if (matrix == 0.0).all():
# The matrix for this IMT is all zeros; there's no reason to
# update `hazard_curve_progress` records.
logs.LOG.debug('* No hazard contribution for IMT=%s' % imt)
continue
else:
# The is some contribution here to the hazard; we need to
# update.
with transaction.commit_on_success():
logs.LOG.debug('> updating hazard for IMT=%s' % imt)
query = """
SELECT * FROM htemp.hazard_curve_progress
WHERE lt_realization_id = %s
AND imt = %s
FOR UPDATE"""
[hc_progress] = models.HazardCurveProgress.objects.raw(
query, [lt_rlz.id, imt])
hc_progress.result_matrix = update_result_matrix(
hc_progress.result_matrix, matrix)
hc_progress.save()
logs.LOG.debug('< done updating hazard for IMT=%s' % imt)
with transaction.commit_on_success():
# Check here if any of records in source progress model
# with parsed_source_id from src_ids are marked as complete,
# and rollback and abort if there is at least one
src_prog = models.SourceProgress.objects.filter(
lt_realization=lt_rlz, parsed_source__in=src_ids)
if any(x.is_complete for x in src_prog):
msg = (
'One or more `source_progress` records were marked as '
'complete. This was unexpected and probably means that the'
' calculation workload was not distributed properly.')
logs.LOG.critical(msg)
transaction.rollback()
raise RuntimeError(msg)
# Mark source_progress records as complete
src_prog.update(is_complete=True)
# Update realiation progress,
# mark realization as complete if it is done
haz_general.update_realization(lt_rlz.id, len(src_ids))
def compute_disagg(job_id, sites, lt_rlz_id, ltp):
"""
Calculate disaggregation histograms and saving the results to the database.
Here is the basic calculation workflow:
1. Get all sources
2. Get IMTs
3. Get the hazard curve for each point, IMT, and realization
4. For each `poes_disagg`, interpolate the IML for each curve.
5. Get GSIMs, TOM (Temporal Occurence Model), and truncation level.
6. Get histogram bin edges.
7. Prepare calculation args.
8. Call the hazardlib calculator
(see :func:`openquake.hazardlib.calc.disagg.disaggregation`
for more info).
:param int job_id:
ID of the currently running :class:`openquake.engine.db.models.OqJob`
:param list sites:
`list` of :class:`openquake.hazardlib.site.Site` objects, which
indicate the locations (and associated soil parameters) for which we
need to compute disaggregation histograms.
:param int lt_rlz_id:
ID of the :class:`openquake.engine.db.models.LtRealization` for which
we want to compute disaggregation histograms. This realization will
determine which hazard curve results to use as a basis for the
calculation.
:param ltp:
a :class:`openquake.engine.input.LogicTreeProcessor` instance
"""
# Silencing 'Too many local variables'
# pylint: disable=R0914
logs.LOG.debug(
'> computing disaggregation for %(np)s sites for realization %(rlz)s' %
dict(np=len(sites), rlz=lt_rlz_id))
job = models.OqJob.objects.get(id=job_id)
hc = job.hazard_calculation
lt_rlz = models.LtRealization.objects.get(id=lt_rlz_id)
apply_uncertainties = ltp.parse_source_model_logictree_path(
lt_rlz.sm_lt_path)
gsims = ltp.parse_gmpe_logictree_path(lt_rlz.gsim_lt_path)
src_ids = models.SourceProgress.objects.filter(lt_realization=lt_rlz)\
.order_by('id').values_list('parsed_source_id', flat=True)
sources = [
apply_uncertainties(s.nrml)
for s in models.ParsedSource.objects.filter(pk__in=src_ids)
]
# Make filters for distance to source and distance to rupture:
# a better approach would be to filter the sources on distance
# before, see the comment in the classical calculator
src_site_filter = openquake.hazardlib.calc.filters.\
source_site_distance_filter(hc.maximum_distance)
rup_site_filter = openquake.hazardlib.calc.filters.\
rupture_site_distance_filter(hc.maximum_distance)
for imt, imls in hc.intensity_measure_types_and_levels.iteritems():
hazardlib_imt = haz_general.imt_to_hazardlib(imt)
hc_im_type, sa_period, sa_damping = models.parse_imt(imt)
imls = numpy.array(imls[::-1])
# loop over sites
for site in sites:
# get curve for this point/IMT/realization
[curve] = models.HazardCurveData.objects.filter(
location=site.location.wkt2d,
hazard_curve__lt_realization=lt_rlz_id,
hazard_curve__imt=hc_im_type,
hazard_curve__sa_period=sa_period,
hazard_curve__sa_damping=sa_damping,
)
# If the hazard curve is all zeros, don't even do the
# disagg calculation.
if all([x == 0.0 for x in curve.poes]):
logs.LOG.debug(
'* hazard curve contained all 0 probability values; '
'skipping')
continue
for poe in hc.poes_disagg:
iml = numpy.interp(poe, curve.poes[::-1], imls)
calc_kwargs = {
'sources': sources,
'site': site,
'imt': hazardlib_imt,
'iml': iml,
'gsims': gsims,
'time_span': hc.investigation_time,
'truncation_level': hc.truncation_level,
'n_epsilons': hc.num_epsilon_bins,
'mag_bin_width': hc.mag_bin_width,
'dist_bin_width': hc.distance_bin_width,
'coord_bin_width': hc.coordinate_bin_width,
'source_site_filter': src_site_filter,
'rupture_site_filter': rup_site_filter,
}
with EnginePerformanceMonitor('computing disaggregation',
job_id, disagg_task):
bin_edges, diss_matrix = openquake.hazardlib.calc.\
disagg.disaggregation_poissonian(**calc_kwargs)
if not bin_edges: # no ruptures generated
continue
with EnginePerformanceMonitor('saving disaggregation', job_id,
disagg_task):
_save_disagg_matrix(job, site, bin_edges, diss_matrix,
lt_rlz, hc.investigation_time,
hc_im_type, iml, poe, sa_period,
sa_damping)
with transaction.commit_on_success():
# Update realiation progress,
# mark realization as complete if it is done
haz_general.update_realization(lt_rlz_id, len(sites))
logs.LOG.debug('< done computing disaggregation')
