def test_im_dict_to_hazardlib(self):
imts_in = {
'PGA': [1, 2],
'PGV': [2, 3],
'PGD': [3, 4],
'SA(0.1)': [0.1, 0.2],
'SA(0.025)': [0.2, 0.3],
'IA': [0.3, 0.4],
'RSD': [0.4, 0.5],
'MMI': [0.5, 0.6],
}
expected = {
openquake.hazardlib.imt.PGA(): [1, 2],
openquake.hazardlib.imt.PGV(): [2, 3],
openquake.hazardlib.imt.PGD(): [3, 4],
openquake.hazardlib.imt.SA(0.1, models.DEFAULT_SA_DAMPING): [
0.1, 0.2],
openquake.hazardlib.imt.SA(0.025, models.DEFAULT_SA_DAMPING): [
0.2, 0.3],
openquake.hazardlib.imt.IA(): [0.3, 0.4],
openquake.hazardlib.imt.RSD(): [0.4, 0.5],
openquake.hazardlib.imt.MMI(): [0.5, 0.6],
}
actual = general.im_dict_to_hazardlib(imts_in)
self.assertEqual(len(expected), len(actual))
for exp_imt, exp_imls in expected.items():
act_imls = actual[exp_imt]
self.assertEqual(exp_imls, act_imls)
def test_im_dict_to_hazardlib(self):
imts_in = {
'PGA': [1, 2],
'PGV': [2, 3],
'PGD': [3, 4],
'SA(0.1)': [0.1, 0.2],
'SA(0.025)': [0.2, 0.3],
'IA': [0.3, 0.4],
'RSD': [0.4, 0.5],
'MMI': [0.5, 0.6],
}
expected = {
openquake.hazardlib.imt.PGA(): [1, 2],
openquake.hazardlib.imt.PGV(): [2, 3],
openquake.hazardlib.imt.PGD(): [3, 4],
openquake.hazardlib.imt.SA(0.1, models.DEFAULT_SA_DAMPING):
[0.1, 0.2],
openquake.hazardlib.imt.SA(0.025, models.DEFAULT_SA_DAMPING):
[0.2, 0.3],
openquake.hazardlib.imt.IA(): [0.3, 0.4],
openquake.hazardlib.imt.RSD(): [0.4, 0.5],
openquake.hazardlib.imt.MMI(): [0.5, 0.6],
}
actual = general.im_dict_to_hazardlib(imts_in)
self.assertEqual(len(expected), len(actual))
for exp_imt, exp_imls in expected.items():
act_imls = actual[exp_imt]
self.assertEqual(exp_imls, act_imls)
def compute_hazard_curves(job_id, src_ids, lt_rlz_id, ltp):
"""
Celery task for hazard curve calculator.
Samples logic trees, gathers site parameters, and calls the hazard curve
calculator.
Once hazard curve data is computed, result progress updated (within a
transaction, to prevent race conditions) in the
`htemp.hazard_curve_progress` table.
Once all of this work is complete, a signal will be sent via AMQP to let
the control node know that the work is complete. (If there is any work left
to be dispatched, this signal will indicate to the control node that more
work can be enqueued.)
:param int job_id:
ID of the currently running job.
:param src_ids:
List of ids of parsed source models to take into account.
:param lt_rlz_id:
Id of logic tree realization model to calculate for.
:param ltp:
a :class:`openquake.engine.input.LogicTreeProcessor` instance
"""
hc = models.HazardCalculation.objects.get(oqjob=job_id)
lt_rlz = models.LtRealization.objects.get(id=lt_rlz_id)
apply_uncertainties = ltp.parse_source_model_logictree_path(
lt_rlz.sm_lt_path)
gsims = ltp.parse_gmpe_logictree_path(lt_rlz.gsim_lt_path)
parsed_sources = models.ParsedSource.objects.filter(pk__in=src_ids)
imts = haz_general.im_dict_to_hazardlib(
hc.intensity_measure_types_and_levels)
# Prepare args for the calculator.
calc_kwargs = {'gsims': gsims,
'truncation_level': hc.truncation_level,
'time_span': hc.investigation_time,
'sources': [apply_uncertainties(s.nrml)
for s in parsed_sources],
'imts': imts,
'sites': hc.site_collection}
if hc.maximum_distance:
dist = hc.maximum_distance
# NB: a better approach could be to filter the sources by distance
# at the beginning and to sore into the database only the relevant
# sources, as we do in the event based calculator: I am not doing that
# for the classical calculators because I wonder about the performance
# impact in in SHARE-like calculations. So at the moment we store
# everything in the database and we filter on the workers. This
# will probably change in the future.
calc_kwargs['source_site_filter'] = (
openquake.hazardlib.calc.filters.source_site_distance_filter(dist))
calc_kwargs['rupture_site_filter'] = (
openquake.hazardlib.calc.filters.rupture_site_distance_filter(
dist))
# mapping "imt" to 2d array of hazard curves: first dimension -- sites,
# second -- IMLs
with EnginePerformanceMonitor(
'computing hazard curves', job_id, hazard_curves, tracing=True):
matrices = openquake.hazardlib.calc.hazard_curve.\
hazard_curves_poissonian(**calc_kwargs)
with EnginePerformanceMonitor(
'saving hazard curves', job_id, hazard_curves, tracing=True):
_update_curves(hc, matrices, lt_rlz, src_ids)
def compute_hazard_curves(job_id, src_ids, lt_rlz_id):
"""
Celery task for hazard curve calculator.
Samples logic trees, gathers site parameters, and calls the hazard curve
calculator.
Once hazard curve data is computed, result progress updated (within a
transaction, to prevent race conditions) in the
`htemp.hazard_curve_progress` table.
Once all of this work is complete, a signal will be sent via AMQP to let
the control node know that the work is complete. (If there is any work left
to be dispatched, this signal will indicate to the control node that more
work can be enqueued.)
:param int job_id:
ID of the currently running job.
:param src_ids:
List of ids of parsed source models to take into account.
:param lt_rlz_id:
Id of logic tree realization model to calculate for.
"""
hc = models.HazardCalculation.objects.get(oqjob=job_id)
lt_rlz = models.LtRealization.objects.get(id=lt_rlz_id)
ltp = logictree.LogicTreeProcessor(hc.id)
apply_uncertainties = ltp.parse_source_model_logictree_path(
lt_rlz.sm_lt_path)
gsims = ltp.parse_gmpe_logictree_path(lt_rlz.gsim_lt_path)
sources = haz_general.gen_sources(
src_ids, apply_uncertainties, hc.rupture_mesh_spacing,
hc.width_of_mfd_bin, hc.area_source_discretization)
imts = haz_general.im_dict_to_hazardlib(
hc.intensity_measure_types_and_levels)
# Prepare args for the calculator.
calc_kwargs = {'gsims': gsims,
'truncation_level': hc.truncation_level,
'time_span': hc.investigation_time,
'sources': sources,
'imts': imts,
'sites': hc.site_collection}
if hc.maximum_distance:
dist = hc.maximum_distance
calc_kwargs['source_site_filter'] = (
openquake.hazardlib.calc.filters.source_site_distance_filter(dist))
calc_kwargs['rupture_site_filter'] = (
openquake.hazardlib.calc.filters.rupture_site_distance_filter(
dist))
# mapping "imt" to 2d array of hazard curves: first dimension -- sites,
# second -- IMLs
with EnginePerformanceMonitor(
'computing hazard curves', job_id, hazard_curves, tracing=True):
matrices = openquake.hazardlib.calc.hazard_curve.\
hazard_curves_poissonian(**calc_kwargs)
with EnginePerformanceMonitor(
'saving hazard curves', job_id, hazard_curves, tracing=True):
_update_curves(hc, matrices, lt_rlz, src_ids)
def compute_hazard_curves(job_id, src_ids, lt_rlz_id):
"""
Celery task for hazard curve calculator.
Samples logic trees, gathers site parameters, and calls the hazard curve
calculator.
Once hazard curve data is computed, result progress updated (within a
transaction, to prevent race conditions) in the
`htemp.hazard_curve_progress` table.
Once all of this work is complete, a signal will be sent via AMQP to let
the control node know that the work is complete. (If there is any work left
to be dispatched, this signal will indicate to the control node that more
work can be enqueued.)
:param int job_id:
ID of the currently running job.
:param src_ids:
List of ids of parsed source models to take into account.
:param lt_rlz_id:
Id of logic tree realization model to calculate for.
"""
hc = models.HazardCalculation.objects.get(oqjob=job_id)
lt_rlz = models.LtRealization.objects.get(id=lt_rlz_id)
ltp = logictree.LogicTreeProcessor(hc.id)
apply_uncertainties = ltp.parse_source_model_logictree_path(
lt_rlz.sm_lt_path)
gsims = ltp.parse_gmpe_logictree_path(lt_rlz.gsim_lt_path)
sources = haz_general.gen_sources(
src_ids, apply_uncertainties, hc.rupture_mesh_spacing,
hc.width_of_mfd_bin, hc.area_source_discretization)
imts = haz_general.im_dict_to_hazardlib(
hc.intensity_measure_types_and_levels)
# Prepare args for the calculator.
calc_kwargs = {'gsims': gsims,
'truncation_level': hc.truncation_level,
'time_span': hc.investigation_time,
'sources': sources,
'imts': imts,
'sites': hc.site_collection}
if hc.maximum_distance:
dist = hc.maximum_distance
calc_kwargs['source_site_filter'] = (
openquake.hazardlib.calc.filters.source_site_distance_filter(dist))
calc_kwargs['rupture_site_filter'] = (
openquake.hazardlib.calc.filters.rupture_site_distance_filter(
dist))
# mapping "imt" to 2d array of hazard curves: first dimension -- sites,
# second -- IMLs
logs.LOG.debug('> computing hazard matrices')
matrices = openquake.hazardlib.calc.hazard_curve.hazard_curves_poissonian(
**calc_kwargs)
logs.LOG.debug('< done computing hazard matrices')
logs.LOG.debug('> starting transaction')
with transaction.commit_on_success():
logs.LOG.debug('looping over IMTs')
for imt in hc.intensity_measure_types_and_levels.keys():
logs.LOG.debug('> updating hazard for IMT=%s' % imt)
hazardlib_imt = haz_general.imt_to_hazardlib(imt)
query = """
SELECT * FROM htemp.hazard_curve_progress
WHERE lt_realization_id = %s
AND imt = %s
FOR UPDATE"""
[hc_progress] = models.HazardCurveProgress.objects.raw(
query, [lt_rlz.id, imt])
hc_progress.result_matrix = update_result_matrix(
hc_progress.result_matrix, matrices[hazardlib_imt])
hc_progress.save()
logs.LOG.debug('< done updating hazard for IMT=%s' % imt)
# Before the transaction completes:
# Check here if any of records in source progress model
# with parsed_source_id from src_ids are marked as complete,
# and rollback and abort if there is at least one
src_prog = models.SourceProgress.objects.filter(
lt_realization=lt_rlz, parsed_source__in=src_ids)
if any(x.is_complete for x in src_prog):
msg = (
'One or more `source_progress` records were marked as '
'complete. This was unexpected and probably means that the'
' calculation workload was not distributed properly.'
)
logs.LOG.critical(msg)
transaction.rollback()
raise RuntimeError(msg)
# Mark source_progress records as complete
src_prog.update(is_complete=True)
# Update realiation progress,
# mark realization as complete if it is done
haz_general.update_realization(lt_rlz.id, len(src_ids))
logs.LOG.debug('< transaction complete')
def compute_hazard_curves(job_id, src_ids, lt_rlz_id, ltp):
"""
Celery task for hazard curve calculator.
Samples logic trees, gathers site parameters, and calls the hazard curve
calculator.
Once hazard curve data is computed, result progress updated (within a
transaction, to prevent race conditions) in the
`htemp.hazard_curve_progress` table.
Once all of this work is complete, a signal will be sent via AMQP to let
the control node know that the work is complete. (If there is any work left
to be dispatched, this signal will indicate to the control node that more
work can be enqueued.)
:param int job_id:
ID of the currently running job.
:param src_ids:
List of ids of parsed source models to take into account.
:param lt_rlz_id:
Id of logic tree realization model to calculate for.
:param ltp:
a :class:`openquake.engine.input.LogicTreeProcessor` instance
"""
hc = models.HazardCalculation.objects.get(oqjob=job_id)
lt_rlz = models.LtRealization.objects.get(id=lt_rlz_id)
apply_uncertainties = ltp.parse_source_model_logictree_path(
lt_rlz.sm_lt_path)
gsims = ltp.parse_gmpe_logictree_path(lt_rlz.gsim_lt_path)
parsed_sources = models.ParsedSource.objects.filter(pk__in=src_ids)
imts = haz_general.im_dict_to_hazardlib(
hc.intensity_measure_types_and_levels)
# Prepare args for the calculator.
calc_kwargs = {
'gsims': gsims,
'truncation_level': hc.truncation_level,
'time_span': hc.investigation_time,
'sources': [apply_uncertainties(s.nrml) for s in parsed_sources],
'imts': imts,
'sites': hc.site_collection
}
if hc.maximum_distance:
dist = hc.maximum_distance
# NB: a better approach could be to filter the sources by distance
# at the beginning and to sore into the database only the relevant
# sources, as we do in the event based calculator: I am not doing that
# for the classical calculators because I wonder about the performance
# impact in in SHARE-like calculations. So at the moment we store
# everything in the database and we filter on the workers. This
# will probably change in the future.
calc_kwargs['source_site_filter'] = (
openquake.hazardlib.calc.filters.source_site_distance_filter(dist))
calc_kwargs['rupture_site_filter'] = (
openquake.hazardlib.calc.filters.rupture_site_distance_filter(dist)
)
# mapping "imt" to 2d array of hazard curves: first dimension -- sites,
# second -- IMLs
with EnginePerformanceMonitor('computing hazard curves',
job_id,
hazard_curves,
tracing=True):
matrices = openquake.hazardlib.calc.hazard_curve.\
hazard_curves_poissonian(**calc_kwargs)
with EnginePerformanceMonitor('saving hazard curves',
job_id,
hazard_curves,
tracing=True):
_update_curves(hc, matrices, lt_rlz, src_ids)
def compute_hazard_curves(
job_id, sitecol, sources, lt_model, gsim_by_rlz, task_no):
"""
This task computes R2 * I hazard curves (each one is a
numpy array of S * L floats) from the given source_ruptures
pairs.
:param job_id:
ID of the currently running job
:param sitecol:
a :class:`openquake.hazardlib.site.SiteCollection` instance
:param sources:
a block of source objects
:param lt_model:
a :class:`openquake.engine.db.LtSourceModel` instance
:param gsim_by_rlz:
a dictionary of gsims, one for each realization
:param int task_no:
the ordinal number of the current task
"""
hc = models.HazardCalculation.objects.get(oqjob=job_id)
total_sites = len(sitecol)
sitemesh = sitecol.mesh
imts = general.im_dict_to_hazardlib(
hc.intensity_measure_types_and_levels)
curves = dict((rlz, dict((imt, numpy.ones([total_sites, len(imts[imt])]))
for imt in imts))
for rlz in gsim_by_rlz)
if hc.poes_disagg: # doing disaggregation
bbs = [BoundingBox(lt_model.id, site_id) for site_id in sitecol.sids]
else:
bbs = []
mon = LightMonitor(
'getting ruptures', job_id, compute_hazard_curves)
make_ctxt_mon = LightMonitor(
'making contexts', job_id, compute_hazard_curves)
calc_poes_mon = LightMonitor(
'computing poes', job_id, compute_hazard_curves)
# NB: rows are a namedtuples with fields (source, rupture, rupture_sites)
for source, rows in itertools.groupby(
hc.gen_ruptures(sources, mon, sitecol),
key=operator.attrgetter('source')):
t0 = time.time()
num_ruptures = 0
for _source, rupture, r_sites in rows:
num_ruptures += 1
if hc.poes_disagg: # doing disaggregation
jb_dists = rupture.surface.get_joyner_boore_distance(sitemesh)
closest_points = rupture.surface.get_closest_points(sitemesh)
for bb, dist, point in zip(bbs, jb_dists, closest_points):
if dist < hc.maximum_distance:
# ruptures too far away are ignored
bb.update([dist], [point.longitude], [point.latitude])
# compute probabilities for all realizations
for rlz, curv in curves.iteritems():
gsim = gsim_by_rlz[rlz]
with make_ctxt_mon:
sctx, rctx, dctx = gsim.make_contexts(r_sites, rupture)
with calc_poes_mon:
for imt in imts:
poes = gsim.get_poes(sctx, rctx, dctx, imt, imts[imt],
hc.truncation_level)
pno = rupture.get_probability_no_exceedance(poes)
curv[imt] *= r_sites.expand(pno, placeholder=1)
logs.LOG.info('job=%d, src=%s:%s, num_ruptures=%d, calc_time=%fs',
job_id, source.source_id, source.__class__.__name__,
num_ruptures, time.time() - t0)
make_ctxt_mon.flush()
calc_poes_mon.flush()
# the 0 here is a shortcut for filtered sources giving no contribution;
# this is essential for performance, we want to avoid returning
# big arrays of zeros (MS)
curve_dict = dict((rlz, [0 if (curv[imt] == 1.0).all() else 1. - curv[imt]
for imt in sorted(imts)])
for rlz, curv in curves.iteritems())
return curve_dict, bbs
