def test_weighted_quantile_with_decimal_weights(self):
# NOTE(LB): This is a test for a bug I found.
# In the case of end-branch enumeration with _more_ than 1 branch,
# numpy.interp (used in `quantile_curves_weighted`) cannot handle the
# `weights` input properly. `weights` is passed as a list of
# `decimal.Decimal` types. Numpy throws back this error:
# TypeError: array cannot be safely cast to required type
# This doesn't appear to be a problem when there is only a single end
# branch in the logic tree (and so the single weight is
# decimal.Decimal(1.0)).
# This test ensures that `weighted_quantile_curve` works when weights
# are passed as `Decimal` types.
expected_curve = numpy.array([0.89556, 0.83045, 0.73646])
quantile = 0.3
curves = [
[9.2439e-01, 8.6700e-01, 7.7785e-01],
[8.9556e-01, 8.3045e-01, 7.3646e-01],
[9.1873e-01, 8.6697e-01, 7.8992e-01],
]
weights = [decimal.Decimal(x) for x in ('0.2', '0.3', '0.5')]
actual_curve = post_processing.weighted_quantile_curve(
curves, weights, quantile)
numpy.testing.assert_allclose(expected_curve, actual_curve)
def asset_statistics(losses, curves_poes, quantiles, weights, poes):
"""
Compute output statistics (mean/quantile loss curves and maps)
for a single asset
:param losses:
the losses on which the loss curves are defined
:param curves_poes:
a numpy matrix suitable to be used with
:func:`openquake.engine.calculators.post_processing`
:param list quantiles:
an iterable over the quantile levels to be considered for
quantile outputs
:param list weights:
the weights associated with each realization. If all the elements are
`None`, implicit weights are taken into account
:param list poes:
the poe taken into account for computing loss maps
:returns:
a tuple with
1) mean loss curve
2) a list of quantile curves
"""
montecarlo = weights[0] is not None
quantile_curves = []
for quantile in quantiles:
if montecarlo:
q_curve = post_processing.weighted_quantile_curve(
curves_poes, weights, quantile)
else:
q_curve = post_processing.quantile_curve(curves_poes, quantile)
quantile_curves.append((losses, q_curve))
# then mean loss curve
mean_curve_poes = post_processing.mean_curve(curves_poes, weights)
mean_curve = (losses, mean_curve_poes)
mean_map = [
scientific.conditional_loss_ratio(losses, mean_curve_poes, poe)
for poe in poes
]
quantile_maps = [[
scientific.conditional_loss_ratio(losses, poes, poe)
for losses, poes in quantile_curves
] for poe in poes]
return (mean_curve, quantile_curves, mean_map, quantile_maps)
def test_compute_weighted_quantile_curve_case2(self):
expected_curve = numpy.array([0.89556, 0.83045, 0.73646])
quantile = 0.3
curves = [
[9.2439e-01, 8.6700e-01, 7.7785e-01],
[8.9556e-01, 8.3045e-01, 7.3646e-01],
[9.1873e-01, 8.6697e-01, 7.8992e-01],
]
weights = [0.2, 0.3, 0.5]
actual_curve = post_processing.weighted_quantile_curve(curves, weights, quantile)
numpy.testing.assert_allclose(expected_curve, actual_curve)
def test_compute_weighted_quantile_curve_case1(self):
expected_curve = numpy.array([0.69909, 0.60859, 0.50328])
quantile = 0.3
curves = [
[9.9996e-01, 9.9962e-01, 9.9674e-01],
[6.9909e-01, 6.0859e-01, 5.0328e-01],
[1.0000e00, 9.9996e-01, 9.9947e-01],
]
weights = [0.5, 0.3, 0.2]
actual_curve = post_processing.weighted_quantile_curve(curves, weights, quantile)
numpy.testing.assert_allclose(expected_curve, actual_curve)
def curve_statistics(asset, loss_ratio_curves, curves_weights,
mean_loss_curve_id, quantile_loss_curve_ids,
explicit_quantiles, assume_equal):
if assume_equal == 'support':
loss_ratios = loss_ratio_curves[0].abscissae
curves_poes = [curve.ordinates for curve in loss_ratio_curves]
elif assume_equal == 'image':
loss_ratios = loss_ratio_curves[0].abscissae
curves_poes = [curve.ordinate_for(loss_ratios)
for curve in loss_ratio_curves]
else:
raise NotImplementedError
for quantile, quantile_loss_curve_id in quantile_loss_curve_ids.items():
if explicit_quantiles:
q_curve = post_processing.weighted_quantile_curve(
curves_poes, curves_weights, quantile)
else:
q_curve = post_processing.quantile_curve(
curves_poes, quantile)
models.LossCurveData.objects.create(
loss_curve_id=quantile_loss_curve_id,
asset_ref=asset.asset_ref,
poes=q_curve.tolist(),
loss_ratios=loss_ratios,
asset_value=asset.value,
location=asset.site.wkt)
# then means
if mean_loss_curve_id:
mean_curve = post_processing.mean_curve(
curves_poes, weights=curves_weights)
models.LossCurveData.objects.create(
loss_curve_id=mean_loss_curve_id,
asset_ref=asset.asset_ref,
poes=mean_curve.tolist(),
loss_ratios=loss_ratios,
asset_value=asset.value,
location=asset.site.wkt)
def do_aggregate_post_proc(self):
"""
Grab hazard data for all realizations and sites from the database and
compute mean and/or quantile aggregates (depending on which options are
enabled in the calculation).
Post-processing results will be stored directly into the database.
"""
num_rlzs = models.LtRealization.objects.filter(
lt_model__hazard_calculation=self.hc).count()
num_site_blocks_per_incr = int(CURVE_CACHE_SIZE) / int(num_rlzs)
if num_site_blocks_per_incr == 0:
# This means we have `num_rlzs` >= `CURVE_CACHE_SIZE`.
# The minimum number of sites should be 1.
num_site_blocks_per_incr = 1
slice_incr = num_site_blocks_per_incr * num_rlzs # unit: num records
if self.hc.mean_hazard_curves:
# create a new `HazardCurve` 'container' record for mean
# curves (virtual container for multiple imts)
models.HazardCurve.objects.create(
output=models.Output.objects.create_output(
self.job, "mean-curves-multi-imt",
"hazard_curve_multi"),
statistics="mean",
imt=None,
investigation_time=self.hc.investigation_time)
if self.hc.quantile_hazard_curves:
for quantile in self.hc.quantile_hazard_curves:
# create a new `HazardCurve` 'container' record for quantile
# curves (virtual container for multiple imts)
models.HazardCurve.objects.create(
output=models.Output.objects.create_output(
self.job, 'quantile(%s)-curves' % quantile,
"hazard_curve_multi"),
statistics="quantile",
imt=None,
quantile=quantile,
investigation_time=self.hc.investigation_time)
for imt, imls in self.hc.intensity_measure_types_and_levels.items():
im_type, sa_period, sa_damping = from_string(imt)
# prepare `output` and `hazard_curve` containers in the DB:
container_ids = dict()
if self.hc.mean_hazard_curves:
mean_output = models.Output.objects.create_output(
job=self.job,
display_name='Mean Hazard Curves %s' % imt,
output_type='hazard_curve'
)
mean_hc = models.HazardCurve.objects.create(
output=mean_output,
investigation_time=self.hc.investigation_time,
imt=im_type,
imls=imls,
sa_period=sa_period,
sa_damping=sa_damping,
statistics='mean'
)
container_ids['mean'] = mean_hc.id
if self.hc.quantile_hazard_curves:
for quantile in self.hc.quantile_hazard_curves:
q_output = models.Output.objects.create_output(
job=self.job,
display_name=(
'%s quantile Hazard Curves %s' % (quantile, imt)
),
output_type='hazard_curve'
)
q_hc = models.HazardCurve.objects.create(
output=q_output,
investigation_time=self.hc.investigation_time,
imt=im_type,
imls=imls,
sa_period=sa_period,
sa_damping=sa_damping,
statistics='quantile',
quantile=quantile
)
container_ids['q%s' % quantile] = q_hc.id
all_curves_for_imt = models.order_by_location(
models.HazardCurveData.objects.all_curves_for_imt(
self.job.id, im_type, sa_period, sa_damping))
with transaction.commit_on_success(using='job_init'):
inserter = writer.CacheInserter(
models.HazardCurveData, CURVE_CACHE_SIZE)
for chunk in models.queryset_iter(all_curves_for_imt,
slice_incr):
# slice each chunk by `num_rlzs` into `site_chunk`
# and compute the aggregate
for site_chunk in block_splitter(chunk, num_rlzs):
site = site_chunk[0].location
curves_poes = [x.poes for x in site_chunk]
curves_weights = [x.weight for x in site_chunk]
# do means and quantiles
# quantiles first:
if self.hc.quantile_hazard_curves:
for quantile in self.hc.quantile_hazard_curves:
if self.hc.number_of_logic_tree_samples == 0:
# explicitly weighted quantiles
q_curve = weighted_quantile_curve(
curves_poes, curves_weights, quantile
)
else:
# implicitly weighted quantiles
q_curve = quantile_curve(
curves_poes, quantile
)
inserter.add(
models.HazardCurveData(
hazard_curve_id=(
container_ids['q%s' % quantile]),
poes=q_curve.tolist(),
location=site.wkt)
)
# then means
if self.hc.mean_hazard_curves:
m_curve = mean_curve(
curves_poes, weights=curves_weights
)
inserter.add(
models.HazardCurveData(
hazard_curve_id=container_ids['mean'],
poes=m_curve.tolist(),
location=site.wkt)
)
inserter.flush()
def do_aggregate_post_proc(self):
"""
Grab hazard data for all realizations and sites from the database and
compute mean and/or quantile aggregates (depending on which options are
enabled in the calculation).
Post-processing results will be stored directly into the database.
"""
del self.source_collector # save memory
weights = [rlz.weight for rlz in models.LtRealization.objects.filter(
lt_model__hazard_calculation=self.hc)]
num_rlzs = len(weights)
if not num_rlzs:
logs.LOG.warn('No realizations for hazard_calculation_id=%d',
self.hc.id)
return
elif num_rlzs == 1 and self.hc.quantile_hazard_curves:
logs.LOG.warn(
'There is only one realization, the configuration parameter '
'quantile_hazard_curves should not be set')
return
if self.hc.mean_hazard_curves:
# create a new `HazardCurve` 'container' record for mean
# curves (virtual container for multiple imts)
models.HazardCurve.objects.create(
output=models.Output.objects.create_output(
self.job, "mean-curves-multi-imt",
"hazard_curve_multi"),
statistics="mean",
imt=None,
investigation_time=self.hc.investigation_time)
if self.hc.quantile_hazard_curves:
for quantile in self.hc.quantile_hazard_curves:
# create a new `HazardCurve` 'container' record for quantile
# curves (virtual container for multiple imts)
models.HazardCurve.objects.create(
output=models.Output.objects.create_output(
self.job, 'quantile(%s)-curves' % quantile,
"hazard_curve_multi"),
statistics="quantile",
imt=None,
quantile=quantile,
investigation_time=self.hc.investigation_time)
for imt, imls in self.hc.intensity_measure_types_and_levels.items():
im_type, sa_period, sa_damping = from_string(imt)
# prepare `output` and `hazard_curve` containers in the DB:
container_ids = dict()
if self.hc.mean_hazard_curves:
mean_output = models.Output.objects.create_output(
job=self.job,
display_name='Mean Hazard Curves %s' % imt,
output_type='hazard_curve'
)
mean_hc = models.HazardCurve.objects.create(
output=mean_output,
investigation_time=self.hc.investigation_time,
imt=im_type,
imls=imls,
sa_period=sa_period,
sa_damping=sa_damping,
statistics='mean'
)
container_ids['mean'] = mean_hc.id
if self.hc.quantile_hazard_curves:
for quantile in self.hc.quantile_hazard_curves:
q_output = models.Output.objects.create_output(
job=self.job,
display_name=(
'%s quantile Hazard Curves %s' % (quantile, imt)
),
output_type='hazard_curve'
)
q_hc = models.HazardCurve.objects.create(
output=q_output,
investigation_time=self.hc.investigation_time,
imt=im_type,
imls=imls,
sa_period=sa_period,
sa_damping=sa_damping,
statistics='quantile',
quantile=quantile
)
container_ids['q%s' % quantile] = q_hc.id
# num_rlzs * num_sites * num_levels
# NB: different IMTs can have different num_levels
all_curves_for_imt = numpy.array(self.curves_by_imt[imt])
del self.curves_by_imt[imt] # save memory
inserter = writer.CacheInserter(
models.HazardCurveData, max_cache_size=10000)
# curve_poes below is an array num_rlzs * num_levels
for i, site in enumerate(self.hc.site_collection):
wkt = site.location.wkt2d
curve_poes = numpy.array(
[c_by_rlz[i] for c_by_rlz in all_curves_for_imt])
# do means and quantiles
# quantiles first:
if self.hc.quantile_hazard_curves:
for quantile in self.hc.quantile_hazard_curves:
if self.hc.number_of_logic_tree_samples == 0:
# explicitly weighted quantiles
q_curve = weighted_quantile_curve(
curve_poes, weights, quantile)
else:
# implicitly weighted quantiles
q_curve = quantile_curve(
curve_poes, quantile)
inserter.add(
models.HazardCurveData(
hazard_curve_id=(
container_ids['q%s' % quantile]),
poes=q_curve.tolist(),
location=wkt)
)
# then means
if self.hc.mean_hazard_curves:
m_curve = mean_curve(curve_poes, weights=weights)
inserter.add(
models.HazardCurveData(
hazard_curve_id=container_ids['mean'],
poes=m_curve.tolist(),
location=wkt)
)
inserter.flush()
def do_aggregate_post_proc(self):
"""
Grab hazard data for all realizations and sites from the database and
compute mean and/or quantile aggregates (depending on which options are
enabled in the calculation).
Post-processing results will be stored directly into the database.
"""
num_rlzs = models.LtRealization.objects.filter(
hazard_calculation=self.hc).count()
num_site_blocks_per_incr = int(CURVE_CACHE_SIZE) / int(num_rlzs)
if num_site_blocks_per_incr == 0:
# This means we have `num_rlzs` >= `CURVE_CACHE_SIZE`.
# The minimum number of sites should be 1.
num_site_blocks_per_incr = 1
slice_incr = num_site_blocks_per_incr * num_rlzs # unit: num records
if self.hc.mean_hazard_curves:
# create a new `HazardCurve` 'container' record for mean
# curves (virtual container for multiple imts)
models.HazardCurve.objects.create(
output=models.Output.objects.create_output(
self.job, "mean-curves-multi-imt", "hazard_curve_multi"),
statistics="mean",
imt=None,
investigation_time=self.hc.investigation_time)
if self.hc.quantile_hazard_curves:
for quantile in self.hc.quantile_hazard_curves:
# create a new `HazardCurve` 'container' record for quantile
# curves (virtual container for multiple imts)
models.HazardCurve.objects.create(
output=models.Output.objects.create_output(
self.job, 'quantile(%s)-curves' % quantile,
"hazard_curve_multi"),
statistics="quantile",
imt=None,
quantile=quantile,
investigation_time=self.hc.investigation_time)
for imt, imls in self.hc.intensity_measure_types_and_levels.items():
im_type, sa_period, sa_damping = models.parse_imt(imt)
# prepare `output` and `hazard_curve` containers in the DB:
container_ids = dict()
if self.hc.mean_hazard_curves:
mean_output = models.Output.objects.create_output(
job=self.job,
display_name='mean-curves-%s' % imt,
output_type='hazard_curve')
mean_hc = models.HazardCurve.objects.create(
output=mean_output,
investigation_time=self.hc.investigation_time,
imt=im_type,
imls=imls,
sa_period=sa_period,
sa_damping=sa_damping,
statistics='mean')
container_ids['mean'] = mean_hc.id
if self.hc.quantile_hazard_curves:
for quantile in self.hc.quantile_hazard_curves:
q_output = models.Output.objects.create_output(
job=self.job,
display_name=('quantile(%s)-curves-%s' %
(quantile, imt)),
output_type='hazard_curve')
q_hc = models.HazardCurve.objects.create(
output=q_output,
investigation_time=self.hc.investigation_time,
imt=im_type,
imls=imls,
sa_period=sa_period,
sa_damping=sa_damping,
statistics='quantile',
quantile=quantile)
container_ids['q%s' % quantile] = q_hc.id
all_curves_for_imt = models.order_by_location(
models.HazardCurveData.objects.all_curves_for_imt(
self.job.id, im_type, sa_period, sa_damping))
with transaction.commit_on_success(using='reslt_writer'):
inserter = writer.CacheInserter(models.HazardCurveData,
CURVE_CACHE_SIZE)
for chunk in models.queryset_iter(all_curves_for_imt,
slice_incr):
# slice each chunk by `num_rlzs` into `site_chunk`
# and compute the aggregate
for site_chunk in block_splitter(chunk, num_rlzs):
site = site_chunk[0].location
curves_poes = [x.poes for x in site_chunk]
curves_weights = [x.weight for x in site_chunk]
# do means and quantiles
# quantiles first:
if self.hc.quantile_hazard_curves:
for quantile in self.hc.quantile_hazard_curves:
if self.hc.number_of_logic_tree_samples == 0:
# explicitly weighted quantiles
q_curve = weighted_quantile_curve(
curves_poes, curves_weights, quantile)
else:
# implicitly weighted quantiles
q_curve = quantile_curve(
curves_poes, quantile)
inserter.add(
models.HazardCurveData(
hazard_curve_id=(
container_ids['q%s' % quantile]),
poes=q_curve.tolist(),
location=site.wkt))
# then means
if self.hc.mean_hazard_curves:
m_curve = mean_curve(curves_poes,
weights=curves_weights)
inserter.add(
models.HazardCurveData(
hazard_curve_id=container_ids['mean'],
poes=m_curve.tolist(),
location=site.wkt))
inserter.flush()