def test_get_list_json_decoded(self):
data = [{u'1': u'one'}, {u'2': u'two'}, {u'3': u'three'}]
for item in data:
kvs.get_client().rpush(TEST_KEY, json.dumps(item))
self.assertEqual(data, kvs.get_list_json_decoded(TEST_KEY))
def test_get_list_json_decoded(self):
data = [{u'1': u'one'}, {u'2': u'two'}, {u'3': u'three'}]
for item in data:
kvs.get_client().rpush(TEST_KEY, json.dumps(item))
self.assertEqual(data, kvs.get_list_json_decoded(TEST_KEY))
def compute_risk(self, block_id, **kwargs): # pylint: disable=W0613
"""This task computes risk for a block of sites. It requires to have
pre-initialized in kvs:
1) list of sites
2) exposure portfolio (=assets)
3) vulnerability
"""
block = general.Block.from_kvs(block_id)
#pylint: disable=W0201
self.vuln_curves = \
vulnerability.load_vuln_model_from_kvs(self.job_id)
for point in block.grid(self.region):
hazard_curve = self._get_db_curve(point.site)
asset_key = kvs.tokens.asset_key(self.job_id,
point.row, point.column)
for asset in kvs.get_list_json_decoded(asset_key):
LOGGER.debug("processing asset %s" % (asset))
loss_ratio_curve = self.compute_loss_ratio_curve(
point, asset, hazard_curve)
self.compute_loss_curve(point, loss_ratio_curve, asset)
return True
def _compute_loss(self, block_id):
"""
Calculate and store in the kvs the loss data.
"""
block = general.Block.from_kvs(self.calc_proxy.job_id, block_id)
vuln_curves = vulnerability.load_vuln_model_from_kvs(
self.calc_proxy.job_id)
for point in block.grid(self.calc_proxy.region):
hazard_curve = self._get_db_curve(point.site)
asset_key = kvs.tokens.asset_key(self.calc_proxy.job_id,
point.row, point.column)
for asset in kvs.get_list_json_decoded(asset_key):
LOGGER.debug("processing asset %s" % (asset))
loss_ratio_curve = self.compute_loss_ratio_curve(
point, asset, hazard_curve, vuln_curves)
if loss_ratio_curve:
loss_curve = self.compute_loss_curve(point,
loss_ratio_curve, asset)
for loss_poe in conditional_loss_poes(
self.calc_proxy.params):
compute_conditional_loss(
self.calc_proxy.job_id, point.column,
point.row, loss_curve, asset, loss_poe)
return True
def compute_bcr_for_block(job_id, points, get_loss_curve,
interest_rate, asset_life_expectancy):
"""
Compute and return Benefit-Cost Ratio data for a number of points.
:param get_loss_curve:
Function that takes three positional arguments: point object,
vulnerability function object and asset object and is supposed
to return a loss curve.
:return:
A list of tuples::
[((site_lat, site_lon), [
({'bcr': 1, 'eal_retrofitted': 2, 'eal_original': 3}, assetID),
({'bcr': 3, 'eal_retrofitted': 4, 'eal_original': 5}, assetID),
...]),
...]
"""
# too many local vars (16/15) -- pylint: disable=R0914
result = defaultdict(list)
vuln_curves = vulnerability.load_vuln_model_from_kvs(job_id)
vuln_curves_retrofitted = vulnerability.load_vuln_model_from_kvs(
job_id, retrofitted=True)
for point in points:
asset_key = kvs.tokens.asset_key(job_id, point.row, point.column)
for asset in kvs.get_list_json_decoded(asset_key):
vuln_function = vuln_curves[asset['taxonomy']]
loss_curve = get_loss_curve(point, vuln_function, asset)
LOG.info('for asset %s loss_curve = %s',
asset['assetID'], loss_curve)
eal_original = compute_mean_loss(loss_curve)
vuln_function = vuln_curves_retrofitted[asset['taxonomy']]
loss_curve = get_loss_curve(point, vuln_function, asset)
LOG.info('for asset %s loss_curve retrofitted = %s',
asset['assetID'], loss_curve)
eal_retrofitted = compute_mean_loss(loss_curve)
bcr = compute_bcr(
eal_original, eal_retrofitted,
interest_rate, asset_life_expectancy,
asset['retrofittingCost']
)
LOG.info('for asset %s EAL original = %f, '
'EAL retrofitted = %f, BCR = %f',
asset['assetID'], eal_original, eal_retrofitted, bcr)
key = (asset['lat'], asset['lon'])
result[key].append(({'bcr': bcr,
'eal_original': eal_original,
'eal_retrofitted': eal_retrofitted},
asset['assetID']))
return result.items()
def load_assets_for_point(job_id, point):
"""
From the KVS, load all assets for the given point.
:param point: :py:class:`openquake.shapes.GridPoint` object
:returns: List of asset dicts at the given location (point) in the
following form::
{u'assetValue': 124.27, u'taxonomy': u'ID'}
"""
assets_key = kvs.tokens.asset_key(job_id, point.row, point.column)
return kvs.get_list_json_decoded(assets_key)
def _compute_loss(self, block_id):
"""Compute risk for a block of sites, that means:
* loss ratio curves
* loss curves
* conditional losses
* (partial) aggregate loss curve
"""
self.slice_gmfs(block_id)
self.vuln_curves = vulnerability.load_vuln_model_from_kvs(
self.calc_proxy.job_id)
block = general.Block.from_kvs(self.calc_proxy.job_id, block_id)
# aggregate the losses for this block
aggregate_curve = general.AggregateLossCurve()
for point in block.grid(self.calc_proxy.region):
key = kvs.tokens.gmf_set_key(self.calc_proxy.job_id, point.column,
point.row)
gmf_slice = kvs.get_value_json_decoded(key)
asset_key = kvs.tokens.asset_key(
self.calc_proxy.job_id, point.row, point.column)
for asset in kvs.get_list_json_decoded(asset_key):
LOGGER.debug("Processing asset %s" % (asset))
# loss ratios, used both to produce the curve
# and to aggregate the losses
loss_ratios = self.compute_loss_ratios(asset, gmf_slice)
loss_ratio_curve = self.compute_loss_ratio_curve(
point.column, point.row, asset, gmf_slice, loss_ratios)
aggregate_curve.append(loss_ratios * asset["assetValue"])
if loss_ratio_curve:
loss_curve = self.compute_loss_curve(
point.column, point.row, loss_ratio_curve, asset)
for loss_poe in general.conditional_loss_poes(
self.calc_proxy.params):
general.compute_conditional_loss(
self.calc_proxy.job_id, point.column,
point.row, loss_curve, asset, loss_poe)
return aggregate_curve.losses
def grid_assets_iterator(self, grid):
"""
Generates the tuples (point, asset) for all assets known to this job
that are contained in grid.
:returns: tuples (point, asset) where:
* point is a :py:class:`openquake.shapes.GridPoint` on the grid
* asset is a :py:class:`dict` representing an asset
"""
for point in grid:
asset_key = kvs.tokens.asset_key(
self.calc_proxy.job_id, point.row, point.column)
for asset in kvs.get_list_json_decoded(asset_key):
yield point, asset
def load_gmvs_for_point(job_id, point):
"""
From the KVS, load all the ground motion values for the given point. We
expect one ground motion value per realization of the job.
Since there can be tens of thousands of realizations, this could return a
large list.
Note(LB): In the future, we may want to refactor this (and the code which
uses the values) to use a generator instead.
:param point: :py:class:`openquake.shapes.GridPoint` object
:returns: List of ground motion values (as floats). Each value represents a
realization of the calculation for a single point.
"""
gmfs_key = kvs.tokens.ground_motion_values_key(job_id, point)
return [float(x['mag']) for x in kvs.get_list_json_decoded(gmfs_key)]
def compute_risk(self, block_id, **kwargs): # pylint: disable=W0613
"""This task computes risk for a block of sites. It requires to have
pre-initialized in kvs:
1) list of sites
2) gmfs
3) exposure portfolio (=assets)
4) vulnerability
TODO(fab): make conditional_loss_poe (set of probabilities of
exceedance for which the loss computation is done)
a list of floats, and read it from the job configuration.
"""
conditional_loss_poes = [float(x) for x in self.params.get(
'CONDITIONAL_LOSS_POE', "0.01").split()]
self.slice_gmfs(block_id)
#pylint: disable=W0201
self.vuln_curves = \
vulnerability.load_vuln_model_from_kvs(self.job_id)
# TODO(jmc): DONT assumes that hazard and risk grid are the same
block = general.Block.from_kvs(block_id)
for point in block.grid(self.region):
key = kvs.tokens.gmf_set_key(self.job_id, point.column, point.row)
gmf_slice = kvs.get_value_json_decoded(key)
asset_key = kvs.tokens.asset_key(
self.job_id, point.row, point.column)
for asset in kvs.get_list_json_decoded(asset_key):
LOGGER.debug("processing asset %s" % (asset))
loss_ratio_curve = self.compute_loss_ratio_curve(
point.column, point.row, asset, gmf_slice)
if loss_ratio_curve is not None:
# compute loss curve
loss_curve = self.compute_loss_curve(
point.column, point.row,
loss_ratio_curve, asset)
for loss_poe in conditional_loss_poes:
self.compute_conditional_loss(point.column, point.row,
loss_curve, asset, loss_poe)
return True
def load_gmvs_at(job_id, site):
"""
From the KVS, load all the ground motion values for the given site. We
expect one ground motion value per realization of the job.
Since there can be tens of thousands of realizations, this could return a
large list.
Note(LB): In the future, we may want to refactor this (and the code which
uses the values) to use a generator instead.
:param site: :py:class:`openquake.shapes.Site` object
:returns: List of ground motion values (as floats). Each value represents a
realization of the calculation for a single site.
"""
gmfs_key = kvs.tokens.ground_motion_values_key(job_id, site)
return [float(x['mag']) for x in kvs.get_list_json_decoded(gmfs_key)]
def test_compute_risk_in_the_classical_psha_mixin(self):
"""
tests ClassicalPSHABasedMixin.compute_risk by retrieving
all the loss curves in the kvs and checks their presence
"""
self._compute_risk_classical_psha_setup()
# mixin "instance"
mixin = ClassicalPSHABasedMixin()
mixin.region = self.region
mixin.job_id = self.job_id
mixin.id = self.job_id
mixin.vuln_curves = {"ID": self.vuln_function}
mixin.params = {}
block = Block.from_kvs(self.block_id)
asset = {"taxonomy": "ID",
"assetID": 22.61, "assetValue": 1}
self._store_asset(asset, 10, 10)
# computes the loss curves and puts them in kvs
self.assertTrue(mixin.compute_risk(self.block_id,
point=shapes.GridPoint(None, 10, 20)))
for point in block.grid(mixin.region):
asset_key = kvs.tokens.asset_key(self.job_id, point.row,
point.column)
for asset in kvs.get_list_json_decoded(asset_key):
loss_ratio_key = kvs.tokens.loss_ratio_key(
self.job_id, point.row, point.column, asset['assetID'])
self.assertTrue(kvs.get(loss_ratio_key))
loss_key = kvs.tokens.loss_curve_key(self.job_id, point.row,
point.column, asset['assetID'])
self.assertTrue(kvs.get(loss_key))