def test_get_list_json_decoded(self): data = [{u'1': u'one'}, {u'2': u'two'}, {u'3': u'three'}] for item in data: kvs.get_client().rpush(TEST_KEY, json.dumps(item)) self.assertEqual(data, kvs.get_list_json_decoded(TEST_KEY))
def test_load_gmvs_at(self): """ Exercise the function :func:`openquake.calculators.risk.general.load_gmvs_at`. """ gmvs = [{ 'site_lon': 0.1, 'site_lat': 0.2, 'mag': 0.117 }, { 'site_lon': 0.1, 'site_lat': 0.2, 'mag': 0.167 }, { 'site_lon': 0.1, 'site_lat': 0.2, 'mag': 0.542 }] expected_gmvs = [0.117, 0.167, 0.542] point = self.region.grid.point_at(shapes.Site(0.1, 0.2)) # we expect this point to be at row 1, column 0 self.assertEqual(1, point.row) self.assertEqual(0, point.column) key = kvs.tokens.ground_motion_values_key(self.job_id, point) # place the test values in kvs for gmv in gmvs: kvs.get_client().rpush(key, json.JSONEncoder().encode(gmv)) actual_gmvs = load_gmvs_at(self.job_id, point) self.assertEqual(expected_gmvs, actual_gmvs)
def compute_conditional_loss(job_id, col, row, loss_curve, asset, loss_poe): """Compute the conditional loss for a loss curve and Probability of Exceedance (PoE).""" loss_conditional = _compute_conditional_loss(loss_curve, loss_poe) key = kvs.tokens.loss_key(job_id, row, col, asset.asset_ref, loss_poe) kvs.get_client().set(key, loss_conditional)
def compute_loss_ratio_curve( self, col, row, asset, gmf_slice, loss_ratios): """Compute the loss ratio curve for a single asset.""" calc_proxy = self.calc_proxy vuln_function = self.vuln_curves.get( asset["taxonomy"], None) if not vuln_function: LOGGER.error( "Unknown vulnerability function %s for asset %s" % (asset["taxonomy"], asset["assetID"])) return None epsilon_provider = general.EpsilonProvider(calc_proxy.params) loss_histogram_bins = calc_proxy.oq_job_profile.loss_histogram_bins loss_ratio_curve = general.compute_loss_ratio_curve( vuln_function, gmf_slice, epsilon_provider, asset, loss_histogram_bins, loss_ratios=loss_ratios) # NOTE (jmc): Early exit if the loss ratio is all zeros if not False in (loss_ratio_curve.ordinates == 0.0): return None key = kvs.tokens.loss_ratio_key( self.calc_proxy.job_id, row, col, asset["assetID"]) kvs.get_client().set(key, loss_ratio_curve.to_json()) LOGGER.debug("Loss ratio curve is %s, write to key %s" % (loss_ratio_curve, key)) return loss_ratio_curve
def test_load_gmvs_at(self): """ Exercise the function :func:`openquake.calculators.risk.general.load_gmvs_at`. """ gmvs = [ {'site_lon': 0.1, 'site_lat': 0.2, 'mag': 0.117}, {'site_lon': 0.1, 'site_lat': 0.2, 'mag': 0.167}, {'site_lon': 0.1, 'site_lat': 0.2, 'mag': 0.542}] expected_gmvs = [0.117, 0.167, 0.542] point = self.region.grid.point_at(shapes.Site(0.1, 0.2)) # we expect this point to be at row 1, column 0 self.assertEqual(1, point.row) self.assertEqual(0, point.column) key = kvs.tokens.ground_motion_values_key(self.job_id, point) # place the test values in kvs for gmv in gmvs: kvs.get_client().rpush(key, json.JSONEncoder().encode(gmv)) actual_gmvs = load_gmvs_at(self.job_id, point) self.assertEqual(expected_gmvs, actual_gmvs)
def _write_output_for_block(self, job_id, block_id): """ Given a job and a block, write out a plotted curve """ loss_ratio_curves = [] loss_curves = [] block = Block.from_kvs(job_id, block_id) for point, asset in self.grid_assets_iterator( block.grid(self.calc_proxy.region)): site = shapes.Site(asset['lon'], asset['lat']) loss_curve = kvs.get_client().get( kvs.tokens.loss_curve_key( job_id, point.row, point.column, asset["assetID"])) loss_ratio_curve = kvs.get_client().get( kvs.tokens.loss_ratio_key( job_id, point.row, point.column, asset["assetID"])) if loss_curve: loss_curve = shapes.Curve.from_json(loss_curve) loss_curves.append((site, (loss_curve, asset))) if loss_ratio_curve: loss_ratio_curve = shapes.Curve.from_json(loss_ratio_curve) loss_ratio_curves.append((site, (loss_ratio_curve, asset))) results = self._serialize(block_id, curves=loss_ratio_curves, curve_mode='loss_ratio') if loss_curves: results.extend( self._serialize( block_id, curves=loss_curves, curve_mode='loss', curve_mode_prefix='loss_curve', render_multi=True)) return results
def setUp(self): self.params = dict( CALCULATION_MODE='Hazard', REFERENCE_VS30_VALUE=500, SOURCE_MODEL_LOGIC_TREE_FILE_PATH=SIMPLE_FAULT_SRC_MODEL_LT, GMPE_LOGIC_TREE_FILE_PATH=SIMPLE_FAULT_GMPE_LT, BASE_PATH=SIMPLE_FAULT_BASE_PATH) self.imls = [ 5.0000e-03, 7.0000e-03, 1.3700e-02, 1.9200e-02, 2.6900e-02, 3.7600e-02, 5.2700e-02, 7.3800e-02, 9.8000e-02, 1.0300e-01, 1.4500e-01, 2.0300e-01, 2.8400e-01, 3.9700e-01, 5.5600e-01, 7.7800e-01, 1.0900e+00, 1.5200e+00, 2.1300e+00 ] self.job_ctxt = helpers.create_job(self.params) self.calculator = classical.ClassicalHazardCalculator(self.job_ctxt) self.job_id = self.job_ctxt.job_id self.empty_mean_curve = [] # deleting server side cached data kvs.get_client().flushall() mean_curve = [ 9.8728e-01, 9.8266e-01, 9.4957e-01, 9.0326e-01, 8.1956e-01, 6.9192e-01, 5.2866e-01, 3.6143e-01, 2.4231e-01, 2.2452e-01, 1.2831e-01, 7.0352e-02, 3.6060e-02, 1.6579e-02, 6.4213e-03, 2.0244e-03, 4.8605e-04, 8.1752e-05, 7.3425e-06 ] self.site = shapes.Site(2.0, 5.0) self._store_curve_at(self.site, mean_curve)
def setUp(self): self.params = dict( CALCULATION_MODE='Hazard', REFERENCE_VS30_VALUE=500, SOURCE_MODEL_LOGIC_TREE_FILE_PATH=SIMPLE_FAULT_SRC_MODEL_LT, GMPE_LOGIC_TREE_FILE_PATH=SIMPLE_FAULT_GMPE_LT, BASE_PATH=SIMPLE_FAULT_BASE_PATH) self.imls = [5.0000e-03, 7.0000e-03, 1.3700e-02, 1.9200e-02, 2.6900e-02, 3.7600e-02, 5.2700e-02, 7.3800e-02, 9.8000e-02, 1.0300e-01, 1.4500e-01, 2.0300e-01, 2.8400e-01, 3.9700e-01, 5.5600e-01, 7.7800e-01, 1.0900e+00, 1.5200e+00, 2.1300e+00] self.job_ctxt = helpers.create_job(self.params) self.calculator = classical.ClassicalHazardCalculator(self.job_ctxt) self.job_id = self.job_ctxt.job_id self.empty_mean_curve = [] # deleting server side cached data kvs.get_client().flushall() mean_curve = [9.8728e-01, 9.8266e-01, 9.4957e-01, 9.0326e-01, 8.1956e-01, 6.9192e-01, 5.2866e-01, 3.6143e-01, 2.4231e-01, 2.2452e-01, 1.2831e-01, 7.0352e-02, 3.6060e-02, 1.6579e-02, 6.4213e-03, 2.0244e-03, 4.8605e-04, 8.1752e-05, 7.3425e-06] self.site = shapes.Site(2.0, 5.0) self._store_curve_at(self.site, mean_curve)
def setUp(self): kvs.get_client().flushall() base_path = helpers.testdata_path("scenario") job = engine.prepare_job() self.job_profile, self.params, self.sections = ( engine.import_job_profile(SCENARIO_SMOKE_TEST, job)) self.job_ctxt = JobContext(self.params, job.id, sections=self.sections, base_path=base_path, oq_job_profile=self.job_profile, oq_job=job) self.job_ctxt.params[NUMBER_OF_CALC_KEY] = "1" self.job_ctxt.params['SERIALIZE_RESULTS_TO'] = 'xml' self.job_ctxt.serialize_results_to = ["xml"] # saving the default java implementation self.default = ( scenario.ScenarioHazardCalculator.compute_ground_motion_field) self.grid = self.job_ctxt.region.grid self.job_ctxt.to_kvs()
def setUp(self): kvs.get_client().flushall() base_path = helpers.testdata_path("scenario") self.job_profile, self.params, self.sections = ( engine.import_job_profile(SCENARIO_SMOKE_TEST)) calculation = OqCalculation(owner=self.job_profile.owner, oq_job_profile=self.job_profile) calculation.save() self.calc_proxy = CalculationProxy( self.params, calculation.id, sections=self.sections, base_path=base_path, oq_job_profile=self.job_profile, oq_calculation=calculation) self.calc_proxy.params[NUMBER_OF_CALC_KEY] = "1" self.calc_proxy.params['SERIALIZE_RESULTS_TO'] = 'xml' # saving the default java implementation self.default = ( scenario.ScenarioHazardCalculator.compute_ground_motion_field) self.grid = self.calc_proxy.region.grid self.calc_proxy.to_kvs()
def test_get_client_same_conn(self): """ get_client() returns redis client instances with the same connection pool. """ obj1 = kvs.get_client() obj2 = kvs.get_client() self.assertIs(obj1.connection_pool, obj2.connection_pool)
def _insured_loss_curve_on_kvs(self, column, row, insured_loss_curve, asset): """ Put the insured loss curve on kvs. """ key_ic = kvs.tokens.insured_loss_curve_key(self.job_ctxt.job_id, row, column, asset.asset_ref) kvs.get_client().set(key_ic, insured_loss_curve.to_json())
def _insured_loss_ratio_curve_on_kvs(self, column, row, insured_loss_ratio_curve, asset): """ Put the insured loss ratio curve on kvs. """ key = kvs.tokens.insured_loss_ratio_curve_key(self.job_ctxt.job_id, row, column, asset.asset_ref) kvs.get_client().set(key, insured_loss_ratio_curve.to_json())
def _loss_ratio_curve_on_kvs(self, column, row, loss_ratio_curve, asset): """ Put the loss ratio curve on kvs. """ key = kvs.tokens.loss_ratio_key(self.job_ctxt.job_id, row, column, asset.asset_ref) kvs.get_client().set(key, loss_ratio_curve.to_json()) LOGGER.debug("Loss ratio curve is %s, write to key %s" % (loss_ratio_curve, key))
def test_load_assets_for_point(self): """ Exercises the function :py:func:`openquake.risk.job.deterministic.load_assets_for_point`. """ kvs.flush() # Fabricate some test data. test_assets = [ {'assetValue': 2129.5, 'assetID': '104', 'listDescription': 'fake_description', 'structureCategory': 'S4L_MC', 'lon': 0.11, 'assetDescription': 'LA building', 'vulnerabilityFunctionReference': 'HAZUS_S4L_MC', 'listID': 'LA01', 'assetValueUnit': 'EUR', 'lat': 0.11}, {'assetValue': 2229.5, 'assetID': '105', 'listDescription': 'fake_description', 'structureCategory': 'S4L_MC', 'lon': 0.11, 'assetDescription': 'LA building', 'vulnerabilityFunctionReference': 'HAZUS_S4L_MC', 'listID': 'LA02', 'assetValueUnit': 'EUR', 'lat': 0.12}] test_site = shapes.Site(0.1, 0.1) test_point = TEST_REGION.grid.point_at(test_site) encoder = json.JSONEncoder() assets_key = kvs.tokens.asset_key( TEST_JOB_ID, test_point.row, test_point.column) # Throw the test data into the KVS. for asset in test_assets: kvs.get_client().rpush(assets_key, encoder.encode(asset)) # The data should now be in the KVS. # Now verify that the load_assets_for_point function returns # the appropriate data. actual_assets = \ risk_job_det.load_assets_for_point(TEST_JOB_ID, test_point) kvs.flush() # They should come out exactly the way they went in. self.assertEqual(test_assets, actual_assets)
def store_exposure_assets(self): """ Load exposure assets and write to kvs """ exposure_parser = exposure.ExposurePortfolioFile("%s/%s" % (self.base_path, self.params[job.EXPOSURE])) for site, asset in exposure_parser.filter(self.region): # TODO(JMC): This is kludgey asset['lat'] = site.latitude asset['lon'] = site.longitude gridpoint = self.region.grid.point_at(site) asset_key = kvs.tokens.asset_key(self.id, gridpoint.row, gridpoint.column) kvs.get_client().rpush(asset_key, json.JSONEncoder().encode(asset))
def compute_conditional_loss(job_id, col, row, loss_curve, asset, loss_poe): """Compute the conditional loss for a loss curve and Probability of Exceedance (PoE).""" loss_conditional = _compute_conditional_loss( loss_curve, loss_poe) key = kvs.tokens.loss_key( job_id, row, col, asset["assetID"], loss_poe) LOG.debug("Conditional loss is %s, write to key %s" % (loss_conditional, key)) kvs.get_client().set(key, loss_conditional)
def get_pattern(regexp): """Get all the values whose keys satisfy the given regexp. Return an empty list if there are no keys satisfying the given regxep. """ values = [] keys = kvs.get_client().keys(regexp) if keys: values = kvs.get_client().mget(keys) return values
def compute_loss_curve(self, column, row, loss_ratio_curve, asset): """Compute the loss curve for a single asset.""" if asset is None: return None loss_curve = loss_ratio_curve.rescale_abscissae(asset.value) key = kvs.tokens.loss_curve_key(self.job_ctxt.job_id, row, column, asset.asset_ref) LOGGER.debug("Loss curve is %s, write to key %s" % (loss_curve, key)) kvs.get_client().set(key, loss_curve.to_json()) return loss_curve
def compute_loss_curve(self, column, row, loss_ratio_curve, asset): """Compute the loss curve for a single asset.""" if asset is None: return None loss_curve = loss_ratio_curve.rescale_abscissae(asset.value) key = kvs.tokens.loss_curve_key( self.job_ctxt.job_id, row, column, asset.asset_ref) LOGGER.debug("Loss curve is %s, write to key %s" % (loss_curve, key)) kvs.get_client().set(key, loss_curve.to_json()) return loss_curve
def _compute_risk_classical_psha_setup(self): SITE = shapes.Site(1.0, 1.0) # deletes all keys from kvs kvs.get_client().flushall() self.job = self.setup_classic_job() # at the moment the hazard part doesn't do exp on the 'x' # so it's done on the risk part. To adapt the calculation # we do the reverse of the exp, i.e. log(x) self.hazard_curve = [ (SITE, {'IMLValues': [0.001, 0.080, 0.170, 0.260, 0.360, 0.550, 0.700], 'PoEValues': [0.99, 0.96, 0.89, 0.82, 0.70, 0.40, 0.01], 'statistics': 'mean'})] # Vitor provided this Vulnerability Function imls_1 = [0.03, 0.04, 0.07, 0.1, 0.12, 0.22, 0.37, 0.52] loss_ratios_1 = [0.001, 0.022, 0.051, 0.08, 0.1, 0.2, 0.405, 0.700] covs_1 = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] self.vuln_function = shapes.VulnerabilityFunction(imls_1, loss_ratios_1, covs_1) imls_2 = [0.1, 0.2, 0.4, 0.6] loss_ratios_2 = [0.05, 0.08, 0.2, 0.4] covs_2 = [0.5, 0.3, 0.2, 0.1] self.vuln_function_2 = shapes.VulnerabilityFunction(imls_2, loss_ratios_2, covs_2) self.job_id = self.job.id self.asset_1 = {"taxonomy": "ID", "assetValue": 124.27} self.region = shapes.RegionConstraint.from_simple( (0.0, 0.0), (2.0, 2.0)) self.block_id = kvs.tokens.risk_block_key(self.job_id, 7) block = Block((SITE, SITE), self.block_id) block.to_kvs() writer = hazard.HazardCurveDBWriter('test_path.xml', self.job_id) writer.serialize(self.hazard_curve) kvs.set_value_json_encoded( kvs.tokens.vuln_key(self.job_id), {"ID": self.vuln_function.to_json()})
def write_loss_map(self, loss_poe): """ Iterates through all the assets and maps losses at loss_poe """ decoder = json.JSONDecoder() # Make a special grid at a higher resolution risk_grid = shapes.Grid(self.region, float(self['RISK_CELL_SIZE'])) filename = "losses_at-%s.tiff" % (loss_poe) path = os.path.join(self.base_path, self['OUTPUT_DIR'], filename) output_generator = geotiff.LossMapGeoTiffFile(path, risk_grid, init_value=0.0, normalize=True) for point in self.region.grid: asset_key = kvs.tokens.asset_key(self.id, point.row, point.column) asset_list = kvs.get_client().lrange(asset_key, 0, -1) for asset in [decoder.decode(x) for x in asset_list]: key = kvs.tokens.loss_key(self.id, point.row, point.column, asset["assetID"], loss_poe) loss = kvs.get(key) LOG.debug("Loss for asset %s at %s %s is %s" % (asset["assetID"], asset['lon'], asset['lat'], loss)) if loss: loss_ratio = float(loss) / float(asset["assetValue"]) risk_site = shapes.Site(asset['lon'], asset['lat']) risk_point = risk_grid.point_at(risk_site) output_generator.write( (risk_point.row, risk_point.column), loss_ratio) output_generator.close() return [path]
def write_loss_map(self, loss_poe): """ Iterates through all the assets and maps losses at loss_poe """ # Make a special grid at a higher resolution risk_grid = shapes.Grid(self.region, float(self['RISK_CELL_SIZE'])) path = os.path.join(self.base_path, self['OUTPUT_DIR'], "losses_at-%s.tiff" % loss_poe) output_generator = geotiff.LossMapGeoTiffFile(path, risk_grid, init_value=0.0, normalize=True) for point in self.region.grid: asset_key = kvs.tokens.asset_key(self.id, point.row, point.column) asset_list = kvs.get_client().lrange(asset_key, 0, -1) for asset in [json.loads(x) for x in asset_list]: key = kvs.tokens.loss_key(self.id, point.row, point.column, asset["assetID"], loss_poe) loss = kvs.get(key) LOG.debug("Loss for asset %s at %s %s is %s" % (asset["assetID"], asset['lon'], asset['lat'], loss)) if loss: loss_ratio = float(loss) / float(asset["assetValue"]) risk_site = shapes.Site(asset['lon'], asset['lat']) risk_point = risk_grid.point_at(risk_site) output_generator.write( (risk_point.row, risk_point.column), loss_ratio) output_generator.close() return [path]
def compute_risk(self, block_id, **kwargs): # pylint: disable=W0613 """This task computes risk for a block of sites. It requires to have pre-initialized in kvs: 1) list of sites 2) exposure portfolio (=assets) 3) vulnerability """ block = job.Block.from_kvs(block_id) #pylint: disable=W0201 self.vuln_curves = \ vulnerability.load_vuln_model_from_kvs(self.job_id) for point in block.grid(self.region): curve_token = kvs.tokens.mean_hazard_curve_key( self.job_id, point.site) decoded_curve = kvs.get_value_json_decoded(curve_token) hazard_curve = Curve([(exp(float(el['x'])), el['y']) for el in decoded_curve['curve']]) asset_key = kvs.tokens.asset_key(self.id, point.row, point.column) assets = kvs.get_client().lrange(asset_key, 0, -1) for asset in [json.JSONDecoder().decode(x) for x in assets]: LOGGER.debug("processing asset %s" % (asset)) loss_ratio_curve = self.compute_loss_ratio_curve( point, asset, hazard_curve) self.compute_loss_curve(point, loss_ratio_curve, asset) return True
def generate_erf(job_id): """ Stubbed ERF generator Takes a job_id, returns a job_id. Connects to the Java HazardEngine using hazardwrapper, waits for an ERF to be generated, and then writes it to KVS. """ # TODO(JM): implement real ERF computation erf_key = kvs.generate_product_key(job_id, kvs.tokens.ERF_KEY_TOKEN) kvs.get_client().set(erf_key, json.JSONEncoder().encode([job_id])) return job_id
def execute(self): """Entry point to trigger the computation.""" random_generator = java.jclass( "Random")(int(self.params["GMF_RANDOM_SEED"])) encoder = json.JSONEncoder() kvs_client = kvs.get_client(binary=False) grid = self.region.grid for _ in xrange(self._number_of_calculations()): gmf = self.compute_ground_motion_field(random_generator) for gmv in gmf_to_dict( gmf, self.params["INTENSITY_MEASURE_TYPE"]): site = shapes.Site(gmv["site_lon"], gmv["site_lat"]) point = grid.point_at(site) key = kvs.tokens.ground_motion_values_key( self.job_id, point) kvs_client.rpush(key, encoder.encode(gmv)) return [True]
def execute(self): """Entry point to trigger the computation.""" random_generator = java.jclass("Random")(int( self.job_ctxt.params["GMF_RANDOM_SEED"])) encoder = json.JSONEncoder() kvs_client = kvs.get_client() num_calculations = self._number_of_calculations() self.initialize_pr_data(num_calculations=num_calculations) for cnum in xrange(num_calculations): try: gmf = self.compute_ground_motion_field(random_generator) stats.pk_inc(self.job_ctxt.job_id, "nhzrd_done", 1) except: # Count failure stats.pk_inc(self.job_ctxt.job_id, "nhzrd_failed", 1) raise logs.log_percent_complete(self.job_ctxt.job_id, "hazard") imt = self.job_ctxt.params["INTENSITY_MEASURE_TYPE"] self._serialize_gmf(gmf, imt, cnum) for gmv in gmf_to_dict(gmf, imt): site = shapes.Site(gmv["site_lon"], gmv["site_lat"]) key = kvs.tokens.ground_motion_values_key( self.job_ctxt.job_id, site) kvs_client.rpush(key, encoder.encode(gmv))
def setUp(self): self.params = {} self.job = helpers.create_job(self.params) self.job_id = self.job.job_id self.expected_mean_curve = numpy.array([9.8542200e-01, 9.8196600e-01, 9.5842000e-01, 9.2639600e-01, 8.6713000e-01, 7.7081800e-01, 6.3448600e-01, 4.7256800e-01, 3.3523400e-01, 3.1255000e-01, 1.7832000e-01, 9.0883400e-02, 4.2189200e-02, 1.7874200e-02, 6.7449200e-03, 2.1658200e-03, 5.3878600e-04, 9.4369400e-05, 8.9830380e-06]) self.empty_curve = [] # deleting server side cached data kvs.get_client().flushall()
def execute(self): """Entry point to trigger the computation.""" random_generator = java.jclass( "Random")(int(self.job_ctxt.params["GMF_RANDOM_SEED"])) encoder = json.JSONEncoder() kvs_client = kvs.get_client() num_calculations = self._number_of_calculations() self.initialize_pr_data(num_calculations=num_calculations) for cnum in xrange(num_calculations): try: gmf = self.compute_ground_motion_field(random_generator) stats.pk_inc(self.job_ctxt.job_id, "nhzrd_done", 1) except: # Count failure stats.pk_inc(self.job_ctxt.job_id, "nhzrd_failed", 1) raise logs.log_percent_complete(self.job_ctxt.job_id, "hazard") imt = self.job_ctxt.params["INTENSITY_MEASURE_TYPE"] self._serialize_gmf(gmf, imt, cnum) for gmv in gmf_to_dict(gmf, imt): site = shapes.Site(gmv["site_lon"], gmv["site_lat"]) key = kvs.tokens.ground_motion_values_key( self.job_ctxt.job_id, site) kvs_client.rpush(key, encoder.encode(gmv))
def compute_hazard_curve(self, site_list, realization): """ Compute hazard curves, write them to KVS as JSON, and return a list of the KVS keys for each curve. """ jsite_list = self.parameterize_sites(site_list) hazard_curves = java.jclass("HazardCalculator").getHazardCurvesAsJson( jsite_list, self.generate_erf(), self.generate_gmpe_map(), self.get_iml_list(), float(self.params['MAXIMUM_DISTANCE'])) # write the curves to the KVS and return a list of the keys kvs_client = kvs.get_client() curve_keys = [] for i in xrange(0, len(hazard_curves)): curve = hazard_curves[i] site = site_list[i] lon = site.longitude lat = site.latitude curve_key = kvs.tokens.hazard_curve_key(self.id, realization, lon, lat) kvs_client.set(curve_key, curve) curve_keys.append(curve_key) return curve_keys
def store_exposure_assets(self): """Load exposure assets and write them to KVS.""" exposure_parser = exposure.ExposurePortfolioFile( os.path.join(self.base_path, self.params[job_config.EXPOSURE])) for site, asset in exposure_parser.filter(self.region): # TODO(ac): This is kludgey (?) asset["lat"] = site.latitude asset["lon"] = site.longitude gridpoint = self.region.grid.point_at(site) asset_key = kvs.tokens.asset_key( self.job_id, gridpoint.row, gridpoint.column) kvs.get_client().rpush(asset_key, json.JSONEncoder().encode(asset))
def _write_output_for_block(self, job_id, block_id): """ Given a job and a block, write out a plotted curve """ decoder = json.JSONDecoder() loss_ratio_curves = [] block = job.Block.from_kvs(block_id) for point in block.grid(self.region): asset_key = kvs.tokens.asset_key(self.id, point.row, point.column) asset_list = kvs.get_client().lrange(asset_key, 0, -1) for asset in [decoder.decode(x) for x in asset_list]: site = shapes.Site(asset["lon"], asset["lat"]) key = kvs.tokens.loss_ratio_key(job_id, point.row, point.column, asset["AssetID"]) loss_ratio_curve = kvs.get(key) if loss_ratio_curve: loss_ratio_curve = shapes.Curve.from_json(loss_ratio_curve) loss_ratio_curves.append((site, (loss_ratio_curve, asset))) LOG.debug("Serializing loss_ratio_curves") filename = "%s-block-%s.xml" % (self["LOSS_CURVES_OUTPUT_PREFIX"], block_id) path = os.path.join(self.base_path, self["OUTPUT_DIR"], filename) output_generator = risk_output.LossRatioCurveXMLWriter(path) # TODO(JMC): Take mean or max for each site output_generator.serialize(loss_ratio_curves) filename = "%s-block-%s.svg" % (self["LOSS_CURVES_OUTPUT_PREFIX"], block_id) curve_path = os.path.join(self.base_path, self["OUTPUT_DIR"], filename) plotter = curve.RiskCurvePlotter(curve_path, path, mode="loss_ratio") plotter.plot(autoscale_y=False) results = [path] results.extend(list(plotter.filenames())) return results
def compute_risk(self, block_id, **kwargs): # pylint: disable=W0613 """This task computes risk for a block of sites. It requires to have pre-initialized in kvs: 1) list of sites 2) exposure portfolio (=assets) 3) vulnerability """ block = job.Block.from_kvs(block_id) #pylint: disable=W0201 self.vuln_curves = \ vulnerability.load_vuln_model_from_kvs(self.job_id) for point in block.grid(self.region): curve_token = kvs.tokens.mean_hazard_curve_key(self.job_id, point.site) decoded_curve = kvs.get_value_json_decoded(curve_token) hazard_curve = Curve([(exp(float(el['x'])), el['y']) for el in decoded_curve['curve']]) asset_key = kvs.tokens.asset_key(self.id, point.row, point.column) assets = kvs.get_client().lrange(asset_key, 0, -1) for asset in [json.JSONDecoder().decode(x) for x in assets]: LOGGER.debug("processing asset %s" % (asset)) loss_ratio_curve = self.compute_loss_ratio_curve( point, asset, hazard_curve) self.compute_loss_curve(point, loss_ratio_curve, asset) return True
def setUp(self): self.generated_files = [] self.kvs_client = kvs.get_client() # We will run a full test using amqp logging, as configured in # openquake.cfg helpers.declare_signalling_exchange()
def _assets_keys_for_gmfs(job_id, gmfs_key): """Return the asset related to the GMFs given.""" column, row = kvs.tokens.column_row_from_gmf_set_key(gmfs_key) key = kvs.tokens.asset_key(job_id, row, column) return kvs.get_client().lrange(key, 0, -1)
def setUp(self): self.params = {} self.job = helpers.create_job(self.params) self.job_id = self.job.job_id self.expected_mean_curve = numpy.array([ 9.8542200e-01, 9.8196600e-01, 9.5842000e-01, 9.2639600e-01, 8.6713000e-01, 7.7081800e-01, 6.3448600e-01, 4.7256800e-01, 3.3523400e-01, 3.1255000e-01, 1.7832000e-01, 9.0883400e-02, 4.2189200e-02, 1.7874200e-02, 6.7449200e-03, 2.1658200e-03, 5.3878600e-04, 9.4369400e-05, 8.9830380e-06 ]) self.empty_curve = [] # deleting server side cached data kvs.get_client().flushall()
def _store_gmvs(self, gmvs): client = kvs.get_client() encoder = json.JSONEncoder() key = ground_motion_values_key(self.job.id, self.site) for gmv in gmvs: client.rpush(key, encoder.encode({"mag": gmv}))
def _keys_found(self, job_id, keys): """Return the keys found in kvs.""" result = [] conn = kvs.get_client() for key in keys: key %= job_id if conn.get(key) is not None: result.append(key) return result
def _assets_keys_for_gmfs(job_id, gmfs_key): """Return the asset related to the GMFs given.""" row = lambda key: key.split(kvs.KVS_KEY_SEPARATOR)[3] column = lambda key: key.split(kvs.KVS_KEY_SEPARATOR)[2] key = kvs.tokens.asset_key(job_id, row(gmfs_key), column(gmfs_key)) return kvs.get_client().lrange(key, 0, -1)
def _write_output_for_block(self, job_id, block_id): """ Write loss / loss ratio curves to xml for a single block. """ loss_curves = [] loss_ratio_curves = [] block = Block.from_kvs(job_id, block_id) for site in block.sites: point = self.job_ctxt.region.grid.point_at(site) assets = BaseRiskCalculator.assets_at(self.job_ctxt.job_id, site) for asset in assets: loss_curve = kvs.get_client().get( kvs.tokens.loss_curve_key(job_id, point.row, point.column, asset.asset_ref)) loss_ratio_curve = kvs.get_client().get( kvs.tokens.loss_ratio_key(job_id, point.row, point.column, asset.asset_ref)) if loss_curve: loss_curve = shapes.Curve.from_json(loss_curve) loss_curves.append((site, (loss_curve, asset))) if loss_ratio_curve: loss_ratio_curve = shapes.Curve.from_json(loss_ratio_curve) loss_ratio_curves.append((site, (loss_ratio_curve, asset))) results = self._serialize(block_id, curves=loss_ratio_curves, curve_mode="loss_ratio") if loss_curves: results.extend( self._serialize(block_id, curves=loss_curves, curve_mode="loss", curve_mode_prefix="loss_curve", render_multi=True)) return results
def setUpClass(cls): cls.client = kvs.get_client() cls.client.delete(tokens.CURRENT_JOBS) cls.client.delete(tokens.NEXT_JOB_ID) # create 3 jobs # this will add job keys to CURRENT_JOBS for i in range(1, 4): tokens.alloc_job_key()
def simple_task_return_name_to_memcache(name, **kwargs): logger = simple_task_return_name_to_memcache.get_logger(**kwargs) memcache_client = kvs.get_client(binary=False) wait_time = _wait_a_bit() logger.info("processing %s, waited %s milliseconds" % (name, wait_time)) memcache_client.set(name, name) logger.info("wrote to memcache key %s" % (name))
def compute_loss_curve(self, point, loss_ratio_curve, asset): """ Computes the loss ratio and store it in kvs to provide data to the @output decorator which does the serialization. :param point: the point of the grid we want to compute :type point: :py:class:`openquake.shapes.GridPoint` :param loss_ratio_curve: the loss ratio curve :type loss_ratio_curve: :py:class `openquake.shapes.Curve` :param asset: the asset for which to compute the loss curve :type asset: :py:class:`dict` as provided by :py:class:`openquake.parser.exposure.ExposureModelFile` """ loss_curve = compute_loss_curve(loss_ratio_curve, asset.value) loss_key = kvs.tokens.loss_curve_key(self.job_ctxt.job_id, point.row, point.column, asset.asset_ref) kvs.get_client().set(loss_key, loss_curve.to_json()) return loss_curve
def setUp(self): self.params = dict( CALCULATION_MODE='Hazard', SOURCE_MODEL_LOGIC_TREE_FILE_PATH=SIMPLE_FAULT_SRC_MODEL_LT, GMPE_LOGIC_TREE_FILE_PATH=SIMPLE_FAULT_GMPE_LT, BASE_PATH=SIMPLE_FAULT_BASE_PATH) self.job_ctxt = helpers.create_job(self.params) self.calculator = classical.ClassicalHazardCalculator(self.job_ctxt) self.job_id = self.job_ctxt.job_id self.expected_curve = numpy.array([ 9.9178000e-01, 9.8892000e-01, 9.6903000e-01, 9.4030000e-01, 8.8405000e-01, 7.8782000e-01, 6.4897250e-01, 4.8284250e-01, 3.4531500e-01, 3.2337000e-01, 1.8880500e-01, 9.5574000e-02, 4.3707250e-02, 1.9643000e-02, 8.1923000e-03, 2.9157000e-03, 7.9955000e-04, 1.5233000e-04, 1.5582000e-05 ]) # deleting server side cached data kvs.get_client().flushall()
def compute_mgm_intensity(job_id, block_id, site_id): """Compute mean ground intensity for a specific site.""" # We don't actually need the JobContext returned by this function # (yet) but this does check if the calculation is still in progress. utils_tasks.get_running_job(job_id) kvs_client = kvs.get_client() mgm_key = kvs.tokens.mgm_key(job_id, block_id, site_id) mgm = kvs_client.get(mgm_key) return json.JSONDecoder().decode(mgm)
def store_gmpe_map(job_id, seed, calc): """Generate a hash map of GMPEs (keyed by Tectonic Region Type) and store it in the KVS. :param int job_id: numeric ID of the job :param int seed: seed for random logic tree sampling :param calc: logic tree processor :type calc: :class:`openquake.input.logictree.LogicTreeProcessor` instance """ LOG.info("Storing GMPE map from job config") key = kvs.tokens.gmpe_key(job_id) calc.sample_and_save_gmpe_logictree(kvs.get_client(), key, seed)
def mget_decoded(keys): """ Retrieve multiple JSON values from the KVS :param keys: keys to retrieve (the corresponding value must be a JSON string) :type keys: list :returns: one value for each key in the list """ decoder = json.JSONDecoder() return [decoder.decode(value) for value in kvs.get_client().mget(keys)]
def setUp(self): # starting the jvm... print "About to start the jvm..." jpype = java.jvm() java_class = jpype.JClass("org.gem.engine.hazard.redis.Cache") print ("Not dead yet, and found the class...") self.java_client = java_class(settings.KVS_HOST, settings.KVS_PORT) self.python_client = kvs.get_client(binary=False) self.reader = reader.Reader(self.python_client) self._delete_test_file()
def compute_loss_ratio_curve(self, col, row, asset, gmf_slice, loss_ratios): """Compute the loss ratio curve for a single asset. :param asset: the asset used to compute loss :type asset: an :py:class:`openquake.db.model.ExposureData` instance """ job_ctxt = self.job_ctxt vuln_function = self.vuln_curves.get(asset.taxonomy, None) if not vuln_function: LOGGER.error("Unknown vulnerability function %s for asset %s" % (asset.taxonomy, asset.asset_ref)) return None epsilon_provider = general.EpsilonProvider(job_ctxt.params) loss_histogram_bins = job_ctxt.oq_job_profile.loss_histogram_bins loss_ratio_curve = general.compute_loss_ratio_curve( vuln_function, gmf_slice, epsilon_provider, asset, loss_histogram_bins, loss_ratios=loss_ratios) # NOTE (jmc): Early exit if the loss ratio is all zeros if not False in (loss_ratio_curve.ordinates == 0.0): return None key = kvs.tokens.loss_ratio_key(self.job_ctxt.job_id, row, col, asset.asset_ref) kvs.get_client().set(key, loss_ratio_curve.to_json()) LOGGER.debug("Loss ratio curve is %s, write to key %s" % (loss_ratio_curve, key)) return loss_ratio_curve
def setUp(self): # starting the jvm... print "About to start the jvm..." jpype = java.jvm() java_class = jpype.JClass("org.gem.engine.hazard.redis.Cache") print "Not dead yet, and found the class..." self.java_client = java_class(config.get("kvs", "host"), int(config.get("kvs", "port"))) self.python_client = kvs.get_client() self.python_client.flushdb() self._delete_test_file()