def test_run_calc_with_description(self):
# Test importing and running a job with a config containing the
# optional DESCRIPTION parameter.
description = 'Classical PSHA hazard test description'
orig_cfg_path = demo_file('PeerTestSet1Case2/config.gem')
mod_cfg_path = os.path.join(demo_file('PeerTestSet1Case2'),
'modified_config.gem')
# Use ConfigParser to add the DESCRIPTION param to an existing config
# profile and write a new temporary config file:
cfg_parser = ConfigParser.ConfigParser()
cfg_parser.readfp(open(orig_cfg_path, 'r'))
cfg_parser.set('general', 'DESCRIPTION', description)
cfg_parser.write(open(mod_cfg_path, 'w'))
run_job(mod_cfg_path)
job = OqJob.objects.latest('id')
job_profile = job.profile()
self.assertEqual(description, job_profile.description)
self.assertEqual(description, job.description)
# Clean up the temporary config file:
os.unlink(mod_cfg_path)
def test_run_calc_with_description(self):
# Test importing and running a job with a config containing the
# optional DESCRIPTION parameter.
description = 'Classical PSHA hazard test description'
orig_cfg_path = demo_file('PeerTestSet1Case2/config.gem')
mod_cfg_path = os.path.join(demo_file('PeerTestSet1Case2'),
'modified_config.gem')
# Use ConfigParser to add the DESCRIPTION param to an existing config
# profile and write a new temporary config file:
cfg_parser = ConfigParser.ConfigParser()
cfg_parser.readfp(open(orig_cfg_path, 'r'))
cfg_parser.set('general', 'DESCRIPTION', description)
cfg_parser.write(open(mod_cfg_path, 'w'))
run_job(mod_cfg_path)
job = OqJob.objects.latest('id')
job_profile = job.profile()
self.assertEqual(description, job_profile.description)
self.assertEqual(description, job.description)
# Clean up the temporary config file:
os.unlink(mod_cfg_path)
def test_uhs_output_type_xml(self):
# Run a calculation with --output-type=xml and check that the expected
# result files are created in the right location.
# This location is based on parameters in the UHS config file:
results_target_dir = demo_file('uhs/computed_output')
# clear the target dir from previous demo/test runs
shutil.rmtree(results_target_dir)
expected_export_files = [
os.path.join(results_target_dir, 'uhs_poe:0.1.hdf5'),
os.path.join(results_target_dir, 'uhs_poe:0.02.hdf5'),
os.path.join(results_target_dir, 'uhs.xml'),
]
for f in expected_export_files:
self.assertFalse(os.path.exists(f))
uhs_cfg = demo_file('uhs/config.gem')
try:
ret_code = run_job(uhs_cfg, ['--output-type=xml'])
self.assertEqual(0, ret_code)
# Check that all the output files were created:
for f in expected_export_files:
self.assertTrue(os.path.exists(f))
finally:
shutil.rmtree(results_target_dir)
def test_uhs_output_type_xml(self):
# Run a calculation with --output-type=xml and check that the expected
# result files are created in the right location.
# This location is based on parameters in the UHS config file:
results_target_dir = demo_file('uhs/computed_output')
# clear the target dir from previous demo/test runs
shutil.rmtree(results_target_dir)
expected_export_files = [
os.path.join(results_target_dir, 'uhs_poe:0.1.hdf5'),
os.path.join(results_target_dir, 'uhs_poe:0.02.hdf5'),
os.path.join(results_target_dir, 'uhs.xml'),
]
for f in expected_export_files:
self.assertFalse(os.path.exists(f))
uhs_cfg = demo_file('uhs/config.gem')
try:
ret_code = run_job(uhs_cfg, ['--output-type=xml'])
self.assertEqual(0, ret_code)
# Check that all the output files were created:
for f in expected_export_files:
self.assertTrue(os.path.exists(f))
finally:
shutil.rmtree(results_target_dir)
def test_scenario_risk_sample_based(self):
# This QA test is a longer-running test of the Scenario Risk
# calculator.
# The vulnerabiilty model has non-zero Coefficients of Variation and
# therefore exercises the 'sample-based' path through the calculator.
# This test is configured to produce 1000 ground motion fields at each
# location of interest (in the test above, only 10 are produced).
# Since we're seeding the random epsilon sampling, we can consistently
# reproduce all result values.
# When these values are compared to the results computed by a similar
# config which takes the 'mean-based' path (with CoVs = 0), we expect
# the following:
# All of the mean values in the 'sample-based' results should be with
# 5%, + or -, of the 'mean-based' results.
# The standard deviation values of the 'sample-based' results should
# simply be greater than those produced with the 'mean-based' method.
# For comparison, mean and stddev values for the region were computed
# with 1000 GMFs using the mean-based approach. These values (rounded
# to 2 decimal places) are:
mb_mean_loss = 1233.26
mb_stddev_loss = 443.63
# Loss map for the mean-based approach:
mb_loss_map = [
dict(asset='a3', pos='15.48 38.25', mean=200.54874638,
stddev=94.2302991022),
dict(asset='a2', pos='15.56 38.17', mean=510.821363253,
stddev=259.964152622),
dict(asset='a1', pos='15.48 38.09', mean=521.885458891,
stddev=244.825980356),
]
# Sanity checks are done. Let's do this.
scen_cfg = helpers.demo_file(
'scenario_risk/config_sample-based_qa.gem')
result = helpers.run_job(scen_cfg, ['--output-type=xml'],
check_output=True)
job = OqJob.objects.latest('id')
self.assertEqual('succeeded', job.status)
expected_loss_map_file = helpers.demo_file(
'scenario_risk/computed_output/loss-map-%s.xml' % job.id)
self.assertTrue(os.path.exists(expected_loss_map_file))
loss_map = helpers.loss_map_result_from_file(expected_loss_map_file)
self._verify_loss_map_within_range(sorted(mb_loss_map),
sorted(loss_map), 0.05)
exp_mean_loss, exp_stddev_loss = helpers.mean_stddev_from_result_line(
result)
self.assertAlmostEqual(mb_mean_loss, exp_mean_loss,
delta=mb_mean_loss * 0.05)
self.assertTrue(exp_stddev_loss > mb_stddev_loss)
def test_scenario_risk(self):
# This test exercises the 'mean-based' path through the Scenario Risk
# calculator. There is no random sampling done here so the results are
# 100% predictable.
scen_cfg = helpers.demo_file('scenario_risk/config.gem')
exp_mean_loss = 1053.09
exp_stddev_loss = 246.62
expected_loss_map = [
dict(asset='a3',
pos='15.48 38.25',
mean=180.717534009275,
stddev=92.2122644809969),
dict(asset='a2',
pos='15.56 38.17',
mean=432.225448142534,
stddev=186.864456949986),
dict(asset='a1',
pos='15.48 38.09',
mean=440.147078317589,
stddev=182.615976701858),
]
result = helpers.run_job(scen_cfg, ['--output-type=xml'],
check_output=True)
job = OqJob.objects.latest('id')
self.assertEqual('succeeded', job.status)
expected_loss_map_file = helpers.demo_file(
'scenario_risk/computed_output/loss-map-%s.xml' % job.id)
self.assertTrue(os.path.exists(expected_loss_map_file))
self._verify_loss_map(expected_loss_map_file, expected_loss_map)
# We expected the shell output to look something like the following
# two lines:
# Mean region loss value: 1053.09
# Standard deviation region loss value: 246.62
# split on newline and filter out empty lines
result = [line for line in result.split('\n') if len(line) > 0]
# we expect 2 lines; 1 for mean, 1 for stddev
self.assertEqual(2, len(result))
actual_mean = float(result[0].split()[-1])
actual_stddev = float(result[1].split()[-1])
self.assertAlmostEqual(exp_mean_loss,
actual_mean,
places=self.TOTAL_LOSS_PRECISION)
self.assertAlmostEqual(exp_stddev_loss,
actual_stddev,
places=self.TOTAL_LOSS_PRECISION)
def _create_job_profiles(self, user_name):
uhs_cfg = helpers.demo_file('uhs/config.gem')
job = engine.prepare_job()
self.uhs_jp, _, _ = engine.import_job_profile(uhs_cfg, job,
user_name=user_name)
cpsha_cfg = helpers.demo_file('classical_psha_based_risk/config.gem')
job = engine.prepare_job()
self.cpsha_jp, _, _ = engine.import_job_profile(cpsha_cfg, job,
user_name=user_name)
def setUp(self):
self.job, _ = helpers.get_risk_job(
demo_file('scenario_risk/job.ini'),
demo_file('scenario_hazard/job.ini'), output_type="gmf_scenario")
self.calculator = scenario.ScenarioRiskCalculator(self.job)
self.calculator.pre_execute()
self.job.is_running = True
self.job.status = 'executing'
self.job.save()
def setUp(self):
self.job, _ = helpers.get_risk_job(
demo_file('event_based_risk/job.ini'),
demo_file('event_based_hazard/job.ini'), output_type="gmf")
self.calculator = event_based.EventBasedRiskCalculator(self.job)
self.calculator.pre_execute()
self.job.is_running = True
self.job.status = 'executing'
self.job.save()
def _create_job_profiles(self, user_name):
uhs_cfg = helpers.demo_file('uhs/config.gem')
job = engine.prepare_job()
self.uhs_jp, _, _ = engine.import_job_profile(uhs_cfg,
job,
user_name=user_name)
cpsha_cfg = helpers.demo_file('classical_psha_based_risk/config.gem')
job = engine.prepare_job()
self.cpsha_jp, _, _ = engine.import_job_profile(cpsha_cfg,
job,
user_name=user_name)
def test_a_few_inputs(self):
job, files = helpers.get_risk_job(
demo_file('classical_psha_based_risk/job.ini'),
demo_file('simple_fault_demo_hazard/job.ini'))
rc = job.risk_calculation
expected_ids = sorted([x.id for x in files.values()])
inputs = models.inputs4rcalc(rc.id)
actual_ids = sorted([x.id for x in inputs])
self.assertEqual(expected_ids, actual_ids)
def test_hazard_map_test(self):
helpers.run_job(helpers.demo_file(
os.path.join("HazardMapTest", "config.gem")))
self.job = models.OqCalculation.objects.latest("id")
path = helpers.demo_file(os.path.join("HazardMapTest",
"expected_results", "meanHazardMap0.1.dat"))
expected_map = load_expected_map(path)
poe = 0.1
statistic_type = "mean"
verify_hazmap_results(self, self.job, expected_map, poe,
statistic_type)
def setUp(self):
job, _ = helpers.get_risk_job(
demo_file('classical_psha_based_risk/job.ini'),
demo_file('simple_fault_demo_hazard/job.ini')
)
self.compulsory_arguments = dict(
calculation_mode="classical",
lrem_steps_per_interval=5)
self.other_args = dict(
owner=helpers.default_user(),
region_constraint=(
'POLYGON((-122.0 38.113, -122.114 38.113, -122.57 38.111, '
'-122.0 38.113))'),
hazard_output=job.risk_calculation.hazard_output)
def test_hazard_map_test(self):
helpers.run_job(helpers.demo_file(
os.path.join("HazardMapTest", "config.gem")))
self.job = models.OqJob.objects.latest("id")
path = helpers.demo_file(os.path.join("HazardMapTest",
"expected_results", "meanHazardMap0.1.dat"))
expected_map = load_expected_map(path)
poe = 0.1
statistic_type = "mean"
verify_hazmap_results(self, self.job, expected_map, poe,
statistic_type)
def test_with_input_type(self):
job, files = helpers.get_risk_job(
demo_file('classical_psha_based_risk/job.ini'),
demo_file('simple_fault_demo_hazard/job.ini'))
rc = job.risk_calculation
# It should only be 1 id, actually.
expected_ids = [x.id for x in files.values()
if x.input_type == 'exposure']
inputs = models.inputs4rcalc(rc.id, input_type='exposure')
actual_ids = sorted([x.id for x in inputs])
self.assertEqual(expected_ids, actual_ids)
def test_hazard_input_on_exposure_sites(self):
cfg = helpers.demo_file("scenario_risk/config_hzr_exposure.gem")
ret_code = helpers.run_job(cfg, ["--output-type=xml"])
self.assertEquals(0, ret_code)
self.assertEqual("succeeded", OqJob.objects.latest("id").status)
def test_calculator_for_task(self):
"""Load up a sample calculation (into the db and cache) and make sure
we can instantiate the correct calculator for a given calculation id.
"""
from openquake.calculators.hazard.classical.core import (
ClassicalHazardCalculator)
job = engine.prepare_job()
job_profile, params, sections = engine.import_job_profile(
demo_file('simple_fault_demo_hazard/config.gem'), job)
job_ctxt = engine.JobContext(params,
job.id,
oq_job_profile=job_profile,
oq_job=job)
job_ctxt.to_kvs()
with patch('openquake.utils.tasks.get_running_job') as grc_mock:
# Loading of the JobContext is done by
# `get_running_job`, which is covered by other tests.
# So, we just want to make sure that it's called here.
grc_mock.return_value = job_ctxt
calculator = tasks.calculator_for_task(job.id, 'hazard')
self.assertTrue(isinstance(calculator, ClassicalHazardCalculator))
self.assertEqual(1, grc_mock.call_count)
def test_bcr_risk_export(self):
# Tests that outputs of a risk classical calculation are
# exported
target_dir = tempfile.mkdtemp()
try:
cfg = helpers.demo_file('classical_bcr/job.ini')
# run the calculation to create something to export
retcode = helpers.run_risk_job_sp(cfg, self.hazard_id,
silence=True)
self.assertEqual(0, retcode)
job = models.OqJob.objects.latest('id')
outputs = export_core.get_outputs(job.id)
expected_outputs = 1 # 1 bcr distribution
self.assertEqual(expected_outputs, len(outputs))
# Export the loss curves:
distribution = outputs.filter(output_type='bcr_distribution')[0]
rc_files = risk.export(distribution.id, target_dir)
self.assertEqual(1, len(rc_files))
for f in rc_files:
self._test_exported_file(f)
finally:
shutil.rmtree(target_dir)
def hazard_id(self):
job = helpers.get_hazard_job(
helpers.demo_file("scenario_hazard/job.ini"))
job.hazard_calculation = models.HazardCalculation.objects.create(
owner=job.hazard_calculation.owner,
truncation_level=job.hazard_calculation.truncation_level,
maximum_distance=job.hazard_calculation.maximum_distance,
intensity_measure_types_and_levels=(
job.hazard_calculation.intensity_measure_types_and_levels),
calculation_mode="scenario")
job.save()
output = models.Output.objects.create_output(
job, "Test Hazard output", "gmf_scenario")
fname = os.path.join(os.path.dirname(__file__), 'gmf_scenario.csv')
with open(fname, 'rb') as csvfile:
gmfreader = csv.reader(csvfile, delimiter=',')
locations = gmfreader.next()
arr = numpy.array([map(float, row) for row in gmfreader])
for i, gmvs in enumerate(arr.transpose()):
models.GmfScenario.objects.create(
output=output,
imt="MMI",
gmvs=gmvs,
result_grp_ordinal=1,
location="POINT(%s)" % locations[i])
return output.id
def test_parameterize_sites_with_site_model(self):
job_ctxt = helpers.prepare_job_context(
helpers.demo_file(
'simple_fault_demo_hazard/config_with_site_model.gem'))
calc = classical.ClassicalHazardCalculator(job_ctxt)
calc.initialize()
# This tests to ensure that the `initialize` implementation for this
# calculator properly stores the site model in the DB.
# NOTE: If this test ever breaks, it's probably because the
# ClassicalHazardCalculator is no longer calling the `initalize` code
# in its super class (BaseHazardCalculator).
site_model = hazard_general.get_site_model(job_ctxt.oq_job.id)
self.assertIsNotNone(site_model)
set_params_patch = helpers.patch(
'openquake.calculators.hazard.general.set_java_site_parameters')
closest_data_patch = helpers.patch(
'openquake.calculators.hazard.general.get_closest_site_model_data')
sp_mock = set_params_patch.start()
cd_mock = closest_data_patch.start()
try:
calc.parameterize_sites(job_ctxt.sites_to_compute())
exp_call_count = len(job_ctxt.sites_to_compute())
self.assertEqual(exp_call_count, sp_mock.call_count)
self.assertEqual(exp_call_count, cd_mock.call_count)
finally:
# tear down the patches
set_params_patch.stop()
closest_data_patch.stop()
def test_parameterize_sites_no_site_model(self):
job_ctxt = helpers.prepare_job_context(
helpers.demo_file('simple_fault_demo_hazard/config.gem'))
calc = classical.ClassicalHazardCalculator(job_ctxt)
jsites = calc.parameterize_sites(job_ctxt.sites_to_compute())
# expected params:
jp = job_ctxt.oq_job_profile
exp_sadigh = job_params.REVERSE_ENUM_MAP[jp.sadigh_site_type]
exp_vs30 = jp.reference_vs30_value
exp_vs30_type = jp.vs30_type
exp_z1pt0 = jp.depth_to_1pt_0km_per_sec
exp_z2pt5 = jp.reference_depth_to_2pt5km_per_sec_param
for jsite in jsites:
self.assertEqual(exp_vs30,
jsite.getParameter('Vs30').getValue().value)
self.assertEqual(exp_vs30_type,
jsite.getParameter('Vs30 Type').getValue())
self.assertEqual(
exp_z1pt0,
jsite.getParameter('Depth 1.0 km/sec').getValue().value)
self.assertEqual(
exp_z2pt5,
jsite.getParameter('Depth 2.5 km/sec').getValue().value)
self.assertEqual(exp_sadigh,
jsite.getParameter('Sadigh Site Type').getValue())
def test_complex_fault_demo_hazard_nrml_written_once(self):
"""
Run the `complex_fault_demo_hazard` demo and verify that the
NRML files are written only once.
"""
def filter_multi():
"""Filter and return files that were written more than once."""
counts = defaultdict(int)
files = stats.kvs_op("lrange", key, 0, -1)
for file in files:
counts[file] += 1
return [(f, c) for f, c in counts.iteritems() if c > 1]
job_cfg = helpers.demo_file(os.path.join(
"complex_fault_demo_hazard", "config.gem"))
helpers.run_job(job_cfg, ['--output-type=xml'])
self.job = models.OqJob.objects.latest("id")
key = stats.key_name(
self.job.id, *stats.STATS_KEYS["hcls_xmlcurvewrites"])
if key:
multi_writes = filter_multi()
self.assertFalse(multi_writes, str(multi_writes))
key = stats.key_name(
self.job.id, *stats.STATS_KEYS["hcls_xmlmapwrites"])
if key:
multi_writes = filter_multi()
self.assertFalse(multi_writes, str(multi_writes))
def test_complex_fault_demo_hazard_nrml_written_once(self):
"""
Run the `complex_fault_demo_hazard` demo and verify that the
NRML files are written only once.
"""
def filter_multi():
"""Filter and return files that were written more than once."""
counts = defaultdict(int)
files = stats.kvs_op("lrange", key, 0, -1)
for file in files:
counts[file] += 1
return [(f, c) for f, c in counts.iteritems() if c > 1]
job_cfg = helpers.demo_file(os.path.join(
"complex_fault_demo_hazard", "config.gem"))
helpers.run_job(job_cfg, output="xml")
self.job = models.OqCalculation.objects.latest("id")
key = stats.key_name(
self.job.id, *stats.STATS_KEYS["hcls_xmlcurvewrites"])
if key:
multi_writes = filter_multi()
self.assertFalse(multi_writes, str(multi_writes))
key = stats.key_name(
self.job.id, *stats.STATS_KEYS["hcls_xmlmapwrites"])
if key:
multi_writes = filter_multi()
self.assertFalse(multi_writes, str(multi_writes))
def test_initialize_site_model(self):
# we need a slightly different config file for this test
cfg = helpers.demo_file(
'simple_fault_demo_hazard/job_with_site_model.ini')
self.job = helpers.get_hazard_job(cfg)
self.calc = core.ClassicalHazardCalculator(self.job)
self.calc.initialize_site_model()
# If the site model isn't valid for the calculation geometry, a
# `RuntimeError` should be raised here
# Okay, it's all good. Now check the count of the site model records.
[site_model_inp] = models.inputs4hcalc(
self.job.hazard_calculation.id, input_type='site_model')
sm_nodes = models.SiteModel.objects.filter(input=site_model_inp)
self.assertEqual(2601, len(sm_nodes))
num_pts_to_compute = len(
self.job.hazard_calculation.points_to_compute())
[site_data] = models.SiteData.objects.filter(
hazard_calculation=self.job.hazard_calculation.id)
# The site model is good. Now test that `site_data` was computed.
# For now, just test the lengths of the site data collections:
self.assertEqual(num_pts_to_compute, len(site_data.lons))
self.assertEqual(num_pts_to_compute, len(site_data.lats))
self.assertEqual(num_pts_to_compute, len(site_data.vs30s))
self.assertEqual(num_pts_to_compute, len(site_data.vs30_measured))
self.assertEqual(num_pts_to_compute, len(site_data.z1pt0s))
self.assertEqual(num_pts_to_compute, len(site_data.z2pt5s))
def hazard_id(self):
job = helpers.get_hazard_job(
helpers.demo_file("scenario_hazard/job.ini"))
hc = job.hazard_calculation
job.hazard_calculation = models.HazardCalculation.objects.create(
owner=hc.owner, truncation_level=hc.truncation_level,
maximum_distance=hc.maximum_distance,
intensity_measure_types=["PGA"],
calculation_mode="scenario")
job.status = "complete"
job.save()
output = models.Output.objects.create_output(
job, "Test Hazard output", "gmf_scenario")
fname = os.path.join(os.path.dirname(__file__), 'gmf_scenario.csv')
with open(fname, 'rb') as csvfile:
gmfreader = csv.reader(csvfile, delimiter=',')
locations = gmfreader.next()
arr = numpy.array([[float(x) for x in row] for row in gmfreader])
for i, gmvs in enumerate(arr):
models.GmfScenario.objects.create(
output=output,
imt="PGA",
gmvs=gmvs,
location="POINT(%s)" % locations[i])
return output.id
def test_peer_test_set_1_case_10(self):
expected_results = self._load_results("PeerTestSet1Case10")
self._run_job(helpers.demo_file(
os.path.join("PeerTestSet1Case10", "config.gem")))
self._assert_results_are(expected_results)
def test_peer_test_set_1_case_5(self):
expected_results = load_exp_hazcurve_results("PeerTestSet1Case5")
helpers.run_job(helpers.demo_file(
os.path.join("PeerTestSet1Case5", "config.gem")))
self._assert_hazcurve_results_are(expected_results)
def test_calculator_for_task(self):
"""Load up a sample calculation (into the db and cache) and make sure
we can instantiate the correct calculator for a given calculation id.
"""
from openquake.calculators.hazard.classical.core import (
ClassicalHazardCalculator)
job = engine.prepare_job()
job_profile, params, sections = engine.import_job_profile(demo_file(
'simple_fault_demo_hazard/config.gem'), job)
job_ctxt = engine.JobContext(params, job.id,
oq_job_profile=job_profile,
oq_job=job)
job_ctxt.to_kvs()
with patch(
'openquake.utils.tasks.get_running_job') as grc_mock:
# Loading of the JobContext is done by
# `get_running_job`, which is covered by other tests.
# So, we just want to make sure that it's called here.
grc_mock.return_value = job_ctxt
calculator = tasks.calculator_for_task(job.id, 'hazard')
self.assertTrue(isinstance(calculator, ClassicalHazardCalculator))
self.assertEqual(1, grc_mock.call_count)
def test_compute_uhs_with_site_model(self):
the_job = helpers.prepare_job_context(
helpers.demo_file('uhs/config_with_site_model.gem'))
the_job.to_kvs()
site = Site(0, 0)
helpers.store_hazard_logic_trees(the_job)
get_sm_patch = helpers.patch(
'openquake.calculators.hazard.general.get_site_model')
get_closest_patch = helpers.patch(
'openquake.calculators.hazard.general.get_closest_site_model_data')
compute_patch = helpers.patch(
'openquake.calculators.hazard.uhs.core._compute_uhs')
get_sm_mock = get_sm_patch.start()
get_closest_mock = get_closest_patch.start()
compute_mock = compute_patch.start()
get_closest_mock.return_value = SiteModel(
vs30=800, vs30_type='measured', z1pt0=100, z2pt5=200)
try:
compute_uhs(the_job, site)
self.assertEqual(1, get_sm_mock.call_count)
self.assertEqual(1, get_closest_mock.call_count)
self.assertEqual(1, compute_mock.call_count)
finally:
get_sm_patch.stop()
get_closest_patch.stop()
compute_patch.stop()
def hazard_id(self):
job = helpers.get_hazard_job(
helpers.demo_file("simple_fault_demo_hazard/job.ini"))
hazard_curve = [
(0.001, 0.0398612669790014),
(0.01, 0.039861266979001400), (0.05, 0.039728757480298900),
(0.10, 0.029613426625612500), (0.15, 0.019827328756491600),
(0.20, 0.013062270161451900), (0.25, 0.008655387950000430),
(0.30, 0.005898520593689670), (0.35, 0.004061698589511780),
(0.40, 0.002811727179526820), (0.45, 0.001995117417776690),
(0.50, 0.001358705972845710), (0.55, 0.000989667841573727),
(0.60, 0.000757544444296432), (0.70, 0.000272824002045979),
(0.80, 0.00), (0.9, 0.00), (1.0, 0.00)]
hd = models.HazardCurveData.objects.create(
hazard_curve=models.HazardCurve.objects.create(
output=models.Output.objects.create_output(
job, "Test Hazard curve", "hazard_curve"),
investigation_time=50,
imt="PGA", imls=[hz[0] for hz in hazard_curve],
statistics="mean"),
poes=[hz[1] for hz in hazard_curve],
location="POINT(1 1)")
return hd.hazard_curve.output.id
def test_calculator_for_task(self):
"""Load up a sample calculation (into the db and cache) and make sure
we can instantiate the correct calculator for a given calculation id.
"""
from openquake.calculators.hazard.classical.core import (
ClassicalHazardCalculator)
job_profile, params, sections = engine.import_job_profile(demo_file(
'simple_fault_demo_hazard/config.gem'))
calculation = OqCalculation(owner=job_profile.owner,
oq_job_profile=job_profile)
calculation.save()
calc_proxy = engine.CalculationProxy(params, calculation.id,
oq_job_profile=job_profile,
oq_calculation=calculation)
calc_proxy.to_kvs()
with patch(
'openquake.utils.tasks.get_running_calculation') as grc_mock:
# Loading of the CalculationProxy is done by
# `get_running_calculation`, which is covered by other tests.
# So, we just want to make sure that it's called here.
grc_mock.return_value = calc_proxy
calculator = tasks.calculator_for_task(calculation.id, 'hazard')
self.assertTrue(isinstance(calculator, ClassicalHazardCalculator))
self.assertEqual(1, grc_mock.call_count)
def test_export_for_scenario(self):
target_dir = tempfile.mkdtemp()
try:
cfg = helpers.demo_file('scenario_hazard/job.ini')
# run the calculation to create something to export
retcode = helpers.run_hazard_job_sp(cfg, silence=True)
self.assertEqual(0, retcode)
job = models.OqJob.objects.latest('id')
outputs = export_core.get_outputs(job.id)
self.assertEqual(1, len(outputs)) # 1 GMF
gmf_outputs = outputs.filter(output_type='gmf_scenario')
self.assertEqual(1, len(gmf_outputs))
exported_files = hazard.export(gmf_outputs[0].id, target_dir)
self.assertEqual(1, len(exported_files))
# Check the file paths exist, is absolute, and the file isn't
# empty.
f = exported_files[0]
self._test_exported_file(f)
# Check for the correct number of GMFs in the file:
tree = etree.parse(f)
self.assertEqual(10, number_of('nrml:gmf', tree))
finally:
shutil.rmtree(target_dir)
def test_peer_test_set_1_case_5(self):
expected_results = load_exp_hazcurve_results("PeerTestSet1Case5")
helpers.run_job(
helpers.demo_file(os.path.join("PeerTestSet1Case5", "config.gem")))
self._assert_hazcurve_results_are(expected_results)
def test_hazard_input_on_exposure_sites(self):
cfg = helpers.demo_file(
"scenario_risk/config_hzr_exposure.gem")
ret_code = helpers.run_job(cfg, ["--output-type=xml"])
self.assertEquals(0, ret_code)
self.assertEqual("succeeded", OqJob.objects.latest("id").status)
def test_scenario_risk(self):
# This test exercises the 'mean-based' path through the Scenario Risk
# calculator. There is no random sampling done here so the results are
# 100% predictable.
scen_cfg = helpers.demo_file('scenario_risk/config.gem')
exp_mean_loss = 1053.09
exp_stddev_loss = 246.62
expected_loss_map = [
dict(asset='a3',
pos='15.48 38.25',
mean=180.717534009275,
stddev=92.2122644809969),
dict(asset='a2',
pos='15.56 38.17',
mean=432.225448142534,
stddev=186.864456949986),
dict(asset='a1',
pos='15.48 38.09',
mean=440.147078317589,
stddev=182.615976701858),
]
result = helpers.run_job(scen_cfg, ['--output-type=xml'],
check_output=True)
job = OqJob.objects.latest('id')
self.assertEqual('succeeded', job.status)
expected_loss_map_file = helpers.demo_file(
'scenario_risk/computed_output/loss-map-%s.xml' % job.id)
self.assertTrue(os.path.exists(expected_loss_map_file))
helpers.verify_loss_map(self, expected_loss_map_file,
expected_loss_map, self.LOSSMAP_PRECISION)
actual_mean, actual_stddev = helpers.mean_stddev_from_result_line(
result)
self.assertAlmostEqual(exp_mean_loss,
actual_mean,
places=self.TOTAL_LOSS_PRECISION)
self.assertAlmostEqual(exp_stddev_loss,
actual_stddev,
places=self.TOTAL_LOSS_PRECISION)
def setUp(self):
cfg = helpers.demo_file('simple_fault_demo_hazard/job.ini')
self.job = helpers.get_hazard_job(cfg, username="******")
for i in range(0, random.randint(1, 10)):
models.LtRealization(
hazard_calculation=self.job.hazard_calculation,
ordinal=i, seed=None, weight=1 / (i + 1), sm_lt_path=[i],
gsim_lt_path=[i], total_items=0, completed_items=0).save()
def test_log_file_access_denied(self):
# Attempt to log to a location for which the user does not have write
# access ('/', for example).
uhs_cfg = helpers.demo_file('uhs/config.gem')
result = helpers.run_job(uhs_cfg, ['--log-file', '/oq.log'],
check_output=True)
self.assertEqual(
'Error writing to log file /oq.log: Permission denied\n', result)
def test_hazard_computed_on_exposure_sites(self):
# slightly different configuration where we
# run the engine triggering the hazard computation
# on sites defined in the exposure file
cfg = helpers.demo_file(
"classical_psha_based_risk/config_hzr_exposure.gem")
self._run_job(cfg)
self._verify_job_succeeded()
def hazard_id(self):
job = helpers.get_hazard_job(helpers.demo_file("event_based_hazard/job.ini"))
job.hazard_calculation = models.HazardCalculation.objects.create(
owner=job.hazard_calculation.owner,
truncation_level=job.hazard_calculation.truncation_level,
maximum_distance=job.hazard_calculation.maximum_distance,
intensity_measure_types_and_levels=(job.hazard_calculation.intensity_measure_types_and_levels),
calculation_mode="event_based",
investigation_time=50,
ses_per_logic_tree_path=1,
)
job.save()
hc = job.hazard_calculation
lt_realization = models.LtRealization.objects.create(
hazard_calculation=job.hazard_calculation,
ordinal=1,
seed=1,
weight=None,
sm_lt_path="test_sm",
gsim_lt_path="test_gsim",
is_complete=False,
total_items=1,
completed_items=1,
)
gmf_set = models.GmfSet.objects.create(
gmf_collection=models.GmfCollection.objects.create(
output=models.Output.objects.create_output(job, "Test Hazard output", "gmf"),
lt_realization=lt_realization,
complete_logic_tree_gmf=False,
),
investigation_time=hc.investigation_time,
ses_ordinal=1,
complete_logic_tree_gmf=False,
)
with open(os.path.join(os.path.dirname(__file__), "gmf.csv"), "rb") as csvfile:
gmfreader = csv.reader(csvfile, delimiter=",")
locations = gmfreader.next()
gmv_matrix = numpy.array([[float(x) for x in row] for row in gmfreader]).transpose()
rupture_ids = helpers.get_rupture_ids(job, hc, lt_realization, len(gmv_matrix[0]))
for i, gmvs in enumerate(gmv_matrix):
models.Gmf.objects.create(
gmf_set=gmf_set,
imt="PGA",
gmvs=gmvs,
rupture_ids=map(str, rupture_ids),
result_grp_ordinal=1,
location="POINT(%s)" % locations[i],
)
return gmf_set.gmf_collection.output.id
def test_probabilistic_risk(self):
cfg = helpers.demo_file("probabilistic_event_based_risk/config_qa.gem")
self._run_job(cfg)
self._verify_job_succeeded()
self._verify_loss_maps()
self._verify_loss_ratio_curves()
self._verify_loss_curves()
self._verify_aggregate_curve()
def test_classical_psha_based_risk(self):
cfg = helpers.demo_file(
'classical_psha_based_risk/config.gem')
self._run_job(cfg)
self._verify_job_succeeded()
self._verify_loss_curve()
self._verify_loss_ratio_curve()
self._verify_loss_maps()
def test_scenario_damage_con(self):
cfg = helpers.demo_file("scenario_damage_risk/config.gem")
self._run_job(cfg)
self._verify_job_succeeded()
self._verify_damage_states()
self._verify_dist_per_asset_con()
self._verify_dist_per_taxonomy_con()
self._verify_total_dist_con()
def test_scenario_damage_con(self):
cfg = helpers.demo_file("scenario_damage_risk/config.gem")
self._run_job(cfg)
self._verify_job_succeeded()
self._verify_damage_states()
self._verify_dist_per_asset_con()
self._verify_dist_per_taxonomy_con()
self._verify_total_dist_con()
def test_scenario_risk(self):
# The rudimentary beginnings of a QA test for the scenario calc.
# For now, just run it end-to-end to make sure it doesn't blow up.
scen_cfg = helpers.demo_file('scenario_risk/config.gem')
ret_code = helpers.run_job(scen_cfg, ['--output-type=xml'])
self.assertEqual(0, ret_code)
job = OqJob.objects.latest('id')
self.assertEqual('succeeded', job.status)
def test_log_file_access_denied(self):
# Attempt to log to a location for which the user does not have write
# access ('/', for example).
uhs_cfg = helpers.demo_file('uhs/config.gem')
result = helpers.run_job(uhs_cfg, ['--log-file', '/oq.log'],
check_output=True)
self.assertEqual(
'Error writing to log file /oq.log: Permission denied\n',
result)
def test_classical_psha_based_risk(self):
cfg = helpers.demo_file(
'classical_psha_based_risk/config.gem')
self._run_job(cfg)
self._verify_job_succeeded()
self._verify_loss_curve()
self._verify_loss_ratio_curve()
self._verify_loss_maps()
def test_export_uhs(self):
# Tests the UHS calculation run and export end-to-end.
# For the export, we only check the quantity, location, and names of
# each exported file. We don't check the contents; that's covered in
# other tests.
uhs_cfg = helpers.demo_file('uhs/config.gem')
export_target_dir = tempfile.mkdtemp()
expected_export_files = [
os.path.join(export_target_dir, 'uhs_poe:0.1.hdf5'),
os.path.join(export_target_dir, 'uhs_poe:0.02.hdf5'),
os.path.join(export_target_dir, 'uhs.xml'),
]
# Sanity check and precondition: these files should not exist yet
for f in expected_export_files:
self.assertFalse(os.path.exists(f))
try:
ret_code = helpers.run_job(uhs_cfg)
self.assertEqual(0, ret_code)
job = models.OqJob.objects.latest('id')
[output] = models.Output.objects.filter(oq_job=job.id)
# Split into a list, 1 result for each row in the output.
# The first row of output (the table header) is discarded.
listed_calcs = helpers.prepare_cli_output(
subprocess.check_output(
['bin/openquake', '--list-calculations']))
check_list_calcs(self, listed_calcs, job.id)
listed_outputs = helpers.prepare_cli_output(
subprocess.check_output(
['bin/openquake', '--list-outputs',
str(job.id)]))
check_list_outputs(self, listed_outputs, output.id, 'uh_spectra')
listed_exports = helpers.prepare_cli_output(
subprocess.check_output([
'bin/openquake', '--export',
str(output.id), export_target_dir
]))
self.assertEqual(expected_export_files, listed_exports)
# Check that the files actually have been created,
# and also verify that the paths are absolute:
for f in listed_exports:
self.assertTrue(os.path.exists(f))
self.assertTrue(os.path.isabs(f))
finally:
shutil.rmtree(export_target_dir)
def test_default_validators_disagg_job(self):
"""Test to ensure that a Disaggregation job always includes the
:class:`openquake.job.config.DisaggregationValidator`.
"""
da_job_path = helpers.demo_file('disaggregation/config.gem')
da_job = helpers.job_from_file(da_job_path)
validators = config.default_validators(da_job.sections, da_job.params)
# test that the default validators include a DisaggregationValidator
self.assertTrue(
any(isinstance(v, DisaggregationValidator) for v in validators))
def test_default_validators_classical_bcr_risk(self):
# For Classical BCR Hazard+Risk calculations, ensure that a
# `ClassicalRiskValidator` is included in the default validators.
cfg_path = helpers.demo_file('benefit_cost_ratio/config.gem')
job_profile, params, sections = engine.import_job_profile(
cfg_path, self.job)
validators = config.default_validators(sections, params)
self.assertTrue(
any(isinstance(v, ClassicalRiskValidator) for v in validators))
def test_default_validators_event_based_bcr_risk(self):
# For Event-Based BCR Risk calculations, ensure that a
# `EventBasedRiskValidator` is included in the default validators.
cfg_path = helpers.demo_file('benefit_cost_ratio/config_ebased.gem')
job_profile, params, sections = engine.import_job_profile(
cfg_path, self.job)
validators = config.default_validators(sections, params)
self.assertTrue(
any(isinstance(v, EventBasedRiskValidator) for v in validators))
def test_complex_fault_demo_hazard(self):
"""Run the `complex_fault_demo_hazard` demo and verify all of the
resulting hazard curve and hazard map data."""
job_cfg = helpers.demo_file(
os.path.join("complex_fault_demo_hazard", "config.gem"))
exp_results_dir = os.path.join("complex_fault_demo_hazard",
"expected_results")
helpers.run_job(job_cfg)
self.job = models.OqJob.objects.latest("id")
# Check hazard curves for sample 0:
# Hazard curve expected results for logic tree sample 0:
hazcurve_0 = helpers.demo_file(
os.path.join(exp_results_dir, "hazardcurve-0.dat"))
verify_hazcurve_results(self, self.job, hazcurve_0, end_branch_label=0)
# Check mean hazard curves:
hazcurve_mean = helpers.demo_file(
os.path.join(exp_results_dir, "hazardcurve-mean.dat"))
verify_hazcurve_results(self,
self.job,
hazcurve_mean,
statistic_type="mean")
# Check hazard map mean 0.02:
hazmap_mean_0_02 = helpers.demo_file(
os.path.join(exp_results_dir, "hazardmap-0.02-mean.dat"))
verify_hazmap_results(self, self.job,
load_expected_map(hazmap_mean_0_02), 0.02,
"mean")
# Check hazard map mean 0.1:
hazmap_mean_0_1 = helpers.demo_file(
os.path.join(exp_results_dir, "hazardmap-0.1-mean.dat"))
verify_hazmap_results(self, self.job,
load_expected_map(hazmap_mean_0_1), 0.1, "mean")
def setUp(self):
self.job = engine.prepare_job()
self.job_profile, self.params, _sections = (engine.import_job_profile(
demo_file('simple_fault_demo_hazard/config.gem'), self.job))
self.params['debug'] = 'warn'
# Cache the calc proxy data into the kvs:
job_ctxt = engine.JobContext(self.params,
self.job.id,
oq_job_profile=self.job_profile,
oq_job=self.job)
job_ctxt.to_kvs()
def test_default_validators_classical_job(self):
"""Test to ensure that a classical always includes the
:class:`openquake.job.config.ClassicalValidator`.
"""
classical_risk_job_path = helpers.demo_file(
'classical_psha_based_risk/config.gem')
classical_risk_job = helpers.job_from_file(classical_risk_job_path)
validators = config.default_validators(classical_risk_job.sections,
classical_risk_job.params)
self.assertTrue(
any(isinstance(v, ClassicalValidator) for v in validators))
def test_default_validators_scenario_job(self):
"""Test to ensure that a Scenario job always includes the
:class:`openquake.job.config.ScenarioComputationValidator`."""
scenario_job_path = helpers.demo_file('scenario_risk/config.gem')
scenario_job = helpers.job_from_file(scenario_job_path)
validators = config.default_validators(scenario_job.sections,
scenario_job.params)
self.assertTrue(
any(
isinstance(v, ScenarioComputationValidator)
for v in validators))
def test__serialize_xml_filenames(self):
# Test that the file names of the loss XML artifacts are correct.
# See https://bugs.launchpad.net/openquake/+bug/894706.
expected_lrc_file_name = (
'losscurves-block-#%(job_id)s-block#%(block)s.xml')
expected_lr_file_name = (
'losscurves-loss-block-#%(job_id)s-block#%(block)s.xml')
cfg_file = helpers.demo_file('classical_psha_based_risk/config.gem')
job = engine.prepare_job()
job_profile, params, sections = engine.import_job_profile(
cfg_file, job)
job_ctxt = engine.JobContext(params,
job.id,
sections=sections,
serialize_results_to=['xml', 'db'],
oq_job_profile=job_profile,
oq_job=job)
calculator = ClassicalRiskCalculator(job_ctxt)
with helpers.patch('openquake.writer.FileWriter.serialize'):
# The 'curves' key in the kwargs just needs to be present;
# because of the serialize mock in place above, it doesn't need
# to have a real value.
# First, we test loss ratio curve output,
# then we'll do the same test for loss curve output.
# We expect to get a single file path back.
[file_path] = calculator._serialize(
0, **dict(curve_mode='loss_ratio', curves=[]))
_dir, file_name = os.path.split(file_path)
self.assertEqual(
expected_lrc_file_name % dict(job_id=job.id, block=0),
file_name)
# The same test again, except for loss curves this time.
[file_path
] = calculator._serialize(0, **dict(curve_mode='loss', curves=[]))
_dir, file_name = os.path.split(file_path)
self.assertEqual(
expected_lr_file_name % dict(job_id=job.id, block=0),
file_name)
def test_disagg(self):
self.setup_config()
os.environ.update(self.orig_env)
cp = ConfigParser.SafeConfigParser()
cp.read('openquake.cfg.test_bakk')
cp.set('nfs', 'base_dir', '/tmp')
cp.write(open('openquake.cfg', 'w'))
try:
self._do_test(
helpers.demo_file('disaggregation/config_with_site_model.gem')
)
finally:
self.teardown_config()
def test_initialize_calls_validate(self):
# Test make sure the calculator `initialize` calls
# `validate_site_model`.
job_ctxt = helpers.prepare_job_context(
helpers.demo_file(
'simple_fault_demo_hazard/config_with_site_model.gem'))
calc = general.BaseHazardCalculator(job_ctxt)
patch_path = 'openquake.calculators.hazard.general.validate_site_model'
with helpers.patch(patch_path) as validate_patch:
calc.initialize()
# validate_site_model itself is tested in another test
# here, we just make sure it gets called
self.assertEqual(1, validate_patch.call_count)
def test_log_file(self):
# Test logging to a file when running bin/oqscript.py.
uhs_cfg = helpers.demo_file('uhs/config.gem')
log_file = './%s.log' % helpers.random_string()
self.assertFalse(os.path.exists(log_file))
ret_code = helpers.run_job(
uhs_cfg, ['--log-level', 'debug', '--log-file', log_file])
self.assertEqual(0, ret_code)
self.assertTrue(os.path.exists(log_file))
# Make sure there is something in it.
self.assertTrue(os.path.getsize(log_file) > 0)
os.unlink(log_file)
