def test_validate_warns(self):
# Test that `validate` raises warnings if unnecessary parameters are
# specified for a given calculation.
# For example, `ses_per_logic_tree_path` is an event-based hazard
# param; if this param is specified for a classical hazard job, a
# warning should be raised.
cfg_file = helpers.get_data_path('simple_fault_demo_hazard/job.ini')
job = engine.prepare_job()
params = engine.parse_config(open(cfg_file, 'r'))
# Add a few superfluous parameters:
params['ses_per_logic_tree_path'] = 5
params['ground_motion_correlation_model'] = 'JB2009'
calculation = engine.create_calculation(
models.HazardCalculation, params)
job.hazard_calculation = calculation
job.save()
with warnings.catch_warnings(record=True) as w:
validation.validate(job, 'hazard', params, ['xml'])
expected_warnings = [
"Unknown parameter '%s' for calculation mode 'classical'."
" Ignoring." % x for x in ('ses_per_logic_tree_path',
'ground_motion_correlation_model')
]
actual_warnings = [m.message.message for m in w]
self.assertEqual(sorted(expected_warnings), sorted(actual_warnings))
def test_parse_config_with_sites_csv(self):
sites_csv = helpers.touch(content='1.0,2.1\n3.0,4.1\n5.0,6.1')
try:
source = StringIO.StringIO("""
[general]
calculation_mode = classical
[geometry]
sites_csv = %s
[misc]
maximum_distance=0
truncation_level=3
random_seed=5
""" % sites_csv)
source.name = 'path/to/some/job.ini'
exp_base_path = os.path.dirname(
os.path.join(os.path.abspath('.'), source.name))
expected_params = {
'base_path': exp_base_path,
'sites': 'MULTIPOINT(1.0 2.1, 3.0 4.1, 5.0 6.1)',
'calculation_mode': 'classical',
'truncation_level': '3',
'random_seed': '5',
'maximum_distance': '0',
'inputs': {},
}
params = engine.parse_config(source)
self.assertEqual(expected_params, params)
finally:
os.unlink(sites_csv)
def test_validate_warns(self):
# Test that `validate` raises warnings if unnecessary parameters are
# specified for a given calculation.
# For example, `ses_per_logic_tree_path` is an event-based hazard
# param; if this param is specified for a classical hazard job, a
# warning should be raised.
cfg_file = helpers.get_data_path('simple_fault_demo_hazard/job.ini')
job = engine.prepare_job()
params = engine.parse_config(open(cfg_file, 'r'))
# Add a few superfluous parameters:
params['ses_per_logic_tree_path'] = 5
params['ground_motion_correlation_model'] = 'JB2009'
calculation = engine.create_calculation(models.HazardCalculation,
params)
job.hazard_calculation = calculation
job.save()
with warnings.catch_warnings(record=True) as w:
validation.validate(job, 'hazard', params, ['xml'])
expected_warnings = [
"Unknown parameter '%s' for calculation mode 'classical'."
" Ignoring." % x for x in ('ses_per_logic_tree_path',
'ground_motion_correlation_model')
]
actual_warnings = [m.message.message for m in w]
self.assertEqual(sorted(expected_warnings), sorted(actual_warnings))
def test_parse_config_with_files(self):
temp_dir = tempfile.mkdtemp()
site_model_input = helpers.touch(dir=temp_dir, content="foo")
job_config = helpers.touch(dir=temp_dir, content="""
[general]
calculation_mode = classical
[site]
site_model_file = %s
maximum_distance=0
truncation_level=0
random_seed=0
""" % site_model_input)
try:
exp_base_path = os.path.dirname(job_config)
expected_params = {
'base_path': exp_base_path,
'calculation_mode': 'classical',
'truncation_level': '0',
'random_seed': '0',
'maximum_distance': '0',
'inputs': {'site_model': site_model_input},
}
params = engine.parse_config(open(job_config, 'r'))
self.assertEqual(expected_params, params)
self.assertEqual(['site_model'], params['inputs'].keys())
self.assertEqual([site_model_input], params['inputs'].values())
finally:
shutil.rmtree(temp_dir)
def test_parse_config_with_sites_csv(self):
sites_csv = helpers.touch(content='1.0,2.1\n3.0,4.1\n5.0,6.1')
try:
source = StringIO.StringIO("""
[general]
calculation_mode = classical
[geometry]
sites_csv = %s
[misc]
maximum_distance=0
truncation_level=3
random_seed=5
""" % sites_csv)
source.name = 'path/to/some/job.ini'
exp_base_path = os.path.dirname(
os.path.join(os.path.abspath('.'), source.name))
expected_params = {
'base_path': exp_base_path,
'sites': 'MULTIPOINT(1.0 2.1, 3.0 4.1, 5.0 6.1)',
'calculation_mode': 'classical',
'truncation_level': '3',
'random_seed': '5',
'maximum_distance': '0',
'inputs': {},
}
params = engine.parse_config(source)
self.assertEqual(expected_params, params)
finally:
os.unlink(sites_csv)
def test_parse_config_no_files(self):
# sections are there just for documentation
# when we parse the file, we ignore these
source = StringIO.StringIO("""
[general]
CALCULATION_MODE = classical
region = 1 1 2 2 3 3
[foo]
bar = baz
""")
# Add a 'name' to make this look like a real file:
source.name = 'path/to/some/job.ini'
exp_base_path = os.path.dirname(
os.path.join(os.path.abspath('.'), source.name))
expected_params = {
'base_path': exp_base_path,
'calculation_mode': 'classical',
'region': '1 1 2 2 3 3',
'bar': 'baz',
'inputs': {},
}
params = engine.parse_config(source)
self.assertEqual(expected_params, params)
def test(self):
# check that if risk models are provided, then the ``points to
# compute`` and the imls are got from there
username = helpers.default_user()
job = engine.prepare_job(username)
cfg = helpers.get_data_path('classical_job-sd-imt.ini')
params = engine.parse_config(open(cfg, 'r'))
haz_calc = engine.create_calculation(models.HazardCalculation, params)
haz_calc = models.HazardCalculation.objects.get(id=haz_calc.id)
job.hazard_calculation = haz_calc
job.is_running = True
job.save()
calc = get_calculator_class(
'hazard',
job.hazard_calculation.calculation_mode)(job)
calc.parse_risk_models()
self.assertEqual([(1.0, -1.0), (0.0, 0.0)],
[(point.latitude, point.longitude)
for point in haz_calc.points_to_compute()])
self.assertEqual(['PGA'], haz_calc.get_imts())
self.assertEqual(
3, haz_calc.oqjob.exposuremodel.exposuredata_set.count())
return job
def test_parse_config_with_files(self):
site_model_input = helpers.touch(content="foo")
try:
source = StringIO.StringIO("""
[general]
calculation_mode = classical
[site]
site_model_file = %s
maximum_distance=0
truncation_level=0
random_seed=0
""" % site_model_input)
# Add a 'name' to make this look like a real file:
source.name = 'path/to/some/job.ini'
exp_base_path = os.path.dirname(
os.path.join(os.path.abspath('.'), source.name))
expected_params = {
'base_path': exp_base_path,
'calculation_mode': 'classical',
'truncation_level': '0',
'random_seed': '0',
'maximum_distance': '0'
}
params, files = engine.parse_config(source)
self.assertEqual(expected_params, params)
self.assertEqual(['site_model_file'], files.keys())
self.assertEqual('acbd18db4cc2f85cedef654fccc4a4d8',
files['site_model_file'].digest)
finally:
os.unlink(site_model_input)
def get_risk_job(cfg, username=None, hazard_calculation_id=None,
hazard_output_id=None):
"""
Given a path to a config file and a hazard_calculation_id
(or, alternatively, a hazard_output_id, create a
:class:`openquake.engine.db.models.OqJob` object for a risk calculation.
"""
username = username if username is not None else default_user().user_name
# You can't specify both a hazard output and hazard calculation
# Pick one
assert not (hazard_calculation_id is not None
and hazard_output_id is not None)
job = engine.prepare_job(username)
params, files = engine.parse_config(open(cfg, 'r'))
params.update(
dict(hazard_output_id=hazard_output_id,
hazard_calculation_id=hazard_calculation_id)
)
risk_calc = engine.create_risk_calculation(
job.owner, params, files)
risk_calc = models.RiskCalculation.objects.get(id=risk_calc.id)
job.risk_calculation = risk_calc
job.save()
return job
def test_pre_execute_check_imts_raises(self):
haz_job = engine.prepare_job()
cfg = helpers.get_data_path('classical_job.ini')
params, files = engine.parse_config(open(cfg, 'r'))
haz_job.hazard_calculation = engine.create_hazard_calculation(
haz_job.owner, params, files.values())
haz_job.save()
hazard_curve_output = models.Output.objects.create_output(
haz_job, 'test_hazard_curve', 'hazard_curve'
)
models.HazardCurve.objects.create(
output=hazard_curve_output,
investigation_time=50.0,
imt='PGV', # the vulnerability model only defines SA(0.1)
statistics='mean'
)
cfg = helpers.get_data_path(
'end-to-end-hazard-risk/job_risk_classical.ini')
risk_job = helpers.get_risk_job(
cfg, hazard_output_id=hazard_curve_output.id
)
models.JobStats.objects.create(oq_job=risk_job)
calc = classical.ClassicalRiskCalculator(risk_job)
# Check for compatibility between the IMTs defined in the vulnerability
# model and the chosen hazard output (--hazard-output-id)
with self.assertRaises(ValueError) as ar:
calc.pre_execute()
self.assertEqual(
"There is no hazard output for: SA(0.1). "
"The available IMTs are: PGA.",
ar.exception.message)
def test_pre_execute_check_imts_no_errors(self):
haz_job = engine.prepare_job()
cfg = helpers.get_data_path(
'end-to-end-hazard-risk/job_haz_classical.ini')
params, files = engine.parse_config(open(cfg, 'r'))
haz_job.hazard_calculation = engine.create_hazard_calculation(
haz_job.owner, params, files.values())
haz_job.save()
hazard_curve_output = models.Output.objects.create_output(
haz_job, 'test_hazard_curve', 'hazard_curve'
)
models.HazardCurve.objects.create(
output=hazard_curve_output,
investigation_time=50.0,
# this imt is compatible with the vuln model
imt='SA',
sa_period=0.025,
sa_damping=5.0,
statistics='mean'
)
cfg = helpers.get_data_path(
'end-to-end-hazard-risk/job_risk_classical.ini')
risk_job = helpers.get_risk_job(
cfg, hazard_output_id=hazard_curve_output.id
)
models.JobStats.objects.create(oq_job=risk_job)
calc = classical.ClassicalRiskCalculator(risk_job)
# In contrast to the test above (`test_pre_execute_check_imts_raises`),
# we expect no errors to be raised.
calc.pre_execute()
def test_parse_config_with_files(self):
temp_dir = tempfile.mkdtemp()
site_model_input = helpers.touch(dir=temp_dir, content="foo")
job_config = helpers.touch(dir=temp_dir, content="""
[general]
calculation_mode = classical
[site]
site_model_file = %s
maximum_distance=0
truncation_level=0
random_seed=0
""" % site_model_input)
try:
exp_base_path = os.path.dirname(job_config)
expected_params = {
'base_path': exp_base_path,
'calculation_mode': 'classical',
'truncation_level': '0',
'random_seed': '0',
'maximum_distance': '0',
'inputs': {'site_model': site_model_input},
}
params = engine.parse_config(open(job_config, 'r'))
self.assertEqual(expected_params, params)
self.assertEqual(['site_model'], params['inputs'].keys())
self.assertEqual([site_model_input], params['inputs'].values())
finally:
shutil.rmtree(temp_dir)
def test_parse_config_no_files(self):
# sections are there just for documentation
# when we parse the file, we ignore these
source = StringIO.StringIO("""
[general]
CALCULATION_MODE = classical
region = 1 1 2 2 3 3
[foo]
bar = baz
""")
# Add a 'name' to make this look like a real file:
source.name = 'path/to/some/job.ini'
exp_base_path = os.path.dirname(
os.path.join(os.path.abspath('.'), source.name))
expected_params = {
'base_path': exp_base_path,
'calculation_mode': 'classical',
'region': '1 1 2 2 3 3',
'bar': 'baz',
'inputs': {},
}
params = engine.parse_config(source)
self.assertEqual(expected_params, params)
def test_read_and_validate_hazard_config(self):
cfg = helpers.get_data_path('simple_fault_demo_hazard/job.ini')
job = engine.prepare_job(getpass.getuser())
params, files = engine.parse_config(open(cfg, 'r'))
calculation = engine.create_hazard_calculation(
job.owner, params, files.values())
form = validation.ClassicalHazardForm(
instance=calculation, files=files
)
self.assertTrue(form.is_valid())
def test_a_few_inputs(self):
cfg = helpers.get_data_path('simple_fault_demo_hazard/job.ini')
params, files = engine.parse_config(open(cfg, 'r'))
owner = helpers.default_user()
hc = engine.create_hazard_calculation(
owner.user_name, params, files
)
inputs = models.inputs4hcalc(hc.id)
# We expect 3: the two logic trees and one source model
self.assertEqual(3, inputs.count())
def test_with_input_type(self):
cfg = helpers.get_data_path('simple_fault_demo_hazard/job.ini')
params, files = engine.parse_config(open(cfg, 'r'))
owner = helpers.default_user()
hc = engine.create_hazard_calculation(
owner.user_name, params, files
)
inputs = models.inputs4hcalc(
hc.id, input_type='source_model_logic_tree'
)
self.assertEqual(1, inputs.count())
def test_a_few_inputs(self):
cfg = helpers.get_data_path('simple_fault_demo_hazard/job.ini')
params, files = engine.parse_config(open(cfg, 'r'))
owner = helpers.default_user()
hc = engine.create_hazard_calculation(owner, params, files.values())
expected_ids = sorted([x.id for x in files.values()])
inputs = models.inputs4hcalc(hc.id)
actual_ids = sorted([x.id for x in inputs])
self.assertEqual(expected_ids, actual_ids)
def test(self):
# check that if risk models are provided, then the ``points to
# compute`` and the imls are got from there
username = helpers.default_user().user_name
job = engine.prepare_job(username)
cfg = helpers.get_data_path('classical_job-sd-imt.ini')
params, files = engine.parse_config(open(cfg, 'r'))
haz_calc = engine.create_hazard_calculation(
job.owner.user_name, params, files)
haz_calc = models.HazardCalculation.objects.get(id=haz_calc.id)
job.hazard_calculation = haz_calc
job.is_running = True
job.save()
base_path = ('openquake.engine.calculators.hazard.classical.core'
'.ClassicalHazardCalculator')
init_src_patch = helpers.patch(
'%s.%s' % (base_path, 'initialize_sources'))
init_sm_patch = helpers.patch(
'%s.%s' % (base_path, 'initialize_site_model'))
init_rlz_patch = helpers.patch(
'%s.%s' % (base_path, 'initialize_realizations'))
record_stats_patch = helpers.patch(
'%s.%s' % (base_path, 'record_init_stats'))
init_pr_data_patch = helpers.patch(
'%s.%s' % (base_path, 'initialize_pr_data'))
patches = (init_src_patch, init_sm_patch, init_rlz_patch,
record_stats_patch, init_pr_data_patch)
mocks = [p.start() for p in patches]
get_calculator_class(
'hazard',
job.hazard_calculation.calculation_mode)(job).pre_execute()
self.assertEqual([(1.0, -1.0), (0.0, 0.0)],
[(point.latitude, point.longitude)
for point in haz_calc.points_to_compute()])
self.assertEqual(['PGA'], haz_calc.get_imts())
self.assertEqual(3, haz_calc.exposure_model.exposuredata_set.count())
for i, m in enumerate(mocks):
m.stop()
patches[i].stop()
return job
def test(self):
# check that if risk models are provided, then the ``points to
# compute`` and the imls are got from there
username = helpers.default_user()
job = engine.prepare_job(username)
cfg = helpers.get_data_path('classical_job-sd-imt.ini')
params = engine.parse_config(open(cfg, 'r'))
haz_calc = engine.create_calculation(models.HazardCalculation, params)
haz_calc = models.HazardCalculation.objects.get(id=haz_calc.id)
job.hazard_calculation = haz_calc
job.is_running = True
job.save()
base_path = ('openquake.engine.calculators.hazard.classical.core'
'.ClassicalHazardCalculator')
init_src_patch = helpers.patch('%s.%s' %
(base_path, 'initialize_sources'))
init_sm_patch = helpers.patch('%s.%s' %
(base_path, 'initialize_site_model'))
init_rlz_patch = helpers.patch('%s.%s' %
(base_path, 'initialize_realizations'))
record_stats_patch = helpers.patch('%s.%s' %
(base_path, 'record_init_stats'))
init_pr_data_patch = helpers.patch('%s.%s' %
(base_path, 'initialize_pr_data'))
patches = (init_src_patch, init_sm_patch, init_rlz_patch,
record_stats_patch, init_pr_data_patch)
mocks = [p.start() for p in patches]
get_calculator_class(
'hazard',
job.hazard_calculation.calculation_mode)(job).pre_execute()
self.assertEqual([(1.0, -1.0), (0.0, 0.0)],
[(point.latitude, point.longitude)
for point in haz_calc.points_to_compute()])
self.assertEqual(['PGA'], haz_calc.get_imts())
self.assertEqual(3,
haz_calc.oqjob.exposuremodel.exposuredata_set.count())
for i, m in enumerate(mocks):
m.stop()
patches[i].stop()
return job
def get_hazard_job(cfg, username=None):
"""
Given a path to a config file, create a
:class:`openquake.engine.db.models.OqJob` object for a hazard calculation.
"""
username = username if username is not None else default_user().user_name
job = engine.prepare_job(username)
params, files = engine.parse_config(open(cfg, 'r'))
haz_calc = engine.create_hazard_calculation(
job.owner, params, files.values())
haz_calc = models.HazardCalculation.objects.get(id=haz_calc.id)
job.hazard_calculation = haz_calc
job.save()
return job
def test_with_input_type(self):
cfg = helpers.get_data_path('simple_fault_demo_hazard/job.ini')
params, files = engine.parse_config(open(cfg, 'r'))
owner = helpers.default_user()
hc = engine.create_hazard_calculation(owner, params, files.values())
# It should only be 1 id, actually.
expected_ids = [x.id for x in files.values()
if x.input_type == 'source_model_logic_tree']
inputs = models.inputs4hcalc(
hc.id, input_type='source_model_logic_tree'
)
actual_ids = sorted([x.id for x in inputs])
self.assertEqual(expected_ids, actual_ids)
def get_job(cfg, username="******", hazard_calculation_id=None,
hazard_output_id=None):
"""
Given a path to a config file and a hazard_calculation_id
(or, alternatively, a hazard_output_id, create a
:class:`openquake.engine.db.models.OqJob` object for a risk calculation.
"""
if hazard_calculation_id is None and hazard_output_id is None:
return engine.job_from_file(cfg, username, 'error', [])
job = engine.prepare_job(username)
params = engine.parse_config(open(cfg, 'r'))
params.update(
dict(hazard_output_id=hazard_output_id,
hazard_calculation_id=hazard_calculation_id)
)
risk_calc = engine.create_calculation(
models.RiskCalculation, params)
risk_calc = models.RiskCalculation.objects.get(id=risk_calc.id)
job.risk_calculation = risk_calc
job.save()
return job
def get_fake_risk_job(risk_cfg, hazard_cfg, output_type="curve",
username=None):
"""
Takes in input the paths to a risk job config file and a hazard job config
file.
Creates fake hazard outputs suitable to be used by a risk
calculation and then creates a :class:`openquake.engine.db.models.OqJob`
object for a risk calculation. It also returns the input files
referenced by the risk config file.
:param output_type: gmf, gmf_scenario, or curve
"""
username = username if username is not None else default_user().user_name
hazard_job = get_hazard_job(hazard_cfg, username)
hc = hazard_job.hazard_calculation
rlz = models.LtRealization.objects.create(
hazard_calculation=hazard_job.hazard_calculation,
ordinal=1, seed=1, weight=None,
sm_lt_path="test_sm", gsim_lt_path="test_gsim",
is_complete=False, total_items=1, completed_items=1)
if output_type == "curve":
models.HazardCurve.objects.create(
lt_realization=rlz,
output=models.Output.objects.create_output(
hazard_job, "Test Hazard output", "hazard_curve_multi"),
investigation_time=hc.investigation_time)
hazard_output = models.HazardCurve.objects.create(
lt_realization=rlz,
output=models.Output.objects.create_output(
hazard_job, "Test Hazard output", "hazard_curve"),
investigation_time=hc.investigation_time,
imt="PGA", imls=[0.1, 0.2, 0.3])
for point in ["POINT(-1.01 1.01)", "POINT(0.9 1.01)",
"POINT(0.01 0.01)", "POINT(0.9 0.9)"]:
models.HazardCurveData.objects.create(
hazard_curve=hazard_output,
poes=[0.1, 0.2, 0.3],
location="%s" % point)
elif output_type == "gmf_scenario":
hazard_output = models.Gmf.objects.create(
output=models.Output.objects.create_output(
hazard_job, "Test gmf scenario output", "gmf_scenario"))
site_ids = hazard_job.hazard_calculation.save_sites(
[(15.48, 38.0900001), (15.565, 38.17), (15.481, 38.25)])
for site_id in site_ids:
models.GmfData.objects.create(
gmf=hazard_output,
imt="PGA",
site_id=site_id,
gmvs=[0.1, 0.2, 0.3])
else:
hazard_output = create_gmf_data_records(
hazard_job, rlz)[0].gmf
hazard_job.status = "complete"
hazard_job.save()
job = engine.prepare_job(username)
params, files = engine.parse_config(open(risk_cfg, 'r'))
params.update(dict(hazard_output_id=hazard_output.output.id))
risk_calc = engine.create_risk_calculation(job.owner, params, files)
job.risk_calculation = risk_calc
job.save()
error_message = validate(job, 'risk', params, files, [])
# reload risk calculation to have all the types converted properly
job.risk_calculation = models.RiskCalculation.objects.get(id=risk_calc.id)
if error_message:
raise RuntimeError(error_message)
return job, files
