def test_parse_config_with_files_no_force_inputs(self):
site_model_input = helpers.touch(content="foo")
source = StringIO.StringIO("""
[general]
calculation_mode = classical
[site]
site_model_file = %s
maximum_distance=0
truncation_level=0
random_seed=0
""" % site_model_input)
# Add a 'name' to make this look like a real file:
source.name = 'path/to/some/job.ini'
exp_base_path = os.path.dirname(
os.path.join(os.path.abspath('.'), source.name))
expected_params = {
'base_path': exp_base_path,
'force_inputs': False,
'calculation_mode': 'classical',
'truncation_level': '0',
'random_seed': '0',
'maximum_distance': '0'
}
# Run first with force_inputs=True to create the new Input.
params, expected_files = engine2.parse_config(
source, force_inputs=True)
# In order for us to reuse the existing input, we need to associate
# each input with a successful job.
job = engine2.prepare_job(getpass.getuser())
job.hazard_calculation = engine2.create_hazard_calculation(
job.owner, params, expected_files.values())
job.status = 'complete'
job.save()
# Now run twice with force_inputs=False (the default).
source.seek(0)
params1, files1 = engine2.parse_config(source)
source.seek(0)
params2, files2 = engine2.parse_config(source)
# Check the params just for sanity.
self.assertEqual(expected_params, params1)
self.assertEqual(expected_params, params2)
# Finally, check that the Input returned by the latest 2 calls matches
# the input we created above.
self.assertEqual(len(expected_files), len(files1))
self.assertEqual(len(expected_files), len(files2))
self.assertEqual(
expected_files['site_model_file'].id, files1['site_model_file'].id)
self.assertEqual(
expected_files['site_model_file'].id, files2['site_model_file'].id)
def test_parse_config_no_files(self):
# sections are there just for documentation
# when we parse the file, we ignore these
source = StringIO.StringIO("""
[general]
CALCULATION_MODE = classical
region = 1 1 2 2 3 3
[foo]
bar = baz
""")
# Add a 'name' to make this look like a real file:
source.name = 'path/to/some/job.ini'
exp_base_path = os.path.dirname(
os.path.join(os.path.abspath('.'), source.name))
expected_params = {
'base_path': exp_base_path,
'force_inputs': False,
'calculation_mode': 'classical',
'region': '1 1 2 2 3 3',
'bar': 'baz',
}
params, _ = engine2.parse_config(source)
self.assertEqual(expected_params, params)
def test_read_and_validate_hazard_config(self):
cfg = helpers.demo_file('simple_fault_demo_hazard/job.ini')
job = engine2.prepare_job(getpass.getuser())
params, files = engine2.parse_config(open(cfg, 'r'))
calculation = engine2.create_hazard_calculation(
job.owner, params, files.values())
form = validation.ClassicalHazardForm(
instance=calculation, files=files
)
self.assertTrue(form.is_valid())
def test_a_few_inputs(self):
cfg = helpers.demo_file('simple_fault_demo_hazard/job.ini')
params, files = engine2.parse_config(open(cfg, 'r'), force_inputs=True)
owner = helpers.default_user()
hc = engine2.create_hazard_calculation(owner, params, files.values())
expected_ids = sorted([x.id for x in files.values()])
inputs = models.inputs4hcalc(hc.id)
actual_ids = sorted([x.id for x in inputs])
self.assertEqual(expected_ids, actual_ids)
def test_with_input_type(self):
cfg = helpers.demo_file("simple_fault_demo_hazard/job.ini")
params, files = engine2.parse_config(open(cfg, "r"), force_inputs=True)
owner = helpers.default_user()
hc = engine2.create_hazard_calculation(owner, params, files.values())
# It should only be 1 id, actually.
expected_ids = [x.id for x in files.values() if x.input_type == "lt_source"]
inputs = models.inputs4hcalc(hc.id, input_type="lt_source")
actual_ids = sorted([x.id for x in inputs])
self.assertEqual(expected_ids, actual_ids)
def get_hazard_job(cfg, username=None):
"""
Given a path to a config file, create a
:class:`openquake.engine.db.models.OqJob` object for a hazard calculation.
"""
username = username if username is not None else default_user().user_name
job = engine2.prepare_job(username)
params, files = engine2.parse_config(open(cfg, 'r'), force_inputs=True)
haz_calc = engine2.create_hazard_calculation(
job.owner, params, files.values())
haz_calc = models.HazardCalculation.objects.get(id=haz_calc.id)
job.hazard_calculation = haz_calc
job.save()
return job
def test_parse_config_with_files_force_inputs(self):
site_model_input = helpers.touch(content="site model")
sm_lt_input = helpers.touch(content="source model logic tree")
gsim_lt_input = helpers.touch(content="gsim logic tree")
source = StringIO.StringIO("""
[hazard_or_whatever]
calculation_mode = classical
gsim_logic_tree_file = %s
source_model_logic_tree_file = %s
site_model_file = %s
not_a_valid_xml = foo.xml
""" % (gsim_lt_input, sm_lt_input, site_model_input))
# Add a 'name' to make this look like a real file:
source.name = 'path/to/some/job.ini'
exp_base_path = os.path.dirname(
os.path.join(os.path.abspath('.'), source.name))
expected_params = {
'base_path': exp_base_path,
'force_inputs': True,
'calculation_mode': 'classical',
'not_a_valid_xml': 'foo.xml',
}
params, files = engine2.parse_config(source, force_inputs=True)
expected_files = {
'site_model_file': models.Input.objects.filter(
input_type='site_model').latest('id'),
'source_model_logic_tree_file': models.Input.objects.filter(
input_type='source_model_logic_tree').latest('id'),
'gsim_logic_tree_file': models.Input.objects.filter(
input_type='gsim_logic_tree').latest('id'),
}
self.assertEqual(expected_params, params)
self.assertEqual(len(expected_files), len(files))
for key in expected_files:
self.assertEqual(expected_files[key].id, files[key].id)
def get_risk_job(risk_demo, hazard_demo, output_type="curve", username=None):
"""
Takes in input the paths (relative to the demos directory) to a
risk and hazard demo file config, respectively.
Creates the hazard outputs suitable to be used by a risk
calculation and then creates a :class:`openquake.engine.db.models.OqJob`
object for a risk calculation. It also returns the input files
referenced by the risk config file.
:param output_type: gmf or curve
"""
username = username if username is not None else default_user().user_name
hazard_cfg = demo_file(hazard_demo)
hazard_job = get_hazard_job(hazard_cfg, username)
hc = hazard_job.hazard_calculation
risk_cfg = demo_file(risk_demo)
rlz = models.LtRealization.objects.create(
hazard_calculation=hazard_job.hazard_calculation,
ordinal=1, seed=1, weight=None,
sm_lt_path="test_sm", gsim_lt_path="test_gsim",
is_complete=False, total_items=1, completed_items=1)
if output_type == "curve":
hazard_output = models.HazardCurveData.objects.create(
hazard_curve=models.HazardCurve.objects.create(
lt_realization=rlz,
output=models.Output.objects.create_output(
hazard_job, "Test Hazard output", "hazard_curve"),
investigation_time=hc.investigation_time,
imt="PGA", imls=[0.1, 0.2, 0.3]),
poes=[0.1, 0.2, 0.3],
location="POINT(1 1)")
else:
hazard_output = models.Gmf.objects.create(
gmf_set=models.GmfSet.objects.create(
gmf_collection=models.GmfCollection.objects.create(
output=models.Output.objects.create_output(
hazard_job, "Test Hazard output", "gmf"),
lt_realization=rlz,
complete_logic_tree_gmf=False),
investigation_time=hc.investigation_time,
ses_ordinal=1,
complete_logic_tree_gmf=False),
imt="PGA", gmvs=[0.1, 0.2, 0.3],
result_grp_ordinal=1,
location="POINT(1 1)")
hazard_job.status = "complete"
hazard_job.save()
job = engine2.prepare_job(username)
params, files = engine2.parse_config(
open(risk_cfg, 'r'), force_inputs=True)
if output_type == "curve":
params.update(
dict(hazard_output_id=hazard_output.hazard_curve.output.id))
else:
output = hazard_output.gmf_set.gmf_collection.output
params.update(dict(hazard_output_id=output.id))
risk_calc = engine2.create_risk_calculation(
job.owner, params, files.values())
risk_calc = models.RiskCalculation.objects.get(id=risk_calc.id)
job.risk_calculation = risk_calc
job.save()
return job, files
def get_risk_job(risk_cfg, hazard_cfg, output_type="curve", username=None):
"""
Takes in input the paths to a risk job config file and a hazard job config
file.
Creates fake hazard outputs suitable to be used by a risk
calculation and then creates a :class:`openquake.engine.db.models.OqJob`
object for a risk calculation. It also returns the input files
referenced by the risk config file.
:param output_type: gmf, gmf_scenario, or curve
"""
username = username if username is not None else default_user().user_name
hazard_job = get_hazard_job(hazard_cfg, username)
hc = hazard_job.hazard_calculation
rlz = models.LtRealization.objects.create(
hazard_calculation=hazard_job.hazard_calculation,
ordinal=1, seed=1, weight=None,
sm_lt_path="test_sm", gsim_lt_path="test_gsim",
is_complete=False, total_items=1, completed_items=1)
if output_type == "curve":
hazard_output = models.HazardCurve.objects.create(
lt_realization=rlz,
output=models.Output.objects.create_output(
hazard_job, "Test Hazard output", "hazard_curve"),
investigation_time=hc.investigation_time,
imt="PGA", imls=[0.1, 0.2, 0.3])
for point in ["POINT(-1.01 1.01)", "POINT(0.9 1.01)",
"POINT(0.01 0.01)", "POINT(0.9 0.9)"]:
models.HazardCurveData.objects.create(
hazard_curve=hazard_output,
poes=[0.1, 0.2, 0.3],
location="%s" % point)
elif output_type == "gmf_scenario":
output = models.Output.objects.create_output(
hazard_job, "Test Hazard output", "gmf_scenario")
for point in ["POINT(15.48 38.0900001)", "POINT(15.565 38.17)",
"POINT(15.481 38.25)"]:
hazard_output = models.GmfScenario.objects.create(
output=output,
imt="PGA",
location=point,
gmvs=[0.1, 0.2, 0.3])
else:
rupture_ids = get_rupture_ids(hazard_job, hc, rlz, 3)
hazard_output = models.GmfCollection.objects.create(
output=models.Output.objects.create_output(
hazard_job, "Test Hazard output", "gmf"),
lt_realization=rlz,
complete_logic_tree_gmf=False)
gmf_set = models.GmfSet.objects.create(
gmf_collection=hazard_output,
investigation_time=hc.investigation_time,
ses_ordinal=1,
complete_logic_tree_gmf=False)
for point in ["POINT(15.310 38.225)", "POINT(15.71 37.225)",
"POINT(15.48 38.091)", "POINT(15.565 38.17)",
"POINT(15.481 38.25)"]:
models.Gmf.objects.create(
gmf_set=gmf_set,
imt="PGA", gmvs=[0.1, 0.2, 0.3],
rupture_ids=rupture_ids,
result_grp_ordinal=1,
location=point)
hazard_job.status = "complete"
hazard_job.save()
job = engine2.prepare_job(username)
params, files = engine2.parse_config(
open(risk_cfg, 'r'), force_inputs=True)
params.update(dict(hazard_output_id=hazard_output.output.id))
risk_calc = engine2.create_risk_calculation(
job.owner, params, files.values())
risk_calc = models.RiskCalculation.objects.get(id=risk_calc.id)
job.risk_calculation = risk_calc
job.save()
return job, files