def test_parse_config_with_files(self):
temp_dir = tempfile.mkdtemp()
site_model_input = general.writetmp(dir=temp_dir, content="foo")
job_config = general.writetmp(dir=temp_dir, content="""
[general]
calculation_mode = classical
[site]
sites = 0 0
site_model_file = %s
maximum_distance=1
truncation_level=0
random_seed=0
""" % site_model_input)
try:
exp_base_path = os.path.dirname(job_config)
expected_params = {
'base_path': exp_base_path,
'calculation_mode': 'classical',
'hazard_calculation_id': None,
'hazard_output_id': None,
'truncation_level': 0.0,
'random_seed': 0,
'maximum_distance': 1.0,
'inputs': {'site_model': site_model_input},
'sites': [(0.0, 0.0)],
}
params = vars(readini.parse_config(open(job_config)))
self.assertEqual(expected_params, params)
self.assertEqual(['site_model'], params['inputs'].keys())
self.assertEqual([site_model_input], params['inputs'].values())
finally:
shutil.rmtree(temp_dir)
def job_from_file(cfg_file_path, username, log_level, exports,
hazard_output_id=None, hazard_calculation_id=None):
"""
Create a full job profile from a job config file.
:param str cfg_file_path:
Path to the job.ini.
:param str username:
The user who will own this job profile and all results.
:param str log_level:
Desired log level.
:param exports:
List of desired export types.
:param int hazard_output_id:
ID of a hazard output to use as input to this calculation. Specify
this xor ``hazard_calculation_id``.
:param int hazard_calculation_id:
ID of a complete hazard calculation to use as input to this
calculation. Specify this xor ``hazard_output_id``.
:returns:
:class:`openquake.engine.db.models.OqJob` object
:raises:
`RuntimeError` if the input job configuration is not valid
"""
# create the job
job = prepare_job(user_name=username, log_level=log_level)
# read calculation params and create the calculation profile
params = readini.parse_config(open(cfg_file_path, 'r'))
missing = set(params['inputs']) - INPUT_TYPES
if missing:
raise ValueError(
'The parameters %s in the .ini file does '
'not correspond to a valid input type' % ', '.join(missing))
if hazard_output_id is None and hazard_calculation_id is None:
# this is a hazard calculation, not a risk one
job.hazard_calculation = create_calculation(
models.HazardCalculation, params)
job.save()
# validate and raise an error if there are any problems
error_message = validate(job, 'hazard', params, exports)
if error_message:
raise RuntimeError(error_message)
return job
# otherwise run a risk calculation
params.update(dict(hazard_output_id=hazard_output_id,
hazard_calculation_id=hazard_calculation_id))
calculation = create_calculation(models.RiskCalculation, params)
job.risk_calculation = calculation
job.save()
# validate and raise an error if there are any problems
error_message = validate(job, 'risk', params, exports)
if error_message:
raise RuntimeError(error_message)
return job
def test_parse_config_no_files(self):
# sections are there just for documentation
# when we parse the file, we ignore these
source = StringIO.StringIO("""
[general]
CALCULATION_MODE = classical_risk
region = 1 1, 2 2, 3 3
[foo]
bar = baz
""")
# Add a 'name' to make this look like a real file:
source.name = 'path/to/some/job.ini'
exp_base_path = os.path.dirname(
os.path.join(os.path.abspath('.'), source.name))
expected_params = {
'base_path': exp_base_path,
'calculation_mode': 'classical_risk',
'hazard_calculation_id': None,
'hazard_output_id': 42,
'region': [(1.0, 1.0), (2.0, 2.0), (3.0, 3.0)],
'inputs': {},
}
params = vars(readini.parse_config(source, hazard_output_id=42))
self.assertEqual(expected_params, params)
def test(self):
# check that if risk models are provided, then the ``points to
# compute`` and the imls are got from there
username = helpers.default_user()
job = engine.prepare_job(username)
cfg = helpers.get_data_path("classical_job-sd-imt.ini")
params = vars(readini.parse_config(open(cfg)))
del params["hazard_calculation_id"]
del params["hazard_output_id"]
haz_calc = engine.create_calculation(models.HazardCalculation, params)
haz_calc = models.HazardCalculation.objects.get(id=haz_calc.id)
job.hazard_calculation = haz_calc
job.is_running = True
job.save()
calc = get_calculator_class("hazard", job.hazard_calculation.calculation_mode)(job)
calc.parse_risk_models()
self.assertEqual(
[(1.0, -1.0), (0.0, 0.0)], [(point.latitude, point.longitude) for point in haz_calc.points_to_compute()]
)
self.assertEqual(["PGA"], haz_calc.get_imts())
self.assertEqual(3, haz_calc.oqjob.exposuremodel.exposuredata_set.count())
return job
def get_job(cfg, username="******", hazard_calculation_id=None,
hazard_output_id=None, **extras):
"""
Given a path to a config file and a hazard_calculation_id
(or, alternatively, a hazard_output_id, create a
:class:`openquake.engine.db.models.OqJob` object for a risk calculation.
"""
if hazard_calculation_id is None and hazard_output_id is None:
return engine.job_from_file(cfg, username, 'error', [], **extras)
job = engine.prepare_job(username)
oqparam = readini.parse_config(
open(cfg), hazard_calculation_id, hazard_output_id)
params = vars(oqparam)
if hazard_calculation_id is None:
params['hazard_calculation_id'] = models.Output.objects.get(
pk=hazard_output_id).oq_job.id
# we are removing intensity_measure_types_and_levels because it is not
# a field of RiskCalculation; this ugliness will disappear when
# RiskCalculation will be removed
del params['intensity_measure_types_and_levels']
job.save_params(params)
risk_calc = engine.create_calculation(models.RiskCalculation, params)
risk_calc = models.RiskCalculation.objects.get(id=risk_calc.id)
job.risk_calculation = risk_calc
job.save()
return job
def test_parse_config_with_sites_csv(self):
sites_csv = general.writetmp(content='1.0,2.1\n3.0,4.1\n5.0,6.1')
try:
source = StringIO.StringIO("""
[general]
calculation_mode = classical
[geometry]
sites_csv = %s
[misc]
maximum_distance=0
truncation_level=3
random_seed=5
""" % sites_csv)
source.name = 'path/to/some/job.ini'
exp_base_path = os.path.dirname(
os.path.join(os.path.abspath('.'), source.name))
expected_params = {
'base_path': exp_base_path,
'sites': 'MULTIPOINT(1.0 2.1, 3.0 4.1, 5.0 6.1)',
'calculation_mode': 'classical',
'truncation_level': '3',
'random_seed': '5',
'maximum_distance': '0',
'inputs': {},
}
params = readini.parse_config(source)
self.assertEqual(expected_params, params)
finally:
os.unlink(sites_csv)
def test_parse_config_no_files(self):
# sections are there just for documentation
# when we parse the file, we ignore these
source = StringIO.StringIO("""
[general]
CALCULATION_MODE = classical
region = 1 1 2 2 3 3
[foo]
bar = baz
""")
# Add a 'name' to make this look like a real file:
source.name = 'path/to/some/job.ini'
exp_base_path = os.path.dirname(
os.path.join(os.path.abspath('.'), source.name))
expected_params = {
'base_path': exp_base_path,
'calculation_mode': 'classical',
'region': '1 1 2 2 3 3',
'bar': 'baz',
'inputs': {},
}
params = readini.parse_config(source)
self.assertEqual(expected_params, params)
def test_parse_config_no_files(self):
# sections are there just for documentation
# when we parse the file, we ignore these
source = StringIO.StringIO("""
[general]
CALCULATION_MODE = classical_risk
region = 1 1, 2 2, 3 3
[foo]
bar = baz
""")
# Add a 'name' to make this look like a real file:
source.name = 'path/to/some/job.ini'
exp_base_path = os.path.dirname(
os.path.join(os.path.abspath('.'), source.name))
expected_params = {
'base_path': exp_base_path,
'calculation_mode': 'classical_risk',
'hazard_calculation_id': None,
'hazard_output_id': 42,
'region': [(1.0, 1.0), (2.0, 2.0), (3.0, 3.0)],
'inputs': {},
}
params = vars(readini.parse_config(source, hazard_output_id=42))
self.assertEqual(expected_params, params)
def get_job(cfg, username="******", hazard_calculation_id=None,
hazard_output_id=None):
"""
Given a path to a config file and a hazard_calculation_id
(or, alternatively, a hazard_output_id, create a
:class:`openquake.engine.db.models.OqJob` object for a risk calculation.
"""
if hazard_calculation_id is None and hazard_output_id is None:
return engine.job_from_file(cfg, username, 'error', [])
job = engine.prepare_job(username)
params = vars(readini.parse_config(
open(cfg), hazard_calculation_id, hazard_output_id))
risk_calc = engine.create_calculation(models.RiskCalculation, params)
risk_calc = models.RiskCalculation.objects.get(id=risk_calc.id)
job.risk_calculation = risk_calc
job.save()
return job
def test_parse_config_with_sites_csv(self):
sites_csv = general.writetmp(content='1.0,2.1\n3.0,4.1\n5.0,6.1')
try:
source = StringIO.StringIO("""
[general]
calculation_mode = classical
[geometry]
sites_csv = %s
[misc]
maximum_distance=1
truncation_level=3
random_seed=5
[site_params]
reference_vs30_type = measured
reference_vs30_value = 600.0
reference_depth_to_2pt5km_per_sec = 5.0
reference_depth_to_1pt0km_per_sec = 100.0
intensity_measure_types = PGA
""" % sites_csv)
source.name = 'path/to/some/job.ini'
exp_base_path = os.path.dirname(
os.path.join(os.path.abspath('.'), source.name))
expected_params = {
'base_path': exp_base_path,
'calculation_mode': 'classical',
'hazard_calculation_id': None,
'hazard_output_id': None,
'truncation_level': 3.0,
'random_seed': 5,
'maximum_distance': 1.0,
'inputs': {'site': sites_csv},
'reference_depth_to_1pt0km_per_sec': 100.0,
'reference_depth_to_2pt5km_per_sec': 5.0,
'reference_vs30_type': 'measured',
'reference_vs30_value': 600.0,
'intensity_measure_types_and_levels': {'PGA': None},
}
params = vars(readini.parse_config(source))
self.assertEqual(expected_params, params)
finally:
os.unlink(sites_csv)
def test_parse_config_with_sites_csv(self):
sites_csv = general.writetmp(content='1.0,2.1\n3.0,4.1\n5.0,6.1')
try:
source = StringIO.StringIO("""
[general]
calculation_mode = classical
[geometry]
sites_csv = %s
[misc]
maximum_distance=1
truncation_level=3
random_seed=5
[site_params]
reference_vs30_type = measured
reference_vs30_value = 600.0
reference_depth_to_2pt5km_per_sec = 5.0
reference_depth_to_1pt0km_per_sec = 100.0
""" % sites_csv)
source.name = 'path/to/some/job.ini'
exp_base_path = os.path.dirname(
os.path.join(os.path.abspath('.'), source.name))
expected_params = {
'base_path': exp_base_path,
'sites': [(1.0, 2.1), (3.0, 4.1), (5.0, 6.1)],
'calculation_mode': 'classical',
'hazard_calculation_id': None,
'hazard_output_id': None,
'truncation_level': 3.0,
'random_seed': 5,
'maximum_distance': 1.0,
'inputs': {},
'reference_depth_to_1pt0km_per_sec': 100.0,
'reference_depth_to_2pt5km_per_sec': 5.0,
'reference_vs30_type': 'measured',
'reference_vs30_value': 600.0,
}
params = vars(readini.parse_config(source))
self.assertEqual(expected_params, params)
finally:
os.unlink(sites_csv)
def test_parse_config_with_files(self):
temp_dir = tempfile.mkdtemp()
site_model_input = general.writetmp(dir=temp_dir, content="foo")
job_config = general.writetmp(dir=temp_dir,
content="""
[general]
calculation_mode = classical
[site]
sites = 0 0
site_model_file = %s
maximum_distance=1
truncation_level=0
random_seed=0
""" % site_model_input)
try:
exp_base_path = os.path.dirname(job_config)
expected_params = {
'base_path': exp_base_path,
'calculation_mode': 'classical',
'hazard_calculation_id': None,
'hazard_output_id': None,
'truncation_level': 0.0,
'random_seed': 0,
'maximum_distance': 1.0,
'inputs': {
'site_model': site_model_input
},
'sites': [(0.0, 0.0)],
}
params = vars(readini.parse_config(open(job_config)))
self.assertEqual(expected_params, params)
self.assertEqual(['site_model'], params['inputs'].keys())
self.assertEqual([site_model_input], params['inputs'].values())
finally:
shutil.rmtree(temp_dir)
def job_from_file(cfg_file_path, username, log_level="info", exports=(), hazard_output_id=None, hazard_job_id=None):
"""
Create a full job profile from a job config file.
:param str cfg_file_path:
Path to the job.ini.
:param str username:
The user who will own this job profile and all results.
:param str log_level:
Desired log level.
:param exports:
List of desired export types.
:param int hazard_output_id:
ID of a hazard output to use as input to this calculation. Specify
this xor ``hazard_calculation_id``.
:param int hazard_job_id:
ID of a complete hazard job to use as input to this
calculation. Specify this xor ``hazard_output_id``.
:returns:
:class:`openquake.engine.db.models.OqJob` object
:raises:
`RuntimeError` if the input job configuration is not valid
"""
# determine the previous hazard job, if any
if hazard_job_id:
haz_job = models.OqJob.objects.get(pk=hazard_job_id)
elif hazard_output_id: # extract the hazard job from the hazard_output_id
haz_job = models.Output.objects.get(pk=hazard_output_id).oq_job
else:
haz_job = None # no previous hazard job
if haz_job:
assert haz_job.job_type == "hazard", haz_job
# create the current job
job = prepare_job(user_name=username, log_level=log_level)
# read calculation params and create the calculation profile
oqparam = readini.parse_config(
open(cfg_file_path),
haz_job.hazard_calculation.id if haz_job and not hazard_output_id else None,
hazard_output_id,
)
missing = set(oqparam.inputs) - INPUT_TYPES
if missing:
raise ValueError(
"The parameters %s in the .ini file does " "not correspond to a valid input type" % ", ".join(missing)
)
params = vars(oqparam).copy()
job.save_params(params)
if hazard_output_id is None and hazard_job_id is None:
# this is a hazard calculation, not a risk one
del params["hazard_calculation_id"]
del params["hazard_output_id"]
job.hazard_calculation = create_calculation(models.HazardCalculation, params)
job.save()
return job
calculation = create_calculation(models.RiskCalculation, params)
job.risk_calculation = calculation
job.save()
return job
def get_fake_risk_job(risk_cfg, hazard_cfg, output_type="curve",
username="******"):
"""
Takes in input the paths to a risk job config file and a hazard job config
file.
Creates fake hazard outputs suitable to be used by a risk
calculation and then creates a :class:`openquake.engine.db.models.OqJob`
object for a risk calculation. It also returns the input files
referenced by the risk config file.
:param output_type: gmf, gmf_scenario, or curve
"""
hazard_job = get_job(hazard_cfg, username)
hc = hazard_job.hazard_calculation
lt_model = models.LtSourceModel.objects.create(
hazard_calculation=hazard_job.hazard_calculation,
ordinal=1, sm_lt_path="test_sm")
rlz = models.LtRealization.objects.create(
lt_model=lt_model, ordinal=1, weight=1,
gsim_lt_path="test_gsim")
if output_type == "curve":
models.HazardCurve.objects.create(
lt_realization=rlz,
output=models.Output.objects.create_output(
hazard_job, "Test Hazard output", "hazard_curve_multi"),
investigation_time=hc.investigation_time)
hazard_output = models.HazardCurve.objects.create(
lt_realization=rlz,
output=models.Output.objects.create_output(
hazard_job, "Test Hazard output", "hazard_curve"),
investigation_time=hc.investigation_time,
imt="PGA", imls=[0.1, 0.2, 0.3])
for point in ["POINT(-1.01 1.01)", "POINT(0.9 1.01)",
"POINT(0.01 0.01)", "POINT(0.9 0.9)"]:
models.HazardSite.objects.create(
hazard_calculation=hc, location=point)
models.HazardCurveData.objects.create(
hazard_curve=hazard_output,
poes=[0.1, 0.2, 0.3],
location="%s" % point)
elif output_type == "gmf_scenario":
hazard_output = models.Gmf.objects.create(
output=models.Output.objects.create_output(
hazard_job, "Test gmf scenario output", "gmf_scenario"))
models.SESCollection.objects.create(
output=models.Output.objects.create_output(
hazard_job, "Test SES Collection", "ses"),
lt_model=None, ordinal=0)
site_ids = hazard_job.hazard_calculation.save_sites(
[(15.48, 38.0900001), (15.565, 38.17), (15.481, 38.25)])
for site_id in site_ids:
models.GmfData.objects.create(
gmf=hazard_output,
task_no=0,
imt="PGA",
site_id=site_id,
gmvs=[0.1, 0.2, 0.3],
rupture_ids=[0, 1, 2])
elif output_type in ("ses", "gmf"):
hazard_output = create_gmf_data_records(hazard_job, rlz)[0].gmf
else:
raise RuntimeError('Unexpected output_type: %s' % output_type)
hazard_job.status = "complete"
hazard_job.save()
job = engine.prepare_job(username)
params = vars(readini.parse_config(open(risk_cfg),
hazard_output_id=hazard_output.output.id))
risk_calc = engine.create_calculation(models.RiskCalculation, params)
job.risk_calculation = risk_calc
job.save()
# reload risk calculation to have all the types converted properly
job.risk_calculation = models.RiskCalculation.objects.get(id=risk_calc.id)
return job, set(params['inputs'])
