def test_risk_mandatory_parameters(self):
sections = [
config.RISK_SECTION, config.HAZARD_SECTION, config.GENERAL_SECTION]
dummy_exposure = self.touch()
params = {}
validator = config.default_validators(sections, params)
self.assertFalse(validator.is_valid()[0])
params = {config.EXPOSURE: dummy_exposure,
config.DEPTHTO1PT0KMPERSEC: "33.33",
config.VS30_TYPE: "measured"}
validator = config.default_validators(sections, params)
self.assertFalse(validator.is_valid()[0])
params = {config.EXPOSURE: dummy_exposure,
config.REGION_GRID_SPACING: '0.5',
config.DEPTHTO1PT0KMPERSEC: "33.33",
config.VS30_TYPE: "measured"}
validator = config.default_validators(sections, params)
self.assertFalse(validator.is_valid()[0])
params = {config.EXPOSURE: dummy_exposure,
config.INPUT_REGION: "1.0, 2.0, 3.0, 4.0, 5.0, 6.0",
config.REGION_GRID_SPACING: '0.5',
config.DEPTHTO1PT0KMPERSEC: "33.33",
config.VS30_TYPE: "measured"}
validator = config.default_validators(sections, params)
self.assertTrue(validator.is_valid()[0])
def test_in_default_validators(self):
sections = ['HAZARD', 'RISK']
params = dict(CALCULATION_MODE='Scenario')
validators = config.default_validators(sections, params)
self.assertTrue(
any(isinstance(v, AssetCorrelationValidator) for v in validators))
params = dict(CALCULATION_MODE='Event Based')
validators = config.default_validators(sections, params)
self.assertTrue(
any(isinstance(v, AssetCorrelationValidator) for v in validators))
def test_in_default_validators(self):
sections = ['HAZARD', 'RISK']
params = dict(CALCULATION_MODE='Scenario')
validators = config.default_validators(sections, params)
self.assertTrue(any(
isinstance(v, AssetCorrelationValidator) for v in validators))
params = dict(CALCULATION_MODE='Event Based')
validators = config.default_validators(sections, params)
self.assertTrue(any(
isinstance(v, AssetCorrelationValidator) for v in validators))
def import_job_profile(path_to_cfg,
job,
user_name='openquake',
force_inputs=False):
"""Given the path to a job config file, create a new
:class:`openquake.db.models.OqJobProfile`, save it to the DB, and return
it.
:param str path_to_cfg:
Path to a job config file.
:param job:
The :class:`openquake.db.models.OqJob` instance to use
:param user_name:
The user performing this action.
:param bool force_inputs: If `True` the model input files will be parsed
and the resulting content written to the database no matter what.
:returns:
A tuple of :class:`openquake.db.models.OqJobProfile` instance,
params dict, and sections list.
NOTE: The params and sections are temporary. These should be removed
from the return value the future whenever possible to keep the API
clean.
"""
params, sections = _parse_config_file(path_to_cfg)
params, sections = _prepare_config_parameters(params, sections)
validator = jobconf.default_validators(sections, params)
is_valid, errors = validator.is_valid()
if not is_valid:
raise jobconf.ValidationException(errors)
job_profile = _prepare_job(params, sections, user_name, job, force_inputs)
return job_profile, params, sections
def test_mandatory_hazard_params_without_java_names(self):
"""
All mandatory hazard parameters must have the 'java_name' property
set.
"""
sections = [config.HAZARD_SECTION]
params = {
config.CALCULATION_MODE: "CLASSICAL",
config.BASE_PATH: "/a/b/c",
config.SITES: "37.9, -121.9",
config.DEPTHTO1PT0KMPERSEC: "33.33",
config.VS30_TYPE: "measured"
}
# Add a paramer *without* a 'java_name' property to the mandatory
# hazard parameters list in order to provoke the error.
HazardMandatoryParamsValidator.MANDATORY_PARAMS.append("BASE_PATH")
# Now validate ..
validator = config.default_validators(sections, params)
result, msgs = validator.is_valid()
# .. and check that the validation blew up due to the lack of the
# 'java_name' property.
self.assertFalse(result)
self.assertEqual([
"The following mandatory hazard parameter(s) lack a 'java_name' "
"property: BASE_PATH"
], msgs)
# Restore the list with the mandatory hazard parameters.
HazardMandatoryParamsValidator.MANDATORY_PARAMS.pop()
def _job_from_file(config_file, output_type, owner_username='******'):
"""
Create a job from external configuration files.
NOTE: This function is deprecated. Please use
:function:`openquake.engine.import_job_profile`.
:param config_file:
The external configuration file path
:param output_type:
Where to store results:
* 'db' database
* 'xml' XML files *plus* database
:param owner_username:
oq_user.user_name which defines the owner of all DB artifacts created
by this function.
"""
# output_type can be set, in addition to 'db' and 'xml', also to
# 'xml_without_db', which has the effect of serializing only to xml
# without requiring a database at all.
# This allows to run tests without requiring a database.
# This is not documented in the public interface because it is
# essentially a detail of our current tests and ci infrastructure.
assert output_type in ('db', 'xml')
params, sections = _parse_config_file(config_file)
params, sections = _prepare_config_parameters(params, sections)
job_profile = _prepare_job(params, sections)
validator = jobconf.default_validators(sections, params)
is_valid, errors = validator.is_valid()
if not is_valid:
raise jobconf.ValidationException(errors)
owner = OqUser.objects.get(user_name=owner_username)
# openquake-server creates the calculation record in advance and stores
# the calculation id in the config file
calculation_id = params.get('OPENQUAKE_JOB_ID')
if not calculation_id:
# create the database record for this calculation
calculation = OqCalculation(owner=owner, path=None)
calculation.oq_job_profile = job_profile
calculation.save()
calculation_id = calculation.id
if output_type == 'db':
serialize_results_to = ['db']
else:
serialize_results_to = ['db', 'xml']
base_path = params['BASE_PATH']
job = CalculationProxy(params, calculation_id, sections=sections,
base_path=base_path,
serialize_results_to=serialize_results_to)
job.to_kvs()
return job
def import_job_profile(path_to_cfg, job, user_name='openquake',
force_inputs=False):
"""Given the path to a job config file, create a new
:class:`openquake.db.models.OqJobProfile`, save it to the DB, and return
it.
:param str path_to_cfg:
Path to a job config file.
:param job:
The :class:`openquake.db.models.OqJob` instance to use
:param user_name:
The user performing this action.
:param bool force_inputs: If `True` the model input files will be parsed
and the resulting content written to the database no matter what.
:returns:
A tuple of :class:`openquake.db.models.OqJobProfile` instance,
params dict, and sections list.
NOTE: The params and sections are temporary. These should be removed
from the return value the future whenever possible to keep the API
clean.
"""
params, sections = _parse_config_file(path_to_cfg)
params, sections = _prepare_config_parameters(params, sections)
validator = jobconf.default_validators(sections, params)
is_valid, errors = validator.is_valid()
if not is_valid:
raise jobconf.ValidationException(errors)
job_profile = _prepare_job(params, sections, user_name, job, force_inputs)
return job_profile, params, sections
def test_mandatory_hazard_params_without_java_names(self):
"""
All mandatory hazard parameters must have the 'java_name' property
set.
"""
sections = [config.HAZARD_SECTION]
params = {config.CALCULATION_MODE: "CLASSICAL",
config.BASE_PATH: "/a/b/c",
config.SITES: "37.9, -121.9",
config.DEPTHTO1PT0KMPERSEC: "33.33",
config.VS30_TYPE: "measured"}
# Add a paramer *without* a 'java_name' property to the mandatory
# hazard parameters list in order to provoke the error.
HazardMandatoryParamsValidator.MANDATORY_PARAMS.append("BASE_PATH")
# Now validate ..
validator = config.default_validators(sections, params)
result, msgs = validator.is_valid()
# .. and check that the validation blew up due to the lack of the
# 'java_name' property.
self.assertFalse(result)
self.assertEqual(
["The following mandatory hazard parameter(s) lack a 'java_name' "
"property: BASE_PATH"], msgs)
# Restore the list with the mandatory hazard parameters.
HazardMandatoryParamsValidator.MANDATORY_PARAMS.pop()
def import_job_profile(path_to_cfg):
"""Given the path to a job config file, create a new
:class:`openquake.db.models.OqJobProfile` and save it to the DB, and return
it.
:param str path_to_cfg:
Path to a job config file.
:returns:
A tuple of :class:`openquake.db.models.OqJobProfile` instance,
params dict, and sections list.
NOTE: The params and sections are temporary. These should be removed
from the return value the future whenever possible to keep the API
clean.
"""
params, sections = _parse_config_file(path_to_cfg)
params, sections = _prepare_config_parameters(params, sections)
validator = jobconf.default_validators(sections, params)
is_valid, errors = validator.is_valid()
if not is_valid:
raise jobconf.ValidationException(errors)
job_profile = _prepare_job(params, sections)
return job_profile, params, sections
def test_hazard_mandatory_parameters(self):
sections = [config.HAZARD_SECTION]
params = {config.CALCULATION_MODE: "CLASSICAL",
config.SITES: "37.9, -121.9",
config.DEPTHTO1PT0KMPERSEC: "33.33"}
validator = config.default_validators(sections, params)
self.assertFalse(validator.is_valid()[0])
params = {config.CALCULATION_MODE: "CLASSICAL",
config.SITES: "37.9, -121.9",
config.DEPTHTO1PT0KMPERSEC: "33.33",
config.VS30_TYPE: "measured"}
validator = config.default_validators(sections, params)
self.assertTrue(validator.is_valid()[0])
def is_valid(self):
"""Return true if this job is valid and can be
processed, false otherwise.
:returns: the status of this job and the related error messages.
:rtype: when valid, a (True, []) tuple is returned. When invalid, a
(False, [ERROR_MESSAGE#1, ERROR_MESSAGE#2, ..., ERROR_MESSAGE#N])
tuple is returned
"""
return conf.default_validators(self.sections, self.params).is_valid()
def test_default_validators_classical_bcr_risk(self):
# For Classical BCR Hazard+Risk calculations, ensure that a
# `ClassicalRiskValidator` is included in the default validators.
cfg_path = helpers.demo_file('benefit_cost_ratio/config.gem')
job_profile, params, sections = engine.import_job_profile(cfg_path)
validators = config.default_validators(sections, params)
self.assertTrue(any(
isinstance(v, ClassicalRiskValidator) for v in validators))
def test_default_validators_scenario_job(self):
"""Test to ensure that a Scenario job always includes the
:class:`openquake.job.config.ScenarioComputationValidator`."""
scenario_job_path = helpers.demo_file('scenario_risk/config.gem')
scenario_job = helpers.job_from_file(scenario_job_path)
validators = config.default_validators(scenario_job.sections,
scenario_job.params)
self.assertTrue(any(
isinstance(v, ScenarioComputationValidator) for v in validators))
def from_file(config_file, output_type):
"""
Create a job from external configuration files.
:param config_file: the external configuration file path
:param output_type: where to store results:
* 'db' database
* 'xml' XML files *plus* database
:param params: optional dictionary of default parameters, overridden by
the ones read from the config file
:type params: :py:class:`dict`
"""
# output_type can be set, in addition to 'db' and 'xml', also to
# 'xml_without_db', which has the effect of serializing only to xml
# without requiring a database at all.
# This allows to run tests without requiring a database.
# This is not documented in the public interface because it is
# essentially a detail of our current tests and ci infrastructure.
assert output_type in ('db', 'xml', 'xml_without_db')
params, sections = parse_config_file(config_file)
params, sections = prepare_config_parameters(params, sections)
validator = conf.default_validators(sections, params)
is_valid, errors = validator.is_valid()
if not is_valid:
raise conf.ValidationException(errors)
if output_type == 'xml_without_db':
# we are running a test
job_id = 0
serialize_results_to = ['xml']
else:
# openquake-server creates the job record in advance and stores the
# job id in the config file
job_id = params.get('OPENQUAKE_JOB_ID')
if not job_id:
# create the database record for this job
job_id = prepare_job(params).id
if output_type == 'db':
serialize_results_to = ['db']
else:
serialize_results_to = ['db', 'xml']
base_path = params['BASE_PATH']
job = Job(params, job_id, sections=sections, base_path=base_path,
serialize_results_to=serialize_results_to)
job.to_kvs()
return job
def test_risk_mandatory_parameters(self):
sections = [
config.RISK_SECTION, config.HAZARD_SECTION, config.GENERAL_SECTION
]
dummy_exposure = helpers.touch()
params = {}
validator = config.default_validators(sections, params)
self.assertFalse(validator.is_valid()[0])
params = {
config.EXPOSURE: dummy_exposure,
config.DEPTHTO1PT0KMPERSEC: "33.33",
config.VS30_TYPE: "measured"
}
validator = config.default_validators(sections, params)
self.assertFalse(validator.is_valid()[0])
params = {
config.EXPOSURE: dummy_exposure,
config.REGION_GRID_SPACING: '0.5',
config.DEPTHTO1PT0KMPERSEC: "33.33",
config.VS30_TYPE: "measured"
}
validator = config.default_validators(sections, params)
self.assertFalse(validator.is_valid()[0])
params = {
config.EXPOSURE: dummy_exposure,
config.INPUT_REGION: "1.0, 2.0, 3.0, 4.0, 5.0, 6.0",
config.REGION_GRID_SPACING: '0.5',
config.DEPTHTO1PT0KMPERSEC: "33.33",
config.VS30_TYPE: "measured"
}
validator = config.default_validators(sections, params)
self.assertTrue(validator.is_valid()[0])
def test_default_validators_classical_bcr_risk(self):
# For Classical BCR Hazard+Risk calculations, ensure that a
# `ClassicalRiskValidator` is included in the default validators.
cfg_path = helpers.demo_file('benefit_cost_ratio/config.gem')
job_profile, params, sections = engine.import_job_profile(
cfg_path, self.job)
validators = config.default_validators(sections, params)
self.assertTrue(
any(isinstance(v, ClassicalRiskValidator) for v in validators))
def test_default_validators_disagg_job(self):
"""Test to ensure that a Disaggregation job always includes the
:class:`openquake.job.config.DisaggregationValidator`.
"""
da_job_path = helpers.demo_file('disaggregation/config.gem')
da_job = helpers.job_from_file(da_job_path)
validators = config.default_validators(da_job.sections, da_job.params)
# test that the default validators include a DisaggregationValidator
self.assertTrue(
any(isinstance(v, DisaggregationValidator) for v in validators))
def test_hazard_mandatory_parameters(self):
sections = [config.HAZARD_SECTION]
params = {
config.CALCULATION_MODE: "CLASSICAL",
config.SITES: "37.9, -121.9",
config.DEPTHTO1PT0KMPERSEC: "33.33"
}
validator = config.default_validators(sections, params)
self.assertFalse(validator.is_valid()[0])
params = {
config.CALCULATION_MODE: "CLASSICAL",
config.SITES: "37.9, -121.9",
config.DEPTHTO1PT0KMPERSEC: "33.33",
config.VS30_TYPE: "measured"
}
validator = config.default_validators(sections, params)
self.assertTrue(validator.is_valid()[0])
def test_default_validators_event_based_bcr_risk(self):
# For Event-Based BCR Risk calculations, ensure that a
# `EventBasedRiskValidator` is included in the default validators.
cfg_path = helpers.demo_file(
'event_based_bcr_risk/config.gem')
job_profile, params, sections = engine.import_job_profile(cfg_path)
validators = config.default_validators(sections, params)
self.assertTrue(any(
isinstance(v, EventBasedRiskValidator) for v in validators))
def test_default_validators_event_based_bcr_risk(self):
# For Event-Based BCR Risk calculations, ensure that a
# `EventBasedRiskValidator` is included in the default validators.
cfg_path = helpers.demo_file('benefit_cost_ratio/config_ebased.gem')
job_profile, params, sections = engine.import_job_profile(
cfg_path, self.job)
validators = config.default_validators(sections, params)
self.assertTrue(
any(isinstance(v, EventBasedRiskValidator) for v in validators))
def test_default_validators_disagg_job(self):
"""Test to ensure that a Disaggregation job always includes the
:class:`openquake.job.config.DisaggregationValidator`.
"""
da_job_path = helpers.demo_file('disaggregation/config.gem')
da_job = helpers.job_from_file(da_job_path)
validators = config.default_validators(da_job.sections, da_job.params)
# test that the default validators include a DisaggregationValidator
self.assertTrue(any(
isinstance(v, DisaggregationValidator) for v in validators))
def test_default_validators_scenario_job(self):
"""Test to ensure that a Scenario job always includes the
:class:`openquake.job.config.ScenarioComputationValidator`."""
scenario_job_path = helpers.demo_file('scenario_risk/config.gem')
scenario_job = helpers.job_from_file(scenario_job_path)
validators = config.default_validators(scenario_job.sections,
scenario_job.params)
self.assertTrue(
any(
isinstance(v, ScenarioComputationValidator)
for v in validators))
def test_default_validators_classical_job(self):
"""Test to ensure that a classical always includes the
:class:`openquake.job.config.ClassicalValidator`.
"""
classical_risk_job_path = helpers.demo_file(
'classical_psha_based_risk/config.gem')
classical_risk_job = helpers.job_from_file(classical_risk_job_path)
validators = config.default_validators(classical_risk_job.sections,
classical_risk_job.params)
self.assertTrue(
any(isinstance(v, ClassicalValidator) for v in validators))
def test_default_validators_classical_job(self):
"""Test to ensure that a classical always includes the
:class:`openquake.job.config.ClassicalValidator`.
"""
classical_risk_job_path = helpers.demo_file(
'classical_psha_based_risk/config.gem')
classical_risk_job = helpers.job_from_file(classical_risk_job_path)
validators = config.default_validators(classical_risk_job.sections,
classical_risk_job.params)
self.assertTrue(
any(isinstance(v, ClassicalValidator) for v in validators))
def test_hazard_tasks(self):
"""
The `HAZARD_TASKS` parameter is not ignored for classical PSHA
calculations.
"""
sections = [config.HAZARD_SECTION]
params = {config.CALCULATION_MODE: "CLASSICAL",
config.SITES: "37.9, -121.9",
config.DEPTHTO1PT0KMPERSEC: "33.33",
config.VS30_TYPE: "measured",
config.HAZARD_TASKS: "96"}
validator = config.default_validators(sections, params)
self.assertTrue(validator.is_valid()[0])
def test_default_validators_scenario_damage_job(self):
# Ensures that a Scenario Damage job always includes the
# :class:`openquake.job.config.ScenarioComputationValidator` and
# the :class:`openquake.job.config.ScenarioDamageComputationValidator`.
scenario_job_path = helpers.demo_file(
"scenario_damage_risk/config.gem")
scenario_job = helpers.job_from_file(scenario_job_path)
validators = config.default_validators(
scenario_job.sections, scenario_job.params)
self.assertTrue(any(isinstance(
v, ScenarioComputationValidator) for v in validators))
self.assertTrue(any(isinstance(
v, ScenarioDamageValidator) for v in validators))
def test_default_validators_scenario_damage_job(self):
# Ensures that a Scenario Damage job always includes the
# :class:`openquake.job.config.ScenarioComputationValidator` and
# the :class:`openquake.job.config.ScenarioDamageComputationValidator`.
scenario_job_path = helpers.demo_file(
"scenario_damage_risk/config.gem")
scenario_job = helpers.job_from_file(scenario_job_path)
validators = config.default_validators(scenario_job.sections,
scenario_job.params)
self.assertTrue(
any(
isinstance(v, ScenarioComputationValidator)
for v in validators))
self.assertTrue(
any(isinstance(v, ScenarioDamageValidator) for v in validators))
def _job_from_file(config_file,
output_type,
owner_username='******',
force_inputs=False):
"""
Create a job from external configuration files.
NOTE: This function is deprecated. Please use
:function:`openquake.engine.import_job_profile`.
:param config_file:
The external configuration file path
:param output_type:
Where to store results:
* 'db' database
* 'xml' XML files *plus* database
:param owner_username:
oq_user.user_name which defines the owner of all DB artifacts created
by this function.
:param bool force_inputs: If `True` the model input files will be parsed
and the resulting content written to the database no matter what.
"""
# output_type can be set, in addition to 'db' and 'xml', also to
# 'xml_without_db', which has the effect of serializing only to xml
# without requiring a database at all.
# This allows to run tests without requiring a database.
# This is not documented in the public interface because it is
# essentially a detail of our current tests and ci infrastructure.
assert output_type in ('db', 'xml')
params, sections = _parse_config_file(config_file)
params, sections = _prepare_config_parameters(params, sections)
validator = jobconf.default_validators(sections, params)
is_valid, errors = validator.is_valid()
if not is_valid:
raise jobconf.ValidationException(errors)
owner = OqUser.objects.get(user_name=owner_username)
# openquake-server creates the job record in advance and stores
# the calculation id in the config file
job_id = params.get('OPENQUAKE_JOB_ID')
if not job_id:
# create the database record for this job
job = OqJob(owner=owner, path=None)
job.save()
job_id = job.id
else:
job = OqJob.objects.get(job_id)
job_profile = _prepare_job(params, sections, owner_username, job,
force_inputs)
if output_type == 'db':
serialize_results_to = ['db']
else:
serialize_results_to = ['db', 'xml']
base_path = params['BASE_PATH']
job_ctxt = JobContext(params,
job_id,
sections=sections,
base_path=base_path,
serialize_results_to=serialize_results_to,
oq_job=job,
oq_job_profile=job_profile)
job_ctxt.to_kvs()
return job_ctxt
def _job_from_file(config_file, output_type, owner_username="******", force_inputs=False):
"""
Create a job from external configuration files.
NOTE: This function is deprecated. Please use
:function:`openquake.engine.import_job_profile`.
:param config_file:
The external configuration file path
:param output_type:
Where to store results:
* 'db' database
* 'xml' XML files *plus* database
:param owner_username:
oq_user.user_name which defines the owner of all DB artifacts created
by this function.
:param bool force_inputs: If `True` the model input files will be parsed
and the resulting content written to the database no matter what.
"""
# output_type can be set, in addition to 'db' and 'xml', also to
# 'xml_without_db', which has the effect of serializing only to xml
# without requiring a database at all.
# This allows to run tests without requiring a database.
# This is not documented in the public interface because it is
# essentially a detail of our current tests and ci infrastructure.
assert output_type in ("db", "xml")
params, sections = _parse_config_file(config_file)
params, sections = _prepare_config_parameters(params, sections)
validator = jobconf.default_validators(sections, params)
is_valid, errors = validator.is_valid()
if not is_valid:
raise jobconf.ValidationException(errors)
owner = OqUser.objects.get(user_name=owner_username)
# openquake-server creates the job record in advance and stores
# the calculation id in the config file
job_id = params.get("OPENQUAKE_JOB_ID")
if not job_id:
# create the database record for this job
job = OqJob(owner=owner, path=None)
job.save()
job_id = job.id
else:
job = OqJob.objects.get(job_id)
job_profile = _prepare_job(params, sections, owner_username, job, force_inputs)
if output_type == "db":
serialize_results_to = ["db"]
else:
serialize_results_to = ["db", "xml"]
base_path = params["BASE_PATH"]
job_ctxt = JobContext(
params,
job_id,
sections=sections,
base_path=base_path,
serialize_results_to=serialize_results_to,
oq_job=job,
oq_job_profile=job_profile,
)
job_ctxt.to_kvs()
return job_ctxt