def setUp(self):
# Patch a few methods here and restore them in the tearDown to avoid
# too many nested with
# See http://www.voidspace.org.uk/python/mock/patch.html \
# #patch-methods-start-and-stop
self.patchers = []
def start_patch(attr_path):
_, attr = attr_path.rsplit('.', 1)
patcher = patch(attr_path)
self.patchers.append(patcher)
setattr(self, attr, patcher.start())
start_patch('openquake.engine.supervising.is_pid_running')
# Patch the actions taken by the supervisor
start_patch('openquake.engine.supervising.supervisor.\
record_job_stop_time')
start_patch(
'openquake.engine.supervising.supervisor.cleanup_after_job')
start_patch('openquake.engine.supervising.supervisor.terminate_job')
start_patch('openquake.engine.supervising.supervisor.get_job_status')
start_patch('openquake.engine.supervising.supervisor'
'.update_job_status_and_error_msg')
logging.root.setLevel(logging.CRITICAL)
self.job = engine2.prepare_job()
def test_prepare_job_explicit_log_level(self):
# By default, a job is created with a log level of 'progress'
# (just to show calculation progress).
# In this test, we'll specify 'debug' as the log level.
job = engine2.prepare_job(log_level='debug')
self.assertEqual('debug', job.log_level)
def setUp(self):
self.job = engine2.prepare_job()
self.calc = disagg_core.DisaggHazardCalculator(self.job)
# Mock `disagg_task_arg_gen`
disagg_path = 'openquake.engine.calculators.hazard.disaggregation'
self.disagg_tag_patch = helpers.patch(
'%s.core.DisaggHazardCalculator.disagg_task_arg_gen'
% disagg_path)
self.disagg_tag_mock = self.disagg_tag_patch.start()
# fake disagg task arg generator:
disagg_tag = iter(xrange(3))
self.disagg_tag_mock.return_value = disagg_tag
# Mock `haz_general.queue_next`
base_path = 'openquake.engine.calculators.base'
self.queue_next_patch = helpers.patch('%s.queue_next' % base_path)
self.queue_next_mock = self.queue_next_patch.start()
# Mock `finalize_hazard_curves`
general_path = 'openquake.engine.calculators.hazard.general'
self.finalize_curves_patch = helpers.patch(
'%s.BaseHazardCalculatorNext.finalize_hazard_curves'
% general_path)
self.finalize_curves_mock = self.finalize_curves_patch.start()
def test_parse_config_with_files_no_force_inputs(self):
site_model_input = helpers.touch(content="foo")
source = StringIO.StringIO("""
[general]
calculation_mode = classical
[site]
site_model_file = %s
maximum_distance=0
truncation_level=0
random_seed=0
""" % site_model_input)
# Add a 'name' to make this look like a real file:
source.name = 'path/to/some/job.ini'
exp_base_path = os.path.dirname(
os.path.join(os.path.abspath('.'), source.name))
expected_params = {
'base_path': exp_base_path,
'force_inputs': False,
'calculation_mode': 'classical',
'truncation_level': '0',
'random_seed': '0',
'maximum_distance': '0'
}
# Run first with force_inputs=True to create the new Input.
params, expected_files = engine2.parse_config(
source, force_inputs=True)
# In order for us to reuse the existing input, we need to associate
# each input with a successful job.
job = engine2.prepare_job(getpass.getuser())
job.hazard_calculation = engine2.create_hazard_calculation(
job.owner, params, expected_files.values())
job.status = 'complete'
job.save()
# Now run twice with force_inputs=False (the default).
source.seek(0)
params1, files1 = engine2.parse_config(source)
source.seek(0)
params2, files2 = engine2.parse_config(source)
# Check the params just for sanity.
self.assertEqual(expected_params, params1)
self.assertEqual(expected_params, params2)
# Finally, check that the Input returned by the latest 2 calls matches
# the input we created above.
self.assertEqual(len(expected_files), len(files1))
self.assertEqual(len(expected_files), len(files2))
self.assertEqual(
expected_files['site_model_file'].id, files1['site_model_file'].id)
self.assertEqual(
expected_files['site_model_file'].id, files2['site_model_file'].id)
def test_read_and_validate_hazard_config(self):
cfg = helpers.demo_file('simple_fault_demo_hazard/job.ini')
job = engine2.prepare_job(getpass.getuser())
params, files = engine2.parse_config(open(cfg, 'r'))
calculation = engine2.create_hazard_calculation(
job.owner, params, files.values())
form = validation.ClassicalHazardForm(
instance=calculation, files=files
)
self.assertTrue(form.is_valid())
def test_prepare_job_specified_user(self):
user_name = helpers.random_string()
job = engine2.prepare_job(user_name=user_name)
self.assertEqual(user_name, job.owner.user_name)
self.assertEqual('pre_executing', job.status)
self.assertEqual('progress', job.log_level)
try:
models.OqJob.objects.get(id=job.id)
except exceptions.ObjectDoesNotExist:
self.fail('Job was not found in the database')
def test_prepare_job_default_user(self):
job = engine2.prepare_job()
self.assertEqual('openquake', job.owner.user_name)
self.assertEqual('pre_executing', job.status)
self.assertEqual('progress', job.log_level)
# Check the make sure it's in the database.
try:
models.OqJob.objects.get(id=job.id)
except exceptions.ObjectDoesNotExist:
self.fail('Job was not found in the database')
def get_hazard_job(cfg, username=None):
"""
Given a path to a config file, create a
:class:`openquake.engine.db.models.OqJob` object for a hazard calculation.
"""
username = username if username is not None else default_user().user_name
job = engine2.prepare_job(username)
params, files = engine2.parse_config(open(cfg, 'r'), force_inputs=True)
haz_calc = engine2.create_hazard_calculation(
job.owner, params, files.values())
haz_calc = models.HazardCalculation.objects.get(id=haz_calc.id)
job.hazard_calculation = haz_calc
job.save()
return job
def get_risk_job(risk_demo, hazard_demo, output_type="curve", username=None):
"""
Takes in input the paths (relative to the demos directory) to a
risk and hazard demo file config, respectively.
Creates the hazard outputs suitable to be used by a risk
calculation and then creates a :class:`openquake.engine.db.models.OqJob`
object for a risk calculation. It also returns the input files
referenced by the risk config file.
:param output_type: gmf or curve
"""
username = username if username is not None else default_user().user_name
hazard_cfg = demo_file(hazard_demo)
hazard_job = get_hazard_job(hazard_cfg, username)
hc = hazard_job.hazard_calculation
risk_cfg = demo_file(risk_demo)
rlz = models.LtRealization.objects.create(
hazard_calculation=hazard_job.hazard_calculation,
ordinal=1, seed=1, weight=None,
sm_lt_path="test_sm", gsim_lt_path="test_gsim",
is_complete=False, total_items=1, completed_items=1)
if output_type == "curve":
hazard_output = models.HazardCurveData.objects.create(
hazard_curve=models.HazardCurve.objects.create(
lt_realization=rlz,
output=models.Output.objects.create_output(
hazard_job, "Test Hazard output", "hazard_curve"),
investigation_time=hc.investigation_time,
imt="PGA", imls=[0.1, 0.2, 0.3]),
poes=[0.1, 0.2, 0.3],
location="POINT(1 1)")
else:
hazard_output = models.Gmf.objects.create(
gmf_set=models.GmfSet.objects.create(
gmf_collection=models.GmfCollection.objects.create(
output=models.Output.objects.create_output(
hazard_job, "Test Hazard output", "gmf"),
lt_realization=rlz,
complete_logic_tree_gmf=False),
investigation_time=hc.investigation_time,
ses_ordinal=1,
complete_logic_tree_gmf=False),
imt="PGA", gmvs=[0.1, 0.2, 0.3],
result_grp_ordinal=1,
location="POINT(1 1)")
hazard_job.status = "complete"
hazard_job.save()
job = engine2.prepare_job(username)
params, files = engine2.parse_config(
open(risk_cfg, 'r'), force_inputs=True)
if output_type == "curve":
params.update(
dict(hazard_output_id=hazard_output.hazard_curve.output.id))
else:
output = hazard_output.gmf_set.gmf_collection.output
params.update(dict(hazard_output_id=output.id))
risk_calc = engine2.create_risk_calculation(
job.owner, params, files.values())
risk_calc = models.RiskCalculation.objects.get(id=risk_calc.id)
job.risk_calculation = risk_calc
job.save()
return job, files
def get_risk_job(risk_cfg, hazard_cfg, output_type="curve", username=None):
"""
Takes in input the paths to a risk job config file and a hazard job config
file.
Creates fake hazard outputs suitable to be used by a risk
calculation and then creates a :class:`openquake.engine.db.models.OqJob`
object for a risk calculation. It also returns the input files
referenced by the risk config file.
:param output_type: gmf, gmf_scenario, or curve
"""
username = username if username is not None else default_user().user_name
hazard_job = get_hazard_job(hazard_cfg, username)
hc = hazard_job.hazard_calculation
rlz = models.LtRealization.objects.create(
hazard_calculation=hazard_job.hazard_calculation,
ordinal=1, seed=1, weight=None,
sm_lt_path="test_sm", gsim_lt_path="test_gsim",
is_complete=False, total_items=1, completed_items=1)
if output_type == "curve":
hazard_output = models.HazardCurve.objects.create(
lt_realization=rlz,
output=models.Output.objects.create_output(
hazard_job, "Test Hazard output", "hazard_curve"),
investigation_time=hc.investigation_time,
imt="PGA", imls=[0.1, 0.2, 0.3])
for point in ["POINT(-1.01 1.01)", "POINT(0.9 1.01)",
"POINT(0.01 0.01)", "POINT(0.9 0.9)"]:
models.HazardCurveData.objects.create(
hazard_curve=hazard_output,
poes=[0.1, 0.2, 0.3],
location="%s" % point)
elif output_type == "gmf_scenario":
output = models.Output.objects.create_output(
hazard_job, "Test Hazard output", "gmf_scenario")
for point in ["POINT(15.48 38.0900001)", "POINT(15.565 38.17)",
"POINT(15.481 38.25)"]:
hazard_output = models.GmfScenario.objects.create(
output=output,
imt="PGA",
location=point,
gmvs=[0.1, 0.2, 0.3])
else:
rupture_ids = get_rupture_ids(hazard_job, hc, rlz, 3)
hazard_output = models.GmfCollection.objects.create(
output=models.Output.objects.create_output(
hazard_job, "Test Hazard output", "gmf"),
lt_realization=rlz,
complete_logic_tree_gmf=False)
gmf_set = models.GmfSet.objects.create(
gmf_collection=hazard_output,
investigation_time=hc.investigation_time,
ses_ordinal=1,
complete_logic_tree_gmf=False)
for point in ["POINT(15.310 38.225)", "POINT(15.71 37.225)",
"POINT(15.48 38.091)", "POINT(15.565 38.17)",
"POINT(15.481 38.25)"]:
models.Gmf.objects.create(
gmf_set=gmf_set,
imt="PGA", gmvs=[0.1, 0.2, 0.3],
rupture_ids=rupture_ids,
result_grp_ordinal=1,
location=point)
hazard_job.status = "complete"
hazard_job.save()
job = engine2.prepare_job(username)
params, files = engine2.parse_config(
open(risk_cfg, 'r'), force_inputs=True)
params.update(dict(hazard_output_id=hazard_output.output.id))
risk_calc = engine2.create_risk_calculation(
job.owner, params, files.values())
risk_calc = models.RiskCalculation.objects.get(id=risk_calc.id)
job.risk_calculation = risk_calc
job.save()
return job, files
