def _create_job_profiles(self, user_name):
uhs_cfg = helpers.demo_file('uhs/config.gem')
job = engine.prepare_job()
self.uhs_jp, _, _ = engine.import_job_profile(uhs_cfg, job,
user_name=user_name)
cpsha_cfg = helpers.demo_file('classical_psha_based_risk/config.gem')
job = engine.prepare_job()
self.cpsha_jp, _, _ = engine.import_job_profile(cpsha_cfg, job,
user_name=user_name)
def _create_job_profiles(self, user_name):
uhs_cfg = helpers.demo_file('uhs/config.gem')
job = engine.prepare_job()
self.uhs_jp, _, _ = engine.import_job_profile(uhs_cfg,
job,
user_name=user_name)
cpsha_cfg = helpers.demo_file('classical_psha_based_risk/config.gem')
job = engine.prepare_job()
self.cpsha_jp, _, _ = engine.import_job_profile(cpsha_cfg,
job,
user_name=user_name)
def test__db_cnode_status_and_two_jobs(self):
job1 = engine.prepare_job()
for node, status in [("P1", "up"), ("P2", "down"), ("P3", "down")]:
ns = models.CNodeStats(oq_job=job1, node=node,
current_status=status)
ns.save(using="job_superv")
job2 = engine.prepare_job()
expected = {}
for node, status in [("Q2", "down"), ("Q3", "down")]:
ns = models.CNodeStats(oq_job=job2, node=node,
current_status=status)
ns.save(using="job_superv")
expected[node] = ns
self.assertEqual(expected, monitor._db_cnode_status(job2.id))
def test_calculator_for_task(self):
"""Load up a sample calculation (into the db and cache) and make sure
we can instantiate the correct calculator for a given calculation id.
"""
from openquake.calculators.hazard.classical.core import (
ClassicalHazardCalculator)
job = engine.prepare_job()
job_profile, params, sections = engine.import_job_profile(demo_file(
'simple_fault_demo_hazard/config.gem'), job)
job_ctxt = engine.JobContext(params, job.id,
oq_job_profile=job_profile,
oq_job=job)
job_ctxt.to_kvs()
with patch(
'openquake.utils.tasks.get_running_job') as grc_mock:
# Loading of the JobContext is done by
# `get_running_job`, which is covered by other tests.
# So, we just want to make sure that it's called here.
grc_mock.return_value = job_ctxt
calculator = tasks.calculator_for_task(job.id, 'hazard')
self.assertTrue(isinstance(calculator, ClassicalHazardCalculator))
self.assertEqual(1, grc_mock.call_count)
def setup_classic_job(cls, create_job_path=True, upload_id=None,
inputs=None):
"""Create a classic job with associated upload and inputs.
:param bool create_job_path: if set the path for the job will be
created and captured in the job record
:param integer upload_id: if set use upload record with given db key.
:param list inputs: a list of 2-tuples where the first and the second
element are the input type and path respectively
:returns: a :py:class:`db.models.OqJob` instance
"""
assert upload_id is None # temporary
job = engine.prepare_job()
oqjp = cls.setup_job_profile(job)
models.Job2profile(oq_job=job, oq_job_profile=oqjp).save()
# Insert input model files
if inputs:
insert_inputs(job, inputs)
if create_job_path:
job.path = os.path.join(tempfile.mkdtemp(), str(job.id))
job.save()
os.mkdir(job.path)
os.chmod(job.path, 0777)
return job
def setUpClass(cls):
cls.job = engine.prepare_job()
jp, _, _ = engine.import_job_profile(RISK_DEMO_CONFIG_FILE, cls.job)
cls.job_ctxt = helpers.create_job({}, job_id=cls.job.id,
oq_job_profile=jp, oq_job=cls.job)
calc = ClassicalRiskCalculator(cls.job_ctxt)
calc.store_exposure_assets()
[input] = models.inputs4job(cls.job.id, input_type="exposure")
model = input.model()
assets = model.exposuredata_set.filter(taxonomy="af/ctc-D/LR")
# Add some more assets.
coos = [(10.000155392289116, 46.546194318563),
(10.222034128255, 46.0071299176413),
(10.520376165581, 46.247463385278)]
for lat, lon in coos:
site = shapes.Site(lat, lon)
cls.sites.append(site)
if assets:
continue
location = geos.GEOSGeometry(site.point.to_wkt())
asset = models.ExposureData(
exposure_model=model, taxonomy="af/ctc-D/LR",
asset_ref=helpers.random_string(6), stco=lat * 2,
site=location, reco=1.1 * lon)
asset.save()
def setUpClass(cls):
cls.job = engine.prepare_job()
jp, _, _ = engine.import_job_profile(RISK_DEMO_CONFIG_FILE, cls.job)
cls.job_ctxt = helpers.create_job({},
job_id=cls.job.id,
oq_job_profile=jp,
oq_job=cls.job)
calc = ClassicalRiskCalculator(cls.job_ctxt)
calc.store_exposure_assets()
[input] = models.inputs4job(cls.job.id, input_type="exposure")
model = input.model()
assets = model.exposuredata_set.filter(taxonomy="af/ctc-D/LR")
# Add some more assets.
coos = [(10.000155392289116, 46.546194318563),
(10.222034128255, 46.0071299176413),
(10.520376165581, 46.247463385278)]
for lat, lon in coos:
site = shapes.Site(lat, lon)
cls.sites.append(site)
if assets:
continue
location = geos.GEOSGeometry(site.point.to_wkt())
asset = models.ExposureData(exposure_model=model,
taxonomy="af/ctc-D/LR",
asset_ref=helpers.random_string(6),
stco=lat * 2,
site=location,
reco=1.1 * lon)
asset.save()
def test_calculator_for_task(self):
"""Load up a sample calculation (into the db and cache) and make sure
we can instantiate the correct calculator for a given calculation id.
"""
from openquake.calculators.hazard.classical.core import (
ClassicalHazardCalculator)
job = engine.prepare_job()
job_profile, params, sections = engine.import_job_profile(
demo_file('simple_fault_demo_hazard/config.gem'), job)
job_ctxt = engine.JobContext(params,
job.id,
oq_job_profile=job_profile,
oq_job=job)
job_ctxt.to_kvs()
with patch('openquake.utils.tasks.get_running_job') as grc_mock:
# Loading of the JobContext is done by
# `get_running_job`, which is covered by other tests.
# So, we just want to make sure that it's called here.
grc_mock.return_value = job_ctxt
calculator = tasks.calculator_for_task(job.id, 'hazard')
self.assertTrue(isinstance(calculator, ClassicalHazardCalculator))
self.assertEqual(1, grc_mock.call_count)
def setUp(self):
# Test 'event-based' job
cfg_path = helpers.testdata_path("simplecase/config.gem")
self.job = engine.prepare_job()
self.jp, self.params, self.sections = engine.import_job_profile(
cfg_path, self.job)
def setUpClass(cls):
cls.job = engine.prepare_job()
jp, _, _ = engine.import_job_profile(RISK_DEMO_CONFIG_FILE, cls.job)
calc_proxy = helpers.create_job({}, job_id=cls.job.id,
oq_job_profile=jp, oq_job=cls.job)
# storing the basic exposure model
ClassicalRiskCalculator(calc_proxy).store_exposure_assets()
[input] = models.inputs4job(cls.job.id, input_type="exposure")
model = input.model()
assets = model.exposuredata_set.filter(taxonomy="aa/aatc-D/LR")
if not assets:
# This model did not exist in the database before.
site = shapes.Site(1.0, 2.0)
# more assets at same location
models.ExposureData(
exposure_model=model, taxonomy="aa/aatc-D/LR",
asset_ref="ASSET_1", stco=1,
site=geos.GEOSGeometry(site.point.to_wkt()), reco=1).save()
models.ExposureData(
exposure_model=model, taxonomy="aa/aatc-D/LR",
asset_ref="ASSET_2", stco=1,
site=geos.GEOSGeometry(site.point.to_wkt()), reco=1).save()
site = shapes.Site(2.0, 2.0)
# just one asset at location
models.ExposureData(
exposure_model=model, taxonomy="aa/aatc-D/LR",
asset_ref="ASSET_3", stco=1,
site=geos.GEOSGeometry(site.point.to_wkt()), reco=1).save()
def setUp(self):
client = kvs.get_client()
# Delete managed job id info so we can predict the job key
# which will be allocated for us
client.delete(kvs.tokens.CURRENT_JOBS)
self.generated_files = []
job = engine.prepare_job()
jp, params, sections = import_job_profile(helpers.get_data_path(CONFIG_FILE), job)
self.job_ctxt = JobContext(params, job.id, sections=sections, oq_job_profile=jp, oq_job=job)
job = engine.prepare_job()
jp, params, sections = import_job_profile(helpers.get_data_path(CONFIG_WITH_INCLUDES), job)
self.job_ctxt_with_includes = JobContext(params, job.id, sections=sections, oq_job_profile=jp, oq_job=job)
def setUp(self):
# Test 'event-based' job
cfg_path = helpers.testdata_path("simplecase/config.gem")
self.job = engine.prepare_job()
self.jp, self.params, self.sections = engine.import_job_profile(
cfg_path, self.job)
def setUp(self):
kvs.get_client().flushall()
base_path = helpers.testdata_path("scenario")
job = engine.prepare_job()
self.job_profile, self.params, self.sections = (
engine.import_job_profile(SCENARIO_SMOKE_TEST, job))
self.job_ctxt = JobContext(self.params,
job.id,
sections=self.sections,
base_path=base_path,
oq_job_profile=self.job_profile,
oq_job=job)
self.job_ctxt.params[NUMBER_OF_CALC_KEY] = "1"
self.job_ctxt.params['SERIALIZE_RESULTS_TO'] = 'xml'
self.job_ctxt.serialize_results_to = ["xml"]
# saving the default java implementation
self.default = (
scenario.ScenarioHazardCalculator.compute_ground_motion_field)
self.grid = self.job_ctxt.region.grid
self.job_ctxt.to_kvs()
def setUpClass(cls):
cls.job = engine.prepare_job()
jp, _, _ = engine.import_job_profile(RISK_DEMO_CONFIG_FILE, cls.job)
calc_proxy = helpers.create_job({}, job_id=cls.job.id,
oq_job_profile=jp, oq_job=cls.job)
# storing the basic exposure model
ClassicalRiskCalculator(calc_proxy).store_exposure_assets()
[em_input] = models.inputs4job(cls.job.id, input_type="exposure")
[model] = em_input.exposuremodel_set.all()
site = shapes.Site(1.0, 2.0)
# more assets at same location
models.ExposureData(
exposure_model=model, taxonomy="NOT_USED",
asset_ref="ASSET_1", stco=1,
site=geos.GEOSGeometry(site.point.to_wkt()), reco=1).save()
models.ExposureData(
exposure_model=model, taxonomy="NOT_USED",
asset_ref="ASSET_2", stco=1,
site=geos.GEOSGeometry(site.point.to_wkt()), reco=1).save()
site = shapes.Site(2.0, 2.0)
# just one asset at location
models.ExposureData(
exposure_model=model, taxonomy="NOT_USED",
asset_ref="ASSET_3", stco=1,
site=geos.GEOSGeometry(site.point.to_wkt()), reco=1).save()
def test__db_cnode_status_and_two_jobs(self):
job1 = engine.prepare_job()
for node, status in [("P1", "up"), ("P2", "down"), ("P3", "down")]:
ns = models.CNodeStats(oq_job=job1,
node=node,
current_status=status)
ns.save(using="job_superv")
job2 = engine.prepare_job()
expected = {}
for node, status in [("Q2", "down"), ("Q3", "down")]:
ns = models.CNodeStats(oq_job=job2,
node=node,
current_status=status)
ns.save(using="job_superv")
expected[node] = ns
self.assertEqual(expected, monitor._db_cnode_status(job2.id))
def test__db_cnode_status_and_wrong_job_id(self):
job = engine.prepare_job()
expected = {}
for node, status in [("O1", "up"), ("O2", "down"), ("O3", "down")]:
ns = models.CNodeStats(oq_job=job, node=node,
current_status=status)
ns.save(using="job_superv")
self.assertEqual(expected, monitor._db_cnode_status(-1))
def setUp(self):
self.job_from_file = engine._job_from_file
self.init_logs_amqp_send = patch('openquake.logs.init_logs_amqp_send')
self.init_logs_amqp_send.start()
self.job = engine.prepare_job()
self.job_profile, self.params, self.sections = (
engine.import_job_profile(helpers.get_data_path(CONFIG_FILE),
self.job))
def setUp(self):
self.job_from_file = engine._job_from_file
self.init_logs_amqp_send = patch('openquake.logs.init_logs_amqp_send')
self.init_logs_amqp_send.start()
self.job = engine.prepare_job()
self.job_profile, self.params, self.sections = (
engine.import_job_profile(helpers.get_data_path(CONFIG_FILE),
self.job))
def test__db_cnode_status(self):
job = engine.prepare_job()
expected = {}
for node, status in [("N1", "up"), ("N2", "down"), ("N3", "down")]:
ns = models.CNodeStats(oq_job=job, node=node,
current_status=status)
ns.save(using="job_superv")
expected[node] = ns
self.assertEqual(expected, monitor._db_cnode_status(job.id))
def test__db_cnode_status_and_wrong_job_id(self):
job = engine.prepare_job()
expected = {}
for node, status in [("O1", "up"), ("O2", "down"), ("O3", "down")]:
ns = models.CNodeStats(oq_job=job,
node=node,
current_status=status)
ns.save(using="job_superv")
self.assertEqual(expected, monitor._db_cnode_status(-1))
def test__db_cnode_status(self):
job = engine.prepare_job()
expected = {}
for node, status in [("N1", "up"), ("N2", "down"), ("N3", "down")]:
ns = models.CNodeStats(oq_job=job,
node=node,
current_status=status)
ns.save(using="job_superv")
expected[node] = ns
self.assertEqual(expected, monitor._db_cnode_status(job.id))
def setUp(self):
# Test 'event-based' job
cfg_path = helpers.testdata_path("simplecase/config.gem")
base_path = helpers.testdata_path("simplecase")
oq_job = engine.prepare_job()
oq_job_profile, params, sections = engine.import_job_profile(cfg_path, oq_job)
self.eb_job = JobContext(
params, oq_job.id, sections=sections, base_path=base_path, oq_job_profile=oq_job_profile, oq_job=oq_job
)
def setUp(self):
self.job = engine.prepare_job()
self.job_profile, params, sections = engine.import_job_profile(
UHS_DEMO_CONFIG_FILE, self.job)
self.job_ctxt = engine.JobContext(
params, self.job.id, sections=sections,
serialize_results_to=['db'], oq_job_profile=self.job_profile,
oq_job=self.job)
self.job_ctxt.to_kvs()
self.job_id = self.job_ctxt.job_id
def setUp(self):
client = kvs.get_client()
# Delete managed job id info so we can predict the job key
# which will be allocated for us
client.delete(kvs.tokens.CURRENT_JOBS)
self.generated_files = []
job = engine.prepare_job()
jp, params, sections = import_job_profile(helpers.get_data_path(
CONFIG_FILE), job)
self.job_ctxt = JobContext(
params, job.id, sections=sections, oq_job_profile=jp, oq_job=job)
job = engine.prepare_job()
jp, params, sections = import_job_profile(helpers.get_data_path(
CONFIG_WITH_INCLUDES), job)
self.job_ctxt_with_includes = JobContext(
params, job.id, sections=sections, oq_job_profile=jp, oq_job=job)
def setUp(self):
self.job = engine.prepare_job()
self.job_profile, params, sections = engine.import_job_profile(
UHS_DEMO_CONFIG_FILE, self.job)
self.job_ctxt = engine.JobContext(
params, self.job.id, sections=sections,
serialize_results_to=['db'], oq_job_profile=self.job_profile,
oq_job=self.job)
self.job_ctxt.to_kvs()
self.job_id = self.job_ctxt.job_id
def setUp(self):
# Test 'event-based' job
cfg_path = helpers.testdata_path("simplecase/config.gem")
base_path = helpers.testdata_path("simplecase")
oq_job = engine.prepare_job()
oq_job_profile, params, sections = engine.import_job_profile(
cfg_path, oq_job)
self.eb_job = JobContext(
params, oq_job.id, sections=sections, base_path=base_path,
oq_job_profile=oq_job_profile, oq_job=oq_job)
def setUp(self):
self.job = engine.prepare_job()
self.job_profile, self.params, _sections = (engine.import_job_profile(
demo_file('simple_fault_demo_hazard/config.gem'), self.job))
self.params['debug'] = 'warn'
# Cache the calc proxy data into the kvs:
job_ctxt = engine.JobContext(self.params,
self.job.id,
oq_job_profile=self.job_profile,
oq_job=self.job)
job_ctxt.to_kvs()
def setUp(self):
self.job = engine.prepare_job()
self.job_profile, self.params, _sections = (
engine.import_job_profile(demo_file(
'simple_fault_demo_hazard/config.gem'), self.job))
self.params['debug'] = 'warn'
# Cache the calc proxy data into the kvs:
job_ctxt = engine.JobContext(
self.params, self.job.id, oq_job_profile=self.job_profile,
oq_job=self.job)
job_ctxt.to_kvs()
def test__serialize_xml_filenames(self):
# Test that the file names of the loss XML artifacts are correct.
# See https://bugs.launchpad.net/openquake/+bug/894706.
expected_lrc_file_name = (
'losscurves-block-#%(job_id)s-block#%(block)s.xml')
expected_lr_file_name = (
'losscurves-loss-block-#%(job_id)s-block#%(block)s.xml')
cfg_file = helpers.demo_file('classical_psha_based_risk/config.gem')
job = engine.prepare_job()
job_profile, params, sections = engine.import_job_profile(
cfg_file, job)
job_ctxt = engine.JobContext(params,
job.id,
sections=sections,
serialize_results_to=['xml', 'db'],
oq_job_profile=job_profile,
oq_job=job)
calculator = ClassicalRiskCalculator(job_ctxt)
with helpers.patch('openquake.writer.FileWriter.serialize'):
# The 'curves' key in the kwargs just needs to be present;
# because of the serialize mock in place above, it doesn't need
# to have a real value.
# First, we test loss ratio curve output,
# then we'll do the same test for loss curve output.
# We expect to get a single file path back.
[file_path] = calculator._serialize(
0, **dict(curve_mode='loss_ratio', curves=[]))
_dir, file_name = os.path.split(file_path)
self.assertEqual(
expected_lrc_file_name % dict(job_id=job.id, block=0),
file_name)
# The same test again, except for loss curves this time.
[file_path
] = calculator._serialize(0, **dict(curve_mode='loss', curves=[]))
_dir, file_name = os.path.split(file_path)
self.assertEqual(
expected_lr_file_name % dict(job_id=job.id, block=0),
file_name)
def test__serialize_xml_filenames(self):
# Test that the file names of the loss XML artifacts are correct.
# See https://bugs.launchpad.net/openquake/+bug/894706.
expected_lrc_file_name = (
'losscurves-block-#%(job_id)s-block#%(block)s.xml')
expected_lr_file_name = (
'losscurves-loss-block-#%(job_id)s-block#%(block)s.xml')
cfg_file = helpers.demo_file('classical_psha_based_risk/config.gem')
job = engine.prepare_job()
job_profile, params, sections = engine.import_job_profile(
cfg_file, job)
job_ctxt = engine.JobContext(
params, job.id, sections=sections,
serialize_results_to=['xml', 'db'], oq_job_profile=job_profile,
oq_job=job)
calculator = ClassicalRiskCalculator(job_ctxt)
with helpers.patch('openquake.writer.FileWriter.serialize'):
# The 'curves' key in the kwargs just needs to be present;
# because of the serialize mock in place above, it doesn't need
# to have a real value.
# First, we test loss ratio curve output,
# then we'll do the same test for loss curve output.
# We expect to get a single file path back.
[file_path] = calculator._serialize(
0, **dict(curve_mode='loss_ratio', curves=[]))
_dir, file_name = os.path.split(file_path)
self.assertEqual(
expected_lrc_file_name % dict(job_id=job.id,
block=0),
file_name)
# The same test again, except for loss curves this time.
[file_path] = calculator._serialize(
0, **dict(curve_mode='loss', curves=[]))
_dir, file_name = os.path.split(file_path)
self.assertEqual(
expected_lr_file_name % dict(job_id=job.id,
block=0),
file_name)
def test_get_site_model(self):
job = engine.prepare_job()
site_model_inp = models.Input(
owner=job.owner, digest='fake', path='fake',
input_type='site_model', size=0
)
site_model_inp.save()
# The link has not yet been made in the input2job table.
self.assertIsNone(general.get_site_model(job.id))
# Complete the link:
models.Input2job(input=site_model_inp, oq_job=job).save()
actual_site_model = general.get_site_model(job.id)
self.assertEqual(site_model_inp, actual_site_model)
def test_get_site_model(self):
job = engine.prepare_job()
site_model_inp = models.Input(
owner=job.owner, digest='fake', path='fake',
input_type='site_model', size=0
)
site_model_inp.save()
# The link has not yet been made in the input2job table.
self.assertIsNone(general.get_site_model(job.id))
# Complete the link:
models.Input2job(input=site_model_inp, oq_job=job).save()
actual_site_model = general.get_site_model(job.id)
self.assertEqual(site_model_inp, actual_site_model)
def setUp(self):
cfg_path = helpers.demo_file(
'probabilistic_event_based_risk/config.gem')
job = engine.prepare_job()
jp, params, sections = engine.import_job_profile(cfg_path, job)
job_ctxt = engine.JobContext(
params, 1, sections=sections, base_path='/tmp',
serialize_results_to=['db', 'xml'], oq_job_profile=jp, oq_job=job)
job_ctxt.blocks_keys = []
self.calculator = EventBasedRiskCalculator(job_ctxt)
self.calculator.store_exposure_assets = lambda: None
self.calculator.store_vulnerability_model = lambda: None
self.calculator.partition = lambda: None
def main():
arg_parser = set_up_arg_parser()
args = arg_parser.parse_args()
if args.version:
print utils_version.info(__version__)
elif args.config_file is not None:
from openquake import job
from openquake import engine
try:
if args.log_file is not None:
# Capture logging messages to a file.
try:
_touch_log_file(args.log_file)
except IOError as e:
raise IOError("Error writing to log file %s: %s" % (args.log_file, e.strerror))
user_name = getpass.getuser()
ajob = engine.prepare_job(user_name)
_, params, sections = engine.import_job_profile(args.config_file, ajob, user_name, args.force_inputs)
engine.run_job(
ajob,
params,
sections,
output_type=args.output_type,
log_level=args.log_level,
force_inputs=args.force_inputs,
log_file=args.log_file,
)
except job.config.ValidationException as e:
print str(e)
except IOError as e:
print str(e)
except Exception as e:
raise
elif args.list_calculations:
list_calculations()
elif args.list_outputs is not None:
list_outputs(args.list_outputs)
elif args.export is not None:
output_id, target_dir = args.export
output_id = int(output_id)
do_export(output_id, target_dir)
else:
arg_parser.print_usage()
def test_write_output(self):
# Test that the loss map writers are properly called when
# write_output is invoked.
cfg_file = helpers.demo_file('classical_psha_based_risk/config.gem')
job = engine.prepare_job()
job_profile, params, sections = engine.import_job_profile(
cfg_file, job)
# Set conditional loss poe so that loss maps are created.
# If this parameter is not specified, no loss maps will be serialized
# at the end of the job.
params['CONDITIONAL_LOSS_POE'] = '0.01'
job_profile.conditional_loss_poe = [0.01]
job_profile.save()
job_ctxt = engine.JobContext(params,
job.id,
sections=sections,
serialize_results_to=['xml', 'db'],
oq_job_profile=job_profile,
oq_job=job)
calculator = ClassicalRiskCalculator(job_ctxt)
# Mock the composed loss map serializer:
with helpers.patch('openquake.writer.CompositeWriter'
'.serialize') as writer_mock:
calculator.write_output()
self.assertEqual(1, writer_mock.call_count)
# Now test that the composite writer got the correct
# 'serialize to' instructions. The composite writer should have
# 1 DB and 1 XML loss map serializer:
composite_writer = writer_mock.call_args[0][0]
writers = composite_writer.writers
self.assertEqual(2, len(writers))
# We don't assume anything about the order of the writers,
# and we don't care anyway in this test:
self.assertTrue(
any(isinstance(w, LossMapDBWriter) for w in writers))
self.assertTrue(
any(
isinstance(w, LossMapNonScenarioXMLWriter)
for w in writers))
def setUp(self):
cfg_path = helpers.demo_file(
'probabilistic_event_based_risk/config.gem')
job = engine.prepare_job()
jp, params, sections = engine.import_job_profile(cfg_path, job)
job_ctxt = engine.JobContext(
params, 1, sections=sections, base_path='/tmp',
serialize_results_to=['db', 'xml'], oq_job_profile=jp, oq_job=job)
job_ctxt.blocks_keys = []
self.calculator = EventBasedRiskCalculator(job_ctxt)
self.calculator.store_exposure_assets = lambda: None
self.calculator.store_fragility_model = lambda: None
self.calculator.store_vulnerability_model = lambda: None
self.calculator.partition = lambda: None
def main():
arg_parser = set_up_arg_parser()
args = arg_parser.parse_args()
if args.version:
print utils_version.info(__version__)
elif args.config_file is not None:
from openquake import job
from openquake import engine
try:
if args.log_file is not None:
# Capture logging messages to a file.
try:
_touch_log_file(args.log_file)
except IOError as e:
raise IOError('Error writing to log file %s: %s' %
(args.log_file, e.strerror))
user_name = getpass.getuser()
ajob = engine.prepare_job(user_name)
_, params, sections = engine.import_job_profile(
args.config_file, ajob, user_name, args.force_inputs)
engine.run_job(ajob,
params,
sections,
output_type=args.output_type,
log_level=args.log_level,
force_inputs=args.force_inputs,
log_file=args.log_file)
except job.config.ValidationException as e:
print str(e)
except IOError as e:
print str(e)
except Exception as e:
raise
elif args.list_calculations:
list_calculations()
elif args.list_outputs is not None:
list_outputs(args.list_outputs)
elif args.export is not None:
output_id, target_dir = args.export
output_id = int(output_id)
do_export(output_id, target_dir)
else:
arg_parser.print_usage()
def test__launch_job_calls_core_calc_methods(self):
# The `Calculator` interface defines 4 general methods:
# - initialize
# - pre_execute
# - execute
# - post_execute
# When `_launch_job` is called, each of these methods should be
# called once per job type (hazard, risk).
# Calculation setup:
cfg_file = helpers.demo_file('classical_psha_based_risk/config.gem')
job = engine.prepare_job()
job_profile, params, sections = engine.import_job_profile(
cfg_file, job)
job_ctxt = engine.JobContext(
params, job.id, sections=sections,
serialize_results_to=['xml', 'db'],
oq_job_profile=job_profile, oq_job=job)
# Mocking setup:
cls_haz_calc = ('openquake.calculators.hazard.classical.core'
'.ClassicalHazardCalculator')
cls_risk_calc = ('openquake.calculators.risk.classical.core'
'.ClassicalRiskCalculator')
methods = ('initialize', 'pre_execute', 'execute', 'post_execute')
haz_patchers = [helpers.patch('%s.%s' % (cls_haz_calc, m))
for m in methods]
risk_patchers = [helpers.patch('%s.%s' % (cls_risk_calc, m))
for m in methods]
haz_mocks = [p.start() for p in haz_patchers]
risk_mocks = [p.start() for p in risk_patchers]
# Call the function under test:
engine._launch_job(job_ctxt, sections)
self.assertTrue(all(x.call_count == 1 for x in haz_mocks))
self.assertTrue(all(x.call_count == 1 for x in risk_mocks))
# Tear down the mocks:
for p in haz_patchers:
p.stop()
for p in risk_patchers:
p.stop()
def test__launch_job_calls_core_calc_methods(self):
# The `Calculator` interface defines 4 general methods:
# - initialize
# - pre_execute
# - execute
# - post_execute
# When `_launch_job` is called, each of these methods should be
# called once per job type (hazard, risk).
# Calculation setup:
cfg_file = helpers.demo_file('classical_psha_based_risk/config.gem')
job = engine.prepare_job()
job_profile, params, sections = engine.import_job_profile(
cfg_file, job)
job_ctxt = engine.JobContext(
params, job.id, sections=sections,
serialize_results_to=['xml', 'db'],
oq_job_profile=job_profile, oq_job=job)
# Mocking setup:
cls_haz_calc = ('openquake.calculators.hazard.classical.core'
'.ClassicalHazardCalculator')
cls_risk_calc = ('openquake.calculators.risk.classical.core'
'.ClassicalRiskCalculator')
methods = ('initialize', 'pre_execute', 'execute', 'post_execute')
haz_patchers = [helpers.patch('%s.%s' % (cls_haz_calc, m))
for m in methods]
risk_patchers = [helpers.patch('%s.%s' % (cls_risk_calc, m))
for m in methods]
haz_mocks = [p.start() for p in haz_patchers]
risk_mocks = [p.start() for p in risk_patchers]
# Call the function under test:
engine._launch_job(job_ctxt, sections)
self.assertTrue(all(x.call_count == 1 for x in haz_mocks))
self.assertTrue(all(x.call_count == 1 for x in risk_mocks))
# Tear down the mocks:
for p in haz_patchers:
p.stop()
for p in risk_patchers:
p.stop()
def prepare_job_context(path_to_cfg):
"""Given a path to a config file, prepare and return a
:class:`openquake.engine.JobContext`. This convenient because it can be
immediately passed to a calculator constructor.
This also creates the necessary job and oq_job_profile records.
"""
job = engine.prepare_job()
cfg = demo_file(path_to_cfg)
job_profile, params, sections = engine.import_job_profile(
cfg, job, force_inputs=True)
job_ctxt = engine.JobContext(
params, job.id, sections=sections, oq_job_profile=job_profile,
oq_job=job)
return job_ctxt
def test_write_output(self):
# Test that the loss map writers are properly called when
# write_output is invoked.
cfg_file = helpers.demo_file('classical_psha_based_risk/config.gem')
job = engine.prepare_job()
job_profile, params, sections = engine.import_job_profile(
cfg_file, job)
# Set conditional loss poe so that loss maps are created.
# If this parameter is not specified, no loss maps will be serialized
# at the end of the job.
params['CONDITIONAL_LOSS_POE'] = '0.01'
job_profile.conditional_loss_poe = [0.01]
job_profile.save()
job_ctxt = engine.JobContext(
params, job.id, sections=sections,
serialize_results_to=['xml', 'db'], oq_job_profile=job_profile,
oq_job=job)
calculator = ClassicalRiskCalculator(job_ctxt)
# Mock the composed loss map serializer:
with helpers.patch('openquake.writer.CompositeWriter'
'.serialize') as writer_mock:
calculator.write_output()
self.assertEqual(1, writer_mock.call_count)
# Now test that the composite writer got the correct
# 'serialize to' instructions. The composite writer should have
# 1 DB and 1 XML loss map serializer:
composite_writer = writer_mock.call_args[0][0]
writers = composite_writer.writers
self.assertEqual(2, len(writers))
# We don't assume anything about the order of the writers,
# and we don't care anyway in this test:
self.assertTrue(any(
isinstance(w, LossMapDBWriter) for w in writers))
self.assertTrue(any(
isinstance(w, LossMapNonScenarioXMLWriter) for w in writers))
def prepare_job_context(path_to_cfg):
"""Given a path to a config file, prepare and return a
:class:`openquake.engine.JobContext`. This convenient because it can be
immediately passed to a calculator constructor.
This also creates the necessary job and oq_job_profile records.
"""
job = engine.prepare_job()
cfg = demo_file(path_to_cfg)
job_profile, params, sections = engine.import_job_profile(
cfg, job, force_inputs=True)
job_ctxt = engine.JobContext(params,
job.id,
sections=sections,
oq_job_profile=job_profile,
oq_job=job)
return job_ctxt
def setup_classic_job(cls,
create_job_path=True,
upload_id=None,
inputs=None,
force_inputs=False,
omit_profile=False,
user_name="openquake"):
"""Create a classic job with associated upload and inputs.
:param bool create_job_path: if set the path for the job will be
created and captured in the job record
:param integer upload_id: if set use upload record with given db key.
:param list inputs: a list of 2-tuples where the first and the second
element are the input type and path respectively
:param bool force_inputs: If `True` the model input files will be
parsed and the resulting content written to the database no matter
what.
:param bool omit_profile: If `True` no job profile will be created.
:param str user_name: The name of the user that is running the job.
:returns: a :py:class:`db.models.OqJob` instance
"""
assert upload_id is None # temporary
job = engine.prepare_job(user_name)
if not omit_profile:
oqjp = cls.setup_job_profile(job, force_inputs)
models.Job2profile(oq_job=job, oq_job_profile=oqjp).save()
# Insert input model files
if inputs:
insert_inputs(job, inputs)
if create_job_path:
job.path = os.path.join(tempfile.mkdtemp(), str(job.id))
job.save()
os.mkdir(job.path)
os.chmod(job.path, 0777)
return job
def setUpClass(cls):
cls.job = engine.prepare_job()
jp, _, _ = engine.import_job_profile(RISK_DEMO_CONFIG_FILE, cls.job)
calc_proxy = helpers.create_job({},
job_id=cls.job.id,
oq_job_profile=jp,
oq_job=cls.job)
# storing the basic exposure model
ClassicalRiskCalculator(calc_proxy).store_exposure_assets()
[input] = models.inputs4job(cls.job.id, input_type="exposure")
model = input.model()
assets = model.exposuredata_set.filter(taxonomy="aa/aatc-D/LR")
if not assets:
# This model did not exist in the database before.
site = shapes.Site(1.0, 2.0)
# more assets at same location
models.ExposureData(exposure_model=model,
taxonomy="aa/aatc-D/LR",
asset_ref="ASSET_1",
stco=1,
site=geos.GEOSGeometry(site.point.to_wkt()),
reco=1).save()
models.ExposureData(exposure_model=model,
taxonomy="aa/aatc-D/LR",
asset_ref="ASSET_2",
stco=1,
site=geos.GEOSGeometry(site.point.to_wkt()),
reco=1).save()
site = shapes.Site(2.0, 2.0)
# just one asset at location
models.ExposureData(exposure_model=model,
taxonomy="aa/aatc-D/LR",
asset_ref="ASSET_3",
stco=1,
site=geos.GEOSGeometry(site.point.to_wkt()),
reco=1).save()
def test_get_site_model_too_many_site_models(self):
job = engine.prepare_job()
site_model_inp1 = models.Input(
owner=job.owner, digest='fake', path='fake',
input_type='site_model', size=0
)
site_model_inp1.save()
site_model_inp2 = models.Input(
owner=job.owner, digest='fake', path='fake',
input_type='site_model', size=0
)
site_model_inp2.save()
# link both site models to the job:
models.Input2job(input=site_model_inp1, oq_job=job).save()
models.Input2job(input=site_model_inp2, oq_job=job).save()
with self.assertRaises(RuntimeError) as assert_raises:
general.get_site_model(job.id)
self.assertEqual('Only 1 site model per job is allowed, found 2.',
assert_raises.exception.message)
def test_get_site_model_too_many_site_models(self):
job = engine.prepare_job()
site_model_inp1 = models.Input(
owner=job.owner, digest='fake', path='fake',
input_type='site_model', size=0
)
site_model_inp1.save()
site_model_inp2 = models.Input(
owner=job.owner, digest='fake', path='fake',
input_type='site_model', size=0
)
site_model_inp2.save()
# link both site models to the job:
models.Input2job(input=site_model_inp1, oq_job=job).save()
models.Input2job(input=site_model_inp2, oq_job=job).save()
with self.assertRaises(RuntimeError) as assert_raises:
general.get_site_model(job.id)
self.assertEqual('Only 1 site model per job is allowed, found 2.',
assert_raises.exception.message)
def setUp(self):
kvs.get_client().flushall()
base_path = helpers.testdata_path("scenario")
job = engine.prepare_job()
self.job_profile, self.params, self.sections = (
engine.import_job_profile(SCENARIO_SMOKE_TEST, job))
self.job_ctxt = JobContext(
self.params, job.id, sections=self.sections,
base_path=base_path, oq_job_profile=self.job_profile,
oq_job=job)
self.job_ctxt.params[NUMBER_OF_CALC_KEY] = "1"
self.job_ctxt.params['SERIALIZE_RESULTS_TO'] = 'xml'
# saving the default java implementation
self.default = (
scenario.ScenarioHazardCalculator.compute_ground_motion_field)
self.grid = self.job_ctxt.region.grid
self.job_ctxt.to_kvs()
def setup_classic_job(cls, create_job_path=True, upload_id=None,
inputs=None, force_inputs=False, omit_profile=False,
user_name="openquake"):
"""Create a classic job with associated upload and inputs.
:param bool create_job_path: if set the path for the job will be
created and captured in the job record
:param integer upload_id: if set use upload record with given db key.
:param list inputs: a list of 2-tuples where the first and the second
element are the input type and path respectively
:param bool force_inputs: If `True` the model input files will be
parsed and the resulting content written to the database no matter
what.
:param bool omit_profile: If `True` no job profile will be created.
:param str user_name: The name of the user that is running the job.
:returns: a :py:class:`db.models.OqJob` instance
"""
assert upload_id is None # temporary
job = engine.prepare_job(user_name)
if not omit_profile:
oqjp = cls.setup_job_profile(job, force_inputs)
models.Job2profile(oq_job=job, oq_job_profile=oqjp).save()
# Insert input model files
if inputs:
insert_inputs(job, inputs)
if create_job_path:
job.path = os.path.join(tempfile.mkdtemp(), str(job.id))
job.save()
os.mkdir(job.path)
os.chmod(job.path, 0777)
return job
def setUp(self):
self.job = engine.prepare_job()
self.job.hazard_calculation = HazardCalculation(no_progress_timeout=99)
def test_generate_hazard_curves_using_classical_psha(self):
def verify_realization_haz_curves_stored_to_kvs(the_job, keys):
""" This just tests to make sure there something in the KVS
for each key in given list of keys. This does NOT test the
actual results. """
# TODO (LB): At some point we need to test the actual
# results to verify they are correct
realizations = int(the_job.params['NUMBER_OF_LOGIC_TREE_SAMPLES'])
for realization in xrange(0, realizations):
for site in the_job.sites_to_compute():
key = tokens.hazard_curve_poes_key(the_job.job_id,
realization, site)
self.assertTrue(key in keys, "Missing key %s" % key)
def verify_mean_haz_curves_stored_to_kvs(the_job, keys):
""" Make sure that the keys and non-empty values for mean
hazard curves have been written to KVS."""
if the_job.params['COMPUTE_MEAN_HAZARD_CURVE'].lower() == 'true':
LOG.debug("verifying KVS entries for mean hazard curves")
for site in the_job.sites_to_compute():
key = tokens.mean_hazard_curve_key(the_job.job_id, site)
self.assertTrue(key in keys, "Missing key %s" % key)
def verify_mean_haz_maps_stored_to_kvs(the_job, calculator, keys):
""" Make sure that the keys and non-empty values for mean
hazard maps have been written to KVS."""
if (the_job.params[hazard_general.POES_PARAM_NAME] != '' and
the_job.params['COMPUTE_MEAN_HAZARD_CURVE'].lower() == \
'true'):
LOG.debug("verifying KVS entries for mean hazard maps")
for poe in calculator.poes_hazard_maps:
for site in the_job.sites_to_compute():
key = tokens.mean_hazard_map_key(
the_job.job_id, site, poe)
self.assertTrue(key in keys, "Missing key %s" % key)
def verify_quantile_haz_curves_stored_to_kvs(the_job, calculator,
keys):
""" Make sure that the keys and non-empty values for quantile
hazard curves have been written to KVS."""
quantiles = calculator.quantile_levels
LOG.debug("verifying KVS entries for quantile hazard curves, "\
"%s quantile values" % len(quantiles))
for quantile in quantiles:
for site in the_job.sites_to_compute():
key = tokens.quantile_hazard_curve_key(
the_job.job_id, site, quantile)
self.assertTrue(key in keys, "Missing key %s" % key)
def verify_quantile_haz_maps_stored_to_kvs(the_job, calculator, keys):
""" Make sure that the keys and non-empty values for quantile
hazard maps have been written to KVS."""
quantiles = calculator.quantile_levels
if (the_job.params[hazard_general.POES_PARAM_NAME] != ''
and len(quantiles) > 0):
poes = calculator.poes_hazard_maps
LOG.debug("verifying KVS entries for quantile hazard maps, "\
"%s quantile values, %s PoEs" % (
len(quantiles), len(poes)))
for quantile in quantiles:
for poe in poes:
for site in the_job.sites_to_compute():
key = tokens.quantile_hazard_map_key(
the_job.job_id, site, poe, quantile)
self.assertTrue(key in keys,
"Missing key %s" % key)
def verify_realization_haz_curves_stored_to_nrml(the_job, calculator):
"""Tests that a NRML file has been written for each realization,
and that this file validates against the NRML schema.
Does NOT test if results in NRML file are correct.
"""
realizations = int(the_job.params['NUMBER_OF_LOGIC_TREE_SAMPLES'])
for realization in xrange(0, realizations):
nrml_path = os.path.join(
"demos/classical_psha_simple/computed_output",
calculator.hazard_curve_filename(realization))
LOG.debug("validating NRML file %s" % nrml_path)
self.assertTrue(xml.validates_against_xml_schema(
nrml_path, NRML_SCHEMA_PATH),
"NRML instance file %s does not validate against schema" \
% nrml_path)
def verify_mean_haz_curves_stored_to_nrml(the_job, calculator):
"""Tests that a mean hazard curve NRML file has been written,
and that this file validates against the NRML schema.
Does NOT test if results in NRML file are correct.
"""
if the_job.params['COMPUTE_MEAN_HAZARD_CURVE'].lower() == 'true':
nrml_path = os.path.join(
"demos/classical_psha_simple/computed_output",
calculator.mean_hazard_curve_filename())
LOG.debug("validating NRML file %s" % nrml_path)
self.assertTrue(xml.validates_against_xml_schema(
nrml_path, NRML_SCHEMA_PATH),
"NRML instance file %s does not validate against schema" \
% nrml_path)
def verify_mean_haz_maps_stored_to_nrml(the_job):
"""Tests that a mean hazard map NRML file has been written,
and that this file validates against the NRML schema.
Does NOT test if results in NRML file are correct.
"""
if (the_job.params[hazard_general.POES_PARAM_NAME] != '' and
the_job.params['COMPUTE_MEAN_HAZARD_CURVE'].lower() == \
'true'):
for poe in calculator.poes_hazard_maps:
nrml_path = os.path.join(
"demos/classical_psha_simple/computed_output",
calculator.mean_hazard_map_filename(poe))
LOG.debug("validating NRML file for mean hazard map %s" \
% nrml_path)
self.assertTrue(xml.validates_against_xml_schema(
nrml_path, NRML_SCHEMA_PATH),
"NRML instance file %s does not validate against "\
"schema" % nrml_path)
def verify_quantile_haz_curves_stored_to_nrml(the_job, calculator):
"""Tests that quantile hazard curve NRML files have been written,
and that these file validate against the NRML schema.
Does NOT test if results in NRML files are correct.
"""
for quantile in calculator.quantile_levels:
nrml_path = os.path.join(
"demos/classical_psha_simple/computed_output",
calculator.quantile_hazard_curve_filename(quantile))
LOG.debug("validating NRML file for quantile hazard curve: "\
"%s" % nrml_path)
self.assertTrue(xml.validates_against_xml_schema(
nrml_path, NRML_SCHEMA_PATH),
"NRML instance file %s does not validate against schema" \
% nrml_path)
def verify_quantile_haz_maps_stored_to_nrml(the_job, calculator):
"""Tests that quantile hazard map NRML files have been written,
and that these file validate against the NRML schema.
Does NOT test if results in NRML files are correct.
"""
quantiles = calculator.quantile_levels
if (the_job.params[hazard_general.POES_PARAM_NAME] != ''
and len(quantiles) > 0):
for poe in calculator.poes_hazard_maps:
for quantile in quantiles:
nrml_path = os.path.join(
"demos/classical_psha_simple/computed_output",
calculator.quantile_hazard_map_filename(
quantile, poe))
LOG.debug("validating NRML file for quantile hazard "\
"map: %s" % nrml_path)
self.assertTrue(xml.validates_against_xml_schema(
nrml_path, NRML_SCHEMA_PATH),
"NRML instance file %s does not validate against "\
"schema" % nrml_path)
base_path = helpers.testdata_path("classical_psha_simple")
path = helpers.testdata_path("classical_psha_simple/config.gem")
job = engine.prepare_job()
job_profile, params, sections = engine.import_job_profile(path, job)
the_job = JobContext(params,
job.id,
sections=sections,
base_path=base_path,
serialize_results_to=['db', 'xml'],
oq_job_profile=job_profile,
oq_job=job)
the_job.to_kvs()
calc_mode = job_profile.calc_mode
calculator = CALCULATORS[calc_mode](the_job)
used_keys = []
calculator.execute(used_keys)
verify_realization_haz_curves_stored_to_kvs(the_job, used_keys)
verify_realization_haz_curves_stored_to_nrml(the_job, calculator)
# hazard curves: check results of mean and quantile computation
verify_mean_haz_curves_stored_to_kvs(the_job, used_keys)
verify_quantile_haz_curves_stored_to_kvs(the_job, calculator,
used_keys)
verify_mean_haz_curves_stored_to_nrml(the_job, calculator)
verify_quantile_haz_curves_stored_to_nrml(the_job, calculator)
# hazard maps: check results of mean and quantile computation
verify_mean_haz_maps_stored_to_kvs(the_job, calculator, used_keys)
verify_quantile_haz_maps_stored_to_kvs(the_job, calculator, used_keys)
verify_mean_haz_maps_stored_to_nrml(the_job)
verify_quantile_haz_maps_stored_to_nrml(the_job, calculator)
def test_generate_hazard_curves_using_classical_psha(self):
def verify_realization_haz_curves_stored_to_kvs(the_job, keys):
""" This just tests to make sure there something in the KVS
for each key in given list of keys. This does NOT test the
actual results. """
# TODO (LB): At some point we need to test the actual
# results to verify they are correct
realizations = int(
the_job.params['NUMBER_OF_LOGIC_TREE_SAMPLES'])
for realization in xrange(0, realizations):
for site in the_job.sites_to_compute():
key = tokens.hazard_curve_poes_key(
the_job.job_id, realization, site)
self.assertTrue(key in keys, "Missing key %s" % key)
def verify_mean_haz_curves_stored_to_kvs(the_job, keys):
""" Make sure that the keys and non-empty values for mean
hazard curves have been written to KVS."""
if the_job.params['COMPUTE_MEAN_HAZARD_CURVE'].lower() == 'true':
LOG.debug("verifying KVS entries for mean hazard curves")
for site in the_job.sites_to_compute():
key = tokens.mean_hazard_curve_key(the_job.job_id, site)
self.assertTrue(key in keys, "Missing key %s" % key)
def verify_mean_haz_maps_stored_to_kvs(the_job, calculator, keys):
""" Make sure that the keys and non-empty values for mean
hazard maps have been written to KVS."""
if (the_job.params[hazard_general.POES_PARAM_NAME] != '' and
the_job.params['COMPUTE_MEAN_HAZARD_CURVE'].lower() == \
'true'):
LOG.debug("verifying KVS entries for mean hazard maps")
for poe in calculator.poes_hazard_maps:
for site in the_job.sites_to_compute():
key = tokens.mean_hazard_map_key(
the_job.job_id, site, poe)
self.assertTrue(key in keys, "Missing key %s" % key)
def verify_quantile_haz_curves_stored_to_kvs(the_job, calculator,
keys):
""" Make sure that the keys and non-empty values for quantile
hazard curves have been written to KVS."""
quantiles = calculator.quantile_levels
LOG.debug("verifying KVS entries for quantile hazard curves, "\
"%s quantile values" % len(quantiles))
for quantile in quantiles:
for site in the_job.sites_to_compute():
key = tokens.quantile_hazard_curve_key(
the_job.job_id, site, quantile)
self.assertTrue(key in keys, "Missing key %s" % key)
def verify_quantile_haz_maps_stored_to_kvs(the_job, calculator, keys):
""" Make sure that the keys and non-empty values for quantile
hazard maps have been written to KVS."""
quantiles = calculator.quantile_levels
if (the_job.params[hazard_general.POES_PARAM_NAME] != '' and
len(quantiles) > 0):
poes = calculator.poes_hazard_maps
LOG.debug("verifying KVS entries for quantile hazard maps, "\
"%s quantile values, %s PoEs" % (
len(quantiles), len(poes)))
for quantile in quantiles:
for poe in poes:
for site in the_job.sites_to_compute():
key = tokens.quantile_hazard_map_key(
the_job.job_id, site, poe, quantile)
self.assertTrue(
key in keys, "Missing key %s" % key)
def verify_realization_haz_curves_stored_to_nrml(the_job, calculator):
"""Tests that a NRML file has been written for each realization,
and that this file validates against the NRML schema.
Does NOT test if results in NRML file are correct.
"""
realizations = int(
the_job.params['NUMBER_OF_LOGIC_TREE_SAMPLES'])
for realization in xrange(0, realizations):
nrml_path = os.path.join(
"demos/classical_psha_simple/computed_output",
calculator.hazard_curve_filename(realization))
LOG.debug("validating NRML file %s" % nrml_path)
self.assertTrue(xml.validates_against_xml_schema(
nrml_path, NRML_SCHEMA_PATH),
"NRML instance file %s does not validate against schema" \
% nrml_path)
def verify_mean_haz_curves_stored_to_nrml(the_job, calculator):
"""Tests that a mean hazard curve NRML file has been written,
and that this file validates against the NRML schema.
Does NOT test if results in NRML file are correct.
"""
if the_job.params['COMPUTE_MEAN_HAZARD_CURVE'].lower() == 'true':
nrml_path = os.path.join(
"demos/classical_psha_simple/computed_output",
calculator.mean_hazard_curve_filename())
LOG.debug("validating NRML file %s" % nrml_path)
self.assertTrue(xml.validates_against_xml_schema(
nrml_path, NRML_SCHEMA_PATH),
"NRML instance file %s does not validate against schema" \
% nrml_path)
def verify_mean_haz_maps_stored_to_nrml(the_job):
"""Tests that a mean hazard map NRML file has been written,
and that this file validates against the NRML schema.
Does NOT test if results in NRML file are correct.
"""
if (the_job.params[hazard_general.POES_PARAM_NAME] != '' and
the_job.params['COMPUTE_MEAN_HAZARD_CURVE'].lower() == \
'true'):
for poe in calculator.poes_hazard_maps:
nrml_path = os.path.join(
"demos/classical_psha_simple/computed_output",
calculator.mean_hazard_map_filename(poe))
LOG.debug("validating NRML file for mean hazard map %s" \
% nrml_path)
self.assertTrue(xml.validates_against_xml_schema(
nrml_path, NRML_SCHEMA_PATH),
"NRML instance file %s does not validate against "\
"schema" % nrml_path)
def verify_quantile_haz_curves_stored_to_nrml(the_job, calculator):
"""Tests that quantile hazard curve NRML files have been written,
and that these file validate against the NRML schema.
Does NOT test if results in NRML files are correct.
"""
for quantile in calculator.quantile_levels:
nrml_path = os.path.join(
"demos/classical_psha_simple/computed_output",
calculator.quantile_hazard_curve_filename(quantile))
LOG.debug("validating NRML file for quantile hazard curve: "\
"%s" % nrml_path)
self.assertTrue(xml.validates_against_xml_schema(
nrml_path, NRML_SCHEMA_PATH),
"NRML instance file %s does not validate against schema" \
% nrml_path)
def verify_quantile_haz_maps_stored_to_nrml(the_job, calculator):
"""Tests that quantile hazard map NRML files have been written,
and that these file validate against the NRML schema.
Does NOT test if results in NRML files are correct.
"""
quantiles = calculator.quantile_levels
if (the_job.params[hazard_general.POES_PARAM_NAME] != '' and
len(quantiles) > 0):
for poe in calculator.poes_hazard_maps:
for quantile in quantiles:
nrml_path = os.path.join(
"demos/classical_psha_simple/computed_output",
calculator.quantile_hazard_map_filename(quantile,
poe))
LOG.debug("validating NRML file for quantile hazard "\
"map: %s" % nrml_path)
self.assertTrue(xml.validates_against_xml_schema(
nrml_path, NRML_SCHEMA_PATH),
"NRML instance file %s does not validate against "\
"schema" % nrml_path)
base_path = helpers.testdata_path("classical_psha_simple")
path = helpers.testdata_path("classical_psha_simple/config.gem")
job = engine.prepare_job()
job_profile, params, sections = engine.import_job_profile(path, job)
the_job = JobContext(
params, job.id, sections=sections, base_path=base_path,
serialize_results_to=['db', 'xml'], oq_job_profile=job_profile,
oq_job=job)
the_job.to_kvs()
calc_mode = job_profile.calc_mode
calculator = CALCULATORS[calc_mode](the_job)
used_keys = []
calculator.execute(used_keys)
verify_realization_haz_curves_stored_to_kvs(the_job, used_keys)
verify_realization_haz_curves_stored_to_nrml(the_job, calculator)
# hazard curves: check results of mean and quantile computation
verify_mean_haz_curves_stored_to_kvs(the_job, used_keys)
verify_quantile_haz_curves_stored_to_kvs(the_job, calculator,
used_keys)
verify_mean_haz_curves_stored_to_nrml(the_job, calculator)
verify_quantile_haz_curves_stored_to_nrml(the_job, calculator)
# hazard maps: check results of mean and quantile computation
verify_mean_haz_maps_stored_to_kvs(the_job, calculator, used_keys)
verify_quantile_haz_maps_stored_to_kvs(the_job, calculator, used_keys)
verify_mean_haz_maps_stored_to_nrml(the_job)
verify_quantile_haz_maps_stored_to_nrml(the_job, calculator)
def setUpClass(cls):
cls.job = engine.prepare_job()
def setUp(self):
self.job = engine.prepare_job()
def setUpClass(cls):
cls.job = engine.prepare_job()
def setUp(self):
self.job = engine.prepare_job()
