def test_insert_map_datum(self):
"""An `uiapi.hazard_map_data` record is inserted correctly."""
self.output = self.setup_output()
session = Session.get()
hmw = HazardMapDBWriter(
session, self.output.path, self.output.oq_job.id)
hmw.output = self.output
# This output has no map data before calling the function under test.
self.assertEqual(0, len(self.output.hazardmapdata_set))
self.assertEqual(0, len(self.output.lossmapdata_set))
# Call the function under test.
data = HAZARD_MAP_DATA[-1]
hmw.insert_map_datum(*data)
# After calling the function under test we see the expected map data.
self.assertEqual(1, len(self.output.hazardmapdata_set))
self.assertEqual(0, len(self.output.lossmapdata_set))
# Make sure the inserted map data is correct.
[hmd] = self.output.hazardmapdata_set
point = data[0].point
self.assertEqual([point.x, point.y], hmd.location.coords(session))
self.assertEqual(round_float(data[1].get("IML")),
round_float(hmd.value))
def setup_output(self,
job_to_use=None,
output_type="hazard_map",
db_backed=True):
"""Create an output object of the given type.
:param job_to_use: if set use the passed
:py:class:`db.alchemy.models.OqJob` instance as opposed to
creating a new one.
:param str output_type: map type, one of "hazard_map", "loss_map"
:param bool db_backed: initialize the property of the newly created
:py:class:`db.alchemy.models.Output` instance with this value.
:returns: a :py:class:`db.alchemy.models.Output` instance
"""
job = job_to_use if job_to_use else self.setup_classic_job()
output = Output(owner=job.owner,
oq_job=job,
output_type=output_type,
db_backed=db_backed)
output.path = self.touch(
dir=os.path.join(job.path, "computed_output"),
suffix=".xml",
prefix="hzrd." if output_type == "hazard_map" else "loss.")
session = Session.get()
session.add(output)
session.commit()
return output
def test_insert_map_datum(self):
"""An `uiapi.hazard_map_data` record is inserted correctly."""
self.output = self.setup_output()
session = Session.get()
hmw = HazardMapDBWriter(session, self.output.path,
self.output.oq_job.id)
hmw.output = self.output
# This output has no map data before calling the function under test.
self.assertEqual(0, len(self.output.hazardmapdata_set))
self.assertEqual(0, len(self.output.lossmapdata_set))
# Call the function under test.
data = HAZARD_MAP_DATA[-1]
hmw.insert_map_datum(*data)
# After calling the function under test we see the expected map data.
self.assertEqual(1, len(self.output.hazardmapdata_set))
self.assertEqual(0, len(self.output.lossmapdata_set))
# Make sure the inserted map data is correct.
[hmd] = self.output.hazardmapdata_set
point = data[0].point
self.assertEqual([point.x, point.y], hmd.location.coords(session))
self.assertEqual(round_float(data[1].get("IML")),
round_float(hmd.value))
def setup_classic_job(self, create_job_path=True, upload_id=None):
"""Create a classic job with associated upload and inputs.
:param integer upload_id: if set use upload record with given db key.
:param bool create_job_path: if set the path for the job will be
created and captured in the job record
:returns: a :py:class:`db.alchemy.models.OqJob` instance
"""
session = Session.get()
upload = self.setup_upload(upload_id)
oqp = OqParams()
oqp.job_type = "classical"
oqp.upload = upload
oqp.region_grid_spacing = 0.01
oqp.min_magnitude = 5.0
oqp.investigation_time = 50.0
oqp.component = "gmroti50"
oqp.imt = "pga"
oqp.truncation_type = "twosided"
oqp.truncation_level = 3
oqp.reference_vs30_value = 760
oqp.imls = [
0.005,
0.007,
0.0098,
0.0137,
0.0192,
0.0269,
0.0376,
0.0527,
0.0738,
0.103,
0.145,
0.203,
0.284,
0.397,
0.556,
0.778,
]
oqp.poes = [0.01, 0.10]
oqp.realizations = 1
oqp.region = "POLYGON((-81.3 37.2, -80.63 38.04, -80.02 37.49, -81.3 37.2))"
session.add(oqp)
job = OqJob(oq_params=oqp, owner=upload.owner, job_type="classical")
session.add(job)
session.commit()
if create_job_path:
job.path = os.path.join(upload.path, str(job.id))
session.add(job)
session.commit()
os.mkdir(job.path)
os.chmod(job.path, 0777)
return job
def setup_upload(self, dbkey=None):
"""Create an upload with associated inputs.
:param integer dbkey: if set use the upload record with given db key.
:returns: a :py:class:`db.alchemy.models.Upload` instance
"""
session = Session.get()
if dbkey:
upload = session.query(Upload).filter(Upload.id == dbkey).one()
return upload
user = session.query(OqUser).filter(OqUser.user_name == "openquake").one()
upload = Upload(owner=user, path=tempfile.mkdtemp())
session.add(upload)
session.commit()
return upload
def setup_upload(self, dbkey=None):
"""Create an upload with associated inputs.
:param integer dbkey: if set use the upload record with given db key.
:returns: a :py:class:`db.alchemy.models.Upload` instance
"""
session = Session.get()
if dbkey:
upload = session.query(Upload).filter(Upload.id == dbkey).one()
return upload
user = session.query(OqUser).filter(
OqUser.user_name == "openquake").one()
upload = Upload(owner=user, path=tempfile.mkdtemp())
session.add(upload)
session.commit()
return upload
def teardown_upload(self, upload, filesystem_only=True):
"""
Tear down the file system (and potentially db) artefacts for the
given upload.
:param upload: the :py:class:`db.alchemy.models.Upload` instance
in question
:param bool filesystem_only: if set the upload/input database records
will be left intact. This saves time and the test db will be
dropped/recreated prior to the next db test suite run anyway.
"""
# This is like "rm -rf path"
shutil.rmtree(upload.path, ignore_errors=True)
if filesystem_only:
return
session = Session.get()
session.delete(upload)
session.commit()
def teardown_upload(self, upload, filesystem_only=True):
"""
Tear down the file system (and potentially db) artefacts for the
given upload.
:param upload: the :py:class:`db.alchemy.models.Upload` instance
in question
:param bool filesystem_only: if set the upload/input database records
will be left intact. This saves time and the test db will be
dropped/recreated prior to the next db test suite run anyway.
"""
# This is like "rm -rf path"
shutil.rmtree(upload.path, ignore_errors=True)
if filesystem_only:
return
session = Session.get()
session.delete(upload)
session.commit()
def test_serialize_sets_min_max_values(self):
"""
serialize() sets the minimum and maximum values on the output record.
"""
self.job = self.setup_classic_job()
session = Session.get()
output_path = self.generate_output_path(self.job)
hmw = HazardMapDBWriter(session, output_path, self.job.id)
# Call the function under test.
hmw.serialize(HAZARD_MAP_DATA)
minimum = min(data[1].get("IML") for data in HAZARD_MAP_DATA)
maximum = max(data[1].get("IML") for data in HAZARD_MAP_DATA)
# After calling the function under test we see the expected map data.
[output] = self.job.output_set
self.assertEqual(round_float(minimum), round_float(output.min_value))
self.assertEqual(round_float(maximum), round_float(output.max_value))
def test_serialize_sets_min_max_values(self):
"""
serialize() sets the minimum and maximum values on the output record.
"""
self.job = self.setup_classic_job()
session = Session.get()
output_path = self.generate_output_path(self.job)
hmw = HazardMapDBWriter(session, output_path, self.job.id)
# Call the function under test.
hmw.serialize(HAZARD_MAP_DATA)
minimum = min(data[1].get("IML") for data in HAZARD_MAP_DATA)
maximum = max(data[1].get("IML") for data in HAZARD_MAP_DATA)
# After calling the function under test we see the expected map data.
[output] = self.job.output_set
self.assertEqual(round_float(minimum), round_float(output.min_value))
self.assertEqual(round_float(maximum), round_float(output.max_value))
def create_hazardmap_writer(params, nrml_path):
"""Create a hazard map writer observing the settings in the config file.
:param dict params: the settings from the OpenQuake engine configuration
file.
:param str nrml_path: the full path of the XML/NRML representation of the
hazard map.
:returns: an :py:class:`output.hazard.HazardMapXMLWriter` or an
:py:class:`output.hazard.HazardMapDBWriter` instance.
"""
db_flag = params.get("SERIALIZE_MAPS_TO_DB")
if not db_flag or db_flag.lower() == "false":
return hazard_output.HazardMapXMLWriter(nrml_path)
else:
job_db_key = params.get("OPENQUAKE_JOB_ID")
assert job_db_key, "No job db key in the configuration parameters"
job_db_key = int(job_db_key)
session = Session.get()
return hazard_output.HazardMapDBWriter(session, nrml_path, job_db_key)
def create_hazardmap_writer(params, nrml_path):
"""Create a hazard map writer observing the settings in the config file.
:param dict params: the settings from the OpenQuake engine configuration
file.
:param str nrml_path: the full path of the XML/NRML representation of the
hazard map.
:returns: an :py:class:`output.hazard.HazardMapXMLWriter` or an
:py:class:`output.hazard.HazardMapDBWriter` instance.
"""
db_flag = params.get("SERIALIZE_MAPS_TO_DB")
if not db_flag or db_flag.lower() == "false":
return hazard_output.HazardMapXMLWriter(nrml_path)
else:
job_db_key = params.get("OPENQUAKE_JOB_ID")
assert job_db_key, "No job db key in the configuration parameters"
job_db_key = int(job_db_key)
session = Session.get()
return hazard_output.HazardMapDBWriter(session, nrml_path, job_db_key)
def test_serialize(self):
"""serialize() inserts the output and the hazard_map_data records."""
self.job = self.setup_classic_job()
session = Session.get()
output_path = self.generate_output_path(self.job)
hmw = HazardMapDBWriter(session, output_path, self.job.id)
# This job has no outputs before calling the function under test.
self.assertEqual(0, len(self.job.output_set))
# Call the function under test.
hmw.serialize(HAZARD_MAP_DATA)
# After calling the function under test we see the expected output.
self.assertEqual(1, len(self.job.output_set))
# After calling the function under test we see the expected map data.
[output] = self.job.output_set
self.assertEqual(len(HAZARD_MAP_DATA), len(output.hazardmapdata_set))
self.assertEqual(0, len(output.lossmapdata_set))
def teardown_output(self, output, teardown_job=True, filesystem_only=True):
"""
Tear down the file system (and potentially db) artefacts for the
given output.
:param output: the :py:class:`db.alchemy.models.Output` instance
in question
:param bool teardown_job: the associated job and its related artefacts
shall be torn down as well.
:param bool filesystem_only: if set the various database records will
be left intact. This saves time and the test db will be
dropped/recreated prior to the next db test suite run anyway.
"""
job = output.oq_job
if not filesystem_only:
session = Session.get()
session.delete(output)
session.commit()
if teardown_job:
self.teardown_job(job, filesystem_only=filesystem_only)
def teardown_job(self, job, filesystem_only=True):
"""
Tear down the file system (and potentially db) artefacts for the
given job.
:param job: the :py:class:`db.alchemy.models.OqJob` instance
in question
:param bool filesystem_only: if set the oq_job/oq_param/upload/input
database records will be left intact. This saves time and the test
db will be dropped/recreated prior to the next db test suite run
anyway.
"""
oqp = job.oq_params
self.teardown_upload(oqp.upload, filesystem_only=filesystem_only)
if filesystem_only:
return
session = Session.get()
session.delete(job)
session.delete(oqp)
session.commit()
def test_serialize(self):
"""serialize() inserts the output and the hazard_map_data records."""
self.job = self.setup_classic_job()
session = Session.get()
output_path = self.generate_output_path(self.job)
hmw = HazardMapDBWriter(session, output_path, self.job.id)
# This job has no outputs before calling the function under test.
self.assertEqual(0, len(self.job.output_set))
# Call the function under test.
hmw.serialize(HAZARD_MAP_DATA)
# After calling the function under test we see the expected output.
self.assertEqual(1, len(self.job.output_set))
# After calling the function under test we see the expected map data.
[output] = self.job.output_set
self.assertEqual(len(HAZARD_MAP_DATA), len(output.hazardmapdata_set))
self.assertEqual(0, len(output.lossmapdata_set))
def teardown_output(self, output, teardown_job=True, filesystem_only=True):
"""
Tear down the file system (and potentially db) artefacts for the
given output.
:param output: the :py:class:`db.alchemy.models.Output` instance
in question
:param bool teardown_job: the associated job and its related artefacts
shall be torn down as well.
:param bool filesystem_only: if set the various database records will
be left intact. This saves time and the test db will be
dropped/recreated prior to the next db test suite run anyway.
"""
job = output.oq_job
if not filesystem_only:
session = Session.get()
session.delete(output)
session.commit()
if teardown_job:
self.teardown_job(job, filesystem_only=filesystem_only)
def teardown_job(self, job, filesystem_only=True):
"""
Tear down the file system (and potentially db) artefacts for the
given job.
:param job: the :py:class:`db.alchemy.models.OqJob` instance
in question
:param bool filesystem_only: if set the oq_job/oq_param/upload/input
database records will be left intact. This saves time and the test
db will be dropped/recreated prior to the next db test suite run
anyway.
"""
oqp = job.oq_params
self.teardown_upload(oqp.upload, filesystem_only=filesystem_only)
if filesystem_only:
return
session = Session.get()
session.delete(job)
session.delete(oqp)
session.commit()
def setup_classic_job(self, create_job_path=True, upload_id=None):
"""Create a classic job with associated upload and inputs.
:param integer upload_id: if set use upload record with given db key.
:param bool create_job_path: if set the path for the job will be
created and captured in the job record
:returns: a :py:class:`db.alchemy.models.OqJob` instance
"""
session = Session.get()
upload = self.setup_upload(upload_id)
oqp = OqParams()
oqp.job_type = "classical"
oqp.upload = upload
oqp.region_grid_spacing = 0.01
oqp.min_magnitude = 5.0
oqp.investigation_time = 50.0
oqp.component = "gmroti50"
oqp.imt = "pga"
oqp.truncation_type = "twosided"
oqp.truncation_level = 3
oqp.reference_vs30_value = 760
oqp.imls = [
0.005, 0.007, 0.0098, 0.0137, 0.0192, 0.0269, 0.0376, 0.0527,
0.0738, 0.103, 0.145, 0.203, 0.284, 0.397, 0.556, 0.778
]
oqp.poes = [0.01, 0.10]
oqp.realizations = 1
oqp.region = (
"POLYGON((-81.3 37.2, -80.63 38.04, -80.02 37.49, -81.3 37.2))")
session.add(oqp)
job = OqJob(oq_params=oqp, owner=upload.owner, job_type="classical")
session.add(job)
session.commit()
if create_job_path:
job.path = os.path.join(upload.path, str(job.id))
session.add(job)
session.commit()
os.mkdir(job.path)
os.chmod(job.path, 0777)
return job
def test_insert_output(self):
"""An `uiapi.output` record is inserted correctly."""
self.job = self.setup_classic_job()
session = Session.get()
output_path = self.generate_output_path(self.job)
hmw = HazardMapDBWriter(session, output_path, self.job.id)
# This job has no outputs before calling the function under test.
self.assertEqual(0, len(self.job.output_set))
# Call the function under test.
hmw.insert_output()
# After calling the function under test we see the expected output.
self.assertEqual(1, len(self.job.output_set))
# Make sure the inserted output record has the right data.
[output] = self.job.output_set
self.assertTrue(output.db_backed)
self.assertEqual(output_path, output.path)
self.assertEqual("hazard_map", output.output_type)
self.assertIs(self.job, output.oq_job)
def test_insert_output(self):
"""An `uiapi.output` record is inserted correctly."""
self.job = self.setup_classic_job()
session = Session.get()
output_path = self.generate_output_path(self.job)
hmw = HazardMapDBWriter(session, output_path, self.job.id)
# This job has no outputs before calling the function under test.
self.assertEqual(0, len(self.job.output_set))
# Call the function under test.
hmw.insert_output()
# After calling the function under test we see the expected output.
self.assertEqual(1, len(self.job.output_set))
# Make sure the inserted output record has the right data.
[output] = self.job.output_set
self.assertTrue(output.db_backed)
self.assertEqual(output_path, output.path)
self.assertEqual("hazard_map", output.output_type)
self.assertIs(self.job, output.oq_job)
def setup_output(self, job_to_use=None, output_type="hazard_map", db_backed=True):
"""Create an output object of the given type.
:param job_to_use: if set use the passed
:py:class:`db.alchemy.models.OqJob` instance as opposed to
creating a new one.
:param str output_type: map type, one of "hazard_map", "loss_map"
:param bool db_backed: initialize the property of the newly created
:py:class:`db.alchemy.models.Output` instance with this value.
:returns: a :py:class:`db.alchemy.models.Output` instance
"""
job = job_to_use if job_to_use else self.setup_classic_job()
output = Output(owner=job.owner, oq_job=job, output_type=output_type, db_backed=db_backed)
output.path = self.touch(
dir=os.path.join(job.path, "computed_output"),
suffix=".xml",
prefix="hzrd." if output_type == "hazard_map" else "loss.",
)
session = Session.get()
session.add(output)
session.commit()
return output