def setUp(self):
self.job = self.setup_classic_job()
self.session = get_uiapi_writer_session()
output_path = self.generate_output_path(self.job)
self.display_name = os.path.basename(output_path)
self.writer = LossCurveDBWriter(self.session, output_path, self.job.id)
def test_insert_map_datum(self):
"""An `uiapi.hazard_map_data` record is inserted correctly."""
self.output = self.setup_output()
session = get_uiapi_writer_session()
hmw = HazardMapDBWriter(
session, self.output.path, self.output.oq_job.id)
hmw.output = self.output
# This output has no map data before calling the function under test.
self.assertEqual(0, len(self.output.hazardmapdata_set))
self.assertEqual(0, len(self.output.lossmapdata_set))
# Call the function under test.
data = HAZARD_MAP_DATA[-1]
hmw.insert_map_datum(*data)
# After calling the function under test we see the expected map data.
self.assertEqual(1, len(self.output.hazardmapdata_set))
self.assertEqual(0, len(self.output.lossmapdata_set))
# Make sure the inserted map data is correct.
[hmd] = self.output.hazardmapdata_set
point = data[0].point
self.assertEqual([point.x, point.y], hmd.location.coords(session))
self.assertEqual(round_float(data[1].get("IML")),
round_float(hmd.value))
def test_serialize(self):
"""serialize() inserts the output and the hazard_map_data records."""
self.job = self.setup_classic_job()
session = get_uiapi_writer_session()
output_path = self.generate_output_path(self.job)
hcw = HazardCurveDBWriter(session, output_path, self.job.id)
# This job has no outputs before calling the function under test.
self.assertEqual(0, len(self.job.output_set))
# Call the function under test.
hcw.serialize(HAZARD_CURVE_DATA)
# After calling the function under test we see the expected output.
self.assertEqual(1, len(self.job.output_set))
# After calling the function under test we see the expected map data.
[output] = self.job.output_set
self.assertEqual(4, len(output.hazardcurvedata_set))
self.assertEqual(0, len(output.lossmapdata_set))
# read data from the DB and check that it's equal to the original data
inserted_data = []
for hcd in output.hazardcurvedata_set:
for hcdn in hcd.hazardcurvenodedata_set:
location = hcdn.location.coords(session)
node = (Site(location[0], location[1]),
{'IMLValues': hcd.imls,
'PoEValues': hcdn.poes})
if hcd.end_branch_label:
node[1]['endBranchLabel'] = hcd.end_branch_label
else:
node[1]['statistics'] = hcd.statistic_type
if hcd.quantile is not None:
node[1]['quantileValue'] = hcd.quantile
inserted_data.append(node)
def normalize(values):
def sort_key(v):
return v[0].longitude, v[0].latitude, v[1]
def norm(dic):
dic = dict(dic)
# remove keys not stored in the database
dic.pop('investigationTimeSpan', None)
dic.pop('IMT', None)
return dic
return sorted([(s, norm(v)) for s, v in values], key=sort_key)
self.assertEquals(normalize(HAZARD_CURVE_DATA),
normalize(inserted_data))
def setup_upload(self, dbkey=None):
"""Create an upload with associated inputs.
:param integer dbkey: if set use the upload record with given db key.
:returns: a :py:class:`db.alchemy.models.Upload` instance
"""
session = get_uiapi_writer_session()
if dbkey:
upload = session.query(Upload).filter(Upload.id == dbkey).one()
return upload
user = session.query(OqUser).filter(
OqUser.user_name == "openquake").one()
upload = Upload(owner=user, path=tempfile.mkdtemp())
session.add(upload)
session.commit()
return upload
def test_serialize_sets_min_max_values(self):
"""
serialize() sets the minimum and maximum values on the output record.
"""
self.job = self.setup_classic_job()
session = get_uiapi_writer_session()
output_path = self.generate_output_path(self.job)
hmw = HazardMapDBWriter(session, output_path, self.job.id)
# Call the function under test.
hmw.serialize(HAZARD_MAP_DATA)
minimum = min(data[1].get("IML") for data in HAZARD_MAP_DATA)
maximum = max(data[1].get("IML") for data in HAZARD_MAP_DATA)
# After calling the function under test we see the expected map data.
[output] = self.job.output_set
self.assertEqual(round_float(minimum), round_float(output.min_value))
self.assertEqual(round_float(maximum), round_float(output.max_value))
def teardown_upload(self, upload, filesystem_only=True):
"""
Tear down the file system (and potentially db) artefacts for the
given upload.
:param upload: the :py:class:`db.alchemy.models.Upload` instance
in question
:param bool filesystem_only: if set the upload/input database records
will be left intact. This saves time and the test db will be
dropped/recreated prior to the next db test suite run anyway.
"""
# This is like "rm -rf path"
shutil.rmtree(upload.path, ignore_errors=True)
if filesystem_only:
return
session = get_uiapi_writer_session()
session.delete(upload)
session.commit()
def create_hazardmap_writer(params, nrml_path):
"""Create a hazard map writer observing the settings in the config file.
:param dict params: the settings from the OpenQuake engine configuration
file.
:param str nrml_path: the full path of the XML/NRML representation of the
hazard map.
:returns: an :py:class:`output.hazard.HazardMapXMLWriter` or an
:py:class:`output.hazard.HazardMapDBWriter` instance.
"""
db_flag = params.get("SERIALIZE_MAPS_TO_DB")
if not db_flag or db_flag.lower() == "false":
return hazard_output.HazardMapXMLWriter(nrml_path)
else:
job_db_key = params.get("OPENQUAKE_JOB_ID")
assert job_db_key, "No job db key in the configuration parameters"
job_db_key = int(job_db_key)
session = get_uiapi_writer_session()
return hazard_output.HazardMapDBWriter(session, nrml_path, job_db_key)
def test_serialize(self):
"""serialize() inserts the output and the hazard_map_data records."""
self.job = self.setup_classic_job()
session = get_uiapi_writer_session()
output_path = self.generate_output_path(self.job)
hmw = HazardMapDBWriter(session, output_path, self.job.id)
# This job has no outputs before calling the function under test.
self.assertEqual(0, len(self.job.output_set))
# Call the function under test.
hmw.serialize(HAZARD_MAP_DATA)
# After calling the function under test we see the expected output.
self.assertEqual(1, len(self.job.output_set))
# After calling the function under test we see the expected map data.
[output] = self.job.output_set
self.assertEqual(len(HAZARD_MAP_DATA), len(output.hazardmapdata_set))
self.assertEqual(0, len(output.lossmapdata_set))
def teardown_output(self, output, teardown_job=True, filesystem_only=True):
"""
Tear down the file system (and potentially db) artefacts for the
given output.
:param output: the :py:class:`db.alchemy.models.Output` instance
in question
:param bool teardown_job: the associated job and its related artefacts
shall be torn down as well.
:param bool filesystem_only: if set the various database records will
be left intact. This saves time and the test db will be
dropped/recreated prior to the next db test suite run anyway.
"""
job = output.oq_job
if not filesystem_only:
session = get_uiapi_writer_session()
session.delete(output)
session.commit()
if teardown_job:
self.teardown_job(job, filesystem_only=filesystem_only)
def teardown_job(self, job, filesystem_only=True):
"""
Tear down the file system (and potentially db) artefacts for the
given job.
:param job: the :py:class:`db.alchemy.models.OqJob` instance
in question
:param bool filesystem_only: if set the oq_job/oq_param/upload/input
database records will be left intact. This saves time and the test
db will be dropped/recreated prior to the next db test suite run
anyway.
"""
oqp = job.oq_params
self.teardown_upload(oqp.upload, filesystem_only=filesystem_only)
if filesystem_only:
return
session = get_uiapi_writer_session()
session.delete(job)
session.delete(oqp)
session.commit()
def setup_output(self, job_to_use=None, output_type="hazard_map",
db_backed=True):
"""Create an output object of the given type.
:param job_to_use: if set use the passed
:py:class:`db.alchemy.models.OqJob` instance as opposed to
creating a new one.
:param str output_type: map type, one of "hazard_map", "loss_map"
:param bool db_backed: initialize the property of the newly created
:py:class:`db.alchemy.models.Output` instance with this value.
:returns: a :py:class:`db.alchemy.models.Output` instance
"""
job = job_to_use if job_to_use else self.setup_classic_job()
output = Output(owner=job.owner, oq_job=job, output_type=output_type,
db_backed=db_backed)
output.path = self.generate_output_path(job, output_type)
output.display_name = os.path.basename(output.path)
session = get_uiapi_writer_session()
session.add(output)
session.commit()
return output
def setup_classic_job(self, create_job_path=True, upload_id=None):
"""Create a classic job with associated upload and inputs.
:param integer upload_id: if set use upload record with given db key.
:param bool create_job_path: if set the path for the job will be
created and captured in the job record
:returns: a :py:class:`db.alchemy.models.OqJob` instance
"""
session = get_uiapi_writer_session()
upload = self.setup_upload(upload_id)
oqp = OqParams()
oqp.job_type = "classical"
oqp.upload = upload
oqp.region_grid_spacing = 0.01
oqp.min_magnitude = 5.0
oqp.investigation_time = 50.0
oqp.component = "gmroti50"
oqp.imt = "pga"
oqp.truncation_type = "twosided"
oqp.truncation_level = 3
oqp.reference_vs30_value = 760
oqp.imls = [
0.005, 0.007, 0.0098, 0.0137, 0.0192, 0.0269, 0.0376, 0.0527,
0.0738, 0.103, 0.145, 0.203, 0.284, 0.397, 0.556, 0.778]
oqp.poes = [0.01, 0.10]
oqp.realizations = 1
oqp.region = (
"POLYGON((-81.3 37.2, -80.63 38.04, -80.02 37.49, -81.3 37.2))")
session.add(oqp)
job = OqJob(oq_params=oqp, owner=upload.owner, job_type="classical")
session.add(job)
session.commit()
if create_job_path:
job.path = os.path.join(upload.path, str(job.id))
session.add(job)
session.commit()
os.mkdir(job.path)
os.chmod(job.path, 0777)
return job
def test_serialize(self):
"""serialize() inserts the output and the gmf_data records."""
self.job = self.setup_classic_job()
session = get_uiapi_writer_session()
output_path = self.generate_output_path(self.job)
gmfw = GMFDBWriter(session, output_path, self.job.id)
# This job has no outputs before calling the function under test.
self.assertEqual(0, len(self.job.output_set))
# Call the function under test.
gmfw.serialize(GMF_DATA)
# After calling the function under test we see the expected output.
self.assertEqual(1, len(self.job.output_set))
# After calling the function under test we see the expected map data.
[output] = self.job.output_set
self.assertEqual(0, len(output.hazardcurvedata_set))
self.assertEqual(0, len(output.lossmapdata_set))
self.assertEqual(4, len(output.gmfdata_set))
# read data from the DB and check that it's equal to the original data
inserted_data = []
for gmfd in output.gmfdata_set:
location = gmfd.location.coords(session)
inserted_data.append((Site(location[0], location[1]),
{'groundMotion': gmfd.ground_motion}))
def normalize(values):
def sort_key(v):
return v[0].longitude, v[0].latitude, v[1]
return sorted(values, key=sort_key)
self.assertEquals(normalize(GMF_DATA.items()),
normalize(inserted_data))
def test_insert_output(self):
"""An `uiapi.output` record is inserted correctly."""
self.job = self.setup_classic_job()
session = get_uiapi_writer_session()
output_path = self.generate_output_path(self.job)
display_name = os.path.basename(output_path)
hmw = HazardMapDBWriter(session, output_path, self.job.id)
# This job has no outputs before calling the function under test.
self.assertEqual(0, len(self.job.output_set))
# Call the function under test.
hmw.insert_output("hazard_map")
# After calling the function under test we see the expected output.
self.assertEqual(1, len(self.job.output_set))
# Make sure the inserted output record has the right data.
[output] = self.job.output_set
self.assertTrue(output.db_backed)
self.assertTrue(output.path is None)
self.assertEqual(display_name, output.display_name)
self.assertEqual("hazard_map", output.output_type)
self.assertTrue(self.job is output.oq_job)
