def setUp(self): self.job = self.setup_classic_job() self.session = get_uiapi_writer_session() output_path = self.generate_output_path(self.job) self.display_name = os.path.basename(output_path) self.writer = LossCurveDBWriter(self.session, output_path, self.job.id)
def test_insert_map_datum(self): """An `uiapi.hazard_map_data` record is inserted correctly.""" self.output = self.setup_output() session = get_uiapi_writer_session() hmw = HazardMapDBWriter( session, self.output.path, self.output.oq_job.id) hmw.output = self.output # This output has no map data before calling the function under test. self.assertEqual(0, len(self.output.hazardmapdata_set)) self.assertEqual(0, len(self.output.lossmapdata_set)) # Call the function under test. data = HAZARD_MAP_DATA[-1] hmw.insert_map_datum(*data) # After calling the function under test we see the expected map data. self.assertEqual(1, len(self.output.hazardmapdata_set)) self.assertEqual(0, len(self.output.lossmapdata_set)) # Make sure the inserted map data is correct. [hmd] = self.output.hazardmapdata_set point = data[0].point self.assertEqual([point.x, point.y], hmd.location.coords(session)) self.assertEqual(round_float(data[1].get("IML")), round_float(hmd.value))
def test_serialize(self): """serialize() inserts the output and the hazard_map_data records.""" self.job = self.setup_classic_job() session = get_uiapi_writer_session() output_path = self.generate_output_path(self.job) hcw = HazardCurveDBWriter(session, output_path, self.job.id) # This job has no outputs before calling the function under test. self.assertEqual(0, len(self.job.output_set)) # Call the function under test. hcw.serialize(HAZARD_CURVE_DATA) # After calling the function under test we see the expected output. self.assertEqual(1, len(self.job.output_set)) # After calling the function under test we see the expected map data. [output] = self.job.output_set self.assertEqual(4, len(output.hazardcurvedata_set)) self.assertEqual(0, len(output.lossmapdata_set)) # read data from the DB and check that it's equal to the original data inserted_data = [] for hcd in output.hazardcurvedata_set: for hcdn in hcd.hazardcurvenodedata_set: location = hcdn.location.coords(session) node = (Site(location[0], location[1]), {'IMLValues': hcd.imls, 'PoEValues': hcdn.poes}) if hcd.end_branch_label: node[1]['endBranchLabel'] = hcd.end_branch_label else: node[1]['statistics'] = hcd.statistic_type if hcd.quantile is not None: node[1]['quantileValue'] = hcd.quantile inserted_data.append(node) def normalize(values): def sort_key(v): return v[0].longitude, v[0].latitude, v[1] def norm(dic): dic = dict(dic) # remove keys not stored in the database dic.pop('investigationTimeSpan', None) dic.pop('IMT', None) return dic return sorted([(s, norm(v)) for s, v in values], key=sort_key) self.assertEquals(normalize(HAZARD_CURVE_DATA), normalize(inserted_data))
def setup_upload(self, dbkey=None): """Create an upload with associated inputs. :param integer dbkey: if set use the upload record with given db key. :returns: a :py:class:`db.alchemy.models.Upload` instance """ session = get_uiapi_writer_session() if dbkey: upload = session.query(Upload).filter(Upload.id == dbkey).one() return upload user = session.query(OqUser).filter( OqUser.user_name == "openquake").one() upload = Upload(owner=user, path=tempfile.mkdtemp()) session.add(upload) session.commit() return upload
def test_serialize_sets_min_max_values(self): """ serialize() sets the minimum and maximum values on the output record. """ self.job = self.setup_classic_job() session = get_uiapi_writer_session() output_path = self.generate_output_path(self.job) hmw = HazardMapDBWriter(session, output_path, self.job.id) # Call the function under test. hmw.serialize(HAZARD_MAP_DATA) minimum = min(data[1].get("IML") for data in HAZARD_MAP_DATA) maximum = max(data[1].get("IML") for data in HAZARD_MAP_DATA) # After calling the function under test we see the expected map data. [output] = self.job.output_set self.assertEqual(round_float(minimum), round_float(output.min_value)) self.assertEqual(round_float(maximum), round_float(output.max_value))
def teardown_upload(self, upload, filesystem_only=True): """ Tear down the file system (and potentially db) artefacts for the given upload. :param upload: the :py:class:`db.alchemy.models.Upload` instance in question :param bool filesystem_only: if set the upload/input database records will be left intact. This saves time and the test db will be dropped/recreated prior to the next db test suite run anyway. """ # This is like "rm -rf path" shutil.rmtree(upload.path, ignore_errors=True) if filesystem_only: return session = get_uiapi_writer_session() session.delete(upload) session.commit()
def create_hazardmap_writer(params, nrml_path): """Create a hazard map writer observing the settings in the config file. :param dict params: the settings from the OpenQuake engine configuration file. :param str nrml_path: the full path of the XML/NRML representation of the hazard map. :returns: an :py:class:`output.hazard.HazardMapXMLWriter` or an :py:class:`output.hazard.HazardMapDBWriter` instance. """ db_flag = params.get("SERIALIZE_MAPS_TO_DB") if not db_flag or db_flag.lower() == "false": return hazard_output.HazardMapXMLWriter(nrml_path) else: job_db_key = params.get("OPENQUAKE_JOB_ID") assert job_db_key, "No job db key in the configuration parameters" job_db_key = int(job_db_key) session = get_uiapi_writer_session() return hazard_output.HazardMapDBWriter(session, nrml_path, job_db_key)
def test_serialize(self): """serialize() inserts the output and the hazard_map_data records.""" self.job = self.setup_classic_job() session = get_uiapi_writer_session() output_path = self.generate_output_path(self.job) hmw = HazardMapDBWriter(session, output_path, self.job.id) # This job has no outputs before calling the function under test. self.assertEqual(0, len(self.job.output_set)) # Call the function under test. hmw.serialize(HAZARD_MAP_DATA) # After calling the function under test we see the expected output. self.assertEqual(1, len(self.job.output_set)) # After calling the function under test we see the expected map data. [output] = self.job.output_set self.assertEqual(len(HAZARD_MAP_DATA), len(output.hazardmapdata_set)) self.assertEqual(0, len(output.lossmapdata_set))
def teardown_output(self, output, teardown_job=True, filesystem_only=True): """ Tear down the file system (and potentially db) artefacts for the given output. :param output: the :py:class:`db.alchemy.models.Output` instance in question :param bool teardown_job: the associated job and its related artefacts shall be torn down as well. :param bool filesystem_only: if set the various database records will be left intact. This saves time and the test db will be dropped/recreated prior to the next db test suite run anyway. """ job = output.oq_job if not filesystem_only: session = get_uiapi_writer_session() session.delete(output) session.commit() if teardown_job: self.teardown_job(job, filesystem_only=filesystem_only)
def teardown_job(self, job, filesystem_only=True): """ Tear down the file system (and potentially db) artefacts for the given job. :param job: the :py:class:`db.alchemy.models.OqJob` instance in question :param bool filesystem_only: if set the oq_job/oq_param/upload/input database records will be left intact. This saves time and the test db will be dropped/recreated prior to the next db test suite run anyway. """ oqp = job.oq_params self.teardown_upload(oqp.upload, filesystem_only=filesystem_only) if filesystem_only: return session = get_uiapi_writer_session() session.delete(job) session.delete(oqp) session.commit()
def setup_output(self, job_to_use=None, output_type="hazard_map", db_backed=True): """Create an output object of the given type. :param job_to_use: if set use the passed :py:class:`db.alchemy.models.OqJob` instance as opposed to creating a new one. :param str output_type: map type, one of "hazard_map", "loss_map" :param bool db_backed: initialize the property of the newly created :py:class:`db.alchemy.models.Output` instance with this value. :returns: a :py:class:`db.alchemy.models.Output` instance """ job = job_to_use if job_to_use else self.setup_classic_job() output = Output(owner=job.owner, oq_job=job, output_type=output_type, db_backed=db_backed) output.path = self.generate_output_path(job, output_type) output.display_name = os.path.basename(output.path) session = get_uiapi_writer_session() session.add(output) session.commit() return output
def setup_classic_job(self, create_job_path=True, upload_id=None): """Create a classic job with associated upload and inputs. :param integer upload_id: if set use upload record with given db key. :param bool create_job_path: if set the path for the job will be created and captured in the job record :returns: a :py:class:`db.alchemy.models.OqJob` instance """ session = get_uiapi_writer_session() upload = self.setup_upload(upload_id) oqp = OqParams() oqp.job_type = "classical" oqp.upload = upload oqp.region_grid_spacing = 0.01 oqp.min_magnitude = 5.0 oqp.investigation_time = 50.0 oqp.component = "gmroti50" oqp.imt = "pga" oqp.truncation_type = "twosided" oqp.truncation_level = 3 oqp.reference_vs30_value = 760 oqp.imls = [ 0.005, 0.007, 0.0098, 0.0137, 0.0192, 0.0269, 0.0376, 0.0527, 0.0738, 0.103, 0.145, 0.203, 0.284, 0.397, 0.556, 0.778] oqp.poes = [0.01, 0.10] oqp.realizations = 1 oqp.region = ( "POLYGON((-81.3 37.2, -80.63 38.04, -80.02 37.49, -81.3 37.2))") session.add(oqp) job = OqJob(oq_params=oqp, owner=upload.owner, job_type="classical") session.add(job) session.commit() if create_job_path: job.path = os.path.join(upload.path, str(job.id)) session.add(job) session.commit() os.mkdir(job.path) os.chmod(job.path, 0777) return job
def test_serialize(self): """serialize() inserts the output and the gmf_data records.""" self.job = self.setup_classic_job() session = get_uiapi_writer_session() output_path = self.generate_output_path(self.job) gmfw = GMFDBWriter(session, output_path, self.job.id) # This job has no outputs before calling the function under test. self.assertEqual(0, len(self.job.output_set)) # Call the function under test. gmfw.serialize(GMF_DATA) # After calling the function under test we see the expected output. self.assertEqual(1, len(self.job.output_set)) # After calling the function under test we see the expected map data. [output] = self.job.output_set self.assertEqual(0, len(output.hazardcurvedata_set)) self.assertEqual(0, len(output.lossmapdata_set)) self.assertEqual(4, len(output.gmfdata_set)) # read data from the DB and check that it's equal to the original data inserted_data = [] for gmfd in output.gmfdata_set: location = gmfd.location.coords(session) inserted_data.append((Site(location[0], location[1]), {'groundMotion': gmfd.ground_motion})) def normalize(values): def sort_key(v): return v[0].longitude, v[0].latitude, v[1] return sorted(values, key=sort_key) self.assertEquals(normalize(GMF_DATA.items()), normalize(inserted_data))
def test_insert_output(self): """An `uiapi.output` record is inserted correctly.""" self.job = self.setup_classic_job() session = get_uiapi_writer_session() output_path = self.generate_output_path(self.job) display_name = os.path.basename(output_path) hmw = HazardMapDBWriter(session, output_path, self.job.id) # This job has no outputs before calling the function under test. self.assertEqual(0, len(self.job.output_set)) # Call the function under test. hmw.insert_output("hazard_map") # After calling the function under test we see the expected output. self.assertEqual(1, len(self.job.output_set)) # Make sure the inserted output record has the right data. [output] = self.job.output_set self.assertTrue(output.db_backed) self.assertTrue(output.path is None) self.assertEqual(display_name, output.display_name) self.assertEqual("hazard_map", output.output_type) self.assertTrue(self.job is output.oq_job)