def setUp(self):
self.job = self.setup_classic_job()
session = get_db_session("reslt", "writer")
for gmf in GMF_DATA():
output_path = self.generate_output_path(self.job)
hcw = GMFDBWriter(session, output_path, self.job.id)
hcw.serialize(gmf)
def run_job(job_file, output_type):
"""Given a job_file, run the job."""
a_job = Job.from_file(job_file, output_type)
is_job_valid = a_job.is_valid()
if is_job_valid[0]:
a_job.set_status('running')
try:
a_job.launch()
except sqlalchemy.exc.SQLAlchemyError:
# Try to cleanup the session status to have a chance to update the
# job record without further errors.
session = get_db_session("reslt", "writer")
if session.is_active:
session.rollback()
a_job.set_status('failed')
raise
except:
a_job.set_status('failed')
raise
else:
a_job.set_status('succeeded')
else:
a_job.set_status('failed')
LOG.critical("The job configuration is inconsistent:")
for error_message in is_job_valid[1]:
LOG.critical(" >>> %s" % error_message)
def create_loss_map_writer(job_id, serialize_to, nrml_path, deterministic):
"""Create a loss map writer observing the settings in the config file.
:param job_id: the id of the job the curve belongs to.
:type job_id: int
:param serialize_to: where to serialize
:type serialize_to: list of strings. Permitted values: 'db', 'xml'.
:param nrml_path: the full path of the XML/NRML representation of the
loss map.
:type nrml_path: string
:param deterministic: Whether the loss map is deterministic (True) or
non-deterministic (False)
:type deterministic: boolean
:returns: None or an instance of
:py:class:`output.risk.LossMapXMLWriter` or
:py:class:`output.risk.LossMapDBWriter`
"""
writers = []
if 'db' in serialize_to:
writers.append(LossMapDBWriter(get_db_session("reslt", "writer"),
nrml_path,
job_id))
if 'xml' in serialize_to:
if deterministic:
writers.append(LossMapXMLWriter(nrml_path))
else:
# No XML schema for non-deterministic maps yet (see bug 805434)
pass
return writer.compose_writers(writers)
def setUp(self):
self.job = self.setup_classic_job()
self.session = get_db_session("reslt", "writer")
output_path = self.generate_output_path(self.job)
self.display_name = os.path.basename(output_path)
self.writer = GMFDBWriter(self.session, output_path, self.job.id)
self.reader = GMFDBReader(self.session)
def get_status_from_db(job_id):
"""
Get the status of the database record belonging to job ``job_id``.
:returns: one of strings 'pending', 'running', 'succeeded', 'failed'.
"""
session = get_db_session("reslt", "reader")
[status] = session.query(OqJob.status).filter(OqJob.id == job_id).one()
return status
def _gmf_db_list(self, job_id): # pylint: disable=R0201
"""Returns a list of the output IDs of all computed GMFs"""
session = get_db_session("reslt", "reader")
ids = session.query(models.Output.id) \
.filter(models.Output.oq_job_id == job_id) \
.filter(models.Output.output_type == 'gmf')
return [row[0] for row in ids]
def test_is_job_completed(self):
job_id = Job.from_file(helpers.get_data_path(CONFIG_FILE), 'db').job_id
session = get_db_session("reslt", "writer")
pairs = [('pending', False), ('running', False),
('succeeded', True), ('failed', True)]
for status, is_completed in pairs:
session.query(OqJob).update({'status': status})
session.commit()
self.assertEqual(Job.is_job_completed(job_id), is_completed)
def prepare_job(params):
"""
Create a new OqJob and fill in the related OpParams entry.
Returns the newly created job object.
"""
session = get_db_session("reslt", "writer")
# TODO specify the owner as a command line parameter
owner = session.query(OqUser).filter(OqUser.user_name == 'openquake').one()
oqp = OqParams(upload=None)
job = OqJob(owner=owner, path=None, oq_params=oqp,
job_type=CALCULATION_MODE[params['CALCULATION_MODE']])
# fill-in parameters
oqp.job_type = job.job_type
oqp.region_grid_spacing = float(params['REGION_GRID_SPACING'])
oqp.component = ENUM_MAP[params['COMPONENT']]
oqp.imt = ENUM_MAP[params['INTENSITY_MEASURE_TYPE']]
oqp.truncation_type = ENUM_MAP[params['GMPE_TRUNCATION_TYPE']]
oqp.truncation_level = float(params['TRUNCATION_LEVEL'])
oqp.reference_vs30_value = float(params['REFERENCE_VS30_VALUE'])
if oqp.imt == 'sa':
oqp.period = float(params.get('PERIOD', 0.0))
if oqp.job_type != 'classical':
oqp.gm_correlated = (
params['GROUND_MOTION_CORRELATION'].lower() != 'false')
else:
oqp.imls = [float(v) for v in
params['INTENSITY_MEASURE_LEVELS'].split(",")]
oqp.poes = [float(v) for v in
params['POES_HAZARD_MAPS'].split(" ")]
if oqp.job_type != 'deterministic':
oqp.investigation_time = float(params.get('INVESTIGATION_TIME', 0.0))
oqp.min_magnitude = float(params.get('MINIMUM_MAGNITUDE', 0.0))
oqp.realizations = int(params['NUMBER_OF_LOGIC_TREE_SAMPLES'])
if oqp.job_type == 'event_based':
oqp.histories = int(params['NUMBER_OF_SEISMICITY_HISTORIES'])
# config lat/lon -> postgis -> lon/lat
coords = [float(v) for v in
params['REGION_VERTEX'].split(",")]
vertices = ["%f %f" % (coords[i + 1], coords[i])
for i in xrange(0, len(coords), 2)]
region = "SRID=4326;POLYGON((%s, %s))" % (", ".join(vertices), vertices[0])
oqp.region = ga.WKTSpatialElement(region)
session.add(oqp)
session.add(job)
session.commit()
return job
def test_get_status_from_db(self):
self.job = Job.from_file(helpers.get_data_path(CONFIG_FILE), 'db')
session = get_db_session("reslt", "writer")
session.query(OqJob).update({'status': 'failed'})
session.commit()
self.assertEqual(Job.get_status_from_db(self.job.job_id), 'failed')
session.query(OqJob).update({'status': 'running'})
session.commit()
self.assertEqual(Job.get_status_from_db(self.job.job_id), 'running')
def test_set_status(self):
self.job = Job.from_file(helpers.get_data_path(CONFIG_FILE), 'db')
session = get_db_session("reslt", "writer")
status = 'running'
self.job.set_status(status)
job = session.query(OqJob).filter(OqJob.id == self.job.job_id).one()
self.assertEqual(status, job.status)
def _toCoordList(polygon):
session = get_db_session("reslt", "writer")
pts = []
# postgis -> lon/lat -> config lat/lon, skip the closing point
for c in polygon.coords(session)[0][:-1]:
pts.append("%.2f" % c[1])
pts.append("%.2f" % c[0])
return ", ".join(pts)
def set_status(self, status):
"""
Set the status of the database record belonging to this job.
:param status: one of 'pending', 'running', 'succeeded', 'failed'
:type status: string
"""
session = get_db_session("reslt", "writer")
db_job = self.get_db_job(session)
db_job.status = status
session.add(db_job)
session.commit()
def test_get_db_session(self):
"""
SessionCache.get() is called with the appropriate environment
variables.
"""
for (usr_var, (schema, role), (user, password)) in self.test_data:
os.environ[usr_var] = "usr12"
os.environ[usr_var + "_PWD"] = "pwd12"
session = get_db_session(schema, role)
self.assertTrue(session is self.expected_session)
(user, passwd), _ = self.mock_method.call_args
self.assertEqual("usr12", user)
self.assertEqual("pwd12", passwd)
def test_serialize_small(self):
data = LOSS_MAP_DATA(['a%d' % i for i in range(5)], 20, 4)
self.job = self.setup_classic_job()
session = get_db_session("hzrdr", "writer")
output_path = self.generate_output_path(self.job)
for i in xrange(0, 10):
lmw = LossMapDBWriter(session, output_path + str(i), self.job.id)
# Call the function under test.
lmw.serialize(data)
session.commit()
def test_serialize_small(self):
data = LOSS_CURVE_DATA(20, 4)
self.job = self.setup_classic_job()
session = get_db_session("hzrdr", "writer")
output_path = self.generate_output_path(self.job)
for i in xrange(0, 20):
lcw = LossCurveDBWriter(session, output_path + str(i), self.job.id)
# Call the function under test.
lcw.serialize(data)
session.commit()
def test_serialize_small(self):
data = GMF_DATA(20, 4)
self.job = self.setup_classic_job()
session = get_db_session("hzrdr", "writer")
output_path = self.generate_output_path(self.job)
for i in xrange(0, 10):
gmfw = GMFDBWriter(session, output_path + str(i), self.job.id)
# Call the function under test.
gmfw.serialize(data)
session.commit()
def test_serialize_small(self):
data = HAZARD_CURVE_DATA(['1_1', '1_2', '2_2', '2'], 20, 4)
self.job = self.setup_classic_job()
session = get_db_session("hzrdr", "writer")
output_path = self.generate_output_path(self.job)
for i in xrange(0, 10):
hcw = HazardCurveDBWriter(session, output_path + str(i),
self.job.id)
# Call the function under test.
hcw.serialize(data)
session.commit()
def test_get_db_session_with_no_env(self):
"""
The default user/passwords will be used.
"""
for (usr_var, (schema, role), (user, password)) in self.test_data:
if os.environ.get(usr_var) is not None:
del os.environ[usr_var]
pwd_var = usr_var + "_PWD"
if os.environ.get(pwd_var) is not None:
del os.environ[pwd_var]
session = get_db_session(schema, role)
self.assertTrue(session is self.expected_session)
(actual_user, actual_password), _ = self.mock_method.call_args
self.assertEqual(user, actual_user)
self.assertEqual(password, actual_password)
def setup_upload(self, dbkey=None):
"""Create an upload with associated inputs.
:param integer dbkey: if set use the upload record with given db key.
:returns: a :py:class:`db.alchemy.models.Upload` instance
"""
session = get_db_session("uiapi", "writer")
if dbkey:
upload = session.query(Upload).filter(Upload.id == dbkey).one()
return upload
user = session.query(OqUser).filter(
OqUser.user_name == "openquake").one()
upload = Upload(owner=user, path=tempfile.mkdtemp())
session.add(upload)
session.commit()
return upload
def test_deserialize_small(self):
data = HAZARD_CURVE_DATA(['1_1', '1_2', '2_2', '2'], 20, 4)
self.job = self.setup_classic_job()
session = get_db_session("reslt", "writer")
output_path = self.generate_output_path(self.job)
hcw = HazardCurveDBWriter(session, output_path, self.job.id)
hcw.serialize(data)
session.commit()
# deserialize
hcr = HazardCurveDBReader(session)
for i in xrange(0, 10):
# Call the function under test.
hcr.deserialize(hcw.output.id)
def teardown_upload(self, upload, filesystem_only=True):
"""
Tear down the file system (and potentially db) artefacts for the
given upload.
:param upload: the :py:class:`db.alchemy.models.Upload` instance
in question
:param bool filesystem_only: if set the upload/input database records
will be left intact. This saves time and the test db will be
dropped/recreated prior to the next db test suite run anyway.
"""
# This is like "rm -rf path"
shutil.rmtree(upload.path, ignore_errors=True)
if filesystem_only:
return
session = get_db_session("uiapi", "writer")
session.delete(upload)
session.commit()
def create_loss_curve_writer(job_id, serialize_to, nrml_path, curve_mode):
"""Create a loss curve writer observing the settings in the config file.
If no writer is available for the given curve_mode and settings, returns
None.
:param job_id: the id of the job the curve belongs to.
:type job_id: int
:param serialize_to: where to serialize
:type serialize_to: list of strings. Permitted values: 'db', 'xml'.
:param str nrml_path: the full path of the XML/NRML representation of the
hazard map.
:param str curve_mode: one of 'loss', 'loss_ratio'
:returns: None or an instance of
:py:class:`output.risk.LossCurveXMLWriter`,
:py:class:`output.risk.LossCurveDBWriter`,
:py:class:`output.risk.LossRatioCurveXMLWriter`
"""
assert curve_mode in ('loss', 'loss_ratio')
writers = []
if 'db' in serialize_to:
assert job_id, "No job_id supplied"
job_id = int(job_id)
if curve_mode == 'loss':
writers.append(LossCurveDBWriter(get_db_session("reslt", "writer"),
nrml_path,
job_id))
elif curve_mode == 'loss_ratio':
# We are non interested in storing loss ratios in the db
pass
if 'xml' in serialize_to:
if curve_mode == 'loss':
writer_class = LossCurveXMLWriter
elif curve_mode == 'loss_ratio':
writer_class = LossRatioCurveXMLWriter
writers.append(writer_class(nrml_path))
return writer.compose_writers(writers)
def _get_db_gmf(self, gmf_id):
"""Returns a field for the given GMF"""
session = get_db_session("reslt", "reader")
grid = self.region.grid
field = zeros((grid.rows, grid.columns))
sites = session.query(
sqlfunc.ST_X(models.GMFData.location),
sqlfunc.ST_Y(models.GMFData.location),
models.GMFData.ground_motion) \
.filter(models.GMFData.output_id == gmf_id)
for x, y, value in sites:
site = shapes.Site(x, y)
grid_point = grid.point_at(site)
field[grid_point.row][grid_point.column] = value
return shapes.Field(field)
def _get_db_curve(self, site):
"""Read hazard curve data from the DB"""
session = get_db_session("reslt", "reader")
iml_query = session.query(models.OqParams.imls) \
.join(models.OqJob) \
.filter(models.OqJob.id == self.job_id)
curve_query = session.query(models.HazardCurveData.poes) \
.join(models.HazardCurve) \
.join(models.Output) \
.filter(models.Output.oq_job_id == self.job_id) \
.filter(models.HazardCurve.statistic_type == 'mean') \
.filter(sqlfunc.ST_GeoHash(models.HazardCurveData.location, 12)
== geohash.encode(site.latitude, site.longitude,
precision=12))
hc = curve_query.one()
pms = iml_query.one()
return Curve(zip(pms.imls, hc.poes))
def test_csv_to_db_loader_end_to_end(self):
"""
* Serializes the csv into the database
* Queries the database for the data just inserted
* Verifies the data against the csv
* Deletes the inserted records from the database
"""
engine = db_utils.get_db_session("eqcat", "writer").connection().engine
csv_loader = db_loader.CsvModelLoader(self.csv_path, engine, 'eqcat')
csv_loader.serialize()
db_rows = self._retrieve_db_data(csv_loader.soup)
# rewind the file
csv_loader.csv_fd.seek(0)
self._verify_db_data(csv_loader, db_rows)
csv_loader.soup.commit()
def teardown_output(self, output, teardown_job=True, filesystem_only=True):
"""
Tear down the file system (and potentially db) artefacts for the
given output.
:param output: the :py:class:`db.alchemy.models.Output` instance
in question
:param bool teardown_job: the associated job and its related artefacts
shall be torn down as well.
:param bool filesystem_only: if set the various database records will
be left intact. This saves time and the test db will be
dropped/recreated prior to the next db test suite run anyway.
"""
job = output.oq_job
if not filesystem_only:
session = get_db_session("uiapi", "writer")
session.delete(output)
session.commit()
if teardown_job:
self.teardown_job(job, filesystem_only=filesystem_only)
def _serialize_test_helper(self, test_file, expected_tables):
engine = db_utils.get_db_session("hzrdi", "writer").connection().engine
java.jvm().java.lang.System.setProperty("openquake.nrml.schema",
xml.nrml_schema_file())
src_loader = db_loader.SourceModelLoader(test_file, engine)
results = src_loader.serialize()
# we should get a 3 item list of results
self.assertEquals(3, len(results))
# We expect there to have been 3 inserts.
# The results are a list of dicts with a single key.
# The key is the table name (including table space);
# the value is the id (as an int) of the new record.
# First, check that the results includes the 3 tables we expect:
result_tables = [x.keys()[0] for x in results]
self.assertEqual(expected_tables, result_tables)
# Everything appears to be fine, but let's query the database to make
# sure the expected records are there.
# At this point, we're not going to check every single value; we just
# want to make sure the records made it into the database.
tables = src_loader.meta.tables
# list of tuples of (table name, id)
table_id_pairs = [x.items()[0] for x in results]
for table_name, record_id in table_id_pairs:
table = tables[table_name]
# run a query against the table object to get a ResultProxy
result_proxy = table.select(table.c.id == record_id).execute()
# there should be 1 record here
self.assertEqual(1, result_proxy.rowcount)
# clean up db resources
src_loader.close()
def teardown_job(self, job, filesystem_only=True):
"""
Tear down the file system (and potentially db) artefacts for the
given job.
:param job: the :py:class:`db.alchemy.models.OqJob` instance
in question
:param bool filesystem_only: if set the oq_job/oq_param/upload/input
database records will be left intact. This saves time and the test
db will be dropped/recreated prior to the next db test suite run
anyway.
"""
oqp = job.oq_params
if oqp.upload is not None:
self.teardown_upload(oqp.upload, filesystem_only=filesystem_only)
if filesystem_only:
return
session = get_db_session("uiapi", "writer")
session.delete(job)
session.delete(oqp)
session.commit()
def setup_output(self, job_to_use=None, output_type="hazard_map",
db_backed=True):
"""Create an output object of the given type.
:param job_to_use: if set use the passed
:py:class:`db.alchemy.models.OqJob` instance as opposed to
creating a new one.
:param str output_type: map type, one of "hazard_map", "loss_map"
:param bool db_backed: initialize the property of the newly created
:py:class:`db.alchemy.models.Output` instance with this value.
:returns: a :py:class:`db.alchemy.models.Output` instance
"""
job = job_to_use if job_to_use else self.setup_classic_job()
output = Output(owner=job.owner, oq_job=job, output_type=output_type,
db_backed=db_backed)
output.path = self.generate_output_path(job, output_type)
output.display_name = os.path.basename(output.path)
session = get_db_session("uiapi", "writer")
session.add(output)
session.commit()
return output
def setup_classic_job(self, create_job_path=True, upload_id=None):
"""Create a classic job with associated upload and inputs.
:param integer upload_id: if set use upload record with given db key.
:param bool create_job_path: if set the path for the job will be
created and captured in the job record
:returns: a :py:class:`db.alchemy.models.OqJob` instance
"""
session = get_db_session("uiapi", "writer")
upload = self.setup_upload(upload_id)
oqp = OqParams()
oqp.job_type = "classical"
oqp.upload = upload
oqp.region_grid_spacing = 0.01
oqp.min_magnitude = 5.0
oqp.investigation_time = 50.0
oqp.component = "gmroti50"
oqp.imt = "pga"
oqp.truncation_type = "twosided"
oqp.truncation_level = 3
oqp.reference_vs30_value = 760
oqp.imls = self.IMLS
oqp.poes = [0.01, 0.10]
oqp.realizations = 1
oqp.region = (
"POLYGON((-81.3 37.2, -80.63 38.04, -80.02 37.49, -81.3 37.2))")
session.add(oqp)
job = OqJob(oq_params=oqp, owner=upload.owner, job_type="classical")
session.add(job)
session.commit()
if create_job_path:
job.path = os.path.join(upload.path, str(job.id))
session.add(job)
session.commit()
os.mkdir(job.path)
os.chmod(job.path, 0777)
return job
