def from_kvs(job_id):
"""Return the job in the underlying kvs system with the given id."""
params = kvs.get_value_json_decoded(kvs.tokens.generate_job_key(job_id))
job_profile = profile4job(job_id)
job = OqJob.objects.get(id=job_id)
job = JobContext(params, job_id, oq_job_profile=job_profile, oq_job=job, log_level=params["debug"])
return job
def get_progress_timing_data(job):
"""Get length of time since the last task completed and the timeout.
:param job: The :class:`openquake.db.models.OqJob` instance to use
:returns: number of seconds since the last task completed (or the
execute phase began) and the value of the `no_progress_timeout`
parameter as a 2-tuple
"""
def epoch(dto):
"""Convert a datetime object to seconds since epoch"""
return int(dto.strftime("%s"))
tstamp = epoch(datetime.utcnow())
# the "last value recorded time stamp" will be zero if not set in the kvs
lvr_ts = pk_get(job.id, "lvr_ts")
# when did the most recent "executing" job phase for this calculation
# start?
jpss = models.JobPhaseStats.objects.filter(oq_job=job,
job_status="executing")
[jps] = jpss.order_by("-start_time")[:1]
jps_ts = epoch(jps.start_time)
# take the more recent of the two time stamps
lvr_ts = jps_ts if jps_ts > lvr_ts else lvr_ts
return (tstamp - lvr_ts, models.profile4job(job.id).no_progress_timeout)
def get_progress_timing_data(job):
"""Get length of time since the last task completed and the timeout.
:param job: The :class:`openquake.db.models.OqJob` instance to use
:returns: number of seconds since the last task completed (or the
execute phase began) and the value of the `no_progress_timeout`
parameter as a 2-tuple
"""
def epoch(dto):
"""Convert a datetime object to seconds since epoch"""
return int(dto.strftime("%s"))
tstamp = epoch(datetime.utcnow())
# the "last value recorded time stamp" will be zero if not set in the kvs
lvr_ts = pk_get(job.id, "lvr_ts")
# when did the most recent "executing" job phase for this calculation
# start?
jpss = models.JobPhaseStats.objects.filter(oq_job=job,
job_status="executing")
[jps] = jpss.order_by("-start_time")[:1]
jps_ts = epoch(jps.start_time)
# take the more recent of the two time stamps
lvr_ts = jps_ts if jps_ts > lvr_ts else lvr_ts
return (tstamp - lvr_ts, models.profile4job(job.id).no_progress_timeout)
def from_kvs(job_id):
"""Return the job in the underlying kvs system with the given id."""
params = kvs.get_value_json_decoded(
kvs.tokens.generate_job_key(job_id))
job_profile = profile4job(job_id)
job = OqJob.objects.get(id=job_id)
job = JobContext(params,
job_id,
oq_job_profile=job_profile,
oq_job=job,
log_level=params['debug'])
return job
def test_grid_assets_iterator(self):
def row_col(item):
return item[0].row, item[0].column
jp = models.profile4job(self.job_ctxt.job_id)
jp.region_grid_spacing = 0.01
jp.save()
calculator = general.BaseRiskCalculator(self.job_ctxt)
expected = sorted(self.grid_assets, key=row_col)
actual = sorted(calculator.grid_assets_iterator(self.grid),
key=row_col)
self.assertEqual(expected, actual)
def test_grid_assets_iterator(self):
def row_col(item):
return item[0].row, item[0].column
jp = models.profile4job(self.job_ctxt.job_id)
jp.region_grid_spacing = 0.01
jp.save()
calculator = general.BaseRiskCalculator(self.job_ctxt)
expected = sorted(self.grid_assets, key=row_col)
actual = sorted(calculator.grid_assets_iterator(self.grid),
key=row_col)
self.assertEqual(expected, actual)
def setUp(self):
self.params = {}
self.params["OUTPUT_DIR"] = helpers.OUTPUT_DIR
self.params["BASE_PATH"] = "."
self.params["INVESTIGATION_TIME"] = 50.0
self.job_ctxt = helpers.create_job(
self.params, base_path=".", job_id=self.job.id,
oq_job=self.job, oq_job_profile=models.profile4job(self.job.id))
self.job_id = self.job_ctxt.job_id
self.job_ctxt.to_kvs()
self.vulnerability_function2 = vulnerability_function.VulnerabilityFunction([
0.0, 0.04, 0.08, 0.12, 0.16, 0.2, 0.24, 0.28, 0.32, 0.36,
0.4, 0.44, 0.48, 0.53, 0.57, 0.61, 0.65, 0.69, 0.73, 0.77, 0.81,
0.85, 0.89, 0.93, 0.97, 1.01, 1.05, 1.09, 1.13, 1.17, 1.21, 1.25,
1.29, 1.33, 1.37, 1.41, 1.45, 1.49, 1.54, 1.58, 1.62, 1.66, 1.7,
1.74, 1.78, 1.82, 1.86, 1.9, 1.94, 1.98, 2.02, 2.06, 2.1, 2.14,
2.18, 2.22, 2.26, 2.3, 2.34, 2.38, 2.42, 2.46, 2.51, 2.55, 2.59,
2.63, 2.67, 2.71, 2.75, 2.79, 2.83, 2.87, 2.91, 2.95, 2.99, 3.03,
3.07, 3.11, 3.15, 3.19, 3.23, 3.27, 3.31, 3.35, 3.39, 3.43, 3.47,
3.52, 3.56, 3.6, 3.64, 3.68, 3.72, 3.76, 3.8, 3.84, 3.88, 3.92,
3.96, 4.0], [0.0, 0.0, 0.0, 0.01, 0.04, 0.07, 0.11, 0.15, 0.2,
0.25, 0.3, 0.35, 0.39, 0.43, 0.47, 0.51, 0.55, 0.58, 0.61, 0.64,
0.67, 0.69, 0.71, 0.73, 0.75, 0.77, 0.79, 0.8, 0.81, 0.83, 0.84,
0.85, 0.86, 0.87, 0.88, 0.89, 0.89, 0.9, 0.91, 0.91, 0.92, 0.92,
0.93, 0.93, 0.94, 0.94, 0.94, 0.95, 0.95, 0.95, 0.95, 0.96, 0.96,
0.96, 0.96, 0.97, 0.97, 0.97, 0.97, 0.97, 0.97, 0.98, 0.98, 0.98,
0.98, 0.98, 0.98, 0.98, 0.98, 0.98, 0.98, 0.99, 0.99, 0.99, 0.99,
0.99, 0.99, 0.99, 0.99, 0.99, 0.99, 0.99, 0.99, 0.99, 0.99, 0.99,
0.99, 0.99, 0.99, 0.99, 0.99, 0.99, 0.99, 0.99, 0.99, 1.0, 1.0,
1.0, 1.0, 1.0], [0.0] * 100, "LN")
# deleting keys in kvs
kvs.get_client().flushall()
kvs.set_value_json_encoded(
kvs.tokens.vuln_key(self.job_id),
{"ID": self.vulnerability_function2.to_json()})
kvs.set_value_json_encoded(
kvs.tokens.vuln_key(self.job_id, retrofitted=True),
{"ID": self.vulnerability_function2.to_json()})
def assets_for_cell(cls, job_id, center):
"""Return exposure assets for the given job and risk cell mid-point.
:param int job_id: the database key of the job in question
:param center: a :py:class:`openquake.shapes.Site` instance
with the location of the risk cell center
:returns: a potentially empty list of
:py:class:`openquake.db.models.ExposureData` instances
"""
jp = models.profile4job(job_id)
assert jp.region_grid_spacing is not None, "Grid spacing not known."
cls._load_exposure_model(job_id)
if not cls._em_inputs:
return []
risk_cell = cls._cell_to_polygon(center, jp.region_grid_spacing)
result = models.ExposureData.objects.filter(
exposure_model__input__in=cls._em_inputs,
site__contained=risk_cell)
return list(result)
def assets_for_cell(cls, job_id, center):
"""Return exposure assets for the given job and risk cell mid-point.
:param int job_id: the database key of the job in question
:param center: a :py:class:`openquake.shapes.Site` instance
with the location of the risk cell center
:returns: a potentially empty list of
:py:class:`openquake.db.models.ExposureData` instances
"""
jp = models.profile4job(job_id)
assert jp.region_grid_spacing is not None, "Grid spacing not known."
cls._load_exposure_model(job_id)
if not cls._em_inputs:
return []
risk_cell = cls._cell_to_polygon(center, jp.region_grid_spacing)
result = models.ExposureData.objects.filter(
exposure_model__input__in=cls._em_inputs,
site__contained=risk_cell)
return list(result)
def teardown_job(cls, job, filesystem_only=True):
"""
Tear down the file system (and potentially db) artefacts for the
given job.
:param job: a :py:class:`db.models.OqJob` instance
:param bool filesystem_only: if set the oq_job/oq_param/upload/
input database records will be left intact. This saves time and the
test db will be dropped/recreated prior to the next db test suite
run anyway.
"""
DbTestCase.teardown_inputs(models.inputs4job(job.id),
filesystem_only=filesystem_only)
if filesystem_only:
return
job.delete()
try:
oqjp = models.profile4job(job.id)
oqjp.delete()
except ValueError:
# no job profile for this job
pass
def teardown_job(job, filesystem_only=True):
"""
Tear down the file system (and potentially db) artefacts for the
given job.
:param job: a :py:class:`db.models.OqJob` instance
:param bool filesystem_only: if set the oq_job/oq_param/upload/
input database records will be left intact. This saves time and the
test db will be dropped/recreated prior to the next db test suite
run anyway.
"""
DbTestCase.teardown_inputs(models.inputs4job(job.id),
filesystem_only=filesystem_only)
if filesystem_only:
return
job.delete()
try:
oqjp = models.profile4job(job.id)
oqjp.delete()
except ValueError:
# no job profile for this job
pass
def _reload_params(self):
return models.profile4job(self.job.id)
def test_profile4job_with_existing(self):
# The correct job profile is found.
job = self.setup_classic_job()
self.assertIsNot(None, models.profile4job(job.id))
def _reload_params(self):
return models.profile4job(self.job.id)
def test_profile4job_with_existing(self):
# The correct job profile is found.
job = self.setup_classic_job()
self.assertIsNot(None, models.profile4job(job.id))
