def log_percent_complete(job_id, ctype):
"""Log a message when the percentage completed changed for a calculation.
:param int job_id: identifier of the job in question
:param str ctype: calculation type, one of: hazard, risk
"""
if ctype not in ("hazard", "risk"):
LOG.warn("Unknown calculation type: '%s'" % ctype)
return -1
key = "nhzrd_total" if ctype == "hazard" else "nrisk_total"
total = stats.pk_get(job_id, key)
key = "nhzrd_done" if ctype == "hazard" else "nrisk_done"
done = stats.pk_get(job_id, key)
if done <= 0 or total <= 0:
return 0
percent = total / 100.0
# Store percentage complete as well as the last value reported as integers
# in order to avoid reporting the same percentage more than once.
percent_complete = int(done / percent)
# Get the last value reported
lvr = stats.pk_get(job_id, "lvr")
# Only report the percentage completed if it is above the last value shown
if percent_complete > lvr:
log_progress("%s %3d%% complete" % (ctype, percent_complete), 2)
stats.pk_set(job_id, "lvr", percent_complete)
return percent_complete
def log_percent_complete(job_id, ctype):
"""Log a message when the percentage completed changed for a calculation.
:param int job_id: identifier of the job in question
:param str ctype: calculation type, one of: hazard, risk
"""
if ctype not in ("hazard", "risk"):
LOG.warn("Unknown calculation type: '%s'" % ctype)
return -1
key = "nhzrd_total" if ctype == "hazard" else "nrisk_total"
total = stats.pk_get(job_id, key)
key = "nhzrd_done" if ctype == "hazard" else "nrisk_done"
done = stats.pk_get(job_id, key)
if done <= 0 or total <= 0:
return 0
percent = total / 100.0
# Store percentage complete as well as the last value reported as integers
# in order to avoid reporting the same percentage more than once.
percent_complete = int(done / percent)
# Get the last value reported
lvr = stats.pk_get(job_id, "lvr")
# Only report the percentage completed if it is above the last value shown
if percent_complete > lvr:
log_progress("%s %3d%% complete" % (ctype, percent_complete), 2)
stats.pk_set(job_id, "lvr", percent_complete)
return percent_complete
def serialize_hazard_map_at_poe(self, sites, poe, key_template,
hm_attrib_update, nrml_file):
"""
Serialize the hazard map for a set of sites at a given PoE.
Depending on the parameters the serialized map will be a mean or
quantile hazard map.
:param sites: the sites of which the map will be serialized
:type sites: list of :py:class:`openquake.shapes.Site`
:param poe: the PoE at which the map will be serialized
:type poe: :py:class:`float`
:param key_template: a template for constructing the key used to get,
for each site, its map from the KVS
:type key_template: :py:class:`string`
:param hc_attrib_update: a dictionary containing metadata for the set
of maps that will be serialized
:type hc_attrib_update: :py:class:`dict`
:param nrml_file: the output filename
:type nrml_file: :py:class:`string`
"""
nrml_path = self.job_ctxt.build_nrml_path(nrml_file)
LOG.info("Generating NRML hazard map file for PoE %s, "
"%s nodes in hazard map: %s" % (poe, len(sites), nrml_file))
map_writer = hazard_output.create_hazardmap_writer(
self.job_ctxt.job_id, self.job_ctxt.serialize_results_to,
nrml_path)
hm_data = []
for site in sites:
key = key_template % hash(site)
# use hazard map IML values from KVS
hm_attrib = {
'investigationTimeSpan': self.job_ctxt['INVESTIGATION_TIME'],
'IMT': self.job_ctxt['INTENSITY_MEASURE_TYPE'],
'vs30': self.job_ctxt['REFERENCE_VS30_VALUE'],
'IML': kvs.get_value_json_decoded(key),
'poE': poe
}
hm_attrib.update(hm_attrib_update)
hm_data.append((site, hm_attrib))
LOG.debug(">> path: %s" % nrml_path)
# XML serialization context
xsc = namedtuple("XSC", "blocks, cblock, i_total, i_done, i_next")(
stats.pk_get(self.job_ctxt.job_id, "blocks"),
stats.pk_get(self.job_ctxt.job_id, "cblock"), len(sites), 0,
len(hm_data))
hazard_output.SerializerContext().update(xsc)
map_writer.serialize(hm_data)
return nrml_path
def serialize_hazard_map_at_poe(self, sites, poe, key_template,
hm_attrib_update, nrml_file):
"""
Serialize the hazard map for a set of sites at a given PoE.
Depending on the parameters the serialized map will be a mean or
quantile hazard map.
:param sites: the sites of which the map will be serialized
:type sites: list of :py:class:`openquake.shapes.Site`
:param poe: the PoE at which the map will be serialized
:type poe: :py:class:`float`
:param key_template: a template for constructing the key used to get,
for each site, its map from the KVS
:type key_template: :py:class:`string`
:param hc_attrib_update: a dictionary containing metadata for the set
of maps that will be serialized
:type hc_attrib_update: :py:class:`dict`
:param nrml_file: the output filename
:type nrml_file: :py:class:`string`
"""
nrml_path = self.job_ctxt.build_nrml_path(nrml_file)
LOG.info("Generating NRML hazard map file for PoE %s, "
"%s nodes in hazard map: %s" % (poe, len(sites), nrml_file))
map_writer = hazard_output.create_hazardmap_writer(
self.job_ctxt.job_id, self.job_ctxt.serialize_results_to,
nrml_path)
hm_data = []
for site in sites:
key = key_template % hash(site)
# use hazard map IML values from KVS
hm_attrib = {
'investigationTimeSpan':
self.job_ctxt['INVESTIGATION_TIME'],
'IMT': self.job_ctxt['INTENSITY_MEASURE_TYPE'],
'vs30': self.job_ctxt['REFERENCE_VS30_VALUE'],
'IML': kvs.get_value_json_decoded(key),
'poE': poe}
hm_attrib.update(hm_attrib_update)
hm_data.append((site, hm_attrib))
LOG.debug(">> path: %s" % nrml_path)
# XML serialization context
xsc = namedtuple("XSC", "blocks, cblock, i_total, i_done, i_next")(
stats.pk_get(self.job_ctxt.job_id, "blocks"),
stats.pk_get(self.job_ctxt.job_id, "cblock"),
len(sites), 0, len(hm_data))
hazard_output.SerializerContext().update(xsc)
map_writer.serialize(hm_data)
return nrml_path
def test_pk_get_with_existing_incremental(self):
"""The correct value is obtained for an existing predefined key."""
job_id = 92
pkey = "cblock"
key = stats.key_name(job_id, *stats.STATS_KEYS[pkey])
stats.delete_job_counters(job_id)
kvs = self.connect()
kvs.set(key, 929)
stats.pk_get(job_id, pkey)
self.assertEqual("929", kvs.get(key))
def test_pk_get_with_existing_incremental(self):
"""The correct value is obtained for an existing predefined key."""
job_id = 92
pkey = "cblock"
key = stats.key_name(job_id, *stats.STATS_KEYS[pkey])
stats.delete_job_counters(job_id)
kvs = self.connect()
kvs.set(key, 929)
stats.pk_get(job_id, pkey)
self.assertEqual("929", kvs.get(key))
def test_compute_uhs_task_pi_failure_counter(self):
# Same as the previous test, except that we want to make sure task
# failure counters are properly incremented if a task fails.
cmpt_uhs = '%s.%s' % (self.UHS_CORE_MODULE, 'compute_uhs')
with helpers.patch(cmpt_uhs) as compute_mock:
# We want to force a failure to occur in the task:
compute_mock.side_effect = RuntimeError('Mock exception')
get_counter = lambda: stats.pk_get(self.job_id, "nhzrd_failed")
# The counter should start out empty:
self.assertEqual(0, get_counter())
self.assertRaises(RuntimeError, compute_uhs_task,
self.job_id, 0, site=Site(0.0, 0.0))
self.assertEqual(1, get_counter())
# Create two more failures:
self.assertRaises(RuntimeError, compute_uhs_task,
self.job_id, 0, site=Site(0.0, 0.0))
self.assertRaises(RuntimeError, compute_uhs_task,
self.job_id, 0, site=Site(0.0, 0.0))
self.assertEqual(3, get_counter())
def test_compute_uhs_task_pi_failure_counter(self):
# Same as the previous test, except that we want to make sure task
# failure counters are properly incremented if a task fails.
cmpt_uhs = '%s.%s' % (self.UHS_CORE_MODULE, 'compute_uhs')
with helpers.patch(cmpt_uhs) as compute_mock:
# We want to force a failure to occur in the task:
compute_mock.side_effect = RuntimeError('Mock exception')
get_counter = lambda: stats.pk_get(self.job_id, "nhzrd_failed")
# The counter should start out empty:
self.assertEqual(0, get_counter())
self.assertRaises(RuntimeError,
compute_uhs_task,
self.job_id,
0,
site=Site(0.0, 0.0))
self.assertEqual(1, get_counter())
# Create two more failures:
self.assertRaises(RuntimeError,
compute_uhs_task,
self.job_id,
0,
site=Site(0.0, 0.0))
self.assertRaises(RuntimeError,
compute_uhs_task,
self.job_id,
0,
site=Site(0.0, 0.0))
self.assertEqual(3, get_counter())
def test_compute_uhs_task_pi(self):
# Test that progress indicators are working properly for
# `compute_uhs_task`.
# Mock out the two 'heavy' functions called by this task;
# we don't need to do these and we don't want to waste the cycles.
cmpt_uhs = '%s.%s' % (self.UHS_CORE_MODULE, 'compute_uhs')
write_uhs_data = '%s.%s' % (self.UHS_CORE_MODULE,
'write_uhs_spectrum_data')
with helpers.patch(cmpt_uhs):
with helpers.patch(write_uhs_data):
get_counter = lambda: stats.pk_get(self.job_id, "nhzrd_done")
# First, check that the completion counter is zero
self.assertEqual(0, get_counter())
realization = 0
site = Site(0.0, 0.0)
# execute the task as a plain old function
compute_uhs_task(self.job_id, realization, site=site)
self.assertEqual(1, get_counter())
compute_uhs_task(self.job_id, realization, site=site)
self.assertEqual(2, get_counter())
def test_compute_uhs_task_pi(self):
# Test that progress indicators are working properly for
# `compute_uhs_task`.
# Mock out the two 'heavy' functions called by this task;
# we don't need to do these and we don't want to waste the cycles.
cmpt_uhs = '%s.%s' % (self.UHS_CORE_MODULE, 'compute_uhs')
write_uhs_data = '%s.%s' % (self.UHS_CORE_MODULE,
'write_uhs_spectrum_data')
with helpers.patch(cmpt_uhs):
with helpers.patch(write_uhs_data):
get_counter = lambda: stats.pk_get(self.job_id, "nhzrd_done")
# First, check that the completion counter is zero
self.assertEqual(0, get_counter())
realization = 0
site = Site(0.0, 0.0)
# execute the task as a plain old function
compute_uhs_task(self.job_id, realization, site=site)
self.assertEqual(1, get_counter())
compute_uhs_task(self.job_id, realization, site=site)
self.assertEqual(2, get_counter())
def completed_task_count(job_id):
"""Given the ID of a currently running calculation, query the stats
counters in Redis to get the number of completed
:function:`compute_uhs_task` task executions.
Successful and failed executions are included in the count.
:param int job_id:
ID of the current job.
:returns:
Number of completed :function:`compute_uhs_task` task executions so
far.
"""
success_count = stats.pk_get(job_id, "nhzrd_done")
fail_count = stats.pk_get(job_id, "nhzrd_failed")
return (success_count or 0) + (fail_count or 0)
def test_with_job_id_and_data_in_args(self):
# The job_id is passed via args
result = _RESULTS.next()
job_id = _JOB_IDS.next()
ctype = "h"
@stats.count_progress(ctype)
def no_exception(job_id, items):
return result
previous_value = stats.pk_get(job_id, _COUNTER[ctype])
# Call the wrapped function.
self.assertEqual(result, no_exception(job_id, range(result)))
value = stats.pk_get(job_id, _COUNTER[ctype])
self.assertEqual(result, (value - previous_value))
def test_with_job_id_and_data_in_args(self):
# The job_id is passed via args
result = _RESULTS.next()
job_id = _JOB_IDS.next()
ctype = "h"
@stats.count_progress(ctype)
def no_exception(job_id, items):
return result
previous_value = stats.pk_get(job_id, _COUNTER[ctype])
# Call the wrapped function.
self.assertEqual(result, no_exception(job_id, range(result)))
value = stats.pk_get(job_id, _COUNTER[ctype])
self.assertEqual(result, (value - previous_value))
def completed_task_count(job_id):
"""Given the ID of a currently running calculation, query the stats
counters in Redis to get the number of completed
:function:`compute_uhs_task` task executions.
Successful and failed executions are included in the count.
:param int job_id:
ID of the current job.
:returns:
Number of completed :function:`compute_uhs_task` task executions so
far.
"""
success_count = stats.pk_get(job_id, "nhzrd_done")
fail_count = stats.pk_get(job_id, "nhzrd_failed")
return (success_count or 0) + (fail_count or 0)
def test_success_stats(self):
"""
The success counter is incremented when the wrapped function
terminates without raising an exception.
"""
area = "h"
@stats.count_progress(area)
def no_exception(job_id, items):
return 999
previous_value = stats.pk_get(11, "nhzrd_done")
# Call the wrapped function.
self.assertEqual(999, no_exception(11, range(5)))
value = stats.pk_get(11, "nhzrd_done")
self.assertEqual(5, (value - previous_value))
def test_failure_stats(self):
"""
The failure counter is incremented when the wrapped function
terminates raises an exception.
"""
area = "r"
@stats.count_progress(area)
def raise_exception(job_id, items):
raise NotImplementedError
previous_value = stats.pk_get(22, "nrisk_failed")
# Call the wrapped function.
self.assertRaises(NotImplementedError, raise_exception, 22, range(6))
value = stats.pk_get(22, "nrisk_failed")
self.assertEqual(6, (value - previous_value))
def test_initialize_pr_data(self):
# The total/done counters for progress reporting are initialized
# correctly.
self.calc.initialize_sources()
self.calc.initialize_realizations(
rlz_callbacks=[self.calc.initialize_hazard_curve_progress])
ltr1, ltr2 = models.LtRealization.objects.filter(
hazard_calculation=self.job.hazard_calculation.id).order_by("id")
ltr1.completed_sources = 11
ltr1.save()
self.calc.initialize_pr_data()
total = stats.pk_get(self.calc.job.id, "nhzrd_total")
self.assertEqual(ltr1.total_sources + ltr2.total_sources, total)
done = stats.pk_get(self.calc.job.id, "nhzrd_done")
self.assertEqual(ltr1.completed_sources + ltr2.completed_sources, done)
def test_with_data_arg_in_kwargs_overriding_2nd_positional_param(self):
# A data parameter passed via kwargs will override the second
# positional parameter.
result = _RESULTS.next()
job_id = _JOB_IDS.next()
ctype = "h"
@stats.count_progress(ctype, data_arg="the_data_arg")
def no_exception(job_id, items, the_data_arg):
return result
previous_value = stats.pk_get(job_id, _COUNTER[ctype])
# Call the wrapped function.
self.assertEqual(result, no_exception(job_id, range(result - 1),
the_data_arg=range(result)))
value = stats.pk_get(job_id, _COUNTER[ctype])
self.assertEqual(result, (value - previous_value))
def test_pk_get_with_existing_debug_and_debug_stats_off(self):
"""`None` is returned when debug stats are off."""
job_id = 95
pkey = "hcls_xmlcurvewrites"
stats.delete_job_counters(job_id)
with helpers.patch("openquake.utils.stats.debug_stats_enabled") as dse:
dse.return_value = False
key = stats._KEY_TEMPLATE % ((job_id,) + stats.STATS_KEYS[pkey])
kvs = self.connect()
kvs.set(key, 959)
self.assertIs(None, stats.pk_get(job_id, pkey))
def test_pk_get_with_existing_debug_and_debug_stats_enabled(self):
"""The value is obtained correctly for an existing debug counter."""
job_id = 94
pkey = "hcls_xmlcurvewrites"
stats.delete_job_counters(job_id)
with helpers.patch("openquake.utils.stats.debug_stats_enabled") as dse:
dse.return_value = True
key = stats.key_name(job_id, *stats.STATS_KEYS[pkey])
kvs = self.connect()
kvs.set(key, 949)
self.assertEqual(949, stats.pk_get(job_id, pkey))
def test_pk_get_with_existing_debug_and_debug_stats_enabled(self):
"""The value is obtained correctly for an existing debug counter."""
job_id = 94
pkey = "hcls_xmlcurvewrites"
stats.delete_job_counters(job_id)
with helpers.patch("openquake.utils.stats.debug_stats_enabled") as dse:
dse.return_value = True
key = stats.key_name(job_id, *stats.STATS_KEYS[pkey])
kvs = self.connect()
kvs.set(key, 949)
self.assertEqual(949, stats.pk_get(job_id, pkey))
def test_pk_get_with_existing_debug_and_debug_stats_off(self):
"""`None` is returned when debug stats are off."""
job_id = 95
pkey = "hcls_xmlcurvewrites"
stats.delete_job_counters(job_id)
with helpers.patch("openquake.utils.stats.debug_stats_enabled") as dse:
dse.return_value = False
key = stats._KEY_TEMPLATE % ((job_id, ) + stats.STATS_KEYS[pkey])
kvs = self.connect()
kvs.set(key, 959)
self.assertIs(None, stats.pk_get(job_id, pkey))
def test_initialize_pr_data_with_ses(self):
hc = self.job.hazard_calculation
# Initialize sources as a setup for the test:
self.calc.initialize_sources()
self.calc.initialize_realizations(
rlz_callbacks=[self.calc.initialize_ses_db_records])
ltr1, ltr2 = models.LtRealization.objects.filter(
hazard_calculation=hc).order_by("id")
ltr1.completed_sources = 12
ltr1.save()
self.calc.initialize_pr_data()
total = stats.pk_get(self.calc.job.id, "nhzrd_total")
self.assertEqual(ltr1.total_sources + ltr2.total_sources, total)
done = stats.pk_get(self.calc.job.id, "nhzrd_done")
self.assertEqual(ltr1.completed_sources + ltr2.completed_sources, done)
def test_with_data_arg_in_kwargs_overriding_2nd_positional_param(self):
# A data parameter passed via kwargs will override the second
# positional parameter.
result = _RESULTS.next()
job_id = _JOB_IDS.next()
ctype = "h"
@stats.count_progress(ctype, data_arg="the_data_arg")
def no_exception(job_id, items, the_data_arg):
return result
previous_value = stats.pk_get(job_id, _COUNTER[ctype])
# Call the wrapped function.
self.assertEqual(
result,
no_exception(job_id, range(result - 1),
the_data_arg=range(result)))
value = stats.pk_get(job_id, _COUNTER[ctype])
self.assertEqual(result, (value - previous_value))
def serialize_hazard_curve(self, nrml_file, key_template, hc_attrib_update,
sites):
"""
Serialize the hazard curves of a set of sites.
Depending on the parameters the serialized curve will be a plain, mean
or quantile hazard curve.
:param nrml_file: the output filename
:type nrml_file: :py:class:`string`
:param key_template: a template for constructing the key to get, for
each site, its curve from the KVS
:type key_template: :py:class:`string`
:param hc_attrib_update: a dictionary containing metadata for the set
of curves that will be serialized
:type hc_attrib_update: :py:class:`dict`
:param sites: the sites of which the curve will be serialized
:type sites: list of :py:class:`openquake.shapes.Site`
"""
def pause_generator(value):
"""
Returns the initial value when called for the first time and
the double value upon each subsequent invocation.
N.B.: the maximum value returned will never exceed 90 (seconds).
"""
yield value
while True:
if value < 45:
value *= 2
yield value
# XML serialization context
xsc = namedtuple("XSC", "blocks, cblock, i_total, i_done, i_next")(
stats.pk_get(self.job_ctxt.job_id, "blocks"),
stats.pk_get(self.job_ctxt.job_id, "cblock"),
len(sites), 0, 0)
nrml_path = self.job_ctxt.build_nrml_path(nrml_file)
curve_writer = hazard_output.create_hazardcurve_writer(
self.job_ctxt.job_id, self.job_ctxt.serialize_results_to,
nrml_path)
sites = set(sites)
accounted_for = set()
min_pause = 0.1
pgen = pause_generator(min_pause)
pause = pgen.next()
while accounted_for != sites:
failures = stats.failure_counters(self.job_ctxt.job_id, "h")
if failures:
raise RuntimeError("hazard failures (%s), aborting" % failures)
hc_data = []
# Sleep a little before checking the availability of additional
# hazard curve results.
time.sleep(pause)
results_found = 0
for site in sites:
if site in accounted_for:
continue
value = kvs.get_value_json_decoded(key_template % hash(site))
if value is None:
# No value yet, proceed to next site.
continue
# Use hazard curve ordinate values (PoE) from KVS and abscissae
# from the IML list in config.
hc_attrib = {
'investigationTimeSpan':
self.job_ctxt['INVESTIGATION_TIME'],
'IMLValues': self.job_ctxt.imls,
'IMT': self.job_ctxt['INTENSITY_MEASURE_TYPE'],
'PoEValues': value}
hc_attrib.update(hc_attrib_update)
hc_data.append((site, hc_attrib))
accounted_for.add(site)
results_found += 1
if not results_found:
# No results found, increase the sleep pause.
pause = pgen.next()
else:
hazard_output.SerializerContext().update(
xsc._replace(i_next=len(hc_data)))
curve_writer.serialize(hc_data)
xsc = xsc._replace(i_done=xsc.i_done + len(hc_data))
pause *= 0.8
pause = min_pause if pause < min_pause else pause
return nrml_path
def serialize_hazard_curve(self, nrml_file, key_template, hc_attrib_update,
sites):
"""
Serialize the hazard curves of a set of sites.
Depending on the parameters the serialized curve will be a plain, mean
or quantile hazard curve.
:param nrml_file: the output filename
:type nrml_file: :py:class:`string`
:param key_template: a template for constructing the key to get, for
each site, its curve from the KVS
:type key_template: :py:class:`string`
:param hc_attrib_update: a dictionary containing metadata for the set
of curves that will be serialized
:type hc_attrib_update: :py:class:`dict`
:param sites: the sites of which the curve will be serialized
:type sites: list of :py:class:`openquake.shapes.Site`
"""
def pause_generator(value):
"""
Returns the initial value when called for the first time and
the double value upon each subsequent invocation.
N.B.: the maximum value returned will never exceed 90 (seconds).
"""
yield value
while True:
if value < 45:
value *= 2
yield value
# XML serialization context
xsc = namedtuple("XSC", "blocks, cblock, i_total, i_done, i_next")(
stats.pk_get(self.job_ctxt.job_id, "blocks"),
stats.pk_get(self.job_ctxt.job_id, "cblock"), len(sites), 0, 0)
nrml_path = self.job_ctxt.build_nrml_path(nrml_file)
curve_writer = hazard_output.create_hazardcurve_writer(
self.job_ctxt.job_id, self.job_ctxt.serialize_results_to,
nrml_path)
sites = set(sites)
accounted_for = set()
min_pause = 0.1
pgen = pause_generator(min_pause)
pause = pgen.next()
while accounted_for != sites:
failures = stats.failure_counters(self.job_ctxt.job_id, "h")
if failures:
raise RuntimeError("hazard failures (%s), aborting" % failures)
hc_data = []
# Sleep a little before checking the availability of additional
# hazard curve results.
time.sleep(pause)
results_found = 0
for site in sites:
if site in accounted_for:
continue
value = kvs.get_value_json_decoded(key_template % hash(site))
if value is None:
# No value yet, proceed to next site.
continue
# Use hazard curve ordinate values (PoE) from KVS and abscissae
# from the IML list in config.
hc_attrib = {
'investigationTimeSpan':
self.job_ctxt['INVESTIGATION_TIME'],
'IMLValues': self.job_ctxt.imls,
'IMT': self.job_ctxt['INTENSITY_MEASURE_TYPE'],
'PoEValues': value
}
hc_attrib.update(hc_attrib_update)
hc_data.append((site, hc_attrib))
accounted_for.add(site)
results_found += 1
if not results_found:
# No results found, increase the sleep pause.
pause = pgen.next()
else:
hazard_output.SerializerContext().update(
xsc._replace(i_next=len(hc_data)))
curve_writer.serialize(hc_data)
xsc = xsc._replace(i_done=xsc.i_done + len(hc_data))
pause *= 0.8
pause = min_pause if pause < min_pause else pause
return nrml_path
