def test_delete_job_counters_deletes_counters_for_job(self):
"""
The progress indication counters for a given job are deleted.
"""
kvs = self.connect()
args = [(55, "h", "a/b/c"), (55, "h", "d/e/f")]
for data in args:
stats.incr_counter(*data)
stats.delete_job_counters(55)
self.assertEqual(0, len(kvs.keys("oqs:55:*")))
def test_delete_job_counters_deletes_counters_for_job(self):
"""
The progress indication counters for a given job are deleted.
"""
kvs = self.connect()
args = [(55, "h", "a/b/c"), (55, "h", "d/e/f")]
for data in args:
stats.incr_counter(*data)
stats.delete_job_counters(55)
self.assertEqual(0, len(kvs.keys("oqs:55:*")))
def test_incr_counter(self):
"""
The counter is incremented for the given key
"""
args = (44, "h", "d/x/z", "i")
kvs = self.connect()
key = stats.key_name(*args)
previous_value = kvs.get(key)
previous_value = int(previous_value) if previous_value else 0
stats.incr_counter(*args[:-1])
value = int(kvs.get(key))
self.assertEqual(1, (value - previous_value))
def test_incr_counter(self):
"""
The counter is incremented for the given key
"""
args = (44, "h", "d/x/z", "i")
kvs = self.connect()
key = stats.key_name(*args)
previous_value = kvs.get(key)
previous_value = int(previous_value) if previous_value else 0
stats.incr_counter(*args[:-1])
value = int(kvs.get(key))
self.assertEqual(1, (value - previous_value))
def test_delete_job_counters_resets_counters(self):
"""
The progress indication counters for a given job are reset.
"""
kvs = self.connect()
args = [(66, "h", "g/h/i", "i"), (66, "h", "j/k/l", "i")]
for data in args:
stats.incr_counter(*data[:-1])
stats.delete_job_counters(66)
# The counters have been reset, after incrementing we expect them all
# to have a value of "1".
for data in args:
stats.incr_counter(*data[:-1])
self.assertEqual("1", kvs.get(stats.key_name(*data)))
def test_delete_job_counters_resets_counters(self):
"""
The progress indication counters for a given job are reset.
"""
kvs = self.connect()
args = [(66, "h", "g/h/i", "i"), (66, "h", "j/k/l", "i")]
for data in args:
stats.incr_counter(*data[:-1])
stats.delete_job_counters(66)
# The counters have been reset, after incrementing we expect them all
# to have a value of "1".
for data in args:
stats.incr_counter(*data[:-1])
self.assertEqual("1", kvs.get(stats.key_name(*data)))
def test_failure_counters_with_no_area(self):
# Failure counters are returned for all computation areas if the
# 'area' parameter is omitted.
stats.delete_job_counters(123)
fcname = itertools.cycle(string.ascii_lowercase)
for cidx, carea in enumerate(["g", "h", "r"]):
stats.incr_counter(123, carea, "%s:failed" % fcname.next())
if not (cidx % 2):
stats.incr_counter(123, carea, "%s:failed" % fcname.next())
self.assertEqual(
[('oqs/123/g/a:failed/i', 1), ('oqs/123/g/b:failed/i', 1),
('oqs/123/h/c:failed/i', 1), ('oqs/123/r/d:failed/i', 1),
('oqs/123/r/e:failed/i', 1)],
sorted(stats.failure_counters(123)))
def test_failure_counters_with_no_area(self):
# Failure counters are returned for all computation areas if the
# 'area' parameter is omitted.
stats.delete_job_counters(123)
fcname = itertools.cycle(string.ascii_lowercase)
for cidx, carea in enumerate(["g", "h", "r"]):
stats.incr_counter(123, carea, "%s:failed" % fcname.next())
if not (cidx % 2):
stats.incr_counter(123, carea, "%s:failed" % fcname.next())
self.assertEqual([('oqs/123/g/a:failed/i', 1),
('oqs/123/g/b:failed/i', 1),
('oqs/123/h/c:failed/i', 1),
('oqs/123/r/d:failed/i', 1),
('oqs/123/r/e:failed/i', 1)],
sorted(stats.failure_counters(123)))
def test_actions_after_job_process_failures(self):
# the job process is running but has some failure counters above zero
# shorten the delay to checking failure counters
supervisor.SupervisorLogMessageConsumer.FCC_DELAY = 2
self.is_pid_running.return_value = True
self.get_job_status.return_value = 'running'
stats.delete_job_counters(self.job.id)
stats.incr_counter(self.job.id, "h", "a:failed")
stats.incr_counter(self.job.id, "r", "b:failed")
stats.incr_counter(self.job.id, "r", "b:failed")
supervisor.supervise(1, self.job.id, timeout=0.1)
# the job process is terminated
self.assertEqual(1, self.terminate_job.call_count)
self.assertEqual(((1,), {}), self.terminate_job.call_args)
# stop time is recorded
self.assertEqual(1, self.record_job_stop_time.call_count)
self.assertEqual(
((self.job.id,), {}),
self.record_job_stop_time.call_args)
# the cleanup is triggered
self.assertEqual(1, self.cleanup_after_job.call_count)
self.assertEqual(
((self.job.id,), {}),
self.cleanup_after_job.call_args)
def test_actions_after_job_process_failures(self):
# the job process is running but has some failure counters above zero
# shorten the delay to checking failure counters
supervisor.SupervisorLogMessageConsumer.FCC_DELAY = 2
self.is_pid_running.return_value = True
self.get_job_status.return_value = 'running'
stats.delete_job_counters(self.job.id)
stats.incr_counter(self.job.id, "h", "a:failed")
stats.incr_counter(self.job.id, "r", "b:failed")
stats.incr_counter(self.job.id, "r", "b:failed")
supervisor.supervise(1, self.job.id, timeout=0.1)
# the job process is terminated
self.assertEqual(1, self.terminate_job.call_count)
self.assertEqual(((1, ), {}), self.terminate_job.call_args)
# stop time is recorded
self.assertEqual(1, self.record_job_stop_time.call_count)
self.assertEqual(((self.job.id, ), {}),
self.record_job_stop_time.call_args)
# the cleanup is triggered
self.assertEqual(1, self.cleanup_after_job.call_count)
self.assertEqual(((self.job.id, terminate), {}),
self.cleanup_after_job.call_args)
