def test_pk_inc_with_non_existent_predef_key(self):
"""`KeyError` is raised for keys that do not exist in `STATS_KEYS`."""
job_id = 83
pkey = "That is the question.."
stats.delete_job_counters(job_id)
self.assertRaises(KeyError, stats.pk_inc, job_id, pkey)
def test_pk_inc_with_non_existent_predef_key(self):
"""`KeyError` is raised for keys that do not exist in `STATS_KEYS`."""
job_id = 83
pkey = "That is the question.."
stats.delete_job_counters(job_id)
self.assertRaises(KeyError, stats.pk_inc, job_id, pkey)
def test_pk_set_with_non_existent_predef_key(self):
"""`KeyError` is raised for keys that do not exist in `STATS_KEYS`."""
job_id = 73
pkey = "To be or not to be!?"
stats.delete_job_counters(job_id)
self.assertRaises(KeyError, stats.pk_set, job_id, pkey, 737)
def test_actions_after_job_process_failures(self):
# the job process is running but has some failure counters above zero
# shorten the delay to checking failure counters
supervisor.SupervisorLogMessageConsumer.FCC_DELAY = 2
self.is_pid_running.return_value = True
self.get_job_status.return_value = 'running'
stats.delete_job_counters(self.job.id)
stats.incr_counter(self.job.id, "h", "a:failed")
stats.incr_counter(self.job.id, "r", "b:failed")
stats.incr_counter(self.job.id, "r", "b:failed")
supervisor.supervise(1, self.job.id, timeout=0.1)
# the job process is terminated
self.assertEqual(1, self.terminate_job.call_count)
self.assertEqual(((1, ), {}), self.terminate_job.call_args)
# stop time is recorded
self.assertEqual(1, self.record_job_stop_time.call_count)
self.assertEqual(((self.job.id, ), {}),
self.record_job_stop_time.call_args)
# the cleanup is triggered
self.assertEqual(1, self.cleanup_after_job.call_count)
self.assertEqual(((self.job.id, terminate), {}),
self.cleanup_after_job.call_args)
def test_pk_set_with_non_existent_predef_key(self):
"""`KeyError` is raised for keys that do not exist in `STATS_KEYS`."""
job_id = 73
pkey = "To be or not to be!?"
stats.delete_job_counters(job_id)
self.assertRaises(KeyError, stats.pk_set, job_id, pkey, 737)
def test_actions_after_job_process_failures(self):
# the job process is running but has some failure counters above zero
# shorten the delay to checking failure counters
supervisor.SupervisorLogMessageConsumer.FCC_DELAY = 2
self.is_pid_running.return_value = True
self.get_job_status.return_value = 'running'
stats.delete_job_counters(self.job.id)
stats.incr_counter(self.job.id, "h", "a:failed")
stats.incr_counter(self.job.id, "r", "b:failed")
stats.incr_counter(self.job.id, "r", "b:failed")
supervisor.supervise(1, self.job.id, timeout=0.1)
# the job process is terminated
self.assertEqual(1, self.terminate_job.call_count)
self.assertEqual(((1,), {}), self.terminate_job.call_args)
# stop time is recorded
self.assertEqual(1, self.record_job_stop_time.call_count)
self.assertEqual(
((self.job.id,), {}),
self.record_job_stop_time.call_args)
# the cleanup is triggered
self.assertEqual(1, self.cleanup_after_job.call_count)
self.assertEqual(
((self.job.id,), {}),
self.cleanup_after_job.call_args)
def test_pk_inc_with_existing_incremental(self):
"""The value is incremented for an existing predefined key."""
job_id = 82
pkey = "cblock"
key = stats.key_name(job_id, *stats.STATS_KEYS[pkey])
stats.delete_job_counters(job_id)
kvs = self.connect()
stats.pk_inc(job_id, pkey)
self.assertEqual("1", kvs.get(key))
def test_pk_set_with_existing_total(self):
"""The value is set correctly for an existing predefined key."""
job_id = 71
pkey = "blocks"
key = stats.key_name(job_id, *stats.STATS_KEYS[pkey])
stats.delete_job_counters(job_id)
kvs = self.connect()
stats.pk_set(job_id, pkey, 717)
self.assertEqual("717", kvs.get(key))
def test_delete_job_counters_deletes_counters_for_job(self):
"""
The progress indication counters for a given job are deleted.
"""
kvs = self.connect()
args = [(55, "h", "a/b/c"), (55, "h", "d/e/f")]
for data in args:
stats.incr_counter(*data)
stats.delete_job_counters(55)
self.assertEqual(0, len(kvs.keys("oqs:55:*")))
def test_delete_job_counters_deletes_counters_for_job(self):
"""
The progress indication counters for a given job are deleted.
"""
kvs = self.connect()
args = [(55, "h", "a/b/c"), (55, "h", "d/e/f")]
for data in args:
stats.incr_counter(*data)
stats.delete_job_counters(55)
self.assertEqual(0, len(kvs.keys("oqs:55:*")))
def test_pk_set_with_existing_total(self):
"""The value is set correctly for an existing predefined key."""
job_id = 71
pkey = "blocks"
key = stats.key_name(job_id, *stats.STATS_KEYS[pkey])
stats.delete_job_counters(job_id)
kvs = self.connect()
stats.pk_set(job_id, pkey, 717)
self.assertEqual("717", kvs.get(key))
def test_pk_inc_with_existing_incremental(self):
"""The value is incremented for an existing predefined key."""
job_id = 82
pkey = "cblock"
key = stats.key_name(job_id, *stats.STATS_KEYS[pkey])
stats.delete_job_counters(job_id)
kvs = self.connect()
stats.pk_inc(job_id, pkey)
self.assertEqual("1", kvs.get(key))
def test_pk_get_with_existing_incremental(self):
"""The correct value is obtained for an existing predefined key."""
job_id = 92
pkey = "cblock"
key = stats.key_name(job_id, *stats.STATS_KEYS[pkey])
stats.delete_job_counters(job_id)
kvs = self.connect()
kvs.set(key, 929)
stats.pk_get(job_id, pkey)
self.assertEqual("929", kvs.get(key))
def test_pk_get_with_existing_incremental(self):
"""The correct value is obtained for an existing predefined key."""
job_id = 92
pkey = "cblock"
key = stats.key_name(job_id, *stats.STATS_KEYS[pkey])
stats.delete_job_counters(job_id)
kvs = self.connect()
kvs.set(key, 929)
stats.pk_get(job_id, pkey)
self.assertEqual("929", kvs.get(key))
def test_delete_job_counters_resets_counters(self):
"""
The progress indication counters for a given job are reset.
"""
kvs = self.connect()
args = [(66, "h", "g/h/i", "i"), (66, "h", "j/k/l", "i")]
for data in args:
stats.incr_counter(*data[:-1])
stats.delete_job_counters(66)
# The counters have been reset, after incrementing we expect them all
# to have a value of "1".
for data in args:
stats.incr_counter(*data[:-1])
self.assertEqual("1", kvs.get(stats.key_name(*data)))
def test_delete_job_counters_resets_counters(self):
"""
The progress indication counters for a given job are reset.
"""
kvs = self.connect()
args = [(66, "h", "g/h/i", "i"), (66, "h", "j/k/l", "i")]
for data in args:
stats.incr_counter(*data[:-1])
stats.delete_job_counters(66)
# The counters have been reset, after incrementing we expect them all
# to have a value of "1".
for data in args:
stats.incr_counter(*data[:-1])
self.assertEqual("1", kvs.get(stats.key_name(*data)))
def test_failure_counters_with_no_area(self):
# Failure counters are returned for all computation areas if the
# 'area' parameter is omitted.
stats.delete_job_counters(123)
fcname = itertools.cycle(string.ascii_lowercase)
for cidx, carea in enumerate(["g", "h", "r"]):
stats.incr_counter(123, carea, "%s:failed" % fcname.next())
if not (cidx % 2):
stats.incr_counter(123, carea, "%s:failed" % fcname.next())
self.assertEqual(
[('oqs/123/g/a:failed/i', 1), ('oqs/123/g/b:failed/i', 1),
('oqs/123/h/c:failed/i', 1), ('oqs/123/r/d:failed/i', 1),
('oqs/123/r/e:failed/i', 1)],
sorted(stats.failure_counters(123)))
def test_failure_counters_with_no_area(self):
# Failure counters are returned for all computation areas if the
# 'area' parameter is omitted.
stats.delete_job_counters(123)
fcname = itertools.cycle(string.ascii_lowercase)
for cidx, carea in enumerate(["g", "h", "r"]):
stats.incr_counter(123, carea, "%s:failed" % fcname.next())
if not (cidx % 2):
stats.incr_counter(123, carea, "%s:failed" % fcname.next())
self.assertEqual([('oqs/123/g/a:failed/i', 1),
('oqs/123/g/b:failed/i', 1),
('oqs/123/h/c:failed/i', 1),
('oqs/123/r/d:failed/i', 1),
('oqs/123/r/e:failed/i', 1)],
sorted(stats.failure_counters(123)))
def cache_gc(job_id):
"""
Garbage collection for the KVS. This works by simply removing all keys
which contain the input job key.
The job key must be a member of the 'CURRENT_JOBS' set. If it isn't, this
function will do nothing and simply return None.
:param job_id: the id of the job
:type job_id: int
:returns: the number of deleted keys (int), or None if the job doesn't
exist in CURRENT_JOBS
"""
client = get_client()
if client.sismember(tokens.CURRENT_JOBS, job_id):
# matches a current job
# do the garbage collection
keys = client.keys('*%s*' % tokens.generate_job_key(job_id))
num_deleted = 0
if len(keys) > 0:
success = client.delete(*keys)
# delete should return True
if not success:
msg = 'Redis failed to delete data for job %s' % job_id
LOG.error(msg)
raise RuntimeError(msg)
# finally, remove the job key from CURRENT_JOBS
client.srem(tokens.CURRENT_JOBS, job_id)
msg = 'KVS garbage collection removed %s keys for job %s'
msg %= (len(keys), job_id)
LOG.debug(msg)
num_deleted += len(keys)
# clear stats counters too:
num_deleted += stats.delete_job_counters(job_id)
return num_deleted
else:
# does not match a current job
msg = 'KVS garbage collection was called with an invalid job key: ' \
'%s is not recognized as a current job.' % job_id
LOG.error(msg)
return None
def cache_gc(job_id):
"""
Garbage collection for the KVS. This works by simply removing all keys
which contain the input job key.
The job key must be a member of the 'CURRENT_JOBS' set. If it isn't, this
function will do nothing and simply return None.
:param job_id: the id of the job
:type job_id: int
:returns: the number of deleted keys (int), or None if the job doesn't
exist in CURRENT_JOBS
"""
client = get_client()
if client.sismember(tokens.CURRENT_JOBS, job_id):
# matches a current job
# do the garbage collection
keys = client.keys('*%s*' % tokens.generate_job_key(job_id))
num_deleted = 0
if len(keys) > 0:
success = client.delete(*keys)
# delete should return True
if not success:
msg = 'Redis failed to delete data for job %s' % job_id
LOG.error(msg)
raise RuntimeError(msg)
# finally, remove the job key from CURRENT_JOBS
client.srem(tokens.CURRENT_JOBS, job_id)
msg = 'KVS garbage collection removed %s keys for job %s'
msg %= (len(keys), job_id)
LOG.debug(msg)
num_deleted += len(keys)
# clear stats counters too:
num_deleted += stats.delete_job_counters(job_id)
return num_deleted
else:
# does not match a current job
msg = 'KVS garbage collection was called with an invalid job key: ' \
'%s is not recognized as a current job.' % job_id
LOG.error(msg)
return None
def test_failure_counters_with_no_failures(self):
# An empty list is returned in the absence of any failure counters
stats.delete_job_counters(123)
self.assertEqual([], stats.failure_counters(123))
def test_pk_get_with_non_existent_predef_key(self):
"""`KeyError` is raised for keys that do not exist in `STATS_KEYS`."""
job_id = 93
pkey = "This is unlikely to exist"
stats.delete_job_counters(job_id)
self.assertRaises(KeyError, stats.pk_get, job_id, pkey)
def test_delete_job_counters_copes_with_nonexistent_counters(self):
"""
stats.delete_job_counters() copes with jobs without progress indication
counters.
"""
stats.delete_job_counters(sys.maxint)
def test_delete_job_counters_copes_with_nonexistent_counters(self):
"""
stats.delete_job_counters() copes with jobs without progress indication
counters.
"""
stats.delete_job_counters(sys.maxint)
def test_failure_counters_with_no_failures(self):
# An empty list is returned in the absence of any failure counters
stats.delete_job_counters(123)
self.assertEqual([], stats.failure_counters(123))
def test_pk_get_with_non_existent_predef_key(self):
"""`KeyError` is raised for keys that do not exist in `STATS_KEYS`."""
job_id = 93
pkey = "This is unlikely to exist"
stats.delete_job_counters(job_id)
self.assertRaises(KeyError, stats.pk_get, job_id, pkey)
