def test_pk_inc_with_non_existent_predef_key(self): """`KeyError` is raised for keys that do not exist in `STATS_KEYS`.""" job_id = 83 pkey = "That is the question.." stats.delete_job_counters(job_id) self.assertRaises(KeyError, stats.pk_inc, job_id, pkey)
def test_pk_set_with_non_existent_predef_key(self): """`KeyError` is raised for keys that do not exist in `STATS_KEYS`.""" job_id = 73 pkey = "To be or not to be!?" stats.delete_job_counters(job_id) self.assertRaises(KeyError, stats.pk_set, job_id, pkey, 737)
def test_actions_after_job_process_failures(self): # the job process is running but has some failure counters above zero # shorten the delay to checking failure counters supervisor.SupervisorLogMessageConsumer.FCC_DELAY = 2 self.is_pid_running.return_value = True self.get_job_status.return_value = 'running' stats.delete_job_counters(self.job.id) stats.incr_counter(self.job.id, "h", "a:failed") stats.incr_counter(self.job.id, "r", "b:failed") stats.incr_counter(self.job.id, "r", "b:failed") supervisor.supervise(1, self.job.id, timeout=0.1) # the job process is terminated self.assertEqual(1, self.terminate_job.call_count) self.assertEqual(((1, ), {}), self.terminate_job.call_args) # stop time is recorded self.assertEqual(1, self.record_job_stop_time.call_count) self.assertEqual(((self.job.id, ), {}), self.record_job_stop_time.call_args) # the cleanup is triggered self.assertEqual(1, self.cleanup_after_job.call_count) self.assertEqual(((self.job.id, terminate), {}), self.cleanup_after_job.call_args)
def test_actions_after_job_process_failures(self): # the job process is running but has some failure counters above zero # shorten the delay to checking failure counters supervisor.SupervisorLogMessageConsumer.FCC_DELAY = 2 self.is_pid_running.return_value = True self.get_job_status.return_value = 'running' stats.delete_job_counters(self.job.id) stats.incr_counter(self.job.id, "h", "a:failed") stats.incr_counter(self.job.id, "r", "b:failed") stats.incr_counter(self.job.id, "r", "b:failed") supervisor.supervise(1, self.job.id, timeout=0.1) # the job process is terminated self.assertEqual(1, self.terminate_job.call_count) self.assertEqual(((1,), {}), self.terminate_job.call_args) # stop time is recorded self.assertEqual(1, self.record_job_stop_time.call_count) self.assertEqual( ((self.job.id,), {}), self.record_job_stop_time.call_args) # the cleanup is triggered self.assertEqual(1, self.cleanup_after_job.call_count) self.assertEqual( ((self.job.id,), {}), self.cleanup_after_job.call_args)
def test_pk_inc_with_existing_incremental(self): """The value is incremented for an existing predefined key.""" job_id = 82 pkey = "cblock" key = stats.key_name(job_id, *stats.STATS_KEYS[pkey]) stats.delete_job_counters(job_id) kvs = self.connect() stats.pk_inc(job_id, pkey) self.assertEqual("1", kvs.get(key))
def test_pk_set_with_existing_total(self): """The value is set correctly for an existing predefined key.""" job_id = 71 pkey = "blocks" key = stats.key_name(job_id, *stats.STATS_KEYS[pkey]) stats.delete_job_counters(job_id) kvs = self.connect() stats.pk_set(job_id, pkey, 717) self.assertEqual("717", kvs.get(key))
def test_delete_job_counters_deletes_counters_for_job(self): """ The progress indication counters for a given job are deleted. """ kvs = self.connect() args = [(55, "h", "a/b/c"), (55, "h", "d/e/f")] for data in args: stats.incr_counter(*data) stats.delete_job_counters(55) self.assertEqual(0, len(kvs.keys("oqs:55:*")))
def test_pk_get_with_existing_incremental(self): """The correct value is obtained for an existing predefined key.""" job_id = 92 pkey = "cblock" key = stats.key_name(job_id, *stats.STATS_KEYS[pkey]) stats.delete_job_counters(job_id) kvs = self.connect() kvs.set(key, 929) stats.pk_get(job_id, pkey) self.assertEqual("929", kvs.get(key))
def test_delete_job_counters_resets_counters(self): """ The progress indication counters for a given job are reset. """ kvs = self.connect() args = [(66, "h", "g/h/i", "i"), (66, "h", "j/k/l", "i")] for data in args: stats.incr_counter(*data[:-1]) stats.delete_job_counters(66) # The counters have been reset, after incrementing we expect them all # to have a value of "1". for data in args: stats.incr_counter(*data[:-1]) self.assertEqual("1", kvs.get(stats.key_name(*data)))
def test_failure_counters_with_no_area(self): # Failure counters are returned for all computation areas if the # 'area' parameter is omitted. stats.delete_job_counters(123) fcname = itertools.cycle(string.ascii_lowercase) for cidx, carea in enumerate(["g", "h", "r"]): stats.incr_counter(123, carea, "%s:failed" % fcname.next()) if not (cidx % 2): stats.incr_counter(123, carea, "%s:failed" % fcname.next()) self.assertEqual( [('oqs/123/g/a:failed/i', 1), ('oqs/123/g/b:failed/i', 1), ('oqs/123/h/c:failed/i', 1), ('oqs/123/r/d:failed/i', 1), ('oqs/123/r/e:failed/i', 1)], sorted(stats.failure_counters(123)))
def test_failure_counters_with_no_area(self): # Failure counters are returned for all computation areas if the # 'area' parameter is omitted. stats.delete_job_counters(123) fcname = itertools.cycle(string.ascii_lowercase) for cidx, carea in enumerate(["g", "h", "r"]): stats.incr_counter(123, carea, "%s:failed" % fcname.next()) if not (cidx % 2): stats.incr_counter(123, carea, "%s:failed" % fcname.next()) self.assertEqual([('oqs/123/g/a:failed/i', 1), ('oqs/123/g/b:failed/i', 1), ('oqs/123/h/c:failed/i', 1), ('oqs/123/r/d:failed/i', 1), ('oqs/123/r/e:failed/i', 1)], sorted(stats.failure_counters(123)))
def cache_gc(job_id): """ Garbage collection for the KVS. This works by simply removing all keys which contain the input job key. The job key must be a member of the 'CURRENT_JOBS' set. If it isn't, this function will do nothing and simply return None. :param job_id: the id of the job :type job_id: int :returns: the number of deleted keys (int), or None if the job doesn't exist in CURRENT_JOBS """ client = get_client() if client.sismember(tokens.CURRENT_JOBS, job_id): # matches a current job # do the garbage collection keys = client.keys('*%s*' % tokens.generate_job_key(job_id)) num_deleted = 0 if len(keys) > 0: success = client.delete(*keys) # delete should return True if not success: msg = 'Redis failed to delete data for job %s' % job_id LOG.error(msg) raise RuntimeError(msg) # finally, remove the job key from CURRENT_JOBS client.srem(tokens.CURRENT_JOBS, job_id) msg = 'KVS garbage collection removed %s keys for job %s' msg %= (len(keys), job_id) LOG.debug(msg) num_deleted += len(keys) # clear stats counters too: num_deleted += stats.delete_job_counters(job_id) return num_deleted else: # does not match a current job msg = 'KVS garbage collection was called with an invalid job key: ' \ '%s is not recognized as a current job.' % job_id LOG.error(msg) return None
def test_failure_counters_with_no_failures(self): # An empty list is returned in the absence of any failure counters stats.delete_job_counters(123) self.assertEqual([], stats.failure_counters(123))
def test_pk_get_with_non_existent_predef_key(self): """`KeyError` is raised for keys that do not exist in `STATS_KEYS`.""" job_id = 93 pkey = "This is unlikely to exist" stats.delete_job_counters(job_id) self.assertRaises(KeyError, stats.pk_get, job_id, pkey)
def test_delete_job_counters_copes_with_nonexistent_counters(self): """ stats.delete_job_counters() copes with jobs without progress indication counters. """ stats.delete_job_counters(sys.maxint)