def test_sleeps(self, mock_sleep, mock_check_celery_processes):
# raising an exception is the only way we have to break out of the
# infinite loop
mock_sleep.side_effect = self.SleepException
self.assertRaises(self.SleepException, scheduler.CeleryProcessTimeoutMonitor().run)
# verify the frequency
mock_sleep.assert_called_once_with(constants.PULP_PROCESS_HEARTBEAT_INTERVAL)
def test_logs_exception(self, mock_sleep, mock_check_celery_processes, mock_log_error):
# raising an exception is the only way we have to break out of the
# infinite loop
mock_check_celery_processes.side_effect = self.SleepException
mock_log_error.side_effect = self.SleepException
self.assertRaises(self.SleepException, scheduler.CeleryProcessTimeoutMonitor().run)
self.assertEqual(mock_log_error.call_count, 1)
def test_checks_workers(self, mock_sleep, mock_check_celery_processes, mock_log_error):
# raising an exception is the only way we have to break out of the
# infinite loop
mock_check_celery_processes.side_effect = self.SleepException
mock_log_error.side_effect = self.SleepException
self.assertRaises(self.SleepException, scheduler.CeleryProcessTimeoutMonitor().run)
mock_check_celery_processes.assert_called_once_with()
def test_deletes_workers(self, mock_worker, mock_delete_worker):
mock_worker.objects.all.return_value = [
Worker(name='name1', last_heartbeat=datetime.utcnow() - timedelta(seconds=400)),
Worker(name='name2', last_heartbeat=datetime.utcnow()),
]
scheduler.CeleryProcessTimeoutMonitor().check_celery_processes()
# make sure _delete_worker is only called for the old worker
mock_delete_worker.assert_has_calls([mock.call('name1')])
def test_logs_resource_manager_missing(self, mock__logger, mock_worker, mock_delete_worker):
mock_worker.objects.all.return_value = [
Worker(name=constants.SCHEDULER_WORKER_NAME, last_heartbeat=datetime.utcnow()),
Worker(name='name2', last_heartbeat=datetime.utcnow()),
]
scheduler.CeleryProcessTimeoutMonitor().check_celery_processes()
mock__logger.error.assert_called_once_with(
'There are 0 pulp_resource_manager processes running. Pulp will not operate '
'correctly without at least one pulp_resource_mananger process running.')
def test_debug_logging(self, mock__logger, mock_worker, mock_delete_worker):
mock_worker.objects.all.return_value = [
Worker(name='name1', last_heartbeat=datetime.utcnow() - timedelta(seconds=400)),
Worker(name='name2', last_heartbeat=datetime.utcnow()),
Worker(name=RESOURCE_MANAGER_WORKER_NAME, last_heartbeat=datetime.utcnow()),
Worker(name=SCHEDULER_WORKER_NAME, last_heartbeat=datetime.utcnow()),
]
scheduler.CeleryProcessTimeoutMonitor().check_celery_processes()
mock__logger.debug.assert_has_calls([
mock.call('Checking if pulp_workers, pulp_celerybeat, or '
'pulp_resource_manager processes are missing for more than 300 seconds'),
mock.call('1 pulp_worker processes, 1 pulp_celerybeat processes, '
'and 1 pulp_resource_manager processes')
])
def test_debug_logging(self, mock__logger, mock_worker, mock_delete_worker):
combined_delay = constants.PULP_PROCESS_TIMEOUT_INTERVAL + \
constants.PULP_PROCESS_HEARTBEAT_INTERVAL
now = datetime.utcnow()
mock_worker.objects.all.return_value = [
Worker(name='name1', last_heartbeat=now - timedelta(seconds=combined_delay)),
Worker(name='name2', last_heartbeat=now),
Worker(name=constants.RESOURCE_MANAGER_WORKER_NAME, last_heartbeat=now),
Worker(name=constants.SCHEDULER_WORKER_NAME, last_heartbeat=now),
]
scheduler.CeleryProcessTimeoutMonitor().check_celery_processes()
mock__logger.debug.assert_has_calls([
mock.call(
'Checking if pulp_workers, pulp_celerybeat, or pulp_resource_manager processes are '
'missing for more than %d seconds' % constants.PULP_PROCESS_TIMEOUT_INTERVAL
),
mock.call(
'1 pulp_worker processes, 1 pulp_celerybeat processes, '
'and 1 pulp_resource_manager processes'
)
])
def test_queries_all_workers(self, mock_worker):
mock_worker.return_value = []
scheduler.CeleryProcessTimeoutMonitor().check_celery_processes()
mock_worker.objects.all.assert_called_once_with()