def test_99_redis_died(self):
""" sometimes the redis server can die while the worker keeps running.
In such cases we have to kill the worker before restarting it.
"""
worker_name = Worker.redis_worker_namespace_prefix + self.args.name
worker = Worker.find_by_key(worker_name)
worker.connection.shutdown()
def test_find_by_key(self):
"""Worker.find_by_key restores queues, state and job_id."""
queues = [Queue('foo'), Queue('bar')]
w = Worker(queues)
w.register_death()
w.register_birth()
w.set_state(WorkerStatus.STARTED)
worker = Worker.find_by_key(w.key)
self.assertEqual(worker.queues, queues)
self.assertEqual(worker.get_state(), WorkerStatus.STARTED)
self.assertEqual(worker._job_id, None)
self.assertTrue(worker.key in Worker.all_keys(worker.connection))
# If worker is gone, its keys should also be removed
worker.connection.delete(worker.key)
Worker.find_by_key(worker.key)
self.assertFalse(worker.key in Worker.all_keys(worker.connection))
def test_find_by_key(self):
"""Worker.find_by_key restores queues, state and job_id."""
queues = [Queue('foo'), Queue('bar')]
w = Worker(queues)
w.register_death()
w.register_birth()
w.set_state(WorkerStatus.STARTED)
worker = Worker.find_by_key(w.key)
self.assertEqual(worker.queues, queues)
self.assertEqual(worker.get_state(), WorkerStatus.STARTED)
self.assertEqual(worker._job_id, None)
self.assertTrue(worker.key in Worker.all_keys(worker.connection))
# If worker is gone, its keys should also be removed
worker.connection.delete(worker.key)
Worker.find_by_key(worker.key)
self.assertFalse(worker.key in Worker.all_keys(worker.connection))
def get_context_data(self, **kwargs):
ctx = super(WorkerDetails, self).get_context_data(**kwargs)
key = Worker.redis_worker_namespace_prefix + self.kwargs['worker']
worker = Worker.find_by_key(key, connection=self.connection)
ctx.update({
'worker': worker,
'title': _('Worker %s') % worker.name,
})
return ctx
def get_context_data(self, **kwargs):
ctx = super(WorkerDetails, self).get_context_data(**kwargs)
key = Worker.redis_worker_namespace_prefix + self.kwargs['worker']
worker = Worker.find_by_key(key, connection=self.connection)
ctx.update({
'worker': worker,
'title': _('Worker %s') % worker.name,
})
return ctx
def test_worker_version(self):
q = Queue()
w = Worker([q])
w.version = '0.0.0'
w.register_birth()
self.assertEqual(w.version, '0.0.0')
w.refresh()
self.assertEqual(w.version, '0.0.0')
# making sure that version is preserved when worker is retrieved by key
worker = Worker.find_by_key(w.key)
self.assertEqual(worker.version, '0.0.0')
def test_heartbeat(self):
"""Heartbeat saves last_heartbeat"""
q = Queue()
w = Worker([q])
w.register_birth()
w.heartbeat()
last_heartbeat = self.testconn.hget(w.key, 'last_heartbeat')
self.assertTrue(last_heartbeat is not None)
w = Worker.find_by_key(w.key)
self.assertIsInstance(w.last_heartbeat, datetime)
def test_find_by_key(self):
"""Worker.find_by_key restores queues, state and job_id."""
queues = [Queue('foo'), Queue('bar')]
w = Worker(queues)
w.register_death()
w.register_birth()
w.set_state(WorkerStatus.STARTED)
worker = Worker.find_by_key(w.key)
self.assertEqual(worker.queues, queues)
self.assertEqual(worker.get_state(), WorkerStatus.STARTED)
self.assertEqual(worker._job_id, None)
w.register_death()
def test_find_by_key(self):
"""Worker.find_by_key restores queues, state and job_id."""
queues = [Queue('foo'), Queue('bar')]
w = Worker(queues)
w.register_death()
w.register_birth()
w.set_state(WorkerStatus.STARTED)
worker = Worker.find_by_key(w.key)
self.assertEqual(worker.queues, queues)
self.assertEqual(worker.get_state(), WorkerStatus.STARTED)
self.assertEqual(worker._job_id, None)
w.register_death()
def worker_details(request, queue_index, key):
queue_index = int(queue_index)
queue = get_queue_by_index(queue_index)
worker = Worker.find_by_key(key, connection=queue.connection)
queue_names = ', '.join(worker.queue_names())
context_data = {
'queue': queue,
'queue_index': queue_index,
'worker': worker,
'queue_names': queue_names,
'job': worker.get_current_job()
}
return render(request, 'django_rq/worker_details.html', context_data)
def worker_details(request, queue_index, key):
queue_index = int(queue_index)
queue = get_queue_by_index(queue_index)
worker = Worker.find_by_key(key, connection=queue.connection)
queue_names = ', '.join(worker.queue_names())
context_data = {
'queue': queue,
'queue_index': queue_index,
'worker': worker,
'queue_names': queue_names,
'job': worker.get_current_job()
}
return render(request, 'django_rq/worker_details.html', context_data)
def test_heartbeat(self):
"""Heartbeat saves last_heartbeat"""
q = Queue()
w = Worker([q])
w.register_birth()
last_heartbeat = self.testconn.hget(w.key, 'last_heartbeat')
self.assertTrue(last_heartbeat is not None)
w = Worker.find_by_key(w.key)
self.assertIsInstance(w.last_heartbeat, datetime)
# worker.refresh() shouldn't fail if last_heartbeat is None
# for compatibility reasons
self.testconn.hdel(w.key, 'last_heartbeat')
w.refresh()
def test_python_version(self):
python_version = sys.version
q = Queue()
w = Worker([q])
w.register_birth()
self.assertEqual(w.python_version, python_version)
# now patching version
python_version = 'X.Y.Z.final' # dummy version
self.assertNotEqual(python_version, sys.version) # otherwise tests are pointless
w2 = Worker([q])
w2.python_version = python_version
w2.register_birth()
self.assertEqual(w2.python_version, python_version)
# making sure that version is preserved when worker is retrieved by key
worker = Worker.find_by_key(w2.key)
self.assertEqual(worker.python_version, python_version)
def processqueue():
args = RedisQueueArgs()
setup_loghandlers(args)
setup_redis(args)
try:
queues = map(Queue, args.queues)
worker_name = Worker.redis_worker_namespace_prefix + args.name
worker = Worker.find_by_key(worker_name)
if worker:
# get the stale worker to stop in order to start a new one
print 'Stopping stale worker.'
worker.register_death()
else:
worker = Worker(queues, name=args.name)
worker.work(burst=args.burst)
except ConnectionError as e:
print(e)
sys.exit(1)
def test_heartbeat(self):
"""Heartbeat saves last_heartbeat"""
q = Queue()
w = Worker([q])
w.register_birth()
birth = self.testconn.hget(w.key, 'birth')
last_heartbeat = self.testconn.hget(w.key, 'last_heartbeat')
self.assertTrue(birth is not None)
self.assertTrue(last_heartbeat is not None)
w = Worker.find_by_key(w.key)
self.assertIsInstance(w.last_heartbeat, datetime)
# worker.refresh() shouldn't fail if last_heartbeat is None
# for compatibility reasons
self.testconn.hdel(w.key, 'last_heartbeat')
w.refresh()
# worker.refresh() shouldn't fail if birth is None
# for compatibility reasons
self.testconn.hdel(w.key, 'birth')
w.refresh()
def workers_status(self):
name_list = [worker['thread'].name for worker in self.worker_table]
workers = [
Worker.find_by_key(worker_key='rq:worker:' + worker['worker'].name,
connection=self.conn)
for worker in self.worker_table
]
#status_list = [worker['worker'].state for worker in self.worker_table]
status_list = [worker.state for worker in workers]
job_list = []
for worker in workers:
try:
sub_job_name = str(worker.get_current_job())
sub_job_name = find_between(sub_job_name, '<Job ', ': ')
job_list.append(sub_job_name)
except AttributeError:
job_list.append("no jobs")
return {'name': name_list, 'status': status_list, 'job': job_list}
def test_heartbeat(self):
"""Heartbeat saves last_heartbeat"""
q = Queue()
w = Worker([q])
w.register_birth()
self.assertEqual(str(w.pid), as_text(self.testconn.hget(w.key, 'pid')))
self.assertEqual(w.hostname,
as_text(self.testconn.hget(w.key, 'hostname')))
last_heartbeat = self.testconn.hget(w.key, 'last_heartbeat')
self.assertIsNotNone(self.testconn.hget(w.key, 'birth'))
self.assertTrue(last_heartbeat is not None)
w = Worker.find_by_key(w.key)
self.assertIsInstance(w.last_heartbeat, datetime)
# worker.refresh() shouldn't fail if last_heartbeat is None
# for compatibility reasons
self.testconn.hdel(w.key, 'last_heartbeat')
w.refresh()
# worker.refresh() shouldn't fail if birth is None
# for compatibility reasons
self.testconn.hdel(w.key, 'birth')
w.refresh()
def delete_worker(self, name):
print("Deleting worker %s ..." % name)
name_list = [worker['thread'].name for worker in self.worker_table]
try:
current_item = self.worker_table[name_list.index(name)]
except ValueError:
print("Cannot find worker %s, cannot terminate" % name)
return False
worker = Worker.find_by_key(worker_key='rq:worker:' +
current_item['worker'].name,
connection=self.conn)
thread = current_item['thread']
worker_status = worker.state
if worker_status != 'busy':
thread.terminate()
thread.join()
self.worker_table.remove(current_item)
print("Worker %s deleted" % name)
return True
else:
print("cannot delete worker %s, it's %s" %
(thread.name, worker.state))
return False
def test_02_assert_redis_worker_running(self):
worker_name = Worker.redis_worker_namespace_prefix + self.args.name
worker = Worker.find_by_key(worker_name)
assert worker is not None, 'Worker is not running. Start the worker.'
