def get(self):
"""获取任务列表"""
args = request.args
job_status = args.get('job_status')
queue_name = args.get('queue_name')
if job_status not in Config.RQ_JOB_STATUS:
return {
'code': StatesCode.JOB_STATUS_NO_EXIST,
'message': '任务状态不存在！'
}
if queue_name not in Config.RQ_QUEUES_ALL:
return {'code': StatesCode.QUEUE_NOT_EXIST, 'message': '任务队列不存在！'}
job_list = []
if job_status == 'queued':
if queue_name == 'all':
for queue_name in Config.RQ_QUEUES:
job_list += queue_dict[queue_name].get_job_ids()
else:
job_list = queue_dict[queue_name].get_job_ids()
elif job_status == 'started':
if queue_name == 'all':
for queue_name in Config.RQ_QUEUES:
started_job_registry = StartedJobRegistry(
queue=queue_dict[queue_name])
job_list += started_job_registry.get_job_ids()
else:
started_job_registry = StartedJobRegistry(
queue=queue_dict[queue_name])
job_list = started_job_registry.get_job_ids()
elif job_status == 'finished':
if queue_name == 'all':
for queue_name in Config.RQ_QUEUES:
finished_job_registry = FinishedJobRegistry(
queue=queue_dict[queue_name])
job_list += finished_job_registry.get_job_ids()
else:
finished_job_registry = FinishedJobRegistry(
queue=queue_dict[queue_name])
job_list = finished_job_registry.get_job_ids()
elif job_status == 'failed':
if queue_name == 'all':
for queue_name in Config.RQ_QUEUES:
failed_job_registry = FailedJobRegistry(
queue=queue_dict[queue_name])
job_list += failed_job_registry.get_job_ids()
else:
failed_job_registry = FailedJobRegistry(
queue=queue_dict[queue_name])
job_list = failed_job_registry.get_job_ids()
elif job_status == 'deferred':
if queue_name == 'all':
for queue_name in Config.RQ_QUEUES:
deferred_job_registry = DeferredJobRegistry(
queue=queue_dict[queue_name])
job_list += deferred_job_registry.get_job_ids()
else:
deferred_job_registry = DeferredJobRegistry(
queue=queue_dict[queue_name])
job_list = deferred_job_registry.get_job_ids()
return {'code': StatesCode.SUCCESS, 'data': job_list}
class TestFinishedJobRegistry(RQTestCase):
def setUp(self):
super(TestFinishedJobRegistry, self).setUp()
self.registry = FinishedJobRegistry(connection=self.testconn)
def test_cleanup(self):
"""Finished job registry removes expired jobs."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, 1, 'foo')
self.testconn.zadd(self.registry.key, timestamp + 10, 'bar')
self.testconn.zadd(self.registry.key, timestamp + 30, 'baz')
self.registry.cleanup()
self.assertEqual(self.registry.get_job_ids(), ['bar', 'baz'])
self.registry.cleanup(timestamp + 20)
self.assertEqual(self.registry.get_job_ids(), ['baz'])
def test_jobs_are_put_in_registry(self):
"""Completed jobs are added to FinishedJobRegistry."""
self.assertEqual(self.registry.get_job_ids(), [])
queue = Queue(connection=self.testconn)
worker = Worker([queue])
# Completed jobs are put in FinishedJobRegistry
job = queue.enqueue(say_hello)
worker.perform_job(job, queue)
self.assertEqual(self.registry.get_job_ids(), [job.id])
# Failed jobs are not put in FinishedJobRegistry
failed_job = queue.enqueue(div_by_zero)
worker.perform_job(failed_job, queue)
self.assertEqual(self.registry.get_job_ids(), [job.id])
def test_reenqueue_object_success(reenqueue_object, session, redis,
museum_object, museum_package):
# Create fake DB entries
museum_package.downloaded = True
museum_package.packaged = True
museum_package.uploaded = True
museum_package.rejected = True
session.commit()
# Create a job that was completed prior to re-enqueuing
queue = get_queue(QueueType.CONFIRM_SIP)
queue.enqueue(successful_job, job_id="confirm_sip_123456")
SimpleWorker([queue], connection=queue.connection).work(burst=True)
finished_registry = FinishedJobRegistry(queue=queue)
assert finished_registry.get_job_ids() == ["confirm_sip_123456"]
result = reenqueue_object(["123456"])
assert "Object 123456 re-enqueued" in result.stdout
# New RQ task was enqueued
queue = get_queue(QueueType.DOWNLOAD_OBJECT)
assert "download_object_123456" in queue.job_ids
# Database was updated
db_museum_object = session.query(MuseumObject).filter_by(id=123456).one()
assert len(db_museum_object.packages) == 1
assert not db_museum_object.latest_package
# Prior finished job was removed
assert finished_registry.get_job_ids() == []
class TestFinishedJobRegistry(RQTestCase):
def setUp(self):
super(TestFinishedJobRegistry, self).setUp()
self.registry = FinishedJobRegistry(connection=self.testconn)
def test_cleanup(self):
"""Finished job registry removes expired jobs."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, 1, 'foo')
self.testconn.zadd(self.registry.key, timestamp + 10, 'bar')
self.testconn.zadd(self.registry.key, timestamp + 30, 'baz')
self.registry.cleanup()
self.assertEqual(self.registry.get_job_ids(), ['bar', 'baz'])
self.registry.cleanup(timestamp + 20)
self.assertEqual(self.registry.get_job_ids(), ['baz'])
def test_jobs_are_put_in_registry(self):
"""Completed jobs are added to FinishedJobRegistry."""
self.assertEqual(self.registry.get_job_ids(), [])
queue = Queue(connection=self.testconn)
worker = Worker([queue])
# Completed jobs are put in FinishedJobRegistry
job = queue.enqueue(say_hello)
worker.perform_job(job)
self.assertEqual(self.registry.get_job_ids(), [job.id])
# Failed jobs are not put in FinishedJobRegistry
failed_job = queue.enqueue(div_by_zero)
worker.perform_job(failed_job)
self.assertEqual(self.registry.get_job_ids(), [job.id])
def get_all_finished_tasks_from(queue_name, app_info):
with Connection(redis.from_url(app_info['REDIS_URL'])):
f_registry = FinishedJobRegistry(queue_name)
job_ids = f_registry.get_job_ids()
# Generate the task info
return generate_task_info(queue_name, job_ids, "finished")
def getfinishedjobs(self, q):
try:
registry = FinishedJobRegistry(q, connection=self.base_connection)
response_object = registry.get_job_ids()
return response_object
except Exception as e:
return e
def get_finished_tasks(request):
current_queue = request.GET.get('queue')
queue = django_rq.get_queue(current_queue)
registry = FinishedJobRegistry(queue.name, queue.connection)
items_per_page = 10
num_jobs = len(registry)
jobs = []
if num_jobs > 0:
offset = 0
job_ids = registry.get_job_ids(offset, items_per_page)
for job_id in job_ids:
try:
jobs.append(Job.fetch(job_id, connection=queue.connection))
except NoSuchJobError:
pass
jobdata = list()
for job in jobs:
job_dict = {
'job_id': job.id,
'func_name': job.func_name,
'ended_at': job.ended_at.strftime("%a, %d %b %Y %H:%M:%S +0000"),
'enqueued_at': job.enqueued_at.strftime("%a, %d %b %Y %H:%M:%S +0000"),
'args': job.args}
jobdata.append(job_dict)
data = json.dumps(jobdata)
return HttpResponse(data, content_type='application/json')
def finished_jobs(request, queue_index):
queue_index = int(queue_index)
queue = get_queue_by_index(queue_index)
registry = FinishedJobRegistry(queue.name, queue.connection)
items_per_page = 100
num_jobs = len(registry)
page = int(request.GET.get('page', 1))
jobs = []
if num_jobs > 0:
last_page = int(ceil(num_jobs / items_per_page))
page_range = range(1, last_page + 1)
offset = items_per_page * (page - 1)
job_ids = registry.get_job_ids(offset, offset + items_per_page - 1)
jobs = get_jobs(queue, job_ids, registry)
else:
page_range = []
context_data = {
'queue': queue,
'queue_index': queue_index,
'jobs': jobs,
'num_jobs': num_jobs,
'page': page,
'page_range': page_range,
'job_status': 'Finished',
}
return render(request, 'django_rq/jobs.html', context_data)
def get_previous_job_from_registry(index=-1):
q = Queue('high', connection=conn)
registry = FinishedJobRegistry(queue=q)
job_id = registry.get_job_ids()[index]
job = q.fetch_job(job_id)
return job
def _remove_jobs(self, project):
started_registry = StartedJobRegistry(self._redis_queue.name, self._redis_queue.connection)
finished_registry = FinishedJobRegistry(self._redis_queue.name, self._redis_queue.connection)
for job_id in started_registry.get_job_ids() + finished_registry.get_job_ids():
job = Job.fetch(job_id, connection=self._redis)
if job.meta.get('project') != project:
continue
logger.info(f'Deleting job_id {job_id}')
job.delete()
def getfinishedjobs(self, q):
"""returns list of finished redis jobs"""
log.info(f"getting finished jobs: {q}")
try:
registry = FinishedJobRegistry(q, connection=self.base_connection)
response_object = registry.get_job_ids()
return response_object
except Exception as e:
return e
def show_results():
registry = FinishedJobRegistry(queue=q)
ids = registry.get_job_ids()
print("Got finished jobs:", len(ids))
res = []
for j in ids:
job = q.fetch_job(j)
if job.result and job.result is not None and isinstance(job.result, ResultField):
print("Result:", job.result.name, job.result.power, job.result.duration)
res.append(job.result)
return render_template('overview.html', rows=res)
def queue_count():
queue_out = {}
csv_out = str(int(time.time()))
csv_out += ','
with Connection(redis.from_url(current_app.config['REDIS_URL'])):
q = Queue('default')
f_registry = FinishedJobRegistry('default')
csv_out += str(len(q))
csv_out += ','
csv_out += str(len(f_registry.get_job_ids()))
csv_out += '\n'
return csv_out
def test_job_from_non_default_queue_completed(self):
"""
Ensure that job from non default queue successfully completed
when run from rqworker.
"""
queue_name = 'django_rq_test'
queue = get_queue(queue_name)
job = queue.enqueue(divide, 42, 1)
finished_job_registry = FinishedJobRegistry(queue.name,
queue.connection)
call_command('rqworker', queue_name, burst=True)
self.assertTrue(job.is_finished)
self.assertIn(job.id, finished_job_registry.get_job_ids())
def _remove_jobs(cls, project):
started_registry = StartedJobRegistry(
cls._redis_queue.name, cls._redis_queue.connection
)
finished_registry = FinishedJobRegistry(
cls._redis_queue.name, cls._redis_queue.connection
)
for job_id in started_registry.get_job_ids() + finished_registry.get_job_ids():
job = Job.fetch(job_id, connection=cls._redis)
if job.meta.get("project") != project:
continue
logger.info("Deleting job_id " + job_id)
job.delete()
def test_04_schedule(self):
"""
Should create a task a put in Redis queue.
"""
redis_conn = db_utils.get_redis_conn()
job_id = upload.schedule("test", "test")
time.sleep(1)
finished = FinishedJobRegistry(RQ_NLP_QUEUE_LOW, connection=redis_conn)
finished_job_ids = finished.get_job_ids()
queue = Queue(RQ_NLP_QUEUE_LOW, connection=redis_conn)
active_job_ids = queue.get_job_ids()
self.assertTrue(job_id in [*finished_job_ids, *active_job_ids])
def flush_redis():
with Connection(redis.from_url(current_app.config['REDIS_URL'])):
queues = ['default', 'aggregator']
for queue in queues:
q = Queue(queue)
f_registry = FinishedJobRegistry(queue)
finished_job_ids = f_registry.get_job_ids()
for f_job_id in finished_job_ids:
j = q.fetch_job(f_job_id)
j.cleanup(0)
return "Flushed"
def monitor():
'''
DOESN'T WORK
Meant to run until all jobs are finished. Monitors queues and adds completed graphs to Blazegraph.
'''
sregistry = StartedJobRegistry(connection=redis_conn)
fregistry = FinishedJobRegistry(connection=redis_conn)
print high.get_job_ids()
print sregistry.get_job_ids()
while sregistry.get_job_ids():
print 'in sregistry...'
print fregistry.get_job_ids()
for job_id in fregistry.get_job_ids():
job = Job.fetch(job_id, connection=redis_conn)
# sanity check
if type(job.result) is Graph:
print ('inserting', job_id, job)
logging.info('inserting', job_id, job)
insert(job.result)
print 'sleeping 5'
time.sleep(5)
print 'all jobs complete'
logging.info('monitor() exiting...all jobs complete')
def get_all_finished_tasks_from(queue_name):
with Connection(redis.from_url(current_app.config['REDIS_URL'])):
# q = Queue(queue_name)
f_registry = FinishedJobRegistry(queue_name)
data = {}
finished_job_ids = f_registry.get_job_ids()
data['status'] = 'success'
data['queue_name'] = queue_name
data['finished'] = {}
data['finished']['count'] = len(finished_job_ids)
data['finished']['finished_tasks_ids'] = []
for finished_job_id in finished_job_ids:
data['finished']['finished_tasks_ids'].append(finished_job_id)
return jsonify(data)
def delete_finished(c, queue=None):
"""Delete finished jobs from given queue
"""
if queue is None:
raise ValueError("Please specify queue")
q = Queue(
queue,
connection=Redis(host=REDIS_HOST,
port=REDIS_PORT,
password=REDIS_PASSWORD,
db=REDIS_DB),
)
registry = FinishedJobRegistry(queue=q)
for job_id in registry.get_job_ids():
job = q.fetch_job(job_id)
if job is not None and job.is_started:
job.delete()
def _get_job_list(redis_conn):
queue_list = _get_queue_list()
job_list = []
for q in queue_list:
queue = Queue(q, connection=redis_conn)
jobs = queue.get_jobs()
for j in jobs:
job_list.append({'id': j.get_id(), 'status': j.get_status()})
registry = FinishedJobRegistry(name=q, connection=redis_conn)
job_ids = registry.get_job_ids()
for jid in job_ids:
job = Job.fetch(jid, connection=redis_conn)
job_list.append({'id': jid, 'status': job.get_status()})
return job_list
def get_average():
# TODO: Code for case where there are no registries yet
avg_time = {}
redis = Redis()
qs = ['high', 'default', 'low']
for q in qs:
registry = FinishedJobRegistry(name=q, connection=redis)
averages = []
for job_id in registry.get_job_ids():
job = Job.fetch(job_id, connection=redis)
averages.append((job.ended_at - job.enqueued_at).total_seconds())
ave = sum(averages) / len(averages)
avg_time[q] = time.strftime('%H:%M:%S', time.gmtime(ave))
response = api.response_class(response=json.dumps(avg_time),
status=200,
mimetype='application/json')
return response
def get_all_task_ids(app_info):
with Connection(redis.from_url(app_info['REDIS_URL'])):
q = Queue('default')
registry = StartedJobRegistry('default')
f_registry = FinishedJobRegistry('default')
if q:
data = {}
running_job_ids = registry.get_job_ids()
expired_job_ids = registry.get_expired_job_ids()
finished_job_ids = f_registry.get_job_ids()
queued_job_ids = q.job_ids
data['status'] = 'success'
data['queue_name'] = 'default' #Make dynamic or parameterized?
data['running'] = {}
data['queued'] = {}
data['expired'] = {}
data['finished'] = {}
data['running']['count'] = len(running_job_ids)
data['running']['running_tasks_ids'] = []
for running_job_id in running_job_ids:
data['running']['running_tasks_ids'].append(running_job_id)
data['queued']['count'] = len(queued_job_ids)
data['queued']['queued_tasks_ids'] = []
for queued_job_id in queued_job_ids:
data['queued']['queued_tasks_ids'].append(queued_job_id)
data['expired']['count'] = len(expired_job_ids)
data['expired']['expired_tasks_ids'] = []
for expired_job_id in expired_job_ids:
data['expired']['expired_tasks_ids'].append(expired_job_id)
data['finished']['count'] = len(finished_job_ids)
data['finished']['finished_tasks_ids'] = []
for finished_job_id in finished_job_ids:
data['finished']['finished_tasks_ids'].append(finished_job_id)
return data
return {'status': 'error'}
def finished_jobs(request, queue_index):
queue_index = int(queue_index)
queue = get_queue_by_index(queue_index)
registry = FinishedJobRegistry(queue.name, queue.connection)
items_per_page = 100
num_jobs = len(registry)
page = int(request.GET.get('page', 1))
jobs = []
if num_jobs > 0:
last_page = int(ceil(num_jobs / items_per_page))
page_range = range(1, last_page + 1)
offset = items_per_page * (page - 1)
job_ids = registry.get_job_ids(offset, items_per_page)
for job_id in job_ids:
try:
jobs.append(Job.fetch(job_id, connection=queue.connection))
except NoSuchJobError:
pass
else:
page_range = []
###
# Custom logic here
for job in jobs:
use_actual_name(job)
##
context_data = {
'queue': queue,
'queue_index': queue_index,
'jobs': jobs,
'num_jobs': num_jobs,
'page': page,
'page_range': page_range,
'job_status': 'Finished',
}
return render(request, 'django_rq/jobs.html', context_data)
def handle(self, *args, **options):
import django_rq
from rq.registry import FinishedJobRegistry
from rq.exceptions import NoSuchJobError
from rq.job import Job
from datetime import datetime, timedelta
day_before_yesterday = datetime.now() - timedelta(days=2)
for index, config in enumerate(django_rq.settings.QUEUES_LIST):
queue = django_rq.queues.get_queue_by_index(index)
registry = FinishedJobRegistry(queue.name, queue.connection)
for job_id in registry.get_job_ids():
try:
job = Job.fetch(job_id, connection=queue.connection)
# delete jobs older 2 days
if job.ended_at > day_before_yesterday:
continue
except NoSuchJobError:
# for some reason job already deleted but job key exists
pass
registry.connection.zrem(registry.key, job_id)
def finished_jobs(request, queue_index):
queue_index = int(queue_index)
queue = get_queue_by_index(queue_index)
registry = FinishedJobRegistry(queue.name, queue.connection)
items_per_page = 100
num_jobs = len(registry)
page = int(request.GET.get('page', 1))
jobs = []
if num_jobs > 0:
last_page = int(ceil(num_jobs / items_per_page))
page_range = range(1, last_page + 1)
offset = items_per_page * (page - 1)
job_ids = registry.get_job_ids(offset, items_per_page)
for job_id in job_ids:
try:
jobs.append(Job.fetch(job_id, connection=queue.connection))
except NoSuchJobError:
pass
else:
page_range = []
context_data = admin.site.each_context(request)
context_data.update({
'title': _("Finished Jobs"),
'queue': queue,
'queue_index': queue_index,
'jobs': jobs,
'num_jobs': num_jobs,
'page': page,
'page_range': page_range,
'job_status': _('Finished'),
})
return render(request, 'django_rq/jobs.html', context_data)
class TestFinishedJobRegistry(RQTestCase):
def setUp(self):
super(TestFinishedJobRegistry, self).setUp()
self.registry = FinishedJobRegistry(connection=self.testconn)
def test_key(self):
self.assertEqual(self.registry.key, 'rq:finished:default')
def test_cleanup(self):
"""Finished job registry removes expired jobs."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, {'foo': 1})
self.testconn.zadd(self.registry.key, {'bar': timestamp + 10})
self.testconn.zadd(self.registry.key, {'baz': timestamp + 30})
self.registry.cleanup()
self.assertEqual(self.registry.get_job_ids(), ['bar', 'baz'])
self.registry.cleanup(timestamp + 20)
self.assertEqual(self.registry.get_job_ids(), ['baz'])
# CanceledJobRegistry now implements noop cleanup, should not raise exception
registry = CanceledJobRegistry(connection=self.testconn)
registry.cleanup()
def test_jobs_are_put_in_registry(self):
"""Completed jobs are added to FinishedJobRegistry."""
self.assertEqual(self.registry.get_job_ids(), [])
queue = Queue(connection=self.testconn)
worker = Worker([queue])
# Completed jobs are put in FinishedJobRegistry
job = queue.enqueue(say_hello)
worker.perform_job(job, queue)
self.assertEqual(self.registry.get_job_ids(), [job.id])
# When job is deleted, it should be removed from FinishedJobRegistry
self.assertEqual(job.get_status(), JobStatus.FINISHED)
job.delete()
self.assertEqual(self.registry.get_job_ids(), [])
# Failed jobs are not put in FinishedJobRegistry
failed_job = queue.enqueue(div_by_zero)
worker.perform_job(failed_job, queue)
self.assertEqual(self.registry.get_job_ids(), [])
def finished_jobs(request, queue_index):
queue_index = int(queue_index)
queue = get_queue_by_index(queue_index)
registry = FinishedJobRegistry(queue.name, queue.connection)
items_per_page = 100
num_jobs = len(registry)
page = int(request.GET.get('page', 1))
jobs = []
if num_jobs > 0:
last_page = int(ceil(num_jobs / items_per_page))
page_range = range(1, last_page + 1)
offset = items_per_page * (page - 1)
job_ids = registry.get_job_ids(offset, offset + items_per_page - 1)
for job_id in job_ids:
try:
jobs.append(Job.fetch(job_id, connection=queue.connection))
except NoSuchJobError:
pass
else:
page_range = []
context_data = {
'queue': queue,
'queue_index': queue_index,
'jobs': jobs,
'num_jobs': num_jobs,
'page': page,
'page_range': page_range,
'job_status': 'Finished',
}
return render(request, 'django_rq/jobs.html', context_data)
class TestFinishedJobRegistry(RQTestCase):
def setUp(self):
super(TestFinishedJobRegistry, self).setUp()
self.registry = FinishedJobRegistry(connection=self.testconn)
def test_key(self):
self.assertEqual(self.registry.key, 'rq:finished:default')
def test_cleanup(self):
"""Finished job registry removes expired jobs."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, {'foo': 1})
self.testconn.zadd(self.registry.key, {'bar': timestamp + 10})
self.testconn.zadd(self.registry.key, {'baz': timestamp + 30})
self.registry.cleanup()
self.assertEqual(self.registry.get_job_ids(), ['bar', 'baz'])
self.registry.cleanup(timestamp + 20)
self.assertEqual(self.registry.get_job_ids(), ['baz'])
def test_jobs_are_put_in_registry(self):
"""Completed jobs are added to FinishedJobRegistry."""
self.assertEqual(self.registry.get_job_ids(), [])
queue = Queue(connection=self.testconn)
worker = Worker([queue])
# Completed jobs are put in FinishedJobRegistry
job = queue.enqueue(say_hello)
worker.perform_job(job, queue)
self.assertEqual(self.registry.get_job_ids(), [job.id])
# When job is deleted, it should be removed from FinishedJobRegistry
self.assertEqual(job.get_status(), JobStatus.FINISHED)
job.delete()
self.assertEqual(self.registry.get_job_ids(), [])
# Failed jobs are not put in FinishedJobRegistry
failed_job = queue.enqueue(div_by_zero)
worker.perform_job(failed_job, queue)
self.assertEqual(self.registry.get_job_ids(), [])
class QueueManager:
retries_before_fail = 10
backoff = 30
def __init__(self, name, connection):
self._name = name
self._logger = logger.bind(queue=name)
self._queue = rq.Queue('vivarium', connection=connection)
self._wip = StartedJobRegistry('vivarium',
connection=connection,
job_class=self._queue.job_class)
self._finished = FinishedJobRegistry('vivarium',
connection=connection,
job_class=self._queue.job_class)
self._status = {
'total': 0,
'pending': 0,
'running': 0,
'failed': 0,
'finished': 0,
'done': 0.,
'workers': 0
}
self._failed = False
self._completed = False
self._retries = QueueManager.retries_before_fail
self._backoff = QueueManager.backoff
@property
def name(self):
return self._name
@property
def failed(self):
return self._failed
@property
def completed(self):
return self._completed
@property
def jobs_to_finish(self):
return not (self.failed or self.completed)
def enqueue(self, jobs):
self._logger.info(f'Enqueuing jobs in queue {self.name}')
for job in jobs:
# TODO: might be nice to have tighter ttls but it's hard to predict how long our jobs
# will take from model to model and the entire system is short lived anyway
self._queue.enqueue(
'vivarium_cluster_tools.psimulate.distributed_worker.worker',
parameters=job,
ttl=60 * 60 * 24 * 2,
result_ttl=60 * 60,
job_timeout='7d')
def get_results(self):
self._logger.info(f'Checking queue {self.name}')
finished_jobs = self._get_finished_jobs()
start = time.time()
results = []
for job_id in finished_jobs:
result = self._get_result(job_id)
if result is not None:
results.append(result)
self._logger.info(
f'Retrieved {len(results)} results from queue {self.name} in {time.time() - start:.2f}s'
)
return results
def update_and_report(self):
self._update_status()
template = (
f"Queue {self.name} - Total jobs: {{total}}, % Done: {{done:.2f}}% "
f"Pending: {{pending}}, Running: {{running}}, Failed: {{failed}}, Finished: {{finished}} "
f"Workers: {{workers}}.")
if not (self.completed or self.failed):
self._logger.info(template.format(**self._status))
return self._status
def _update_status(self):
if self.jobs_to_finish and self._retries > 0:
self._logger.info(f'Updating status for queue {self.name}')
# TODO: Sometimes there are duplicate job_ids, why?
try:
q_pending = len(set(self._queue.job_ids)) if self._queue else 0
q_running = len(self._wip) if self._wip else 0
q_to_write = len(self._finished) if self._finished else 0
q_failed = len(self._queue.failed_job_registry
) if self._queue else self._status['failed']
q_finished = self._status['finished']
q_total = q_pending + q_running + q_failed + q_to_write + q_finished
q_workers = len(ResilientWorker.all(
queue=self._queue)) if self._queue else 0
self._status['pending'] = q_pending
self._status['running'] = q_running + q_to_write
self._status['failed'] = q_failed
self._status['finished'] = q_finished
self._status['total'] = q_total
self._status['workers'] = q_workers
self._status['done'] = q_finished / q_total * 100
if sum([q_pending, q_running, q_to_write]) == 0:
self._mark_complete()
elif sum([q_pending, q_workers, q_to_write]) == 0:
self._logger.info(
f'Queue {self.name} ran out of workers with running jobs. Marking finished.'
)
self._mark_complete()
except redis.connection.ConnectionError:
self._sleep_on_it()
self._update_status()
else:
self._mark_failed()
def _get_finished_jobs(self):
if self._retries:
try:
return self._finished.get_job_ids()
except redis.exceptions.ConnectionError:
self._sleep_on_it()
return self._get_finished_jobs()
else:
self._mark_failed()
return []
def _get_result(self, job_id):
job = self._get_job(job_id)
result = None
if job is not None:
start = time.time()
result = pd.read_msgpack(job.result[0])
end = time.time()
self._logger.debug(
f'Read {job_id} from msgpack from queue {self.name} in {end - start:.2f}s.'
)
self._status['finished'] += 1
self._remove_finished_job(job)
return result
def _get_job(self, job_id):
if self._retries:
try:
start = time.time()
job = self._queue.fetch_job(job_id)
end = time.time()
self._logger.debug(
f'Fetched job {job_id} from queue {self.name} in {end - start:.2f}s'
)
return job
except redis.exceptions.ConnectionError:
self._sleep_on_it()
return self._get_job(job_id)
else:
self._mark_failed()
return None
def _remove_finished_job(self, job):
if self._retries:
try:
self._finished.remove(job)
except redis.exceptions.ConnectionError:
self._sleep_on_it()
self._remove_finished_job(job)
else:
self._mark_failed()
def _sleep_on_it(self):
backoff = self._backoff * random.random()
self._logger.warning(
f'Failed to connect to redis for queue {self.name}. Retrying in {backoff}s.'
)
self._retries -= 1
time.sleep(backoff)
def _mark_failed(self):
# TODO: Probably should cleanup redis stuff, but not sure how yet.
if not (self.failed or self.completed):
self._logger.warning(
f'Queue {self.name} has run out of retries. Marking as failed.'
)
self._status[
'failed'] += self._status['pending'] + self._status['running']
self._status['pending'] = 0
self._status['running'] = 0
self._status['workers'] = 0
self._failed = True
def _mark_complete(self):
if not (self.failed or self.completed):
self._logger.info(f'All jobs on queue {self.name} complete.')
self._status['finished'] += self._status['pending'] + self._status[
'running']
self._status['pending'] = 0
self._status['running'] = 0
self._status['workers'] = 0
self._completed = True
def list_all_jobs():
registry = FinishedJobRegistry(os.environ["RQ_QUEUE"], connection=redis)
print(f"{len(registry.get_job_ids())} record(s) succesfully saved to './output/data.json'.")
return Job.fetch_many(registry.get_job_ids(), connection=redis)
def GetJobProgress(app):
"""Infinite thread that sends job updates to client."""
with app.app_context():
redis = Redis()
queue = Queue(connection=redis)
running_registry = StartedJobRegistry(queue=queue)
finished_registry = FinishedJobRegistry(queue=queue)
showing_running_job_bar = False
toggle_on_running_job_bar = False
toggle_off_running_job_bar = False
while True:
time.sleep(2)
running_msg = "Running jobs:"
finished_msg = ""
running_job_id_list = running_registry.get_job_ids()
finished_job_id_list = finished_registry.get_job_ids()
if running_job_id_list:
# Get all job progress and emit update
for job_id in running_job_id_list:
running_msg += "<br>Job ID: {}".format(job_id)
running_job = Job.fetch(id=job_id, connection=redis)
if 'process' in running_job.meta:
running_msg += "<UL><LI>Process: {}".format(
running_job.meta['process'])
if 'progress' in running_job.meta:
running_msg += "<LI>Progress: {}</UL>".format(
running_job.meta['progress'])
socketio.emit('update job progress', {'data': running_msg})
toggle_on_running_job_bar = True
else:
toggle_off_running_job_bar = True
if finished_job_id_list:
# Display finished job notification and save to db
for job_id in finished_job_id_list:
finished_msg += "<br>Job ID: {}".format(job_id)
finished_job = Job.fetch(job_id, connection=redis)
if 'process' in finished_job.meta:
finished_msg += "<UL><LI>Process: {}".format(
finished_job.meta['process'])
rq_job = RQJob()
rq_job.save_job(finished_job)
finished_registry.remove(finished_job)
finished_msg += "<LI>Result: {}".format(
rq_job.get_result())
finished_msg += "<LI>Time elapsed: {} seconds</UL>".format(
rq_job.get_elapsed_time())
socketio.emit('update job finished', {'data': finished_msg})
socketio.emit('show finished job bar')
if showing_running_job_bar and toggle_off_running_job_bar:
toggle_off_running_job_bar = False
toggle_on_running_job_bar = False
showing_running_job_bar = False
print('---------Server: Hide running job bar')
socketio.emit('hide running job bar')
elif not showing_running_job_bar and toggle_on_running_job_bar:
toggle_off_running_job_bar = False
toggle_on_running_job_bar = False
showing_running_job_bar = True
print('---------Server: Show running job bar')
socketio.emit('show running job bar')
return
def clear_registry():
registry = FinishedJobRegistry(os.environ["RQ_QUEUE"], connection=redis)
for job_id in registry.get_job_ids():
registry.remove(job_id)
