def getstartedjobs(self, q):
"""returns list of started redis jobs"""
log.info(f"getting started jobs: {q}")
try:
registry = StartedJobRegistry(q, connection=self.base_connection)
response_object = registry.get_job_ids()
return response_object
except Exception as e:
return e
def test_clean_queue_registries(self):
"""worker.clean_registries sets last_cleaned_at and cleans registries."""
foo_queue = Queue('foo', connection=self.testconn)
foo_registry = StartedJobRegistry('foo', connection=self.testconn)
self.testconn.zadd(foo_registry.key, 1, 'foo')
self.assertEqual(self.testconn.zcard(foo_registry.key), 1)
bar_queue = Queue('bar', connection=self.testconn)
bar_registry = StartedJobRegistry('bar', connection=self.testconn)
self.testconn.zadd(bar_registry.key, 1, 'bar')
self.assertEqual(self.testconn.zcard(bar_registry.key), 1)
worker = Worker([foo_queue, bar_queue])
self.assertEqual(worker.last_cleaned_at, None)
worker.clean_registries()
self.assertNotEqual(worker.last_cleaned_at, None)
self.assertEqual(self.testconn.zcard(foo_registry.key), 0)
self.assertEqual(self.testconn.zcard(bar_registry.key), 0)
def _remove_jobs(self, project):
started_registry = StartedJobRegistry(self._redis_queue.name, self._redis_queue.connection)
finished_registry = FinishedJobRegistry(self._redis_queue.name, self._redis_queue.connection)
for job_id in started_registry.get_job_ids() + finished_registry.get_job_ids():
job = Job.fetch(job_id, connection=self._redis)
if job.meta.get('project') != project:
continue
logger.info(f'Deleting job_id {job_id}')
job.delete()
def test_worker_calls_clean_registries(self):
"""Worker calls clean_registries when run."""
queue = Queue(connection=self.testconn)
registry = StartedJobRegistry(connection=self.testconn)
self.testconn.zadd(registry.key, 1, 'foo')
worker = Worker(queue, connection=self.testconn)
worker.work(burst=True)
self.assertEqual(self.testconn.zcard(registry.key), 0)
def getRunningExperiments(cls):
"""Returns experiments currently being run
Returns:
jobs(list): list of jobs that are being run
"""
with Connection(redis.from_url(current_app.config['REDIS_URL'])) as conn:
registry = StartedJobRegistry('default', connection=conn)
return [Job.fetch(id, connection=conn) for id in registry.get_job_ids()]
def rq_queues():
return {
q.name: {
"name": q.name,
"started": fetch_jobs(StartedJobRegistry(queue=q).get_job_ids()),
"queued": len(q.job_ids),
}
for q in sorted(Queue.all(), key=lambda q: q.name)
}
def get_statistics():
queues = []
workers_collections = collect_workers_by_connection(QUEUES_LIST)
for index, config in enumerate(QUEUES_LIST):
queue = get_queue_by_index(index)
connection = queue.connection
connection_kwargs = connection.connection_pool.connection_kwargs
# Raw access to the first item from left of the redis list.
# This might not be accurate since new job can be added from the left
# with `at_front` parameters.
# Ideally rq should supports Queue.oldest_job
last_job_id = connection.lindex(queue.key, 0)
last_job = queue.fetch_job(last_job_id.decode('utf-8')) if last_job_id else None
if last_job:
oldest_job_timestamp = to_localtime(last_job.enqueued_at)\
.strftime('%Y-%m-%d, %H:%M:%S')
else:
oldest_job_timestamp = "-"
# parse_class and connection_pool are not needed and not JSON serializable
connection_kwargs.pop('parser_class', None)
connection_kwargs.pop('connection_pool', None)
queue_data = {
'name': queue.name,
'jobs': queue.count,
'oldest_job_timestamp': oldest_job_timestamp,
'index': index,
'connection_kwargs': connection_kwargs
}
if queue.name == 'failed':
queue_data['workers'] = '-'
queue_data['finished_jobs'] = '-'
queue_data['started_jobs'] = '-'
queue_data['deferred_jobs'] = '-'
else:
connection = get_connection(queue.name)
all_workers = get_all_workers_by_configuration(
config['connection_config'],
workers_collections
)
queue_workers = [worker for worker in all_workers if queue in worker.queues]
queue_data['workers'] = len(queue_workers)
finished_job_registry = FinishedJobRegistry(queue.name, connection)
started_job_registry = StartedJobRegistry(queue.name, connection)
deferred_job_registry = DeferredJobRegistry(queue.name, connection)
queue_data['finished_jobs'] = len(finished_job_registry)
queue_data['started_jobs'] = len(started_job_registry)
queue_data['deferred_jobs'] = len(deferred_job_registry)
queues.append(queue_data)
return {'queues': queues}
def rq_queues():
return {
q.name: {
'name': q.name,
'started': fetch_jobs(q,
StartedJobRegistry(queue=q).get_job_ids()),
'queued': len(q.job_ids)
}
for q in Queue.all(connection=redis_connection)
}
def get(self):
pending = queue.get_job_ids()
registry = StartedJobRegistry('default', connection=broker)
started = registry.get_job_ids()
fail_queue = FailedJobRegistry(connection=broker)
failed = fail_queue.get_job_ids()
return {"jobs": started + pending, "failed": failed}
def pending_or_running_jobs(queue):
"""
Chequea si hay trabajos encolados o corriendo, en la cola
pasada por parámetro
"""
rq_queue = get_queue(queue)
pending = bool(rq_queue.jobs)
registry = StartedJobRegistry(name=queue, connection=get_connection(queue))
running = bool(len(registry))
return pending or running
def rq_queues():
return {
q.name: {
"name": q.name,
"started": fetch_jobs(q,
StartedJobRegistry(queue=q).get_job_ids()),
"queued": len(q.job_ids),
}
for q in Queue.all()
}
def test_get_expiration_time(self):
"""registry.get_expiration_time() returns correct datetime objects"""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
registry.add(job, 5)
self.assertEqual(registry.get_expiration_time(job),
(datetime.utcnow() +
timedelta(seconds=5)).replace(microsecond=0))
def perform_job(self, job, queue, heartbeat_ttl=None):
"""Performs the actual work of a job. Will/should only be called
inside the work horse's process.
"""
self.prepare_job_execution(job, heartbeat_ttl)
push_connection(self.connection)
started_job_registry = StartedJobRegistry(job.origin,
self.connection,
job_class=self.job_class)
try:
job.started_at = utcnow()
timeout = job.timeout or self.queue_class.DEFAULT_TIMEOUT
with self.death_penalty_class(timeout, JobTimeoutException, job_id=job.id):
rv = job.perform(self.workerKwargs)
job.ended_at = utcnow()
# Pickle the result in the same try-except block since we need
# to use the same exc handling when pickling fails
job._result = rv
self.handle_job_success(job=job,
queue=queue,
started_job_registry=started_job_registry)
except:
job.ended_at = utcnow()
self.handle_job_failure(job=job,
started_job_registry=started_job_registry)
self.handle_exception(job, *sys.exc_info())
return False
finally:
pop_connection()
self.log.info('{0}: {1} ({2})'.format(
green(job.origin), blue('Job OK'), job.id))
if rv is not None:
log_result = "{0!r}".format(as_text(text_type(rv)))
self.log.debug('Result: %s', yellow(log_result))
if self.log_result_lifespan:
result_ttl = job.get_result_ttl(self.default_result_ttl)
if result_ttl == 0:
self.log.info('Result discarded immediately')
elif result_ttl > 0:
self.log.info(
'Result is kept for {0} seconds'.format(result_ttl))
else:
self.log.warning(
'Result will never expire, clean up result key manually')
return True
def request_fresh_data():
registry = StartedJobRegistry(settings.QUEUE_NAME, connection=redis_conn)
running_job_ids = registry.get_job_ids()
if len(running_job_ids) == 0: # no job currently running
job = queue.enqueue(get_fresh_movie_data)
job_id = job.id
else:
job_id = running_job_ids.pop()
return jsonify(job_id=job_id)
def test_add_and_remove_with_serializer(self):
"""Adding and removing job to StartedJobRegistry (with serializer)."""
# delete_job = True also works with job.id and custom serializer
queue = Queue(connection=self.testconn, serializer=JSONSerializer)
registry = StartedJobRegistry(connection=self.testconn,
serializer=JSONSerializer)
job = queue.enqueue(say_hello)
registry.add(job, -1)
registry.remove(job.id, delete_job=True)
self.assertIsNone(self.testconn.zscore(registry.key, job.id))
self.assertFalse(self.testconn.exists(job.key))
def test_started_jobs(self):
"""Ensure that active jobs page works properly."""
queue = get_queue('django_rq_test')
queue_index = get_queue_index('django_rq_test')
job = queue.enqueue(access_self)
registry = StartedJobRegistry(queue.name, queue.connection)
registry.add(job, 2)
response = self.client.get(
reverse('rq_started_jobs', args=[queue_index]))
self.assertEqual(response.context['jobs'], [job])
def jobs():
from rq.registry import BaseRegistry, StartedJobRegistry, FinishedJobRegistry, FailedJobRegistry
BaseRegistry.get_jobs = lambda s: [
app.queue.fetch_job(job_id) for job_id in s.get_job_ids()
]
return {
'queued': app.queue.get_jobs(),
'started': StartedJobRegistry(queue=app.queue).get_jobs(),
'finished': FinishedJobRegistry(queue=app.queue).get_jobs(),
'failed': FailedJobRegistry(queue=app.queue).get_jobs(),
}
def get(self):
registry = StartedJobRegistry(queue=queue, connection=r_connection)
running_job_ids = registry.get_job_ids(
) # Jobs which are exactly running.
expired_job_ids = registry.get_expired_job_ids()
response = jsonify({
'running': running_job_ids,
'expired': expired_job_ids
})
response.status_code = 200
return response
def handle(self, *args, **options):
conn = redis.StrictRedis()
if options["job_id"]:
self.stop_job(options["job_id"][0], conn)
else:
started_jobs = StartedJobRegistry("job_queue", connection=conn)
for job_id in started_jobs.get_job_ids():
self.stop_job(job_id, conn)
def quantity(self, **kwargs):
"""
Returns the aggregated number of tasks of the proc queues.
"""
count = 0
# Total count should be what's queued plus the started jobs.
for queue in self.clients:
registry = StartedJobRegistry(queue.name, queue.connection)
count += (queue.count + len(registry))
return count
def test_contains(self):
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
self.assertFalse(job in registry)
self.assertFalse(job.id in registry)
registry.add(job, 5)
self.assertTrue(job in registry)
self.assertTrue(job.id in registry)
def test_get_expiration_time(self):
"""registry.get_expiration_time() returns correct datetime objects"""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
registry.add(job, 5)
time = registry.get_expiration_time(job)
expected_time = (datetime.utcnow() +
timedelta(seconds=5)).replace(microsecond=0)
self.assertGreaterEqual(time, expected_time - timedelta(seconds=2))
self.assertLessEqual(time, expected_time + timedelta(seconds=2))
def active_analyses():
"""
Returns a list of job IDs corresponding to active (pending or running)
analysis tasks.
"""
pending_jobs = Job.fetch_many(binary_analysis.job_ids, connection=redis)
pending_ids = [
job.id for job in pending_jobs if job.func_name != "analyze_binary"
]
started = StartedJobRegistry("binary_analysis", connection=redis)
return pending_ids + started.get_job_ids()
def get_statistics(run_maintenance_tasks=False):
queues = []
for index, config in enumerate(QUEUES_LIST):
queue = get_queue_by_index(index)
connection = queue.connection
connection_kwargs = connection.connection_pool.connection_kwargs
if run_maintenance_tasks:
clean_registries(queue)
clean_worker_registry(queue)
# Raw access to the first item from left of the redis list.
# This might not be accurate since new job can be added from the left
# with `at_front` parameters.
# Ideally rq should supports Queue.oldest_job
last_job_id = connection.lindex(queue.key, 0)
last_job = queue.fetch_job(
last_job_id.decode('utf-8')) if last_job_id else None
if last_job:
oldest_job_timestamp = to_localtime(last_job.enqueued_at)\
.strftime('%Y-%m-%d, %H:%M:%S')
else:
oldest_job_timestamp = "-"
# parse_class and connection_pool are not needed and not JSON serializable
connection_kwargs.pop('parser_class', None)
connection_kwargs.pop('connection_pool', None)
queue_data = {
'name': queue.name,
'jobs': queue.count,
'oldest_job_timestamp': oldest_job_timestamp,
'index': index,
'connection_kwargs': connection_kwargs
}
connection = get_connection(queue.name)
queue_data['workers'] = Worker.count(queue=queue)
finished_job_registry = FinishedJobRegistry(queue.name, connection)
started_job_registry = StartedJobRegistry(queue.name, connection)
deferred_job_registry = DeferredJobRegistry(queue.name, connection)
failed_job_registry = FailedJobRegistry(queue.name, connection)
scheduled_job_registry = ScheduledJobRegistry(queue.name, connection)
queue_data['finished_jobs'] = len(finished_job_registry)
queue_data['started_jobs'] = len(started_job_registry)
queue_data['deferred_jobs'] = len(deferred_job_registry)
queue_data['failed_jobs'] = len(failed_job_registry)
queue_data['scheduled_jobs'] = len(scheduled_job_registry)
queues.append(queue_data)
return {'queues': queues}
def info(request, token):
"""
Return the HireFire json data needed to scale worker dynos
"""
if not settings.HIREFIRE_TOKEN:
return HttpResponseBadRequest('Hirefire not configured. Set the HIREFIRE_TOKEN environment variable on the app to use Hirefire for dyno scaling')
if token != settings.HIREFIRE_TOKEN:
return HttpResponseForbidden('Invalid token')
current_tasks = 0
queues = []
for index, config in enumerate(QUEUES_LIST):
queue = get_queue_by_index(index)
connection = queue.connection
# Only look at the default queue
if queue.name != 'default':
continue
queue_data = {
'name': queue.name,
'jobs': queue.count,
'index': index,
'connection_kwargs': connection.connection_pool.connection_kwargs
}
connection = get_connection(queue.name)
all_workers = Worker.all(connection=connection)
queue_workers = [worker for worker in all_workers if queue in worker.queues]
queue_data['workers'] = len(queue_workers)
finished_job_registry = FinishedJobRegistry(queue.name, connection)
started_job_registry = StartedJobRegistry(queue.name, connection)
deferred_job_registry = DeferredJobRegistry(queue.name, connection)
queue_data['finished_jobs'] = len(finished_job_registry)
queue_data['started_jobs'] = len(started_job_registry)
queue_data['deferred_jobs'] = len(deferred_job_registry)
current_tasks += queue_data['jobs']
current_tasks += queue_data['started_jobs']
queues.append(queue_data)
payload = [{
'quantity': current_tasks,
'name': 'worker',
}]
payload = json.dumps(payload)
return HttpResponse(payload, content_type='application/json')
async def get_jobs(md5_hashes: List[str]):
"""
Returns the prediction status for a list of jobs, specified by md5 hash.
:param md5_hashes: List of image md5 hashes
:return: Array of image prediction results.
"""
results = []
if not md5_hashes:
return []
for md5_hash in md5_hashes:
# If there are any pending predictions, alert user and return existing ones
# Since job_id is a composite hash+model, we must loop and find all jobs that have the
# hash we want to find. We must get all running and pending jobs to return the correct value
all_jobs = StartedJobRegistry(
'model_prediction',
connection=redis).get_job_ids() + prediction_queue.job_ids
image = get_image_by_md5_hash_db(md5_hash) # Get image object
found_pending_job = False
for job_id in all_jobs:
if md5_hash in job_id and Job.fetch(
job_id, connection=redis).get_status() != 'finished':
found_pending_job = True
results.append({
'status': 'success',
'detail':
'Image has pending predictions. Check back later for all model results.',
**image.dict()
})
break # Don't look for more jobs since we have found one that is pending
# If we have found a job that is pending, then move on to next image
if found_pending_job:
continue
# If we haven't found a pending job for this image, and it doesn't exist in our database, then that
# means that the image hash must be invalid.
if not image:
results.append({
'status': 'failure',
'detail': 'Unknown md5 hash specified.',
'hash_md5': md5_hash
})
continue
# If everything is successful with image, return data
results.append({'status': 'success', **image.dict()})
return results
def serialize_queues(instance_number, queues):
return [
dict(
name=q.name,
count=q.count,
queued_url=url_for(
".jobs_overview",
instance_number=instance_number,
queue_name=q.name,
registry_name="queued",
per_page="8",
page="1",
),
failed_job_registry_count=FailedJobRegistry(q.name).count,
failed_url=url_for(
".jobs_overview",
instance_number=instance_number,
queue_name=q.name,
registry_name="failed",
per_page="8",
page="1",
),
started_job_registry_count=StartedJobRegistry(q.name).count,
started_url=url_for(
".jobs_overview",
instance_number=instance_number,
queue_name=q.name,
registry_name="started",
per_page="8",
page="1",
),
deferred_job_registry_count=DeferredJobRegistry(q.name).count,
deferred_url=url_for(
".jobs_overview",
instance_number=instance_number,
queue_name=q.name,
registry_name="deferred",
per_page="8",
page="1",
),
finished_job_registry_count=FinishedJobRegistry(q.name).count,
finished_url=url_for(
".jobs_overview",
instance_number=instance_number,
queue_name=q.name,
registry_name="finished",
per_page="8",
page="1",
),
)
for q in queues
]
def _remove_jobs(cls, project):
started_registry = StartedJobRegistry(
cls._redis_queue.name, cls._redis_queue.connection
)
finished_registry = FinishedJobRegistry(
cls._redis_queue.name, cls._redis_queue.connection
)
for job_id in started_registry.get_job_ids() + finished_registry.get_job_ids():
job = Job.fetch(job_id, connection=cls._redis)
if job.meta.get("project") != project:
continue
logger.info("Deleting job_id " + job_id)
job.delete()
def test_getting_registries(self):
"""Getting job registries from queue object"""
queue = Queue('example')
self.assertEqual(queue.scheduled_job_registry,
ScheduledJobRegistry(queue=queue))
self.assertEqual(queue.started_job_registry,
StartedJobRegistry(queue=queue))
self.assertEqual(queue.failed_job_registry,
FailedJobRegistry(queue=queue))
self.assertEqual(queue.deferred_job_registry,
DeferredJobRegistry(queue=queue))
self.assertEqual(queue.finished_job_registry,
FinishedJobRegistry(queue=queue))
def main():
with Connection(conn):
# queue = Queue(connection=conn)
# for job_id in queue.job_ids:
# queue.fetch_job(job_id)
# if queue:
#
# queue.fetch
queue = Queue(connection=conn)
registry = StartedJobRegistry(queue=queue)
pdb.set_trace()
worker = Worker(list(map(Queue, listen)))
worker.work()
