def getstartedjobs(self, q): """returns list of started redis jobs""" log.info(f"getting started jobs: {q}") try: registry = StartedJobRegistry(q, connection=self.base_connection) response_object = registry.get_job_ids() return response_object except Exception as e: return e
def test_clean_queue_registries(self): """worker.clean_registries sets last_cleaned_at and cleans registries.""" foo_queue = Queue('foo', connection=self.testconn) foo_registry = StartedJobRegistry('foo', connection=self.testconn) self.testconn.zadd(foo_registry.key, 1, 'foo') self.assertEqual(self.testconn.zcard(foo_registry.key), 1) bar_queue = Queue('bar', connection=self.testconn) bar_registry = StartedJobRegistry('bar', connection=self.testconn) self.testconn.zadd(bar_registry.key, 1, 'bar') self.assertEqual(self.testconn.zcard(bar_registry.key), 1) worker = Worker([foo_queue, bar_queue]) self.assertEqual(worker.last_cleaned_at, None) worker.clean_registries() self.assertNotEqual(worker.last_cleaned_at, None) self.assertEqual(self.testconn.zcard(foo_registry.key), 0) self.assertEqual(self.testconn.zcard(bar_registry.key), 0)
def _remove_jobs(self, project): started_registry = StartedJobRegistry(self._redis_queue.name, self._redis_queue.connection) finished_registry = FinishedJobRegistry(self._redis_queue.name, self._redis_queue.connection) for job_id in started_registry.get_job_ids() + finished_registry.get_job_ids(): job = Job.fetch(job_id, connection=self._redis) if job.meta.get('project') != project: continue logger.info(f'Deleting job_id {job_id}') job.delete()
def test_worker_calls_clean_registries(self): """Worker calls clean_registries when run.""" queue = Queue(connection=self.testconn) registry = StartedJobRegistry(connection=self.testconn) self.testconn.zadd(registry.key, 1, 'foo') worker = Worker(queue, connection=self.testconn) worker.work(burst=True) self.assertEqual(self.testconn.zcard(registry.key), 0)
def getRunningExperiments(cls): """Returns experiments currently being run Returns: jobs(list): list of jobs that are being run """ with Connection(redis.from_url(current_app.config['REDIS_URL'])) as conn: registry = StartedJobRegistry('default', connection=conn) return [Job.fetch(id, connection=conn) for id in registry.get_job_ids()]
def rq_queues(): return { q.name: { "name": q.name, "started": fetch_jobs(StartedJobRegistry(queue=q).get_job_ids()), "queued": len(q.job_ids), } for q in sorted(Queue.all(), key=lambda q: q.name) }
def get_statistics(): queues = [] workers_collections = collect_workers_by_connection(QUEUES_LIST) for index, config in enumerate(QUEUES_LIST): queue = get_queue_by_index(index) connection = queue.connection connection_kwargs = connection.connection_pool.connection_kwargs # Raw access to the first item from left of the redis list. # This might not be accurate since new job can be added from the left # with `at_front` parameters. # Ideally rq should supports Queue.oldest_job last_job_id = connection.lindex(queue.key, 0) last_job = queue.fetch_job(last_job_id.decode('utf-8')) if last_job_id else None if last_job: oldest_job_timestamp = to_localtime(last_job.enqueued_at)\ .strftime('%Y-%m-%d, %H:%M:%S') else: oldest_job_timestamp = "-" # parse_class and connection_pool are not needed and not JSON serializable connection_kwargs.pop('parser_class', None) connection_kwargs.pop('connection_pool', None) queue_data = { 'name': queue.name, 'jobs': queue.count, 'oldest_job_timestamp': oldest_job_timestamp, 'index': index, 'connection_kwargs': connection_kwargs } if queue.name == 'failed': queue_data['workers'] = '-' queue_data['finished_jobs'] = '-' queue_data['started_jobs'] = '-' queue_data['deferred_jobs'] = '-' else: connection = get_connection(queue.name) all_workers = get_all_workers_by_configuration( config['connection_config'], workers_collections ) queue_workers = [worker for worker in all_workers if queue in worker.queues] queue_data['workers'] = len(queue_workers) finished_job_registry = FinishedJobRegistry(queue.name, connection) started_job_registry = StartedJobRegistry(queue.name, connection) deferred_job_registry = DeferredJobRegistry(queue.name, connection) queue_data['finished_jobs'] = len(finished_job_registry) queue_data['started_jobs'] = len(started_job_registry) queue_data['deferred_jobs'] = len(deferred_job_registry) queues.append(queue_data) return {'queues': queues}
def rq_queues(): return { q.name: { 'name': q.name, 'started': fetch_jobs(q, StartedJobRegistry(queue=q).get_job_ids()), 'queued': len(q.job_ids) } for q in Queue.all(connection=redis_connection) }
def get(self): pending = queue.get_job_ids() registry = StartedJobRegistry('default', connection=broker) started = registry.get_job_ids() fail_queue = FailedJobRegistry(connection=broker) failed = fail_queue.get_job_ids() return {"jobs": started + pending, "failed": failed}
def pending_or_running_jobs(queue): """ Chequea si hay trabajos encolados o corriendo, en la cola pasada por parámetro """ rq_queue = get_queue(queue) pending = bool(rq_queue.jobs) registry = StartedJobRegistry(name=queue, connection=get_connection(queue)) running = bool(len(registry)) return pending or running
def rq_queues(): return { q.name: { "name": q.name, "started": fetch_jobs(q, StartedJobRegistry(queue=q).get_job_ids()), "queued": len(q.job_ids), } for q in Queue.all() }
def test_get_expiration_time(self): """registry.get_expiration_time() returns correct datetime objects""" registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) registry.add(job, 5) self.assertEqual(registry.get_expiration_time(job), (datetime.utcnow() + timedelta(seconds=5)).replace(microsecond=0))
def perform_job(self, job, queue, heartbeat_ttl=None): """Performs the actual work of a job. Will/should only be called inside the work horse's process. """ self.prepare_job_execution(job, heartbeat_ttl) push_connection(self.connection) started_job_registry = StartedJobRegistry(job.origin, self.connection, job_class=self.job_class) try: job.started_at = utcnow() timeout = job.timeout or self.queue_class.DEFAULT_TIMEOUT with self.death_penalty_class(timeout, JobTimeoutException, job_id=job.id): rv = job.perform(self.workerKwargs) job.ended_at = utcnow() # Pickle the result in the same try-except block since we need # to use the same exc handling when pickling fails job._result = rv self.handle_job_success(job=job, queue=queue, started_job_registry=started_job_registry) except: job.ended_at = utcnow() self.handle_job_failure(job=job, started_job_registry=started_job_registry) self.handle_exception(job, *sys.exc_info()) return False finally: pop_connection() self.log.info('{0}: {1} ({2})'.format( green(job.origin), blue('Job OK'), job.id)) if rv is not None: log_result = "{0!r}".format(as_text(text_type(rv))) self.log.debug('Result: %s', yellow(log_result)) if self.log_result_lifespan: result_ttl = job.get_result_ttl(self.default_result_ttl) if result_ttl == 0: self.log.info('Result discarded immediately') elif result_ttl > 0: self.log.info( 'Result is kept for {0} seconds'.format(result_ttl)) else: self.log.warning( 'Result will never expire, clean up result key manually') return True
def request_fresh_data(): registry = StartedJobRegistry(settings.QUEUE_NAME, connection=redis_conn) running_job_ids = registry.get_job_ids() if len(running_job_ids) == 0: # no job currently running job = queue.enqueue(get_fresh_movie_data) job_id = job.id else: job_id = running_job_ids.pop() return jsonify(job_id=job_id)
def test_add_and_remove_with_serializer(self): """Adding and removing job to StartedJobRegistry (with serializer).""" # delete_job = True also works with job.id and custom serializer queue = Queue(connection=self.testconn, serializer=JSONSerializer) registry = StartedJobRegistry(connection=self.testconn, serializer=JSONSerializer) job = queue.enqueue(say_hello) registry.add(job, -1) registry.remove(job.id, delete_job=True) self.assertIsNone(self.testconn.zscore(registry.key, job.id)) self.assertFalse(self.testconn.exists(job.key))
def test_started_jobs(self): """Ensure that active jobs page works properly.""" queue = get_queue('django_rq_test') queue_index = get_queue_index('django_rq_test') job = queue.enqueue(access_self) registry = StartedJobRegistry(queue.name, queue.connection) registry.add(job, 2) response = self.client.get( reverse('rq_started_jobs', args=[queue_index])) self.assertEqual(response.context['jobs'], [job])
def jobs(): from rq.registry import BaseRegistry, StartedJobRegistry, FinishedJobRegistry, FailedJobRegistry BaseRegistry.get_jobs = lambda s: [ app.queue.fetch_job(job_id) for job_id in s.get_job_ids() ] return { 'queued': app.queue.get_jobs(), 'started': StartedJobRegistry(queue=app.queue).get_jobs(), 'finished': FinishedJobRegistry(queue=app.queue).get_jobs(), 'failed': FailedJobRegistry(queue=app.queue).get_jobs(), }
def get(self): registry = StartedJobRegistry(queue=queue, connection=r_connection) running_job_ids = registry.get_job_ids( ) # Jobs which are exactly running. expired_job_ids = registry.get_expired_job_ids() response = jsonify({ 'running': running_job_ids, 'expired': expired_job_ids }) response.status_code = 200 return response
def handle(self, *args, **options): conn = redis.StrictRedis() if options["job_id"]: self.stop_job(options["job_id"][0], conn) else: started_jobs = StartedJobRegistry("job_queue", connection=conn) for job_id in started_jobs.get_job_ids(): self.stop_job(job_id, conn)
def quantity(self, **kwargs): """ Returns the aggregated number of tasks of the proc queues. """ count = 0 # Total count should be what's queued plus the started jobs. for queue in self.clients: registry = StartedJobRegistry(queue.name, queue.connection) count += (queue.count + len(registry)) return count
def test_contains(self): registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) self.assertFalse(job in registry) self.assertFalse(job.id in registry) registry.add(job, 5) self.assertTrue(job in registry) self.assertTrue(job.id in registry)
def test_get_expiration_time(self): """registry.get_expiration_time() returns correct datetime objects""" registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) registry.add(job, 5) time = registry.get_expiration_time(job) expected_time = (datetime.utcnow() + timedelta(seconds=5)).replace(microsecond=0) self.assertGreaterEqual(time, expected_time - timedelta(seconds=2)) self.assertLessEqual(time, expected_time + timedelta(seconds=2))
def active_analyses(): """ Returns a list of job IDs corresponding to active (pending or running) analysis tasks. """ pending_jobs = Job.fetch_many(binary_analysis.job_ids, connection=redis) pending_ids = [ job.id for job in pending_jobs if job.func_name != "analyze_binary" ] started = StartedJobRegistry("binary_analysis", connection=redis) return pending_ids + started.get_job_ids()
def get_statistics(run_maintenance_tasks=False): queues = [] for index, config in enumerate(QUEUES_LIST): queue = get_queue_by_index(index) connection = queue.connection connection_kwargs = connection.connection_pool.connection_kwargs if run_maintenance_tasks: clean_registries(queue) clean_worker_registry(queue) # Raw access to the first item from left of the redis list. # This might not be accurate since new job can be added from the left # with `at_front` parameters. # Ideally rq should supports Queue.oldest_job last_job_id = connection.lindex(queue.key, 0) last_job = queue.fetch_job( last_job_id.decode('utf-8')) if last_job_id else None if last_job: oldest_job_timestamp = to_localtime(last_job.enqueued_at)\ .strftime('%Y-%m-%d, %H:%M:%S') else: oldest_job_timestamp = "-" # parse_class and connection_pool are not needed and not JSON serializable connection_kwargs.pop('parser_class', None) connection_kwargs.pop('connection_pool', None) queue_data = { 'name': queue.name, 'jobs': queue.count, 'oldest_job_timestamp': oldest_job_timestamp, 'index': index, 'connection_kwargs': connection_kwargs } connection = get_connection(queue.name) queue_data['workers'] = Worker.count(queue=queue) finished_job_registry = FinishedJobRegistry(queue.name, connection) started_job_registry = StartedJobRegistry(queue.name, connection) deferred_job_registry = DeferredJobRegistry(queue.name, connection) failed_job_registry = FailedJobRegistry(queue.name, connection) scheduled_job_registry = ScheduledJobRegistry(queue.name, connection) queue_data['finished_jobs'] = len(finished_job_registry) queue_data['started_jobs'] = len(started_job_registry) queue_data['deferred_jobs'] = len(deferred_job_registry) queue_data['failed_jobs'] = len(failed_job_registry) queue_data['scheduled_jobs'] = len(scheduled_job_registry) queues.append(queue_data) return {'queues': queues}
def info(request, token): """ Return the HireFire json data needed to scale worker dynos """ if not settings.HIREFIRE_TOKEN: return HttpResponseBadRequest('Hirefire not configured. Set the HIREFIRE_TOKEN environment variable on the app to use Hirefire for dyno scaling') if token != settings.HIREFIRE_TOKEN: return HttpResponseForbidden('Invalid token') current_tasks = 0 queues = [] for index, config in enumerate(QUEUES_LIST): queue = get_queue_by_index(index) connection = queue.connection # Only look at the default queue if queue.name != 'default': continue queue_data = { 'name': queue.name, 'jobs': queue.count, 'index': index, 'connection_kwargs': connection.connection_pool.connection_kwargs } connection = get_connection(queue.name) all_workers = Worker.all(connection=connection) queue_workers = [worker for worker in all_workers if queue in worker.queues] queue_data['workers'] = len(queue_workers) finished_job_registry = FinishedJobRegistry(queue.name, connection) started_job_registry = StartedJobRegistry(queue.name, connection) deferred_job_registry = DeferredJobRegistry(queue.name, connection) queue_data['finished_jobs'] = len(finished_job_registry) queue_data['started_jobs'] = len(started_job_registry) queue_data['deferred_jobs'] = len(deferred_job_registry) current_tasks += queue_data['jobs'] current_tasks += queue_data['started_jobs'] queues.append(queue_data) payload = [{ 'quantity': current_tasks, 'name': 'worker', }] payload = json.dumps(payload) return HttpResponse(payload, content_type='application/json')
async def get_jobs(md5_hashes: List[str]): """ Returns the prediction status for a list of jobs, specified by md5 hash. :param md5_hashes: List of image md5 hashes :return: Array of image prediction results. """ results = [] if not md5_hashes: return [] for md5_hash in md5_hashes: # If there are any pending predictions, alert user and return existing ones # Since job_id is a composite hash+model, we must loop and find all jobs that have the # hash we want to find. We must get all running and pending jobs to return the correct value all_jobs = StartedJobRegistry( 'model_prediction', connection=redis).get_job_ids() + prediction_queue.job_ids image = get_image_by_md5_hash_db(md5_hash) # Get image object found_pending_job = False for job_id in all_jobs: if md5_hash in job_id and Job.fetch( job_id, connection=redis).get_status() != 'finished': found_pending_job = True results.append({ 'status': 'success', 'detail': 'Image has pending predictions. Check back later for all model results.', **image.dict() }) break # Don't look for more jobs since we have found one that is pending # If we have found a job that is pending, then move on to next image if found_pending_job: continue # If we haven't found a pending job for this image, and it doesn't exist in our database, then that # means that the image hash must be invalid. if not image: results.append({ 'status': 'failure', 'detail': 'Unknown md5 hash specified.', 'hash_md5': md5_hash }) continue # If everything is successful with image, return data results.append({'status': 'success', **image.dict()}) return results
def serialize_queues(instance_number, queues): return [ dict( name=q.name, count=q.count, queued_url=url_for( ".jobs_overview", instance_number=instance_number, queue_name=q.name, registry_name="queued", per_page="8", page="1", ), failed_job_registry_count=FailedJobRegistry(q.name).count, failed_url=url_for( ".jobs_overview", instance_number=instance_number, queue_name=q.name, registry_name="failed", per_page="8", page="1", ), started_job_registry_count=StartedJobRegistry(q.name).count, started_url=url_for( ".jobs_overview", instance_number=instance_number, queue_name=q.name, registry_name="started", per_page="8", page="1", ), deferred_job_registry_count=DeferredJobRegistry(q.name).count, deferred_url=url_for( ".jobs_overview", instance_number=instance_number, queue_name=q.name, registry_name="deferred", per_page="8", page="1", ), finished_job_registry_count=FinishedJobRegistry(q.name).count, finished_url=url_for( ".jobs_overview", instance_number=instance_number, queue_name=q.name, registry_name="finished", per_page="8", page="1", ), ) for q in queues ]
def _remove_jobs(cls, project): started_registry = StartedJobRegistry( cls._redis_queue.name, cls._redis_queue.connection ) finished_registry = FinishedJobRegistry( cls._redis_queue.name, cls._redis_queue.connection ) for job_id in started_registry.get_job_ids() + finished_registry.get_job_ids(): job = Job.fetch(job_id, connection=cls._redis) if job.meta.get("project") != project: continue logger.info("Deleting job_id " + job_id) job.delete()
def test_getting_registries(self): """Getting job registries from queue object""" queue = Queue('example') self.assertEqual(queue.scheduled_job_registry, ScheduledJobRegistry(queue=queue)) self.assertEqual(queue.started_job_registry, StartedJobRegistry(queue=queue)) self.assertEqual(queue.failed_job_registry, FailedJobRegistry(queue=queue)) self.assertEqual(queue.deferred_job_registry, DeferredJobRegistry(queue=queue)) self.assertEqual(queue.finished_job_registry, FinishedJobRegistry(queue=queue))
def main(): with Connection(conn): # queue = Queue(connection=conn) # for job_id in queue.job_ids: # queue.fetch_job(job_id) # if queue: # # queue.fetch queue = Queue(connection=conn) registry = StartedJobRegistry(queue=queue) pdb.set_trace() worker = Worker(list(map(Queue, listen))) worker.work()