def test_job_execution(self): """Job is removed from StartedJobRegistry after execution.""" registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) worker = Worker([queue]) job = queue.enqueue(say_hello) self.assertTrue(job.is_queued) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) self.assertTrue(job.is_started) worker.perform_job(job, queue) self.assertNotIn(job.id, registry.get_job_ids()) self.assertTrue(job.is_finished) # Job that fails job = queue.enqueue(div_by_zero) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) worker.perform_job(job, queue) self.assertNotIn(job.id, registry.get_job_ids())
def test_get_queue(self): """registry.get_queue() returns the right Queue object.""" registry = StartedJobRegistry(connection=self.testconn) self.assertEqual(registry.get_queue(), Queue(connection=self.testconn)) registry = StartedJobRegistry('foo', connection=self.testconn) self.assertEqual(registry.get_queue(), Queue('foo', connection=self.testconn))
def test_contains(self): registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) self.assertFalse(job in registry) self.assertFalse(job.id in registry) registry.add(job, 5) self.assertTrue(job in registry) self.assertTrue(job.id in registry)
def test_started_jobs(self): """Ensure that active jobs page works properly.""" queue = get_queue('django_rq_test') queue_index = get_queue_index('django_rq_test') job = queue.enqueue(access_self) registry = StartedJobRegistry(queue.name, queue.connection) registry.add(job, 2) response = self.client.get( reverse('rq_started_jobs', args=[queue_index]) ) self.assertEqual(response.context['jobs'], [job])
def test_job_deletion(self): """Ensure job is removed from StartedJobRegistry when deleted.""" registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) worker = Worker([queue]) job = queue.enqueue(say_hello) self.assertTrue(job.is_queued) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) job.delete() self.assertNotIn(job.id, registry.get_job_ids())
def test_prepare_job_execution(self): """Prepare job execution does the necessary bookkeeping.""" queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) worker = Worker([queue]) worker.prepare_job_execution(job) # Updates working queue registry = StartedJobRegistry(connection=self.testconn) self.assertEqual(registry.get_job_ids(), [job.id]) # Updates worker statuses self.assertEqual(worker.get_state(), 'busy') self.assertEqual(worker.get_current_job_id(), job.id)
def test_prepare_job_execution_inf_timeout(self): """Prepare job execution handles infinite job timeout""" queue = Queue(connection=self.testconn) job = queue.enqueue(long_running_job, args=(1,), job_timeout=-1) worker = Worker([queue]) worker.prepare_job_execution(job) # Updates working queue registry = StartedJobRegistry(connection=self.testconn) self.assertEqual(registry.get_job_ids(), [job.id]) # Score in queue is +inf self.assertEqual(self.testconn.zscore(registry.key, job.id), float('Inf'))
def started_jobs(request, queue_index): queue_index = int(queue_index) queue = get_queue_by_index(queue_index) registry = StartedJobRegistry(queue.name, queue.connection) items_per_page = 100 num_jobs = len(registry) page = int(request.GET.get('page', 1)) jobs = [] if num_jobs > 0: last_page = int(ceil(num_jobs / items_per_page)) page_range = range(1, last_page + 1) offset = items_per_page * (page - 1) job_ids = registry.get_job_ids(offset, offset + items_per_page - 1) for job_id in job_ids: try: jobs.append(Job.fetch(job_id, connection=queue.connection)) except NoSuchJobError: pass else: page_range = [] context_data = { 'queue': queue, 'queue_index': queue_index, 'jobs': jobs, 'num_jobs': num_jobs, 'page': page, 'page_range': page_range, 'job_status': 'Started', } return render(request, 'django_rq/jobs.html', context_data)
class TestRegistry(RQTestCase): def setUp(self): super(TestRegistry, self).setUp() self.registry = StartedJobRegistry(connection=self.testconn) def test_add_and_remove(self): """Adding and removing job to StartedJobRegistry.""" timestamp = current_timestamp() job = Job() # Test that job is added with the right score self.registry.add(job, 1000) self.assertLess(self.testconn.zscore(self.registry.key, job.id), timestamp + 1002) # Ensure that a timeout of -1 results in a score of -1 self.registry.add(job, -1) self.assertEqual(self.testconn.zscore(self.registry.key, job.id), -1) # Ensure that job is properly removed from sorted set self.registry.remove(job) self.assertIsNone(self.testconn.zscore(self.registry.key, job.id)) def test_get_job_ids(self): """Getting job ids from StartedJobRegistry.""" timestamp = current_timestamp() self.testconn.zadd(self.registry.key, timestamp + 10, 'foo') self.testconn.zadd(self.registry.key, timestamp + 20, 'bar') self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar']) def test_get_expired_job_ids(self): """Getting expired job ids form StartedJobRegistry.""" timestamp = current_timestamp() self.testconn.zadd(self.registry.key, 1, 'foo') self.testconn.zadd(self.registry.key, timestamp + 10, 'bar') self.testconn.zadd(self.registry.key, timestamp + 30, 'baz') self.assertEqual(self.registry.get_expired_job_ids(), ['foo']) self.assertEqual(self.registry.get_expired_job_ids(timestamp + 20), ['foo', 'bar']) def test_cleanup(self): """Moving expired jobs to FailedQueue.""" failed_queue = FailedQueue(connection=self.testconn) self.assertTrue(failed_queue.is_empty()) self.testconn.zadd(self.registry.key, 2, 'foo') self.registry.cleanup(1) self.assertNotIn('foo', failed_queue.job_ids) self.assertEqual(self.testconn.zscore(self.registry.key, 'foo'), 2) self.registry.cleanup() self.assertIn('foo', failed_queue.job_ids) self.assertEqual(self.testconn.zscore(self.registry.key, 'foo'), None) def test_job_execution(self): """Job is removed from StartedJobRegistry after execution.""" registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) worker = Worker([queue]) job = queue.enqueue(say_hello) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) worker.perform_job(job) self.assertNotIn(job.id, registry.get_job_ids()) # Job that fails job = queue.enqueue(div_by_zero) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) worker.perform_job(job) self.assertNotIn(job.id, registry.get_job_ids()) def test_get_job_count(self): """StartedJobRegistry returns the right number of job count.""" timestamp = current_timestamp() + 10 self.testconn.zadd(self.registry.key, timestamp, 'foo') self.testconn.zadd(self.registry.key, timestamp, 'bar') self.assertEqual(self.registry.count, 2) self.assertEqual(len(self.registry), 2)
def requeue_interrupted_jobs(queue_name=None): # type: (str) -> None """ Requeues jobs found in the given queue's started job registry. Only restarts those that aren't already queued or being run. When rq starts a job, it records it in the queue's started job registry. If the server is rebooted before the job completes, the job is not automatically restarted from the information in the registry. For tasks like secure deletion of files, this means that information thought to be deleted is still present in the case of seizure or compromise. We have manage.py tasks to clean such files up, but this utility attempts to reduce the need for manual intervention by automatically resuming interrupted jobs. This function is predicated on a risky assumption: that all jobs are idempotent. At time of writing, we use rq for securely deleting submission files and hashing submissions for the ETag header. Both of these can be safely repeated. If we add rq tasks that cannot, this function should be improved to omit those. """ queue = create_queue(queue_name) started_job_registry = StartedJobRegistry(queue=queue) queued_job_ids = queue.get_job_ids() logging.debug("queued jobs: {}".format(queued_job_ids)) started_job_ids = started_job_registry.get_job_ids() logging.debug("started jobs: {}".format(started_job_ids)) job_ids = [j for j in started_job_ids if j not in queued_job_ids] logging.debug("candidate job ids: {}".format(job_ids)) if not job_ids: logging.debug("No interrupted jobs found in started job registry.") for job_id in job_ids: logging.debug("Considering job %s", job_id) try: job = started_job_registry.job_class.fetch( job_id, started_job_registry.connection) except NoSuchJobError as e: logging.error("Could not find details for job %s: %s", job_id, e) continue logging.debug("Job %s enqueued at %s, started at %s", job_id, job.enqueued_at, job.started_at) worker = worker_for_job(job_id) if worker: logging.info( "Skipping job %s, which is already being run by worker %s", job_id, worker.key) continue logging.info("Requeuing job %s", job) try: started_job_registry.remove(job) except InvalidJobOperation as e: logging.error( "Could not remove job %s from started job registry: %s", job, e) continue try: queue.enqueue_job(job) logging.debug("Job now enqueued at %s, started at %s", job.enqueued_at, job.started_at) except Exception as e: logging.error("Could not requeue job %s: %s", job, e) continue
def poll_and_update_on_processing(n_intervals, session_id, jobs): if n_intervals == 0: return '#', '', {'display': 'none'}, '#', {'display': 'none'}, '' res = None dbconn = eng.connect() if os.environ.get( 'STACK') else sqlite3.connect('output.db') query = f'select distinct table_name \ as name from information_schema.tables' \ if os.environ.get('STACK') \ else 'select distinct name from sqlite_master' def _check_for_output(n_intervals, dbconn): df = pd.read_sql(query, con=dbconn, columns=['name']) # print(df.columns) # print('NAMES', df) if jobs and any(df.name.str.contains(f'output_{session_id}_{jobs[-1]}')): print('found output') return True else: print('didn\'t find output') return False if _check_for_output(n_intervals, dbconn): # Allow user to download the results of the most recent job df = pd.read_sql(f'select * from output_{session_id}_{jobs[-1]}', con=dbconn) df.rename({'index': 'filename'}, axis=1, inplace=True) csv_string = df.to_csv(index=False) output = join(BASE_DIR, 'output') session = join(output, f'output_{session_id}') analysis = join(session, 'analysis') job = join(analysis, f'job_{jobs[-1]}') images = join(job, 'images') for dir_ in [output, session, analysis, job, images]: if not isdir(dir_): os.mkdir(dir_) # don't download if imagedir already full # download s3 images if len(os.listdir(images)) > 0: # print(os.listdir(imagedir)) if not isfile(join(job, 'images.zip')): tmp = shutil.make_archive(images, 'zip', images) print(f'made archive {tmp}') # for f in os.listdir(imagedir): # os.unlink(join(imagedir, f)) # {'display': 'none'} res = ("data:text/csv;charset=utf-8," + quote(csv_string), 'output.csv', {}, f'/dash/download?session_id={session_id}&job_id={jobs[-1]}', {}, dbc.Alert('Your results are ready!', color='success')) else: if not os.environ.get('STACK'): with open(join(BASE_DIR, '.aws'), 'r') as f: print('getting aws creds') creds = json.loads(f.read()) AWS_ACCESS = creds['access'] AWS_SECRET = creds['secret'] else: AWS_ACCESS = os.environ['AWS_ACCESS'] AWS_SECRET = os.environ['AWS_SECRET'] s3 = get_s3(AWS_ACCESS, AWS_SECRET) # select bucket bucket = s3.Bucket('voigt') # download file into current directory print(f'found S3 buckets {bucket.objects.all()}') for s3_object in bucket.objects.all(): print(f'found S3 object {s3_object.key}') do_zip = False # Need to split s3_object.key into path and file name, else it will # give error file not found. if f'output_{session_id}/job_{jobs[-1]}' not in s3_object.key: continue path, filename = os.path.split(s3_object.key) fpth = join(images, filename) print(f'checking for file {s3_object.key}') if not isfile(fpth): print(f'downloading file {s3_object.key}') bucket.download_file(s3_object.key, fpth) do_zip = True if do_zip: tmp = shutil.make_archive(images, 'zip', images) print(f'made archive {tmp}') res = ("data:text/csv;charset=utf-8," + quote(csv_string), 'output.csv', {}, f'/dash/download?session_id={session_id}&job_id={jobs[-1]}', {}, dbc.Alert('Your results are ready!', color='success')) else: registry = StartedJobRegistry('default', connection=conn) input_dir = join(BASE_DIR, 'input', f'input_{session_id}', 'analysis') # TODO concurrency? what if mutliple ppl use app at same time? if (jobs and jobs[-1] not in registry.get_job_ids()) or not jobs: msg = dbc.Alert('Ready.', color='primary') if len(os.listdir(input_dir)) > 0 \ else dbc.Alert('Upload some peak files first!', color='warning') res = ('#', '', {'display': 'none'}, '#', {'display': 'none'}, msg) elif jobs and jobs[-1] in registry.get_job_ids(): res = ('#', '', {'display': 'none'}, '#', {'display': 'none'}, dbc.Alert( [ 'Please wait while your request is processed.', dbc.Spinner(type='grow') ], color='danger') ) dbconn.close() return res
class TestRegistry(RQTestCase): def setUp(self): super(TestRegistry, self).setUp() self.registry = StartedJobRegistry(connection=self.testconn) def test_key(self): self.assertEqual(self.registry.key, 'rq:wip:default') def test_custom_job_class(self): registry = StartedJobRegistry(job_class=CustomJob) self.assertFalse(registry.job_class == self.registry.job_class) def test_add_and_remove(self): """Adding and removing job to StartedJobRegistry.""" timestamp = current_timestamp() job = Job() # Test that job is added with the right score self.registry.add(job, 1000) self.assertLess(self.testconn.zscore(self.registry.key, job.id), timestamp + 1002) # Ensure that a timeout of -1 results in a score of -1 self.registry.add(job, -1) self.assertEqual(self.testconn.zscore(self.registry.key, job.id), -1) # Ensure that job is properly removed from sorted set self.registry.remove(job) self.assertIsNone(self.testconn.zscore(self.registry.key, job.id)) def test_get_job_ids(self): """Getting job ids from StartedJobRegistry.""" timestamp = current_timestamp() self.testconn.zadd(self.registry.key, timestamp + 10, 'foo') self.testconn.zadd(self.registry.key, timestamp + 20, 'bar') self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar']) def test_get_expired_job_ids(self): """Getting expired job ids form StartedJobRegistry.""" timestamp = current_timestamp() self.testconn.zadd(self.registry.key, 1, 'foo') self.testconn.zadd(self.registry.key, timestamp + 10, 'bar') self.testconn.zadd(self.registry.key, timestamp + 30, 'baz') self.assertEqual(self.registry.get_expired_job_ids(), ['foo']) self.assertEqual(self.registry.get_expired_job_ids(timestamp + 20), ['foo', 'bar']) def test_cleanup(self): """Moving expired jobs to FailedQueue.""" failed_queue = FailedQueue(connection=self.testconn) self.assertTrue(failed_queue.is_empty()) queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) self.testconn.zadd(self.registry.key, 2, job.id) self.registry.cleanup(1) self.assertNotIn(job.id, failed_queue.job_ids) self.assertEqual(self.testconn.zscore(self.registry.key, job.id), 2) self.registry.cleanup() self.assertIn(job.id, failed_queue.job_ids) self.assertEqual(self.testconn.zscore(self.registry.key, job.id), None) job.refresh() self.assertEqual(job.get_status(), JobStatus.FAILED) def test_job_execution(self): """Job is removed from StartedJobRegistry after execution.""" registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) worker = Worker([queue]) job = queue.enqueue(say_hello) self.assertTrue(job.is_queued) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) self.assertTrue(job.is_started) worker.perform_job(job, queue) self.assertNotIn(job.id, registry.get_job_ids()) self.assertTrue(job.is_finished) # Job that fails job = queue.enqueue(div_by_zero) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) worker.perform_job(job, queue) self.assertNotIn(job.id, registry.get_job_ids()) def test_job_deletion(self): """Ensure job is removed from StartedJobRegistry when deleted.""" registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) worker = Worker([queue]) job = queue.enqueue(say_hello) self.assertTrue(job.is_queued) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) job.delete() self.assertNotIn(job.id, registry.get_job_ids()) def test_get_job_count(self): """StartedJobRegistry returns the right number of job count.""" timestamp = current_timestamp() + 10 self.testconn.zadd(self.registry.key, timestamp, 'foo') self.testconn.zadd(self.registry.key, timestamp, 'bar') self.assertEqual(self.registry.count, 2) self.assertEqual(len(self.registry), 2) def test_clean_registries(self): """clean_registries() cleans Started and Finished job registries.""" queue = Queue(connection=self.testconn) finished_job_registry = FinishedJobRegistry(connection=self.testconn) self.testconn.zadd(finished_job_registry.key, 1, 'foo') started_job_registry = StartedJobRegistry(connection=self.testconn) self.testconn.zadd(started_job_registry.key, 1, 'foo') clean_registries(queue) self.assertEqual(self.testconn.zcard(finished_job_registry.key), 0) self.assertEqual(self.testconn.zcard(started_job_registry.key), 0)
def get_current_rq_jobs(): registry = StartedJobRegistry('default', connection=get_redis_connection()) running_job_ids = registry.get_job_ids() return len(running_job_ids)
class QueueManager: retries_before_fail = 10 backoff = 30 def __init__(self, name: str, connection: redis.Redis): self._name = name self._logger = logger.bind(queue=name) self._queue = rq.Queue("vivarium", connection=connection) self._wip = StartedJobRegistry( "vivarium", connection=connection, job_class=self._queue.job_class ) self._finished = FinishedJobRegistry( "vivarium", connection=connection, job_class=self._queue.job_class ) self._status = { "total": 0, "pending": 0, "running": 0, "failed": 0, "finished": 0, "done": 0.0, "workers": 0, } self._failed = False self._completed = False self._retries = QueueManager.retries_before_fail self._backoff = QueueManager.backoff @property def name(self) -> str: return self._name @property def failed(self) -> bool: return self._failed @property def completed(self) -> bool: return self._completed @property def jobs_to_finish(self) -> bool: return not (self.failed or self.completed) def enqueue(self, jobs: List[dict], workhorse_import_path: str) -> None: self._logger.info(f"Enqueuing jobs in queue {self.name}") for job in jobs: # TODO: might be nice to have tighter ttls but it's hard to predict # how long our jobs will take from model to model and the entire # system is short lived anyway self._queue.enqueue( workhorse_import_path, job_parameters=job, ttl=60 * 60 * 24 * 2, result_ttl=60 * 60, job_timeout="7d", ) def get_results(self) -> List: self._logger.info(f"Checking queue {self.name}") finished_jobs = self._get_finished_jobs() start = time.time() results = [] for job_id in finished_jobs: result = self._get_result(job_id) if result is not None: results.append(result) self._logger.info( f"Retrieved {len(results)} results from queue {self.name} in {time.time() - start:.2f}s" ) return results def update_and_report(self) -> Dict[str, int]: self._update_status() template = ( f"Queue {self.name} - Total jobs: {{total}}, % Done: {{done:.2f}}% " f"Pending: {{pending}}, Running: {{running}}, Failed: {{failed}}, Finished: {{finished}} " f"Workers: {{workers}}." ) if not (self.completed or self.failed): self._logger.info(template.format(**self._status)) return self._status def _update_status(self) -> None: if self.jobs_to_finish and self._retries > 0: self._logger.info(f"Updating status for queue {self.name}") # TODO: Sometimes there are duplicate job_ids, why? try: # Account for timing discrepancies in accounting and the # fact that a job might legit be in more than one of these. finished_ids = set(self._finished.get_job_ids()) if self._queue else set() running_ids = set(self._wip.get_job_ids()) if self._queue else set() # "Finished" for our purposes means the job is done # and the results have been retrieved, whereas for rq # finished means the job is done and the results have # not been retrieved. Once retrieved, jobs disappear # from the finished job registry and we'd see all our # account eventually just go to zero. running_ids = running_ids | finished_ids pending_ids = set(self._queue.job_ids) if self._queue else set() pending_ids = pending_ids - running_ids failed_ids = ( set(self._queue.failed_job_registry.get_job_ids()) if self._queue else set() ) failed_ids = failed_ids - (running_ids | pending_ids) q_workers = len(rq.Worker.all(queue=self._queue)) if self._queue else 0 self._status["pending"] = len(pending_ids) self._status["running"] = len(running_ids) + len(finished_ids) self._status["failed"] = len(failed_ids) self._status["finished"] = self._status["finished"] self._status["total"] = ( len(pending_ids) + len(running_ids) + len(failed_ids) + self._status["finished"] ) self._status["workers"] = q_workers self._status["done"] = 100 * self._status["finished"] / self._status["total"] if len(pending_ids) + len(running_ids) == 0: self._mark_complete() elif len(pending_ids) + len(finished_ids) + q_workers == 0: self._logger.info( f"Queue {self.name} ran out of workers with running jobs. Marking finished." ) self._mark_complete() except redis.connection.ConnectionError: self._sleep_on_it() self._update_status() else: self._mark_failed() def _get_finished_jobs(self) -> List[str]: if self._retries: try: return self._finished.get_job_ids() except redis.exceptions.ConnectionError: self._sleep_on_it() return self._get_finished_jobs() else: self._mark_failed() return [] def _get_result(self, job_id: str) -> Any: job = self._get_job(job_id) result = None if job is not None: start = time.time() result = job.result end = time.time() self._logger.debug( f"Deserialized {job_id} result from queue {self.name} in {end - start:.2f}s." ) self._status["finished"] += 1 self._remove_finished_job(job) return result def _get_job(self, job_id: str) -> Union[None, Job]: if self._retries: try: start = time.time() job = self._queue.fetch_job(job_id) end = time.time() self._logger.debug( f"Fetched job {job_id} from queue {self.name} in {end - start:.2f}s" ) return job except redis.exceptions.ConnectionError: self._sleep_on_it() return self._get_job(job_id) else: self._mark_failed() return None def _remove_finished_job(self, job: Job) -> None: if self._retries: try: self._finished.remove(job) except redis.exceptions.ConnectionError: self._sleep_on_it() self._remove_finished_job(job) else: self._mark_failed() def _sleep_on_it(self) -> None: backoff = self._backoff * random.random() self._logger.warning( f"Failed to connect to redis for queue {self.name}. Retrying in {backoff}s." ) self._retries -= 1 time.sleep(backoff) def _mark_failed(self) -> None: # TODO: Probably should cleanup redis stuff, but not sure how yet. if not (self.failed or self.completed): self._logger.warning( f"Queue {self.name} has run out of retries. Marking as failed." ) self._status["failed"] += self._status["pending"] + self._status["running"] self._status["pending"] = 0 self._status["running"] = 0 self._status["workers"] = 0 self._failed = True def _mark_complete(self) -> None: if not (self.failed or self.completed): self._logger.info(f"All jobs on queue {self.name} complete.") self._status["finished"] += self._status["pending"] + self._status["running"] self._status["pending"] = 0 self._status["running"] = 0 self._status["workers"] = 0 self._completed = True
def GetJobProgress(app): """Infinite thread that sends job updates to client.""" with app.app_context(): redis = Redis() queue = Queue(connection=redis) running_registry = StartedJobRegistry(queue=queue) finished_registry = FinishedJobRegistry(queue=queue) showing_running_job_bar = False toggle_on_running_job_bar = False toggle_off_running_job_bar = False while True: time.sleep(2) running_msg = "Running jobs:" finished_msg = "" running_job_id_list = running_registry.get_job_ids() finished_job_id_list = finished_registry.get_job_ids() if running_job_id_list: # Get all job progress and emit update for job_id in running_job_id_list: running_msg += "<br>Job ID: {}".format(job_id) running_job = Job.fetch(id=job_id, connection=redis) if 'process' in running_job.meta: running_msg += "<UL><LI>Process: {}".format( running_job.meta['process']) if 'progress' in running_job.meta: running_msg += "<LI>Progress: {}</UL>".format( running_job.meta['progress']) socketio.emit('update job progress', {'data': running_msg}) toggle_on_running_job_bar = True else: toggle_off_running_job_bar = True if finished_job_id_list: # Display finished job notification and save to db for job_id in finished_job_id_list: finished_msg += "<br>Job ID: {}".format(job_id) finished_job = Job.fetch(job_id, connection=redis) if 'process' in finished_job.meta: finished_msg += "<UL><LI>Process: {}".format( finished_job.meta['process']) rq_job = RQJob() rq_job.save_job(finished_job) finished_registry.remove(finished_job) finished_msg += "<LI>Result: {}".format( rq_job.get_result()) finished_msg += "<LI>Time elapsed: {} seconds</UL>".format( rq_job.get_elapsed_time()) socketio.emit('update job finished', {'data': finished_msg}) socketio.emit('show finished job bar') if showing_running_job_bar and toggle_off_running_job_bar: toggle_off_running_job_bar = False toggle_on_running_job_bar = False showing_running_job_bar = False print('---------Server: Hide running job bar') socketio.emit('hide running job bar') elif not showing_running_job_bar and toggle_on_running_job_bar: toggle_off_running_job_bar = False toggle_on_running_job_bar = False showing_running_job_bar = True print('---------Server: Show running job bar') socketio.emit('show running job bar') return
from words.datamuse_json import DatamuseWordNotRecognizedError from words.forms import RelatedWordsForm, WordSetCreateForm, WordSetChoice, ScatterplotWordSetChoice from words.models import WordSet, Word # Get an instance of a logger logger = logging.getLogger(__name__) # se up django-rq queue redis_url = os.getenv('REDISTOGO_URL', 'redis://localhost:6379') redis_cursor = redis.from_url(redis_url) # singleton Redis client # queue for running rq worker for WordSetCreateForm save rq_queue = django_rq.get_queue('default', connection=redis_cursor) # register to track running rq jobs rq_started_job_registry = StartedJobRegistry( queue=rq_queue) # connection=redis_cursor) #, queue=rq_queue) def job_json(request, job_id): """Given the job_id for a django-rq job, exposes the job status, meta and result (if any) in json format""" job = Job.fetch(job_id, connection=redis_cursor) reply = { 'status': job.get_status(), 'meta': job.meta, 'result': job.result } return JsonResponse(reply)
def current_jobs(self): """ Track currently active and queued jobs """ registry = StartedJobRegistry(queue=self.generate_queue) jobs = registry.get_job_ids() + self.current_jobs() return jobs
import logging from rq import Queue from rq.job import Job from collections import deque from rq import get_current_job from django.conf import settings import django_rq from rq.registry import FinishedJobRegistry, StartedJobRegistry logger = logging.getLogger('uprilogger') q = django_rq.get_queue() # rqq = Queue(connection=django_rq.get_connection()) finished_job_registry = FinishedJobRegistry(connection=django_rq.get_connection()) started_job_registry = StartedJobRegistry(connection=django_rq.get_connection()) failed_queue = django_rq.get_failed_queue() # # Job management # def job_message(message): job = get_current_job(connection=django_rq.get_connection()) if not job.meta.get('messages'): job.meta['messages'] = deque() job.meta['messages'].append(message) job.save()
from rq import Queue from rq.job import Job from rq.registry import StartedJobRegistry, FinishedJobRegistry from redisQworker import conn from model import train, predict import config import datetime id_time_format = "%Y-%m-%d %H:%M:%S" train_job_id = "train_job" train_new_job_id = "new_train_job" log = logging.getLogger(__name__) q_train = Queue(connection=conn, name='train') registr_train = StartedJobRegistry('train', connection=conn) started_registry = StartedJobRegistry('train', connection=conn) finished_registry = FinishedJobRegistry('train', connection=conn) def init_time_id(id_time_format=id_time_format): return datetime.datetime.utcnow().strftime(id_time_format) def sort_id_time(list_id_time, id_time_format=id_time_format, reverse=False): list_datetime = [ datetime.datetime.strptime(x, id_time_format) for x in list_id_time ] list_datetime = sorted(list_datetime, reverse=reverse) sorted_list = [x.strftime(id_time_format) for x in list_datetime]
class TestRegistry(RQTestCase): def setUp(self): super(TestRegistry, self).setUp() self.registry = StartedJobRegistry(connection=self.testconn) def test_init(self): """Registry can be instantiated with queue or name/Redis connection""" queue = Queue('foo', connection=self.testconn) registry = StartedJobRegistry(queue=queue) self.assertEqual(registry.name, queue.name) self.assertEqual(registry.connection, queue.connection) registry = StartedJobRegistry('bar', self.testconn) self.assertEqual(registry.name, 'bar') self.assertEqual(registry.connection, self.testconn) def test_key(self): self.assertEqual(self.registry.key, 'rq:wip:default') def test_custom_job_class(self): registry = StartedJobRegistry(job_class=CustomJob) self.assertFalse(registry.job_class == self.registry.job_class) def test_contains(self): registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) self.assertFalse(job in registry) self.assertFalse(job.id in registry) registry.add(job, 5) self.assertTrue(job in registry) self.assertTrue(job.id in registry) def test_get_expiration_time(self): """registry.get_expiration_time() returns correct datetime objects""" registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) registry.add(job, 5) self.assertEqual( registry.get_expiration_time(job), (datetime.utcnow() + timedelta(seconds=5)).replace(microsecond=0) ) def test_add_and_remove(self): """Adding and removing job to StartedJobRegistry.""" timestamp = current_timestamp() queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) # Test that job is added with the right score self.registry.add(job, 1000) self.assertLess(self.testconn.zscore(self.registry.key, job.id), timestamp + 1002) # Ensure that a timeout of -1 results in a score of inf self.registry.add(job, -1) self.assertEqual(self.testconn.zscore(self.registry.key, job.id), float('inf')) # Ensure that job is removed from sorted set, but job key is not deleted self.registry.remove(job) self.assertIsNone(self.testconn.zscore(self.registry.key, job.id)) self.assertTrue(self.testconn.exists(job.key)) self.registry.add(job, -1) # registry.remove() also accepts job.id self.registry.remove(job.id) self.assertIsNone(self.testconn.zscore(self.registry.key, job.id)) self.registry.add(job, -1) # delete_job = True deletes job key self.registry.remove(job, delete_job=True) self.assertIsNone(self.testconn.zscore(self.registry.key, job.id)) self.assertFalse(self.testconn.exists(job.key)) job = queue.enqueue(say_hello) self.registry.add(job, -1) # delete_job = True also works with job.id self.registry.remove(job.id, delete_job=True) self.assertIsNone(self.testconn.zscore(self.registry.key, job.id)) self.assertFalse(self.testconn.exists(job.key)) def test_get_job_ids(self): """Getting job ids from StartedJobRegistry.""" timestamp = current_timestamp() self.testconn.zadd(self.registry.key, {'foo': timestamp + 10}) self.testconn.zadd(self.registry.key, {'bar': timestamp + 20}) self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar']) def test_get_expired_job_ids(self): """Getting expired job ids form StartedJobRegistry.""" timestamp = current_timestamp() self.testconn.zadd(self.registry.key, {'foo': 1}) self.testconn.zadd(self.registry.key, {'bar': timestamp + 10}) self.testconn.zadd(self.registry.key, {'baz': timestamp + 30}) self.assertEqual(self.registry.get_expired_job_ids(), ['foo']) self.assertEqual(self.registry.get_expired_job_ids(timestamp + 20), ['foo', 'bar']) def test_cleanup_moves_jobs_to_failed_job_registry(self): """Moving expired jobs to FailedJobRegistry.""" queue = Queue(connection=self.testconn) failed_job_registry = FailedJobRegistry(connection=self.testconn) job = queue.enqueue(say_hello) self.testconn.zadd(self.registry.key, {job.id: 2}) # Job has not been moved to FailedJobRegistry self.registry.cleanup(1) self.assertNotIn(job, failed_job_registry) self.assertIn(job, self.registry) self.registry.cleanup() self.assertIn(job.id, failed_job_registry) self.assertNotIn(job, self.registry) job.refresh() self.assertEqual(job.get_status(), JobStatus.FAILED) self.assertTrue(job.exc_info) # explanation is written to exc_info def test_job_execution(self): """Job is removed from StartedJobRegistry after execution.""" registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) worker = Worker([queue]) job = queue.enqueue(say_hello) self.assertTrue(job.is_queued) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) self.assertTrue(job.is_started) worker.perform_job(job, queue) self.assertNotIn(job.id, registry.get_job_ids()) self.assertTrue(job.is_finished) # Job that fails job = queue.enqueue(div_by_zero) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) worker.perform_job(job, queue) self.assertNotIn(job.id, registry.get_job_ids()) def test_job_deletion(self): """Ensure job is removed from StartedJobRegistry when deleted.""" registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) worker = Worker([queue]) job = queue.enqueue(say_hello) self.assertTrue(job.is_queued) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) job.delete() self.assertNotIn(job.id, registry.get_job_ids()) def test_get_job_count(self): """StartedJobRegistry returns the right number of job count.""" timestamp = current_timestamp() + 10 self.testconn.zadd(self.registry.key, {'foo': timestamp}) self.testconn.zadd(self.registry.key, {'bar': timestamp}) self.assertEqual(self.registry.count, 2) self.assertEqual(len(self.registry), 2) def test_clean_registries(self): """clean_registries() cleans Started and Finished job registries.""" queue = Queue(connection=self.testconn) finished_job_registry = FinishedJobRegistry(connection=self.testconn) self.testconn.zadd(finished_job_registry.key, {'foo': 1}) started_job_registry = StartedJobRegistry(connection=self.testconn) self.testconn.zadd(started_job_registry.key, {'foo': 1}) failed_job_registry = FailedJobRegistry(connection=self.testconn) self.testconn.zadd(failed_job_registry.key, {'foo': 1}) clean_registries(queue) self.assertEqual(self.testconn.zcard(finished_job_registry.key), 0) self.assertEqual(self.testconn.zcard(started_job_registry.key), 0) self.assertEqual(self.testconn.zcard(failed_job_registry.key), 0) def test_get_queue(self): """registry.get_queue() returns the right Queue object.""" registry = StartedJobRegistry(connection=self.testconn) self.assertEqual(registry.get_queue(), Queue(connection=self.testconn)) registry = StartedJobRegistry('foo', connection=self.testconn) self.assertEqual(registry.get_queue(), Queue('foo', connection=self.testconn))
def get_statistics(): queues = [] workers_collections = collect_workers_by_connection(QUEUES_LIST) for index, config in enumerate(QUEUES_LIST): queue = get_queue_by_index(index) connection = queue.connection connection_kwargs = connection.connection_pool.connection_kwargs # Raw access to the first item from left of the redis list. # This might not be accurate since new job can be added from the left # with `at_front` parameters. # Ideally rq should supports Queue.oldest_job last_job_id = connection.lindex(queue.key, 0) last_job = queue.fetch_job( last_job_id.decode('utf-8')) if last_job_id else None if last_job: oldest_job_timestamp = to_localtime(last_job.enqueued_at)\ .strftime('%Y-%m-%d, %H:%M:%S') else: oldest_job_timestamp = "-" # parse_class is not needed and not JSON serializable try: del (connection_kwargs['parser_class']) except KeyError: pass queue_data = { 'name': queue.name, 'jobs': queue.count, 'oldest_job_timestamp': oldest_job_timestamp, 'index': index, 'connection_kwargs': connection_kwargs } if queue.name == 'failed': queue_data['workers'] = '-' queue_data['finished_jobs'] = '-' queue_data['started_jobs'] = '-' queue_data['deferred_jobs'] = '-' elif RQ_SCHEDULER_INSTALLED and queue.name == 'scheduled': scheduler = get_scheduler(connection=queue.connection) queue_data['jobs'] = scheduler.count() queue_data['workers'] = '-' queue_data['finished_jobs'] = '-' queue_data['started_jobs'] = '-' queue_data['deferred_jobs'] = '-' else: connection = get_connection(queue.name) all_workers = get_all_workers_by_configuration( config['connection_config'], workers_collections) queue_workers = [ worker for worker in all_workers if queue in worker.queues ] queue_data['workers'] = len(queue_workers) finished_job_registry = FinishedJobRegistry(queue.name, connection) started_job_registry = StartedJobRegistry(queue.name, connection) deferred_job_registry = DeferredJobRegistry(queue.name, connection) queue_data['finished_jobs'] = len(finished_job_registry) queue_data['started_jobs'] = len(started_job_registry) queue_data['deferred_jobs'] = len(deferred_job_registry) queues.append(queue_data) return {'queues': queues, 'display_scheduled_jobs': RQ_SCHEDULER_INSTALLED}
sys.path.insert(0, "/home/christophe/workspace/velona/rq") from time import sleep from rqtest.functions import wait_a_bit from redis import Redis from rq import Queue from rq.registry import StartedJobRegistry """ Test enqueing a job but deleting it before it has finished. The reason is to Test that the job is correctly remove from the redis StartedJobRegistry """ conn = Redis('localhost', 6379) q = Queue(connection=conn) job = q.enqueue(wait_a_bit, 10) registry = StartedJobRegistry(job.origin, conn) print('Enqueued', job) sleep(2) print('StartedJobRegistry, before delete:') print(registry.get_job_ids()) job.delete() print('StartedJobRegistry, after delete:') print(registry.get_job_ids())
request, make_response ) from redis import Redis from stocks import ( StockDatabase, ) queue_name = "trading-tasks" db = StockDatabase('84WDI082Z0HOREL6') tasks = [] app = Flask(__name__) redis = Redis.from_url("redis://*****:*****@app.route('/api/trading/stocks/suggest/', methods=["POST"]) def suggest_stock(): return make_response(jsonify(None), 404) @app.route('/api/trading/stocks/search/', methods=["GET"]) def search_stock(): """ Describes a specific stock
def test_job_delete_removes_itself_from_registries(self): """job.delete() should remove itself from job registries""" connection = self.testconn job = Job.create(func=fixtures.say_hello, status=JobStatus.FAILED, connection=self.testconn, origin='default') job.save() registry = FailedJobRegistry(connection=self.testconn) registry.add(job, 500) job.delete() self.assertFalse(job in registry) job = Job.create(func=fixtures.say_hello, status=JobStatus.FINISHED, connection=self.testconn, origin='default') job.save() registry = FinishedJobRegistry(connection=self.testconn) registry.add(job, 500) job.delete() self.assertFalse(job in registry) job = Job.create(func=fixtures.say_hello, status=JobStatus.STARTED, connection=self.testconn, origin='default') job.save() registry = StartedJobRegistry(connection=self.testconn) registry.add(job, 500) job.delete() self.assertFalse(job in registry) job = Job.create(func=fixtures.say_hello, status=JobStatus.DEFERRED, connection=self.testconn, origin='default') job.save() registry = DeferredJobRegistry(connection=self.testconn) registry.add(job, 500) job.delete() self.assertFalse(job in registry) job = Job.create(func=fixtures.say_hello, status=JobStatus.SCHEDULED, connection=self.testconn, origin='default') job.save() registry = ScheduledJobRegistry(connection=self.testconn) registry.add(job, 500) job.delete() self.assertFalse(job in registry)
ParoptManager.start() app.run(debug=True, host="0.0.0.0", port=8080, use_reloader=False, ssl_context='adhoc') elif args.setupaws: # run func to configure aws provider state setupAWS() else: if args.workers <= 0: print("Error: --workers must be an integer > 0") sys.exit(1) redis_url = app.config['REDIS_URL'] # clear previously started started jobs - if shut down while running a job, the job will remain in StartedJobsRegistry # when it's restarted, which is a problem because it's not actually running anymore with Connection(redis.from_url(redis_url)) as conn: registry = StartedJobRegistry('default', connection=conn) for job_id in registry.get_job_ids(): registry.remove(Job.fetch(job_id, connection=conn)) procs = [] for i in range(args.workers): procs.append( Process(target=startWorker, args=(redis_url, app.config['QUEUES']))) procs[i].start() for proc in procs: proc.join()
def _repr(q): return "running:%d pending:%d finished:%d" % (StartedJobRegistry( q.name, conn).count, q.count, FinishedJobRegistry(q.name, conn).count)
from rq import Queue from rq.job import Job from rq.exceptions import NoSuchJobError from rq.registry import StartedJobRegistry from sitesearch.config import Config from sitesearch.connections import get_search_connection, get_rq_redis_client from sitesearch.tasks import JOB_ID, JOB_STARTED, JOB_NOT_QUEUED, index, INDEXING_TIMEOUT from .resource import Resource config = Config() redis_client = get_rq_redis_client() search_client = get_search_connection(config.default_search_site.index_name) log = logging.getLogger(__name__) queue = Queue(connection=redis_client) registry = StartedJobRegistry('default', connection=redis_client) API_KEY = os.environ['API_KEY'] class IndexerResource(Resource): def on_get(self, req, resp): """Start an indexing job.""" try: status = Job.fetch(JOB_ID, connection=redis_client).get_status() except NoSuchJobError: if JOB_ID in registry.get_job_ids(): status = JOB_STARTED else: status = JOB_NOT_QUEUED
def test_custom_job_class(self): registry = StartedJobRegistry(job_class=CustomJob) self.assertFalse(registry.job_class == self.registry.job_class)
def poll_and_update_on_processing(n_intervals, session_id, fit_jobs, file_format, min_, max_): if n_intervals == 0: return '#', {'display': 'none'}, '', '', True res = None def job_status(): if len(fit_jobs) == 0: return 'ready' try: job = Job.fetch(fit_jobs[-1], connection=conn) except NoSuchJobError: return 'finished' print(f'Job status: {job.get_status()}') return job.get_status() input_dir = join(BASE_DIR, 'input', f'input_{session_id}', 'fitting') if job_status() == 'queued': msg = dbc.Alert(['Waiting for compute resources...', dbc.Spinner(type='grow')], color='warning') res = ('#', '', {'display': 'none'}, msg, '', True) elif job_status() == 'ready': msg, flag = (dbc.Alert('Ready.', color='primary'), False) if len(os.listdir(input_dir)) > 0 \ else (dbc.Alert('Upload some TGA measurements first!', color='warning'), True) res = ('#', '', {'display': 'none'}, msg, '', flag) elif job_status() == 'finished': fitting = join(BASE_DIR, 'output', f'output_{session_id}', 'fitting') outputdir = join(fitting, f'job_{fit_jobs[-1]}') if not exists(outputdir): os.mkdir(outputdir) # already downloaded results if len(os.listdir(outputdir)) > 0: if not isfile(join(fitting, f'job_{fit_jobs[-1]}.zip')): tmp = shutil.make_archive(outputdir, 'zip', outputdir) print(f'made archive {tmp}') res = (f'/dash/download-fit?session_id={session_id}&job_id={fit_jobs[-1]}', 'fits.csv', {}, dbc.Alert('Your results are ready!', color='success'), '', False) # download results else: if not os.environ.get('STACK'): with open(join(BASE_DIR, '.aws'), 'r') as f: print('getting aws creds') creds = json.loads(f.read()) AWS_ACCESS = creds['access'] AWS_SECRET = creds['secret'] else: AWS_ACCESS = os.environ['AWS_ACCESS'] AWS_SECRET = os.environ['AWS_SECRET'] s3 = get_s3(AWS_ACCESS, AWS_SECRET) # select bucket bucket = s3.Bucket('voigt') # download file into current directory # print(f'found S3 buckets {bucket.objects.all()}') for s3_object in bucket.objects.all(): do_zip = False # Need to split s3_object.key into path and file name, else it will # give error file not found. if f'output_{session_id}/fitting/job_{fit_jobs[-1]}' not in s3_object.key: continue print(f'found S3 object {s3_object.key}') path, filename = os.path.split(s3_object.key) fpth = join(outputdir, filename) print(f'checking for file {s3_object.key}') if not isfile(fpth): print(f'downloading file {s3_object.key}') bucket.download_file(s3_object.key, fpth) do_zip = True if do_zip: tmp = shutil.make_archive(outputdir, 'zip', outputdir) print(f'made archive {tmp}') res = (f'/dash/download-fit?session_id={session_id}&job_id={fit_jobs[-1]}', 'fits.csv', {}, dbc.Alert('Your results are ready!', color='success'), '', False) elif job_status() == 'failed': job = Job.fetch(fit_jobs[-1], connection=conn) job_id = fit_jobs[-1] if 'JobTimeoutException' in job.exc_info: res = ('#', '', {'display': 'none'}, dbc.Alert(f'Job {session_id}:{job_id} failed due to timeout!', color='danger'), '', False) else: res = ('#', '', {'display': 'none'}, dbc.Alert(f'Job {session_id}:{job_id} failed!', color='danger'), '', False) jobdir = join(BASE_DIR, 'output', f'output_{session_id}', 'fitting', f'job_{job_id}') with open(join(jobdir, 'log.txt'), 'w') as f: f.write(job.exc_info) # upload log to S3 if os.environ.get('STACK'): AWS_ACCESS = os.environ['AWS_ACCESS'] AWS_SECRET = os.environ['AWS_SECRET'] else: with open(join(BASE_DIR, '.aws'), 'r') as f: creds = json.loads(f.read()) AWS_ACCESS = creds['access'] AWS_SECRET = creds['secret'] from voigt.common.amazon import upload_file s3_pth = join(f'output_{session_id}', 'fitting', f'job_{job_id}', 'log', 'log.txt') upload_file(join(jobdir, 'log.txt'), object_name=s3_pth, aws_access_key_id=AWS_ACCESS, aws_secret_access_key=AWS_SECRET) elif job_status() == 'started': registry = StartedJobRegistry('default', connection=conn) # TODO concurrency? what if mutliple ppl use app at same time? if (fit_jobs and fit_jobs[-1] not in registry.get_job_ids()) or not fit_jobs: msg = dbc.Alert('Ready.', color='primary') if len(os.listdir(input_dir)) > 0 \ else dbc.Alert('Upload some TGA measurements first!', color='warning') res = ('#', '', {'display': 'none'}, msg, '', True) elif fit_jobs and fit_jobs[-1] in registry.get_job_ids(): res = ('#', '', {'display': 'none'}, dbc.Alert( [ 'Please wait while your request is processed.', dbc.Spinner(type='grow') ], color='warning'), '', True ) res = list(res) data = [read_data(join(input_dir, f), format=file_format) for f in os.listdir(input_dir)] if any([x[2][0] > min_ for x in data]): msg = dbc.Alert( 'Run Start Temp outside of actual bounds!', color='warning') disabled = True elif any([x[2][-1] < max_ for x in data]): msg = dbc.Alert( 'Run End Temp outside of actual bounds!', color='warning') disabled = True else: msg = '' disabled = False res[-2] = msg res[-1] = disabled # print(res) return tuple(res)
def jobs_running(): registry = StartedJobRegistry(name=sjs.get_sjs_config()['queue'], connection=sjs.get_redis_conn()) return registry.get_job_ids()
class TestQueue(RQTestCase): def setUp(self): super(TestQueue, self).setUp() self.registry = StartedJobRegistry(connection=self.testconn) def test_add_and_remove(self): """Adding and removing job to StartedJobRegistry.""" timestamp = current_timestamp() job = Job() # Test that job is added with the right score self.registry.add(job, 1000) self.assertLess(self.testconn.zscore(self.registry.key, job.id), timestamp + 1002) # Ensure that job is properly removed from sorted set self.registry.remove(job) self.assertIsNone(self.testconn.zscore(self.registry.key, job.id)) def test_get_job_ids(self): """Getting job ids from StartedJobRegistry.""" self.testconn.zadd(self.registry.key, 1, 'foo') self.testconn.zadd(self.registry.key, 10, 'bar') self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar']) def test_get_expired_job_ids(self): """Getting expired job ids form StartedJobRegistry.""" timestamp = current_timestamp() self.testconn.zadd(self.registry.key, 1, 'foo') self.testconn.zadd(self.registry.key, timestamp + 10, 'bar') self.assertEqual(self.registry.get_expired_job_ids(), ['foo']) def test_cleanup(self): """Moving expired jobs to FailedQueue.""" failed_queue = FailedQueue(connection=self.testconn) self.assertTrue(failed_queue.is_empty()) self.testconn.zadd(self.registry.key, 1, 'foo') self.registry.move_expired_jobs_to_failed_queue() self.assertIn('foo', failed_queue.job_ids) def test_job_execution(self): """Job is removed from StartedJobRegistry after execution.""" registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) worker = Worker([queue]) job = queue.enqueue(say_hello) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) worker.perform_job(job) self.assertNotIn(job.id, registry.get_job_ids()) # Job that fails job = queue.enqueue(div_by_zero) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) worker.perform_job(job) self.assertNotIn(job.id, registry.get_job_ids()) def test_get_job_count(self): """StartedJobRegistry returns the right number of job count.""" self.testconn.zadd(self.registry.key, 1, 'foo') self.testconn.zadd(self.registry.key, 10, 'bar') self.assertEqual(self.registry.count, 2) self.assertEqual(len(self.registry), 2)
#! /usr/bin/env python import sys from redis import StrictRedis from cvgmeasure.conf import REDIS_URL_RQ from rq.registry import StartedJobRegistry from rq.utils import current_timestamp r = StrictRedis.from_url(REDIS_URL_RQ) sjr = StartedJobRegistry(sys.argv[1], connection=r) #sjr.cleanup() sjr.cleanup(timestamp=current_timestamp()+100000000000)
def perform_job(self, job): """Performs the actual work of a job. Will/should only be called inside the work horse's process. """ self.prepare_job_execution(job) with self.connection._pipeline() as pipeline: started_job_registry = StartedJobRegistry(job.origin, self.connection) try: logging.debug('perform_job in sw') job.matlab_engine = self.matlab_engine logging.debug('pj engine:' + str(self.matlab_engine)) # logging.debug('pj args,kwargs:'+str(job._args)+','+str(job._kwargs)) if len(job._args) > 0: new_args = (self.matlab_engine, ) + job._args logging.debug('tg pj new args:' + str(new_args)) job._args = new_args elif len(job._kwargs) > 0: job._kwargs['matlab_engine'] = self.matlab_engine logging.debug('tg pj new kwargs:' + str(job._kwargs)) with self.death_penalty_class( job.timeout or self.queue_class.DEFAULT_TIMEOUT): rv = job.perform() # Pickle the result in the same try-except block since we need # to use the same exc handling when pickling fails job._result = rv self.set_current_job_id(None, pipeline=pipeline) result_ttl = job.get_result_ttl(self.default_result_ttl) if result_ttl != 0: job.ended_at = utcnow() job._status = JobStatus.FINISHED job.save(pipeline=pipeline) finished_job_registry = FinishedJobRegistry( job.origin, self.connection) finished_job_registry.add(job, result_ttl, pipeline) job.cleanup(result_ttl, pipeline=pipeline) started_job_registry.remove(job, pipeline=pipeline) pipeline.execute() except Exception: job.set_status(JobStatus.FAILED, pipeline=pipeline) started_job_registry.remove(job, pipeline=pipeline) try: pipeline.execute() except Exception: pass self.handle_exception(job, *sys.exc_info()) return False if rv is None: self.log.info('Job OK') else: self.log.info('Job OK, result = {0!r}'.format(yellow( text_type(rv)))) if result_ttl == 0: self.log.info('Result discarded immediately') elif result_ttl > 0: self.log.info('Result is kept for {0} seconds'.format(result_ttl)) else: self.log.warning( 'Result will never expire, clean up result key manually') return True
def started_job_registry(self): """Returns this queue's StartedJobRegistry.""" from rq.registry import StartedJobRegistry return StartedJobRegistry(queue=self, job_class=self.job_class)
if 'REDIS_SERVER' in os.environ: redis_server = os.environ["REDIS_SERVER"] if 'REDIS_PORT' in os.environ: redis_port = int(os.environ["REDIS_PORT"]) if 'REDIS_PASSWORD' in os.environ: redis_password = os.environ["REDIS_PASSWORD"] if 'ACCESS_KEY' in os.environ: access_key = os.environ["ACCESS_KEY"] if 'SECRET_KEY' in os.environ: access_key = os.environ["SECRET_KEY"] if 'S3_URL' in os.environ: s3_url = os.environ["S3_URL"] redis_con = Redis(redis_server, redis_port, password=redis_password) train_queue = Queue('train', connection=redis_con) registry = StartedJobRegistry('train', connection=redis_con) fregistry = FinishedJobRegistry('train', connection=redis_con) flregistry = train_queue.failed_job_registry ns = Namespace('train', description='Train a vision corpus on a prepared dataset') train_job = ns.model( 'Train_Job', { 'name': fields.String(required=True, description='The name of the model that will be trained', example='project0'), 'bucket': fields.String(required=False, description='The s3 bucket target', example='bucket0'),
def setUp(self): super(TestRegistry, self).setUp() self.registry = StartedJobRegistry(connection=self.testconn)
class TestRegistry(RQTestCase): def setUp(self): super(TestRegistry, self).setUp() self.registry = StartedJobRegistry(connection=self.testconn) def test_init(self): """Registry can be instantiated with queue or name/Redis connection""" queue = Queue('foo', connection=self.testconn) registry = StartedJobRegistry(queue=queue) self.assertEqual(registry.name, queue.name) self.assertEqual(registry.connection, queue.connection) registry = StartedJobRegistry('bar', self.testconn) self.assertEqual(registry.name, 'bar') self.assertEqual(registry.connection, self.testconn) def test_key(self): self.assertEqual(self.registry.key, 'rq:wip:default') def test_custom_job_class(self): registry = StartedJobRegistry(job_class=CustomJob) self.assertFalse(registry.job_class == self.registry.job_class) def test_contains(self): registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) self.assertFalse(job in registry) self.assertFalse(job.id in registry) registry.add(job, 5) self.assertTrue(job in registry) self.assertTrue(job.id in registry) def test_add_and_remove(self): """Adding and removing job to StartedJobRegistry.""" timestamp = current_timestamp() job = Job() # Test that job is added with the right score self.registry.add(job, 1000) self.assertLess(self.testconn.zscore(self.registry.key, job.id), timestamp + 1002) # Ensure that a timeout of -1 results in a score of inf self.registry.add(job, -1) self.assertEqual(self.testconn.zscore(self.registry.key, job.id), float('inf')) # Ensure that job is properly removed from sorted set self.registry.remove(job) self.assertIsNone(self.testconn.zscore(self.registry.key, job.id)) def test_get_job_ids(self): """Getting job ids from StartedJobRegistry.""" timestamp = current_timestamp() self.testconn.zadd(self.registry.key, {'foo': timestamp + 10}) self.testconn.zadd(self.registry.key, {'bar': timestamp + 20}) self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar']) def test_get_expired_job_ids(self): """Getting expired job ids form StartedJobRegistry.""" timestamp = current_timestamp() self.testconn.zadd(self.registry.key, {'foo': 1}) self.testconn.zadd(self.registry.key, {'bar': timestamp + 10}) self.testconn.zadd(self.registry.key, {'baz': timestamp + 30}) self.assertEqual(self.registry.get_expired_job_ids(), ['foo']) self.assertEqual(self.registry.get_expired_job_ids(timestamp + 20), ['foo', 'bar']) def test_cleanup_moves_jobs_to_failed_job_registry(self): """Moving expired jobs to FailedJobRegistry.""" queue = Queue(connection=self.testconn) failed_job_registry = FailedJobRegistry(connection=self.testconn) job = queue.enqueue(say_hello) self.testconn.zadd(self.registry.key, {job.id: 2}) # Job has not been moved to FailedJobRegistry self.registry.cleanup(1) self.assertNotIn(job, failed_job_registry) self.assertIn(job, self.registry) self.registry.cleanup() self.assertIn(job.id, failed_job_registry) self.assertNotIn(job, self.registry) job.refresh() self.assertEqual(job.get_status(), JobStatus.FAILED) def test_job_execution(self): """Job is removed from StartedJobRegistry after execution.""" registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) worker = Worker([queue]) job = queue.enqueue(say_hello) self.assertTrue(job.is_queued) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) self.assertTrue(job.is_started) worker.perform_job(job, queue) self.assertNotIn(job.id, registry.get_job_ids()) self.assertTrue(job.is_finished) # Job that fails job = queue.enqueue(div_by_zero) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) worker.perform_job(job, queue) self.assertNotIn(job.id, registry.get_job_ids()) def test_job_deletion(self): """Ensure job is removed from StartedJobRegistry when deleted.""" registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) worker = Worker([queue]) job = queue.enqueue(say_hello) self.assertTrue(job.is_queued) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) job.delete() self.assertNotIn(job.id, registry.get_job_ids()) def test_get_job_count(self): """StartedJobRegistry returns the right number of job count.""" timestamp = current_timestamp() + 10 self.testconn.zadd(self.registry.key, {'foo': timestamp}) self.testconn.zadd(self.registry.key, {'bar': timestamp}) self.assertEqual(self.registry.count, 2) self.assertEqual(len(self.registry), 2) def test_clean_registries(self): """clean_registries() cleans Started and Finished job registries.""" queue = Queue(connection=self.testconn) finished_job_registry = FinishedJobRegistry(connection=self.testconn) self.testconn.zadd(finished_job_registry.key, {'foo': 1}) started_job_registry = StartedJobRegistry(connection=self.testconn) self.testconn.zadd(started_job_registry.key, {'foo': 1}) failed_job_registry = FailedJobRegistry(connection=self.testconn) self.testconn.zadd(failed_job_registry.key, {'foo': 1}) clean_registries(queue) self.assertEqual(self.testconn.zcard(finished_job_registry.key), 0) self.assertEqual(self.testconn.zcard(started_job_registry.key), 0) self.assertEqual(self.testconn.zcard(failed_job_registry.key), 0) def test_get_queue(self): """registry.get_queue() returns the right Queue object.""" registry = StartedJobRegistry(connection=self.testconn) self.assertEqual(registry.get_queue(), Queue(connection=self.testconn)) registry = StartedJobRegistry('foo', connection=self.testconn) self.assertEqual(registry.get_queue(), Queue('foo', connection=self.testconn))
def serialize_queues(instance_number, queues): if scheduler_is_here: result_list = [ dict( name=q.name, count=q.count, queued_url=url_for( ".jobs_overview", instance_number=instance_number, queue_name=q.name, registry_name="queued", per_page="8", page="1", ), failed_job_registry_count=FailedJobRegistry(q.name).count, failed_url=url_for( ".jobs_overview", instance_number=instance_number, queue_name=q.name, registry_name="failed", per_page="8", page="1", ), started_job_registry_count=StartedJobRegistry(q.name).count, started_url=url_for( ".jobs_overview", instance_number=instance_number, queue_name=q.name, registry_name="started", per_page="8", page="1", ), deferred_job_registry_count=DeferredJobRegistry(q.name).count, deferred_url=url_for( ".jobs_overview", instance_number=instance_number, queue_name=q.name, registry_name="deferred", per_page="8", page="1", ), finished_job_registry_count=FinishedJobRegistry(q.name).count, finished_url=url_for( ".jobs_overview", instance_number=instance_number, queue_name=q.name, registry_name="finished", per_page="8", page="1", ), scheduled_job_registry_count=ScheduledJobRegistry(q.name).count, scheduled_url=url_for( ".jobs_overview", instance_number=instance_number, queue_name=q.name, registry_name="scheduled", per_page="8", page="1", ), ) for q in queues ] else: result_list = [ dict( name=q.name, count=q.count, queued_url=url_for( ".jobs_overview", instance_number=instance_number, queue_name=q.name, registry_name="queued", per_page="8", page="1", ), failed_job_registry_count=FailedJobRegistry(q.name).count, failed_url=url_for( ".jobs_overview", instance_number=instance_number, queue_name=q.name, registry_name="failed", per_page="8", page="1", ), started_job_registry_count=StartedJobRegistry(q.name).count, started_url=url_for( ".jobs_overview", instance_number=instance_number, queue_name=q.name, registry_name="started", per_page="8", page="1", ), deferred_job_registry_count=DeferredJobRegistry(q.name).count, deferred_url=url_for( ".jobs_overview", instance_number=instance_number, queue_name=q.name, registry_name="deferred", per_page="8", page="1", ), finished_job_registry_count=FinishedJobRegistry(q.name).count, finished_url=url_for( ".jobs_overview", instance_number=instance_number, queue_name=q.name, registry_name="finished", per_page="8", page="1", ), ) for q in queues ] return result_list
class TestRegistry(RQTestCase): def setUp(self): super(TestRegistry, self).setUp() self.registry = StartedJobRegistry(connection=self.testconn) def test_add_and_remove(self): """Adding and removing job to StartedJobRegistry.""" timestamp = current_timestamp() job = Job() # Test that job is added with the right score self.registry.add(job, 1000) self.assertLess(self.testconn.zscore(self.registry.key, job.id), timestamp + 1002) # Ensure that a timeout of -1 results in a score of -1 self.registry.add(job, -1) self.assertEqual(self.testconn.zscore(self.registry.key, job.id), -1) # Ensure that job is properly removed from sorted set self.registry.remove(job) self.assertIsNone(self.testconn.zscore(self.registry.key, job.id)) def test_get_job_ids(self): """Getting job ids from StartedJobRegistry.""" timestamp = current_timestamp() self.testconn.zadd(self.registry.key, timestamp + 10, 'foo') self.testconn.zadd(self.registry.key, timestamp + 20, 'bar') self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar']) def test_get_expired_job_ids(self): """Getting expired job ids form StartedJobRegistry.""" timestamp = current_timestamp() self.testconn.zadd(self.registry.key, 1, 'foo') self.testconn.zadd(self.registry.key, timestamp + 10, 'bar') self.testconn.zadd(self.registry.key, timestamp + 30, 'baz') self.assertEqual(self.registry.get_expired_job_ids(), ['foo']) self.assertEqual(self.registry.get_expired_job_ids(timestamp + 20), ['foo', 'bar']) def test_cleanup(self): """Moving expired jobs to FailedQueue.""" failed_queue = FailedQueue(connection=self.testconn) self.assertTrue(failed_queue.is_empty()) queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) self.testconn.zadd(self.registry.key, 2, job.id) self.registry.cleanup(1) self.assertNotIn(job.id, failed_queue.job_ids) self.assertEqual(self.testconn.zscore(self.registry.key, job.id), 2) self.registry.cleanup() self.assertIn(job.id, failed_queue.job_ids) self.assertEqual(self.testconn.zscore(self.registry.key, job.id), None) job.refresh() self.assertEqual(job.status, JobStatus.FAILED) def test_job_execution(self): """Job is removed from StartedJobRegistry after execution.""" registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) worker = Worker([queue]) job = queue.enqueue(say_hello) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) worker.perform_job(job) self.assertNotIn(job.id, registry.get_job_ids()) # Job that fails job = queue.enqueue(div_by_zero) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) worker.perform_job(job) self.assertNotIn(job.id, registry.get_job_ids()) def test_get_job_count(self): """StartedJobRegistry returns the right number of job count.""" timestamp = current_timestamp() + 10 self.testconn.zadd(self.registry.key, timestamp, 'foo') self.testconn.zadd(self.registry.key, timestamp, 'bar') self.assertEqual(self.registry.count, 2) self.assertEqual(len(self.registry), 2) def test_clean_registries(self): """clean_registries() cleans Started and Finished job registries.""" queue = Queue(connection=self.testconn) finished_job_registry = FinishedJobRegistry(connection=self.testconn) self.testconn.zadd(finished_job_registry.key, 1, 'foo') started_job_registry = StartedJobRegistry(connection=self.testconn) self.testconn.zadd(started_job_registry.key, 1, 'foo') clean_registries(queue) self.assertEqual(self.testconn.zcard(finished_job_registry.key), 0) self.assertEqual(self.testconn.zcard(started_job_registry.key), 0)
def getmetas(): with Connection(redis.from_url(current_app.config['REDIS_URL'])): q = Queue('default') registry = StartedJobRegistry('default') f_registry = FinishedJobRegistry('default') all_task_ids = getalltasksID() # if request.method == 'GET': response_text = all_task_ids.get_data(as_text=True) response_json = all_task_ids.get_json() # data = json.load(response_json) running_tasks_ids = response_json["running"]["running_tasks_ids"] finished_tasks_ids = response_json["finished"]["finished_tasks_ids"] queued_tasks_ids = response_json["queued"]["queued_tasks_ids"] data = {} data['running_tasks'] = [] for task_id in running_tasks_ids: d = {} job = q.fetch_job(task_id) job.refresh() job.meta['result'] = 'null' d[task_id] = job.meta data['running_tasks'].append(d) data['queued_tasks'] = [] for task_id in queued_tasks_ids: d = {} job = q.fetch_job(task_id) job.refresh() job.meta['result'] = 'null' d[task_id] = job.meta data['queued_tasks'].append(d) data['finished_tasks'] = [] for task_id in finished_tasks_ids: d = {} job = q.fetch_job(task_id) job.refresh() job.meta['result'] = job.result d[task_id] = job.meta data['finished_tasks'].append(d) with Connection(redis.from_url(current_app.config['REDIS_URL'])): q = Queue('aggregator') agg_fin_task_ids = get_all_finished_tasks_from('aggregator') temp = agg_fin_task_ids.get_json() agg_fin_tasks_ids = temp["finished"]["finished_tasks_ids"] agg_que_task_ids = get_all_queued_tasks_from('aggregator') temp = agg_que_task_ids.get_json() agg_que_tasks_ids = temp["queued"]["queued_tasks_ids"] agg_run_task_ids = get_all_running_tasks_from('aggregator') temp = agg_run_task_ids.get_json() agg_run_tasks_ids = temp["running"]["running_tasks_ids"] data['agg_finished_tasks'] = [] for task_id in agg_fin_tasks_ids: d = {} job = q.fetch_job(task_id) if job is not None: job.refresh() job.meta['result'] = job.result d[task_id] = job.meta data['agg_finished_tasks'].append(d) data['agg_queued_tasks'] = [] for task_id in agg_que_tasks_ids: d = {} job = q.fetch_job(task_id) if job is not None: job.refresh() job.meta['result'] = job.result d[task_id] = job.meta data['agg_queued_tasks'].append(d) data['agg_running_tasks'] = [] for task_id in agg_run_tasks_ids: d = {} job = q.fetch_job(task_id) if job is not None: job.refresh() job.meta['result'] = job.result d[task_id] = job.meta data['agg_running_tasks'].append(d) return jsonify(data)
def count_builds(self, queue): return queue.count + len(StartedJobRegistry(queue=queue))
from rq.job import Job from rq.registry import StartedJobRegistry from webapp.apis import query from datetime import datetime from functools import partial import pandas as pd jobs_api = Blueprint('jobs_api', __name__, url_prefix='/api/jobs') redis_connection = Redis(host='redis', port=6379) q = Queue('matching', connection=redis_connection) registry = StartedJobRegistry('matching', connection=redis_connection) @jobs_api.route('/get_current_jobs', methods=['GET']) @login_required def get_current_jobs(): queued_job_ids = q.job_ids queued_jobs = q.jobs q_time = [job.enqueued_at for job in queued_jobs] q_ids = [job.meta['upload_id'] for job in queued_jobs] try: current_job_id = registry.get_job_ids() current_job_created_at = [Job.fetch(job_id, connection=redis_connection).created_at for job_id in current_job_id] current_job_upload_id = [Job.fetch(job_id, connection=redis_connection).meta['upload_id'] for job_id in current_job_id] current_job = [ { 'job_id': job_id,