def setUp(self): user = User.objects.create_user('foo', password='******') user.is_staff = True user.is_active = True user.save() self.client = Client() self.client.login(username=user.username, password='******') get_queue('django_rq_test').connection.flushall()
def test_async(self): """ Checks whether asynchronous settings work """ # Make sure async is not set by default defaultQueue = get_queue('default') self.assertTrue(defaultQueue._async) # Make sure async override works defaultQueueAsync = get_queue('default', async=False) self.assertFalse(defaultQueueAsync._async) # Make sure async setting works asyncQueue = get_queue('async') self.assertFalse(asyncQueue._async)
def job(func_or_queue=None, connection=None, *args, **kwargs): """ The same as RQ's job decorator, but it works automatically works out the ``connection`` argument from RQ_QUEUES. And also, it allows simplified ``@job`` syntax to put job into default queue. """ if callable(func_or_queue): func = func_or_queue queue = 'default' else: func = None queue = func_or_queue or 'default' if not isinstance(queue, basestring): queue = unicode(queue) try: queue = get_queue(queue) if connection is None: connection = queue.connection except KeyError: pass decorator = _job(queue, connection=connection, *args, **kwargs) if func: return decorator(func) return decorator
def test_action_requeue_jobs(self): def failing_job(): raise ValueError queue = get_queue('django_rq_test') failed_queue_index = get_failed_queue_index('django_rq_test') # enqueue some jobs that will fail jobs = [] job_ids = [] for _ in range(0, 3): job = queue.enqueue(failing_job) jobs.append(job) job_ids.append(job.id) # do those jobs = fail them worker = get_worker('django_rq_test') worker.work(burst=True) # check if all jobs are really failed for job in jobs: self.assertTrue(job.is_failed) # renqueue failed jobs from failed queue self.client.post(reverse('rq_actions', args=[failed_queue_index]), {'action': 'requeue', 'job_ids': job_ids}) # check if we requeue all failed jobs for job in jobs: self.assertFalse(job.is_failed)
def job(func_or_queue, connection=None, *args, **kwargs): """ The same as RQ's job decorator, but it works automatically works out the ``connection`` argument from RQ_QUEUES. And also, it allows simplified ``@job`` syntax to put job into default queue. """ if callable(func_or_queue): func = func_or_queue queue = 'default' else: func = None queue = func_or_queue try: from django.utils import six string_type = six.string_types except ImportError: # for django lt v1.5 and python 2 string_type = basestring if isinstance(queue, string_type): try: queue = get_queue(queue) if connection is None: connection = queue.connection except KeyError: pass decorator = _rq_job(queue, connection=connection, *args, **kwargs) if func: return decorator(func) return decorator
def check_redis(app_configs=None, **kwargs): from django_rq.queues import get_queue from django_rq.workers import Worker errors = [] try: queue = get_queue() workers = Worker.all(queue.connection) except Exception as e: conn_settings = queue.connection.connection_pool.connection_kwargs errors.append(checks.Critical(_("Could not connect to Redis (%s)") % (e), hint=_("Make sure Redis is running on %(host)s:%(port)s") % (conn_settings), id="pootle.C001", )) else: if not workers or workers[0].stopped: # We need to check we're not running manage.py rqworker right now.. import sys if len(sys.argv) > 1 and sys.argv[1] in RQWORKER_WHITELIST: errors.append(checks.Warning( _("No RQ Worker running."), hint=_("Run new workers with manage.py rqworker"), id="pootle.W001", )) redis_version = queue.connection.info()["redis_version"].split(".") if tuple(int(x) for x in redis_version) < REDIS_MINIMUM_REQUIRED_VERSION: errors.append(checks.Warning( _("Your version of Redis is too old."), hint=_("Update your system's Redis server package"), id="pootle.W002", )) return errors
def test_clear(self): queue = get_queue() delayed_queue = thread_queue.get_queue() delayed_queue.append((queue, divide, (1,), {'b': 1})) thread_queue.clear() delayed_queue = thread_queue.get_queue() self.assertEqual(delayed_queue, [])
def test_success(self): queue = get_queue() queue.empty() thread_queue.clear() self.assertEqual(queue.count, 0) self.client.get(reverse('success')) self.assertEqual(queue.count, 1)
def test_error(self): queue = get_queue() queue.empty() self.assertEqual(queue.count, 0) url = reverse('error') self.assertRaises(ValueError, self.client.get, url) self.assertEqual(queue.count, 0)
def rq_stats(): queue = get_queue() failed_queue = get_failed_queue() try: workers = Worker.all(queue.connection) except ConnectionError: return None num_workers = len(workers) is_running = len(queue.connection.smembers(Worker.redis_workers_keys)) > 0 if is_running: # Translators: this refers to the status of the background job worker status_msg = ungettext('Running (%d worker)', 'Running (%d workers)', num_workers) % num_workers else: # Translators: this refers to the status of the background job worker status_msg = _('Stopped') result = { 'job_count': queue.count, 'failed_job_count': failed_queue.count, 'is_running': is_running, 'status_msg': status_msg, } return result
def test_get_queue_django_redis(self, mocked): """ Test that the USE_REDIS_CACHE option for configuration works. """ queue = get_queue('django-redis') queue.enqueue(access_self) self.assertEqual(len(queue), 1) self.assertEqual(mocked.call_count, 1)
def test_to_localtime(self): with self.settings(TIME_ZONE='Asia/Jakarta'): queue = get_queue() job = queue.enqueue(access_self) time = to_localtime(job.created_at) self.assertIsNotNone(time.tzinfo) self.assertEqual(time.strftime("%z"), '+0700')
def test_clear_queue(self): """Test that the queue clear actually clears the queue.""" queue = get_queue("django_rq_test") queue_index = get_queue_index("django_rq_test") job = queue.enqueue(access_self) self.client.post(reverse("rq_clear", args=[queue_index]), {"post": "yes"}) self.assertFalse(Job.exists(job.id, connection=queue.connection)) self.assertNotIn(job.id, queue.job_ids)
def enqueue(func, *args, **kwargs): """ A convenience function to put a job in the default queue. Usage:: from django_rq import enqueue enqueue(func, *args, **kwargs) """ return get_queue().enqueue(func, *args, **kwargs)
def create_update_cache_job_wrapper(instance, keys, decrement=1): queue = get_queue('default') if queue._async: def _create_update_cache_job(): create_update_cache_job(queue, instance, keys, decrement=decrement) connection.on_commit(_create_update_cache_job) else: instance._update_cache_job(keys, decrement=decrement)
def test_autocommit(self): """ Checks whether autocommit is set properly. """ queue = get_queue(autocommit=True) self.assertTrue(queue._autocommit) queue = get_queue(autocommit=False) self.assertFalse(queue._autocommit) # Falls back to default AUTOCOMMIT mode queue = get_queue() self.assertFalse(queue._autocommit) queues = get_queues(autocommit=True) self.assertTrue(queues[0]._autocommit) queues = get_queues(autocommit=False) self.assertFalse(queues[0]._autocommit) queues = get_queues() self.assertFalse(queues[0]._autocommit)
def test_enqueue_autocommit_on(self): """ Running ``enqueue`` when AUTOCOMMIT is on should immediately persist job into Redis. """ queue = get_queue() job = queue.enqueue(divide, 1, 1) self.assertTrue(job.id in queue.job_ids) job.delete()
def rq_workers_are_running(): """Checks if there are any rq workers running :returns: `True` if there are rq workers running, `False` otherwise. """ if redis_is_running(): queue = get_queue() if len(queue.connection.smembers(Worker.redis_workers_keys)): return True return False
def test_get_current_job(self): """ Ensure that functions using RQ's ``get_current_job`` doesn't fail when run from rqworker (the job id is not in the failed queue). """ queue = get_queue() job = queue.enqueue(access_self) call_command('rqworker', burst=True) failed_queue = Queue(name='failed', connection=queue.connection) self.assertFalse(job.id in failed_queue.job_ids)
def test_job_decorator(self): # Ensure that decorator passes in the right queue from settings.py queue_name = 'test3' config = QUEUES[queue_name] @job(queue_name) def test(): pass result = test.delay() queue = get_queue(queue_name) self.assertEqual(result.origin, queue_name)
def commit(*args, **kwargs): """ Processes all jobs in the delayed queue. """ delayed_queue = get_queue() try: while delayed_queue: queue, args, kwargs = delayed_queue.pop(0) queue.original_enqueue_call(*args, **kwargs) finally: clear()
def test_delete_job(self): """ In addition to deleting job from Redis, the job id also needs to be deleted from Queue. """ queue = get_queue('django_rq_test') job = queue.enqueue(access_self) self.client.post(reverse('rq_delete_job', args=[queue.connection_name, queue.name, job.id]), {'post': 'yes'}) self.assertFalse(Job.exists(job.id, connection=queue.connection)) self.assertNotIn(job.id, queue.job_ids)
def test_delete_job(self): """ In addition to deleting job from Redis, the job id also needs to be deleted from Queue. """ queue = get_queue("django_rq_test") queue_index = get_queue_index("django_rq_test") job = queue.enqueue(access_self) self.client.post(reverse("rq_delete_job", args=[queue_index, job.id]), {"post": "yes"}) self.assertFalse(Job.exists(job.id, connection=queue.connection)) self.assertNotIn(job.id, queue.job_ids)
def redis_is_running(): """Checks is redis is running :returns: `True` if redis is running, `False` otherwise. """ try: queue = get_queue() Worker.all(queue.connection) except ConnectionError: return False return True
def test_get_queue_test(self): """ Test that get_queue use the right parameters for `test` connection. """ config = QUEUES["test"] queue = get_queue("test") connection_kwargs = queue.connection.connection_pool.connection_kwargs self.assertEqual(queue.name, "test") self.assertEqual(connection_kwargs["host"], config["HOST"]) self.assertEqual(connection_kwargs["port"], config["PORT"]) self.assertEqual(connection_kwargs["db"], config["DB"])
def test_get_queue_test(self): """ Test that get_queue use the right parameters for `test` connection. """ config = QUEUES['test'] queue = get_queue('test') connection_kwargs = queue.connection.connection_pool.connection_kwargs self.assertEqual(queue.name, 'test') self.assertEqual(connection_kwargs['host'], config['HOST']) self.assertEqual(connection_kwargs['port'], config['PORT']) self.assertEqual(connection_kwargs['db'], config['DB'])
def test_worker_details(self): """Worker index page should show workers for a specific queue""" queue = get_queue('django_rq_test') queue_index = get_queue_index('django_rq_test') worker = get_worker('django_rq_test', name=uuid.uuid4().hex) worker.register_birth() response = self.client.get( reverse('rq_worker_details', args=[queue_index, worker.key]) ) self.assertEqual(response.context['worker'], worker)
def test_deferred_jobs(self): """Ensure that active jobs page works properly.""" queue = get_queue('django_rq_test') queue_index = get_queue_index('django_rq_test') job = queue.enqueue(access_self) registry = DeferredJobRegistry(queue.name, queue.connection) registry.add(job, 2) response = self.client.get( reverse('rq_deferred_jobs', args=[queue_index]) ) self.assertEqual(response.context['jobs'], [job])
def get_queues(*queue_names, **kwargs): """ Return queue instances from specified queue names. All instances must use the same Redis connection. """ from .settings import QUEUES autocommit = kwargs.get("autocommit", None) if len(queue_names) == 0: # Return "default" queue if no queue name is specified return [get_queue(autocommit=autocommit)] if len(queue_names) > 1: connection_params = QUEUES[queue_names[0]] for name in queue_names: if QUEUES[name] != connection_params: raise ValueError( "Queues must have the same redis connection." '"{0}" and "{1}" have ' "different connections".format(name, queue_names[0]) ) return [get_queue(name, autocommit=autocommit) for name in queue_names]
def test_get_connection_url(self): """ Test that get_queue use the right parameters for queues using URL for connection. """ queue = get_queue('default', connection_name='url') connection_kwargs = queue.connection.connection_pool.connection_kwargs self.assertEqual(queue.name, 'default') self.assertEqual(connection_kwargs['host'], 'host') self.assertEqual(connection_kwargs['port'], 1234) self.assertEqual(connection_kwargs['db'], 4) self.assertEqual(connection_kwargs['password'], 'password')
def test_for_queue(self): queue = get_queue('test1') self.assertIsInstance(queue, DummyQueue)
def test_rq_workers_running(): from django_rq.queues import get_queue from django_rq.workers import Worker queue = get_queue() workers = Worker.all(queue.connection) return len(workers) >= 1 and not workers[0].stopped, len(workers)
def tearDown(self): get_queue().empty()
def test_default_timeout(self): """Ensure DEFAULT_TIMEOUT are properly parsed.""" queue = get_queue() self.assertEqual(queue._default_timeout, 500) queue = get_queue('test1') self.assertEqual(queue._default_timeout, 400)
def upload_view(request): """ :param request: Django Request :return: Http Response """ if preferences.Setting.active_release is None: all_msgs = messages.get_messages(request) if len(all_msgs) == 0: messages.error(request, mark_safe( _("Active release not set: you cannot publish your " "repository without an active release. <a href=\"%s\">Add Release</a>") % reverse("admin:WEIPDCRM_release_add") )) # POST if request.method == 'POST': # action: upload if 'action' in request.POST and request.POST['action'] == 'upload': if 'ajax' in request.POST and request.POST['ajax'] == 'true': result_dict = {} if 'job' in request.POST: job_id = request.POST['job'] result_dict = {} m_job = queues.get_queue('high').fetch_job(job_id) if m_job is None: result_dict.update({ 'result': False, 'msg': _('No such job'), 'job': None }) else: result_dict.update({ 'result': True, 'msg': '', 'job': { 'id': m_job.id, 'is_failed': m_job.is_failed, 'is_finished': m_job.is_finished, 'result': m_job.result } }) else: form = UploadForm(request.POST, request.FILES) if form.is_valid(): # Handle File if settings.ENABLE_REDIS is True: m_job = handle_uploaded_file(request) result_dict.update({ 'status': True, 'msg': _('Upload succeed, proceeding...'), 'job': { 'id': m_job.id, 'result': m_job.result } }) else: m_result = handle_uploaded_file(request) succeed = m_result['success'] if succeed: result_dict.update({ 'status': True, 'msg': _('Upload succeed, proceeding...'), 'job': { 'id': None, 'result': { 'version': m_result['version'] } } }) else: result_dict.update({ 'status': False, 'msg': m_result['exception'], 'job': None }) else: result_dict.update({ 'status': False, 'msg': _('Upload failed, invalid form.'), 'job': None }) return HttpResponse(json.dumps(result_dict), content_type='application/json') else: # render upload result form = UploadForm(request.POST, request.FILES) if form.is_valid(): # Handle File if settings.ENABLE_REDIS is True: m_job = handle_uploaded_file(request) job_id = m_job.id msg = _('Upload succeed, proceeding...') else: m_result = handle_uploaded_file(request) if m_result["success"] is True: return redirect(Version.objects.get(id=int(m_result["version"])).get_admin_url()) else: job_id = '' msg = m_result["exception"] else: job_id = '' msg = _('Upload failed, invalid form.') form = UploadForm() context = admin.site.each_context(request) context.update({ 'title': _('Upload New Packages'), 'form': form, 'job_id': job_id, 'msg': msg }) template = 'admin/upload.html' return render(request, template, context) # action: async-import elif 'action' in request.POST and request.POST['action'] == 'async-import': if not settings.ENABLE_REDIS: messages.error(request, mark_safe( _("To use this action, you must enable <b>Redis Queue</b>.") )) else: items = os.listdir(settings.UPLOAD_ROOT) import_items = [] for item in items: if item[-4:] == ".deb": item_path = os.path.join(settings.UPLOAD_ROOT, item) import_items.append(item_path) if len(import_items) > 0: temp_root = settings.TEMP_ROOT if not os.path.exists(temp_root): try: mkdir_p(temp_root) except OSError: pass import_jobs = [] queue = django_rq.get_queue('high') for import_item in import_items: package_temp_path = os.path.join(temp_root, str(uuid.uuid1()) + '.deb') shutil.copy(import_item, package_temp_path) os.chmod(package_temp_path, 0o755) import_job = queue.enqueue(handle_uploaded_package, package_temp_path) import_jobs.append(import_job) if len(import_jobs) == 1: messages.info(request, mark_safe(_("{job_count} package importing job have been added to the \"<a href=\"{jobs}\">high</a>\" queue.").format( job_count=str(len(import_jobs)), jobs=reverse('rq_jobs', args=(1, )), ))) else: messages.info(request, mark_safe(_("{job_count} package importing jobs have been added to the \"<a href=\"{jobs}\">high</a>\" queue.").format( job_count=str(len(import_jobs)), jobs=reverse('rq_jobs', args=(1, )), ))) else: messages.warning(request, _("There is no package to import.")) return redirect('upload') # GET elif request.method == 'GET': form = UploadForm() context = admin.site.each_context(request) context.update({ 'title': _('Upload New Packages'), 'form': form, 'job_id': '' }) template = 'admin/upload.html' return render(request, template, context)
def test_default_queue_class(self): queue = get_queue('test') self.assertIsInstance(queue, DjangoRQ)
def _enqueue_job(): queue = get_queue('default') queue.enqueue(update_translation_project, tp, initialize_from_templates, response_url)
def upload_screenshots_view(request, package_id): """ :param request: Django Request :return: Http Response """ if request.method == "POST": # Save Images To Resource Base if 'ajax' in request.POST and request.POST['ajax'] == 'true': result_dict = {} if 'job' in request.POST: job_id = request.POST['job'] result_dict = {} m_job = queues.get_queue('high').fetch_job(job_id) if m_job is None: result_dict.update({ 'result': False, 'msg': _('No such job'), 'job': None }) else: result_dict.update({ 'result': True, 'msg': '', 'job': { 'id': m_job.id, 'is_failed': m_job.is_failed, 'is_finished': m_job.is_finished, 'result': m_job.result } }) else: form = ImageForm(request.POST, request.FILES) if form.is_valid(): # Handle File if settings.ENABLE_REDIS is True: m_job = handle_uploaded_image(request, package_id) result_dict.update({ 'status': True, 'msg': _('Upload succeed, proceeding...'), 'job': { 'id': m_job.id, 'result': m_job.result } }) else: m_result = handle_uploaded_image(request, package_id) succeed = m_result['success'] if succeed: result_dict.update({ 'status': True, 'msg': _('Upload succeed, proceeding...'), 'job': { 'id': None, 'result': { 'version': m_result['version'] } } }) else: result_dict.update({ 'status': False, 'msg': m_result['exception'], 'job': None }) else: result_dict.update({ 'status': False, 'msg': _('Upload failed, invalid form.'), 'job': None }) return HttpResponse(json.dumps(result_dict), content_type='application/json') else: # render upload result form = UploadForm(request.POST, request.FILES) if form.is_valid(): # Handle File if settings.ENABLE_REDIS is True: m_job = handle_uploaded_image(request, package_id) job_id = m_job.id msg = _('Upload succeed, proceeding...') else: m_result = handle_uploaded_image(request, package_id) if m_result["success"] is True: return redirect( Version.objects.get( id=int(m_result["version"])).get_admin_url()) else: job_id = '' msg = m_result["exception"] else: job_id = '' msg = _('Upload failed, invalid form.') form = ImageForm() context = admin.site.each_context(request) context.update({ 'title': _('Upload Screenshots'), 'form': form, 'job_id': job_id, 'msg': msg }) template = 'admin/upload_image.html' return render(request, template, context) else: version = Version.objects.get(id=int(package_id)) name = version.c_name + " " + version.c_version form = ImageForm() context = admin.site.each_context(request) context.update({ 'title': _('Upload Screenshots'), 'form': form, 'drop_title': name, 'job_id': '' }) template = 'admin/upload_image.html' return render(request, template, context)
def test_in_kwargs(self): queue = get_queue('test', queue_class=DummyQueue) self.assertIsInstance(queue, DummyQueue)
def upload_view(request): if preferences.Setting.active_release is None: messages.error( request, _("Active release not set: you cannot publish your " "repository without an active release.")) """ :param request: Django Request :return: Http Response """ if request.method == 'POST': # Save Package File To Resource Base if 'ajax' in request.POST and request.POST['ajax'] == 'true': result_dict = {} if 'job' in request.POST: job_id = request.POST['job'] result_dict = {} m_job = queues.get_queue('high').fetch_job(job_id) if m_job is None: result_dict.update({ 'result': False, 'msg': _('No such job'), 'job': None }) else: result_dict.update({ 'result': True, 'msg': '', 'job': { 'id': m_job.id, 'is_failed': m_job.is_failed, 'is_finished': m_job.is_finished, 'result': m_job.result } }) else: form = UploadForm(request.POST, request.FILES) if form.is_valid(): # Handle File if settings.ENABLE_REDIS is True: m_job = handle_uploaded_file(request) result_dict.update({ 'status': True, 'msg': _('Upload succeed, proceeding...'), 'job': { 'id': m_job.id, 'result': m_job.result } }) else: m_result = handle_uploaded_file(request) succeed = m_result['success'] if succeed: result_dict.update({ 'status': True, 'msg': _('Upload succeed, proceeding...'), 'job': { 'id': None, 'result': { 'version': m_result['version'] } } }) else: result_dict.update({ 'status': False, 'msg': m_result['exception'], 'job': None }) else: result_dict.update({ 'status': False, 'msg': _('Upload failed, invalid form.'), 'job': None }) return HttpResponse(json.dumps(result_dict), content_type='application/json') else: # render upload result form = UploadForm(request.POST, request.FILES) if form.is_valid(): # Handle File if settings.ENABLE_REDIS is True: m_job = handle_uploaded_file(request) job_id = m_job.id msg = _('Upload succeed, proceeding...') else: m_result = handle_uploaded_file(request) if m_result["success"] is True: return redirect( Version.objects.get( id=int(m_result["version"])).get_admin_url()) else: job_id = '' msg = m_result["exception"] else: job_id = '' msg = _('Upload failed, invalid form.') form = UploadForm() context = admin.site.each_context(request) context.update({ 'title': _('Upload New Packages'), 'form': form, 'job_id': job_id, 'msg': msg }) template = 'admin/upload.html' return render(request, template, context) else: form = UploadForm() context = admin.site.each_context(request) context.update({ 'title': _('Upload New Packages'), 'form': form, 'job_id': '' }) template = 'admin/upload.html' return render(request, template, context)
def get_context_data(self, *args, **kwargs): can_translate = False can_translate_stats = False User = get_user_model() if self.request.user.is_superuser or self.language: can_translate = True can_translate_stats = True elif self.project: can_translate = True ctx = super().get_context_data(*args, **kwargs) lang_code, proj_code = split_pootle_path(self.pootle_path)[:2] top_scorers = User.top_scorers( project=proj_code, language=lang_code, limit=TOP_CONTRIBUTORS_CHUNK_SIZE + 1, ) top_scorers = get_top_scorers_data(top_scorers, TOP_CONTRIBUTORS_CHUNK_SIZE) can_admin_due_dates = bool(self.project and self.has_admin_access) due_date = None if can_admin_due_dates: try: due_date_obj = DueDate.objects.get(pootle_path=self.pootle_path,) due_date = { "id": due_date_obj.id, "due_on": due_date_obj.due_on, "pootle_path": due_date_obj.pootle_path, } except DueDate.DoesNotExist: due_date = { "id": 0, "due_on": 0, "pootle_path": self.pootle_path, } pending_tasks = None PENDING_TASKS_LIMIT = 3 if lang_code and self.request.user.is_authenticated: tasks = DueDate.tasks(lang_code, user=self.request.user) pending_tasks = { "total": tasks.total, "items": tasks[:PENDING_TASKS_LIMIT], } pending_jobs = 0 if self.request.user.is_superuser: from django_rq.queues import get_queue queue = get_queue() pending_jobs = queue.count has_disabled_items = self.request.user.is_superuser and any( ( getattr(item, "disabled", False) or not self.stats["total"] or self.stats["total"] < 1 ) for item in self.object.children ) ctx.update( { "page": "browse", "stats_pending_jobs": pending_jobs, "stats_refresh_attempts_count": STATS_REFRESH_ATTEMPTS_COUNT, "browsing_data": self.get_browsing_data(), "can_translate": can_translate, "can_translate_stats": can_translate_stats, "has_disabled_items": has_disabled_items, "top_scorers": remove_empty_from_dict(top_scorers), "browser_extends": self.template_extends, "can_admin_due_dates": can_admin_due_dates, "due_date": due_date, "pending_tasks": pending_tasks, } ) return ctx
def create_update_cache_job(instance, keys, decrement=1): queue = get_queue('default') queue.connection.sadd(queue.redis_queues_keys, queue.key) job_wrapper = JobWrapper.create(update_cache_job, instance=instance, keys=keys, decrement=decrement, connection=queue.connection, origin=queue.name, timeout=queue.DEFAULT_TIMEOUT) last_job_key = instance.get_last_job_key() with queue.connection.pipeline() as pipe: while True: try: pipe.watch(last_job_key) last_job_id = queue.connection.get(last_job_key) depends_on_wrapper = None if last_job_id is not None: pipe.watch(Job.key_for(last_job_id), JobWrapper.params_key_for(last_job_id)) depends_on_wrapper = JobWrapper(last_job_id, queue.connection) pipe.multi() depends_on_status = None if depends_on_wrapper is not None: depends_on = depends_on_wrapper.job depends_on_status = depends_on.get_status() if depends_on_status is None: # enqueue without dependencies pipe.set(last_job_key, job_wrapper.id) job_wrapper.save_enqueued(pipe) pipe.execute() break if depends_on_status in [JobStatus.QUEUED, JobStatus.DEFERRED]: new_job_params = \ depends_on_wrapper.merge_job_params(keys, decrement, pipeline=pipe) pipe.execute() msg = 'SKIP %s (decrement=%s, job_status=%s, job_id=%s)' msg = msg % (last_job_key, new_job_params[1], depends_on_status, last_job_id) logger.debug(msg) # skip this job return None pipe.set(last_job_key, job_wrapper.id) if depends_on_status not in [JobStatus.FINISHED]: # add job as a dependent job = job_wrapper.save_deferred(last_job_id, pipe) pipe.execute() logger.debug('ADD AS DEPENDENT for %s (job_id=%s) OF %s' % (last_job_key, job.id, last_job_id)) return job job_wrapper.save_enqueued(pipe) pipe.execute() break except WatchError: logger.debug('RETRY after WatchError for %s' % last_job_key) continue logger.debug('ENQUEUE %s (job_id=%s)' % (last_job_key, job_wrapper.id)) queue.push_job_id(job_wrapper.id)
def test_get_queues_same_connection(self): """ Checks that getting queues with the same redis connection is ok. """ self.assertEqual(get_queues('test', 'test2'), [get_queue('test'), get_queue('test2')])
def handle(self, **options): queue = get_queue() failed_job_registry = FailedJobRegistry(queue.name, queue.connection) for job_id in failed_job_registry.get_job_ids(): failed_job_registry.requeue(job_id)