def test_service(service): task_uuid = str(uuid4()) monitored_phrases, cached_wordchecks = _get_monitored_phrases(service) cases = _get_test_cases(service, monitored_phrases) start_time = int(time.time()) cases_cache = {} jobs = [] for case in cases: agent_queue = django_rq.get_queue(case['agent_queue_name']) case.update({'uuid': task_uuid}) job = agent_queue.enqueue_call( func=run_test, kwargs={'config': case, 'start_time': start_time}, timeout=45, result_ttl=180, ) jobs.append((job.id, case['agent_queue_name'])) cases_cache[job.id] = case queue = django_rq.get_queue( name='monitors' if 'monitors' in settings.RQ_QUEUES else 'default', ) queue.enqueue_call( func=_test_service_summary, kwargs={ 'service_id': service.id, 'task_uuid': task_uuid, 'start_time': start_time, 'jobs': jobs, 'sensitivity': service.sensitivity, 'cases': cases_cache, 'wordchecks': cached_wordchecks, }, timeout=45, result_ttl=0, )
def summary_export(self, bundle, request): self.method_check(request, allowed=['get']) # check from cache cache_key = "summary_export_{type}_{query}" \ .format( type = request.GET.get("type", "all"), query = hashlib.md5(request.GET.get("q", "null")).hexdigest()) response_in_cache = utils.topic_cache.get(self.topic, cache_key) if response_in_cache: # could be empty or str("<filename>") logger.debug("export already exist from cache") response = dict(status="ok", file_name=response_in_cache) else: # return a quick response response = dict( status = "enqueued") # check if a job already exist for job in django_rq.get_queue('high').jobs: if job.meta["cache_key"] == cache_key: response["token"] = job.id logger.debug("job_already_exist") break else: # enqueue the job queue = django_rq.get_queue('high', default_timeout=360) job = queue.enqueue(render_csv_zip_file, topic = self.topic, model_type = request.GET.get("type"), query = json.loads(request.GET.get('q', 'null')), cache_key = cache_key) # save the cache_key in the meta data in order to check if a job already exist for this key later job.meta["cache_key"] = cache_key job.save() response['token'] = job.id self.log_throttled_access(request) return response
def delete_all_tasks_from_queue(queue_name): if queue_name == "failed": q = django_rq.get_failed_queue() elif queue_name == "parser": q = django_rq.get_queue("parser") else: q = django_rq.get_queue("default") while True: job = q.dequeue() if not job: break job.delete()
def showStats(self): import pprint q = django_rq.get_queue() qp = django_rq.get_queue('needs_passphrase') s = django_rq.get_scheduler() pp = pprint.PrettyPrinter() print print 'Job Queue:' pp.pprint( q.jobs ) pp.pprint( qp.jobs ) print print 'Scheduled tasks:' pp.pprint( s.get_jobs(with_times=True) ) print
def handle(self, *args, **kwargs): queue = django_rq.get_queue('default') return queue.enqueue_call( func=foo, args=(random.randint(1, 100),), ).id
def fetch_job(job_id): queue = django_rq.get_queue() try: job = queue.fetch_job(job_id) except: job = queue.safe_fetch_job(job_id) return job
def handle(self, *args, **kwargs): queue = django_rq.get_queue('default') return queue.enqueue_call( func=evil, timeout=15, ).id
def get_data(self, *args, **kwargs): cache_key = get_cache_key( self.data_provider.func_name, *args, **kwargs ) cache = get_cache( self.data_provider.async_report_cache_alias, ) data = cache.get(cache_key) if data is not None: return None if data == 'in progress' else data cache.set( cache_key, 'in progress', self.data_provider.async_report_results_expiration, ) queue = django_rq.get_queue( name='reports' if 'reports' in settings.RQ_QUEUES else 'default', ) queue.enqueue_call( func='%s.%s' % ( self.data_provider.__module__, self.data_provider.func_name, ), args=args, kwargs=kwargs, timeout=3600, result_ttl=0, )
def old_fashioned_uploads(self, request): """ Admin view to upload file from archiprod-uploads server and monitor archive currently transfered. See archive command for effective archive transfert. """ opts = self.model._meta app_label = opts.app_label if request.method == 'POST': form = UploadFileFromServer(request.POST) if form.is_valid(): file_path = form.cleaned_data['file_path'] media = form.cleaned_data['media'] media_id = None if media is not None: media_id = media.id queue = django_rq.get_queue('archive') queue.enqueue(call_command, args=('archive', file_path, media_id), timeout=86400) form = UploadFileFromServer() return render_to_response('admin/archives/media/upload-from-server.html', {'form': form, 'app_label': app_label, 'opts':opts, 'current_files_copied_in_queue': self._current_files_copied_in_queue(), 'current_file_copied_in_progress': self._current_file_copied_in_progress(), 'failed_copies': self._failed_copies() }, context_instance=RequestContext(request))
def healthcheck(request): ''' Return a JSON response with health-related information about the current state of the app. For more details, see `docs/monitoring.md`. ''' canonical_url = get_canonical_url(request) request_url = request.build_absolute_uri() results = { 'version': __version__, 'canonical_url': canonical_url, 'request_url': request_url, 'canonical_url_matches_request_url': canonical_url == request_url, 'rq_jobs': len(django_rq.get_queue().jobs), **get_database_info(), } ok = True if not (results['is_database_synchronized'] and results['canonical_url_matches_request_url']): ok = False # We're always returning 200 but indicating whether everything # is *really* ok in the `is_everything_ok` key. We used to # return 500 if the healthcheck failed, but this ended up # causing odd behavior with CloudFront. For more details, see: # # https://github.com/18F/calc/issues/1516 results['is_everything_ok'] = ok return JsonResponse(results, status=200)
def async_get_result_rows(context, view, query_options, job_options=None): """ Creates a new job to asynchronously get result rows and returns the job ID. Args: See get_result_rows argument list. Keyword Arugments: Set as properties on the returned job's meta. Returns: The ID of the created job. """ if not job_options: job_options = {} queue = get_queue(settings.ASYNC_QUEUE) job = queue.enqueue(get_result_rows, context, view, query_options, evaluate_rows=True) job.meta.update(job_options) job.save() return job.id
def start_searchable_activities_task(counter=0): workers = Worker.all(connection=redis_conn) queue = django_rq.get_queue("parser") has_other_jobs = False already_running_update = False for w in workers: if len(w.queues): if w.queues[0].name == "parser": current_job = w.get_current_job() if current_job: if ('start_searchable_activities_task' not in current_job.description): has_other_jobs = True if ('update_searchable_activities' in current_job.description): already_running_update = True if already_running_update: # update_searchable_activities already running or other # start_searchable_activities_task running, invalidate task pass elif not has_other_jobs: queue.enqueue(update_searchable_activities) elif counter > 180: raise Exception( "Waited for 30 min, still jobs runnings so invalidating this task. \ If this happens please contact OIPA devs!") else: counter += 1 time.sleep(120) queue.enqueue(start_searchable_activities_task, args=(counter,), timeout=300)
def discover_all(interactive=False, update_existing=False, outputs=None): """Runs discovery on all networks defined in the database.""" sanity_check() if outputs: stdout, stdout_verbose, stderr = outputs else: stdout = output.get(interactive) nets = Network.objects.filter( environment__isnull=False, environment__queue__isnull=False, ) for net in nets: if interactive: discover_network( net.network, interactive=True, update_existing=True, ) else: queue = django_rq.get_queue() queue.enqueue( discover_network, net.network, update_existing=update_existing, ) stdout()
def force_parse_by_publisher_ref(org_ref): queue = django_rq.get_queue("parser") for e in Dataset.objects.filter(publisher__publisher_iati_id=org_ref): queue.enqueue(force_parse_source_by_id, args=(e.id,), timeout=14400) if settings.ROOT_ORGANISATIONS: queue.enqueue(start_searchable_activities_task, args=(0,), timeout=300)
def _select_run_method(context, interactive, function, after): """Return a function that either executes the task directly (if `interactive` is True), enqueues it right away or schedules its enqueueing (if `after` is given). """ if interactive: return function set_queue(context) if after: # FIXME: what about timeout= and result_ttl= for scheduled tasks? scheduler = django_rq.get_scheduler(context['queue'], ) if isinstance(after, timedelta): enqueue = scheduler.enqueue_in elif isinstance(after, datetime): enqueue = scheduler.enqueue_at else: raise NotImplementedError( "after={!r} not supported.".format(after), ) return partial(enqueue, after, function) queue = django_rq.get_queue( context['queue'], ) return partial(_enqueue, queue, function)
def test_media_encoded_state(self): #we create media without encoding self.media = MediaFactory.build(title='test_media_encoded_state', file__from_path=os.path.join(os.path.dirname(__file__), 'tests/data/audio-mini.mp3')) self.media.save(encode=False) #check it has been well created and encoding state is not encoded self.assertEqual(self.media.id, 1) self.assertEqual(self.media.encoding_state, ENCODING_NOT_ENCODED) #put it in encoding queue queue = django_rq.get_queue('default') job = queue.enqueue(call_command, args=('encode', self.media.id)) #check job is in encoding queue self.assertTrue(job.is_queued) #start encoding worker = get_worker('default') worker.work(burst=True) #check encoding state is encoded, job is now in finished status self.assertEqual(self.media.encoding_state, ENCODING_ENCODED) self.assertFalse(job.is_queued) self.assertTrue(job.is_finished) failed_queue = Queue(name='failed', connection=queue.connection) self.assertFalse(job.id in failed_queue.job_ids)
def Redeem(self): if self.status == Gift.GIFT_STATUS_CREATED: try: card = Card.Card.objects.filter(vendor=self.product.vendor, user=None, master=False)[:1] except Card.Card.DoesNotExist: card = None expireTimeUTC = None if card: card = card[0] card.user = self.receiver card.save() queue = rq.get_queue('high') load = queue.enqueue(Card.SetBalance, card.id, self.product.price) scheduler = rq.get_scheduler('low') expireTimeUTC = datetime.utcnow() + timedelta(minutes=5) expireTime = datetime.now() + timedelta(minutes=5) unload = scheduler.enqueue_at(expireTime, CheckRedemption, self.id) self.activated = datetime.utcnow().replace(tzinfo=utc) self.status = self.GIFT_STATUS_ACTIVE self.payment = card self.save() elif self.status == Gift.GIFT_STATUS_ACTIVE: card = self.payment expireTimeUTC = self.activated + timedelta(minutes=5) return (card, expireTimeUTC)
def obj_get(self, bundle, **kwargs): """ Returns redis document from provided id. """ queue = django_rq.get_queue('default') job = Job.fetch(kwargs['pk'], connection=queue.connection) return Document(**job.__dict__)
def save(self, *args, **kwargs): if self.name: self.name = self.name.strip() if (self.status == 'ready for hathi') or (self.status == 'retry'): if self.status == 'retry': # Reset the status on the failed KDips so they will be retried. for k in self.kdip_set.all(): if k.status == 'upload_fail': k.status = 'new' k.save() # Send volumes to the upload task. self.status = 'uploading' # Add the celery task. # At this point the work is passed off to rq and executes # `tasks.py` from tasks import upload_for_ht queue = django_rq.get_queue('high') queue.enqueue(upload_for_ht, self) elif self.status == 'ready for zephir': zephir_status = send_to_zephir(self) # Set status self.status = zephir_status super(Job, self).save(*args, **kwargs)
def test_process_header_only(self): """ Test that the task will properly process the file and extract location and scan_number from the filename, with the rest of the values being default. """ file_data = ('\\*File list\n' '\\Version: 0x05120130\n' '\\*File list end\n') scan_file = SimpleUploadedFile('s0001a_r.001', file_data.encode('cp1252'), content_type='text/plain') queue = get_queue('default', async=False) job = queue.enqueue(tasks.process_nanoscope_file, scan_file) self.assertEqual(len(job.result), 1) data = { 'rms': 0.0, 'zrange': 0.0, 'size': 0.0, 'scan_number': 1, 'location': 'r', 'image_type': 'Raw', 'state': 'raw', 'content_type': 'application/octet-stream', } self.assertDictEqual(job.result[0].kwargs, data)
def import_file(request): """Import a CSV or XLS file into the database. This view will create a report with the uploaded file and the email address especified on the form, then it will save the querystrings (if any) and get the full path to the uploaded file. After that we just queue the processing task with django-rq. Args: request: The main request Returns: A template confirming that the task has been queued """ form = ReportForm(request.POST, request.FILES) if request.method == 'POST': try: # Save the data report = Report(email=request.POST.get("email", ''), original_file=request.FILES['original_file'], label=str(datetime.now())) report.save() # Start the task queue queue = django_rq.get_queue('importer') queue.enqueue(trigger_queue, args=(request.META['QUERY_STRING'], report)) return render_to_response('thanks.html') except Exception as e: logger.error("CRITICAL ERROR: THE TASK COULDN'T BE EXECUTED.") logger.error(e) return render_to_response('error.html') else: return render_to_response('import.html', {'form': form}, context_instance=RequestContext(request))
def handle(self, *args, **kwargs): if sum([kwargs['limit'], kwargs['job_id']]) > 1: raise CommandError("You can't mix limit and job_id options.") elif sum([kwargs['limit'], kwargs['job_id']]) == 0: raise CommandError("Choose something...") if not args: raise CommandError("Specify limit or job_id.") queue = django_rq.get_queue('default') if kwargs.get('limit', False): try: limit = int(args[0]) except (IndexError, ValueError) as e: raise CommandError(e) job = queue.enqueue_call( func=prime_numbers, args=(limit,), timeout=3600, result_ttl=60, ) self.stdout.write("JOB ID = %s" % job.id) if kwargs.get('job_id', False): job = rq.job.Job.fetch(args[0], django_rq.get_connection()) while job.result is None and not job.is_failed: time.sleep(0.5) sys.stdout.write('.') sys.stdout.flush() if job.is_failed: self.stdout.write('\nJOB FAILED\n') else: self.stdout.write('\n%s\n' % job.result)
def ChangeInitiatorHelper(requestDic,owner): ''' Change the initiator for SCST (do not change it in the saturnring DB though) ''' logger = getLogger(__name__) try: user = User.objects.get(username=owner); iqntar = requestDic['iqntar'] newini = requestDic['newini'] target = Target.objects.get(owner=user,iqntar=iqntar); except: errorstring = format_exc() logger.error(errorstring) return (-1,errorstring) config = ConfigReader() numqueues = config.get('saturnring','numqueues') queuename = 'queue'+str(hash(target.targethost)%int(numqueues)) queue = get_queue(queuename) job = queue.enqueue(ExecChangeInitiator,args=(iqntar,newini),timeout=45,ttl=60) while (job.result != 0) and (job.result != -1) : sleep(1) logger.info("...Working on changing target %s initiator name to %s" %(iqntar,newini)) #logger.info(str(job)) return (job.result,str(job))
def save(self, encode=True, *args, **kwargs): # Set slug to the media if not self.slug: if self.work: if self.work.composers : composers = '-'.join(["%s" % (slugify(c.__unicode__())) for c in self.work.composers.all()]) self.slug = 'x%s_%s-%s' % (hexlify(os.urandom(3)), slugify(self.work.title), slugify(composers)) else: self.slug = 'x%s_%s' % (hexlify(os.urandom(3)), slugify(self.work.title)) self.slug = self.slug[:50] elif self.title: self.slug = 'x%s_%s' % (hexlify(os.urandom(3)), slugify(self.title[:42])) else: self.slug = 'x%s' % hexlify(os.urandom(3)) if self.file: # Set media duration (if file exists) self.duration = get_media_duration(self.file.path) # Set mimetype self.mime_type = mimetypes.guess_type(self.file.path)[0] if self.mime_type is None: # TODO: this is due to original archiprod archives files # guesstype doesn't find the mimetype because theses archives # doesn't have extension name if self.file.name.find('VI') != -1: self.mime_type = 'video' super(Media, self).save(*args, **kwargs) # encode param allow to bypass the encoding process if self.file and encode: # Call asynchronous encode command if self.encoding_state == ENCODING_NOT_ENCODED: queue = django_rq.get_queue('default') queue.enqueue(call_command, args=('encode', self.id, ), timeout=86400)
def _current_files_copied_in_queue(self): """ return current archives being processed (from server to archiprod) """ _in_queue = [] queue = django_rq.get_queue('archive') for index, job in enumerate(queue.jobs): _in_queue.append(job.args[0]) return _in_queue
def scan_address(address, plugins): """Queue manual discovery on the specified address.""" try: network = Network.from_ip(address) except IndexError: raise NoQueueError( "Address {0} doesn't belong to any configured " "network.".format(address), ) if not network.queue: raise NoQueueError( "The network {0} has no discovery queue.".format(network), ) ipaddress, created = IPAddress.objects.get_or_create(address=address) queue_name = network.queue.name queue = django_rq.get_queue(queue_name) job = queue.enqueue_call( func=_scan_address, args=( address, plugins, ), kwargs={ 'snmp_community': ipaddress.snmp_community, # 'snmp_version': ipaddress.snmp_version, 'snmp_version': '2c', 'http_family': ipaddress.http_family, 'snmp_name': ipaddress.snmp_name, }, timeout=60, result_ttl=3600, ) return job
def receive_resources_async(self): aws_user = AwsUser.objects.get(user=self.user) aws_user.resources_last_updated = datetime.now(timezone.utc) aws_user.save() self.initialize_connections() aws_accounts = aws_user.aws_accounts.all() regions = Region.objects.all() params = {'connections': self.__CONNECTIONS, 'regions': regions, 'aws_accounts': aws_accounts} queue = django_rq.get_queue('high') queue.enqueue(receive_keypairs_async, **params) queue.enqueue(receive_security_groups_async, **params) queue.enqueue(receive_amis_async, **params) queue.enqueue(receive_instances_async, **params) queue.enqueue(add_instance_prices) queue.enqueue(receive_snapshots_async, **params) queue.enqueue(receive_volumes_async, **params) queue.enqueue(receive_elastic_ips_async, **params) queue.enqueue(receive_load_balancers_async, self.__ELB_CONNECTIONS, regions, aws_accounts)
def clear_queue(self): """Clear all queued analyze_frame jobs, and their corresponding frames""" frame_class = self.get_frame_class() queue = django_rq.get_queue() jobs = queue.get_jobs() cleared = 0 frames_deleted = 0 for job in jobs: # Delete jobs for this task but not scheduler jobs if (job.meta.get('analysis.task.key') == self.key) and not job.meta.get('analysis.task.schedule'): cleared += 1 job.cancel() frame_id = job.meta.get('analysis.frame.id') if frame_id: # Delete the corresponding frame try: frame_class.objects.filter(pk=frame_id, calculated=False).delete() frames_deleted += 1 except Exception as e: logger.warn(e, exc_info=True) return cleared, frames_deleted
def DeleteTarget(requestDic,owner): ''' Delete iSCSI target This function dispatches a request to the worker queue of the Saturn host to delete the object (iSCSI target object). R: RequestDic - which may contain one of the following iqntarget name, initiator name (all targets provisioned for that initiator), targethost (DNS name of Saturn server). ''' logger = getLogger(__name__) queryset = None if 'iqntar' in requestDic: queryset=Target.objects.filter(iqntar=requestDic['iqntar'],owner=owner) if 'iqnini' in requestDic: if queryset is None: queryset=Target.objects.filter(iqnini=requestDic['iqnini'],owner=owner) else: queryset=queryset.objects.filter(iqnini=requestDic['iqnini']) if 'targethost' in requestDic: if queryset is None: queryset=Target.objects.filter(targethost=requestDic['targethost'],owner=owner) else: queryset=queryset.objects.filter(targethost=requestDic['targethost']) if queryset is None: return (1,"No targets to delete, or check delete API call") config = ConfigReader() numqueues = config.get('saturnring','numqueues') jobs =[] logger.info("DeleteTarget has %d targets to delete" % (queryset.count())) for obj in queryset: logger.info("DeleteTarget Working on deleting target %s" % (obj.iqntar,)) queuename = 'queue'+str(hash(obj.targethost)%int(numqueues)) queue = get_queue(queuename) jobs = [] jobs.append( (queue.enqueue(DeleteTargetObject,args=(obj.iqntar,),timeout=45,ttl=60), obj.iqntar) ) logger.info("Using queue %s for deletion" %(queuename,)) rtnStatus= {} rtnFlag=0 numDone=0 while numDone < len(jobs): ii=0 sleep(1) for ii in range(0,len(jobs)): if jobs[ii] == 0: continue (job,target) = jobs[ii] if (job.result == 0) or (job.result == 1) or job.is_failed: if job.result==1 or job.is_failed: logger.error("Failed deletion of " + target) rtnStatus[target] = "Failed deletion of " + target rtnFlag=rtnFlag + job.result + int(job.is_failed) jobs[ii]=0 numDone=numDone+1 else: logger.info('...Working on deleting target '+target) break return (rtnFlag,str(rtnStatus))
def cron_update_index(sender, **kwargs): index_redis = get_redis_connection('djrq') index_count = int(index_redis.incr(settings.INDEX_NAME)) if index_count > settings.INDEX_COUNT + 1: index_redis.set(settings.INDEX_NAME, 0) index_queue = django_rq.get_queue(settings.INDEX_QUEUE) if index_queue.count < 1: index_queue.enqueue(warp_update_index)
def send_mails_smtp(request): queue_name = os.path.basename(settings.BASE_DIR) queue = django_rq.get_queue(queue_name) for (idx, row) in enumerate(request.session['csv']): kwargs = { 'idx': idx, 'row': row, 'session': request.session, 'mode': 'smtp', } queue.enqueue(send_msg, ttl=100, **kwargs)
def add_task(request): task = request.GET.get('task') parameters = request.GET.get('parameters') queue_to_be_added_to = request.GET.get('queue') queue = django_rq.get_queue(queue_to_be_added_to) func = getattr(tasks, task) if parameters: queue.enqueue(func, args=(parameters,)) else: queue.enqueue(func) return HttpResponse(json.dumps(True), content_type='application/json')
def test_deferred_jobs(self): """Ensure that active jobs page works properly.""" queue = get_queue('django_rq_test') queue_index = get_queue_index('django_rq_test') job = queue.enqueue(access_self) registry = DeferredJobRegistry(queue.name, queue.connection) registry.add(job, 2) response = self.client.get( reverse('rq_deferred_jobs', args=[queue_index]) ) self.assertEqual(response.context['jobs'], [job])
def test_delete_job(self): """ In addition to deleting job from Redis, the job id also needs to be deleted from Queue. """ queue = get_queue('django_rq_test') queue_index = get_queue_index('django_rq_test') job = queue.enqueue(access_self) self.client.post(reverse('rq_delete_job', args=[queue_index, job.id]), {'post': 'yes'}) self.assertFalse(Job.exists(job.id, connection=queue.connection)) self.assertNotIn(job.id, queue.job_ids)
def add_to_parse_queue(self, request): xml_id = request.GET.get('xml_id') obj = get_object_or_404(Dataset, pk=xml_id) queue = django_rq.get_queue("parser") queue.enqueue(force_parse_source_by_url, args=(obj.source_url, True), timeout=7200) # This is needed for direct debugging # force_parse_source_by_url(obj.source_url, True) return HttpResponse('Success')
def test_job_details_on_deleted_dependency(self): """Page doesn't crash even if job.dependency has been deleted""" queue = get_queue('default') queue_index = get_queue_index('default') job = queue.enqueue(access_self) second_job = queue.enqueue(access_self, depends_on=job) job.delete() url = reverse('rq_job_detail', args=[queue_index, second_job.id]) response = self.client.get(url) self.assertEqual(response.status_code, 200) self.assertIn(second_job._dependency_id, response.content.decode())
def generate_pdf_async(source_path, output_path, data): queue = django_rq.get_queue('inkpy') return queue.enqueue_call( func=generate_pdf, args=( source_path, output_path, data, ), timeout=600, ).id
def update_storage(self): """ Update control fields and write to deb files This method is executed async. """ control = self.get_control_dict() path = self.storage.name if settings.ENABLE_REDIS is True: queue = django_rq.get_queue('high') queue.enqueue(write_to_package_job, control, path, self.id) else: write_to_package_job(control, path, self.id)
def enqueue_webhooks(instance, user, request_id, action): """ Find Webhook(s) assigned to this instance + action and enqueue them to be processed """ # Determine whether this type of object supports webhooks app_label = instance._meta.app_label model_name = instance._meta.model_name if model_name not in registry['model_features']['webhooks'].get( app_label, []): return # Retrieve any applicable Webhooks content_type = ContentType.objects.get_for_model(instance) action_flag = { ObjectChangeActionChoices.ACTION_CREATE: 'type_create', ObjectChangeActionChoices.ACTION_UPDATE: 'type_update', ObjectChangeActionChoices.ACTION_DELETE: 'type_delete', }[action] webhooks = Webhook.objects.filter(content_types=content_type, enabled=True, **{action_flag: True}) if webhooks.exists(): # Get the Model's API serializer class and serialize the object serializer_class = get_serializer_for_model(instance.__class__) serializer_context = { 'request': None, } serializer = serializer_class(instance, context=serializer_context) # Gather pre- and post-change snapshots snapshots = { 'prechange': getattr(instance, '_prechange_snapshot', None), 'postchange': serialize_object(instance) if action != ObjectChangeActionChoices.ACTION_DELETE else None, } # Enqueue the webhooks webhook_queue = get_queue('default') for webhook in webhooks: webhook_queue.enqueue("extras.webhooks_worker.process_webhook", webhook=webhook, model_name=instance._meta.model_name, event=action, data=serializer.data, snapshots=snapshots, timestamp=str(timezone.now()), username=user.username, request_id=request_id)
def add_task(request): import django_rq task = request.GET.get('task') parameters = request.GET.get('parameters') queue_to_be_added_to = request.GET.get('queue') queue = django_rq.get_queue(queue_to_be_added_to) if parameters: queue.enqueue(getattr(tasks, task), args=(parameters, ), timeout=7200) else: queue.enqueue(getattr(tasks, task), timeout=7200) return HttpResponse('Success')
def get_filings(request): for j in django_rq.get_queue('default').jobs: if j.func_name == 'filings.parser.parse_filing' and j.result == None: return HttpResponse('parsing filings, please wait a minute and refresh...') state = request.GET.get('state') or 'ALL' if r.exists(state): state_data = json.loads(r.get(state)) return JsonResponse(state_data, json_dumps_params={'indent': 2}) django_rq.enqueue(_get_filings_async, state) return HttpResponse('generating JSON output, please wait a few seconds and refresh...')
def run(*args): print("Empty queues") django_rq.get_queue('bot').empty() django_rq.get_queue('listen').empty() # Stop existing jobs registry = StartedJobRegistry('bot', connection=django_rq.get_connection('bot')) running_ids = registry.get_job_ids() if len(running_ids) > 1: for i in running_ids: current_job = django_rq.get_queue('bot').fetch_job(i) print("Delete : ", current_job) current_job.delete() else: for i in running_ids: current_job = django_rq.get_queue('bot').fetch_job(i) print("Send kill : ", current_job) current_job.meta['kill'] = "true" current_job.save_meta() if args and len(args) > 0 and args[0] == "stop": return print("Launch bot job") print(launch_bot.delay())
def bulk_send_new_attempt_email(**kwargs): """ Queue RQ job for sending out notifications to users when they have been given a new attempt. Adds :meth:`bulk_deadline_email` to the RQ-queue. """ kwargs.update({ 'template_name': 'devilry_email/deadline_email/new_attempt.txt', 'deadline_type': 'new_attempt' }) queue = django_rq.get_queue(name='email') queue.enqueue(bulk_deadline_email, **kwargs)
def force_parse_source_by_id(source_id, update_searchable=False): try: xml_source = Dataset.objects.get(pk=source_id) xml_source.process(force_reparse=True) queue = django_rq.get_queue("parser") if update_searchable and settings.ROOT_ORGANISATIONS: queue.enqueue(start_searchable_activities_task, args=(0, ), timeout=300) except Dataset.DoesNotExist: return False
def run(self): while True: self.logger.info('cron update index start') index_queue = django_rq.get_queue(settings.INDEX_QUEUE) if index_queue.count < 1 : index_redis = get_redis_connection('djrq') index_count = int(index_redis.get(settings.INDEX_NAME)) if index_redis.get(settings.INDEX_NAME) else 0 if index_count > 0: self.logger.info('index count is ' + str(index_count) + ', cron update index enqueue') index_redis.set(settings.INDEX_NAME, 0) index_queue.enqueue(warp_update_index) self.logger.info('cron update index done, sleep ' + str(settings.INDEX_TIME) + '\n*********************') time.sleep(settings.INDEX_TIME)
def post(self, request, format=None): queue = django_rq.get_queue('high') job = queue.enqueue(regimes_clustering_run, request.data['start'], request.data['end'], request.data['capital_base'], request.data['ticker'], request.data['use_clf'], request.data['no_shorts'], request.data['log_channel']) json = {'success': True, 'job_id': job.key} return Response(json)
def get_context_data(self, *args, **kwargs): context = super(QueuedJobsMixin, self).get_context_data(*args, **kwargs) # Add a list of job items currently active for the object obj = self.object lname = generate_object_index(obj) # Retrieve a list of jobs conn = get_connection() queue = django_rq.get_queue() context['queued_jobs'] = [queue.fetch_job(job_id) for job_id in conn.lrange(lname, 0, 10)] return context
def start(self, sync=False): if self.state == states.PENDING: if sync: method = self._get_function_from_string(self.name) method(self.pk, sync=True) else: queue = django_rq.get_queue(self.queue or "default") queue.enqueue(self.name, self.pk) self.state = states.STARTED self.save(update_fields=["state", "updated_at"]) signals.job_started.send(sender=self.__class__, job=self) return True return False
def post(self, request, format=None): """ Some description for posts""" data = self.serialize(request) if not isinstance(data, dict): return data data["domain_name"] = request.META.get('HTTP_HOST', '') # if request.POST.get('synchronous_job', False): # return self.worker_class.run(data) # else: job = django_rq.get_queue("default").enqueue(self.worker_class.run, data, result_ttl=180) return Response({"job_id": job.id}, status=status.HTTP_200_OK)
def test_jobs(self): """Jobs in queue are displayed properly""" queue = get_queue('default') job = queue.enqueue(access_self) queue_index = get_queue_index('default') response = self.client.get(reverse('rq_jobs', args=[queue_index])) self.assertEqual(response.context['jobs'], [job]) # This page shouldn't fail when job.data is corrupt queue.connection.hset(job.key, 'data', 'unpickleable data') response = self.client.get(reverse('rq_jobs', args=[queue_index])) self.assertEqual(response.status_code, 200) self.assertIn('UnpicklingError', response.content.decode('utf-8'))
def cancel_rq_job(self): """ Cancels the RQ job corresponding to me. """ try: logger.debug(f"cancel_rq_job(): Started: {self}") queue = django_rq.get_queue(UPLOAD_FILE_QUEUE_NAME) job = queue.fetch_job(self.rq_job_id()) job.cancel( ) # NB: just removes it from the queue and won't will kill it if is already executing logger.debug(f"cancel_rq_job(): done: {self}") except Exception as ex: logger.debug(f"cancel_rq_job(): Failed: {ex}, {self}")
def bulk_send_deadline_moved_email(**kwargs): """ Queue RQ job for sending out notifications to users when their deadline is moved. Adds :meth:`bulk_deadline_email` to the RQ-queue. """ kwargs.update({ 'template_name': 'devilry_email/deadline_email/deadline_moved.txt', 'deadline_type': 'moved' }) queue = django_rq.get_queue(name='email') queue.enqueue(bulk_deadline_email, **kwargs)
def healthcheck(request): results = { 'version': __version__, 'is_database_synchronized': is_database_synchronized(), 'rq_jobs': len(django_rq.get_queue().jobs), } status_code = 200 if not results['is_database_synchronized']: status_code = 500 return JsonResponse(results, status=status_code)
def dummy_horde(interactive=False, how_many=1000): if interactive: for i in xrange(how_many): dummy_task(interactive=interactive, index=i + 1) else: queue = django_rq.get_queue() for i in xrange(how_many): queue.enqueue_call( func=dummy_task, kwargs=dict(interactive=interactive, index=i + 1), timeout=60, result_ttl=0, )
def update_user(self): Subscription = apps.get_model('bhs.subscription') User = apps.get_model('api.user') queue = django_rq.get_queue('low') try: subscription = self.subscriptions.filter( items_editable=True, ).latest('modified') queue.enqueue( User.objects.update_or_create_from_subscription, subscription, ) except Subscription.DoesNotExist: pass
def enqueue_task(kwargs, task_options=None): task_queue = getattr(settings, 'CACHEBACK_TASK_QUEUE', 'celery') if task_queue == 'rq' and rq_refresh_cache is not None: return django_rq.get_queue(**task_options or {}).enqueue( rq_refresh_cache, **kwargs) elif task_queue == 'celery' and celery_refresh_cache is not None: return celery_refresh_cache.apply_async(kwargs=kwargs, **task_options or {}) raise ImproperlyConfigured( 'Unkown task queue configured: {0}'.format(task_queue))
def delete_sources_not_found_in_registry_in_x_days(days): if int(days) < 6: raise Exception( "The task queue only allows deletion of sources when not found \ for +5 days") for source in Dataset.objects.all(): current_date = float(datetime.datetime.now().strftime('%s')) if source.last_found_in_registry: last_found_in_registry = float( source.last_found_in_registry.strftime('%s')) update_interval_time = 24 * 60 * 60 * int(days) if (current_date - update_interval_time) > last_found_in_registry: queue = django_rq.get_queue("parser") queue.enqueue(delete_source_by_id, args=(source.id,)) else: if not source.added_manually: # Old source, delete queue = django_rq.get_queue("parser") queue.enqueue(delete_source_by_id, args=(source.id,))
def get_job(job_id): """ Return the job for the specified ID or None if it cannot be found. Args: job_id(uuid): The ID of the job to retrieve. Returns: The job with the matching ID or None if no job with the supplied job ID could be found. """ queue = get_queue(settings.ASYNC_QUEUE) return queue.fetch_job(job_id)
def update(score_abbrev, project_pk, model_abbrev, no_enqueue): """ A subcommand that enqueues or (executes immediately) updating model scores, controlled by the args. NB: Does NOT exclude those that do not need updating according to how ForecastModel.forecasts_changed_at compares to ScoreLastUpdate.updated_at . :param score_abbrev: if a valid Score abbreviation then only that score is updated. o/w all scores are updated :param project_pk: if a valid Project pk then only that project's models are updated. o/w defers to `model_abbrev` arg :param model_abbrev: if a valid ForecastModel abbreviation then only that model is updated. o/w all models are updated :param no_enqueue: controls whether the update will be immediate in the calling thread (blocks), or enqueued for RQ """ from forecast_repo.settings.base import UPDATE_MODEL_SCORES_QUEUE_NAME # avoid circular imports Score.ensure_all_scores_exist() logger.info( f"update(): score_abbrev={score_abbrev!r}, project_pk={project_pk}, model_abbrev={model_abbrev!r}, " f"no_enqueue={no_enqueue}") scores = [get_object_or_404(Score, abbreviation=score_abbrev) ] if score_abbrev else Score.objects.all() # set models project = get_object_or_404(Project, pk=project_pk) if project_pk else None model = get_object_or_404(ForecastModel, project__id=project_pk, abbreviation=model_abbrev) \ if model_abbrev and project_pk else None if model: models = [model] elif project: models = project.models.all() else: models = ForecastModel.objects.all() logger.info( f"update(): project={project}, scores=({len(scores)}) {scores}, model={model}, " f"models=({len(models)}) {models}") queue = django_rq.get_queue(UPDATE_MODEL_SCORES_QUEUE_NAME) for score in scores: logger.info(f"* {score}") for forecast_model in models: if no_enqueue: logger.info( f"** (no enqueue) calculating score={score}, forecast_model={forecast_model}" ) _update_model_scores_worker(score.pk, forecast_model.pk) else: logger.info( f"** enqueuing score={score}, forecast_model={forecast_model}" ) queue.enqueue(_update_model_scores_worker, score.pk, forecast_model.pk) logger.info("update done")
def batch_hash_update(self, request, queryset): """ :type queryset: QuerySet """ if settings.ENABLE_REDIS is True: queue = django_rq.get_queue('high') queue.enqueue(hash_update_job, queryset) self.message_user( request, _("Hash updating job has been added to the \"high\" queue.")) else: hash_update_job(queryset) self.message_user(request, _("Hash updating job has been finished."))