def post(self, request, *args, **kwargs): """ handle http POST requests """ import time from webui.scheduler.log import get_redis_key from webui.cnmain.utils import get_redis obj = self.get_object() args = [obj] if self.last_executed_required: args.append(int(request.POST.get('last_executed', 0))) task_kwargs = {} if request.POST.get('force') == 'true': task_kwargs['force'] = True task_id = str(self.task.delay(*args, **task_kwargs)) redis_key = get_redis_key(task_id) red = get_redis() for _ in range(10): if red.exists(redis_key): return HttpResponseRedirect("{url}?{class_}={pk}".format( url=reverse('scheduler_result_detail_view', args=[task_id]), class_=self.model.__name__.lower(), pk=obj.pk)) time.sleep(1) messages.error( request, 'Something went wrong, ask Sentry (task id {} )..'.format(task_id)) return redirect(obj)
def process_source(source, older_than=0): """Processes a source""" red = redis.Redis() task_id = process_source.request.id local_manager.cleanup() loggy = get_redis_logger(task_id) local.logger = loggy red.zadd( 'source:{}'.format(source.pk), get_redis_key(task_id), timestamp_now() ) # Init Handler, if an error occurs Source will not be processed. loggy.info('Evaluating Init Handler') try: wf_output = _process_init_handler(source) except: loggy.exception( 'An error occurred while processing Init Handler for source [%s]', unicode(source) ) raise else: loggy.info( 'Init handler executed successfully. Output %s', wf_output ) # Select never scheduled datasets. the_date = timezone.now() - datetime.timedelta(seconds=older_than) dataset_ctype = ContentType.objects.get_for_model(Dataset) already_scheduled_datasets = Scheduler.objects.filter( content_type=dataset_ctype, created__gte=the_date).values('object_id') datasets = Dataset.objects.filter(source=source)\ .exclude(pk__in=already_scheduled_datasets) count = datasets.count() if count: loggy.info('Processing %d datasets', datasets.count()) result = group( [process_dataset.s(ds, logger_name=task_id) for ds in datasets] ).apply() if result.successful(): dispose_sequence.delay(result.join(), source, task_id).get() else: loggy.info('An error occurred in a process_dataset') else: loggy.info('No datasets to process') loggy.info(END)
def process_source(source, older_than=0): """Processes a source""" red = redis.Redis() task_id = process_source.request.id local_manager.cleanup() loggy = get_redis_logger(task_id) local.logger = loggy red.zadd('source:{}'.format(source.pk), get_redis_key(task_id), timestamp_now()) # Init Handler, if an error occurs Source will not be processed. loggy.info('Evaluating Init Handler') try: wf_output = _process_init_handler(source) except: loggy.exception( 'An error occurred while processing Init Handler for source [%s]', unicode(source)) raise else: loggy.info('Init handler executed successfully. Output %s', wf_output) # Select never scheduled datasets. the_date = timezone.now() - datetime.timedelta(seconds=older_than) dataset_ctype = ContentType.objects.get_for_model(Dataset) already_scheduled_datasets = Scheduler.objects.filter( content_type=dataset_ctype, created__gte=the_date).values('object_id') datasets = Dataset.objects.filter(source=source)\ .exclude(pk__in=already_scheduled_datasets) count = datasets.count() if count: loggy.info('Processing %d datasets', datasets.count()) result = group([ process_dataset.s(ds, logger_name=task_id) for ds in datasets ]).apply() if result.successful(): dispose_sequence.delay(result.join(), source, task_id).get() else: loggy.info('An error occurred in a process_dataset') else: loggy.info('No datasets to process') loggy.info(END)
def post(self, request, *args, **kwargs): """ handle http POST requests """ import time from webui.scheduler.log import get_redis_key from webui.cnmain.utils import get_redis obj = self.get_object() args = [obj] if self.last_executed_required: args.append(int(request.POST.get('last_executed', 0))) task_kwargs = {} if request.POST.get('force') == 'true': task_kwargs['force'] = True task_id = str(self.task.delay(*args, **task_kwargs)) redis_key = get_redis_key(task_id) red = get_redis() for _ in range(10): if red.exists(redis_key): return HttpResponseRedirect( "{url}?{class_}={pk}".format( url=reverse( 'scheduler_result_detail_view', args=[task_id] ), class_=self.model.__name__.lower(), pk=obj.pk ) ) time.sleep(1) messages.error( request, 'Something went wrong, ask Sentry (task id {} )..'.format(task_id) ) return redirect(obj)