def task_status(request, task_id): """ get task status info :param request: request object :param task_id: task id :return: """ if request.method == 'GET': result = [] task = Task.objects.get(id=task_id) clients = clients_of_task(task) for client in clients: job_id = get_job_id(client, task) jobs = DjangoJob.objects.filter(id=job_id) logger.debug('jobs from djangojob %s', jobs) # if job does not exist, for date mode exceed time if not jobs: continue job = DjangoJob.objects.get(id=job_id) executions = serialize('json', DjangoJobExecution.objects.filter(job=job)) result.append({ 'client': model_to_dict(client), 'next': job.next_run_time, 'executions': json.loads(executions) }) return JsonResponse({'data': result})
def _add_or_modify_new_jobs(self, task, force=False): """ add new jobs or modify existed jobs :return: """ if not task.modified and not force: return task_job_ids = [] clients = list(clients_of_task(task)) # update for every client for client in clients: # get job id job_id = get_job_id(client, task) # add job_id to array task_job_ids.append(job_id) configuration = json.loads(task.configuration) trigger = task.trigger configuration = {arg: configuration.get(arg) for arg in args_map.get(trigger) if configuration.get(arg)} logger.debug('adding or modifying job %s, trigger %s, configuration %s', job_id, trigger, configuration) # if job doesn't exist, add it. otherwise replace it self.scheduler.add_job(execute, task.trigger, args=[client, task.project, task.spider], id=job_id, replace_existing=True, **configuration)
def realtime_jobs(self): """ get real-time jobs :return: """ tasks = self.realtime_tasks() for task in tasks: clients = list(clients_of_task(task)) for client in clients: job_id = get_job_id(client, task) yield job_id
def task_remove(request, task_id): """ remove task by task_id :param request: :return: """ if request.method == 'POST': # delete job from DjangoJob task = Task.objects.get(id=task_id) clients = clients_of_task(task) for client in clients: job_id = get_job_id(client, task) DjangoJob.objects.filter(id=job_id).delete() # delete task Task.objects.filter(id=task_id).delete() return JsonResponse({'result': '1'})
def task_status(request, task_id): """ get task status info :param request: request object :param task_id: task id :return: """ if request.method == 'GET': result = [] task = Task.objects.get(id=task_id) clients = clients_of_task(task) for client in clients: job_id = get_job_id(client, task) job = DjangoJob.objects.get(name=job_id) executions = serialize('json', DjangoJobExecution.objects.filter(job=job)) result.append({ 'client': model_to_dict(client), 'next': job.next_run_time, 'executions': json.loads(executions) }) return JsonResponse({'data': result})
def _add_or_modify_jobs(self): """ add new jobs or modify existed jobs :return: """ tasks = self.realtime_tasks() for task in tasks: # if modified if task.modified: clients = list(clients_of_task(task)) # update for every client for client in clients: # get job id job_id = get_job_id(client, task) configuration = json.loads(task.configuration) trigger = task.trigger configuration = {arg: configuration.get(arg) for arg in args_map.get(trigger) if configuration.get(arg)} # if job doesn't exist, add it. otherwise replace it self.scheduler.add_job(execute, task.trigger, args=[client, task.project, task.spider], id=job_id, replace_existing=True, **configuration) task.modified = 0 task.save()