def post(self, job_id): job = Job.find(job_id) job.pipeline.start_single_job(job) tracker = insight.GAProvider() tracker.track_event(category='jobs', action='manual_run', label=job.worker_class) return job
def get(self, pipeline_id): args = log_parser.parse_args() entries = [] urlfetch.set_default_fetch_deadline(300) next_page_token = args.get('next_page_token') page_size = 20 from core import cloud_logging project_id = app_identity.get_application_id() filter_ = 'logName="projects/%s/logs/%s"' % (project_id, cloud_logging.logger_name) filter_ += ' AND jsonPayload.labels.pipeline_id="%s"' % pipeline_id if args.get('worker_class'): filter_ += ' AND jsonPayload.labels.worker_class="%s"' \ % args.get('worker_class') if args.get('job_id'): filter_ += ' AND jsonPayload.labels.job_id="%s"' % args.get( 'job_id') if args.get('log_level'): filter_ += ' AND jsonPayload.log_level="%s"' % args.get( 'log_level') if args.get('query'): filter_ += ' AND jsonPayload.message:"%s"' % args.get('query') if args.get('fromdate'): filter_ += ' AND timestamp>="%s"' % args.get('fromdate') if args.get('todate'): filter_ += ' AND timestamp<="%s"' % args.get('todate') iterator = cloud_logging.client.list_entries( projects=[project_id], filter_=filter_, order_by=DESCENDING, page_size=page_size, page_token=next_page_token) page = next(iterator.pages) for entry in page: # print ' Page number: %d' % (iterator.page_number,) # print ' Items in page: %d' % (page.num_items,) # print 'Items remaining: %d' % (page.remaining,) # print 'Next page token: %s' % (iterator.next_page_token,) # print '----------------------------' if isinstance(entry.payload, dict) \ and entry.payload.get('labels') \ and entry.payload.get('labels').get('job_id'): job = Job.find(entry.payload.get('labels').get('job_id')) if job: log = { 'timestamp': entry.timestamp.__str__(), 'payload': entry.payload, 'job_name': job.name, 'log_level': entry.payload.get('log_level', 'INFO') } entries.append(log) next_page_token = iterator.next_page_token return {'entries': entries, 'next_page_token': next_page_token}
def delete(self, job_id): job = Job.find(job_id) abort_if_job_doesnt_exist(job, job_id) if job.pipeline.is_blocked(): return { 'message': 'Removing of job for active pipeline is unavailable' }, 422 job.destroy() return {}, 204
def delete(self, job_id): job = Job.find(job_id) abort_if_job_doesnt_exist(job, job_id) if job.pipeline.is_blocked(): return { 'message': 'Removing of job for active pipeline is unavailable' }, 422 job.destroy() tracker = insight.GAProvider() tracker.track_event(category='jobs', action='delete') return {}, 204
def put(self, job_id): job = Job.find(job_id) abort_if_job_doesnt_exist(job, job_id) if job.pipeline.is_blocked(): return { 'message': 'Editing of job for active pipeline is unavailable' }, 422 args = parser.parse_args() job.assign_attributes(args) job.save() job.save_relations(args) return job, 200
def post(self): """ NB: you want retrieve the task name with this snippet task_name = request.headers.get('X-AppEngine-TaskName')[11:] """ # Clear the memcache client, mainly to avoid memory overflow of # the internal hashmap. cache.clear_memcache_client() retries = int(request.headers.get('X-AppEngine-TaskExecutionCount')) args = parser.parse_args() logger.debug(args) task_name = args['task_name'] job = Job.find(args['job_id']) worker_class = getattr(workers, args['worker_class']) worker_params = json.loads(args['worker_params']) worker = worker_class(worker_params, job.pipeline_id, job.id) if retries >= worker_class.MAX_ATTEMPTS: worker.log_error('Execution canceled after %i failed attempts', retries) job.task_failed(task_name) elif job.status == 'stopping': worker.log_warn( 'Execution canceled as parent job is going to stop') job.task_failed(task_name) else: try: workers_to_enqueue = worker.execute() except workers.WorkerException as e: worker.log_error('Execution failed: %s: %s', e.__class__.__name__, e) job.task_failed(task_name) except Exception as e: worker.log_error('Unexpected error: %s: %s', e.__class__.__name__, e) raise e else: for worker_class_name, worker_params, delay in workers_to_enqueue: job.enqueue(worker_class_name, worker_params, delay) job.task_succeeded(task_name) return 'OK', 200
def post(self): """ NB: you want retrieve the task name with this snippet task_name = request.headers.get('X-AppEngine-TaskName')[11:] """ urlfetch.set_default_fetch_deadline(300) retries = int(request.headers.get('X-AppEngine-TaskExecutionCount')) args = parser.parse_args() logger.debug(args) task_name = args['task_name'] job = Job.find(args['job_id']) worker_class = getattr(workers, args['worker_class']) worker_params = json.loads(args['worker_params']) for setting in worker_class.GLOBAL_SETTINGS: worker_params[setting] = GeneralSetting.where(name=setting).first().value worker = worker_class(worker_params, job.pipeline_id, job.id) if retries >= worker_class.MAX_ATTEMPTS: worker.log_error('Execution canceled after %i failed attempts', retries) job.task_failed(task_name) elif job.status == 'stopping': worker.log_warn('Execution canceled as parent job is going to stop') job.task_failed(task_name) else: try: workers_to_enqueue = worker.execute() except workers.WorkerException as e: worker.log_error('Execution failed: %s: %s', e.__class__.__name__, e) job.task_failed(task_name) except Exception as e: worker.log_error('Unexpected error: %s: %s', e.__class__.__name__, e) raise e else: for worker_class_name, worker_params, delay in workers_to_enqueue: job.enqueue(worker_class_name, worker_params, delay) job.task_succeeded(task_name) return 'OK', 200
def get(self, job_id): job = Job.find(job_id) abort_if_job_doesnt_exist(job, job_id) return job
def post(self, job_id): job = Job.find(job_id) job.pipeline.start_single_job(job) return job