def tx(): package = db.get(package_key) if package is not None: counter.incr('Packages.DuplicatePackageError') raise DuplicatePackageError() package = Package(key=package_key, name=name, version=version, created_by=created_by) # TODO(jeff.carollo): Handle conflicting package name/version. db.put(package) package_files = [] # Create PackageFiles with blob refs. for (blob_info, destination, file_mode, download_url) in files: # TODO(jeff.carollo): Create PackageFile.key from destination. package_files.append( PackageFile(parent=package_key, destination=destination, file_mode=file_mode, download_url=download_url, blob=blob_info)) # Create PackageFiles with urls instead of blobrefs. for urlfile in urlfiles: package_files.append( PackageFile(parent=package_key, destination=urlfile['file_destination'], file_mode=urlfile['file_mode'], download_url=urlfile['url'])) db.put(package_files) return package
def tx(): package = db.get(package_key) if package is not None: counter.incr('Packages.DuplicatePackageError') raise DuplicatePackageError() package = Package(key=package_key, name=name, version=version, created_by=created_by) # TODO(jeff.carollo): Handle conflicting package name/version. db.put(package) package_files = [] # Create PackageFiles with blob refs. for (blob_info, destination, file_mode, download_url) in files: # TODO(jeff.carollo): Create PackageFile.key from destination. package_files.append(PackageFile(parent=package_key, destination=destination, file_mode=file_mode, download_url=download_url, blob=blob_info)) # Create PackageFiles with urls instead of blobrefs. for urlfile in urlfiles: package_files.append(PackageFile(parent=package_key, destination=urlfile['file_destination'], file_mode=urlfile['file_mode'], download_url=urlfile['url'])) db.put(package_files) return package
def DeleteById(task_id): """Deletes Task with given integer task_id.""" task = GetById(task_id) if task is None: return False task.delete() counter.incr('Tasks.Deleted') return True
def DeleteById(task_id): """Deletes Task with given integer task_id.""" task = GetById(task_id) if task is None: return False task.delete() counter.incr('Tasks.Deleted') return True
def tx(executor_capability): task = GetOldestTaskForCapability(executor_capability) if task is None: return None logging.info('Assigning task %s to %s for %s.', task.key().id_or_name(), worker, executor_capability) AssignTaskToWorker(task, worker) logging.info('Assignment successful.') ScheduleTaskTimeout(task) counter.incr('Tasks.Assigned') counter.incr('Executors.%s.Assigned' % executor_capability) return task
def Schedule(name, config, scheduled_by, executor_requirements, priority=0): """Adds a new Task with given name, config, user and requirements.""" webhook = json.loads(config)['task'].get('webhook', None) task = Task(parent=MakeParentKey(), name=name, config=config, scheduled_by=scheduled_by, executor_requirements=executor_requirements, priority=priority, webhook=webhook) db.put(task) counter.incr('Tasks.Scheduled') return task
def Schedule(name, config, scheduled_by, executor_requirements, priority=0): """Adds a new Task with given name, config, user and requirements.""" webhook = json.loads(config)['task'].get('webhook', None) task = Task(parent=MakeParentKey(), name=name, config=config, scheduled_by=scheduled_by, executor_requirements=executor_requirements, priority=priority, webhook=webhook) db.put(task) counter.incr('Tasks.Scheduled') return task
def tx(executor_capability): task = GetOldestTaskForCapability(executor_capability) if task is None: return None logging.info('Assigning task %s to %s for %s.', task.key().id_or_name(), worker, executor_capability) AssignTaskToWorker(task, worker) logging.info('Assignment successful.') ScheduleTaskTimeout(task) counter.incr('Tasks.Assigned') counter.incr('Executors.%s.Assigned' % executor_capability) return task
class InvokeWebhookHandler(webapp2.RequestHandler): def post(self, task_id): task = GetById(int(task_id)) if not task: return config = json.loads(task.config) try: webhook = config['task']['webhook'] except Exception, e: logging.exception(e) logging.info('No webhook, or error invoking webhook.') return logging.info('invoking webhook: %s', webhook) payload = urllib.urlencode({'task_id': task_id}).encode('utf-8') fetched = urlfetch.fetch( url=webhook, method='POST', payload=payload, headers={ 'Content-Type': 'application/x-www-form-urlencoded;encoding=utf-8' }) logging.info('Webhook invoked with status %d: %s.', fetched.status_code, fetched.content) self.response.set_status(fetched.status_code) counter.incr('Tasks.WebhookInvoked%s' % fetched.status_code)
def tx(): task = GetById(task_id) counters = counter.Batch() # Validate that task is in a state to accept results from worker. if not task: raise TaskNotFoundError() counters.incr('Tasks.Completed') if device_serial_number: counters.incr('Executors.%s.Completed' % device_serial_number) if task.attempts != attempt: logging.info('Attempts: %d, attempt: %d', task.attempts, attempt) counter.incr('Tasks.Completed.TimedOut') if device_serial_number: counter.incr('Executors.%s.TimedOut' % device_serial_number) raise TaskTimedOutError() # Here we allow a timed out task to publish results if it hasn't # been scheduled to another worker yet. if task.state not in [TaskStates.ASSIGNED, TaskStates.SCHEDULED]: logging.info('task.state: %s', task.state) counter.incr('Tasks.Completed.TimedOut') if device_serial_number: counter.incr('Executors.%s.TimedOut' % device_serial_number) raise TaskTimedOutError() # Mark task as complete and place results. task.completed_time = datetime.datetime.now() if exit_code == 0: counters.incr('Tasks.Completed.Success') if device_serial_number: counters.incr('Executors.%s.Success' % device_serial_number) task.outcome = TaskOutcomes.SUCCESS else: counters.incr('Tasks.Completed.Failed') if device_serial_number: counters.incr('Executors.%s.Failed' % device_serial_number) task.outcome = TaskOutcomes.FAILED task.state = TaskStates.COMPLETE task_result = TaskResult(parent=task, exit_code=exit_code, execution_time=execution_time, stdout=stdout, stderr=stderr, stdout_download_url=stdout_download_url, stderr_download_url=stderr_download_url, device_serial_number=device_serial_number, result_metadata=result_metadata, worker_log=worker_log) task_result = db.put(task_result) task.result = task_result db.put(task) taskqueue.add(url='/tasks/%d/invoke_webhook' % task.key().id(), transactional=True) return (task, counters)
def tx(): task = GetById(task_id) counters = counter.Batch() # Validate that task is in a state to accept results from worker. if not task: raise TaskNotFoundError() counters.incr('Tasks.Completed') if device_serial_number: counters.incr('Executors.%s.Completed' % device_serial_number) if task.attempts != attempt: logging.info('Attempts: %d, attempt: %d', task.attempts, attempt) counter.incr('Tasks.Completed.TimedOut') if device_serial_number: counter.incr('Executors.%s.TimedOut' % device_serial_number) raise TaskTimedOutError() # Here we allow a timed out task to publish results if it hasn't # been scheduled to another worker yet. if task.state not in [TaskStates.ASSIGNED, TaskStates.SCHEDULED]: logging.info('task.state: %s', task.state) counter.incr('Tasks.Completed.TimedOut') if device_serial_number: counter.incr('Executors.%s.TimedOut' % device_serial_number) raise TaskTimedOutError() # Mark task as complete and place results. task.completed_time = datetime.datetime.now() if exit_code == 0: counters.incr('Tasks.Completed.Success') if device_serial_number: counters.incr('Executors.%s.Success' % device_serial_number) task.outcome = TaskOutcomes.SUCCESS else: counters.incr('Tasks.Completed.Failed') if device_serial_number: counters.incr('Executors.%s.Failed' % device_serial_number) task.outcome = TaskOutcomes.FAILED task.state = TaskStates.COMPLETE task_result = TaskResult(parent=task, exit_code=exit_code, execution_time=execution_time, stdout=stdout, stderr=stderr, stdout_download_url=stdout_download_url, stderr_download_url=stderr_download_url, device_serial_number=device_serial_number, result_metadata=result_metadata, worker_log=worker_log) task_result = db.put(task_result) task.result = task_result db.put(task) taskqueue.add(url='/tasks/%d/invoke_webhook' % task.key().id(), transactional=True) return (task, counters)
def post(self, task_id): """Uploads results of a task, including STDOUT and STDERR.""" counter.incr('Tasks.Update') logging.info('Request: %s', self.request.body) task_id = int(task_id) blob_infos = self.GetBlobInfosFromPostBody() task_result = self.request.get('task_result', None) if task_result: task_result = quopri.decodestring(task_result).decode('ISO-8859-1') if task_result: try: task_result = json.loads(task_result, 'ISO-8859-1') except ValueError, e: self.response.out.write('Field "task_result" must be valid JSON.\n') logging.info(e) task_result = None
def post(self, task_id): """Uploads results of a task, including STDOUT and STDERR.""" counter.incr('Tasks.Update') logging.info('Request: %s', self.request.body) task_id = int(task_id) blob_infos = self.GetBlobInfosFromPostBody() task_result = self.request.get('task_result', None) if task_result: task_result = quopri.decodestring(task_result).decode('ISO-8859-1') if task_result: try: task_result = json.loads(task_result, 'ISO-8859-1') except ValueError, e: self.response.out.write( 'Field "task_result" must be valid JSON.\n') logging.info(e) task_result = None
def post(self): """TODO(jeff.carollo): Specify request and response format.""" counter.incr('Tasks.Schedule') content_type = self.request.headers['Content-Type'] if 'application/json' not in content_type: logging.info('Content-Type: %s', content_type) self.response.out.write('Content-Type must be application/json.\n') self.response.set_status(415) return body = self.request.body.decode('utf-8') if body is None: counter.incr('Tasks.Schedule.400') self.response.out.write('Config is required in message body\n') self.response.set_status(400) return config = urllib.unquote(body) logging.info('Config: %s', config) try: parsed_config = json.loads(config, 'utf-8') if not parsed_config: raise Exception('json could not parse config.') except Exception, e: counter.incr('Tasks.Schedule.400') self.response.out.write('Failure parsing config.\n') self.response.out.write(e) self.response.out.write('\n') self.response.set_status(400) return
def post(self): """TODO(jeff.carollo): Specify request and response format.""" counter.incr('Tasks.Schedule') content_type = self.request.headers['Content-Type'] if 'application/json' not in content_type: logging.info('Content-Type: %s', content_type) self.response.out.write('Content-Type must be application/json.\n') self.response.set_status(415) return body = self.request.body.decode('utf-8') if body is None: counter.incr('Tasks.Schedule.400') self.response.out.write('Config is required in message body\n') self.response.set_status(400) return config = urllib.unquote(body) logging.info('Config: %s', config) try: parsed_config = json.loads(config, 'utf-8') if not parsed_config: raise Exception('json could not parse config.') except Exception, e: counter.incr('Tasks.Schedule.400') self.response.out.write('Failure parsing config.\n') self.response.out.write(e) self.response.out.write('\n') self.response.set_status(400) return
def put(self): # TODO(jeff.carollo): Specify request and response format.""" counter.incr('Tasks.Assign') body = self.request.body.decode('utf-8') if body is None: counter.incr('Tasks.Assign.400') self.response.out.write( 'AssignRequest is required in message body\n') self.response.set_status(400) return assign_request = urllib.unquote(body) logging.info('assign_request: %s', assign_request) try: parsed_request = json.loads(assign_request) if not parsed_request: raise Exception('json could not parse AssignRequest.') except Exception, e: counter.incr('Tasks.Assign.400') self.response.out.write('Failure parsing AssignRequest.\n') self.response.out.write(e) self.response.out.write('\n') self.response.set_status(400) return
def delete(self, task_id): """Removes a single task given by task_id.""" counter.incr('Tasks.Delete') task_id = int(task_id) success = tasks.DeleteById(task_id) if not success: counter.incr('Tasks.Delete.404') self.error(404) return # 200 OK. counter.incr('Tasks.Delete.200')
def delete(self, task_id): """Removes a single task given by task_id.""" counter.incr('Tasks.Delete') task_id = int(task_id) success = tasks.DeleteById(task_id) if not success: counter.incr('Tasks.Delete.404') self.error(404) return # 200 OK. counter.incr('Tasks.Delete.200')
class TasksScheduleHandler(webapp2.RequestHandler): """Handles the creation of a new Task, also known as scheduling.""" def post(self): """TODO(jeff.carollo): Specify request and response format.""" counter.incr('Tasks.Schedule') content_type = self.request.headers['Content-Type'] if 'application/json' not in content_type: logging.info('Content-Type: %s', content_type) self.response.out.write('Content-Type must be application/json.\n') self.response.set_status(415) return body = self.request.body.decode('utf-8') if body is None: counter.incr('Tasks.Schedule.400') self.response.out.write('Config is required in message body\n') self.response.set_status(400) return config = urllib.unquote(body) logging.info('Config: %s', config) try: parsed_config = json.loads(config, 'utf-8') if not parsed_config: raise Exception('json could not parse config.') except Exception, e: counter.incr('Tasks.Schedule.400') self.response.out.write('Failure parsing config.\n') self.response.out.write(e) self.response.out.write('\n') self.response.set_status(400) return try: name = parsed_config['task']['name'] except KeyError, e: counter.incr('Tasks.Schedule.400') self.response.out.write('Failure parsing config.\n') self.response.out.write('task.name is required\n') self.response.set_status(400) return
class TasksAssignHandler(webapp2.RequestHandler): """Handles /tasks/assign, which hands off tasks to workers.""" def put(self): # TODO(jeff.carollo): Specify request and response format.""" counter.incr('Tasks.Assign') body = self.request.body.decode('utf-8') if body is None: counter.incr('Tasks.Assign.400') self.response.out.write( 'AssignRequest is required in message body\n') self.response.set_status(400) return assign_request = urllib.unquote(body) logging.info('assign_request: %s', assign_request) try: parsed_request = json.loads(assign_request) if not parsed_request: raise Exception('json could not parse AssignRequest.') except Exception, e: counter.incr('Tasks.Assign.400') self.response.out.write('Failure parsing AssignRequest.\n') self.response.out.write(e) self.response.out.write('\n') self.response.set_status(400) return # TODO(jeff.carollo): Make these real objects with validate methods. try: worker = parsed_request['worker'] except KeyError, e: counter.incr('Tasks.Assign.400') self.response.out.write('AssignRequest.worker is required.\n') self.response.out.write(e) self.response.out.write('\n') self.response.set_status(400) return
def get(self, task_id): counter.incr('Tasks.GetTaskCompleteUrl') try: task_id = int(task_id) except: counter.incr('Tasks.GetTaskCompleteUrl.400') self.response.set_status(400) self.response.out.write('task_id must be an integer.') return response = {} response['kind'] = 'mrtaskman#task_complete_url' response['task_complete_url'] = MakeTaskCompleteUrl(task_id) json.dump(response, self.response.out, indent=2) self.response.headers['Content-Type'] = 'application/json' self.response.out.write('\n') counter.incr('Tasks.GetTaskCompleteUrl.200')
def get(self, task_id): counter.incr('Tasks.GetTaskCompleteUrl') try: task_id = int(task_id) except: counter.incr('Tasks.GetTaskCompleteUrl.400') self.response.set_status(400) self.response.out.write('task_id must be an integer.') return response = {} response['kind'] = 'mrtaskman#task_complete_url' response['task_complete_url'] = MakeTaskCompleteUrl(task_id) json.dump(response, self.response.out, indent=2) self.response.headers['Content-Type'] = 'application/json' self.response.out.write('\n') counter.incr('Tasks.GetTaskCompleteUrl.200')
def get(self, task_id): """Retrieves a single task given by task_id.""" counter.incr('Tasks.Get') # TODO(jeff.carollo): Specify request and response format.""" task_id = int(task_id) task = tasks.GetById(task_id) if task is None: counter.incr('Tasks.Get.404') self.error(404) return # Success. Write response. self.response.headers['Content-Type'] = 'application/json' response = model_to_dict.ModelToDict(task) response['kind'] = 'mrtaskman#task' json.dump(response, self.response.out, indent=2) self.response.out.write('\n') counter.incr('Tasks.Get.200')
def get(self, task_id): """Retrieves a single task given by task_id.""" counter.incr('Tasks.Get') # TODO(jeff.carollo): Specify request and response format.""" task_id = int(task_id) task = tasks.GetById(task_id) if task is None: counter.incr('Tasks.Get.404') self.error(404) return # Success. Write response. self.response.headers['Content-Type'] = 'application/json' response = model_to_dict.ModelToDict(task) response['kind'] = 'mrtaskman#task' json.dump(response, self.response.out, indent=2) self.response.out.write('\n') counter.incr('Tasks.Get.200')
def put(self): # TODO(jeff.carollo): Specify request and response format.""" counter.incr('Tasks.Assign') body = self.request.body.decode('utf-8') if body is None: counter.incr('Tasks.Assign.400') self.response.out.write('AssignRequest is required in message body\n') self.response.set_status(400) return assign_request = urllib.unquote(body) logging.info('assign_request: %s', assign_request) try: parsed_request = json.loads(assign_request) if not parsed_request: raise Exception('json could not parse AssignRequest.') except Exception, e: counter.incr('Tasks.Assign.400') self.response.out.write('Failure parsing AssignRequest.\n') self.response.out.write(e) self.response.out.write('\n') self.response.set_status(400) return
except KeyError, e: counter.incr('Tasks.Schedule.400') self.response.out.write('Failure parsing config.\n') self.response.out.write('task.name is required\n') self.response.set_status(400) return try: executor_requirements = parsed_config['task']['requirements'][ 'executor'] assert executor_requirements assert isinstance(executor_requirements, list) for executor_req in executor_requirements: assert isinstance(executor_req, basestring) except KeyError, e: counter.incr('Tasks.Schedule.400') self.response.out.write('Failure parsing config.\n') self.response.out.write('task.requirements.executor is required\n') self.response.set_status(400) return except AssertionError, e: counter.incr('Tasks.Schedule.400') self.response.out.write('Failure parsing config.\n') self.response.out.write( 'task.requirements.executor must be a non-empty list of strings.\n' ) self.response.set_status(400) return priority = parsed_config['task'].get('priority', 0) try:
class TasksHandler(webapp2.RequestHandler): def get(self, task_id): """Retrieves a single task given by task_id.""" counter.incr('Tasks.Get') # TODO(jeff.carollo): Specify request and response format.""" task_id = int(task_id) task = tasks.GetById(task_id) if task is None: counter.incr('Tasks.Get.404') self.error(404) return # Success. Write response. self.response.headers['Content-Type'] = 'application/json' response = model_to_dict.ModelToDict(task) response['kind'] = 'mrtaskman#task' json.dump(response, self.response.out, indent=2) self.response.out.write('\n') counter.incr('Tasks.Get.200') def delete(self, task_id): """Removes a single task given by task_id.""" counter.incr('Tasks.Delete') task_id = int(task_id) success = tasks.DeleteById(task_id) if not success: counter.incr('Tasks.Delete.404') self.error(404) return # 200 OK. counter.incr('Tasks.Delete.200') def post(self, task_id): """Uploads results of a task, including STDOUT and STDERR.""" counter.incr('Tasks.Update') logging.info('Request: %s', self.request.body) task_id = int(task_id) blob_infos = self.GetBlobInfosFromPostBody() task_result = self.request.get('task_result', None) if task_result: task_result = quopri.decodestring(task_result).decode('ISO-8859-1') if task_result: try: task_result = json.loads(task_result, 'ISO-8859-1') except ValueError, e: self.response.out.write( 'Field "task_result" must be valid JSON.\n') logging.info(e) task_result = None if not task_result: counter.incr('Tasks.Update.400') self.DeleteBlobs(blob_infos) self.response.out.write('Field "task_result" is required.\n') self.response.set_status(400) return # Get required attempt and exit_code. try: # TODO(jeff.carollo): Make attempt a required path parameter, # and make it restful by delivering it in the tasks.assign response. attempt = task_result['attempt'] exit_code = task_result['exit_code'] assert isinstance(attempt, int) assert isinstance(exit_code, int) except KeyError, AssertionError: counter.incr('Tasks.Update.400') self.DeleteBlobs(blob_infos) self.response.out.write( 'task_result must contain integers "attempt" and "exit_code".') self.response.set_status(400) return
name = parsed_config['task']['name'] except KeyError, e: counter.incr('Tasks.Schedule.400') self.response.out.write('Failure parsing config.\n') self.response.out.write('task.name is required\n') self.response.set_status(400) return try: executor_requirements = parsed_config['task']['requirements']['executor'] assert executor_requirements assert isinstance(executor_requirements, list) for executor_req in executor_requirements: assert isinstance(executor_req, basestring) except KeyError, e: counter.incr('Tasks.Schedule.400') self.response.out.write('Failure parsing config.\n') self.response.out.write('task.requirements.executor is required\n') self.response.set_status(400) return except AssertionError, e: counter.incr('Tasks.Schedule.400') self.response.out.write('Failure parsing config.\n') self.response.out.write( 'task.requirements.executor must be a non-empty list of strings.\n') self.response.set_status(400) return priority = parsed_config['task'].get('priority', 0) try: logging.info('priority is %s', priority)