def run_execution(self, execution_id, function_id, version, rlimit=None, input=None, identifier=None, service_url=None, entry='main.main', trust_id=None): """Run execution. Return a tuple including the result and the output. """ if service_url: func_url = '%s/execute' % service_url data = utils.get_request_data(self.conf, function_id, version, execution_id, rlimit, input, entry, trust_id, self.qinling_endpoint) LOG.debug('Invoke function %s(version %s), url: %s, data: %s', function_id, version, func_url, data) return utils.url_request(self.session, func_url, body=data) else: # Wait for image type function execution to be finished def _wait_complete(): pod = self.v1.read_namespaced_pod( identifier, self.conf.kubernetes.namespace) status = pod.status.phase return True if status == 'Succeeded' else False try: r = tenacity.Retrying(wait=tenacity.wait_fixed(1), stop=tenacity.stop_after_delay(180), retry=tenacity.retry_if_result( lambda result: result is False)) r.call(_wait_complete) except Exception as e: LOG.exception("Failed to get pod output, pod: %s, error: %s", identifier, str(e)) return False, {'error': 'Function execution failed.'} output = self.v1.read_namespaced_pod_log( identifier, self.conf.kubernetes.namespace, ) return True, output
def run_execution(self, execution_id, function_id, version, rlimit=None, input=None, identifier=None, service_url=None, entry='main.main', trust_id=None, timeout=None): """Run execution. Return a tuple including the result and the output. """ if service_url: func_url = '%s/execute' % service_url data = utils.get_request_data(self.conf, function_id, version, execution_id, rlimit, input, entry, trust_id, self.qinling_endpoint, timeout) LOG.debug('Invoke function %s(version %s), url: %s, data: %s', function_id, version, func_url, data) return utils.url_request(self.session, func_url, body=data) else: # Wait for image type function execution to be finished def _wait_complete(): pod = self.v1.read_namespaced_pod( identifier, self.conf.kubernetes.namespace) status = pod.status.phase if status == 'Succeeded': return pod raise exc.TimeoutException() duration = 0 try: r = tenacity.Retrying(wait=tenacity.wait_fixed(1), stop=tenacity.stop_after_delay(timeout), retry=tenacity.retry_if_exception_type( exc.TimeoutException), reraise=True) pod = r.call(_wait_complete) statuses = pod.status.container_statuses for s in statuses: if hasattr(s.state, "terminated"): end_time = s.state.terminated.finished_at start_time = s.state.terminated.started_at delta = end_time - start_time duration = delta.seconds break except exc.TimeoutException: LOG.exception("Timeout for function execution %s, pod %s", execution_id, identifier) self.v1.delete_namespaced_pod(identifier, self.conf.kubernetes.namespace, {}) LOG.debug('Pod %s deleted.', identifier) return False, { 'output': 'Function execution timeout.', 'duration': timeout } except Exception: LOG.exception("Failed to wait for pod %s", identifier) return False, { 'output': 'Function execution failed.', 'duration': duration } log = self.v1.read_namespaced_pod_log( identifier, self.conf.kubernetes.namespace, ) return True, {'duration': duration, 'logs': log}
def create_execution(self, ctx, execution_id, function_id, function_version, runtime_id, input=None): LOG.info( 'Creating execution. execution_id=%s, function_id=%s, ' 'function_version=%s, runtime_id=%s, input=%s', execution_id, function_id, function_version, runtime_id, input) function = db_api.get_function(function_id) source = function.code['source'] rlimit = {'cpu': function.cpu, 'memory_size': function.memory_size} image = None identifier = None labels = None svc_url = None is_image_source = source == constants.IMAGE_FUNCTION # Auto scale workers if needed if not is_image_source: try: svc_url = self.function_load_check(function_id, function_version, runtime_id) except exc.OrchestratorException as e: utils.handle_execution_exception(execution_id, str(e)) return temp_url = etcd_util.get_service_url(function_id, function_version) svc_url = svc_url or temp_url if svc_url: func_url = '%s/execute' % svc_url LOG.debug( 'Found service url for function: %s(version %s), execution: ' '%s, url: %s', function_id, function_version, execution_id, func_url) data = utils.get_request_data(CONF, function_id, function_version, execution_id, rlimit, input, function.entry, function.trust_id, self.qinling_endpoint) success, res = utils.url_request(self.session, func_url, body=data) utils.finish_execution(execution_id, success, res, is_image_source=is_image_source) return if source == constants.IMAGE_FUNCTION: image = function.code['image'] # Be consistent with k8s naming convention identifier = ( '%s-%s' % (common.generate_unicode_uuid(dashed=False), function_id))[:63] else: identifier = runtime_id labels = {'runtime_id': runtime_id} try: # For image function, it will be executed inside this method; for # package type function it only sets up underlying resources and # get a service url. If the service url is already created # beforehand, nothing happens. _, svc_url = self.orchestrator.prepare_execution( function_id, function_version, rlimit=rlimit, image=image, identifier=identifier, labels=labels, input=input, ) except exc.OrchestratorException as e: utils.handle_execution_exception(execution_id, str(e)) return # For image type function, read the worker log; For package type # function, invoke and get log success, res = self.orchestrator.run_execution( execution_id, function_id, function_version, rlimit=rlimit if svc_url else None, input=input, identifier=identifier, service_url=svc_url, entry=function.entry, trust_id=function.trust_id) utils.finish_execution(execution_id, success, res, is_image_source=is_image_source)
def create_execution(self, ctx, execution_id, function_id, runtime_id, input=None): LOG.info( 'Creating execution. execution_id=%s, function_id=%s, ' 'runtime_id=%s, input=%s', execution_id, function_id, runtime_id, input) function = db_api.get_function(function_id) source = function.code['source'] image = None identifier = None labels = None svc_url = None # Auto scale workers if needed if source != constants.IMAGE_FUNCTION: svc_url = self.function_load_check(function_id, runtime_id) temp_url = etcd_util.get_service_url(function_id) svc_url = svc_url or temp_url if svc_url: func_url = '%s/execute' % svc_url LOG.debug( 'Found service url for function: %s, execution: %s, url: %s', function_id, execution_id, func_url) data = utils.get_request_data(CONF, function_id, execution_id, input, function.entry, function.trust_id, self.qinling_endpoint) success, res = utils.url_request(self.session, func_url, body=data) success = success and res.pop('success') LOG.debug('Finished execution %s, success: %s', execution_id, success) db_api.update_execution( execution_id, { 'status': status.SUCCESS if success else status.FAILED, 'logs': res.pop('logs', ''), 'result': res }) return if source == constants.IMAGE_FUNCTION: image = function.code['image'] identifier = ( '%s-%s' % (common.generate_unicode_uuid(dashed=False), function_id))[:63] labels = {'function_id': function_id} else: identifier = runtime_id labels = {'runtime_id': runtime_id} _, svc_url = self.orchestrator.prepare_execution( function_id, image=image, identifier=identifier, labels=labels, input=input, ) success, res = self.orchestrator.run_execution( execution_id, function_id, input=input, identifier=identifier, service_url=svc_url, entry=function.entry, trust_id=function.trust_id) logs = '' # Execution log is only available for non-image source execution. if svc_url: logs = res.pop('logs', '') success = success and res.pop('success') else: # If the function is created from docker image, the result is # direct output, here we convert to a dict to fit into the db # schema. res = {'output': res} LOG.debug('Finished execution %s, success: %s', execution_id, success) db_api.update_execution( execution_id, { 'status': status.SUCCESS if success else status.FAILED, 'logs': logs, 'result': res })