def run_execution(self,
execution_id,
function_id,
version,
rlimit=None,
input=None,
identifier=None,
service_url=None,
entry='main.main',
trust_id=None):
"""Run execution.
Return a tuple including the result and the output.
"""
if service_url:
func_url = '%s/execute' % service_url
data = utils.get_request_data(self.conf, function_id, version,
execution_id, rlimit, input, entry,
trust_id, self.qinling_endpoint)
LOG.debug('Invoke function %s(version %s), url: %s, data: %s',
function_id, version, func_url, data)
return utils.url_request(self.session, func_url, body=data)
else:
# Wait for image type function execution to be finished
def _wait_complete():
pod = self.v1.read_namespaced_pod(
identifier, self.conf.kubernetes.namespace)
status = pod.status.phase
return True if status == 'Succeeded' else False
try:
r = tenacity.Retrying(wait=tenacity.wait_fixed(1),
stop=tenacity.stop_after_delay(180),
retry=tenacity.retry_if_result(
lambda result: result is False))
r.call(_wait_complete)
except Exception as e:
LOG.exception("Failed to get pod output, pod: %s, error: %s",
identifier, str(e))
return False, {'error': 'Function execution failed.'}
output = self.v1.read_namespaced_pod_log(
identifier,
self.conf.kubernetes.namespace,
)
return True, output
def run_execution(self,
execution_id,
function_id,
version,
rlimit=None,
input=None,
identifier=None,
service_url=None,
entry='main.main',
trust_id=None,
timeout=None):
"""Run execution.
Return a tuple including the result and the output.
"""
if service_url:
func_url = '%s/execute' % service_url
data = utils.get_request_data(self.conf, function_id, version,
execution_id, rlimit, input, entry,
trust_id, self.qinling_endpoint,
timeout)
LOG.debug('Invoke function %s(version %s), url: %s, data: %s',
function_id, version, func_url, data)
return utils.url_request(self.session, func_url, body=data)
else:
# Wait for image type function execution to be finished
def _wait_complete():
pod = self.v1.read_namespaced_pod(
identifier, self.conf.kubernetes.namespace)
status = pod.status.phase
if status == 'Succeeded':
return pod
raise exc.TimeoutException()
duration = 0
try:
r = tenacity.Retrying(wait=tenacity.wait_fixed(1),
stop=tenacity.stop_after_delay(timeout),
retry=tenacity.retry_if_exception_type(
exc.TimeoutException),
reraise=True)
pod = r.call(_wait_complete)
statuses = pod.status.container_statuses
for s in statuses:
if hasattr(s.state, "terminated"):
end_time = s.state.terminated.finished_at
start_time = s.state.terminated.started_at
delta = end_time - start_time
duration = delta.seconds
break
except exc.TimeoutException:
LOG.exception("Timeout for function execution %s, pod %s",
execution_id, identifier)
self.v1.delete_namespaced_pod(identifier,
self.conf.kubernetes.namespace,
{})
LOG.debug('Pod %s deleted.', identifier)
return False, {
'output': 'Function execution timeout.',
'duration': timeout
}
except Exception:
LOG.exception("Failed to wait for pod %s", identifier)
return False, {
'output': 'Function execution failed.',
'duration': duration
}
log = self.v1.read_namespaced_pod_log(
identifier,
self.conf.kubernetes.namespace,
)
return True, {'duration': duration, 'logs': log}
def create_execution(self,
ctx,
execution_id,
function_id,
function_version,
runtime_id,
input=None):
LOG.info(
'Creating execution. execution_id=%s, function_id=%s, '
'function_version=%s, runtime_id=%s, input=%s', execution_id,
function_id, function_version, runtime_id, input)
function = db_api.get_function(function_id)
source = function.code['source']
rlimit = {'cpu': function.cpu, 'memory_size': function.memory_size}
image = None
identifier = None
labels = None
svc_url = None
is_image_source = source == constants.IMAGE_FUNCTION
# Auto scale workers if needed
if not is_image_source:
try:
svc_url = self.function_load_check(function_id,
function_version,
runtime_id)
except exc.OrchestratorException as e:
utils.handle_execution_exception(execution_id, str(e))
return
temp_url = etcd_util.get_service_url(function_id, function_version)
svc_url = svc_url or temp_url
if svc_url:
func_url = '%s/execute' % svc_url
LOG.debug(
'Found service url for function: %s(version %s), execution: '
'%s, url: %s', function_id, function_version, execution_id,
func_url)
data = utils.get_request_data(CONF, function_id, function_version,
execution_id, rlimit, input,
function.entry, function.trust_id,
self.qinling_endpoint)
success, res = utils.url_request(self.session, func_url, body=data)
utils.finish_execution(execution_id,
success,
res,
is_image_source=is_image_source)
return
if source == constants.IMAGE_FUNCTION:
image = function.code['image']
# Be consistent with k8s naming convention
identifier = (
'%s-%s' %
(common.generate_unicode_uuid(dashed=False), function_id))[:63]
else:
identifier = runtime_id
labels = {'runtime_id': runtime_id}
try:
# For image function, it will be executed inside this method; for
# package type function it only sets up underlying resources and
# get a service url. If the service url is already created
# beforehand, nothing happens.
_, svc_url = self.orchestrator.prepare_execution(
function_id,
function_version,
rlimit=rlimit,
image=image,
identifier=identifier,
labels=labels,
input=input,
)
except exc.OrchestratorException as e:
utils.handle_execution_exception(execution_id, str(e))
return
# For image type function, read the worker log; For package type
# function, invoke and get log
success, res = self.orchestrator.run_execution(
execution_id,
function_id,
function_version,
rlimit=rlimit if svc_url else None,
input=input,
identifier=identifier,
service_url=svc_url,
entry=function.entry,
trust_id=function.trust_id)
utils.finish_execution(execution_id,
success,
res,
is_image_source=is_image_source)
def create_execution(self,
ctx,
execution_id,
function_id,
runtime_id,
input=None):
LOG.info(
'Creating execution. execution_id=%s, function_id=%s, '
'runtime_id=%s, input=%s', execution_id, function_id, runtime_id,
input)
function = db_api.get_function(function_id)
source = function.code['source']
image = None
identifier = None
labels = None
svc_url = None
# Auto scale workers if needed
if source != constants.IMAGE_FUNCTION:
svc_url = self.function_load_check(function_id, runtime_id)
temp_url = etcd_util.get_service_url(function_id)
svc_url = svc_url or temp_url
if svc_url:
func_url = '%s/execute' % svc_url
LOG.debug(
'Found service url for function: %s, execution: %s, url: %s',
function_id, execution_id, func_url)
data = utils.get_request_data(CONF, function_id, execution_id,
input, function.entry,
function.trust_id,
self.qinling_endpoint)
success, res = utils.url_request(self.session, func_url, body=data)
success = success and res.pop('success')
LOG.debug('Finished execution %s, success: %s', execution_id,
success)
db_api.update_execution(
execution_id, {
'status': status.SUCCESS if success else status.FAILED,
'logs': res.pop('logs', ''),
'result': res
})
return
if source == constants.IMAGE_FUNCTION:
image = function.code['image']
identifier = (
'%s-%s' %
(common.generate_unicode_uuid(dashed=False), function_id))[:63]
labels = {'function_id': function_id}
else:
identifier = runtime_id
labels = {'runtime_id': runtime_id}
_, svc_url = self.orchestrator.prepare_execution(
function_id,
image=image,
identifier=identifier,
labels=labels,
input=input,
)
success, res = self.orchestrator.run_execution(
execution_id,
function_id,
input=input,
identifier=identifier,
service_url=svc_url,
entry=function.entry,
trust_id=function.trust_id)
logs = ''
# Execution log is only available for non-image source execution.
if svc_url:
logs = res.pop('logs', '')
success = success and res.pop('success')
else:
# If the function is created from docker image, the result is
# direct output, here we convert to a dict to fit into the db
# schema.
res = {'output': res}
LOG.debug('Finished execution %s, success: %s', execution_id, success)
db_api.update_execution(
execution_id, {
'status': status.SUCCESS if success else status.FAILED,
'logs': logs,
'result': res
})