def prepare_execution(self,
function_id,
image=None,
identifier=None,
labels=None,
input=None):
"""Prepare service URL for function.
For image function, create a single pod with input, so the function
will be executed.
For normal function, choose a pod from the pool and expose a service,
return the service URL.
"""
pod = None
if image:
self._create_pod(image, identifier, labels, input)
return identifier, None
else:
pod = self._choose_available_pod(labels, function_id=function_id)
if not pod:
LOG.critical('No worker available.')
raise exc.OrchestratorException('Execution preparation failed.')
try:
pod_name, url = self._prepare_pod(pod[0], identifier, function_id,
labels)
return pod_name, url
except Exception:
LOG.exception('Pod preparation failed.')
self.delete_function(function_id, labels)
raise exc.OrchestratorException('Execution preparation failed.')
def prepare_execution(self,
function_id,
version,
rlimit=None,
image=None,
identifier=None,
labels=None,
input=None):
"""Prepare service URL for function version.
:param rlimit: optional argument passed to limit cpu/mem resources.
For image function, create a single pod with rlimit and input, so the
function will be executed in the resource limited pod.
For normal function, choose a pod from the pool and expose a service,
return the service URL.
return a tuple includes pod name and the servise url.
"""
pods = None
labels = labels or {'function_id': function_id}
if image:
if not rlimit:
LOG.critical('Param rlimit is required for image function.')
raise exc.OrchestratorException(
'Execution preparation failed.')
self._create_pod(image, rlimit, identifier, labels, input)
return identifier, None
else:
pods = self._choose_available_pods(labels,
function_id=function_id,
function_version=version)
if not pods:
LOG.critical('No worker available.')
raise exc.OrchestratorException('Execution preparation failed.')
try:
pod_name, url = self._prepare_pod(pods[0], identifier, function_id,
version, labels)
return pod_name, url
except Exception:
LOG.exception('Pod preparation failed.')
self.delete_function(function_id, version, labels)
raise exc.OrchestratorException('Execution preparation failed.')
def _wait_deployment_available(self, name):
ret = self.v1extension.read_namespaced_deployment(
name, self.conf.kubernetes.namespace)
if (not ret.status.replicas
or ret.status.replicas != ret.status.available_replicas):
raise exc.OrchestratorException('Deployment %s not ready.' % name)
def prepare_execution(self,
function_id,
image=None,
identifier=None,
labels=None,
input=None,
entry='main.main'):
"""Prepare service URL for function.
For image function, create a single pod with input, so the function
will be executed.
For normal function, choose a pod from the pool and expose a service,
return the service URL.
"""
pod = None
if image:
self._create_pod(image, identifier, labels, input)
return identifier, None
else:
pod = self._choose_available_pod(labels)
if not pod:
raise exc.OrchestratorException('No pod available.')
return self._prepare_pod(pod[0], identifier, function_id, labels,
entry)
def test_create_execution_prepare_execution_exception(
self, etcd_util_get_service_url_mock):
"""test_create_execution_prepare_execution_exception
Create execution for image type function, prepare_execution method
raises exception.
"""
function = self.create_function()
function_id = function.id
runtime_id = function.runtime_id
db_api.update_function(
function_id, {
'code': {
'source': constants.IMAGE_FUNCTION,
'image': self.rand_name('image', prefix=self.prefix)
}
})
function = db_api.get_function(function_id)
execution = self.create_execution(function_id=function_id)
execution_id = execution.id
prepare_execution = self.orchestrator.prepare_execution
prepare_execution.side_effect = exc.OrchestratorException(
'Exception in prepare_execution')
etcd_util_get_service_url_mock.return_value = None
self.default_engine.create_execution(mock.Mock(), execution_id,
function_id, 0, runtime_id)
execution = db_api.get_execution(execution_id)
self.assertEqual(execution.status, status.ERROR)
self.assertEqual(execution.logs, '')
self.assertEqual(execution.result,
{'error': 'Function execution failed.'})
def prepare_execution(self, function_id, identifier=None, labels=None):
pod = self._choose_available_pod(labels)
if not pod:
raise exc.OrchestratorException('No pod available.')
return self._prepare_pod(pod, identifier, function_id, labels)
def _wait_complete():
pod = self.v1.read_namespaced_pod(
identifier, self.conf.kubernetes.namespace)
status = pod.status.phase
if status == 'Succeeded':
return pod
raise exc.OrchestratorException()
def scaleup_function(self, function_id, identifier=None, count=1):
pod_names = []
labels = {'runtime_id': identifier}
pods = self._choose_available_pod(labels, count=count)
if not pods:
raise exc.OrchestratorException('Not enough workers available.')
for pod in pods:
pod_name, service_url = self._prepare_pod(pod, identifier,
function_id, labels)
pod_names.append(pod_name)
LOG.info('Pods scaled up for function %s: %s', function_id, pod_names)
return pod_names, service_url
def load_orchestrator(conf, qinling_endpoint):
global ORCHESTRATOR
if not ORCHESTRATOR:
try:
mgr = driver.DriverManager('qinling.orchestrator',
conf.engine.orchestrator,
invoke_on_load=True,
invoke_args=[conf, qinling_endpoint])
ORCHESTRATOR = mgr.driver
except Exception as e:
raise exc.OrchestratorException(
'Failed to load orchestrator: %s. Error: %s' %
(conf.engine.orchestrator, str(e)))
return ORCHESTRATOR
def test_create_execution_loadcheck_exception(self):
function = self.create_function()
function_id = function.id
runtime_id = function.runtime_id
execution = self.create_execution(function_id=function_id)
execution_id = execution.id
self.default_engine.function_load_check = mock.Mock(
side_effect=exc.OrchestratorException(
'Exception in scaleup_function'))
self.default_engine.create_execution(mock.Mock(), execution_id,
function_id, 0, runtime_id)
execution = db_api.get_execution(execution_id)
self.assertEqual(execution.status, status.ERROR)
self.assertEqual(execution.logs, '')
self.assertEqual(execution.result,
{'error': 'Function execution failed.'})
def scaleup_function(self,
function_id,
identifier=None,
entry='main.main',
count=1):
pod_names = []
labels = {'runtime_id': identifier}
pods = self._choose_available_pod(labels, count=count)
if not pods:
raise exc.OrchestratorException('Not enough pods available.')
temp_function = '%s-temp' % function_id
for pod in pods:
self._prepare_pod(pod,
identifier,
temp_function,
labels,
entry,
actual_function=function_id)
# Delete temporary service
selector = common.convert_dict_to_string(
{'function_id': temp_function})
ret = self.v1.list_namespaced_service(
self.conf.kubernetes.namespace, label_selector=selector)
svc_names = [i.metadata.name for i in ret.items]
for svc_name in svc_names:
self.v1.delete_namespaced_service(
svc_name,
self.conf.kubernetes.namespace,
)
# Modify pod labels to fit into correct service
self._update_pod_label(pod, {'function_id': function_id})
pod_names.append(pod.metadata.name)
LOG.info('Pods scaled up for function %s: %s', function_id, pod_names)
return pod_names
def _create_pod(self, image, rlimit, pod_name, labels, input):
"""Create pod for image type function."""
if not input:
input_list = []
elif isinstance(input, dict) and input.get('__function_input'):
input_list = input.get('__function_input').split()
else:
input_list = list(json.loads(input))
pod_body = self.pod_template.render({
"pod_name":
pod_name,
"labels":
labels,
"pod_image":
image,
"input":
input_list,
"req_cpu":
str(rlimit['cpu']),
"limit_cpu":
str(rlimit['cpu']),
"req_memory":
str(rlimit['memory_size']),
"limit_memory":
str(rlimit['memory_size'])
})
LOG.info("Creating pod %s for image function:\n%s", pod_name, pod_body)
try:
self.v1.create_namespaced_pod(
self.conf.kubernetes.namespace,
body=yaml.safe_load(pod_body),
)
except Exception:
LOG.exception("Failed to create pod.")
raise exc.OrchestratorException('Execution preparation failed.')
def _wait_for_upgrade(self, deploy_name):
ret = self.v1extension.read_namespaced_deployment(
deploy_name, self.conf.kubernetes.namespace)
if ret.status.unavailable_replicas is not None:
raise exc.OrchestratorException("Deployment %s upgrade not "
"ready." % deploy_name)
def _prepare_pod(self,
pod,
deployment_name,
function_id,
labels=None,
entry=None,
actual_function=None):
"""Pod preparation.
1. Update pod labels.
2. Expose service and trigger package download.
"""
name = pod.metadata.name
actual_function = actual_function or function_id
LOG.info('Prepare pod %s in deployment %s for function %s', name,
deployment_name, function_id)
# Update pod label.
pod_labels = self._update_pod_label(pod, {'function_id': function_id})
# Create service for the chosen pod.
service_name = "service-%s" % function_id
labels.update({'function_id': function_id})
service_body = self.service_template.render({
"service_name": service_name,
"labels": labels,
"selector": pod_labels
})
ret = self.v1.create_namespaced_service(self.conf.kubernetes.namespace,
yaml.safe_load(service_body))
node_port = ret.spec.ports[0].node_port
LOG.debug(
'Service created for pod %s, service name: %s, node port: %s',
name, service_name, node_port)
# Get external ip address for an arbitrary node.
ret = self.v1.list_node()
addresses = ret.items[0].status.addresses
node_ip = None
for addr in addresses:
if addr.type == 'ExternalIP':
node_ip = addr.address
# FIXME: test purpose using minikube
if not node_ip:
for addr in addresses:
if addr.type == 'InternalIP':
node_ip = addr.address
# Download code package into container.
pod_service_url = 'http://%s:%s' % (node_ip, node_port)
request_url = '%s/download' % pod_service_url
download_url = ('http://%s:%s/v1/functions/%s?download=true' %
(self.conf.kubernetes.qinling_service_address,
self.conf.api.port, actual_function))
data = {
'download_url': download_url,
'function_id': actual_function,
'entry': entry,
'token': context.get_ctx().auth_token,
}
LOG.debug('Send request to pod %s, request_url: %s, data: %s', name,
request_url, data)
# TODO(kong): Here we sleep some time to avoid 'Failed to establish a
# new connection' error for some reason. Needs to find a better
# solution.
time.sleep(1)
r = requests.post(request_url, json=data)
if r.status_code != requests.codes.ok:
raise exc.OrchestratorException(
'Failed to download function code package.')
return name, pod_service_url
