def test_run_execution_image_type_function_timeout(self):
execution_id = common.generate_unicode_uuid()
function_id = common.generate_unicode_uuid()
pod1 = mock.Mock()
pod1.status.phase = ''
self.k8s_v1_api.read_namespaced_pod.return_value = pod1
result, output = self.manager.run_execution(
execution_id, function_id, 0,
identifier='fake_identifier',
timeout=1
)
self.assertFalse(result)
expected_output = {
'output': 'Function execution timeout.',
'duration': 1
}
self.assertEqual(expected_output, output)
def test_run_execution_image_type_function_read_pod_exception(self):
self.k8s_v1_api.read_namespaced_pod.side_effect = RuntimeError
execution_id = common.generate_unicode_uuid()
function_id = common.generate_unicode_uuid()
result, output = self.manager.run_execution(execution_id,
function_id,
0,
timeout=5)
self.k8s_v1_api.read_namespaced_pod.assert_called_once_with(
None, self.fake_namespace)
self.k8s_v1_api.read_namespaced_pod_log.assert_not_called()
self.assertFalse(result)
expected_output = {
'output': 'Function execution failed.',
'duration': 0
}
self.assertEqual(expected_output, output)
def test_scaleup_function_service_internal_ip(self):
pod = mock.Mock()
pod.metadata.name = self.rand_name('pod', prefix=self.prefix)
pod.metadata.labels = None
list_pod_ret = mock.Mock()
list_pod_ret.items = [pod]
self.k8s_v1_api.list_namespaced_pod.return_value = list_pod_ret
self.k8s_v1_api.create_namespaced_service.return_value = (
self._create_service())
self.k8s_v1_api.list_node.return_value = (
self._create_nodes_with_internal_ip())
runtime_id = common.generate_unicode_uuid()
function_id = common.generate_unicode_uuid()
pod_names, service_url = self.manager.scaleup_function(
function_id, 0, identifier=runtime_id)
self.assertEqual([pod.metadata.name], pod_names)
self.assertEqual(
'http://%s:%s' % (SERVICE_ADDRESS_INTERNAL, SERVICE_PORT),
service_url)
def test_scaleup_function_service_already_exists(self):
pod = mock.Mock()
pod.metadata.name = self.rand_name('pod', prefix=self.prefix)
pod.metadata.labels = {'pod1_key1': 'pod1_value1'}
list_pod_ret = mock.Mock()
list_pod_ret.items = [pod]
self.k8s_v1_api.list_namespaced_pod.return_value = list_pod_ret
exception = RuntimeError()
exception.status = 409
self.k8s_v1_api.create_namespaced_service.side_effect = exception
self.k8s_v1_api.read_namespaced_service.return_value = (
self._create_service())
self.k8s_v1_api.list_node.return_value = (
self._create_nodes_with_external_ip())
runtime_id = common.generate_unicode_uuid()
function_id = common.generate_unicode_uuid()
pod_names, service_url = self.manager.scaleup_function(
function_id, 0, identifier=runtime_id)
# in _prepare_pod
self.k8s_v1_api.read_namespaced_service.assert_called_once_with(
'service-%s-0' % function_id, self.fake_namespace)
def test_scaledown_function(self, etcd_util_get_workers_mock,
etcd_util_delete_workers_mock):
function_id = common.generate_unicode_uuid()
etcd_util_get_workers_mock.return_value = [
'worker_%d' % i for i in range(4)
]
self.default_engine.scaledown_function(mock.Mock(), function_id)
etcd_util_get_workers_mock.assert_called_once_with(function_id, 0)
self.orchestrator.delete_worker.assert_called_once_with('worker_0')
etcd_util_delete_workers_mock.assert_called_once_with(function_id,
'worker_0',
version=0)
def test_prepare_execution_with_image(self):
function_id = common.generate_unicode_uuid()
image = self.rand_name('image', prefix=self.prefix)
identifier = (
'%s-%s' %
(common.generate_unicode_uuid(dashed=False), function_id))[:63]
pod_name, url = self.manager.prepare_execution(function_id,
0,
rlimit=self.rlimit,
image=image,
identifier=identifier)
self.assertEqual(identifier, pod_name)
self.assertIsNone(url)
# in _create_pod
pod_body = self.manager.pod_template.render({
'pod_name':
identifier,
'labels': {
'function_id': function_id
},
'pod_image':
image,
'input': [],
'req_cpu':
str(cfg.CONF.resource_limits.default_cpu),
'req_memory':
str(cfg.CONF.resource_limits.default_memory),
'limit_cpu':
str(cfg.CONF.resource_limits.default_cpu),
'limit_memory':
str(cfg.CONF.resource_limits.default_memory)
})
self.k8s_v1_api.create_namespaced_pod.assert_called_once_with(
self.fake_namespace, body=yaml.safe_load(pod_body))
def test_prepare_execution_service_internal_ip(self):
pod = mock.Mock()
pod.metadata.name = self.rand_name('pod', prefix=self.prefix)
pod.metadata.labels = {'pod1_key1': 'pod1_value1'}
list_pod_ret = mock.Mock()
list_pod_ret.items = [pod]
self.k8s_v1_api.list_namespaced_pod.return_value = list_pod_ret
self.k8s_v1_api.create_namespaced_service.return_value = (
self._create_service()
)
self.k8s_v1_api.list_node.return_value = (
self._create_nodes_with_internal_ip()
)
runtime_id = common.generate_unicode_uuid()
function_id = common.generate_unicode_uuid()
pod_names, service_url = self.manager.prepare_execution(
function_id, 0, rlimit=None, image=None, identifier=runtime_id,
labels={'runtime_id': runtime_id})
self.assertEqual(pod.metadata.name, pod_names)
self.assertEqual(
'http://%s:%s' % (SERVICE_ADDRESS_INTERNAL, SERVICE_PORT),
service_url)
def test_prepare_execution_with_image_json_input(self):
function_id = common.generate_unicode_uuid()
image = self.rand_name('image', prefix='TestKubernetesManager')
identifier = (
'%s-%s' %
(common.generate_unicode_uuid(dashed=False), function_id))[:63]
fake_input = '["input_item3", "input_item4"]'
pod_name, url = self.manager.prepare_execution(function_id,
0,
rlimit=self.rlimit,
image=image,
identifier=identifier,
input=fake_input)
# in _create_pod
pod_body = self.manager.pod_template.render({
'pod_name':
identifier,
'labels': {
'function_id': function_id
},
'pod_image':
image,
'input': ['input_item3', 'input_item4'],
'req_cpu':
str(cfg.CONF.resource_limits.default_cpu),
'req_memory':
str(cfg.CONF.resource_limits.default_memory),
'limit_cpu':
str(cfg.CONF.resource_limits.default_cpu),
'limit_memory':
str(cfg.CONF.resource_limits.default_memory)
})
self.k8s_v1_api.create_namespaced_pod.assert_called_once_with(
self.fake_namespace, body=yaml.safe_load(pod_body))
def test_scaleup_function_multiple_workers(
self,
etcd_util_create_worker_mock,
etcd_util_create_service_url_mock
):
function_id = common.generate_unicode_uuid()
runtime_id = common.generate_unicode_uuid()
self.orchestrator.scaleup_function.return_value = (
['worker0', 'worker1'], 'url')
self.default_engine.scaleup_function(
mock.Mock(), function_id, 0, runtime_id, count=2
)
self.orchestrator.scaleup_function.assert_called_once_with(
function_id, 0, identifier=runtime_id, count=2
)
# Two new workers are created.
expected = [mock.call(function_id, 'worker0', version=0),
mock.call(function_id, 'worker1', version=0)]
etcd_util_create_worker_mock.assert_has_calls(expected)
etcd_util_create_service_url_mock.assert_called_once_with(
function_id, 'url', version=0
)
def test_delete_function_with_labels(self):
services = mock.Mock()
services.items = []
labels = {'key1': 'value1', 'key2': 'value2'}
selector = common.convert_dict_to_string(labels)
self.k8s_v1_api.list_namespaced_service.return_value = services
function_id = common.generate_unicode_uuid()
self.manager.delete_function(function_id, 0, labels=labels)
self.k8s_v1_api.list_namespaced_service.assert_called_once_with(
self.fake_namespace, label_selector=selector)
self.k8s_v1_api.delete_namespaced_service.assert_not_called()
delete_pod = self.k8s_v1_api.delete_collection_namespaced_pod
delete_pod.assert_called_once_with(self.fake_namespace,
label_selector=selector)
def test_delete_function(self):
# Deleting namespaced service is also tested in this.
svc1 = mock.Mock()
svc1_name = self.rand_name('service', prefix=self.prefix)
svc1.metadata.name = svc1_name
svc2 = mock.Mock()
svc2_name = self.rand_name('service', prefix=self.prefix)
svc2.metadata.name = svc2_name
services = mock.Mock()
services.items = [svc1, svc2]
self.k8s_v1_api.list_namespaced_service.return_value = services
function_id = common.generate_unicode_uuid()
self.manager.delete_function(function_id, 0)
args, kwargs = self.k8s_v1_api.list_namespaced_service.call_args
self.assertIn(self.fake_namespace, args)
self.assertIn(
"function_id=%s" % function_id,
kwargs.get("label_selector")
)
self.assertIn(
"function_version=0",
kwargs.get("label_selector")
)
delete_service_calls = [
mock.call(svc1_name, self.fake_namespace, mock.ANY),
mock.call(svc2_name, self.fake_namespace, mock.ANY)
]
self.k8s_v1_api.delete_namespaced_service.assert_has_calls(
delete_service_calls)
self.assertEqual(
2, self.k8s_v1_api.delete_namespaced_service.call_count
)
args, kwargs = self.k8s_v1_api.delete_collection_namespaced_pod. \
call_args
self.assertIn(self.fake_namespace, args)
self.assertIn(
"function_id=%s" % function_id,
kwargs.get("label_selector")
)
self.assertIn(
"function_version=0",
kwargs.get("label_selector")
)
def test_scaledown_function_multiple_workers(
self, etcd_util_get_workers_mock, etcd_util_delete_workers_mock):
function_id = common.generate_unicode_uuid()
etcd_util_get_workers_mock.return_value = [
'worker_%d' % i for i in range(4)
]
self.default_engine.scaledown_function(mock.Mock(),
function_id,
count=2)
etcd_util_get_workers_mock.assert_called_once_with(function_id, 0)
# First two workers will be deleted.
expected = [mock.call('worker_0'), mock.call('worker_1')]
self.orchestrator.delete_worker.assert_has_calls(expected)
self.assertEqual(2, self.orchestrator.delete_worker.call_count)
expected = [
mock.call(function_id, 'worker_0', version=0),
mock.call(function_id, 'worker_1', version=0)
]
etcd_util_delete_workers_mock.assert_has_calls(expected)
self.assertEqual(2, etcd_util_delete_workers_mock.call_count)
def create_execution(self,
ctx,
execution_id,
function_id,
runtime_id,
input=None):
LOG.info(
'Creating execution. execution_id=%s, function_id=%s, '
'runtime_id=%s, input=%s', execution_id, function_id, runtime_id,
input)
# FIXME(kong): Make the transaction range smaller.
with db_api.transaction():
execution = db_api.get_execution(execution_id)
function = db_api.get_function(function_id)
if function.service:
func_url = '%s/execute' % function.service.service_url
LOG.debug('Found service url for function: %s, url: %s',
function_id, func_url)
data = {'input': input, 'execution_id': execution_id}
r = self.session.post(func_url, json=data)
res = r.json()
LOG.debug('Finished execution %s', execution_id)
success = res.pop('success')
execution.status = status.SUCCESS if success else status.FAILED
execution.logs = res.pop('logs', '')
execution.output = res
return
source = function.code['source']
image = None
identifier = None
labels = None
if source == constants.IMAGE_FUNCTION:
image = function.code['image']
identifier = ('%s-%s' %
(common.generate_unicode_uuid(dashed=False),
function_id))[:63]
labels = {'function_id': function_id}
else:
identifier = runtime_id
labels = {'runtime_id': runtime_id}
worker_name, service_url = self.orchestrator.prepare_execution(
function_id,
image=image,
identifier=identifier,
labels=labels,
input=input,
entry=function.entry,
trust_id=function.trust_id)
output = self.orchestrator.run_execution(
execution_id,
function_id,
input=input,
identifier=identifier,
service_url=service_url,
)
logs = ''
# Execution log is only available for non-image source execution.
if service_url:
logs = output.pop('logs', '')
success = output.pop('success')
else:
# If the function is created from docker image, the output is
# direct output, here we convert to a dict to fit into the db
# schema.
output = {'output': output}
success = True
LOG.debug('Finished execution. execution_id=%s, output=%s',
execution_id, output)
execution.output = output
execution.logs = logs
execution.status = status.SUCCESS if success else status.FAILED
# No service is created in orchestrator for single container.
if not image:
mapping = {
'function_id': function_id,
'service_url': service_url,
}
db_api.create_function_service_mapping(mapping)
worker = {
'function_id': function_id,
'worker_name': worker_name
}
db_api.create_function_worker(worker)
def create_execution(self,
ctx,
execution_id,
function_id,
function_version,
runtime_id,
input=None):
LOG.info(
'Creating execution. execution_id=%s, function_id=%s, '
'function_version=%s, runtime_id=%s, input=%s', execution_id,
function_id, function_version, runtime_id, input)
function = db_api.get_function(function_id)
source = function.code['source']
rlimit = {'cpu': function.cpu, 'memory_size': function.memory_size}
image = None
identifier = None
labels = None
svc_url = None
is_image_source = source == constants.IMAGE_FUNCTION
# Auto scale workers if needed
if not is_image_source:
try:
svc_url = self.function_load_check(function_id,
function_version,
runtime_id)
except exc.OrchestratorException as e:
utils.handle_execution_exception(execution_id, str(e))
return
temp_url = etcd_util.get_service_url(function_id, function_version)
svc_url = svc_url or temp_url
if svc_url:
func_url = '%s/execute' % svc_url
LOG.debug(
'Found service url for function: %s(version %s), execution: '
'%s, url: %s', function_id, function_version, execution_id,
func_url)
data = utils.get_request_data(CONF, function_id, function_version,
execution_id, rlimit, input,
function.entry, function.trust_id,
self.qinling_endpoint)
success, res = utils.url_request(self.session, func_url, body=data)
utils.finish_execution(execution_id,
success,
res,
is_image_source=is_image_source)
return
if source == constants.IMAGE_FUNCTION:
image = function.code['image']
# Be consistent with k8s naming convention
identifier = (
'%s-%s' %
(common.generate_unicode_uuid(dashed=False), function_id))[:63]
else:
identifier = runtime_id
labels = {'runtime_id': runtime_id}
try:
# For image function, it will be executed inside this method; for
# package type function it only sets up underlying resources and
# get a service url. If the service url is already created
# beforehand, nothing happens.
_, svc_url = self.orchestrator.prepare_execution(
function_id,
function_version,
rlimit=rlimit,
image=image,
identifier=identifier,
labels=labels,
input=input,
)
except exc.OrchestratorException as e:
utils.handle_execution_exception(execution_id, str(e))
return
# For image type function, read the worker log; For package type
# function, invoke and get log
success, res = self.orchestrator.run_execution(
execution_id,
function_id,
function_version,
rlimit=rlimit if svc_url else None,
input=input,
identifier=identifier,
service_url=svc_url,
entry=function.entry,
trust_id=function.trust_id)
utils.finish_execution(execution_id,
success,
res,
is_image_source=is_image_source)
def create_execution(self,
ctx,
execution_id,
function_id,
runtime_id,
input=None):
LOG.info(
'Creating execution. execution_id=%s, function_id=%s, '
'runtime_id=%s, input=%s', execution_id, function_id, runtime_id,
input)
function = db_api.get_function(function_id)
source = function.code['source']
image = None
identifier = None
labels = None
svc_url = None
# Auto scale workers if needed
if source != constants.IMAGE_FUNCTION:
svc_url = self.function_load_check(function_id, runtime_id)
temp_url = etcd_util.get_service_url(function_id)
svc_url = svc_url or temp_url
if svc_url:
func_url = '%s/execute' % svc_url
LOG.debug(
'Found service url for function: %s, execution: %s, url: %s',
function_id, execution_id, func_url)
data = utils.get_request_data(CONF, function_id, execution_id,
input, function.entry,
function.trust_id,
self.qinling_endpoint)
success, res = utils.url_request(self.session, func_url, body=data)
success = success and res.pop('success')
LOG.debug('Finished execution %s, success: %s', execution_id,
success)
db_api.update_execution(
execution_id, {
'status': status.SUCCESS if success else status.FAILED,
'logs': res.pop('logs', ''),
'result': res
})
return
if source == constants.IMAGE_FUNCTION:
image = function.code['image']
identifier = (
'%s-%s' %
(common.generate_unicode_uuid(dashed=False), function_id))[:63]
labels = {'function_id': function_id}
else:
identifier = runtime_id
labels = {'runtime_id': runtime_id}
_, svc_url = self.orchestrator.prepare_execution(
function_id,
image=image,
identifier=identifier,
labels=labels,
input=input,
)
success, res = self.orchestrator.run_execution(
execution_id,
function_id,
input=input,
identifier=identifier,
service_url=svc_url,
entry=function.entry,
trust_id=function.trust_id)
logs = ''
# Execution log is only available for non-image source execution.
if svc_url:
logs = res.pop('logs', '')
success = success and res.pop('success')
else:
# If the function is created from docker image, the result is
# direct output, here we convert to a dict to fit into the db
# schema.
res = {'output': res}
LOG.debug('Finished execution %s, success: %s', execution_id, success)
db_api.update_execution(
execution_id, {
'status': status.SUCCESS if success else status.FAILED,
'logs': logs,
'result': res
})