def test_start_pod_retries_three_times(self, mock_run_pod_async):
mock_run_pod_async.side_effect = [
ApiException(status=409),
ApiException(status=409),
ApiException(status=409),
ApiException(status=409),
]
with pytest.raises(ApiException):
self.pod_launcher.start_pod(mock.sentinel)
assert mock_run_pod_async.call_count == 3
def delete_storage_class(self, name, body):
if self.name == 'fail':
raise ApiException('Delete storage class fail')
if self.name == '404' or name == '404':
raise ApiException(reason='Not Found')
if self.name == 'test1' or name == 'test1':
my_response = namedtuple('my_response', 'message')
return my_response(message='Failed')
if self.name == 'test2' or name == 'test2':
my_response = namedtuple('my_response', 'message')
return my_response(message=None)
return {'key1': 'value1'}
def create(self, namespace: str, body, **kwargs):
if not body:
raise ValueError
body.metadata = dict(body.metadata, **{"namespace": namespace})
body.metadata = V1ObjectMeta(**body.metadata)
if namespace not in self._namespaced_items:
raise ApiException(404, "Not Found")
if body in self._namespaced_items[namespace]:
raise ApiException(409, "AlreadyExists")
self._items.add(body)
self._namespaced_items[namespace].add(body)
return body
def delete_namespaced_deployment(self, name, body, namespace):
if self.name == 'fail':
raise ApiException('Delete deployment fail')
if self.name == '404' or name == '404':
raise ApiException(reason='Not Found')
if self.name == 'test1' or name == 'test1':
my_response = namedtuple('my_response', 'message')
return my_response(message='Failed')
if self.name == 'test2' or name == 'test2':
my_response = namedtuple('my_response', 'message')
return my_response(message=None)
return {'key1': 'value1'}
def test_update_secret_valid_key(credstash_get_secret_mock):
cont = CredStashController("none", "none", "none", "none", "none")
cont.v1core = MagicMock()
cont.v1core.read_namespaced_secret = MagicMock(side_effect=ApiException(
status=404))
credstash_secret = {
"metadata": {
"namespace": "test",
"name": "boom"
},
"spec": [{
"from": "ba",
"name": "lala",
"version": "0001"
}],
}
cont.update_secret(credstash_secret, resource_version=1)
assert (
cont.v1core.create_namespaced_secret.call_args_list[0][0][0] == "test")
assert cont.v1core.create_namespaced_secret.call_args_list[0][0][
1].to_dict() == {
"api_version": "v1",
"data": {
"lala": "MTIz"
},
"kind": "Secret",
"metadata": {
"annotations": {
"credstash-fully-managed": "true",
"credstash-resourceversion": "1",
},
"cluster_name": None,
"creation_timestamp": None,
"deletion_grace_period_seconds": None,
"deletion_timestamp": None,
"finalizers": None,
"generate_name": None,
"generation": None,
"initializers": None,
"labels": None,
"name": "boom",
"namespace": "test",
"owner_references": None,
"resource_version": None,
"self_link": None,
"uid": None,
},
"string_data": None,
"type": None,
}
credstash_get_secret_mock.assert_called_once_with(
aws_access_key_id="none",
aws_secret_access_key="none",
name="ba",
region="none",
table="none",
version="0001",
)
def get_cr(self, name: str, use_cache: bool = True) -> Dict:
"""Retrieve a specific custom resource by name."""
cls = self.__class__
if use_cache is True:
try:
return copy.deepcopy(self._cr_cache[name])
except KeyError as err:
_LOGGER.error(
'%s/%s: Exception when retrieving CR %s: not found',
self.group, self.plural, name)
raise ApiException(status=404) from err
try:
api_response = self.co_api.get_namespaced_custom_object(
self.group,
self.version,
self.namespace,
self.plural,
name,
_request_timeout=cls.REQUEST_TIMEOUT)
except ApiException as err:
_LOGGER.error('%s/%s: Exception when retrieving CR %s: %s',
self.group, self.plural, name, err)
raise
else:
_LOGGER.debug('%s/%s: Successfully retrieved CR %s', self.group,
self.plural, name)
return api_response
def test_creating_cro_allows_conflicts(cl, client, has_conf):
has_conf.return_value = False
resp_data = {
"kind": "Status",
"apiVersion": "v1",
"metadata": {},
"status": "Failure",
"message":
'crontabs.stable.example.com "my-new-cron-object" already exists',
"reason": "AlreadyExists",
"details": {
"name": "my-new-cron-object",
"group": "stable.example.com",
"kind": "crontabs",
},
"code": 409,
}
resp = MagicMock()
resp.status = 409
resp.reason = "Conflicts"
resp.data = json.dumps(resp_data)
resource = {
"group": "stable.example.com",
"version": "v1",
"plural": "crontabs",
"resource": {
"apiVersion": "stable.example.com/v1",
"kind": "CronTab",
"metadata": {
"name": "my-new-cron-object"
},
"spec": {
"cronSpec": "* * * * */5",
"image": "my-awesome-cron-image"
},
},
}
v1 = MagicMock()
client.CustomObjectsApi.return_value = v1
v1.create_namespaced_custom_object.side_effect = ApiException(
http_resp=resp)
o = create_custom_object(group="stable.example.com",
version="v1",
plural="crontabs",
resource=resource)
assert o == resp_data
assert v1.create_namespaced_custom_object.call_count == 1
v1.create_namespaced_custom_object.assert_called_with(
"stable.example.com",
"v1",
"default",
"crontabs",
resource,
_preload_content=False,
)
def websocket_call(configuration, *args, **kwargs):
"""An internal function to be called in api-client when a websocket
connection is required. args and kwargs are the parameters of
apiClient.request method."""
url = args[1]
_request_timeout = kwargs.get("_request_timeout", 60)
_preload_content = kwargs.get("_preload_content", True)
capture_all = kwargs.get("capture_all", True)
headers = kwargs.get("headers")
# Expand command parameter list to indivitual command params
query_params = []
for key, value in kwargs.get("query_params", {}):
if key == 'command' and isinstance(value, list):
for command in value:
query_params.append((key, command))
else:
query_params.append((key, value))
if query_params:
url += '?' + urlencode(query_params)
try:
client = WSClient(configuration, get_websocket_url(url), headers, capture_all)
if not _preload_content:
return client
client.run_forever(timeout=_request_timeout)
return WSResponse('%s' % ''.join(client.read_all()))
except (Exception, KeyboardInterrupt, SystemExit) as e:
raise ApiException(status=0, reason=str(e))
def create_namespaced_service(body, namespace):
name = '{}-{}'.format(body.metadata.name, namespace)
if name in MockCoreV1Api.service_map.keys():
raise ApiException(status=409) # mock conflict
else:
MockCoreV1Api.service_map[name] = body
return body
def delete_deployment(source):
'''
Deletes the kubernetes deployment defined by name and namespace
'''
src_obj = __read_and_render_yaml_file(source)
metadata = src_obj['metadata']
name = metadata['name']
namespace = metadata['namespace']
body = kubernetes.client.V1DeleteOptions(api_version="v1",
grace_period_seconds=50,
propagation_policy="Background")
try:
api_instance = kubernetes.client.AppsV1Api(
kubernetes.client.ApiClient(configuration))
api_response = api_instance.delete_namespaced_deployment(
pretty=pretty, name=name, namespace=namespace, body=body)
mutable_api_response = api_response.to_dict()
if mutable_api_response['code'] != 200:
logging.warning(
'Reached polling time limit. Deployment is not yet '
'deleted, but we are backing off. Sorry, but you\'ll '
'have to check manually.')
return mutable_api_response
except (ApiException, HTTPError) as exc:
if isinstance(exc, ApiException) and exc.status == 404:
return None
else:
logging.exception(
'Exception when calling '
'ExtensionsV1beta1Api->delete_namespaced_deployment: '
'{0}'.format(exc))
raise ApiException(exc)
def test_ns_create_new(self, mock_ns_read, mock_api, mock_print):
# TODO: We should ideally replicate the correct API exception
mock_ns_read.side_effect = ApiException()
ns_create('a-namespace')
mock_ns_read.assert_called_once_with('a-namespace', verbose=False)
mock_api.create_namespace.assert_called_once()
mock_print.assert_not_called()
def test_createconfigmapfromlocaldir(self):
local_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'..', 'resources', 'localDirTest')
local_config = LocalDirConfig(local_dir)
api_instance = Mock()
api_instance.close = Mock()
api_core_v1_mock = Mock()
api_core_v1_mock.create_namespaced_config_map = Mock(return_value={
'api_version': 'v1',
'binary_data': None,
'data': {}
})
api_core_v1_mock.patch_namespaced_config_map = Mock(return_value={
'api_version': 'v1',
'binary_data': None,
'data': {}
})
api_core_v1_mock.create_namespaced_config_map.side_effect = Mock(
side_effect=ApiException('409'))
config_map = K8sConfigMap('collection-ingester',
'sdap',
local_config,
api_instance=api_instance,
api_core_v1_instance=api_core_v1_mock)
config_map.publish()
def test_lock_expiration(self, _):
# Timestamp on the 'lock' is old to ensure lock is expired
self.resp['data']['lastUpdated'] = "2018-01-22T16:20:14Z"
# When the lock already exists, Kubernetes responds with a 409
self.mock_create.side_effect = ApiException(status=409)
# Getting the lock should return the 'lock' above
self.mock_read.return_value = self.resp
# New return value of create should have a newer timestamp
new_resp = copy.deepcopy(self.resp)
new_time = str(datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'))
new_resp['metadata']['creationTimestamp'] = new_time
mock_create = self.mock_create
def clear_side_effect(*args, **kwargs):
mock_create.side_effect = None
mock_create.return_value = new_resp
# Once the lock is 'deleted' we need to stop create from raising err
self.mock_delete.side_effect = clear_side_effect
try:
self.test_lock.acquire_lock()
except lock.LockException:
self.fail("acquire_lock() raised LockException unexpectedly")
def read_namespaced_pod(self, name, namespace):
print(self, namespace)
for item_list in MockCoreV1ApiForTTL.pod_map.values():
for item in item_list:
if item.metadata.name == name:
return item
raise ApiException(status=404)
def test_secret_from_files(self, mock_secret_read, mock_secret_create,
mock_open):
mock_secret_read.side_effect = ApiException()
secret_from_files("a_secret", "a-namespace", {"a": "path_a"})
mock_secret_read.assert_called_once()
mock_secret_create.assert_called_once()
mock_open.assert_called_once()
def test_edi_deploy_one_model(self):
with EDITestServer() as edi:
with m_func('kubernetes.client.CoreV1Api.read_namespaced_service',
[ApiException(404), 'demo_abc_model_1_0']), \
m_func('kubernetes.client.ExtensionsV1beta1Api.read_namespaced_deployment', 'demo_abc_model_1_0'), \
m_func('kubernetes.client.ExtensionsV1beta1Api.create_namespaced_deployment', 'deploy_done'), \
m_func('kubernetes.client.CoreV1Api.create_namespaced_service', 'deploy_done'), \
mock.patch('legion.k8s.utils.build_client', return_value=None), \
mock.patch('legion.k8s.enclave.Enclave.graphite_service', return_value=None), \
mock.patch('legion.k8s.utils.get_docker_image_labels', return_value=DOCKER_IMAGE_LABELS):
deployments = edi.edi_client.deploy(
'127.0.0.1/legion/test-bare-model-api-model-1:0.9.0-20181106123540.560.3b9739a')
# Test count of returned deployments
self.assertIsInstance(deployments, list)
self.assertEqual(len(deployments), 1)
# Check deployed model fields
expected_result = {
'image': '127.0.0.1/legion/test-bare-model-api-model-1:0.9.0-20181106123540.560.3b9739a',
'model': 'demo-abc-model',
'version': '1.0',
'scale': 1,
'ready_replicas': 1,
'status': 'ok',
'namespace': 'debug-enclave'
}
actual_result = {
'image': deployments[0].image,
'model': deployments[0].model,
'version': deployments[0].version,
'scale': deployments[0].scale,
'ready_replicas': deployments[0].ready_replicas,
'status': deployments[0].status,
'namespace': deployments[0].namespace
}
self.assertDictEqual(expected_result, actual_result)
def test_job_logs_not_ready(self, mock_core_client):
namespace = "notready"
manager = JobManager(
namespace=namespace, signer=Mock(), register=StaticJobDefinitionsRegister()
)
pod_name = "p"
container_name = "c"
mock_core_client.list_namespaced_pod.return_value.items = [
V1Pod(
metadata=V1ObjectMeta(name=pod_name),
spec=V1PodSpec(containers=[V1Container(name=container_name)]),
)
]
mock_core_client.read_namespaced_pod_log.side_effect = ApiException(
http_resp=Mock(
data={
"message": f'container "{container_name}" in pod "{pod_name}" is waiting to start: ContainerCreating'
}
)
)
# No exception
logs = manager.job_logs("whatever")
assert logs == {pod_name: {container_name: ["ContainerCreating"]}}
def portforward_call(configuration, _method, url, **kwargs):
"""An internal function to be called in api-client when a websocket
connection is required for port forwarding. args and kwargs are the
parameters of apiClient.request method."""
query_params = kwargs.get("query_params")
ports = []
for param, value in query_params:
if param == 'ports':
for port in value.split(','):
try:
port_number = int(port)
except ValueError:
raise ApiValueError("Invalid port number: %s" % port)
if not (0 < port_number < 65536):
raise ApiValueError("Port number must be between 0 and 65536: %s" % port)
if port_number in ports:
raise ApiValueError("Duplicate port numbers: %s" % port)
ports.append(port_number)
if not ports:
raise ApiValueError("Missing required parameter `ports`")
url = get_websocket_url(url, query_params)
headers = kwargs.get("headers")
try:
websocket = create_websocket(configuration, url, headers)
return PortForward(websocket, ports)
except (Exception, KeyboardInterrupt, SystemExit) as e:
raise ApiException(status=0, reason=str(e))
def test_get_lock(self, _):
try:
# read needs to raise a 404 when the lock doesn't exist
self.mock_read.side_effect = ApiException(status=404)
mock_read = self.mock_read
resp = self.resp
def update_get_and_set_return(*args, **kwargs):
# Once the lock is 'created' it should no longer raise err
mock_read.read_custom_resource.side_effect = None
mock_read.read_custom_resource.return_value = resp
# Set the mock_create return to return the new lock
return resp
self.mock_create.side_effect = update_get_and_set_return
self.test_lock.acquire_lock()
except lock.LockException:
self.fail("acquire_lock() raised LockException unexpectedly")
except ApiException:
self.fail("acquire_lock() raised ApiException unexpectedly")
try:
self.test_lock.release_lock()
except lock.LockException:
self.fail("release_lock() raised LockException unexpectedly")
except ApiException:
self.fail("acquire_lock() raised ApiException unexpectedly")
def test_is_current_user_administrator_(mocker):
gcr_mock = mocker.patch("util.k8s.k8s_info.get_cluster_roles", side_effect=ApiException(status=404))
with pytest.raises(ApiException):
is_current_user_administrator()
assert gcr_mock.call_count == 1
def test_ns_create_new_verbose(self, mock_ns_read, mock_api, mock_print):
# TODO: We should ideally replicate the correct API exception
mock_ns_read.side_effect = ApiException()
ns_create("a-namespace", verbose=True)
mock_ns_read.assert_called_once_with("a-namespace", verbose=True)
mock_api.create_namespace.assert_called_once()
mock_print.assert_called_once_with('Created namespace "a-namespace"')
def replace(self, name: str, namespace: str, body, **kwargs):
if namespace not in self._namespaced_items:
raise ApiException(404, "Not Found")
if not body:
raise ValueError
for service in self._namespaced_items[namespace]:
if service.metadata.name == name:
self._namespaced_items[namespace].remove(service)
self._items.remove(service)
break
else:
raise ApiException(404, "Not Found")
return self.create(namespace, body)
def test_workflow_finish_and_kubernetes_not_available(
in_memory_queue_connection,
sample_serial_workflow_in_db,
consume_queue,
):
"""Test workflow finish with a Kubernetes connection troubles."""
sample_serial_workflow_in_db.status = RunStatus.running
next_status = RunStatus.failed
job_status_consumer = JobStatusConsumer(
connection=in_memory_queue_connection)
workflow_status_publisher = WorkflowStatusPublisher(
connection=in_memory_queue_connection, queue=job_status_consumer.queue)
workflow_status_publisher.publish_workflow_status(
str(sample_serial_workflow_in_db.id_),
next_status.value,
)
k8s_corev1_api_client_mock = Mock()
k8s_corev1_api_client_mock.delete_namespaced_job = Mock(
side_effect=ApiException(reason="Could not delete job.", status=404))
with patch(
"reana_workflow_controller.consumer.current_k8s_corev1_api_client",
k8s_corev1_api_client_mock,
):
consume_queue(job_status_consumer, limit=1)
assert sample_serial_workflow_in_db.status == next_status
def list(self, namespace: str, **kwargs):
if namespace not in self._namespaced_items:
raise ApiException(404, "Not Found")
label_selector = kwargs.get("label_selector")
field_selector = kwargs.get("field_selector")
services = []
if not label_selector and not field_selector:
services = self._namespaced_items[namespace]
else:
for service in self._namespaced_items[namespace]:
if label_selector and field_selector:
if self._label_in_resource(
service,
label_selector) and self._field_in_resource(
service, field_selector):
services.append(service)
elif label_selector:
if self._label_in_resource(service, label_selector):
services.append(service)
elif field_selector:
if self._field_in_resource(service, field_selector):
services.append(service)
return services
def read_namespaced_persistent_volume_claim_status(name, **_):
if name == 'nonexistent-pvc':
raise ApiException(status=404, reason="nonexistent-pvc")
elif name == 'pending-pvc':
return DotDict({'status': DotDict({'phase': 'Pending'})})
else:
return DotDict({'status': DotDict({'phase': 'Bound'})})
def test_list_endpoints(mocker, apps_client_mock_endpoint_utils,
tenant_exception, k8s_exception):
tenant_exists_mock = mocker.patch(
'management_api.endpoints.endpoint_utils.tenant_exists')
create_apps_client_mock, apps_client = apps_client_mock_endpoint_utils
if tenant_exception:
with pytest.raises(TenantDoesNotExistException):
tenant_exists_mock.return_value = False
list_endpoints(namespace="test", id_token=user_token)
else:
tenant_exists_mock.return_value = True
if k8s_exception:
with pytest.raises(KubernetesGetException):
apps_client.list_namespaced_deployment.side_effect = ApiException(
)
list_endpoints(namespace="test", id_token=user_token)
else:
endpoints_name_status_mock = mocker.patch(
'management_api.endpoints.endpoint_utils.get_endpoints_name_status'
)
endpoints_name_status_mock.return_value = {}
apps_client.list_namespaced_deployment.return_value = {}
list_endpoints(namespace="test", id_token=user_token)
endpoints_name_status_mock.assert_called_once()
create_apps_client_mock.assert_called_once()
apps_client.list_namespaced_deployment.assert_called_once()
tenant_exists_mock.assert_called_once()
def create_namespaced_ingress(namespace, body):
name = '{}-{}'.format(namespace, body.metadata['name'])
if name in MockExtensionsV1beta1Api.ingress_map.keys():
raise ApiException(status=409)
else:
MockExtensionsV1beta1Api.ingress_map[name] = body
return body
def test_ensure_pod_disruption_budget_create(
bounce_margin_factor_set,
mock_pdr_for_service_instance,
mock_load_system_paasta_config,
):
mock_load_system_paasta_config.return_value.get_pdb_max_unavailable.return_value = 3
mock_req_pdr = mock.Mock()
mock_req_pdr.spec.max_unavailable = 10 if bounce_margin_factor_set else 3
mock_pdr_for_service_instance.return_value = mock_req_pdr
mock_client = mock.MagicMock()
mock_client.policy.read_namespaced_pod_disruption_budget.side_effect = ApiException(
status=404)
app = mock.MagicMock()
if bounce_margin_factor_set:
app.soa_config.config_dict = {"bounce_margin_factor": 0.1}
app.soa_config.get_bounce_margin_factor.return_value = 0.1
app.kube_deployment.service.return_value = "fake_service"
app.kube_deployment.instance.return_value = "fake_instance"
Application.ensure_pod_disruption_budget(self=app, kube_client=mock_client)
mock_client.policy.create_namespaced_pod_disruption_budget.assert_called_once_with(
namespace="paasta", body=mock_req_pdr)
def test_is_pod_already_deployed(init_openshift_deployer):
od = init_openshift_deployer
flexmock(od).should_receive("get_pod").and_raise(
ApiException(status=200, reason="POD already exists")
)
with pytest.raises(SandcastleExecutionError):
od.is_pod_already_deployed()
def test_start_pod_retries_on_409_error(self, mock_run_pod_async):
mock_run_pod_async.side_effect = [
ApiException(status=409),
mock.MagicMock(),
]
self.pod_launcher.start_pod(mock.sentinel)
assert mock_run_pod_async.call_count == 2
