def test_list_pods(self, list_all_p, list_p):
pod = V1Pod(metadata=V1ObjectMeta(name='test'))
list_p.return_value = V1PodList(items=[pod])
list_all_p.return_value = V1PodList(items=[pod])
res = list_pods(namespace='test', label_selector='test')
self.assertIsInstance(res, V1PodList)
self.assertEqual(1, len(res.items))
res = list_pods(label_selector='test')
self.assertIsInstance(res, V1PodList)
self.assertEqual(1, len(res.items))
list_p.side_effect = ApiException('Test')
list_all_p.side_effect = ApiException('Test')
with self.assertRaises(api.ApiError) as e:
list_pods(namespace='test')
expected = ("Error when listing pods in namespace test: (Test)\n"
"Reason: None\n")
self.assertEqual(expected, str(e.exception))
self.assertEqual(400, e.exception.status_code)
with self.assertRaises(api.ApiError) as e:
list_pods()
expected = ("Error when listing pods in namespace All: (Test)\n"
"Reason: None\n")
self.assertEqual(expected, str(e.exception))
self.assertEqual(400, e.exception.status_code)
def test_create_flapp(self):
test_yaml = {'metadata': {'name': 'test app'}}
mock_crds = MagicMock()
self._k8s_client.crds = mock_crds
# Test create successfully
mock_crds.create_namespaced_custom_object = MagicMock()
self._k8s_client.create_flapp(test_yaml)
mock_crds.create_namespaced_custom_object.assert_called_once_with(
group='fedlearner.k8s.io',
namespace='default',
plural='flapps',
version='v1alpha1',
body=test_yaml)
# Test that flapp exists
mock_crds.create_namespaced_custom_object = MagicMock(
side_effect=[ApiException(status=409), None])
self._k8s_client.delete_flapp = MagicMock()
self._k8s_client.create_flapp(test_yaml)
self._k8s_client.delete_flapp.assert_called_once_with('test app')
self.assertEqual(mock_crds.create_namespaced_custom_object.call_count,
2)
# Test with other exceptions
mock_crds.create_namespaced_custom_object = MagicMock(
side_effect=ApiException(status=114))
with self.assertRaises(ApiException):
self._k8s_client.create_flapp(test_yaml)
self.assertEqual(mock_crds.create_namespaced_custom_object.call_count,
3)
def test_delete_flapp(self):
mock_crds = MagicMock()
self._k8s_client.crds = mock_crds
# Test delete successfully
mock_crds.delete_namespaced_custom_object = MagicMock()
self._k8s_client.delete_flapp('test_flapp')
mock_crds.delete_namespaced_custom_object.assert_called_once_with(
group='fedlearner.k8s.io',
name='test_flapp',
namespace='default',
plural='flapps',
version='v1alpha1')
# Tests that the flapp has been deleted
mock_crds.delete_namespaced_custom_object = MagicMock(
side_effect=ApiException(status=404))
self._k8s_client.delete_flapp('test_flapp2')
self.assertEqual(mock_crds.delete_namespaced_custom_object.call_count,
1)
# Tests with other exceptions
mock_crds.delete_namespaced_custom_object = MagicMock(
side_effect=ApiException(status=500))
with self.assertRaises(RuntimeError):
self._k8s_client.delete_flapp('test_flapp3')
self.assertEqual(mock_crds.delete_namespaced_custom_object.call_count,
3)
def read_namespaced_secret(self, name: str, namespace: str) -> V1Secret:
self._maybe_error("read_namespaced_secret", name, namespace)
if namespace not in self.objects:
reason = f"{namespace}/{name} not found"
raise ApiException(status=404, reason=reason)
if name not in self.objects[namespace]:
reason = f"{namespace}/{name} not found"
raise ApiException(status=404, reason=reason)
assert isinstance(
self.objects[namespace][name],
V1Secret), "Name conflicts between objects not supported"
return self.objects[namespace][name]
def test_get(self, status_p, logs_p, call_p):
call_p.return_value = 100
status_p.return_value = 'Ready'
logs_p.return_value = ['bla']
res = cubegens.get(user_id='drwho', cubegen_id='id')
self.assertDictEqual(
{
'cubegen_id': 'id',
'status': 'Ready',
'output': ['bla'],
'progress': 100
}, res)
status_p.return_value = None
with self.assertRaises(api.ApiError) as e:
cubegens.get(user_id='drwho', cubegen_id='id')
self.assertEqual("Cubegen id not found", str(e.exception))
self.assertEqual(404, e.exception.status_code)
status_p.return_value = 'Ready'
status_p.side_effect = ApiException(401, 'Unauthorized')
with self.assertRaises(api.ApiError) as e:
cubegens.get(user_id='drwho', cubegen_id='id')
self.assertIn("Reason: Unauthorized", str(e.exception))
self.assertIn("401", str(e.exception))
self.assertEqual(400, e.exception.status_code)
def test_list(self, batch_p, get_p):
batch_p.return_value = V1JobList(
items=[V1Pod(metadata=V1ObjectMeta(name='drwho-cate-sdav'))])
get_p.return_value = {
'cubegen_id': 'id',
'status': 'Ready',
'output': ['bla'],
'progress': 100
}
res = cubegens.list('drwho')
self.assertEqual(1, len(res))
self.assertDictEqual(
{
'cubegen_id': 'id',
'status': 'Ready',
'output': ['bla'],
'progress': 100
}, res[0])
batch_p.side_effect = ApiException(401, 'Unauthorized')
with self.assertRaises(api.ApiError) as e:
cubegens.list(user_id='drwho')
self.assertIn("Reason: Unauthorized", str(e.exception))
self.assertIn("401", str(e.exception))
self.assertEqual(400, e.exception.status_code)
def test_run_failed_exception(k8s_executor):
task_config = KubernetesTaskConfig(name="fake_task_name",
uuid="fake_id",
image="fake_docker_image",
command="fake_command")
k8s_executor.kube_client.core.create_namespaced_pod.side_effect = ApiException(
status=403, reason="Fake unauthorized message")
assert k8s_executor.run(task_config) is None
def create_namespaced_secret(self, namespace: str,
secret: V1Secret) -> None:
self._maybe_error("create_namespaced_secret", namespace, secret)
assert namespace == secret.metadata.namespace
name = secret.metadata.name
if namespace not in self.objects:
self.objects[namespace] = {}
if name in self.objects[namespace]:
raise ApiException(status=500, reason=f"{namespace}/{name} exists")
self.objects[namespace][name] = secret
def test_list_ingresses(self):
self._networking_api.list_namespaced_ingress = Mock(
side_effect=ApiException(500, 'Test'))
with self.assertRaises(api.ApiError) as e:
list_ingresses(namespace='test', core_api=self._networking_api)
expected = ("Error when listing ingresses in namespace test: (500)\n"
"Reason: Test\n")
self.assertEqual(expected, str(e.exception))
self.assertEqual(400, e.exception.status_code)
def test_create_pvc(self):
self._core_v1_api.create_namespaced_persistent_volume_claim = Mock(
side_effect=ApiException(500, 'Test'))
pvc = client.V1PersistentVolumeClaim(
metadata=V1ObjectMeta(name='workspace-nfs-pvc'), )
with self.assertRaises(api.ApiError) as e:
create_pvc(pvc, 'test', core_api=self._core_v1_api)
expected = ("Error when creating the pvc workspace-nfs-pvc: (500)\n"
"Reason: Test\n")
self.assertEqual(expected, str(e.exception))
self.assertEqual(400, e.exception.status_code)
def test_delete_service(self):
self._core_v1_api.delete_namespaced_service = Mock(
side_effect=ApiException(500, 'Test'))
with self.assertRaises(api.ApiError) as e:
delete_service(name="service",
namespace='test',
core_api=self._core_v1_api)
expected = ("Error when deleting the service service: (500)\n"
"Reason: Test\n")
self.assertEqual(expected, str(e.exception))
self.assertEqual(400, e.exception.status_code)
def test_delete_deployment(self, delete_p):
delete_deployment('deployment', core_api=self._apps_api)
delete_p.assert_called_once()
delete_p.side_effect = ApiException('Test')
with self.assertRaises(api.ApiError) as e:
delete_deployment('deployment', core_api=self._apps_api)
expected = ("Error when deleting the deployment deployment: (Test)\n"
"Reason: None\n")
self.assertEqual(expected, str(e.exception))
self.assertEqual(400, e.exception.status_code)
def test_delete(self, delete_p):
res = user_namespaces.delete('drwho')
delete_p.assert_called_once()
self.assertTrue(res)
delete_p.side_effect = ApiException(500, 'Error')
with self.assertRaises(api.ApiError) as e:
user_namespaces.delete('drwho')
self.assertEqual(400, e.exception.status_code)
self.assertIn('500', str(e.exception))
self.assertIn('Error', str(e.exception))
def test_create_service(self):
self._core_v1_api.create_namespaced_service = Mock(
side_effect=ApiException(500, 'Test'))
service = client.V1Service(metadata=V1ObjectMeta(name='service'), )
with self.assertRaises(api.ApiError) as e:
create_service(service=service,
namespace='test',
core_api=self._core_v1_api)
expected = ("Error when creating the service service: (500)\n"
"Reason: Test\n")
self.assertEqual(expected, str(e.exception))
self.assertEqual(400, e.exception.status_code)
def test_create_deployment(self, create_p):
pvc = client.V1Deployment(metadata=V1ObjectMeta(name='deployment'), )
k8s.create_deployment(pvc, 'test')
create_p.assert_called_once()
create_p.side_effect = MagicMock(side_effect=ApiException(500, 'Test'))
with self.assertRaises(api.ApiError) as e:
k8s.create_deployment(pvc, 'test')
expected = ("Error when creating the deployment deployment: (500)\n"
"Reason: Test\n")
self.assertEqual(expected, str(e.exception))
self.assertEqual(400, e.exception.status_code)
def test_create_ingress(self):
self._networking_api.create_namespaced_ingress = Mock(
side_effect=ApiException(500, 'Test'))
ingress = client.NetworkingV1beta1Ingress(
metadata=V1ObjectMeta(name='ingress'), )
with self.assertRaises(api.ApiError) as e:
create_ingress(ingress=ingress,
namespace='test',
core_api=self._networking_api)
expected = ("Error when creating the ingress ingress: (500)\n"
"Reason: Test\n")
self.assertEqual(expected, str(e.exception))
self.assertEqual(400, e.exception.status_code)
def test_tail_pod_log(self, mocker: MockFixture) -> None:
mocked_core_v1_api = mocker.Mock(spec=CoreV1Api)
mocked_core_v1_api_call = mocker.patch(
"opta.core.kubernetes.CoreV1Api", return_value=mocked_core_v1_api)
mocked_watch = mocker.Mock(spec=Watch)
mocked_watch_call = mocker.patch("opta.core.kubernetes.Watch",
return_value=mocked_watch)
mocked_watch.stream.side_effect = [
"hello_world",
ApiException(status=400),
ApiException(status=400),
ApiException(status=400),
Exception(),
ApiException(status=400),
]
mocked_pod = mocker.Mock(spec=V1Pod)
mocked_pod.metadata = mocker.Mock()
mocked_pod.metadata.name = "pod1"
mocked_time = mocker.patch("opta.core.kubernetes.time")
tail_pod_log("mocked_namespace", mocked_pod, 2, 3)
mocked_watch_call.assert_called_once_with()
mocked_core_v1_api_call.assert_called_once_with()
mocked_time.sleep.assert_has_calls([
mocker.call(0),
mocker.call(1),
mocker.call(2),
mocker.call(3),
mocker.call(4),
mocker.call(5),
mocker.call(6),
mocker.call(7),
mocker.call(8),
mocker.call(9),
mocker.call(10),
mocker.call(11),
mocker.call(12),
mocker.call(13),
mocker.call(14),
])
assert mocked_time.sleep.call_count == 15
# Tailing should not retry upon encountering a 404 API exception.
mocked_watch.stream.side_effect = [
"hello_world",
ApiException(status=400),
ApiException(status=404),
]
mocked_time = mocker.patch("opta.core.kubernetes.time")
tail_pod_log("mocked_namespace", mocked_pod, 2, 3)
assert mocked_time.sleep.call_count == 1
def recycle_pvc(client, pvc_name, namespace='default', timeout=60):
k8s_client = kubernetes.client
v1 = k8s_client.CoreV1Api(client)
try:
v1.read_namespaced_persistent_volume_claim(name=pvc_name,
namespace=namespace)
except ApiException as e:
if e.status == 404:
raise ApiException(
f"Couldn't find pvc {pvc_name} in namespace {namespace}")
container = kubernetes.client.V1Container(
name="pv-cleaner",
command=["/bin/sh", "-c", "rm -rf /scrub/*"],
image="k8s.gcr.io/busybox",
volume_mounts=[
k8s_client.V1VolumeMount(name="pvc-volume", mount_path="/scrub")
])
volume = k8s_client.V1Volume(
name="pvc-volume",
persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource(
claim_name=pvc_name))
pod_spec = k8s_client.V1PodSpec(volumes=[volume],
containers=[container],
restart_policy="Never")
pod_name = f"pv-cleaner-{str(uuid.uuid4())[:6]}"
pod = k8s_client.V1Pod(metadata=k8s_client.V1ObjectMeta(name=pod_name),
spec=pod_spec)
v1 = k8s_client.CoreV1Api(client)
v1.create_namespaced_pod(namespace=namespace, body=pod)
try:
waiter.wait_for_predicate(lambda: v1.read_namespaced_pod(
name=pod_name, namespace=namespace).status.phase == "Succeeded",
timeout=timeout)
except TimeoutError as e:
logging.debug(
v1.read_namespaced_pod(name=pod_name, namespace=namespace).status)
raise e
v1.delete_namespaced_pod(name=pod_name, namespace=namespace)
def test_logs(self, pod_read_p, pod_p):
pod_p.return_value = V1JobList(
items=[V1Pod(metadata=V1ObjectMeta(name='drwho-cate-sdav'))])
pod_read_p.return_value = 'bla'
res = cubegens.logs('drwho')
self.assertEqual(1, len(res))
self.assertEqual([
'bla',
], res)
pod_p.side_effect = ApiException(401, 'Unauthorized')
with self.assertRaises(api.ApiError) as e:
cubegens.logs('drwho', raises=True)
self.assertIn("Reason: Unauthorized", str(e.exception))
self.assertIn("401", str(e.exception))
self.assertEqual(400, e.exception.status_code)
def test_list(self, list_p):
list_p.return_value = V1NamespaceList(items=[])
res = user_namespaces.list()
list_p.assert_called_once()
self.assertEqual(0, len(res))
list_p.return_value = V1NamespaceList(
items=[V1Namespace(metadata=V1ObjectMeta(name='drwho'))])
res = user_namespaces.list()
self.assertEqual(1, len(res))
list_p.side_effect = ApiException(500, 'Error')
with self.assertRaises(api.ApiError) as e:
user_namespaces.list()
self.assertEqual(400, e.exception.status_code)
self.assertIn('500', str(e.exception))
self.assertIn('Error', str(e.exception))
def test_create_if_not_exists(self, create_p):
with patch('xcube_hub.core.user_namespaces.exists') as exists_p:
exists_p.return_value = True
res = user_namespaces.create_if_not_exists('drwho')
create_p.assert_not_called()
self.assertFalse(res)
exists_p.return_value = False
res = user_namespaces.create_if_not_exists('drwho')
create_p.assert_called_once()
self.assertTrue(res)
create_p.side_effect = ApiException(500, 'Error')
with self.assertRaises(api.ApiError) as e:
user_namespaces.create_if_not_exists('drwho')
self.assertEqual(400, e.exception.status_code)
self.assertIn('500', str(e.exception))
self.assertIn('Error', str(e.exception))
def test_create_info_only(self, namespace_p, create_p, status_p):
res = cubegens.create('drwho', '*****@*****.**', _CFG, info_only=True)
self.assertIn("drwho-", res['cubegen_id'])
self.assertEqual(24, len(res['cubegen_id']))
create_p.side_effect = ApiException(401, 'Unauthorized')
with self.assertRaises(api.ApiError) as e:
cubegens.create('drwho', '*****@*****.**', _CFG, info_only=True)
self.assertEqual(400, e.exception.status_code)
self.assertIn("Reason: Unauthorized", str(e.exception))
self.assertIn("401", str(e.exception))
create_p.side_effect = ApiValueError('Error')
with self.assertRaises(api.ApiError) as e:
cubegens.create('drwho', '*****@*****.**', _CFG, info_only=True)
self.assertEqual(400, e.exception.status_code)
self.assertEqual("Error", str(e.exception))
del os.environ['XCUBE_HUB_CALLBACK_URL']
with self.assertRaises(api.ApiError) as e:
cubegens.create('drwho', '*****@*****.**', _CFG, info_only=True)
self.assertEqual('XCUBE_HUB_CALLBACK_URL must be given',
str(e.exception))
cfg = _CFG.copy()
del cfg['input_config']
with self.assertRaises(api.ApiError) as e:
cubegens.create('drwho', '*****@*****.**', cfg, info_only=True)
self.assertEqual(
"Either 'input_config' or 'input_configs' must be given",
str(e.exception))
def test_launch_cate(self, namespace_p, ct_p, deployment_p,
deployment_create_p, service_create_p, get_ingress_p,
ingress_p, poll_p):
with self.assertRaises(api.ApiError) as e:
cate.launch_cate('drwho#######')
self.assertEqual('Invalid user name.', str(e.exception))
self.assertEqual('400', str(e.exception.status_code))
ct_p.return_value = 100
with self.assertRaises(api.ApiError) as e:
cate.launch_cate('drwho')
self.assertEqual('Too many pods running.', str(e.exception))
self.assertEqual('413', str(e.exception.status_code))
ct_p.return_value = 10
deployment_p.return_value = V1Deployment(metadata=V1ObjectMeta(
name='drwho-cate'))
res = cate.launch_cate('drwho')
self.assertDictEqual(
{'serverUrl': 'https://stage.catehub.climate.esa.int/drwho'}, res)
get_ingress_p.return_value = None
res = cate.launch_cate('drwho')
self.assertDictEqual(
{'serverUrl': 'https://stage.catehub.climate.esa.int/drwho'}, res)
ct_p.side_effect = ApiException(400, 'test')
with self.assertRaises(api.ApiError) as e:
cate.launch_cate('drwho')
self.assertIn('Reason: test', str(e.exception))
self.assertEqual('400', str(e.exception.status_code))
def raise_api_exception(namespace, body):
raise ApiException(500, 'Test')
def error_callback(method: str, *args: Any) -> None:
if method in ("delete_namespaced_secret", "patch_namespaced_secret"):
raise ApiException(status=500, reason="Some error")
def error_callback_read(method: str, *args: Any) -> None:
if method == "read_namespaced_secret":
if args[1] != "elsewhere":
raise ApiException(status=500, reason="Some error")
def test_tail_namespace_events(self, mocker: MockFixture) -> None:
mocker.patch("opta.core.kubernetes.load_kube_config")
mocked_events_v1_api = mocker.Mock(spec=EventsV1Api)
mocked_events_v1_api_call = mocker.patch(
"opta.core.kubernetes.EventsV1Api",
return_value=mocked_events_v1_api)
mocked_watch = mocker.Mock(spec=Watch)
mocked_watch_call = mocker.patch("opta.core.kubernetes.Watch",
return_value=mocked_watch)
layer = mocker.Mock(spec=Layer)
layer.name = "mocked_layer"
layer.parent = None
layer.providers = {
"aws": {
"region": "us-east-1",
"account_id": "111111111111"
}
}
mocked_event_1 = mocker.Mock(spec=EventsV1Event)
mocked_event_1.series = None
mocked_event_1.event_time = datetime.datetime.now(
pytz.utc) - datetime.timedelta(seconds=1)
mocked_event_1.note = "blah1"
mocked_event_2 = mocker.Mock(spec=EventsV1Event)
mocked_event_2.series = None
mocked_event_2.event_time = datetime.datetime.now(
pytz.utc) - datetime.timedelta(seconds=100)
mocked_event_2.note = "blah2"
mocked_event_3 = mocker.Mock(spec=EventsV1Event)
mocked_event_3.series = None
mocked_event_3.event_time = datetime.datetime.now(
pytz.utc) - datetime.timedelta(seconds=10)
mocked_event_3.note = "blah3"
mocked_old_events = mocker.Mock(spec=EventsV1EventList)
mocked_old_events.items = [
mocked_event_1, mocked_event_2, mocked_event_3
]
mocked_events_v1_api.list_namespaced_event.return_value = mocked_old_events
mocked_event_4 = mocker.Mock(spec=EventsV1Event)
mocked_event_4.series = None
mocked_event_4.event_time = datetime.datetime.now(
pytz.utc) - datetime.timedelta(seconds=100)
mocked_event_4.note = "blah2"
mocked_event_5 = mocker.Mock(spec=EventsV1Event)
mocked_event_5.series = None
mocked_event_5.event_time = datetime.datetime.now(
pytz.utc) - datetime.timedelta(seconds=10)
mocked_event_5.note = "blah3"
mocked_watch.stream.side_effect = [
[{
"object": mocked_event_4
}],
[{
"object": mocked_event_5
}],
ApiException(status=400),
ApiException(status=400),
ApiException(status=400),
ApiException(status=400),
ApiException(status=400),
]
mocked_time = mocker.patch("opta.core.kubernetes.time")
start_time = datetime.datetime.now(
pytz.utc) - datetime.timedelta(seconds=2)
tail_namespace_events(layer, start_time, 3)
mocked_events_v1_api.list_namespaced_event.assert_called_once_with(
namespace="mocked_layer")
mocked_watch_call.assert_called_once_with()
mocked_events_v1_api_call.assert_called_once_with()
mocked_time.sleep.assert_has_calls([
mocker.call(1),
mocker.call(2),
mocker.call(4),
mocker.call(8),
mocker.call(16),
])
def error_callback(method: str, *args: Any) -> None:
if method == "list_secret_for_all_namespaces":
raise ApiException(status=500, reason="Some error")