def test_updateService(self, client_mock):
service = KubernetesService()
client_mock.reset_mock()
expected_body = V1Service(metadata=self._createMeta(self.name),
spec=V1ServiceSpec(
cluster_ip="None",
ports=[
V1ServicePort(name='mongod',
port=27017,
protocol='TCP')
],
selector={
'heritage': 'mongos',
'name': self.name,
'operated-by':
'operators.ultimaker.com'
},
))
result = service.updateService(self.cluster_object)
expected_calls = [
call.CoreV1Api().patch_namespaced_service(self.name,
self.namespace,
expected_body)
]
self.assertEqual(expected_calls, client_mock.mock_calls)
self.assertEqual(
client_mock.CoreV1Api().patch_namespaced_service.return_value,
result)
def test_listMongoObjects_404(self, client_mock):
service = KubernetesService()
client_mock.reset_mock()
item = MagicMock()
item.spec.names.plural = "mongos"
client_mock.ApiextensionsV1beta1Api.return_value.list_custom_resource_definition.return_value.items = [
item
]
client_mock.CustomObjectsApi.return_value.list_cluster_custom_object.side_effect = ApiException(
404)
with self.assertRaises(TimeoutError) as context:
service.listMongoObjects(param="value")
expected_calls = [
call.ApiextensionsV1beta1Api().list_custom_resource_definition(),
call.CustomObjectsApi().list_cluster_custom_object(
'operators.ultimaker.com', 'v1', "mongos", param='value'),
call.CustomObjectsApi().list_cluster_custom_object(
'operators.ultimaker.com', 'v1', "mongos", param='value'),
call.CustomObjectsApi().list_cluster_custom_object(
'operators.ultimaker.com', 'v1', "mongos", param='value'),
]
self.assertEquals(expected_calls, client_mock.mock_calls)
self.assertEquals(
"Could not list the custom mongo objects after 3 retries",
str(context.exception))
def test_createMongoObjectDefinition(self, client_mock):
service = KubernetesService()
client_mock.reset_mock()
expected_def = V1beta1CustomResourceDefinition(
api_version="apiextensions.k8s.io/v1beta1",
kind="CustomResourceDefinition",
metadata=V1ObjectMeta(name="mongos.operators.ultimaker.com"),
spec=V1beta1CustomResourceDefinitionSpec(
group="operators.ultimaker.com",
version="v1",
scope="Namespaced",
names=V1beta1CustomResourceDefinitionNames(plural="mongos",
singular="mongo",
kind="Mongo",
short_names=["mng"
])))
client_mock.ApiextensionsV1beta1Api.return_value.list_custom_resource_definition.return_value = \
V1beta1CustomResourceDefinitionList(items=[])
client_mock.ApiextensionsV1beta1Api.return_value.create_custom_resource_definition.return_value = expected_def
result = service.createMongoObjectDefinition()
self.assertEquals(expected_def, result)
expected_calls = [
call.ApiextensionsV1beta1Api().list_custom_resource_definition(),
call.ApiextensionsV1beta1Api().create_custom_resource_definition(
expected_def),
]
self.assertEqual(expected_calls, client_mock.mock_calls)
def test_updateService(self, client_mock):
service = KubernetesService()
client_mock.reset_mock()
expected_body = V1Service(metadata=self._createMeta(self.name),
spec=V1ServiceSpec(
cluster_ip="None",
ports=[
V1ServicePort(name="mongod",
port=27017,
protocol="TCP")
],
selector={
"heritage": "mongos",
"name": self.name,
"operated-by":
"operators.javamachr.cz"
},
))
result = service.updateService(self.cluster_object)
expected_calls = [
call.CoreV1Api().patch_namespaced_service(self.name,
self.namespace,
expected_body)
]
self.assertEqual(expected_calls, client_mock.mock_calls)
self.assertEqual(
client_mock.CoreV1Api().patch_namespaced_service.return_value,
result)
def test_createService(self, client_mock):
service = KubernetesService()
client_mock.reset_mock()
client_mock.CoreV1Api.return_value.create_namespaced_service.return_value = V1Service(
kind="unit")
expected_body = V1Service(metadata=self._createMeta(self.name),
spec=V1ServiceSpec(
cluster_ip="None",
ports=[
V1ServicePort(name="mongod",
port=27017,
protocol="TCP")
],
selector={
"heritage": "mongos",
"name": self.name,
"operated-by":
"operators.ultimaker.com"
},
))
expected_calls = [
call.CoreV1Api().create_namespaced_service(self.namespace,
expected_body)
]
result = service.createService(self.cluster_object)
self.assertEqual(expected_calls, client_mock.mock_calls)
self.assertEqual(V1Service(kind="unit"), result)
def __init__(self) -> None:
self._cluster_versions: Dict[Tuple[str, str], str] = {} # format: {(cluster_name, namespace): resource_version}
self._kubernetes_service = KubernetesService()
self._mongo_service = MongoService(self._kubernetes_service)
self._backup_checker = BackupHelper(self._kubernetes_service)
self._resource_checkers: List[BaseResourceChecker] = [
ServiceChecker(self._kubernetes_service),
StatefulSetChecker(self._kubernetes_service),
AdminSecretChecker(self._kubernetes_service),
]
def test_deleteStatefulSet(self, client_mock):
service = KubernetesService()
client_mock.reset_mock()
result = service.deleteStatefulSet(self.name, self.namespace)
expected_calls = [
call.AppsV1beta1Api().delete_namespaced_stateful_set(
self.name, self.namespace, V1DeleteOptions())
]
self.assertEqual(expected_calls, client_mock.mock_calls)
self.assertEqual(
client_mock.AppsV1beta1Api().delete_namespaced_stateful_set.
return_value, result)
def test_getService(self, client_mock):
service = KubernetesService()
client_mock.reset_mock()
result = service.getService(self.name, self.namespace)
expected_calls = [
call.CoreV1Api().read_namespaced_service(self.name, self.namespace)
]
self.assertEqual(expected_calls, client_mock.mock_calls)
self.assertEqual(
client_mock.CoreV1Api.return_value.read_namespaced_service.
return_value, result)
def test_updateStatefulSet(self, client_mock):
service = KubernetesService()
client_mock.reset_mock()
result = service.updateStatefulSet(self.cluster_object)
expected_calls = [
call.AppsV1beta1Api().patch_namespaced_stateful_set(
self.name, self.namespace, self.stateful_set)
]
self.assertEqual(expected_calls, client_mock.mock_calls)
self.assertEqual(
client_mock.AppsV1beta1Api().patch_namespaced_stateful_set.
return_value, result)
def test_getMongoObject(self, client_mock):
service = KubernetesService()
client_mock.reset_mock()
result = service.getMongoObject(self.name, self.namespace)
expected_calls = [
call.CustomObjectsApi().get_namespaced_custom_object(
"javamachr.cz", "v1", self.namespace, "mongos", self.name)
]
self.assertEqual(expected_calls, client_mock.mock_calls)
self.assertEqual(
client_mock.CustomObjectsApi().get_namespaced_custom_object.
return_value, result)
def __init__(self):
self.kubernetes_service = KubernetesService()
self.mongo_service = MongoService(self.kubernetes_service)
self.resource_checkers = [
ServiceChecker(self.kubernetes_service),
StatefulSetChecker(self.kubernetes_service),
AdminSecretChecker(self.kubernetes_service),
] # type: List[BaseResourceChecker]
self.backup_checker = BackupChecker(self.kubernetes_service)
self.cluster_versions = {
} # type: Dict[Tuple[str, str], str] # format: {(cluster_name, namespace): resource_version}
def test_listAllSecretsWithLabels_custom(self, client_mock):
service = KubernetesService()
client_mock.reset_mock()
labels = {"operated-by": "me", "heritage": "mongo", "name": "name"}
result = service.listAllSecretsWithLabels(labels)
expected_calls = [
call.CoreV1Api().list_secret_for_all_namespaces(
label_selector="operated-by=me,heritage=mongo,name=name")
]
self.assertEqual(expected_calls, client_mock.mock_calls)
self.assertEqual(
client_mock.CoreV1Api().list_secret_for_all_namespaces.
return_value, result)
def test_listAllSecretsWithLabels_default(self, client_mock):
service = KubernetesService()
client_mock.reset_mock()
result = service.listAllSecretsWithLabels()
expected_calls = [
call.CoreV1Api().list_secret_for_all_namespaces(
label_selector=
"operated-by=operators.ultimaker.com,heritage=mongos")
]
self.assertEqual(expected_calls, client_mock.mock_calls)
self.assertEqual(
client_mock.CoreV1Api().list_secret_for_all_namespaces.
return_value, result)
def test_getMongoObject(self, client_mock):
service = KubernetesService()
client_mock.reset_mock()
result = service.getMongoObject(self.name, self.namespace)
expected_calls = [
call.CustomObjectsApi().get_namespaced_custom_object(
'operators.ultimaker.com', 'v1', self.namespace, 'mongos',
self.name)
]
self.assertEqual(expected_calls, client_mock.mock_calls)
self.assertEqual(
client_mock.CustomObjectsApi().get_namespaced_custom_object.
return_value, result)
def test_listAllStatefulSetsWithLabels_default(self, client_mock):
service = KubernetesService()
client_mock.reset_mock()
result = service.listAllStatefulSetsWithLabels()
expected_calls = [
call.AppsV1beta1Api().list_stateful_set_for_all_namespaces(
label_selector=
"operated-by=operators.javamachr.cz,heritage=mongos")
]
self.assertEqual(expected_calls, client_mock.mock_calls)
self.assertEqual(
client_mock.AppsV1beta1Api().list_stateful_set_for_all_namespaces.
return_value, result)
def test_createSecret(self, client_mock):
service = KubernetesService()
client_mock.reset_mock()
secret_data = {"username": "******", "password": "******"}
expected_body = V1Secret(metadata=self._createMeta("secret-name"),
string_data=secret_data)
result = service.createSecret("secret-name", self.namespace,
secret_data)
self.assertEqual([
call.CoreV1Api().create_namespaced_secret(self.namespace,
expected_body)
], client_mock.mock_calls)
self.assertEqual(
client_mock.CoreV1Api.return_value.create_namespaced_secret.
return_value, result)
def test_createStatefulSet_no_optional_fields(self, client_mock):
service = KubernetesService()
client_mock.reset_mock()
del self.cluster_dict["spec"]["mongodb"]["cpu_limit"]
del self.cluster_dict["spec"]["mongodb"]["memory_limit"]
self.cluster_object = V1MongoClusterConfiguration(**self.cluster_dict)
expected_calls = [
call.AppsV1beta1Api().create_namespaced_stateful_set(
self.namespace, self.stateful_set)
]
result = service.createStatefulSet(self.cluster_object)
self.assertEqual(expected_calls, client_mock.mock_calls)
self.assertEqual(
client_mock.AppsV1beta1Api().create_namespaced_stateful_set.
return_value, result)
def test_execInPod(self, stream_mock, client_mock):
service = KubernetesService()
client_mock.reset_mock()
result = service.execInPod("container", "pod_name", self.namespace,
"ls")
stream_mock.assert_called_once_with(
client_mock.CoreV1Api.return_value.connect_get_namespaced_pod_exec,
'pod_name',
'default',
command='ls',
container='container',
stderr=True,
stdin=False,
stdout=True,
tty=False)
self.assertEquals(stream_mock.return_value, result)
self.assertEquals([], client_mock.mock_calls)
def test_deleteService_TypeError(self, client_mock):
service = KubernetesService()
client_mock.reset_mock()
client_mock.CoreV1Api.return_value.delete_namespaced_service.side_effect = TypeError, V1Status(
)
result = service.deleteService(self.name, self.namespace)
expected_calls = [
call.CoreV1Api().delete_namespaced_service(self.name,
self.namespace,
V1DeleteOptions()),
call.CoreV1Api().delete_namespaced_service(self.name,
self.namespace),
]
self.assertEqual(expected_calls, client_mock.mock_calls)
self.assertEqual(V1Status(), result)
def test_createMongoObjectDefinition_existing(self, client_mock):
service = KubernetesService()
client_mock.reset_mock()
item = MagicMock()
item.spec.names.plural = "mongos"
client_mock.ApiextensionsV1beta1Api.return_value.list_custom_resource_definition.return_value.items = [
item
]
result = service.createMongoObjectDefinition()
self.assertEquals(item, result)
expected_calls = [
call.ApiextensionsV1beta1Api().list_custom_resource_definition()
]
self.assertEqual(expected_calls, client_mock.mock_calls)
def test_createSecret_exists(self, client_mock):
service = KubernetesService()
client_mock.reset_mock()
client_mock.CoreV1Api.return_value.create_namespaced_secret.side_effect = ApiException(
status=409)
client_mock.CoreV1Api.return_value.create_namespaced_secret.side_effect.body = "{}"
secret_data = {"username": "******", "password": "******"}
result = service.createSecret(self.name, self.namespace, secret_data)
expected_body = V1Secret(metadata=self._createMeta(self.name),
string_data=secret_data)
expected_calls = [
call.CoreV1Api().create_namespaced_secret(self.namespace,
expected_body)
]
self.assertEqual(expected_calls, client_mock.mock_calls)
self.assertIsNone(result)
def test_listMongoObjects(self, client_mock):
service = KubernetesService()
client_mock.reset_mock()
item = MagicMock()
item.spec.names.plural = "mongos"
client_mock.ApiextensionsV1beta1Api.return_value.list_custom_resource_definition.return_value.items = [
item
]
result = service.listMongoObjects(param="value")
expected_calls = [
call.ApiextensionsV1beta1Api().list_custom_resource_definition(),
call.CustomObjectsApi().list_cluster_custom_object(
'operators.ultimaker.com', 'v1', "mongos", param='value')
]
self.assertEqual(expected_calls, client_mock.mock_calls)
self.assertEqual(
client_mock.CustomObjectsApi().list_cluster_custom_object.
return_value, result)
def test_updateSecret(self, client_mock):
service = KubernetesService()
client_mock.reset_mock()
client_mock.CoreV1Api.return_value.read_namespaced_secret.return_value = V1Secret(
kind="unit")
secret_data = {"username": "******", "password": "******"}
expected_body = V1Secret(kind="unit", string_data=secret_data)
expected_calls = [
call.CoreV1Api().read_namespaced_secret(self.name, self.namespace),
call.CoreV1Api().patch_namespaced_secret(self.name, self.namespace,
expected_body),
]
result = service.updateSecret(self.name, self.namespace, secret_data)
self.assertEqual(expected_calls, client_mock.mock_calls)
self.assertEqual(
client_mock.CoreV1Api.return_value.patch_namespaced_secret.
return_value, result)
def test_listMongoObjects_400(self, client_mock):
service = KubernetesService()
client_mock.reset_mock()
item = MagicMock()
item.spec.names.plural = "mongos"
client_mock.ApiextensionsV1beta1Api.return_value.list_custom_resource_definition.return_value.items = [
item
]
client_mock.CustomObjectsApi.return_value.list_cluster_custom_object.side_effect = ApiException(
400)
with self.assertRaises(ApiException):
service.listMongoObjects(param="value")
expected_calls = [
call.ApiextensionsV1beta1Api().list_custom_resource_definition(),
call.CustomObjectsApi().list_cluster_custom_object(
"operators.ultimaker.com", "v1", "mongos", param="value")
]
self.assertEqual(expected_calls, client_mock.mock_calls)
def test___init__(self, client_mock):
KubernetesService()
config = Configuration()
config.debug = False
expected = [
call.ApiClient(config),
call.CoreV1Api(client_mock.ApiClient.return_value),
call.CustomObjectsApi(client_mock.ApiClient.return_value),
call.ApiextensionsV1beta1Api(client_mock.ApiClient.return_value),
call.AppsV1beta1Api(client_mock.ApiClient.return_value),
]
with patch("kubernetes.client.configuration.Configuration.__eq__",
dict_eq):
self.assertEqual(expected, client_mock.mock_calls)
class ClusterChecker:
"""
Manager that periodically checks the status of the MongoDB objects in the cluster.
"""
STREAM_REQUEST_TIMEOUT = (15.0, 5.0) # connect, read timeout
def __init__(self):
self.kubernetes_service = KubernetesService()
self.mongo_service = MongoService(self.kubernetes_service)
self.resource_checkers = [
ServiceChecker(self.kubernetes_service),
StatefulSetChecker(self.kubernetes_service),
AdminSecretChecker(self.kubernetes_service),
] # type: List[BaseResourceChecker]
self.backup_checker = BackupChecker(self.kubernetes_service)
self.cluster_versions = {
} # type: Dict[Tuple[str, str], str] # format: {(cluster_name, namespace): resource_version}
@staticmethod
def _parseConfiguration(
cluster_dict: Dict[str,
any]) -> Optional[V1MongoClusterConfiguration]:
"""
Tries to parse the given cluster configuration, returning None if the object cannot be parsed.
:param cluster_dict: The dictionary containing the configuration.
:return: The cluster configuration model, if valid, or None.
"""
try:
result = V1MongoClusterConfiguration(**cluster_dict)
result.validate()
return result
except ValueError as err:
meta = cluster_dict.get("metadata", {})
logging.error(
"Could not validate cluster configuration for {} @ ns/{}: {}. The cluster will be ignored."
.format(meta.get("name"), meta.get("namespace"), err))
def checkExistingClusters(self) -> None:
"""
Check all Mongo objects and see if the sub objects are available.
If they are not, they should be (re-)created to ensure the cluster is in the expected state.
"""
mongo_objects = self.kubernetes_service.listMongoObjects()
logging.info("Checking %s mongo objects.", len(mongo_objects["items"]))
for cluster_dict in mongo_objects["items"]:
cluster_object = self._parseConfiguration(cluster_dict)
if cluster_object:
self.checkCluster(cluster_object)
def streamEvents(self) -> None:
"""
Watches for changes to the mongo objects in Kubernetes and processes any changes immediately.
"""
event_watcher = Watch()
# start watching from the latest version that we have
if self.cluster_versions:
event_watcher.resource_version = max(
self.cluster_versions.values())
for event in event_watcher.stream(
self.kubernetes_service.listMongoObjects,
_request_timeout=self.STREAM_REQUEST_TIMEOUT):
logging.info("Received event %s", event)
if event["type"] in ("ADDED", "MODIFIED"):
cluster_object = self._parseConfiguration(event["object"])
if cluster_object:
self.checkCluster(cluster_object)
else:
logging.warning(
"Could not validate cluster object, stopping event watcher."
)
event_watcher.stop = True
elif event["type"] in ("DELETED", ):
self.collectGarbage()
else:
logging.warning(
"Could not parse event, stopping event watcher.")
event_watcher.stop = True
# Change the resource version manually because of a bug fixed in a later version of the K8s client:
# https://github.com/kubernetes-client/python-base/pull/64
if isinstance(event.get('object'),
dict) and 'resourceVersion' in event['object'].get(
'metadata', {}):
event_watcher.resource_version = event['object']['metadata'][
'resourceVersion']
def collectGarbage(self) -> None:
"""
Cleans up any resources that are left after a cluster has been removed.
"""
for checker in self.resource_checkers:
checker.cleanResources()
def checkCluster(self,
cluster_object: V1MongoClusterConfiguration,
force: bool = False) -> None:
"""
Checks whether the given cluster is configured and updated.
:param cluster_object: The cluster object from the YAML file.
:param force: If this is True, we will re-update the cluster even if it has been checked before.
"""
key = (cluster_object.metadata.name, cluster_object.metadata.namespace)
if self.cluster_versions.get(
key) == cluster_object.metadata.resource_version and not force:
logging.debug(
"Cluster object %s has been checked already in version %s.",
key, cluster_object.metadata.resource_version)
# we still want to check the replicas to make sure everything is working.
self.mongo_service.checkReplicaSetOrInitialize(cluster_object)
else:
for checker in self.resource_checkers:
checker.checkResource(cluster_object)
self.mongo_service.checkReplicaSetOrInitialize(cluster_object)
self.mongo_service.createUsers(cluster_object)
self.cluster_versions[
key] = cluster_object.metadata.resource_version
self.backup_checker.backupIfNeeded(cluster_object)
class ClusterManager:
""" Manager that periodically checks the status of the MongoDB objects in the cluster. """
def __init__(self) -> None:
self._cluster_versions: Dict[Tuple[str, str], str] = {} # format: {(cluster_name, namespace): resource_version}
self._kubernetes_service = KubernetesService()
self._mongo_service = MongoService(self._kubernetes_service)
self._backup_checker = BackupHelper(self._kubernetes_service)
self._resource_checkers: List[BaseResourceChecker] = [
ServiceChecker(self._kubernetes_service),
StatefulSetChecker(self._kubernetes_service),
AdminSecretChecker(self._kubernetes_service),
]
def checkExistingClusters(self) -> None:
"""
Check all Mongo objects and see if the sub objects are available.
If they are not, they should be (re-)created to ensure the cluster is in the expected state.
"""
mongo_objects = self._kubernetes_service.listMongoObjects()
logging.info("Checking %s mongo objects.", len(mongo_objects["items"]))
for cluster_dict in mongo_objects["items"]:
cluster_object = self._parseConfiguration(cluster_dict)
if cluster_object:
self._checkCluster(cluster_object)
def collectGarbage(self) -> None:
"""
Cleans up any resources that are left after a cluster has been removed.
"""
for checker in self._resource_checkers:
checker.cleanResources()
def _checkCluster(self, cluster_object: V1MongoClusterConfiguration, force: bool = False) -> None:
"""
Checks whether the given cluster is configured and updated.
:param cluster_object: The cluster object from the YAML file.
:param force: If this is True, we will re-update the cluster even if it has been checked before.
"""
key = cluster_object.metadata.name, cluster_object.metadata.namespace
if self._cluster_versions.get(key) == cluster_object.metadata.resource_version and not force:
logging.debug("Cluster object %s has been checked already in version %s.",
key, cluster_object.metadata.resource_version)
# we still want to check the replicas to make sure everything is working.
self._mongo_service.checkOrCreateReplicaSet(cluster_object)
else:
for checker in self._resource_checkers:
checker.checkResource(cluster_object)
self._mongo_service.checkOrCreateReplicaSet(cluster_object)
self._mongo_service.createUsers(cluster_object)
self._cluster_versions[key] = cluster_object.metadata.resource_version
self._backup_checker.backupIfNeeded(cluster_object)
@staticmethod
def _parseConfiguration(cluster_dict: Dict[str, any]) -> Optional[V1MongoClusterConfiguration]:
"""
Tries to parse the given cluster configuration, returning None if the object cannot be parsed.
:param cluster_dict: The dictionary containing the configuration.
:return: The cluster configuration model, if valid, or None.
"""
try:
result = V1MongoClusterConfiguration(**cluster_dict)
result.validate()
return result
except ValueError as err:
meta = cluster_dict.get("metadata", {})
logging.error("Could not validate cluster configuration for %s @ ns/%s: %s. The cluster will be ignored.",
meta.get("name"), meta.get("namespace"), err)
def test_createSecret_error(self, client_mock):
service = KubernetesService()
client_mock.CoreV1Api.return_value.create_namespaced_secret.side_effect = ApiException(
status=400)
with self.assertRaises(ApiException):
service.createSecret(self.name, self.namespace, secret_data={})
