def test_updateService(self, client_mock): service = KubernetesService() client_mock.reset_mock() expected_body = V1Service(metadata=self._createMeta(self.name), spec=V1ServiceSpec( cluster_ip="None", ports=[ V1ServicePort(name='mongod', port=27017, protocol='TCP') ], selector={ 'heritage': 'mongos', 'name': self.name, 'operated-by': 'operators.ultimaker.com' }, )) result = service.updateService(self.cluster_object) expected_calls = [ call.CoreV1Api().patch_namespaced_service(self.name, self.namespace, expected_body) ] self.assertEqual(expected_calls, client_mock.mock_calls) self.assertEqual( client_mock.CoreV1Api().patch_namespaced_service.return_value, result)
def test_listMongoObjects_404(self, client_mock): service = KubernetesService() client_mock.reset_mock() item = MagicMock() item.spec.names.plural = "mongos" client_mock.ApiextensionsV1beta1Api.return_value.list_custom_resource_definition.return_value.items = [ item ] client_mock.CustomObjectsApi.return_value.list_cluster_custom_object.side_effect = ApiException( 404) with self.assertRaises(TimeoutError) as context: service.listMongoObjects(param="value") expected_calls = [ call.ApiextensionsV1beta1Api().list_custom_resource_definition(), call.CustomObjectsApi().list_cluster_custom_object( 'operators.ultimaker.com', 'v1', "mongos", param='value'), call.CustomObjectsApi().list_cluster_custom_object( 'operators.ultimaker.com', 'v1', "mongos", param='value'), call.CustomObjectsApi().list_cluster_custom_object( 'operators.ultimaker.com', 'v1', "mongos", param='value'), ] self.assertEquals(expected_calls, client_mock.mock_calls) self.assertEquals( "Could not list the custom mongo objects after 3 retries", str(context.exception))
def test_createMongoObjectDefinition(self, client_mock): service = KubernetesService() client_mock.reset_mock() expected_def = V1beta1CustomResourceDefinition( api_version="apiextensions.k8s.io/v1beta1", kind="CustomResourceDefinition", metadata=V1ObjectMeta(name="mongos.operators.ultimaker.com"), spec=V1beta1CustomResourceDefinitionSpec( group="operators.ultimaker.com", version="v1", scope="Namespaced", names=V1beta1CustomResourceDefinitionNames(plural="mongos", singular="mongo", kind="Mongo", short_names=["mng" ]))) client_mock.ApiextensionsV1beta1Api.return_value.list_custom_resource_definition.return_value = \ V1beta1CustomResourceDefinitionList(items=[]) client_mock.ApiextensionsV1beta1Api.return_value.create_custom_resource_definition.return_value = expected_def result = service.createMongoObjectDefinition() self.assertEquals(expected_def, result) expected_calls = [ call.ApiextensionsV1beta1Api().list_custom_resource_definition(), call.ApiextensionsV1beta1Api().create_custom_resource_definition( expected_def), ] self.assertEqual(expected_calls, client_mock.mock_calls)
def test_updateService(self, client_mock): service = KubernetesService() client_mock.reset_mock() expected_body = V1Service(metadata=self._createMeta(self.name), spec=V1ServiceSpec( cluster_ip="None", ports=[ V1ServicePort(name="mongod", port=27017, protocol="TCP") ], selector={ "heritage": "mongos", "name": self.name, "operated-by": "operators.javamachr.cz" }, )) result = service.updateService(self.cluster_object) expected_calls = [ call.CoreV1Api().patch_namespaced_service(self.name, self.namespace, expected_body) ] self.assertEqual(expected_calls, client_mock.mock_calls) self.assertEqual( client_mock.CoreV1Api().patch_namespaced_service.return_value, result)
def test_createService(self, client_mock): service = KubernetesService() client_mock.reset_mock() client_mock.CoreV1Api.return_value.create_namespaced_service.return_value = V1Service( kind="unit") expected_body = V1Service(metadata=self._createMeta(self.name), spec=V1ServiceSpec( cluster_ip="None", ports=[ V1ServicePort(name="mongod", port=27017, protocol="TCP") ], selector={ "heritage": "mongos", "name": self.name, "operated-by": "operators.ultimaker.com" }, )) expected_calls = [ call.CoreV1Api().create_namespaced_service(self.namespace, expected_body) ] result = service.createService(self.cluster_object) self.assertEqual(expected_calls, client_mock.mock_calls) self.assertEqual(V1Service(kind="unit"), result)
def __init__(self) -> None: self._cluster_versions: Dict[Tuple[str, str], str] = {} # format: {(cluster_name, namespace): resource_version} self._kubernetes_service = KubernetesService() self._mongo_service = MongoService(self._kubernetes_service) self._backup_checker = BackupHelper(self._kubernetes_service) self._resource_checkers: List[BaseResourceChecker] = [ ServiceChecker(self._kubernetes_service), StatefulSetChecker(self._kubernetes_service), AdminSecretChecker(self._kubernetes_service), ]
def test_deleteStatefulSet(self, client_mock): service = KubernetesService() client_mock.reset_mock() result = service.deleteStatefulSet(self.name, self.namespace) expected_calls = [ call.AppsV1beta1Api().delete_namespaced_stateful_set( self.name, self.namespace, V1DeleteOptions()) ] self.assertEqual(expected_calls, client_mock.mock_calls) self.assertEqual( client_mock.AppsV1beta1Api().delete_namespaced_stateful_set. return_value, result)
def test_getService(self, client_mock): service = KubernetesService() client_mock.reset_mock() result = service.getService(self.name, self.namespace) expected_calls = [ call.CoreV1Api().read_namespaced_service(self.name, self.namespace) ] self.assertEqual(expected_calls, client_mock.mock_calls) self.assertEqual( client_mock.CoreV1Api.return_value.read_namespaced_service. return_value, result)
def test_updateStatefulSet(self, client_mock): service = KubernetesService() client_mock.reset_mock() result = service.updateStatefulSet(self.cluster_object) expected_calls = [ call.AppsV1beta1Api().patch_namespaced_stateful_set( self.name, self.namespace, self.stateful_set) ] self.assertEqual(expected_calls, client_mock.mock_calls) self.assertEqual( client_mock.AppsV1beta1Api().patch_namespaced_stateful_set. return_value, result)
def test_getMongoObject(self, client_mock): service = KubernetesService() client_mock.reset_mock() result = service.getMongoObject(self.name, self.namespace) expected_calls = [ call.CustomObjectsApi().get_namespaced_custom_object( "javamachr.cz", "v1", self.namespace, "mongos", self.name) ] self.assertEqual(expected_calls, client_mock.mock_calls) self.assertEqual( client_mock.CustomObjectsApi().get_namespaced_custom_object. return_value, result)
def __init__(self): self.kubernetes_service = KubernetesService() self.mongo_service = MongoService(self.kubernetes_service) self.resource_checkers = [ ServiceChecker(self.kubernetes_service), StatefulSetChecker(self.kubernetes_service), AdminSecretChecker(self.kubernetes_service), ] # type: List[BaseResourceChecker] self.backup_checker = BackupChecker(self.kubernetes_service) self.cluster_versions = { } # type: Dict[Tuple[str, str], str] # format: {(cluster_name, namespace): resource_version}
def test_listAllSecretsWithLabels_custom(self, client_mock): service = KubernetesService() client_mock.reset_mock() labels = {"operated-by": "me", "heritage": "mongo", "name": "name"} result = service.listAllSecretsWithLabels(labels) expected_calls = [ call.CoreV1Api().list_secret_for_all_namespaces( label_selector="operated-by=me,heritage=mongo,name=name") ] self.assertEqual(expected_calls, client_mock.mock_calls) self.assertEqual( client_mock.CoreV1Api().list_secret_for_all_namespaces. return_value, result)
def test_listAllSecretsWithLabels_default(self, client_mock): service = KubernetesService() client_mock.reset_mock() result = service.listAllSecretsWithLabels() expected_calls = [ call.CoreV1Api().list_secret_for_all_namespaces( label_selector= "operated-by=operators.ultimaker.com,heritage=mongos") ] self.assertEqual(expected_calls, client_mock.mock_calls) self.assertEqual( client_mock.CoreV1Api().list_secret_for_all_namespaces. return_value, result)
def test_getMongoObject(self, client_mock): service = KubernetesService() client_mock.reset_mock() result = service.getMongoObject(self.name, self.namespace) expected_calls = [ call.CustomObjectsApi().get_namespaced_custom_object( 'operators.ultimaker.com', 'v1', self.namespace, 'mongos', self.name) ] self.assertEqual(expected_calls, client_mock.mock_calls) self.assertEqual( client_mock.CustomObjectsApi().get_namespaced_custom_object. return_value, result)
def test_listAllStatefulSetsWithLabels_default(self, client_mock): service = KubernetesService() client_mock.reset_mock() result = service.listAllStatefulSetsWithLabels() expected_calls = [ call.AppsV1beta1Api().list_stateful_set_for_all_namespaces( label_selector= "operated-by=operators.javamachr.cz,heritage=mongos") ] self.assertEqual(expected_calls, client_mock.mock_calls) self.assertEqual( client_mock.AppsV1beta1Api().list_stateful_set_for_all_namespaces. return_value, result)
def test_createSecret(self, client_mock): service = KubernetesService() client_mock.reset_mock() secret_data = {"username": "******", "password": "******"} expected_body = V1Secret(metadata=self._createMeta("secret-name"), string_data=secret_data) result = service.createSecret("secret-name", self.namespace, secret_data) self.assertEqual([ call.CoreV1Api().create_namespaced_secret(self.namespace, expected_body) ], client_mock.mock_calls) self.assertEqual( client_mock.CoreV1Api.return_value.create_namespaced_secret. return_value, result)
def test_createStatefulSet_no_optional_fields(self, client_mock): service = KubernetesService() client_mock.reset_mock() del self.cluster_dict["spec"]["mongodb"]["cpu_limit"] del self.cluster_dict["spec"]["mongodb"]["memory_limit"] self.cluster_object = V1MongoClusterConfiguration(**self.cluster_dict) expected_calls = [ call.AppsV1beta1Api().create_namespaced_stateful_set( self.namespace, self.stateful_set) ] result = service.createStatefulSet(self.cluster_object) self.assertEqual(expected_calls, client_mock.mock_calls) self.assertEqual( client_mock.AppsV1beta1Api().create_namespaced_stateful_set. return_value, result)
def test_execInPod(self, stream_mock, client_mock): service = KubernetesService() client_mock.reset_mock() result = service.execInPod("container", "pod_name", self.namespace, "ls") stream_mock.assert_called_once_with( client_mock.CoreV1Api.return_value.connect_get_namespaced_pod_exec, 'pod_name', 'default', command='ls', container='container', stderr=True, stdin=False, stdout=True, tty=False) self.assertEquals(stream_mock.return_value, result) self.assertEquals([], client_mock.mock_calls)
def test_deleteService_TypeError(self, client_mock): service = KubernetesService() client_mock.reset_mock() client_mock.CoreV1Api.return_value.delete_namespaced_service.side_effect = TypeError, V1Status( ) result = service.deleteService(self.name, self.namespace) expected_calls = [ call.CoreV1Api().delete_namespaced_service(self.name, self.namespace, V1DeleteOptions()), call.CoreV1Api().delete_namespaced_service(self.name, self.namespace), ] self.assertEqual(expected_calls, client_mock.mock_calls) self.assertEqual(V1Status(), result)
def test_createMongoObjectDefinition_existing(self, client_mock): service = KubernetesService() client_mock.reset_mock() item = MagicMock() item.spec.names.plural = "mongos" client_mock.ApiextensionsV1beta1Api.return_value.list_custom_resource_definition.return_value.items = [ item ] result = service.createMongoObjectDefinition() self.assertEquals(item, result) expected_calls = [ call.ApiextensionsV1beta1Api().list_custom_resource_definition() ] self.assertEqual(expected_calls, client_mock.mock_calls)
def test_createSecret_exists(self, client_mock): service = KubernetesService() client_mock.reset_mock() client_mock.CoreV1Api.return_value.create_namespaced_secret.side_effect = ApiException( status=409) client_mock.CoreV1Api.return_value.create_namespaced_secret.side_effect.body = "{}" secret_data = {"username": "******", "password": "******"} result = service.createSecret(self.name, self.namespace, secret_data) expected_body = V1Secret(metadata=self._createMeta(self.name), string_data=secret_data) expected_calls = [ call.CoreV1Api().create_namespaced_secret(self.namespace, expected_body) ] self.assertEqual(expected_calls, client_mock.mock_calls) self.assertIsNone(result)
def test_listMongoObjects(self, client_mock): service = KubernetesService() client_mock.reset_mock() item = MagicMock() item.spec.names.plural = "mongos" client_mock.ApiextensionsV1beta1Api.return_value.list_custom_resource_definition.return_value.items = [ item ] result = service.listMongoObjects(param="value") expected_calls = [ call.ApiextensionsV1beta1Api().list_custom_resource_definition(), call.CustomObjectsApi().list_cluster_custom_object( 'operators.ultimaker.com', 'v1', "mongos", param='value') ] self.assertEqual(expected_calls, client_mock.mock_calls) self.assertEqual( client_mock.CustomObjectsApi().list_cluster_custom_object. return_value, result)
def test_updateSecret(self, client_mock): service = KubernetesService() client_mock.reset_mock() client_mock.CoreV1Api.return_value.read_namespaced_secret.return_value = V1Secret( kind="unit") secret_data = {"username": "******", "password": "******"} expected_body = V1Secret(kind="unit", string_data=secret_data) expected_calls = [ call.CoreV1Api().read_namespaced_secret(self.name, self.namespace), call.CoreV1Api().patch_namespaced_secret(self.name, self.namespace, expected_body), ] result = service.updateSecret(self.name, self.namespace, secret_data) self.assertEqual(expected_calls, client_mock.mock_calls) self.assertEqual( client_mock.CoreV1Api.return_value.patch_namespaced_secret. return_value, result)
def test_listMongoObjects_400(self, client_mock): service = KubernetesService() client_mock.reset_mock() item = MagicMock() item.spec.names.plural = "mongos" client_mock.ApiextensionsV1beta1Api.return_value.list_custom_resource_definition.return_value.items = [ item ] client_mock.CustomObjectsApi.return_value.list_cluster_custom_object.side_effect = ApiException( 400) with self.assertRaises(ApiException): service.listMongoObjects(param="value") expected_calls = [ call.ApiextensionsV1beta1Api().list_custom_resource_definition(), call.CustomObjectsApi().list_cluster_custom_object( "operators.ultimaker.com", "v1", "mongos", param="value") ] self.assertEqual(expected_calls, client_mock.mock_calls)
def test___init__(self, client_mock): KubernetesService() config = Configuration() config.debug = False expected = [ call.ApiClient(config), call.CoreV1Api(client_mock.ApiClient.return_value), call.CustomObjectsApi(client_mock.ApiClient.return_value), call.ApiextensionsV1beta1Api(client_mock.ApiClient.return_value), call.AppsV1beta1Api(client_mock.ApiClient.return_value), ] with patch("kubernetes.client.configuration.Configuration.__eq__", dict_eq): self.assertEqual(expected, client_mock.mock_calls)
class ClusterChecker: """ Manager that periodically checks the status of the MongoDB objects in the cluster. """ STREAM_REQUEST_TIMEOUT = (15.0, 5.0) # connect, read timeout def __init__(self): self.kubernetes_service = KubernetesService() self.mongo_service = MongoService(self.kubernetes_service) self.resource_checkers = [ ServiceChecker(self.kubernetes_service), StatefulSetChecker(self.kubernetes_service), AdminSecretChecker(self.kubernetes_service), ] # type: List[BaseResourceChecker] self.backup_checker = BackupChecker(self.kubernetes_service) self.cluster_versions = { } # type: Dict[Tuple[str, str], str] # format: {(cluster_name, namespace): resource_version} @staticmethod def _parseConfiguration( cluster_dict: Dict[str, any]) -> Optional[V1MongoClusterConfiguration]: """ Tries to parse the given cluster configuration, returning None if the object cannot be parsed. :param cluster_dict: The dictionary containing the configuration. :return: The cluster configuration model, if valid, or None. """ try: result = V1MongoClusterConfiguration(**cluster_dict) result.validate() return result except ValueError as err: meta = cluster_dict.get("metadata", {}) logging.error( "Could not validate cluster configuration for {} @ ns/{}: {}. The cluster will be ignored." .format(meta.get("name"), meta.get("namespace"), err)) def checkExistingClusters(self) -> None: """ Check all Mongo objects and see if the sub objects are available. If they are not, they should be (re-)created to ensure the cluster is in the expected state. """ mongo_objects = self.kubernetes_service.listMongoObjects() logging.info("Checking %s mongo objects.", len(mongo_objects["items"])) for cluster_dict in mongo_objects["items"]: cluster_object = self._parseConfiguration(cluster_dict) if cluster_object: self.checkCluster(cluster_object) def streamEvents(self) -> None: """ Watches for changes to the mongo objects in Kubernetes and processes any changes immediately. """ event_watcher = Watch() # start watching from the latest version that we have if self.cluster_versions: event_watcher.resource_version = max( self.cluster_versions.values()) for event in event_watcher.stream( self.kubernetes_service.listMongoObjects, _request_timeout=self.STREAM_REQUEST_TIMEOUT): logging.info("Received event %s", event) if event["type"] in ("ADDED", "MODIFIED"): cluster_object = self._parseConfiguration(event["object"]) if cluster_object: self.checkCluster(cluster_object) else: logging.warning( "Could not validate cluster object, stopping event watcher." ) event_watcher.stop = True elif event["type"] in ("DELETED", ): self.collectGarbage() else: logging.warning( "Could not parse event, stopping event watcher.") event_watcher.stop = True # Change the resource version manually because of a bug fixed in a later version of the K8s client: # https://github.com/kubernetes-client/python-base/pull/64 if isinstance(event.get('object'), dict) and 'resourceVersion' in event['object'].get( 'metadata', {}): event_watcher.resource_version = event['object']['metadata'][ 'resourceVersion'] def collectGarbage(self) -> None: """ Cleans up any resources that are left after a cluster has been removed. """ for checker in self.resource_checkers: checker.cleanResources() def checkCluster(self, cluster_object: V1MongoClusterConfiguration, force: bool = False) -> None: """ Checks whether the given cluster is configured and updated. :param cluster_object: The cluster object from the YAML file. :param force: If this is True, we will re-update the cluster even if it has been checked before. """ key = (cluster_object.metadata.name, cluster_object.metadata.namespace) if self.cluster_versions.get( key) == cluster_object.metadata.resource_version and not force: logging.debug( "Cluster object %s has been checked already in version %s.", key, cluster_object.metadata.resource_version) # we still want to check the replicas to make sure everything is working. self.mongo_service.checkReplicaSetOrInitialize(cluster_object) else: for checker in self.resource_checkers: checker.checkResource(cluster_object) self.mongo_service.checkReplicaSetOrInitialize(cluster_object) self.mongo_service.createUsers(cluster_object) self.cluster_versions[ key] = cluster_object.metadata.resource_version self.backup_checker.backupIfNeeded(cluster_object)
class ClusterManager: """ Manager that periodically checks the status of the MongoDB objects in the cluster. """ def __init__(self) -> None: self._cluster_versions: Dict[Tuple[str, str], str] = {} # format: {(cluster_name, namespace): resource_version} self._kubernetes_service = KubernetesService() self._mongo_service = MongoService(self._kubernetes_service) self._backup_checker = BackupHelper(self._kubernetes_service) self._resource_checkers: List[BaseResourceChecker] = [ ServiceChecker(self._kubernetes_service), StatefulSetChecker(self._kubernetes_service), AdminSecretChecker(self._kubernetes_service), ] def checkExistingClusters(self) -> None: """ Check all Mongo objects and see if the sub objects are available. If they are not, they should be (re-)created to ensure the cluster is in the expected state. """ mongo_objects = self._kubernetes_service.listMongoObjects() logging.info("Checking %s mongo objects.", len(mongo_objects["items"])) for cluster_dict in mongo_objects["items"]: cluster_object = self._parseConfiguration(cluster_dict) if cluster_object: self._checkCluster(cluster_object) def collectGarbage(self) -> None: """ Cleans up any resources that are left after a cluster has been removed. """ for checker in self._resource_checkers: checker.cleanResources() def _checkCluster(self, cluster_object: V1MongoClusterConfiguration, force: bool = False) -> None: """ Checks whether the given cluster is configured and updated. :param cluster_object: The cluster object from the YAML file. :param force: If this is True, we will re-update the cluster even if it has been checked before. """ key = cluster_object.metadata.name, cluster_object.metadata.namespace if self._cluster_versions.get(key) == cluster_object.metadata.resource_version and not force: logging.debug("Cluster object %s has been checked already in version %s.", key, cluster_object.metadata.resource_version) # we still want to check the replicas to make sure everything is working. self._mongo_service.checkOrCreateReplicaSet(cluster_object) else: for checker in self._resource_checkers: checker.checkResource(cluster_object) self._mongo_service.checkOrCreateReplicaSet(cluster_object) self._mongo_service.createUsers(cluster_object) self._cluster_versions[key] = cluster_object.metadata.resource_version self._backup_checker.backupIfNeeded(cluster_object) @staticmethod def _parseConfiguration(cluster_dict: Dict[str, any]) -> Optional[V1MongoClusterConfiguration]: """ Tries to parse the given cluster configuration, returning None if the object cannot be parsed. :param cluster_dict: The dictionary containing the configuration. :return: The cluster configuration model, if valid, or None. """ try: result = V1MongoClusterConfiguration(**cluster_dict) result.validate() return result except ValueError as err: meta = cluster_dict.get("metadata", {}) logging.error("Could not validate cluster configuration for %s @ ns/%s: %s. The cluster will be ignored.", meta.get("name"), meta.get("namespace"), err)
def test_createSecret_error(self, client_mock): service = KubernetesService() client_mock.CoreV1Api.return_value.create_namespaced_secret.side_effect = ApiException( status=400) with self.assertRaises(ApiException): service.createSecret(self.name, self.namespace, secret_data={})