def test_parseMongoResponse_bad_json(self):
with open("tests/fixtures/mongo_responses/replica-status-ok.txt") as f:
with self.assertRaises(ValueError) as context:
MongoResources.parseMongoResponse(f.read().replace(
"Timestamp", "TimeStamp"))
self.assertIn("Cannot parse JSON because of error",
str(context.exception))
def _initializeReplicaSet(
cluster_object: V1MongoClusterConfiguration) -> None:
"""
Initializes the replica set by sending an `initiate` command to the 1st Mongo pod.
:param cluster_object: The cluster object from the YAML file.
:raise ValueError: In case we receive an unexpected response from Mongo.
:raise ApiException: In case we receive an unexpected response from Kubernetes.
"""
cluster_name = cluster_object.metadata.name
namespace = cluster_object.metadata.namespace
logging.debug("Will initialize replicaset now.")
master_connection = MongoClient(
MongoResources.getMemberHostname(0, cluster_name, namespace),
username='******',
password=cluster_object.spec.users.admin_password,
authSource='admin')
create_replica_command, create_replica_args = MongoResources.createReplicaInitiateCommand(
cluster_object)
create_replica_response = master_connection.admin.command(
create_replica_command, create_replica_args)
if create_replica_response["ok"] == 1:
logging.info("Initialized replica set %s @ ns/%s", cluster_name,
namespace)
return
logging.error("Initializing replica set failed, received %s",
repr(create_replica_response))
raise ValueError(
"Unexpected response initializing replica set {} @ ns/{}:\n{}".
format(cluster_name, namespace, create_replica_response))
def _execInPod(self, pod_index: int, name: str, namespace: str,
mongo_command: str) -> Dict[str, any]:
"""
Executes the given mongo command inside the pod with the given name.
Retries a few times in case we receive a handshake failure.
:param pod_index: The index of the pod.
:param name: The name of the cluster.
:param namespace: The namespace of the cluster.
:param mongo_command: The command to be executed in mongo.
:return: The response from MongoDB. See files in `tests/fixtures/mongo_responses` for examples.
:raise ValueError: If the result could not be parsed.
:raise TimeoutError: If we could not connect to the pod after retrying.
"""
exec_command = MongoResources.createMongoExecCommand(mongo_command)
pod_name = "{}-{}".format(name, pod_index)
for _ in range(self.EXEC_IN_POD_RETRIES):
try:
exec_response = self.kubernetes_service.execInPod(
self.CONTAINER, pod_name, namespace, exec_command)
response = MongoResources.parseMongoResponse(exec_response)
if response.get("ok") == 0 and response.get(
"codeName") == "NodeNotFound":
logging.info(
"Waiting for replica set members for %s @ ns/%s: %s",
pod_name, namespace, response)
else:
return response
except ValueError as e:
if str(e) not in ("connection attempt failed",
"connect failed"):
raise
logging.info(
"Could not connect to Mongo in pod %s @ ns/%s: %s",
pod_name, namespace, e)
except ApiException as e:
if "Handshake status" not in e.reason:
logging.error(
"Error sending following command to pod %s: %s",
pod_name, repr(mongo_command))
raise
logging.info(
"Could not check the replica set or initialize it because of %s. The service is probably "
"starting up. We wait %s seconds before retrying.",
e.reason, self.EXEC_IN_POD_WAIT)
sleep(self.EXEC_IN_POD_WAIT)
raise TimeoutError(
"Could not check the replica set after {} retries!".format(
self.EXEC_IN_POD_RETRIES))
def checkReplicaSetOrInitialize(
self, cluster_object: V1MongoClusterConfiguration) -> None:
"""
Checks that the replica set is initialized, or initializes it otherwise.
:param cluster_object: The cluster object from the YAML file.
:raise ValueError: In case we receive an unexpected response from Mongo.
:raise ApiException: In case we receive an unexpected response from Kubernetes.
"""
cluster_name = cluster_object.metadata.name
namespace = cluster_object.metadata.namespace
replicas = cluster_object.spec.mongodb.replicas
create_status_command = MongoResources.createStatusCommand()
create_status_response = self._execInPod(0, cluster_name, namespace,
create_status_command)
logging.debug("Checking replicas, received %s",
repr(create_status_response))
# If the replica set is not initialized yet, we initialize it
if create_status_response["ok"] == 0 and create_status_response[
"codeName"] == "NotYetInitialized":
return self.initializeReplicaSet(cluster_object)
elif create_status_response["ok"] == 1:
logging.info(
"The replica set %s @ ns/%s seems to be working properly with %s/%s pods.",
cluster_name, namespace,
len(create_status_response["members"]), replicas)
if replicas != len(create_status_response["members"]):
self.reconfigureReplicaSet(cluster_object)
else:
raise ValueError(
"Unexpected response trying to check replicas: '{}'".format(
repr(create_status_response)))
def _createMongoClientForReplicaSet(
self, cluster_object: V1MongoClusterConfiguration) -> MongoClient:
"""
Creates a new MongoClient instance for a replica set.
:return: The mongo client.
"""
logging.info("Creating MongoClient for replicaset %s.",
cluster_object.metadata.name)
client = MongoClient(
MongoResources.getMemberHostnames(cluster_object),
connectTimeoutMS=120000,
serverSelectionTimeoutMS=120000,
replicaSet=cluster_object.metadata.name,
username='******',
password=cluster_object.spec.users.admin_password,
authSource='admin',
event_listeners=[
CommandLogger(),
ServerLogger(),
TopologyListener(
cluster_object,
replica_set_ready_callback=self._onReplicaSetReady),
HeartbeatListener(
cluster_object,
all_hosts_ready_callback=self._onAllHostsReady)
])
logging.info("Created mongoclient connected to %s.", client.address)
return client
def restore(self, cluster_object: V1MongoClusterConfiguration, backup_file: str) -> bool:
"""
Attempts to restore the latest backup in the specified location to the given cluster.
Creates a new backup for the given cluster saving it in the NFS storage.
:param cluster_object: The cluster object from the YAML file.
:param backup_file: The filename of the backup we want to restore.
"""
hostnames = MongoResources.getMemberHostnames(cluster_object)
logging.info("Restoring backup file %s to cluster %s @ ns/%s.", backup_file, cluster_object.metadata.name,
cluster_object.metadata.namespace)
# Wait for the replica set to become ready
for _ in range(self.RESTORE_RETRIES):
try:
logging.info("Running mongorestore --host %s --gzip --archive=%s", ",".join(hostnames), backup_file)
restore_output = check_output(["/opt/rh/rh-mongodb36/root/usr/bin/mongorestore", "--authenticationDatabase=admin", "-u", "admin",
"-p", cluster_object.spec.users.admin_password, "--host", ",".join(hostnames), "--gzip",
"--archive=" + backup_file])
logging.info("Restore output: %s", restore_output)
try:
os.remove(backup_file)
except OSError as err:
logging.error("Unable to remove '%s': %s", backup_file, err.strerror)
return True
except CalledProcessError as err:
logging.error("Could not restore '%s', attempt %d. Return code: %s stderr: '%s' stdout: '%s'",
backup_file, _, err.returncode, err.stderr, err.stdout)
sleep(self.RESTORE_WAIT)
raise TimeoutError("Could not restore '{}' after {} retries!".format(backup_file, self.RESTORE_RETRIES))
def _reconfigureReplicaSet(
self, cluster_object: V1MongoClusterConfiguration) -> None:
"""
Initializes the replica set by sending a `reconfig` command to the 1st Mongo pod.
:param cluster_object: The cluster object from the YAML file.
:raise ValueError: In case we receive an unexpected response from Mongo.
:raise ApiException: In case we receive an unexpected response from Kubernetes.
"""
cluster_name = cluster_object.metadata.name
namespace = cluster_object.metadata.namespace
replicas = cluster_object.spec.mongodb.replicas
reconfigure_command, reconfigure_args = MongoResources.createReplicaReconfigureCommand(
cluster_object)
reconfigure_response = self._executeAdminCommand(
cluster_object, reconfigure_command, reconfigure_args)
logging.debug("Reconfiguring replica, received %s",
repr(reconfigure_response))
if reconfigure_response["ok"] != 1:
raise ValueError(
"Unexpected response reconfiguring replica set {} @ ns/{}:\n{}"
.format(cluster_name, namespace, reconfigure_response))
logging.info("Reconfigured replica set %s @ ns/%s to %s pods",
cluster_name, namespace, replicas)
def backup(self, cluster_object: V1MongoClusterConfiguration, now: datetime):
"""
Creates a new backup for the given cluster saving it in the cloud storage.
:param cluster_object: The cluster object from the YAML file.
:param now: The current date, used in the date format.
"""
backup_file = "/tmp/" + self.BACKUP_FILE_FORMAT.format(namespace=cluster_object.metadata.namespace,
name=cluster_object.metadata.name,
date=now.strftime("%Y-%m-%d_%H%M%S"))
pod_index = cluster_object.spec.mongodb.replicas - 1 # take last pod
hostname = MongoResources.getMemberHostname(pod_index, cluster_object.metadata.name,
cluster_object.metadata.namespace)
logging.info("Backing up cluster %s @ ns/%s from %s to %s.", cluster_object.metadata.name,
cluster_object.metadata.namespace, hostname, backup_file)
try:
backup_output = check_output(["mongodump", "--host", hostname, "--gzip", "--archive=" + backup_file])
except CalledProcessError as err:
raise SubprocessError("Could not backup '{}' to '{}'. Return code: {}\n stderr: '{}'\n stdout: '{}'"
.format(hostname, backup_file, err.returncode, err.stderr, err.stdout))
logging.debug("Backup output: %s", backup_output)
self._uploadBackup(cluster_object, backup_file)
os.remove(backup_file)
def initializeReplicaSet(
self, cluster_object: V1MongoClusterConfiguration) -> None:
"""
Initializes the replica set by sending an `initiate` command to the 1st Mongo pod.
:param cluster_object: The cluster object from the YAML file.
:raise ValueError: In case we receive an unexpected response from Mongo.
:raise ApiException: In case we receive an unexpected response from Kubernetes.
"""
cluster_name = cluster_object.metadata.name
namespace = cluster_object.metadata.namespace
create_replica_command = MongoResources.createReplicaInitiateCommand(
cluster_object)
create_replica_response = self._execInPod(0, cluster_name, namespace,
create_replica_command)
logging.debug("Initializing replica, received %s",
repr(create_replica_response))
if create_replica_response["ok"] == 1:
logging.info("Initialized replica set %s @ ns/%s", cluster_name,
namespace)
else:
raise ValueError(
"Unexpected response initializing replica set {} @ ns/{}:\n{}".
format(cluster_name, namespace, create_replica_response))
def test__mongoAdminCommand_NodeNotFound(self, mongo_client_mock):
mongo_client_mock.return_value.admin.command.side_effect = OperationFailure(
"replSetInitiate quorum check failed because not all proposed set members responded affirmatively:"
)
with self.assertRaises(OperationFailure) as ex:
mongo_command, mongo_args = MongoResources.createReplicaInitiateCommand(
self.cluster_object)
self.service._executeAdminCommand(self.cluster_object,
mongo_command, mongo_args)
self.assertIn("replSetInitiate quorum check failed", str(ex.exception))
def test_parseMongoResponse_not_initialized(self):
with open(
"tests/fixtures/mongo_responses/replica-status-not-initialized.txt"
) as f:
response = MongoResources.parseMongoResponse(f.read())
expected = {
"info": "run rs.initiate(...) if not yet done for the set",
"ok": 0,
"errmsg": "no replset config has been received",
"code": 94,
"codeName": "NotYetInitialized"
}
self.assertEqual(expected, response)
def userExists(self, cluster_object: V1MongoClusterConfiguration,
username: str) -> bool:
"""
Runs a Mongo command to determine whether the specified user exists in this cluster.
:param cluster_object: The cluster object from the YAML file.
:param username: The user we want to lookup.
:return: A boolean value indicating whether the user exists.
"""
find_admin_command, find_admin_kwargs = MongoResources.createFindAdminCommand(
username)
find_result = self._executeAdminCommand(cluster_object,
find_admin_command,
find_admin_kwargs)
logging.debug("Result of user find_one is %s", repr(find_result))
return find_result is not None
def checkOrCreateReplicaSet(
self, cluster_object: V1MongoClusterConfiguration) -> None:
"""
Checks that the replica set is initialized, or initializes it otherwise.
:param cluster_object: The cluster object from the YAML file.
:raise ValueError: In case we receive an unexpected response from Mongo.
:raise ApiException: In case we receive an unexpected response from Kubernetes.
"""
cluster_name = cluster_object.metadata.name
namespace = cluster_object.metadata.namespace
replicas = cluster_object.spec.mongodb.replicas
create_status_command = MongoResources.createStatusCommand()
try:
logging.debug("Will execute status command.")
create_status_response = self._executeAdminCommand(
cluster_object, create_status_command)
logging.debug("Checking replicas, received %s",
repr(create_status_response))
# The replica set could not be checked
if create_status_response["ok"] != 1:
raise ValueError(
"Unexpected response trying to check replicas: '{}'".
format(repr(create_status_response)))
logging.info(
"The replica set %s @ ns/%s seems to be working properly with %s/%s pods.",
cluster_name, namespace,
len(create_status_response["members"]), replicas)
# The amount of replicas is not the same as configured, we need to fix this
if replicas != len(create_status_response["members"]):
self._reconfigureReplicaSet(cluster_object)
except OperationFailure as err:
logging.debug("Failed with %s", err)
if str(err) != self.NO_REPLICA_SET_RESPONSE:
logging.debug("No replicaset response.")
raise
# If the replica set is not initialized yet, we initialize it
logging.debug(
"Replicaset is not initialized, will initialize now.")
self._initializeReplicaSet(cluster_object)
def _createMongoClientForReplicaSet(
self, cluster_object: V1MongoClusterConfiguration) -> MongoClient:
"""
Creates a new MongoClient instance for a replica set.
:return: The mongo client.
"""
return MongoClient(
MongoResources.getMemberHostnames(cluster_object),
connectTimeoutMS=120000,
serverSelectionTimeoutMS=120000,
replicaSet=cluster_object.metadata.name,
event_listeners=[
CommandLogger(),
ServerLogger(),
TopologyListener(
cluster_object,
replica_set_ready_callback=self._onReplicaSetReady),
HeartbeatListener(
cluster_object,
all_hosts_ready_callback=self._onAllHostsReady)
])
def createUsers(self, cluster_object: V1MongoClusterConfiguration) -> None:
"""
Creates the users required for each of the pods in the replica.
:param cluster_object: The cluster object from the YAML file.
:raise ValueError: In case we receive an unexpected response from Mongo.
:raise ApiException: In case we receive an unexpected response from Kubernetes.
"""
cluster_name = cluster_object.metadata.name
namespace = cluster_object.metadata.namespace
secret_name = AdminSecretChecker.getSecretName(cluster_name)
admin_credentials = self._kubernetes_service.getSecret(
secret_name, namespace)
create_admin_command, create_admin_args, create_admin_kwargs = MongoResources.createCreateAdminCommand(
admin_credentials)
if not self.userExists(cluster_object, create_admin_args):
create_admin_response = self._executeAdminCommand(
cluster_object, create_admin_command, create_admin_args,
**create_admin_kwargs)
logging.info("Created admin user: %s", create_admin_response)
else:
logging.info("No need to create admin user, it already exists")
def test_parseMongoResponse_version_twice(self):
self.assertEqual({},
MongoResources.parseMongoResponse(
"MongoDB shell version v3.6.4\n"
"connecting to: mongodb://localhost:27017/admin\n"
"MongoDB server version: 3.6.4\n"))
def test_parseMongoResponse_only_version(self):
self.assertEqual({},
MongoResources.parseMongoResponse(
"MongoDB shell version v3.6.4\n"))
def test_parseMongoResponse_empty(self):
self.assertEqual({}, MongoResources.parseMongoResponse(''))
def test_parseMongoResponse_error(self):
with open("tests/fixtures/mongo_responses/replica-status-error.txt"
) as f:
with self.assertRaises(ValueError) as context:
MongoResources.parseMongoResponse(f.read())
self.assertEqual("connect failed", str(context.exception))
def test_parseMongoResponse_ok(self):
with open("tests/fixtures/mongo_responses/replica-status-ok.txt") as f:
response = MongoResources.parseMongoResponse(f.read())
expected = {
'$clusterTime': {
'clusterTime': 1528362785.0,
'signature': {
'hash': 'AAAAAAAAAAAAAAAAAAAAAAAAAAA=',
'keyId': 0
}
},
'date':
'2018-06-07T09:13:07.663Z',
'heartbeatIntervalMillis':
2000,
'members': [{
'_id': 0,
'configVersion': 1,
'electionDate': '2018-06-07T09:10:22Z',
'electionTime': 1528362622.1,
'health': 1,
'name': 'some-db-0.some-db.default.svc.cluster.local:27017',
'optime': {
't': 1,
'ts': 1528362783.1
},
'optimeDate': '2018-06-07T09:13:03Z',
'self': True,
'state': 1,
'stateStr': 'PRIMARY',
'uptime': 210
}, {
'_id': 1,
'configVersion': 1,
'health': 1,
'lastHeartbeat': '2018-06-07T09:13:07.162Z',
'lastHeartbeatRecv': '2018-06-07T09:13:07.265Z',
'name': 'some-db-1.some-db.default.svc.cluster.local:27017',
'optime': {
't': 1,
'ts': 1528362783.1
},
'optimeDate': '2018-06-07T09:13:03Z',
'optimeDurable': {
't': 1,
'ts': 1528362783.1
},
'optimeDurableDate': '2018-06-07T09:13:03Z',
'pingMs': 0,
'state': 2,
'stateStr': 'SECONDARY',
'syncingTo':
'some-db-2.some-db.default.svc.cluster.local:27017',
'uptime': 178
}, {
'_id': 2,
'configVersion': 1,
'health': 1,
'lastHeartbeat': '2018-06-07T09:13:06.564Z',
'lastHeartbeatRecv': '2018-06-07T09:13:06.760Z',
'name': 'some-db-2.some-db.default.svc.cluster.local:27017',
'optime': {
't': 1,
'ts': 1528362783.1
},
'optimeDate': '2018-06-07T09:13:03Z',
'optimeDurable': {
't': 1,
'ts': 1528362783.1
},
'optimeDurableDate': '2018-06-07T09:13:03Z',
'pingMs': 6,
'state': 2,
'stateStr': 'SECONDARY',
'syncingTo':
'some-db-0.some-db.default.svc.cluster.local:27017',
'uptime': 178
}],
'myState':
1,
'ok':
1,
'operationTime':
1528362783.1,
'optimes': {
'appliedOpTime': {
't': 1,
'ts': 1528362783.1
},
'durableOpTime': {
't': 1,
'ts': 1528362783.1
},
'lastCommittedOpTime': {
't': 1,
'ts': 1528362783.1
},
'readConcernMajorityOpTime': {
't': 1,
'ts': 1528362783.1
}
},
'set':
'some-db',
'term':
1
}
self.assertEqual(expected, response)
def createUsers(self, cluster_object: V1MongoClusterConfiguration) -> None:
"""
Creates the users required for each of the pods in the replica.
:param cluster_object: The cluster object from the YAML file.
:raise ValueError: In case we receive an unexpected response from Mongo.
:raise ApiException: In case we receive an unexpected response from Kubernetes.
"""
cluster_name = cluster_object.metadata.name
namespace = cluster_object.metadata.namespace
replicas = cluster_object.spec.mongodb.replicas
secret_name = AdminSecretChecker.getSecretName(cluster_name)
admin_credentials = self.kubernetes_service.getSecret(
secret_name, namespace)
create_admin_command = MongoResources.createCreateAdminCommand(
admin_credentials)
logging.info("Creating users for %s pods", replicas)
for _ in range(self.EXEC_IN_POD_RETRIES):
for i in range(replicas):
# see tests for examples of these responses.
try:
exec_response = self._execInPod(i, cluster_name, namespace,
create_admin_command)
if "user" in exec_response:
logging.info("Created users for pod %s-%s @ ns/%s",
cluster_name, i, namespace)
return
raise ValueError(
"Unexpected response creating users for pod {}-{} @ ns/{}:\n{}"
.format(cluster_name, i, namespace, exec_response))
except ValueError as err:
err_str = str(err)
if "couldn't add user: not master" in err_str:
# most of the time member 0 is elected master, otherwise we get this error and need to loop through
# members until we find the master
logging.info(
"The user could not be created in pod %s-%s because it's not master.",
cluster_name, i)
continue
if "already exists" in err_str:
logging.info("User creation not necessary: %s",
err_str)
return
raise
logging.info(
"Could not create users in any of the %s pods of cluster %s @ ns/%s. We wait %s seconds "
"before retrying.", replicas, cluster_name, namespace,
self.EXEC_IN_POD_WAIT)
sleep(self.EXEC_IN_POD_WAIT)
raise TimeoutError(
"Could not create users in any of the {} pods of cluster {} @ ns/{}."
.format(replicas, cluster_name, namespace))
def test_parseMongoResponse_user_created(self):
with open("tests/fixtures/mongo_responses/createUser-ok.txt") as f:
response = MongoResources.parseMongoResponse(f.read())
expected = {"user": "******", "roles": [{"role": "root", "db": "admin"}]}
self.assertEqual(expected, response)
def test_parseMongoResponse_empty(self):
with self.assertRaises(ValueError) as context:
MongoResources.parseMongoResponse("")
self.assertEqual("Cannot parse MongoDB status response: ''",
str(context.exception))
