def can_function(self):
status = self.get_status()
if status["connection"]:
if "error" not in status:
return True
else:
log_verbose("Error while connecting to server '%s': %s " % (self.id, status["error"]))
def _apply_alt_address_mapping(self, member_conf):
# Not applicable to arbiters
if self.is_arbiter():
return
tag_mapping = get_cluster_member_alt_address_mapping()
if not tag_mapping:
return
tags = member_conf.get("tags", {})
for tag_name, alt_address_prop in tag_mapping.items():
alt_address = self.get_server().get_property(alt_address_prop)
# set the alt address if it is different than host
if alt_address and alt_address != member_conf['host']:
tags[tag_name] = alt_address
else:
log_verbose("No alt address tag value created for alt address"
" mapping '%s=%s' for member \n%s" %
(tag_name, alt_address_prop, self))
# set the tags property of the member config if there are any
if tags:
log_verbose("Member '%s' tags : %s" % (member_conf['host'], tags))
member_conf['tags'] = tags
def disconnecting_db_command(self, cmd, dbname):
try:
result = self.db_command(cmd, dbname)
return result
except AutoReconnect,e:
log_verbose("This is an expected exception that happens after "
"disconnecting db commands: %s" % e)
def disconnecting_db_command(self, cmd, dbname):
try:
result = self.db_command(cmd, dbname)
return result
except AutoReconnect,e:
log_verbose("This is an expected exception that happens after "
"disconnecting db commands: %s" % e)
def get_connection_address(self):
if self._connection_address:
return self._connection_address
# try to get the first working connection address
# only use this technique if the server is not assumed locally
if not is_assumed_local_server(self.id):
if (self.is_use_local()
and self.has_connectivity_on(self.get_local_address())):
self._connection_address = self.get_local_address()
elif self.has_connectivity_on(self.get_address()):
self._connection_address = self.get_address()
# use old logic
if not self._connection_address:
if self.is_use_local():
self._connection_address = self.get_local_address()
else:
self._connection_address = self.get_address()
log_verbose("Using connection address '%s' for server '%s'" %
(self._connection_address, self.id))
return self._connection_address
def _apply_alt_address_mapping(self, member_conf):
# Not applicable to arbiters
if self.is_arbiter():
return
tag_mapping = get_cluster_member_alt_address_mapping()
if not tag_mapping:
return
tags = member_conf.get("tags", {})
for tag_name, alt_address_prop in tag_mapping.items():
alt_address = self.get_server().get_property(alt_address_prop)
# set the alt address if it is different than host
if alt_address and alt_address != member_conf['host']:
tags[tag_name] = alt_address
else:
log_verbose("No alt address tag value created for alt address"
" mapping '%s=%s' for member \n%s" %
(tag_name, alt_address_prop, self))
# set the tags property of the member config if there are any
if tags:
log_verbose("Member '%s' tags : %s" % (member_conf['host'], tags))
member_conf['tags'] = tags
def can_function(self):
status = self.get_status()
if status['connection']:
if 'error' not in status:
return True
else:
log_verbose("Error while connecting to server '%s': %s " %
(self.id, status['error']))
def is_master_command(self):
try:
if self.is_online():
result = self.db_command({"isMaster": 1}, "admin")
return result
except (Exception, RuntimeError), e:
log_verbose("isMaster command failed on server '%s'. Cause %s" %
(self.id, e))
def is_master_command(self):
try:
if self.is_online():
result = self.db_command({"isMaster" : 1}, "admin")
return result
except(Exception, RuntimeError),e:
log_verbose("isMaster command failed on server '%s'. Cause %s" %
(self.id, e))
def get_rs_config(self):
try:
return self.get_db("local")["system.replset"].find_one()
except (Exception, RuntimeError), e:
log_exception(e)
if type(e) == MongoctlException:
raise e
else:
log_verbose("Cannot get rs config from server '%s'. " "cause: %s" % (self.id, e))
return None
def has_connectivity_on(self, address):
try:
log_verbose("Checking if server '%s' is accessible on " "address '%s'" % (self.id, address))
self.make_db_connection(address)
return True
except Exception, e:
log_exception(e)
log_verbose("Check failed for server '%s' is accessible on " "address '%s': %s" % (self.id, address, e))
return False
def get_rs_config(self):
try:
return self.get_db('local')['system.replset'].find_one()
except (Exception, RuntimeError), e:
log_exception(e)
if type(e) == MongoctlException:
raise e
else:
log_verbose("Cannot get rs config from server '%s'. "
"cause: %s" % (self.id, e))
return None
def has_connectivity_on(self, address):
try:
log_verbose("Checking if server '%s' is accessible on "
"address '%s'" % (self.id, address))
ping(pymongo.MongoClient(address, **DEFAULT_CLIENT_OPTIONS))
return True
except Exception, e:
log_exception(e)
log_verbose("Check failed for server '%s' is accessible on "
"address '%s': %s" % (self.id, address, e))
return False
def has_connectivity_on(self, address):
try:
log_verbose("Checking if server '%s' is accessible on "
"address '%s'" % (self.id, address))
ping(pymongo.MongoClient(address, **DEFAULT_CLIENT_OPTIONS))
return True
except Exception, e:
log_exception(e)
log_verbose("Check failed for server '%s' is accessible on "
"address '%s': %s" % (self.id, address, e))
return False
def log_server_activity(self, activity):
if is_logging_activity():
log_record = {"op": activity,
"ts": datetime.datetime.utcnow(),
"serverDoc": self.get_document(),
"server": self.id,
"serverDisplayName": self.get_description()}
log_verbose("Logging server activity \n%s" %
document_pretty_string(log_record))
repository.get_activity_collection().insert(log_record)
def has_connectivity_on(self, address):
try:
log_verbose("Checking if server '%s' is accessible on "
"address '%s'" % (self.id, address))
mongo_utils.mongo_client(address)
return True
except Exception, e:
log_exception(e)
log_verbose("Check failed for server '%s' is accessible on "
"address '%s': %s" % (self.id, address, e))
return False
def log_server_activity(self, activity):
if is_logging_activity():
log_record = {"op": activity,
"ts": datetime.datetime.utcnow(),
"serverDoc": self.get_document(),
"server": self.id,
"serverDisplayName": self.get_description()}
log_verbose("Logging server activity \n%s" %
document_pretty_string(log_record))
repository.get_activity_collection().insert(log_record)
def get_rs_config(self):
try:
return self.get_db('local')['system.replset'].find_one()
except (Exception,RuntimeError), e:
log_debug("Error whille trying to read rs config from "
"server '%s': %s" % (self.id, e))
log_exception(e)
if type(e) == MongoctlException:
raise e
else:
log_verbose("Cannot get rs config from server '%s'. "
"cause: %s" % (self.id, e))
return None
def validate_local_op(self, op):
# If the server has been assumed to be local then skip validation
if is_assumed_local_server(self.id):
log_verbose("Skipping validation of server's '%s' address '%s' to be"
" local because --assume-local is on" %
(self.id, self.get_host_address()))
return
log_verbose("Validating server address: "
"Ensuring that server '%s' address '%s' is local on this "
"machine" % (self.id, self.get_host_address()))
if not self.is_local():
log_verbose("Server address validation failed.")
raise MongoctlException("Cannot %s server '%s' on this machine "
"because server's address '%s' does not appear "
"to be local to this machine. Pass the "
"--assume-local option if you are sure that "
"this server should be running on this "
"machine." % (op,
self.id,
self.get_host_address()))
else:
log_verbose("Server address validation passed. "
"Server '%s' address '%s' is local on this "
"machine !" % (self.id, self.get_host_address()))
def validate_local_op(self, op):
# If the server has been assumed to be local then skip validation
if is_assumed_local_server(self.id):
log_verbose("Skipping validation of server's '%s' address '%s' to be"
" local because --assume-local is on" %
(self.id, self.get_host_address()))
return
log_verbose("Validating server address: "
"Ensuring that server '%s' address '%s' is local on this "
"machine" % (self.id, self.get_host_address()))
if not self.is_local():
log_verbose("Server address validation failed.")
raise MongoctlException("Cannot %s server '%s' on this machine "
"because server's address '%s' does not appear "
"to be local to this machine. Pass the "
"--assume-local option if you are sure that "
"this server should be running on this "
"machine." % (op,
self.id,
self.get_host_address()))
else:
log_verbose("Server address validation passed. "
"Server '%s' address '%s' is local on this "
"machine !" % (self.id, self.get_host_address()))
def authenticate_db(self, db, dbname, retry=True):
"""
Returns True if we manage to auth to the given db, else False.
"""
log_verbose("Server '%s' attempting to authenticate to db '%s'" %
(self.id, dbname))
login_user = self.get_login_user(dbname)
username = None
password = None
auth_success = False
if login_user:
username = login_user["username"]
if "password" in login_user:
password = login_user["password"]
# have three attempts to authenticate
no_tries = 0
while not auth_success and no_tries < 3:
if not username:
username = read_username(dbname)
if not password:
password = self.lookup_password(dbname, username)
if not password:
password = read_password(
"Enter password for user '%s\%s'" % (dbname, username))
# if auth success then exit loop and memoize login
try:
auth_success = db.authenticate(username, password)
log_verbose(
"Authentication attempt #%s to db '%s' result: %s" %
(no_tries, dbname, auth_success))
except OperationFailure, ofe:
if "auth fails" in str(ofe):
auth_success = False
if auth_success or not retry:
break
else:
log_error("Invalid login!")
username = None
password = None
no_tries += 1
def get_rs_config(self):
rs_conf = None
try:
if self.version_greater_than_3_0():
rs_conf = self.db_command(SON([('replSetGetConfig', 1)]), "admin")["config"]
else:
rs_conf = self.get_db('local')['system.replset'].find_one()
except (Exception,RuntimeError), e:
log_debug("Error whille trying to read rs config from "
"server '%s': %s" % (self.id, e))
log_exception(e)
if type(e) == MongoctlException:
raise e
else:
log_verbose("Cannot get rs config from server '%s'. "
"cause: %s" % (self.id, e))
def get_rs_config(self):
rs_conf = None
try:
if self.version_greater_than_3_0():
rs_conf = self.db_command(SON([('replSetGetConfig', 1)]), "admin")["config"]
else:
rs_conf = self.get_db('local')['system.replset'].find_one()
except (Exception,RuntimeError), e:
log_debug("Error whille trying to read rs config from "
"server '%s': %s" % (self.id, e))
log_exception(e)
if type(e) == MongoctlException:
raise e
else:
log_verbose("Cannot get rs config from server '%s'. "
"cause: %s" % (self.id, e))
def db_command(self, cmd, dbname):
# try without auth first if server allows it (i.e. version >= 3.0.0)
if self.try_on_auth_failures():
need_auth = False
else:
need_auth = self.command_needs_auth(dbname, cmd)
log_verbose("Server '%s': DB Command requested on db %s, need auth ? %s, command: %s" %
(self.id, dbname, need_auth, document_pretty_string(cmd)))
db = self.get_db(dbname, no_auth=not need_auth)
try:
return db.command(cmd)
except (RuntimeError,Exception), e:
if is_auth_error(e) and self.try_on_auth_failures():
db = self.get_db(dbname, no_auth=False)
return db.command(cmd)
else:
raise
def db_command(self, cmd, dbname):
# try without auth first if server allows it (i.e. version >= 3.0.0)
if self.try_on_auth_failures():
need_auth = False
else:
need_auth = self.command_needs_auth(dbname, cmd)
log_verbose("Server '%s': DB Command requested on db %s, need auth ? %s, command: %s" %
(self.id, dbname, need_auth, document_pretty_string(cmd)))
db = self.get_db(dbname, no_auth=not need_auth)
try:
return db.command(cmd)
except (RuntimeError,Exception), e:
if is_auth_error(e) and self.try_on_auth_failures():
db = self.get_db(dbname, no_auth=False)
return db.command(cmd)
else:
raise
def authenticate_db(self, db, dbname, retry=True):
"""
Returns True if we manage to auth to the given db, else False.
"""
log_verbose("Server '%s' attempting to authenticate to db '%s'" % (self.id, dbname))
login_user = self.get_login_user(dbname)
username = None
password = None
auth_success = False
if login_user:
username = login_user["username"]
if "password" in login_user:
password = login_user["password"]
# have three attempts to authenticate
no_tries = 0
while not auth_success and no_tries < 3:
if not username:
username = read_username(dbname)
if not password:
password = self.lookup_password(dbname, username)
if not password:
password = read_password("Enter password for user '%s\%s'"%
(dbname, username))
# if auth success then exit loop and memoize login
try:
auth_success = db.authenticate(username, password)
log_verbose("Authentication attempt #%s to db '%s' result: %s" % (no_tries, dbname, auth_success))
except OperationFailure, ofe:
if "auth fails" in str(ofe):
auth_success = False
if auth_success or not retry:
break
else:
log_error("Invalid login!")
username = None
password = None
no_tries += 1
def get_connection_address(self):
if self._connection_address:
return self._connection_address
# try to get the first working connection address
# only use this technique if the server is not assumed locally
if not is_assumed_local_server(self.id):
if self.is_use_local() and self.has_connectivity_on(self.get_local_address()):
self._connection_address = self.get_local_address()
elif self.has_connectivity_on(self.get_address()):
self._connection_address = self.get_address()
# use old logic
if not self._connection_address:
if self.is_use_local():
self._connection_address = self.get_local_address()
else:
self._connection_address = self.get_address()
log_verbose("Using connection address '%s' for server '%s'" % (self._connection_address, self.id))
return self._connection_address
def assume_local_server(server_id):
global __assumed_local_servers__
if server_id not in __assumed_local_servers__:
log_verbose("Assuming server '%s' to be local" % server_id)
__assumed_local_servers__.append(server_id)
except OperationFailure, ofe:
if "auth fails" in str(ofe):
auth_success = False
if auth_success or not retry:
break
else:
log_error("Invalid login!")
username = None
password = None
no_tries += 1
if auth_success:
self.set_login_user(dbname, username, password)
log_verbose("Authentication Succeeded!")
else:
log_verbose("Authentication failed")
return auth_success
###########################################################################
def get_working_login(self, database, username=None, password=None):
"""
authenticate to the specified database starting with specified
username/password (if present), try to return a successful login
within 3 attempts
"""
login_user = None
def configure_replicaset(self, add_server=None, force_primary_server=None):
# Check if this is an init VS an update
if not self.is_replicaset_initialized():
self.initialize_replicaset()
return
primary_member = self.get_primary_member()
# force server validation and setup
if force_primary_server:
force_primary_member = self.get_member_for(force_primary_server)
# validate is cluster member
if not force_primary_member:
msg = ("Server '%s' is not a member of cluster '%s'" %
(force_primary_server.id, self.id))
raise MongoctlException(msg)
# validate is administrable
if not force_primary_server.is_administrable():
msg = ("Server '%s' is not running or has connection problems."
" For more details, Run 'mongoctl status %s'" %
(force_primary_server.id,
force_primary_server.id))
raise MongoctlException(msg)
if not force_primary_member.can_become_primary():
msg = ("Server '%s' cannot become primary. Reconfiguration of"
" a replica set must be sent to a node that can become"
" primary" % force_primary_server.id)
raise MongoctlException(msg)
if primary_member:
msg = ("Cluster '%s' currently has server '%s' as primary. "
"Proceed with force-reconfigure on server '%s'?" %
(self.id,
primary_member.get_server().id,
force_primary_server.id))
if not prompt_confirm(msg):
return
else:
log_info("No primary server found for cluster '%s'" %
self.id)
elif primary_member is None:
raise MongoctlException("Unable to determine primary server"
" for replica set cluster '%s'" %
self.id)
cmd_server = (force_primary_server if force_primary_server
else primary_member.get_server())
log_info("Re-configuring replica set cluster '%s'..." % self.id)
force = force_primary_server is not None
rs_reconfig_cmd = \
self.get_replicaset_reconfig_db_command(add_server=add_server,
force=force)
desired_config = rs_reconfig_cmd['replSetReconfig']
try:
log_info("Executing the following command on server '%s':"
"\n%s" % (cmd_server.id,
document_pretty_string(rs_reconfig_cmd)))
cmd_server.disconnecting_db_command(rs_reconfig_cmd, "admin")
log_info("Re-configuration command for replica set cluster '%s'"
" issued successfully." % self.id)
# Probably need to reconnect. May not be primary any more.
realized_config = self.read_rs_config()
if realized_config['version'] != desired_config['version']:
log_verbose("Really? Config version unchanged? "
"Let me double-check that ...")
def got_the_memo():
version_diff = (self.read_rs_config()['version'] -
desired_config['version'])
return ((version_diff == 0) or
# force => mongo adds large random # to 'version'.
(force and version_diff >= 0))
if not wait_for(got_the_memo, timeout=45, sleep_duration=5):
raise Exception("New config version not detected!")
# Finally! Resample.
realized_config = self.read_rs_config()
log_info("New replica set configuration:\n %s" %
document_pretty_string(realized_config))
return True
except Exception, e:
log_exception(e)
raise MongoctlException("Unable to reconfigure "
"replica set cluster '%s'. Cause: %s " %
(self.id,e) )
except OperationFailure, ofe:
if "auth fails" in str(ofe):
auth_success = False
if auth_success or not retry:
break
else:
log_error("Invalid login!")
username = None
password = None
no_tries += 1
if auth_success:
self.set_login_user(dbname, username, password)
log_verbose("Authentication Succeeded!")
else:
log_verbose("Authentication failed")
return auth_success
###########################################################################
def get_working_login(self, database, username=None, password=None):
"""
authenticate to the specified database starting with specified
username/password (if present), try to return a successful login
within 3 attempts
"""
login_user = None
def assume_local_server(server_id):
global __assumed_local_servers__
if server_id not in __assumed_local_servers__:
log_verbose("Assuming server '%s' to be local" % server_id)
__assumed_local_servers__.append(server_id)
except OperationFailure, ofe:
if "auth fails" in str(ofe):
auth_success = False
if auth_success or not retry:
break
else:
log_error("Invalid login!")
username = None
password = None
no_tries += 1
if auth_success:
self.set_login_user(dbname, username, password)
log_verbose("Authentication Succeeded!")
else:
log_verbose("Authentication failed")
return auth_success
###########################################################################
def get_working_login(self, database, username=None, password=None):
"""
authenticate to the specified database starting with specified
username/password (if present), try to return a successful login
within 3 attempts
"""
login_user = None
# this will authenticate and update login user
def configure_replicaset(self, add_server=None, force_primary_server=None):
# Check if this is an init VS an update
if not self.is_replicaset_initialized():
self.initialize_replicaset()
return
primary_member = self.get_primary_member()
# force server validation and setup
if force_primary_server:
force_primary_member = self.get_member_for(force_primary_server)
# validate is cluster member
if not force_primary_member:
msg = ("Server '%s' is not a member of cluster '%s'" %
(force_primary_server.id, self.id))
raise MongoctlException(msg)
# validate is administrable
if not force_primary_server.is_administrable():
msg = ("Server '%s' is not running or has connection problems."
" For more details, Run 'mongoctl status %s'" %
(force_primary_server.id, force_primary_server.id))
raise MongoctlException(msg)
if not force_primary_member.can_become_primary():
msg = ("Server '%s' cannot become primary. Reconfiguration of"
" a replica set must be sent to a node that can become"
" primary" % force_primary_server.id)
raise MongoctlException(msg)
if primary_member:
msg = ("Cluster '%s' currently has server '%s' as primary. "
"Proceed with force-reconfigure on server '%s'?" %
(self.id, primary_member.get_server().id,
force_primary_server.id))
if not prompt_confirm(msg):
return
else:
log_info("No primary server found for cluster '%s'" % self.id)
elif primary_member is None:
raise MongoctlException("Unable to determine primary server"
" for replica set cluster '%s'" % self.id)
cmd_server = (force_primary_server
if force_primary_server else primary_member.get_server())
log_info("Re-configuring replica set cluster '%s'..." % self.id)
force = force_primary_server is not None
rs_reconfig_cmd = \
self.get_replicaset_reconfig_db_command(add_server=add_server,
force=force)
desired_config = rs_reconfig_cmd['replSetReconfig']
try:
log_info("Executing the following command on server '%s':"
"\n%s" %
(cmd_server.id, document_pretty_string(rs_reconfig_cmd)))
cmd_server.disconnecting_db_command(rs_reconfig_cmd, "admin")
log_info("Re-configuration command for replica set cluster '%s'"
" issued successfully." % self.id)
# wait until there is a primary
log_info("Wait for the new primary to be elected...")
def has_primary():
return self.get_primary_member() is not None
if not wait_for(has_primary, timeout=60, sleep_duration=1):
raise Exception(
"No primary elected 60 seconds after reconfiguration!")
# Probably need to reconnect. May not be primary any more.
desired_cfg_version = desired_config['version']
def got_the_memo(cur_cfg=None):
current_config = cur_cfg or self.read_rs_config()
# might've gotten None if nobody answers & tells us, so:
current_cfg_version = (current_config['version']
if current_config else 0)
version_diff = (current_cfg_version - desired_cfg_version)
return ((version_diff == 0) or
# force => mongo adds large random # to 'version'.
(force and version_diff >= 0))
realized_config = self.read_rs_config()
if not got_the_memo(realized_config):
log_verbose("Really? Config version %s? "
"Let me double-check that ..." %
"unchanged" if realized_config else "unavailable")
if not wait_for(got_the_memo, timeout=45, sleep_duration=5):
raise Exception("New config version not detected!")
# Finally! Resample.
realized_config = self.read_rs_config()
log_info("New replica set configuration:\n %s" %
document_pretty_string(realized_config))
return True
except Exception, e:
log_exception(e)
raise MongoctlException("Unable to reconfigure "
"replica set cluster '%s'. Cause: %s " %
(self.id, e))
