def dump_command(parsed_options):
# get and validate dump target
target = parsed_options.target
use_best_secondary = parsed_options.useBestSecondary
#max_repl_lag = parsed_options.maxReplLag
is_addr = is_db_address(target)
is_path = is_dbpath(target)
if is_addr and is_path:
msg = ("Ambiguous target value '%s'. Your target matches both a dbpath"
" and a db address. Use prefix 'file://', 'cluster://' or"
" 'server://' to make it more specific" % target)
raise MongoctlException(msg)
elif not (is_addr or is_path):
raise MongoctlException("Invalid target value '%s'. Target has to be"
" a valid db address or dbpath." % target)
dump_options = extract_mongo_dump_options(parsed_options)
if is_addr:
mongo_dump_db_address(target,
username=parsed_options.username,
password=parsed_options.password,
use_best_secondary=use_best_secondary,
max_repl_lag=None,
dump_options=dump_options)
else:
dbpath = resolve_path(target)
mongo_dump_db_path(dbpath, dump_options=dump_options)
def mongo_exe_version(mongo_exe):
mongod_path = os.path.join(os.path.dirname(mongo_exe), "mongod")
try:
re_expr = "v?((([0-9]+)\.([0-9]+)\.([0-9]+))([^, ]*))"
vers_spew = execute_command([mongod_path, "--version"])
# only take first line of spew
vers_spew_line = vers_spew.split('\n')[0]
vers_grep = re.findall(re_expr, vers_spew_line)
help_spew = execute_command([mongod_path, "--help"])
full_version = vers_grep[-1][0]
if "subscription" in vers_spew or "enterprise" in vers_spew:
edition = MongoDBEdition.ENTERPRISE
elif "SSL" in help_spew:
edition = MongoDBEdition.COMMUNITY_SSL
else:
edition = MongoDBEdition.COMMUNITY
result = make_version_info(full_version, edition=edition)
if result is not None:
return result
else:
raise MongoctlException("Cannot parse mongo version from the"
" output of '%s --version'" % mongod_path)
except Exception, e:
log_exception(e)
raise MongoctlException("Unable to get mongo version of '%s'."
" Cause: %s" % (mongod_path, e))
def restore_command(parsed_options):
# get and validate source/destination
source = parsed_options.source
destination = parsed_options.destination
is_addr = is_db_address(destination)
is_path = is_dbpath(destination)
if is_addr and is_path:
msg = ("Ambiguous destination value '%s'. Your destination matches"
" both a dbpath and a db address. Use prefix 'file://',"
" 'cluster://' or 'server://' to make it more specific" %
destination)
raise MongoctlException(msg)
elif not (is_addr or is_path):
raise MongoctlException(
"Invalid destination value '%s'. Destination has to be"
" a valid db address or dbpath." % destination)
restore_options = extract_mongo_restore_options(parsed_options)
if is_addr:
mongo_restore_db_address(destination,
source,
username=parsed_options.username,
password=parsed_options.password,
restore_options=restore_options)
else:
dbpath = resolve_path(destination)
mongo_restore_db_path(dbpath, source, restore_options=restore_options)
def remove_shard_command(parsed_options):
shard_id = parsed_options.shardId
# determine if the shard is a replicaset cluster or a server
shard = repository.lookup_cluster(shard_id)
if not shard:
shard = repository.lookup_server(shard_id)
if not shard:
raise MongoctlException("Unknown shard '%s'" % shard_id)
sharded_cluster = repository.lookup_cluster_by_shard(shard)
if not sharded_cluster:
raise MongoctlException("'%s' is not a shard" % shard_id)
dest = getattr(parsed_options, "unshardedDataDestination")
synchronized = getattr(parsed_options, "synchronized")
if parsed_options.dryRun:
dry_run_remove_shard(shard, sharded_cluster)
else:
sharded_cluster.remove_shard(shard,
unsharded_data_dest_id=dest,
synchronized=synchronized)
def do_start_server(server, options_override=None, rs_add=False, no_init=False, standalone=False):
# ensure that the start was issued locally. Fail otherwise
server.validate_local_op("start")
log_info("Checking to see if server '%s' is already running"
" before starting it..." % server.id)
status = server.get_status()
if status['connection']:
log_info("Server '%s' is already running." %
server.id)
# always call post server start if the server is already started
# the post server start steps should be idempotent
_post_server_start(server, server.get_pid(), rs_add=rs_add, no_init=no_init,
standalone=standalone)
return
elif "timedOut" in status:
raise MongoctlException("Unable to start server: Server '%s' seems to"
" be already started but is"
" not responding (connection timeout)."
" Or there might some server running on the"
" same port %s" %
(server.id, server.get_port()))
# check if there is another process running on the same port
elif "error" in status and ("closed" in status["error"] or
"reset" in status["error"] or
"ids don't match" in status["error"]):
raise MongoctlException("Unable to start server: Either server '%s' is "
"started but not responding or port %s is "
"already in use." %
(server.id, server.get_port()))
elif "error" in status and "SSL handshake failed" in status["error"]:
raise MongoctlException(
"Unable to start server '%s'. "
"The server appears to be configured with SSL but is not "
"currently running with SSL (SSL handshake failed). "
"Try running mongoctl with --ssl-off." % server.id)
# do necessary work before starting the mongod process
_pre_server_start(server, options_override=options_override)
server.log_server_activity("start")
server_pid = start_server_process(server, options_override, standalone=standalone)
_post_server_start(server, server_pid, rs_add=rs_add, no_init=no_init, standalone=standalone)
# Note: The following block has to be the last block
# because server_process.communicate() will not return unless you
# interrupt the server process which will kill mongoctl, so nothing after
# this block will be executed. Almost never...
if not server.is_fork():
communicate_to_child_process(server_pid)
def show_server_command(parsed_options):
server = repository.lookup_server(parsed_options.server)
if server is None:
raise MongoctlException("Could not find server '%s'." %
parsed_options.server)
log_info("Configuration for server '%s':" % parsed_options.server)
print server
def open_mongo_shell_to(db_address,
username=None,
password=None,
shell_options=None,
js_files=None):
if is_mongo_uri(db_address):
open_mongo_shell_to_uri(db_address, username, password, shell_options,
js_files)
return
# db_address is an id string
id_path = db_address.split("/")
id = id_path[0]
database = id_path[1] if len(id_path) == 2 else None
server = repository.lookup_server(id)
if server:
open_mongo_shell_to_server(server, database, username, password,
shell_options, js_files)
return
# Maybe cluster?
cluster = repository.lookup_cluster(id)
if cluster:
open_mongo_shell_to_cluster(cluster, database, username, password,
shell_options, js_files)
return
# Unknown destination
raise MongoctlException("Unknown db address '%s'" % db_address)
def show_cluster_command(parsed_options):
cluster = repository.lookup_cluster(parsed_options.cluster)
if cluster is None:
raise MongoctlException("Could not find cluster '%s'." %
parsed_options.cluster)
log_info("Configuration for cluster '%s':" % parsed_options.cluster)
print cluster
def mongo_dump_db_address(db_address,
username=None,
password=None,
use_best_secondary=False,
max_repl_lag=None,
dump_options=None):
if is_mongo_uri(db_address):
mongo_dump_uri(uri=db_address, username=username, password=password,
use_best_secondary=use_best_secondary,
dump_options=dump_options)
return
# db_address is an id string
id_path = db_address.split("/")
id = id_path[0]
database = id_path[1] if len(id_path) == 2 else None
server = repository.lookup_server(id)
if server:
mongo_dump_server(server, database=database, username=username,
password=password, dump_options=dump_options)
return
else:
cluster = repository.lookup_cluster(id)
if cluster:
mongo_dump_cluster(cluster, database=database, username=username,
password=password,
use_best_secondary=use_best_secondary,
max_repl_lag=max_repl_lag,
dump_options=dump_options)
return
# Unknown destination
raise MongoctlException("Unknown db address '%s'" % db_address)
def server_started():
# check if the command failed
if not is_pid_alive(mongod_pid):
raise MongoctlException("Could not start the server. Please check"
" the log file.")
return server.is_online()
def validate_local_op(self, op):
# If the server has been assumed to be local then skip validation
if is_assumed_local_server(self.id):
log_verbose("Skipping validation of server's '%s' address '%s' to be"
" local because --assume-local is on" %
(self.id, self.get_host_address()))
return
log_verbose("Validating server address: "
"Ensuring that server '%s' address '%s' is local on this "
"machine" % (self.id, self.get_host_address()))
if not self.is_local():
log_verbose("Server address validation failed.")
raise MongoctlException("Cannot %s server '%s' on this machine "
"because server's address '%s' does not appear "
"to be local to this machine. Pass the "
"--assume-local option if you are sure that "
"this server should be running on this "
"machine." % (op,
self.id,
self.get_host_address()))
else:
log_verbose("Server address validation passed. "
"Server '%s' address '%s' is local on this "
"machine !" % (self.id, self.get_host_address()))
def get_validate_cluster(self):
cluster = self.get_cluster()
if not cluster:
raise MongoctlException("No cluster found for server '%s'" %
self.id)
repository.validate_cluster(cluster)
return cluster
def _pre_mongod_server_start(server, options_override=None):
"""
Does necessary work before starting a server
1- An efficiency step for arbiters running with --no-journal
* there is a lock file ==>
* server must not have exited cleanly from last run, and does not know
how to auto-recover (as a journalled server would)
* however: this is an arbiter, therefore
* there is no need to repair data files in any way ==>
* i can rm this lockfile and start my server
"""
lock_file_path = server.get_lock_file_path()
no_journal = (server.get_cmd_option("nojournal")
or (options_override and "nojournal" in options_override))
if (os.path.exists(lock_file_path) and server.is_arbiter_server()
and no_journal):
log_warning("WARNING: Detected a lock file ('%s') for your server '%s'"
" ; since this server is an arbiter, there is no need for"
" repair or other action. Deleting mongod.lock and"
" proceeding..." % (lock_file_path, server.id))
try:
os.remove(lock_file_path)
except Exception, e:
log_exception(e)
raise MongoctlException("Error while trying to delete '%s'. "
"Cause: %s" % (lock_file_path, e))
def uninstall_mongodb(version_number, edition=None):
version_info = make_version_info(version_number, edition=edition)
# validate version string
if not is_valid_version_info(version_info):
raise MongoctlException("Invalid version '%s'. Please provide a"
" valid MongoDB version." % version_info)
mongo_installation = get_mongo_installation(version_info)
if mongo_installation is None: # no-op
msg = ("Cannot find a MongoDB installation for version '%s'. Please"
" use list-versions to see all possible versions " %
version_info)
log_info(msg)
return
log_info("Found MongoDB '%s' in '%s'" % (version_info, mongo_installation))
def rm_mongodb():
# make sure that the mongo installation to be removed does not have
# any running processes
ensure_mongo_home_not_used(mongo_installation)
log_info("Deleting '%s'" % mongo_installation)
shutil.rmtree(mongo_installation)
log_info("MongoDB '%s' Uninstalled successfully!" % version_info)
prompt_execute_task("Proceed uninstall?" , rm_mongodb)
def fetch_latest_stable_version():
response = urllib.urlopen(LATEST_VERSION_FILE_URL)
if response.getcode() == 200:
return response.read().strip()
else:
raise MongoctlException("Unable to fetch MongoDB latest stable version"
" from '%s' (Response code %s)" %
(LATEST_VERSION_FILE_URL, response.getcode()))
def get_environment_variables(self):
env_vars = self.get_property('environmentVariables') or {}
allowed = self.get_allowed_environment_variables()
for v in env_vars.keys():
if v not in allowed:
raise MongoctlException("Unknown environment variable '%s'" % v)
return env_vars
def do_stop_server(server, force=False):
# ensure that the stop was issued locally. Fail otherwise
server.validate_local_op("stop")
log_info("Checking to see if server '%s' is actually running before"
" stopping it..." % server.id)
# init local flags
can_stop_mongoly = True
shutdown_success = False
status = server.get_status()
if not status['connection']:
if "timedOut" in status:
log_info("Unable to issue 'shutdown' command to server '%s'. "
"The server is not responding (connection timed out) "
"although port %s is open, possibly for mongod." %
(server.id, server.get_port()))
can_stop_mongoly = False
elif "error" in status and "SSL handshake failed" in status["error"]:
log_info("Unable to issue 'shutdown' command to server '%s'. "
"The server appears to be configured with SSL but is not "
"currently running with SSL (SSL handshake failed). "
"Try running mongoctl with --ssl-off." % server.id)
can_stop_mongoly = False
elif "error" in status and "connection closed" in status["error"]:
log_info(
"Unable to issue 'shutdown' command to server '%s'. "
"The server appears to have reached max # of connections." %
server.id)
can_stop_mongoly = False
else:
log_info("Server '%s' is not running." % server.id)
return
pid = server.get_pid()
pid_disp = pid if pid else "[Cannot be determined]"
log_info("Stopping server '%s' (pid=%s)..." % (server.id, pid_disp))
# log server activity stop
server.log_server_activity("stop")
# TODO: Enable this again when stabilized
# step_down_if_needed(server, force)
if can_stop_mongoly:
log_verbose(" ... issuing db 'shutdown' command ... ")
shutdown_success = mongo_stop_server(server, pid, force=False)
if not can_stop_mongoly or not shutdown_success:
log_verbose(" ... taking more forceful measures ... ")
shutdown_success = \
prompt_or_force_stop_server(server, pid, force,
try_mongo_force=can_stop_mongoly)
if shutdown_success:
log_info("Server '%s' has stopped." % server.id)
else:
raise MongoctlException("Unable to stop server '%s'." % server.id)
def extract_archive(archive_name):
log_info("Extracting %s..." % archive_name)
if not which("tar"):
msg = ("Cannot extract archive.You need to have 'tar' command in your"
" path in order to proceed.")
raise MongoctlException(msg)
tar_cmd = ['tar', 'xvf', archive_name]
call_command(tar_cmd)
def get_forked_mongod_pid(parent_mongod):
output = parent_mongod.communicate()[0]
pid_re_expr = "forked process: ([0-9]+)"
pid_str_search = re.search(pid_re_expr, output)
if pid_str_search:
pid_str = pid_str_search.groups()[0]
return int(pid_str)
else:
raise MongoctlException("Could not start the server. Check output: "
"%s" % output)
def get_db(self,
dbname,
no_auth=False,
username=None,
password=None,
retry=True,
never_auth_with_admin=False):
conn = self.get_db_connection()
db = conn[dbname]
# If the DB doesn't need to be authenticated to (or at least yet)
# then don't authenticate. this piece of code is important for the case
# where you are connecting to the DB on local host where --auth is on
# but there are no admin users yet
if no_auth:
return db
if (not username and (not self.needs_to_auth(dbname))):
return db
if username:
self.set_login_user(dbname, username, password)
login_user = self.get_login_user(dbname)
is_system_user = (login_user
and login_user.get("username") == "__system")
# if there is no login user for this database then use admin db unless
# it was specified not to
# ALSO use admin if this is 'local' db for mongodb >= 2.6.0
if ((not never_auth_with_admin and not login_user
and dbname != "admin")
or (dbname == "local" and not is_system_user
and not users.server_supports_local_users(self))):
# if this passes then we are authed!
admin_db = self.get_db("admin", retry=retry)
return admin_db.connection[dbname]
# no retries on local db, so if we fail to auth to local we always
# attempt to use admin
retry = retry and dbname != "local"
auth_success = self.authenticate_db(db, dbname, retry=retry)
# If auth failed then give it a try by auth into admin db unless it
# was specified not to
if (not never_auth_with_admin and not auth_success
and dbname != "admin"):
admin_db = self.get_db("admin", retry=retry)
return admin_db.connection[dbname]
if auth_success:
return db
else:
raise MongoctlException("Failed to authenticate to %s db" % dbname)
def make_db_connection(self, address):
try:
return Connection(address,
socketTimeoutMS=CONN_TIMEOUT,
connectTimeoutMS=CONN_TIMEOUT)
except Exception, e:
log_exception(e)
error_msg = "Cannot connect to '%s'. Cause: %s" % \
(address, e)
raise MongoctlException(error_msg,cause=e)
def ensure_mongo_home_not_used(mongo_installation):
output = execute_command([
"ps",
"-eaf"
])
if mongo_installation in output:
msg = ("ERROR: Cannot uninstall '%s' because its currently being used. "
"Please terminate all running processes then try again." %
mongo_installation)
raise MongoctlException(msg)
def set_client_ssl_mode(mode):
allowed_modes = [ClientSslMode.DISABLED,
ClientSslMode.ALLOW,
ClientSslMode.REQUIRE,
ClientSslMode.PREFER]
if mode not in allowed_modes:
raise MongoctlException("Invalid ssl mode '%s'. Mush choose from %s" %
(mode, allowed_modes))
global CLIENT_SSL_MODE
CLIENT_SSL_MODE = mode
def validate_mongodb_install(install_dir):
log_info("Verifying mongodb installation %s" % install_dir)
mongod_exe = os.path.join(install_dir, "bin", "mongod")
cmd = [mongod_exe, "--version"]
try:
execute_command(cmd)
log_info("Validation passed!")
except CalledProcessError, cpe:
log_exception(cpe)
raise MongoctlException("MongoDB installation failed. Validation command %s failed with error: %s" %
(" ".join(cmd), cpe.output))
def configure_sharded_cluster_command(parsed_options):
cluster_id = parsed_options.cluster
cluster = repository.lookup_and_validate_cluster(cluster_id)
if not isinstance(cluster, ShardedCluster):
raise MongoctlException(
"Cluster '%s' is not a ShardedCluster cluster" % cluster.id)
if parsed_options.dryRun:
dry_run_configure_sharded_cluster(cluster)
else:
configure_sharded_cluster(cluster)
def add_shard_command(parsed_options):
shard_id = parsed_options.shardId
# determine if the shard is a replicaset cluster or a server
shard = repository.lookup_cluster(shard_id)
if not shard:
shard = repository.lookup_server(shard_id)
if not shard:
raise MongoctlException("Unknown shard '%s'" % shard_id)
sharded_cluster = repository.lookup_cluster_by_shard(shard)
if not sharded_cluster:
raise MongoctlException("'%s' is not a shard" % shard_id)
if parsed_options.dryRun:
dry_run_add_shard(shard, sharded_cluster)
else:
add_shard(shard, sharded_cluster)
def get_repl_lag(self, master_status):
"""
Given two 'members' elements from rs.status(),
return lag between their optimes (in secs).
"""
member_status = self.get_member_rs_status()
if not member_status:
raise MongoctlException("Unable to determine replicaset status for"
" member '%s'" % self.id)
return get_member_repl_lag(member_status, master_status)
def push_mongodb(repo_name,
mongodb_version,
mongodb_edition=None,
access_key=None,
secret_key=None):
"""
:param repo_name:
:param mongodb_version:
:param mongodb_edition:
:return:
"""
mongodb_edition = mongodb_edition or MongoDBEdition.COMMUNITY
repo = get_binary_repository(repo_name)
if access_key and isinstance(repo, S3MongoDBBinaryRepository):
repo.access_key = access_key
repo.secret_key = secret_key
repo.validate()
version_info = make_version_info(mongodb_version, mongodb_edition)
mongodb_install_dir = get_mongo_installation(version_info)
if not mongodb_install_dir:
raise MongoctlException("No mongodb installation found for '%s'" %
version_info)
mongodb_install_home = os.path.dirname(mongodb_install_dir)
target_archive_name = repo.get_archive_name(mongodb_version,
mongodb_edition)
target_archive_path = os.path.join(mongodb_install_home,
target_archive_name)
mongodb_install_dir_name = os.path.basename(mongodb_install_dir)
log_info("Taring MongoDB at '%s'" % mongodb_install_dir_name)
tar_exe = which("tar")
tar_cmd = [tar_exe, "-cvzf", target_archive_name, mongodb_install_dir_name]
call_command(tar_cmd, cwd=mongodb_install_home)
log_info("Uploading tar to repo")
repo.upload_file(mongodb_version, mongodb_edition, target_archive_path)
# cleanup
log_info("Cleanup")
try:
os.remove(target_archive_path)
except Exception, e:
log_error(str(e))
log_exception(e)
def get_validate_platform_spec(os_name, bits):
if os_name not in ["linux", "osx", "win32", "sunos5"]:
raise MongoctlException("Unsupported OS %s" % os_name)
if bits == "64":
return "%s-x86_64" % os_name
else:
if os_name == "linux":
return "linux-i686"
elif os_name in ["osx" , "win32"]:
return "%s-i386" % os_name
elif os_name == "sunos5":
return "i86pc"
def configure_cluster_command(parsed_options):
cluster_id = parsed_options.cluster
cluster = repository.lookup_and_validate_cluster(cluster_id)
if not isinstance(cluster, ReplicaSetCluster):
raise MongoctlException("Cluster '%s' is not a replicaset cluster" %
cluster.id)
force_primary_server_id = parsed_options.forcePrimaryServer
if parsed_options.dryRun:
dry_run_configure_cluster(
cluster, force_primary_server_id=force_primary_server_id)
else:
configure_cluster(cluster,
force_primary_server_id=force_primary_server_id)
