def dump_command(parsed_options): # get and validate dump target target = parsed_options.target use_best_secondary = parsed_options.useBestSecondary #max_repl_lag = parsed_options.maxReplLag is_addr = is_db_address(target) is_path = is_dbpath(target) if is_addr and is_path: msg = ("Ambiguous target value '%s'. Your target matches both a dbpath" " and a db address. Use prefix 'file://', 'cluster://' or" " 'server://' to make it more specific" % target) raise MongoctlException(msg) elif not (is_addr or is_path): raise MongoctlException("Invalid target value '%s'. Target has to be" " a valid db address or dbpath." % target) dump_options = extract_mongo_dump_options(parsed_options) if is_addr: mongo_dump_db_address(target, username=parsed_options.username, password=parsed_options.password, use_best_secondary=use_best_secondary, max_repl_lag=None, dump_options=dump_options) else: dbpath = resolve_path(target) mongo_dump_db_path(dbpath, dump_options=dump_options)
def mongo_exe_version(mongo_exe): mongod_path = os.path.join(os.path.dirname(mongo_exe), "mongod") try: re_expr = "v?((([0-9]+)\.([0-9]+)\.([0-9]+))([^, ]*))" vers_spew = execute_command([mongod_path, "--version"]) # only take first line of spew vers_spew_line = vers_spew.split('\n')[0] vers_grep = re.findall(re_expr, vers_spew_line) help_spew = execute_command([mongod_path, "--help"]) full_version = vers_grep[-1][0] if "subscription" in vers_spew or "enterprise" in vers_spew: edition = MongoDBEdition.ENTERPRISE elif "SSL" in help_spew: edition = MongoDBEdition.COMMUNITY_SSL else: edition = MongoDBEdition.COMMUNITY result = make_version_info(full_version, edition=edition) if result is not None: return result else: raise MongoctlException("Cannot parse mongo version from the" " output of '%s --version'" % mongod_path) except Exception, e: log_exception(e) raise MongoctlException("Unable to get mongo version of '%s'." " Cause: %s" % (mongod_path, e))
def restore_command(parsed_options): # get and validate source/destination source = parsed_options.source destination = parsed_options.destination is_addr = is_db_address(destination) is_path = is_dbpath(destination) if is_addr and is_path: msg = ("Ambiguous destination value '%s'. Your destination matches" " both a dbpath and a db address. Use prefix 'file://'," " 'cluster://' or 'server://' to make it more specific" % destination) raise MongoctlException(msg) elif not (is_addr or is_path): raise MongoctlException( "Invalid destination value '%s'. Destination has to be" " a valid db address or dbpath." % destination) restore_options = extract_mongo_restore_options(parsed_options) if is_addr: mongo_restore_db_address(destination, source, username=parsed_options.username, password=parsed_options.password, restore_options=restore_options) else: dbpath = resolve_path(destination) mongo_restore_db_path(dbpath, source, restore_options=restore_options)
def remove_shard_command(parsed_options): shard_id = parsed_options.shardId # determine if the shard is a replicaset cluster or a server shard = repository.lookup_cluster(shard_id) if not shard: shard = repository.lookup_server(shard_id) if not shard: raise MongoctlException("Unknown shard '%s'" % shard_id) sharded_cluster = repository.lookup_cluster_by_shard(shard) if not sharded_cluster: raise MongoctlException("'%s' is not a shard" % shard_id) dest = getattr(parsed_options, "unshardedDataDestination") synchronized = getattr(parsed_options, "synchronized") if parsed_options.dryRun: dry_run_remove_shard(shard, sharded_cluster) else: sharded_cluster.remove_shard(shard, unsharded_data_dest_id=dest, synchronized=synchronized)
def do_start_server(server, options_override=None, rs_add=False, no_init=False, standalone=False): # ensure that the start was issued locally. Fail otherwise server.validate_local_op("start") log_info("Checking to see if server '%s' is already running" " before starting it..." % server.id) status = server.get_status() if status['connection']: log_info("Server '%s' is already running." % server.id) # always call post server start if the server is already started # the post server start steps should be idempotent _post_server_start(server, server.get_pid(), rs_add=rs_add, no_init=no_init, standalone=standalone) return elif "timedOut" in status: raise MongoctlException("Unable to start server: Server '%s' seems to" " be already started but is" " not responding (connection timeout)." " Or there might some server running on the" " same port %s" % (server.id, server.get_port())) # check if there is another process running on the same port elif "error" in status and ("closed" in status["error"] or "reset" in status["error"] or "ids don't match" in status["error"]): raise MongoctlException("Unable to start server: Either server '%s' is " "started but not responding or port %s is " "already in use." % (server.id, server.get_port())) elif "error" in status and "SSL handshake failed" in status["error"]: raise MongoctlException( "Unable to start server '%s'. " "The server appears to be configured with SSL but is not " "currently running with SSL (SSL handshake failed). " "Try running mongoctl with --ssl-off." % server.id) # do necessary work before starting the mongod process _pre_server_start(server, options_override=options_override) server.log_server_activity("start") server_pid = start_server_process(server, options_override, standalone=standalone) _post_server_start(server, server_pid, rs_add=rs_add, no_init=no_init, standalone=standalone) # Note: The following block has to be the last block # because server_process.communicate() will not return unless you # interrupt the server process which will kill mongoctl, so nothing after # this block will be executed. Almost never... if not server.is_fork(): communicate_to_child_process(server_pid)
def show_server_command(parsed_options): server = repository.lookup_server(parsed_options.server) if server is None: raise MongoctlException("Could not find server '%s'." % parsed_options.server) log_info("Configuration for server '%s':" % parsed_options.server) print server
def open_mongo_shell_to(db_address, username=None, password=None, shell_options=None, js_files=None): if is_mongo_uri(db_address): open_mongo_shell_to_uri(db_address, username, password, shell_options, js_files) return # db_address is an id string id_path = db_address.split("/") id = id_path[0] database = id_path[1] if len(id_path) == 2 else None server = repository.lookup_server(id) if server: open_mongo_shell_to_server(server, database, username, password, shell_options, js_files) return # Maybe cluster? cluster = repository.lookup_cluster(id) if cluster: open_mongo_shell_to_cluster(cluster, database, username, password, shell_options, js_files) return # Unknown destination raise MongoctlException("Unknown db address '%s'" % db_address)
def show_cluster_command(parsed_options): cluster = repository.lookup_cluster(parsed_options.cluster) if cluster is None: raise MongoctlException("Could not find cluster '%s'." % parsed_options.cluster) log_info("Configuration for cluster '%s':" % parsed_options.cluster) print cluster
def mongo_dump_db_address(db_address, username=None, password=None, use_best_secondary=False, max_repl_lag=None, dump_options=None): if is_mongo_uri(db_address): mongo_dump_uri(uri=db_address, username=username, password=password, use_best_secondary=use_best_secondary, dump_options=dump_options) return # db_address is an id string id_path = db_address.split("/") id = id_path[0] database = id_path[1] if len(id_path) == 2 else None server = repository.lookup_server(id) if server: mongo_dump_server(server, database=database, username=username, password=password, dump_options=dump_options) return else: cluster = repository.lookup_cluster(id) if cluster: mongo_dump_cluster(cluster, database=database, username=username, password=password, use_best_secondary=use_best_secondary, max_repl_lag=max_repl_lag, dump_options=dump_options) return # Unknown destination raise MongoctlException("Unknown db address '%s'" % db_address)
def server_started(): # check if the command failed if not is_pid_alive(mongod_pid): raise MongoctlException("Could not start the server. Please check" " the log file.") return server.is_online()
def validate_local_op(self, op): # If the server has been assumed to be local then skip validation if is_assumed_local_server(self.id): log_verbose("Skipping validation of server's '%s' address '%s' to be" " local because --assume-local is on" % (self.id, self.get_host_address())) return log_verbose("Validating server address: " "Ensuring that server '%s' address '%s' is local on this " "machine" % (self.id, self.get_host_address())) if not self.is_local(): log_verbose("Server address validation failed.") raise MongoctlException("Cannot %s server '%s' on this machine " "because server's address '%s' does not appear " "to be local to this machine. Pass the " "--assume-local option if you are sure that " "this server should be running on this " "machine." % (op, self.id, self.get_host_address())) else: log_verbose("Server address validation passed. " "Server '%s' address '%s' is local on this " "machine !" % (self.id, self.get_host_address()))
def get_validate_cluster(self): cluster = self.get_cluster() if not cluster: raise MongoctlException("No cluster found for server '%s'" % self.id) repository.validate_cluster(cluster) return cluster
def _pre_mongod_server_start(server, options_override=None): """ Does necessary work before starting a server 1- An efficiency step for arbiters running with --no-journal * there is a lock file ==> * server must not have exited cleanly from last run, and does not know how to auto-recover (as a journalled server would) * however: this is an arbiter, therefore * there is no need to repair data files in any way ==> * i can rm this lockfile and start my server """ lock_file_path = server.get_lock_file_path() no_journal = (server.get_cmd_option("nojournal") or (options_override and "nojournal" in options_override)) if (os.path.exists(lock_file_path) and server.is_arbiter_server() and no_journal): log_warning("WARNING: Detected a lock file ('%s') for your server '%s'" " ; since this server is an arbiter, there is no need for" " repair or other action. Deleting mongod.lock and" " proceeding..." % (lock_file_path, server.id)) try: os.remove(lock_file_path) except Exception, e: log_exception(e) raise MongoctlException("Error while trying to delete '%s'. " "Cause: %s" % (lock_file_path, e))
def uninstall_mongodb(version_number, edition=None): version_info = make_version_info(version_number, edition=edition) # validate version string if not is_valid_version_info(version_info): raise MongoctlException("Invalid version '%s'. Please provide a" " valid MongoDB version." % version_info) mongo_installation = get_mongo_installation(version_info) if mongo_installation is None: # no-op msg = ("Cannot find a MongoDB installation for version '%s'. Please" " use list-versions to see all possible versions " % version_info) log_info(msg) return log_info("Found MongoDB '%s' in '%s'" % (version_info, mongo_installation)) def rm_mongodb(): # make sure that the mongo installation to be removed does not have # any running processes ensure_mongo_home_not_used(mongo_installation) log_info("Deleting '%s'" % mongo_installation) shutil.rmtree(mongo_installation) log_info("MongoDB '%s' Uninstalled successfully!" % version_info) prompt_execute_task("Proceed uninstall?" , rm_mongodb)
def fetch_latest_stable_version(): response = urllib.urlopen(LATEST_VERSION_FILE_URL) if response.getcode() == 200: return response.read().strip() else: raise MongoctlException("Unable to fetch MongoDB latest stable version" " from '%s' (Response code %s)" % (LATEST_VERSION_FILE_URL, response.getcode()))
def get_environment_variables(self): env_vars = self.get_property('environmentVariables') or {} allowed = self.get_allowed_environment_variables() for v in env_vars.keys(): if v not in allowed: raise MongoctlException("Unknown environment variable '%s'" % v) return env_vars
def do_stop_server(server, force=False): # ensure that the stop was issued locally. Fail otherwise server.validate_local_op("stop") log_info("Checking to see if server '%s' is actually running before" " stopping it..." % server.id) # init local flags can_stop_mongoly = True shutdown_success = False status = server.get_status() if not status['connection']: if "timedOut" in status: log_info("Unable to issue 'shutdown' command to server '%s'. " "The server is not responding (connection timed out) " "although port %s is open, possibly for mongod." % (server.id, server.get_port())) can_stop_mongoly = False elif "error" in status and "SSL handshake failed" in status["error"]: log_info("Unable to issue 'shutdown' command to server '%s'. " "The server appears to be configured with SSL but is not " "currently running with SSL (SSL handshake failed). " "Try running mongoctl with --ssl-off." % server.id) can_stop_mongoly = False elif "error" in status and "connection closed" in status["error"]: log_info( "Unable to issue 'shutdown' command to server '%s'. " "The server appears to have reached max # of connections." % server.id) can_stop_mongoly = False else: log_info("Server '%s' is not running." % server.id) return pid = server.get_pid() pid_disp = pid if pid else "[Cannot be determined]" log_info("Stopping server '%s' (pid=%s)..." % (server.id, pid_disp)) # log server activity stop server.log_server_activity("stop") # TODO: Enable this again when stabilized # step_down_if_needed(server, force) if can_stop_mongoly: log_verbose(" ... issuing db 'shutdown' command ... ") shutdown_success = mongo_stop_server(server, pid, force=False) if not can_stop_mongoly or not shutdown_success: log_verbose(" ... taking more forceful measures ... ") shutdown_success = \ prompt_or_force_stop_server(server, pid, force, try_mongo_force=can_stop_mongoly) if shutdown_success: log_info("Server '%s' has stopped." % server.id) else: raise MongoctlException("Unable to stop server '%s'." % server.id)
def extract_archive(archive_name): log_info("Extracting %s..." % archive_name) if not which("tar"): msg = ("Cannot extract archive.You need to have 'tar' command in your" " path in order to proceed.") raise MongoctlException(msg) tar_cmd = ['tar', 'xvf', archive_name] call_command(tar_cmd)
def get_forked_mongod_pid(parent_mongod): output = parent_mongod.communicate()[0] pid_re_expr = "forked process: ([0-9]+)" pid_str_search = re.search(pid_re_expr, output) if pid_str_search: pid_str = pid_str_search.groups()[0] return int(pid_str) else: raise MongoctlException("Could not start the server. Check output: " "%s" % output)
def get_db(self, dbname, no_auth=False, username=None, password=None, retry=True, never_auth_with_admin=False): conn = self.get_db_connection() db = conn[dbname] # If the DB doesn't need to be authenticated to (or at least yet) # then don't authenticate. this piece of code is important for the case # where you are connecting to the DB on local host where --auth is on # but there are no admin users yet if no_auth: return db if (not username and (not self.needs_to_auth(dbname))): return db if username: self.set_login_user(dbname, username, password) login_user = self.get_login_user(dbname) is_system_user = (login_user and login_user.get("username") == "__system") # if there is no login user for this database then use admin db unless # it was specified not to # ALSO use admin if this is 'local' db for mongodb >= 2.6.0 if ((not never_auth_with_admin and not login_user and dbname != "admin") or (dbname == "local" and not is_system_user and not users.server_supports_local_users(self))): # if this passes then we are authed! admin_db = self.get_db("admin", retry=retry) return admin_db.connection[dbname] # no retries on local db, so if we fail to auth to local we always # attempt to use admin retry = retry and dbname != "local" auth_success = self.authenticate_db(db, dbname, retry=retry) # If auth failed then give it a try by auth into admin db unless it # was specified not to if (not never_auth_with_admin and not auth_success and dbname != "admin"): admin_db = self.get_db("admin", retry=retry) return admin_db.connection[dbname] if auth_success: return db else: raise MongoctlException("Failed to authenticate to %s db" % dbname)
def make_db_connection(self, address): try: return Connection(address, socketTimeoutMS=CONN_TIMEOUT, connectTimeoutMS=CONN_TIMEOUT) except Exception, e: log_exception(e) error_msg = "Cannot connect to '%s'. Cause: %s" % \ (address, e) raise MongoctlException(error_msg,cause=e)
def ensure_mongo_home_not_used(mongo_installation): output = execute_command([ "ps", "-eaf" ]) if mongo_installation in output: msg = ("ERROR: Cannot uninstall '%s' because its currently being used. " "Please terminate all running processes then try again." % mongo_installation) raise MongoctlException(msg)
def set_client_ssl_mode(mode): allowed_modes = [ClientSslMode.DISABLED, ClientSslMode.ALLOW, ClientSslMode.REQUIRE, ClientSslMode.PREFER] if mode not in allowed_modes: raise MongoctlException("Invalid ssl mode '%s'. Mush choose from %s" % (mode, allowed_modes)) global CLIENT_SSL_MODE CLIENT_SSL_MODE = mode
def validate_mongodb_install(install_dir): log_info("Verifying mongodb installation %s" % install_dir) mongod_exe = os.path.join(install_dir, "bin", "mongod") cmd = [mongod_exe, "--version"] try: execute_command(cmd) log_info("Validation passed!") except CalledProcessError, cpe: log_exception(cpe) raise MongoctlException("MongoDB installation failed. Validation command %s failed with error: %s" % (" ".join(cmd), cpe.output))
def configure_sharded_cluster_command(parsed_options): cluster_id = parsed_options.cluster cluster = repository.lookup_and_validate_cluster(cluster_id) if not isinstance(cluster, ShardedCluster): raise MongoctlException( "Cluster '%s' is not a ShardedCluster cluster" % cluster.id) if parsed_options.dryRun: dry_run_configure_sharded_cluster(cluster) else: configure_sharded_cluster(cluster)
def add_shard_command(parsed_options): shard_id = parsed_options.shardId # determine if the shard is a replicaset cluster or a server shard = repository.lookup_cluster(shard_id) if not shard: shard = repository.lookup_server(shard_id) if not shard: raise MongoctlException("Unknown shard '%s'" % shard_id) sharded_cluster = repository.lookup_cluster_by_shard(shard) if not sharded_cluster: raise MongoctlException("'%s' is not a shard" % shard_id) if parsed_options.dryRun: dry_run_add_shard(shard, sharded_cluster) else: add_shard(shard, sharded_cluster)
def get_repl_lag(self, master_status): """ Given two 'members' elements from rs.status(), return lag between their optimes (in secs). """ member_status = self.get_member_rs_status() if not member_status: raise MongoctlException("Unable to determine replicaset status for" " member '%s'" % self.id) return get_member_repl_lag(member_status, master_status)
def push_mongodb(repo_name, mongodb_version, mongodb_edition=None, access_key=None, secret_key=None): """ :param repo_name: :param mongodb_version: :param mongodb_edition: :return: """ mongodb_edition = mongodb_edition or MongoDBEdition.COMMUNITY repo = get_binary_repository(repo_name) if access_key and isinstance(repo, S3MongoDBBinaryRepository): repo.access_key = access_key repo.secret_key = secret_key repo.validate() version_info = make_version_info(mongodb_version, mongodb_edition) mongodb_install_dir = get_mongo_installation(version_info) if not mongodb_install_dir: raise MongoctlException("No mongodb installation found for '%s'" % version_info) mongodb_install_home = os.path.dirname(mongodb_install_dir) target_archive_name = repo.get_archive_name(mongodb_version, mongodb_edition) target_archive_path = os.path.join(mongodb_install_home, target_archive_name) mongodb_install_dir_name = os.path.basename(mongodb_install_dir) log_info("Taring MongoDB at '%s'" % mongodb_install_dir_name) tar_exe = which("tar") tar_cmd = [tar_exe, "-cvzf", target_archive_name, mongodb_install_dir_name] call_command(tar_cmd, cwd=mongodb_install_home) log_info("Uploading tar to repo") repo.upload_file(mongodb_version, mongodb_edition, target_archive_path) # cleanup log_info("Cleanup") try: os.remove(target_archive_path) except Exception, e: log_error(str(e)) log_exception(e)
def get_validate_platform_spec(os_name, bits): if os_name not in ["linux", "osx", "win32", "sunos5"]: raise MongoctlException("Unsupported OS %s" % os_name) if bits == "64": return "%s-x86_64" % os_name else: if os_name == "linux": return "linux-i686" elif os_name in ["osx" , "win32"]: return "%s-i386" % os_name elif os_name == "sunos5": return "i86pc"
def configure_cluster_command(parsed_options): cluster_id = parsed_options.cluster cluster = repository.lookup_and_validate_cluster(cluster_id) if not isinstance(cluster, ReplicaSetCluster): raise MongoctlException("Cluster '%s' is not a replicaset cluster" % cluster.id) force_primary_server_id = parsed_options.forcePrimaryServer if parsed_options.dryRun: dry_run_configure_cluster( cluster, force_primary_server_id=force_primary_server_id) else: configure_cluster(cluster, force_primary_server_id=force_primary_server_id)