def do_mongo_restore(source, host=None, port=None, dbpath=None, database=None, username=None, password=None, version_info=None, restore_options=None): # create restore command with host and port restore_cmd = [get_mongo_restore_executable(version_info)] if host: restore_cmd.extend(["--host", host]) if port: restore_cmd.extend(["--port", str(port)]) # dbpath if dbpath: restore_cmd.extend(["--dbpath", dbpath]) # database if database: restore_cmd.extend(["-d", database]) # username and password if username: restore_cmd.extend(["-u", username, "-p"]) if password: restore_cmd.append(password) # ignore authenticationDatabase option is version_info is less than 2.4.0 if (restore_options and "authenticationDatabase" in restore_options and version_info and version_info < make_version_info("2.4.0")): restore_options.pop("authenticationDatabase", None) # ignore restoreDbUsersAndRoles option is version_info is less than 2.6.0 if (restore_options and "restoreDbUsersAndRoles" in restore_options and version_info and version_info < make_version_info("2.6.0")): restore_options.pop("restoreDbUsersAndRoles", None) # append shell options if restore_options: restore_cmd.extend(options_to_command_args(restore_options)) # pass source arg restore_cmd.append(source) cmd_display = restore_cmd[:] # mask user/password if username: cmd_display[cmd_display.index("-u") + 1] = "****" if password: cmd_display[cmd_display.index("-p") + 1] = "****" # execute! log_info("Executing command: \n%s" % " ".join(cmd_display)) call_command(restore_cmd, bubble_exit_code=True)
def is_wired_tiger(self): version = self.get_mongo_version_info() storage_engine = self.get_cmd_option("storageEngine") if version >= make_version_info("3.2.0"): return not storage_engine or storage_engine == "wiredTiger" elif version >= make_version_info("3.0.0"): return storage_engine == "wiredTiger"
def is_wired_tiger(self): version = self.get_mongo_version_info() storage_engine = self.get_cmd_option("storageEngine") if version >= make_version_info("3.4.0"): #TODO XXX when we figure out 3.4.x is out. See SERVER-24303 raise Exception("is_wired_tiger() not supported >= 3.4.x yet.") elif version >= make_version_info("3.2.0"): return not storage_engine or storage_engine == "wiredTiger" elif version >= make_version_info("3.0.0"): return storage_engine == "wiredTiger"
def get_mongo_version_info(self): version_number = self.get_mongo_version() if version_number is not None: return make_version_info(version_number, edition=self.get_mongodb_edition()) else: return None
def uninstall_mongodb(version_number, edition=None): version_info = make_version_info(version_number, edition=edition) # validate version string if not is_valid_version_info(version_info): raise MongoctlException("Invalid version '%s'. Please provide a" " valid MongoDB version." % version_info) mongo_installation = get_mongo_installation(version_info) if mongo_installation is None: # no-op msg = ("Cannot find a MongoDB installation for version '%s'. Please" " use list-versions to see all possible versions " % version_info) log_info(msg) return log_info("Found MongoDB '%s' in '%s'" % (version_info, mongo_installation)) def rm_mongodb(): # make sure that the mongo installation to be removed does not have # any running processes ensure_mongo_home_not_used(mongo_installation) log_info("Deleting '%s'" % mongo_installation) shutil.rmtree(mongo_installation) log_info("MongoDB '%s' Uninstalled successfully!" % version_info) prompt_execute_task("Proceed uninstall?" , rm_mongodb)
def mongo_exe_version(mongo_exe): mongod_path = os.path.join(os.path.dirname(mongo_exe), "mongod") try: re_expr = "v?((([0-9]+)\.([0-9]+)\.([0-9]+))([^, ]*))" vers_spew = execute_command([mongod_path, "--version"]) # only take first line of spew vers_spew_line = vers_spew.split('\n')[0] vers_grep = re.findall(re_expr, vers_spew_line) help_spew = execute_command([mongod_path, "--help"]) full_version = vers_grep[-1][0] if "subscription" in vers_spew or "enterprise" in vers_spew: edition = MongoDBEdition.ENTERPRISE elif "SSL" in help_spew: edition = MongoDBEdition.COMMUNITY_SSL else: edition = MongoDBEdition.COMMUNITY result = make_version_info(full_version, edition=edition) if result is not None: return result else: raise MongoctlException("Cannot parse mongo version from the" " output of '%s --version'" % mongod_path) except Exception, e: log_exception(e) raise MongoctlException("Unable to get mongo version of '%s'." " Cause: %s" % (mongod_path, e))
def supports_repl_key(self): """ We need a repl key if you are auth + a cluster member + version is None or >= 2.0.0 """ version = self.get_mongo_version_info() return version is None or version >= make_version_info(REPL_KEY_SUPPORTED_VERSION)
def supports_repl_key(self): """ We need a repl key if you are auth + a cluster member + version is None or >= 2.0.0 """ version = self.get_mongo_version_info() return (version is None or version >= make_version_info(REPL_KEY_SUPPORTED_VERSION))
def push_mongodb(repo_name, mongodb_version, mongodb_edition=None, access_key=None, secret_key=None): """ :param repo_name: :param mongodb_version: :param mongodb_edition: :return: """ mongodb_edition = mongodb_edition or MongoDBEdition.COMMUNITY repo = get_binary_repository(repo_name) if access_key and isinstance(repo, S3MongoDBBinaryRepository): repo.access_key = access_key repo.secret_key = secret_key repo.validate() version_info = make_version_info(mongodb_version, mongodb_edition) mongodb_install_dir = get_mongo_installation(version_info) if not mongodb_install_dir: raise MongoctlException("No mongodb installation found for '%s'" % version_info) mongodb_install_home = os.path.dirname(mongodb_install_dir) target_archive_name = repo.get_archive_name(mongodb_version, mongodb_edition) target_archive_path = os.path.join(mongodb_install_home, target_archive_name) mongodb_install_dir_name = os.path.basename(mongodb_install_dir) log_info("Taring MongoDB at '%s'" % mongodb_install_dir_name) tar_exe = which("tar") tar_cmd = [tar_exe, "-cvzf", target_archive_name, mongodb_install_dir_name] call_command(tar_cmd, cwd=mongodb_install_home) log_info("Uploading tar to repo") repo.upload_file(mongodb_version, mongodb_edition, target_archive_path) # cleanup log_info("Cleanup") try: os.remove(target_archive_path) except Exception, e: log_error(str(e)) log_exception(e)
def get_mongo_restore_executable(version_info): version_check_pref = VersionPreference.EXACT_OR_MINOR if version_info and version_info.equals_ignore_edition(make_version_info("3.0.7")): log_info("Requiring version 3.0.6 instead of 3.0.7 b/c of mongorestore " "bug TOOLS-939") version_info = make_version_info("3.0.6", version_info.edition) version_check_pref = VersionPreference.EXACT restore_exe = get_mongo_executable(version_info, 'mongorestore', version_check_pref= version_check_pref) # Warn the user if it is not an exact match (minor match) if version_info and version_info != restore_exe.version: log_warning("Using mongorestore '%s' that does not exactly match" "server version '%s'" % (restore_exe.version, version_info)) return restore_exe.path
def get_mongo_restore_executable(version_info): version_check_pref = VersionPreference.EXACT_OR_MINOR if version_info and version_info.equals_ignore_edition( make_version_info("3.0.7")): log_info( "Requiring version 3.0.6 instead of 3.0.7 b/c of mongorestore " "bug TOOLS-939") version_info = make_version_info("3.0.6", version_info.edition) version_check_pref = VersionPreference.EXACT restore_exe = get_mongo_executable(version_info, 'mongorestore', version_check_pref=version_check_pref) # Warn the user if it is not an exact match (minor match) if version_info and version_info != restore_exe.version: log_warning("Using mongorestore '%s' that does not exactly match" "server version '%s'" % (restore_exe.version, version_info)) return restore_exe.path
def test_version_functions(self): self.assertTrue(make_version_info("1.2.0") < make_version_info("1.2.1")) self.assertFalse(make_version_info("1.2.0") < make_version_info("1.2.0")) self.assertFalse(make_version_info("1.2.0") < make_version_info("1.2.0C")) self.assertFalse(make_version_info("1.2.0-rc1") < make_version_info("1.2.0-rc0")) self.assertTrue(make_version_info("1.2.0-rc0") < make_version_info("1.2.0-rc1")) self.assertTrue(make_version_info("1.2.0-rc0") < make_version_info("1.2.0")) self.assertTrue(make_version_info("1.2.0-rc0") < make_version_info("1.2.1")) self.assertTrue(is_valid_version("1.0.1")) self.assertTrue(is_valid_version("0.1")) self.assertFalse(is_valid_version("1")) self.assertTrue(is_valid_version("0.1a")) self.assertTrue(is_valid_version("0.1c")) self.assertTrue(is_valid_version("1.8.9-rc0")) self.assertFalse(is_valid_version("a.1.2.3.4"))
def do_mongo_restore(source, host=None, port=None, dbpath=None, database=None, username=None, password=None, version_info=None, parsed_options=None, ssl=False): restore_options = extract_mongo_restore_options(parsed_options) # create restore command with host and port restore_cmd = [get_mongo_restore_executable(version_info)] # ssl options if ssl: restore_cmd.append("--ssl") if host: restore_cmd.extend(["--host", host]) if port: restore_cmd.extend(["--port", str(port)]) # dbpath if dbpath: restore_cmd.extend(["--dbpath", dbpath]) # database if database: restore_cmd.extend(["-d", database]) # username and password if username: restore_cmd.extend(["-u", username, "-p"]) if password: restore_cmd.append(password) # ignore authenticationDatabase option is version_info is less than 2.4.0 if (restore_options and "authenticationDatabase" in restore_options and version_info and version_info < make_version_info("2.4.0")): restore_options.pop("authenticationDatabase", None) # ignore restoreDbUsersAndRoles option is version_info is less than 2.6.0 if (restore_options and "restoreDbUsersAndRoles" in restore_options and version_info and version_info < make_version_info("2.6.0")): restore_options.pop("restoreDbUsersAndRoles", None) # for 3.0 default writeConcern to '{w:1}' unless overridden by restore_options # default stopOnError to true if version_info and version_info >= make_version_info("3.0.0"): if not restore_options or "writeConcern" not in restore_options: restore_cmd.extend(["--writeConcern", "{w:1}"]) if not parsed_options.continueOnError and "stopOnError" not in restore_options: restore_cmd.append("--stopOnError") # append shell options if restore_options: restore_cmd.extend(options_to_command_args(restore_options)) # pass source arg restore_cmd.append(source) cmd_display = restore_cmd[:] # mask user/password if username: cmd_display[cmd_display.index("-u") + 1] = "****" if password: cmd_display[cmd_display.index("-p") + 1] = "****" # execute! log_info("Executing command: \n%s" % " ".join(cmd_display)) call_command(restore_cmd, bubble_exit_code=True)
# This is mongodb's default port DEFAULT_PORT = 27017 REPL_KEY_SUPPORTED_VERSION = '2.0.0' # CLIENT_SSL_MODE global flag to for turning off ssl # TODO this is temporary and should be deleted CLIENT_SSL_MODE = None # A global config that is set through --use-alt-address option that will use # a different "address" property of when making connections to servers USE_ALT_ADDRESS = None ############################################################################### VERSION_2_6 = make_version_info("2.6.0") VERSION_3_0 = make_version_info("3.0.0") VERSION_3_2 = make_version_info("3.2.0") ############################################################################### class ClientSslMode(object): DISABLED = "disabled" ALLOW = "allow" REQUIRE = "require" PREFER = "prefer" ############################################################################### # Server Class ############################################################################### class Server(DocumentWrapper):
def install_mongodb(mongodb_version=None, mongodb_edition=None, from_source=False, build_threads=1, build_tmp_dir=None, include_only=None): if mongodb_version is None: mongodb_version = fetch_latest_stable_version() log_info("Installing latest stable MongoDB version '%s'..." % mongodb_version) version_info = make_version_info(mongodb_version, mongodb_edition) mongo_installation = get_mongo_installation(version_info) mongodb_edition = version_info.edition if mongo_installation is not None: # no-op log_info("You already have MongoDB %s installed ('%s'). " "Nothing to do." % (version_info, mongo_installation)) return mongo_installation target_dir = get_install_target_dir(mongodb_version, mongodb_edition) if os.path.exists(target_dir): raise MongoctlException("Target directory '%s' already exists" % target_dir) if mongodb_edition not in MongoDBEdition.ALL: raise MongoctlException("Unknown edition '%s'. Please select from %s" % (mongodb_edition, MongoDBEdition.ALL)) if from_source: install_from_source(mongodb_version, mongodb_edition, build_threads=build_threads, build_tmp_dir=build_tmp_dir) return bits = platform.architecture()[0].replace("bit", "") os_name = platform.system().lower() if os_name == 'darwin' and platform.mac_ver(): os_name = "osx" mongodb_installs_dir = config.get_mongodb_installs_dir() if not mongodb_installs_dir: raise MongoctlException("No mongoDBInstallationsDirectory configured" " in mongoctl.config") # ensure the mongo installs dir ensure_dir(mongodb_installs_dir) platform_spec = get_validate_platform_spec(os_name, bits) log_verbose("INSTALL_MONGODB: OS='%s' , BITS='%s' , VERSION='%s', " "PLATFORM_SPEC='%s'" % (os_name, bits, version_info, platform_spec)) # XXX LOOK OUT! Two processes installing same version simultaneously => BAD. # TODO: mutex to protect the following try: ## download the url archive_path = download_mongodb_binary(mongodb_version, mongodb_edition) archive_name = os.path.basename(archive_path) mongo_dir_name = extract_archive(archive_name) # apply include_only if specified if include_only: apply_include_only(mongo_dir_name, include_only) log_info("Deleting archive %s" % archive_name) os.remove(archive_name) target_dir_name = os.path.basename(target_dir) os.rename(mongo_dir_name, target_dir_name) # move target to mongodb install dir (Unless target is already there! # i.e current working dir == mongodb_installs_dir if os.getcwd() != mongodb_installs_dir: log_info("Moving extracted folder to %s" % mongodb_installs_dir) shutil.move(target_dir_name, mongodb_installs_dir) install_dir = os.path.join(mongodb_installs_dir, mongo_dir_name) # install validation validate_mongodb_install(target_dir) log_info("MongoDB %s installed successfully!" % version_info) return install_dir except Exception, e: log_exception(e) msg = "Failed to install MongoDB '%s'. Cause: %s" % (version_info, e) raise MongoctlException(msg)
def install_from_source(mongodb_version, mongodb_edition, build_threads=None, build_tmp_dir=None): """ :param version: :param ssl: :param repo_name: The repo to use to generate archive name :return: """ version_info = make_version_info(mongodb_version, mongodb_edition) if build_tmp_dir: ensure_dir(build_tmp_dir) os.chdir(build_tmp_dir) allowed_build_editions = [MongoDBEdition.COMMUNITY, MongoDBEdition.COMMUNITY_SSL] if mongodb_edition not in allowed_build_editions: raise MongoctlException("build is only allowed for %s editions" % allowed_build_editions) log_info("Installing MongoDB '%s %s' from source" % (mongodb_version, mongodb_edition)) source_archive_name = "r%s.tar.gz" % mongodb_version target_dir = get_install_target_dir(mongodb_version, mongodb_edition) source_url = ("https://github.com/mongodb/mongo/archive/%s" % source_archive_name) response = urllib.urlopen(source_url) if response.getcode() != 200: msg = ("Unable to find a mongodb release for version '%s' in MongoDB" " github repo. See https://github.com/mongodb/mongo/releases " "for possible releases (response code '%s'). " % (mongodb_version, response.getcode())) raise MongoctlException(msg) log_info("Downloading MongoDB '%s' source from github '%s' ..." % (mongodb_version, source_url)) download_url(source_url) log_info("Extract source archive ...") source_dir = extract_archive(source_archive_name) log_info("Building with scons!") scons_exe = which("scons") if not scons_exe: raise MongoctlException("scons command not found in your path") scons_cmd = [scons_exe, "core", "tools", "install"] if build_threads: scons_cmd.extend(["-j", str(build_threads)]) scons_cmd.append("--prefix=%s" % target_dir) if mongodb_edition == MongoDBEdition.COMMUNITY_SSL: validate_openssl() scons_cmd.append("--ssl") log_info("Running scons command: %s" % " ".join(scons_cmd)) call_command(scons_cmd, cwd=source_dir) # cleanup log_info("Cleanup") try: os.remove(source_archive_name) shutil.rmtree(source_dir) except Exception, e: log_error(str(e)) log_exception(e)
CONN_TIMEOUT = 10000 REPL_KEY_SUPPORTED_VERSION = '2.0.0' # CLIENT_SSL_MODE global flag to for turning off ssl # TODO this is temporary and should be deleted CLIENT_SSL_MODE = None # A global config that is set through --use-alt-address option that will use # a different "address" property of when making connections to servers USE_ALT_ADDRESS = None ############################################################################### VERSION_2_6 = make_version_info("2.6.0") VERSION_3_0 = make_version_info("3.0.0") ############################################################################### class ClientSslMode(object): DISABLED = "disabled" ALLOW = "allow" REQUIRE = "require" PREFER = "prefer" ############################################################################### # Server Class ############################################################################### class Server(DocumentWrapper):
def needs_tcmalloc_aggressive_decommit(self): version = self.get_mongo_version_info() return self.is_wired_tiger() and version < make_version_info("3.2.10") and system_memory_size_gbs() < 30
def needs_tcmalloc_aggressive_decommit(self): version = self.get_mongo_version_info() return self.is_wired_tiger() and version < make_version_info( "3.2.10") and system_memory_size_gbs() < 30