def read_json_string(path_or_url, validate_exists=True): path_or_url = to_full_config_path(path_or_url) # if the path is just filename then append config root # check if its a file if not is_url(path_or_url): if os.path.isfile(path_or_url): return open(path_or_url).read() elif validate_exists: raise MongoctlException("Config file %s does not exist." % path_or_url) else: return None # Then its url response = urllib.urlopen(path_or_url) if response.getcode() != 200: msg = ("Unable to open url '%s' (response code '%s')." % (path_or_url, response.getcode())) if validate_exists: raise MongoctlException(msg) else: log_verbose(msg) return None else: return response.read()
def lookup_and_validate_server(server_id): server = lookup_server(server_id) if server is None: raise MongoctlException("Cannot find configuration for a server " "with _id of '%s'." % server_id) validation_errors = validate_server(server) if len(validation_errors) > 0: raise MongoctlException( "Server '%s' configuration is not valid. Please fix errors below" " and try again.\n%s" % (server_id, "\n".join(validation_errors))) return server
def get_configured_clusters(): global __configured_clusters__, __commandline_clusters__ if __configured_clusters__ is None: __configured_clusters__ = {} cluster_documents = [] if has_file_repository(): file_repo_conf = config.get_file_repository_conf() clusters_path_or_url = file_repo_conf.get("clusters", DEFAULT_CLUSTERS_FILE) cluster_documents = config.read_config_json("clusters", clusters_path_or_url) if not isinstance(cluster_documents, list): raise MongoctlException("Cluster list in '%s' must be an array" % clusters_path_or_url) if __commandline_clusters__: cluster_documents.extend(__commandline_clusters__) for document in cluster_documents: cluster = new_cluster(document) __configured_clusters__[cluster.id] = cluster return __configured_clusters__
def get_configured_servers(): global __configured_servers__, __commandline_servers__ if __configured_servers__ is None: __configured_servers__ = {} server_documents = [] if has_file_repository(): file_repo_conf = config.get_file_repository_conf() servers_path_or_url = file_repo_conf.get("servers", DEFAULT_SERVERS_FILE) server_documents = config.read_config_json("servers", servers_path_or_url) if not isinstance(server_documents, list): raise MongoctlException("Server list in '%s' must be an array" % servers_path_or_url) if __commandline_servers__: server_documents.extend(__commandline_servers__) for document in server_documents: server = new_server(document) __configured_servers__[server.id] = server return __configured_servers__
def setup_server_admin_users(server): admin_users = server.get_admin_users() if server.is_auth(): admin_users = prepend_global_admin_user(admin_users, server) if admin_users is None or len(admin_users) < 1: log_verbose("No users configured for admin DB...") return 0 root_user_added = setup_root_admin_user(server, admin_users) if not root_user_added: log_verbose("Not seeding users for database 'admin'") return 0 log_verbose("Checking setup for other admin users...") count_new_users = 1 try: admin_db = server.get_db("admin") count_new_users += setup_db_users(server, admin_db, admin_users[1:]) return count_new_users except Exception, e: log_exception(e) raise MongoctlException( "Error while setting up admin users on server '%s'." "\n Cause: %s" % (server.id, e))
def get_download_url(self, mongodb_version, mongodb_edition): template_args = get_template_args(mongodb_version, mongodb_edition) platform_spec = template_args["platform_spec"] os_dist_name = template_args["os_dist_name"] os_dist_version_no_dots = template_args["os_dist_version_no_dots"] os_name = template_args["os_name"] version_info = make_version_info(mongodb_version) if mongodb_edition == MongoDBEdition.COMMUNITY: archive_name = "mongodb-%s-%s.tgz" % (platform_spec, mongodb_version) domain = "fastdl.mongodb.org" elif mongodb_edition == MongoDBEdition.ENTERPRISE: if version_info and version_info >= VERSION_2_6_1: domain = "downloads.10gen.com" rel_name = "enterprise" else: rel_name = "subscription" domain = "downloads.mongodb.com" archive_name = ("mongodb-%s-%s-%s%s-%s.tgz" % (platform_spec, rel_name, os_dist_name, os_dist_version_no_dots, mongodb_version)) else: raise MongoctlException("Unsupported edition '%s'" % mongodb_edition) return "http://%s/%s/%s" % (domain, os_name, archive_name)
def read_username(dbname): # If we are running in a noninteractive mode then fail if not is_interactive_mode(): msg = ("mongoctl needs username in order to proceed. Please pass the" " username using the -u option or run without --noninteractive") raise MongoctlException(msg) return read_input("Enter username for database '%s': " % dbname)
def read_password(message=''): if not is_interactive_mode(): msg = ("mongoctl needs password in order to proceed. Please pass the" " password using the -p option or run without --noninteractive") raise MongoctlException(msg) print >> sys.stderr, message return getpass.getpass()
def lookup_and_validate_cluster(cluster_id): cluster = lookup_cluster(cluster_id) if cluster is None: raise MongoctlException("Unknown cluster: %s" % cluster_id) validate_cluster(cluster) return cluster
def set_commandline_servers_and_clusters(servers_json_str, clusters_json_str): global __commandline_servers__, __commandline_clusters__ try: if servers_json_str: __commandline_servers__ = json_util.loads(servers_json_str) if clusters_json_str: __commandline_clusters__ = json_util.loads(clusters_json_str) except Exception, ex: log_exception(ex) raise MongoctlException("--servers/--clusters must be a valid json string: %s" % ex)
def read_input(message): # If we are running in a noninteractive mode then fail if not is_interactive_mode(): msg = ("Error while trying to prompt you for '%s'. Prompting is not " "allowed when running with --noninteractive mode. Please pass" " enough arguments to bypass prompting or run without " "--noninteractive" % message) raise MongoctlException(msg) print >> sys.stderr, message, return raw_input()
def new_server(server_doc): _type = server_doc.get("_type") if _type is None or _type in SERVER_TYPE_MAP: clazz = resolve_class( SERVER_TYPE_MAP.get(_type, MONGOD_SERVER_CLASS_NAME)) else: raise MongoctlException("Unknown server _type '%s' for server:\n%s" % (_type, document_pretty_string(server_doc))) return clazz(server_doc)
def new_cluster(cluster_doc): _type = cluster_doc.get("_type") if _type is None or _type == "ReplicaSetCluster": clazz = replicaset_cluster_type() elif _type == "ShardedCluster": clazz = sharded_cluster_type() else: raise MongoctlException("Unknown cluster _type '%s' for server:\n%s" % (_type, document_pretty_string(cluster_doc))) return clazz(cluster_doc)
def make_version_info(version_number, edition=None): if version_number is None: return None version_number = version_number.strip() version_number = version_number.replace("-pre-" , "-pre") version_info = MongoDBVersionInfo(version_number, edition=edition) # validate version string if not is_valid_version_info(version_info): raise MongoctlException("Invalid version '%s." % version_info) else: return version_info
def new_server(server_doc): _type = server_doc.get("_type") if _type is None or _type == "mongod": server_type = "mongoctl.objects.mongod.MongodServer" elif _type == "mongos": server_type = "mongoctl.objects.mongos.MongosServer" else: raise MongoctlException("Unknown server _type '%s' for server:\n%s" % (_type, document_pretty_string(server_doc))) clazz = resolve_class(server_type) return clazz(server_doc)
def get_download_url(self, mongodb_version, mongodb_edition): template_args = get_template_args(mongodb_version, mongodb_edition) platform_spec = template_args["platform_spec"] os_dist_name = template_args["os_dist_name"] os_dist_version_no_dots = template_args["os_dist_version_no_dots"] os_name = template_args["os_name"] version_info = make_version_info(mongodb_version) if mongodb_edition == MongoDBEdition.COMMUNITY: archive_name = "mongodb-%s-%s.tgz" % (platform_spec, mongodb_version) domain = "fastdl.mongodb.org" # community ssl from mongodb.org are supported only for version >= 3.0 elif mongodb_edition == MongoDBEdition.COMMUNITY_SSL: if version_info >= VERSION_3_0: # Platforms that lack OS dist data (eg OS X) apparently don't # need it dist_bits = "" # Ones that have it, need it injected, i.e. # 'mongodb-linux-x86_64-debian71-3.2.9.tgz if os_dist_name is not None: dist_bits = "-{0}{1}".format(os_dist_name, os_dist_version_no_dots) archive_name = "mongodb-{0}{1}-{2}.tgz".format( platform_spec, dist_bits, mongodb_version) domain = "fastdl.mongodb.org" else: return None elif mongodb_edition == MongoDBEdition.ENTERPRISE: if version_info and version_info >= VERSION_2_6_1: domain = "downloads.10gen.com" rel_name = "enterprise" else: rel_name = "subscription" domain = "downloads.mongodb.com" archive_name = ("mongodb-%s-%s-%s%s-%s.tgz" % (platform_spec, rel_name, os_dist_name, os_dist_version_no_dots, mongodb_version)) else: raise MongoctlException("Unsupported edition '%s'" % mongodb_edition) url = "http://%s/%s/%s" % (domain, os_name, archive_name) log_verbose( "Download url for MongoDB %s (%s edition) OS %s, dist '%s': %s" % (mongodb_version, mongodb_edition, os_name, os_dist_name, url)) return url
def validate_cluster(cluster): log_info("Validating cluster '%s'..." % cluster.id) errors = [] if isinstance(cluster, replicaset_cluster_type()): errors.extend(validate_replicaset_cluster(cluster)) elif isinstance(cluster, sharded_cluster_type()): errors.extend(validate_sharded_cluster(cluster)) if len(errors) > 0: raise MongoctlException("Cluster %s configuration is not valid. " "Please fix errors below and try again.\n%s" % (cluster.id, "\n".join(errors))) return cluster
def validate_openssl(): """ Validates OpenSSL to ensure it has TLS_FALLBACK_SCSV supported """ try: open_ssl_exe = which("openssl") if not open_ssl_exe: raise Exception("No openssl exe found in path") try: # execute a an invalid command to get output with available options # since openssl does not have a --help option unfortunately execute_command([open_ssl_exe, "s_client", "invalidDummyCommand"]) except subprocess.CalledProcessError as e: if "fallback_scsv" not in e.output: raise Exception("openssl does not support TLS_FALLBACK_SCSV") except Exception as e: raise MongoctlException("Unsupported OpenSSL. %s" % e)
def extract_archive(archive_name): log_info("Extracting %s..." % archive_name) if not which("tar"): msg = ("Cannot extract archive.You need to have 'tar' command in your" " path in order to proceed.") raise MongoctlException(msg) dir_name = archive_name.replace(".tgz", "").replace(".tar.gz", "") ensure_dir(dir_name) tar_cmd = [ 'tar', 'xvf', archive_name, "-C", dir_name, "--strip-components", "1" ] call_command(tar_cmd) return dir_name
def read_config_json(name, path_or_url): try: log_verbose("Reading %s configuration" " from '%s'..." % (name, path_or_url)) json_str = read_json_string(path_or_url) # minify the json/remove comments and sh*t json_str = minify_json.json_minify(json_str) json_val = json.loads(json_str, object_hook=json_util.object_hook) if not json_val and not isinstance(json_val, list): # b/c [] is not True raise MongoctlException("Unable to load %s " "config file: %s" % (name, path_or_url)) else: return json_val except MongoctlException, e: raise e
def download_url(url, destination=None, show_errors=False): destination = destination or os.getcwd() log_info("Downloading %s..." % url) if which("curl"): download_cmd = ['curl', '-O', '-L'] if show_errors: download_cmd.append('-Ss') elif which("wget"): download_cmd = ['wget'] else: msg = ("Cannot download file.You need to have 'curl' or 'wget" "' command in your path in order to proceed.") raise MongoctlException(msg) download_cmd.append(url) call_command(download_cmd, cwd=destination) file_name = url.split("/")[-1] return os.path.join(destination, file_name)
def setup_server_db_users(server, dbname, db_users): log_verbose("Checking if there are any users that needs to be added for " "database '%s'..." % dbname) if not should_seed_db_users(server, dbname): log_verbose("Not seeding users for database '%s'" % dbname) return 0 db = server.get_db(dbname) try: any_new_user_added = setup_db_users(server, db, db_users) if not any_new_user_added: log_verbose("No new users added for database '%s'" % dbname) return any_new_user_added except Exception, e: log_exception(e) raise MongoctlException( "Error while setting up users for '%s'" \ " database on server '%s'." "\n Cause: %s" % (dbname, server.id, e))
def setup_server_admin_users(server): if not should_seed_db_users(server, "admin"): log_verbose("Not seeding users for database 'admin'") return 0 admin_users = server.get_admin_users() if server.is_auth(): admin_users = prepend_global_admin_user(admin_users, server) if (admin_users is None or len(admin_users) < 1): log_verbose("No users configured for admin DB...") return 0 log_verbose("Checking setup for admin users...") count_new_users = 0 try: admin_db = server.get_db("admin") # potentially create the 1st admin user count_new_users += setup_db_users(server, admin_db, admin_users[0:1]) # the 1st-time init case: # BEFORE adding 1st admin user, auth. is not possible -- # only localhost cxn gets a magic pass. # AFTER adding 1st admin user, authentication is required; # so, to be sure we now have authenticated cxn, re-pull admin db: admin_db = server.get_db("admin") # create the rest of the users count_new_users += setup_db_users(server, admin_db, admin_users[1:]) return count_new_users except Exception, e: log_exception(e) raise MongoctlException( "Error while setting up admin users on server '%s'." "\n Cause: %s" % (server.id, e))
def download_file(self, mongodb_version, mongodb_edition, destination=None): destination or os.getcwd() url = self.get_download_url(mongodb_version, mongodb_edition) response = urllib.urlopen(url) if response.code == 404: raise FileNotInRepoError("File not found in repo") if response.getcode() != 200: msg = ( "Unable to download from url '%s' (response code '%s'). " "It could be that version '%s' you specified does not exist." " Please double check the version you provide" % (url, response.getcode(), mongodb_version)) raise MongoctlException(msg) return download_url(url, destination, show_errors=not is_interactive_mode())
for elem in addr_info: ip = elem[4] if ip not in ips: ips.append(ip) # TODO remove this temp hack that works around the case where # host X has more IPs than X.foo.com. if len(host.split(".")) == 3: try: ips.extend(get_host_ips(host.split(".")[0])) except Exception, ex: pass return ips except Exception, e: raise MongoctlException("Invalid host '%s'. Cause: %s" % (host, e)) ############################################################################### def resolve_class(kls): if kls == "dict": return dict try: parts = kls.split('.') module = ".".join(parts[:-1]) m = __import__(module) for comp in parts[1:]: m = getattr(m, comp) return m except Exception, e:
def do_main(args): init_mongoctl_signal_handler() header = """ ------------------------------------------------------------------------------------------- __ _ ___ ___ ___ ____ ____/ /_/ / / ' \/ _ \/ _ \/ _ `/ _ \/ __/ __/ / /_/_/_/\___/_//_/\_, /\___/\__/\__/_/ /___/ ------------------------------------------------------------------------------------------- """ # Parse options parser = get_mongoctl_cmd_parser() if len(args) < 1: print(header) parser.print_help() return # Parse the arguments and call the function of the selected cmd parsed_args = parser.parse_args(args) # turn on verbose if specified if namespace_get_property(parsed_args,"mongoctlVerbose"): turn_logging_verbose_on() # set interactive mode non_interactive = namespace_get_property(parsed_args,'noninteractive') non_interactive = False if non_interactive is None else non_interactive set_interactive_mode(not non_interactive) if not is_interactive_mode(): log_verbose("Running with noninteractive mode") # set global prompt value yes_all = parsed_args.yesToEverything no_all = parsed_args.noToEverything if yes_all and no_all: raise MongoctlException("Cannot have --yes and --no at the same time. " "Please choose either --yes or --no") elif yes_all: say_yes_to_everything() elif no_all: say_no_to_everything() # set the global SSL_OFF flag if parsed_args.clientSslMode: objects.server.set_client_ssl_mode(parsed_args.clientSslMode) # set the global USE_ALT_ADDRESS field if parsed_args.useAltAddress: use_alt_address = parsed_args.useAltAddress log_info("Using alternative address '%s'..." % use_alt_address) objects.server.USE_ALT_ADDRESS = use_alt_address # set conf root if specified if parsed_args.configRoot is not None: config.set_config_root(parsed_args.configRoot) elif os.getenv(CONF_ROOT_ENV_VAR) is not None: config.set_config_root(os.getenv(CONF_ROOT_ENV_VAR)) # set cmd arg servers/clusters if parsed_args.servers or parsed_args.clusters: repository.set_commandline_servers_and_clusters(parsed_args.servers, parsed_args.clusters) # get the function to call from the parser framework command_function = parsed_args.func # parse global login if present username = namespace_get_property(parsed_args, "username") password = namespace_get_property(parsed_args, "password") server_id = namespace_get_property(parsed_args, SERVER_ID_PARAM) parse_global_login_user_arg(username, password, server_id) if server_id is not None: # check if assumeLocal was specified assume_local = namespace_get_property(parsed_args,"assumeLocal") if assume_local: objects.server.assume_local_server(server_id) # execute command log_info("") return command_function(parsed_args)
def set_config_root(root_path): if not is_url(root_path) and not dir_exists(root_path): raise MongoctlException("Invalid config-root value: %s does not" " exist or is not a directory" % root_path) global __config_root__ __config_root__ = root_path
else: log_verbose( "Repository '%s' doesnt have this version. Skipping..." % repo.name) continue except FileNotInRepoError, e: log_verbose("No mongodb binary (version: '%s', edition: '%s' )" "found in repo '%s'" % (mongodb_version, mongodb_edition, repo.name)) else: log_verbose("Binary repository '%s' does not support edition '%s'." " Supported editions %s" % (repo.name, mongodb_edition, repo.supported_editions)) raise MongoctlException( "No mongodb binary (version: '%s', edition: '%s')" % (mongodb_version, mongodb_edition)) ############################################################################### # HELPERS ############################################################################### def _make_binary_repo(name, repo_config): repo_type = repo_config.get("_type") if repo_type == "s3": repo = S3MongoDBBinaryRepository() repo.bucket_name = repo_config["bucketName"] repo.access_key = repo_config["accessKey"] repo.secret_key = repo_config["secretKey"]
def get_binary_repository(name): for repo in get_registered_binary_repositories(): if repo.name == name: return repo raise MongoctlException("Unknown repository '%s'" % name)
def validate_repositories(): if ((not has_file_repository()) and (not has_db_repository())): raise MongoctlException("Invalid 'mongoctl.config': No fileRepository" " or databaseRepository configured. At least" " one repository has to be configured.")