def read_json_string(path_or_url, validate_exists=True):
path_or_url = to_full_config_path(path_or_url)
# if the path is just filename then append config root
# check if its a file
if not is_url(path_or_url):
if os.path.isfile(path_or_url):
return open(path_or_url).read()
elif validate_exists:
raise MongoctlException("Config file %s does not exist." %
path_or_url)
else:
return None
# Then its url
response = urllib.urlopen(path_or_url)
if response.getcode() != 200:
msg = ("Unable to open url '%s' (response code '%s')." %
(path_or_url, response.getcode()))
if validate_exists:
raise MongoctlException(msg)
else:
log_verbose(msg)
return None
else:
return response.read()
def lookup_and_validate_server(server_id):
server = lookup_server(server_id)
if server is None:
raise MongoctlException("Cannot find configuration for a server "
"with _id of '%s'." % server_id)
validation_errors = validate_server(server)
if len(validation_errors) > 0:
raise MongoctlException(
"Server '%s' configuration is not valid. Please fix errors below"
" and try again.\n%s" % (server_id, "\n".join(validation_errors)))
return server
def get_configured_clusters():
global __configured_clusters__, __commandline_clusters__
if __configured_clusters__ is None:
__configured_clusters__ = {}
cluster_documents = []
if has_file_repository():
file_repo_conf = config.get_file_repository_conf()
clusters_path_or_url = file_repo_conf.get("clusters",
DEFAULT_CLUSTERS_FILE)
cluster_documents = config.read_config_json("clusters",
clusters_path_or_url)
if not isinstance(cluster_documents, list):
raise MongoctlException("Cluster list in '%s' must be an array" %
clusters_path_or_url)
if __commandline_clusters__:
cluster_documents.extend(__commandline_clusters__)
for document in cluster_documents:
cluster = new_cluster(document)
__configured_clusters__[cluster.id] = cluster
return __configured_clusters__
def get_configured_servers():
global __configured_servers__, __commandline_servers__
if __configured_servers__ is None:
__configured_servers__ = {}
server_documents = []
if has_file_repository():
file_repo_conf = config.get_file_repository_conf()
servers_path_or_url = file_repo_conf.get("servers",
DEFAULT_SERVERS_FILE)
server_documents = config.read_config_json("servers",
servers_path_or_url)
if not isinstance(server_documents, list):
raise MongoctlException("Server list in '%s' must be an array" %
servers_path_or_url)
if __commandline_servers__:
server_documents.extend(__commandline_servers__)
for document in server_documents:
server = new_server(document)
__configured_servers__[server.id] = server
return __configured_servers__
def setup_server_admin_users(server):
admin_users = server.get_admin_users()
if server.is_auth():
admin_users = prepend_global_admin_user(admin_users, server)
if admin_users is None or len(admin_users) < 1:
log_verbose("No users configured for admin DB...")
return 0
root_user_added = setup_root_admin_user(server, admin_users)
if not root_user_added:
log_verbose("Not seeding users for database 'admin'")
return 0
log_verbose("Checking setup for other admin users...")
count_new_users = 1
try:
admin_db = server.get_db("admin")
count_new_users += setup_db_users(server, admin_db, admin_users[1:])
return count_new_users
except Exception, e:
log_exception(e)
raise MongoctlException(
"Error while setting up admin users on server '%s'."
"\n Cause: %s" % (server.id, e))
def get_download_url(self, mongodb_version, mongodb_edition):
template_args = get_template_args(mongodb_version, mongodb_edition)
platform_spec = template_args["platform_spec"]
os_dist_name = template_args["os_dist_name"]
os_dist_version_no_dots = template_args["os_dist_version_no_dots"]
os_name = template_args["os_name"]
version_info = make_version_info(mongodb_version)
if mongodb_edition == MongoDBEdition.COMMUNITY:
archive_name = "mongodb-%s-%s.tgz" % (platform_spec,
mongodb_version)
domain = "fastdl.mongodb.org"
elif mongodb_edition == MongoDBEdition.ENTERPRISE:
if version_info and version_info >= VERSION_2_6_1:
domain = "downloads.10gen.com"
rel_name = "enterprise"
else:
rel_name = "subscription"
domain = "downloads.mongodb.com"
archive_name = ("mongodb-%s-%s-%s%s-%s.tgz" %
(platform_spec, rel_name, os_dist_name,
os_dist_version_no_dots, mongodb_version))
else:
raise MongoctlException("Unsupported edition '%s'" %
mongodb_edition)
return "http://%s/%s/%s" % (domain, os_name, archive_name)
def read_username(dbname):
# If we are running in a noninteractive mode then fail
if not is_interactive_mode():
msg = ("mongoctl needs username in order to proceed. Please pass the"
" username using the -u option or run without --noninteractive")
raise MongoctlException(msg)
return read_input("Enter username for database '%s': " % dbname)
def read_password(message=''):
if not is_interactive_mode():
msg = ("mongoctl needs password in order to proceed. Please pass the"
" password using the -p option or run without --noninteractive")
raise MongoctlException(msg)
print >> sys.stderr, message
return getpass.getpass()
def lookup_and_validate_cluster(cluster_id):
cluster = lookup_cluster(cluster_id)
if cluster is None:
raise MongoctlException("Unknown cluster: %s" % cluster_id)
validate_cluster(cluster)
return cluster
def set_commandline_servers_and_clusters(servers_json_str, clusters_json_str):
global __commandline_servers__, __commandline_clusters__
try:
if servers_json_str:
__commandline_servers__ = json_util.loads(servers_json_str)
if clusters_json_str:
__commandline_clusters__ = json_util.loads(clusters_json_str)
except Exception, ex:
log_exception(ex)
raise MongoctlException("--servers/--clusters must be a valid json string: %s" % ex)
def read_input(message):
# If we are running in a noninteractive mode then fail
if not is_interactive_mode():
msg = ("Error while trying to prompt you for '%s'. Prompting is not "
"allowed when running with --noninteractive mode. Please pass"
" enough arguments to bypass prompting or run without "
"--noninteractive" % message)
raise MongoctlException(msg)
print >> sys.stderr, message,
return raw_input()
def new_server(server_doc):
_type = server_doc.get("_type")
if _type is None or _type in SERVER_TYPE_MAP:
clazz = resolve_class(
SERVER_TYPE_MAP.get(_type, MONGOD_SERVER_CLASS_NAME))
else:
raise MongoctlException("Unknown server _type '%s' for server:\n%s" %
(_type, document_pretty_string(server_doc)))
return clazz(server_doc)
def new_cluster(cluster_doc):
_type = cluster_doc.get("_type")
if _type is None or _type == "ReplicaSetCluster":
clazz = replicaset_cluster_type()
elif _type == "ShardedCluster":
clazz = sharded_cluster_type()
else:
raise MongoctlException("Unknown cluster _type '%s' for server:\n%s" %
(_type, document_pretty_string(cluster_doc)))
return clazz(cluster_doc)
def make_version_info(version_number, edition=None):
if version_number is None:
return None
version_number = version_number.strip()
version_number = version_number.replace("-pre-" , "-pre")
version_info = MongoDBVersionInfo(version_number, edition=edition)
# validate version string
if not is_valid_version_info(version_info):
raise MongoctlException("Invalid version '%s." % version_info)
else:
return version_info
def new_server(server_doc):
_type = server_doc.get("_type")
if _type is None or _type == "mongod":
server_type = "mongoctl.objects.mongod.MongodServer"
elif _type == "mongos":
server_type = "mongoctl.objects.mongos.MongosServer"
else:
raise MongoctlException("Unknown server _type '%s' for server:\n%s" %
(_type, document_pretty_string(server_doc)))
clazz = resolve_class(server_type)
return clazz(server_doc)
def get_download_url(self, mongodb_version, mongodb_edition):
template_args = get_template_args(mongodb_version, mongodb_edition)
platform_spec = template_args["platform_spec"]
os_dist_name = template_args["os_dist_name"]
os_dist_version_no_dots = template_args["os_dist_version_no_dots"]
os_name = template_args["os_name"]
version_info = make_version_info(mongodb_version)
if mongodb_edition == MongoDBEdition.COMMUNITY:
archive_name = "mongodb-%s-%s.tgz" % (platform_spec,
mongodb_version)
domain = "fastdl.mongodb.org"
# community ssl from mongodb.org are supported only for version >= 3.0
elif mongodb_edition == MongoDBEdition.COMMUNITY_SSL:
if version_info >= VERSION_3_0:
# Platforms that lack OS dist data (eg OS X) apparently don't
# need it
dist_bits = ""
# Ones that have it, need it injected, i.e.
# 'mongodb-linux-x86_64-debian71-3.2.9.tgz
if os_dist_name is not None:
dist_bits = "-{0}{1}".format(os_dist_name,
os_dist_version_no_dots)
archive_name = "mongodb-{0}{1}-{2}.tgz".format(
platform_spec, dist_bits, mongodb_version)
domain = "fastdl.mongodb.org"
else:
return None
elif mongodb_edition == MongoDBEdition.ENTERPRISE:
if version_info and version_info >= VERSION_2_6_1:
domain = "downloads.10gen.com"
rel_name = "enterprise"
else:
rel_name = "subscription"
domain = "downloads.mongodb.com"
archive_name = ("mongodb-%s-%s-%s%s-%s.tgz" %
(platform_spec, rel_name, os_dist_name,
os_dist_version_no_dots, mongodb_version))
else:
raise MongoctlException("Unsupported edition '%s'" %
mongodb_edition)
url = "http://%s/%s/%s" % (domain, os_name, archive_name)
log_verbose(
"Download url for MongoDB %s (%s edition) OS %s, dist '%s': %s" %
(mongodb_version, mongodb_edition, os_name, os_dist_name, url))
return url
def validate_cluster(cluster):
log_info("Validating cluster '%s'..." % cluster.id)
errors = []
if isinstance(cluster, replicaset_cluster_type()):
errors.extend(validate_replicaset_cluster(cluster))
elif isinstance(cluster, sharded_cluster_type()):
errors.extend(validate_sharded_cluster(cluster))
if len(errors) > 0:
raise MongoctlException("Cluster %s configuration is not valid. "
"Please fix errors below and try again.\n%s" %
(cluster.id, "\n".join(errors)))
return cluster
def validate_openssl():
"""
Validates OpenSSL to ensure it has TLS_FALLBACK_SCSV supported
"""
try:
open_ssl_exe = which("openssl")
if not open_ssl_exe:
raise Exception("No openssl exe found in path")
try:
# execute a an invalid command to get output with available options
# since openssl does not have a --help option unfortunately
execute_command([open_ssl_exe, "s_client", "invalidDummyCommand"])
except subprocess.CalledProcessError as e:
if "fallback_scsv" not in e.output:
raise Exception("openssl does not support TLS_FALLBACK_SCSV")
except Exception as e:
raise MongoctlException("Unsupported OpenSSL. %s" % e)
def extract_archive(archive_name):
log_info("Extracting %s..." % archive_name)
if not which("tar"):
msg = ("Cannot extract archive.You need to have 'tar' command in your"
" path in order to proceed.")
raise MongoctlException(msg)
dir_name = archive_name.replace(".tgz", "").replace(".tar.gz", "")
ensure_dir(dir_name)
tar_cmd = [
'tar', 'xvf', archive_name, "-C", dir_name, "--strip-components", "1"
]
call_command(tar_cmd)
return dir_name
def read_config_json(name, path_or_url):
try:
log_verbose("Reading %s configuration"
" from '%s'..." % (name, path_or_url))
json_str = read_json_string(path_or_url)
# minify the json/remove comments and sh*t
json_str = minify_json.json_minify(json_str)
json_val = json.loads(json_str, object_hook=json_util.object_hook)
if not json_val and not isinstance(json_val,
list): # b/c [] is not True
raise MongoctlException("Unable to load %s "
"config file: %s" % (name, path_or_url))
else:
return json_val
except MongoctlException, e:
raise e
def download_url(url, destination=None, show_errors=False):
destination = destination or os.getcwd()
log_info("Downloading %s..." % url)
if which("curl"):
download_cmd = ['curl', '-O', '-L']
if show_errors:
download_cmd.append('-Ss')
elif which("wget"):
download_cmd = ['wget']
else:
msg = ("Cannot download file.You need to have 'curl' or 'wget"
"' command in your path in order to proceed.")
raise MongoctlException(msg)
download_cmd.append(url)
call_command(download_cmd, cwd=destination)
file_name = url.split("/")[-1]
return os.path.join(destination, file_name)
def setup_server_db_users(server, dbname, db_users):
log_verbose("Checking if there are any users that needs to be added for "
"database '%s'..." % dbname)
if not should_seed_db_users(server, dbname):
log_verbose("Not seeding users for database '%s'" % dbname)
return 0
db = server.get_db(dbname)
try:
any_new_user_added = setup_db_users(server, db, db_users)
if not any_new_user_added:
log_verbose("No new users added for database '%s'" % dbname)
return any_new_user_added
except Exception, e:
log_exception(e)
raise MongoctlException(
"Error while setting up users for '%s'" \
" database on server '%s'."
"\n Cause: %s" % (dbname, server.id, e))
def setup_server_admin_users(server):
if not should_seed_db_users(server, "admin"):
log_verbose("Not seeding users for database 'admin'")
return 0
admin_users = server.get_admin_users()
if server.is_auth():
admin_users = prepend_global_admin_user(admin_users, server)
if (admin_users is None or len(admin_users) < 1):
log_verbose("No users configured for admin DB...")
return 0
log_verbose("Checking setup for admin users...")
count_new_users = 0
try:
admin_db = server.get_db("admin")
# potentially create the 1st admin user
count_new_users += setup_db_users(server, admin_db, admin_users[0:1])
# the 1st-time init case:
# BEFORE adding 1st admin user, auth. is not possible --
# only localhost cxn gets a magic pass.
# AFTER adding 1st admin user, authentication is required;
# so, to be sure we now have authenticated cxn, re-pull admin db:
admin_db = server.get_db("admin")
# create the rest of the users
count_new_users += setup_db_users(server, admin_db, admin_users[1:])
return count_new_users
except Exception, e:
log_exception(e)
raise MongoctlException(
"Error while setting up admin users on server '%s'."
"\n Cause: %s" % (server.id, e))
def download_file(self,
mongodb_version,
mongodb_edition,
destination=None):
destination or os.getcwd()
url = self.get_download_url(mongodb_version, mongodb_edition)
response = urllib.urlopen(url)
if response.code == 404:
raise FileNotInRepoError("File not found in repo")
if response.getcode() != 200:
msg = (
"Unable to download from url '%s' (response code '%s'). "
"It could be that version '%s' you specified does not exist."
" Please double check the version you provide" %
(url, response.getcode(), mongodb_version))
raise MongoctlException(msg)
return download_url(url,
destination,
show_errors=not is_interactive_mode())
for elem in addr_info:
ip = elem[4]
if ip not in ips:
ips.append(ip)
# TODO remove this temp hack that works around the case where
# host X has more IPs than X.foo.com.
if len(host.split(".")) == 3:
try:
ips.extend(get_host_ips(host.split(".")[0]))
except Exception, ex:
pass
return ips
except Exception, e:
raise MongoctlException("Invalid host '%s'. Cause: %s" % (host, e))
###############################################################################
def resolve_class(kls):
if kls == "dict":
return dict
try:
parts = kls.split('.')
module = ".".join(parts[:-1])
m = __import__(module)
for comp in parts[1:]:
m = getattr(m, comp)
return m
except Exception, e:
def do_main(args):
init_mongoctl_signal_handler()
header = """
-------------------------------------------------------------------------------------------
__ _ ___ ___ ___ ____ ____/ /_/ /
/ ' \/ _ \/ _ \/ _ `/ _ \/ __/ __/ /
/_/_/_/\___/_//_/\_, /\___/\__/\__/_/
/___/
-------------------------------------------------------------------------------------------
"""
# Parse options
parser = get_mongoctl_cmd_parser()
if len(args) < 1:
print(header)
parser.print_help()
return
# Parse the arguments and call the function of the selected cmd
parsed_args = parser.parse_args(args)
# turn on verbose if specified
if namespace_get_property(parsed_args,"mongoctlVerbose"):
turn_logging_verbose_on()
# set interactive mode
non_interactive = namespace_get_property(parsed_args,'noninteractive')
non_interactive = False if non_interactive is None else non_interactive
set_interactive_mode(not non_interactive)
if not is_interactive_mode():
log_verbose("Running with noninteractive mode")
# set global prompt value
yes_all = parsed_args.yesToEverything
no_all = parsed_args.noToEverything
if yes_all and no_all:
raise MongoctlException("Cannot have --yes and --no at the same time. "
"Please choose either --yes or --no")
elif yes_all:
say_yes_to_everything()
elif no_all:
say_no_to_everything()
# set the global SSL_OFF flag
if parsed_args.clientSslMode:
objects.server.set_client_ssl_mode(parsed_args.clientSslMode)
# set the global USE_ALT_ADDRESS field
if parsed_args.useAltAddress:
use_alt_address = parsed_args.useAltAddress
log_info("Using alternative address '%s'..." % use_alt_address)
objects.server.USE_ALT_ADDRESS = use_alt_address
# set conf root if specified
if parsed_args.configRoot is not None:
config.set_config_root(parsed_args.configRoot)
elif os.getenv(CONF_ROOT_ENV_VAR) is not None:
config.set_config_root(os.getenv(CONF_ROOT_ENV_VAR))
# set cmd arg servers/clusters
if parsed_args.servers or parsed_args.clusters:
repository.set_commandline_servers_and_clusters(parsed_args.servers, parsed_args.clusters)
# get the function to call from the parser framework
command_function = parsed_args.func
# parse global login if present
username = namespace_get_property(parsed_args, "username")
password = namespace_get_property(parsed_args, "password")
server_id = namespace_get_property(parsed_args, SERVER_ID_PARAM)
parse_global_login_user_arg(username, password, server_id)
if server_id is not None:
# check if assumeLocal was specified
assume_local = namespace_get_property(parsed_args,"assumeLocal")
if assume_local:
objects.server.assume_local_server(server_id)
# execute command
log_info("")
return command_function(parsed_args)
def set_config_root(root_path):
if not is_url(root_path) and not dir_exists(root_path):
raise MongoctlException("Invalid config-root value: %s does not"
" exist or is not a directory" % root_path)
global __config_root__
__config_root__ = root_path
else:
log_verbose(
"Repository '%s' doesnt have this version. Skipping..."
% repo.name)
continue
except FileNotInRepoError, e:
log_verbose("No mongodb binary (version: '%s', edition: '%s' )"
"found in repo '%s'" %
(mongodb_version, mongodb_edition, repo.name))
else:
log_verbose("Binary repository '%s' does not support edition '%s'."
" Supported editions %s" %
(repo.name, mongodb_edition, repo.supported_editions))
raise MongoctlException(
"No mongodb binary (version: '%s', edition: '%s')" %
(mongodb_version, mongodb_edition))
###############################################################################
# HELPERS
###############################################################################
def _make_binary_repo(name, repo_config):
repo_type = repo_config.get("_type")
if repo_type == "s3":
repo = S3MongoDBBinaryRepository()
repo.bucket_name = repo_config["bucketName"]
repo.access_key = repo_config["accessKey"]
repo.secret_key = repo_config["secretKey"]
def get_binary_repository(name):
for repo in get_registered_binary_repositories():
if repo.name == name:
return repo
raise MongoctlException("Unknown repository '%s'" % name)
def validate_repositories():
if ((not has_file_repository()) and (not has_db_repository())):
raise MongoctlException("Invalid 'mongoctl.config': No fileRepository"
" or databaseRepository configured. At least"
" one repository has to be configured.")
