def generate_ssl_key_files(self):
ssl_dir = os.path.join(get_test_dir(), "ssl")
ensure_dir(ssl_dir)
ssl_cmd = [which("openssl"), "req", "-newkey","rsa:2048", "-new", "-x509", "-days", "1", "-nodes", "-out",
"test-mongodb-cert.crt", "-keyout", "test-mongodb-cert.key", "-subj",
"/C=US/ST=CA/L=SF/O=mongoctl/CN=test"
]
call_command(ssl_cmd, cwd=ssl_dir)
# create the .pem file
crt_file = os.path.join(ssl_dir, "test-mongodb-cert.crt")
key_file = os.path.join(ssl_dir, "test-mongodb-cert.key")
pem_file = os.path.join(ssl_dir, "test-mongodb.pem")
with open(pem_file, 'w') as outfile:
with open(crt_file) as infile1:
outfile.write(infile1.read())
with open(key_file) as infile2:
outfile.write(infile2.read())
for server_id in self.get_my_test_servers():
server = repository.lookup_server(server_id)
server.set_cmd_option("sslPEMKeyFile", pem_file)
server.set_cmd_option("sslCAFile", pem_file)
def do_mongo_restore(source,
host=None,
port=None,
dbpath=None,
database=None,
username=None,
password=None,
version_info=None,
restore_options=None):
# create restore command with host and port
restore_cmd = [get_mongo_restore_executable(version_info)]
if host:
restore_cmd.extend(["--host", host])
if port:
restore_cmd.extend(["--port", str(port)])
# dbpath
if dbpath:
restore_cmd.extend(["--dbpath", dbpath])
# database
if database:
restore_cmd.extend(["-d", database])
# username and password
if username:
restore_cmd.extend(["-u", username, "-p"])
if password:
restore_cmd.append(password)
# ignore authenticationDatabase option is version_info is less than 2.4.0
if (restore_options and "authenticationDatabase" in restore_options and
version_info and version_info < make_version_info("2.4.0")):
restore_options.pop("authenticationDatabase", None)
# ignore restoreDbUsersAndRoles option is version_info is less than 2.6.0
if (restore_options and "restoreDbUsersAndRoles" in restore_options and
version_info and version_info < make_version_info("2.6.0")):
restore_options.pop("restoreDbUsersAndRoles", None)
# append shell options
if restore_options:
restore_cmd.extend(options_to_command_args(restore_options))
# pass source arg
restore_cmd.append(source)
cmd_display = restore_cmd[:]
# mask user/password
if username:
cmd_display[cmd_display.index("-u") + 1] = "****"
if password:
cmd_display[cmd_display.index("-p") + 1] = "****"
# execute!
log_info("Executing command: \n%s" % " ".join(cmd_display))
call_command(restore_cmd, bubble_exit_code=True)
def do_mongo_restore(source,
host=None,
port=None,
dbpath=None,
database=None,
username=None,
password=None,
version_info=None,
restore_options=None):
# create restore command with host and port
restore_cmd = [get_mongo_restore_executable(version_info)]
if host:
restore_cmd.extend(["--host", host])
if port:
restore_cmd.extend(["--port", str(port)])
# dbpath
if dbpath:
restore_cmd.extend(["--dbpath", dbpath])
# database
if database:
restore_cmd.extend(["-d", database])
# username and password
if username:
restore_cmd.extend(["-u", username, "-p"])
if password:
restore_cmd.append(password)
# ignore authenticationDatabase option is version_info is less than 2.4.0
if (restore_options and "authenticationDatabase" in restore_options
and version_info and version_info < make_version_info("2.4.0")):
restore_options.pop("authenticationDatabase", None)
# ignore restoreDbUsersAndRoles option is version_info is less than 2.6.0
if (restore_options and "restoreDbUsersAndRoles" in restore_options
and version_info and version_info < make_version_info("2.6.0")):
restore_options.pop("restoreDbUsersAndRoles", None)
# append shell options
if restore_options:
restore_cmd.extend(options_to_command_args(restore_options))
# pass source arg
restore_cmd.append(source)
cmd_display = restore_cmd[:]
# mask user/password
if username:
cmd_display[cmd_display.index("-u") + 1] = "****"
if password:
cmd_display[cmd_display.index("-p") + 1] = "****"
# execute!
log_info("Executing command: \n%s" % " ".join(cmd_display))
call_command(restore_cmd, bubble_exit_code=True)
def do_mongo_dump(host=None,
port=None,
dbpath=None,
database=None,
username=None,
password=None,
version_info=None,
dump_options=None):
# create dump command with host and port
dump_cmd = [get_mongo_dump_executable(version_info)]
if host:
dump_cmd.extend(["--host", host])
if port:
dump_cmd.extend(["--port", str(port)])
# dbpath
if dbpath:
dump_cmd.extend(["--dbpath", dbpath])
# database
if database:
dump_cmd.extend(["-d", database])
# username and password
if username:
dump_cmd.extend(["-u", username, "-p"])
if password:
dump_cmd.append(password)
# ignore authenticationDatabase option is version_info is less than 2.4.0
if (dump_options and "authenticationDatabase" in dump_options and
version_info and version_info < VersionInfo("2.4.0")):
dump_options.pop("authenticationDatabase", None)
# ignore dumpDbUsersAndRoles option is version_info is less than 2.6.0
if (dump_options and "dumpDbUsersAndRoles" in dump_options and
version_info and version_info < VersionInfo("2.6.0")):
dump_options.pop("dumpDbUsersAndRoles", None)
# append shell options
if dump_options:
dump_cmd.extend(options_to_command_args(dump_options))
cmd_display = dump_cmd[:]
# mask user/password
if username:
cmd_display[cmd_display.index("-u") + 1] = "****"
if password:
cmd_display[cmd_display.index("-p") + 1] = "****"
log_info("Executing command: \n%s" % " ".join(cmd_display))
call_command(dump_cmd, bubble_exit_code=True)
def extract_archive(archive_name):
log_info("Extracting %s..." % archive_name)
if not which("tar"):
msg = ("Cannot extract archive.You need to have 'tar' command in your"
" path in order to proceed.")
raise MongoctlException(msg)
tar_cmd = ['tar', 'xvf', archive_name]
call_command(tar_cmd)
def extract_archive(archive_name):
log_info("Extracting %s..." % archive_name)
if not which("tar"):
msg = ("Cannot extract archive.You need to have 'tar' command in your"
" path in order to proceed.")
raise MongoctlException(msg)
tar_cmd = ['tar', 'xvf', archive_name]
call_command(tar_cmd)
def do_mongo_dump(host=None,
port=None,
dbpath=None,
database=None,
username=None,
password=None,
version_info=None,
dump_options=None):
# create dump command with host and port
dump_cmd = [get_mongo_dump_executable(version_info)]
if host:
dump_cmd.extend(["--host", host])
if port:
dump_cmd.extend(["--port", str(port)])
# dbpath
if dbpath:
dump_cmd.extend(["--dbpath", dbpath])
# database
if database:
dump_cmd.extend(["-d", database])
# username and password
if username:
dump_cmd.extend(["-u", username, "-p"])
if password:
dump_cmd.append(password)
# ignore authenticationDatabase option is version_info is less than 2.4.0
if (dump_options and "authenticationDatabase" in dump_options
and version_info and version_info < MongoDBVersionInfo("2.4.0")):
dump_options.pop("authenticationDatabase", None)
# ignore dumpDbUsersAndRoles option is version_info is less than 2.6.0
if (dump_options and "dumpDbUsersAndRoles" in dump_options and version_info
and version_info < MongoDBVersionInfo("2.6.0")):
dump_options.pop("dumpDbUsersAndRoles", None)
# append shell options
if dump_options:
dump_cmd.extend(options_to_command_args(dump_options))
cmd_display = dump_cmd[:]
# mask user/password
if username:
cmd_display[cmd_display.index("-u") + 1] = "****"
if password:
cmd_display[cmd_display.index("-p") + 1] = "****"
log_info("Executing command: \n%s" % " ".join(cmd_display))
call_command(dump_cmd, bubble_exit_code=True)
def push_mongodb(repo_name,
mongodb_version,
mongodb_edition=None,
access_key=None,
secret_key=None):
"""
:param repo_name:
:param mongodb_version:
:param mongodb_edition:
:return:
"""
mongodb_edition = mongodb_edition or MongoDBEdition.COMMUNITY
repo = get_binary_repository(repo_name)
if access_key and isinstance(repo, S3MongoDBBinaryRepository):
repo.access_key = access_key
repo.secret_key = secret_key
repo.validate()
version_info = make_version_info(mongodb_version, mongodb_edition)
mongodb_install_dir = get_mongo_installation(version_info)
if not mongodb_install_dir:
raise MongoctlException("No mongodb installation found for '%s'" %
version_info)
mongodb_install_home = os.path.dirname(mongodb_install_dir)
target_archive_name = repo.get_archive_name(mongodb_version,
mongodb_edition)
target_archive_path = os.path.join(mongodb_install_home,
target_archive_name)
mongodb_install_dir_name = os.path.basename(mongodb_install_dir)
log_info("Taring MongoDB at '%s'" % mongodb_install_dir_name)
tar_exe = which("tar")
tar_cmd = [tar_exe, "-cvzf", target_archive_name, mongodb_install_dir_name]
call_command(tar_cmd, cwd=mongodb_install_home)
log_info("Uploading tar to repo")
repo.upload_file(mongodb_version, mongodb_edition, target_archive_path)
# cleanup
log_info("Cleanup")
try:
os.remove(target_archive_path)
except Exception, e:
log_error(str(e))
log_exception(e)
def do_open_mongo_shell_to(address,
database=None,
username=None,
password=None,
server_version=None,
shell_options=None,
js_files=None,
ssl=False):
# default database to admin
database = database if database else "admin"
shell_options = shell_options or {}
js_files = js_files or []
# override port if specified in --port
if "port" in shell_options:
address = "%s:%s" % (address.split(":")[0], shell_options["port"])
# remove port from options since passing address + port is disallowed in mongo
del shell_options["port"]
connect_cmd = [
get_mongo_shell_executable(server_version),
"%s/%s" % (address, database)
]
if username:
connect_cmd.extend(["-u", username, "-p"])
if password:
connect_cmd.extend([password])
# append shell options
if shell_options:
connect_cmd.extend(options_to_command_args(shell_options))
# append js files
if js_files:
connect_cmd.extend(js_files)
# ssl options
if ssl and "--ssl" not in connect_cmd:
connect_cmd.append("--ssl")
cmd_display = connect_cmd[:]
# mask user/password
if username:
cmd_display[cmd_display.index("-u") + 1] = "****"
if password:
cmd_display[cmd_display.index("-p") + 1] = "****"
log_info("Executing command: \n%s" % " ".join(cmd_display))
call_command(connect_cmd, bubble_exit_code=True)
def push_mongodb(repo_name, mongodb_version, mongodb_edition=None,
access_key=None, secret_key=None):
"""
:param repo_name:
:param mongodb_version:
:param mongodb_edition:
:return:
"""
mongodb_edition = mongodb_edition or MongoDBEdition.COMMUNITY
repo = get_binary_repository(repo_name)
if access_key and isinstance(repo, S3MongoDBBinaryRepository):
repo.access_key = access_key
repo.secret_key = secret_key
repo.validate()
version_info = make_version_info(mongodb_version, mongodb_edition)
mongodb_install_dir = get_mongo_installation(version_info)
if not mongodb_install_dir:
raise MongoctlException("No mongodb installation found for '%s'" %
version_info)
mongodb_install_home = os.path.dirname(mongodb_install_dir)
target_archive_name = repo.get_archive_name(mongodb_version,
mongodb_edition)
target_archive_path = os.path.join(mongodb_install_home,
target_archive_name)
mongodb_install_dir_name = os.path.basename(mongodb_install_dir)
log_info("Taring MongoDB at '%s'" % mongodb_install_dir_name)
tar_exe = which("tar")
tar_cmd = [tar_exe, "-cvzf", target_archive_name, mongodb_install_dir_name]
call_command(tar_cmd, cwd=mongodb_install_home)
log_info("Uploading tar to repo")
repo.upload_file(mongodb_version, mongodb_edition, target_archive_path)
# cleanup
log_info("Cleanup")
try:
os.remove(target_archive_path)
except Exception, e:
log_error(str(e))
log_exception(e)
def do_open_mongo_shell_to(address,
database=None,
username=None,
password=None,
server_version=None,
shell_options=None,
js_files=None,
ssl=False):
# default database to admin
database = database if database else "admin"
shell_options = shell_options or {}
js_files = js_files or []
# override port if specified in --port
if "port" in shell_options:
address = "%s:%s" % (address.split(":")[0], shell_options["port"])
# remove port from options since passing address + port is disallowed in mongo
del shell_options["port"]
connect_cmd = [get_mongo_shell_executable(server_version),
"%s/%s" % (address, database)]
if username:
connect_cmd.extend(["-u",username, "-p"])
if password:
connect_cmd.extend([password])
# append shell options
if shell_options:
connect_cmd.extend(options_to_command_args(shell_options))
# append js files
if js_files:
connect_cmd.extend(js_files)
# ssl options
if ssl and "--ssl" not in connect_cmd:
connect_cmd.append("--ssl")
cmd_display = connect_cmd[:]
# mask user/password
if username:
cmd_display[cmd_display.index("-u") + 1] = "****"
if password:
cmd_display[cmd_display.index("-p") + 1] = "****"
log_info("Executing command: \n%s" % " ".join(cmd_display))
call_command(connect_cmd, bubble_exit_code=True)
def download(url):
log_info("Downloading %s..." % url)
if which("curl"):
download_cmd = ['curl', '-O']
if not is_interactive_mode():
download_cmd.append('-Ss')
elif which("wget"):
download_cmd = ['wget']
else:
msg = ("Cannot download file.You need to have 'curl' or 'wget"
"' command in your path in order to proceed.")
raise MongoctlException(msg)
download_cmd.append(url)
call_command(download_cmd)
def download(url):
log_info("Downloading %s..." % url)
if which("curl"):
download_cmd = ['curl', '-O']
if not is_interactive_mode():
download_cmd.append('-Ss')
elif which("wget"):
download_cmd = ['wget']
else:
msg = ("Cannot download file.You need to have 'curl' or 'wget"
"' command in your path in order to proceed.")
raise MongoctlException(msg)
download_cmd.append(url)
call_command(download_cmd)
def do_open_mongo_shell_to(address,
database=None,
username=None,
password=None,
server_version=None,
shell_options={},
js_files=[],
ssl=False):
# default database to admin
database = database if database else "admin"
connect_cmd = [
get_mongo_shell_executable(server_version),
"%s/%s" % (address, database)
]
if username:
connect_cmd.extend(["-u", username, "-p"])
if password:
connect_cmd.extend([password])
# append shell options
if shell_options:
connect_cmd.extend(options_to_command_args(shell_options))
# append js files
if js_files:
connect_cmd.extend(js_files)
# ssl options
if ssl:
connect_cmd.append("--ssl")
cmd_display = connect_cmd[:]
# mask user/password
if username:
cmd_display[cmd_display.index("-u") + 1] = "****"
if password:
cmd_display[cmd_display.index("-p") + 1] = "****"
log_info("Executing command: \n%s" % " ".join(cmd_display))
call_command(connect_cmd, bubble_exit_code=True)
def do_open_mongo_shell_to(address,
database=None,
username=None,
password=None,
server_version=None,
shell_options={},
js_files=[],
ssl=False):
# default database to admin
database = database if database else "admin"
connect_cmd = [get_mongo_shell_executable(server_version),
"%s/%s" % (address, database)]
if username:
connect_cmd.extend(["-u",username, "-p"])
if password:
connect_cmd.extend([password])
# append shell options
if shell_options:
connect_cmd.extend(options_to_command_args(shell_options))
# append js files
if js_files:
connect_cmd.extend(js_files)
# ssl options
if ssl:
connect_cmd.append("--ssl")
cmd_display = connect_cmd[:]
# mask user/password
if username:
cmd_display[cmd_display.index("-u") + 1] = "****"
if password:
cmd_display[cmd_display.index("-p") + 1] = "****"
log_info("Executing command: \n%s" % " ".join(cmd_display))
call_command(connect_cmd, bubble_exit_code=True)
def install_from_source(mongodb_version, mongodb_edition, build_threads=None,
build_tmp_dir=None):
"""
:param version:
:param ssl:
:param repo_name: The repo to use to generate archive name
:return:
"""
version_info = make_version_info(mongodb_version, mongodb_edition)
if build_tmp_dir:
ensure_dir(build_tmp_dir)
os.chdir(build_tmp_dir)
allowed_build_editions = [MongoDBEdition.COMMUNITY,
MongoDBEdition.COMMUNITY_SSL]
if mongodb_edition not in allowed_build_editions:
raise MongoctlException("build is only allowed for %s editions" %
allowed_build_editions)
log_info("Installing MongoDB '%s %s' from source" % (mongodb_version,
mongodb_edition))
source_archive_name = "r%s.tar.gz" % mongodb_version
target_dir = get_install_target_dir(mongodb_version, mongodb_edition)
source_url = ("https://github.com/mongodb/mongo/archive/%s" %
source_archive_name)
response = urllib.urlopen(source_url)
if response.getcode() != 200:
msg = ("Unable to find a mongodb release for version '%s' in MongoDB"
" github repo. See https://github.com/mongodb/mongo/releases "
"for possible releases (response code '%s'). " %
(mongodb_version, response.getcode()))
raise MongoctlException(msg)
log_info("Downloading MongoDB '%s' source from github '%s' ..." %
(mongodb_version, source_url))
download_url(source_url)
log_info("Extract source archive ...")
source_dir = extract_archive(source_archive_name)
log_info("Building with scons!")
scons_exe = which("scons")
if not scons_exe:
raise MongoctlException("scons command not found in your path")
scons_cmd = [scons_exe, "core", "tools", "install"]
if build_threads:
scons_cmd.extend(["-j", str(build_threads)])
scons_cmd.append("--prefix=%s" % target_dir)
if mongodb_edition == MongoDBEdition.COMMUNITY_SSL:
validate_openssl()
scons_cmd.append("--ssl")
log_info("Running scons command: %s" % " ".join(scons_cmd))
call_command(scons_cmd, cwd=source_dir)
# cleanup
log_info("Cleanup")
try:
os.remove(source_archive_name)
shutil.rmtree(source_dir)
except Exception, e:
log_error(str(e))
log_exception(e)
def do_mongo_restore(source,
host=None,
port=None,
dbpath=None,
database=None,
username=None,
password=None,
version_info=None,
parsed_options=None,
ssl=False):
restore_options = extract_mongo_restore_options(parsed_options)
# create restore command with host and port
restore_cmd = [get_mongo_restore_executable(version_info)]
# ssl options
if ssl:
restore_cmd.append("--ssl")
if host:
restore_cmd.extend(["--host", host])
if port:
restore_cmd.extend(["--port", str(port)])
# dbpath
if dbpath:
restore_cmd.extend(["--dbpath", dbpath])
# database
if database:
restore_cmd.extend(["-d", database])
# username and password
if username:
restore_cmd.extend(["-u", username, "-p"])
if password:
restore_cmd.append(password)
# ignore authenticationDatabase option is version_info is less than 2.4.0
if (restore_options and "authenticationDatabase" in restore_options
and version_info and version_info < make_version_info("2.4.0")):
restore_options.pop("authenticationDatabase", None)
# ignore restoreDbUsersAndRoles option is version_info is less than 2.6.0
if (restore_options and "restoreDbUsersAndRoles" in restore_options
and version_info and version_info < make_version_info("2.6.0")):
restore_options.pop("restoreDbUsersAndRoles", None)
# for 3.0 default writeConcern to '{w:1}' unless overridden by restore_options
# default stopOnError to true
if version_info and version_info >= make_version_info("3.0.0"):
if not restore_options or "writeConcern" not in restore_options:
restore_cmd.extend(["--writeConcern", "{w:1}"])
if not parsed_options.continueOnError and "stopOnError" not in restore_options:
restore_cmd.append("--stopOnError")
# append shell options
if restore_options:
restore_cmd.extend(options_to_command_args(restore_options))
# pass source arg
restore_cmd.append(source)
cmd_display = restore_cmd[:]
# mask user/password
if username:
cmd_display[cmd_display.index("-u") + 1] = "****"
if password:
cmd_display[cmd_display.index("-p") + 1] = "****"
# execute!
log_info("Executing command: \n%s" % " ".join(cmd_display))
call_command(restore_cmd, bubble_exit_code=True)
def do_mongo_restore(source,
host=None,
port=None,
dbpath=None,
database=None,
username=None,
password=None,
version_info=None,
parsed_options=None,
ssl=False):
restore_options = extract_mongo_restore_options(parsed_options)
# create restore command with host and port
restore_cmd = [get_mongo_restore_executable(version_info)]
# ssl options
if ssl:
restore_cmd.append("--ssl")
if host:
restore_cmd.extend(["--host", host])
if port:
restore_cmd.extend(["--port", str(port)])
# dbpath
if dbpath:
restore_cmd.extend(["--dbpath", dbpath])
# database
if database:
restore_cmd.extend(["-d", database])
# username and password
if username:
restore_cmd.extend(["-u", username, "-p"])
if password:
restore_cmd.append(password)
# ignore authenticationDatabase option is version_info is less than 2.4.0
if (restore_options and "authenticationDatabase" in restore_options and
version_info and version_info < make_version_info("2.4.0")):
restore_options.pop("authenticationDatabase", None)
# ignore restoreDbUsersAndRoles option is version_info is less than 2.6.0
if (restore_options and "restoreDbUsersAndRoles" in restore_options and
version_info and version_info < make_version_info("2.6.0")):
restore_options.pop("restoreDbUsersAndRoles", None)
# for 3.0 default writeConcern to '{w:1}' unless overridden by restore_options
# default stopOnError to true
if version_info and version_info >= make_version_info("3.0.0"):
if not restore_options or "writeConcern" not in restore_options:
restore_cmd.extend(["--writeConcern", "{w:1}"])
if not parsed_options.continueOnError and "stopOnError" not in restore_options:
restore_cmd.append("--stopOnError")
# append shell options
if restore_options:
restore_cmd.extend(options_to_command_args(restore_options))
# pass source arg
restore_cmd.append(source)
cmd_display = restore_cmd[:]
# mask user/password
if username:
cmd_display[cmd_display.index("-u") + 1] = "****"
if password:
cmd_display[cmd_display.index("-p") + 1] = "****"
# execute!
log_info("Executing command: \n%s" % " ".join(cmd_display))
call_command(restore_cmd, bubble_exit_code=True)
