def extract_tarball_to_directory(archive_file: str, dest_directory: str, strip_root: bool = False) -> None:
"""Extract Tar archive (.tar, .tar.gz or .tgz) to destination directory, optionally stripping the root directory
first."""
archive_file = decode_object_from_bytes_if_needed(archive_file)
dest_directory = decode_object_from_bytes_if_needed(dest_directory)
if not os.path.isfile(archive_file):
raise McExtractTarballToDirectoryException("Archive at '%s' does not exist" % archive_file)
archive_file_extension = file_extension(archive_file)
if archive_file_extension in [".gz", ".tgz"]:
tar_args = "-zxf"
elif archive_file_extension in [".tar"]:
tar_args = "-xf"
else:
raise McExtractTarballToDirectoryException("Unsupported archive '%s' with extension '%s'" %
(archive_file, archive_file_extension))
args = ["tar",
tar_args, archive_file,
"-C", dest_directory]
if strip_root:
args += ['--strip', '1']
try:
run_command_in_foreground(args)
except McRunCommandInForegroundException as ex:
raise McExtractTarballToDirectoryException("Error while extracting archive '%s': %s" % (archive_file, str(ex)))
def __upgrade_lucene_index(instance_data_dir: str,
dist_directory: str = MC_DIST_DIR,
solr_version: str = MC_SOLR_VERSION):
"""Upgrade Solr (Lucene) index using the IndexUpgrader tool in a given instance directory."""
if not __solr_is_installed(dist_directory=dist_directory,
solr_version=solr_version):
l.info("Solr is not installed, installing...")
__install_solr(dist_directory=dist_directory,
solr_version=solr_version)
if not os.path.isdir(instance_data_dir):
raise McSolrRunException(
"Instance data directory '%s' does not exist." % instance_data_dir)
solr_path = __solr_path(dist_directory=dist_directory,
solr_version=solr_version)
lucene_lib_path = os.path.join(solr_path, "server", "solr-webapp",
"webapp", "WEB-INF", "lib")
if not os.path.isdir(lucene_lib_path):
raise McSolrRunException(
"Lucene library directory '%s' does not exist.")
lucene_core_jar = glob.glob(lucene_lib_path + "/lucene-core-*.jar")
if len(lucene_core_jar) != 1:
raise McSolrRunException("lucene-core JAR was not found in '%s'." %
lucene_lib_path)
lucene_core_jar = lucene_core_jar[0]
lucene_backward_codecs_jar = glob.glob(lucene_lib_path +
"/lucene-backward-codecs-*.jar")
if len(lucene_backward_codecs_jar) != 1:
raise McSolrRunException(
"lucene-backward-codecs JAR was not found in '%s'." %
lucene_lib_path)
lucene_backward_codecs_jar = lucene_backward_codecs_jar[0]
collections = __collections().keys()
for collection_name in collections:
collection_path = os.path.join(instance_data_dir, collection_name)
if not os.path.isdir(collection_path):
raise McSolrRunException(
"Collection data directory '%s' does not exist." %
collection_path)
index_path = os.path.join(collection_path, "data", "index")
if not os.path.isdir(index_path):
raise McSolrRunException("Index directory '%s' does not exist." %
index_path)
l.info("Upgrading index at path '%s'..." % index_path)
args = [
"java",
"-cp",
":".join([lucene_core_jar, lucene_backward_codecs_jar]),
"org.apache.lucene.index.IndexUpgrader",
"-verbose",
index_path,
]
run_command_in_foreground(args)
l.info("Upgraded index at path '%s'." % index_path)
def extract_tarball_to_directory(archive_file: str, dest_directory: str, strip_root: bool = False) -> None:
"""Extract Tar archive (.tar, .tar.gz or .tgz) to destination directory, optionally stripping the root directory
first."""
archive_file = decode_object_from_bytes_if_needed(archive_file)
dest_directory = decode_object_from_bytes_if_needed(dest_directory)
if not os.path.isfile(archive_file):
raise McExtractTarballToDirectoryException("Archive at '%s' does not exist" % archive_file)
archive_file_extension = file_extension(archive_file)
if archive_file_extension in [".gz", ".tgz"]:
tar_args = "-zxf"
elif archive_file_extension in [".tar"]:
tar_args = "-xf"
else:
raise McExtractTarballToDirectoryException("Unsupported archive '%s' with extension '%s'" %
(archive_file, archive_file_extension))
args = ["tar",
tar_args, archive_file,
"-C", dest_directory]
if strip_root:
args += ['--strip', '1']
try:
run_command_in_foreground(args)
except McRunCommandInForegroundException as ex:
raise McExtractTarballToDirectoryException("Error while extracting archive '%s': %s" % (archive_file, str(ex)))
def download_file(source_url: str, target_path: str) -> None:
"""Download URL to path."""
# FIXME reimplement using Python's "requests", don't use cURL
source_url = decode_object_from_bytes_if_needed(source_url)
target_path = decode_object_from_bytes_if_needed(target_path)
args = ["curl",
"--silent",
"--show-error",
"--fail",
"--retry", "3",
"--retry-delay", "5",
"--output", target_path,
source_url]
try:
run_command_in_foreground(args)
except McRunCommandInForegroundException as ex:
raise McDownloadFileException(
"Error while downloading file from '%(source_url)s' to '%(target_path)s': %(exception)s" % {
'source_url': source_url,
'target_path': target_path,
'exception': str(ex),
})
def __run_solr_zkcli(zkcli_args: List[str],
zookeeper_host: str = MC_SOLR_CLUSTER_ZOOKEEPER_HOST,
zookeeper_port: int = MC_SOLR_CLUSTER_ZOOKEEPER_PORT,
dist_directory: str = MC_DIST_DIR,
solr_version: str = MC_SOLR_VERSION) -> None:
"""Run Solr's zkcli.sh helper script."""
solr_path = __solr_path(dist_directory=dist_directory,
solr_version=solr_version)
jetty_home_path = __jetty_home_path(dist_directory=dist_directory,
solr_version=solr_version)
log4j_properties_path = None
log4j_properties_expected_paths = [
# Solr 4.6
os.path.join(jetty_home_path, "cloud-scripts", "log4j.properties"),
# Solr 4.10+
os.path.join(jetty_home_path, "scripts", "cloud-scripts",
"log4j.properties"),
]
for expected_path in log4j_properties_expected_paths:
if os.path.isfile(expected_path):
log4j_properties_path = expected_path
break
if log4j_properties_path is None:
raise McSolrRunException(
"Unable to find log4j.properties file for zkcli.sh script in paths: %s"
% str(log4j_properties_expected_paths))
if not tcp_port_is_open(hostname=zookeeper_host, port=zookeeper_port):
raise McSolrRunException("ZooKeeper is not running at %s:%d." %
(zookeeper_host, zookeeper_port))
jetty_home_path = __jetty_home_path(dist_directory=dist_directory,
solr_version=solr_version)
zkhost = "%s:%d" % (zookeeper_host, zookeeper_port)
java_classpath_dirs = [
# Solr 4
os.path.join(solr_path, "dist", "*"),
os.path.join(jetty_home_path, "solr-webapp", "webapp", "WEB-INF",
"lib", "*"),
os.path.join(jetty_home_path, "lib", "ext", "*"),
]
args = [
"java", "-classpath", ":".join(java_classpath_dirs),
"-Dlog4j.configuration=file://" +
os.path.abspath(log4j_properties_path), "org.apache.solr.cloud.ZkCLI",
"-zkhost", zkhost
] + zkcli_args
run_command_in_foreground(args)
def __run_solr_zkcli(
zkcli_args: List[str],
zookeeper_host: str = MC_SOLR_CLUSTER_ZOOKEEPER_HOST,
zookeeper_port: int = MC_SOLR_CLUSTER_ZOOKEEPER_PORT,
dist_directory: str = MC_DIST_DIR,
solr_version: str = MC_SOLR_VERSION,
) -> None:
"""Run Solr's zkcli.sh helper script."""
solr_path = __solr_path(dist_directory=dist_directory, solr_version=solr_version)
jetty_home_path = __jetty_home_path(dist_directory=dist_directory, solr_version=solr_version)
log4j_properties_path = None
log4j_properties_expected_paths = [
# Solr 4.6
os.path.join(jetty_home_path, "cloud-scripts", "log4j.properties"),
# Solr 4.10+
os.path.join(jetty_home_path, "scripts", "cloud-scripts", "log4j.properties"),
]
for expected_path in log4j_properties_expected_paths:
if os.path.isfile(expected_path):
log4j_properties_path = expected_path
break
if log4j_properties_path is None:
raise Exception(
"Unable to find log4j.properties file for zkcli.sh script in paths: %s"
% str(log4j_properties_expected_paths)
)
if not tcp_port_is_open(hostname=zookeeper_host, port=zookeeper_port):
raise Exception("ZooKeeper is not running at %s:%d." % (zookeeper_host, zookeeper_port))
jetty_home_path = __jetty_home_path(dist_directory=dist_directory, solr_version=solr_version)
zkhost = "%s:%d" % (zookeeper_host, zookeeper_port)
java_classpath_dirs = [
# Solr 4
os.path.join(solr_path, "dist", "*"),
os.path.join(jetty_home_path, "solr-webapp", "webapp", "WEB-INF", "lib", "*"),
os.path.join(jetty_home_path, "lib", "ext", "*"),
]
args = [
"java",
"-classpath",
":".join(java_classpath_dirs),
"-Dlog4j.configuration=file://" + os.path.abspath(log4j_properties_path),
"org.apache.solr.cloud.ZkCLI",
"-zkhost",
zkhost,
] + zkcli_args
run_command_in_foreground(args)
def __upgrade_lucene_index(
instance_data_dir: str, dist_directory: str = MC_DIST_DIR, solr_version: str = MC_SOLR_VERSION
):
"""Upgrade Solr (Lucene) index using the IndexUpgrader tool in a given instance directory."""
if not __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version):
l.info("Solr is not installed, installing...")
__install_solr(dist_directory=dist_directory, solr_version=solr_version)
if not os.path.isdir(instance_data_dir):
raise Exception("Instance data directory '%s' does not exist." % instance_data_dir)
solr_path = __solr_path(dist_directory=dist_directory, solr_version=solr_version)
lucene_lib_path = os.path.join(solr_path, "server", "solr-webapp", "webapp", "WEB-INF", "lib")
if not os.path.isdir(lucene_lib_path):
raise Exception("Lucene library directory '%s' does not exist.")
lucene_core_jar = glob.glob(lucene_lib_path + "/lucene-core-*.jar")
if len(lucene_core_jar) != 1:
raise Exception("lucene-core JAR was not found in '%s'." % lucene_lib_path)
lucene_core_jar = lucene_core_jar[0]
lucene_backward_codecs_jar = glob.glob(lucene_lib_path + "/lucene-backward-codecs-*.jar")
if len(lucene_backward_codecs_jar) != 1:
raise Exception("lucene-backward-codecs JAR was not found in '%s'." % lucene_lib_path)
lucene_backward_codecs_jar = lucene_backward_codecs_jar[0]
collections = __collections().keys()
for collection_name in collections:
collection_path = os.path.join(instance_data_dir, collection_name)
if not os.path.isdir(collection_path):
raise Exception("Collection data directory '%s' does not exist." % collection_path)
index_path = os.path.join(collection_path, "data", "index")
if not os.path.isdir(index_path):
raise Exception("Index directory '%s' does not exist." % index_path)
l.info("Upgrading index at path '%s'..." % index_path)
args = [
"java",
"-cp",
":".join([lucene_core_jar, lucene_backward_codecs_jar]),
"org.apache.lucene.index.IndexUpgrader",
"-verbose",
index_path,
]
run_command_in_foreground(args)
l.info("Upgraded index at path '%s'." % index_path)
def extract_zip_to_directory(archive_file: str, dest_directory: str) -> None:
"""Extract ZIP archive (.zip or .war) to destination directory."""
archive_file = decode_object_from_bytes_if_needed(archive_file)
dest_directory = decode_object_from_bytes_if_needed(dest_directory)
if not os.path.isfile(archive_file):
raise McExtractZipToDirectoryException("Archive at '%s' does not exist" % archive_file)
archive_file_extension = file_extension(archive_file)
if archive_file_extension not in [".zip", ".war"]:
raise McExtractZipToDirectoryException(
"Unsupported archive '%s' with extension '%s'" % (archive_file, archive_file_extension))
args = ["unzip", "-q", archive_file, "-d", dest_directory]
try:
run_command_in_foreground(args)
except McRunCommandInForegroundException as ex:
raise McExtractZipToDirectoryException("Error while extracting archive '%s': %s" % (archive_file, str(ex)))
def extract_zip_to_directory(archive_file: str, dest_directory: str) -> None:
"""Extract ZIP archive (.zip or .war) to destination directory."""
archive_file = decode_object_from_bytes_if_needed(archive_file)
dest_directory = decode_object_from_bytes_if_needed(dest_directory)
if not os.path.isfile(archive_file):
raise McExtractZipToDirectoryException("Archive at '%s' does not exist" % archive_file)
archive_file_extension = file_extension(archive_file)
if archive_file_extension not in [".zip", ".war"]:
raise McExtractZipToDirectoryException(
"Unsupported archive '%s' with extension '%s'" % (archive_file, archive_file_extension))
args = ["unzip", "-q", archive_file, "-d", dest_directory]
try:
run_command_in_foreground(args)
except McRunCommandInForegroundException as ex:
raise McExtractZipToDirectoryException("Error while extracting archive '%s': %s" % (archive_file, str(ex)))
def download_file(source_url: str, target_path: str) -> None:
"""Download URL to path."""
source_url = decode_string_from_bytes_if_needed(source_url)
target_path = decode_string_from_bytes_if_needed(target_path)
args = ["curl",
"--silent",
"--show-error",
"--fail",
"--retry", "3",
"--retry-delay", "5",
"--output", target_path,
source_url]
try:
run_command_in_foreground(args)
except McRunCommandInForegroundException as ex:
raise McDownloadFileException(
"Error while downloading file from '%(source_url)s' to '%(target_path)s': %(exception)s" % {
'source_url': source_url,
'target_path': target_path,
'exception': str(ex),
})
def test_run_command_in_foreground():
temp_dir = tempfile.mkdtemp()
test_file_to_create = os.path.join(temp_dir, 'test.txt')
assert os.path.isfile(test_file_to_create) is False
run_command_in_foreground(['touch', test_file_to_create])
assert os.path.isfile(test_file_to_create) is True
run_command_in_foreground(['rm', test_file_to_create])
assert os.path.isfile(test_file_to_create) is False
# Environment variables
test_env_variable = 'MC_RUN_COMMAND_IN_FOREGROUND_ENV_TEST'
test_file_with_env = os.path.join(temp_dir, 'env.txt')
test_file_without_env = os.path.join(temp_dir, 'no-env.txt')
run_command_in_foreground(['/bin/bash', '-c', 'env > %s' % test_file_with_env],
env={test_env_variable: '1'})
run_command_in_foreground(['/bin/bash', '-c', 'env > %s' % test_file_without_env], env={})
with open(test_file_with_env, 'r') as f:
contents = f.read()
assert test_env_variable in contents
with open(test_file_without_env, 'r') as f:
contents = f.read()
assert test_env_variable not in contents
# cwd
test_file_with_cwd = os.path.join(temp_dir, 'cwd.txt')
test_file_without_cwd = os.path.join(temp_dir, 'no-cwd.txt')
run_command_in_foreground(['/bin/bash', '-c', 'pwd > %s' % test_file_with_cwd], cwd=temp_dir)
run_command_in_foreground(['/bin/bash', '-c', 'pwd > %s' % test_file_without_cwd])
with open(test_file_with_cwd, 'r') as f:
contents = f.read()
assert temp_dir in contents
with open(test_file_without_cwd, 'r') as f:
contents = f.read()
assert temp_dir not in contents
# Faulty command
with pytest.raises(McRunCommandInForegroundException):
run_command_in_foreground(['this_command_totally_doesnt_exist'])
