def test_download_file_to_temp_path():
# Existent URL
temp_file = download_file_to_temp_path(source_url='https://www.google.com/robots.txt')
assert os.path.isfile(temp_file)
assert os.path.getsize(temp_file) > 0
# Nonexistent URL
with pytest.raises(McDownloadFileToTempPathException):
download_file_to_temp_path('https://mediacloud.org/should-not-exist.txt')
def test_download_file_to_temp_path():
# Existent URL
temp_file = download_file_to_temp_path(
source_url='https://www.google.com/robots.txt')
assert os.path.isfile(temp_file)
assert os.path.getsize(temp_file) > 0
# Nonexistent URL
with pytest.raises(McDownloadFileToTempPathException):
download_file_to_temp_path(
'https://mediacloud.org/should-not-exist.txt')
def __install_solr(dist_directory: str = MC_DIST_DIR,
solr_version: str = MC_SOLR_VERSION) -> None:
"""Install Solr to distribution directory; lock directory before installing and unlock afterwards."""
if __solr_is_installed(dist_directory=dist_directory,
solr_version=solr_version):
raise McSolrRunException(
"Solr %s is already installed in distribution directory '%s'." %
(solr_version, dist_directory))
solr_path = __solr_path(dist_directory=dist_directory,
solr_version=solr_version)
log.info("Creating Solr directory...")
mkdir_p(solr_path)
installing_file_path = __solr_installing_file_path(
dist_directory=dist_directory, solr_version=solr_version)
log.info("Locking Solr directory for installation...")
lock_file(installing_file_path, timeout=MC_INSTALL_TIMEOUT)
# Waited for concurrent installation to finish?
if __solr_is_installed(dist_directory=dist_directory,
solr_version=solr_version):
log.info(
"While waiting for Solr directory to unlock, Solr got installed to said directory."
)
return
solr_dist_url = __solr_dist_url(solr_version=solr_version)
log.info("Downloading Solr %s from %s..." % (solr_version, solr_dist_url))
solr_tarball_dest_path = download_file_to_temp_path(solr_dist_url)
log.info("Extracting %s to %s..." % (solr_tarball_dest_path, solr_path))
extract_tarball_to_directory(archive_file=solr_tarball_dest_path,
dest_directory=solr_path,
strip_root=True)
# Solr needs its .war extracted first before ZkCLI is usable
jetty_home_path = __jetty_home_path(dist_directory=dist_directory,
solr_version=solr_version)
solr_war_dest_dir = os.path.join(jetty_home_path, "solr-webapp", "webapp")
if not os.path.exists(os.path.join(solr_war_dest_dir, "index.html")):
raise McSolrRunException("Solr's .war is not extracted at path %s" %
solr_war_dest_dir)
log.info("Creating 'installed' file...")
installed_file_path = __solr_installed_file_path(
dist_directory=dist_directory, solr_version=solr_version)
lock_file(installed_file_path)
log.info("Removing lock file...")
unlock_file(installing_file_path)
if not __solr_is_installed(dist_directory=dist_directory,
solr_version=solr_version):
raise McSolrRunException(
"I've done everything but Solr is still not installed.")
def __install_zookeeper(dist_directory: str = MC_DIST_DIR,
zookeeper_version: str = MC_ZOOKEEPER_VERSION) -> None:
"""Install ZooKeeper to distribution directory; lock directory before installing and unlock afterwards."""
if __zookeeper_is_installed(dist_directory=dist_directory,
zookeeper_version=zookeeper_version):
raise McZooKeeperRunException(
"ZooKeeper %s is already installed in distribution directory '%s'."
% (zookeeper_version, dist_directory))
zookeeper_path = __zookeeper_path(dist_directory=dist_directory,
zookeeper_version=zookeeper_version)
log.info("Creating ZooKeeper directory...")
mkdir_p(zookeeper_path)
installing_file_path = __zookeeper_installing_file_path(
dist_directory=dist_directory, zookeeper_version=zookeeper_version)
log.info("Locking ZooKeeper directory for installation...")
lock_file(installing_file_path, timeout=MC_INSTALL_TIMEOUT)
# Waited for concurrent installation to finish?
if __zookeeper_is_installed(dist_directory=dist_directory,
zookeeper_version=zookeeper_version):
log.info(
"While waiting for ZooKeeper directory to unlock, ZooKeeper got installed to said directory."
)
return
zookeeper_dist_url = __zookeeper_dist_url(
zookeeper_version=zookeeper_version)
log.info("Downloading ZooKeeper %s from %s..." %
(zookeeper_version, zookeeper_dist_url))
zookeeper_tarball_dest_path = download_file_to_temp_path(
source_url=zookeeper_dist_url)
log.info("Extracting %s to %s..." %
(zookeeper_tarball_dest_path, zookeeper_path))
extract_tarball_to_directory(archive_file=zookeeper_tarball_dest_path,
dest_directory=zookeeper_path,
strip_root=True)
log.info("Creating 'installed' file...")
installed_file_path = __zookeeper_installed_file_path(
dist_directory=dist_directory, zookeeper_version=zookeeper_version)
lock_file(installed_file_path)
log.info("Removing lock file...")
unlock_file(installing_file_path)
if not __zookeeper_is_installed(dist_directory=dist_directory,
zookeeper_version=zookeeper_version):
raise McZooKeeperRunException(
"I've done everything but ZooKeeper is still not installed.")
def __install_solr(dist_directory: str = MC_DIST_DIR, solr_version: str = MC_SOLR_VERSION) -> None:
"""Install Solr to distribution directory; lock directory before installing and unlock afterwards."""
if __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version):
raise Exception("Solr %s is already installed in distribution directory '%s'." % (solr_version, dist_directory))
solr_path = __solr_path(dist_directory=dist_directory, solr_version=solr_version)
l.info("Creating Solr directory...")
mkdir_p(solr_path)
installing_file_path = __solr_installing_file_path(dist_directory=dist_directory, solr_version=solr_version)
l.info("Locking Solr directory for installation...")
lock_file(installing_file_path, timeout=MC_INSTALL_TIMEOUT)
# Waited for concurrent installation to finish?
if __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version):
l.info("While waiting for Solr directory to unlock, Solr got installed to said directory.")
return
solr_dist_url = __solr_dist_url(solr_version=solr_version)
l.info("Downloading Solr %s from %s..." % (solr_version, solr_dist_url))
solr_tarball_dest_path = download_file_to_temp_path(solr_dist_url)
l.info("Extracting %s to %s..." % (solr_tarball_dest_path, solr_path))
extract_tarball_to_directory(archive_file=solr_tarball_dest_path, dest_directory=solr_path, strip_root=True)
# Solr needs its .war extracted first before ZkCLI is usable
jetty_home_path = __jetty_home_path(dist_directory=dist_directory, solr_version=solr_version)
solr_war_dest_dir = os.path.join(jetty_home_path, "solr-webapp", "webapp")
# Solr 5.5.2+ already has the .war extracted
if not os.path.exists(os.path.join(solr_war_dest_dir, "index.html")):
solr_war_path = os.path.join(jetty_home_path, "webapps", "solr.war")
if not os.path.isfile(solr_war_path):
raise Exception("Solr's .war file does not exist at path %s" % solr_war_path)
solr_war_dest_dir = os.path.join(jetty_home_path, "solr-webapp", "webapp")
l.info("Extracting solr.war at '%s' to '%s'..." % (solr_war_path, solr_war_dest_dir))
mkdir_p(solr_war_dest_dir)
extract_zip_to_directory(archive_file=solr_war_path, dest_directory=solr_war_dest_dir)
l.info("Creating 'installed' file...")
installed_file_path = __solr_installed_file_path(dist_directory=dist_directory, solr_version=solr_version)
lock_file(installed_file_path)
l.info("Removing lock file...")
unlock_file(installing_file_path)
if not __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version):
raise Exception("I've done everything but Solr is still not installed.")
def __install_zookeeper(dist_directory: str = MC_DIST_DIR, zookeeper_version: str = MC_ZOOKEEPER_VERSION) -> None:
"""Install ZooKeeper to distribution directory; lock directory before installing and unlock afterwards."""
if __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version):
raise McZooKeeperRunException("ZooKeeper %s is already installed in distribution directory '%s'." % (
zookeeper_version, dist_directory
))
zookeeper_path = __zookeeper_path(dist_directory=dist_directory, zookeeper_version=zookeeper_version)
log.info("Creating ZooKeeper directory...")
mkdir_p(zookeeper_path)
installing_file_path = __zookeeper_installing_file_path(dist_directory=dist_directory,
zookeeper_version=zookeeper_version)
log.info("Locking ZooKeeper directory for installation...")
lock_file(installing_file_path, timeout=MC_INSTALL_TIMEOUT)
# Waited for concurrent installation to finish?
if __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version):
log.info("While waiting for ZooKeeper directory to unlock, ZooKeeper got installed to said directory.")
return
zookeeper_dist_url = __zookeeper_dist_url(zookeeper_version=zookeeper_version)
log.info("Downloading ZooKeeper %s from %s..." % (zookeeper_version, zookeeper_dist_url))
zookeeper_tarball_dest_path = download_file_to_temp_path(source_url=zookeeper_dist_url)
log.info("Extracting %s to %s..." % (zookeeper_tarball_dest_path, zookeeper_path))
extract_tarball_to_directory(archive_file=zookeeper_tarball_dest_path,
dest_directory=zookeeper_path,
strip_root=True)
log.info("Creating 'installed' file...")
installed_file_path = __zookeeper_installed_file_path(dist_directory=dist_directory,
zookeeper_version=zookeeper_version)
lock_file(installed_file_path)
log.info("Removing lock file...")
unlock_file(installing_file_path)
if not __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version):
raise McZooKeeperRunException("I've done everything but ZooKeeper is still not installed.")
