def __install_solr(dist_directory: str = MC_DIST_DIR, solr_version: str = MC_SOLR_VERSION) -> None: """Install Solr to distribution directory; lock directory before installing and unlock afterwards.""" if __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version): raise McSolrRunException( "Solr %s is already installed in distribution directory '%s'." % (solr_version, dist_directory)) solr_path = __solr_path(dist_directory=dist_directory, solr_version=solr_version) log.info("Creating Solr directory...") mkdir_p(solr_path) installing_file_path = __solr_installing_file_path( dist_directory=dist_directory, solr_version=solr_version) log.info("Locking Solr directory for installation...") lock_file(installing_file_path, timeout=MC_INSTALL_TIMEOUT) # Waited for concurrent installation to finish? if __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version): log.info( "While waiting for Solr directory to unlock, Solr got installed to said directory." ) return solr_dist_url = __solr_dist_url(solr_version=solr_version) log.info("Downloading Solr %s from %s..." % (solr_version, solr_dist_url)) solr_tarball_dest_path = download_file_to_temp_path(solr_dist_url) log.info("Extracting %s to %s..." % (solr_tarball_dest_path, solr_path)) extract_tarball_to_directory(archive_file=solr_tarball_dest_path, dest_directory=solr_path, strip_root=True) # Solr needs its .war extracted first before ZkCLI is usable jetty_home_path = __jetty_home_path(dist_directory=dist_directory, solr_version=solr_version) solr_war_dest_dir = os.path.join(jetty_home_path, "solr-webapp", "webapp") if not os.path.exists(os.path.join(solr_war_dest_dir, "index.html")): raise McSolrRunException("Solr's .war is not extracted at path %s" % solr_war_dest_dir) log.info("Creating 'installed' file...") installed_file_path = __solr_installed_file_path( dist_directory=dist_directory, solr_version=solr_version) lock_file(installed_file_path) log.info("Removing lock file...") unlock_file(installing_file_path) if not __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version): raise McSolrRunException( "I've done everything but Solr is still not installed.")
def __install_zookeeper(dist_directory: str = MC_DIST_DIR, zookeeper_version: str = MC_ZOOKEEPER_VERSION) -> None: """Install ZooKeeper to distribution directory; lock directory before installing and unlock afterwards.""" if __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version): raise McZooKeeperRunException( "ZooKeeper %s is already installed in distribution directory '%s'." % (zookeeper_version, dist_directory)) zookeeper_path = __zookeeper_path(dist_directory=dist_directory, zookeeper_version=zookeeper_version) log.info("Creating ZooKeeper directory...") mkdir_p(zookeeper_path) installing_file_path = __zookeeper_installing_file_path( dist_directory=dist_directory, zookeeper_version=zookeeper_version) log.info("Locking ZooKeeper directory for installation...") lock_file(installing_file_path, timeout=MC_INSTALL_TIMEOUT) # Waited for concurrent installation to finish? if __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version): log.info( "While waiting for ZooKeeper directory to unlock, ZooKeeper got installed to said directory." ) return zookeeper_dist_url = __zookeeper_dist_url( zookeeper_version=zookeeper_version) log.info("Downloading ZooKeeper %s from %s..." % (zookeeper_version, zookeeper_dist_url)) zookeeper_tarball_dest_path = download_file_to_temp_path( source_url=zookeeper_dist_url) log.info("Extracting %s to %s..." % (zookeeper_tarball_dest_path, zookeeper_path)) extract_tarball_to_directory(archive_file=zookeeper_tarball_dest_path, dest_directory=zookeeper_path, strip_root=True) log.info("Creating 'installed' file...") installed_file_path = __zookeeper_installed_file_path( dist_directory=dist_directory, zookeeper_version=zookeeper_version) lock_file(installed_file_path) log.info("Removing lock file...") unlock_file(installing_file_path) if not __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version): raise McZooKeeperRunException( "I've done everything but ZooKeeper is still not installed.")
def __install_solr(dist_directory: str = MC_DIST_DIR, solr_version: str = MC_SOLR_VERSION) -> None: """Install Solr to distribution directory; lock directory before installing and unlock afterwards.""" if __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version): raise Exception("Solr %s is already installed in distribution directory '%s'." % (solr_version, dist_directory)) solr_path = __solr_path(dist_directory=dist_directory, solr_version=solr_version) l.info("Creating Solr directory...") mkdir_p(solr_path) installing_file_path = __solr_installing_file_path(dist_directory=dist_directory, solr_version=solr_version) l.info("Locking Solr directory for installation...") lock_file(installing_file_path, timeout=MC_INSTALL_TIMEOUT) # Waited for concurrent installation to finish? if __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version): l.info("While waiting for Solr directory to unlock, Solr got installed to said directory.") return solr_dist_url = __solr_dist_url(solr_version=solr_version) l.info("Downloading Solr %s from %s..." % (solr_version, solr_dist_url)) solr_tarball_dest_path = download_file_to_temp_path(solr_dist_url) l.info("Extracting %s to %s..." % (solr_tarball_dest_path, solr_path)) extract_tarball_to_directory(archive_file=solr_tarball_dest_path, dest_directory=solr_path, strip_root=True) # Solr needs its .war extracted first before ZkCLI is usable jetty_home_path = __jetty_home_path(dist_directory=dist_directory, solr_version=solr_version) solr_war_dest_dir = os.path.join(jetty_home_path, "solr-webapp", "webapp") # Solr 5.5.2+ already has the .war extracted if not os.path.exists(os.path.join(solr_war_dest_dir, "index.html")): solr_war_path = os.path.join(jetty_home_path, "webapps", "solr.war") if not os.path.isfile(solr_war_path): raise Exception("Solr's .war file does not exist at path %s" % solr_war_path) solr_war_dest_dir = os.path.join(jetty_home_path, "solr-webapp", "webapp") l.info("Extracting solr.war at '%s' to '%s'..." % (solr_war_path, solr_war_dest_dir)) mkdir_p(solr_war_dest_dir) extract_zip_to_directory(archive_file=solr_war_path, dest_directory=solr_war_dest_dir) l.info("Creating 'installed' file...") installed_file_path = __solr_installed_file_path(dist_directory=dist_directory, solr_version=solr_version) lock_file(installed_file_path) l.info("Removing lock file...") unlock_file(installing_file_path) if not __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version): raise Exception("I've done everything but Solr is still not installed.")
def test_extract_tarball_to_directory(): src_temp_dir = tempfile.mkdtemp() dst_temp_dir = tempfile.mkdtemp() # Nonexistent archive with pytest.raises(McExtractTarballToDirectoryException): extract_tarball_to_directory( os.path.join(src_temp_dir, 'nonexistent-archive.tgz'), dst_temp_dir) # Unsupported archive unsupported_archive_path = os.path.join(src_temp_dir, 'unsupported-archive.tar.bz2') run_command_in_foreground(['touch', unsupported_archive_path]) with pytest.raises(McExtractTarballToDirectoryException): extract_tarball_to_directory(unsupported_archive_path, dst_temp_dir) # Faulty archive faulty_archive_path = os.path.join(src_temp_dir, 'faulty-archive.tgz') with open(faulty_archive_path, 'w') as fh: fh.write('Totally not valid Gzip data.') with pytest.raises(McExtractTarballToDirectoryException): extract_tarball_to_directory(faulty_archive_path, dst_temp_dir) # .tar.gz archive tar_archive_path = os.path.join(src_temp_dir, 'tar-gz-archive.tar.gz') tar_archive_contents_dir = os.path.join(src_temp_dir, 'test-contents') os.mkdir(tar_archive_contents_dir) with open(os.path.join(tar_archive_contents_dir, 'test.txt'), 'w') as fh: fh.write('Test contents') with tarfile.open(tar_archive_path, "w:gz") as tar: tar.add(tar_archive_contents_dir, arcname=os.path.basename(tar_archive_contents_dir)) extract_tarball_to_directory(archive_file=tar_archive_path, dest_directory=dst_temp_dir) assert os.path.isdir(os.path.join(dst_temp_dir, 'test-contents')) assert os.path.isfile( os.path.join(dst_temp_dir, 'test-contents', 'test.txt')) # Strip root dst_strip_root_temp_dir = tempfile.mkdtemp() extract_tarball_to_directory(archive_file=tar_archive_path, dest_directory=dst_strip_root_temp_dir, strip_root=True) assert os.path.isdir(os.path.join(dst_strip_root_temp_dir, 'test-contents')) is False assert os.path.isfile(os.path.join(dst_strip_root_temp_dir, 'test.txt'))
def __install_zookeeper(dist_directory: str = MC_DIST_DIR, zookeeper_version: str = MC_ZOOKEEPER_VERSION) -> None: """Install ZooKeeper to distribution directory; lock directory before installing and unlock afterwards.""" if __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version): raise McZooKeeperRunException("ZooKeeper %s is already installed in distribution directory '%s'." % ( zookeeper_version, dist_directory )) zookeeper_path = __zookeeper_path(dist_directory=dist_directory, zookeeper_version=zookeeper_version) log.info("Creating ZooKeeper directory...") mkdir_p(zookeeper_path) installing_file_path = __zookeeper_installing_file_path(dist_directory=dist_directory, zookeeper_version=zookeeper_version) log.info("Locking ZooKeeper directory for installation...") lock_file(installing_file_path, timeout=MC_INSTALL_TIMEOUT) # Waited for concurrent installation to finish? if __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version): log.info("While waiting for ZooKeeper directory to unlock, ZooKeeper got installed to said directory.") return zookeeper_dist_url = __zookeeper_dist_url(zookeeper_version=zookeeper_version) log.info("Downloading ZooKeeper %s from %s..." % (zookeeper_version, zookeeper_dist_url)) zookeeper_tarball_dest_path = download_file_to_temp_path(source_url=zookeeper_dist_url) log.info("Extracting %s to %s..." % (zookeeper_tarball_dest_path, zookeeper_path)) extract_tarball_to_directory(archive_file=zookeeper_tarball_dest_path, dest_directory=zookeeper_path, strip_root=True) log.info("Creating 'installed' file...") installed_file_path = __zookeeper_installed_file_path(dist_directory=dist_directory, zookeeper_version=zookeeper_version) lock_file(installed_file_path) log.info("Removing lock file...") unlock_file(installing_file_path) if not __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version): raise McZooKeeperRunException("I've done everything but ZooKeeper is still not installed.")
def test_extract_tarball_to_directory(): src_temp_dir = tempfile.mkdtemp() dst_temp_dir = tempfile.mkdtemp() # Nonexistent archive with pytest.raises(McExtractTarballToDirectoryException): extract_tarball_to_directory(os.path.join(src_temp_dir, 'nonexistent-archive.tgz'), dst_temp_dir) # Unsupported archive unsupported_archive_path = os.path.join(src_temp_dir, 'unsupported-archive.tar.bz2') run_command_in_foreground(['touch', unsupported_archive_path]) with pytest.raises(McExtractTarballToDirectoryException): extract_tarball_to_directory(unsupported_archive_path, dst_temp_dir) # Faulty archive faulty_archive_path = os.path.join(src_temp_dir, 'faulty-archive.tgz') with open(faulty_archive_path, 'w') as fh: fh.write('Totally not valid Gzip data.') with pytest.raises(McExtractTarballToDirectoryException): extract_tarball_to_directory(faulty_archive_path, dst_temp_dir) # .tar.gz archive tar_archive_path = os.path.join(src_temp_dir, 'tar-gz-archive.tar.gz') tar_archive_contents_dir = os.path.join(src_temp_dir, 'test-contents') os.mkdir(tar_archive_contents_dir) with open(os.path.join(tar_archive_contents_dir, 'test.txt'), 'w') as fh: fh.write('Test contents') with tarfile.open(tar_archive_path, "w:gz") as tar: tar.add(tar_archive_contents_dir, arcname=os.path.basename(tar_archive_contents_dir)) extract_tarball_to_directory(archive_file=tar_archive_path, dest_directory=dst_temp_dir) assert os.path.isdir(os.path.join(dst_temp_dir, 'test-contents')) assert os.path.isfile(os.path.join(dst_temp_dir, 'test-contents', 'test.txt')) # Strip root dst_strip_root_temp_dir = tempfile.mkdtemp() extract_tarball_to_directory(archive_file=tar_archive_path, dest_directory=dst_strip_root_temp_dir, strip_root=True) assert os.path.isdir(os.path.join(dst_strip_root_temp_dir, 'test-contents')) is False assert os.path.isfile(os.path.join(dst_strip_root_temp_dir, 'test.txt'))