def test_download_file_to_temp_path(): # Existent URL temp_file = download_file_to_temp_path(source_url='https://www.google.com/robots.txt') assert os.path.isfile(temp_file) assert os.path.getsize(temp_file) > 0 # Nonexistent URL with pytest.raises(McDownloadFileToTempPathException): download_file_to_temp_path('https://mediacloud.org/should-not-exist.txt')
def test_download_file_to_temp_path(): # Existent URL temp_file = download_file_to_temp_path( source_url='https://www.google.com/robots.txt') assert os.path.isfile(temp_file) assert os.path.getsize(temp_file) > 0 # Nonexistent URL with pytest.raises(McDownloadFileToTempPathException): download_file_to_temp_path( 'https://mediacloud.org/should-not-exist.txt')
def __install_solr(dist_directory: str = MC_DIST_DIR, solr_version: str = MC_SOLR_VERSION) -> None: """Install Solr to distribution directory; lock directory before installing and unlock afterwards.""" if __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version): raise McSolrRunException( "Solr %s is already installed in distribution directory '%s'." % (solr_version, dist_directory)) solr_path = __solr_path(dist_directory=dist_directory, solr_version=solr_version) log.info("Creating Solr directory...") mkdir_p(solr_path) installing_file_path = __solr_installing_file_path( dist_directory=dist_directory, solr_version=solr_version) log.info("Locking Solr directory for installation...") lock_file(installing_file_path, timeout=MC_INSTALL_TIMEOUT) # Waited for concurrent installation to finish? if __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version): log.info( "While waiting for Solr directory to unlock, Solr got installed to said directory." ) return solr_dist_url = __solr_dist_url(solr_version=solr_version) log.info("Downloading Solr %s from %s..." % (solr_version, solr_dist_url)) solr_tarball_dest_path = download_file_to_temp_path(solr_dist_url) log.info("Extracting %s to %s..." % (solr_tarball_dest_path, solr_path)) extract_tarball_to_directory(archive_file=solr_tarball_dest_path, dest_directory=solr_path, strip_root=True) # Solr needs its .war extracted first before ZkCLI is usable jetty_home_path = __jetty_home_path(dist_directory=dist_directory, solr_version=solr_version) solr_war_dest_dir = os.path.join(jetty_home_path, "solr-webapp", "webapp") if not os.path.exists(os.path.join(solr_war_dest_dir, "index.html")): raise McSolrRunException("Solr's .war is not extracted at path %s" % solr_war_dest_dir) log.info("Creating 'installed' file...") installed_file_path = __solr_installed_file_path( dist_directory=dist_directory, solr_version=solr_version) lock_file(installed_file_path) log.info("Removing lock file...") unlock_file(installing_file_path) if not __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version): raise McSolrRunException( "I've done everything but Solr is still not installed.")
def __install_zookeeper(dist_directory: str = MC_DIST_DIR, zookeeper_version: str = MC_ZOOKEEPER_VERSION) -> None: """Install ZooKeeper to distribution directory; lock directory before installing and unlock afterwards.""" if __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version): raise McZooKeeperRunException( "ZooKeeper %s is already installed in distribution directory '%s'." % (zookeeper_version, dist_directory)) zookeeper_path = __zookeeper_path(dist_directory=dist_directory, zookeeper_version=zookeeper_version) log.info("Creating ZooKeeper directory...") mkdir_p(zookeeper_path) installing_file_path = __zookeeper_installing_file_path( dist_directory=dist_directory, zookeeper_version=zookeeper_version) log.info("Locking ZooKeeper directory for installation...") lock_file(installing_file_path, timeout=MC_INSTALL_TIMEOUT) # Waited for concurrent installation to finish? if __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version): log.info( "While waiting for ZooKeeper directory to unlock, ZooKeeper got installed to said directory." ) return zookeeper_dist_url = __zookeeper_dist_url( zookeeper_version=zookeeper_version) log.info("Downloading ZooKeeper %s from %s..." % (zookeeper_version, zookeeper_dist_url)) zookeeper_tarball_dest_path = download_file_to_temp_path( source_url=zookeeper_dist_url) log.info("Extracting %s to %s..." % (zookeeper_tarball_dest_path, zookeeper_path)) extract_tarball_to_directory(archive_file=zookeeper_tarball_dest_path, dest_directory=zookeeper_path, strip_root=True) log.info("Creating 'installed' file...") installed_file_path = __zookeeper_installed_file_path( dist_directory=dist_directory, zookeeper_version=zookeeper_version) lock_file(installed_file_path) log.info("Removing lock file...") unlock_file(installing_file_path) if not __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version): raise McZooKeeperRunException( "I've done everything but ZooKeeper is still not installed.")
def __install_solr(dist_directory: str = MC_DIST_DIR, solr_version: str = MC_SOLR_VERSION) -> None: """Install Solr to distribution directory; lock directory before installing and unlock afterwards.""" if __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version): raise Exception("Solr %s is already installed in distribution directory '%s'." % (solr_version, dist_directory)) solr_path = __solr_path(dist_directory=dist_directory, solr_version=solr_version) l.info("Creating Solr directory...") mkdir_p(solr_path) installing_file_path = __solr_installing_file_path(dist_directory=dist_directory, solr_version=solr_version) l.info("Locking Solr directory for installation...") lock_file(installing_file_path, timeout=MC_INSTALL_TIMEOUT) # Waited for concurrent installation to finish? if __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version): l.info("While waiting for Solr directory to unlock, Solr got installed to said directory.") return solr_dist_url = __solr_dist_url(solr_version=solr_version) l.info("Downloading Solr %s from %s..." % (solr_version, solr_dist_url)) solr_tarball_dest_path = download_file_to_temp_path(solr_dist_url) l.info("Extracting %s to %s..." % (solr_tarball_dest_path, solr_path)) extract_tarball_to_directory(archive_file=solr_tarball_dest_path, dest_directory=solr_path, strip_root=True) # Solr needs its .war extracted first before ZkCLI is usable jetty_home_path = __jetty_home_path(dist_directory=dist_directory, solr_version=solr_version) solr_war_dest_dir = os.path.join(jetty_home_path, "solr-webapp", "webapp") # Solr 5.5.2+ already has the .war extracted if not os.path.exists(os.path.join(solr_war_dest_dir, "index.html")): solr_war_path = os.path.join(jetty_home_path, "webapps", "solr.war") if not os.path.isfile(solr_war_path): raise Exception("Solr's .war file does not exist at path %s" % solr_war_path) solr_war_dest_dir = os.path.join(jetty_home_path, "solr-webapp", "webapp") l.info("Extracting solr.war at '%s' to '%s'..." % (solr_war_path, solr_war_dest_dir)) mkdir_p(solr_war_dest_dir) extract_zip_to_directory(archive_file=solr_war_path, dest_directory=solr_war_dest_dir) l.info("Creating 'installed' file...") installed_file_path = __solr_installed_file_path(dist_directory=dist_directory, solr_version=solr_version) lock_file(installed_file_path) l.info("Removing lock file...") unlock_file(installing_file_path) if not __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version): raise Exception("I've done everything but Solr is still not installed.")
def __install_zookeeper(dist_directory: str = MC_DIST_DIR, zookeeper_version: str = MC_ZOOKEEPER_VERSION) -> None: """Install ZooKeeper to distribution directory; lock directory before installing and unlock afterwards.""" if __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version): raise McZooKeeperRunException("ZooKeeper %s is already installed in distribution directory '%s'." % ( zookeeper_version, dist_directory )) zookeeper_path = __zookeeper_path(dist_directory=dist_directory, zookeeper_version=zookeeper_version) log.info("Creating ZooKeeper directory...") mkdir_p(zookeeper_path) installing_file_path = __zookeeper_installing_file_path(dist_directory=dist_directory, zookeeper_version=zookeeper_version) log.info("Locking ZooKeeper directory for installation...") lock_file(installing_file_path, timeout=MC_INSTALL_TIMEOUT) # Waited for concurrent installation to finish? if __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version): log.info("While waiting for ZooKeeper directory to unlock, ZooKeeper got installed to said directory.") return zookeeper_dist_url = __zookeeper_dist_url(zookeeper_version=zookeeper_version) log.info("Downloading ZooKeeper %s from %s..." % (zookeeper_version, zookeeper_dist_url)) zookeeper_tarball_dest_path = download_file_to_temp_path(source_url=zookeeper_dist_url) log.info("Extracting %s to %s..." % (zookeeper_tarball_dest_path, zookeeper_path)) extract_tarball_to_directory(archive_file=zookeeper_tarball_dest_path, dest_directory=zookeeper_path, strip_root=True) log.info("Creating 'installed' file...") installed_file_path = __zookeeper_installed_file_path(dist_directory=dist_directory, zookeeper_version=zookeeper_version) lock_file(installed_file_path) log.info("Removing lock file...") unlock_file(installing_file_path) if not __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version): raise McZooKeeperRunException("I've done everything but ZooKeeper is still not installed.")