def download_from_mirror(self, mirror_path: str) -> Tuple[str, str, Optional[str]]: """Download a Dynamite service from a mirror Args: mirror_path: The path to the mirror Returns: The mirror url, archive name, and directory name (once the archive has been extracted) """ with open(mirror_path) as mirror_f: res, err = None, None for mirror in mirror_f.readlines(): try: url, archive_name, dir_name = [token.strip() for token in mirror.split(',')] except ValueError: url = mirror archive_name = os.path.basename(url) dir_name = None self.logger.info("Downloading {} from {}".format(archive_name, url)) fqdn_dir_name = f'{const.INSTALL_CACHE}/{str(dir_name)}' if os.path.exists(fqdn_dir_name): shutil.rmtree(fqdn_dir_name, ignore_errors=True) try: res = utilities.download_file(url, archive_name, stdout=self.stdout) except Exception as e: res, err = False, e self.logger.warning(f'Failed to download {archive_name} from {url}; {e}') if res: break if not res: self.logger.error(f'An error occurred while attempting to download: {err}') raise exceptions.DownloadError( f'General error while attempting to download {archive_name} from all mirrors.') return url, archive_name, dir_name
def update_default_configurations(): """ Retrieves the latest skeleton configurations for setting up ElasticSearch, LogStash, Zeek, and Suricata """ shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True) makedirs(const.DEFAULT_CONFIGS, exist_ok=True) try: download_file(const.DEFAULT_CONFIGS_URL, const.DEFAULT_CONFIGS_ARCHIVE_NAME, stdout=True) except Exception as e: raise exceptions.DownloadError( "General error occurred while downloading archive: {}; {}".format( os.path.join(const.INSTALL_CACHE, 'default_configs.tar.gz'), e)) shutil.rmtree(const.DEFAULT_CONFIGS, ignore_errors=True) time.sleep(1) try: extract_archive( os.path.join(const.INSTALL_CACHE, 'default_configs.tar.gz'), const.CONFIG_PATH) except IOError as e: raise exceptions.ArchiveExtractionError( "General error occurred while extracting archive: {}; {}".format( os.path.join(const.INSTALL_CACHE, 'default_configs.tar.gz'), e))
def update_mirrors(): """ Retrieves the latest mirrors which contain the download locations for all components """ shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True) makedirs(const.MIRRORS, exist_ok=True) try: download_file(const.MIRRORS_CONFIG_URL, const.MIRRORS_CONFIG_ARCHIVE_NAME, stdout=True) except Exception as e: raise exceptions.DownloadError( "General error occurred while downloading archive: {}; {}".format( os.path.join(const.INSTALL_CACHE, 'mirrors.tar.gz'), e)) shutil.rmtree(const.MIRRORS, ignore_errors=True) try: extract_archive(os.path.join(const.INSTALL_CACHE, 'mirrors.tar.gz'), const.CONFIG_PATH) return True except IOError as e: raise exceptions.DownloadError( "General error occurred while extracting archive: {}; {}".format( os.path.join(const.INSTALL_CACHE, 'mirrors.tar.gz'), e))
def download_configurations_package( url: Optional[str] = const.DEFAULT_CONFIGURATIONS_URL) -> None: """Download a configurations package from a URL Args: url: A URL where a configurations archive can be downloaded Returns: None """ response = requests.get(url, stream=True) if response.status_code == 200: with open(f'{const.INSTALL_CACHE}/configurations.tar.gz', 'wb') as f: f.write(response.raw.read()) else: raise exceptions.DownloadError( f'Download {url} failed; status: {response.status_code}')
def download_dynamite_sdk(stdout=False): """ Download DynamiteSDK archive :param stdout: Print output to console """ url = None try: with open(const.DYNAMITE_SDK_MIRRORS, 'r') as sdk_archive: for url in sdk_archive.readlines(): if utilities.download_file(url, const.DYNAMITE_SDK_ARCHIVE_NAME, stdout=stdout): break except Exception as e: raise general_exceptions.DownloadError( "General error while downloading DynamiteSDK from {}; {}". format(url, e))
def download_from_mirror(mirror_path, fname, stdout=False, verbose=False): log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('BASESVC', level=log_level, stdout=stdout) with open(mirror_path) as mirror_f: res, err = None, None for url in mirror_f.readlines(): logger.info("Downloading {} from {}".format(fname, url)) try: res = utilities.download_file(url, fname, stdout=stdout) except Exception as e: res, err = False, e logger.warning("Failed to download {} from {}; {}".format(fname, url, e)) if res: break if not res: raise general_exceptions.DownloadError( "General error while attempting to download {} from all mirrors ;".format(fname))