def download_from_mirror(self, mirror_path: str) -> Tuple[str, str, Optional[str]]:
"""Download a Dynamite service from a mirror
Args:
mirror_path: The path to the mirror
Returns:
The mirror url, archive name, and directory name (once the archive has been extracted)
"""
with open(mirror_path) as mirror_f:
res, err = None, None
for mirror in mirror_f.readlines():
try:
url, archive_name, dir_name = [token.strip() for token in mirror.split(',')]
except ValueError:
url = mirror
archive_name = os.path.basename(url)
dir_name = None
self.logger.info("Downloading {} from {}".format(archive_name, url))
fqdn_dir_name = f'{const.INSTALL_CACHE}/{str(dir_name)}'
if os.path.exists(fqdn_dir_name):
shutil.rmtree(fqdn_dir_name, ignore_errors=True)
try:
res = utilities.download_file(url, archive_name, stdout=self.stdout)
except Exception as e:
res, err = False, e
self.logger.warning(f'Failed to download {archive_name} from {url}; {e}')
if res:
break
if not res:
self.logger.error(f'An error occurred while attempting to download: {err}')
raise exceptions.DownloadError(
f'General error while attempting to download {archive_name} from all mirrors.')
return url, archive_name, dir_name
def update_default_configurations():
"""
Retrieves the latest skeleton configurations for setting up ElasticSearch, LogStash, Zeek, and Suricata
"""
shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True)
makedirs(const.DEFAULT_CONFIGS, exist_ok=True)
try:
download_file(const.DEFAULT_CONFIGS_URL,
const.DEFAULT_CONFIGS_ARCHIVE_NAME,
stdout=True)
except Exception as e:
raise exceptions.DownloadError(
"General error occurred while downloading archive: {}; {}".format(
os.path.join(const.INSTALL_CACHE, 'default_configs.tar.gz'),
e))
shutil.rmtree(const.DEFAULT_CONFIGS, ignore_errors=True)
time.sleep(1)
try:
extract_archive(
os.path.join(const.INSTALL_CACHE, 'default_configs.tar.gz'),
const.CONFIG_PATH)
except IOError as e:
raise exceptions.ArchiveExtractionError(
"General error occurred while extracting archive: {}; {}".format(
os.path.join(const.INSTALL_CACHE, 'default_configs.tar.gz'),
e))
def update_mirrors():
"""
Retrieves the latest mirrors which contain the download locations for all components
"""
shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True)
makedirs(const.MIRRORS, exist_ok=True)
try:
download_file(const.MIRRORS_CONFIG_URL,
const.MIRRORS_CONFIG_ARCHIVE_NAME,
stdout=True)
except Exception as e:
raise exceptions.DownloadError(
"General error occurred while downloading archive: {}; {}".format(
os.path.join(const.INSTALL_CACHE, 'mirrors.tar.gz'), e))
shutil.rmtree(const.MIRRORS, ignore_errors=True)
try:
extract_archive(os.path.join(const.INSTALL_CACHE, 'mirrors.tar.gz'),
const.CONFIG_PATH)
return True
except IOError as e:
raise exceptions.DownloadError(
"General error occurred while extracting archive: {}; {}".format(
os.path.join(const.INSTALL_CACHE, 'mirrors.tar.gz'), e))
def download_configurations_package(
url: Optional[str] = const.DEFAULT_CONFIGURATIONS_URL) -> None:
"""Download a configurations package from a URL
Args:
url: A URL where a configurations archive can be downloaded
Returns:
None
"""
response = requests.get(url, stream=True)
if response.status_code == 200:
with open(f'{const.INSTALL_CACHE}/configurations.tar.gz',
'wb') as f:
f.write(response.raw.read())
else:
raise exceptions.DownloadError(
f'Download {url} failed; status: {response.status_code}')
def download_dynamite_sdk(stdout=False):
"""
Download DynamiteSDK archive
:param stdout: Print output to console
"""
url = None
try:
with open(const.DYNAMITE_SDK_MIRRORS, 'r') as sdk_archive:
for url in sdk_archive.readlines():
if utilities.download_file(url,
const.DYNAMITE_SDK_ARCHIVE_NAME,
stdout=stdout):
break
except Exception as e:
raise general_exceptions.DownloadError(
"General error while downloading DynamiteSDK from {}; {}".
format(url, e))
def download_from_mirror(mirror_path, fname, stdout=False, verbose=False):
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logger = get_logger('BASESVC', level=log_level, stdout=stdout)
with open(mirror_path) as mirror_f:
res, err = None, None
for url in mirror_f.readlines():
logger.info("Downloading {} from {}".format(fname, url))
try:
res = utilities.download_file(url, fname, stdout=stdout)
except Exception as e:
res, err = False, e
logger.warning("Failed to download {} from {}; {}".format(fname, url, e))
if res:
break
if not res:
raise general_exceptions.DownloadError(
"General error while attempting to download {} from all mirrors ;".format(fname))