def _galaxy_download_error(msg, url):
# url kw but no message
log.debug('msg: %s %s', msg, type(msg))
log.debug('url: %s %s', url, type(url))
exc = exceptions.GalaxyDownloadError(url=url)
log.debug('exc: %s', exc)
assert exc.url == url
assert url in str(exc)
# multiple args, but no url kwarg
exc = exceptions.GalaxyDownloadError(url, msg)
log.debug('exc: %s', exc)
assert exc.url is None
# a message arg and a url kwarg
exc = exceptions.GalaxyDownloadError(msg, url=url)
log.debug('exc: %s', exc)
assert exc.url == url
assert url in str(exc)
assert msg in str(exc)
# just a single message arg, no url
exc = exceptions.GalaxyDownloadError(msg)
log.debug('exc: %s', exc)
assert exc.url is None
assert msg in '%s' % exc
def fetch_url(archive_url,
validate_certs=True,
filename=None,
dest_dir=None,
chunk_size=None):
"""
Downloads the archived content from github to a temp location
"""
request_headers = {}
request_id = uuid.uuid4().hex
request_headers['X-Request-ID'] = request_id
request_headers['User-Agent'] = user_agent.user_agent()
log.debug('Downloading archive_url: %s', archive_url)
try:
resp = requests.get(archive_url,
verify=validate_certs,
headers=request_headers,
stream=True)
except Exception as e:
log.exception(e)
raise exceptions.GalaxyDownloadError(e, url=archive_url)
# Let tmp filenames begin with the expected artifact filename before '::', except
# if we don't know it, then it's the UNKNOWN...
_prefix = filename or 'UNKNOWN-UNKNOWN-UNKNOWN.tar.gz'
prefix = '%s::' % _prefix
# TODO: add a configured spool or cache dir and/or default to using
# one in MAZER_HOME or maybe ANSIBLE_TMP?
temp_fd = tempfile.NamedTemporaryFile(
delete=False, suffix='-tmp-mazer-artifact-download', prefix=prefix)
try:
resp.raise_for_status()
except requests.exceptions.HTTPError as http_exc:
temp_fd.close()
os.unlink(temp_fd.name)
raise exceptions.GalaxyDownloadError(http_exc,
url=http_exc.response.url)
if resp.history:
for redirect in resp.history:
log.debug(
'Original request for %s redirected. %s is redirected to %s',
archive_url, redirect.url, redirect.headers['Location'])
for chunk in resp.iter_content(chunk_size=chunk_size):
log.debug('read chunk')
temp_fd.write(chunk)
temp_fd.close()
return temp_fd.name
return False
def fetch_url(archive_url, validate_certs=True):
"""
Downloads the archived content from github to a temp location
"""
# TODO: should probably be based on/shared with rest API client code, so that
# content downloads could support any thing the rest code does
# (ie, any TLS cert setup, proxy config, auth options, etc)
# WHEN: if we change the underlying http client impl at least
try:
url_file = open_url(archive_url, validate_certs=validate_certs)
temp_file = tempfile.NamedTemporaryFile(
delete=False,
prefix='tmp-ansible-galaxy-content-archive-',
suffix='.tar.gz')
data = url_file.read()
while data:
temp_file.write(data)
data = url_file.read()
temp_file.close()
return temp_file.name
except Exception as e:
# FIXME: there is a ton of reasons a download and save could fail so could likely provided better errors here
log.exception(e)
raise exceptions.GalaxyDownloadError(
"failed to download the file: %s" % str(e))
return False
def _text_encoding():
for valid_string, txtrpr, encoding in VALID_STRINGS:
log.debug('valid_string: %s txtrpr: %s encoding: %s', valid_string,
txtrpr, encoding)
exc = exceptions.GalaxyDownloadError(to_text(valid_string))
log.debug('exc: %s', exc)
log.debug(exc)
log.debug('exc(str): %s', str(exc))
log.debug('exc(repr): %s', repr(exc))
ttext = to_text(valid_string)
log.debug('%s in %s: %s', ttext, '%s' % exc, ttext in '%s' % exc)
assert ttext in '%s' % exc
def test_fetch_download_error(mocker):
mock_fetcher = mocker.MagicMock(name='MockFetch')
mock_fetcher.fetch.side_effect = exceptions.GalaxyDownloadError(
url='http://example.invalid')
repo_spec = RepositorySpec(namespace='some_namespace',
name='some_name',
version='86.75.30')
find_results = {}
with pytest.raises(exceptions.GalaxyDownloadError) as exc_info:
install.fetch(mock_fetcher, repo_spec, find_results)
log.debug("exc_info: %s", exc_info)
def fetch_url(archive_url, validate_certs=True):
"""
Downloads the archived content from github to a temp location
"""
# TODO: should probably be based on/shared with rest API client code, so that
# content downloads could support any thing the rest code does
# (ie, any TLS cert setup, proxy config, auth options, etc)
# WHEN: if we change the underlying http client impl at least
request_headers = {}
request_id = uuid.uuid4().hex
request_headers['X-Request-ID'] = request_id
request_headers['User-Agent'] = user_agent.user_agent()
try:
log.debug('Downloading archive_url: %s', archive_url)
resp = requests.get(archive_url,
verify=validate_certs,
headers=request_headers,
stream=True)
temp_file = tempfile.NamedTemporaryFile(
delete=False,
prefix='tmp-ansible-galaxy-content-archive-',
suffix='.tar.gz')
if resp.history:
for redirect in resp.history:
log.debug(
'Original request for %s redirected. %s is redirected to %s',
archive_url, redirect.url, redirect.headers['Location'])
# TODO: test for short reads
for chunk in resp.iter_content(chunk_size=None):
temp_file.write(chunk)
temp_file.close()
return temp_file.name
except Exception as e:
# FIXME: there is a ton of reasons a download and save could fail so could likely provided better errors here
log.exception(e)
raise exceptions.GalaxyDownloadError(e, url=archive_url)
return False
def test_galaxy_download_error_no_args():
exc = exceptions.GalaxyDownloadError()
log.debug('exc: %s', exc)
