def _galaxy_download_error(msg, url): # url kw but no message log.debug('msg: %s %s', msg, type(msg)) log.debug('url: %s %s', url, type(url)) exc = exceptions.GalaxyDownloadError(url=url) log.debug('exc: %s', exc) assert exc.url == url assert url in str(exc) # multiple args, but no url kwarg exc = exceptions.GalaxyDownloadError(url, msg) log.debug('exc: %s', exc) assert exc.url is None # a message arg and a url kwarg exc = exceptions.GalaxyDownloadError(msg, url=url) log.debug('exc: %s', exc) assert exc.url == url assert url in str(exc) assert msg in str(exc) # just a single message arg, no url exc = exceptions.GalaxyDownloadError(msg) log.debug('exc: %s', exc) assert exc.url is None assert msg in '%s' % exc
def fetch_url(archive_url, validate_certs=True, filename=None, dest_dir=None, chunk_size=None): """ Downloads the archived content from github to a temp location """ request_headers = {} request_id = uuid.uuid4().hex request_headers['X-Request-ID'] = request_id request_headers['User-Agent'] = user_agent.user_agent() log.debug('Downloading archive_url: %s', archive_url) try: resp = requests.get(archive_url, verify=validate_certs, headers=request_headers, stream=True) except Exception as e: log.exception(e) raise exceptions.GalaxyDownloadError(e, url=archive_url) # Let tmp filenames begin with the expected artifact filename before '::', except # if we don't know it, then it's the UNKNOWN... _prefix = filename or 'UNKNOWN-UNKNOWN-UNKNOWN.tar.gz' prefix = '%s::' % _prefix # TODO: add a configured spool or cache dir and/or default to using # one in MAZER_HOME or maybe ANSIBLE_TMP? temp_fd = tempfile.NamedTemporaryFile( delete=False, suffix='-tmp-mazer-artifact-download', prefix=prefix) try: resp.raise_for_status() except requests.exceptions.HTTPError as http_exc: temp_fd.close() os.unlink(temp_fd.name) raise exceptions.GalaxyDownloadError(http_exc, url=http_exc.response.url) if resp.history: for redirect in resp.history: log.debug( 'Original request for %s redirected. %s is redirected to %s', archive_url, redirect.url, redirect.headers['Location']) for chunk in resp.iter_content(chunk_size=chunk_size): log.debug('read chunk') temp_fd.write(chunk) temp_fd.close() return temp_fd.name return False
def fetch_url(archive_url, validate_certs=True): """ Downloads the archived content from github to a temp location """ # TODO: should probably be based on/shared with rest API client code, so that # content downloads could support any thing the rest code does # (ie, any TLS cert setup, proxy config, auth options, etc) # WHEN: if we change the underlying http client impl at least try: url_file = open_url(archive_url, validate_certs=validate_certs) temp_file = tempfile.NamedTemporaryFile( delete=False, prefix='tmp-ansible-galaxy-content-archive-', suffix='.tar.gz') data = url_file.read() while data: temp_file.write(data) data = url_file.read() temp_file.close() return temp_file.name except Exception as e: # FIXME: there is a ton of reasons a download and save could fail so could likely provided better errors here log.exception(e) raise exceptions.GalaxyDownloadError( "failed to download the file: %s" % str(e)) return False
def _text_encoding(): for valid_string, txtrpr, encoding in VALID_STRINGS: log.debug('valid_string: %s txtrpr: %s encoding: %s', valid_string, txtrpr, encoding) exc = exceptions.GalaxyDownloadError(to_text(valid_string)) log.debug('exc: %s', exc) log.debug(exc) log.debug('exc(str): %s', str(exc)) log.debug('exc(repr): %s', repr(exc)) ttext = to_text(valid_string) log.debug('%s in %s: %s', ttext, '%s' % exc, ttext in '%s' % exc) assert ttext in '%s' % exc
def test_fetch_download_error(mocker): mock_fetcher = mocker.MagicMock(name='MockFetch') mock_fetcher.fetch.side_effect = exceptions.GalaxyDownloadError( url='http://example.invalid') repo_spec = RepositorySpec(namespace='some_namespace', name='some_name', version='86.75.30') find_results = {} with pytest.raises(exceptions.GalaxyDownloadError) as exc_info: install.fetch(mock_fetcher, repo_spec, find_results) log.debug("exc_info: %s", exc_info)
def fetch_url(archive_url, validate_certs=True): """ Downloads the archived content from github to a temp location """ # TODO: should probably be based on/shared with rest API client code, so that # content downloads could support any thing the rest code does # (ie, any TLS cert setup, proxy config, auth options, etc) # WHEN: if we change the underlying http client impl at least request_headers = {} request_id = uuid.uuid4().hex request_headers['X-Request-ID'] = request_id request_headers['User-Agent'] = user_agent.user_agent() try: log.debug('Downloading archive_url: %s', archive_url) resp = requests.get(archive_url, verify=validate_certs, headers=request_headers, stream=True) temp_file = tempfile.NamedTemporaryFile( delete=False, prefix='tmp-ansible-galaxy-content-archive-', suffix='.tar.gz') if resp.history: for redirect in resp.history: log.debug( 'Original request for %s redirected. %s is redirected to %s', archive_url, redirect.url, redirect.headers['Location']) # TODO: test for short reads for chunk in resp.iter_content(chunk_size=None): temp_file.write(chunk) temp_file.close() return temp_file.name except Exception as e: # FIXME: there is a ton of reasons a download and save could fail so could likely provided better errors here log.exception(e) raise exceptions.GalaxyDownloadError(e, url=archive_url) return False
def test_galaxy_download_error_no_args(): exc = exceptions.GalaxyDownloadError() log.debug('exc: %s', exc)