def test_init_http_request_ok(self, mock_req):
utils.init_http_request('http://fake_url')
mock_req.assert_called_once_with('http://fake_url',
stream=True,
timeout=CONF.http_request_timeout,
headers={'Range': 'bytes=0-'},
proxies=None)
def test_init_http_request_ok(self, mock_req):
utils.init_http_request("http://fake_url")
mock_req.assert_called_once_with(
"http://fake_url",
stream=True,
timeout=CONF.http_request_timeout,
headers={"Range": "bytes=0-"},
proxies=None,
)
def test_init_http_requests_bypass_proxy(self, mock_req):
proxies = {"http": "proxy1"}
hostname = "fake.hostname.11.42"
url = "http://{0}/web".format(hostname)
noproxy_addrs = [hostname]
utils.init_http_request(url, proxies=proxies, noproxy_addrs=noproxy_addrs)
mock_req.assert_called_once_with(
url, stream=True, timeout=CONF.http_request_timeout, headers={"Range": "bytes=0-"}, proxies=None
)
def test_init_http_request_proxy(self, mock_req):
proxies = {'http': 'proxy1'}
noproxy_addrs = ['fake.hostname.11.42']
utils.init_http_request('http://fake_url',
proxies=proxies,
noproxy_addrs=noproxy_addrs)
mock_req.assert_called_once_with(
'http://fake_url', stream=True, timeout=CONF.http_request_timeout,
headers={'Range': 'bytes=0-'}, proxies=proxies)
def test_init_http_request_proxy(self, mock_req):
proxies = {"http": "proxy1"}
noproxy_addrs = ["fake.hostname.11.42"]
utils.init_http_request("http://fake_url", proxies=proxies, noproxy_addrs=noproxy_addrs)
mock_req.assert_called_once_with(
"http://fake_url",
stream=True,
timeout=CONF.http_request_timeout,
headers={"Range": "bytes=0-"},
proxies=proxies,
)
def test_init_http_request_proxy(self, mock_req):
proxies = {'http': 'proxy1'}
noproxy_addrs = ['fake.hostname.11.42']
utils.init_http_request('http://fake_url',
proxies=proxies,
noproxy_addrs=noproxy_addrs)
mock_req.assert_called_once_with('http://fake_url',
stream=True,
timeout=CONF.http_request_timeout,
headers={'Range': 'bytes=0-'},
proxies=proxies)
def test_init_http_requests_bypass_proxy(self, mock_req):
proxies = {'http': 'proxy1'}
hostname = 'fake.hostname.11.42'
url = "http://{0}/web".format(hostname)
noproxy_addrs = [hostname]
utils.init_http_request(url,
proxies=proxies,
noproxy_addrs=noproxy_addrs)
mock_req.assert_called_once_with(url,
stream=True,
timeout=CONF.http_request_timeout,
headers={'Range': 'bytes=0-'},
proxies=None)
def get_release_file(uri,
suite,
section,
proxies=None,
direct_repo_addrs=None):
"""Download and parse repo's Release file
Returns an apt preferences for specified repo.
:param proxies: Dict protocol:uri format
:param direct_repo_addrs: List of addresses which should be bypassed
by proxy
:returns: a string with apt preferences rules
"""
if section:
# We can't use urljoin here because it works pretty bad in
# cases when 'uri' doesn't have a trailing slash.
download_uri = os.path.join(uri, 'dists', suite, 'Release')
else:
# Well, we have a flat repo case, so we should download Release
# file from a different place. Please note, we have to strip
# a leading slash from suite because otherwise the download
# link will be wrong.
download_uri = os.path.join(uri, suite.lstrip('/'), 'Release')
return utils.init_http_request(download_uri,
proxies=proxies,
noproxy_addrs=direct_repo_addrs).text
def parse_image_meta(self):
LOG.debug('--- Preparing image metadata ---')
data = self.data
# FIXME(agordeev): this piece of code for fetching additional image
# meta data should be factored out of this particular nailgun driver
# into more common and absract data getter which should be able to deal
# with various data sources (local file, http(s), etc.) and different
# data formats ('blob', json, yaml, etc.).
# So, the manager will combine and manipulate all those multiple data
# getter instances.
# Also, the initial data source should be set to sort out chicken/egg
# problem. Command line option may be useful for such a case.
# BUG: https://bugs.launchpad.net/fuel/+bug/1430418
root_uri = data['ks_meta']['image_data']['/']['uri']
filename = os.path.basename(urlparse(root_uri).path).split('.')[0] + \
'.yaml'
metadata_url = urljoin(root_uri, filename)
try:
image_meta = yaml.load(
utils.init_http_request(metadata_url).text)
except Exception as e:
LOG.exception(e)
LOG.debug('Failed to fetch/decode image meta data')
image_meta = {}
return image_meta
def test_init_http_request_non_critical_errors(self, mock_req, mock_s):
mock_ok = mock.Mock()
mock_req.side_effect = [urllib3.exceptions.DecodeError(),
urllib3.exceptions.ProxyError(),
requests.exceptions.ConnectionError(),
requests.exceptions.Timeout(),
requests.exceptions.TooManyRedirects(),
socket.timeout(),
mock_ok]
req_obj = utils.init_http_request('http://fake_url')
self.assertEqual(mock_ok, req_obj)
def test_init_http_request_non_critical_errors(self, mock_req, mock_s):
mock_ok = mock.Mock()
mock_req.side_effect = [
urllib3.exceptions.DecodeError(),
urllib3.exceptions.ProxyError(),
requests.exceptions.ConnectionError(),
requests.exceptions.Timeout(),
requests.exceptions.TooManyRedirects(),
socket.timeout(), mock_ok
]
req_obj = utils.init_http_request('http://fake_url')
self.assertEqual(mock_ok, req_obj)
def __init__(self, url):
self.url = str(url)
self.response_obj = utils.init_http_request(self.url)
self.processed_bytes = 0
try:
self.length = int(self.response_obj.headers['content-length'])
except (ValueError, KeyError):
raise errors.HttpUrlInvalidContentLength(
'Can not get content length for %s' % self.url)
else:
LOG.debug('Expected content length %s for %s' % (self.length,
self.url))
def next(self):
while self.processed_bytes < self.length:
try:
data = self.response_obj.raw.read(CONF.data_chunk_size)
if not data:
raise errors.HttpUrlConnectionError(
'Could not receive data: URL=%s, range=%s' %
(self.url, self.processed_bytes))
except Exception as exc:
LOG.exception(exc)
self.response_obj = utils.init_http_request(
self.url, byte_range=self.processed_bytes)
continue
else:
self.processed_bytes += len(data)
return data
raise StopIteration()
def parse_image_meta(self):
LOG.debug('--- Preparing image metadata ---')
data = self.data
# FIXME(agordeev): this piece of code for fetching additional image
# meta data should be factored out of this particular nailgun driver
# into more common and absract data getter which should be able to deal
# with various data sources (local file, http(s), etc.) and different
# data formats ('blob', json, yaml, etc.).
# So, the manager will combine and manipulate all those multiple data
# getter instances.
# Also, the initial data source should be set to sort out chicken/egg
# problem. Command line option may be useful for such a case.
# BUG: https://bugs.launchpad.net/fuel/+bug/1430418
root_uri = data['ks_meta']['image_data']['/']['uri']
filename = os.path.basename(urlparse(root_uri).path).split('.')[0] + \
'.yaml'
metadata_url = urljoin(root_uri, filename)
try:
image_meta = yaml.load(utils.init_http_request(metadata_url).text)
except Exception as e:
LOG.exception(e)
LOG.debug('Failed to fetch/decode image meta data')
image_meta = {}
return image_meta
def get_release_file(uri, suite, section, proxies=None,
direct_repo_addrs=None):
"""Download and parse repo's Release file
Returns an apt preferences for specified repo.
:param proxies: Dict protocol:uri format
:param direct_repo_addrs: List of addresses which should be bypassed
by proxy
:returns: a string with apt preferences rules
"""
if section:
# We can't use urljoin here because it works pretty bad in
# cases when 'uri' doesn't have a trailing slash.
download_uri = os.path.join(uri, 'dists', suite, 'Release')
else:
# Well, we have a flat repo case, so we should download Release
# file from a different place. Please note, we have to strip
# a leading slash from suite because otherwise the download
# link will be wrong.
download_uri = os.path.join(uri, suite.lstrip('/'), 'Release')
return utils.init_http_request(download_uri, proxies=proxies,
noproxy_addrs=direct_repo_addrs).text
def test_init_http_request_ok(self, mock_req):
utils.init_http_request('http://fake_url')
mock_req.assert_called_once_with(
'http://fake_url', stream=True, timeout=CONF.http_request_timeout,
headers={'Range': 'bytes=0-'}, proxies=None)
