def test_init_http_request_proxy(self, mock_req):
proxies = {'http': 'proxy1'}
noproxy_addrs = ['fake.hostname.11.42']
utils.init_http_request('http://fake_url',
proxies=proxies,
noproxy_addrs=noproxy_addrs)
mock_req.assert_called_once_with(
'http://fake_url', stream=True, timeout=CONF.http_request_timeout,
headers={'Range': 'bytes=0-'}, proxies=proxies)
def test_init_http_request_ok(self, mock_req):
utils.init_http_request("http://fake_url")
mock_req.assert_called_once_with(
"http://fake_url",
stream=True,
timeout=CONF.http_request_timeout,
headers={"Range": "bytes=0-"},
proxies=None,
)
def test_init_http_requests_bypass_proxy(self, mock_req):
proxies = {"http": "proxy1"}
hostname = "fake.hostname.11.42"
url = "http://{0}/web".format(hostname)
noproxy_addrs = [hostname]
utils.init_http_request(url, proxies=proxies, noproxy_addrs=noproxy_addrs)
mock_req.assert_called_once_with(
url, stream=True, timeout=CONF.http_request_timeout, headers={"Range": "bytes=0-"}, proxies=None
)
def test_init_http_requests_bypass_proxy(self, mock_req):
proxies = {'http': 'proxy1'}
hostname = 'fake.hostname.11.42'
url = "http://{0}/web".format(hostname)
noproxy_addrs = [hostname]
utils.init_http_request(url,
proxies=proxies,
noproxy_addrs=noproxy_addrs)
mock_req.assert_called_once_with(
url, stream=True, timeout=CONF.http_request_timeout,
headers={'Range': 'bytes=0-'}, proxies=None)
def test_init_http_request_proxy(self, mock_req):
proxies = {"http": "proxy1"}
noproxy_addrs = ["fake.hostname.11.42"]
utils.init_http_request("http://fake_url", proxies=proxies, noproxy_addrs=noproxy_addrs)
mock_req.assert_called_once_with(
"http://fake_url",
stream=True,
timeout=CONF.http_request_timeout,
headers={"Range": "bytes=0-"},
proxies=proxies,
)
def get_release_file(uri,
suite,
section,
proxies=None,
direct_repo_addrs=None):
"""Download and parse repo's Release file
Returns an apt preferences for specified repo.
:param proxies: Dict protocol:uri format
:param direct_repo_addrs: List of addresses which should be bypassed
by proxy
:returns: a string with apt preferences rules
"""
if section:
# We can't use urljoin here because it works pretty bad in
# cases when 'uri' doesn't have a trailing slash.
download_uri = os.path.join(uri, 'dists', suite, 'Release')
else:
# Well, we have a flat repo case, so we should download Release
# file from a different place. Please note, we have to strip
# a leading slash from suite because otherwise the download
# link will be wrong.
download_uri = os.path.join(uri, suite.lstrip('/'), 'Release')
return utils.init_http_request(download_uri,
proxies=proxies,
noproxy_addrs=direct_repo_addrs).text
def parse_image_meta(self):
LOG.debug('--- Preparing image metadata ---')
data = self.data
# FIXME(agordeev): this piece of code for fetching additional image
# meta data should be factored out of this particular nailgun driver
# into more common and absract data getter which should be able to deal
# with various data sources (local file, http(s), etc.) and different
# data formats ('blob', json, yaml, etc.).
# So, the manager will combine and manipulate all those multiple data
# getter instances.
# Also, the initial data source should be set to sort out chicken/egg
# problem. Command line option may be useful for such a case.
# BUG: https://bugs.launchpad.net/fuel/+bug/1430418
root_uri = data['ks_meta']['image_data']['/']['uri']
filename = os.path.basename(urlparse(root_uri).path).split('.')[0] + \
'.yaml'
metadata_url = urljoin(root_uri, filename)
try:
image_meta = yaml.load(
utils.init_http_request(metadata_url).text)
except Exception as e:
LOG.exception(e)
LOG.debug('Failed to fetch/decode image meta data')
image_meta = {}
return image_meta
def test_init_http_request_non_critical_errors(self, mock_req, mock_s):
mock_ok = mock.Mock()
mock_req.side_effect = [urllib3.exceptions.DecodeError(),
urllib3.exceptions.ProxyError(),
requests.exceptions.ConnectionError(),
requests.exceptions.Timeout(),
requests.exceptions.TooManyRedirects(),
mock_ok]
req_obj = utils.init_http_request('fake_url')
self.assertEqual(mock_ok, req_obj)
def image_scheme(self, partition_scheme):
LOG.debug('--- Preparing image scheme ---')
data = self.data
image_scheme = objects.ImageScheme()
#FIXME(agordeev): this piece of code for fetching additional image
# meta data should be factored out of this particular nailgun driver
# into more common and absract data getter which should be able to deal
# with various data sources (local file, http(s), etc.) and different
# data formats ('blob', json, yaml, etc.).
# So, the manager will combine and manipulate all those multiple data
# getter instances.
# Also, the initial data source should be set to sort out chicken/egg
# problem. Command line option may be useful for such a case.
# BUG: https://bugs.launchpad.net/fuel/+bug/1430418
root_uri = data['ks_meta']['image_data']['/']['uri']
filename = os.path.basename(urlparse(root_uri).path).split('.')[0] + \
'.yaml'
metadata_url = urljoin(root_uri, filename)
image_meta = {}
raw_image_meta = None
try:
raw_image_meta = yaml.load(
utils.init_http_request(metadata_url).text)
except Exception as e:
LOG.exception(e)
LOG.debug('Failed to fetch/decode image meta data')
if raw_image_meta:
[image_meta.update(img_info) for img_info in raw_image_meta]
# We assume for every file system user may provide a separate
# file system image. For example if partitioning scheme has
# /, /boot, /var/lib file systems then we will try to get images
# for all those mount points. Images data are to be defined
# at provision.json -> ['ks_meta']['image_data']
LOG.debug('Looping over all file systems in partition scheme')
for fs in partition_scheme.fss:
LOG.debug('Processing fs %s' % fs.mount)
if fs.mount not in data['ks_meta']['image_data']:
LOG.debug('There is no image for fs %s. Skipping.' % fs.mount)
continue
image_data = data['ks_meta']['image_data'][fs.mount]
LOG.debug('Adding image for fs %s: uri=%s format=%s container=%s' %
(fs.mount, image_data['uri'],
image_data['format'], image_data['container']))
image_scheme.add_image(
uri=image_data['uri'],
target_device=fs.device,
# In the future we will get format and container
# from provision.json, but currently it is hard coded.
format=image_data['format'],
container=image_data['container'],
size=image_meta.get(fs.mount, {}).get('size'),
md5=image_meta.get(fs.mount, {}).get('md5'),
)
return image_scheme
def test_init_http_request_non_critical_errors(self, mock_req, mock_s):
mock_ok = mock.Mock()
mock_req.side_effect = [urllib3.exceptions.DecodeError(),
urllib3.exceptions.ProxyError(),
requests.exceptions.ConnectionError(),
requests.exceptions.Timeout(),
requests.exceptions.TooManyRedirects(),
socket.timeout(),
mock_ok]
req_obj = utils.init_http_request('http://fake_url')
self.assertEqual(mock_ok, req_obj)
def __init__(self, url):
self.url = str(url)
self.response_obj = utils.init_http_request(self.url)
self.processed_bytes = 0
try:
self.length = int(self.response_obj.headers['content-length'])
except (ValueError, KeyError):
raise errors.HttpUrlInvalidContentLength(
'Can not get content length for %s' % self.url)
else:
LOG.debug('Expected content length %s for %s' % (self.length,
self.url))
def parse_image_scheme(self):
LOG.debug('--- Preparing image scheme ---')
data = self.data
image_scheme = objects.ImageScheme()
# FIXME(agordeev): this piece of code for fetching additional image
# meta data should be factored out of this particular nailgun driver
# into more common and absract data getter which should be able to deal
# with various data sources (local file, http(s), etc.) and different
# data formats ('blob', json, yaml, etc.).
# So, the manager will combine and manipulate all those multiple data
# getter instances.
# Also, the initial data source should be set to sort out chicken/egg
# problem. Command line option may be useful for such a case.
# BUG: https://bugs.launchpad.net/fuel/+bug/1430418
root_uri = data['ks_meta']['image_data']['/']['uri']
filename = os.path.basename(urlparse(root_uri).path).split('.')[0] + \
'.yaml'
metadata_url = urljoin(root_uri, filename)
try:
image_meta = yaml.load(
utils.init_http_request(metadata_url).text)
except Exception as e:
LOG.exception(e)
LOG.debug('Failed to fetch/decode image meta data')
image_meta = {}
# We assume for every file system user may provide a separate
# file system image. For example if partitioning scheme has
# /, /boot, /var/lib file systems then we will try to get images
# for all those mount points. Images data are to be defined
# at provision.json -> ['ks_meta']['image_data']
LOG.debug('Looping over all images in provision data')
for mount_point, image_data in six.iteritems(
data['ks_meta']['image_data']):
LOG.debug('Adding image for fs %s: uri=%s format=%s container=%s' %
(mount_point, image_data['uri'],
image_data['format'], image_data['container']))
iname = os.path.basename(urlparse(image_data['uri']).path)
imeta = next(itertools.chain(
(img for img in image_meta.get('images', [])
if img['container_name'] == iname), [{}]))
image_scheme.add_image(
uri=image_data['uri'],
target_device=self.partition_scheme.fs_by_mount(
mount_point).device,
format=image_data['format'],
container=image_data['container'],
size=imeta.get('raw_size'),
md5=imeta.get('raw_md5'),
)
return image_scheme
def parse_image_scheme(self):
LOG.debug('--- Preparing image scheme ---')
data = self.data
image_scheme = objects.ImageScheme()
#FIXME(agordeev): this piece of code for fetching additional image
# meta data should be factored out of this particular nailgun driver
# into more common and absract data getter which should be able to deal
# with various data sources (local file, http(s), etc.) and different
# data formats ('blob', json, yaml, etc.).
# So, the manager will combine and manipulate all those multiple data
# getter instances.
# Also, the initial data source should be set to sort out chicken/egg
# problem. Command line option may be useful for such a case.
# BUG: https://bugs.launchpad.net/fuel/+bug/1430418
root_uri = data['ks_meta']['image_data']['/']['uri']
filename = os.path.basename(urlparse(root_uri).path).split('.')[0] + \
'.yaml'
metadata_url = urljoin(root_uri, filename)
try:
image_meta = yaml.load(
utils.init_http_request(metadata_url).text)
except Exception as e:
LOG.exception(e)
LOG.debug('Failed to fetch/decode image meta data')
image_meta = {}
# We assume for every file system user may provide a separate
# file system image. For example if partitioning scheme has
# /, /boot, /var/lib file systems then we will try to get images
# for all those mount points. Images data are to be defined
# at provision.json -> ['ks_meta']['image_data']
LOG.debug('Looping over all images in provision data')
for mount_point, image_data in six.iteritems(
data['ks_meta']['image_data']):
LOG.debug('Adding image for fs %s: uri=%s format=%s container=%s' %
(mount_point, image_data['uri'],
image_data['format'], image_data['container']))
iname = os.path.basename(urlparse(image_data['uri']).path)
imeta = next(itertools.chain(
(img for img in image_meta.get('images', [])
if img['container_name'] == iname), [{}]))
image_scheme.add_image(
uri=image_data['uri'],
target_device=self.partition_scheme.fs_by_mount(
mount_point).device,
format=image_data['format'],
container=image_data['container'],
size=imeta.get('raw_size'),
md5=imeta.get('raw_md5'),
)
return image_scheme
def next(self):
while self.processed_bytes < self.length:
try:
data = self.response_obj.raw.read(CONF.data_chunk_size)
if not data:
raise errors.HttpUrlConnectionError(
'Could not receive data: URL=%s, range=%s' %
(self.url, self.processed_bytes))
except Exception as exc:
LOG.exception(exc)
self.response_obj = utils.init_http_request(
self.url, byte_range=self.processed_bytes)
continue
else:
self.processed_bytes += len(data)
return data
raise StopIteration()
def get_release_file(uri, suite, section):
"""Download repo's Release file, parse it and returns an apt
preferences line for this repo.
:param repo: a repo as dict
:returns: a string with apt preferences rules
"""
if section:
# We can't use urljoin here because it works pretty bad in
# cases when 'uri' doesn't have a trailing slash.
download_uri = os.path.join(uri, 'dists', suite, 'Release')
else:
# Well, we have a flat repo case, so we should download Release
# file from a different place. Please note, we have to strip
# a leading slash from suite because otherwise the download
# link will be wrong.
download_uri = os.path.join(uri, suite.lstrip('/'), 'Release')
return utils.init_http_request(download_uri).text
def get_release_file(uri, suite, section):
"""Download and parse repo's Release file
It and returns an apt preferences line for specified repo.
:param repo: a repo as dict
:returns: a string with apt preferences rules
"""
if section:
# We can't use urljoin here because it works pretty bad in
# cases when 'uri' doesn't have a trailing slash.
download_uri = os.path.join(uri, 'dists', suite, 'Release')
else:
# Well, we have a flat repo case, so we should download Release
# file from a different place. Please note, we have to strip
# a leading slash from suite because otherwise the download
# link will be wrong.
download_uri = os.path.join(uri, suite.lstrip('/'), 'Release')
return utils.init_http_request(download_uri).text
def get_release_file(uri, suite, section, proxies=None,
direct_repo_addrs=None):
"""Download and parse repo's Release file
Returns an apt preferences for specified repo.
:param proxies: Dict protocol:uri format
:param direct_repo_addrs: List of addresses which should be bypassed
by proxy
:returns: a string with apt preferences rules
"""
if section:
# We can't use urljoin here because it works pretty bad in
# cases when 'uri' doesn't have a trailing slash.
download_uri = os.path.join(uri, 'dists', suite, 'Release')
else:
# Well, we have a flat repo case, so we should download Release
# file from a different place. Please note, we have to strip
# a leading slash from suite because otherwise the download
# link will be wrong.
download_uri = os.path.join(uri, suite.lstrip('/'), 'Release')
return utils.init_http_request(download_uri, proxies=proxies,
noproxy_addrs=direct_repo_addrs).text
def test_init_http_request_ok(self, mock_req):
utils.init_http_request('http://fake_url')
mock_req.assert_called_once_with(
'http://fake_url', stream=True, timeout=CONF.http_request_timeout,
headers={'Range': 'bytes=0-'}, proxies=None)