def test_init_http_request_proxy(self, mock_req): proxies = {'http': 'proxy1'} noproxy_addrs = ['fake.hostname.11.42'] utils.init_http_request('http://fake_url', proxies=proxies, noproxy_addrs=noproxy_addrs) mock_req.assert_called_once_with( 'http://fake_url', stream=True, timeout=CONF.http_request_timeout, headers={'Range': 'bytes=0-'}, proxies=proxies)
def test_init_http_request_ok(self, mock_req): utils.init_http_request("http://fake_url") mock_req.assert_called_once_with( "http://fake_url", stream=True, timeout=CONF.http_request_timeout, headers={"Range": "bytes=0-"}, proxies=None, )
def test_init_http_requests_bypass_proxy(self, mock_req): proxies = {"http": "proxy1"} hostname = "fake.hostname.11.42" url = "http://{0}/web".format(hostname) noproxy_addrs = [hostname] utils.init_http_request(url, proxies=proxies, noproxy_addrs=noproxy_addrs) mock_req.assert_called_once_with( url, stream=True, timeout=CONF.http_request_timeout, headers={"Range": "bytes=0-"}, proxies=None )
def test_init_http_requests_bypass_proxy(self, mock_req): proxies = {'http': 'proxy1'} hostname = 'fake.hostname.11.42' url = "http://{0}/web".format(hostname) noproxy_addrs = [hostname] utils.init_http_request(url, proxies=proxies, noproxy_addrs=noproxy_addrs) mock_req.assert_called_once_with( url, stream=True, timeout=CONF.http_request_timeout, headers={'Range': 'bytes=0-'}, proxies=None)
def test_init_http_request_proxy(self, mock_req): proxies = {"http": "proxy1"} noproxy_addrs = ["fake.hostname.11.42"] utils.init_http_request("http://fake_url", proxies=proxies, noproxy_addrs=noproxy_addrs) mock_req.assert_called_once_with( "http://fake_url", stream=True, timeout=CONF.http_request_timeout, headers={"Range": "bytes=0-"}, proxies=proxies, )
def get_release_file(uri, suite, section, proxies=None, direct_repo_addrs=None): """Download and parse repo's Release file Returns an apt preferences for specified repo. :param proxies: Dict protocol:uri format :param direct_repo_addrs: List of addresses which should be bypassed by proxy :returns: a string with apt preferences rules """ if section: # We can't use urljoin here because it works pretty bad in # cases when 'uri' doesn't have a trailing slash. download_uri = os.path.join(uri, 'dists', suite, 'Release') else: # Well, we have a flat repo case, so we should download Release # file from a different place. Please note, we have to strip # a leading slash from suite because otherwise the download # link will be wrong. download_uri = os.path.join(uri, suite.lstrip('/'), 'Release') return utils.init_http_request(download_uri, proxies=proxies, noproxy_addrs=direct_repo_addrs).text
def parse_image_meta(self): LOG.debug('--- Preparing image metadata ---') data = self.data # FIXME(agordeev): this piece of code for fetching additional image # meta data should be factored out of this particular nailgun driver # into more common and absract data getter which should be able to deal # with various data sources (local file, http(s), etc.) and different # data formats ('blob', json, yaml, etc.). # So, the manager will combine and manipulate all those multiple data # getter instances. # Also, the initial data source should be set to sort out chicken/egg # problem. Command line option may be useful for such a case. # BUG: https://bugs.launchpad.net/fuel/+bug/1430418 root_uri = data['ks_meta']['image_data']['/']['uri'] filename = os.path.basename(urlparse(root_uri).path).split('.')[0] + \ '.yaml' metadata_url = urljoin(root_uri, filename) try: image_meta = yaml.load( utils.init_http_request(metadata_url).text) except Exception as e: LOG.exception(e) LOG.debug('Failed to fetch/decode image meta data') image_meta = {} return image_meta
def test_init_http_request_non_critical_errors(self, mock_req, mock_s): mock_ok = mock.Mock() mock_req.side_effect = [urllib3.exceptions.DecodeError(), urllib3.exceptions.ProxyError(), requests.exceptions.ConnectionError(), requests.exceptions.Timeout(), requests.exceptions.TooManyRedirects(), mock_ok] req_obj = utils.init_http_request('fake_url') self.assertEqual(mock_ok, req_obj)
def image_scheme(self, partition_scheme): LOG.debug('--- Preparing image scheme ---') data = self.data image_scheme = objects.ImageScheme() #FIXME(agordeev): this piece of code for fetching additional image # meta data should be factored out of this particular nailgun driver # into more common and absract data getter which should be able to deal # with various data sources (local file, http(s), etc.) and different # data formats ('blob', json, yaml, etc.). # So, the manager will combine and manipulate all those multiple data # getter instances. # Also, the initial data source should be set to sort out chicken/egg # problem. Command line option may be useful for such a case. # BUG: https://bugs.launchpad.net/fuel/+bug/1430418 root_uri = data['ks_meta']['image_data']['/']['uri'] filename = os.path.basename(urlparse(root_uri).path).split('.')[0] + \ '.yaml' metadata_url = urljoin(root_uri, filename) image_meta = {} raw_image_meta = None try: raw_image_meta = yaml.load( utils.init_http_request(metadata_url).text) except Exception as e: LOG.exception(e) LOG.debug('Failed to fetch/decode image meta data') if raw_image_meta: [image_meta.update(img_info) for img_info in raw_image_meta] # We assume for every file system user may provide a separate # file system image. For example if partitioning scheme has # /, /boot, /var/lib file systems then we will try to get images # for all those mount points. Images data are to be defined # at provision.json -> ['ks_meta']['image_data'] LOG.debug('Looping over all file systems in partition scheme') for fs in partition_scheme.fss: LOG.debug('Processing fs %s' % fs.mount) if fs.mount not in data['ks_meta']['image_data']: LOG.debug('There is no image for fs %s. Skipping.' % fs.mount) continue image_data = data['ks_meta']['image_data'][fs.mount] LOG.debug('Adding image for fs %s: uri=%s format=%s container=%s' % (fs.mount, image_data['uri'], image_data['format'], image_data['container'])) image_scheme.add_image( uri=image_data['uri'], target_device=fs.device, # In the future we will get format and container # from provision.json, but currently it is hard coded. format=image_data['format'], container=image_data['container'], size=image_meta.get(fs.mount, {}).get('size'), md5=image_meta.get(fs.mount, {}).get('md5'), ) return image_scheme
def test_init_http_request_non_critical_errors(self, mock_req, mock_s): mock_ok = mock.Mock() mock_req.side_effect = [urllib3.exceptions.DecodeError(), urllib3.exceptions.ProxyError(), requests.exceptions.ConnectionError(), requests.exceptions.Timeout(), requests.exceptions.TooManyRedirects(), socket.timeout(), mock_ok] req_obj = utils.init_http_request('http://fake_url') self.assertEqual(mock_ok, req_obj)
def __init__(self, url): self.url = str(url) self.response_obj = utils.init_http_request(self.url) self.processed_bytes = 0 try: self.length = int(self.response_obj.headers['content-length']) except (ValueError, KeyError): raise errors.HttpUrlInvalidContentLength( 'Can not get content length for %s' % self.url) else: LOG.debug('Expected content length %s for %s' % (self.length, self.url))
def parse_image_scheme(self): LOG.debug('--- Preparing image scheme ---') data = self.data image_scheme = objects.ImageScheme() # FIXME(agordeev): this piece of code for fetching additional image # meta data should be factored out of this particular nailgun driver # into more common and absract data getter which should be able to deal # with various data sources (local file, http(s), etc.) and different # data formats ('blob', json, yaml, etc.). # So, the manager will combine and manipulate all those multiple data # getter instances. # Also, the initial data source should be set to sort out chicken/egg # problem. Command line option may be useful for such a case. # BUG: https://bugs.launchpad.net/fuel/+bug/1430418 root_uri = data['ks_meta']['image_data']['/']['uri'] filename = os.path.basename(urlparse(root_uri).path).split('.')[0] + \ '.yaml' metadata_url = urljoin(root_uri, filename) try: image_meta = yaml.load( utils.init_http_request(metadata_url).text) except Exception as e: LOG.exception(e) LOG.debug('Failed to fetch/decode image meta data') image_meta = {} # We assume for every file system user may provide a separate # file system image. For example if partitioning scheme has # /, /boot, /var/lib file systems then we will try to get images # for all those mount points. Images data are to be defined # at provision.json -> ['ks_meta']['image_data'] LOG.debug('Looping over all images in provision data') for mount_point, image_data in six.iteritems( data['ks_meta']['image_data']): LOG.debug('Adding image for fs %s: uri=%s format=%s container=%s' % (mount_point, image_data['uri'], image_data['format'], image_data['container'])) iname = os.path.basename(urlparse(image_data['uri']).path) imeta = next(itertools.chain( (img for img in image_meta.get('images', []) if img['container_name'] == iname), [{}])) image_scheme.add_image( uri=image_data['uri'], target_device=self.partition_scheme.fs_by_mount( mount_point).device, format=image_data['format'], container=image_data['container'], size=imeta.get('raw_size'), md5=imeta.get('raw_md5'), ) return image_scheme
def parse_image_scheme(self): LOG.debug('--- Preparing image scheme ---') data = self.data image_scheme = objects.ImageScheme() #FIXME(agordeev): this piece of code for fetching additional image # meta data should be factored out of this particular nailgun driver # into more common and absract data getter which should be able to deal # with various data sources (local file, http(s), etc.) and different # data formats ('blob', json, yaml, etc.). # So, the manager will combine and manipulate all those multiple data # getter instances. # Also, the initial data source should be set to sort out chicken/egg # problem. Command line option may be useful for such a case. # BUG: https://bugs.launchpad.net/fuel/+bug/1430418 root_uri = data['ks_meta']['image_data']['/']['uri'] filename = os.path.basename(urlparse(root_uri).path).split('.')[0] + \ '.yaml' metadata_url = urljoin(root_uri, filename) try: image_meta = yaml.load( utils.init_http_request(metadata_url).text) except Exception as e: LOG.exception(e) LOG.debug('Failed to fetch/decode image meta data') image_meta = {} # We assume for every file system user may provide a separate # file system image. For example if partitioning scheme has # /, /boot, /var/lib file systems then we will try to get images # for all those mount points. Images data are to be defined # at provision.json -> ['ks_meta']['image_data'] LOG.debug('Looping over all images in provision data') for mount_point, image_data in six.iteritems( data['ks_meta']['image_data']): LOG.debug('Adding image for fs %s: uri=%s format=%s container=%s' % (mount_point, image_data['uri'], image_data['format'], image_data['container'])) iname = os.path.basename(urlparse(image_data['uri']).path) imeta = next(itertools.chain( (img for img in image_meta.get('images', []) if img['container_name'] == iname), [{}])) image_scheme.add_image( uri=image_data['uri'], target_device=self.partition_scheme.fs_by_mount( mount_point).device, format=image_data['format'], container=image_data['container'], size=imeta.get('raw_size'), md5=imeta.get('raw_md5'), ) return image_scheme
def next(self): while self.processed_bytes < self.length: try: data = self.response_obj.raw.read(CONF.data_chunk_size) if not data: raise errors.HttpUrlConnectionError( 'Could not receive data: URL=%s, range=%s' % (self.url, self.processed_bytes)) except Exception as exc: LOG.exception(exc) self.response_obj = utils.init_http_request( self.url, byte_range=self.processed_bytes) continue else: self.processed_bytes += len(data) return data raise StopIteration()
def get_release_file(uri, suite, section): """Download repo's Release file, parse it and returns an apt preferences line for this repo. :param repo: a repo as dict :returns: a string with apt preferences rules """ if section: # We can't use urljoin here because it works pretty bad in # cases when 'uri' doesn't have a trailing slash. download_uri = os.path.join(uri, 'dists', suite, 'Release') else: # Well, we have a flat repo case, so we should download Release # file from a different place. Please note, we have to strip # a leading slash from suite because otherwise the download # link will be wrong. download_uri = os.path.join(uri, suite.lstrip('/'), 'Release') return utils.init_http_request(download_uri).text
def get_release_file(uri, suite, section): """Download and parse repo's Release file It and returns an apt preferences line for specified repo. :param repo: a repo as dict :returns: a string with apt preferences rules """ if section: # We can't use urljoin here because it works pretty bad in # cases when 'uri' doesn't have a trailing slash. download_uri = os.path.join(uri, 'dists', suite, 'Release') else: # Well, we have a flat repo case, so we should download Release # file from a different place. Please note, we have to strip # a leading slash from suite because otherwise the download # link will be wrong. download_uri = os.path.join(uri, suite.lstrip('/'), 'Release') return utils.init_http_request(download_uri).text
def test_init_http_request_ok(self, mock_req): utils.init_http_request('http://fake_url') mock_req.assert_called_once_with( 'http://fake_url', stream=True, timeout=CONF.http_request_timeout, headers={'Range': 'bytes=0-'}, proxies=None)