def _handle_response(self, resp): if not resp.ok: LOG.debug("Request returned failure status %s." % resp.status_code) raise exc.from_response(resp, resp.content) elif (resp.status_code == requests.codes.MULTIPLE_CHOICES and resp.request.path_url != '/versions'): # NOTE(flaper87): Eventually, we'll remove the check on `versions` # which is a bug (1491350) on the server. raise exc.from_response(resp) content_type = resp.headers.get('Content-Type') # Read body into string if it isn't obviously image data if content_type == 'application/octet-stream': # Do not read all response in memory when downloading an image. body_iter = _close_after_stream(resp, CHUNKSIZE) else: content = resp.text if content_type and content_type.startswith('application/json'): # Let's use requests json method, it should take care of # response encoding body_iter = resp.json() else: body_iter = six.StringIO(content) try: body_iter = json.loads(''.join([c for c in body_iter])) except ValueError: body_iter = None return resp, body_iter
def _handle_response(self, resp): if not resp.ok: LOG.debug("Request returned failure status %s." % resp.status_code) raise exc.from_response(resp, resp.content) elif resp.status_code == requests.codes.MULTIPLE_CHOICES: raise exc.from_response(resp) content_type = resp.headers.get("Content-Type") # Read body into string if it isn't obviously image data if content_type == "application/octet-stream": # Do not read all response in memory when downloading an image. body_iter = _close_after_stream(resp, CHUNKSIZE) else: content = resp.text if content_type and content_type.startswith("application/json"): # Let's use requests json method, it should take care of # response encoding body_iter = resp.json() else: body_iter = six.StringIO(content) try: body_iter = json.loads("".join([c for c in body_iter])) except ValueError: body_iter = None return resp, body_iter
def request(self, url, method, **kwargs): headers = kwargs.setdefault('headers', {}) content_type = headers.get('Content-Type', 'application/octet-stream') data = kwargs.pop("data", None) if data is not None and not isinstance(data, six.string_types): try: data = json.dumps(data) content_type = 'application/json' except TypeError: # Here we assume it's a file-like object and we'll chunk it data = chunk_body(data) headers['Content-Type'] = content_type kwargs['data'] = data kwargs['raise_exc'] = False kwargs['stream'] = content_type == 'application/octet-stream' try: resp = super(SessionClient, self).request(url, method, **kwargs) except ksc_exc.RequestTimeout as e: message = ("Error communicating with %(endpoint)s %(e)s" % dict(url=conn_url, e=e)) raise exc.InvalidEndpoint(message=message) except ksc_exc.ConnectionRefused as e: conn_url = self.get_endpoint(auth=kwargs.get('auth')) message = ("Error finding address for %(url)s: %(e)s" % dict(url=conn_url, e=e)) raise exc.CommunicationError(message=message) if not resp.ok: LOG.debug("Request returned failure status %s." % resp.status_code) raise exc.from_response(resp, resp.content) elif resp.status_code == requests.codes.MULTIPLE_CHOICES: raise exc.from_response(resp) content_type = resp.headers.get('Content-Type') # Read body into string if it isn't obviously image data if content_type == 'application/octet-stream': # Do not read all response in memory when downloading an image. body_iter = _close_after_stream(resp, CHUNKSIZE) else: content = resp.content if content_type and content_type.startswith('application/json'): # Let's use requests json method, it should take care of # response encoding body_iter = resp.json() else: body_iter = six.StringIO(content) try: body_iter = json.loads(''.join([c for c in body_iter])) except ValueError: body_iter = None return resp, body_iter
def test_format_no_content_type(self): mock_resp = mock.Mock() mock_resp.status_code = 400 mock_resp.headers = {'content-type': 'application/octet-stream'} body = b'Error \n\n' err = exc.from_response(mock_resp, body) self.assertEqual('Error \n', err.details)
def _http_request(self, url, method, **kwargs): """ Send an http request with the specified characteristics. Wrapper around httplib.HTTP(S)Connection.request to handle tasks such as setting headers and error handling. """ # Copy the kwargs so we can reuse the original in case of redirects kwargs['headers'] = copy.deepcopy(kwargs.get('headers', {})) kwargs['headers'].setdefault('User-Agent', USER_AGENT) if self.auth_token: kwargs['headers'].setdefault('X-Auth-Token', self.auth_token) self.log_curl_request(method, url, kwargs) conn = self.get_connection() try: conn_url = os.path.normpath('%s/%s' % (self.endpoint_path, url)) conn.request(method, conn_url, **kwargs) resp = conn.getresponse() except socket.gaierror as e: message = "Error finding address for %(url)s: %(e)s" % locals() raise exc.InvalidEndpoint(message=message) except (socket.error, socket.timeout) as e: endpoint = self.endpoint message = "Error communicating with %(endpoint)s %(e)s" % locals() raise exc.CommunicationError(message=message) body_iter = ResponseBodyIterator(resp) # Read body into string if it isn't obviously image data if resp.getheader('content-type', None) != 'application/octet-stream': body_str = ''.join([chunk for chunk in body_iter]) self.log_http_response(resp, body_str) body_iter = StringIO.StringIO(body_str) else: self.log_http_response(resp) if 400 <= resp.status < 600: LOG.error("Request returned failure status.") raise exc.from_response(resp) elif resp.status in (301, 302, 305): # Redirected. Reissue the request to the new location. return self._http_request(resp['location'], method, **kwargs) elif resp.status == 300: raise exc.from_response(resp) return resp, body_iter
def test_handles_html(self): """exc.from_response should not print HTML.""" mock_resp = mock.Mock() mock_resp.status_code = 404 mock_resp.text = HTML_MSG mock_resp.headers = {"content-type": "text/html"} err = exc.from_response(mock_resp, HTML_MSG) self.assertIsInstance(err, exc.HTTPNotFound) self.assertEqual("404 Entity Not Found: Entity could not be found", err.details)
def test_handles_html(self): """exc.from_response should not print HTML.""" mock_resp = mock.Mock() mock_resp.status_code = 404 mock_resp.text = HTML_MSG mock_resp.headers = { "content-type": "text/html" } err = exc.from_response(mock_resp, HTML_MSG) self.assertIsInstance(err, exc.HTTPNotFound) self.assertEqual("404 Entity Not Found: Entity could not be found", err.details)
def test_handles_json(self): """exc.from_response should not print JSON.""" mock_resp = mock.Mock() mock_resp.status_code = 413 mock_resp.json.return_value = { "overLimit": { "code": 413, "message": "OverLimit Retry...", "details": "Error Details...", "retryAt": "2014-12-03T13:33:06Z" } } mock_resp.headers = {"content-type": "application/json"} err = exc.from_response(mock_resp, "Non-empty body") self.assertIsInstance(err, exc.HTTPOverLimit) self.assertEqual("OverLimit Retry...", err.details)
def test_handles_json(self): """exc.from_response should not print JSON.""" mock_resp = mock.Mock() mock_resp.status_code = 413 mock_resp.json.return_value = { "overLimit": { "code": 413, "message": "OverLimit Retry...", "details": "Error Details...", "retryAt": "2014-12-03T13:33:06Z" } } mock_resp.headers = { "content-type": "application/json" } err = exc.from_response(mock_resp, "Non-empty body") self.assertIsInstance(err, exc.HTTPOverLimit) self.assertEqual("OverLimit Retry...", err.details)
def test_from_response(self): """exc.from_response should return instance of an HTTP exception.""" mock_resp = mock.Mock() mock_resp.status_code = 400 out = exc.from_response(mock_resp) self.assertIsInstance(out, exc.HTTPBadRequest)
def _request(self, method, url, **kwargs): """Send an http request with the specified characteristics. Wrapper around httplib.HTTP(S)Connection.request to handle tasks such as setting headers and error handling. """ # Copy the kwargs so we can reuse the original in case of redirects headers = kwargs.pop("headers", {}) headers = headers and copy.deepcopy(headers) or {} # Default Content-Type is octet-stream content_type = headers.get('Content-Type', 'application/octet-stream') def chunk_body(body): chunk = body while chunk: chunk = body.read(CHUNKSIZE) yield chunk data = kwargs.pop("data", None) if data is not None and not isinstance(data, six.string_types): try: data = json.dumps(data) content_type = 'application/json' except TypeError: # Here we assume it's # a file-like object # and we'll chunk it data = chunk_body(data) headers['Content-Type'] = content_type # Note(flaper87): Before letting headers / url fly, # they should be encoded otherwise httplib will # complain. headers = self.encode_headers(headers) try: conn_url = "%s/%s" % (self.endpoint, url) self.log_curl_request(method, conn_url, headers, data, kwargs) resp = self.session.request(method, conn_url, data=data, stream=True, headers=headers, **kwargs) except requests.exceptions.Timeout as e: message = ("Error communicating with %(endpoint)s %(e)s" % dict(url=conn_url, e=e)) raise exc.InvalidEndpoint(message=message) except requests.exceptions.ConnectionError as e: message = ("Error finding address for %(url)s: %(e)s" % dict(url=conn_url, e=e)) raise exc.CommunicationError(message=message) except socket.gaierror as e: message = "Error finding address for %s: %s" % ( self.endpoint_hostname, e) raise exc.InvalidEndpoint(message=message) except (socket.error, socket.timeout) as e: endpoint = self.endpoint message = ("Error communicating with %(endpoint)s %(e)s" % { 'endpoint': endpoint, 'e': e }) raise exc.CommunicationError(message=message) if not resp.ok: LOG.error("Request returned failure status %s." % resp.status_code) raise exc.from_response(resp, resp.content) elif resp.status_code == requests.codes.MULTIPLE_CHOICES: raise exc.from_response(resp) content_type = resp.headers.get('Content-Type') # Read body into string if it isn't obviously image data if content_type == 'application/octet-stream': # Do not read all response in memory when # downloading an image. body_iter = resp.iter_content(chunk_size=CHUNKSIZE) self.log_http_response(resp) else: content = resp.content self.log_http_response(resp, content) if content_type and content_type.startswith('application/json'): # Let's use requests json method, # it should take care of response # encoding body_iter = resp.json() else: body_iter = six.StringIO(content) return resp, body_iter
def _http_request(self, url, method, **kwargs): """Send an http request with the specified characteristics. Wrapper around httplib.HTTP(S)Connection.request to handle tasks such as setting headers and error handling. """ # Copy the kwargs so we can reuse the original in case of redirects kwargs["headers"] = copy.deepcopy(kwargs.get("headers", {})) kwargs["headers"].setdefault("User-Agent", USER_AGENT) if self.auth_token: kwargs["headers"].setdefault("X-Auth-Token", self.auth_token) if self.identity_headers: for k, v in self.identity_headers.iteritems(): kwargs["headers"].setdefault(k, v) self.log_curl_request(method, url, kwargs) conn = self.get_connection() # Note(flaper87): Before letting headers / url fly, # they should be encoded otherwise httplib will # complain. If we decide to rely on python-request # this wont be necessary anymore. kwargs["headers"] = self.encode_headers(kwargs["headers"]) try: if self.endpoint_path: url = "%s/%s" % (self.endpoint_path, url) conn_url = posixpath.normpath(url) # Note(flaper87): Ditto, headers / url # encoding to make httplib happy. conn_url = strutils.safe_encode(conn_url) if kwargs["headers"].get("Transfer-Encoding") == "chunked": conn.putrequest(method, conn_url) for header, value in kwargs["headers"].items(): conn.putheader(header, value) conn.endheaders() chunk = kwargs["body"].read(CHUNKSIZE) # Chunk it, baby... while chunk: conn.send("%x\r\n%s\r\n" % (len(chunk), chunk)) chunk = kwargs["body"].read(CHUNKSIZE) conn.send("0\r\n\r\n") else: conn.request(method, conn_url, **kwargs) resp = conn.getresponse() except socket.gaierror as e: message = "Error finding address for %s: %s" % (self.endpoint_hostname, e) raise exc.InvalidEndpoint(message=message) except (socket.error, socket.timeout) as e: endpoint = self.endpoint message = "Error communicating with %(endpoint)s %(e)s" % locals() raise exc.CommunicationError(message=message) body_iter = ResponseBodyIterator(resp) # Read body into string if it isn't obviously image data if resp.getheader("content-type", None) != "application/octet-stream": body_str = "".join([chunk for chunk in body_iter]) self.log_http_response(resp, body_str) body_iter = StringIO.StringIO(body_str) else: self.log_http_response(resp) if 400 <= resp.status < 600: LOG.error("Request returned failure status.") raise exc.from_response(resp, body_str) elif resp.status in (301, 302, 305): # Redirected. Reissue the request to the new location. return self._http_request(resp["location"], method, **kwargs) elif resp.status == 300: raise exc.from_response(resp) return resp, body_iter
def _request(self, method, url, **kwargs): """Send an http request with the specified characteristics. Wrapper around httplib.HTTP(S)Connection.request to handle tasks such as setting headers and error handling. """ # Copy the kwargs so we can reuse the original in case of redirects headers = kwargs.pop("headers", {}) headers = headers and copy.deepcopy(headers) or {} if self.identity_headers: for k, v in six.iteritems(self.identity_headers): headers.setdefault(k, v) # Default Content-Type is octet-stream content_type = headers.get("Content-Type", "application/octet-stream") def chunk_body(body): chunk = body while chunk: chunk = body.read(CHUNKSIZE) if chunk == "": break yield chunk data = kwargs.pop("data", None) if data is not None and not isinstance(data, six.string_types): try: data = json.dumps(data) content_type = "application/json" except TypeError: # Here we assume it's # a file-like object # and we'll chunk it data = chunk_body(data) headers["Content-Type"] = content_type stream = True if content_type == "application/octet-stream" else False if osprofiler_web: headers.update(osprofiler_web.get_trace_id_headers()) # Note(flaper87): Before letting headers / url fly, # they should be encoded otherwise httplib will # complain. headers = self.encode_headers(headers) try: if self.endpoint.endswith("/") or url.startswith("/"): conn_url = "%s%s" % (self.endpoint, url) else: conn_url = "%s/%s" % (self.endpoint, url) self.log_curl_request(method, conn_url, headers, data, kwargs) resp = self.session.request(method, conn_url, data=data, stream=stream, headers=headers, **kwargs) except requests.exceptions.Timeout as e: message = "Error communicating with %(endpoint)s %(e)s" % dict(url=conn_url, e=e) raise exc.InvalidEndpoint(message=message) except (requests.exceptions.ConnectionError, ProtocolError) as e: message = "Error finding address for %(url)s: %(e)s" % dict(url=conn_url, e=e) raise exc.CommunicationError(message=message) except socket.gaierror as e: message = "Error finding address for %s: %s" % (self.endpoint_hostname, e) raise exc.InvalidEndpoint(message=message) except (socket.error, socket.timeout) as e: endpoint = self.endpoint message = "Error communicating with %(endpoint)s %(e)s" % {"endpoint": endpoint, "e": e} raise exc.CommunicationError(message=message) if not resp.ok: LOG.debug("Request returned failure status %s." % resp.status_code) raise exc.from_response(resp, resp.text) elif resp.status_code == requests.codes.MULTIPLE_CHOICES: raise exc.from_response(resp) content_type = resp.headers.get("Content-Type") # Read body into string if it isn't obviously image data if content_type == "application/octet-stream": # Do not read all response in memory when # downloading an image. body_iter = _close_after_stream(resp, CHUNKSIZE) self.log_http_response(resp) else: content = resp.text self.log_http_response(resp, content) if content_type and content_type.startswith("application/json"): # Let's use requests json method, # it should take care of response # encoding body_iter = resp.json() else: body_iter = six.StringIO(content) try: body_iter = json.loads("".join([c for c in body_iter])) except ValueError: body_iter = None return resp, body_iter
def _http_request(self, url, method, **kwargs): """Send an http request with the specified characteristics. Wrapper around httplib.HTTP(S)Connection.request to handle tasks such as setting headers and error handling. """ # Copy the kwargs so we can reuse the original in case of redirects kwargs['headers'] = copy.deepcopy(kwargs.get('headers', {})) kwargs['headers'].setdefault('User-Agent', USER_AGENT) if self.auth_token: kwargs['headers'].setdefault('X-Auth-Token', self.auth_token) self.log_curl_request(method, url, kwargs) conn = self.get_connection() # Note(flaper87): Before letting headers / url fly, # they should be encoded otherwise httplib will # complain. If we decide to rely on python-request # this wont be necessary anymore. kwargs['headers'] = self.encode_headers(kwargs['headers']) try: if self.endpoint_path: url = '%s/%s' % (self.endpoint_path, url) conn_url = posixpath.normpath(url) # Note(flaper87): Ditto, headers / url # encoding to make httplib happy. conn_url = strutils.safe_encode(conn_url) if kwargs['headers'].get('Transfer-Encoding') == 'chunked': conn.putrequest(method, conn_url) for header, value in kwargs['headers'].items(): conn.putheader(header, value) conn.endheaders() chunk = kwargs['body'].read(CHUNKSIZE) # Chunk it, baby... while chunk: conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) chunk = kwargs['body'].read(CHUNKSIZE) conn.send('0\r\n\r\n') else: conn.request(method, conn_url, **kwargs) resp = conn.getresponse() except socket.gaierror as e: message = "Error finding address for %s: %s" % ( self.endpoint_hostname, e) raise exc.InvalidEndpoint(message=message) except (socket.error, socket.timeout) as e: endpoint = self.endpoint message = "Error communicating with %(endpoint)s %(e)s" % locals() raise exc.CommunicationError(message=message) body_iter = ResponseBodyIterator(resp) # Read body into string if it isn't obviously image data if resp.getheader('content-type', None) != 'application/octet-stream': body_str = ''.join([chunk for chunk in body_iter]) self.log_http_response(resp, body_str) body_iter = StringIO.StringIO(body_str) else: self.log_http_response(resp) if 400 <= resp.status < 600: LOG.error("Request returned failure status.") raise exc.from_response(resp, body_str) elif resp.status in (301, 302, 305): # Redirected. Reissue the request to the new location. return self._http_request(resp['location'], method, **kwargs) elif resp.status == 300: raise exc.from_response(resp) return resp, body_iter
def _http_request(self, url, method, **kwargs): """Send an http request with the specified characteristics. Wrapper around httplib.HTTP(S)Connection.request to handle tasks such as setting headers and error handling. """ # Copy the kwargs so we can reuse the original in case of redirects kwargs["headers"] = copy.deepcopy(kwargs.get("headers", {})) kwargs["headers"].setdefault("User-Agent", USER_AGENT) if self.auth_token: kwargs["headers"].setdefault("X-Auth-Token", self.auth_token) if self.identity_headers: for k, v in six.iteritems(self.identity_headers): kwargs["headers"].setdefault(k, v) self.log_curl_request(method, url, kwargs) conn = self.get_connection() # Note(flaper87): Before letting headers / url fly, # they should be encoded otherwise httplib will # complain. If we decide to rely on python-request # this wont be necessary anymore. kwargs["headers"] = self.encode_headers(kwargs["headers"]) try: if self.endpoint_path: # NOTE(yuyangbj): this method _http_request could either be # called by API layer, or be called recursively with # redirection. For example, url would be '/v1/images/detail' # from API layer, but url would be 'https://example.com:92/ # v1/images/detail' from recursion. # See bug #1230032 and bug #1208618. if url is not None: all_parts = parse.urlparse(url) if not (all_parts.scheme and all_parts.netloc): norm_parse = posixpath.normpath url = norm_parse("/".join([self.endpoint_path, url])) else: url = self.endpoint_path conn_url = parse.urlsplit(url).geturl() # Note(flaper87): Ditto, headers / url # encoding to make httplib happy. conn_url = strutils.safe_encode(conn_url) if kwargs["headers"].get("Transfer-Encoding") == "chunked": conn.putrequest(method, conn_url) for header, value in kwargs["headers"].items(): conn.putheader(header, value) conn.endheaders() chunk = kwargs["body"].read(CHUNKSIZE) # Chunk it, baby... while chunk: conn.send("%x\r\n%s\r\n" % (len(chunk), chunk)) chunk = kwargs["body"].read(CHUNKSIZE) conn.send("0\r\n\r\n") else: conn.request(method, conn_url, **kwargs) resp = conn.getresponse() except socket.gaierror as e: message = "Error finding address for %s: %s" % (self.endpoint_hostname, e) raise exc.InvalidEndpoint(message=message) except (socket.error, socket.timeout) as e: endpoint = self.endpoint message = "Error communicating with %(endpoint)s %(e)s" % {"endpoint": endpoint, "e": e} raise exc.CommunicationError(message=message) body_iter = ResponseBodyIterator(resp) # Read body into string if it isn't obviously image data if resp.getheader("content-type", None) != "application/octet-stream": body_str = b"".join([to_bytes(chunk) for chunk in body_iter]) self.log_http_response(resp, body_str) body_iter = six.BytesIO(body_str) else: self.log_http_response(resp) if 400 <= resp.status < 600: LOG.debug("Request returned failure status: %d" % resp.status) raise exc.from_response(resp, body_str) elif resp.status in (301, 302, 305): # Redirected. Reissue the request to the new location. return self._http_request(resp.getheader("location", None), method, **kwargs) elif resp.status == 300: raise exc.from_response(resp) return resp, body_iter
def test_from_response(self): """exc.from_response should return instance of an HTTP exception.""" out = exc.from_response(FakeResponse(400)) self.assertTrue(isinstance(out, exc.HTTPBadRequest))
def _http_request(self, url, method, **kwargs): """Send an http request with the specified characteristics. Wrapper around httplib.HTTP(S)Connection.request to handle tasks such as setting headers and error handling. """ # Copy the kwargs so we can reuse the original in case of redirects kwargs['headers'] = copy.deepcopy(kwargs.get('headers', {})) kwargs['headers'].setdefault('User-Agent', USER_AGENT) if self.auth_token: kwargs['headers'].setdefault('X-Auth-Token', self.auth_token) if self.identity_headers: for k, v in self.identity_headers.iteritems(): kwargs['headers'].setdefault(k, v) self.log_curl_request(method, url, kwargs) conn = self.get_connection() # Note(flaper87): Before letting headers / url fly, # they should be encoded otherwise httplib will # complain. If we decide to rely on python-request # this wont be necessary anymore. kwargs['headers'] = self.encode_headers(kwargs['headers']) try: if self.endpoint_path: # NOTE(yuyangbj): this method _http_request could either be # called by API layer, or be called recursively with # redirection. For example, url would be '/v1/images/detail' # from API layer, but url would be 'https://example.com:92/ # v1/images/detail' from recursion. # See bug #1230032 and bug #1208618. if url is not None: all_parts = urlparse.urlparse(url) if not (all_parts.scheme and all_parts.netloc): norm_parse = posixpath.normpath url = norm_parse('/'.join([self.endpoint_path, url])) else: url = self.endpoint_path conn_url = urlparse.urlsplit(url).geturl() # Note(flaper87): Ditto, headers / url # encoding to make httplib happy. conn_url = strutils.safe_encode(conn_url) if kwargs['headers'].get('Transfer-Encoding') == 'chunked': conn.putrequest(method, conn_url) for header, value in kwargs['headers'].items(): conn.putheader(header, value) conn.endheaders() chunk = kwargs['body'].read(CHUNKSIZE) # Chunk it, baby... while chunk: conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) chunk = kwargs['body'].read(CHUNKSIZE) conn.send('0\r\n\r\n') else: conn.request(method, conn_url, **kwargs) resp = conn.getresponse() except socket.gaierror as e: message = "Error finding address for %s: %s" % ( self.endpoint_hostname, e) raise exc.InvalidEndpoint(message=message) except (socket.error, socket.timeout) as e: endpoint = self.endpoint message = ("Error communicating with %(endpoint)s %(e)s" % { 'endpoint': endpoint, 'e': e }) raise exc.CommunicationError(message=message) body_iter = ResponseBodyIterator(resp) # Read body into string if it isn't obviously image data if resp.getheader('content-type', None) != 'application/octet-stream': body_str = ''.join([chunk for chunk in body_iter]) self.log_http_response(resp, body_str) body_iter = StringIO.StringIO(body_str) else: self.log_http_response(resp) if 400 <= resp.status < 600: LOG.error("Request returned failure status.") raise exc.from_response(resp, body_str) elif resp.status in (301, 302, 305): # Redirected. Reissue the request to the new location. return self._http_request(resp.getheader('location', None), method, **kwargs) elif resp.status == 300: raise exc.from_response(resp) return resp, body_iter
def _request(self, method, url, **kwargs): """Send an http request with the specified characteristics. Wrapper around httplib.HTTP(S)Connection.request to handle tasks such as setting headers and error handling. """ # Copy the kwargs so we can reuse the original in case of redirects headers = kwargs.pop("headers", {}) headers = headers and copy.deepcopy(headers) or {} # Default Content-Type is octet-stream content_type = headers.get('Content-Type', 'application/octet-stream') def chunk_body(body): chunk = body while chunk: chunk = body.read(CHUNKSIZE) yield chunk data = kwargs.pop("data", None) if data is not None and not isinstance(data, six.string_types): try: data = json.dumps(data) content_type = 'application/json' except TypeError: # Here we assume it's # a file-like object # and we'll chunk it data = chunk_body(data) headers['Content-Type'] = content_type # Note(flaper87): Before letting headers / url fly, # they should be encoded otherwise httplib will # complain. headers = self.encode_headers(headers) try: conn_url = "%s/%s" % (self.endpoint, url) self.log_curl_request(method, conn_url, headers, data, kwargs) resp = self.session.request(method, conn_url, data=data, stream=True, headers=headers, **kwargs) except requests.exceptions.Timeout as e: message = ("Error communicating with %(endpoint)s %(e)s" % dict(url=conn_url, e=e)) raise exc.InvalidEndpoint(message=message) except requests.exceptions.ConnectionError as e: message = ("Error finding address for %(url)s: %(e)s" % dict(url=conn_url, e=e)) raise exc.CommunicationError(message=message) except socket.gaierror as e: message = "Error finding address for %s: %s" % ( self.endpoint_hostname, e) raise exc.InvalidEndpoint(message=message) except (socket.error, socket.timeout) as e: endpoint = self.endpoint message = ("Error communicating with %(endpoint)s %(e)s" % {'endpoint': endpoint, 'e': e}) raise exc.CommunicationError(message=message) if not resp.ok: LOG.error("Request returned failure status %s." % resp.status_code) raise exc.from_response(resp, resp.content) elif resp.status_code == requests.codes.MULTIPLE_CHOICES: raise exc.from_response(resp) content_type = resp.headers.get('Content-Type') # Read body into string if it isn't obviously image data if content_type == 'application/octet-stream': # Do not read all response in memory when # downloading an image. body_iter = resp.iter_content(chunk_size=CHUNKSIZE) self.log_http_response(resp) else: content = resp.content self.log_http_response(resp, content) if content_type and content_type.startswith('application/json'): # Let's use requests json method, # it should take care of response # encoding body_iter = resp.json() else: body_iter = six.StringIO(content) return resp, body_iter