def test_invocation_flow(client): client.ping.return_value = SBusResponse(True, 'OK') client.stop_daemon.return_value = SBusResponse(True, 'OK') client.start_daemon.return_value = SBusResponse(True, 'OK') sresp = self.gateway.invocation_flow(st_req, extra_sources) eventlet.sleep(0.1) file_like = FileLikeIter(sresp.data_iter) self.assertEqual(b'something', file_like.read())
def test_invocation_flow(): sresp = self.gateway.invocation_flow(st_req) file_like = FileLikeIter(sresp.data_iter) self.assertEqual('something', file_like.read())
class FileWrapper(object): def __init__(self, swift_client, account, container, key, headers={}): self._swift = swift_client self._account = account self._container = container self._key = key self.swift_req_hdrs = headers self._bytes_read = 0 self.open_object_stream() def open_object_stream(self): status, self._headers, body = self._swift.get_object( self._account, self._container, self._key, headers=self.swift_req_hdrs) if status != 200: raise RuntimeError('Failed to get the object') self._bytes_read = 0 self._swift_stream = body self._iter = FileLikeIter(body) self._s3_headers = convert_to_s3_headers(self._headers) def tell(self): return self._bytes_read def seek(self, pos, flag=0): if pos != 0: raise RuntimeError('Arbitrary seeks are not supported') if self._bytes_read == 0: return self._swift_stream.close() self.open_object_stream() def reset(self, *args, **kwargs): self.seek(0) def read(self, size=-1): if self._bytes_read == self.__len__(): return '' data = self._iter.read(size) self._bytes_read += len(data) # TODO: we do not need to read an extra byte after # https://review.openstack.org/#/c/363199/ is released if self._bytes_read == self.__len__(): self._iter.read(1) self._swift_stream.close() return data def __len__(self): if 'Content-Length' not in self._headers: raise RuntimeError('Length is not implemented') return int(self._headers['Content-Length']) def __iter__(self): return self._iter def get_s3_headers(self): return self._s3_headers def get_headers(self): return self._headers def close(self): self._swift_stream.close()
def test_invocation_flow(): sresp = self.gateway.invocation_flow(st_req, extra_sources) eventlet.sleep(0.1) file_like = FileLikeIter(sresp.data_iter) self.assertEqual('something', file_like.read())
def direct_put_object(node, part, account, container, name, contents, content_length=None, etag=None, content_type=None, headers=None, conn_timeout=5, response_timeout=15, chunk_size=65535): """ Put object directly from the object server. :param node: node dictionary from the ring :param part: partition the container is on :param account: account name :param container: container name :param name: object name :param contents: an iterable or string to read object data from :param content_length: value to send as content-length header :param etag: etag of contents :param content_type: value to send as content-type header :param headers: additional headers to include in the request :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :param chunk_size: if defined, chunk size of data to send. :returns: etag from the server response :raises ClientException: HTTP PUT request failed """ path = '/%s/%s/%s' % (account, container, name) if headers is None: headers = {} if etag: headers['ETag'] = etag.strip('"') if content_length is not None: headers['Content-Length'] = str(content_length) else: for n, v in headers.items(): if n.lower() == 'content-length': content_length = int(v) if content_type is not None: headers['Content-Type'] = content_type else: headers['Content-Type'] = 'application/octet-stream' if not contents: headers['Content-Length'] = '0' if isinstance(contents, six.string_types): contents = [contents] # Incase the caller want to insert an object with specific age add_ts = 'X-Timestamp' not in headers if content_length is None: headers['Transfer-Encoding'] = 'chunked' with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'PUT', path, headers=gen_headers(headers, add_ts)) contents_f = FileLikeIter(contents) if content_length is None: chunk = contents_f.read(chunk_size) while chunk: conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) chunk = contents_f.read(chunk_size) conn.send('0\r\n\r\n') else: left = content_length while left > 0: size = chunk_size if size > left: size = left chunk = contents_f.read(size) if not chunk: break conn.send(chunk) left -= len(chunk) with Timeout(response_timeout): resp = conn.getresponse() resp.read() if not is_success(resp.status): raise DirectClientException('Object', 'PUT', node, part, path, resp) return resp.getheader('etag').strip('"')
def direct_put_object(node, part, account, container, name, contents, content_length=None, etag=None, content_type=None, headers=None, conn_timeout=5, response_timeout=15, chunk_size=65535): """ Put object directly from the object server. :param node: node dictionary from the ring :param part: partition the container is on :param account: account name :param container: container name :param name: object name :param contents: an iterable or string to read object data from :param content_length: value to send as content-length header :param etag: etag of contents :param content_type: value to send as content-type header :param headers: additional headers to include in the request :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :param chunk_size: if defined, chunk size of data to send. :returns: etag from the server response :raises ClientException: HTTP PUT request failed """ path = '/%s/%s/%s' % (account, container, name) if headers is None: headers = {} if etag: headers['ETag'] = etag.strip('"') if content_length is not None: headers['Content-Length'] = str(content_length) else: for n, v in headers.items(): if n.lower() == 'content-length': content_length = int(v) if content_type is not None: headers['Content-Type'] = content_type else: headers['Content-Type'] = 'application/octet-stream' if not contents: headers['Content-Length'] = '0' if isinstance(contents, basestring): contents = [contents] #Incase the caller want to insert an object with specific age add_ts = 'X-Timestamp' not in headers if content_length is None: headers['Transfer-Encoding'] = 'chunked' with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'PUT', path, headers=gen_headers(headers, add_ts)) contents_f = FileLikeIter(contents) if content_length is None: chunk = contents_f.read(chunk_size) while chunk: conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) chunk = contents_f.read(chunk_size) conn.send('0\r\n\r\n') else: left = content_length while left > 0: size = chunk_size if size > left: size = left chunk = contents_f.read(size) if not chunk: break conn.send(chunk) left -= len(chunk) with Timeout(response_timeout): resp = conn.getresponse() resp.read() if not is_success(resp.status): raise DirectClientException('Object', 'PUT', node, part, path, resp) return resp.getheader('etag').strip('"')
def _make_req(node, part, method, path, headers, stype, conn_timeout=5, response_timeout=15, send_timeout=15, contents=None, content_length=None, chunk_size=65535): """ Make request to backend storage node. (i.e. 'Account', 'Container', 'Object') :param node: a node dict from a ring :param part: an integer, the partition number :param method: a string, the HTTP method (e.g. 'PUT', 'DELETE', etc) :param path: a string, the request path :param headers: a dict, header name => value :param stype: a string, describing the type of service :param conn_timeout: timeout while waiting for connection; default is 5 seconds :param response_timeout: timeout while waiting for response; default is 15 seconds :param send_timeout: timeout for sending request body; default is 15 seconds :param contents: an iterable or string to read object data from :param content_length: value to send as content-length header :param chunk_size: if defined, chunk size of data to send :returns: an HTTPResponse object :raises DirectClientException: if the response status is not 2xx :raises eventlet.Timeout: if either conn_timeout or response_timeout is exceeded """ if contents is not None: if content_length is not None: headers['Content-Length'] = str(content_length) else: for n, v in headers.items(): if n.lower() == 'content-length': content_length = int(v) if not contents: headers['Content-Length'] = '0' if isinstance(contents, six.string_types): contents = [contents] if content_length is None: headers['Transfer-Encoding'] = 'chunked' with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, method, path, headers=headers) if contents is not None: contents_f = FileLikeIter(contents) with Timeout(send_timeout): if content_length is None: chunk = contents_f.read(chunk_size) while chunk: conn.send(b'%x\r\n%s\r\n' % (len(chunk), chunk)) chunk = contents_f.read(chunk_size) conn.send(b'0\r\n\r\n') else: left = content_length while left > 0: size = chunk_size if size > left: size = left chunk = contents_f.read(size) if not chunk: break conn.send(chunk) left -= len(chunk) with Timeout(response_timeout): resp = conn.getresponse() resp.read() if not is_success(resp.status): raise DirectClientException(stype, method, node, part, path, resp) return resp