def put(self, content, basename="", bucket=DEFAULT_BUCKET, content_md5=None): """Put a blob in persistent storage :param content: A file-like object in binary read mode. :param basename: Optional name from which the blob name will be derived. This is used to make the unique blob name somewhat recognizable. :param bucket: Optional bucket name used to partition blob data in the persistent storage medium. This may be delimited with slashes (/). It must be a valid relative path. :param content_md5: RFC-1864-compliant Content-MD5 header value. If this parameter is omitted or its value is `None` then content must be a seekable file-like object. NOTE: the value should not be prefixed with `md5-` even though we store it that way. :returns: A `BlobInfo` named tuple. The returned object has a `name` member that must be used to get or delete the blob. It should not be confused with the optional `basename` parameter. """ name = self.get_unique_name(basename) path = self.get_path(name, bucket) if content_md5 is None: params = {"body": content, "headers": {}} calculate_md5(params) content_md5 = params["headers"]["Content-MD5"] obj = self._s3_bucket(create=True).put_object( Key=path, Body=content, ContentMD5=content_md5, ) return BlobInfo(name, obj.content_length, "md5-" + content_md5)
def _send_request(self, request_dict, operation_model, callback=None): # add md5 signature to ensure api calls (such as put bucket policy) will work body = request_dict.get('body') if body and isinstance(body, (bytes, bytearray)): calculate_md5(request_dict) request = self.endpoint.create_request(request_dict, operation_model) req_body = getattr(request.body, 'buf', request.body) request = HTTPRequest(url=request.url, headers=request.headers, method=request.method, body=req_body, validate_cert=False, proxy_host=self.proxy_host, proxy_port=self.proxy_port, connect_timeout=self.connect_timeout, request_timeout=self.request_timeout) if callback is None: # sync return self._process_response(HTTPClient().fetch(request), operation_model=operation_model) # async self.http_client.fetch(request, callback=partial( self._process_response, callback=callback, operation_model=operation_model))
def test_adds_md5_with_file_like_body(self): request_dict = { 'body': six.BytesIO(b'foobar'), 'headers': {} } handlers.calculate_md5(request_dict) self.assertEqual(request_dict['headers']['Content-MD5'], 'OFj2IjCsPJFfMAxmQxLGPw==')
def test_adds_md5_with_bytes_object(self): request_dict = { 'body': b'foobar', 'headers': {} } handlers.calculate_md5(request_dict) self.assertEqual( request_dict['headers']['Content-MD5'], 'OFj2IjCsPJFfMAxmQxLGPw==')
def test_add_md5_with_file_like_body(self): request_dict = { 'body': six.BytesIO(b'foobar'), 'headers': {} } self.md5_digest.return_value = b'8X\xf6"0\xac<\x91_0\x0cfC\x12\xc6?' handlers.calculate_md5(request_dict) self.assertEqual(request_dict['headers']['Content-MD5'], 'OFj2IjCsPJFfMAxmQxLGPw==')
def get_content_md5(content, offset=0, filepos=0): pos = content.tell() print "TELL %s (%s, %s)" % (pos, offset, filepos) try: content.seek(0) params = {"body": content, "headers": {}} calculate_md5(params) return params["headers"]["Content-MD5"] finally: print "SEEK", content.tell(), "->", pos, params["headers"]["Content-MD5"] content.seek(pos)
def test_add_md5_raises_error_when_md5_unavailable(self): credentials = Credentials('key', 'secret') request_signer = RequestSigner( 's3', 'us-east-1', 's3', 's3', credentials, mock.Mock()) request_dict = {'body': b'bar', 'url': 'https://s3.us-east-1.amazonaws.com', 'method': 'PUT', 'headers': {}} self.set_md5_available(False) with self.assertRaises(MD5UnavailableError): handlers.calculate_md5( request_dict, request_signer=request_signer)
def getBodyMd5(self, body): """ Returns the md5 for the bytes from body. :param body: :return: md5(body) :rtype: str """ req = { 'body': body, 'headers': {} } calculate_md5(req) return req['headers']['Content-MD5']
def copy_blob(self, content, info, bucket): """Copy blob from other blob database :param content: File-like blob content object. :param info: `BlobInfo` object. :param bucket: Bucket name. """ if info.digest and info.digest.startswith("md5-"): content_md5 = info.digest[4:] else: params = {"body": content, "headers": {}} calculate_md5(params) content_md5 = params["headers"]["Content-MD5"] self._s3_bucket(create=True).put_object( Key=self.get_path(info.name, bucket), Body=content, ContentMD5=content_md5, )
def test_adds_md5_with_bytes_object(self): request_dict = {'body': b'foobar', 'headers': {}} handlers.calculate_md5(request_dict) self.assertEqual(request_dict['headers']['Content-MD5'], 'OFj2IjCsPJFfMAxmQxLGPw==')
def test_adds_md5_with_file_like_body(self): request_dict = {'body': six.BytesIO(b'foobar'), 'headers': {}} handlers.calculate_md5(request_dict) self.assertEqual(request_dict['headers']['Content-MD5'], 'OFj2IjCsPJFfMAxmQxLGPw==')
def get_content_md5(content): params = {"body": content, "headers": {}} calculate_md5(params) return params["headers"]["Content-MD5"]
def test_adds_md5_with_bytes_object(self): request_dict = {"body": b"foobar", "headers": {}} handlers.calculate_md5(request_dict) self.assertEqual(request_dict["headers"]["Content-MD5"], "OFj2IjCsPJFfMAxmQxLGPw==")
def test_adds_md5_with_file_like_body(self): request_dict = {"body": six.BytesIO(b"foobar"), "headers": {}} handlers.calculate_md5(request_dict) self.assertEqual(request_dict["headers"]["Content-MD5"], "OFj2IjCsPJFfMAxmQxLGPw==")
def get_content_md5(content): # legacy: can be removed with old API params = {"body": content, "headers": {}} calculate_md5(params) return params["headers"]["Content-MD5"]