def put(self, content, basename="", bucket=DEFAULT_BUCKET, content_md5=None):
"""Put a blob in persistent storage
:param content: A file-like object in binary read mode.
:param basename: Optional name from which the blob name will be
derived. This is used to make the unique blob name somewhat
recognizable.
:param bucket: Optional bucket name used to partition blob data
in the persistent storage medium. This may be delimited with
slashes (/). It must be a valid relative path.
:param content_md5: RFC-1864-compliant Content-MD5 header value.
If this parameter is omitted or its value is `None` then content
must be a seekable file-like object. NOTE: the value should not
be prefixed with `md5-` even though we store it that way.
:returns: A `BlobInfo` named tuple. The returned object has a
`name` member that must be used to get or delete the blob. It
should not be confused with the optional `basename` parameter.
"""
name = self.get_unique_name(basename)
path = self.get_path(name, bucket)
if content_md5 is None:
params = {"body": content, "headers": {}}
calculate_md5(params)
content_md5 = params["headers"]["Content-MD5"]
obj = self._s3_bucket(create=True).put_object(
Key=path,
Body=content,
ContentMD5=content_md5,
)
return BlobInfo(name, obj.content_length, "md5-" + content_md5)
def _send_request(self, request_dict, operation_model, callback=None):
# add md5 signature to ensure api calls (such as put bucket policy) will work
body = request_dict.get('body')
if body and isinstance(body, (bytes, bytearray)):
calculate_md5(request_dict)
request = self.endpoint.create_request(request_dict, operation_model)
req_body = getattr(request.body, 'buf', request.body)
request = HTTPRequest(url=request.url,
headers=request.headers,
method=request.method,
body=req_body,
validate_cert=False,
proxy_host=self.proxy_host,
proxy_port=self.proxy_port,
connect_timeout=self.connect_timeout,
request_timeout=self.request_timeout)
if callback is None:
# sync
return self._process_response(HTTPClient().fetch(request),
operation_model=operation_model)
# async
self.http_client.fetch(request,
callback=partial(
self._process_response,
callback=callback,
operation_model=operation_model))
def test_adds_md5_with_file_like_body(self):
request_dict = {
'body': six.BytesIO(b'foobar'),
'headers': {}
}
handlers.calculate_md5(request_dict)
self.assertEqual(request_dict['headers']['Content-MD5'],
'OFj2IjCsPJFfMAxmQxLGPw==')
def test_adds_md5_with_bytes_object(self):
request_dict = {
'body': b'foobar',
'headers': {}
}
handlers.calculate_md5(request_dict)
self.assertEqual(
request_dict['headers']['Content-MD5'],
'OFj2IjCsPJFfMAxmQxLGPw==')
def test_add_md5_with_file_like_body(self):
request_dict = {
'body': six.BytesIO(b'foobar'),
'headers': {}
}
self.md5_digest.return_value = b'8X\xf6"0\xac<\x91_0\x0cfC\x12\xc6?'
handlers.calculate_md5(request_dict)
self.assertEqual(request_dict['headers']['Content-MD5'],
'OFj2IjCsPJFfMAxmQxLGPw==')
def get_content_md5(content, offset=0, filepos=0):
pos = content.tell()
print "TELL %s (%s, %s)" % (pos, offset, filepos)
try:
content.seek(0)
params = {"body": content, "headers": {}}
calculate_md5(params)
return params["headers"]["Content-MD5"]
finally:
print "SEEK", content.tell(), "->", pos, params["headers"]["Content-MD5"]
content.seek(pos)
def test_add_md5_raises_error_when_md5_unavailable(self):
credentials = Credentials('key', 'secret')
request_signer = RequestSigner(
's3', 'us-east-1', 's3', 's3', credentials, mock.Mock())
request_dict = {'body': b'bar',
'url': 'https://s3.us-east-1.amazonaws.com',
'method': 'PUT',
'headers': {}}
self.set_md5_available(False)
with self.assertRaises(MD5UnavailableError):
handlers.calculate_md5(
request_dict, request_signer=request_signer)
def getBodyMd5(self, body):
"""
Returns the md5 for the bytes from body.
:param body:
:return: md5(body)
:rtype: str
"""
req = {
'body': body,
'headers': {}
}
calculate_md5(req)
return req['headers']['Content-MD5']
def copy_blob(self, content, info, bucket):
"""Copy blob from other blob database
:param content: File-like blob content object.
:param info: `BlobInfo` object.
:param bucket: Bucket name.
"""
if info.digest and info.digest.startswith("md5-"):
content_md5 = info.digest[4:]
else:
params = {"body": content, "headers": {}}
calculate_md5(params)
content_md5 = params["headers"]["Content-MD5"]
self._s3_bucket(create=True).put_object(
Key=self.get_path(info.name, bucket),
Body=content,
ContentMD5=content_md5,
)
def test_adds_md5_with_bytes_object(self):
request_dict = {'body': b'foobar', 'headers': {}}
handlers.calculate_md5(request_dict)
self.assertEqual(request_dict['headers']['Content-MD5'],
'OFj2IjCsPJFfMAxmQxLGPw==')
def test_adds_md5_with_file_like_body(self):
request_dict = {'body': six.BytesIO(b'foobar'), 'headers': {}}
handlers.calculate_md5(request_dict)
self.assertEqual(request_dict['headers']['Content-MD5'],
'OFj2IjCsPJFfMAxmQxLGPw==')
def get_content_md5(content):
params = {"body": content, "headers": {}}
calculate_md5(params)
return params["headers"]["Content-MD5"]
def test_adds_md5_with_bytes_object(self):
request_dict = {"body": b"foobar", "headers": {}}
handlers.calculate_md5(request_dict)
self.assertEqual(request_dict["headers"]["Content-MD5"], "OFj2IjCsPJFfMAxmQxLGPw==")
def test_adds_md5_with_file_like_body(self):
request_dict = {"body": six.BytesIO(b"foobar"), "headers": {}}
handlers.calculate_md5(request_dict)
self.assertEqual(request_dict["headers"]["Content-MD5"], "OFj2IjCsPJFfMAxmQxLGPw==")
def get_content_md5(content):
params = {"body": content, "headers": {}}
calculate_md5(params)
return params["headers"]["Content-MD5"]
def get_content_md5(content):
# legacy: can be removed with old API
params = {"body": content, "headers": {}}
calculate_md5(params)
return params["headers"]["Content-MD5"]
