def test_transfer_methods_do_not_use_threads(self):
# This is just a smoke test to make sure that
# setting use_threads to False has no issues transferring files as
# the non-threaded implementation is ran under the same integration
# and functional tests in ibm_s3transfer as the normal threaded
# implementation
#
# The methods used are arbitrary other than one of the methods
# use ``ibm_boto3.s3.transfer.S3Transfer`` and the other should be
# using ``ibm_s3transfer.manager.TransferManager`` directly
content = b'my content'
filename = self.files.create_file('myfile', content.decode('utf-8'))
key = 'foo'
config = ibm_boto3.s3.transfer.TransferConfig(use_threads=False)
self.client.upload_file(Bucket=self.bucket_name,
Key=key,
Filename=filename,
Config=config)
self.addCleanup(self.delete_object, key)
self.assertTrue(self.object_exists(key))
fileobj = six.BytesIO()
self.client.download_fileobj(Bucket=self.bucket_name,
Key='foo',
Fileobj=fileobj,
Config=config)
self.assertEqual(fileobj.getvalue(), content)
def add_glacier_checksums(params, **kwargs):
"""Add glacier checksums to the http request.
This will add two headers to the http request:
* x-amz-content-sha256
* x-amz-sha256-tree-hash
These values will only be added if they are not present
in the HTTP request.
"""
request_dict = params
headers = request_dict['headers']
body = request_dict['body']
if isinstance(body, six.binary_type):
# If the user provided a bytes type instead of a file
# like object, we're temporarily create a BytesIO object
# so we can use the util functions to calculate the
# checksums which assume file like objects. Note that
# we're not actually changing the body in the request_dict.
body = six.BytesIO(body)
starting_position = body.tell()
if 'x-amz-content-sha256' not in headers:
headers['x-amz-content-sha256'] = utils.calculate_sha256(body,
as_hex=True)
body.seek(starting_position)
if 'x-amz-sha256-tree-hash' not in headers:
headers['x-amz-sha256-tree-hash'] = utils.calculate_tree_hash(body)
body.seek(starting_position)
def test_streaming_line_iterator_ends_newline(self):
body = six.BytesIO(b'1234567890\n1234567890\n12345\n')
stream = response.StreamingBody(body, content_length=28)
self.assert_lines(
stream.iter_lines(),
[b'1234567890', b'1234567890', b'12345'],
)
def test_get_response_streaming_ng(self):
http_response = Response()
http_response.headers = {
'content-type': 'application/xml',
'date': 'Sat, 08 Mar 2014 12:05:44 GMT',
'server': 'AmazonS3',
'transfer-encoding': 'chunked',
'x-amz-id-2': 'AAAAAAAAAAAAAAAAAAA',
'x-amz-request-id': 'XXXXXXXXXXXXXXXX'}
http_response.raw = six.BytesIO(XMLBODY1)
http_response.status_code = 403
http_response.reason = 'Forbidden'
session = ibm_botocore.session.get_session()
service_model = session.get_service_model('s3')
operation_model = service_model.operation_model('GetObject')
self.assert_response_with_subset_metadata(
response.get_response(operation_model, http_response)[1],
{'Error': {'Message': 'Access Denied',
'Code': 'AccessDenied'},
'ResponseMetadata': {'HostId': 'AAAAAAAAAAAAAAAAAAA',
'RequestId': 'XXXXXXXXXXXXXXXX',
'HTTPStatusCode': 403},
}
)
def test_streaming_line_iterator_keepends(self):
body = six.BytesIO(b'1234567890\n1234567890\n12345')
stream = response.StreamingBody(body, content_length=27)
self.assert_lines(
stream.iter_lines(keepends=True),
[b'1234567890\n', b'1234567890\n', b'12345'],
)
def test_upload_fileobj_progress(self):
# This has to be an integration test because the fileobj will never
# actually be read from when using the stubber and therefore the
# progress callbacks will not be invoked.
chunksize = 5 * (1024**2)
config = ibm_boto3.s3.transfer.TransferConfig(
multipart_chunksize=chunksize,
multipart_threshold=chunksize,
max_concurrency=1,
)
fileobj = six.BytesIO(b'0' * (chunksize * 3))
def progress_callback(amount):
self.progress += amount
self.client.upload_fileobj(
Fileobj=fileobj,
Bucket=self.bucket_name,
Key='foo',
Config=config,
Callback=progress_callback,
)
self.addCleanup(self.delete_object, 'foo')
self.object_exists('foo')
self.assertEqual(self.progress, chunksize * 3)
def test_streaming_body_is_an_iterator(self):
body = six.BytesIO(b'a' * 1024 + b'b' * 1024 + b'c' * 2)
stream = response.StreamingBody(body, content_length=2050)
self.assertEqual(b'a' * 1024, next(stream))
self.assertEqual(b'b' * 1024, next(stream))
self.assertEqual(b'c' * 2, next(stream))
with self.assertRaises(StopIteration):
next(stream)
def test_streaming_body_with_invalid_length(self):
body = six.BytesIO(b'123456789')
stream = response.StreamingBody(body, content_length=10)
with self.assertRaises(IncompleteReadError):
self.assertEqual(stream.read(9), b'123456789')
# The next read will have nothing returned and raise
# an IncompleteReadError because we were expectd 10 bytes, not 9.
stream.read()
def test_streaming_line_iter_chunk_sizes(self):
for chunk_size in range(1, 30):
body = six.BytesIO(b'1234567890\n1234567890\n12345')
stream = response.StreamingBody(body, content_length=27)
self.assert_lines(
stream.iter_lines(chunk_size),
[b'1234567890', b'1234567890', b'12345'],
)
def test_payload_is_binary_file(self):
request = AWSRequest()
request.data = six.BytesIO(u'\u2713'.encode('utf-8'))
auth = self.create_signer()
payload = auth.payload(request)
self.assertEqual(
payload,
'1dabba21cdad44541f6b15796f8d22978fc7ea10c46aeceeeeb66c23b3ac7604')
def __init__(self, raw_request):
if isinstance(raw_request, six.text_type):
raw_request = raw_request.encode('utf-8')
self.rfile = six.BytesIO(raw_request)
self.raw_requestline = self.rfile.readline()
self.error_code = None
self.error_message = None
self.parse_request()
def test_upload_fileobj(self):
fileobj = six.BytesIO(b'foo')
self.client.upload_fileobj(Fileobj=fileobj,
Bucket=self.bucket_name,
Key='foo')
self.addCleanup(self.delete_object, 'foo')
self.object_exists('foo')
def test_streaming_line_abstruse_newline_standard(self):
for chunk_size in range(1, 30):
body = six.BytesIO(b'1234567890\r\n1234567890\r\n12345\r\n')
stream = response.StreamingBody(body, content_length=31)
self.assert_lines(
stream.iter_lines(chunk_size),
[b'1234567890', b'1234567890', b'12345'],
)
def test_add_md5_with_file_like_body(self):
request_dict = {
'body': six.BytesIO(b'foobar'),
'headers': {}
}
self.md5_digest.return_value = b'8X\xf6"0\xac<\x91_0\x0cfC\x12\xc6?'
handlers.calculate_md5(request_dict)
self.assertEqual(request_dict['headers']['Content-MD5'],
'OFj2IjCsPJFfMAxmQxLGPw==')
def test_validates_file_like_object(self):
value = six.BytesIO(b'foo')
errors = self.get_validation_error_message(
given_shapes=self.shapes,
input_params={'Blob': value},
)
error_msg = errors.generate_report()
self.assertEqual(error_msg, '')
def test_checksum_added_only_if_not_exists(self):
request_dict = {
'headers': {
'x-amz-content-sha256': 'pre-exists',
},
'body': six.BytesIO(b'hello world'),
}
handlers.add_glacier_checksums(request_dict)
self.assertEqual(request_dict['headers']['x-amz-content-sha256'],
'pre-exists')
def test_upload_fileobj(self):
fileobj = six.BytesIO(b'foo')
inject.bucket_upload_fileobj(self.bucket, Key='key', Fileobj=fileobj)
self.bucket.meta.client.upload_fileobj.assert_called_with(
Bucket=self.bucket.name,
Fileobj=fileobj,
Key='key',
ExtraArgs=None,
Callback=None,
Config=None)
def test_download_fileobj(self):
obj = six.BytesIO()
inject.bucket_download_fileobj(self.bucket, Key='key', Fileobj=obj)
self.bucket.meta.client.download_fileobj.assert_called_with(
Bucket=self.bucket.name,
Key='key',
Fileobj=obj,
ExtraArgs=None,
Callback=None,
Config=None)
def test_download_fileobj(self):
fileobj = six.BytesIO()
inject.object_download_fileobj(self.obj, Fileobj=fileobj)
self.obj.meta.client.download_fileobj.assert_called_with(
Bucket=self.obj.bucket_name,
Key=self.obj.key,
Fileobj=fileobj,
ExtraArgs=None,
Callback=None,
Config=None)
def test_content_sha256_set_if_payload_signing_disabled(self):
request = AWSRequest()
request.data = six.BytesIO(u'\u2713'.encode('utf-8'))
request.url = 'https://amazonaws.com'
request.context['payload_signing_enabled'] = False
request.method = 'PUT'
auth = self.create_signer()
auth.add_auth(request)
sha_header = request.headers['X-Amz-Content-SHA256']
self.assertEqual(sha_header, 'UNSIGNED-PAYLOAD')
def test_can_set_utf_8_headers(self):
bucket_name = _SHARED_BUCKET
body = six.BytesIO(b"Hello world!")
response = self.client.put_object(
Bucket=bucket_name,
Key="foo.txt",
Body=body,
ContentDisposition="attachment; filename=5小時接力起跑.jpg;")
self.assert_status_code(response, 200)
self.addCleanup(self.client.delete_object,
Bucket=bucket_name,
Key="foo.txt")
def test_download_fileobj(self):
fileobj = six.BytesIO()
self.client.put_object(Bucket=self.bucket_name,
Key='foo',
Body=b'beach')
self.addCleanup(self.delete_object, 'foo')
self.wait_until_object_exists('foo')
self.client.download_fileobj(Bucket=self.bucket_name,
Key='foo',
Fileobj=fileobj)
self.assertEqual(fileobj.getvalue(), b'beach')
def stub_get_object(self, full_contents, start_byte=0, end_byte=None):
"""
Stubs out the get_object operation.
:param full_contents: The FULL contents of the object
:param start_byte: The first byte to grab.
:param end_byte: The last byte to grab.
"""
get_object_response = {}
expected_params = {}
contents = full_contents
end_byte_range = end_byte
# If the start byte is set and the end byte is not, the end byte is
# the last byte.
if start_byte != 0 and end_byte is None:
end_byte = len(full_contents) - 1
# The range on get object where the the end byte is the last byte
# should set the input range as e.g. Range='bytes=3-'
if end_byte == len(full_contents) - 1:
end_byte_range = ''
# If this is a ranged get, ContentRange needs to be returned,
# contents needs to be pruned, and Range needs to be an expected param.
if end_byte is not None:
contents = full_contents[start_byte:end_byte+1]
part_range = 'bytes=%s-%s' % (start_byte, end_byte_range)
content_range = 'bytes=%s-%s/%s' % (
start_byte, end_byte, len(full_contents))
get_object_response['ContentRange'] = content_range
expected_params['Range'] = part_range
get_object_response.update({
"AcceptRanges": "bytes",
"ETag": self.etag,
"ContentLength": len(contents),
"ContentType": "binary/octet-stream",
"Body": six.BytesIO(contents),
"ResponseMetadata": {
"HTTPStatusCode": 200
}
})
expected_params.update({
"Bucket": self.bucket,
"Key": self.key
})
self.stubber.add_response(
method='get_object', service_response=get_object_response,
expected_params=expected_params)
def test_make_request_with_sse(self):
key_bytes = os.urandom(32)
# Obviously a bad key here, but we just want to ensure we can use
# a str/unicode type as a key.
key_str = 'abcd' * 8
# Put two objects with an sse key, one with random bytes,
# one with str/unicode. Then verify we can GetObject() both
# objects.
self.client.put_object(Bucket=self.bucket_name,
Key='foo.txt',
Body=six.BytesIO(b'mycontents'),
SSECustomerAlgorithm='AES256',
SSECustomerKey=key_bytes)
self.addCleanup(self.client.delete_object,
Bucket=self.bucket_name,
Key='foo.txt')
self.client.put_object(Bucket=self.bucket_name,
Key='foo2.txt',
Body=six.BytesIO(b'mycontents2'),
SSECustomerAlgorithm='AES256',
SSECustomerKey=key_str)
self.addCleanup(self.client.delete_object,
Bucket=self.bucket_name,
Key='foo2.txt')
self.assertEqual(
self.client.get_object(Bucket=self.bucket_name,
Key='foo.txt',
SSECustomerAlgorithm='AES256',
SSECustomerKey=key_bytes)['Body'].read(),
b'mycontents')
self.assertEqual(
self.client.get_object(Bucket=self.bucket_name,
Key='foo2.txt',
SSECustomerAlgorithm='AES256',
SSECustomerKey=key_str)['Body'].read(),
b'mycontents2')
def _get_upload_part_fileobj_with_full_size(self, fileobj, **kwargs):
# Note: It is unfortunate that in order to do a multithreaded
# multipart upload we cannot simply copy the filelike object
# since there is not really a mechanism in python (i.e. os.dup
# points to the same OS filehandle which causes concurrency
# issues). So instead we need to read from the fileobj and
# chunk the data out to separate file-like objects in memory.
data = fileobj.read(kwargs['part_size'])
# We return the length of the data instead of the full_file_size
# because we partitioned the data into separate BytesIO objects
# meaning the BytesIO object has no knowledge of its start position
# relative the input source nor access to the rest of the input
# source. So we must treat it as its own standalone file.
return six.BytesIO(data), len(data)
def test_multipart_upload(self):
chunksize = 8 * (1024 ** 2)
contents = six.BytesIO(b'0' * (chunksize * 3))
self.stub_multipart_upload(num_parts=3)
transfer_config = TransferConfig(
multipart_chunksize=chunksize, multipart_threshold=1,
max_concurrency=1)
with self.stubber:
# The stubber will assert that all the right parameters are called.
self.s3.meta.client.upload_fileobj(
Fileobj=contents, Bucket=self.bucket, Key=self.key,
Config=transfer_config)
self.stubber.assert_no_pending_responses()
def setUp(self):
super(TestS3SigV4Auth, self).setUp()
self.credentials = ibm_botocore.credentials.Credentials(
access_key='foo', secret_key='bar', token='baz')
self.auth = ibm_botocore.auth.S3SigV4Auth(self.credentials, 'ec2',
'eu-central-1')
self.request = AWSRequest(data=six.BytesIO(b"foo bar baz"))
self.request.method = 'PUT'
self.request.url = 'https://s3.eu-central-1.amazonaws.com/'
self.client_config = mock.Mock()
self.s3_config = {}
self.client_config.s3 = self.s3_config
self.request.context = {'client_config': self.client_config}
def test_glacier_checksums_added(self):
request_dict = {
'headers': {},
'body': six.BytesIO(b'hello world'),
}
handlers.add_glacier_checksums(request_dict)
self.assertIn('x-amz-content-sha256', request_dict['headers'])
self.assertIn('x-amz-sha256-tree-hash', request_dict['headers'])
self.assertEqual(
request_dict['headers']['x-amz-content-sha256'],
'b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9')
self.assertEqual(
request_dict['headers']['x-amz-sha256-tree-hash'],
'b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9')
# And verify that the body can still be read.
self.assertEqual(request_dict['body'].read(), b'hello world')
def test_handles_expect_100_with_different_reason_phrase(self):
with patch('urllib3.util.wait_for_read') as wait_mock:
# Shows the server first sending a 100 continue response
# then a 200 ok response.
s = FakeSocket(b'HTTP/1.1 100 (Continue)\r\n\r\nHTTP/1.1 200 OK\r\n')
conn = AWSHTTPConnection('s3.amazonaws.com', 443)
conn.sock = s
wait_mock.return_value = True
conn.request('GET', '/bucket/foo', six.BytesIO(b'body'),
{'Expect': b'100-continue', 'Content-Length': b'4'})
response = conn.getresponse()
# Now we should verify that our final response is the 200 OK.
self.assertEqual(response.status, 200)
# Assert that we waited for the 100-continue response
self.assertEqual(wait_mock.call_count, 1)
# Verify that we went the request body because we got a 100
# continue.
self.assertIn(b'body', s.sent_data)
def _wrap_data(self, data, callbacks, close_callbacks):
"""
Wraps data with the interrupt reader and the file chunk reader.
:type data: bytes
:param data: The data to wrap.
:type callbacks: list
:param callbacks: The callbacks associated with the transfer future.
:type close_callbacks: list
:param close_callbacks: The callbacks to be called when closing the
wrapper for the data.
:return: Fully wrapped data.
"""
fileobj = self._wrap_with_interrupt_reader(six.BytesIO(data))
return self._osutil.open_file_chunk_reader_from_fileobj(
fileobj=fileobj, chunk_size=len(data), full_file_size=len(data),
callbacks=callbacks, close_callbacks=close_callbacks)
