def test_transfer_methods_do_not_use_threads(self): # This is just a smoke test to make sure that # setting use_threads to False has no issues transferring files as # the non-threaded implementation is ran under the same integration # and functional tests in ibm_s3transfer as the normal threaded # implementation # # The methods used are arbitrary other than one of the methods # use ``ibm_boto3.s3.transfer.S3Transfer`` and the other should be # using ``ibm_s3transfer.manager.TransferManager`` directly content = b'my content' filename = self.files.create_file('myfile', content.decode('utf-8')) key = 'foo' config = ibm_boto3.s3.transfer.TransferConfig(use_threads=False) self.client.upload_file(Bucket=self.bucket_name, Key=key, Filename=filename, Config=config) self.addCleanup(self.delete_object, key) self.assertTrue(self.object_exists(key)) fileobj = six.BytesIO() self.client.download_fileobj(Bucket=self.bucket_name, Key='foo', Fileobj=fileobj, Config=config) self.assertEqual(fileobj.getvalue(), content)
def add_glacier_checksums(params, **kwargs): """Add glacier checksums to the http request. This will add two headers to the http request: * x-amz-content-sha256 * x-amz-sha256-tree-hash These values will only be added if they are not present in the HTTP request. """ request_dict = params headers = request_dict['headers'] body = request_dict['body'] if isinstance(body, six.binary_type): # If the user provided a bytes type instead of a file # like object, we're temporarily create a BytesIO object # so we can use the util functions to calculate the # checksums which assume file like objects. Note that # we're not actually changing the body in the request_dict. body = six.BytesIO(body) starting_position = body.tell() if 'x-amz-content-sha256' not in headers: headers['x-amz-content-sha256'] = utils.calculate_sha256(body, as_hex=True) body.seek(starting_position) if 'x-amz-sha256-tree-hash' not in headers: headers['x-amz-sha256-tree-hash'] = utils.calculate_tree_hash(body) body.seek(starting_position)
def test_streaming_line_iterator_ends_newline(self): body = six.BytesIO(b'1234567890\n1234567890\n12345\n') stream = response.StreamingBody(body, content_length=28) self.assert_lines( stream.iter_lines(), [b'1234567890', b'1234567890', b'12345'], )
def test_get_response_streaming_ng(self): http_response = Response() http_response.headers = { 'content-type': 'application/xml', 'date': 'Sat, 08 Mar 2014 12:05:44 GMT', 'server': 'AmazonS3', 'transfer-encoding': 'chunked', 'x-amz-id-2': 'AAAAAAAAAAAAAAAAAAA', 'x-amz-request-id': 'XXXXXXXXXXXXXXXX'} http_response.raw = six.BytesIO(XMLBODY1) http_response.status_code = 403 http_response.reason = 'Forbidden' session = ibm_botocore.session.get_session() service_model = session.get_service_model('s3') operation_model = service_model.operation_model('GetObject') self.assert_response_with_subset_metadata( response.get_response(operation_model, http_response)[1], {'Error': {'Message': 'Access Denied', 'Code': 'AccessDenied'}, 'ResponseMetadata': {'HostId': 'AAAAAAAAAAAAAAAAAAA', 'RequestId': 'XXXXXXXXXXXXXXXX', 'HTTPStatusCode': 403}, } )
def test_streaming_line_iterator_keepends(self): body = six.BytesIO(b'1234567890\n1234567890\n12345') stream = response.StreamingBody(body, content_length=27) self.assert_lines( stream.iter_lines(keepends=True), [b'1234567890\n', b'1234567890\n', b'12345'], )
def test_upload_fileobj_progress(self): # This has to be an integration test because the fileobj will never # actually be read from when using the stubber and therefore the # progress callbacks will not be invoked. chunksize = 5 * (1024**2) config = ibm_boto3.s3.transfer.TransferConfig( multipart_chunksize=chunksize, multipart_threshold=chunksize, max_concurrency=1, ) fileobj = six.BytesIO(b'0' * (chunksize * 3)) def progress_callback(amount): self.progress += amount self.client.upload_fileobj( Fileobj=fileobj, Bucket=self.bucket_name, Key='foo', Config=config, Callback=progress_callback, ) self.addCleanup(self.delete_object, 'foo') self.object_exists('foo') self.assertEqual(self.progress, chunksize * 3)
def test_streaming_body_is_an_iterator(self): body = six.BytesIO(b'a' * 1024 + b'b' * 1024 + b'c' * 2) stream = response.StreamingBody(body, content_length=2050) self.assertEqual(b'a' * 1024, next(stream)) self.assertEqual(b'b' * 1024, next(stream)) self.assertEqual(b'c' * 2, next(stream)) with self.assertRaises(StopIteration): next(stream)
def test_streaming_body_with_invalid_length(self): body = six.BytesIO(b'123456789') stream = response.StreamingBody(body, content_length=10) with self.assertRaises(IncompleteReadError): self.assertEqual(stream.read(9), b'123456789') # The next read will have nothing returned and raise # an IncompleteReadError because we were expectd 10 bytes, not 9. stream.read()
def test_streaming_line_iter_chunk_sizes(self): for chunk_size in range(1, 30): body = six.BytesIO(b'1234567890\n1234567890\n12345') stream = response.StreamingBody(body, content_length=27) self.assert_lines( stream.iter_lines(chunk_size), [b'1234567890', b'1234567890', b'12345'], )
def test_payload_is_binary_file(self): request = AWSRequest() request.data = six.BytesIO(u'\u2713'.encode('utf-8')) auth = self.create_signer() payload = auth.payload(request) self.assertEqual( payload, '1dabba21cdad44541f6b15796f8d22978fc7ea10c46aeceeeeb66c23b3ac7604')
def __init__(self, raw_request): if isinstance(raw_request, six.text_type): raw_request = raw_request.encode('utf-8') self.rfile = six.BytesIO(raw_request) self.raw_requestline = self.rfile.readline() self.error_code = None self.error_message = None self.parse_request()
def test_upload_fileobj(self): fileobj = six.BytesIO(b'foo') self.client.upload_fileobj(Fileobj=fileobj, Bucket=self.bucket_name, Key='foo') self.addCleanup(self.delete_object, 'foo') self.object_exists('foo')
def test_streaming_line_abstruse_newline_standard(self): for chunk_size in range(1, 30): body = six.BytesIO(b'1234567890\r\n1234567890\r\n12345\r\n') stream = response.StreamingBody(body, content_length=31) self.assert_lines( stream.iter_lines(chunk_size), [b'1234567890', b'1234567890', b'12345'], )
def test_add_md5_with_file_like_body(self): request_dict = { 'body': six.BytesIO(b'foobar'), 'headers': {} } self.md5_digest.return_value = b'8X\xf6"0\xac<\x91_0\x0cfC\x12\xc6?' handlers.calculate_md5(request_dict) self.assertEqual(request_dict['headers']['Content-MD5'], 'OFj2IjCsPJFfMAxmQxLGPw==')
def test_validates_file_like_object(self): value = six.BytesIO(b'foo') errors = self.get_validation_error_message( given_shapes=self.shapes, input_params={'Blob': value}, ) error_msg = errors.generate_report() self.assertEqual(error_msg, '')
def test_checksum_added_only_if_not_exists(self): request_dict = { 'headers': { 'x-amz-content-sha256': 'pre-exists', }, 'body': six.BytesIO(b'hello world'), } handlers.add_glacier_checksums(request_dict) self.assertEqual(request_dict['headers']['x-amz-content-sha256'], 'pre-exists')
def test_upload_fileobj(self): fileobj = six.BytesIO(b'foo') inject.bucket_upload_fileobj(self.bucket, Key='key', Fileobj=fileobj) self.bucket.meta.client.upload_fileobj.assert_called_with( Bucket=self.bucket.name, Fileobj=fileobj, Key='key', ExtraArgs=None, Callback=None, Config=None)
def test_download_fileobj(self): obj = six.BytesIO() inject.bucket_download_fileobj(self.bucket, Key='key', Fileobj=obj) self.bucket.meta.client.download_fileobj.assert_called_with( Bucket=self.bucket.name, Key='key', Fileobj=obj, ExtraArgs=None, Callback=None, Config=None)
def test_download_fileobj(self): fileobj = six.BytesIO() inject.object_download_fileobj(self.obj, Fileobj=fileobj) self.obj.meta.client.download_fileobj.assert_called_with( Bucket=self.obj.bucket_name, Key=self.obj.key, Fileobj=fileobj, ExtraArgs=None, Callback=None, Config=None)
def test_content_sha256_set_if_payload_signing_disabled(self): request = AWSRequest() request.data = six.BytesIO(u'\u2713'.encode('utf-8')) request.url = 'https://amazonaws.com' request.context['payload_signing_enabled'] = False request.method = 'PUT' auth = self.create_signer() auth.add_auth(request) sha_header = request.headers['X-Amz-Content-SHA256'] self.assertEqual(sha_header, 'UNSIGNED-PAYLOAD')
def test_can_set_utf_8_headers(self): bucket_name = _SHARED_BUCKET body = six.BytesIO(b"Hello world!") response = self.client.put_object( Bucket=bucket_name, Key="foo.txt", Body=body, ContentDisposition="attachment; filename=5小時接力起跑.jpg;") self.assert_status_code(response, 200) self.addCleanup(self.client.delete_object, Bucket=bucket_name, Key="foo.txt")
def test_download_fileobj(self): fileobj = six.BytesIO() self.client.put_object(Bucket=self.bucket_name, Key='foo', Body=b'beach') self.addCleanup(self.delete_object, 'foo') self.wait_until_object_exists('foo') self.client.download_fileobj(Bucket=self.bucket_name, Key='foo', Fileobj=fileobj) self.assertEqual(fileobj.getvalue(), b'beach')
def stub_get_object(self, full_contents, start_byte=0, end_byte=None): """ Stubs out the get_object operation. :param full_contents: The FULL contents of the object :param start_byte: The first byte to grab. :param end_byte: The last byte to grab. """ get_object_response = {} expected_params = {} contents = full_contents end_byte_range = end_byte # If the start byte is set and the end byte is not, the end byte is # the last byte. if start_byte != 0 and end_byte is None: end_byte = len(full_contents) - 1 # The range on get object where the the end byte is the last byte # should set the input range as e.g. Range='bytes=3-' if end_byte == len(full_contents) - 1: end_byte_range = '' # If this is a ranged get, ContentRange needs to be returned, # contents needs to be pruned, and Range needs to be an expected param. if end_byte is not None: contents = full_contents[start_byte:end_byte+1] part_range = 'bytes=%s-%s' % (start_byte, end_byte_range) content_range = 'bytes=%s-%s/%s' % ( start_byte, end_byte, len(full_contents)) get_object_response['ContentRange'] = content_range expected_params['Range'] = part_range get_object_response.update({ "AcceptRanges": "bytes", "ETag": self.etag, "ContentLength": len(contents), "ContentType": "binary/octet-stream", "Body": six.BytesIO(contents), "ResponseMetadata": { "HTTPStatusCode": 200 } }) expected_params.update({ "Bucket": self.bucket, "Key": self.key }) self.stubber.add_response( method='get_object', service_response=get_object_response, expected_params=expected_params)
def test_make_request_with_sse(self): key_bytes = os.urandom(32) # Obviously a bad key here, but we just want to ensure we can use # a str/unicode type as a key. key_str = 'abcd' * 8 # Put two objects with an sse key, one with random bytes, # one with str/unicode. Then verify we can GetObject() both # objects. self.client.put_object(Bucket=self.bucket_name, Key='foo.txt', Body=six.BytesIO(b'mycontents'), SSECustomerAlgorithm='AES256', SSECustomerKey=key_bytes) self.addCleanup(self.client.delete_object, Bucket=self.bucket_name, Key='foo.txt') self.client.put_object(Bucket=self.bucket_name, Key='foo2.txt', Body=six.BytesIO(b'mycontents2'), SSECustomerAlgorithm='AES256', SSECustomerKey=key_str) self.addCleanup(self.client.delete_object, Bucket=self.bucket_name, Key='foo2.txt') self.assertEqual( self.client.get_object(Bucket=self.bucket_name, Key='foo.txt', SSECustomerAlgorithm='AES256', SSECustomerKey=key_bytes)['Body'].read(), b'mycontents') self.assertEqual( self.client.get_object(Bucket=self.bucket_name, Key='foo2.txt', SSECustomerAlgorithm='AES256', SSECustomerKey=key_str)['Body'].read(), b'mycontents2')
def _get_upload_part_fileobj_with_full_size(self, fileobj, **kwargs): # Note: It is unfortunate that in order to do a multithreaded # multipart upload we cannot simply copy the filelike object # since there is not really a mechanism in python (i.e. os.dup # points to the same OS filehandle which causes concurrency # issues). So instead we need to read from the fileobj and # chunk the data out to separate file-like objects in memory. data = fileobj.read(kwargs['part_size']) # We return the length of the data instead of the full_file_size # because we partitioned the data into separate BytesIO objects # meaning the BytesIO object has no knowledge of its start position # relative the input source nor access to the rest of the input # source. So we must treat it as its own standalone file. return six.BytesIO(data), len(data)
def test_multipart_upload(self): chunksize = 8 * (1024 ** 2) contents = six.BytesIO(b'0' * (chunksize * 3)) self.stub_multipart_upload(num_parts=3) transfer_config = TransferConfig( multipart_chunksize=chunksize, multipart_threshold=1, max_concurrency=1) with self.stubber: # The stubber will assert that all the right parameters are called. self.s3.meta.client.upload_fileobj( Fileobj=contents, Bucket=self.bucket, Key=self.key, Config=transfer_config) self.stubber.assert_no_pending_responses()
def setUp(self): super(TestS3SigV4Auth, self).setUp() self.credentials = ibm_botocore.credentials.Credentials( access_key='foo', secret_key='bar', token='baz') self.auth = ibm_botocore.auth.S3SigV4Auth(self.credentials, 'ec2', 'eu-central-1') self.request = AWSRequest(data=six.BytesIO(b"foo bar baz")) self.request.method = 'PUT' self.request.url = 'https://s3.eu-central-1.amazonaws.com/' self.client_config = mock.Mock() self.s3_config = {} self.client_config.s3 = self.s3_config self.request.context = {'client_config': self.client_config}
def test_glacier_checksums_added(self): request_dict = { 'headers': {}, 'body': six.BytesIO(b'hello world'), } handlers.add_glacier_checksums(request_dict) self.assertIn('x-amz-content-sha256', request_dict['headers']) self.assertIn('x-amz-sha256-tree-hash', request_dict['headers']) self.assertEqual( request_dict['headers']['x-amz-content-sha256'], 'b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9') self.assertEqual( request_dict['headers']['x-amz-sha256-tree-hash'], 'b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9') # And verify that the body can still be read. self.assertEqual(request_dict['body'].read(), b'hello world')
def test_handles_expect_100_with_different_reason_phrase(self): with patch('urllib3.util.wait_for_read') as wait_mock: # Shows the server first sending a 100 continue response # then a 200 ok response. s = FakeSocket(b'HTTP/1.1 100 (Continue)\r\n\r\nHTTP/1.1 200 OK\r\n') conn = AWSHTTPConnection('s3.amazonaws.com', 443) conn.sock = s wait_mock.return_value = True conn.request('GET', '/bucket/foo', six.BytesIO(b'body'), {'Expect': b'100-continue', 'Content-Length': b'4'}) response = conn.getresponse() # Now we should verify that our final response is the 200 OK. self.assertEqual(response.status, 200) # Assert that we waited for the 100-continue response self.assertEqual(wait_mock.call_count, 1) # Verify that we went the request body because we got a 100 # continue. self.assertIn(b'body', s.sent_data)
def _wrap_data(self, data, callbacks, close_callbacks): """ Wraps data with the interrupt reader and the file chunk reader. :type data: bytes :param data: The data to wrap. :type callbacks: list :param callbacks: The callbacks associated with the transfer future. :type close_callbacks: list :param close_callbacks: The callbacks to be called when closing the wrapper for the data. :return: Fully wrapped data. """ fileobj = self._wrap_with_interrupt_reader(six.BytesIO(data)) return self._osutil.open_file_chunk_reader_from_fileobj( fileobj=fileobj, chunk_size=len(data), full_file_size=len(data), callbacks=callbacks, close_callbacks=close_callbacks)