def test_make_request_with_sse(self): key_bytes = os.urandom(32) # Obviously a bad key here, but we just want to ensure we can use # a str/unicode type as a key. key_str = 'abcd' * 8 # Put two objects with an sse key, one with random bytes, # one with str/unicode. Then verify we can GetObject() both # objects. self.client.put_object( Bucket=self.bucket_name, Key='foo.txt', Body=six.BytesIO(b'mycontents'), SSECustomerAlgorithm='AES256', SSECustomerKey=key_bytes) self.addCleanup(self.client.delete_object, Bucket=self.bucket_name, Key='foo.txt') self.client.put_object( Bucket=self.bucket_name, Key='foo2.txt', Body=six.BytesIO(b'mycontents2'), SSECustomerAlgorithm='AES256', SSECustomerKey=key_str) self.addCleanup(self.client.delete_object, Bucket=self.bucket_name, Key='foo2.txt') self.assertEqual( self.client.get_object(Bucket=self.bucket_name, Key='foo.txt', SSECustomerAlgorithm='AES256', SSECustomerKey=key_bytes)['Body'].read(), b'mycontents') self.assertEqual( self.client.get_object(Bucket=self.bucket_name, Key='foo2.txt', SSECustomerAlgorithm='AES256', SSECustomerKey=key_str)['Body'].read(), b'mycontents2')
def test_streaming_line_iterator_keepends(self): body = six.BytesIO(b'1234567890\n1234567890\n12345') stream = response.StreamingBody(body, content_length=27) self.assert_lines( stream.iter_lines(keepends=True), [b'1234567890\n', b'1234567890\n', b'12345'], )
def test_upload_fileobj(self): fileobj = six.BytesIO(b'foo') self.client.upload_fileobj( Fileobj=fileobj, Bucket=self.bucket_name, Key='foo') self.addCleanup(self.delete_object, 'foo') self.object_exists('foo')
def add_glacier_checksums(params, **kwargs): """Add glacier checksums to the http request. This will add two headers to the http request: * x-amz-content-sha256 * x-amz-sha256-tree-hash These values will only be added if they are not present in the HTTP request. """ request_dict = params headers = request_dict['headers'] body = request_dict['body'] if isinstance(body, six.binary_type): # If the user provided a bytes type instead of a file # like object, we're temporarily create a BytesIO object # so we can use the util functions to calculate the # checksums which assume file like objects. Note that # we're not actually changing the body in the request_dict. body = six.BytesIO(body) starting_position = body.tell() if 'x-amz-content-sha256' not in headers: headers['x-amz-content-sha256'] = utils.calculate_sha256(body, as_hex=True) body.seek(starting_position) if 'x-amz-sha256-tree-hash' not in headers: headers['x-amz-sha256-tree-hash'] = utils.calculate_tree_hash(body) body.seek(starting_position)
def test_make_request_with_sse_copy_source(self): encrypt_key = 'a' * 32 other_encrypt_key = 'b' * 32 # Upload the object using one encrypt key self.client.put_object( Bucket=self.bucket_name, Key='foo.txt', Body=six.BytesIO(b'mycontents'), SSECustomerAlgorithm='AES256', SSECustomerKey=encrypt_key) self.addCleanup(self.client.delete_object, Bucket=self.bucket_name, Key='foo.txt') # Copy the object using the original encryption key as the copy source # and encrypt with a new encryption key. self.client.copy_object( Bucket=self.bucket_name, CopySource=self.bucket_name+'/foo.txt', Key='bar.txt', CopySourceSSECustomerAlgorithm='AES256', CopySourceSSECustomerKey=encrypt_key, SSECustomerAlgorithm='AES256', SSECustomerKey=other_encrypt_key ) self.addCleanup(self.client.delete_object, Bucket=self.bucket_name, Key='bar.txt') # Download the object using the new encryption key. # The content should not have changed. self.assertEqual( self.client.get_object( Bucket=self.bucket_name, Key='bar.txt', SSECustomerAlgorithm='AES256', SSECustomerKey=other_encrypt_key)['Body'].read(), b'mycontents')
def test_streaming_line_iterator_ends_newline(self): body = six.BytesIO(b'1234567890\n1234567890\n12345\n') stream = response.StreamingBody(body, content_length=28) self.assert_lines( stream.iter_lines(), [b'1234567890', b'1234567890', b'12345'], )
def test_upload_fileobj_progress(self): # This has to be an integration test because the fileobj will never # actually be read from when using the stubber and therefore the # progress callbacks will not be invoked. chunksize = 5 * (1024**2) config = boto3.s3.transfer.TransferConfig( multipart_chunksize=chunksize, multipart_threshold=chunksize, max_concurrency=1, ) fileobj = six.BytesIO(b'0' * (chunksize * 3)) def progress_callback(amount): self.progress += amount self.client.upload_fileobj( Fileobj=fileobj, Bucket=self.bucket_name, Key='foo', Config=config, Callback=progress_callback, ) self.addCleanup(self.delete_object, 'foo') self.object_exists('foo') self.assertEqual(self.progress, chunksize * 3)
def test_transfer_methods_do_not_use_threads(self): # This is just a smoke test to make sure that # setting use_threads to False has no issues transferring files as # the non-threaded implementation is ran under the same integration # and functional tests in s3transfer as the normal threaded # implementation # # The methods used are arbitrary other than one of the methods # use ``boto3_wasabi.s3.transfer.S3Transfer`` and the other should be # using ``s3transfer.manager.TransferManager`` directly content = b'my content' filename = self.files.create_file('myfile', content.decode('utf-8')) key = 'foo' config = boto3_wasabi.s3.transfer.TransferConfig(use_threads=False) self.client.upload_file( Bucket=self.bucket_name, Key=key, Filename=filename, Config=config) self.addCleanup(self.delete_object, key) self.assertTrue(self.object_exists(key)) fileobj = six.BytesIO() self.client.download_fileobj( Bucket=self.bucket_name, Key='foo', Fileobj=fileobj, Config=config) self.assertEqual(fileobj.getvalue(), content)
def test_get_response_streaming_ng(self): http_response = Response() http_response.headers = { 'content-type': 'application/xml', 'date': 'Sat, 08 Mar 2014 12:05:44 GMT', 'server': 'AmazonS3', 'transfer-encoding': 'chunked', 'x-amz-id-2': 'AAAAAAAAAAAAAAAAAAA', 'x-amz-request-id': 'XXXXXXXXXXXXXXXX' } http_response.raw = six.BytesIO(XMLBODY1) http_response.status_code = 403 http_response.reason = 'Forbidden' session = botocore.session.get_session() s3 = session.get_service('s3') operation = s3.get_operation('GetObject') # streaming operation self.assertEqual( response.get_response(operation.model, http_response)[1], { 'Error': { 'Message': 'Access Denied', 'Code': 'AccessDenied', }, 'ResponseMetadata': { 'HostId': 'AAAAAAAAAAAAAAAAAAA', 'RequestId': 'XXXXXXXXXXXXXXXX', 'HTTPStatusCode': 403 }, })
def test_get_response_nonstreaming_ok(self): http_response = Response() http_response.headers = { 'content-type': 'application/xml', 'date': 'Sun, 09 Mar 2014 02:55:43 GMT', 'server': 'AmazonS3', 'transfer-encoding': 'chunked', 'x-amz-id-2': 'AAAAAAAAAAAAAAAAAAA', 'x-amz-request-id': 'XXXXXXXXXXXXXXXX' } http_response.raw = six.BytesIO(XMLBODY1) http_response.status_code = 403 http_response.reason = 'Forbidden' http_response.request = Request() session = botocore.session.get_session() service_model = session.get_service_model('s3') operation_model = service_model.operation_model('ListObjects') self.assert_response_with_subset_metadata( response.get_response(operation_model, http_response)[1], { 'ResponseMetadata': { 'RequestId': 'XXXXXXXXXXXXXXXX', 'HostId': 'AAAAAAAAAAAAAAAAAAA', 'HTTPStatusCode': 403 }, 'Error': { 'Message': 'Access Denied', 'Code': 'AccessDenied' } })
def test_streaming_line_iter_chunk_sizes(self): for chunk_size in range(1, 30): body = six.BytesIO(b'1234567890\n1234567890\n12345') stream = response.StreamingBody(body, content_length=27) self.assert_lines( stream.iter_lines(chunk_size), [b'1234567890', b'1234567890', b'12345'], )
def __init__(self, raw_request): if isinstance(raw_request, six.text_type): raw_request = raw_request.encode('utf-8') self.rfile = six.BytesIO(raw_request) self.raw_requestline = self.rfile.readline() self.error_code = None self.error_message = None self.parse_request()
def test_streaming_line_abstruse_newline_standard(self): for chunk_size in range(1, 30): body = six.BytesIO(b'1234567890\r\n1234567890\r\n12345\r\n') stream = response.StreamingBody(body, content_length=31) self.assert_lines( stream.iter_lines(chunk_size), [b'1234567890', b'1234567890', b'12345'], )
def test_payload_is_binary_file(self): request = AWSRequest() request.data = six.BytesIO(u'\u2713'.encode('utf-8')) auth = self.create_signer() payload = auth.payload(request) self.assertEqual( payload, '1dabba21cdad44541f6b15796f8d22978fc7ea10c46aeceeeeb66c23b3ac7604')
def test_streaming_body_is_an_iterator(self): body = six.BytesIO(b'a' * 1024 + b'b' * 1024 + b'c' * 2) stream = response.StreamingBody(body, content_length=2050) self.assertEqual(b'a' * 1024, next(stream)) self.assertEqual(b'b' * 1024, next(stream)) self.assertEqual(b'c' * 2, next(stream)) with self.assertRaises(StopIteration): next(stream)
def test_streaming_body_with_invalid_length(self): body = six.BytesIO(b'123456789') stream = response.StreamingBody(body, content_length=10) with self.assertRaises(IncompleteReadError): self.assertEqual(stream.read(9), b'123456789') # The next read will have nothing returned and raise # an IncompleteReadError because we were expectd 10 bytes, not 9. stream.read()
def test_request_retried_for_sigv4(self): body = six.BytesIO(b"Hello world!") exception = ConnectionClosedError(endpoint_url='') self.http_stubber.responses.append(exception) self.http_stubber.responses.append(None) with self.http_stubber: response = self.client.put_object(Bucket=self.bucket_name, Key='foo.txt', Body=body) self.assert_status_code(response, 200)
def test_add_md5_with_file_like_body(self): request_dict = { 'body': six.BytesIO(b'foobar'), 'headers': {} } self.md5_digest.return_value = b'8X\xf6"0\xac<\x91_0\x0cfC\x12\xc6?' handlers.calculate_md5(request_dict) self.assertEqual(request_dict['headers']['Content-MD5'], 'OFj2IjCsPJFfMAxmQxLGPw==')
def test_validates_file_like_object(self): value = six.BytesIO(b'foo') errors = self.get_validation_error_message( given_shapes=self.shapes, input_params={'Blob': value}, ) error_msg = errors.generate_report() self.assertEqual(error_msg, '')
def setUp(self): self.credentials = botocore.credentials.Credentials(access_key='foo', secret_key='bar', token='baz') self.auth = botocore.auth.S3SigV4Auth(self.credentials, 'ec2', 'eu-central-1') self.request = AWSRequest(data=six.BytesIO(b"foo bar baz")) self.request.method = 'PUT' self.request.url = 'https://s3.eu-central-1.amazonaws.com/'
def test_can_set_utf_8_headers(self): bucket_name = self.create_bucket(self.region) body = six.BytesIO(b"Hello world!") response = self.client.put_object( Bucket=bucket_name, Key="foo.txt", Body=body, ContentDisposition="attachment; filename=5小時接力起跑.jpg;") self.assert_status_code(response, 200) self.addCleanup(self.client.delete_object, Bucket=bucket_name, Key="foo.txt")
def test_content_sha256_set_if_payload_signing_disabled(self): request = AWSRequest() request.data = six.BytesIO(u'\u2713'.encode('utf-8')) request.url = 'https://amazonaws.com' request.context['payload_signing_enabled'] = False request.method = 'PUT' auth = self.create_signer() auth.add_auth(request) sha_header = request.headers['X-Amz-Content-SHA256'] self.assertEqual(sha_header, 'UNSIGNED-PAYLOAD')
def test_streaming_json_upload(self): stream = six.BytesIO(b'{"fakejson": true}') service = self.session.get_service('cloudsearchdomain') operation = service.get_operation('UploadDocuments') built = operation.build_parameters(contentType='application/json', documents=stream) endpoint = service.get_endpoint(region_name='us-east-1', endpoint_url='http://example.com') request = endpoint.create_request(built, signer=None) self.assertEqual(request.body, stream)
def test_download_fileobj(self): obj = six.BytesIO() inject.bucket_download_fileobj(self.bucket, Key='key', Fileobj=obj) self.bucket.meta.client.download_fileobj.assert_called_with( Bucket=self.bucket.name, Key='key', Fileobj=obj, ExtraArgs=None, Callback=None, Config=None)
def test_checksum_added_only_if_not_exists(self): request_dict = { 'headers': { 'x-amz-content-sha256': 'pre-exists', }, 'body': six.BytesIO(b'hello world'), } handlers.add_glacier_checksums(request_dict) self.assertEqual(request_dict['headers']['x-amz-content-sha256'], 'pre-exists')
def test_download_fileobj(self): fileobj = six.BytesIO() inject.object_download_fileobj(self.obj, Fileobj=fileobj) self.obj.meta.client.download_fileobj.assert_called_with( Bucket=self.obj.bucket_name, Key=self.obj.key, Fileobj=fileobj, ExtraArgs=None, Callback=None, Config=None)
def test_download_fileobj(self): fileobj = six.BytesIO() self.client.put_object( Bucket=self.bucket_name, Key='foo', Body=b'beach') self.addCleanup(self.delete_object, 'foo') self.client.download_fileobj( Bucket=self.bucket_name, Key='foo', Fileobj=fileobj) self.assertEqual(fileobj.getvalue(), b'beach')
def test_upload_fileobj(self): fileobj = six.BytesIO(b'foo') inject.bucket_upload_fileobj(self.bucket, Key='key', Fileobj=fileobj) self.bucket.meta.client.upload_fileobj.assert_called_with( Bucket=self.bucket.name, Fileobj=fileobj, Key='key', ExtraArgs=None, Callback=None, Config=None)
def test_get_response_nonstreaming_ng(self): http_response = Response() http_response.headers = { 'content-type': 'application/xml', 'date': 'Sat, 08 Mar 2014 12:05:44 GMT', 'server': 'AmazonS3', 'transfer-encoding': 'chunked', 'x-amz-id-2': 'AAAAAAAAAAAAAAAAAAA', 'x-amz-request-id': 'XXXXXXXXXXXXXXXX' } http_response.raw = six.BytesIO(XMLBODY2) http_response.status_code = 200 http_response.reason = 'ok' http_response.request = Request() session = botocore.session.get_session() s3 = session.get_service('s3') operation = s3.get_operation('ListObjects') # non-streaming operation self.assertEqual( response.get_response(operation.model, http_response)[1], { u'Contents': [{ u'ETag': '"00000000000000000000000000000000"', u'Key': 'test.png', u'LastModified': datetime.datetime(2014, 3, 1, 17, 6, 40, tzinfo=tzutc()), u'Owner': { u'DisplayName': 'dummy', u'ID': 'AAAAAAAAAAAAAAAAAAA' }, u'Size': 6702, u'StorageClass': 'STANDARD' }], u'IsTruncated': False, u'Marker': "", u'MaxKeys': 1000, u'Name': 'mybucket', u'Prefix': "", 'ResponseMetadata': { 'RequestId': 'XXXXXXXXXXXXXXXX', 'HostId': 'AAAAAAAAAAAAAAAAAAA', 'HTTPStatusCode': 200, } })
def stub_get_object(self, full_contents, start_byte=0, end_byte=None): """ Stubs out the get_object operation. :param full_contents: The FULL contents of the object :param start_byte: The first byte to grab. :param end_byte: The last byte to grab. """ get_object_response = {} expected_params = {} contents = full_contents end_byte_range = end_byte # If the start byte is set and the end byte is not, the end byte is # the last byte. if start_byte != 0 and end_byte is None: end_byte = len(full_contents) - 1 # The range on get object where the the end byte is the last byte # should set the input range as e.g. Range='bytes=3-' if end_byte == len(full_contents) - 1: end_byte_range = '' # If this is a ranged get, ContentRange needs to be returned, # contents needs to be pruned, and Range needs to be an expected param. if end_byte is not None: contents = full_contents[start_byte:end_byte + 1] part_range = 'bytes=%s-%s' % (start_byte, end_byte_range) content_range = 'bytes=%s-%s/%s' % ( start_byte, end_byte, len(full_contents)) get_object_response['ContentRange'] = content_range expected_params['Range'] = part_range get_object_response.update({ "AcceptRanges": "bytes", "ETag": self.etag, "ContentLength": len(contents), "ContentType": "binary/octet-stream", "Body": six.BytesIO(contents), "ResponseMetadata": { "HTTPStatusCode": 200 } }) expected_params.update({ "Bucket": self.bucket, "Key": self.key }) self.stubber.add_response( method='get_object', service_response=get_object_response, expected_params=expected_params)