def test_head_object_with_json(self):
http_response = Mock()
http_response.encoding = 'utf-8'
http_response.headers = CaseInsensitiveDict(
{'Date': 'Thu, 22 Aug 2013 02:11:57 GMT',
'Content-Length': '0',
'x-amz-request-id': '2B74ECB010FF029E',
'ETag': '"40d06eb6194712ac1c915783004ef730"',
'Server': 'AmazonS3',
'content-type': 'application/json',
'Content-Type': 'application/json',
'accept-ranges': 'bytes',
'Last-Modified': 'Tue, 20 Aug 2013 18:33:25 GMT',
'x-amz-server-side-encryption': 'AES256'})
http_response.content = ''
put_object = self.s3.get_operation('HeadObject')
expected = {"AcceptRanges": "bytes",
"ContentType": "application/json",
"LastModified": "Tue, 20 Aug 2013 18:33:25 GMT",
"ContentLength": "0",
"ETag": '"40d06eb6194712ac1c915783004ef730"',
"ServerSideEncryption": "AES256"
}
response_data = get_response(self.session, put_object,
http_response)[1]
self.assertEqual(response_data, expected)
def test_non_unicode_string_error_strict(self):
my_action = action.CmdAction("", decode_error='strict')
not_unicode = BytesIO(six.b('\xa9'))
realtime = Mock()
realtime.encoding = 'utf-8'
pytest.raises(UnicodeDecodeError, my_action._print_process_output,
Mock(), not_unicode, Mock(), realtime)
def test_list_objects_with_invalid_content_length(self):
http_response = Mock()
http_response.encoding = 'utf-8'
http_response.headers = CaseInsensitiveDict(
{'Date': 'Thu, 22 Aug 2013 02:11:57 GMT',
# We say we have 265 bytes but we're returning 0,
# this should raise an exception because this is not
# a HEAD request.
'Content-Length': '265',
'x-amz-request-id': '2B74ECB010FF029E',
'ETag': '"40d06eb6194712ac1c915783004ef730"',
'Server': 'AmazonS3',
'content-type': 'binary/octet-stream',
'Content-Type': 'binary/octet-stream',
'accept-ranges': 'bytes',
'Last-Modified': 'Tue, 20 Aug 2013 18:33:25 GMT',
'x-amz-server-side-encryption': 'AES256'
})
http_response.content = ''
http_response.request.method = 'GET'
list_objects = self.s3.get_operation('ListObjects')
expected = {"AcceptRanges": "bytes",
"ContentType": "binary/octet-stream",
"LastModified": "Tue, 20 Aug 2013 18:33:25 GMT",
"ContentLength": "265",
"ETag": '"40d06eb6194712ac1c915783004ef730"',
"ServerSideEncryption": "AES256"
}
with self.assertRaises(IncompleteReadError):
response_data = get_response(self.session, list_objects,
http_response)[1]
def test_decode_gzip(self, r_g):
def compressed_gzip_body():
stream = cStringIO.StringIO()
compressor = gzip.GzipFile(fileobj=stream, mode='w')
compressor.write(u"é".encode('utf-8') * 100)
compressor.close()
stream.seek(0)
return stream
normal_response_object = Mock()
# Build an HTTPResponse object like the one requests uses, with
# a gzip compressed body. `decode_content` needs to be False to
# properly emulate requests behaviour : it's the caller's
# responsability to decode, since it's supposed to be a raw stream.
body = compressed_gzip_body()
normal_response_object.raw = HTTPResponse(
status=200, preload_content=False, headers={
'content-encoding': 'gzip',
'content-type': 'application/blah; charset=utf-8'
}, body=body, decode_content=False)
normal_response_object.encoding = 'UTF-8'
normal_response_object.status_code = 200
r_g.return_value = normal_response_object
eq_(appbase.try_get_resource(self.err, None, "http://foo.bar/", ""),
u"é" * 100)
self.assert_silent()
def test_head_object(self):
http_response = Mock()
http_response.encoding = 'utf-8'
http_response.headers = CaseInsensitiveDict(
{'Date': 'Thu, 22 Aug 2013 02:11:57 GMT',
'Content-Length': '265',
'x-amz-request-id': '2B74ECB010FF029E',
'ETag': '"40d06eb6194712ac1c915783004ef730"',
'Server': 'AmazonS3',
'content-type': 'binary/octet-stream',
'Content-Type': 'binary/octet-stream',
'accept-ranges': 'bytes',
'Last-Modified': 'Tue, 20 Aug 2013 18:33:25 GMT',
'x-amz-server-side-encryption': 'AES256',
'x-amz-meta-mykey1': 'value1',
'x-amz-meta-mykey2': 'value2',
})
http_response.content = ''
http_response.request.method = 'HEAD'
put_object = self.s3.get_operation('HeadObject')
expected = {"AcceptRanges": "bytes",
"ContentType": "binary/octet-stream",
"LastModified": "Tue, 20 Aug 2013 18:33:25 GMT",
"ContentLength": "265",
"ETag": '"40d06eb6194712ac1c915783004ef730"',
"ServerSideEncryption": "AES256",
"Metadata": {
'mykey1': 'value1',
'mykey2': 'value2',
}}
response_data = get_response(self.session, put_object,
http_response)[1]
self.assertEqual(response_data, expected)
def test_eventual_404(self, r_g):
error_response = Mock()
error_response.raw.read.side_effect = ["x" * 100, ""]
error_response.encoding = ""
error_response.status_code = 404
r_g.return_value = error_response
appbase.try_get_resource(self.err, None, "http://foo.bar/", "")
self.assert_failed(with_errors=True)
def test_non_unicode_string_error_replace(self):
my_action = action.CmdAction("") # default is decode_error = 'replace'
not_unicode = BytesIO("\xa9".encode("latin-1"))
realtime = Mock()
realtime.encoding = "utf-8"
capture = StringIO()
my_action._print_process_output(Mock(), not_unicode, capture, realtime)
# get the replacement char
expected = "�"
assert expected == capture.getvalue()
def test_non_unicode_string_ok(self):
my_action = action.CmdAction("", encoding="iso-8859-1")
not_unicode = BytesIO(six.b("\xa9"))
realtime = Mock()
realtime.encoding = "utf-8"
capture = StringIO()
my_action._print_process_output(Mock(), not_unicode, capture, realtime)
# get the correct char from latin-1 encoding
expected = "©" if six.PY3 else "©".decode("utf-8")
assert expected == capture.getvalue()
def test_just_right(self, r_g):
normal_response_object = Mock()
normal_response_object.raw.read.side_effect = ["x" * 100, ""]
normal_response_object.encoding = ""
normal_response_object.status_code = 200
r_g.return_value = normal_response_object
eq_(appbase.try_get_resource(self.err, None, "http://foo.bar/", ""),
"x" * 100)
self.assert_silent()
def test_non_unicode_string_ok(self):
my_action = action.CmdAction("", encoding='iso-8859-1')
not_unicode = BytesIO(six.b('\xa9'))
realtime = Mock()
realtime.encoding = 'utf-8'
capture = StringIO()
my_action._print_process_output(Mock(), not_unicode, capture, realtime)
# get the correct char from latin-1 encoding
expected = '©' if six.PY3 else unicode('©', 'utf-8')
assert expected == capture.getvalue()
def test_non_unicode_string_ok(self):
my_action = action.CmdAction("", encoding='iso-8859-1')
not_unicode = BytesIO('\xa9'.encode("latin-1"))
realtime = Mock()
realtime.encoding = 'utf-8'
capture = StringIO()
my_action._print_process_output(Mock(), not_unicode, capture, realtime)
# get the correct char from latin-1 encoding
expected = '©'
assert expected == capture.getvalue()
def test_just_right(self, r_g):
normal_response_object = Mock()
normal_response_object.raw.read.side_effect = ["x" * 100, ""]
normal_response_object.encoding = ""
normal_response_object.status_code = 200
r_g.return_value = normal_response_object
eq_(appbase.try_get_resource(self.err, None, "http://foo.bar/", ""),
"x" * 100)
self.assert_silent()
def test_non_unicode_string_error_replace(self):
my_action = action.CmdAction("") # default is decode_error = 'replace'
not_unicode = BytesIO('\xa9'.encode("latin-1"))
realtime = Mock()
realtime.encoding = 'utf-8'
capture = StringIO()
my_action._print_process_output(Mock(), not_unicode, capture, realtime)
# get the replacement char
expected = '�'
assert expected == capture.getvalue()
def test_empty(self, r_g):
empty_response = Mock()
empty_response.raw.read.return_value = ""
empty_response.encoding = ""
empty_response.status_code = 200
r_g.return_value = empty_response
eq_(appbase.try_get_resource(
self.err, None, "http://foo.bar/", ""), "")
self.assert_failed(with_errors=True)
def test_empty(self, r_g):
empty_response = Mock()
empty_response.raw.read.return_value = ""
empty_response.encoding = ""
empty_response.status_code = 200
r_g.return_value = empty_response
eq_(appbase.try_get_resource(self.err, None, "http://foo.bar/", ""),
"")
self.assert_failed(with_errors=True)
def test_non_unicode_string_error_replace(self):
my_action = action.CmdAction("") # default is decode_error = 'replace'
not_unicode = BytesIO(six.b('\xa9'))
realtime = Mock()
realtime.encoding = 'utf-8'
capture = StringIO()
my_action._print_process_output(Mock(), not_unicode, capture, realtime)
# get the replacement char
expected = '�' if six.PY3 else unicode('�', 'utf-8')
assert expected == capture.getvalue()
def test_unicodeness(self, r_g):
normal_response_object = Mock()
normal_response_object.raw.read.side_effect = [u"é".encode('utf-8')
* 100, ""]
normal_response_object.encoding = "UTF-8"
normal_response_object.status_code = 200
r_g.return_value = normal_response_object
eq_(appbase.try_get_resource(self.err, None, "http://foo.bar/", ""),
u"é" * 100)
self.assert_silent()
def test_unicode_binary(self, r_g):
normal_response_object = Mock()
# The u"é" is *not* encoded in UTF-8 this time, so it would throw an
# UnicodeEncodeError if we'd try to decode it.
normal_response_object.raw.read.side_effect = [u"é" * 100, ""]
normal_response_object.encoding = "UTF-8"
normal_response_object.status_code = 200
r_g.return_value = normal_response_object
eq_(appbase.try_get_resource(self.err, None, "http://foo.bar/", "",
binary=True), u"é" * 100)
self.assert_silent()
def test_unicodeness(self, r_g):
normal_response_object = Mock()
normal_response_object.raw.read.side_effect = [
u"é".encode('utf-8') * 100, ""
]
normal_response_object.encoding = "UTF-8"
normal_response_object.status_code = 200
r_g.return_value = normal_response_object
eq_(appbase.try_get_resource(self.err, None, "http://foo.bar/", ""),
u"é" * 100)
self.assert_silent()
def request_generator(*args, **kwargs):
url = kwargs.get("url", args[0])
if "://" not in url:
raise requests.exceptions.MissingSchema
request = Mock()
request.text = "foo bar"
request.status_code = 200
request.encoding = 'UTF-8'
# The first bit is the return value. The second bit tells whatever
# is requesting the data that there's no more data.
request.raw.read.side_effect = [request.text, ""]
return request
def request_generator(*args, **kwargs):
url = kwargs.get("url", args[0])
if "://" not in url:
raise requests.exceptions.MissingSchema
request = Mock()
request.text = "foo bar"
request.status_code = 200
request.encoding = 'UTF-8'
# The first bit is the return value. The second bit tells whatever
# is requesting the data that there's no more data.
request.raw.read.side_effect = [request.text, ""]
return request
def test_put_object(self):
http_response = Mock()
http_response.encoding = 'utf-8'
http_response.headers = {'Date': 'Thu, 22 Aug 2013 02:11:57 GMT',
'Content-Length': '0',
'x-amz-request-id': '2B74ECB010FF029E',
'ETag': '"b081e66e7e0c314285c655cafb4d1e71"',
'x-amz-id-2': 'bKECRRBFttBRVbJPIVBLQwwipI0i+s9HMvNFdttR17ouR0pvQSKEJUR+1c6cW1nQ',
'Server': 'AmazonS3',
'content-type': 'text/xml'}
http_response.content = ''
put_object = self.s3.get_operation('PutObject')
expected = {"ETag": '"b081e66e7e0c314285c655cafb4d1e71"'}
response_data = get_response(self.session, put_object, http_response)[1]
self.assertEqual(response_data, expected)
def test_unicode_binary(self, r_g):
normal_response_object = Mock()
# The u"é" is *not* encoded in UTF-8 this time, so it would throw an
# UnicodeEncodeError if we'd try to decode it.
normal_response_object.raw.read.side_effect = [u"é" * 100, ""]
normal_response_object.encoding = "UTF-8"
normal_response_object.status_code = 200
r_g.return_value = normal_response_object
eq_(
appbase.try_get_resource(self.err,
None,
"http://foo.bar/",
"",
binary=True), u"é" * 100)
self.assert_silent()
def test_head_object(self):
http_response = Mock()
http_response.encoding = 'utf-8'
http_response.headers = CaseInsensitiveDict({
'Date':
'Thu, 22 Aug 2013 02:11:57 GMT',
'Content-Length':
'265',
'x-amz-request-id':
'2B74ECB010FF029E',
'ETag':
'"40d06eb6194712ac1c915783004ef730"',
'Server':
'AmazonS3',
'content-type':
'binary/octet-stream',
'Content-Type':
'binary/octet-stream',
'accept-ranges':
'bytes',
'Last-Modified':
'Tue, 20 Aug 2013 18:33:25 GMT',
'x-amz-server-side-encryption':
'AES256',
'x-amz-meta-mykey1':
'value1',
'x-amz-meta-mykey2':
'value2',
})
http_response.content = ''
http_response.request.method = 'HEAD'
put_object = self.s3.get_operation('HeadObject')
expected = {
"AcceptRanges": "bytes",
"ContentType": "binary/octet-stream",
"LastModified": "Tue, 20 Aug 2013 18:33:25 GMT",
"ContentLength": "265",
"ETag": '"40d06eb6194712ac1c915783004ef730"',
"ServerSideEncryption": "AES256",
"Metadata": {
'mykey1': 'value1',
'mykey2': 'value2',
}
}
response_data = get_response(self.session, put_object,
http_response)[1]
self.assertEqual(response_data, expected)
def test_put_object(self):
http_response = Mock()
http_response.encoding = 'utf-8'
http_response.headers = {
'Date': 'Thu, 22 Aug 2013 02:11:57 GMT',
'Content-Length': '0',
'x-amz-request-id': '2B74ECB010FF029E',
'ETag': '"b081e66e7e0c314285c655cafb4d1e71"',
'x-amz-id-2':
'bKECRRBFttBRVbJPIVBLQwwipI0i+s9HMvNFdttR17ouR0pvQSKEJUR+1c6cW1nQ',
'Server': 'AmazonS3',
'content-type': 'text/xml'
}
http_response.content = ''
put_object = self.s3.get_operation('PutObject')
expected = {"ETag": '"b081e66e7e0c314285c655cafb4d1e71"'}
response_data = get_response(self.session, put_object,
http_response)[1]
self.assertEqual(response_data, expected)
def test_list_objects_with_invalid_content_length(self):
http_response = Mock()
http_response.encoding = 'utf-8'
http_response.headers = CaseInsensitiveDict({
'Date':
'Thu, 22 Aug 2013 02:11:57 GMT',
# We say we have 265 bytes but we're returning 0,
# this should raise an exception because this is not
# a HEAD request.
'Content-Length':
'265',
'x-amz-request-id':
'2B74ECB010FF029E',
'ETag':
'"40d06eb6194712ac1c915783004ef730"',
'Server':
'AmazonS3',
'content-type':
'binary/octet-stream',
'Content-Type':
'binary/octet-stream',
'accept-ranges':
'bytes',
'Last-Modified':
'Tue, 20 Aug 2013 18:33:25 GMT',
'x-amz-server-side-encryption':
'AES256'
})
http_response.content = ''
http_response.request.method = 'GET'
list_objects = self.s3.get_operation('ListObjects')
expected = {
"AcceptRanges": "bytes",
"ContentType": "binary/octet-stream",
"LastModified": "Tue, 20 Aug 2013 18:33:25 GMT",
"ContentLength": "265",
"ETag": '"40d06eb6194712ac1c915783004ef730"',
"ServerSideEncryption": "AES256"
}
with self.assertRaises(IncompleteReadError):
response_data = get_response(self.session, list_objects,
http_response)[1]
def test(self, inspect, util, quote_ident, tmp_table, cols, data):
mock_cursor = Mock()
connection = Mock()
connection.cursor.return_value = mock_cursor
connection.encoding = 'UTF8'
Record = namedtuple(
"Record",
"attname,type_category,type_name,type_mod,not_null,typelem")
inspect.get_types.return_value = {
'id':
Record(attname='id',
type_category='N',
type_name='int8',
type_mod=-1,
not_null=False,
typelem=0),
'location':
Record(attname='location',
type_category='S',
type_name='varchar',
type_mod=259,
not_null=False,
typelem=0)
}
import_data(connection, tmp_table, cols, data)
# assert connection.cursor.call_count == mock_cursor.close.call_count
mock_cursor.copy_expert.assert_called_once()
expected = [
call(
'COPY "public"."src_tbl" ("id", "location") FROM STDIN WITH BINARY',
ANY)
]
assert mock_cursor.copy_expert.call_args_list == expected
def test_head_object_with_json(self):
http_response = Mock()
http_response.encoding = 'utf-8'
http_response.headers = CaseInsensitiveDict({
'Date':
'Thu, 22 Aug 2013 02:11:57 GMT',
'Content-Length':
'0',
'x-amz-request-id':
'2B74ECB010FF029E',
'ETag':
'"40d06eb6194712ac1c915783004ef730"',
'Server':
'AmazonS3',
'content-type':
'application/json',
'Content-Type':
'application/json',
'accept-ranges':
'bytes',
'Last-Modified':
'Tue, 20 Aug 2013 18:33:25 GMT',
'x-amz-server-side-encryption':
'AES256'
})
http_response.content = ''
put_object = self.s3.get_operation('HeadObject')
expected = {
"AcceptRanges": "bytes",
"ContentType": "application/json",
"LastModified": "Tue, 20 Aug 2013 18:33:25 GMT",
"ContentLength": "0",
"ETag": '"40d06eb6194712ac1c915783004ef730"',
"ServerSideEncryption": "AES256"
}
response_data = get_response(self.session, put_object,
http_response)[1]
self.assertEqual(response_data, expected)
def test_non_unicode_string_error_strict(self):
my_action = action.CmdAction("", decode_error="strict")
not_unicode = BytesIO("\xa9".encode("latin-1"))
realtime = Mock()
realtime.encoding = "utf-8"
pytest.raises(UnicodeDecodeError, my_action._print_process_output, Mock(), not_unicode, Mock(), realtime)
