def test_read_never_returns_none(self):
"""read should never return None."""
conn = boto.connect_s3()
conn.create_bucket("mybucket")
test_string = u"ветер по морю гуляет..."
with smart_open.smart_open("s3://mybucket/mykey", "wb") as fout:
fout.write(test_string.encode('utf8'))
r = smart_open.S3OpenRead(conn.get_bucket("mybucket").get_key("mykey"))
self.assertEquals(r.read(), test_string.encode("utf-8"))
self.assertEquals(r.read(), b"")
self.assertEquals(r.read(), b"")
def test_readline_eof(self):
"""Does readline() return empty string on EOF?"""
conn = boto.connect_s3()
conn.create_bucket("mybucket")
with smart_open.smart_open("s3://mybucket/mykey", "wb"):
pass
mykey = conn.get_bucket("mybucket").get_key("mykey")
reader = smart_open.S3OpenRead(mykey)
self.assertEquals(reader.readline(), b"")
self.assertEquals(reader.readline(), b"")
self.assertEquals(reader.readline(), b"")
def test_readline(self):
"""Does readline() return the correct file content?"""
conn = boto.connect_s3()
conn.create_bucket("mybucket")
test_string = u"hello žluťoučký world!\nhow are you?".encode('utf8')
with smart_open.smart_open("s3://mybucket/mykey", "wb") as fout:
fout.write(test_string)
mykey = conn.get_bucket("mybucket").get_key("mykey")
reader = smart_open.S3OpenRead(mykey)
self.assertEquals(reader.readline(),
u"hello žluťoučký world!\n".encode("utf-8"))
self.assertEquals(reader.readline(), b"how are you?")
def test_readline_iter(self):
"""Does __iter__ return the correct file content?"""
conn = boto.connect_s3()
conn.create_bucket("mybucket")
lines = [u"всем привет!\n", u"что нового?"]
with smart_open.smart_open("s3://mybucket/mykey", "wb") as fout:
fout.write("".join(lines).encode("utf-8"))
mykey = conn.get_bucket("mybucket").get_key("mykey")
reader = smart_open.S3OpenRead(mykey)
actual_lines = [l.decode("utf-8") for l in reader]
self.assertEquals(2, len(actual_lines))
self.assertEquals(lines[0], actual_lines[0])
self.assertEquals(lines[1], actual_lines[1])
def test_s3_boto(self, mock_s3_iter_lines, mock_boto):
"""Is S3 line iterator called correctly?"""
# no credentials
smart_open_object = smart_open.S3OpenRead(
smart_open.ParseUri("s3://mybucket/mykey"))
smart_open_object.__iter__()
mock_boto.connect_s3.assert_called_with(aws_access_key_id=None,
aws_secret_access_key=None)
# with credential
smart_open_object = smart_open.S3OpenRead(
smart_open.ParseUri("s3://access_id:access_secret@mybucket/mykey"))
smart_open_object.__iter__()
mock_boto.connect_s3.assert_called_with(
aws_access_key_id="access_id",
aws_secret_access_key="access_secret")
# lookup bucket, key; call s3_iter_lines
smart_open_object = smart_open.S3OpenRead(
smart_open.ParseUri("s3://access_id:access_secret@mybucket/mykey"))
smart_open_object.__iter__()
mock_boto.connect_s3().get_bucket.assert_called_with("mybucket")
mock_boto.connect_s3().get_bucket().lookup.assert_called_with("mykey")
self.assertTrue(mock_s3_iter_lines.called)
def test_s3_iter_lines_with_key(self):
"""Does s3_iter_lines give correct content?"""
# create fake bucket and fake key
conn = boto.connect_s3()
conn.create_bucket("mybucket")
test_string = u"hello žluťoučký world!\nhow are you?".encode('utf8')
with smart_open.smart_open("s3://mybucket/mykey", "wb") as fin:
fin.write(test_string)
# obtain boto key object
mykey = conn.get_bucket("mybucket").get_key("mykey")
# call s3_iter_lines and check output
output = list(smart_open.S3OpenRead(mykey))
self.assertEqual(b''.join(output), test_string)
def test_s3_read_moto(self):
"""Are S3 files read correctly?"""
conn = boto.connect_s3()
conn.create_bucket("mybucket")
# write some bogus key so we can check it below
content = "hello wořld\nhow are you?"
with smart_open.smart_open("s3://mybucket/mykey", "wb") as fout:
fout.write(content)
smart_open_object = smart_open.S3OpenRead(smart_open.ParseUri("s3://mybucket/mykey"))
self.assertEqual(content[:6], smart_open_object.read(6))
self.assertEqual(content[6:14], smart_open_object.read(8)) # ř is 2 bytes
# make sure iteration does not affect read()
for line in smart_open_object:
pass
self.assertEqual(content[14:], smart_open_object.read()) # read the rest
def test_s3_seek_moto(self):
"""Does seeking in S3 files work correctly?"""
conn = boto.connect_s3()
conn.create_bucket("mybucket")
# write some bogus key so we can check it below
content = "hello wořld\nhow are you?"
with smart_open.smart_open("s3://mybucket/mykey", "wb") as fout:
fout.write(content)
smart_open_object = smart_open.S3OpenRead(smart_open.ParseUri("s3://mybucket/mykey"))
self.assertEqual(content[:6], smart_open_object.read(6))
self.assertEqual(content[6:14], smart_open_object.read(8)) # ř is 2 bytes
smart_open_object.seek(0)
self.assertEqual(content, smart_open_object.read()) # no size given => read whole file
smart_open_object.seek(0)
self.assertEqual(content, smart_open_object.read(-1)) # same thing