def test_read_never_returns_none(self): """read should never return None.""" conn = boto.connect_s3() conn.create_bucket("mybucket") test_string = u"ветер по морю гуляет..." with smart_open.smart_open("s3://mybucket/mykey", "wb") as fout: fout.write(test_string.encode('utf8')) r = smart_open.S3OpenRead(conn.get_bucket("mybucket").get_key("mykey")) self.assertEquals(r.read(), test_string.encode("utf-8")) self.assertEquals(r.read(), b"") self.assertEquals(r.read(), b"")
def test_readline_eof(self): """Does readline() return empty string on EOF?""" conn = boto.connect_s3() conn.create_bucket("mybucket") with smart_open.smart_open("s3://mybucket/mykey", "wb"): pass mykey = conn.get_bucket("mybucket").get_key("mykey") reader = smart_open.S3OpenRead(mykey) self.assertEquals(reader.readline(), b"") self.assertEquals(reader.readline(), b"") self.assertEquals(reader.readline(), b"")
def test_readline(self): """Does readline() return the correct file content?""" conn = boto.connect_s3() conn.create_bucket("mybucket") test_string = u"hello žluťoučký world!\nhow are you?".encode('utf8') with smart_open.smart_open("s3://mybucket/mykey", "wb") as fout: fout.write(test_string) mykey = conn.get_bucket("mybucket").get_key("mykey") reader = smart_open.S3OpenRead(mykey) self.assertEquals(reader.readline(), u"hello žluťoučký world!\n".encode("utf-8")) self.assertEquals(reader.readline(), b"how are you?")
def test_readline_iter(self): """Does __iter__ return the correct file content?""" conn = boto.connect_s3() conn.create_bucket("mybucket") lines = [u"всем привет!\n", u"что нового?"] with smart_open.smart_open("s3://mybucket/mykey", "wb") as fout: fout.write("".join(lines).encode("utf-8")) mykey = conn.get_bucket("mybucket").get_key("mykey") reader = smart_open.S3OpenRead(mykey) actual_lines = [l.decode("utf-8") for l in reader] self.assertEquals(2, len(actual_lines)) self.assertEquals(lines[0], actual_lines[0]) self.assertEquals(lines[1], actual_lines[1])
def test_s3_boto(self, mock_s3_iter_lines, mock_boto): """Is S3 line iterator called correctly?""" # no credentials smart_open_object = smart_open.S3OpenRead( smart_open.ParseUri("s3://mybucket/mykey")) smart_open_object.__iter__() mock_boto.connect_s3.assert_called_with(aws_access_key_id=None, aws_secret_access_key=None) # with credential smart_open_object = smart_open.S3OpenRead( smart_open.ParseUri("s3://access_id:access_secret@mybucket/mykey")) smart_open_object.__iter__() mock_boto.connect_s3.assert_called_with( aws_access_key_id="access_id", aws_secret_access_key="access_secret") # lookup bucket, key; call s3_iter_lines smart_open_object = smart_open.S3OpenRead( smart_open.ParseUri("s3://access_id:access_secret@mybucket/mykey")) smart_open_object.__iter__() mock_boto.connect_s3().get_bucket.assert_called_with("mybucket") mock_boto.connect_s3().get_bucket().lookup.assert_called_with("mykey") self.assertTrue(mock_s3_iter_lines.called)
def test_s3_iter_lines_with_key(self): """Does s3_iter_lines give correct content?""" # create fake bucket and fake key conn = boto.connect_s3() conn.create_bucket("mybucket") test_string = u"hello žluťoučký world!\nhow are you?".encode('utf8') with smart_open.smart_open("s3://mybucket/mykey", "wb") as fin: fin.write(test_string) # obtain boto key object mykey = conn.get_bucket("mybucket").get_key("mykey") # call s3_iter_lines and check output output = list(smart_open.S3OpenRead(mykey)) self.assertEqual(b''.join(output), test_string)
def test_s3_read_moto(self): """Are S3 files read correctly?""" conn = boto.connect_s3() conn.create_bucket("mybucket") # write some bogus key so we can check it below content = "hello wořld\nhow are you?" with smart_open.smart_open("s3://mybucket/mykey", "wb") as fout: fout.write(content) smart_open_object = smart_open.S3OpenRead(smart_open.ParseUri("s3://mybucket/mykey")) self.assertEqual(content[:6], smart_open_object.read(6)) self.assertEqual(content[6:14], smart_open_object.read(8)) # ř is 2 bytes # make sure iteration does not affect read() for line in smart_open_object: pass self.assertEqual(content[14:], smart_open_object.read()) # read the rest
def test_s3_seek_moto(self): """Does seeking in S3 files work correctly?""" conn = boto.connect_s3() conn.create_bucket("mybucket") # write some bogus key so we can check it below content = "hello wořld\nhow are you?" with smart_open.smart_open("s3://mybucket/mykey", "wb") as fout: fout.write(content) smart_open_object = smart_open.S3OpenRead(smart_open.ParseUri("s3://mybucket/mykey")) self.assertEqual(content[:6], smart_open_object.read(6)) self.assertEqual(content[6:14], smart_open_object.read(8)) # ř is 2 bytes smart_open_object.seek(0) self.assertEqual(content, smart_open_object.read()) # no size given => read whole file smart_open_object.seek(0) self.assertEqual(content, smart_open_object.read(-1)) # same thing