def test_readline(self):
f = GridIn(self.db.fs, chunkSize=5)
f.write(
b(
"""Hello world,
How are you?
Hope all is well.
Bye"""
)
)
f.close()
# Try read(), then readline().
g = GridOut(self.db.fs, f._id)
self.assertEqual(b("H"), g.read(1))
self.assertEqual(b("ello world,\n"), g.readline())
self.assertEqual(b("How a"), g.readline(5))
self.assertEqual(b(""), g.readline(0))
self.assertEqual(b("re you?\n"), g.readline())
self.assertEqual(b("Hope all is well.\n"), g.readline(1000))
self.assertEqual(b("Bye"), g.readline())
self.assertEqual(b(""), g.readline())
# Try readline() first, then read().
g = GridOut(self.db.fs, f._id)
self.assertEqual(b("He"), g.readline(2))
self.assertEqual(b("l"), g.read(1))
self.assertEqual(b("lo"), g.readline(2))
self.assertEqual(b(" world,\n"), g.readline())
# Only readline().
g = GridOut(self.db.fs, f._id)
self.assertEqual(b("H"), g.readline(1))
self.assertEqual(b("e"), g.readline(1))
self.assertEqual(b("llo world,\n"), g.readline())
def test_grid_out_lazy_connect(self):
fs = self.db.fs
outfile = GridOut(fs, file_id=-1, _connect=False)
self.assertRaises(NoFile, outfile.read)
self.assertRaises(NoFile, getattr, outfile, 'filename')
infile = GridIn(fs, filename=1)
infile.close()
outfile = GridOut(fs, infile._id, _connect=False)
outfile.read()
outfile.filename
def test_grid_out_file_document(self):
one = GridIn(self.db.fs)
one.write(b("foo bar"))
one.close()
two = GridOut(self.db.fs, file_document=self.db.fs.files.find_one())
self.assertEqual(b("foo bar"), two.read())
three = GridOut(self.db.fs, 5, file_document=self.db.fs.files.find_one())
self.assertEqual(b("foo bar"), three.read())
self.assertRaises(NoFile, GridOut, self.db.fs, file_document={})
def test_grid_out_file_document(self):
a = GridIn(self.db.fs)
a.write("foo bar")
a.close()
b = GridOut(self.db.fs, file_document=self.db.fs.files.find_one())
self.assertEqual("foo bar", b.read())
c = GridOut(self.db.fs, 5, file_document=self.db.fs.files.find_one())
self.assertEqual("foo bar", c.read())
self.assertRaises(NoFile, GridOut, self.db.fs, file_document={})
def test_multiple_reads(self):
f = GridIn(self.db.fs, chunkSize=3)
f.write("hello world")
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual("he", g.read(2))
self.assertEqual("ll", g.read(2))
self.assertEqual("o ", g.read(2))
self.assertEqual("wo", g.read(2))
self.assertEqual("rl", g.read(2))
self.assertEqual("d", g.read(2))
self.assertEqual("", g.read(2))
def test_grid_out_file_document(self):
one = GridIn(self.db.fs)
one.write(b"foo bar")
one.close()
two = GridOut(self.db.fs, file_document=self.db.fs.files.find_one())
self.assertEqual(b"foo bar", two.read())
three = GridOut(self.db.fs, 5, file_document=self.db.fs.files.find_one())
self.assertEqual(b"foo bar", three.read())
four = GridOut(self.db.fs, file_document={})
with self.assertRaises(NoFile):
four.name
def test_readchunk(self):
in_data = b('a') * 10
f = GridIn(self.db.fs, chunkSize=3)
f.write(in_data)
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual(3, len(g.readchunk()))
self.assertEqual(2, len(g.read(2)))
self.assertEqual(1, len(g.readchunk()))
self.assertEqual(3, len(g.read(3)))
self.assertEqual(1, len(g.readchunk()))
self.assertEqual(0, len(g.readchunk()))
def test_write_unicode(self):
f = GridIn(self.db.fs)
self.assertRaises(TypeError, f.write, u"foo")
f = GridIn(self.db.fs, encoding="utf-8")
f.write(u"foo")
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual("foo", g.read())
f = GridIn(self.db.fs, encoding="iso-8859-1")
f.write(u"aé")
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual(u"aé".encode("iso-8859-1"), g.read())
def test_write_unicode(self):
f = GridIn(self.db.fs)
self.assertRaises(TypeError, f.write, u"foo")
f = GridIn(self.db.fs, encoding="utf-8")
f.write(u"foo")
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual(b("foo"), g.read())
f = GridIn(self.db.fs, encoding="iso-8859-1")
f.write(u"aé")
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual(u"aé".encode("iso-8859-1"), g.read())
def helper(data):
f = GridIn(self.db.fs, chunkSize=1)
f.write(data)
f.close()
self.files += 1
self.chunks += len(data)
self.assertEqual(self.files, self.db.fs.files.find().count())
self.assertEqual(self.chunks, self.db.fs.chunks.find().count())
g = GridOut(self.db.fs, f._id)
self.assertEqual(data, g.read())
g = GridOut(self.db.fs, f._id)
self.assertEqual(data, g.read(10) + g.read(10))
return True
def test_readchunk(self):
in_data = b("a") * 10
f = GridIn(self.db.fs, chunkSize=3)
f.write(in_data)
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual(3, len(g.readchunk()))
self.assertEqual(2, len(g.read(2)))
self.assertEqual(1, len(g.readchunk()))
self.assertEqual(3, len(g.read(3)))
self.assertEqual(1, len(g.readchunk()))
self.assertEqual(0, len(g.readchunk()))
def helper(data):
f = GridIn(self.db.fs, chunkSize=1)
f.write(data)
f.close()
self.files += 1
self.chunks += len(data)
self.assertEqual(self.files, self.db.fs.files.find().count())
self.assertEqual(self.chunks, self.db.fs.chunks.find().count())
g = GridOut(self.db.fs, f._id)
self.assertEqual(data, g.read())
g = GridOut(self.db.fs, f._id)
self.assertEqual(data, g.read(10) + g.read(10))
return True
def test_iterator(self):
f = GridIn(self.db.fs)
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual([], list(g))
f = GridIn(self.db.fs)
f.write(b"hello world")
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual([b"hello world"], list(g))
self.assertEqual(b"hello", g.read(5))
self.assertEqual([b"hello world"], list(g))
self.assertEqual(b" worl", g.read(5))
f = GridIn(self.db.fs, chunk_size=2)
f.write(b"hello world")
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual([b"he", b"ll", b"o ", b"wo", b"rl", b"d"], list(g))
def test_iterator(self):
f = GridIn(self.db.fs)
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual([], list(g))
f = GridIn(self.db.fs)
f.write(b("hello world"))
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual([b("hello world")], list(g))
self.assertEqual(b("hello"), g.read(5))
self.assertEqual([b("hello world")], list(g))
self.assertEqual(b(" worl"), g.read(5))
f = GridIn(self.db.fs, chunk_size=2)
f.write(b("hello world"))
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual([b("he"), b("ll"), b("o "), b("wo"), b("rl"), b("d")], list(g))
def test_iterator(self):
f = GridIn(self.db.fs)
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual([], list(g))
f = GridIn(self.db.fs)
f.write("hello world")
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual(["hello world"], list(g))
self.assertEqual("hello", g.read(5))
self.assertEqual(["hello world"], list(g))
self.assertEqual(" worl", g.read(5))
f = GridIn(self.db.fs, chunk_size=2)
f.write("hello world")
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual(["he", "ll", "o ", "wo", "rl", "d"], list(g))
def test_multi_chunk_file(self):
random_string = b("a") * (DEFAULT_CHUNK_SIZE + 1000)
f = GridIn(self.db.fs)
f.write(random_string)
f.close()
self.assertEqual(1, self.db.fs.files.find().count())
self.assertEqual(2, self.db.fs.chunks.find().count())
g = GridOut(self.db.fs, f._id)
self.assertEqual(random_string, g.read())
def test_basic(self):
f = GridIn(self.db.fs, filename="test")
f.write(b("hello world"))
f.close()
self.assertEqual(1, self.db.fs.files.find().count())
self.assertEqual(1, self.db.fs.chunks.find().count())
g = GridOut(self.db.fs, f._id)
self.assertEqual(b("hello world"), g.read())
# make sure it's still there...
g = GridOut(self.db.fs, f._id)
self.assertEqual(b("hello world"), g.read())
f = GridIn(self.db.fs, filename="test")
f.close()
self.assertEqual(2, self.db.fs.files.find().count())
self.assertEqual(1, self.db.fs.chunks.find().count())
g = GridOut(self.db.fs, f._id)
self.assertEqual(b(""), g.read())
def test_iterator(self):
f = GridIn(self.db.fs)
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual([], list(g))
f = GridIn(self.db.fs)
f.write(b("hello world"))
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual([b("hello world")], list(g))
self.assertEqual(b("hello"), g.read(5))
self.assertEqual([b("hello world")], list(g))
self.assertEqual(b(" worl"), g.read(5))
f = GridIn(self.db.fs, chunk_size=2)
f.write(b("hello world"))
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual([b("he"), b("ll"), b("o "),
b("wo"), b("rl"), b("d")], list(g))
def test_multi_chunk_file(self):
random_string = qcheck.gen_string(qcheck.lift(300000))()
f = GridIn(self.db.fs)
f.write(random_string)
f.close()
self.assertEqual(1, self.db.fs.files.find().count())
self.assertEqual(2, self.db.fs.chunks.find().count())
g = GridOut(self.db.fs, f._id)
self.assertEqual(random_string, g.read())
def test_multi_chunk_file(self):
random_string = qcheck.gen_string(qcheck.lift(300000))()
f = GridIn(self.db.fs)
f.write(random_string)
f.close()
self.assertEqual(1, self.db.fs.files.find().count())
self.assertEqual(2, self.db.fs.chunks.find().count())
g = GridOut(self.db.fs, f._id)
self.assertEqual(random_string, g.read())
def test_multi_chunk_file(self):
random_string = b'a' * (DEFAULT_CHUNK_SIZE + 1000)
f = GridIn(self.db.fs)
f.write(random_string)
f.close()
self.assertEqual(1, self.db.fs.files.count_documents({}))
self.assertEqual(2, self.db.fs.chunks.count_documents({}))
g = GridOut(self.db.fs, f._id)
self.assertEqual(random_string, g.read())
def test_alternate_collection(self):
self.db.alt.files.delete_many({})
self.db.alt.chunks.delete_many({})
f = GridIn(self.db.alt)
f.write(b"hello world")
f.close()
self.assertEqual(1, self.db.alt.files.count_documents({}))
self.assertEqual(1, self.db.alt.chunks.count_documents({}))
g = GridOut(self.db.alt, f._id)
self.assertEqual(b"hello world", g.read())
def test_tell(self):
f = GridIn(self.db.fs, chunkSize=3)
f.write("hello world")
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual(0, g.tell())
g.read(0)
self.assertEqual(0, g.tell())
g.read(1)
self.assertEqual(1, g.tell())
g.read(2)
self.assertEqual(3, g.tell())
g.read()
self.assertEqual(g.length, g.tell())
def test_tell(self):
f = GridIn(self.db.fs, chunkSize=3)
f.write(b"hello world")
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual(0, g.tell())
g.read(0)
self.assertEqual(0, g.tell())
g.read(1)
self.assertEqual(1, g.tell())
g.read(2)
self.assertEqual(3, g.tell())
g.read()
self.assertEqual(g.length, g.tell())
def write_me(s, chunk_size):
buf = StringIO(s)
infile = GridIn(self.db.fs)
while True:
to_write = buf.read(chunk_size)
if to_write == b(''):
break
infile.write(to_write)
infile.close()
buf.close()
outfile = GridOut(self.db.fs, infile._id)
data = outfile.read()
self.assertEqual(s, data)
def write_me(s, chunk_size):
buf = StringIO(s)
infile = GridIn(self.db.fs)
while True:
to_write = buf.read(chunk_size)
if to_write == b'':
break
infile.write(to_write)
infile.close()
buf.close()
outfile = GridOut(self.db.fs, infile._id)
data = outfile.read()
self.assertEqual(s, data)
def test_read_chunks_unaligned_buffer_size(self):
in_data = "This is a text that doesn't quite fit in a single 16-byte chunk."
f = GridIn(self.db.fs, chunkSize=16)
f.write(in_data)
f.close()
g = GridOut(self.db.fs, f._id)
out_data = ''
while 1:
s = g.read(13)
if not s:
break
out_data += s
self.assertEqual(in_data, out_data)
def test_read_chunks_unaligned_buffer_size(self):
in_data = "This is a text that doesn't quite fit in a single 16-byte chunk."
f = GridIn(self.db.fs, chunkSize=16)
f.write(in_data)
f.close()
g = GridOut(self.db.fs, f._id)
out_data = ''
while 1:
s = g.read(13)
if not s:
break
out_data += s
self.assertEqual(in_data, out_data)
def test_basic(self):
f = GridIn(self.db.fs, filename="test")
f.write(b"hello world")
f.close()
self.assertEqual(1, self.db.fs.files.find().count())
self.assertEqual(1, self.db.fs.chunks.find().count())
g = GridOut(self.db.fs, f._id)
self.assertEqual(b"hello world", g.read())
# make sure it's still there...
g = GridOut(self.db.fs, f._id)
self.assertEqual(b"hello world", g.read())
f = GridIn(self.db.fs, filename="test")
f.close()
self.assertEqual(2, self.db.fs.files.find().count())
self.assertEqual(1, self.db.fs.chunks.find().count())
g = GridOut(self.db.fs, f._id)
self.assertEqual(b"", g.read())
# test that reading 0 returns proper type
self.assertEqual(b"", g.read(0))
def test_basic(self):
f = GridIn(self.db.fs, filename="test")
f.write(b"hello world")
f.close()
self.assertEqual(1, self.db.fs.files.count_documents({}))
self.assertEqual(1, self.db.fs.chunks.count_documents({}))
g = GridOut(self.db.fs, f._id)
self.assertEqual(b"hello world", g.read())
# make sure it's still there...
g = GridOut(self.db.fs, f._id)
self.assertEqual(b"hello world", g.read())
f = GridIn(self.db.fs, filename="test")
f.close()
self.assertEqual(2, self.db.fs.files.count_documents({}))
self.assertEqual(1, self.db.fs.chunks.count_documents({}))
g = GridOut(self.db.fs, f._id)
self.assertEqual(b"", g.read())
# test that reading 0 returns proper type
self.assertEqual(b"", g.read(0))
def test_alternate_collection(self):
self.db.alt.files.remove({})
self.db.alt.chunks.remove({})
f = GridIn(self.db.alt)
f.write("hello world")
f.close()
self.assertEqual(1, self.db.alt.files.find().count())
self.assertEqual(1, self.db.alt.chunks.find().count())
g = GridOut(self.db.alt, f._id)
self.assertEqual("hello world", g.read())
# test that md5 still works...
self.assertEqual("5eb63bbbe01eeed093cb22bb8f5acdc3", g.md5)
def test_alternate_collection(self):
self.db.alt.files.delete_many({})
self.db.alt.chunks.delete_many({})
f = GridIn(self.db.alt)
f.write(b"hello world")
f.close()
self.assertEqual(1, self.db.alt.files.count_documents({}))
self.assertEqual(1, self.db.alt.chunks.count_documents({}))
g = GridOut(self.db.alt, f._id)
self.assertEqual(b"hello world", g.read())
# test that md5 still works...
self.assertEqual("5eb63bbbe01eeed093cb22bb8f5acdc3", g.md5)
def test_alternate_collection(self):
self.db.alt.files.remove({})
self.db.alt.chunks.remove({})
f = GridIn(self.db.alt)
f.write(b("hello world"))
f.close()
self.assertEqual(1, self.db.alt.files.find().count())
self.assertEqual(1, self.db.alt.chunks.find().count())
g = GridOut(self.db.alt, f._id)
self.assertEqual(b("hello world"), g.read())
# test that md5 still works...
self.assertEqual("5eb63bbbe01eeed093cb22bb8f5acdc3", g.md5)
def read_bytes_io(obj: GridOut) -> np.array:
"""reads a binary file stored in mongodb and returns a numpy array
Parameters
----------
obj : GridOut
output from a mongodb girdfs file
Returns
-------
np.array
numpy array containing the information loaded from gridfs file
"""
return np.load(io.BytesIO(obj.read()), allow_pickle=True)
def test_readline(self):
f = GridIn(self.db.fs, chunkSize=5)
f.write(b("""Hello world,
How are you?
Hope all is well.
Bye"""))
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual(b("H"), g.read(1))
self.assertEqual(b("ello world,\n"), g.readline())
self.assertEqual(b("How a"), g.readline(5))
self.assertEqual(b(""), g.readline(0))
self.assertEqual(b("re you?\n"), g.readline())
self.assertEqual(b("Hope all is well.\n"), g.readline(1000))
self.assertEqual(b("Bye"), g.readline())
self.assertEqual(b(""), g.readline())
def test_readline(self):
f = GridIn(self.db.fs, chunkSize=5)
f.write("""Hello world,
How are you?
Hope all is well.
Bye""")
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual("H", g.read(1))
self.assertEqual("ello world,\n", g.readline())
self.assertEqual("How a", g.readline(5))
self.assertEqual("", g.readline(0))
self.assertEqual("re you?\n", g.readline())
self.assertEqual("Hope all is well.\n", g.readline(1000))
self.assertEqual("Bye", g.readline())
self.assertEqual("", g.readline())
def test_write_file_like(self):
a = GridIn(self.db.fs)
a.write(b"hello world")
a.close()
b = GridOut(self.db.fs, a._id)
c = GridIn(self.db.fs)
c.write(b)
c.close()
d = GridOut(self.db.fs, c._id)
self.assertEqual(b"hello world", d.read())
e = GridIn(self.db.fs, chunk_size=2)
e.write(b"hello")
buffer = BytesIO(b" world")
e.write(buffer)
e.write(b" and mongodb")
e.close()
self.assertEqual(b"hello world and mongodb", GridOut(self.db.fs, e._id).read())
def test_write_file_like(self):
one = GridIn(self.db.fs)
one.write(b("hello world"))
one.close()
two = GridOut(self.db.fs, one._id)
three = GridIn(self.db.fs)
three.write(two)
three.close()
four = GridOut(self.db.fs, three._id)
self.assertEqual(b("hello world"), four.read())
five = GridIn(self.db.fs, chunk_size=2)
five.write(b("hello"))
buffer = StringIO(b(" world"))
five.write(buffer)
five.write(b(" and mongodb"))
five.close()
self.assertEqual(b("hello world and mongodb"), GridOut(self.db.fs, five._id).read())
def test_write_file_like(self):
a = GridIn(self.db.fs)
a.write("hello world")
a.close()
b = GridOut(self.db.fs, a._id)
c = GridIn(self.db.fs)
c.write(b)
c.close()
d = GridOut(self.db.fs, c._id)
self.assertEqual("hello world", d.read())
e = GridIn(self.db.fs, chunk_size=2)
e.write("hello")
buffer = StringIO(" world")
e.write(buffer)
e.write(" and mongodb")
e.close()
self.assertEqual("hello world and mongodb",
GridOut(self.db.fs, e._id).read())
def test_write_file_like(self):
one = GridIn(self.db.fs)
one.write(b"hello world")
one.close()
two = GridOut(self.db.fs, one._id)
three = GridIn(self.db.fs)
three.write(two)
three.close()
four = GridOut(self.db.fs, three._id)
self.assertEqual(b"hello world", four.read())
five = GridIn(self.db.fs, chunk_size=2)
five.write(b"hello")
buffer = StringIO(b" world")
five.write(buffer)
five.write(b" and mongodb")
five.close()
self.assertEqual(b"hello world and mongodb",
GridOut(self.db.fs, five._id).read())
def test_seek(self):
f = GridIn(self.db.fs, chunkSize=3)
f.write(b"hello world")
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual(b"hello world", g.read())
g.seek(0)
self.assertEqual(b"hello world", g.read())
g.seek(1)
self.assertEqual(b"ello world", g.read())
self.assertRaises(IOError, g.seek, -1)
g.seek(-3, _SEEK_END)
self.assertEqual(b"rld", g.read())
g.seek(0, _SEEK_END)
self.assertEqual(b"", g.read())
self.assertRaises(IOError, g.seek, -100, _SEEK_END)
g.seek(3)
g.seek(3, _SEEK_CUR)
self.assertEqual(b"world", g.read())
self.assertRaises(IOError, g.seek, -100, _SEEK_CUR)
def test_seek(self):
f = GridIn(self.db.fs, chunkSize=3)
f.write("hello world")
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual("hello world", g.read())
g.seek(0)
self.assertEqual("hello world", g.read())
g.seek(1)
self.assertEqual("ello world", g.read())
self.assertRaises(IOError, g.seek, -1)
g.seek(-3, _SEEK_END)
self.assertEqual("rld", g.read())
g.seek(0, _SEEK_END)
self.assertEqual("", g.read())
self.assertRaises(IOError, g.seek, -100, _SEEK_END)
g.seek(3)
g.seek(3, _SEEK_CUR)
self.assertEqual("world", g.read())
self.assertRaises(IOError, g.seek, -100, _SEEK_CUR)
