def test_grid_out_lazy_connect(self):
fs = self.db.fs
outfile = GridOut(fs, file_id=-1)
self.assertRaises(NoFile, outfile.read)
self.assertRaises(NoFile, getattr, outfile, 'filename')
infile = GridIn(fs, filename=1)
infile.close()
outfile = GridOut(fs, infile._id)
outfile.read()
outfile.filename
outfile = GridOut(fs, infile._id)
outfile.readchunk()
def test_basic(self):
f = GridIn(self.db.fs, filename="test")
f.write(b"hello world")
f.close()
self.assertEqual(1, self.db.fs.files.find().count())
self.assertEqual(1, self.db.fs.chunks.find().count())
g = GridOut(self.db.fs, f._id)
self.assertEqual(b"hello world", g.read())
# make sure it's still there...
g = GridOut(self.db.fs, f._id)
self.assertEqual(b"hello world", g.read())
f = GridIn(self.db.fs, filename="test")
f.close()
self.assertEqual(2, self.db.fs.files.find().count())
self.assertEqual(1, self.db.fs.chunks.find().count())
g = GridOut(self.db.fs, f._id)
self.assertEqual(b"", g.read())
# test that reading 0 returns proper type
self.assertEqual(b"", g.read(0))
def test_tell(self):
f = GridIn(self.db.fs, chunkSize=3)
f.write(b"hello world")
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual(0, g.tell())
g.read(0)
self.assertEqual(0, g.tell())
g.read(1)
self.assertEqual(1, g.tell())
g.read(2)
self.assertEqual(3, g.tell())
g.read()
self.assertEqual(g.length, g.tell())
def test_alternate_collection(self):
self.db.alt.files.remove({})
self.db.alt.chunks.remove({})
f = GridIn(self.db.alt)
f.write(b("hello world"))
f.close()
self.assertEqual(1, self.db.alt.files.find().count())
self.assertEqual(1, self.db.alt.chunks.find().count())
g = GridOut(self.db.alt, f._id)
self.assertEqual(b("hello world"), g.read())
# test that md5 still works...
self.assertEqual("5eb63bbbe01eeed093cb22bb8f5acdc3", g.md5)
def test_alternate_collection(self):
self.db.alt.files.delete_many({})
self.db.alt.chunks.delete_many({})
f = GridIn(self.db.alt)
f.write(b"hello world")
f.close()
self.assertEqual(1, self.db.alt.files.count_documents({}))
self.assertEqual(1, self.db.alt.chunks.count_documents({}))
g = GridOut(self.db.alt, f._id)
self.assertEqual(b"hello world", g.read())
# test that md5 still works...
self.assertEqual("5eb63bbbe01eeed093cb22bb8f5acdc3", g.md5)
def test_grid_out_file_document(self):
one = GridIn(self.db.fs)
one.write(b"foo bar")
one.close()
two = GridOut(self.db.fs, file_document=self.db.fs.files.find_one())
self.assertEqual(b"foo bar", two.read())
three = GridOut(self.db.fs,
5,
file_document=self.db.fs.files.find_one())
self.assertEqual(b"foo bar", three.read())
four = GridOut(self.db.fs, file_document={})
with self.assertRaises(NoFile):
four.name
def test_read_unaligned_buffer_size(self):
in_data = (b"This is a text that doesn't "
b"quite fit in a single 16-byte chunk.")
f = GridIn(self.db.fs, chunkSize=16)
f.write(in_data)
f.close()
g = GridOut(self.db.fs, f._id)
out_data = b''
while 1:
s = g.read(13)
if not s:
break
out_data += s
self.assertEqual(in_data, out_data)
def put(self, data, **kwargs):
"""Put data in GridFS as a new file.
Equivalent to doing::
try:
f = new_file(**kwargs)
f.write(data)
finally:
f.close()
`data` can be either an instance of :class:`str` (:class:`bytes`
in python 3) or a file-like object providing a :meth:`read` method.
If an `encoding` keyword argument is passed, `data` can also be a
:class:`unicode` (:class:`str` in python 3) instance, which will
be encoded as `encoding` before being written. Any keyword arguments
will be passed through to the created file - see
:meth:`~gridfs.grid_file.GridIn` for possible arguments. Returns the
``"_id"`` of the created file.
If the ``"_id"`` of the file is manually specified, it must
not already exist in GridFS. Otherwise
:class:`~gridfs.errors.FileExists` is raised.
:Parameters:
- `data`: data to be written as a file.
- `**kwargs` (optional): keyword arguments for file creation
.. versionadded:: 1.9
The ability to write :class:`unicode`, if an `encoding` has
been specified as a keyword argument.
.. versionadded:: 1.6
"""
grid_file = GridIn(self.__collection, **kwargs)
# Start a request - necessary if w=0, harmless otherwise
request = self.__collection.database.connection.start_request()
try:
try:
grid_file.write(data)
finally:
grid_file.close()
finally:
# Ensure request is ended even if close() throws error
request.end()
return grid_file._id
def new_file(self, **kwargs):
"""Create a new file in GridFS.
Returns a new :class:`~gridfs.grid_file.GridIn` instance to
which data can be written. Any keyword arguments will be
passed through to :meth:`~gridfs.grid_file.GridIn`.
If the ``"_id"`` of the file is manually specified, it must
not already exist in GridFS. Otherwise
:class:`~gridfs.errors.FileExists` is raised.
:Parameters:
- `**kwargs` (optional): keyword arguments for file creation
.. versionadded:: 1.6
"""
return GridIn(self.__collection, **kwargs)
def open_upload_stream(self,
filename,
chunk_size_bytes=None,
metadata=None):
"""Opens a Stream that the application can write the contents of the
file to.
The user must specify the filename, and can choose to add any
additional information in the metadata field of the file document or
modify the chunk size.
For example::
my_db = MongoClient().test
fs = GridFSBucket(my_db)
grid_in, file_id = fs.open_upload_stream(
"test_file", chunk_size_bytes=4,
metadata={"contentType": "text/plain"})
grid_in.write("data I want to store!")
grid_in.close() # uploaded on close
Returns an instance of :class:`~gridfs.grid_file.GridIn`.
Raises :exc:`~gridfs.errors.NoFile` if no such version of
that file exists.
Raises :exc:`~ValueError` if `filename` is not a string.
:Parameters:
- `filename`: The name of the file to upload.
- `chunk_size_bytes` (options): The number of bytes per chunk of this
file. Defaults to the chunk_size_bytes in :class:`GridFSBucket`.
- `metadata` (optional): User data for the 'metadata' field of the
files collection document. If not provided the metadata field will
be omitted from the files collection document.
"""
validate_string("filename", filename)
opts = {
"filename":
filename,
"chunk_size": (chunk_size_bytes if chunk_size_bytes is not None
else self._chunk_size_bytes)
}
if metadata is not None:
opts["metadata"] = metadata
return GridIn(self._collection, **opts)
def test_zip(self):
zf = BytesIO()
z = zipfile.ZipFile(zf, "w")
z.writestr("test.txt", b"hello world")
z.close()
zf.seek(0)
f = GridIn(self.db.fs, filename="test.zip")
f.write(zf)
f.close()
self.assertEqual(1, self.db.fs.files.count_documents({}))
self.assertEqual(1, self.db.fs.chunks.count_documents({}))
g = GridOut(self.db.fs, f._id)
z = zipfile.ZipFile(g)
self.assertSequenceEqual(z.namelist(), ["test.txt"])
self.assertEqual(z.read("test.txt"), b"hello world")
def helper(data):
f = GridIn(self.db.fs, chunkSize=1)
f.write(data)
f.close()
self.files += 1
self.chunks += len(data)
self.assertEqual(self.files, self.db.fs.files.find().count())
self.assertEqual(self.chunks, self.db.fs.chunks.find().count())
g = GridOut(self.db.fs, f._id)
self.assertEqual(data, g.read())
g = GridOut(self.db.fs, f._id)
self.assertEqual(data, g.read(10) + g.read(10))
return True
def test_readline(self):
f = GridIn(self.db.fs, chunkSize=5)
f.write(b("""Hello world,
How are you?
Hope all is well.
Bye"""))
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual(b("H"), g.read(1))
self.assertEqual(b("ello world,\n"), g.readline())
self.assertEqual(b("How a"), g.readline(5))
self.assertEqual(b(""), g.readline(0))
self.assertEqual(b("re you?\n"), g.readline())
self.assertEqual(b("Hope all is well.\n"), g.readline(1000))
self.assertEqual(b("Bye"), g.readline())
self.assertEqual(b(""), g.readline())
def new_file(self, **kwargs):
"""Create a new file in GridFS.
Returns a new :class:`~gridfs.grid_file.GridIn` instance to
which data can be written. Any keyword arguments will be
passed through to :meth:`~gridfs.grid_file.GridIn`.
If the ``"_id"`` of the file is manually specified, it must
not already exist in GridFS. Otherwise
:class:`~gridfs.errors.FileExists` is raised.
:Parameters:
- `**kwargs` (optional): keyword arguments for file creation
"""
# No need for __ensure_index_files_id() here; GridIn ensures
# the (files_id, n) index when needed.
return GridIn(self.__collection, **kwargs)
def test_readchunk(self):
in_data = b'a' * 10
f = GridIn(self.db.fs, chunkSize=3)
f.write(in_data)
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual(3, len(g.readchunk()))
self.assertEqual(2, len(g.read(2)))
self.assertEqual(1, len(g.readchunk()))
self.assertEqual(3, len(g.read(3)))
self.assertEqual(1, len(g.readchunk()))
self.assertEqual(0, len(g.readchunk()))
def put(self, data, **kwargs):
"""Put data in GridFS as a new file.
Equivalent to doing::
try:
f = new_file(**kwargs)
f.write(data)
finally:
f.close()
`data` can be either an instance of :class:`str` (:class:`bytes`
in python 3) or a file-like object providing a :meth:`read` method.
If an `encoding` keyword argument is passed, `data` can also be a
:class:`unicode` (:class:`str` in python 3) instance, which will
be encoded as `encoding` before being written. Any keyword arguments
will be passed through to the created file - see
:meth:`~gridfs.grid_file.GridIn` for possible arguments. Returns the
``"_id"`` of the created file.
If the ``"_id"`` of the file is manually specified, it must
not already exist in GridFS. Otherwise
:class:`~gridfs.errors.FileExists` is raised.
:Parameters:
- `data`: data to be written as a file.
- `**kwargs` (optional): keyword arguments for file creation
.. versionchanged:: 3.0
w=0 writes to GridFS are now prohibited.
"""
grid_file = GridIn(self.__collection,
disable_md5=self.__disable_md5,
**kwargs)
try:
grid_file.write(data)
finally:
grid_file.close()
return grid_file._id
def put(self, data, **kwargs):
"""Put data in GridFS as a new file.
Equivalent to doing:
>>> f = new_file(**kwargs)
>>> try:
>>> f.write(data)
>>> finally:
>>> f.close()
`data` can be either an instance of :class:`str` or a
file-like object providing a :meth:`read` method. If an
`encoding` keyword argument is passed, `data` can also be a
:class:`unicode` instance, which will be encoded as `encoding`
before being written. Any keyword arguments will be passed
through to the created file - see
:meth:`~gridfs.grid_file.GridIn` for possible
arguments. Returns the ``"_id"`` of the created file.
If the ``"_id"`` of the file is manually specified, it must
not already exist in GridFS. Otherwise
:class:`~gridfs.errors.FileExists` is raised.
:Parameters:
- `data`: data to be written as a file.
- `**kwargs` (optional): keyword arguments for file creation
.. versionadded:: 1.9
The ability to write :class:`unicode`, if an `encoding` has
been specified as a keyword argument.
.. versionadded:: 1.6
"""
grid_file = GridIn(self.__collection, **kwargs)
try:
grid_file.write(data)
finally:
grid_file.close()
return grid_file._id
def test_grid_out_default_opts(self):
self.assertRaises(TypeError, GridOut, "foo")
self.assertRaises(NoFile, GridOut, self.db.fs, 5)
a = GridIn(self.db.fs)
a.close()
b = GridOut(self.db.fs, a._id)
self.assertEqual(a._id, b._id)
self.assertEqual(0, b.length)
self.assertEqual(None, b.content_type)
self.assertEqual(256 * 1024, b.chunk_size)
self.assert_(isinstance(b.upload_date, datetime.datetime))
self.assertEqual(None, b.aliases)
self.assertEqual(None, b.metadata)
self.assertEqual("d41d8cd98f00b204e9800998ecf8427e", b.md5)
for attr in ["_id", "name", "content_type", "length", "chunk_size",
"upload_date", "aliases", "metadata", "md5"]:
self.assertRaises(AttributeError, setattr, b, attr, 5)
def test_grid_out_custom_opts(self):
a = GridIn(self.db.fs, _id=5, filename="my_file",
contentType="text/html", chunkSize=1000, aliases=["foo"],
metadata={"foo": 1, "bar": 2}, bar=3, baz="hello")
a.write("hello world")
a.close()
b = GridOut(self.db.fs, 5)
self.assertEqual(5, b._id)
self.assertEqual(11, b.length)
self.assertEqual("text/html", b.content_type)
self.assertEqual(1000, b.chunk_size)
self.assert_(isinstance(b.upload_date, datetime.datetime))
self.assertEqual(["foo"], b.aliases)
self.assertEqual({"foo": 1, "bar": 2}, b.metadata)
self.assertEqual(3, b.bar)
self.assertEqual("5eb63bbbe01eeed093cb22bb8f5acdc3", b.md5)
for attr in ["_id", "name", "content_type", "length", "chunk_size",
"upload_date", "aliases", "metadata", "md5"]:
self.assertRaises(AttributeError, setattr, b, attr, 5)
def test_seek(self):
f = GridIn(self.db.fs, chunkSize=3)
f.write(b"hello world")
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual(b"hello world", g.read())
g.seek(0)
self.assertEqual(b"hello world", g.read())
g.seek(1)
self.assertEqual(b"ello world", g.read())
self.assertRaises(IOError, g.seek, -1)
g.seek(-3, _SEEK_END)
self.assertEqual(b"rld", g.read())
g.seek(0, _SEEK_END)
self.assertEqual(b"", g.read())
self.assertRaises(IOError, g.seek, -100, _SEEK_END)
g.seek(3)
g.seek(3, _SEEK_CUR)
self.assertEqual(b"world", g.read())
self.assertRaises(IOError, g.seek, -100, _SEEK_CUR)
def test_unacknowledged(self):
# w=0 is prohibited.
with self.assertRaises(ConfigurationError):
GridIn(rs_or_single_client(w=0).pymongo_test.fs)
def test_close(self):
f = GridIn(self.db.fs)
f.close()
self.assertRaises(ValueError, f.write, "test")
f.close()
def test_write_lines(self):
a = GridIn(self.db.fs)
a.writelines([b"hello ", b"world"])
a.close()
self.assertEqual(b"hello world", GridOut(self.db.fs, a._id).read())
def test_grid_in_default_opts(self):
self.assertRaises(TypeError, GridIn, "foo")
a = GridIn(self.db.fs)
self.assertTrue(isinstance(a._id, ObjectId))
self.assertRaises(AttributeError, setattr, a, "_id", 5)
self.assertEqual(None, a.filename)
self.assertEqual(None, a.name)
a.filename = "my_file"
self.assertEqual("my_file", a.filename)
self.assertEqual("my_file", a.name)
self.assertEqual(None, a.content_type)
a.content_type = "text/html"
self.assertEqual("text/html", a.content_type)
self.assertRaises(AttributeError, getattr, a, "length")
self.assertRaises(AttributeError, setattr, a, "length", 5)
self.assertEqual(255 * 1024, a.chunk_size)
self.assertRaises(AttributeError, setattr, a, "chunk_size", 5)
self.assertRaises(AttributeError, getattr, a, "upload_date")
self.assertRaises(AttributeError, setattr, a, "upload_date", 5)
self.assertRaises(AttributeError, getattr, a, "aliases")
a.aliases = ["foo"]
self.assertEqual(["foo"], a.aliases)
self.assertRaises(AttributeError, getattr, a, "metadata")
a.metadata = {"foo": 1}
self.assertEqual({"foo": 1}, a.metadata)
self.assertRaises(AttributeError, setattr, a, "md5", 5)
a.close()
a.forty_two = 42
self.assertEqual(42, a.forty_two)
self.assertTrue(isinstance(a._id, ObjectId))
self.assertRaises(AttributeError, setattr, a, "_id", 5)
self.assertEqual("my_file", a.filename)
self.assertEqual("my_file", a.name)
self.assertEqual("text/html", a.content_type)
self.assertEqual(0, a.length)
self.assertRaises(AttributeError, setattr, a, "length", 5)
self.assertEqual(255 * 1024, a.chunk_size)
self.assertRaises(AttributeError, setattr, a, "chunk_size", 5)
self.assertTrue(isinstance(a.upload_date, datetime.datetime))
self.assertRaises(AttributeError, setattr, a, "upload_date", 5)
self.assertEqual(["foo"], a.aliases)
self.assertEqual({"foo": 1}, a.metadata)
self.assertEqual("d41d8cd98f00b204e9800998ecf8427e", a.md5)
self.assertRaises(AttributeError, setattr, a, "md5", 5)
# Make sure custom attributes that were set both before and after
# a.close() are reflected in b. PYTHON-411.
b = GridFS(self.db).get_last_version(filename=a.filename)
self.assertEqual(a.metadata, b.metadata)
self.assertEqual(a.aliases, b.aliases)
self.assertEqual(a.forty_two, b.forty_two)
def test_md5(self):
f = GridIn(self.db.fs)
f.write(b"hello world\n")
f.close()
self.assertEqual(None, f.md5)
def test_md5(self):
f = GridIn(self.db.fs)
f.write(b"hello world\n")
f.close()
self.assertEqual("6f5902ac237024bdd0c176cb93063dc4", f.md5)
def test_grid_in_lazy_connect(self):
client = MongoClient('badhost', _connect=False)
fs = client.db.fs
infile = GridIn(fs, file_id=-1, chunk_size=1)
self.assertRaises(ConnectionFailure, infile.write, b('data goes here'))
self.assertRaises(ConnectionFailure, infile.close)
