async def test_iter_gridfs(self):
gfs = AsyncIOMotorGridFS(self.db)
async def cleanup():
await self.db.fs.files.remove()
await self.db.fs.chunks.remove()
await cleanup()
# Empty iterator.
async for _ in gfs.find({'_id': 1}):
self.fail()
data = b'data'
for n_files in 1, 2, 10:
for i in range(n_files):
await gfs.put(data, filename='filename')
# Force extra batches to test iteration.
j = 0
async for _ in gfs.find({'filename': 'filename'}).batch_size(3):
j += 1
self.assertEqual(j, n_files)
await cleanup()
def test_put_unacknowledged(self):
client = self.asyncio_client(w=0)
fs = AsyncIOMotorGridFS(client.motor_test)
with self.assertRaises(ConfigurationError):
yield from fs.put(b"hello")
client.close()
def test_gridfs_secondary(self):
primary_host, primary_port = test.env.primary
primary = self.asyncio_client(primary_host, primary_port)
if test.env.auth:
yield from primary.admin.authenticate(db_user, db_password)
secondary_host, secondary_port = test.env.secondaries[0]
secondary = self.asyncio_client(
secondary_host, secondary_port,
read_preference=ReadPreference.SECONDARY)
if test.env.auth:
yield from secondary.admin.authenticate(db_user, db_password)
yield from primary.motor_test.drop_collection("fs.files")
yield from primary.motor_test.drop_collection("fs.chunks")
# Should detect it's connected to secondary and not attempt to
# create index
fs = AsyncIOMotorGridFS(secondary.motor_test)
# This won't detect secondary, raises error
with self.assertRaises(AutoReconnect):
yield from fs.put(b'foo')
def test_gridfs_secondary(self):
primary_host, primary_port = test.env.primary
primary = self.asyncio_client(primary_host, primary_port)
if test.env.auth:
yield from primary.admin.authenticate(db_user, db_password)
secondary_host, secondary_port = test.env.secondaries[0]
secondary = self.asyncio_client(
secondary_host,
secondary_port,
read_preference=ReadPreference.SECONDARY)
if test.env.auth:
yield from secondary.admin.authenticate(db_user, db_password)
yield from primary.motor_test.drop_collection("fs.files")
yield from primary.motor_test.drop_collection("fs.chunks")
# Should detect it's connected to secondary and not attempt to
# create index
fs = AsyncIOMotorGridFS(secondary.motor_test)
# This won't detect secondary, raises error
with self.assertRaises(AutoReconnect):
yield from fs.put(b'foo')
async def test_iter_gridfs(self):
gfs = AsyncIOMotorGridFS(self.db)
async def cleanup():
await self.db.fs.files.remove()
await self.db.fs.chunks.remove()
await cleanup()
# Empty iterator.
async for _ in gfs.find({'_id': 1}):
self.fail()
data = b'data'
for n_files in 1, 2, 10:
for i in range(n_files):
await gfs.put(data, filename='filename')
# Force extra batches to test iteration.
j = 0
async for _ in gfs.find({'filename': 'filename'}).batch_size(3):
j += 1
self.assertEqual(j, n_files)
await cleanup()
def test_put_unacknowledged(self):
client = self.asyncio_client(w=0)
fs = AsyncIOMotorGridFS(client.motor_test)
with self.assertRaises(ConfigurationError):
yield from fs.put(b"hello")
client.close()
def test_gridfs_replica_set(self):
rsc = self.asyncio_rsc(w=test.env.w,
wtimeout=5000,
read_preference=ReadPreference.SECONDARY)
fs = AsyncIOMotorGridFS(rsc.motor_test)
oid = yield from fs.put(b'foo')
gridout = yield from fs.get(oid)
content = yield from gridout.read()
self.assertEqual(b'foo', content)
def test_stream_to_handler(self):
fs = AsyncIOMotorGridFS(self.db)
for content_length in (0, 1, 100, 100 * 1000):
_id = yield from fs.put(b'a' * content_length)
gridout = yield from fs.get(_id)
handler = test.MockRequestHandler()
yield from gridout.stream_to_handler(handler)
self.assertEqual(content_length, handler.n_written)
yield from fs.delete(_id)
def test_stream_to_handler(self):
fs = AsyncIOMotorGridFS(self.db)
for content_length in (0, 1, 100, 100 * 1000):
_id = yield from fs.put(b'a' * content_length)
gridout = yield from fs.get(_id)
handler = test.MockRequestHandler()
yield from gridout.stream_to_handler(handler)
self.assertEqual(content_length, handler.n_written)
yield from fs.delete(_id)
def test_gridfs_replica_set(self):
rsc = self.asyncio_rsc(
w=test.env.w, wtimeout=5000,
read_preference=ReadPreference.SECONDARY)
fs = AsyncIOMotorGridFS(rsc.motor_test)
oid = yield from fs.put(b'foo')
gridout = yield from fs.get(oid)
content = yield from gridout.read()
self.assertEqual(b'foo', content)
async def test_stream_to_handler(self):
# Sort of Tornado-specific, but it does work with asyncio.
fs = AsyncIOMotorGridFS(self.db)
content_length = 1000
await fs.delete(1)
self.assertEqual(1, await fs.put(b'a' * content_length, _id=1))
gridout = await fs.get(1)
handler = test.MockRequestHandler()
await gridout.stream_to_handler(handler)
self.assertEqual(content_length, handler.n_written)
await fs.delete(1)
def test_stream_to_handler(self):
# TODO: Sort of Tornado-specific, but it does work with asyncio.
class MockRequestHandler(object):
def __init__(self):
self.n_written = 0
def write(self, data):
self.n_written += len(data)
def flush(self):
pass
fs = AsyncIOMotorGridFS(self.db)
for content_length in (0, 1, 100, 100 * 1000):
_id = yield from fs.put(b'a' * content_length)
gridout = yield from fs.get(_id)
handler = MockRequestHandler()
yield from gridout.stream_to_handler(handler)
self.assertEqual(content_length, handler.n_written)
yield from fs.delete(_id)
async def put_gridfile():
with tempfile.TemporaryFile() as tmp:
with gzip.GzipFile(mode='wb', fileobj=tmp) as gzfile:
for _ in range(10):
gzfile.write(b'Nonesuch nonsense\n')
gfs = AsyncIOMotorGridFS(client.my_database)
tmp.seek(0)
await gfs.put(tmp,
filename='my_file',
content_type='text',
metadata={'compressed': True})
def test_stream_to_handler(self):
# TODO: Sort of Tornado-specific, but it does work with asyncio.
class MockRequestHandler(object):
def __init__(self):
self.n_written = 0
def write(self, data):
self.n_written += len(data)
def flush(self):
pass
fs = AsyncIOMotorGridFS(self.db)
for content_length in (0, 1, 100, 100 * 1000):
_id = yield from fs.put(b'a' * content_length)
gridout = yield from fs.get(_id)
handler = MockRequestHandler()
yield from gridout.stream_to_handler(handler)
self.assertEqual(content_length, handler.n_written)
yield from fs.delete(_id)
async def test_iter_gridfs(self):
gfs = AsyncIOMotorGridFS(self.db)
async def cleanup():
await self.db.fs.files.delete_many({})
await self.db.fs.chunks.delete_many({})
await cleanup()
# Empty iterator.
async for _ in gfs.find({'_id': 1}):
self.fail()
data = b'data'
for n_files in 1, 2, 10:
for i in range(n_files):
await gfs.put(data, filename='filename')
# Force extra batches to test iteration.
j = 0
async for _ in gfs.find({'filename': 'filename'}).batch_size(3):
j += 1
self.assertEqual(j, n_files)
await cleanup()
async with await gfs.new_file(_id=1, chunk_size=1) as f:
await f.write(data)
gout = await gfs.find_one({'_id': 1})
chunks = []
async for chunk in gout:
chunks.append(chunk)
self.assertEqual(len(chunks), len(data))
self.assertEqual(b''.join(chunks), data)
class TestAsyncIOGridFS(AsyncIOTestCase):
@asyncio.coroutine
def _reset(self):
yield from self.db.drop_collection("fs.files")
yield from self.db.drop_collection("fs.chunks")
yield from self.db.drop_collection("alt.files")
yield from self.db.drop_collection("alt.chunks")
def setUp(self):
super().setUp()
self.loop.run_until_complete(self._reset())
self.fs = AsyncIOMotorGridFS(self.db)
def tearDown(self):
self.loop.run_until_complete(self._reset())
super().tearDown()
@asyncio_test
def test_get_version(self):
# new_file creates a MotorGridIn.
gin = yield from self.fs.new_file(_id=1, filename='foo', field=0)
yield from gin.write(b'a')
yield from gin.close()
yield from self.fs.put(b'', filename='foo', field=1)
yield from self.fs.put(b'', filename='foo', field=2)
gout = yield from self.fs.get_version('foo')
self.assertEqual(2, gout.field)
gout = yield from self.fs.get_version('foo', -3)
self.assertEqual(0, gout.field)
gout = yield from self.fs.get_last_version('foo')
self.assertEqual(2, gout.field)
@asyncio_test
def test_basic(self):
oid = yield from self.fs.put(b"hello world")
out = yield from self.fs.get(oid)
self.assertEqual(b"hello world", (yield from out.read()))
self.assertEqual(1, (yield from self.db.fs.files.count()))
self.assertEqual(1, (yield from self.db.fs.chunks.count()))
yield from self.fs.delete(oid)
with self.assertRaises(NoFile):
yield from self.fs.get(oid)
self.assertEqual(0, (yield from self.db.fs.files.count()))
self.assertEqual(0, (yield from self.db.fs.chunks.count()))
with self.assertRaises(NoFile):
yield from self.fs.get("foo")
self.assertEqual(
"foo", (yield from self.fs.put(b"hello world", _id="foo")))
gridout = yield from self.fs.get("foo")
self.assertEqual(b"hello world", (yield from gridout.read()))
@asyncio_test
def test_list(self):
self.assertEqual([], (yield from self.fs.list()))
yield from self.fs.put(b"hello world")
self.assertEqual([], (yield from self.fs.list()))
yield from self.fs.put(b"", filename="mike")
yield from self.fs.put(b"foo", filename="test")
yield from self.fs.put(b"", filename="hello world")
self.assertEqual(set(["mike", "test", "hello world"]),
set((yield from self.fs.list())))
@asyncio_test(timeout=30)
def test_put_filelike(self):
oid = yield from self.fs.put(StringIO(b"hello world"), chunk_size=1)
self.assertEqual(11, (yield from self.cx.motor_test.fs.chunks.count()))
gridout = yield from self.fs.get(oid)
self.assertEqual(b"hello world", (yield from gridout.read()))
@asyncio_test
def test_put_duplicate(self):
oid = yield from self.fs.put(b"hello")
with self.assertRaises(FileExists):
yield from self.fs.put(b"world", _id=oid)
@asyncio_test
def test_put_unacknowledged(self):
client = self.asyncio_client(w=0)
with self.assertRaises(ConfigurationError):
AsyncIOMotorGridFS(client.motor_test)
client.close()
@asyncio_test
def test_gridfs_find(self):
yield from self.fs.put(b"test2", filename="two")
yield from self.fs.put(b"test2+", filename="two")
yield from self.fs.put(b"test1", filename="one")
yield from self.fs.put(b"test2++", filename="two")
cursor = self.fs.find().sort("_id", -1).skip(1).limit(2)
self.assertTrue((yield from cursor.fetch_next))
grid_out = cursor.next_object()
self.assertTrue(isinstance(grid_out, AsyncIOMotorGridOut))
self.assertEqual(b"test1", (yield from grid_out.read()))
self.assertRaises(TypeError, self.fs.find, {}, {"_id": True})
def setUp(self):
super().setUp()
self.loop.run_until_complete(self._reset())
self.fs = AsyncIOMotorGridFS(self.db)
def test_put_unacknowledged(self):
client = self.asyncio_client(w=0)
with self.assertRaises(ConfigurationError):
AsyncIOMotorGridFS(client.motor_test)
client.close()
def setUp(self):
super().setUp()
self.fs = AsyncIOMotorGridFS(self.db)
class TestAsyncIOGridFS(AsyncIOTestCase):
@asyncio.coroutine
def _reset(self):
yield from self.db.drop_collection("fs.files")
yield from self.db.drop_collection("fs.chunks")
yield from self.db.drop_collection("alt.files")
yield from self.db.drop_collection("alt.chunks")
def setUp(self):
super().setUp()
self.loop.run_until_complete(self._reset())
self.fs = AsyncIOMotorGridFS(self.db)
def tearDown(self):
self.loop.run_until_complete(self._reset())
super().tearDown()
@asyncio_test
def test_get_version(self):
# new_file creates a MotorGridIn.
gin = yield from self.fs.new_file(_id=1, filename='foo', field=0)
yield from gin.write(b'a')
yield from gin.close()
yield from self.fs.put(b'', filename='foo', field=1)
yield from self.fs.put(b'', filename='foo', field=2)
gout = yield from self.fs.get_version('foo')
self.assertEqual(2, gout.field)
gout = yield from self.fs.get_version('foo', -3)
self.assertEqual(0, gout.field)
gout = yield from self.fs.get_last_version('foo')
self.assertEqual(2, gout.field)
@asyncio_test
def test_basic(self):
oid = yield from self.fs.put(b"hello world")
out = yield from self.fs.get(oid)
self.assertEqual(b"hello world", (yield from out.read()))
self.assertEqual(1, (yield from self.db.fs.files.count()))
self.assertEqual(1, (yield from self.db.fs.chunks.count()))
yield from self.fs.delete(oid)
with self.assertRaises(NoFile):
yield from self.fs.get(oid)
self.assertEqual(0, (yield from self.db.fs.files.count()))
self.assertEqual(0, (yield from self.db.fs.chunks.count()))
with self.assertRaises(NoFile):
yield from self.fs.get("foo")
self.assertEqual("foo",
(yield from self.fs.put(b"hello world", _id="foo")))
gridout = yield from self.fs.get("foo")
self.assertEqual(b"hello world", (yield from gridout.read()))
@asyncio_test
def test_list(self):
self.assertEqual([], (yield from self.fs.list()))
yield from self.fs.put(b"hello world")
self.assertEqual([], (yield from self.fs.list()))
yield from self.fs.put(b"", filename="mike")
yield from self.fs.put(b"foo", filename="test")
yield from self.fs.put(b"", filename="hello world")
self.assertEqual(set(["mike", "test", "hello world"]),
set((yield from self.fs.list())))
@asyncio_test
def test_put_filelike(self):
oid = yield from self.fs.put(StringIO(b"hello world"), chunk_size=1)
self.assertEqual(11, (yield from self.cx.motor_test.fs.chunks.count()))
gridout = yield from self.fs.get(oid)
self.assertEqual(b"hello world", (yield from gridout.read()))
@asyncio_test
def test_put_duplicate(self):
oid = yield from self.fs.put(b"hello")
with self.assertRaises(FileExists):
yield from self.fs.put(b"world", _id=oid)
@asyncio_test
def test_put_unacknowledged(self):
client = self.asyncio_client(w=0)
fs = AsyncIOMotorGridFS(client.motor_test)
with self.assertRaises(ConfigurationError):
yield from fs.put(b"hello")
client.close()
@asyncio_test
def test_gridfs_find(self):
yield from self.fs.put(b"test2", filename="two")
yield from self.fs.put(b"test2+", filename="two")
yield from self.fs.put(b"test1", filename="one")
yield from self.fs.put(b"test2++", filename="two")
cursor = self.fs.find().sort("_id", -1).skip(1).limit(2)
self.assertTrue((yield from cursor.fetch_next))
grid_out = cursor.next_object()
self.assertTrue(isinstance(grid_out, AsyncIOMotorGridOut))
self.assertEqual(b"test1", (yield from grid_out.read()))
self.assertRaises(TypeError, self.fs.find, {}, {"_id": True})
def setUp(self):
super().setUp()
self.loop.run_until_complete(self._reset())
self.fs = AsyncIOMotorGridFS(self.db)
