def test_GridFsIteration(self):
""" Tests gridfs iterator """
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
yield self._drop_gridfs(db)
gfs = GridFS(db) # Default collection
new_file = gfs.new_file(filename="testName", contentType="text/plain", length=1048576,
chunk_size=4096)
yield new_file.write(b"0xDEADBEEF"*4096*2)
yield new_file.close()
fake_doc = {"_id": new_file._id, "name": "testName", "length": 4096*2, "chunkSize": 4096,
"contentType": "text/plain"}
grid_out_file = GridOut(db.fs, fake_doc)
iterator = GridOutIterator(grid_out_file, db.fs.chunks)
next_it = yield next(iterator)
self.assertEqual(len(next_it), 4096)
_ = yield next(iterator)
next_it = yield next(iterator)
self.assertEqual(next_it, None)
fake_bad_doc = {"_id": "bad_id", "name": "testName", "length": 4096*2,
"chunkSize": 4096, "contentType": "text/plain"}
grid_bad_out_file = GridOut(db.fs, fake_bad_doc)
bad_iterator = GridOutIterator(grid_bad_out_file, db.fs.chunks)
if twisted_version.major >= 15:
with self.assertRaises(errors.CorruptGridFile):
next_it = yield bad_iterator.next()
# disconnect
yield conn.disconnect()
def test_GridFsIteration(self):
""" Tests gridfs iterator """
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
db.fs.files.remove({}) # drop all objects there first
db.fs.chunks.remove({})
gfs = GridFS(db) # Default collection
new_file = gfs.new_file(filename="testName", contentType="text/plain", length=1048576,
chunk_size=4096)
yield new_file.write(b"0xDEADBEEF"*4096*2)
yield new_file.close()
fake_doc = {"_id": new_file._id, "name": "testName", "length": 4096*2, "chunkSize": 4096,
"contentType": "text/plain"}
grid_out_file = GridOut(db.fs, fake_doc)
iterator = GridOutIterator(grid_out_file, db.fs.chunks)
next_it = yield next(iterator)
self.assertEqual(len(next_it), 4096)
_ = yield next(iterator)
next_it = yield next(iterator)
self.assertEqual(next_it, None)
fake_bad_doc = {"_id": "bad_id", "name": "testName", "length": 4096*2,
"chunkSize": 4096, "contentType": "text/plain"}
grid_bad_out_file = GridOut(db.fs, fake_bad_doc)
bad_iterator = GridOutIterator(grid_bad_out_file, db.fs.chunks)
if _version.version.major >= 15:
with self.assertRaises(errors.CorruptGridFile):
next_it = yield bad_iterator.next()
# disconnect
yield conn.disconnect()
def test_GridFsIndexesCreation(self):
""" Tests gridfs indexes creation"""
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
yield self._drop_gridfs(db)
# Create a new GridFS instance should trigger indexes creation
gfs = GridFS(db)
# Multiple calls should return multiple defer not to mix between them
self.assertNotEqual(gfs.indexes_created(), gfs.indexes_created())
yield gfs.indexes_created()
indexes = yield db.fs.files.index_information()
self.assertTrue(
any(key["key"] == SON([("filename", 1), ("uploadDate", 1)])
for key in indexes.values()))
indexes = yield db.fs.chunks.index_information()
self.assertTrue(
any(key["key"] == SON([("files_id", 1), ("n", 1)])
for key in indexes.values()))
yield conn.disconnect()
def test_GridFsObjects(self):
""" Tests gridfs objects """
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
db.fs.files.remove({}) # drop all objects there first
db.fs.chunks.remove({})
gfs = GridFS(db) # Default collection
yield gfs.delete(u"test")
_ = gfs.new_file(filename="test_1", contentType="text/plain", chunk_size=65536)
yield conn.disconnect()
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
db.fs.files.remove({}) # drop all objects there first
gfs = GridFS(db) # Default collection
_ = yield gfs.put(b"0xDEADBEEF", filename="test_2", contentType="text/plain",
chunk_size=65536)
# disconnect
yield conn.disconnect()
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
gfs = GridFS(db) # Default collection
_ = yield gfs.get("test_3")
# disconnect
yield conn.disconnect()
def test_GridFsObjects(self):
""" Tests gridfs objects """
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
yield self._drop_gridfs(db)
gfs = GridFS(db) # Default collection
yield gfs.delete(u"test")
_ = gfs.new_file(filename="test_1",
contentType="text/plain",
chunk_size=65536)
yield conn.disconnect()
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
yield self._drop_gridfs(db)
gfs = GridFS(db) # Default collection
_ = yield gfs.put(b"0xDEADBEEF",
filename="test_2",
contentType="text/plain",
chunk_size=65536)
# disconnect
yield conn.disconnect()
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
gfs = GridFS(db) # Default collection
# Missing file raises error
yield self.assertFailure(gfs.get("test_3"), NoFile)
# disconnect
yield conn.disconnect()
def _add_image_to_gridfs(self, image_id, image_content,
image_content_type):
d = GridFS(mongo_connection['rotator_database']).put(
image_content, content_type=image_content_type, md5hash=image_id)
d.addCallback(self._image_to_gridfs_success, image_id)
log_me('_add_image_to_gridfs', image_id, image_content_type)
def test_GridFsIndexesCreation(self):
""" Tests gridfs indexes creation"""
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
yield self._drop_gridfs(db)
# Create a new GridFS instance should trigger indexes creation
gfs = GridFS(db)
# Multiple calls should return multiple defer not to mix between them
self.assertNotEqual(gfs.indexes_created(), gfs.indexes_created())
yield gfs.indexes_created()
yield conn.disconnect()
def test_GridFsIndexesCreation(self):
""" Tests gridfs indexes creation"""
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
yield self._drop_gridfs(db)
# Create a new GridFS instance should trigger indexes creation
gfs = GridFS(db)
# Multiple calls should return multiple defer not to mix between them
self.assertNotEqual(gfs.indexes_created(), gfs.indexes_created())
yield gfs.indexes_created()
yield conn.disconnect()
def test_GridFileObjects(self):
""" Tests gridfs objects """
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
db.fs.files.remove({}) # drop all objects there first
db.fs.chunks.remove({})
self.assertRaises(TypeError, GridFS, None)
_ = GridFS(db) # Default collection
self.assertRaises(TypeError, GridIn, None)
with GridIn(db.fs, filename="test_with", contentType="text/plain", chunk_size=1024):
pass
grid_in_file = GridIn(db.fs, filename="test_1", contentType="text/plain",
content_type="text/plain", chunk_size=65536, length=1048576,
upload_date="20150101")
self.assertFalse(grid_in_file.closed)
if _version.version.major >= 15:
with self.assertRaises(TypeError):
yield grid_in_file.write(1)
with self.assertRaises(TypeError):
yield grid_in_file.write(u"0xDEADBEEF")
with self.assertRaises(AttributeError):
_ = grid_in_file.test
grid_in_file.test = 1
yield grid_in_file.write(b"0xDEADBEEF")
yield grid_in_file.write(b"0xDEADBEEF"*1048576)
fake_doc = {"_id": "test_id", "length": 1048576, "filename": "test",
"upload_date": "20150101"}
self.assertRaises(TypeError, GridOut, None, None)
grid_out_file = GridOut(db.fs, fake_doc)
if _version.version.major >= 15:
with self.assertRaises(AttributeError):
_ = grid_out_file.testing
self.assertEqual("test", grid_out_file.filename)
self.assertEqual(0, grid_out_file.tell())
grid_out_file.seek(1024)
self.assertEqual(1024, grid_out_file.tell())
grid_out_file.seek(1024, os.SEEK_CUR)
self.assertEqual(2048, grid_out_file.tell())
grid_out_file.seek(0, os.SEEK_END)
self.assertEqual(1048576, grid_out_file.tell())
self.assertRaises(IOError, grid_out_file.seek, 0, 4)
self.assertRaises(IOError, grid_out_file.seek, -1)
self.assertTrue("'_id': 'test_id'" in repr(grid_out_file))
self.assertTrue("20150101", grid_in_file.upload_date)
yield grid_in_file.writelines([b"0xDEADBEEF", b"0xDEADBEAF"])
yield grid_in_file.close()
if _version.version.major >= 15:
with self.assertRaises(AttributeError):
grid_in_file.length = 1
self.assertEqual(1, grid_in_file.test)
if _version.version.major >= 15:
with self.assertRaises(AttributeError):
_ = grid_in_file.test_none
self.assertTrue(grid_in_file.closed)
if _version.version.major >= 15:
with self.assertRaises(ValueError):
yield grid_in_file.write(b"0xDEADBEEF")
yield conn.disconnect()
def test_GridFsObjects(self):
""" Tests gridfs objects """
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
yield self._drop_gridfs(db)
gfs = GridFS(db) # Default collection
yield gfs.delete(u"test")
_ = gfs.new_file(filename="test_1", contentType="text/plain", chunk_size=65536)
yield conn.disconnect()
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
yield self._drop_gridfs(db)
gfs = GridFS(db) # Default collection
_ = yield gfs.put(b"0xDEADBEEF", filename="test_2", contentType="text/plain",
chunk_size=65536)
# disconnect
yield conn.disconnect()
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
gfs = GridFS(db) # Default collection
# Missing file raises error
yield self.assertFailure(gfs.get("test_3"), NoFile)
# disconnect
yield conn.disconnect()
def test_GridFsOperations(self):
""" Tests gridfs operations """
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
# Drop files first TODO: iterate through files and delete them
yield self._drop_gridfs(db)
# Don't forget to disconnect
self.addCleanup(self._disconnect, conn)
try:
in_file = StringIO(b"Test input string")
out_file = StringIO()
except Exception as e:
self.fail("Failed to create memory files for testing: %s" % e)
g_out = None
try:
# Tests writing to a new gridfs file
gfs = GridFS(db) # Default collection
if twisted_version.major >= 15:
with self.assertRaises(NoFile):
yield gfs.get_last_version("optest")
g_in = gfs.new_file(
filename="optest", contentType="text/plain",
chunk_size=65536) # non-default chunk size used
# yielding to ensure writes complete before we close and close before we try to read
yield g_in.write(in_file.read())
yield g_in.close()
# Tests reading from an existing gridfs file
g_out = yield gfs.get_last_version("optest")
data = yield g_out.read()
out_file.write(data)
_id = g_out._id
except Exception as e:
self.fail("Failed to communicate with the GridFS. " +
"Is MongoDB running? %s" % e)
else:
self.assertEqual(in_file.getvalue(), out_file.getvalue(),
"Could not read the value from writing an input")
finally:
in_file.close()
out_file.close()
if g_out:
g_out.close()
listed_files = yield gfs.list()
self.assertEqual(
["optest"], listed_files,
"`optest` is the only expected file and we received %s" %
listed_files)
yield gfs.delete(_id)
def _image_delete_success(self, value, request, file_ids):
dl = []
for file_id in file_ids:
dl.append(
GridFS(mongo_connection['rotator_database']).delete(file_id))
d = defer.DeferredList(dl)
d.addCallback(self._output_delete_success, request)
d.addErrback(self._image_delete_failure, request)
log_me('_image_delete_success', value, file_ids)
return server.NOT_DONE_YET
def test_GridFsIndexesCreation(self):
""" Tests gridfs indexes creation"""
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
yield self._drop_gridfs(db)
# Create a new GridFS instance should trigger indexes creation
gfs = GridFS(db)
# Multiple calls should return multiple defer not to mix between them
self.assertNotEqual(gfs.indexes_created(), gfs.indexes_created())
yield gfs.indexes_created()
indexes = yield db.fs.files.index_information()
self.assertTrue(any(key["key"] == SON([("filename", 1), ("uploadDate", 1)])
for key in indexes.values()))
indexes = yield db.fs.chunks.index_information()
self.assertTrue(any(key["key"] == SON([("files_id", 1), ("n", 1)])
for key in indexes.values()))
yield conn.disconnect()
def test_GridFsOperations(self):
""" Tests gridfs operations """
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
# Drop files first TODO: iterate through files and delete them
yield self._drop_gridfs(db)
# Don't forget to disconnect
self.addCleanup(self._disconnect, conn)
try:
in_file = StringIO(b"Test input string")
out_file = StringIO()
except Exception as e:
self.fail("Failed to create memory files for testing: %s" % e)
g_out = None
try:
# Tests writing to a new gridfs file
gfs = GridFS(db) # Default collection
if twisted_version.major >= 15:
with self.assertRaises(NoFile):
yield gfs.get_last_version("optest")
g_in = gfs.new_file(filename="optest", contentType="text/plain",
chunk_size=65536) # non-default chunk size used
# yielding to ensure writes complete before we close and close before we try to read
yield g_in.write(in_file.read())
yield g_in.close()
# Tests reading from an existing gridfs file
g_out = yield gfs.get_last_version("optest")
data = yield g_out.read()
out_file.write(data)
_id = g_out._id
except Exception as e:
self.fail("Failed to communicate with the GridFS. " +
"Is MongoDB running? %s" % e)
else:
self.assertEqual(in_file.getvalue(), out_file.getvalue(),
"Could not read the value from writing an input")
finally:
in_file.close()
out_file.close()
if g_out:
g_out.close()
listed_files = yield gfs.list()
self.assertEqual(["optest"], listed_files,
"`optest` is the only expected file and we received %s" % listed_files)
yield gfs.delete(_id)
def _get_image_content_success(self, value, request):
log_me('_get_image_content_success', value)
if value and len(value):
gridfs_id = value[0].get('gridfs_id')
log_me('gridfs_id', gridfs_id)
d = GridFS(mongo_connection['rotator_database']).get(gridfs_id)
d.addCallback(self._get_gridfs_info, request)
d.addErrback(self._get_image_content_failure, request)
return server.NOT_DONE_YET
else:
return self._get_image_content_failure(
ERROR_IMAGE_NOT_FOUND_IN_GRIDFS, request)
def test_GridFileObjects(self):
""" Tests gridfs objects """
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
yield self._drop_gridfs(db)
self.assertRaises(TypeError, GridFS, None)
_ = GridFS(db) # Default collection
self.assertRaises(TypeError, GridIn, None)
with GridIn(db.fs,
filename="test_with",
contentType="text/plain",
chunk_size=1024):
pass
grid_in_file = GridIn(db.fs,
filename="test_1",
contentType="text/plain",
content_type="text/plain",
chunk_size=65536,
length=1048576,
upload_date="20150101")
self.assertFalse(grid_in_file.closed)
if twisted_version.major >= 15:
with self.assertRaises(TypeError):
yield grid_in_file.write(1)
with self.assertRaises(TypeError):
yield grid_in_file.write(u"0xDEADBEEF")
with self.assertRaises(AttributeError):
_ = grid_in_file.test
grid_in_file.test = 1
yield grid_in_file.write(b"0xDEADBEEF")
yield grid_in_file.write(b"0xDEADBEEF" * 1048576)
self.assertTrue("20150101", grid_in_file.upload_date)
yield grid_in_file.writelines([b"0xDEADBEEF", b"0xDEADBEEF"])
yield grid_in_file.close()
if twisted_version.major >= 15:
with self.assertRaises(AttributeError):
grid_in_file.length = 1
self.assertEqual(1, grid_in_file.test)
if twisted_version.major >= 15:
with self.assertRaises(AttributeError):
_ = grid_in_file.test_none
self.assertTrue(grid_in_file.closed)
if twisted_version.major >= 15:
with self.assertRaises(ValueError):
yield grid_in_file.write(b"0xDEADBEEF")
doc = yield db.fs.files.find_one({"_id": grid_in_file._id})
grid_out_file = GridOut(db.fs, doc)
grid_out_file.seek(0, os.SEEK_END)
self.assertEqual(10 * (1 + 1048576 + 2), grid_out_file.tell())
yield grid_out_file.close()
data = b''
for block_dfr in GridOutIterator(grid_out_file, db.fs.chunks):
block = yield block_dfr
if block:
data += block
else:
break
self.assertEqual(data, b"0xDEADBEEF" * (1 + 1048576 + 2))
fake_doc = {
"_id": "test_id",
"length": 1048576,
"filename": "test",
"upload_date": "20150101"
}
self.assertRaises(TypeError, GridOut, None, None)
grid_out_file = GridOut(db.fs, fake_doc)
if twisted_version.major >= 15:
with self.assertRaises(AttributeError):
_ = grid_out_file.testing
self.assertEqual("test", grid_out_file.filename)
self.assertEqual(0, grid_out_file.tell())
grid_out_file.seek(1024)
self.assertEqual(1024, grid_out_file.tell())
grid_out_file.seek(1024, os.SEEK_CUR)
self.assertEqual(2048, grid_out_file.tell())
grid_out_file.seek(0, os.SEEK_END)
self.assertEqual(1048576, grid_out_file.tell())
self.assertRaises(IOError, grid_out_file.seek, 0, 4)
self.assertRaises(IOError, grid_out_file.seek, -1)
self.assertTrue("'_id': 'test_id'" in repr(grid_out_file))
yield conn.disconnect()
