def test_GridFsIteration(self): """ Tests gridfs iterator """ conn = yield txmongo.MongoConnection(mongo_host, mongo_port) db = conn.test yield self._drop_gridfs(db) gfs = GridFS(db) # Default collection new_file = gfs.new_file(filename="testName", contentType="text/plain", length=1048576, chunk_size=4096) yield new_file.write(b"0xDEADBEEF"*4096*2) yield new_file.close() fake_doc = {"_id": new_file._id, "name": "testName", "length": 4096*2, "chunkSize": 4096, "contentType": "text/plain"} grid_out_file = GridOut(db.fs, fake_doc) iterator = GridOutIterator(grid_out_file, db.fs.chunks) next_it = yield next(iterator) self.assertEqual(len(next_it), 4096) _ = yield next(iterator) next_it = yield next(iterator) self.assertEqual(next_it, None) fake_bad_doc = {"_id": "bad_id", "name": "testName", "length": 4096*2, "chunkSize": 4096, "contentType": "text/plain"} grid_bad_out_file = GridOut(db.fs, fake_bad_doc) bad_iterator = GridOutIterator(grid_bad_out_file, db.fs.chunks) if twisted_version.major >= 15: with self.assertRaises(errors.CorruptGridFile): next_it = yield bad_iterator.next() # disconnect yield conn.disconnect()
def test_GridFsIteration(self): """ Tests gridfs iterator """ conn = yield txmongo.MongoConnection(mongo_host, mongo_port) db = conn.test db.fs.files.remove({}) # drop all objects there first db.fs.chunks.remove({}) gfs = GridFS(db) # Default collection new_file = gfs.new_file(filename="testName", contentType="text/plain", length=1048576, chunk_size=4096) yield new_file.write(b"0xDEADBEEF"*4096*2) yield new_file.close() fake_doc = {"_id": new_file._id, "name": "testName", "length": 4096*2, "chunkSize": 4096, "contentType": "text/plain"} grid_out_file = GridOut(db.fs, fake_doc) iterator = GridOutIterator(grid_out_file, db.fs.chunks) next_it = yield next(iterator) self.assertEqual(len(next_it), 4096) _ = yield next(iterator) next_it = yield next(iterator) self.assertEqual(next_it, None) fake_bad_doc = {"_id": "bad_id", "name": "testName", "length": 4096*2, "chunkSize": 4096, "contentType": "text/plain"} grid_bad_out_file = GridOut(db.fs, fake_bad_doc) bad_iterator = GridOutIterator(grid_bad_out_file, db.fs.chunks) if _version.version.major >= 15: with self.assertRaises(errors.CorruptGridFile): next_it = yield bad_iterator.next() # disconnect yield conn.disconnect()
def test_GridFsIndexesCreation(self): """ Tests gridfs indexes creation""" conn = yield txmongo.MongoConnection(mongo_host, mongo_port) db = conn.test yield self._drop_gridfs(db) # Create a new GridFS instance should trigger indexes creation gfs = GridFS(db) # Multiple calls should return multiple defer not to mix between them self.assertNotEqual(gfs.indexes_created(), gfs.indexes_created()) yield gfs.indexes_created() indexes = yield db.fs.files.index_information() self.assertTrue( any(key["key"] == SON([("filename", 1), ("uploadDate", 1)]) for key in indexes.values())) indexes = yield db.fs.chunks.index_information() self.assertTrue( any(key["key"] == SON([("files_id", 1), ("n", 1)]) for key in indexes.values())) yield conn.disconnect()
def test_GridFsObjects(self): """ Tests gridfs objects """ conn = yield txmongo.MongoConnection(mongo_host, mongo_port) db = conn.test db.fs.files.remove({}) # drop all objects there first db.fs.chunks.remove({}) gfs = GridFS(db) # Default collection yield gfs.delete(u"test") _ = gfs.new_file(filename="test_1", contentType="text/plain", chunk_size=65536) yield conn.disconnect() conn = yield txmongo.MongoConnection(mongo_host, mongo_port) db = conn.test db.fs.files.remove({}) # drop all objects there first gfs = GridFS(db) # Default collection _ = yield gfs.put(b"0xDEADBEEF", filename="test_2", contentType="text/plain", chunk_size=65536) # disconnect yield conn.disconnect() conn = yield txmongo.MongoConnection(mongo_host, mongo_port) db = conn.test gfs = GridFS(db) # Default collection _ = yield gfs.get("test_3") # disconnect yield conn.disconnect()
def test_GridFsObjects(self): """ Tests gridfs objects """ conn = yield txmongo.MongoConnection(mongo_host, mongo_port) db = conn.test yield self._drop_gridfs(db) gfs = GridFS(db) # Default collection yield gfs.delete(u"test") _ = gfs.new_file(filename="test_1", contentType="text/plain", chunk_size=65536) yield conn.disconnect() conn = yield txmongo.MongoConnection(mongo_host, mongo_port) db = conn.test yield self._drop_gridfs(db) gfs = GridFS(db) # Default collection _ = yield gfs.put(b"0xDEADBEEF", filename="test_2", contentType="text/plain", chunk_size=65536) # disconnect yield conn.disconnect() conn = yield txmongo.MongoConnection(mongo_host, mongo_port) db = conn.test gfs = GridFS(db) # Default collection # Missing file raises error yield self.assertFailure(gfs.get("test_3"), NoFile) # disconnect yield conn.disconnect()
def _add_image_to_gridfs(self, image_id, image_content, image_content_type): d = GridFS(mongo_connection['rotator_database']).put( image_content, content_type=image_content_type, md5hash=image_id) d.addCallback(self._image_to_gridfs_success, image_id) log_me('_add_image_to_gridfs', image_id, image_content_type)
def test_GridFsIndexesCreation(self): """ Tests gridfs indexes creation""" conn = yield txmongo.MongoConnection(mongo_host, mongo_port) db = conn.test yield self._drop_gridfs(db) # Create a new GridFS instance should trigger indexes creation gfs = GridFS(db) # Multiple calls should return multiple defer not to mix between them self.assertNotEqual(gfs.indexes_created(), gfs.indexes_created()) yield gfs.indexes_created() yield conn.disconnect()
def test_GridFileObjects(self): """ Tests gridfs objects """ conn = yield txmongo.MongoConnection(mongo_host, mongo_port) db = conn.test db.fs.files.remove({}) # drop all objects there first db.fs.chunks.remove({}) self.assertRaises(TypeError, GridFS, None) _ = GridFS(db) # Default collection self.assertRaises(TypeError, GridIn, None) with GridIn(db.fs, filename="test_with", contentType="text/plain", chunk_size=1024): pass grid_in_file = GridIn(db.fs, filename="test_1", contentType="text/plain", content_type="text/plain", chunk_size=65536, length=1048576, upload_date="20150101") self.assertFalse(grid_in_file.closed) if _version.version.major >= 15: with self.assertRaises(TypeError): yield grid_in_file.write(1) with self.assertRaises(TypeError): yield grid_in_file.write(u"0xDEADBEEF") with self.assertRaises(AttributeError): _ = grid_in_file.test grid_in_file.test = 1 yield grid_in_file.write(b"0xDEADBEEF") yield grid_in_file.write(b"0xDEADBEEF"*1048576) fake_doc = {"_id": "test_id", "length": 1048576, "filename": "test", "upload_date": "20150101"} self.assertRaises(TypeError, GridOut, None, None) grid_out_file = GridOut(db.fs, fake_doc) if _version.version.major >= 15: with self.assertRaises(AttributeError): _ = grid_out_file.testing self.assertEqual("test", grid_out_file.filename) self.assertEqual(0, grid_out_file.tell()) grid_out_file.seek(1024) self.assertEqual(1024, grid_out_file.tell()) grid_out_file.seek(1024, os.SEEK_CUR) self.assertEqual(2048, grid_out_file.tell()) grid_out_file.seek(0, os.SEEK_END) self.assertEqual(1048576, grid_out_file.tell()) self.assertRaises(IOError, grid_out_file.seek, 0, 4) self.assertRaises(IOError, grid_out_file.seek, -1) self.assertTrue("'_id': 'test_id'" in repr(grid_out_file)) self.assertTrue("20150101", grid_in_file.upload_date) yield grid_in_file.writelines([b"0xDEADBEEF", b"0xDEADBEAF"]) yield grid_in_file.close() if _version.version.major >= 15: with self.assertRaises(AttributeError): grid_in_file.length = 1 self.assertEqual(1, grid_in_file.test) if _version.version.major >= 15: with self.assertRaises(AttributeError): _ = grid_in_file.test_none self.assertTrue(grid_in_file.closed) if _version.version.major >= 15: with self.assertRaises(ValueError): yield grid_in_file.write(b"0xDEADBEEF") yield conn.disconnect()
def test_GridFsOperations(self): """ Tests gridfs operations """ conn = yield txmongo.MongoConnection(mongo_host, mongo_port) db = conn.test # Drop files first TODO: iterate through files and delete them yield self._drop_gridfs(db) # Don't forget to disconnect self.addCleanup(self._disconnect, conn) try: in_file = StringIO(b"Test input string") out_file = StringIO() except Exception as e: self.fail("Failed to create memory files for testing: %s" % e) g_out = None try: # Tests writing to a new gridfs file gfs = GridFS(db) # Default collection if twisted_version.major >= 15: with self.assertRaises(NoFile): yield gfs.get_last_version("optest") g_in = gfs.new_file( filename="optest", contentType="text/plain", chunk_size=65536) # non-default chunk size used # yielding to ensure writes complete before we close and close before we try to read yield g_in.write(in_file.read()) yield g_in.close() # Tests reading from an existing gridfs file g_out = yield gfs.get_last_version("optest") data = yield g_out.read() out_file.write(data) _id = g_out._id except Exception as e: self.fail("Failed to communicate with the GridFS. " + "Is MongoDB running? %s" % e) else: self.assertEqual(in_file.getvalue(), out_file.getvalue(), "Could not read the value from writing an input") finally: in_file.close() out_file.close() if g_out: g_out.close() listed_files = yield gfs.list() self.assertEqual( ["optest"], listed_files, "`optest` is the only expected file and we received %s" % listed_files) yield gfs.delete(_id)
def _image_delete_success(self, value, request, file_ids): dl = [] for file_id in file_ids: dl.append( GridFS(mongo_connection['rotator_database']).delete(file_id)) d = defer.DeferredList(dl) d.addCallback(self._output_delete_success, request) d.addErrback(self._image_delete_failure, request) log_me('_image_delete_success', value, file_ids) return server.NOT_DONE_YET
def test_GridFsIndexesCreation(self): """ Tests gridfs indexes creation""" conn = yield txmongo.MongoConnection(mongo_host, mongo_port) db = conn.test yield self._drop_gridfs(db) # Create a new GridFS instance should trigger indexes creation gfs = GridFS(db) # Multiple calls should return multiple defer not to mix between them self.assertNotEqual(gfs.indexes_created(), gfs.indexes_created()) yield gfs.indexes_created() indexes = yield db.fs.files.index_information() self.assertTrue(any(key["key"] == SON([("filename", 1), ("uploadDate", 1)]) for key in indexes.values())) indexes = yield db.fs.chunks.index_information() self.assertTrue(any(key["key"] == SON([("files_id", 1), ("n", 1)]) for key in indexes.values())) yield conn.disconnect()
def test_GridFsOperations(self): """ Tests gridfs operations """ conn = yield txmongo.MongoConnection(mongo_host, mongo_port) db = conn.test # Drop files first TODO: iterate through files and delete them yield self._drop_gridfs(db) # Don't forget to disconnect self.addCleanup(self._disconnect, conn) try: in_file = StringIO(b"Test input string") out_file = StringIO() except Exception as e: self.fail("Failed to create memory files for testing: %s" % e) g_out = None try: # Tests writing to a new gridfs file gfs = GridFS(db) # Default collection if twisted_version.major >= 15: with self.assertRaises(NoFile): yield gfs.get_last_version("optest") g_in = gfs.new_file(filename="optest", contentType="text/plain", chunk_size=65536) # non-default chunk size used # yielding to ensure writes complete before we close and close before we try to read yield g_in.write(in_file.read()) yield g_in.close() # Tests reading from an existing gridfs file g_out = yield gfs.get_last_version("optest") data = yield g_out.read() out_file.write(data) _id = g_out._id except Exception as e: self.fail("Failed to communicate with the GridFS. " + "Is MongoDB running? %s" % e) else: self.assertEqual(in_file.getvalue(), out_file.getvalue(), "Could not read the value from writing an input") finally: in_file.close() out_file.close() if g_out: g_out.close() listed_files = yield gfs.list() self.assertEqual(["optest"], listed_files, "`optest` is the only expected file and we received %s" % listed_files) yield gfs.delete(_id)
def _get_image_content_success(self, value, request): log_me('_get_image_content_success', value) if value and len(value): gridfs_id = value[0].get('gridfs_id') log_me('gridfs_id', gridfs_id) d = GridFS(mongo_connection['rotator_database']).get(gridfs_id) d.addCallback(self._get_gridfs_info, request) d.addErrback(self._get_image_content_failure, request) return server.NOT_DONE_YET else: return self._get_image_content_failure( ERROR_IMAGE_NOT_FOUND_IN_GRIDFS, request)
def test_GridFileObjects(self): """ Tests gridfs objects """ conn = yield txmongo.MongoConnection(mongo_host, mongo_port) db = conn.test yield self._drop_gridfs(db) self.assertRaises(TypeError, GridFS, None) _ = GridFS(db) # Default collection self.assertRaises(TypeError, GridIn, None) with GridIn(db.fs, filename="test_with", contentType="text/plain", chunk_size=1024): pass grid_in_file = GridIn(db.fs, filename="test_1", contentType="text/plain", content_type="text/plain", chunk_size=65536, length=1048576, upload_date="20150101") self.assertFalse(grid_in_file.closed) if twisted_version.major >= 15: with self.assertRaises(TypeError): yield grid_in_file.write(1) with self.assertRaises(TypeError): yield grid_in_file.write(u"0xDEADBEEF") with self.assertRaises(AttributeError): _ = grid_in_file.test grid_in_file.test = 1 yield grid_in_file.write(b"0xDEADBEEF") yield grid_in_file.write(b"0xDEADBEEF" * 1048576) self.assertTrue("20150101", grid_in_file.upload_date) yield grid_in_file.writelines([b"0xDEADBEEF", b"0xDEADBEEF"]) yield grid_in_file.close() if twisted_version.major >= 15: with self.assertRaises(AttributeError): grid_in_file.length = 1 self.assertEqual(1, grid_in_file.test) if twisted_version.major >= 15: with self.assertRaises(AttributeError): _ = grid_in_file.test_none self.assertTrue(grid_in_file.closed) if twisted_version.major >= 15: with self.assertRaises(ValueError): yield grid_in_file.write(b"0xDEADBEEF") doc = yield db.fs.files.find_one({"_id": grid_in_file._id}) grid_out_file = GridOut(db.fs, doc) grid_out_file.seek(0, os.SEEK_END) self.assertEqual(10 * (1 + 1048576 + 2), grid_out_file.tell()) yield grid_out_file.close() data = b'' for block_dfr in GridOutIterator(grid_out_file, db.fs.chunks): block = yield block_dfr if block: data += block else: break self.assertEqual(data, b"0xDEADBEEF" * (1 + 1048576 + 2)) fake_doc = { "_id": "test_id", "length": 1048576, "filename": "test", "upload_date": "20150101" } self.assertRaises(TypeError, GridOut, None, None) grid_out_file = GridOut(db.fs, fake_doc) if twisted_version.major >= 15: with self.assertRaises(AttributeError): _ = grid_out_file.testing self.assertEqual("test", grid_out_file.filename) self.assertEqual(0, grid_out_file.tell()) grid_out_file.seek(1024) self.assertEqual(1024, grid_out_file.tell()) grid_out_file.seek(1024, os.SEEK_CUR) self.assertEqual(2048, grid_out_file.tell()) grid_out_file.seek(0, os.SEEK_END) self.assertEqual(1048576, grid_out_file.tell()) self.assertRaises(IOError, grid_out_file.seek, 0, 4) self.assertRaises(IOError, grid_out_file.seek, -1) self.assertTrue("'_id': 'test_id'" in repr(grid_out_file)) yield conn.disconnect()