class SqlaFilesTests(unittest.TestCase): Model = ObjWithFile CustomModel = CustomObjWithFile def setUp(self): self.transient_root = tempfile.mkdtemp() self.persistent_root = tempfile.mkdtemp() self.transient_url = '/transient/' self.persistent_url = '/media/' self.file_manager = FileManager(self.transient_root, self.persistent_root, self.transient_url, self.persistent_url) self.metadata_transient_root = tempfile.mkdtemp() self.metadata_persistent_root = tempfile.mkdtemp() self.metadata_transient_url = '/metadata/transient/' self.metadata_persistent_url = '/metadata/media/' self.metadata_file_manager = FileManager(self.metadata_transient_root, self.metadata_persistent_root, self.metadata_transient_url, self.metadata_persistent_url) self.model_transient_root = tempfile.mkdtemp() self.model_persistent_root = tempfile.mkdtemp() self.model_transient_url = '/model/transient/' self.model_persistent_url = '/model/media/' self.model_file_manager = FileManager(self.model_transient_root, self.model_persistent_root, self.model_transient_url, self.model_persistent_url) self.field_transient_root = tempfile.mkdtemp() self.field_persistent_root = tempfile.mkdtemp() self.field_transient_url = '/field/transient/' self.field_persistent_url = '/field/media/' self.field_file_manager = FileManager(self.field_transient_root, self.field_persistent_root, self.field_transient_url, self.field_persistent_url) Session = filesessionmaker(orm.sessionmaker(), self.file_manager, file_managers={ CustomObjWithFile.metadata: self.metadata_file_manager, CustomObjWithFile: self.model_file_manager, CustomObjWithFile.field_file: self.field_file_manager }) engine = create_engine('sqlite://') Base.metadata.create_all(engine) CustomBase.metadata.create_all(engine) self.db = Session(bind=engine) def tearDown(self): shutil.rmtree(self.transient_root) shutil.rmtree(self.persistent_root) shutil.rmtree(self.metadata_transient_root) shutil.rmtree(self.metadata_persistent_root) shutil.rmtree(self.model_transient_root) shutil.rmtree(self.model_persistent_root) shutil.rmtree(self.field_transient_root) shutil.rmtree(self.field_persistent_root) def test_session(self): self.assertTrue(hasattr(self.db, 'file_manager')) self.assertIsInstance(self.db.file_manager, FileManager) def test_find_file_manager(self): obj = self.Model() custom_obj = self.CustomModel() self.assertTrue(hasattr(self.db, 'find_file_manager')) self.assertEqual(self.db.find_file_manager(self.Model.file), self.file_manager) self.assertEqual(self.db.find_file_manager(self.CustomModel.metadata_file), self.model_file_manager) self.assertEqual(self.db.find_file_manager(self.CustomModel.model_file), self.model_file_manager) self.assertEqual(self.db.find_file_manager(self.CustomModel.field_file), self.field_file_manager) def test_create(self): obj = self.Model() obj.file = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test') self.assertIsInstance(obj.file, TransientFile) self.assertIsNotNone(obj.file_name) self.db.add(obj) self.db.commit() self.assertIsInstance(obj.file, PersistentFile) self.assertFalse(os.path.exists(f.path)) self.assertTrue(os.path.isfile(obj.file.path)) self.assertEqual(open(obj.file.path).read(), 'test') def test_create_custom(self): obj = self.CustomModel() def populate(field_name): attr = getattr(obj.__class__, field_name) f = self.db.find_file_manager(attr).new_transient() with open(f.path, 'wb') as fp: fp.write('test') setattr(obj, field_name, f) populate('metadata_file') populate('model_file') populate('field_file') self.assertTrue(obj.field_file.path.startswith( self.field_transient_root)) self.assertTrue(obj.model_file.path.startswith( self.model_transient_root)) self.db.add(obj) self.db.commit() self.assertTrue(obj.field_file.path.startswith( self.field_persistent_root)) self.assertTrue(obj.model_file.path.startswith( self.model_persistent_root)) def test_update_none2file(self): obj = self.Model() self.db.add(obj) self.db.commit() obj.file = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test') self.assertIsInstance(obj.file, TransientFile) self.assertIsNotNone(obj.file_name) self.db.commit() self.assertIsInstance(obj.file, PersistentFile) self.assertFalse(os.path.exists(f.path)) self.assertTrue(os.path.isfile(obj.file.path)) self.assertEqual(open(obj.file.path).read(), 'test') def test_update_file2none(self): obj = self.Model() obj.file = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test') self.db.add(obj) self.db.commit() pf = obj.file obj.file = None self.assertIsNone(obj.file_name) self.assertTrue(os.path.exists(pf.path)) self.db.commit() self.assertFalse(os.path.exists(pf.path)) def test_update_file2file(self): obj = self.Model() obj.file = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test1') self.db.add(obj) self.db.commit() pf1 = obj.file obj.file = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test2') self.assertIsInstance(obj.file, TransientFile) self.assertIsNotNone(obj.file_name) self.db.commit() self.assertIsInstance(obj.file, PersistentFile) self.assertFalse(os.path.exists(f.path)) self.assertFalse(os.path.exists(pf1.path)) self.assertTrue(os.path.isfile(obj.file.path)) self.assertEqual(open(obj.file.path).read(), 'test2') def test_update_file2self(self): obj = self.Model() obj.file = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test1') self.db.add(obj) self.db.commit() pf1 = obj.file obj.file = self.file_manager.get_persistent(obj.file.name) self.db.commit() self.assertIsInstance(obj.file, PersistentFile) self.assertTrue(os.path.exists(obj.file.path)) self.assertEqual(pf1.path, obj.file.path) @unittest.skip('Not implemented') def test_update_file2file_not_random(self): obj = self.Model() obj.file_by_id = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test1') self.db.add(obj) self.db.commit() self.assertEqual(obj.file_by_id_name, self.Model.file_by_id.name_template.format(item=obj)) pf1 = obj.file_by_id obj.file_by_id = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test2') self.assertIsInstance(obj.file_by_id, TransientFile) self.assertIsNotNone(obj.file_by_id_name) self.db.commit() self.assertIsInstance(obj.file_by_id, PersistentFile) self.assertFalse(os.path.exists(f.path)) self.assertFalse(os.path.exists(pf1.path)) self.assertTrue(os.path.isfile(obj.file_by_id.path)) self.assertEqual(open(obj.file_by_id.path).read(), 'test2') @unittest.skip('Not implemented') def test_update_random_collision(self): obj = self.Model() self.db.add(obj) self.db.commit() obj.file = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test') persistent = self.file_manager.get_persistent(obj.file_name) dirname = os.path.dirname(persistent.path) if not os.path.isdir(dirname): os.makedirs(dirname) with open(persistent.path, 'wb') as fp: fp.write('taken') self.assertIsInstance(obj.file, TransientFile) self.assertIsNotNone(obj.file_name) self.db.commit() self.assertIsInstance(obj.file, PersistentFile) self.assertFalse(os.path.exists(f.path)) self.assertTrue(os.path.isfile(obj.file.path)) self.assertEqual(open(obj.file.path).read(), 'test') self.assertNotEqual(persistent.path, obj.file.path) self.assertEqual(open(persistent.path).read(), 'taken') def test_update_none2persistent(self): f = self.file_manager.get_persistent('persistent.txt') with open(f.path, 'wb') as fp: fp.write('test1') obj = self.Model() obj.file = f self.db.add(obj) self.db.commit() self.assertIsInstance(obj.file, PersistentFile) self.assertTrue(os.path.exists(obj.file.path)) self.assertEqual(obj.file.name, 'persistent.txt') def test_delete(self): obj = self.Model() obj.file = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test') self.db.add(obj) self.db.commit() pf = obj.file self.db.delete(obj) self.db.commit() self.assertFalse(os.path.exists(pf.path))
class SqlaFilesTests(unittest.TestCase): Model = ObjWithFile def setUp(self): self.transient_root = tempfile.mkdtemp() self.persistent_root = tempfile.mkdtemp() self.transient_url = '/transient/' self.persistent_url = '/media/' self.file_manager = FileManager(self.transient_root, self.persistent_root, self.transient_url, self.persistent_url) self.metadata_transient_root = tempfile.mkdtemp() self.metadata_persistent_root = tempfile.mkdtemp() self.metadata_transient_url = '/metadata/transient/' self.metadata_persistent_url = '/metadata/media/' self.metadata_file_manager = FileManager(self.metadata_transient_root, self.metadata_persistent_root, self.metadata_transient_url, self.metadata_persistent_url) self.model_transient_root = tempfile.mkdtemp() self.model_persistent_root = tempfile.mkdtemp() self.model_transient_url = '/model/transient/' self.model_persistent_url = '/model/media/' self.model_file_manager = FileManager(self.model_transient_root, self.model_persistent_root, self.model_transient_url, self.model_persistent_url) Session = filesessionmaker(orm.sessionmaker(), self.file_manager, file_managers={ MetadataLevelObj.metadata: self.metadata_file_manager, ModelLevelObj: self.model_file_manager, }) engine = create_engine('sqlite://') Base.metadata.create_all(engine) CustomBase.metadata.create_all(engine) self.db = Session(bind=engine) def tearDown(self): shutil.rmtree(self.transient_root) shutil.rmtree(self.persistent_root) shutil.rmtree(self.metadata_transient_root) shutil.rmtree(self.metadata_persistent_root) shutil.rmtree(self.model_transient_root) shutil.rmtree(self.model_persistent_root) def test_session(self): self.assertTrue(hasattr(self.db, 'file_manager')) self.assertIsInstance(self.db.file_manager, FileManager) def test_find_file_manager(self): self.assertTrue(hasattr(self.db, 'find_file_manager')) self.assertEqual(self.db.find_file_manager(self.Model.file), self.file_manager) self.assertEqual(self.db.find_file_manager(ModelLevelObj.file), self.model_file_manager) self.assertEqual(self.db.find_file_manager(MetadataLevelObj.file), self.metadata_file_manager) def test_create(self): obj = self.Model() obj.file = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test') self.assertIsInstance(obj.file, TransientFile) self.assertIsNotNone(obj.file_name) self.db.add(obj) self.db.commit() self.assertIsInstance(obj.file, PersistentFile) self.assertFalse(os.path.exists(f.path)) self.assertTrue(os.path.isfile(obj.file.path)) self.assertEqual(open(obj.file.path).read(), 'test') def test_create_metadata_obj(self): metadata_obj = MetadataLevelObj() metadata_filemanager = self.db.find_file_manager(metadata_obj) metadata_file = metadata_filemanager.new_transient() with open(metadata_file.path, 'wb') as fp: fp.write('test') metadata_obj.file = metadata_file self.assertTrue(metadata_obj.file.path.startswith(self.metadata_transient_root)) self.db.add(metadata_obj) self.db.commit() self.assertTrue(metadata_obj.file.path.startswith(self.metadata_persistent_root)) def test_create_model_obj(self): model_obj = ModelLevelObj() model_filemanager = self.db.find_file_manager(model_obj) model_file = model_filemanager.new_transient() with open(model_file.path, 'wb') as fp: fp.write('test') model_obj.file = model_file self.assertTrue(model_obj.file.path.startswith(self.model_transient_root)) self.db.add(model_obj) self.db.commit() self.assertTrue(model_obj.file.path.startswith(self.model_persistent_root)) def test_update_none2file(self): obj = self.Model() self.db.add(obj) self.db.commit() obj.file = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test') self.assertIsInstance(obj.file, TransientFile) self.assertIsNotNone(obj.file_name) self.db.commit() # cleanup self.Model.file._states.items() to get the result # from scratch, not from cache obj_id = obj.id obj = None gc.collect() obj = self.db.query(self.Model).get(obj_id) self.assertIsInstance(obj.file, PersistentFile) self.assertFalse(os.path.exists(f.path)) self.assertTrue(os.path.isfile(obj.file.path)) self.assertEqual(open(obj.file.path).read(), 'test') self.assertEqual(obj.file.size, 4) self.assertEqual(obj.file_size, 4) def test_update_file2none(self): obj = self.Model() obj.file = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test') self.db.add(obj) self.db.commit() pf = obj.file obj.file = None self.assertIsNone(obj.file_name) self.assertTrue(os.path.exists(pf.path)) self.assertEqual(obj.file_size, 4) self.db.commit() self.assertFalse(os.path.exists(pf.path)) self.assertEqual(obj.file_size, None) def test_update_file2file(self): obj = self.Model() obj.file = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test1') self.db.add(obj) self.db.commit() pf1 = obj.file self.assertEqual(obj.file_size, 5) obj.file = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test22') self.assertIsInstance(obj.file, TransientFile) self.assertIsNotNone(obj.file_name) self.db.commit() self.assertIsInstance(obj.file, PersistentFile) self.assertFalse(os.path.exists(f.path)) self.assertFalse(os.path.exists(pf1.path)) self.assertTrue(os.path.isfile(obj.file.path)) self.assertEqual(open(obj.file.path).read(), 'test22') self.assertEqual(obj.file_size, 6) def test_update_file2self(self): obj = self.Model() obj.file = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test1') self.db.add(obj) self.db.commit() pf1 = obj.file obj.file = self.file_manager.get_persistent(obj.file.name) self.db.commit() self.assertIsInstance(obj.file, PersistentFile) self.assertTrue(os.path.exists(obj.file.path)) self.assertEqual(pf1.path, obj.file.path) # XXX for test coverage # have no idea what extra check can be performed obj.file = obj.file self.assertTrue(os.path.exists(obj.file.path)) self.assertEqual(pf1.path, obj.file.path) @unittest.expectedFailure def test_update_file2file_not_random(self): obj = self.Model() obj.file_by_id = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test1') self.db.add(obj) self.db.commit() self.assertEqual(obj.file_by_id_name, self.Model.file_by_id.name_template.format(item=obj)) pf1 = obj.file_by_id obj.file_by_id = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test2') self.assertIsInstance(obj.file_by_id, TransientFile) self.assertIsNotNone(obj.file_by_id_name) self.db.commit() self.assertIsInstance(obj.file_by_id, PersistentFile) self.assertFalse(os.path.exists(f.path)) self.assertFalse(os.path.exists(pf1.path)) self.assertTrue(os.path.isfile(obj.file_by_id.path)) self.assertEqual(open(obj.file_by_id.path).read(), 'test2') @unittest.expectedFailure def test_update_random_collision(self): obj = self.Model() self.db.add(obj) self.db.commit() obj.file = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test') persistent = self.file_manager.get_persistent(obj.file_name) dirname = os.path.dirname(persistent.path) if not os.path.isdir(dirname): os.makedirs(dirname) with open(persistent.path, 'wb') as fp: fp.write('taken') self.assertIsInstance(obj.file, TransientFile) self.assertIsNotNone(obj.file_name) self.db.commit() self.assertIsInstance(obj.file, PersistentFile) self.assertFalse(os.path.exists(f.path)) self.assertTrue(os.path.isfile(obj.file.path)) self.assertEqual(open(obj.file.path).read(), 'test') self.assertNotEqual(persistent.path, obj.file.path) self.assertEqual(open(persistent.path).read(), 'taken') def test_update_none2persistent(self): f = self.file_manager.get_persistent('persistent.txt') with open(f.path, 'wb') as fp: fp.write('test1') obj = self.Model() obj.file = f self.db.add(obj) self.db.commit() self.assertIsInstance(obj.file, PersistentFile) self.assertTrue(os.path.exists(obj.file.path)) self.assertEqual(obj.file.name, 'persistent.txt') def test_delete(self): obj = self.Model() obj.file = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test') self.db.add(obj) self.db.commit() pf = obj.file self.db.delete(obj) self.db.commit() self.assertFalse(os.path.exists(pf.path)) def test_set_invalid(self): obj = self.Model() self.assertRaises(ValueError, lambda: setattr(obj, 'file', 'test')) def test_cached_lost(self): obj = self.Model() self.db.add(obj) self.db.commit() obj.file = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test') self.db.commit() os.unlink(obj.file.path) self.assertEqual(obj.file.size, 4) self.assertEqual(obj.file_size, 4) del obj.file.size self.assertEqual(obj.file.size, None) def test_file2none_lost(self): obj = self.Model() obj.file = f = self.file_manager.new_transient() with open(f.path, 'wb') as fp: fp.write('test') self.db.add(obj) self.db.commit() os.unlink(obj.file.path) obj.file = None self.db.commit() self.assertEqual(obj.file_size, None) def test_file_manager_for_field(self): def make(): file_manager = FileManager(self.transient_root, self.persistent_root, self.transient_url, self.persistent_url) filesessionmaker(orm.sessionmaker(), self.file_manager, file_managers={ ObjWithFile.file: file_manager, }) self.assertRaises(NotImplementedError, make)
class FormFilesTests(unittest.TestCase): def setUp(self): self.transient_root = tempfile.mkdtemp() self.persistent_root = tempfile.mkdtemp() self.transient_url = '/transient/' self.persistent_url = '/media/' self.file_manager = FileManager(self.transient_root, self.persistent_root, self.transient_url, self.persistent_url) self.env = VersionedStorage(file_manager=self.file_manager) def tearDown(self): shutil.rmtree(self.transient_root) shutil.rmtree(self.persistent_root) def _create_persistent(self): f = self.file_manager.get_persistent('test.txt') with open(f.path, 'wb') as fp: fp.write('test') return f def _create_transient(self, content, original_name='test.txt'): f = self.file_manager.new_transient(ext=os.path.splitext(original_name)[1]) with open(f.path, 'wb') as fp: fp.write(content) return f def _create_fs(self, mimetype, content, filename='uploaded.txt'): fs = cgi.FieldStorage() fs.file = fs.make_file() fs.type = mimetype fs.file.write(content) fs.file.seek(0) fs.filename = filename return fs def test_none2empty(self): form = FormWithFile(self.env) form.accept(MultiDict({'file.file': None, 'file.original_name': '', 'file.transient_name': '', 'file.mode': 'empty',})) data = form.python_data self.assertEqual(data['file'], None) def test_transient2empty(self): transient = self._create_transient('transient1') form = FormWithFile(self.env) form.accept(MultiDict({'file.file': None, 'file.original_name': 'test.txt', 'file.transient_name': transient.name, 'file.mode': 'transient',})) data = form.python_data self.assertIsInstance(data['file'], TransientFile) self.assertEqual(data['file'].name, transient.name) def test_transient_lost(self): form = FormWithFile(self.env) form.accept(MultiDict({'file.file': None, 'file.original_name': 'test.txt', 'file.transient_name': 'lost.txt', 'file.mode': 'transient',})) self.assertIn('file', form.errors) def test_transient_invalid(self): illegal = ['more/../../../lost.txt', '../pass', '/etc/pass', '..', '~', ] for filename in illegal: form = FormWithFile(self.env) form.accept(MultiDict({'file.file': None, 'file.original_name': 'test.txt', 'file.transient_name': filename, 'file.mode': 'transient',})) self.assertIn('file.transient_name', form.errors, 'file name is not filtered: %s' % filename) def test_file2empty(self): form = FormWithFile(self.env) fs = self._create_fs('text', 'file-content', 'filename.ttt') form.accept(MultiDict({'file.file': fs, 'file.mode': 'existing',})) data = form.python_data self.assertIsInstance(data['file'], TransientFile) self.assertEqual(os.path.splitext(data['file'].name)[1], '.ttt') with open(data['file'].path) as fp: self.assertEqual(fp.read(), 'file-content') def test_none2persistent(self): persistent = self._create_persistent() form = FormWithFile(self.env, initial={'file': persistent}) form.accept(MultiDict({'file.file': None, 'file.original_name': '', 'file.transient_name': '', 'file.mode': 'existing',})) data = form.python_data self.assertEqual(data['file'], persistent) def test_empty2persistent(self): persistent = self._create_persistent() form = FormWithFile(self.env, initial={'file': persistent}) form.accept(MultiDict({'file.file': None, 'file.original_name': '', 'file.transient_name': '', 'file.mode': 'empty',})) data = form.python_data self.assertEqual(data['file'], None) def test_transient2persistent(self): persistent = self._create_persistent() transient = self._create_transient('transient1') form = FormWithFile(self.env, initial={'file': persistent}) form.accept(MultiDict({'file.file': None, 'file.original_name': 'test.txt', 'file.transient_name': transient.name, 'file.mode': 'transient',})) data = form.python_data self.assertIsInstance(data['file'], TransientFile) self.assertEqual(data['file'].name, transient.name) def test_file2persistent(self): persistent = self._create_persistent() form = FormWithFile(self.env, initial={'file': persistent}) fs = self._create_fs('text', 'file-content', 'filename.ttt') form.accept(MultiDict({'file.file': fs, 'file.mode': 'existing',})) data = form.python_data self.assertIsInstance(data['file'], TransientFile) self.assertEqual(os.path.splitext(data['file'].name)[1], '.ttt') with open(data['file'].path) as fp: self.assertEqual(fp.read(), 'file-content')