class SqlaImagesTests(unittest.TestCase):
def setUp(self):
self.transient_root = tempfile.mkdtemp()
self.persistent_root = tempfile.mkdtemp()
self.transient_url = '/transient/'
self.persistent_url = '/media/'
self.file_manager = FileManager(self.transient_root,
self.persistent_root,
self.transient_url,
self.persistent_url)
Session = filesessionmaker(orm.sessionmaker(), self.file_manager)
engine = create_engine('sqlite://')
Base.metadata.create_all(engine)
self.db = Session(bind=engine)
def tearDown(self):
shutil.rmtree(self.transient_root)
shutil.rmtree(self.persistent_root)
def test_create(self):
obj = ObjWithImage()
obj.image = f = self.file_manager.new_transient('.gif')
_create_image(f.path)
def return_image(image, filter):
# asserting that filter was called
self.assertEqual(filter, ImageFilter.BLUR)
return image
with mock.patch('PIL.Image.Image.filter',
side_effect=return_image,
autospec=True):
self.db.add(obj)
self.db.commit()
self.assertIsInstance(obj.image, PersistentFile)
self.assertIsInstance(obj.thumb, PersistentFile)
self.assertIsInstance(obj.thumb_filter, PersistentFile)
self.assertIsInstance(obj.thumb_optimize, PersistentFile)
img = Image.open(obj.image.path)
self.assertEqual(img.size, (200, 200))
self.assertEqual(obj.image.width, img.width)
self.assertEqual(obj.image.height, img.height)
thumb = Image.open(obj.thumb.path)
self.assertEqual(thumb.size, (100, 100))
self.assertEqual(obj.thumb.height, thumb.height)
self.assertEqual(obj.thumb.width, thumb.width)
pixels = thumb.load()
self.assertEqual(pixels[50, 50], (186, 186, 186))
self.assertLessEqual(
os.stat(obj.thumb_optimize.path).st_size,
os.stat(obj.thumb.path).st_size)
def test_no_original_image(self):
obj = ObjWithImage()
obj.image = f = self.file_manager.new_transient()
_create_image(f.path, format='PNG')
warn = []
with mock.patch('os.path.isfile', return_value=False):
with mock.patch(
'logging.Logger.warn',
side_effect=lambda m, *args: warn.append(m % args)):
self.db.add(obj)
self.db.commit()
self.assertEqual(len(warn), 3)
self.assertEqual(warn[0], warn[1])
self.assertEqual(warn[1], warn[2])
self.assertIn("Original file is absent", warn[0])
self.assertIn(obj.image.path, warn[0])
def test_no_ext(self):
# test for extraction image extension from image instead of file path
obj = ObjWithImage()
obj.image = f = self.file_manager.new_transient()
_create_image(f.path, format='PNG')
self.db.add(obj)
self.db.commit()
self.assertIsInstance(obj.image, PersistentFile)
self.assertTrue(obj.image.path.endswith('.png'))
def test_no_size(self):
obj = ObjWithImage()
obj.icon = f = self.file_manager.new_transient('.gif')
_create_image(f.path, 200, 300)
self.db.add(obj)
self.db.commit()
self.assertIsNone(obj.image)
self.assertIsNone(obj.thumb)
self.assertIsInstance(obj.icon, PersistentFile)
img = Image.open(obj.icon.path)
self.assertEqual(img.size, (200, 300))
def test_no_img(self):
obj = ObjWithImage()
self.db.add(obj)
self.db.commit()
self.assertEqual(obj.image, None)
self.assertEqual(obj.thumb, None)
self.assertEqual(obj.icon, None)
def test_update(self):
obj = ObjWithImage()
self.db.add(obj)
self.db.commit()
obj.image = f = self.file_manager.new_transient('.gif')
_create_image(f.path)
self.db.commit()
self.assertIsInstance(obj.image, PersistentFile)
self.assertIsInstance(obj.thumb, PersistentFile)
img = Image.open(obj.image.path)
self.assertEqual(img.size, (200, 200))
thumb = Image.open(obj.thumb.path)
self.assertEqual(thumb.size, (100, 100))
def test_invalid(self):
obj = ObjWithImage()
obj.image = f = self.file_manager.new_transient('.gif')
with open(f.path, 'wb') as fp:
fp.write(b'test')
self.db.add(obj)
with self.assertRaises(IOError):
self.db.commit()
def test_fill_without_size(self):
with self.assertRaises(AssertionError):
class Test(Base):
id = Column(Integer, primary_key=True)
image_name = Column(VARBINARY(250))
image = ImageProperty(image_name,
name_template='image/{random}')
thumb_name = Column(VARBINARY(250))
thumb = ImageProperty(thumb_name,
name_template='thumb/{random}',
fill_from='image')
class SqlaFilesTests(unittest.TestCase):
Model = ObjWithFile
def setUp(self):
self.transient_root = tempfile.mkdtemp()
self.persistent_root = tempfile.mkdtemp()
self.transient_url = '/transient/'
self.persistent_url = '/media/'
self.file_manager = FileManager(self.transient_root,
self.persistent_root,
self.transient_url,
self.persistent_url)
self.metadata_transient_root = tempfile.mkdtemp()
self.metadata_persistent_root = tempfile.mkdtemp()
self.metadata_transient_url = '/metadata/transient/'
self.metadata_persistent_url = '/metadata/media/'
self.metadata_file_manager = FileManager(self.metadata_transient_root,
self.metadata_persistent_root,
self.metadata_transient_url,
self.metadata_persistent_url)
self.model_transient_root = tempfile.mkdtemp()
self.model_persistent_root = tempfile.mkdtemp()
self.model_transient_url = '/model/transient/'
self.model_persistent_url = '/model/media/'
self.model_file_manager = FileManager(self.model_transient_root,
self.model_persistent_root,
self.model_transient_url,
self.model_persistent_url)
Session = filesessionmaker(orm.sessionmaker(),
self.file_manager,
file_managers={
MetadataLevelObj.metadata:
self.metadata_file_manager,
ModelLevelObj: self.model_file_manager,
})
engine = create_engine('sqlite://')
Base.metadata.create_all(engine)
CustomBase.metadata.create_all(engine)
self.db = Session(bind=engine)
def tearDown(self):
shutil.rmtree(self.transient_root)
shutil.rmtree(self.persistent_root)
shutil.rmtree(self.metadata_transient_root)
shutil.rmtree(self.metadata_persistent_root)
shutil.rmtree(self.model_transient_root)
shutil.rmtree(self.model_persistent_root)
def test_session(self):
self.assertTrue(hasattr(self.db, 'file_manager'))
self.assertIsInstance(self.db.file_manager, FileManager)
def test_find_file_manager(self):
self.assertTrue(hasattr(self.db, 'find_file_manager'))
self.assertEqual(self.db.find_file_manager(self.Model.file),
self.file_manager)
self.assertEqual(self.db.find_file_manager(ModelLevelObj.file),
self.model_file_manager)
self.assertEqual(self.db.find_file_manager(MetadataLevelObj.file),
self.metadata_file_manager)
def test_create(self):
obj = self.Model()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test')
self.assertIsInstance(obj.file, TransientFile)
self.assertIsNotNone(obj.file_name)
self.db.add(obj)
self.db.commit()
self.assertIsInstance(obj.file, PersistentFile)
self.assertFalse(os.path.exists(f.path))
self.assertTrue(os.path.isfile(obj.file.path))
self.assertEqual(open(obj.file.path).read(), 'test')
def test_create_metadata_obj(self):
metadata_obj = MetadataLevelObj()
metadata_filemanager = self.db.find_file_manager(metadata_obj)
metadata_file = metadata_filemanager.new_transient()
with open(metadata_file.path, 'wb') as fp:
fp.write(b'test')
metadata_obj.file = metadata_file
self.assertTrue(
metadata_obj.file.path.startswith(self.metadata_transient_root))
self.db.add(metadata_obj)
self.db.commit()
self.assertTrue(
metadata_obj.file.path.startswith(self.metadata_persistent_root))
def test_create_model_obj(self):
model_obj = ModelLevelObj()
model_filemanager = self.db.find_file_manager(model_obj)
model_file = model_filemanager.new_transient()
with open(model_file.path, 'wb') as fp:
fp.write(b'test')
model_obj.file = model_file
self.assertTrue(
model_obj.file.path.startswith(self.model_transient_root))
self.db.add(model_obj)
self.db.commit()
self.assertTrue(
model_obj.file.path.startswith(self.model_persistent_root))
def test_update_none2file(self):
obj = self.Model()
self.db.add(obj)
self.db.commit()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test')
self.assertIsInstance(obj.file, TransientFile)
self.assertIsNotNone(obj.file_name)
self.db.commit()
# cleanup self.Model.file._states.items() to get the result
# from scratch, not from cache
obj_id = obj.id
obj = None
gc.collect()
obj = self.db.query(self.Model).get(obj_id)
self.assertIsInstance(obj.file, PersistentFile)
self.assertFalse(os.path.exists(f.path))
self.assertTrue(os.path.isfile(obj.file.path))
self.assertEqual(open(obj.file.path).read(), 'test')
self.assertEqual(obj.file.size, 4)
self.assertEqual(obj.file_size, 4)
def test_update_file2none(self):
obj = self.Model()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test')
self.db.add(obj)
self.db.commit()
pf = obj.file
obj.file = None
self.assertIsNone(obj.file_name)
self.assertTrue(os.path.exists(pf.path))
self.assertEqual(obj.file_size, 4)
self.db.commit()
self.assertFalse(os.path.exists(pf.path))
self.assertEqual(obj.file_size, None)
def test_update_file2file(self):
obj = self.Model()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test1')
self.db.add(obj)
self.db.commit()
pf1 = obj.file
self.assertEqual(obj.file_size, 5)
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test22')
self.assertIsInstance(obj.file, TransientFile)
self.assertIsNotNone(obj.file_name)
self.db.commit()
self.assertIsInstance(obj.file, PersistentFile)
self.assertFalse(os.path.exists(f.path))
self.assertFalse(os.path.exists(pf1.path))
self.assertTrue(os.path.isfile(obj.file.path))
self.assertEqual(open(obj.file.path).read(), 'test22')
self.assertEqual(obj.file_size, 6)
def test_update_file2self(self):
obj = self.Model()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test1')
self.db.add(obj)
self.db.commit()
pf1 = obj.file
obj.file = self.file_manager.get_persistent(obj.file.name)
self.db.commit()
self.assertIsInstance(obj.file, PersistentFile)
self.assertTrue(os.path.exists(obj.file.path))
self.assertEqual(pf1.path, obj.file.path)
# XXX for test coverage
# have no idea what extra check can be performed
obj.file = obj.file
self.assertTrue(os.path.exists(obj.file.path))
self.assertEqual(pf1.path, obj.file.path)
@unittest.expectedFailure
def test_update_file2file_not_random(self):
obj = self.Model()
obj.file_by_id = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test1')
self.db.add(obj)
self.db.commit()
self.assertEqual(obj.file_by_id_name,
self.Model.file_by_id.name_template.format(item=obj))
pf1 = obj.file_by_id
obj.file_by_id = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test2')
self.assertIsInstance(obj.file_by_id, TransientFile)
self.assertIsNotNone(obj.file_by_id_name)
self.db.commit()
self.assertIsInstance(obj.file_by_id, PersistentFile)
self.assertFalse(os.path.exists(f.path))
self.assertFalse(os.path.exists(pf1.path))
self.assertTrue(os.path.isfile(obj.file_by_id.path))
self.assertEqual(open(obj.file_by_id.path).read(), 'test2')
@unittest.expectedFailure
def test_update_random_collision(self):
obj = self.Model()
self.db.add(obj)
self.db.commit()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test')
persistent = self.file_manager.get_persistent(obj.file_name)
dirname = os.path.dirname(persistent.path)
if not os.path.isdir(dirname):
os.makedirs(dirname)
with open(persistent.path, 'wb') as fp:
fp.write(b'taken')
self.assertIsInstance(obj.file, TransientFile)
self.assertIsNotNone(obj.file_name)
self.db.commit()
self.assertIsInstance(obj.file, PersistentFile)
self.assertFalse(os.path.exists(f.path))
self.assertTrue(os.path.isfile(obj.file.path))
self.assertEqual(open(obj.file.path).read(), 'test')
self.assertNotEqual(persistent.path, obj.file.path)
self.assertEqual(open(persistent.path).read(), 'taken')
def test_update_none2persistent(self):
f = self.file_manager.get_persistent('persistent.txt')
with open(f.path, 'wb') as fp:
fp.write(b'test1')
obj = self.Model()
obj.file = f
self.db.add(obj)
self.db.commit()
self.assertIsInstance(obj.file, PersistentFile)
self.assertTrue(os.path.exists(obj.file.path))
self.assertEqual(obj.file.name, 'persistent.txt')
def test_delete(self):
obj = self.Model()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test')
self.db.add(obj)
self.db.commit()
pf = obj.file
self.db.delete(obj)
self.db.commit()
self.assertFalse(os.path.exists(pf.path))
def test_set_invalid(self):
obj = self.Model()
self.assertRaises(ValueError, lambda: setattr(obj, 'file', 'test'))
def test_cached_lost(self):
obj = self.Model()
self.db.add(obj)
self.db.commit()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test')
self.db.commit()
os.unlink(obj.file.path)
self.assertEqual(obj.file.size, 4)
self.assertEqual(obj.file_size, 4)
del obj.file.size
self.assertEqual(obj.file.size, None)
def test_file2none_lost(self):
obj = self.Model()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test')
self.db.add(obj)
self.db.commit()
os.unlink(obj.file.path)
obj.file = None
self.db.commit()
self.assertEqual(obj.file_size, None)
def test_file_manager_for_field(self):
def make():
file_manager = FileManager(self.transient_root,
self.persistent_root,
self.transient_url, self.persistent_url)
filesessionmaker(orm.sessionmaker(),
self.file_manager,
file_managers={
ObjWithFile.file: file_manager,
})
self.assertRaises(NotImplementedError, make)
def test_detached_object(self):
obj = self.Model()
self.db.add(obj)
self.db.commit()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test')
self.db.commit()
# cleanup self.Model.file._states.items() to get the result
# from scratch, not from cache
obj_id = obj.id
obj = None
gc.collect()
obj = self.db.query(self.Model).get(obj_id)
with mock.patch.object(iktomi.db.sqla.files,
'object_session',
return_value=None):
with self.assertRaises(RuntimeError) as exc:
obj.file
self.assertEqual('Object is detached', str(exc.exception))
def test_absent_file_manager(self):
obj = self.Model()
del self.db.file_manager
self.db.add(obj)
self.db.commit()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test')
self.db.commit()
# cleanup self.Model.file._states.items() to get the result
# from scratch, not from cache
obj_id = obj.id
obj = None
gc.collect()
obj = self.db.query(self.Model).get(obj_id)
with self.assertRaises(RuntimeError) as exc:
obj.file
self.assertEqual("Session doesn't support file management",
str(exc.exception))
class FormFilesTests(unittest.TestCase):
def setUp(self):
self.transient_root = tempfile.mkdtemp()
self.persistent_root = tempfile.mkdtemp()
self.transient_url = '/transient/'
self.persistent_url = '/media/'
self.file_manager = FileManager(self.transient_root,
self.persistent_root,
self.transient_url,
self.persistent_url)
self.env = AppEnvironment.create(file_manager=self.file_manager)
def tearDown(self):
shutil.rmtree(self.transient_root)
shutil.rmtree(self.persistent_root)
def _create_persistent(self):
f = self.file_manager.get_persistent('test.txt')
with open(f.path, 'wb') as fp:
fp.write(b'test')
return f
def _create_transient(self, content, original_name='test.txt'):
f = self.file_manager.new_transient(ext=os.path.splitext(original_name)[1])
with open(f.path, 'wb') as fp:
fp.write(content)
return f
def _create_fs(self, mimetype, content, filename='uploaded.txt', name="file"):
# http://stackoverflow.com/questions/12032807/how-to-create-cgi-fieldstorage-for-testing-purposes
content = content.encode('utf-8')
m = {u'content-disposition': u'form-data; name="{}"; filename="{}"'.format(name, filename),
u'content-length': len(content),
u'content-type': mimetype}
environ = {'REQUEST_METHOD': 'POST'}
fp = BytesIO(content)
return cgi.FieldStorage(fp=fp, headers=m, environ=environ)
def test_none2empty(self):
form = FormWithFile(self.env)
form.accept(MultiDict({'file.file': None,
'file.original_name': '',
'file.transient_name': '',
'file.mode': 'empty',}))
data = form.python_data
self.assertEqual(data['file'], None)
def test_transient2empty(self):
transient = self._create_transient(b'transient1')
form = FormWithFile(self.env)
form.accept(MultiDict({'file.file': None,
'file.original_name': 'test.txt',
'file.transient_name': transient.name,
'file.mode': 'transient',}))
data = form.python_data
self.assertIsInstance(data['file'], TransientFile)
self.assertEqual(data['file'].name, transient.name)
def test_transient_lost(self):
form = FormWithFile(self.env)
form.accept(MultiDict({'file.file': None,
'file.original_name': 'test.txt',
'file.transient_name': 'lost.txt',
'file.mode': 'transient',}))
self.assertIn('file', form.errors)
def test_transient_invalid(self):
illegal = ['more/../../../lost.txt',
'../pass',
'/etc/pass',
'..',
'~',
]
for filename in illegal:
form = FormWithFile(self.env)
form.accept(MultiDict({'file.file': None,
'file.original_name': 'test.txt',
'file.transient_name': filename,
'file.mode': 'transient',}))
self.assertIn('file.transient_name', form.errors,
'file name is not filtered: %s' % filename)
def test_file2empty(self):
form = FormWithFile(self.env)
fs = self._create_fs('text', 'file-content', 'filename.ttt')
form.accept(MultiDict({'file.file': fs,
'file.mode': 'existing',}))
data = form.python_data
self.assertIsInstance(data['file'], TransientFile)
self.assertEqual(os.path.splitext(data['file'].name)[1],
'.ttt')
with open(data['file'].path) as fp:
self.assertEqual(fp.read(), 'file-content')
def test_none2persistent(self):
persistent = self._create_persistent()
form = FormWithFile(self.env, initial={'file': persistent})
form.accept(MultiDict({'file.file': None,
'file.original_name': '',
'file.transient_name': '',
'file.mode': 'existing',}))
data = form.python_data
self.assertEqual(data['file'], persistent)
def test_empty2persistent(self):
persistent = self._create_persistent()
form = FormWithFile(self.env, initial={'file': persistent})
form.accept(MultiDict({'file.file': None,
'file.original_name': '',
'file.transient_name': '',
'file.mode': 'empty',}))
data = form.python_data
self.assertEqual(data['file'], None)
def test_transient2persistent(self):
persistent = self._create_persistent()
transient = self._create_transient(b'transient1')
form = FormWithFile(self.env, initial={'file': persistent})
form.accept(MultiDict({'file.file': None,
'file.original_name': 'test.txt',
'file.transient_name': transient.name,
'file.mode': 'transient',}))
data = form.python_data
self.assertIsInstance(data['file'], TransientFile)
self.assertEqual(data['file'].name, transient.name)
def test_file2persistent(self):
persistent = self._create_persistent()
form = FormWithFile(self.env, initial={'file': persistent})
fs = self._create_fs('text', 'file-content', 'filename.ttt')
form.accept(MultiDict({'file.file': fs,
'file.mode': 'existing',}))
data = form.python_data
self.assertIsInstance(data['file'], TransientFile)
self.assertEqual(os.path.splitext(data['file'].name)[1],
'.ttt')
with open(data['file'].path) as fp:
self.assertEqual(fp.read(), 'file-content')
class FormFilesTests(unittest.TestCase):
def setUp(self):
self.transient_root = tempfile.mkdtemp()
self.persistent_root = tempfile.mkdtemp()
self.transient_url = "/transient/"
self.persistent_url = "/media/"
self.file_manager = FileManager(
self.transient_root, self.persistent_root, self.transient_url, self.persistent_url
)
self.env = AppEnvironment.create(file_manager=self.file_manager)
def tearDown(self):
shutil.rmtree(self.transient_root)
shutil.rmtree(self.persistent_root)
def _create_persistent(self):
f = self.file_manager.get_persistent("test.txt")
with open(f.path, "wb") as fp:
fp.write(b"test")
return f
def _create_transient(self, content, original_name="test.txt"):
f = self.file_manager.new_transient(ext=os.path.splitext(original_name)[1])
with open(f.path, "wb") as fp:
fp.write(content)
return f
def _create_fs(self, mimetype, content, filename="uploaded.txt", name="file"):
# http://stackoverflow.com/questions/12032807/how-to-create-cgi-fieldstorage-for-testing-purposes
content = content.encode("utf-8")
m = {
u"content-disposition": u'form-data; name="{}"; filename="{}"'.format(name, filename),
u"content-length": len(content),
u"content-type": mimetype,
}
environ = {"REQUEST_METHOD": "POST"}
fp = BytesIO(content)
return cgi.FieldStorage(fp=fp, headers=m, environ=environ)
def test_none2empty(self):
form = FormWithFile(self.env)
form.accept(
MultiDict({"file.file": None, "file.original_name": "", "file.transient_name": "", "file.mode": "empty"})
)
data = form.python_data
self.assertEqual(data["file"], None)
def test_transient2empty(self):
transient = self._create_transient(b"transient1")
form = FormWithFile(self.env)
form.accept(
MultiDict(
{
"file.file": None,
"file.original_name": "test.txt",
"file.transient_name": transient.name,
"file.mode": "transient",
}
)
)
data = form.python_data
self.assertIsInstance(data["file"], TransientFile)
self.assertEqual(data["file"].name, transient.name)
def test_transient_lost(self):
form = FormWithFile(self.env)
form.accept(
MultiDict(
{
"file.file": None,
"file.original_name": "test.txt",
"file.transient_name": "lost.txt",
"file.mode": "transient",
}
)
)
self.assertIn("file", form.errors)
def test_transient_invalid(self):
illegal = ["more/../../../lost.txt", "../pass", "/etc/pass", "..", "~"]
for filename in illegal:
form = FormWithFile(self.env)
form.accept(
MultiDict(
{
"file.file": None,
"file.original_name": "test.txt",
"file.transient_name": filename,
"file.mode": "transient",
}
)
)
self.assertIn("file.transient_name", form.errors, "file name is not filtered: %s" % filename)
def test_file2empty(self):
form = FormWithFile(self.env)
fs = self._create_fs("text", "file-content", "filename.ttt")
form.accept(MultiDict({"file.file": fs, "file.mode": "existing"}))
data = form.python_data
self.assertIsInstance(data["file"], TransientFile)
self.assertEqual(os.path.splitext(data["file"].name)[1], ".ttt")
with open(data["file"].path) as fp:
self.assertEqual(fp.read(), "file-content")
def test_none2persistent(self):
persistent = self._create_persistent()
form = FormWithFile(self.env, initial={"file": persistent})
form.accept(
MultiDict({"file.file": None, "file.original_name": "", "file.transient_name": "", "file.mode": "existing"})
)
data = form.python_data
self.assertEqual(data["file"], persistent)
def test_empty2persistent(self):
persistent = self._create_persistent()
form = FormWithFile(self.env, initial={"file": persistent})
form.accept(
MultiDict({"file.file": None, "file.original_name": "", "file.transient_name": "", "file.mode": "empty"})
)
data = form.python_data
self.assertEqual(data["file"], None)
def test_transient2persistent(self):
persistent = self._create_persistent()
transient = self._create_transient(b"transient1")
form = FormWithFile(self.env, initial={"file": persistent})
form.accept(
MultiDict(
{
"file.file": None,
"file.original_name": "test.txt",
"file.transient_name": transient.name,
"file.mode": "transient",
}
)
)
data = form.python_data
self.assertIsInstance(data["file"], TransientFile)
self.assertEqual(data["file"].name, transient.name)
def test_file2persistent(self):
persistent = self._create_persistent()
form = FormWithFile(self.env, initial={"file": persistent})
fs = self._create_fs("text", "file-content", "filename.ttt")
form.accept(MultiDict({"file.file": fs, "file.mode": "existing"}))
data = form.python_data
self.assertIsInstance(data["file"], TransientFile)
self.assertEqual(os.path.splitext(data["file"].name)[1], ".ttt")
with open(data["file"].path) as fp:
self.assertEqual(fp.read(), "file-content")
class SqlaImagesTests(unittest.TestCase):
def setUp(self):
self.transient_root = tempfile.mkdtemp()
self.persistent_root = tempfile.mkdtemp()
self.transient_url = '/transient/'
self.persistent_url = '/media/'
self.file_manager = FileManager(self.transient_root,
self.persistent_root,
self.transient_url,
self.persistent_url)
Session = filesessionmaker(orm.sessionmaker(), self.file_manager)
engine = create_engine('sqlite://')
Base.metadata.create_all(engine)
self.db = Session(bind=engine)
def tearDown(self):
shutil.rmtree(self.transient_root)
shutil.rmtree(self.persistent_root)
def test_create(self):
obj = ObjWithImage()
obj.image = f = self.file_manager.new_transient('.gif')
_create_image(f.path)
def return_image(image, filter):
# asserting that filter was called
self.assertEqual(filter, ImageFilter.BLUR)
return image
with mock.patch('PIL.Image.Image.filter',
side_effect=return_image,
autospec=True):
self.db.add(obj)
self.db.commit()
self.assertIsInstance(obj.image, PersistentFile)
self.assertIsInstance(obj.thumb, PersistentFile)
self.assertIsInstance(obj.thumb_filter, PersistentFile)
self.assertIsInstance(obj.thumb_optimize, PersistentFile)
img = Image.open(obj.image.path)
self.assertEqual(img.size, (200, 200))
self.assertEqual(obj.image.width, img.width)
self.assertEqual(obj.image.height, img.height)
thumb = Image.open(obj.thumb.path)
self.assertEqual(thumb.size, (100, 100))
self.assertEqual(obj.thumb.height, thumb.height)
self.assertEqual(obj.thumb.width, thumb.width)
pixels = thumb.load()
self.assertEqual(pixels[50, 50], (186, 186, 186))
self.assertLessEqual(os.stat(obj.thumb_optimize.path).st_size,
os.stat(obj.thumb.path).st_size)
def test_no_original_image(self):
obj = ObjWithImage()
obj.image = f = self.file_manager.new_transient()
_create_image(f.path, format='PNG')
warn = []
with mock.patch('os.path.isfile', return_value=False):
with mock.patch('logging.Logger.warn',
side_effect=lambda m, *args: warn.append(m % args)):
self.db.add(obj)
self.db.commit()
self.assertEqual(len(warn), 3)
self.assertEqual(warn[0], warn[1])
self.assertEqual(warn[1], warn[2])
self.assertIn("Original file is absent", warn[0])
self.assertIn(obj.image.path, warn[0])
def test_no_ext(self):
# test for extraction image extension from image instead of file path
obj = ObjWithImage()
obj.image = f = self.file_manager.new_transient()
_create_image(f.path, format='PNG')
self.db.add(obj)
self.db.commit()
self.assertIsInstance(obj.image, PersistentFile)
self.assertTrue(obj.image.path.endswith('.png'))
def test_no_size(self):
obj = ObjWithImage()
obj.icon = f = self.file_manager.new_transient('.gif')
_create_image(f.path, 200, 300)
self.db.add(obj)
self.db.commit()
self.assertIsNone(obj.image)
self.assertIsNone(obj.thumb)
self.assertIsInstance(obj.icon, PersistentFile)
img = Image.open(obj.icon.path)
self.assertEqual(img.size, (200, 300))
def test_no_img(self):
obj = ObjWithImage()
self.db.add(obj)
self.db.commit()
self.assertEqual(obj.image, None)
self.assertEqual(obj.thumb, None)
self.assertEqual(obj.icon, None)
def test_update(self):
obj = ObjWithImage()
self.db.add(obj)
self.db.commit()
obj.image = f = self.file_manager.new_transient('.gif')
_create_image(f.path)
self.db.commit()
self.assertIsInstance(obj.image, PersistentFile)
self.assertIsInstance(obj.thumb, PersistentFile)
img = Image.open(obj.image.path)
self.assertEqual(img.size, (200, 200))
thumb = Image.open(obj.thumb.path)
self.assertEqual(thumb.size, (100, 100))
def test_invalid(self):
obj = ObjWithImage()
obj.image = f = self.file_manager.new_transient('.gif')
with open(f.path, 'wb') as fp:
fp.write(b'test')
self.db.add(obj)
with self.assertRaises(IOError):
self.db.commit()
def test_fill_without_size(self):
with self.assertRaises(AssertionError):
class Test(Base):
id = Column(Integer, primary_key=True)
image_name = Column(VARBINARY(250))
image = ImageProperty(image_name,
name_template='image/{random}')
thumb_name = Column(VARBINARY(250))
thumb = ImageProperty(thumb_name,
name_template='thumb/{random}',
fill_from='image')
class SqlaFilesTests(unittest.TestCase):
Model = ObjWithFile
def setUp(self):
self.transient_root = tempfile.mkdtemp()
self.persistent_root = tempfile.mkdtemp()
self.transient_url = '/transient/'
self.persistent_url = '/media/'
self.file_manager = FileManager(self.transient_root,
self.persistent_root,
self.transient_url,
self.persistent_url)
self.metadata_transient_root = tempfile.mkdtemp()
self.metadata_persistent_root = tempfile.mkdtemp()
self.metadata_transient_url = '/metadata/transient/'
self.metadata_persistent_url = '/metadata/media/'
self.metadata_file_manager = FileManager(self.metadata_transient_root,
self.metadata_persistent_root,
self.metadata_transient_url,
self.metadata_persistent_url)
self.model_transient_root = tempfile.mkdtemp()
self.model_persistent_root = tempfile.mkdtemp()
self.model_transient_url = '/model/transient/'
self.model_persistent_url = '/model/media/'
self.model_file_manager = FileManager(self.model_transient_root,
self.model_persistent_root,
self.model_transient_url,
self.model_persistent_url)
Session = filesessionmaker(orm.sessionmaker(), self.file_manager,
file_managers={
MetadataLevelObj.metadata: self.metadata_file_manager,
ModelLevelObj: self.model_file_manager,
})
engine = create_engine('sqlite://')
Base.metadata.create_all(engine)
CustomBase.metadata.create_all(engine)
self.db = Session(bind=engine)
def tearDown(self):
shutil.rmtree(self.transient_root)
shutil.rmtree(self.persistent_root)
shutil.rmtree(self.metadata_transient_root)
shutil.rmtree(self.metadata_persistent_root)
shutil.rmtree(self.model_transient_root)
shutil.rmtree(self.model_persistent_root)
def test_session(self):
self.assertTrue(hasattr(self.db, 'file_manager'))
self.assertIsInstance(self.db.file_manager, FileManager)
def test_find_file_manager(self):
self.assertTrue(hasattr(self.db, 'find_file_manager'))
self.assertEqual(self.db.find_file_manager(self.Model.file),
self.file_manager)
self.assertEqual(self.db.find_file_manager(ModelLevelObj.file),
self.model_file_manager)
self.assertEqual(self.db.find_file_manager(MetadataLevelObj.file),
self.metadata_file_manager)
def test_create(self):
obj = self.Model()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test')
self.assertIsInstance(obj.file, TransientFile)
self.assertIsNotNone(obj.file_name)
self.db.add(obj)
self.db.commit()
self.assertIsInstance(obj.file, PersistentFile)
self.assertFalse(os.path.exists(f.path))
self.assertTrue(os.path.isfile(obj.file.path))
self.assertEqual(open(obj.file.path).read(), 'test')
def test_create_metadata_obj(self):
metadata_obj = MetadataLevelObj()
metadata_filemanager = self.db.find_file_manager(metadata_obj)
metadata_file = metadata_filemanager.new_transient()
with open(metadata_file.path, 'wb') as fp:
fp.write(b'test')
metadata_obj.file = metadata_file
self.assertTrue(metadata_obj.file.path.startswith(self.metadata_transient_root))
self.db.add(metadata_obj)
self.db.commit()
self.assertTrue(metadata_obj.file.path.startswith(self.metadata_persistent_root))
def test_create_model_obj(self):
model_obj = ModelLevelObj()
model_filemanager = self.db.find_file_manager(model_obj)
model_file = model_filemanager.new_transient()
with open(model_file.path, 'wb') as fp:
fp.write(b'test')
model_obj.file = model_file
self.assertTrue(model_obj.file.path.startswith(self.model_transient_root))
self.db.add(model_obj)
self.db.commit()
self.assertTrue(model_obj.file.path.startswith(self.model_persistent_root))
def test_update_none2file(self):
obj = self.Model()
self.db.add(obj)
self.db.commit()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test')
self.assertIsInstance(obj.file, TransientFile)
self.assertIsNotNone(obj.file_name)
self.db.commit()
# cleanup self.Model.file._states.items() to get the result
# from scratch, not from cache
obj_id = obj.id
obj = None
gc.collect()
obj = self.db.query(self.Model).get(obj_id)
self.assertIsInstance(obj.file, PersistentFile)
self.assertFalse(os.path.exists(f.path))
self.assertTrue(os.path.isfile(obj.file.path))
self.assertEqual(open(obj.file.path).read(), 'test')
self.assertEqual(obj.file.size, 4)
self.assertEqual(obj.file_size, 4)
def test_update_file2none(self):
obj = self.Model()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test')
self.db.add(obj)
self.db.commit()
pf = obj.file
obj.file = None
self.assertIsNone(obj.file_name)
self.assertTrue(os.path.exists(pf.path))
self.assertEqual(obj.file_size, 4)
self.db.commit()
self.assertFalse(os.path.exists(pf.path))
self.assertEqual(obj.file_size, None)
def test_update_file2file(self):
obj = self.Model()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test1')
self.db.add(obj)
self.db.commit()
pf1 = obj.file
self.assertEqual(obj.file_size, 5)
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test22')
self.assertIsInstance(obj.file, TransientFile)
self.assertIsNotNone(obj.file_name)
self.db.commit()
self.assertIsInstance(obj.file, PersistentFile)
self.assertFalse(os.path.exists(f.path))
self.assertFalse(os.path.exists(pf1.path))
self.assertTrue(os.path.isfile(obj.file.path))
self.assertEqual(open(obj.file.path).read(), 'test22')
self.assertEqual(obj.file_size, 6)
def test_update_file2self(self):
obj = self.Model()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test1')
self.db.add(obj)
self.db.commit()
pf1 = obj.file
obj.file = self.file_manager.get_persistent(obj.file.name)
self.db.commit()
self.assertIsInstance(obj.file, PersistentFile)
self.assertTrue(os.path.exists(obj.file.path))
self.assertEqual(pf1.path, obj.file.path)
# XXX for test coverage
# have no idea what extra check can be performed
obj.file = obj.file
self.assertTrue(os.path.exists(obj.file.path))
self.assertEqual(pf1.path, obj.file.path)
@unittest.expectedFailure
def test_update_file2file_not_random(self):
obj = self.Model()
obj.file_by_id = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test1')
self.db.add(obj)
self.db.commit()
self.assertEqual(obj.file_by_id_name,
self.Model.file_by_id.name_template.format(item=obj))
pf1 = obj.file_by_id
obj.file_by_id = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test2')
self.assertIsInstance(obj.file_by_id, TransientFile)
self.assertIsNotNone(obj.file_by_id_name)
self.db.commit()
self.assertIsInstance(obj.file_by_id, PersistentFile)
self.assertFalse(os.path.exists(f.path))
self.assertFalse(os.path.exists(pf1.path))
self.assertTrue(os.path.isfile(obj.file_by_id.path))
self.assertEqual(open(obj.file_by_id.path).read(), 'test2')
@unittest.expectedFailure
def test_update_random_collision(self):
obj = self.Model()
self.db.add(obj)
self.db.commit()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test')
persistent = self.file_manager.get_persistent(obj.file_name)
dirname = os.path.dirname(persistent.path)
if not os.path.isdir(dirname):
os.makedirs(dirname)
with open(persistent.path, 'wb') as fp:
fp.write(b'taken')
self.assertIsInstance(obj.file, TransientFile)
self.assertIsNotNone(obj.file_name)
self.db.commit()
self.assertIsInstance(obj.file, PersistentFile)
self.assertFalse(os.path.exists(f.path))
self.assertTrue(os.path.isfile(obj.file.path))
self.assertEqual(open(obj.file.path).read(), 'test')
self.assertNotEqual(persistent.path, obj.file.path)
self.assertEqual(open(persistent.path).read(), 'taken')
def test_update_none2persistent(self):
f = self.file_manager.get_persistent('persistent.txt')
with open(f.path, 'wb') as fp:
fp.write(b'test1')
obj = self.Model()
obj.file = f
self.db.add(obj)
self.db.commit()
self.assertIsInstance(obj.file, PersistentFile)
self.assertTrue(os.path.exists(obj.file.path))
self.assertEqual(obj.file.name, 'persistent.txt')
def test_delete(self):
obj = self.Model()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test')
self.db.add(obj)
self.db.commit()
pf = obj.file
self.db.delete(obj)
self.db.commit()
self.assertFalse(os.path.exists(pf.path))
def test_set_invalid(self):
obj = self.Model()
self.assertRaises(ValueError, lambda: setattr(obj, 'file', 'test'))
def test_cached_lost(self):
obj = self.Model()
self.db.add(obj)
self.db.commit()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test')
self.db.commit()
os.unlink(obj.file.path)
self.assertEqual(obj.file.size, 4)
self.assertEqual(obj.file_size, 4)
del obj.file.size
self.assertEqual(obj.file.size, None)
def test_file2none_lost(self):
obj = self.Model()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test')
self.db.add(obj)
self.db.commit()
os.unlink(obj.file.path)
obj.file = None
self.db.commit()
self.assertEqual(obj.file_size, None)
def test_file_manager_for_field(self):
def make():
file_manager = FileManager(self.transient_root,
self.persistent_root,
self.transient_url,
self.persistent_url)
filesessionmaker(orm.sessionmaker(), self.file_manager,
file_managers={
ObjWithFile.file: file_manager,
})
self.assertRaises(NotImplementedError, make)
def test_detached_object(self):
obj = self.Model()
self.db.add(obj)
self.db.commit()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test')
self.db.commit()
# cleanup self.Model.file._states.items() to get the result
# from scratch, not from cache
obj_id = obj.id
obj = None
gc.collect()
obj = self.db.query(self.Model).get(obj_id)
with mock.patch.object(iktomi.db.sqla.files,
'object_session', return_value=None):
with self.assertRaises(RuntimeError) as exc:
obj.file
self.assertEqual('Object is detached', str(exc.exception))
def test_absent_file_manager(self):
obj = self.Model()
del self.db.file_manager
self.db.add(obj)
self.db.commit()
obj.file = f = self.file_manager.new_transient()
with open(f.path, 'wb') as fp:
fp.write(b'test')
self.db.commit()
# cleanup self.Model.file._states.items() to get the result
# from scratch, not from cache
obj_id = obj.id
obj = None
gc.collect()
obj = self.db.query(self.Model).get(obj_id)
with self.assertRaises(RuntimeError) as exc:
obj.file
self.assertEqual("Session doesn't support file management", str(exc.exception))