def test_validate_signature(self):
""" Validating signature contained within filename """
auto_id = utils.generate_id('test.jpg')
auto_params = dict(id=auto_id,
size='100x200',
factor='fill',
output_format='jpg',
upscale=True,
quality=80)
manual_id = utils.generate_id('test.jpg')
manual_params = dict(id=manual_id,
sample_size='200x400',
target_size='100x200',
output_format='jpg',
upscale=True,
quality=80)
pb = PathBuilder('12345')
auto = pb.get_auto_crop_filename(**auto_params)
manual = pb.get_manual_crop_filename(**manual_params)
bad = manual.split('-')
bad[4] = 'ZZZ' + bad[4]
bad = '-'.join(bad)
self.assertTrue(pb.validate_signature(auto_id, auto))
self.assertTrue(pb.validate_signature(manual_id, manual))
self.assertFalse(pb.validate_signature(manual_id, bad))
def put(self, src, delete_local=True, fix_orientation=False):
"""
Put local file to storage
Generates a uuid for the file, tells backend to accept
it by that id and removes original on success.
"""
if not os.path.exists(src):
msg = 'Unable to find local file [{}]'
raise x.LocalFileNotFound(msg.format(src))
path = Path(src)
extension = ''.join(path.suffixes)[1:]
name = path.name.replace('.' + extension, '')
extension = utils.normalize_extension(extension)
filename = name + '.' + extension
id = utils.generate_id(filename)
# fix image orientation before accepting
if fix_orientation:
Resizer.fix_orientation_and_save(src)
self.backend.put_variant(src, id, filename.lower())
if delete_local:
os.remove(src)
return id
def test_delete_file(self):
""" Deleting file from local storage """
# put file
self.prepare_uploads()
backend = BackendS3(**self.config)
uploads = self.upload_path
src = os.path.join(uploads, 'test.tar.gz')
id1 = utils.generate_id('test.tar.gz')
backend.put(src, id1)
# regression testing (only delete what requested)
id2 = id1.split('-')
id2[4] += 'ZZZ'
id2 = '-'.join(id2)
backend.put(src, id1, True)
backend.put_variant(src, id1, 'demo.txt')
backend.put(src, id2, True)
backend.delete(id1)
path1 = '/'.join(backend.id_to_path(id1)) + '/test.tar.gz'
path2 = '/'.join(backend.id_to_path(id1)) + '/demo.txt'
self.assertFalse(backend.exists(path1))
self.assertFalse(backend.exists(path2))
# assume only proper file deleted
path3 = '/'.join(backend.id_to_path(id2)) + '/test.tar.gz'
self.assertTrue(backend.exists(path3))
def test_delete_file(self):
""" Deleting file from local storage """
# put file
self.prepare_uploads()
backend = BackendLocal(self.path)
uploads = self.upload_path
src = os.path.join(uploads, 'test.tar.gz')
id1 = utils.generate_id('test.tar.gz')
# regression testing
id2 = id1.split('-')
id2[4] += 'ZZZ'
id2 = '-'.join(id2)
backend.put_variant(src, id1, 'original.tar.gz')
backend.put_variant(src, id2, 'original.tar.gz')
backend.delete(id1)
path1 = os.path.join(self.path,
*id1.split('-')[0:6], 'original.tar.gz')
self.assertFalse(os.path.exists(path1))
# assume only proper file deleted
path2 = os.path.join(self.path,
*id2.split('-')[0:6], 'original.tar.gz')
self.assertTrue(os.path.exists(path2))
def test_generate_signature(self):
""" Generating hash signature"""
filename = 'example.jpg'
id = utils.generate_id('jpg')
pb = PathBuilder('12345')
signature1 = pb.generate_signature(id, filename)
signature2 = pb.generate_signature(id, filename)
self.assertEquals(signature1, signature2)
def test_convert_id_to_path(self):
""" Converting id to path """
backend = BackendS3(**self.config)
filename = 'demo-test.tar.gz'
id = utils.generate_id(filename)
parts = backend.id_to_path(id)
self.assertEquals(6, len(parts))
self.assertEquals(filename, parts[5])
def test_lowercase_filename_when_putting(self):
"""REGRESSEION (#9): Convert file names to lowercase """
self.prepare_uploads()
backend = BackendS3(**self.config)
uploads = self.upload_path
src = os.path.join(uploads, 'MiXeDcAsE.jpg')
id = utils.generate_id('MiXeDcAsE.jpg')
backend.put(src, id)
path = '/'.join(backend.id_to_path(id)) + '/MiXeDcAsE.jpg'
self.assertTrue(backend.exists(path.lower()))
def test_put_file_variant(self):
""" Put file to storage by filename"""
self.prepare_uploads()
backend = BackendS3(**self.config)
uploads = self.upload_path
src = os.path.join(uploads, 'demo-test.tar.gz')
id = utils.generate_id('demo-test.tar.gz')
backend.put_variant(src, id, 'variant.tar.gz')
path = '/'.join(backend.id_to_path(id)) + '/variant.tar.gz'
self.assertTrue(backend.exists(path))
def test_missing_autocrop_format_defaults_to_original_format(self):
""" Autocrop defaults to original format when format not specified"""
params = dict(id=utils.generate_id('zz.gif'),
size='100x200',
factor='fill',
upscale=True,
quality=80)
pb = PathBuilder('12345')
filename = pb.get_auto_crop_filename(**params)
self.assertTrue(filename.endswith('.gif'))
def test_manual_crop_filename_generator_raises_on_bad_target_size(self):
""" Manual crop filename generator raises on bad target size"""
params = dict(id=utils.generate_id('jpg'),
sample_size='200x400',
target_size='100xCRAP',
output_format='JPEG',
upscale=True,
quality=80)
pb = PathBuilder('12345')
with assert_raises(x.InvalidArgumentException):
pb.get_manual_crop_filename(**params)
def test_auto_crop_filename_generator_raises_on_bad_quality(self):
""" Auto crop filename generator raises exception on bad quality """
params = dict(id=utils.generate_id('jpg'),
size='100x200',
factor='fit',
output_format='jpg',
upscale=True,
quality="CRAP")
pb = PathBuilder('12345')
with assert_raises(x.InvalidArgumentException):
pb.get_auto_crop_filename(**params)
def test_put_raises_on_overwriting(self):
""" Put raises exception on attempt to overwrite existing path """
self.prepare_uploads()
backend = BackendS3(**self.config)
uploads = self.upload_path
src1 = os.path.join(uploads, 'demo-test.tar.gz')
src2 = os.path.join(uploads, 'test.jpg')
id = utils.generate_id('demo-test.tar.gz')
backend.put_variant(src1, id, 'demo-test.tar.gz')
with assert_raises(x.FileExists):
backend.put_variant(src2, id, 'demo-test.tar.gz')
def test_generate_manual_crop_filename(self):
""" Generating manual crop filename"""
params = dict(id=utils.generate_id('jpg'),
sample_size='200x400',
target_size='100x200',
output_format='jpg',
upscale=True,
quality=80)
pb = PathBuilder('12345')
filename = pb.get_manual_crop_filename(**params)
start = '100x200-200x400-80-upscale'
self.assertTrue(filename.startswith(start))
def test_create_auto_crop_filename(self):
""" Creating filename for auto crop"""
params = dict(id=utils.generate_id('jpg'),
size='100x200',
factor='fill',
output_format='jpg',
upscale=True,
quality=80)
pb = PathBuilder('12345')
filename = pb.get_auto_crop_filename(**params)
start = '100x200-fill-80-upscale'
self.assertTrue(filename.startswith(start))
def test_get_original_url(self):
""" Generating original url """
path = self.path
base_url = 'http://test.url'
backend = BackendLocal(path, base_url)
storage = Storage(backend,
secret_key=self.config.SECRET_KEY,
local_temp=self.config.LOCAL_TEMP)
filename = 'example-object.tar.gz'
id = utils.generate_id(filename)
url = storage.get_original_url(id)
self.assertTrue(url.startswith(base_url))
self.assertTrue(url.endswith(filename + '/' + filename))
def test_resize_filename_parser_raises_on_bad_signature(self):
""" Resize filename parser raises on bad signature"""
id = utils.generate_id('test.jpg')
auto_params = dict(id=id + 'SOME-CRAP',
size='100x200',
factor='fill',
output_format='jpg',
upscale=True,
quality=80)
pb = PathBuilder('12345')
filename = pb.get_auto_crop_filename(**auto_params)
with assert_raises(x.InvalidArgumentException):
pb.filename_to_resize_params(id, filename)
def test_lowercase_filename_when_putting(self):
"""REGRESSEION (#9): Convert file names to lowercase """
self.prepare_uploads()
backend = BackendLocal(self.path)
uploads = self.upload_path
src = os.path.join(uploads, 'MiXeDcAsE.jpg')
id = utils.generate_id('MiXeDcAsE.jpg')
result = backend.put(src, id)
current = self.path
for dir in id.split('-')[0:5]:
current = os.path.join(current, dir)
full_file_path = os.path.join(current, 'MiXeDcAsE.jpg')
self.assertTrue(os.path.exists(full_file_path.lower()))
def test_force_put_to_overwrite_existing(self):
""" Using force option to overwrite existing file """
self.prepare_uploads()
backend = BackendLocal(self.path)
uploads = self.upload_path
filename = 'demo-test.tar.gz'
src1 = os.path.join(uploads, filename)
src2 = os.path.join(uploads, 'test.jpg')
id = utils.generate_id(filename)
backend.put_variant(src1, id, filename)
backend.put_variant(src2, id, filename, True)
path = os.path.join(backend.path, *backend.id_to_path(id), filename)
# assert overwritten with src2
self.assertEquals(os.path.getsize(path), os.path.getsize(src2))
def test_retrieve_original_to_temp(self):
""" Retrieving from backend to local temp """
# put file
self.prepare_uploads()
backend = BackendS3(**self.config)
src = os.path.join(self.upload_path, 'demo-test.tar.gz')
id = utils.generate_id('demo-test.tar.gz')
backend.put(src, id)
# retrieve file
result = backend.retrieve_original(id, self.tmp_path)
expected_dst = os.path.join(self.tmp_path, id, 'demo-test.tar.gz')
self.assertEquals(expected_dst, result)
self.assertTrue(os.path.exists(expected_dst))
def test_clear_variants(self):
""" Clearing generated variants"""
self.prepare_uploads()
backend = BackendS3(**self.config)
src1 = os.path.join(self.upload_path, 'demo-test.tar.gz')
id1 = utils.generate_id('demo-test.tar.gz')
backend.put(src1, id1)
backend.put_variant(src1, id1, 'variant1.tar.gz')
backend.put_variant(src1, id1, 'variant2.tar.gz')
src2 = os.path.join(self.upload_path, 'demo-test.tar.gz')
id2 = utils.generate_id('demo-test.tar.gz')
backend.put(src2, id2)
backend.put_variant(src2, id2, 'variant3.jpg')
backend.put_variant(src2, id2, 'variant4.jpg')
backend.clear_variants()
path1 = '/'.join(backend.id_to_path(id1))
original1 = path1 + '/demo-test.tar.gz'
variant1 = path1 + '/variant1.tar.gz'
variant2 = path1 + '/variant2.tar.gz'
path2 = '/'.join(backend.id_to_path(id2))
original2 = path2 + '/demo-test.tar.gz'
variant3 = path2 + '/variant3.tar.gz'
variant4 = path2 + '/variant4.tar.gz'
self.assertTrue(backend.exists(original1))
self.assertFalse(backend.exists(variant1))
self.assertFalse(backend.exists(variant2))
self.assertTrue(backend.exists(original2))
self.assertFalse(backend.exists(variant3))
self.assertFalse(backend.exists(variant4))
def test_resize_filename_parser_raises_on_bad_quality(self):
""" Resize filename parser raises if target size invalid """
id = utils.generate_id('test.jpg')
params = dict(size='100x200',
factor='100x200',
format='jpg',
upscale='upscale',
quality='VRAP')
pb = PathBuilder('12345')
schema = '{size}-{factor}-{quality}-{upscale}.{format}'
signed_schema = '{size}-{factor}-{quality}-{upscale}-{sig}.{format}'
params['sig'] = pb.generate_signature(id, schema.format(**params))
signed_filename = signed_schema.format(**params)
with assert_raises(x.InvalidArgumentException):
pb.filename_to_resize_params(id, signed_filename)
def test_put_with_sequential_ids(self):
""" Putting two items in sequence"""
filename = 'demo-test.tar.gz'
base_id = utils.generate_id(filename).replace('-' + filename, '')
id1 = base_id + '1-' + filename
id2 = base_id + '2-' + filename
self.prepare_uploads()
backend = BackendS3(**self.config)
uploads = self.upload_path
src = os.path.join(uploads, 'demo-test.tar.gz')
backend.put_variant(src, id1, 'demo-test.tar.gz')
backend.put_variant(src, id2, 'demo-test.tar.gz')
path1 = '/'.join(backend.id_to_path(id1)) + '/demo-test.tar.gz'
path2 = '/'.join(backend.id_to_path(id2)) + '/demo-test.tar.gz'
self.assertTrue(backend.exists(path1))
self.assertTrue(backend.exists(path2))
def test_guess_content_type(self):
""" Guess content type for objects when putting to S3 """
self.prepare_uploads()
backend = BackendS3(**self.config)
src = os.path.join(self.upload_path, 'test.jpg')
id = utils.generate_id('demo.jpg')
backend.put(src, id, True)
path = '/'.join(backend.id_to_path(id)) + '/demo.jpg'
client = boto3.client('s3', **backend.credentials)
res = client.head_object(
Bucket=backend.bucket_name,
Key=path
)
headers = res['ResponseMetadata']['HTTPHeaders']
self.assertEquals('image/jpeg', headers['content-type'])
def test_put_file(self):
""" Put file to storage """
self.prepare_uploads()
backend = BackendLocal(self.path)
uploads = self.upload_path
src = os.path.join(uploads, 'demo-test.tar.gz')
id = utils.generate_id('demo-test.tar.gz')
backend.put(src, id)
# assert directories created
current = self.path
for dir in id.split('-')[0:5]:
current = os.path.join(current, dir)
self.assertTrue(os.path.exists(current))
# assert file put
full_file_path = os.path.join(current, 'demo-test.tar.gz')
self.assertTrue(os.path.exists(full_file_path))
def test_put_variant(self):
""" Put file variant to storage by filename"""
self.prepare_uploads()
backend = BackendLocal(self.path)
uploads = self.upload_path
src = os.path.join(uploads, 'demo-test.tar.gz')
id = utils.generate_id('demo-test.tar.gz')
backend.put_variant(src, id, 'demo-test.tar.gz')
# assert directories created
current = self.path
for dir in backend.id_to_path(id):
current = os.path.join(current, dir)
self.assertTrue(os.path.exists(current))
# assert file put
full_file_path = os.path.join(current, 'demo-test.tar.gz')
self.assertTrue(os.path.exists(full_file_path))
def test_parse_url(self):
""" Parsing object url into id and filename """
filename = 'demo-file.tar.gz'
backend = BackendS3(**self.config)
pb = PathBuilder('123456')
base_url = backend.get_url()
id = utils.generate_id(filename)
parts = backend.id_to_path(id)
path = '/'.join(parts)
object_url = base_url + '/' + path + '/'
original = object_url + filename
crop_filename = pb.get_auto_crop_filename(id, '100x100', 'fit', 'jpg')
resize = object_url + crop_filename
result1 = backend.parse_url(original)
result2 = backend.parse_url(resize)
self.assertEquals(id, result1[0])
self.assertEquals(filename, result1[1])
self.assertEquals(id, result2[0])
self.assertEquals(crop_filename, result2[1])
def test_force_put_to_overwrite_existing(self):
""" Using force option to overwrite existing file """
self.prepare_uploads()
backend = BackendS3(**self.config)
uploads = self.upload_path
filename = 'demo-test.tar.gz'
src1 = os.path.join(uploads, filename)
src2 = os.path.join(uploads, 'test.jpg')
id = utils.generate_id(filename)
backend.put_variant(src1, id, filename)
backend.put_variant(src2, id, filename, True)
path = '/'.join(backend.id_to_path(id)) + '/' + filename
client = boto3.client('s3', **backend.credentials)
res = client.head_object(Bucket=backend.bucket_name, Key=path)
self.assertEquals(
str(os.path.getsize(src2)),
str(res['ResponseMetadata']['HTTPHeaders']['content-length'])
)
def test_parse_manual_resize_filename(self):
""" Parse manual resize filename into a set of parameters """
id = utils.generate_id('test.jpg')
params = dict(id=id,
sample_size='100x200',
target_size='200x400',
output_format='jpg',
upscale=True,
quality=80)
pb = PathBuilder('12345')
filename = pb.get_manual_crop_filename(**params)
result = pb.filename_to_resize_params(id, filename)
self.assertEquals(id, result['id'])
self.assertEquals(filename, result['filename'])
self.assertEquals(params['target_size'], result['target_size'])
self.assertEquals(params['sample_size'], result['sample_size'])
self.assertEquals(params['output_format'], result['output_format'])
self.assertEquals(params['quality'], result['quality'])
self.assertEquals(params['upscale'], result['upscale'])
def test_get_manual_crop_url(self):
""" Generating manual crop url """
path = self.path
base_url = 'http://test.url'
backend = BackendLocal(path, base_url)
storage = Storage(backend,
secret_key=self.config.SECRET_KEY,
local_temp=self.config.LOCAL_TEMP)
filename = 'example-object.tar.gz'
id = utils.generate_id(filename)
url = storage.get_manual_crop_url(id=id,
sample_size='200x400',
target_size='100x200',
output_format='gif',
upscale=True,
quality=80)
self.assertTrue(url.startswith(base_url))
url = url.replace(base_url, '').strip('/').split('/')
self.assertEquals(filename, url[5])
self.assertTrue(url[6].startswith('100x200-200x400-80-upscale'))
self.assertTrue(url[6].endswith('.gif'))
def test_put_raises_on_nonexistent_file(self):
""" Put raises exception if source file does not exist """
backend = BackendS3(**self.config)
id = utils.generate_id('test.tar.gz')
with assert_raises(x.LocalFileNotFound):
backend.put_variant('nonexistent', id, 'random.tar.gz')
