def test_validate_signature(self): """ Validating signature contained within filename """ auto_id = utils.generate_id('test.jpg') auto_params = dict(id=auto_id, size='100x200', factor='fill', output_format='jpg', upscale=True, quality=80) manual_id = utils.generate_id('test.jpg') manual_params = dict(id=manual_id, sample_size='200x400', target_size='100x200', output_format='jpg', upscale=True, quality=80) pb = PathBuilder('12345') auto = pb.get_auto_crop_filename(**auto_params) manual = pb.get_manual_crop_filename(**manual_params) bad = manual.split('-') bad[4] = 'ZZZ' + bad[4] bad = '-'.join(bad) self.assertTrue(pb.validate_signature(auto_id, auto)) self.assertTrue(pb.validate_signature(manual_id, manual)) self.assertFalse(pb.validate_signature(manual_id, bad))
def put(self, src, delete_local=True, fix_orientation=False): """ Put local file to storage Generates a uuid for the file, tells backend to accept it by that id and removes original on success. """ if not os.path.exists(src): msg = 'Unable to find local file [{}]' raise x.LocalFileNotFound(msg.format(src)) path = Path(src) extension = ''.join(path.suffixes)[1:] name = path.name.replace('.' + extension, '') extension = utils.normalize_extension(extension) filename = name + '.' + extension id = utils.generate_id(filename) # fix image orientation before accepting if fix_orientation: Resizer.fix_orientation_and_save(src) self.backend.put_variant(src, id, filename.lower()) if delete_local: os.remove(src) return id
def test_delete_file(self): """ Deleting file from local storage """ # put file self.prepare_uploads() backend = BackendS3(**self.config) uploads = self.upload_path src = os.path.join(uploads, 'test.tar.gz') id1 = utils.generate_id('test.tar.gz') backend.put(src, id1) # regression testing (only delete what requested) id2 = id1.split('-') id2[4] += 'ZZZ' id2 = '-'.join(id2) backend.put(src, id1, True) backend.put_variant(src, id1, 'demo.txt') backend.put(src, id2, True) backend.delete(id1) path1 = '/'.join(backend.id_to_path(id1)) + '/test.tar.gz' path2 = '/'.join(backend.id_to_path(id1)) + '/demo.txt' self.assertFalse(backend.exists(path1)) self.assertFalse(backend.exists(path2)) # assume only proper file deleted path3 = '/'.join(backend.id_to_path(id2)) + '/test.tar.gz' self.assertTrue(backend.exists(path3))
def test_delete_file(self): """ Deleting file from local storage """ # put file self.prepare_uploads() backend = BackendLocal(self.path) uploads = self.upload_path src = os.path.join(uploads, 'test.tar.gz') id1 = utils.generate_id('test.tar.gz') # regression testing id2 = id1.split('-') id2[4] += 'ZZZ' id2 = '-'.join(id2) backend.put_variant(src, id1, 'original.tar.gz') backend.put_variant(src, id2, 'original.tar.gz') backend.delete(id1) path1 = os.path.join(self.path, *id1.split('-')[0:6], 'original.tar.gz') self.assertFalse(os.path.exists(path1)) # assume only proper file deleted path2 = os.path.join(self.path, *id2.split('-')[0:6], 'original.tar.gz') self.assertTrue(os.path.exists(path2))
def test_generate_signature(self): """ Generating hash signature""" filename = 'example.jpg' id = utils.generate_id('jpg') pb = PathBuilder('12345') signature1 = pb.generate_signature(id, filename) signature2 = pb.generate_signature(id, filename) self.assertEquals(signature1, signature2)
def test_convert_id_to_path(self): """ Converting id to path """ backend = BackendS3(**self.config) filename = 'demo-test.tar.gz' id = utils.generate_id(filename) parts = backend.id_to_path(id) self.assertEquals(6, len(parts)) self.assertEquals(filename, parts[5])
def test_lowercase_filename_when_putting(self): """REGRESSEION (#9): Convert file names to lowercase """ self.prepare_uploads() backend = BackendS3(**self.config) uploads = self.upload_path src = os.path.join(uploads, 'MiXeDcAsE.jpg') id = utils.generate_id('MiXeDcAsE.jpg') backend.put(src, id) path = '/'.join(backend.id_to_path(id)) + '/MiXeDcAsE.jpg' self.assertTrue(backend.exists(path.lower()))
def test_put_file_variant(self): """ Put file to storage by filename""" self.prepare_uploads() backend = BackendS3(**self.config) uploads = self.upload_path src = os.path.join(uploads, 'demo-test.tar.gz') id = utils.generate_id('demo-test.tar.gz') backend.put_variant(src, id, 'variant.tar.gz') path = '/'.join(backend.id_to_path(id)) + '/variant.tar.gz' self.assertTrue(backend.exists(path))
def test_missing_autocrop_format_defaults_to_original_format(self): """ Autocrop defaults to original format when format not specified""" params = dict(id=utils.generate_id('zz.gif'), size='100x200', factor='fill', upscale=True, quality=80) pb = PathBuilder('12345') filename = pb.get_auto_crop_filename(**params) self.assertTrue(filename.endswith('.gif'))
def test_manual_crop_filename_generator_raises_on_bad_target_size(self): """ Manual crop filename generator raises on bad target size""" params = dict(id=utils.generate_id('jpg'), sample_size='200x400', target_size='100xCRAP', output_format='JPEG', upscale=True, quality=80) pb = PathBuilder('12345') with assert_raises(x.InvalidArgumentException): pb.get_manual_crop_filename(**params)
def test_auto_crop_filename_generator_raises_on_bad_quality(self): """ Auto crop filename generator raises exception on bad quality """ params = dict(id=utils.generate_id('jpg'), size='100x200', factor='fit', output_format='jpg', upscale=True, quality="CRAP") pb = PathBuilder('12345') with assert_raises(x.InvalidArgumentException): pb.get_auto_crop_filename(**params)
def test_put_raises_on_overwriting(self): """ Put raises exception on attempt to overwrite existing path """ self.prepare_uploads() backend = BackendS3(**self.config) uploads = self.upload_path src1 = os.path.join(uploads, 'demo-test.tar.gz') src2 = os.path.join(uploads, 'test.jpg') id = utils.generate_id('demo-test.tar.gz') backend.put_variant(src1, id, 'demo-test.tar.gz') with assert_raises(x.FileExists): backend.put_variant(src2, id, 'demo-test.tar.gz')
def test_generate_manual_crop_filename(self): """ Generating manual crop filename""" params = dict(id=utils.generate_id('jpg'), sample_size='200x400', target_size='100x200', output_format='jpg', upscale=True, quality=80) pb = PathBuilder('12345') filename = pb.get_manual_crop_filename(**params) start = '100x200-200x400-80-upscale' self.assertTrue(filename.startswith(start))
def test_create_auto_crop_filename(self): """ Creating filename for auto crop""" params = dict(id=utils.generate_id('jpg'), size='100x200', factor='fill', output_format='jpg', upscale=True, quality=80) pb = PathBuilder('12345') filename = pb.get_auto_crop_filename(**params) start = '100x200-fill-80-upscale' self.assertTrue(filename.startswith(start))
def test_get_original_url(self): """ Generating original url """ path = self.path base_url = 'http://test.url' backend = BackendLocal(path, base_url) storage = Storage(backend, secret_key=self.config.SECRET_KEY, local_temp=self.config.LOCAL_TEMP) filename = 'example-object.tar.gz' id = utils.generate_id(filename) url = storage.get_original_url(id) self.assertTrue(url.startswith(base_url)) self.assertTrue(url.endswith(filename + '/' + filename))
def test_resize_filename_parser_raises_on_bad_signature(self): """ Resize filename parser raises on bad signature""" id = utils.generate_id('test.jpg') auto_params = dict(id=id + 'SOME-CRAP', size='100x200', factor='fill', output_format='jpg', upscale=True, quality=80) pb = PathBuilder('12345') filename = pb.get_auto_crop_filename(**auto_params) with assert_raises(x.InvalidArgumentException): pb.filename_to_resize_params(id, filename)
def test_lowercase_filename_when_putting(self): """REGRESSEION (#9): Convert file names to lowercase """ self.prepare_uploads() backend = BackendLocal(self.path) uploads = self.upload_path src = os.path.join(uploads, 'MiXeDcAsE.jpg') id = utils.generate_id('MiXeDcAsE.jpg') result = backend.put(src, id) current = self.path for dir in id.split('-')[0:5]: current = os.path.join(current, dir) full_file_path = os.path.join(current, 'MiXeDcAsE.jpg') self.assertTrue(os.path.exists(full_file_path.lower()))
def test_force_put_to_overwrite_existing(self): """ Using force option to overwrite existing file """ self.prepare_uploads() backend = BackendLocal(self.path) uploads = self.upload_path filename = 'demo-test.tar.gz' src1 = os.path.join(uploads, filename) src2 = os.path.join(uploads, 'test.jpg') id = utils.generate_id(filename) backend.put_variant(src1, id, filename) backend.put_variant(src2, id, filename, True) path = os.path.join(backend.path, *backend.id_to_path(id), filename) # assert overwritten with src2 self.assertEquals(os.path.getsize(path), os.path.getsize(src2))
def test_retrieve_original_to_temp(self): """ Retrieving from backend to local temp """ # put file self.prepare_uploads() backend = BackendS3(**self.config) src = os.path.join(self.upload_path, 'demo-test.tar.gz') id = utils.generate_id('demo-test.tar.gz') backend.put(src, id) # retrieve file result = backend.retrieve_original(id, self.tmp_path) expected_dst = os.path.join(self.tmp_path, id, 'demo-test.tar.gz') self.assertEquals(expected_dst, result) self.assertTrue(os.path.exists(expected_dst))
def test_clear_variants(self): """ Clearing generated variants""" self.prepare_uploads() backend = BackendS3(**self.config) src1 = os.path.join(self.upload_path, 'demo-test.tar.gz') id1 = utils.generate_id('demo-test.tar.gz') backend.put(src1, id1) backend.put_variant(src1, id1, 'variant1.tar.gz') backend.put_variant(src1, id1, 'variant2.tar.gz') src2 = os.path.join(self.upload_path, 'demo-test.tar.gz') id2 = utils.generate_id('demo-test.tar.gz') backend.put(src2, id2) backend.put_variant(src2, id2, 'variant3.jpg') backend.put_variant(src2, id2, 'variant4.jpg') backend.clear_variants() path1 = '/'.join(backend.id_to_path(id1)) original1 = path1 + '/demo-test.tar.gz' variant1 = path1 + '/variant1.tar.gz' variant2 = path1 + '/variant2.tar.gz' path2 = '/'.join(backend.id_to_path(id2)) original2 = path2 + '/demo-test.tar.gz' variant3 = path2 + '/variant3.tar.gz' variant4 = path2 + '/variant4.tar.gz' self.assertTrue(backend.exists(original1)) self.assertFalse(backend.exists(variant1)) self.assertFalse(backend.exists(variant2)) self.assertTrue(backend.exists(original2)) self.assertFalse(backend.exists(variant3)) self.assertFalse(backend.exists(variant4))
def test_resize_filename_parser_raises_on_bad_quality(self): """ Resize filename parser raises if target size invalid """ id = utils.generate_id('test.jpg') params = dict(size='100x200', factor='100x200', format='jpg', upscale='upscale', quality='VRAP') pb = PathBuilder('12345') schema = '{size}-{factor}-{quality}-{upscale}.{format}' signed_schema = '{size}-{factor}-{quality}-{upscale}-{sig}.{format}' params['sig'] = pb.generate_signature(id, schema.format(**params)) signed_filename = signed_schema.format(**params) with assert_raises(x.InvalidArgumentException): pb.filename_to_resize_params(id, signed_filename)
def test_put_with_sequential_ids(self): """ Putting two items in sequence""" filename = 'demo-test.tar.gz' base_id = utils.generate_id(filename).replace('-' + filename, '') id1 = base_id + '1-' + filename id2 = base_id + '2-' + filename self.prepare_uploads() backend = BackendS3(**self.config) uploads = self.upload_path src = os.path.join(uploads, 'demo-test.tar.gz') backend.put_variant(src, id1, 'demo-test.tar.gz') backend.put_variant(src, id2, 'demo-test.tar.gz') path1 = '/'.join(backend.id_to_path(id1)) + '/demo-test.tar.gz' path2 = '/'.join(backend.id_to_path(id2)) + '/demo-test.tar.gz' self.assertTrue(backend.exists(path1)) self.assertTrue(backend.exists(path2))
def test_guess_content_type(self): """ Guess content type for objects when putting to S3 """ self.prepare_uploads() backend = BackendS3(**self.config) src = os.path.join(self.upload_path, 'test.jpg') id = utils.generate_id('demo.jpg') backend.put(src, id, True) path = '/'.join(backend.id_to_path(id)) + '/demo.jpg' client = boto3.client('s3', **backend.credentials) res = client.head_object( Bucket=backend.bucket_name, Key=path ) headers = res['ResponseMetadata']['HTTPHeaders'] self.assertEquals('image/jpeg', headers['content-type'])
def test_put_file(self): """ Put file to storage """ self.prepare_uploads() backend = BackendLocal(self.path) uploads = self.upload_path src = os.path.join(uploads, 'demo-test.tar.gz') id = utils.generate_id('demo-test.tar.gz') backend.put(src, id) # assert directories created current = self.path for dir in id.split('-')[0:5]: current = os.path.join(current, dir) self.assertTrue(os.path.exists(current)) # assert file put full_file_path = os.path.join(current, 'demo-test.tar.gz') self.assertTrue(os.path.exists(full_file_path))
def test_put_variant(self): """ Put file variant to storage by filename""" self.prepare_uploads() backend = BackendLocal(self.path) uploads = self.upload_path src = os.path.join(uploads, 'demo-test.tar.gz') id = utils.generate_id('demo-test.tar.gz') backend.put_variant(src, id, 'demo-test.tar.gz') # assert directories created current = self.path for dir in backend.id_to_path(id): current = os.path.join(current, dir) self.assertTrue(os.path.exists(current)) # assert file put full_file_path = os.path.join(current, 'demo-test.tar.gz') self.assertTrue(os.path.exists(full_file_path))
def test_parse_url(self): """ Parsing object url into id and filename """ filename = 'demo-file.tar.gz' backend = BackendS3(**self.config) pb = PathBuilder('123456') base_url = backend.get_url() id = utils.generate_id(filename) parts = backend.id_to_path(id) path = '/'.join(parts) object_url = base_url + '/' + path + '/' original = object_url + filename crop_filename = pb.get_auto_crop_filename(id, '100x100', 'fit', 'jpg') resize = object_url + crop_filename result1 = backend.parse_url(original) result2 = backend.parse_url(resize) self.assertEquals(id, result1[0]) self.assertEquals(filename, result1[1]) self.assertEquals(id, result2[0]) self.assertEquals(crop_filename, result2[1])
def test_force_put_to_overwrite_existing(self): """ Using force option to overwrite existing file """ self.prepare_uploads() backend = BackendS3(**self.config) uploads = self.upload_path filename = 'demo-test.tar.gz' src1 = os.path.join(uploads, filename) src2 = os.path.join(uploads, 'test.jpg') id = utils.generate_id(filename) backend.put_variant(src1, id, filename) backend.put_variant(src2, id, filename, True) path = '/'.join(backend.id_to_path(id)) + '/' + filename client = boto3.client('s3', **backend.credentials) res = client.head_object(Bucket=backend.bucket_name, Key=path) self.assertEquals( str(os.path.getsize(src2)), str(res['ResponseMetadata']['HTTPHeaders']['content-length']) )
def test_parse_manual_resize_filename(self): """ Parse manual resize filename into a set of parameters """ id = utils.generate_id('test.jpg') params = dict(id=id, sample_size='100x200', target_size='200x400', output_format='jpg', upscale=True, quality=80) pb = PathBuilder('12345') filename = pb.get_manual_crop_filename(**params) result = pb.filename_to_resize_params(id, filename) self.assertEquals(id, result['id']) self.assertEquals(filename, result['filename']) self.assertEquals(params['target_size'], result['target_size']) self.assertEquals(params['sample_size'], result['sample_size']) self.assertEquals(params['output_format'], result['output_format']) self.assertEquals(params['quality'], result['quality']) self.assertEquals(params['upscale'], result['upscale'])
def test_get_manual_crop_url(self): """ Generating manual crop url """ path = self.path base_url = 'http://test.url' backend = BackendLocal(path, base_url) storage = Storage(backend, secret_key=self.config.SECRET_KEY, local_temp=self.config.LOCAL_TEMP) filename = 'example-object.tar.gz' id = utils.generate_id(filename) url = storage.get_manual_crop_url(id=id, sample_size='200x400', target_size='100x200', output_format='gif', upscale=True, quality=80) self.assertTrue(url.startswith(base_url)) url = url.replace(base_url, '').strip('/').split('/') self.assertEquals(filename, url[5]) self.assertTrue(url[6].startswith('100x200-200x400-80-upscale')) self.assertTrue(url[6].endswith('.gif'))
def test_put_raises_on_nonexistent_file(self): """ Put raises exception if source file does not exist """ backend = BackendS3(**self.config) id = utils.generate_id('test.tar.gz') with assert_raises(x.LocalFileNotFound): backend.put_variant('nonexistent', id, 'random.tar.gz')