def post(self): # Check if the image uploaded is a multipart/form-data if self.multipart_form_data(): file_data = self.request.files['media'][0] body = file_data['body'] # Retrieve filename from 'filename' field filename = file_data['filename'] else: body = self.request.body # Retrieve filename from 'Slug' header filename = self.request.headers.get('Slug') # Check if the image uploaded is valid if self.validate(body): # Use the default filename for the uploaded images if not filename: content_type = self.request.headers.get('Content-Type', BaseEngine.get_mimetype(body)) extension = mimetypes.guess_extension(content_type.split(';', 1)[0], False) if extension is None: # Content-Type is unknown, try with body extension = mimetypes.guess_extension(BaseEngine.get_mimetype(body), False) if extension == '.jpe': extension = '.jpg' # Hack because mimetypes return .jpe by default if extension is None: # Even body is unknown, return an empty string to be contat extension = '' filename = self.context.config.UPLOAD_DEFAULT_FILENAME + extension # Build image id based on a random uuid (32 characters) id = str(uuid.uuid4().hex) self.write_file(id, body) self.set_status(201) self.set_header('Location', self.location(id, filename))
def post(self): # Check if the image uploaded is a multipart/form-data if self.multipart_form_data(): file_data = self.request.files['media'][0] body = file_data['body'] # Retrieve filename from 'filename' field filename = file_data['filename'] else: body = self.request.body # Retrieve filename from 'Slug' header filename = self.request.headers.get('Slug') # Check if the image uploaded is valid if self.validate(body): # Use the default filename for the uploaded images if not filename: content_type = self.request.headers.get('Content-Type', BaseEngine.get_mimetype(body)) extension = mimetypes.guess_extension(content_type.split(';',1)[0], False) if extension is None: # Content-Type is unknown, try with body extension = mimetypes.guess_extension(BaseEngine.get_mimetype(body), False) if extension == '.jpe': extension = '.jpg' # Hack because mimetypes return .jpe by default if extension is None: # Even body is unknown, return an empty string to be contat extension = '' filename = self.context.config.UPLOAD_DEFAULT_FILENAME + extension # Build image id based on a random uuid (32 characters) id = str(uuid.uuid4().hex) self.write_file(id, body) self.set_status(201) self.set_header('Location', self.location(id, filename))
def _fetch(self, url): storage = self.context.modules.storage buffer = yield gen.maybe_future(storage.get(url)) mime = None if buffer is not None: self.context.statsd_client.incr('storage.hit') mime = BaseEngine.get_mimetype(buffer) self.context.request.extension = EXTENSION.get(mime, '.jpg') if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE: self.context.request.engine = self.context.modules.gif_engine else: self.context.request.engine = self.context.modules.engine raise gen.Return([False, buffer, None]) else: self.context.statsd_client.incr('storage.miss') buffer = yield self.context.modules.loader.load(self.context, url) if buffer is None: raise gen.Return([False, None, None]) if mime is None: mime = BaseEngine.get_mimetype(buffer) if mime is None: raise gen.Return([False, None, None]) self.context.request.extension = EXTENSION.get(mime, '.jpg') original_preserve = self.context.config.PRESERVE_EXIF_INFO self.context.config.PRESERVE_EXIF_INFO = True try: mime = BaseEngine.get_mimetype(buffer) self.context.request.extension = extension = EXTENSION.get(mime, None) if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE: self.context.request.engine = self.context.modules.gif_engine else: self.context.request.engine = self.context.modules.engine self.context.request.engine.load(buffer, extension) normalized = self.context.request.engine.normalize() is_no_storage = isinstance(storage, NoStorage) is_mixed_storage = isinstance(storage, MixedStorage) is_mixed_no_file_storage = is_mixed_storage and isinstance(storage.file_storage, NoStorage) if not (is_no_storage or is_mixed_no_file_storage): buffer = self.context.request.engine.read(extension) storage.put(url, buffer) storage.put_crypto(url) finally: self.context.config.PRESERVE_EXIF_INFO = original_preserve raise gen.Return([normalized, None, self.context.request.engine])
def _fetch(self, url): storage = self.context.modules.storage buffer = yield gen.maybe_future(storage.get(url)) mime = None if buffer is not None: self.context.statsd_client.incr('storage.hit') mime = BaseEngine.get_mimetype(buffer) self.context.request.extension = EXTENSION.get(mime, '.jpg') if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE: self.context.request.engine = self.context.modules.gif_engine else: self.context.request.engine = self.context.modules.engine raise gen.Return([False, buffer, None]) else: self.context.statsd_client.incr('storage.miss') buffer = yield self.context.modules.loader.load(self.context, url) if buffer is None: raise gen.Return([False, None, None]) if mime is None: mime = BaseEngine.get_mimetype(buffer) self.context.request.extension = EXTENSION.get(mime, '.jpg') original_preserve = self.context.config.PRESERVE_EXIF_INFO self.context.config.PRESERVE_EXIF_INFO = True try: mime = BaseEngine.get_mimetype(buffer) self.context.request.extension = extension = EXTENSION.get( mime, None) if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE: self.context.request.engine = self.context.modules.gif_engine else: self.context.request.engine = self.context.modules.engine self.context.request.engine.load(buffer, extension) normalized = self.context.request.engine.normalize() is_no_storage = isinstance(storage, NoStorage) is_mixed_storage = isinstance(storage, MixedStorage) is_mixed_no_file_storage = is_mixed_storage and isinstance( storage.file_storage, NoStorage) if not (is_no_storage or is_mixed_no_file_storage): buffer = self.context.request.engine.read(extension) storage.put(url, buffer) storage.put_crypto(url) finally: self.context.config.PRESERVE_EXIF_INFO = original_preserve raise gen.Return([normalized, None, self.context.request.engine])
def check_resource(self, id): id = id[:self.context.config.MAX_ID_LENGTH] # Check if image exists exists = yield gen.maybe_future( self.context.modules.storage.exists(id)) if exists: body = yield gen.maybe_future(self.context.modules.storage.get(id)) self.set_status(200) mime = BaseEngine.get_mimetype(body) if mime: self.set_header('Content-Type', mime) max_age = self.context.config.MAX_AGE if max_age: self.set_header('Cache-Control', 'max-age=' + str(max_age) + ',public') self.set_header( 'Expires', datetime.datetime.utcnow() + datetime.timedelta(seconds=max_age)) self.write(body) self.finish() else: self._error(404, 'Image not found at the given URL')
def _load_results(self, context): image_extension, content_type = self.define_image_type(context, None) quality = self.context.request.quality if quality is None: if image_extension == '.webp' and self.context.config.WEBP_QUALITY is not None: quality = self.context.config.get('WEBP_QUALITY') else: quality = self.context.config.QUALITY results = context.request.engine.read(image_extension, quality) if context.request.max_bytes is not None: results = self.reload_to_fit_in_kb( context.request.engine, results, image_extension, quality, context.request.max_bytes ) if not context.request.meta: results = self.optimize(context, image_extension, results) # An optimizer might have modified the image format. content_type = BaseEngine.get_mimetype(results) return results, content_type
def _get(self, key): stored = next( self.storage.find( { 'key': key, 'created_at': { '$gte': datetime.utcnow() - timedelta(seconds=self.get_max_age()) }, }, { 'file_id': True, 'created_at': True, 'metadata': True }).limit(1), None) if not stored: return None file_storage = gridfs.GridFS(self.database) contents = file_storage.get(stored['file_id']).read() metadata = stored['metadata'] metadata['LastModified'] = stored['created_at'].replace( tzinfo=pytz.utc) metadata['ContentLength'] = len(contents) metadata['ContentType'] = BaseEngine.get_mimetype(contents) result = ResultStorageResult(buffer=contents, metadata=metadata, successful=True) return result
def put(self, path, data, metadata={}, reduced_redundancy=False, encrypt_key=False, callback=None): """ Stores data at given path :param string path: Path or 'key' for created/updated object :param bytes data: Data to write :param dict metadata: Metadata to store with this data :param bool reduced_redundancy: Whether to reduce storage redundancy or not? :param bool encrypt_key: Encrypt data? :param callable callback: Called function once done """ storage_class = 'REDUCED_REDUNDANCY' if reduced_redundancy else 'STANDARD' content_type = BaseEngine.get_mimetype(data) or 'application/octet-stream' args = dict( callback=callback, Bucket=self._bucket, Key=self._clean_key(path), Body=data, ContentType=content_type, Metadata=metadata, StorageClass=storage_class, ) if encrypt_key: args['ServerSideEncryption'] = 'AES256' my_session = session_handler.get_session(self._endpoint is not None) session = Botocore(service='s3', region_name=self._region, operation='PutObject', session=my_session, endpoint_url=self._endpoint) session.call(**args)
def define_image_type(self, context, result): if result is not None: if isinstance(result, ResultStorageResult): buffer = result.buffer else: buffer = result image_extension = EXTENSION.get(BaseEngine.get_mimetype(buffer), '.jpg') else: image_extension = context.request.format if image_extension is not None: image_extension = '.%s' % image_extension logger.debug('Image format specified as %s.' % image_extension) elif self.is_webp(context): image_extension = '.webp' logger.debug('Image format set by AUTO_WEBP as %s.' % image_extension) elif self.can_auto_convert_png_to_jpg(): image_extension = '.jpg' logger.debug('Image format set by AUTO_PNG_TO_JPG as %s.' % image_extension) else: image_extension = context.request.engine.extension logger.debug('No image format specified. Retrieving from the image extension: %s.' % image_extension) content_type = CONTENT_TYPE.get(image_extension, CONTENT_TYPE['.jpg']) if context.request.meta: context.request.meta_callback = context.config.META_CALLBACK_NAME or self.request.arguments.get('callback', [None])[0] content_type = 'text/javascript' if context.request.meta_callback else 'application/json' logger.debug('Metadata requested. Serving content type of %s.' % content_type) logger.debug('Content Type of %s detected.' % content_type) return (image_extension, content_type)
def validate(self, body): conf = self.context.config mime = BaseEngine.get_mimetype(body) if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE: engine = self.context.modules.gif_engine else: engine = self.context.modules.engine # Check if image is valid try: engine.load(body, None) except IOError: self._error(415, 'Unsupported Media Type') return False # Check weight constraints if (conf.UPLOAD_MAX_SIZE != 0 and len(self.request.body) > conf.UPLOAD_MAX_SIZE): self._error( 412, 'Image exceed max weight (Expected : %s, Actual : %s)' % (conf.UPLOAD_MAX_SIZE, len(self.request.body))) return False # Check size constraints size = engine.size if (conf.MIN_WIDTH > size[0] or conf.MIN_HEIGHT > size[1]): self._error( 412, 'Image is too small (Expected: %s/%s , Actual : %s/%s) % (conf.MIN_WIDTH, conf.MIN_HEIGHT, size[0], size[1])') return False return True
def define_image_type(self, context, result): if result is not None: image_extension = BaseEngine.get_mimetype(result) elif context.config.AUTO_WEBP and context.request.accepts_webp and not context.modules.engine.is_multiple(): image_extension = ".webp" else: image_extension = context.request.format if image_extension is None: image_extension = context.modules.engine.extension logger.debug("No image format specified. Retrieving from the image extension: %s." % image_extension) else: image_extension = ".%s" % image_extension logger.debug("Image format specified as %s." % image_extension) content_type = CONTENT_TYPE.get(image_extension, CONTENT_TYPE[".jpg"]) if context.request.meta: context.request.meta_callback = ( context.config.META_CALLBACK_NAME or self.request.arguments.get("callback", [None])[0] ) content_type = "text/javascript" if context.request.meta_callback else "application/json" logger.debug("Metadata requested. Serving content type of %s." % content_type) logger.debug("Content Type of %s detected." % content_type) return image_extension, content_type
def get(self, callback): path = self.context.request.url file_abspath = self.normalize_path(path) if not self.validate_path(file_abspath): logger.warn("[RESULT_STORAGE] unable to read from outside root path: %s" % file_abspath) return None logger.debug("[RESULT_STORAGE] getting from %s" % file_abspath) if not exists(file_abspath) or self.is_expired(file_abspath): logger.debug("[RESULT_STORAGE] image not found at %s" % file_abspath) callback(None) else: with open(file_abspath, 'r') as f: buffer = f.read() result = ResultStorageResult( buffer=buffer, metadata={ 'LastModified': datetime.fromtimestamp(getmtime(file_abspath)).replace(tzinfo=pytz.utc), 'ContentLength': len(buffer), 'ContentType': BaseEngine.get_mimetype(buffer) } ) callback(result)
def test_auto_should_optimize_jpeg_to_jpeg(self): optimizer = AutoOptimizer(self.get_context()) temp = tempfile.NamedTemporaryFile() optimizer.optimize(None, fixtures_folder + '/img/bend.jpg', temp.name) temp_buffer = open(temp.name).read() self.assertTrue(BaseEngine.get_mimetype(temp_buffer) == 'image/jpeg', "MIME type should be image/jpeg")
def handle_loader_loaded(buffer): if buffer is None: callback(False, None) return original_preserve = self.context.config.PRESERVE_EXIF_INFO self.context.config.PRESERVE_EXIF_INFO = True try: mime = BaseEngine.get_mimetype(buffer) if mime == '.gif' and self.context.config.USE_GIFSICLE_ENGINE: self.context.request.engine = GifEngine(self.context) else: self.context.request.engine = self.context.modules.engine self.context.request.engine.load(buffer, extension) normalized = self.context.request.engine.normalize() is_no_storage = isinstance(storage, NoStorage) is_mixed_storage = isinstance(storage, MixedStorage) is_mixed_no_file_storage = is_mixed_storage and isinstance(storage.file_storage, NoStorage) if not (is_no_storage or is_mixed_no_file_storage): buffer = self.context.request.engine.read() storage.put(url, buffer) storage.put_crypto(url) finally: self.context.config.PRESERVE_EXIF_INFO = original_preserve callback(normalized, engine=self.context.request.engine)
def validate(self, body): conf = self.context.config mime = BaseEngine.get_mimetype(body) if mime == '.gif' and self.context.config.USE_GIFSICLE_ENGINE: engine = GifEngine(self.context) else: engine = self.context.modules.engine # Check if image is valid try: engine.load(body, None) except IOError: self._error(415, 'Unsupported Media Type') return False # Check weight constraints if (conf.UPLOAD_MAX_SIZE != 0 and len(self.request.body) > conf.UPLOAD_MAX_SIZE): self._error( 412, 'Image exceed max weight (Expected : %s, Actual : %s)' % (conf.UPLOAD_MAX_SIZE, len(self.request.body))) return False # Check size constraints size = engine.size if (conf.MIN_WIDTH > size[0] or conf.MIN_HEIGHT > size[1]): self._error( 412, 'Image is too small (Expected: %s/%s , Actual : %s/%s) % (conf.MIN_WIDTH, conf.MIN_HEIGHT, size[0], size[1])') return False return True
async def put(self, path, data, metadata={}, reduced_redundancy=False, encrypt_key=False): """ Stores data at given path :param string path: Path or 'key' for created/updated object :param bytes data: Data to write :param dict metadata: Metadata to store with this data :param bool reduced_redundancy: Whether to reduce storage redundancy or not? :param bool encrypt_key: Encrypt data? :param callable callback: Called function once done """ storage_class = 'REDUCED_REDUNDANCY' if reduced_redundancy else 'STANDARD' content_type = BaseEngine.get_mimetype(data) or 'application/octet-stream' args = dict( Bucket=self._bucket, Key=self._clean_key(path), Body=data, ContentType=content_type, Metadata=metadata, StorageClass=storage_class, ) if encrypt_key: args['ServerSideEncryption'] = 'AES256' return self._put_client.call(**args)
async def check_resource(self, file_id): file_id = file_id[:self.context.config.MAX_ID_LENGTH] # Check if image exists exists = await self.context.modules.storage.exists(file_id) if exists: body = await self.context.modules.storage.get(file_id) self.set_status(200) mime = BaseEngine.get_mimetype(body) if mime: self.set_header("Content-Type", mime) max_age = self.context.config.MAX_AGE if max_age: self.set_header("Cache-Control", "max-age=" + str(max_age) + ",public") self.set_header( "Expires", datetime.datetime.utcnow() + datetime.timedelta(seconds=max_age), ) self.write(body) self.finish() else: self._error(404, "Image not found at the given URL")
def get(self, callback): path = self.context.request.url file_abspath = self.normalize_path(path) if not self.validate_path(file_abspath): logger.warn( "[RESULT_STORAGE] unable to read from outside root path: %s" % file_abspath) return None logger.debug("[RESULT_STORAGE] getting from %s" % file_abspath) if not exists(file_abspath) or self.is_expired(file_abspath): logger.debug("[RESULT_STORAGE] image not found at %s" % file_abspath) callback(None) else: with open(file_abspath, 'r') as f: buffer = f.read() result = ResultStorageResult( buffer=buffer, metadata={ 'LastModified': datetime.fromtimestamp( getmtime(file_abspath)).replace(tzinfo=pytz.utc), 'ContentLength': len(buffer), 'ContentType': BaseEngine.get_mimetype(buffer) }) callback(result)
def _load_results(self, context): image_extension, content_type = self.define_image_type(context, None) quality = self.context.request.quality if quality is None: if (image_extension == ".webp" and self.context.config.WEBP_QUALITY is not None): quality = self.context.config.get("WEBP_QUALITY") else: quality = self.context.config.QUALITY results = context.request.engine.read(image_extension, quality) if context.request.max_bytes is not None: results = self.reload_to_fit_in_kb( context.request.engine, results, image_extension, quality, context.request.max_bytes, ) if not context.request.meta: results = self.optimize(context, image_extension, results) # An optimizer might have modified the image format. content_type = BaseEngine.get_mimetype(results) return results, content_type
def put(self, path, data, metadata={}, reduced_redundancy=False, encrypt_key=False, callback=None): """ Stores data at given path :param string path: Path or 'key' for created/updated object :param bytes data: Data to write :param dict metadata: Metadata to store with this data :param bool reduced_redundancy: Whether to reduce storage redundancy or not? :param bool encrypt_key: Encrypt data? :param callable callback: Called function once done """ storage_class = 'REDUCED_REDUNDANCY' if reduced_redundancy else 'STANDARD' content_type = BaseEngine.get_mimetype(data) or 'application/octet-stream' args = dict( callback=callback, Bucket=self._bucket, Key=self._clean_key(path), Body=data, ContentType=content_type, Metadata=metadata, StorageClass=storage_class, ) if encrypt_key: args['ServerSideEncryption'] = 'AES256' self._put_client.call(**args)
def execute_image_operations(self): self.context.request.quality = None req = self.context.request conf = self.context.config req.extension = splitext(req.image_url)[-1].lower() should_store = self.context.config.RESULT_STORAGE_STORES_UNSAFE or not self.context.request.unsafe if self.context.modules.result_storage and should_store: result = self.context.modules.result_storage.get() if result is not None: mime = BaseEngine.get_mimetype(result) if mime == '.gif' and self.context.config.USE_GIFSICLE_ENGINE: self.context.request.engine = GifEngine(self.context) else: self.context.request.engine = self.context.modules.engine logger.debug('[RESULT_STORAGE] IMAGE FOUND: %s' % req.url) self.finish_request(self.context, result) return if conf.MAX_WIDTH and (not isinstance(req.width, basestring)) and req.width > conf.MAX_WIDTH: req.width = conf.MAX_WIDTH if conf.MAX_HEIGHT and (not isinstance(req.height, basestring)) and req.height > conf.MAX_HEIGHT: req.height = conf.MAX_HEIGHT req.meta_callback = conf.META_CALLBACK_NAME or self.request.arguments.get('callback', [None])[0] self.filters_runner = self.context.filters_factory.create_instances(self.context, self.context.request.filters) self.filters_runner.apply_filters(thumbor.filters.PHASE_PRE_LOAD, self.get_image)
def get_engine(self, buffer, extension): mime = BaseEngine.get_mimetype(buffer) is_gif = extension == '.gif' is_webp = extension == '.webp' accepts_video = getattr(self.context.request, "accepts_video", False) accepts_webp = self.context.request.accepts_webp if is_webp and self.ffmpeg_handle_animated_webp and is_animated( buffer): return self.ffmpeg_engine elif is_gif and self.ffmpeg_handle_animated_gif and is_animated_gif( buffer): if self.context.config.FFMPEG_GIF_AUTO_H265: self.context.request.should_vary = True if accepts_video: logger.debug("FFMPEG_GIF_AUTO_H265 setting format to h264") self.context.request.format = 'h265' elif self.context.config.FFMPEG_GIF_AUTO_H264: self.context.request.should_vary = True if accepts_video: logger.debug("FFMPEG_GIF_AUTO_H264 setting format to h264") self.context.request.format = 'h264' elif self.context.config.FFMPEG_GIF_AUTO_WEBP: self.context.request.should_vary = True if accepts_webp: logger.debug("FFMPEG_GIF_AUTO_WEBP setting format to webp") self.context.request.format = 'webp' return self.ffmpeg_engine elif is_gif and self.use_gif_engine: return self.context.modules.gif_engine elif mime.startswith('video/'): return self.ffmpeg_engine else: return self.image_engine
def define_image_type(self, context, result): if result is not None: if isinstance(result, ResultStorageResult): buffer = result.buffer else: buffer = result image_extension = EXTENSION.get(BaseEngine.get_mimetype(buffer), ".jpg") else: image_extension = context.request.format if image_extension is not None: image_extension = ".%s" % image_extension logger.debug("Image format specified as %s." % image_extension) elif self.is_webp(context): image_extension = ".webp" logger.debug("Image format set by AUTO_WEBP as %s." % image_extension) else: image_extension = context.request.engine.extension logger.debug("No image format specified. Retrieving from the image extension: %s." % image_extension) content_type = CONTENT_TYPE.get(image_extension, CONTENT_TYPE[".jpg"]) if context.request.meta: context.request.meta_callback = ( context.config.META_CALLBACK_NAME or self.request.arguments.get("callback", [None])[0] ) content_type = "text/javascript" if context.request.meta_callback else "application/json" logger.debug("Metadata requested. Serving content type of %s." % content_type) logger.debug("Content Type of %s detected." % content_type) return (image_extension, content_type)
def define_image_type(self, context, result): if result is not None: image_extension = BaseEngine.get_mimetype(result) else: image_extension = context.request.format if image_extension is not None: image_extension = '.%s' % image_extension logger.debug('Image format specified as %s.' % image_extension) elif context.config.AUTO_WEBP and context.request.accepts_webp and not context.request.engine.is_multiple(): image_extension = '.webp' logger.debug('Image format set by AUTO_WEBP as %s.' % image_extension) else: image_extension = context.request.engine.extension logger.debug('No image format specified. Retrieving from the image extension: %s.' % image_extension) content_type = CONTENT_TYPE.get(image_extension, CONTENT_TYPE['.jpg']) if context.request.meta: context.request.meta_callback = context.config.META_CALLBACK_NAME or self.request.arguments.get('callback', [None])[0] content_type = 'text/javascript' if context.request.meta_callback else 'application/json' logger.debug('Metadata requested. Serving content type of %s.' % content_type) logger.debug('Content Type of %s detected.' % content_type) return image_extension, content_type
def test_still_filter(http_client, base_url, pos): response = yield http_client.fetch( "%s/unsafe/filters:still(%s)/hotdog.mp4" % (base_url, pos)) assert response.code == 200 assert response.headers.get('content-type') == 'image/jpeg' assert BaseEngine.get_mimetype(response.body) == 'image/jpeg'
def test_dispatch_to_video_engine(mocker, http_client, base_url): mocker.spy(FFmpegEngine, 'load') response = yield http_client.fetch("%s/unsafe/hotdog.mp4" % base_url) assert response.code == 200 assert FFmpegEngine.load.call_count == 1 assert BaseEngine.get_mimetype(response.body) == 'video/mp4'
def test_dispatch_to_image_engine(mocker, http_client, base_url): mocker.spy(PilEngine, 'load') response = yield http_client.fetch("%s/unsafe/filters:format(jpg)/hotdog.png" % base_url) assert response.code == 200 assert PilEngine.load.call_count == 1 assert BaseEngine.get_mimetype(response.body) == 'image/jpeg'
def test_still_filter_with_format(http_client, base_url, format, mime_type): response = yield http_client.fetch( "%s/unsafe/filters:still():format(%s)/hotdog.mp4" % (base_url, format)) assert response.code == 200 assert response.headers.get('content-type') == mime_type assert BaseEngine.get_mimetype(response.body) == mime_type
def mime(self): ''' Retrieves mime metadata if available :return: ''' return self.metadata[ 'ContentType'] if 'ContentType' in self.metadata else BaseEngine.get_mimetype( self.buffer)
async def get(self): path = self.context.request.url file_abspath = self.normalize_path(path) if not self.validate_path(file_abspath): logger.warning( "[RESULT_STORAGE] unable to read from outside root path: %s", file_abspath, ) return None logger.debug("[RESULT_STORAGE] getting from %s", file_abspath) if isdir(file_abspath): logger.warning( "[RESULT_STORAGE] cache location is a directory: %s", file_abspath) return None if not exists(file_abspath): legacy_path = self.normalize_path_legacy(path) if isfile(legacy_path): logger.debug( "[RESULT_STORAGE] migrating image from old location at %s", legacy_path, ) self.ensure_dir(dirname(file_abspath)) move(legacy_path, file_abspath) else: logger.debug("[RESULT_STORAGE] image not found at %s", file_abspath) return None expire_time = self.get_expire_time(file_abspath) if self.is_expired(file_abspath, expire_time): logger.debug("[RESULT_STORAGE] cached image has expired") return None with open(file_abspath, "rb") as image_file: buffer = image_file.read() if expire_time is not None: self.context.request.max_age = expire_time result = ResultStorageResult( buffer=buffer, metadata={ "LastModified": datetime.fromtimestamp( getmtime(file_abspath)).replace(tzinfo=pytz.utc), "ContentLength": len(buffer), "ContentType": BaseEngine.get_mimetype(buffer), }, ) return result
def execute_image_operations(self): self.context.request.quality = None req = self.context.request conf = self.context.config should_store = self.context.config.RESULT_STORAGE_STORES_UNSAFE or not self.context.request.unsafe if self.context.modules.result_storage and should_store: start = datetime.datetime.now() result = yield gen.maybe_future( self.context.modules.result_storage.get()) finish = datetime.datetime.now() self.context.metrics.timing('result_storage.incoming_time', (finish - start).total_seconds() * 1000) if result is None: self.context.metrics.incr('result_storage.miss') else: self.context.metrics.incr('result_storage.hit') self.context.metrics.incr('result_storage.bytes_read', len(result)) if result is not None: buffer = result.buffer if isinstance( result, ResultStorageResult) else result mime = BaseEngine.get_mimetype(buffer) if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE: self.context.request.engine = self.context.modules.gif_engine else: self.context.request.engine = self.context.modules.engine self.context.request.engine.load(buffer, EXTENSION.get(mime, '.jpg')) logger.debug('[RESULT_STORAGE] IMAGE FOUND: %s' % req.url) self.finish_request(self.context, result) return if conf.MAX_WIDTH and (not isinstance( req.width, basestring)) and req.width > conf.MAX_WIDTH: req.width = conf.MAX_WIDTH if conf.MAX_HEIGHT and (not isinstance( req.height, basestring)) and req.height > conf.MAX_HEIGHT: req.height = conf.MAX_HEIGHT req.meta_callback = conf.META_CALLBACK_NAME or self.request.arguments.get( 'callback', [None])[0] self.filters_runner = self.context.filters_factory.create_instances( self.context, self.context.request.filters) # Apply all the filters from the PRE_LOAD phase and call get_image() afterwards. self.filters_runner.apply_filters(thumbor.filters.PHASE_PRE_LOAD, self.get_image)
def mime(self): """ Retrieves mime metadata if available :return: """ return ( self.metadata["ContentType"] if "ContentType" in self.metadata else BaseEngine.get_mimetype(self.buffer) )
async def post(self): # Check if the image uploaded is a multipart/form-data if self.multipart_form_data(): file_data = self.request.files["media"][0] body = file_data["body"] # Retrieve filename from 'filename' field filename = file_data["filename"] else: body = self.request.body # Retrieve filename from 'Slug' header filename = self.request.headers.get("Slug") # Check if the image uploaded is valid if self.validate(body): # Use the default filename for the uploaded images if not filename: content_type = self.request.headers.get( "Content-Type", BaseEngine.get_mimetype(body)) extension = mimetypes.guess_extension( content_type.split(";", 1)[0], False) if extension is None: # Content-Type is unknown, try with body extension = mimetypes.guess_extension( BaseEngine.get_mimetype(body), False) if extension == ".jpe": extension = ( ".jpg" # Hack because mimetypes return .jpe by default ) if ( extension is None ): # Even body is unknown, return an empty string to be contat extension = "" filename = (self.context.config.UPLOAD_DEFAULT_FILENAME + extension) # Build image id based on a random uuid (32 characters) image_id = str(uuid.uuid4().hex) await self.write_file(image_id, body) self.set_status(201) self.set_header("Location", self.location(image_id, filename))
def test_s3_result_storage_save(mocker, config, http_client, base_url, auto_suffix, mime_type, s3_client): mocker.spy(Bucket, "put") response = yield http_client.fetch("%s/unsafe/hotdog.gif" % base_url, headers={'Accept': mime_type}) assert response.code == 200 bucket_key = "unsafe/hotdog.gif%s" % auto_suffix Bucket.put.assert_called_once() Bucket.put.mock_calls[0].args == (mocker.ANY, bucket_key, mocker.ANY) assert BaseEngine.get_mimetype(response.body) == mime_type assert response.headers.get("vary") == "Accept"
def _fetch(self, url, extension, callback): storage = self.context.modules.storage buffer = storage.get(url) if buffer is not None: self.context.statsd_client.incr('storage.hit') mime = BaseEngine.get_mimetype(buffer) if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE: self.context.request.engine = self.context.modules.gif_engine else: self.context.request.engine = self.context.modules.engine callback(False, buffer=buffer) else: self.context.statsd_client.incr('storage.miss') def handle_loader_loaded(buffer): if buffer is None: callback(False, None) return original_preserve = self.context.config.PRESERVE_EXIF_INFO self.context.config.PRESERVE_EXIF_INFO = True try: mime = BaseEngine.get_mimetype(buffer) extension = EXTENSION.get(mime,None) if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE: self.context.request.engine = self.context.modules.gif_engine else: self.context.request.engine = self.context.modules.engine self.context.request.engine.load(buffer, extension) normalized = self.context.request.engine.normalize() is_no_storage = isinstance(storage, NoStorage) is_mixed_storage = isinstance(storage, MixedStorage) is_mixed_no_file_storage = is_mixed_storage and isinstance(storage.file_storage, NoStorage) if not (is_no_storage or is_mixed_no_file_storage): buffer = self.context.request.engine.read(extension) storage.put(url, buffer) storage.put_crypto(url) finally: self.context.config.PRESERVE_EXIF_INFO = original_preserve callback(normalized, engine=self.context.request.engine) self.context.modules.loader.load(self.context, url, handle_loader_loaded)
def test_config_handle_animated_gif_false(mocker, config, http_client, base_url): config.FFMPEG_USE_GIFSICLE_ENGINE = True config.FFMPEG_HANDLE_ANIMATED_GIF = False mocker.spy(GifEngine, 'load') response = yield http_client.fetch("%s/unsafe/100x75/hotdog.gif" % base_url) assert response.code == 200 assert GifEngine.load.call_count == 1 assert BaseEngine.get_mimetype(response.body) == 'image/gif' im = Image.open(BytesIO(response.body)) assert im.size == (100, 75)
def define_image_type(self, context, result): if result is not None: if isinstance(result, ResultStorageResult): buffer = result.buffer else: buffer = result image_extension = EXTENSION.get(BaseEngine.get_mimetype(buffer), ".jpg") content_type = CONTENT_TYPE.get(image_extension, CONTENT_TYPE[".jpg"]) return image_extension, content_type image_extension = context.request.format if image_extension is not None: image_extension = f".{image_extension}" logger.debug("Image format specified as %s.", image_extension) elif self.is_webp(context): image_extension = ".webp" logger.debug("Image format set by AUTO_WEBP as %s.", image_extension) elif self.can_auto_convert_png_to_jpg(): image_extension = ".jpg" logger.debug( "Image format set by AUTO_PNG_TO_JPG as %s.", image_extension, ) else: image_extension = context.request.engine.extension logger.debug( "No image format specified. Retrieving " "from the image extension: %s.", image_extension, ) content_type = CONTENT_TYPE.get(image_extension, CONTENT_TYPE[".jpg"]) if context.request.meta: context.request.meta_callback = (context.config.META_CALLBACK_NAME or self.request.arguments.get( "callback", [None])[0]) content_type = ("text/javascript" if context.request.meta_callback else "application/json") logger.debug("Metadata requested. Serving content type of %s.", content_type) logger.debug("Content Type of %s detected.", content_type) return (image_extension, content_type)
def parse_image(self, image_buffer): tmp = BytesIO(image_buffer) result = BytesIO() img = PilImage.open(tmp) ext = BaseEngine.get_mimetype(image_buffer) try: img.load() except IOError: pass img.save(result, FORMATS.get(ext, FORMATS['.jpg'])) result_bytes = result.getvalue() result.close() tmp.close() return result_bytes
def get(self, callback): path = self.context.request.url file_abspath = self.normalize_path(path) if not self.validate_path(file_abspath): logger.warn( "[RESULT_STORAGE] unable to read from outside root path: %s" % file_abspath) return callback(None) logger.debug("[RESULT_STORAGE] getting from %s" % file_abspath) if isdir(file_abspath): logger.warn("[RESULT_STORAGE] cache location is a directory: %s" % file_abspath) return callback(None) if not exists(file_abspath): legacy_path = self.normalize_path_legacy(path) if isfile(legacy_path): logger.debug( "[RESULT_STORAGE] migrating image from old location at %s" % legacy_path) self.ensure_dir(dirname(file_abspath)) move(legacy_path, file_abspath) else: logger.debug("[RESULT_STORAGE] image not found at %s" % file_abspath) return callback(None) if self.is_expired(file_abspath): logger.debug("[RESULT_STORAGE] cached image has expired") return callback(None) with open(file_abspath, 'r') as f: buffer = f.read() result = ResultStorageResult( buffer=buffer, metadata={ 'LastModified': datetime.fromtimestamp( getmtime(file_abspath)).replace(tzinfo=pytz.utc), 'ContentLength': len(buffer), 'ContentType': BaseEngine.get_mimetype(buffer) }) callback(result)
def execute_image_operations(self): self.context.request.quality = None req = self.context.request conf = self.context.config req.extension = splitext(req.image_url)[-1].lower() should_store = self.context.config.RESULT_STORAGE_STORES_UNSAFE or not self.context.request.unsafe if self.context.modules.result_storage and should_store: start = datetime.datetime.now() result = self.context.modules.result_storage.get() finish = datetime.datetime.now() self.context.statsd_client.timing( 'result_storage.incoming_time', (finish - start).total_seconds() * 1000) if result is None: self.context.statsd_client.incr('result_storage.miss') else: self.context.statsd_client.incr('result_storage.hit') self.context.statsd_client.incr('result_storage.bytes_read', len(result)) if result is not None: mime = BaseEngine.get_mimetype(result) if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE: self.context.request.engine = self.context.modules.gif_engine self.context.request.engine.load(result, '.gif') else: self.context.request.engine = self.context.modules.engine logger.debug('[RESULT_STORAGE] IMAGE FOUND: %s' % req.url) self.finish_request(self.context, result) return if conf.MAX_WIDTH and (not isinstance( req.width, basestring)) and req.width > conf.MAX_WIDTH: req.width = conf.MAX_WIDTH if conf.MAX_HEIGHT and (not isinstance( req.height, basestring)) and req.height > conf.MAX_HEIGHT: req.height = conf.MAX_HEIGHT req.meta_callback = conf.META_CALLBACK_NAME or self.request.arguments.get( 'callback', [None])[0] self.filters_runner = self.context.filters_factory.create_instances( self.context, self.context.request.filters) self.filters_runner.apply_filters(thumbor.filters.PHASE_PRE_LOAD, self.get_image)
def execute_image_operations(self): self.context.request.quality = None req = self.context.request conf = self.context.config should_store = self.context.config.RESULT_STORAGE_STORES_UNSAFE or not self.context.request.unsafe if self.context.modules.result_storage and should_store: start = datetime.datetime.now() result = yield gen.maybe_future(self.context.modules.result_storage.get()) finish = datetime.datetime.now() self.context.metrics.timing('result_storage.incoming_time', (finish - start).total_seconds() * 1000) if result is None: self.context.metrics.incr('result_storage.miss') else: self.context.metrics.incr('result_storage.hit') self.context.metrics.incr('result_storage.bytes_read', len(result)) if result is not None: buffer = result.buffer if isinstance(result, ResultStorageResult) else result mime = BaseEngine.get_mimetype(buffer) if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE: self.context.request.engine = self.context.modules.gif_engine else: self.context.request.engine = self.context.modules.engine self.context.request.engine.load(buffer, EXTENSION.get(mime, '.jpg')) logger.debug('[RESULT_STORAGE] IMAGE FOUND: %s' % req.url) self.finish_request(self.context, result) return if conf.MAX_WIDTH and (not isinstance(req.width, basestring)) and req.width > conf.MAX_WIDTH: req.width = conf.MAX_WIDTH if conf.MAX_HEIGHT and (not isinstance(req.height, basestring)) and req.height > conf.MAX_HEIGHT: req.height = conf.MAX_HEIGHT req.meta_callback = conf.META_CALLBACK_NAME or self.request.arguments.get('callback', [None])[0] self.filters_runner = self.context.filters_factory.create_instances(self.context, self.context.request.filters) # Apply all the filters from the PRE_LOAD phase and call get_image() afterwards. self.filters_runner.apply_filters(thumbor.filters.PHASE_PRE_LOAD, self.get_image)
def get(self, id): id = id[:32] # Check if image exists if self.context.modules.storage.exists(id): body = self.context.modules.storage.get(id) self.set_status(200) mime = BaseEngine.get_mimetype(body) if mime: self.set_header('Content-Type', mime) max_age = self.context.config.MAX_AGE if max_age: self.set_header('Cache-Control', 'max-age=' + str(max_age) + ',public') self.set_header('Expires', datetime.datetime.utcnow() + datetime.timedelta(seconds=max_age)) self.write(body) else: self._error(404, 'Image not found at the given URL')
def put(self, bytes): file_abspath = self.normalize_path(self.context.request.url) logger.debug("[RESULT_STORAGE] putting at %s" % file_abspath) bucket = self.get_bucket() blob = bucket.blob(file_abspath) blob.upload_from_string(bytes) max_age = self.context.config.MAX_AGE blob.cache_control = "public,max-age=%s" % max_age if bytes: try: mime = BaseEngine.get_mimetype(bytes) blob.content_type = mime except: logger.debug("[RESULT_STORAGE] Couldn't determine mimetype") blob.patch()
def get(self, callback): path = self.context.request.url file_abspath = self.normalize_path(path) if not self.validate_path(file_abspath): logger.warn("[RESULT_STORAGE] unable to read from outside root path: %s" % file_abspath) return callback(None) logger.debug("[RESULT_STORAGE] getting from %s" % file_abspath) if isdir(file_abspath): logger.warn("[RESULT_STORAGE] cache location is a directory: %s" % file_abspath) return callback(None) if not exists(file_abspath): legacy_path = self.normalize_path_legacy(path) if isfile(legacy_path): logger.debug("[RESULT_STORAGE] migrating image from old location at %s" % legacy_path) self.ensure_dir(dirname(file_abspath)) move(legacy_path, file_abspath) else: logger.debug("[RESULT_STORAGE] image not found at %s" % file_abspath) return callback(None) if self.is_expired(file_abspath): logger.debug("[RESULT_STORAGE] cached image has expired") return callback(None) with open(file_abspath, 'r') as f: buffer = f.read() result = ResultStorageResult( buffer=buffer, metadata={ 'LastModified': datetime.fromtimestamp(getmtime(file_abspath)).replace(tzinfo=pytz.utc), 'ContentLength': len(buffer), 'ContentType': BaseEngine.get_mimetype(buffer) } ) callback(result)
def execute_image_operations(self): self.context.request.quality = None req = self.context.request conf = self.context.config req.extension = splitext(req.image_url)[-1].lower() should_store = self.context.config.RESULT_STORAGE_STORES_UNSAFE or not self.context.request.unsafe if self.context.modules.result_storage and should_store: start = datetime.datetime.now() result = self.context.modules.result_storage.get() finish = datetime.datetime.now() self.context.statsd_client.timing('result_storage.incoming_time', (finish - start).total_seconds() * 1000) if result is None: self.context.statsd_client.incr('result_storage.miss') else: self.context.statsd_client.incr('result_storage.hit') self.context.statsd_client.incr('result_storage.bytes_read', len(result)) if result is not None: mime = BaseEngine.get_mimetype(result) if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE: self.context.request.engine = self.context.modules.gif_engine self.context.request.engine.load(result, '.gif') else: self.context.request.engine = self.context.modules.engine logger.debug('[RESULT_STORAGE] IMAGE FOUND: %s' % req.url) self.finish_request(self.context, result) return if conf.MAX_WIDTH and (not isinstance(req.width, basestring)) and req.width > conf.MAX_WIDTH: req.width = conf.MAX_WIDTH if conf.MAX_HEIGHT and (not isinstance(req.height, basestring)) and req.height > conf.MAX_HEIGHT: req.height = conf.MAX_HEIGHT req.meta_callback = conf.META_CALLBACK_NAME or self.request.arguments.get('callback', [None])[0] self.filters_runner = self.context.filters_factory.create_instances(self.context, self.context.request.filters) self.filters_runner.apply_filters(thumbor.filters.PHASE_PRE_LOAD, self.get_image)
def check_resource(self, id): id = id[:self.context.config.MAX_ID_LENGTH] # Check if image exists exists = yield gen.maybe_future(self.context.modules.storage.exists(id)) if exists: body = yield gen.maybe_future(self.context.modules.storage.get(id)) self.set_status(200) mime = BaseEngine.get_mimetype(body) if mime: self.set_header('Content-Type', mime) max_age = self.context.config.MAX_AGE if max_age: self.set_header('Cache-Control', 'max-age=' + str(max_age) + ',public') self.set_header('Expires', datetime.datetime.utcnow() + datetime.timedelta(seconds=max_age)) self.write(body) self.finish() else: self._error(404, 'Image not found at the given URL')
def _fetch(self, url): """ :param url: :type url: :return: :rtype: """ fetch_result = FetchResult() storage = self.context.modules.storage yield self.acquire_url_lock(url) try: fetch_result.buffer = yield gen.maybe_future(storage.get(url)) mime = None if fetch_result.buffer is not None: self.release_url_lock(url) fetch_result.successful = True self.context.metrics.incr('storage.hit') mime = BaseEngine.get_mimetype(fetch_result.buffer) self.context.request.extension = EXTENSION.get(mime, '.jpg') if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE: self.context.request.engine = self.context.modules.gif_engine else: self.context.request.engine = self.context.modules.engine raise gen.Return(fetch_result) else: self.context.metrics.incr('storage.miss') loader_result = yield self.context.modules.loader.load(self.context, url) finally: self.release_url_lock(url) if isinstance(loader_result, LoaderResult): # TODO _fetch should probably return a result object vs a list to # to allow returning metadata if not loader_result.successful: fetch_result.buffer = None fetch_result.loader_error = loader_result.error raise gen.Return(fetch_result) fetch_result.buffer = loader_result.buffer else: # Handle old loaders fetch_result.buffer = loader_result if fetch_result.buffer is None: raise gen.Return(fetch_result) fetch_result.successful = True if mime is None: mime = BaseEngine.get_mimetype(fetch_result.buffer) self.context.request.extension = extension = EXTENSION.get(mime, '.jpg') original_preserve = self.context.config.PRESERVE_EXIF_INFO try: if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE: self.context.request.engine = self.context.modules.gif_engine else: self.context.request.engine = self.context.modules.engine self.context.request.engine.load(fetch_result.buffer, extension) if self.context.request.engine.image is None: fetch_result.successful = False fetch_result.buffer = None fetch_result.engine = self.context.request.engine fetch_result.engine_error = EngineResult.COULD_NOT_LOAD_IMAGE raise gen.Return(fetch_result) fetch_result.normalized = self.context.request.engine.normalize() # Allows engine or loader to override storage on the fly for the purpose of # marking a specific file as unstoreable storage = self.context.modules.storage is_no_storage = isinstance(storage, NoStorage) is_mixed_storage = isinstance(storage, MixedStorage) is_mixed_no_file_storage = is_mixed_storage and isinstance(storage.file_storage, NoStorage) if not (is_no_storage or is_mixed_no_file_storage): # in the case that we have local storage, it's important that we don't try # and strip the exif data - so that we can make the choice about stripping # it on output, instead of input. # unfortunately, the only interface to the engine is the global config # referenced by our context, which has the current setting of preserving # the exif info or not. this is also being read by concurrent # coroutine-threads, so just setting this value outright causes a race. # we solve this in order not to influence these other coroutine-threads # by doing an effective copy-on-write for the config until the end of this # context (ie. this request). # at some point in the future - the API should change, and this horrid hack # should go away. self.context.config = copy.deepcopy(self.context.config) self.context.config.PRESERVE_EXIF_INFO = True fetch_result.buffer = self.context.request.engine.read(extension) storage.put(url, fetch_result.buffer) storage.put_crypto(url) except Exception: fetch_result.successful = False finally: self.context.config.PRESERVE_EXIF_INFO = original_preserve if not fetch_result.successful: raise fetch_result.buffer = None fetch_result.engine = self.context.request.engine raise gen.Return(fetch_result)
def _fetch(self, url): """ :param url: :type url: :return: :rtype: """ fetch_result = FetchResult() storage = self.context.modules.storage yield self.acquire_url_lock(url) try: fetch_result.buffer = yield gen.maybe_future(storage.get(url)) mime = None if fetch_result.buffer is not None: self.release_url_lock(url) fetch_result.successful = True self.context.metrics.incr('storage.hit') mime = BaseEngine.get_mimetype(fetch_result.buffer) self.context.request.extension = EXTENSION.get(mime, '.jpg') if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE: self.context.request.engine = self.context.modules.gif_engine else: self.context.request.engine = self.context.modules.engine raise gen.Return(fetch_result) else: self.context.metrics.incr('storage.miss') loader_result = yield self.context.modules.loader.load(self.context, url) finally: self.release_url_lock(url) if isinstance(loader_result, LoaderResult): # TODO _fetch should probably return a result object vs a list to # to allow returning metadata if not loader_result.successful: fetch_result.buffer = None fetch_result.loader_error = loader_result.error raise gen.Return(fetch_result) fetch_result.buffer = loader_result.buffer else: # Handle old loaders fetch_result.buffer = loader_result if fetch_result.buffer is None: raise gen.Return(fetch_result) fetch_result.successful = True if mime is None: mime = BaseEngine.get_mimetype(fetch_result.buffer) self.context.request.extension = extension = EXTENSION.get(mime, '.jpg') try: if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE: self.context.request.engine = self.context.modules.gif_engine else: self.context.request.engine = self.context.modules.engine self.context.request.engine.load(fetch_result.buffer, extension) if self.context.request.engine.image is None: fetch_result.successful = False fetch_result.buffer = None fetch_result.engine = self.context.request.engine fetch_result.engine_error = EngineResult.COULD_NOT_LOAD_IMAGE raise gen.Return(fetch_result) fetch_result.normalized = self.context.request.engine.normalize() # Allows engine or loader to override storage on the fly for the purpose of # marking a specific file as unstoreable storage = self.context.modules.storage is_no_storage = isinstance(storage, NoStorage) is_mixed_storage = isinstance(storage, MixedStorage) is_mixed_no_file_storage = is_mixed_storage and isinstance(storage.file_storage, NoStorage) if not (is_no_storage or is_mixed_no_file_storage): storage.put(url, fetch_result.buffer) storage.put_crypto(url) except Exception: fetch_result.successful = False finally: if not fetch_result.successful: raise fetch_result.buffer = None fetch_result.engine = self.context.request.engine raise gen.Return(fetch_result)
def mime(self): ''' Retrieves mime metadata if available :return: ''' return self.metadata['ContentType'] if 'ContentType' in self.metadata else BaseEngine.get_mimetype(self.buffer)
def mime(self): """ Retrieves mime metadata if available :return: """ return self.metadata["ContentType"] if "ContentType" in self.metadata else BaseEngine.get_mimetype(self.buffer)
class BaseEngineTestCase(TestCase): def get_context(self): cfg = Config( SECURITY_KEY='ACME-SEC', ENGINE='thumbor.engines', ) cfg.STORAGE = 'thumbor.storages.no_storage' return Context(config=cfg) def flip_horizontally(self): ((a, b), (c, d)) = self.image self.image = ( (b, a), (d, c) ) def flip_vertically(self): ((a, b), (c, d)) = self.image self.image = ( (c, d), (a, b) ) def rotate(self, value): ((a, b), (c, d)) = self.image if value == 270: self.image = ( (c, a), (d, b) ) elif value == 180: self.image = ( (d, c), (b, a) ) elif value == 90: self.image = ( (b, d), (a, c) ) def setUp(self): self.image = ( (1, 2), (3, 4) ) self.context = self.get_context() self.engine = BaseEngine(self.context) self.engine.flip_horizontally = mock.MagicMock() self.engine.flip_horizontally.side_effect = self.flip_horizontally self.engine.flip_vertically = mock.MagicMock() self.engine.flip_vertically.side_effect = self.flip_vertically self.engine.rotate = mock.MagicMock() self.engine.rotate.side_effect = self.rotate def test_create_engine(self): expect(self.engine).to_be_instance_of(BaseEngine) def test_convert_svg_to_png(self): buffer = """<svg width="10px" height="20px" viewBox="0 0 10 20" xmlns="http://www.w3.org/2000/svg"> <rect width="100%" height="10" x="0" y="0"/> </svg>""" self.engine.convert_svg_to_png(buffer) expect(self.engine.extension).to_equal('.png') def test_convert_svg_with_xml_preamble_to_png(self): buffer = """<?xml version="1.0" encoding="utf-8"?> <svg width="10px" height="20px" viewBox="0 0 10 20" xmlns="http://www.w3.org/2000/svg"> <rect width="100%" height="10" x="0" y="0"/> </svg>""".encode('utf-8') self.engine.convert_svg_to_png(buffer) expect(self.engine.extension).to_equal('.png') def test_convert_svg_utf16_to_png(self): buffer = """<?xml version="1.0" encoding="utf-16"?> <svg width="10px" height="20px" viewBox="0 0 10 20" xmlns="http://www.w3.org/2000/svg"> <rect width="100%" height="10" x="0" y="0"/> </svg>""".encode('utf-16') self.engine.convert_svg_to_png(buffer) expect(self.engine.extension).to_equal('.png') @mock.patch('thumbor.engines.cairosvg', new=None) @mock.patch('thumbor.engines.logger.error') def test_not_imported_cairosvg_failed_to_convert_svg_to_png(self, mockLogError): buffer = """<svg width="10px" height="20px" viewBox="0 0 10 20" xmlns="http://www.w3.org/2000/svg"> <rect width="100%" height="10" x="0" y="0"/> </svg>""" returned_buffer = self.engine.convert_svg_to_png(buffer) expect(mockLogError.called).to_be_true() expect(buffer).to_equal(returned_buffer) def test_can_identify_msb_tiff(self): with open(join(STORAGE_PATH, 'gradient_msb_16bperchannel.tif'), 'r') as im: buffer = im.read() mime = self.engine.get_mimetype(buffer) expect(mime).to_equal('image/tiff') def test_can_identify_lsb_tiff(self): with open(join(STORAGE_PATH, 'gradient_lsb_16bperchannel.tif'), 'r') as im: buffer = im.read() mime = self.engine.get_mimetype(buffer) expect(mime).to_equal('image/tiff') def test_get_orientation(self): self.engine.exif = exif_str(1) expect(self.engine.get_orientation()).to_equal(1) expect(self.engine.get_orientation()).to_equal(1) self.engine.exif = exif_str(6) expect(self.engine.get_orientation()).to_equal(6) expect(self.engine.get_orientation()).to_equal(6) self.engine.exif = exif_str(8) expect(self.engine.get_orientation()).to_equal(8) expect(self.engine.get_orientation()).to_equal(8) def test_reorientate1(self): # No rotation self.engine.exif = exif_str(1) self.engine.reorientate() expect(self.engine.rotate.called).to_be_false() expect(self.engine.flip_horizontally.called).to_be_false() expect(self.engine.flip_vertically.called).to_be_false() expect(self.image).to_equal(((1, 2), (3, 4))) expect(self.engine.get_orientation()).to_equal(1) def test_reorientate2(self): self.image = ( (2, 1), (4, 3) ) # Flipped horizontally self.engine.exif = exif_str(2) self.engine.reorientate() expect(self.engine.rotate.called).to_be_false() expect(self.engine.flip_horizontally.called).to_be_true() expect(self.engine.flip_vertically.called).to_be_false() expect(self.image).to_equal(((1, 2), (3, 4))) expect(self.engine.get_orientation()).to_equal(1) def test_reorientate3(self): # Rotated 180° Ⅎ self.image = ( (4, 3), (2, 1) ) self.engine.exif = exif_str(3) self.engine.reorientate() expect(self.engine.rotate.call_args[0]).to_equal((180,)) expect(self.engine.flip_horizontally.called).to_be_false() expect(self.engine.flip_vertically.called).to_be_false() expect(self.image).to_equal(((1, 2), (3, 4))) expect(self.engine.get_orientation()).to_equal(1) def test_reorientate4(self): # Flipped vertically self.image = ( (3, 4), (1, 2) ) self.engine.exif = exif_str(4) self.engine.reorientate() expect(self.engine.rotate.called).to_be_false() expect(self.engine.flip_horizontally.called).to_be_false() expect(self.engine.flip_vertically.called).to_be_true() expect(self.image).to_equal(((1, 2), (3, 4))) expect(self.engine.get_orientation()).to_equal(1) def test_reorientate5(self): # Horizontal Mirror + Rotation 270 # or Vertical Mirror + Rotation 90 self.image = ( (1, 3), (2, 4) ) self.engine.exif = exif_str(5) self.engine.reorientate() expect(self.image).to_equal(((1, 2), (3, 4))) expect(self.engine.get_orientation()).to_equal(1) def test_reorientate6(self): # Rotate 270° self.image = ( (2, 4), (1, 3) ) self.engine.exif = exif_str(6) self.engine.reorientate() expect(self.engine.rotate.call_args[0]).to_equal((270,)) expect(self.engine.flip_horizontally.called).to_be_false() expect(self.engine.flip_vertically.called).to_be_false() expect(self.image).to_equal(((1, 2), (3, 4))) expect(self.engine.get_orientation()).to_equal(1) def test_reorientate7(self): # Flipped horizontally and rotate 90° self.image = ( (4, 2), (3, 1) ) self.engine.exif = exif_str(7) self.engine.reorientate() expect(self.image).to_equal(((1, 2), (3, 4))) expect(self.engine.get_orientation()).to_equal(1) def test_reorientate8(self): # Rotate 90° self.image = ( (3, 1), (4, 2) ) self.engine.exif = exif_str(8) self.engine.reorientate() expect(self.engine.rotate.call_args[0]).to_equal((90,)) expect(self.engine.flip_horizontally.called).to_be_false() expect(self.engine.flip_vertically.called).to_be_false() expect(self.image).to_equal(((1, 2), (3, 4))) expect(self.engine.get_orientation()).to_equal(1)
def _fetch(self, url): fetch_result = FetchResult() storage = self.context.modules.storage fetch_result.buffer = yield gen.maybe_future(storage.get(url)) mime = None if fetch_result.buffer is not None: fetch_result.successful = True self.context.metrics.incr('storage.hit') mime = BaseEngine.get_mimetype(fetch_result.buffer) self.context.request.extension = EXTENSION.get(mime, '.jpg') if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE: self.context.request.engine = self.context.modules.gif_engine else: self.context.request.engine = self.context.modules.engine raise gen.Return(fetch_result) else: self.context.metrics.incr('storage.miss') loader_result = yield self.context.modules.loader.load(self.context, url) if isinstance(loader_result, LoaderResult): # TODO _fetch should probably return a result object vs a list to # to allow returning metadata if not loader_result.successful: fetch_result.buffer = None fetch_result.loader_error = loader_result.error raise gen.Return(fetch_result) fetch_result.buffer = loader_result.buffer else: # Handle old loaders fetch_result.buffer = loader_result if fetch_result.buffer is None: raise gen.Return(fetch_result) fetch_result.successful = True if mime is None: mime = BaseEngine.get_mimetype(fetch_result.buffer) self.context.request.extension = EXTENSION.get(mime, '.jpg') original_preserve = self.context.config.PRESERVE_EXIF_INFO self.context.config.PRESERVE_EXIF_INFO = True try: mime = BaseEngine.get_mimetype(fetch_result.buffer) self.context.request.extension = extension = EXTENSION.get(mime, None) if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE: self.context.request.engine = self.context.modules.gif_engine else: self.context.request.engine = self.context.modules.engine self.context.request.engine.load(fetch_result.buffer, extension) fetch_result.normalized = self.context.request.engine.normalize() is_no_storage = isinstance(storage, NoStorage) is_mixed_storage = isinstance(storage, MixedStorage) is_mixed_no_file_storage = is_mixed_storage and isinstance(storage.file_storage, NoStorage) if not (is_no_storage or is_mixed_no_file_storage): fetch_result.buffer = self.context.request.engine.read(extension) storage.put(url, fetch_result.buffer) storage.put_crypto(url) finally: self.context.config.PRESERVE_EXIF_INFO = original_preserve fetch_result.buffer = None fetch_result.engine = self.context.request.engine raise gen.Return(fetch_result)
class BaseEngineTestCase(TestCase): def get_context(self): cfg = Config( SECURITY_KEY='ACME-SEC', ENGINE='thumbor.engines', ) cfg.STORAGE = 'thumbor.storages.no_storage' return Context(config=cfg) def flip_horizontally(self): ((a, b), (c, d)) = self.image self.image = ( (b, a), (d, c) ) def flip_vertically(self): ((a, b), (c, d)) = self.image self.image = ( (c, d), (a, b) ) def rotate(self, value): ((a, b), (c, d)) = self.image if value == 270: self.image = ( (c, a), (d, b) ) elif value == 180: self.image = ( (d, c), (b, a) ) elif value == 90: self.image = ( (b, d), (a, c) ) def setUp(self): self.image = ( (1, 2), (3, 4) ) self.context = self.get_context() self.engine = BaseEngine(self.context) self.engine.flip_horizontally = mock.MagicMock() self.engine.flip_horizontally.side_effect = self.flip_horizontally self.engine.flip_vertically = mock.MagicMock() self.engine.flip_vertically.side_effect = self.flip_vertically self.engine.rotate = mock.MagicMock() self.engine.rotate.side_effect = self.rotate def test_create_engine(self): expect(self.engine).to_be_instance_of(BaseEngine) def test_convert_svg_to_png(self): buffer = """<svg width="10px" height="20px" viewBox="0 0 10 20" xmlns="http://www.w3.org/2000/svg"> <rect width="100%" height="10" x="0" y="0"/> </svg>""" self.engine.convert_svg_to_png(buffer) expect(self.engine.extension).to_equal('.png') def test_convert_svg_with_xml_preamble_to_png(self): buffer = """<?xml version="1.0" encoding="utf-8"?> <svg width="10px" height="20px" viewBox="0 0 10 20" xmlns="http://www.w3.org/2000/svg"> <rect width="100%" height="10" x="0" y="0"/> </svg>""".encode('utf-8') self.engine.convert_svg_to_png(buffer) expect(self.engine.extension).to_equal('.png') def test_convert_svg_utf16_to_png(self): buffer = """<?xml version="1.0" encoding="utf-16"?> <svg width="10px" height="20px" viewBox="0 0 10 20" xmlns="http://www.w3.org/2000/svg"> <rect width="100%" height="10" x="0" y="0"/> </svg>""".encode('utf-16') self.engine.convert_svg_to_png(buffer) expect(self.engine.extension).to_equal('.png') @mock.patch('thumbor.engines.cairosvg', new=None) @mock.patch('thumbor.engines.logger.error') def test_not_imported_cairosvg_failed_to_convert_svg_to_png(self, mockLogError): buffer = """<svg width="10px" height="20px" viewBox="0 0 10 20" xmlns="http://www.w3.org/2000/svg"> <rect width="100%" height="10" x="0" y="0"/> </svg>""" returned_buffer = self.engine.convert_svg_to_png(buffer) expect(mockLogError.called).to_be_true() expect(buffer).to_equal(returned_buffer) def test_can_identify_msb_tiff(self): with open(join(STORAGE_PATH, 'gradient_msb_16bperchannel.tif'), 'r') as im: buffer = im.read() mime = self.engine.get_mimetype(buffer) expect(mime).to_equal('image/tiff') def test_can_identify_lsb_tiff(self): with open(join(STORAGE_PATH, 'gradient_lsb_16bperchannel.tif'), 'r') as im: buffer = im.read() mime = self.engine.get_mimetype(buffer) expect(mime).to_equal('image/tiff') def test_can_identify_svg_with_xml_namespace_other_than_w3(self): buffer = """<svg width="10px" height="20px" viewBox="0 0 10 20" xmlns="http://ns.foo.com/FooSVGViewerExtensions/3.0/"> <rect width="100%" height="10" x="0" y="0"/> </svg>""".encode('utf-8') mime = self.engine.get_mimetype(buffer) expect(mime).to_equal('image/svg+xml') def test_can_identify_svg_with_xml_preamble_and_lots_of_gibberish(self): buffer = """<?xml version="1.0" encoding="utf-8"?> <!-- Generator: Proprietary Drawing Software, SVG Export Plug-In. SVG Version: 3.0.0 Build 77) --> <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN" "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd" [ <!ENTITY ns_flows "http://ns.foo.com/Flows/1.0/"> <!ENTITY ns_extend "http://ns.foo.com/Extensibility/1.0/"> <!ENTITY ns_ai "http://ns.foo.com/foobar/10.0/"> <!ENTITY ns_graphs "http://ns.foo.com/Graphs/1.0/"> <!ENTITY ns_vars "http://ns.foo.com/Variables/1.0/"> <!ENTITY ns_imrep "http://ns.foo.com/ImageReplacement/1.0/"> <!ENTITY ns_sfw "http://ns.foo.com/SaveForWeb/1.0/"> <!ENTITY ns_custom "http://ns.foo.com/GenericCustomNamespace/1.0/"> <!ENTITY ns_foo_xpath "http://ns.foo.com/XPath/1.0/"> <!ENTITY ns_svg "http://www.w3.org/2000/svg"> <!ENTITY ns_xlink "http://www.w3.org/1999/xlink"> ]> <svg width="10px" height="20px" viewBox="0 0 10 20" xmlns="http://www.w3.org/2000/svg"> <rect width="100%" height="10" x="0" y="0"/> </svg>""".encode('utf-8') mime = self.engine.get_mimetype(buffer) expect(mime).to_equal('image/svg+xml') def test_convert_svg_already_converted_to_png(self): svg_buffer = """<svg width="10px" height="20px" viewBox="0 0 10 20" xmlns="http://www.w3.org/2000/svg"> <rect width="100%" height="10" x="0" y="0"/> </svg>""" png_buffer = self.engine.convert_svg_to_png(svg_buffer) png_buffer_dupe = self.engine.convert_svg_to_png(png_buffer) expect(self.engine.extension).to_equal('.png') expect(png_buffer).to_equal(png_buffer_dupe) def test_convert_not_well_formed_svg_to_png(self): buffer = """<<svg width="10px" height="20px" viewBox="0 0 10 20" xmlns="http://www.w3.org/2000/svg"> <rect width="100%" height="10" x="0" y="0"/> </svg>""".encode('utf-8') with expect.error_to_happen(ParseError): self.engine.convert_svg_to_png(buffer) expect(self.engine.extension).to_be_null() def test_get_orientation_no_exif(self): expect(hasattr(self.engine, 'exif')).to_be_false() expect(self.engine.get_orientation()).to_be_null() def test_get_orientation_null_exif(self): self.engine.exif = None expect(self.engine.get_orientation()).to_be_null() def test_get_orientation_without_orientation_in_exif(self): self.engine.exif = piexif.load(exif_str(1)) self.engine.exif['0th'].pop(piexif.ImageIFD.Orientation, None) expect(self.engine.get_orientation()).to_be_null() def test_get_orientation(self): self.engine.exif = exif_str(1) expect(self.engine.get_orientation()).to_equal(1) expect(self.engine.get_orientation()).to_equal(1) self.engine.exif = exif_str(6) expect(self.engine.get_orientation()).to_equal(6) expect(self.engine.get_orientation()).to_equal(6) self.engine.exif = exif_str(8) expect(self.engine.get_orientation()).to_equal(8) expect(self.engine.get_orientation()).to_equal(8) def test_reorientate_no_exif(self): expect(hasattr(self.engine, 'exif')).to_be_false() self.engine.reorientate() expect(self.engine.get_orientation()).to_be_null() def test_reorientate_null_exif(self): self.engine.exif = None self.engine.reorientate() expect(self.engine.get_orientation()).to_be_null() def test_reorientate1(self): # No rotation self.engine.exif = exif_str(1) self.engine.reorientate() expect(self.engine.rotate.called).to_be_false() expect(self.engine.flip_horizontally.called).to_be_false() expect(self.engine.flip_vertically.called).to_be_false() expect(self.image).to_equal(((1, 2), (3, 4))) expect(self.engine.get_orientation()).to_equal(1) def test_reorientate2(self): self.image = ( (2, 1), (4, 3) ) # Flipped horizontally self.engine.exif = exif_str(2) self.engine.reorientate() expect(self.engine.rotate.called).to_be_false() expect(self.engine.flip_horizontally.called).to_be_true() expect(self.engine.flip_vertically.called).to_be_false() expect(self.image).to_equal(((1, 2), (3, 4))) expect(self.engine.get_orientation()).to_equal(1) def test_reorientate3(self): # Rotated 180° Ⅎ self.image = ( (4, 3), (2, 1) ) self.engine.exif = exif_str(3) self.engine.reorientate() expect(self.engine.rotate.call_args[0]).to_equal((180,)) expect(self.engine.flip_horizontally.called).to_be_false() expect(self.engine.flip_vertically.called).to_be_false() expect(self.image).to_equal(((1, 2), (3, 4))) expect(self.engine.get_orientation()).to_equal(1) def test_reorientate4(self): # Flipped vertically self.image = ( (3, 4), (1, 2) ) self.engine.exif = exif_str(4) self.engine.reorientate() expect(self.engine.rotate.called).to_be_false() expect(self.engine.flip_horizontally.called).to_be_false() expect(self.engine.flip_vertically.called).to_be_true() expect(self.image).to_equal(((1, 2), (3, 4))) expect(self.engine.get_orientation()).to_equal(1) def test_reorientate5(self): # Horizontal Mirror + Rotation 270 # or Vertical Mirror + Rotation 90 self.image = ( (1, 3), (2, 4) ) self.engine.exif = exif_str(5) self.engine.reorientate() expect(self.image).to_equal(((1, 2), (3, 4))) expect(self.engine.get_orientation()).to_equal(1) def test_reorientate6(self): # Rotate 270° self.image = ( (2, 4), (1, 3) ) self.engine.exif = exif_str(6) self.engine.reorientate() expect(self.engine.rotate.call_args[0]).to_equal((270,)) expect(self.engine.flip_horizontally.called).to_be_false() expect(self.engine.flip_vertically.called).to_be_false() expect(self.image).to_equal(((1, 2), (3, 4))) expect(self.engine.get_orientation()).to_equal(1) def test_reorientate7(self): # Flipped horizontally and rotate 90° self.image = ( (4, 2), (3, 1) ) self.engine.exif = exif_str(7) self.engine.reorientate() expect(self.image).to_equal(((1, 2), (3, 4))) expect(self.engine.get_orientation()).to_equal(1) def test_reorientate8(self): # Rotate 90° self.image = ( (3, 1), (4, 2) ) self.engine.exif = exif_str(8) self.engine.reorientate() expect(self.engine.rotate.call_args[0]).to_equal((90,)) expect(self.engine.flip_horizontally.called).to_be_false() expect(self.engine.flip_vertically.called).to_be_false() expect(self.image).to_equal(((1, 2), (3, 4))) expect(self.engine.get_orientation()).to_equal(1)