def _fetch(self, url):
storage = self.context.modules.storage
buffer = yield gen.maybe_future(storage.get(url))
mime = None
if buffer is not None:
self.context.statsd_client.incr('storage.hit')
mime = BaseEngine.get_mimetype(buffer)
self.context.request.extension = EXTENSION.get(mime, '.jpg')
if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
self.context.request.engine = self.context.modules.gif_engine
else:
self.context.request.engine = self.context.modules.engine
raise gen.Return([False, buffer, None])
else:
self.context.statsd_client.incr('storage.miss')
buffer = yield self.context.modules.loader.load(self.context, url)
if buffer is None:
raise gen.Return([False, None, None])
if mime is None:
mime = BaseEngine.get_mimetype(buffer)
self.context.request.extension = EXTENSION.get(mime, '.jpg')
original_preserve = self.context.config.PRESERVE_EXIF_INFO
self.context.config.PRESERVE_EXIF_INFO = True
try:
mime = BaseEngine.get_mimetype(buffer)
self.context.request.extension = extension = EXTENSION.get(
mime, None)
if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
self.context.request.engine = self.context.modules.gif_engine
else:
self.context.request.engine = self.context.modules.engine
self.context.request.engine.load(buffer, extension)
normalized = self.context.request.engine.normalize()
is_no_storage = isinstance(storage, NoStorage)
is_mixed_storage = isinstance(storage, MixedStorage)
is_mixed_no_file_storage = is_mixed_storage and isinstance(
storage.file_storage, NoStorage)
if not (is_no_storage or is_mixed_no_file_storage):
buffer = self.context.request.engine.read(extension)
storage.put(url, buffer)
storage.put_crypto(url)
finally:
self.context.config.PRESERVE_EXIF_INFO = original_preserve
raise gen.Return([normalized, None, self.context.request.engine])
def _fetch(self, url):
storage = self.context.modules.storage
buffer = yield gen.maybe_future(storage.get(url))
mime = None
if buffer is not None:
self.context.statsd_client.incr('storage.hit')
mime = BaseEngine.get_mimetype(buffer)
self.context.request.extension = EXTENSION.get(mime, '.jpg')
if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
self.context.request.engine = self.context.modules.gif_engine
else:
self.context.request.engine = self.context.modules.engine
raise gen.Return([False, buffer, None])
else:
self.context.statsd_client.incr('storage.miss')
buffer = yield self.context.modules.loader.load(self.context, url)
if buffer is None:
raise gen.Return([False, None, None])
if mime is None:
mime = BaseEngine.get_mimetype(buffer)
if mime is None:
raise gen.Return([False, None, None])
self.context.request.extension = EXTENSION.get(mime, '.jpg')
original_preserve = self.context.config.PRESERVE_EXIF_INFO
self.context.config.PRESERVE_EXIF_INFO = True
try:
mime = BaseEngine.get_mimetype(buffer)
self.context.request.extension = extension = EXTENSION.get(mime, None)
if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
self.context.request.engine = self.context.modules.gif_engine
else:
self.context.request.engine = self.context.modules.engine
self.context.request.engine.load(buffer, extension)
normalized = self.context.request.engine.normalize()
is_no_storage = isinstance(storage, NoStorage)
is_mixed_storage = isinstance(storage, MixedStorage)
is_mixed_no_file_storage = is_mixed_storage and isinstance(storage.file_storage, NoStorage)
if not (is_no_storage or is_mixed_no_file_storage):
buffer = self.context.request.engine.read(extension)
storage.put(url, buffer)
storage.put_crypto(url)
finally:
self.context.config.PRESERVE_EXIF_INFO = original_preserve
raise gen.Return([normalized, None, self.context.request.engine])
def load(self, buffer, extension):
self.extension = extension
if extension is None:
mime = self.get_mimetype(buffer)
self.extension = EXTENSION.get(mime, '.jpg')
if self.extension == '.svg':
buffer = self.convert_svg_to_png(buffer)
image_or_frames = self.create_image(buffer)
if METADATA_AVAILABLE:
try:
self.metadata = ImageMetadata.from_buffer(buffer)
self.metadata.read()
except Exception as e:
logger.error('Error reading image metadata: %s' % e)
if self.context.config.ALLOW_ANIMATED_GIFS and isinstance(
image_or_frames, (list, tuple)):
self.image = image_or_frames[0]
if len(image_or_frames) > 1:
self.multiple_engine = MultipleEngine(self)
for frame in image_or_frames:
self.multiple_engine.add_frame(frame)
self.wrap(self.multiple_engine)
else:
self.image = image_or_frames
if self.source_width is None:
self.source_width = self.size[0]
if self.source_height is None:
self.source_height = self.size[1]
def load(self, buffer, extension):
self.extension = extension
if extension is None:
mime = self.get_mimetype(buffer)
self.extension = EXTENSION.get(mime, '.jpg')
if self.extension == '.svg':
buffer = self.convert_svg_to_png(buffer)
image_or_frames = self.create_image(buffer)
if METADATA_AVAILABLE:
try:
self.metadata = ImageMetadata.from_buffer(buffer)
self.metadata.read()
except Exception as e:
logger.error('Error reading image metadata: %s' % e)
if self.context.config.ALLOW_ANIMATED_GIFS and isinstance(
image_or_frames, (list, tuple)):
self.image = image_or_frames[0]
if len(image_or_frames) > 1:
self.multiple_engine = MultipleEngine(self)
for frame in image_or_frames:
self.multiple_engine.add_frame(frame)
self.wrap(self.multiple_engine)
else:
self.image = image_or_frames
if self.source_width is None:
self.source_width = self.size[0]
if self.source_height is None:
self.source_height = self.size[1]
def define_image_type(self, context, result):
if result is not None:
if isinstance(result, ResultStorageResult):
buffer = result.buffer
else:
buffer = result
image_extension = EXTENSION.get(BaseEngine.get_mimetype(buffer), ".jpg")
else:
image_extension = context.request.format
if image_extension is not None:
image_extension = ".%s" % image_extension
logger.debug("Image format specified as %s." % image_extension)
elif self.is_webp(context):
image_extension = ".webp"
logger.debug("Image format set by AUTO_WEBP as %s." % image_extension)
else:
image_extension = context.request.engine.extension
logger.debug("No image format specified. Retrieving from the image extension: %s." % image_extension)
content_type = CONTENT_TYPE.get(image_extension, CONTENT_TYPE[".jpg"])
if context.request.meta:
context.request.meta_callback = (
context.config.META_CALLBACK_NAME or self.request.arguments.get("callback", [None])[0]
)
content_type = "text/javascript" if context.request.meta_callback else "application/json"
logger.debug("Metadata requested. Serving content type of %s." % content_type)
logger.debug("Content Type of %s detected." % content_type)
return (image_extension, content_type)
def define_image_type(self, context, result):
if result is not None:
image_extension = EXTENSION.get(BaseEngine.get_mimetype(result),
'.jpg')
else:
image_extension = context.request.format
if image_extension is not None:
image_extension = '.%s' % image_extension
logger.debug('Image format specified as %s.' % image_extension)
elif self.is_webp(context):
image_extension = '.webp'
logger.debug('Image format set by AUTO_WEBP as %s.' %
image_extension)
else:
image_extension = context.request.engine.extension
logger.debug(
'No image format specified. Retrieving from the image extension: %s.'
% image_extension)
content_type = CONTENT_TYPE.get(image_extension, CONTENT_TYPE['.jpg'])
if context.request.meta:
context.request.meta_callback = context.config.META_CALLBACK_NAME or self.request.arguments.get(
'callback', [None])[0]
content_type = 'text/javascript' if context.request.meta_callback else 'application/json'
logger.debug('Metadata requested. Serving content type of %s.' %
content_type)
logger.debug('Content Type of %s detected.' % content_type)
return (image_extension, content_type)
def define_image_type(self, context, result):
if result is not None:
if isinstance(result, ResultStorageResult):
buffer = result.buffer
else:
buffer = result
image_extension = EXTENSION.get(BaseEngine.get_mimetype(buffer), '.jpg')
else:
image_extension = context.request.format
if image_extension is not None:
image_extension = '.%s' % image_extension
logger.debug('Image format specified as %s.' % image_extension)
elif self.is_webp(context):
image_extension = '.webp'
logger.debug('Image format set by AUTO_WEBP as %s.' % image_extension)
elif self.can_auto_convert_png_to_jpg():
image_extension = '.jpg'
logger.debug('Image format set by AUTO_PNG_TO_JPG as %s.' % image_extension)
else:
image_extension = context.request.engine.extension
logger.debug('No image format specified. Retrieving from the image extension: %s.' % image_extension)
content_type = CONTENT_TYPE.get(image_extension, CONTENT_TYPE['.jpg'])
if context.request.meta:
context.request.meta_callback = context.config.META_CALLBACK_NAME or self.request.arguments.get('callback', [None])[0]
content_type = 'text/javascript' if context.request.meta_callback else 'application/json'
logger.debug('Metadata requested. Serving content type of %s.' % content_type)
logger.debug('Content Type of %s detected.' % content_type)
return (image_extension, content_type)
def load(self, buffer, extension):
self.extension = extension
if extension is None:
mime = self.get_mimetype(buffer)
self.extension = EXTENSION.get(mime, '.jpg')
if self.extension == '.svg':
buffer = self.convert_svg_to_png(buffer)
image_or_frames = self.create_image(buffer)
if self.context.config.ALLOW_ANIMATED_GIFS and isinstance(
image_or_frames, (list, tuple)):
self.image = image_or_frames[0]
if len(image_or_frames) > 1:
self.multiple_engine = MultipleEngine(self)
for frame in image_or_frames:
self.multiple_engine.add_frame(frame)
self.wrap(self.multiple_engine)
else:
self.image = image_or_frames
if self.source_width is None:
self.source_width = self.size[0]
if self.source_height is None:
self.source_height = self.size[1]
def test_can_get_extension(self):
expect(EXTENSION.get('image/jpeg')).to_equal('.jpg')
expect(EXTENSION.get('image/gif')).to_equal('.gif')
expect(EXTENSION.get('image/png')).to_equal('.png')
expect(EXTENSION.get('image/webp')).to_equal('.webp')
expect(EXTENSION.get('video/mp4')).to_equal('.mp4')
expect(EXTENSION.get('video/webm')).to_equal('.webm')
expect(EXTENSION.get('image/svg+xml')).to_equal('.svg')
expect(EXTENSION.get('image/tiff')).to_equal('.tif')
def test_can_get_extension(self):
expect(EXTENSION.get('image/jpeg')).to_equal('.jpg')
expect(EXTENSION.get('image/gif')).to_equal('.gif')
expect(EXTENSION.get('image/png')).to_equal('.png')
expect(EXTENSION.get('image/webp')).to_equal('.webp')
expect(EXTENSION.get('video/mp4')).to_equal('.mp4')
expect(EXTENSION.get('video/webm')).to_equal('.webm')
expect(EXTENSION.get('image/svg+xml')).to_equal('.svg')
expect(EXTENSION.get('image/tiff')).to_equal('.tif')
def test_can_get_extension():
expect(EXTENSION.get("image/jpeg")).to_equal(".jpg")
expect(EXTENSION.get("image/gif")).to_equal(".gif")
expect(EXTENSION.get("image/png")).to_equal(".png")
expect(EXTENSION.get("image/webp")).to_equal(".webp")
expect(EXTENSION.get("video/mp4")).to_equal(".mp4")
expect(EXTENSION.get("video/webm")).to_equal(".webm")
expect(EXTENSION.get("image/svg+xml")).to_equal(".svg")
expect(EXTENSION.get("image/tiff")).to_equal(".tif")
async def _write_results_to_client(self, results, content_type):
max_age = self.context.config.MAX_AGE
if self.context.request.max_age is not None:
max_age = self.context.request.max_age
if (self.context.request.prevent_result_storage
or self.context.request.detection_error):
max_age = self.context.config.MAX_AGE_TEMP_IMAGE
if max_age:
self.set_header("Cache-Control",
"max-age=" + str(max_age) + ",public")
self.set_header(
"Expires",
datetime.datetime.utcnow() +
datetime.timedelta(seconds=max_age),
)
if hasattr(self.context.config, "ACCESS_CONTROL_ALLOW_ORIGIN_HEADER"):
ac_header = self.context.config.ACCESS_CONTROL_ALLOW_ORIGIN_HEADER
if ac_header is not False:
self.set_header("Access-Control-Allow-Origin", ac_header)
logger.debug("CORS header found. Set to: %s", ac_header)
self.set_header("Server", f"Thumbor/{__version__}")
self.set_header("Content-Type", content_type)
if isinstance(results, ResultStorageResult):
buffer = results.buffer
else:
buffer = results
# auto-convert configured?
should_vary = self.context.config.AUTO_WEBP
# we have image (not video)
should_vary = should_vary and content_type.startswith("image/")
# output format is not requested via format filter
should_vary = should_vary and not (
self.context.request.format
and bool( # format is supported by filter
re.search(
r"format\([^)]+\)",
self.context.request.filters)) # filter is in request
)
# our image is not animated gif
should_vary = should_vary and not self.is_animated_gif(buffer)
if should_vary:
self.set_header("Vary", "Accept")
self.context.headers = self._headers.copy()
self._response_ext = EXTENSION.get(content_type)
self._response_length = len(buffer)
self.write(buffer)
self.finish()
def execute_image_operations(self):
self.context.request.quality = None
req = self.context.request
conf = self.context.config
should_store = self.context.config.RESULT_STORAGE_STORES_UNSAFE or not self.context.request.unsafe
if self.context.modules.result_storage and should_store:
start = datetime.datetime.now()
result = yield gen.maybe_future(
self.context.modules.result_storage.get())
finish = datetime.datetime.now()
self.context.metrics.timing('result_storage.incoming_time',
(finish - start).total_seconds() *
1000)
if result is None:
self.context.metrics.incr('result_storage.miss')
else:
self.context.metrics.incr('result_storage.hit')
self.context.metrics.incr('result_storage.bytes_read',
len(result))
if result is not None:
buffer = result.buffer if isinstance(
result, ResultStorageResult) else result
mime = BaseEngine.get_mimetype(buffer)
if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
self.context.request.engine = self.context.modules.gif_engine
else:
self.context.request.engine = self.context.modules.engine
self.context.request.engine.load(buffer,
EXTENSION.get(mime, '.jpg'))
logger.debug('[RESULT_STORAGE] IMAGE FOUND: %s' % req.url)
self.finish_request(self.context, result)
return
if conf.MAX_WIDTH and (not isinstance(
req.width, basestring)) and req.width > conf.MAX_WIDTH:
req.width = conf.MAX_WIDTH
if conf.MAX_HEIGHT and (not isinstance(
req.height, basestring)) and req.height > conf.MAX_HEIGHT:
req.height = conf.MAX_HEIGHT
req.meta_callback = conf.META_CALLBACK_NAME or self.request.arguments.get(
'callback', [None])[0]
self.filters_runner = self.context.filters_factory.create_instances(
self.context, self.context.request.filters)
# Apply all the filters from the PRE_LOAD phase and call get_image() afterwards.
self.filters_runner.apply_filters(thumbor.filters.PHASE_PRE_LOAD,
self.get_image)
def convert_svg_to_png(self, buffer):
if not cairosvg:
msg = """[BaseEngine] convert_svg_to_png failed cairosvg not
imported (if you want svg conversion to png please install cairosvg)
"""
logger.error(msg)
return buffer
try:
buffer = cairosvg.svg2png(bytestring=buffer, dpi=self.context.config.SVG_DPI)
mime = self.get_mimetype(buffer)
self.extension = EXTENSION.get(mime, '.jpg')
except ParseError:
mime = self.get_mimetype(buffer)
extension = EXTENSION.get(mime)
if extension is None or extension == '.svg':
raise
self.extension = extension
return buffer
def load(self, buffer, extension):
self.extension = extension
if extension is None:
mime = self.get_mimetype(buffer)
self.extension = EXTENSION.get(mime, '.jpg')
if self.extension == '.tif': # Pillow does not support 16bit per channel TIFF images
buffer = self.convert_tif_to_png(buffer)
super(Engine, self).load(buffer, self.extension)
def load(self, buffer, extension):
self.extension = extension
if extension is None:
mime = self.get_mimetype(buffer)
self.extension = EXTENSION.get(mime, '.jpg')
if self.extension == '.tif': # Pillow does not support 16bit per channel TIFF images
buffer = self.convert_tif_to_png(buffer)
super(Engine, self).load(buffer, self.extension)
def convert_svg_to_png(self, buffer):
if not cairosvg:
msg = """[BaseEngine] convert_svg_to_png failed cairosvg not
imported (if you want svg conversion to png please install cairosvg)
"""
logger.error(msg)
return buffer
try:
buffer = cairosvg.svg2png(bytestring=buffer, dpi=self.context.config.SVG_DPI)
mime = self.get_mimetype(buffer)
self.extension = EXTENSION.get(mime, '.jpg')
except ParseError:
mime = self.get_mimetype(buffer)
extension = EXTENSION.get(mime)
if extension is None or extension == '.svg':
raise
self.extension = extension
return buffer
def convert_svg_to_png(self, buffer):
if not cairosvg:
msg = """[BaseEngine] convert_svg_to_png failed cairosvg not
imported (if you want svg conversion to png please install cairosvg)
"""
logger.error(msg)
return buffer
buffer = cairosvg.svg2png(bytestring=buffer, dpi=self.context.config.SVG_DPI)
mime = self.get_mimetype(buffer)
self.extension = EXTENSION.get(mime, ".jpg")
return buffer
def convert_svg_to_png(self, buffer):
if not cairosvg:
msg = """[BaseEngine] convert_svg_to_png failed cairosvg not
imported (if you want svg conversion to png please install cairosvg)
"""
logger.error(msg)
return buffer
buffer = cairosvg.svg2png(bytestring=buffer)
mime = self.get_mimetype(buffer)
self.extension = EXTENSION.get(mime, '.jpg')
self.transformed_body = buffer
return buffer
def define_image_type(self, context, result):
if result is not None:
if isinstance(result, ResultStorageResult):
buffer = result.buffer
else:
buffer = result
image_extension = EXTENSION.get(BaseEngine.get_mimetype(buffer),
".jpg")
content_type = CONTENT_TYPE.get(image_extension,
CONTENT_TYPE[".jpg"])
return image_extension, content_type
image_extension = context.request.format
if image_extension is not None:
image_extension = f".{image_extension}"
logger.debug("Image format specified as %s.", image_extension)
elif self.is_webp(context):
image_extension = ".webp"
logger.debug("Image format set by AUTO_WEBP as %s.",
image_extension)
elif self.can_auto_convert_png_to_jpg():
image_extension = ".jpg"
logger.debug(
"Image format set by AUTO_PNG_TO_JPG as %s.",
image_extension,
)
else:
image_extension = context.request.engine.extension
logger.debug(
"No image format specified. Retrieving "
"from the image extension: %s.",
image_extension,
)
content_type = CONTENT_TYPE.get(image_extension, CONTENT_TYPE[".jpg"])
if context.request.meta:
context.request.meta_callback = (context.config.META_CALLBACK_NAME
or self.request.arguments.get(
"callback", [None])[0])
content_type = ("text/javascript" if context.request.meta_callback
else "application/json")
logger.debug("Metadata requested. Serving content type of %s.",
content_type)
logger.debug("Content Type of %s detected.", content_type)
return (image_extension, content_type)
def _write_results_to_client(self, context, results, content_type):
max_age = context.config.MAX_AGE
if context.request.max_age is not None:
max_age = context.request.max_age
if context.request.prevent_result_storage or context.request.detection_error:
max_age = context.config.MAX_AGE_TEMP_IMAGE
if max_age:
self.set_header('Cache-Control',
'max-age=' + str(max_age) + ',public')
self.set_header(
'Expires',
datetime.datetime.utcnow() +
datetime.timedelta(seconds=max_age))
self.set_header('Server', 'Thumbor/%s' % __version__)
self.set_header('Content-Type', content_type)
if isinstance(results, ResultStorageResult):
buffer = results.buffer
else:
buffer = results
# auto-convert configured?
should_vary = context.config.AUTO_WEBP
# we have image (not video)
should_vary = should_vary and content_type.startswith("image/")
# output format is not requested via format filter
should_vary = should_vary and not (
context.request.format # format is supported by filter
and bool(
re.search(r"format\([^)]+\)",
context.request.filters)) # filter is in request
)
# our image is not animated gif
should_vary = should_vary and not self.is_animated_gif(buffer)
if should_vary:
self.set_header('Vary', 'Accept')
context.headers = self._headers.copy()
self._response_ext = EXTENSION.get(content_type)
self._response_length = len(buffer)
self.write(buffer)
self.finish()
def convert_tif_to_png(self, buffer):
if not cv2:
msg = """[PILEngine] convert_tif_to_png failed: opencv not imported"""
logger.error(msg)
return buffer
if not numpy:
msg = """[PILEngine] convert_tif_to_png failed: opencv not imported"""
logger.error(msg)
return buffer
img = cv2.imdecode(numpy.fromstring(buffer, dtype='uint16'), -1)
buffer = cv2.imencode('.png', img)[1].tostring()
mime = self.get_mimetype(buffer)
self.extension = EXTENSION.get(mime, '.jpg')
return buffer
def convert_tif_to_png(self, buffer):
if not cv2:
msg = """[PILEngine] convert_tif_to_png failed: opencv not imported"""
logger.error(msg)
return buffer
if not numpy:
msg = """[PILEngine] convert_tif_to_png failed: opencv not imported"""
logger.error(msg)
return buffer
img = cv2.imdecode(numpy.fromstring(buffer, dtype='uint16'), -1)
buffer = cv2.imencode('.png', img)[1].tostring()
mime = self.get_mimetype(buffer)
self.extension = EXTENSION.get(mime, '.jpg')
return buffer
def execute_image_operations(self):
self.context.request.quality = None
req = self.context.request
conf = self.context.config
should_store = self.context.config.RESULT_STORAGE_STORES_UNSAFE or not self.context.request.unsafe
if self.context.modules.result_storage and should_store:
start = datetime.datetime.now()
result = yield gen.maybe_future(self.context.modules.result_storage.get())
finish = datetime.datetime.now()
self.context.metrics.timing('result_storage.incoming_time', (finish - start).total_seconds() * 1000)
if result is None:
self.context.metrics.incr('result_storage.miss')
else:
self.context.metrics.incr('result_storage.hit')
self.context.metrics.incr('result_storage.bytes_read', len(result))
if result is not None:
buffer = result.buffer if isinstance(result, ResultStorageResult) else result
mime = BaseEngine.get_mimetype(buffer)
if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
self.context.request.engine = self.context.modules.gif_engine
else:
self.context.request.engine = self.context.modules.engine
self.context.request.engine.load(buffer, EXTENSION.get(mime, '.jpg'))
logger.debug('[RESULT_STORAGE] IMAGE FOUND: %s' % req.url)
self.finish_request(self.context, result)
return
if conf.MAX_WIDTH and (not isinstance(req.width, basestring)) and req.width > conf.MAX_WIDTH:
req.width = conf.MAX_WIDTH
if conf.MAX_HEIGHT and (not isinstance(req.height, basestring)) and req.height > conf.MAX_HEIGHT:
req.height = conf.MAX_HEIGHT
req.meta_callback = conf.META_CALLBACK_NAME or self.request.arguments.get('callback', [None])[0]
self.filters_runner = self.context.filters_factory.create_instances(self.context, self.context.request.filters)
# Apply all the filters from the PRE_LOAD phase and call get_image() afterwards.
self.filters_runner.apply_filters(thumbor.filters.PHASE_PRE_LOAD, self.get_image)
def convert_tif_to_png(self, buffer):
if not cv:
msg = """[PILEngine] convert_tif_to_png failed: opencv not imported"""
logger.error(msg)
return buffer
# can not use cv2 here, because ubuntu precise shipped with python-opencv 2.3 which has bug with imencode
# requires 3rd parameter buf which could not be created in python. Could be replaced with these lines:
# img = cv2.imdecode(numpy.fromstring(buffer, dtype='uint16'), -1)
# buffer = cv2.imencode('.png', img)[1].tostring()
mat_data = cv.CreateMatHeader(1, len(buffer), cv.CV_8UC1)
cv.SetData(mat_data, buffer, len(buffer))
img = cv.DecodeImage(mat_data, -1)
buffer = cv.EncodeImage(".png", img).tostring()
mime = self.get_mimetype(buffer)
self.extension = EXTENSION.get(mime, '.jpg')
return buffer
def convert_tif_to_png(self, buffer):
if not cv:
msg = """[PILEngine] convert_tif_to_png failed: opencv not imported"""
logger.error(msg)
return buffer
# can not use cv2 here, because ubuntu precise shipped with python-opencv 2.3 which has bug with imencode
# requires 3rd parameter buf which could not be created in python. Could be replaced with these lines:
# img = cv2.imdecode(numpy.fromstring(buffer, dtype='uint16'), -1)
# buffer = cv2.imencode('.png', img)[1].tostring()
mat_data = cv.CreateMatHeader(1, len(buffer), cv.CV_8UC1)
cv.SetData(mat_data, buffer, len(buffer))
img = cv.DecodeImage(mat_data, -1)
buffer = cv.EncodeImage(".png", img).tostring()
mime = self.get_mimetype(buffer)
self.extension = EXTENSION.get(mime, '.jpg')
return buffer
def _write_results_to_client(self, context, results, content_type):
max_age = context.config.MAX_AGE
if context.request.max_age is not None:
max_age = context.request.max_age
if context.request.prevent_result_storage or context.request.detection_error:
max_age = context.config.MAX_AGE_TEMP_IMAGE
if max_age:
self.set_header('Cache-Control', 'max-age=' + str(max_age) + ',public')
self.set_header('Expires', datetime.datetime.utcnow() + datetime.timedelta(seconds=max_age))
self.set_header('Server', 'Thumbor/%s' % __version__)
self.set_header('Content-Type', content_type)
if isinstance(results, ResultStorageResult):
buffer = results.buffer
else:
buffer = results
# auto-convert configured?
should_vary = context.config.AUTO_WEBP
# we have image (not video)
should_vary = should_vary and content_type.startswith("image/")
# output format is not requested via format filter
should_vary = should_vary and not (
context.request.format # format is supported by filter
and bool(re.search(r"format\([^)]+\)", context.request.filters)) # filter is in request
)
# our image is not animated gif
should_vary = should_vary and not self.is_animated_gif(buffer)
if should_vary:
self.set_header('Vary', 'Accept')
context.headers = self._headers.copy()
self._response_ext = EXTENSION.get(content_type)
self._response_length = len(buffer)
self.write(buffer)
self.finish()
def load(self, buffer, extension):
self.extension = extension
if extension is None:
mime = self.get_mimetype(buffer)
self.extension = EXTENSION.get(mime, '.jpg')
image_or_frames = self.create_image(buffer)
if self.context.config.ALLOW_ANIMATED_GIFS and isinstance(
image_or_frames, (list, tuple)):
self.image = image_or_frames[0]
if len(image_or_frames) > 1:
self.multiple_engine = MultipleEngine(self)
for frame in image_or_frames:
self.multiple_engine.add_frame(frame)
self.wrap(self.multiple_engine)
else:
self.image = image_or_frames
if self.source_width is None:
self.source_width = self.size[0]
if self.source_height is None:
self.source_height = self.size[1]
def _fetch(self, url):
"""
:param url:
:type url:
:return:
:rtype:
"""
fetch_result = FetchResult()
storage = self.context.modules.storage
yield self.acquire_url_lock(url)
try:
fetch_result.buffer = yield gen.maybe_future(storage.get(url))
mime = None
if fetch_result.buffer is not None:
self.release_url_lock(url)
fetch_result.successful = True
self.context.metrics.incr('storage.hit')
mime = BaseEngine.get_mimetype(fetch_result.buffer)
self.context.request.extension = EXTENSION.get(mime, '.jpg')
if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
self.context.request.engine = self.context.modules.gif_engine
else:
self.context.request.engine = self.context.modules.engine
raise gen.Return(fetch_result)
else:
self.context.metrics.incr('storage.miss')
loader_result = yield self.context.modules.loader.load(self.context, url)
finally:
self.release_url_lock(url)
if isinstance(loader_result, LoaderResult):
# TODO _fetch should probably return a result object vs a list to
# to allow returning metadata
if not loader_result.successful:
fetch_result.buffer = None
fetch_result.loader_error = loader_result.error
raise gen.Return(fetch_result)
fetch_result.buffer = loader_result.buffer
else:
# Handle old loaders
fetch_result.buffer = loader_result
if fetch_result.buffer is None:
raise gen.Return(fetch_result)
fetch_result.successful = True
if mime is None:
mime = BaseEngine.get_mimetype(fetch_result.buffer)
self.context.request.extension = extension = EXTENSION.get(mime, '.jpg')
try:
if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
self.context.request.engine = self.context.modules.gif_engine
else:
self.context.request.engine = self.context.modules.engine
self.context.request.engine.load(fetch_result.buffer, extension)
if self.context.request.engine.image is None:
fetch_result.successful = False
fetch_result.buffer = None
fetch_result.engine = self.context.request.engine
fetch_result.engine_error = EngineResult.COULD_NOT_LOAD_IMAGE
raise gen.Return(fetch_result)
fetch_result.normalized = self.context.request.engine.normalize()
# Allows engine or loader to override storage on the fly for the purpose of
# marking a specific file as unstoreable
storage = self.context.modules.storage
is_no_storage = isinstance(storage, NoStorage)
is_mixed_storage = isinstance(storage, MixedStorage)
is_mixed_no_file_storage = is_mixed_storage and isinstance(storage.file_storage, NoStorage)
if not (is_no_storage or is_mixed_no_file_storage):
storage.put(url, fetch_result.buffer)
storage.put_crypto(url)
except Exception:
fetch_result.successful = False
finally:
if not fetch_result.successful:
raise
fetch_result.buffer = None
fetch_result.engine = self.context.request.engine
raise gen.Return(fetch_result)
def _fetch(self, url):
"""
:param url:
:type url:
:return:
:rtype:
"""
fetch_result = FetchResult()
storage = self.context.modules.storage
yield self.acquire_url_lock(url)
try:
fetch_result.buffer = yield gen.maybe_future(storage.get(url))
mime = None
if fetch_result.buffer is not None:
self.release_url_lock(url)
fetch_result.successful = True
self.context.metrics.incr('storage.hit')
mime = BaseEngine.get_mimetype(fetch_result.buffer)
self.context.request.extension = EXTENSION.get(mime, '.jpg')
if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
self.context.request.engine = self.context.modules.gif_engine
else:
self.context.request.engine = self.context.modules.engine
raise gen.Return(fetch_result)
else:
self.context.metrics.incr('storage.miss')
loader_result = yield self.context.modules.loader.load(
self.context, url)
finally:
self.release_url_lock(url)
if isinstance(loader_result, LoaderResult):
# TODO _fetch should probably return a result object vs a list to
# to allow returning metadata
if not loader_result.successful:
fetch_result.buffer = None
fetch_result.loader_error = loader_result.error
raise gen.Return(fetch_result)
fetch_result.buffer = loader_result.buffer
else:
# Handle old loaders
fetch_result.buffer = loader_result
if fetch_result.buffer is None:
raise gen.Return(fetch_result)
fetch_result.successful = True
if mime is None:
mime = BaseEngine.get_mimetype(fetch_result.buffer)
self.context.request.extension = extension = EXTENSION.get(
mime, '.jpg')
original_preserve = self.context.config.PRESERVE_EXIF_INFO
try:
if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
self.context.request.engine = self.context.modules.gif_engine
else:
self.context.request.engine = self.context.modules.engine
self.context.request.engine.load(fetch_result.buffer, extension)
if self.context.request.engine.image is None:
fetch_result.successful = False
fetch_result.buffer = None
fetch_result.engine = self.context.request.engine
fetch_result.engine_error = EngineResult.COULD_NOT_LOAD_IMAGE
raise gen.Return(fetch_result)
fetch_result.normalized = self.context.request.engine.normalize()
# Allows engine or loader to override storage on the fly for the purpose of
# marking a specific file as unstoreable
storage = self.context.modules.storage
is_no_storage = isinstance(storage, NoStorage)
is_mixed_storage = isinstance(storage, MixedStorage)
is_mixed_no_file_storage = is_mixed_storage and isinstance(
storage.file_storage, NoStorage)
if not (is_no_storage or is_mixed_no_file_storage):
# in the case that we have local storage, it's important that we don't try
# and strip the exif data - so that we can make the choice about stripping
# it on output, instead of input.
# unfortunately, the only interface to the engine is the global config
# referenced by our context, which has the current setting of preserving
# the exif info or not. this is also being read by concurrent
# coroutine-threads, so just setting this value outright causes a race.
# we solve this in order not to influence these other coroutine-threads
# by doing an effective copy-on-write for the config until the end of this
# context (ie. this request).
# at some point in the future - the API should change, and this horrid hack
# should go away.
self.context.config = copy.deepcopy(self.context.config)
self.context.config.PRESERVE_EXIF_INFO = True
fetch_result.buffer = self.context.request.engine.read(
extension)
storage.put(url, fetch_result.buffer)
storage.put_crypto(url)
except Exception:
fetch_result.successful = False
finally:
self.context.config.PRESERVE_EXIF_INFO = original_preserve
if not fetch_result.successful:
raise
fetch_result.buffer = None
fetch_result.engine = self.context.request.engine
raise gen.Return(fetch_result)
def _fetch(self, url):
fetch_result = FetchResult()
storage = self.context.modules.storage
fetch_result.buffer = yield gen.maybe_future(storage.get(url))
mime = None
if fetch_result.buffer is not None:
fetch_result.successful = True
self.context.metrics.incr('storage.hit')
mime = BaseEngine.get_mimetype(fetch_result.buffer)
self.context.request.extension = EXTENSION.get(mime, '.jpg')
if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
self.context.request.engine = self.context.modules.gif_engine
else:
self.context.request.engine = self.context.modules.engine
raise gen.Return(fetch_result)
else:
self.context.metrics.incr('storage.miss')
loader_result = yield self.context.modules.loader.load(self.context, url)
if isinstance(loader_result, LoaderResult):
# TODO _fetch should probably return a result object vs a list to
# to allow returning metadata
if not loader_result.successful:
fetch_result.buffer = None
fetch_result.loader_error = loader_result.error
raise gen.Return(fetch_result)
fetch_result.buffer = loader_result.buffer
else:
# Handle old loaders
fetch_result.buffer = loader_result
if fetch_result.buffer is None:
raise gen.Return(fetch_result)
fetch_result.successful = True
if mime is None:
mime = BaseEngine.get_mimetype(fetch_result.buffer)
self.context.request.extension = EXTENSION.get(mime, '.jpg')
original_preserve = self.context.config.PRESERVE_EXIF_INFO
self.context.config.PRESERVE_EXIF_INFO = True
try:
mime = BaseEngine.get_mimetype(fetch_result.buffer)
self.context.request.extension = extension = EXTENSION.get(mime, None)
if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
self.context.request.engine = self.context.modules.gif_engine
else:
self.context.request.engine = self.context.modules.engine
self.context.request.engine.load(fetch_result.buffer, extension)
fetch_result.normalized = self.context.request.engine.normalize()
is_no_storage = isinstance(storage, NoStorage)
is_mixed_storage = isinstance(storage, MixedStorage)
is_mixed_no_file_storage = is_mixed_storage and isinstance(storage.file_storage, NoStorage)
if not (is_no_storage or is_mixed_no_file_storage):
fetch_result.buffer = self.context.request.engine.read(extension)
storage.put(url, fetch_result.buffer)
storage.put_crypto(url)
finally:
self.context.config.PRESERVE_EXIF_INFO = original_preserve
fetch_result.buffer = None
fetch_result.engine = self.context.request.engine
raise gen.Return(fetch_result)
def _fetch(self, url):
"""
:param url:
:type url:
:return:
:rtype:
"""
fetch_result = FetchResult()
storage = self.context.modules.storage
yield self.acquire_url_lock(url)
try:
fetch_result.buffer = yield gen.maybe_future(storage.get(url))
mime = None
if fetch_result.buffer is not None:
self.release_url_lock(url)
fetch_result.successful = True
self.context.metrics.incr('storage.hit')
mime = BaseEngine.get_mimetype(fetch_result.buffer)
self.context.request.extension = EXTENSION.get(mime, '.jpg')
if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
self.context.request.engine = self.context.modules.gif_engine
else:
self.context.request.engine = self.context.modules.engine
raise gen.Return(fetch_result)
else:
self.context.metrics.incr('storage.miss')
loader_result = yield self.context.modules.loader.load(self.context, url)
finally:
self.release_url_lock(url)
if isinstance(loader_result, LoaderResult):
# TODO _fetch should probably return a result object vs a list to
# to allow returning metadata
if not loader_result.successful:
fetch_result.buffer = None
fetch_result.loader_error = loader_result.error
raise gen.Return(fetch_result)
fetch_result.buffer = loader_result.buffer
else:
# Handle old loaders
fetch_result.buffer = loader_result
if fetch_result.buffer is None:
raise gen.Return(fetch_result)
fetch_result.successful = True
if mime is None:
mime = BaseEngine.get_mimetype(fetch_result.buffer)
self.context.request.extension = extension = EXTENSION.get(mime, '.jpg')
original_preserve = self.context.config.PRESERVE_EXIF_INFO
try:
if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
self.context.request.engine = self.context.modules.gif_engine
else:
self.context.request.engine = self.context.modules.engine
self.context.request.engine.load(fetch_result.buffer, extension)
if self.context.request.engine.image is None:
fetch_result.successful = False
fetch_result.buffer = None
fetch_result.engine = self.context.request.engine
fetch_result.engine_error = EngineResult.COULD_NOT_LOAD_IMAGE
raise gen.Return(fetch_result)
fetch_result.normalized = self.context.request.engine.normalize()
# Allows engine or loader to override storage on the fly for the purpose of
# marking a specific file as unstoreable
storage = self.context.modules.storage
is_no_storage = isinstance(storage, NoStorage)
is_mixed_storage = isinstance(storage, MixedStorage)
is_mixed_no_file_storage = is_mixed_storage and isinstance(storage.file_storage, NoStorage)
if not (is_no_storage or is_mixed_no_file_storage):
# in the case that we have local storage, it's important that we don't try
# and strip the exif data - so that we can make the choice about stripping
# it on output, instead of input.
# unfortunately, the only interface to the engine is the global config
# referenced by our context, which has the current setting of preserving
# the exif info or not. this is also being read by concurrent
# coroutine-threads, so just setting this value outright causes a race.
# we solve this in order not to influence these other coroutine-threads
# by doing an effective copy-on-write for the config until the end of this
# context (ie. this request).
# at some point in the future - the API should change, and this horrid hack
# should go away.
self.context.config = copy.deepcopy(self.context.config)
self.context.config.PRESERVE_EXIF_INFO = True
fetch_result.buffer = self.context.request.engine.read(extension)
storage.put(url, fetch_result.buffer)
storage.put_crypto(url)
except Exception:
fetch_result.successful = False
finally:
self.context.config.PRESERVE_EXIF_INFO = original_preserve
if not fetch_result.successful:
raise
fetch_result.buffer = None
fetch_result.engine = self.context.request.engine
raise gen.Return(fetch_result)
def convert_svg_to_png(self, buffer):
buffer = cairosvg.svg2png(bytestring=buffer)
mime = self.get_mimetype(buffer)
self.extension = EXTENSION.get(mime, '.jpg')
self.transformed_body = buffer
return buffer
def _fetch(self, url):
"""
:param url:
:type url:
:return:
:rtype:
"""
fetch_result = FetchResult()
storage = self.context.modules.storage
yield self.acquire_url_lock(url)
try:
fetch_result.buffer = yield gen.maybe_future(storage.get(url))
mime = None
if fetch_result.buffer is not None:
self.release_url_lock(url)
fetch_result.successful = True
self.context.metrics.incr('storage.hit')
mime = BaseEngine.get_mimetype(fetch_result.buffer)
self.context.request.extension = EXTENSION.get(mime, '.jpg')
if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
self.context.request.engine = self.context.modules.gif_engine
else:
self.context.request.engine = self.context.modules.engine
raise gen.Return(fetch_result)
else:
self.context.metrics.incr('storage.miss')
loader_result = yield self.context.modules.loader.load(
self.context, url)
finally:
self.release_url_lock(url)
if isinstance(loader_result, LoaderResult):
# TODO _fetch should probably return a result object vs a list to
# to allow returning metadata
if not loader_result.successful:
fetch_result.buffer = None
fetch_result.loader_error = loader_result.error
raise gen.Return(fetch_result)
fetch_result.buffer = loader_result.buffer
else:
# Handle old loaders
fetch_result.buffer = loader_result
if fetch_result.buffer is None:
raise gen.Return(fetch_result)
fetch_result.successful = True
if mime is None:
mime = BaseEngine.get_mimetype(fetch_result.buffer)
self.context.request.extension = EXTENSION.get(mime, '.jpg')
original_preserve = self.context.config.PRESERVE_EXIF_INFO
self.context.config.PRESERVE_EXIF_INFO = True
try:
mime = BaseEngine.get_mimetype(fetch_result.buffer)
self.context.request.extension = extension = EXTENSION.get(
mime, None)
if mime == 'image/gif' and self.context.config.USE_GIFSICLE_ENGINE:
self.context.request.engine = self.context.modules.gif_engine
else:
self.context.request.engine = self.context.modules.engine
self.context.request.engine.load(fetch_result.buffer, extension)
fetch_result.normalized = self.context.request.engine.normalize()
# Allows engine or loader to override storage on the fly for the purpose of
# marking a specific file as unstoreable
storage = self.context.modules.storage
is_no_storage = isinstance(storage, NoStorage)
is_mixed_storage = isinstance(storage, MixedStorage)
is_mixed_no_file_storage = is_mixed_storage and isinstance(
storage.file_storage, NoStorage)
if not (is_no_storage or is_mixed_no_file_storage):
fetch_result.buffer = self.context.request.engine.read(
extension)
storage.put(url, fetch_result.buffer)
storage.put_crypto(url)
finally:
self.context.config.PRESERVE_EXIF_INFO = original_preserve
fetch_result.buffer = None
fetch_result.engine = self.context.request.engine
raise gen.Return(fetch_result)
def test_should_not_run_for_not_gif(self):
optimizer = Optimizer(self.get_context())
for ext in EXTENSION.itervalues():
if ext != '.gif':
self.assertFalse(optimizer.should_run(ext, ''))
def test_should_not_run_for_not_gif(self):
optimizer = Optimizer(self.get_context())
for ext in EXTENSION.itervalues():
if ext != '.gif':
self.assertFalse(optimizer.should_run(ext, ''))
async def _fetch(self, url):
"""
:param url:
:type url:
:return:
:rtype:
"""
fetch_result = FetchResult()
storage = self.context.modules.storage
await self.acquire_url_lock(url)
try:
fetch_result.buffer = await storage.get(url)
mime = None
if fetch_result.buffer is not None:
self.release_url_lock(url)
fetch_result.successful = True
self.context.metrics.incr("storage.hit")
mime = BaseEngine.get_mimetype(fetch_result.buffer)
self.context.request.extension = EXTENSION.get(mime, ".jpg")
if mime == "image/gif" and self.context.config.USE_GIFSICLE_ENGINE:
self.context.request.engine = self.context.modules.gif_engine
else:
self.context.request.engine = self.context.modules.engine
return fetch_result
self.context.metrics.incr("storage.miss")
loader_result = await self.context.modules.loader.load(
self.context, url)
finally:
self.release_url_lock(url)
if isinstance(loader_result, LoaderResult):
# TODO _fetch should probably return a result object vs a list to
# to allow returning metadata
if not loader_result.successful:
fetch_result.buffer = None
fetch_result.loader_error = loader_result.error
return fetch_result
fetch_result.buffer = loader_result.buffer
else:
# Handle old loaders
fetch_result.buffer = loader_result
if fetch_result.buffer is None:
return fetch_result
fetch_result.successful = True
if mime is None:
mime = BaseEngine.get_mimetype(fetch_result.buffer)
self.context.request.extension = extension = EXTENSION.get(
mime, ".jpg")
try:
if mime == "image/gif" and self.context.config.USE_GIFSICLE_ENGINE:
self.context.request.engine = self.context.modules.gif_engine
else:
self.context.request.engine = self.context.modules.engine
self.context.request.engine.load(fetch_result.buffer, extension)
if self.context.request.engine.image is None:
fetch_result.successful = False
fetch_result.buffer = None
fetch_result.engine = self.context.request.engine
fetch_result.engine_error = EngineResult.COULD_NOT_LOAD_IMAGE
return fetch_result
fetch_result.normalized = self.context.request.engine.normalize()
# Allows engine or loader to override storage
# on the fly for the purpose of
# marking a specific file as unstoreable
storage = self.context.modules.storage
is_no_storage = isinstance(storage, NoStorage)
is_mixed_storage = isinstance(storage, MixedStorage)
is_mixed_no_file_storage = is_mixed_storage and isinstance(
storage.file_storage, NoStorage)
if not (is_no_storage or is_mixed_no_file_storage):
await storage.put(url, fetch_result.buffer)
await storage.put_crypto(url)
except Exception as error:
fetch_result.successful = False
fetch_result.exception = error
if not fetch_result.successful and fetch_result.exception is not None:
raise fetch_result.exception
fetch_result.buffer = None
fetch_result.engine = self.context.request.engine
return fetch_result
def test_should_not_run_for_not_gif(self):
optimizer = Optimizer(self.get_context())
for ext in EXTENSION.items():
if ext != ".gif":
self.assertFalse(optimizer.should_run(ext, ""))
