Esempio n. 1
0
 def put(self, path, contents):
     key = self.key_for(path)
     try:
         self.storage.set(key, contents, time=self.context.config.STORAGE_EXPIRATION_SECONDS)
     except:
         logger.exception("[MEMCACHED] failed to set key '{0}'".format(key));
     return path
Esempio n. 2
0
    async def smart_detect(self):
        is_gifsicle = (
            self.context.request.engine.extension == ".gif"
            and self.context.config.USE_GIFSICLE_ENGINE
        )
        if (
            not (self.context.modules.detectors and self.context.request.smart)
        ) or is_gifsicle:
            return

        try:
            await self.do_smart_detection()
        except Exception:  # pylint: disable=broad-except
            if not self.context.config.IGNORE_SMART_ERRORS:
                raise

            logger.exception("Ignored error during smart detection")
            if self.context.config.USE_CUSTOM_ERROR_HANDLING:
                self.context.modules.importer.error_handler.handle_error(
                    context=self.context,
                    handler=self.context.request_handler,
                    exception=sys.exc_info(),
                )

            self.context.request.prevent_result_storage = True
            self.context.request.detection_error = True
Esempio n. 3
0
 def put_detector_data(self, path, data):
     key = self.detector_key_for(path)
     try:
         self.storage.set(key, dumps(data))
     except:
         logger.exception("[MEMCACHED] failed to set key '{0}'".format(key));
     return key
Esempio n. 4
0
def get_video_frame(context, file_path):
    """
    A context manager that extracts a single frame out of a video file and 
    stores it in a temporary file. Returns the path of the temporary file
    or None in case of failure.
    Depends on FFMPEG_PATH from Thumbor's configuration.
    """
    import subprocess
    import tempfile
    import os
    # Fail nicely when ffmpeg cannot be found
    if not os.path.exists(context.config.FFMPEG_PATH):
        logger.error('%s does not exist, please configure FFMPEG_PATH',
                     context.config.FFMPEG_PATH)
        yield None
        return
    # Prepare temporary file
    f, image_path = tempfile.mkstemp('.jpg')
    os.close(f)
    # Extract image
    try:
        cmd = [
            context.config.FFMPEG_PATH, '-i', file_path, '-ss', '00:00:01.000',
            '-vframes', '1', '-y', '-nostats', '-loglevel', 'error', image_path
        ]
        subprocess.check_call(cmd)
        yield image_path
    except:
        logger.exception('Cannot extract image frame from %s', file_path)
        yield None
    finally:
        # Cleanup
        try_to_delete(image_path)
Esempio n. 5
0
    def smart_detect(self):
        if not (self.context.modules.detectors and self.context.request.smart):
            self.do_image_operations()
            return

        try:
            # Beware! Boolean hell ahead.
            #
            # The `running_smart_detection` flag is needed so we can know
            # whether `after_smart_detect()` is running synchronously or not.
            #
            # If we're running it in a sync fashion it will set
            # `should_run_image_operations` to True so we can avoid running
            # image operation inside the try block.
            self.should_run_image_operations = False
            self.running_smart_detection = True
            self.do_smart_detection()
            self.running_smart_detection = False
        except Exception:
            if not self.context.config.IGNORE_SMART_ERRORS:
                raise

            logger.exception("Ignored error during smart detection")
            if self.context.config.USE_CUSTOM_ERROR_HANDLING:
                self.context.modules.importer.error_handler.handle_error(context=self.context, handler=self.context.request_handler, exception=sys.exc_info())

            self.context.request.prevent_result_storage = True
            self.context.request.detection_error = True
            self.do_image_operations()

        if self.should_run_image_operations:
            self.do_image_operations()
Esempio n. 6
0
    def detect(self, callback):
        engine = self.context.modules.engine
        try:
            img = np.array(
                engine.convert_to_grayscale(update_image=False,
                                            with_alpha=False))
        except Exception as e:
            logger.exception(e)
            logger.warn(
                'Error during feature detection; skipping to next detector')
            self.next(callback)
            return

        points = cv2.goodFeaturesToTrack(
            img,
            maxCorners=20,
            qualityLevel=0.04,
            minDistance=1.0,
            useHarrisDetector=False,
        )
        if points is not None:
            for point in points:
                x, y = point.ravel()
                self.context.request.focal_points.append(
                    FocalPoint(x.item(), y.item(), 1))
            callback()
        else:
            self.next(callback)
Esempio n. 7
0
    def srgb(self):
        try:
            if not isinstance(self.engine, PILEngine):
                logger.warn('Could not perform profileToProfile conversion: engine is not PIL engine')
                return

            if (ImageCms is None):
                logger.warn('ImageCms is not installed. Could not perform profileToProfile conversion')
                return

            image = self.engine.image

            embedded_profile = image.info.get('icc_profile')

            if not embedded_profile:
                logger.debug('Image does not have embedded profile. Assuming already in sRGB')
                return

            embedded_profile = BytesIO(embedded_profile)
            srgb_profile = BytesIO(tiny_srgb)

            output_mode = 'RGBA' if 'A' in image.mode else 'RGB'
            image = ImageCms.profileToProfile(image, embedded_profile, srgb_profile, renderingIntent=0,
                                              outputMode=output_mode)

            self.engine.image = image
            self.engine.icc_profile = image.info.get('icc_profile')
        except Exception as err:
            logger.exception(err)
Esempio n. 8
0
 def put_detector_data(self, path, data):
     key = self.detector_key_for(path)
     try:
         self.storage.set(key, dumps(data))
     except:
         logger.exception("[MEMCACHED] failed to set key '{0}'".format(key))
     return key
Esempio n. 9
0
    def smart_detect(self):
        if not (self.context.modules.detectors and self.context.request.smart):
            self.do_image_operations()
            return

        try:
            # Beware! Boolean hell ahead.
            #
            # The `running_smart_detection` flag is needed so we can know
            # whether `after_smart_detect()` is running synchronously or not.
            #
            # If we're running it in a sync fashion it will set
            # `should_run_image_operations` to True so we can avoid running
            # image operation inside the try block.
            self.should_run_image_operations = False
            self.running_smart_detection = True
            self.do_smart_detection()
            self.running_smart_detection = False
        except Exception:
            if not self.context.config.IGNORE_SMART_ERRORS:
                raise

            logger.exception("Ignored error during smart detection")
            if self.context.config.USE_CUSTOM_ERROR_HANDLING:
                self.context.modules.importer.error_handler.handle_error(
                    context=self.context,
                    handler=self.context.request_handler,
                    exception=sys.exc_info())

            self.context.request.prevent_result_storage = True
            self.context.request.detection_error = True
            self.do_image_operations()

        if self.should_run_image_operations:
            self.do_image_operations()
Esempio n. 10
0
    async def finish_request(self, result_from_storage=None):
        if result_from_storage is not None:
            await self._process_result_from_storage(result_from_storage)

            _, content_type = self.define_image_type(self.context,
                                                     result_from_storage)
            await self._write_results_to_client(result_from_storage,
                                                content_type)

            return

        context = self.context
        result_storage = context.modules.result_storage
        metrics = context.metrics

        should_store = (result_storage
                        and not context.request.prevent_result_storage
                        and (context.config.RESULT_STORAGE_STORES_UNSAFE
                             or not context.request.unsafe))

        try:
            result = await self.context.thread_pool.queue(
                operation=functools.partial(self._load_results, context), )
        except Exception as error:  # pylint: disable=broad-except
            logger.exception("[BaseHander.finish_request] %s", error)
            self._error(
                500, "Error while trying to fetch the image: {}".format(error))
            return

        (results, content_type) = result
        await self._write_results_to_client(results, content_type)

        if should_store:
            await self._store_results(result_storage, metrics, results)
Esempio n. 11
0
    def watermark(self,
                  callback,
                  url,
                  x,
                  y,
                  alpha,
                  w_ratio=False,
                  h_ratio=False):
        self.url = url
        self.x = x
        self.y = y
        self.alpha = alpha
        self.w_ratio = float(
            w_ratio) / 100.0 if w_ratio and w_ratio != 'none' else False
        self.h_ratio = float(
            h_ratio) / 100.0 if h_ratio and h_ratio != 'none' else False
        self.callback = callback
        self.watermark_engine = self.context.modules.engine.__class__(
            self.context)
        self.storage = self.context.modules.storage

        try:
            buffer = yield tornado.gen.maybe_future(self.storage.get(self.url))
            if buffer is not None:
                self.on_image_ready(buffer)
            else:
                self.context.modules.loader.load(self.context, self.url,
                                                 self.on_fetch_done)
        except Exception as e:
            logger.exception(e)
            logger.warn("bad watermark")
            raise tornado.web.HTTPError(500)
Esempio n. 12
0
 def request(self):
     try:
         self.thumbor_filter.context.modules.loader.load(
             self.thumbor_filter.context, self.url, self.on_fetch_done)
     except Exception as err:
         self.failed = True
         logger.exception(err)
Esempio n. 13
0
    async def detect(self):
        self.context.request.prevent_result_storage = True
        try:
            if not QueuedDetector.queue:
                redis = Redis(
                    host=self.context.config.REDIS_QUEUE_SERVER_HOST,
                    port=self.context.config.REDIS_QUEUE_SERVER_PORT,
                    db=self.context.config.REDIS_QUEUE_SERVER_DB,
                    password=self.context.config.REDIS_QUEUE_SERVER_PASSWORD,
                )
                QueuedDetector.queue = UniqueQueue(server=redis)

            QueuedDetector.queue.enqueue_unique_from_string(
                "remotecv.pyres_tasks.DetectTask",
                "Detect",
                args=[self.detection_type, self.context.request.image_url],
                key=self.context.request.image_url,
            )
        except RedisError:
            self.context.request.detection_error = True
            QueuedDetector.queue = None
            logger.exception("Redis Error")

        # Error or not we return an empty list as detection
        # will be done later
        return []
Esempio n. 14
0
    async def detect(self):
        engine = self.context.modules.engine
        try:
            img = np.array(
                engine.convert_to_grayscale(update_image=False, alpha=False))
        except Exception as error:
            logger.exception(error)
            logger.warning(
                "Error during feature detection; skipping to next detector")
            return await self.next()  # pylint: disable=not-callable

        points = cv2.goodFeaturesToTrack(  # pylint: disable=no-member
            img,
            maxCorners=20,
            qualityLevel=0.04,
            minDistance=1.0,
            useHarrisDetector=False,
        )
        if points is not None:
            for point in points:
                x_pos, y_pos = point.ravel()
                self.context.request.focal_points.append(
                    FocalPoint(x_pos.item(), y_pos.item(), 1))
            return

        await self.next()  # pylint: disable=not-callable
Esempio n. 15
0
 def request(self):
     try:
         self.thumbor_filter.context.modules.loader.load(
             self.thumbor_filter.context, self.url, self.on_fetch_done)
     except Exception as err:
         self.failed = True
         logger.exception(err)
Esempio n. 16
0
    async def detect(self):
        self.context.request.prevent_result_storage = True
        try:
            if not Detector.detect_task:
                celery_tasks = CeleryTasks(
                    self.context.config.SQS_QUEUE_KEY_ID,
                    self.context.config.SQS_QUEUE_KEY_SECRET,
                    self.context.config.SQS_QUEUE_REGION,
                    None,
                )
                Detector.detect_task = celery_tasks.get_detect_task()

            Detector.detect_task.delay(
                "all",
                self.context.request.image_url,
                self.context.request.image_url,
            )
        except RuntimeError:
            self.context.request.detection_error = True
            Detector.detect_task = None
            logger.exception("Celery Error")

        # Error or not we return an empty list as detection
        # will be done later
        return []
Esempio n. 17
0
    def detect(self, callback):
        engine = self.context.modules.engine
        try:
            img = np.array(
                engine.convert_to_grayscale(
                    update_image=False,
                    with_alpha=False
                )
            )
        except Exception as e:
            logger.exception(e)
            logger.warn('Error during feature detection; skipping to next detector')
            self.next(callback)
            return

        points = cv2.goodFeaturesToTrack(
            img,
            maxCorners=20,
            qualityLevel=0.04,
            minDistance=1.0,
            useHarrisDetector=False,
        )
        if points is not None:
            for point in points:
                x, y = point.ravel()
                self.context.request.focal_points.append(FocalPoint(x.item(), y.item(), 1))
            callback()
        else:
            self.next(callback)
    def detect(self, callback):
        engine = self.context.modules.engine
        try:
            engine.image_data_as_rgb()
            img = np.array(engine.image)
            self.net.setInput(
                cv2.dnn.blobFromImage(img, size=(300, 300), swapRB=True))
            detections = self.net.forward()
        except Exception as e:
            logger.exception(e)
            logger.warn(
                'Error during feature detection; skipping to next detector')
            self.next(callback)
            return

        confidence_threshold = 0.2
        num_detections = 0
        for detection in detections[0, 0, :, :]:
            confidence = float(detection[2])
            if confidence < confidence_threshold:
                continue
            num_detections += 1
            class_id = int(detection[1]) - 1  # make it zero-indexed
            class_name = coco_classes[class_id]
            left = int(detection[3] * img.shape[1])
            top = int(detection[4] * img.shape[0])
            right = int(detection[5] * img.shape[1])
            bottom = int(detection[6] * img.shape[0])
            width = right - left
            height = bottom - top
            # If the detection is of a person,
            # and the person is vertically oriented,
            # this uses the upper 1/4 of the box to focus on the face.
            # In the case the person is horizontal, perhaps reclining,
            # then the focal point will remain at their center.
            # In the case the person is upside down, this would focus on the feet instead of the face.
            # But consider - whoever is publishing a picture of an upside down person
            # might appreciate that it focuses on the feet.
            if class_name == 'person' and height > width:
                height = int(height * 0.25)
            self.context.request.focal_points.append(
                FocalPoint.from_dict({
                    'x':
                    left + (width / 2),
                    'y':
                    top + (height / 2),
                    'width':
                    width,
                    'height':
                    height,
                    'z':
                    confidence,
                    'origin':
                    'DNN Object Detection (class: {})'.format(class_name)
                }))
        if num_detections > 0:
            callback()
        else:
            self.next(callback)
Esempio n. 19
0
 def _get_exif_segment(self):
     try:
         segment = ExifSegment(None, None, self.exif, 'ro')
     except Exception:
         logger.exception('Ignored error handling exif for reorientation')
     else:
         return segment
     return None
Esempio n. 20
0
 def _get_exif_segment(self):
     try:
         segment = ExifSegment(None, None, self.exif, 'ro')
     except Exception:
         logger.exception('Ignored error handling exif for reorientation')
     else:
         return segment
     return None
Esempio n. 21
0
    async def execute_image_operations(self):
        self.context.request.quality = None

        req = self.context.request
        conf = self.context.config

        should_store = (self.context.config.RESULT_STORAGE_STORES_UNSAFE
                        or not self.context.request.unsafe)
        if self.context.modules.result_storage and should_store:
            start = datetime.datetime.now()

            try:
                result = await self.context.modules.result_storage.get()
            except Exception as error:
                logger.exception("[BaseHander.execute_image_operations] %s",
                                 error)
                self._error(
                    500,
                    "Error while trying to get the image "
                    "from the result storage: {}".format(error),
                )
                return

            finish = datetime.datetime.now()

            self.context.metrics.timing(
                "result_storage.incoming_time",
                (finish - start).total_seconds() * 1000,
            )

            if result is None:
                self.context.metrics.incr("result_storage.miss")
            else:
                self.context.metrics.incr("result_storage.hit")
                self.context.metrics.incr("result_storage.bytes_read",
                                          len(result))
                logger.debug("[RESULT_STORAGE] IMAGE FOUND: %s", req.url)
                await self.finish_request(result)
                return

        if (conf.MAX_WIDTH and (not isinstance(req.width, str))
                and req.width > conf.MAX_WIDTH):
            req.width = conf.MAX_WIDTH
        if (conf.MAX_HEIGHT and (not isinstance(req.height, str))
                and req.height > conf.MAX_HEIGHT):
            req.height = conf.MAX_HEIGHT

        req.meta_callback = (conf.META_CALLBACK_NAME or
                             self.request.arguments.get("callback", [None])[0])

        self.filters_runner = self.context.filters_factory.create_instances(
            self.context, self.context.request.filters)
        # Apply all the filters from the PRE_LOAD phase
        # and call get_image() afterwards.
        await self.filters_runner.apply_filters(
            thumbor.filters.PHASE_PRE_LOAD, )
        await self.get_image()
Esempio n. 22
0
    def read(self, extension=None, quality=None):
        #returns image buffer in byte format.
        img_buffer = BytesIO()

        ext = extension or self.extension

        options = {
            'quality': quality
        }

        if ext == '.jpg' or ext == '.jpeg':
            options['optimize'] = True
            options['progressive'] = True

            if quality is None:
                options['quality'] = 'keep'

        if options['quality'] is None:
            options['quality'] = self.context.config.QUALITY

        if self.icc_profile is not None:
            options['icc_profile'] = self.icc_profile

        if self.context.config.PRESERVE_EXIF_INFO:
            exif = self.image.info.get('exif', None)
            if exif is not None:
                options['exif'] = exif

        if self.image.mode == 'P' and self.transparency:
            options['transparency'] = self.transparency

        try:
            if ext == '.webp':
                if self.image.mode not in ['RGB', 'RGBA']:
                    mode = None
                    if self.image.mode != 'P':
                        mode = 'RGBA' if self.image.mode[-1] == 'A' else 'RGB'
                    self.image = self.image.convert(mode)

            self.image.save(img_buffer, FORMATS[ext], **options)
        except IOError:
            logger.exception('Could not save as improved image, consider to increase ImageFile.MAXBLOCK')
            self.image.save(img_buffer, FORMATS[ext])
        except KeyError:
            logger.exception('Image format not found in PIL: %s' % ext)

            #extension is not present or could not help determine format => force JPEG
            if self.image.mode in ['P', 'RGBA', 'LA']:
                self.image.format = FORMATS['.png']
                self.image.save(img_buffer, FORMATS['.png'])
            else:
                self.image.format = FORMATS['.jpg']
                self.image.save(img_buffer, FORMATS['.jpg'])

        results = img_buffer.getvalue()
        img_buffer.close()
        return results
    def _load_results(self, context):
        try:
            results, content_type = BaseHandler._load_results(self, context)
        except Exception:
            logger.exception('[ImagesHandler] Exception during _load_results', extra=log_extra(context))
            self._error(500)
            return None, None

        return results, content_type
Esempio n. 24
0
 def put(self, path, contents):
     key = self.key_for(path)
     try:
         self.storage.set(
             key,
             contents,
             time=self.context.config.STORAGE_EXPIRATION_SECONDS)
     except:
         logger.exception("[MEMCACHED] failed to set key '{0}'".format(key))
     return path
Esempio n. 25
0
    def detect_faces(self, raw_bytes):

        try:
            resp = self.rekognition_client().detect_faces(
                Image={'Bytes': raw_bytes}, Attributes=['DEFAULT'])
            return [Face(fd) for fd in resp.get('FaceDetails', [])]

        except Exception as e:
            logger.exception(e)
            return []
Esempio n. 26
0
    def _get_exif_object(self):
        if (not hasattr(self, "exif")) or self.exif is None:
            return None

        try:
            return ExifOrientationEditor(self.exif)
        except Exception as error:
            logger.exception("[exif] %s", error)

        return None
Esempio n. 27
0
 def _execute_in_foreground(self, operation, callback):
     result = Future()
     returned = None
     try:
         returned = operation()
     except Exception as e:
         # just log exception and release ioloop
         returned = e
         logger.exception(e)
     result.set_result(returned)
     callback(result)
Esempio n. 28
0
 def _execute_in_foreground(self, operation, callback):
     result = Future()
     returned = None
     try:
         returned = operation()
     except Exception as e:
         # just log exception and release ioloop
         returned = e
         logger.exception(e)
     result.set_result(returned)
     callback(result)
Esempio n. 29
0
    def _get_exif_object(self):
        if (not hasattr(self, 'exif')) or self.exif is None:
            return None

        try:
            return ExifOrientationEditor(self.exif)
        except Exception as e:
            msg = """[exif] %s""" % e
            logger.exception(msg)

        return None
Esempio n. 30
0
    def _get_exif_segment(self):
        if (not hasattr(self, 'exif')) or self.exif is None:
            return None

        try:
            segment = ExifSegment(None, None, self.exif, 'ro')
        except Exception:
            logger.exception('Ignored error handling exif for reorientation')
        else:
            return segment
        return None
Esempio n. 31
0
    def _get_exif_segment(self):
        if (not hasattr(self, 'exif')) or self.exif is None:
            return None

        try:
            exif_dict = piexif.load(self.exif)
        except Exception:
            logger.exception('Ignored error handling exif for reorientation')
        else:
            return exif_dict
        return None
Esempio n. 32
0
    def _get_exif_segment(self):
        if (not hasattr(self, 'exif')) or self.exif is None:
            return None

        try:
            segment = ExifSegment(None, None, self.exif, 'ro')
        except Exception:
            logger.exception('Ignored error handling exif for reorientation')
        else:
            return segment
        return None
Esempio n. 33
0
    def _execute_in_foreground(self, operation, callback):
        result = Future()

        try:
            returned = operation()
        except Exception as e:
            logger.exception('[ThreadPool] %s', e)
            result.set_exception(e)
        else:
            result.set_result(returned)

        callback(result)
Esempio n. 34
0
    def _execute_in_foreground(self, operation, callback):
        result = Future()

        try:
            returned = operation()
        except Exception as e:
            logger.exception('[ThreadPool] %s', e)
            result.set_exception(e)
        else:
            result.set_result(returned)

        callback(result)
Esempio n. 35
0
    def execute_image_operations(self):
        self.context.request.quality = None

        req = self.context.request
        conf = self.context.config

        should_store = self.context.config.RESULT_STORAGE_STORES_UNSAFE or not self.context.request.unsafe
        if self.context.modules.result_storage and should_store:
            start = datetime.datetime.now()

            try:
                result = yield gen.maybe_future(
                    self.context.modules.result_storage.get())
            except Exception as e:
                logger.exception('[BaseHander.execute_image_operations] %s', e)
                self._error(
                    500,
                    'Error while trying to get the image from the result storage: {}'
                    .format(e))
                return

            finish = datetime.datetime.now()

            self.context.metrics.timing('result_storage.incoming_time',
                                        (finish - start).total_seconds() *
                                        1000)

            if result is None:
                self.context.metrics.incr('result_storage.miss')
            else:
                self.context.metrics.incr('result_storage.hit')
                self.context.metrics.incr('result_storage.bytes_read',
                                          len(result))
                logger.debug('[RESULT_STORAGE] IMAGE FOUND: %s' % req.url)
                self.finish_request(result)
                return

        if conf.MAX_WIDTH and (not isinstance(
                req.width, basestring)) and req.width > conf.MAX_WIDTH:
            req.width = conf.MAX_WIDTH
        if conf.MAX_HEIGHT and (not isinstance(
                req.height, basestring)) and req.height > conf.MAX_HEIGHT:
            req.height = conf.MAX_HEIGHT

        req.meta_callback = conf.META_CALLBACK_NAME or self.request.arguments.get(
            'callback', [None])[0]

        self.filters_runner = self.context.filters_factory.create_instances(
            self.context, self.context.request.filters)
        # Apply all the filters from the PRE_LOAD phase and call get_image() afterwards.
        self.filters_runner.apply_filters(thumbor.filters.PHASE_PRE_LOAD,
                                          self.get_image)
Esempio n. 36
0
        def inner(future):
            try:
                future_result = future.result()
            except Exception as e:
                logger.exception('[BaseHander.finish_request] %s', e)
                self._error(500, 'Error while trying to fetch the image: {}'.format(e))
                return

            results, content_type = future_result
            self._write_results_to_client(context, results, content_type)

            if should_store:
                self._store_results(context, results)
Esempio n. 37
0
    def put_crypto(self, path):
        if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE:
            return

        if not self.context.server.security_key:
            raise RuntimeError("STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified")

        key = self.crypto_key_for(path)
        try:
            self.storage.set(key, self.context.server.security_key)
        except:
            logger.exception("[MEMCACHED] failed to set key '{0}'".format(key));
        return key
Esempio n. 38
0
        def inner(future):
            try:
                future_result = future.result()
            except Exception as e:
                logger.exception('[BaseHander.finish_request] %s', e)
                self._error(500, 'Error while trying to fetch the image: {}'.format(e))
                return

            results, content_type = future_result
            self._write_results_to_client(results, content_type)

            if should_store:
                tornado.ioloop.IOLoop.instance().add_callback(self._store_results, result_storage, metrics, results)
Esempio n. 39
0
        def inner(future):
            try:
                future_result = future.result()
            except Exception as e:
                logger.exception('[BaseHander.finish_request] %s', e)
                self._error(500, 'Error while trying to fetch the image: {}'.format(e))
                return

            results, content_type = future_result
            self._write_results_to_client(results, content_type)

            if should_store:
                tornado.ioloop.IOLoop.instance().add_callback(self._store_results, result_storage, metrics, results)
Esempio n. 40
0
    def icc_profile_apply(self, profile=None):
        # Check whether input image has color management.
        if not self.engine.icc_profile:
            logger.info('ICC: Image has no embedded profile. Skipping this image.')
            return

        # Sanitize profile parameter.
        if profile != None:
            profile = os.path.basename(profile).lstrip('.')
            if len(profile) == 0:
                logger.warning('ICC: Invalid profile name.')
                return

        # Find output profile.
        outprofile = self._find_profile(profile)
        if not outprofile:
            logger.warning('ICC: Failed to load profile: {:s}'.format(profile))
            return

        try:
            ext = self.engine.extension
            fmt = Image.EXTENSION[ext.lower()]
        except:
            logger.exception('ICC: Failed to determine image format and extension before attempting to apply profile: {:s}'.format(profile))
            return

        try:
            inmode = self.engine.get_image_mode()
            insize = self.engine.size
            inimg = Image.frombytes(inmode, insize, self.engine.get_image_data())

            # In PIL>=3.0.0 / Thumbor 6, icc_profile is sometimes a tuple :/
            # https://github.com/python-pillow/Pillow/issues/1462
            profile_data = self.engine.icc_profile
            if type(profile_data) == tuple:
                profile_data = profile_data[0]
            inprofile = StringIO(profile_data)

            outmode = 'RGBA' if 'A' in inmode else 'RGB'
        except:
            logger.exception('ICC: Failed to determine image properties before attempting to apply profile: {:s}'.format(profile))
            return

        logger.info('ICC: Attempting to apply profile: {:s}, inmode: {:s}, outmode: {:s}'.format(profile, inmode, outmode))
        try:
            outimg = ImageCms.profileToProfile(inimg, inprofile, outprofile, outputMode=outmode)
        except:
            logger.exception('ICC: Failed to apply profile: {:s}, inmode: {:s}, outmode: {:s}'.format(profile, inmode, outmode))
            return

        # Reload the image into the engine.
        outbuf = StringIO()
        try:
            outimg.save(outbuf, fmt)
            self.engine.load(outbuf.getvalue(), ext)
        except:
            logger.exception('ICC: Failed load the image with an applied profile: {:s}, inmode: {:s}, outmode: {:s}'.format(profile, inmode, outmode))
            return
        finally:
            outbuf.close()
Esempio n. 41
0
    def read(self, extension=None, quality=None):
        if quality is None:
            quality = self.context.request.quality
        #returns image buffer in byte format.
        img_buffer = BytesIO()

        ext = extension or self.extension
        options = {
            'quality': quality
        }
        if ext == '.jpg' or ext == '.jpeg':
            options['optimize'] = True
            options['progressive'] = True

        if self.icc_profile is not None:
            options['icc_profile'] = self.icc_profile

        if self.context.config.PRESERVE_EXIF_INFO:
            exif = self.image.info.get('exif', None)
            if exif is not None:
                options['exif'] = exif

        image_format = self.context.request.format
        if image_format is None:
            image_format = FORMATS[ext]
        image_format = str(image_format).upper()

        try:
            if image_format == 'WEBP' and self.image.mode in ['L', 'LA', 'P', 'RGBA']:
                self.image = self.image.convert('RGB')

            self.image.save(img_buffer, image_format, **options)
        except IOError:
            logger.exception('Could not save as improved image, consider to increase ImageFile.MAXBLOCK')
            self.image.save(img_buffer, FORMATS[ext])
        except KeyError:
            logger.exception('Image format not found in PIL: %s' % image_format)
            #extension is not present or could not help determine format => force JPEG
            #TODO : guess format by image headers maybe
            if self.image.mode in ['P', 'RGBA', 'LA']:
                self.image.format = FORMATS['.png']
                self.image.save(img_buffer, FORMATS['.png'])
            else:
                self.image.format = FORMATS['.jpg']
                self.image.save(img_buffer, FORMATS['.jpg'])

        results = img_buffer.getvalue()
        img_buffer.close()
        return results
Esempio n. 42
0
    def put_crypto(self, path):
        if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE:
            return

        if not self.context.server.security_key:
            raise RuntimeError(
                "STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified"
            )

        key = self.crypto_key_for(path)
        try:
            self.storage.set(key, self.context.server.security_key)
        except:
            logger.exception("[MEMCACHED] failed to set key '{0}'".format(key))
        return key
Esempio n. 43
0
        def inner(future):
            try:
                future_result = future.result()
            except Exception as e:
                logger.exception('[BaseHander.finish_request] %s', e)
                self._error(500, 'Error while trying to fetch the image: {}'.format(e))
                return

            results, content_type = future_result
            self._write_results_to_client(context, results, content_type)

            if should_store:
                self._store_results(context, results)

            schedule.run_pending()
Esempio n. 44
0
 def process(self, canvas_width, canvas_height, size):
     try:
         self.engine.load(self.buffer, self.extension)
         width, height = self.engine.size
         new_width, new_height = calc_new_size_by_height(width, height, canvas_height)
         focal_points = StandaloneFaceDetector.features_to_focal_points(
             StandaloneFaceDetector.get_features(self.thumbor_filter.context, self.engine))
         if focal_points:
             self.resize_focal_points(focal_points, float(new_width) / width)
         else:
             focal_points.append(FocalPoint.from_alignment('center', 'top', new_width, new_height))
         self.engine.resize(new_width, new_height)
         self.engine.focus(focal_points)
         StandaloneFaceDetector.auto_crop(self.engine, focal_points, size, canvas_height)
     except Exception as err:
         logger.exception(err)
Esempio n. 45
0
    def save_on_disc(self):
        if self.fetched:
            try:
                self.engine.load(self.buffer, self.extension)
            except Exception as err:
                self.failed = True
                logger.exception(err)

            try:
                self.thumbor_filter.storage.put(self.url, self.engine.read())
                self.thumbor_filter.storage.put_crypto(self.url)
            except Exception as err:
                self.failed = True
                logger.exception(err)
        else:
            self.failed = True
            logger.error("filters.distributed_collage: Can't save unfetched image")
Esempio n. 46
0
    def detect(self, callback):
        self.context.request.prevent_result_storage = True
        try:
            if not Detector.detect_task:
                celery_tasks = CeleryTasks(
                    self.context.config.SQS_QUEUE_KEY_ID,
                    self.context.config.SQS_QUEUE_KEY_SECRET,
                    self.context.config.SQS_QUEUE_REGION, None
                )
                Detector.detect_task = celery_tasks.get_detect_task()

            Detector.detect_task.delay('all', self.context.request.image_url, self.context.request.image_url)
        except RuntimeError:
            self.context.request.detection_error = True
            Detector.detect_task = None
            logger.exception('Celery Error')
        finally:
            callback([])
Esempio n. 47
0
    def detect(self, callback):
        try:
            features = self.get_features()
        except Exception as e:
            logger.exception(e)
            logger.warn('Error during face detection; skipping to next detector')
            self.next(callback)
            return

        if features:
            for (left, top, width, height), neighbors in features:
                top = self.__add_hair_offset(top, height)
                self.context.request.focal_points.append(
                    FocalPoint.from_square(left, top, width, height, origin="Face Detection")
                )
            callback()
        else:
            self.next(callback)
Esempio n. 48
0
    def save_on_disc(self):
        if self.fetched:
            try:
                self.engine.load(self.buffer, None)
            except Exception as err:
                self.failed = True
                logger.exception(err)

            try:
                self.thumbor_filter.storage.put(self.url, self.engine.read())
                self.thumbor_filter.storage.put_crypto(self.url)
            except Exception as err:
                self.failed = True
                logger.exception(err)
        else:
            self.failed = True
            logger.error(
                "filters.distributed_collage: Can't save unfetched image")
Esempio n. 49
0
    def detect(self, callback):
        self.context.request.prevent_result_storage = True
        try:
            if not QueuedDetector.redis:
                QueuedDetector.redis = Redis(host=self.context.config.REDIS_QUEUE_SERVER_HOST,
                                             port=self.context.config.REDIS_QUEUE_SERVER_PORT,
                                             db=self.context.config.REDIS_QUEUE_SERVER_DB,
                                             password=self.context.config.REDIS_QUEUE_SERVER_PASSWORD)

            queue = UniqueQueue(server=QueuedDetector.redis)
            queue.enqueue_unique_from_string('remotecv.pyres_tasks.DetectTask', 'Detect',
                    args=[self.detection_type, self.context.request.image_url],
                    key=self.context.request.image_url)
        except RedisError:
            self.context.request.detection_error = True
            QueuedDetector.redis = None
            logger.exception('Redis Error')
        finally:
            callback([])
Esempio n. 50
0
    def get_orientation(self, override_exif=True):
        if (not hasattr(self, 'exif')) or self.exif is None:
            return

        orientation = None

        try:
            segment = ExifSegment(None, None, self.exif, 'ro')
            primary = segment.primary
            orientation = primary['Orientation']
            if orientation:
                orientation = orientation[0]
                if orientation != 1 and override_exif:
                    primary['Orientation'] = [1]
                    self.exif = segment.get_data()
        except Exception:
            logger.exception('Ignored error handling exif for reorientation')
        finally:
            return orientation
Esempio n. 51
0
def get_video_frame(context, file_path):
    """
    A context manager that extracts a single frame out of a video file and 
    stores it in a temporary file. Returns the path of the temporary file
    or None in case of failure.
    Depends on FFMPEG_PATH from Thumbor's configuration.
    """
    import subprocess, tempfile, os

    # Fail nicely when ffmpeg cannot be found
    if not os.path.exists(context.config.FFMPEG_PATH):
        logger.error("%s does not exist, please configure FFMPEG_PATH", context.config.FFMPEG_PATH)
        yield None
        return
    # Prepare temporary file
    f, image_path = tempfile.mkstemp(".jpg")
    os.close(f)
    # Extract image
    try:
        cmd = [
            context.config.FFMPEG_PATH,
            "-i",
            file_path,
            "-ss",
            "00:00:01.000",
            "-vframes",
            "1",
            "-y",
            "-nostats",
            "-loglevel",
            "error",
            image_path,
        ]
        subprocess.check_call(cmd)
        yield image_path
    except:
        logger.exception("Cannot extract image frame from %s", file_path)
        yield None
    finally:
        # Cleanup
        try_to_delete(image_path)
Esempio n. 52
0
    def watermark(self, callback, url, x, y, alpha, w_ratio=False, h_ratio=False):
        self.url = url
        self.x = x
        self.y = y
        self.alpha = alpha
        self.w_ratio = float(w_ratio) / 100.0 if w_ratio and w_ratio != 'none' else False
        self.h_ratio = float(h_ratio) / 100.0 if h_ratio and h_ratio != 'none' else False
        self.callback = callback
        self.watermark_engine = self.context.modules.engine.__class__(self.context)
        self.storage = self.context.modules.storage

        try:
            buffer = yield tornado.gen.maybe_future(self.storage.get(self.url))
            if buffer is not None:
                self.on_image_ready(buffer)
            else:
                self.context.modules.loader.load(self.context, self.url, self.on_fetch_done)
        except Exception as e:
            logger.exception(e)
            logger.warn("bad watermark")
            raise tornado.web.HTTPError(500)
Esempio n. 53
0
    def assembly(self):
        logger.debug('filters.distributed_collage: assembly started')
        canvas = self.create_engine()
        canvas_width, canvas_height = self.engine.size
        canvas.image = canvas.gen_image((canvas_width, canvas_height), '#fff')

        slice_size, major_slice_size = self.divide_size(canvas_width, len(self.images))
        for i, url in enumerate(self.urls):
            image = self.images[url]
            x, y = i * slice_size, 0
            if i == len(self.images) - 1:
                slice_size = major_slice_size
            try:
                image.process(canvas_width, canvas_height, slice_size)
                canvas.paste(image.engine, (x, y), merge=True)
            except Exception as err:
                logger.exception(err)

        self.engine.image = canvas.image
        logger.debug('filters.distributed_collage: assembly finished')
        self.callback()
Esempio n. 54
0
    def execute_image_operations(self):
        self.context.request.quality = None

        req = self.context.request
        conf = self.context.config

        should_store = self.context.config.RESULT_STORAGE_STORES_UNSAFE or not self.context.request.unsafe
        if self.context.modules.result_storage and should_store:
            start = datetime.datetime.now()

            try:
                result = yield gen.maybe_future(self.context.modules.result_storage.get())
            except Exception as e:
                logger.exception('[BaseHander.execute_image_operations] %s', e)
                self._error(500, 'Error while trying to get the image from the result storage: {}'.format(e))
                return

            finish = datetime.datetime.now()

            self.context.metrics.timing('result_storage.incoming_time', (finish - start).total_seconds() * 1000)

            if result is None:
                self.context.metrics.incr('result_storage.miss')
            else:
                self.context.metrics.incr('result_storage.hit')
                self.context.metrics.incr('result_storage.bytes_read', len(result))
                logger.debug('[RESULT_STORAGE] IMAGE FOUND: %s' % req.url)
                self.finish_request(result)
                return

        if conf.MAX_WIDTH and (not isinstance(req.width, basestring)) and req.width > conf.MAX_WIDTH:
            req.width = conf.MAX_WIDTH
        if conf.MAX_HEIGHT and (not isinstance(req.height, basestring)) and req.height > conf.MAX_HEIGHT:
            req.height = conf.MAX_HEIGHT

        req.meta_callback = conf.META_CALLBACK_NAME or self.request.arguments.get('callback', [None])[0]

        self.filters_runner = self.context.filters_factory.create_instances(self.context, self.context.request.filters)
        # Apply all the filters from the PRE_LOAD phase and call get_image() afterwards.
        self.filters_runner.apply_filters(thumbor.filters.PHASE_PRE_LOAD, self.get_image)
Esempio n. 55
0
    def reorientate(self):
        if (not hasattr(self, 'exif')) or self.exif is None:
            return

        orientation = None

        try:
            segment = ExifSegment(None, None, self.exif, 'ro')
            primary = segment.primary
            orientation = primary['Orientation']
            if orientation:
                orientation = orientation[0]
                if orientation != 1:
                    primary['Orientation'] = [1]
                    self.exif = segment.get_data()
        except Exception:
            logger.exception('Ignored error handling exif for reorientation')

        if orientation == 2:
            self.flip_horizontally()
        elif orientation == 3:
            self.rotate(180)
        elif orientation == 4:
            self.flip_vertically()
        elif orientation == 5:
            # Horizontal Mirror + Rotation 270
            self.flip_vertically()
            self.rotate(270)
        elif orientation == 6:
            self.rotate(270)
        elif orientation == 7:
            # Vertical Mirror + Rotation 270
            self.flip_horizontally()
            self.rotate(270)
        elif orientation == 8:
            self.rotate(90)
Esempio n. 56
0
    def read(self, extension=None, quality=None):  # NOQA
        # returns image buffer in byte format.
        img_buffer = BytesIO()
        ext = extension or self.extension or self.get_default_extension()

        options = {
            'quality': quality
        }
        if ext == '.jpg' or ext == '.jpeg':
            options['optimize'] = True
            if self.context.config.PROGRESSIVE_JPEG:
                # Can't simply set options['progressive'] to the value
                # of self.context.config.PROGRESSIVE_JPEG because save
                # operates on the presence of the key in **options, not
                # the value of that setting.
                options['progressive'] = True

            if self.image.mode != 'RGB':
                self.image = self.image.convert('RGB')
            else:
                subsampling_config = self.context.config.PILLOW_JPEG_SUBSAMPLING
                qtables_config = self.context.config.PILLOW_JPEG_QTABLES

                if subsampling_config is not None or qtables_config is not None:
                    options['quality'] = 0  # can't use 'keep' here as Pillow would try to extract qtables/subsampling and fail
                    orig_subsampling = self.subsampling
                    orig_qtables = self.qtables

                    if (subsampling_config == 'keep' or subsampling_config is None) and (orig_subsampling is not None):
                        options['subsampling'] = orig_subsampling
                    else:
                        options['subsampling'] = subsampling_config

                    if (qtables_config == 'keep' or qtables_config is None) and (orig_qtables and 2 <= len(orig_qtables) <= 4):
                        options['qtables'] = orig_qtables
                    else:
                        options['qtables'] = qtables_config

        if options['quality'] is None:
            options['quality'] = self.context.config.QUALITY

        if self.icc_profile is not None:
            options['icc_profile'] = self.icc_profile

        if self.context.config.PRESERVE_EXIF_INFO:
            if self.exif is not None:
                options['exif'] = self.exif

        if self.image.mode == 'P' and self.transparency:
            options['transparency'] = self.transparency

        try:
            if ext == '.webp':
                if self.image.mode not in ['RGB', 'RGBA']:
                    mode = None
                    if self.image.mode == 'P':
                        mode = 'RGBA'
                    else:
                        mode = 'RGBA' if self.image.mode[-1] == 'A' else 'RGB'
                    self.image = self.image.convert(mode)

            if ext in ['.png', '.gif'] and self.image.mode == 'CMYK':
                self.image = self.image.convert('RGBA')
            self.image.format = FORMATS[ext]
            self.image.save(img_buffer, FORMATS[ext], **options)
        except IOError:
            logger.exception('Could not save as improved image, consider to increase ImageFile.MAXBLOCK')
            self.image.save(img_buffer, FORMATS[ext])
        except KeyError:
            logger.exception('Image format not found in PIL: %s' % ext)
            ext = self.get_default_extension()
            # extension could not help determine format => use default
            self.image.format = FORMATS[ext]
            self.image.save(img_buffer, FORMATS[ext])

        results = img_buffer.getvalue()
        img_buffer.close()
        self.extension = ext
        return results
Esempio n. 57
0
    def read(self, extension=None, quality=None):
        #returns image buffer in byte format.
        img_buffer = BytesIO()

        ext = extension or self.extension

        options = {
            'quality': quality
        }

        if ext == '.jpg' or ext == '.jpeg':
            options['optimize'] = True
            if self.context.config.PROGRESSIVE_JPEG:
                # Can't simply set options['progressive'] to the value
                # of self.context.config.PROGRESSIVE_JPEG because save
                # operates on the presence of the key in **options, not
                # the value of that setting.
                options['progressive'] = True

            if self.image.mode != 'RGB':
                self.image = self.image.convert('RGB')
            else:
                if self.extension == '.jpg':
                    quantization = getattr(self.image, 'quantization', None)
                    if quality is None and quantization and 2 <= len(quantization) <= 4:
                        options['quality'] = 'keep'

        if options['quality'] is None:
            options['quality'] = self.context.config.QUALITY

        if self.icc_profile is not None:
            options['icc_profile'] = self.icc_profile

        if self.context.config.PRESERVE_EXIF_INFO:
            if self.exif is not None:
                options['exif'] = self.exif

        if self.image.mode == 'P' and self.transparency:
            options['transparency'] = self.transparency

        try:
            if ext == '.webp':
                if self.image.mode not in ['RGB', 'RGBA']:
                    mode = None
                    if self.image.mode != 'P':
                        mode = 'RGBA' if self.image.mode[-1] == 'A' else 'RGB'
                    self.image = self.image.convert(mode)

            if ext == '.png' and self.image.mode == 'CMYK':
                self.image = self.image.convert('RGBA')

            self.image.save(img_buffer, FORMATS[ext], **options)
        except IOError:
            logger.exception('Could not save as improved image, consider to increase ImageFile.MAXBLOCK')
            self.image.save(img_buffer, FORMATS[ext])
        except KeyError:
            logger.exception('Image format not found in PIL: %s' % ext)

            #extension is not present or could not help determine format => force JPEG
            if self.image.mode in ['P', 'RGBA', 'LA']:
                self.image.format = FORMATS['.png']
                self.image.save(img_buffer, FORMATS['.png'])
            else:
                self.image.format = FORMATS['.jpg']
                self.image.save(img_buffer, FORMATS['.jpg'])

        results = img_buffer.getvalue()
        img_buffer.close()
        return results