コード例 #1
0
ファイル: jpegtran.py プロジェクト: Bladrak/thumbor
    def run_optimizer(self, image_extension, buffer):
        if not self.should_run(image_extension, buffer):
            return buffer

        command = [
            self.context.config.JPEGTRAN_PATH,
            '-copy',
            'comments',
            '-optimize',
        ]

        if self.context.config.PROGRESSIVE_JPEG:
            command += [
                '-progressive'
            ]

        jpg_process = Popen(command, stdin=PIPE, stdout=PIPE, stderr=PIPE)
        output_stdout, output_stderr = jpg_process.communicate(buffer)

        if jpg_process.returncode != 0:
            logger.warn('jpegtran finished with non-zero return code (%d): %s'
                        % (jpg_process.returncode, output_stderr))
            return buffer

        return output_stdout
コード例 #2
0
    def import_item(self, config_key=None, class_name=None, is_multiple=False, item_value=None, ignore_errors=False):
        if item_value is None:
            conf_value = getattr(self.config, config_key)
        else:
            conf_value = item_value

        if is_multiple:
            modules = []
            if conf_value:
                for module_name in conf_value:
                    try:
                        if class_name is not None:
                            module = self.import_class('%s.%s' % (module_name, class_name))
                        else:
                            module = self.import_class(module_name, get_module=True)
                        modules.append(module)
                    except ImportError:
                        if ignore_errors:
                            logger.warn('Module %s could not be imported.' % module_name)
                        else:
                            raise
            setattr(self, config_key.lower(), tuple(modules))
        else:
            if class_name is not None:
                module = self.import_class('%s.%s' % (conf_value, class_name))
            else:
                module = self.import_class(conf_value, get_module=True)
            setattr(self, config_key.lower(), module)
コード例 #3
0
    def _process_result_from_storage(self, result):
        if self.context.config.SEND_IF_MODIFIED_LAST_MODIFIED_HEADERS:
            # Handle If-Modified-Since & Last-Modified header
            try:
                if isinstance(result, ResultStorageResult):
                    result_last_modified = result.last_modified
                else:
                    result_last_modified = yield gen.maybe_future(
                        self.context.modules.result_storage.last_updated())

                if result_last_modified:
                    if 'If-Modified-Since' in self.request.headers:
                        logger.debug('LASTMODIFIED HEADER CHECKED')
                        date_modified_since = datetime.datetime.strptime(
                            self.request.headers['If-Modified-Since'],
                            HTTP_DATE_FMT).replace(tzinfo=pytz.utc)

                        logger.debug(date_modified_since)
                        logger.debug(result_last_modified)
                        if result_last_modified <= date_modified_since:
                            logger.debug('LASTMODIFIED SET STATUS')
                            self.set_status(304)
                            self.finish()
                            return

                    self.set_header(
                        'Last-Modified',
                        result_last_modified.strftime(HTTP_DATE_FMT))
            except NotImplementedError:
                logger.warn(
                    'last_updated method is not supported by your result storage service, hence If-Modified-Since & '
                    'Last-Updated headers support is disabled.')
コード例 #4
0
ファイル: file_storage.py プロジェクト: sshyran/thumbor
    def get(self, callback):
        path = self.context.request.url
        file_abspath = self.normalize_path(path)
        if not self.validate_path(file_abspath):
            logger.warn(
                "[RESULT_STORAGE] unable to read from outside root path: %s" %
                file_abspath)
            return None
        logger.debug("[RESULT_STORAGE] getting from %s" % file_abspath)

        if not exists(file_abspath) or self.is_expired(file_abspath):
            logger.debug("[RESULT_STORAGE] image not found at %s" %
                         file_abspath)
            callback(None)
        else:
            with open(file_abspath, 'r') as f:
                buffer = f.read()

            result = ResultStorageResult(
                buffer=buffer,
                metadata={
                    'LastModified':
                    datetime.fromtimestamp(
                        getmtime(file_abspath)).replace(tzinfo=pytz.utc),
                    'ContentLength':
                    len(buffer),
                    'ContentType':
                    BaseEngine.get_mimetype(buffer)
                })

            callback(result)
コード例 #5
0
def load(context, url, callback):
    enable_http_loader = context.config.get('AWS_ENABLE_HTTP_LOADER',
                                            default=False)

    if enable_http_loader and url.startswith('http'):
        return http_loader.load_sync(context,
                                     url,
                                     callback,
                                     normalize_url_func=_normalize_url)

    url = urllib2.unquote(url)

    bucket = context.config.get('S3_LOADER_BUCKET', default=None)

    if not bucket:
        bucket, url = _get_bucket(url,
                                  root_path=context.config.S3_LOADER_ROOT_PATH)

    if _validate_bucket(context, bucket):
        bucket_loader = Bucket(connection=get_connection(context), name=bucket)
        file_key = None
        try:
            file_key = bucket_loader.get_key(url)
        except Exception, e:
            logger.warn("ERROR retrieving image from S3 {0}: {1}".format(
                url, str(e)))

        if file_key:
            callback(file_key.read())
            return
コード例 #6
0
ファイル: http_loader.py プロジェクト: fu2re/thumbor-video
def return_contents(response, url, callback, context, req_start=None):
    if req_start:
        finish = datetime.datetime.now()
        res = urlparse(url)
        context.metrics.timing(
            'original_image.fetch.{0}.{1}'.format(response.code, res.netloc),
            (finish - req_start).total_seconds() * 1000
        )

    result = LoaderResult()
    context.metrics.incr('original_image.status.' + str(response.code))
    if response.error:
        result.successful = False
        if response.code == 599:
            # Return a Gateway Timeout status downstream if upstream times out
            result.error = LoaderResult.ERROR_TIMEOUT
        else:
            result.error = LoaderResult.ERROR_NOT_FOUND

        logger.warn(u"ERROR retrieving image {0}: {1}".format(url, str(response.error)))

    elif response.body is None or len(response.body) == 0:
        result.successful = False
        result.error = LoaderResult.ERROR_UPSTREAM

        logger.warn(u"ERROR retrieving image {0}: Empty response.".format(url))
    else:
        if response.time_info:
            for x in response.time_info:
                context.metrics.timing('original_image.time_info.' + x, response.time_info[x] * 1000)
            context.metrics.timing('original_image.time_info.bytes_per_second', len(response.body) / response.time_info['total'])
        result.buffer = response.body
        context.metrics.incr('original_image.response_bytes', len(response.body))

    callback(result)
コード例 #7
0
    def _process_result_from_storage(self, result):
        if self.context.config.SEND_IF_MODIFIED_LAST_MODIFIED_HEADERS:
            # Handle If-Modified-Since & Last-Modified header
            try:
                result_last_modified = self.context.modules.result_storage.last_updated(
                )
            except NotImplementedError:
                logger.warn(
                    'last_updated method is not supported by your result storage service, hence If-Modified-Since & Last-Updated headers support is disabled.'
                )

            if result_last_modified:
                if 'If-Modified-Since' in self.request.headers:
                    date_modified_since = datetime.datetime.strptime(
                        self.request.headers['If-Modified-Since'],
                        HTTP_DATE_FMT)

                    if result_last_modified <= date_modified_since:
                        self.set_status(304)
                        self.finish()
                        return

                self.set_header('Last-Modified',
                                result_last_modified.strftime(HTTP_DATE_FMT))

        return result
コード例 #8
0
ファイル: http_loader.py プロジェクト: gi11es/thumbor-debian
def return_contents(response, url, callback, context, req_start=None):
    if req_start:
        finish = datetime.datetime.now()
        res = urlparse(url)
        context.metrics.timing(
            'original_image.fetch.{0}.{1}'.format(response.code, res.netloc),
            (finish - req_start).total_seconds() * 1000
        )

    result = LoaderResult()
    context.metrics.incr('original_image.status.' + str(response.code))
    if response.error:
        result.successful = False
        if response.code == 599:
            # Return a Gateway Timeout status downstream if upstream times out
            result.error = LoaderResult.ERROR_TIMEOUT
        else:
            result.error = LoaderResult.ERROR_NOT_FOUND

        logger.warn(u"ERROR retrieving image {0}: {1}".format(url, str(response.error)))

    elif response.body is None or len(response.body) == 0:
        result.successful = False
        result.error = LoaderResult.ERROR_UPSTREAM

        logger.warn(u"ERROR retrieving image {0}: Empty response.".format(url))
    else:
        if response.time_info:
            for x in response.time_info:
                context.metrics.timing('original_image.time_info.' + x, response.time_info[x] * 1000)
            context.metrics.timing('original_image.time_info.bytes_per_second', len(response.body) / response.time_info['total'])
        result.buffer = response.body
        context.metrics.incr('original_image.response_bytes', len(response.body))

    callback(result)
コード例 #9
0
ファイル: __init__.py プロジェクト: antoinebqain/thumbor3
    def reload_to_fit_in_kb(self, engine, initial_results, extension,
                            initial_quality, max_bytes):
        logger.warn('This is the reload step')
        if extension not in ['.webp', '.jpg', '.jpeg'
                             ] or len(initial_results) <= max_bytes:
            return initial_results

        results = initial_results
        quality = initial_quality

        while len(results) > max_bytes:
            quality = int(quality * 0.75)

            if quality < 10:
                logger.debug(
                    'Could not find any reduction that matches required size of %d bytes.'
                    % max_bytes)
                return initial_results

            logger.debug('Trying to downsize image with quality of %d...' %
                         quality)
            results = engine.read(extension, quality)

        prev_result = results
        while len(results) <= max_bytes:
            quality = int(quality * 1.1)
            logger.debug('Trying to upsize image with quality of %d...' %
                         quality)
            prev_result = results
            results = engine.read(extension, quality)

        return prev_result
コード例 #10
0
ファイル: __init__.py プロジェクト: kivicode/Navigation
    def detect(self, callback):
        engine = self.context.modules.engine
        try:
            img = np.array(
                engine.convert_to_grayscale(update_image=False,
                                            with_alpha=False))
        except Exception as e:
            logger.exception(e)
            logger.warn(
                'Error during feature detection; skipping to next detector')
            self.next(callback)
            return

        points = cv2.goodFeaturesToTrack(
            img,
            maxCorners=20,
            qualityLevel=0.04,
            minDistance=1.0,
            useHarrisDetector=False,
        )
        if points is not None:
            for point in points:
                x, y = point.ravel()
                self.context.request.focal_points.append(
                    FocalPoint(x.item(), y.item(), 1))
            callback()
        else:
            self.next(callback)
コード例 #11
0
ファイル: http_loader.py プロジェクト: 5um1th/thumbor
def return_contents(response, url, callback, context):
    result = LoaderResult()

    context.metrics.incr('original_image.status.' + str(response.code))
    if response.error:
        result.successful = False
        if response.code == 599:
            # Return a Gateway Timeout status downstream if upstream times out
            result.error = LoaderResult.ERROR_TIMEOUT
        else:
            result.error = LoaderResult.ERROR_NOT_FOUND

        logger.warn(u"ERROR retrieving image {0}: {1}".format(url, str(response.error)))

    elif response.body is None or len(response.body) == 0:
        result.successful = False
        result.error = LoaderResult.ERROR_UPSTREAM

        logger.warn(u"ERROR retrieving image {0}: Empty response.".format(url))
    else:
        if response.time_info:
            for x in response.time_info:
                context.metrics.timing('original_image.time_info.' + x, response.time_info[x] * 1000)
            context.metrics.timing('original_image.time_info.bytes_per_second', len(response.body) / response.time_info['total'])
        result.buffer = response.body

    callback(result)
コード例 #12
0
 def return_data(file_key):
     if not file_key or self._get_error(file_key) or self.is_expired(
             file_key) or 'Body' not in file_key:
         logger.warn("[AwsStorage] s3 key not found at %s" % path)
         return None
     else:
         return loads(file_key['Body'].read())
コード例 #13
0
ファイル: storage.py プロジェクト: eduherraiz/aws
 def _handle_error(self, response):
     """
     Logs error if necessary
     :param dict response: AWS Response
     """
     if self._get_error(response):
         logger.warn("[STORAGE] error occured while storing data: %s" % self._get_error(response))
コード例 #14
0
 def _handle_error(self, response):
     """
     Logs error if necessary
     :param dict response: AWS Response
     """
     if self._get_error(response):
         logger.warn("[STORAGE] error occured while storing data: %s" % self._get_error(response))
コード例 #15
0
ファイル: srgb.py プロジェクト: PopSugar/thumbor-plugins
    def srgb(self):
        try:
            if not isinstance(self.engine, PILEngine):
                logger.warn('Could not perform profileToProfile conversion: engine is not PIL engine')
                return

            if (ImageCms is None):
                logger.warn('ImageCms is not installed. Could not perform profileToProfile conversion')
                return

            image = self.engine.image

            embedded_profile = image.info.get('icc_profile')

            if not embedded_profile:
                logger.debug('Image does not have embedded profile. Assuming already in sRGB')
                return

            embedded_profile = BytesIO(embedded_profile)
            srgb_profile = BytesIO(tiny_srgb)

            output_mode = 'RGBA' if 'A' in image.mode else 'RGB'
            image = ImageCms.profileToProfile(image, embedded_profile, srgb_profile, renderingIntent=0,
                                              outputMode=output_mode)

            self.engine.image = image
            self.engine.icc_profile = image.info.get('icc_profile')
        except Exception as err:
            logger.exception(err)
コード例 #16
0
ファイル: watermark.py プロジェクト: kivicode/Navigation
    def watermark(self,
                  callback,
                  url,
                  x,
                  y,
                  alpha,
                  w_ratio=False,
                  h_ratio=False):
        self.url = url
        self.x = x
        self.y = y
        self.alpha = alpha
        self.w_ratio = float(
            w_ratio) / 100.0 if w_ratio and w_ratio != 'none' else False
        self.h_ratio = float(
            h_ratio) / 100.0 if h_ratio and h_ratio != 'none' else False
        self.callback = callback
        self.watermark_engine = self.context.modules.engine.__class__(
            self.context)
        self.storage = self.context.modules.storage

        try:
            buffer = yield tornado.gen.maybe_future(self.storage.get(self.url))
            if buffer is not None:
                self.on_image_ready(buffer)
            else:
                self.context.modules.loader.load(self.context, self.url,
                                                 self.on_fetch_done)
        except Exception as e:
            logger.exception(e)
            logger.warn("bad watermark")
            raise tornado.web.HTTPError(500)
コード例 #17
0
ファイル: __init__.py プロジェクト: gi11es/thumbor-debian
    def detect(self, callback):
        engine = self.context.modules.engine
        try:
            img = np.array(
                engine.convert_to_grayscale(
                    update_image=False,
                    with_alpha=False
                )
            )
        except Exception:
            logger.warn('Error during feature detection; skipping to next detector')
            self.next(callback)
            return

        points = cv2.goodFeaturesToTrack(
            img,
            maxCorners=20,
            qualityLevel=0.04,
            minDistance=1.0,
            useHarrisDetector=False,
        )
        if points is not None:
            for x, y in points.squeeze():
                self.context.request.focal_points.append(FocalPoint(x.item(), y.item(), 1))
            callback()
        else:
            self.next(callback)
コード例 #18
0
def return_contents(response, url, callback, context):
    result = LoaderResult()

    context.metrics.incr('original_image.status.' + str(response.code))
    if response.error:
        result.successful = False
        if response.code == 599:
            # Return a Gateway Timeout status downstream if upstream times out
            result.error = LoaderResult.ERROR_TIMEOUT
        else:
            result.error = LoaderResult.ERROR_NOT_FOUND

        logger.warn("ERROR retrieving image {0}: {1}".format(
            url, str(response.error)))

    elif response.body is None or len(response.body) == 0:
        result.successful = False
        result.error = LoaderResult.ERROR_UPSTREAM

        logger.warn("ERROR retrieving image {0}: Empty response.".format(url))
    else:
        if response.time_info:
            for x in response.time_info:
                context.metrics.timing('original_image.time_info.' + x,
                                       response.time_info[x] * 1000)
            context.metrics.timing(
                'original_image.time_info.bytes_per_second',
                len(response.body) / response.time_info['total'])
        result.buffer = response.body

    callback(result)
コード例 #19
0
 def return_data(file_key):
     if not file_key or self._get_error(file_key) or self.is_expired(
             file_key) or 'Body' not in file_key:
         logger.warn("[STORAGE] s3 key not found at %s" % crypto_path)
         return None
     else:
         return file_key['Body']
コード例 #20
0
ファイル: __init__.py プロジェクト: nyimbi/thumbor
    def _process_result_from_storage(self, result):
        if self.context.config.SEND_IF_MODIFIED_LAST_MODIFIED_HEADERS:
            # Handle If-Modified-Since & Last-Modified header
            try:
                if isinstance(result, ResultStorageResult):
                    result_last_modified = result.last_modified
                else:
                    result_last_modified = yield gen.maybe_future(self.context.modules.result_storage.last_updated())

                if result_last_modified:
                    if "If-Modified-Since" in self.request.headers:
                        date_modified_since = datetime.datetime.strptime(
                            self.request.headers["If-Modified-Since"], HTTP_DATE_FMT
                        )

                        if result_last_modified <= date_modified_since:
                            self.set_status(304)
                            self.finish()
                            return

                    self.set_header("Last-Modified", result_last_modified.strftime(HTTP_DATE_FMT))
            except NotImplementedError:
                logger.warn(
                    "last_updated method is not supported by your result storage service, hence If-Modified-Since & "
                    "Last-Updated headers support is disabled."
                )
コード例 #21
0
ファイル: jpegtran.py プロジェクト: varshithr/pixrezol
    def run_optimizer(self, image_extension, buffer):
        if not self.should_run(image_extension, buffer):
            return buffer

        if 'strip_icc' in self.context.request.filters:
            copy_chunks = 'comments'
        else:
            # have to copy everything to preserve icc profile
            copy_chunks = 'all'

        command = [
            self.context.config.JPEGTRAN_PATH,
            '-copy',
            copy_chunks,
            '-optimize',
        ]

        if self.context.config.PROGRESSIVE_JPEG:
            command += ['-progressive']

        jpg_process = Popen(command, stdin=PIPE, stdout=PIPE, stderr=PIPE)
        output_stdout, output_stderr = jpg_process.communicate(buffer)

        if jpg_process.returncode != 0:
            logger.warn(
                'jpegtran finished with non-zero return code (%d): %s' %
                (jpg_process.returncode, output_stderr))
            return buffer

        return output_stdout
コード例 #22
0
ファイル: file_storage.py プロジェクト: RealGeeks/thumbor
    def get(self, callback):
        path = self.context.request.url
        file_abspath = self.normalize_path(path)
        if not self.validate_path(file_abspath):
            logger.warn("[RESULT_STORAGE] unable to read from outside root path: %s" % file_abspath)
            return None
        logger.debug("[RESULT_STORAGE] getting from %s" % file_abspath)

        if not exists(file_abspath) or self.is_expired(file_abspath):
            logger.debug("[RESULT_STORAGE] image not found at %s" % file_abspath)
            callback(None)
        else:
            with open(file_abspath, 'r') as f:
                buffer = f.read()

            result = ResultStorageResult(
                buffer=buffer,
                metadata={
                    'LastModified':  datetime.fromtimestamp(getmtime(file_abspath)).replace(tzinfo=pytz.utc),
                    'ContentLength': len(buffer),
                    'ContentType':   BaseEngine.get_mimetype(buffer)
                }
            )

            callback(result)
コード例 #23
0
ファイル: transformer.py プロジェクト: yswang0927/thumbor
    def trim(self):
        is_gifsicle = (self.context.request.engine.extension == '.gif' and self.context.config.USE_GIFSICLE_ENGINE)
        if self.context.request.trim is None or not trim_enabled or is_gifsicle:
            return

        mode, data = self.engine.image_data_as_rgb()
        box = _bounding_box.apply(
            mode,
            self.engine.size[0],
            self.engine.size[1],
            self.context.request.trim_pos,
            self.context.request.trim_tolerance,
            data
        )

        if box[2] < box[0] or box[3] < box[1]:
            logger.warn("Ignoring trim, there wouldn't be any image left, check the tolerance.")
            return

        self.engine.crop(box[0], box[1], box[2] + 1, box[3] + 1)
        if self.context.request.should_crop:
            self.context.request.crop['left'] -= box[0]
            self.context.request.crop['top'] -= box[1]
            self.context.request.crop['right'] -= box[0]
            self.context.request.crop['bottom'] -= box[1]
コード例 #24
0
def load(context, url, callback, normalize_url_func=_normalize_url):
    result = LoaderResult()
    start = time.perf_counter()

    try:
        result.buffer = ffmpeg(context, normalize_url_func(url))
    except subprocess.CalledProcessError as err:
        result.successful = False
        result.error = err.stderr.decode('utf-8').strip()

        logger.warn(f'ERROR retrieving image {url}: {result.error}')
        if result.error.lower().endswith(
                'Server returned 404 not found'.lower()):
            result.error = LoaderResult.ERROR_NOT_FOUND
    except Exception as err:
        result.successful = False
        result.error = str(err)
        logger.warn(f'ERROR retrieving image {url}: {err}')
    else:
        total_time = (time.perf_counter() - start)
        total_bytes = len(result.buffer)

        result.metadata.update({
            'size': total_bytes,
            # 'updated_at': datetime.datetime.utcnow(),
        })

        context.metrics.incr('original_image.status.200')
        context.metrics.incr('original_image.response_bytes', total_bytes)
        context.metrics.timing(f'original_image.fetch.{url}',
                               total_time * 1000)
        context.metrics.timing('original_image.time_info.bytes_per_second',
                               total_bytes / total_time)

    return callback(result)
コード例 #25
0
    def dispatch(self, file_key):
        """ Callback method for getObject from s3 """
        if not file_key or 'Error' in file_key or 'Body' not in file_key:

            logger.warn(
                "ERROR retrieving image from S3 {0}: {1}".
                format(self.key, str(file_key)))

            # If we got here, there was a failure.
            # We will return 404 if S3 returned a 404, otherwise 502.
            result = LoaderResult()
            result.successful = False

            if not file_key:
                result.error = LoaderResult.ERROR_UPSTREAM
                self.callback(result)
                return

            response_metadata = file_key.get('ResponseMetadata', {})
            status_code = response_metadata.get('HTTPStatusCode')

            if status_code == 404:
                result.error = LoaderResult.ERROR_NOT_FOUND
                self.callback(result)
                return

            if self.max_retries_counter < self.limit_max_retries:
                self.__increment_retry_counter()
                self.bucket_loader.get(self.key,
                                       callback=self.dispatch)
            else:
                result.error = LoaderResult.ERROR_UPSTREAM
                self.callback(result)
        else:
            self.callback(file_key['Body'].read())
コード例 #26
0
ファイル: s3_loader.py プロジェクト: Bladrak/aws
def load(context, url, callback):
    enable_http_loader = context.config.get('AWS_ENABLE_HTTP_LOADER', default=False)

    if enable_http_loader and url.startswith('http'):
        return http_loader.load_sync(context, url, callback, normalize_url_func=_normalize_url)

    url = urllib2.unquote(url)

    bucket = context.config.get('S3_LOADER_BUCKET', default=None)

    if not bucket:
        bucket, url = _get_bucket(url)

    if _validate_bucket(context, bucket):
        bucket_loader = Bucket(
            connection=thumbor_aws.connection.get_connection(context),
            name=bucket
        )
        file_key = None
        try:
            file_key = bucket_loader.get_key(url)
        except Exception, e:
            logger.warn("ERROR retrieving image from S3 {0}: {1}".format(url, str(e)))

        if file_key:
            callback(file_key.read())
            return
コード例 #27
0
ファイル: importer.py プロジェクト: dotscreen/thumbor
    def import_item(self, config_key=None, class_name=None, is_multiple=False, item_value=None, ignore_errors=False):
        if item_value is None:
            conf_value = getattr(self.config, config_key)
        else:
            conf_value = item_value

        if is_multiple:
            modules = []
            if conf_value:
                for module_name in conf_value:
                    try:
                        if class_name is not None:
                            module = self.import_class('%s.%s' % (module_name, class_name))
                        else:
                            module = self.import_class(module_name, get_module=True)
                        modules.append(module)
                    except ImportError:
                        if ignore_errors:
                            logger.warn('Module %s could not be imported.' % module_name)
                        else:
                            raise
            setattr(self, config_key.lower(), tuple(modules))
        else:
            if class_name is not None:
                module = self.import_class('%s.%s' % (conf_value, class_name))
            else:
                module = self.import_class(conf_value, get_module=True)
            setattr(self, config_key.lower(), module)
コード例 #28
0
    def detect(self, callback):
        engine = self.context.modules.engine
        try:
            engine.image_data_as_rgb()
            img = np.array(engine.image)
            self.net.setInput(
                cv2.dnn.blobFromImage(img, size=(300, 300), swapRB=True))
            detections = self.net.forward()
        except Exception as e:
            logger.exception(e)
            logger.warn(
                'Error during feature detection; skipping to next detector')
            self.next(callback)
            return

        confidence_threshold = 0.2
        num_detections = 0
        for detection in detections[0, 0, :, :]:
            confidence = float(detection[2])
            if confidence < confidence_threshold:
                continue
            num_detections += 1
            class_id = int(detection[1]) - 1  # make it zero-indexed
            class_name = coco_classes[class_id]
            left = int(detection[3] * img.shape[1])
            top = int(detection[4] * img.shape[0])
            right = int(detection[5] * img.shape[1])
            bottom = int(detection[6] * img.shape[0])
            width = right - left
            height = bottom - top
            # If the detection is of a person,
            # and the person is vertically oriented,
            # this uses the upper 1/4 of the box to focus on the face.
            # In the case the person is horizontal, perhaps reclining,
            # then the focal point will remain at their center.
            # In the case the person is upside down, this would focus on the feet instead of the face.
            # But consider - whoever is publishing a picture of an upside down person
            # might appreciate that it focuses on the feet.
            if class_name == 'person' and height > width:
                height = int(height * 0.25)
            self.context.request.focal_points.append(
                FocalPoint.from_dict({
                    'x':
                    left + (width / 2),
                    'y':
                    top + (height / 2),
                    'width':
                    width,
                    'height':
                    height,
                    'z':
                    confidence,
                    'origin':
                    'DNN Object Detection (class: {})'.format(class_name)
                }))
        if num_detections > 0:
            callback()
        else:
            self.next(callback)
コード例 #29
0
ファイル: jpegtran.py プロジェクト: caeugusmao/thumbor
 def should_run(self, image_extension, buffer):
     if image_extension in ['.jpg', '.jpeg']:
         if not exists(self.context.config.JPEGTRAN_PATH):
             logger.warn(
                 'jpegtran optimizer enabled but binary JPEGTRAN_PATH does not exist')
             return False
         return True
     return False
コード例 #30
0
 def on_file_fetched(file):
     if not file or self._get_error(file) or self.is_expired(
             file) or 'LastModified' not in file:
         logger.warn("[AwsStorage] s3 key not found at %s" %
                     file_abspath)
         return None
     else:
         return file['LastModified']
コード例 #31
0
ファイル: config.py プロジェクト: bguided/thumbor
 def __setattr__(self, name, value):
     if name in Config.class_aliased_items:
         logger.warn(
             "Option %s is marked as deprecated please use %s instead." % (name, Config.class_aliased_items[name])
         )
         self.__setattr__(Config.class_aliased_items[name], value)
     else:
         super(Config, self).__setattr__(name, value)
コード例 #32
0
 def on_file_fetched(file):
     if not file or self._get_error(file) or self.is_expired(
             file) or 'LastModified' not in file:
         logger.warn("[AwsStorage] s3 key not found at %s" %
                     file_abspath)
         callback(None)
     else:
         callback(self._utc_to_local(file['LastModified']))
コード例 #33
0
 def format(self, format):
     logger.warn('Setting format to %s' % format)
     if format.lower() not in ALLOWED_FORMATS:
         logger.debug('Format not allowed: %s' % format.lower())
         self.context.request.format = None
     else:
         logger.debug('Format specified: %s' % format.lower())
         self.context.request.format = format.lower()
コード例 #34
0
ファイル: gifv.py プロジェクト: caeugusmao/thumbor
 def should_run(self, image_extension, buffer):
     if 'gif' in image_extension and 'gifv' in self.context.request.filters:
         if not exists(self.context.config.FFMPEG_PATH):
             logger.warn(
                 'gifv optimizer enabled but binary FFMPEG_PATH does not exist')
             return False
         return True
     return False
コード例 #35
0
ファイル: __init__.py プロジェクト: antoinebqain/thumbor3
    def optimize(self, context, image_extension, results):
        logger.warn('This is the opt step')
        for optimizer in context.modules.optimizers:
            new_results = optimizer(context).run_optimizer(
                image_extension, results)
            if new_results is not None:
                results = new_results

        return results
コード例 #36
0
ファイル: http_loader.py プロジェクト: iperelivskiy/thumbor
def return_contents(response, url, callback):
    if response.error:
        logger.warn("ERROR retrieving image {0}: {1}".format(url, str(response.error)))
        callback(None)
    elif response.body is None or len(response.body) == 0:
        logger.warn("ERROR retrieving image {0}: Empty response.".format(url))
        callback(None)
    else:
        callback(response.body)
コード例 #37
0
ファイル: http_loader.py プロジェクト: joevandyk/thumbor
def return_contents(response, url, callback):
    if response.error:
        logger.warn("ERROR retrieving image {0}: {1}".format(url, str(response.error)))
        callback(None)
    elif response.body is None or len(response.body) == 0:
        logger.warn("ERROR retrieving image {0}: Empty response.".format(url))
        callback(None)
    else:
        callback(response.body)
コード例 #38
0
 def should_run(self, image_extension, buffer):
     if 'gif' in image_extension and 'gifv' in self.context.request.filters:
         if not exists(self.context.config.FFMPEG_PATH):
             logger.warn(
                 'gifv optimizer enabled but binary FFMPEG_PATH does not exist'
             )
             return False
         return True
     return False
コード例 #39
0
 def should_run(self, image_extension, buffer):
     if image_extension in ['.jpg', '.jpeg']:
         if not exists(self.context.config.JPEGTRAN_PATH):
             logger.warn(
                 'jpegtran optimizer enabled but binary JPEGTRAN_PATH does not exist'
             )
             return False
         return True
     return False
コード例 #40
0
ファイル: __init__.py プロジェクト: caeugusmao/thumbor
    def validate(self, path):
        if not hasattr(self.context.modules.loader, 'validate'):
            return True

        is_valid = self.context.modules.loader.validate(self.context, path)

        if not is_valid:
            logger.warn('Request denied because the specified path "%s" was not identified by the loader as a valid path' % path)

        return is_valid
コード例 #41
0
    def validate(self, path):
        if not hasattr(self.context.modules.loader, 'validate'):
            return True

        is_valid = self.context.modules.loader.validate(self.context, path)

        if not is_valid:
            logger.warn('Request denied because the specified path "%s" was not identified by the loader as a valid path' % path)

        return is_valid
コード例 #42
0
ファイル: file_storage.py プロジェクト: RealGeeks/thumbor
    def last_updated(self):
        path = self.context.request.url
        file_abspath = self.normalize_path(path)
        if not self.validate_path(file_abspath):
            logger.warn("[RESULT_STORAGE] unable to read from outside root path: %s" % file_abspath)
            return True
        logger.debug("[RESULT_STORAGE] getting from %s" % file_abspath)

        if not exists(file_abspath) or self.is_expired(file_abspath):
            logger.debug("[RESULT_STORAGE] image not found at %s" % file_abspath)
            return True

        return datetime.fromtimestamp(getmtime(file_abspath)).replace(tzinfo=pytz.utc)
コード例 #43
0
ファイル: s3_loader.py プロジェクト: sayeghr/aws
 def handle_data(file_key):
     if not file_key or 'Error' in file_key or 'Body' not in file_key:
         logger.warn("ERROR retrieving image from S3 {0}: {1}".format(key, str(file_key)))
         # If we got here, there was a failure. We will return 404 if S3 returned a 404, otherwise 502.
         result = LoaderResult()
         result.successful = False
         if file_key and file_key.get('ResponseMetadata', {}).get('HTTPStatusCode') == 404:
             result.error = LoaderResult.ERROR_NOT_FOUND
         else:
             result.error = LoaderResult.ERROR_UPSTREAM
         callback(result)
     else:
         callback(file_key['Body'].read())
コード例 #44
0
ファイル: file_storage.py プロジェクト: HyperionGG/thumbor
    def get(self):
        path = self.context.request.url
        file_abspath = self.normalize_path(path)
        if not self.validate_path(file_abspath):
            logger.warn("[RESULT_STORAGE] unable to read from outside root path: %s" % file_abspath)
            return None
        logger.debug("[RESULT_STORAGE] getting from %s" % file_abspath)

        if not exists(file_abspath) or self.is_expired(file_abspath):
            logger.debug("[RESULT_STORAGE] image not found at %s" % file_abspath)
            return None
        with open(file_abspath, 'r') as f:
            return f.read()
コード例 #45
0
ファイル: file_storage.py プロジェクト: zanui/thumbor
    def get(self):
        path = self.context.request.url
        file_abspath = self.normalize_path(path)
        if not self.validate_path(file_abspath):
            logger.warn("[RESULT_STORAGE] unable to read from outside root path: %s" % file_abspath)
            return None
        logger.debug("[RESULT_STORAGE] getting from %s" % file_abspath)

        if not exists(file_abspath) or self.is_expired(file_abspath):
            logger.debug("[RESULT_STORAGE] image not found at %s" % file_abspath)
            return None
        with open(file_abspath, 'r') as f:
            return f.read()
コード例 #46
0
ファイル: file_storage.py プロジェクト: thanhliem89dn/thumbor
    def last_updated(self):
        path = self.context.request.url
        file_abspath = self.normalize_path(path)
        if not self.validate_path(file_abspath):
            logger.warn("[RESULT_STORAGE] unable to read from outside root path: %s" % file_abspath)
            return True
        logger.debug("[RESULT_STORAGE] getting from %s" % file_abspath)

        if not exists(file_abspath) or self.is_expired(file_abspath):
            logger.debug("[RESULT_STORAGE] image not found at %s" % file_abspath)
            return True

        return datetime.fromtimestamp(getmtime(file_abspath)).replace(tzinfo=pytz.utc)
コード例 #47
0
    def get(self, callback):
        path = self.context.request.url
        file_abspath = self.normalize_path(path)

        if not self.validate_path(file_abspath):
            logger.warn(
                "[RESULT_STORAGE] unable to read from outside root path: %s" %
                file_abspath)
            return callback(None)

        logger.debug("[RESULT_STORAGE] getting from %s" % file_abspath)

        if isdir(file_abspath):
            logger.warn("[RESULT_STORAGE] cache location is a directory: %s" %
                        file_abspath)
            return callback(None)

        if not exists(file_abspath):
            legacy_path = self.normalize_path_legacy(path)
            if isfile(legacy_path):
                logger.debug(
                    "[RESULT_STORAGE] migrating image from old location at %s"
                    % legacy_path)
                self.ensure_dir(dirname(file_abspath))
                move(legacy_path, file_abspath)
            else:
                logger.debug("[RESULT_STORAGE] image not found at %s" %
                             file_abspath)
                return callback(None)

        if self.is_expired(file_abspath):
            logger.debug("[RESULT_STORAGE] cached image has expired")
            return callback(None)

        with open(file_abspath, 'r') as f:
            buffer = f.read()

        result = ResultStorageResult(
            buffer=buffer,
            metadata={
                'LastModified':
                datetime.fromtimestamp(
                    getmtime(file_abspath)).replace(tzinfo=pytz.utc),
                'ContentLength':
                len(buffer),
                'ContentType':
                BaseEngine.get_mimetype(buffer)
            })

        callback(result)
コード例 #48
0
ファイル: transformer.py プロジェクト: achellies/thumbor
    def trim(self):
        if self.context.request.trim is None or not trim_enabled:
            return
        box = _bounding_box.apply(self.engine.get_image_mode(), self.engine.size[0], self.engine.size[1], self.context.request.trim_pos, self.context.request.trim_tolerance, self.engine.get_image_data())
        if box[2] < box[0] or box[3] < box[1]:
            logger.warn("Ignoring trim, there wouldn't be any image left, check the tolerance.")
            return

        self.engine.crop(box[0], box[1], box[2] + 1, box[3] + 1)
        if self.context.request.should_crop:
            self.context.request.crop['left'] -= box[0]
            self.context.request.crop['top'] -= box[1]
            self.context.request.crop['right'] -= box[0]
            self.context.request.crop['bottom'] -= box[1]
コード例 #49
0
ファイル: http_loader.py プロジェクト: dhardy92/thumbor
def return_contents(response, url, callback, context):
    context.statsd_client.incr('original_image.status.' + str(response.code))
    if response.error:
        logger.warn("ERROR retrieving image {0}: {1}".format(url, str(response.error)))
        callback(None)
    elif response.body is None or len(response.body) == 0:
        logger.warn("ERROR retrieving image {0}: Empty response.".format(url))
        callback(None)
    else:
        if response.time_info:
          for x in response.time_info:
              context.statsd_client.timing('original_image.time_info.' + x, response.time_info[x] * 1000)
          context.statsd_client.timing('original_image.time_info.bytes_per_second', len(response.body) / response.time_info['total'])
        callback(response.body)
コード例 #50
0
ファイル: watermark.py プロジェクト: kivicode/Navigation
    def on_fetch_done(self, result):
        if isinstance(result, LoaderResult) and not result.successful:
            logger.warn('bad watermark result error=%s metadata=%s' %
                        (result.error, result.metadata))
            raise tornado.web.HTTPError(400)

        if isinstance(result, LoaderResult):
            buffer = result.buffer
        else:
            buffer = result

        self.storage.put(self.url, buffer)
        self.storage.put_crypto(self.url)
        self.on_image_ready(buffer)
コード例 #51
0
ファイル: file_storage.py プロジェクト: RealGeeks/thumbor
    def put(self, bytes):
        file_abspath = self.normalize_path(self.context.request.url)
        if not self.validate_path(file_abspath):
            logger.warn("[RESULT_STORAGE] unable to write outside root path: %s" % file_abspath)
            return
        temp_abspath = "%s.%s" % (file_abspath, str(uuid4()).replace('-', ''))
        file_dir_abspath = dirname(file_abspath)
        logger.debug("[RESULT_STORAGE] putting at %s (%s)" % (file_abspath, file_dir_abspath))

        self.ensure_dir(file_dir_abspath)

        with open(temp_abspath, 'w') as _file:
            _file.write(bytes)

        move(temp_abspath, file_abspath)
コード例 #52
0
ファイル: watermark.py プロジェクト: raphamorim/thumbor
    def on_fetch_done(self, result):
        if not result.successful:
            logger.warn(
                    'bad watermark result error=%s metadata=%s' %
                    (result.error, result.metadata))
            raise tornado.web.HTTPError(400)

        if isinstance(result, LoaderResult):
            buffer = result.buffer
        else:
            buffer = result

        self.watermark_engine.load(buffer, self.extension)
        self.storage.put(self.url, self.watermark_engine.read())
        self.storage.put_crypto(self.url)
        self.on_image_ready(buffer)
コード例 #53
0
    def poolcounter_throttle(self, filename, extension):
        self.pc = None

        if not self.context.config.get('POOLCOUNTER_SERVER', False):
            raise tornado.gen.Return(False)

        self.pc = PoolCounter(self.context)

        cfg = self.context.config.get('POOLCOUNTER_CONFIG_PER_IP', False)
        if cfg:
            ff = self.request.headers.get('X-Forwarded-For', False)
            if not ff:
                logger.warn('[ImagesHandler] No X-Forwarded-For header in request, cannot throttle per IP')
            else:
                ff = ff.split(', ')[0]
                throttled = yield self.poolcounter_throttle_key('thumbor-ip-%s' % ff, cfg)

                if throttled:
                    raise tornado.gen.Return(True)

        cfg = self.context.config.get('POOLCOUNTER_CONFIG_PER_ORIGINAL', False)
        if cfg:
            name_sha1 = hashlib.sha1(filename).hexdigest()

            throttled = yield self.poolcounter_throttle_key('thumbor-render-%s' % name_sha1, cfg)

            if throttled:
                raise tornado.gen.Return(True)

        cfg = self.context.config.get('POOLCOUNTER_CONFIG_EXPENSIVE', False)
        if cfg and extension.lower() in cfg['extensions']:
            throttled = yield self.poolcounter_throttle_key('thumbor-render-expensive', cfg)

            if throttled:
                raise tornado.gen.Return(True)

        # This closes the PoolCounter connection in case it hasn't been closed normally.
        # Which can happen if an exception occured while processing the file, for example.
        release_timeout = self.context.config.get('POOLCOUNTER_RELEASE_TIMEOUT', False)
        if release_timeout:
            logger.debug('[ImagesHandler] Setting up PoolCounter cleanup callback')
            tornado.ioloop.IOLoop.instance().call_later(
                release_timeout,
                partial(close_poolcounter, self.pc)
            )

        raise tornado.gen.Return(False)
コード例 #54
0
ファイル: __init__.py プロジェクト: gi11es/thumbor-debian
    def detect(self, callback):
        try:
            features = self.get_features()
        except Exception:
            logger.warn('Error during face detection; skipping to next detector')
            self.next(callback)
            return

        if features:
            for (left, top, width, height), neighbors in features:
                top = self.__add_hair_offset(top, height)
                self.context.request.focal_points.append(
                    FocalPoint.from_square(left, top, width, height, origin="Face Detection")
                )
            callback()
        else:
            self.next(callback)
コード例 #55
0
ファイル: config.py プロジェクト: bguided/thumbor
    def __getattr__(self, name):
        if name in self.__dict__:
            return self.__dict__[name]

        if name in Config.class_aliased_items:
            logger.warn(
                "Option %s is marked as deprecated please use %s instead." % (name, Config.class_aliased_items[name])
            )
            return self.__getattr__(Config.class_aliased_items[name])

        if "defaults" in self.__dict__ and name in self.__dict__["defaults"]:
            return self.__dict__["defaults"][name]

        if name in Config.class_defaults:
            return Config.class_defaults[name]

        raise AttributeError(name)
コード例 #56
0
def _error(self, status, msg=None):
    # If the error isn't due to throttling, we increase a
    # failure counter for the given xkey, which will let us
    # avoid re-trying thumbnails bound to fail too many times
    xkey = self._headers.get('xkey', False)
    mc = self.failure_memcache()

    if status != 429 and status != 404 and xkey and mc:
        key = self._mc_encode_key(xkey)

        start = datetime.datetime.now()

        counter = mc.get(key)
        if not counter:
            # We add randomness to the expiry to avoid stampedes
            duration = self.context.config.get('FAILURE_THROTTLING_DURATION', 3600)
            mc.set(key, '1', duration + random.randint(0, 300))
        else:
            mc.incr(key)

        record_timing(self.context, datetime.datetime.now() - start, 'memcache.set', 'Thumbor-Memcache-Set-Time')

    # Errors should explicitely not be cached
    self.set_header('Cache-Control', 'no-cache')

    self.clear_header('xkey')
    self.clear_header('Content-Disposition')
    self.clear_header('Thumbor-Wikimedia-Original-Container')
    self.clear_header('Thumbor-Wikimedia-Thumbnail-Container')
    self.clear_header('Thumbor-Wikimedia-Original-Path')
    self.clear_header('Thumbor-Wikimedia-Thumbnail-Path')
    self.clear_header('Thumbor-Parameters')

    if status == 429:
        # 429 is missing from httplib.responses
        self.set_status(status, 'Too Many Requests')
    else:
        self.set_status(status)

    if msg is not None:
        try:
            logger.warn(msg, extra=log_extra(self.context.request.url))
        except AttributeError:
            logger.warn(msg)
    self.finish()
コード例 #57
0
ファイル: jpegtran.py プロジェクト: caeugusmao/thumbor
    def run_optimizer(self, image_extension, buffer):
        if not self.should_run(image_extension, buffer):
            return buffer

        if 'strip_icc' in self.context.request.filters:
            copy_chunks = 'comments'
        else:
            # have to copy everything to preserve icc profile
            copy_chunks = 'all'

        command = [
            self.context.config.JPEGTRAN_PATH,
            '-copy',
            copy_chunks,
            '-optimize',
        ]

        if self.context.config.PROGRESSIVE_JPEG:
            command += [
                '-progressive'
            ]

        if self.context.config.JPEGTRAN_SCANS_FILE:
            if exists(self.context.config.JPEGTRAN_SCANS_FILE):
                command += [
                    '-scans',
                    self.context.config.JPEGTRAN_SCANS_FILE
                ]
            else:
                logger.warn('jpegtran optimizer scans file does not exist')

        # close_fds would have a sane default on Python 3 but with Python 2.7 it is False
        # per default but setting it to True + setting any of stdin, stdout, stderr will
        # make this crash on Windows
        # TODO remove close_fds=True if code was upgraded to Python 3
        jpg_process = Popen(command, stdin=PIPE, stdout=PIPE,
                            stderr=PIPE, close_fds=True)
        output_stdout, output_stderr = jpg_process.communicate(buffer)

        if jpg_process.returncode != 0:
            logger.warn('jpegtran finished with non-zero return code (%d): %s'
                        % (jpg_process.returncode, output_stderr))
            return buffer

        return output_stdout
コード例 #58
0
ファイル: s3_loader.py プロジェクト: pgr0ss/aws
def load_sync(context, url, callback):
    if _use_http_loader(context, url):
        return http_loader.load_sync(context, url, callback, normalize_url_func=_normalize_url)

    bucket, key = _get_bucket_and_key(context, url)

    if _validate_bucket(context, bucket):
        bucket_loader = Bucket(
            connection=get_connection(context),
            name=bucket
        )
        file_key = None
        try:
            file_key = bucket_loader.get_key(url)
        except Exception, e:
            logger.warn("ERROR retrieving image from S3 {0}: {1}".format(url, str(e)))

        if file_key:
            callback(file_key.read())
            return
コード例 #59
0
ファイル: __init__.py プロジェクト: lfalcao/thumbor
    def _process_result_from_storage(self, result):
        if self.context.config.SEND_IF_MODIFIED_LAST_MODIFIED_HEADERS:
            # Handle If-Modified-Since & Last-Modified header
            try:
                result_last_modified = self.context.modules.result_storage.last_updated()
            except NotImplementedError:
                logger.warn('last_updated method is not supported by your result storage service, hence If-Modified-Since & Last-Updated headers support is disabled.')

            if result_last_modified:
                if 'If-Modified-Since' in self.request.headers:
                    date_modified_since = datetime.datetime.strptime(self.request.headers['If-Modified-Since'], HTTP_DATE_FMT)

                    if result_last_modified <= date_modified_since:
                        self.set_status(304)
                        self.finish()
                        return

                self.set_header('Last-Modified', result_last_modified.strftime(HTTP_DATE_FMT))

        return result