def _connect(self): if hasattr(self.context.config, 'HBASE_STORAGE_SERVER_HOSTS'): host = self.context.config.HBASE_STORAGE_SERVER_HOSTS[ (self.context.server.port + self.hbase_server_offset) % len(self.context.config.HBASE_STORAGE_SERVER_HOSTS)] else: host = self.context.config.HBASE_STORAGE_SERVER_HOST transport = TBufferedTransport( TSocket(host=host, port=self.context.config.HBASE_STORAGE_SERVER_PORT)) socket = TSocket(host=host, port=self.context.config.HBASE_STORAGE_SERVER_PORT) # Timeout is sum of HTTP timeouts, plus a bit. try: timeout = 5 socket.setTimeout(timeout * 1000) except: pass try: transport = TBufferedTransport(socket) transport.open() protocol = TBinaryProtocol.TBinaryProtocol(transport) self.storage = Hbase.Client(protocol) logger.info("Connected to HBase server " + host + ":" + str(self.context.config.HBASE_STORAGE_SERVER_PORT)) except: logger.error("Error connecting to HBase server " + host + ":" + str(self.context.config.HBASE_STORAGE_SERVER_PORT)) self.hbase_server_offset = self.hbase_server_offset + 1
def icc_profile_apply(self, profile=None): # Check whether input image has color management. if not self.engine.icc_profile: logger.info('ICC: Image has no embedded profile. Skipping this image.') return # Sanitize profile parameter. if profile != None: profile = os.path.basename(profile).lstrip('.') if len(profile) == 0: logger.warning('ICC: Invalid profile name.') return # Find output profile. outprofile = self._find_profile(profile) if not outprofile: logger.warning('ICC: Failed to load profile: {:s}'.format(profile)) return try: ext = self.engine.extension fmt = Image.EXTENSION[ext.lower()] except: logger.exception('ICC: Failed to determine image format and extension before attempting to apply profile: {:s}'.format(profile)) return try: inmode = self.engine.get_image_mode() insize = self.engine.size inimg = Image.frombytes(inmode, insize, self.engine.get_image_data()) # In PIL>=3.0.0 / Thumbor 6, icc_profile is sometimes a tuple :/ # https://github.com/python-pillow/Pillow/issues/1462 profile_data = self.engine.icc_profile if type(profile_data) == tuple: profile_data = profile_data[0] inprofile = StringIO(profile_data) outmode = 'RGBA' if 'A' in inmode else 'RGB' except: logger.exception('ICC: Failed to determine image properties before attempting to apply profile: {:s}'.format(profile)) return logger.info('ICC: Attempting to apply profile: {:s}, inmode: {:s}, outmode: {:s}'.format(profile, inmode, outmode)) try: outimg = ImageCms.profileToProfile(inimg, inprofile, outprofile, outputMode=outmode) except: logger.exception('ICC: Failed to apply profile: {:s}, inmode: {:s}, outmode: {:s}'.format(profile, inmode, outmode)) return # Reload the image into the engine. outbuf = StringIO() try: outimg.save(outbuf, fmt) self.engine.load(outbuf.getvalue(), ext) except: logger.exception('ICC: Failed load the image with an applied profile: {:s}, inmode: {:s}, outmode: {:s}'.format(profile, inmode, outmode)) return finally: outbuf.close()
def callback_wrapper(result): if result.successful: logger.info("Image " + path + " found on C5") callback(result) else: # If file_loader failed try http_loader logger.info("Image " + path + " NOT found on C5") http_loader.load(context, path, callback)
def __init__(self, context): super(Engine, self).__init__(context) self.subsampling = None self.qtables = None self.original_mode = None try: if self.context.config.MAX_PIXELS is None or int(self.context.config.MAX_PIXELS): Image.MAX_IMAGE_PIXELS = self.context.config.MAX_PIXELS except (AttributeError, TypeError, ValueError): # invalid type logger.info('MAX_PIXELS config variable set to invalid type. Has to be int on None')
def callback_wrapper(result): if result.successful: callback(result) else: logger.info('s3 {0}'.format( os.path.join(match.group('bucket').rstrip('/'), match.group('path').lstrip('/'))) ) # If not on efs, try s3 S3Loader.load(context, os.path.join(match.group('bucket').rstrip('/'), match.group('path').lstrip('/')), callback)
def run_optimizer(self, image_extension, buffer): img = Image.open(cStringIO.StringIO(buffer)) logger.info('Image format is %s', img.format) if img.format.upper() not in ('JPEG', 'JPG', 'PNG'): return buffer out = cStringIO.StringIO() if img.mode == 'P' or img.mode == 'RGBA': img = img.convert('RGB') img.save(out, format='JPEG', optimize=True) out.seek(0) return out.getvalue()
def __init__(self, conf_file=None, security_key=None, custom_handlers=None): if conf_file is None: conf_file = ThumborServiceApp.get_conf_file(conf_file) logger.info('Config file: %s' % conf_file) parse_config_file(conf_file) self.loader = real_import(conf.LOADER) self.storage = real_import(conf.STORAGE) self.engine = real_import(conf.ENGINE) self.detectors = [real_import(detector_name).Detector for detector_name in conf.DETECTORS] self.filters = [real_import(filter_name).Filter for filter_name in conf.FILTERS] filters.compile_filters(self.filters) # run again to overwrite the default settings on the # imported modules with the ones defined into the config file parse_config_file(conf_file) #storage = storage.Storage() #self.engine = self.engine.Engine() if security_key: options.SECURITY_KEY = security_key handler_context = { 'loader': self.loader, 'storage': self.storage, 'engine': self.engine, 'detectors': self.detectors, 'filters': self.filters } handlers = [ (r'/healthcheck', HealthcheckHandler) ] if conf.ALLOW_UNSAFE_URL: handlers.append( (Url.regex(), MainHandler, handler_context), ) if custom_handlers: for handler in custom_handlers: handlers.append((handler[0], handler[1], handler_context)) else: handlers.append( (r'/(?P<crypto>[^/]+)/(?P<image>(.+))', CryptoHandler, handler_context) ) super(ThumborServiceApp, self).__init__(handlers)
def __init__(self, conf_file=None, custom_handlers=None): if conf_file is None: conf_file = ThumborServiceApp.get_conf_file(conf_file) logger.info('Config file: %s' % conf_file) parse_config_file(conf_file) loader = real_import(options.LOADER) storage = real_import(options.STORAGE) engine = real_import(options.ENGINE) detectors = [ real_import(detector_name).Detector for detector_name in options.DETECTORS ] filters = [ real_import(filter_name).Filter for filter_name in options.FILTERS ] # run again to overwrite the default settings on the # imported modules with the ones defined into the config file parse_config_file(conf_file) storage = storage.Storage() engine = engine.Engine() handler_context = { 'loader': loader, 'storage': storage, 'engine': engine, 'detectors': detectors, 'filters': filters } handlers = [(r'/healthcheck', HealthcheckHandler)] if options.ALLOW_UNSAFE_URL: handlers.append((Url.regex(), MainHandler, handler_context), ) if custom_handlers: for handler in custom_handlers: handlers.append((handler[0], handler[1], handler_context)) else: handlers.append((r'/(?P<crypto>[^/]+)/(?P<image>(.+))', EncryptedHandler, handler_context)) super(ThumborServiceApp, self).__init__(handlers)
def import_community_modules(cls, instance): for module in cls._community_modules: setattr(instance, module['config_key'].lower(), None) # Autoload for module in cls._community_modules: config_key = module['config_key'] if hasattr(instance.config, config_key): instance.import_item(config_key, module['class_name']) else: logger.info( "Configuration `{config_key}` not found for module (using default value: `{class_name}`)".format( config_key=config_key, class_name=module['class_name'] ) )
def __init__(self, conf_file=None): if conf_file is None: conf_file = ThumborServiceApp.get_conf_file(conf_file) logger.info('Config file: %s' % conf_file) parse_config_file(conf_file) loader = real_import(options.LOADER) storage = real_import(options.STORAGE) engine = real_import(options.ENGINE) detectors = [] for detector_name in options.DETECTORS: detectors.append(real_import(detector_name).Detector) # run again to overwrite the default settings on the # imported modules with the ones defined into the config file parse_config_file(conf_file) storage = storage.Storage() engine = engine.Engine() handler_context = { 'loader': loader, 'storage': storage, 'engine': engine, 'detectors': detectors } handlers = [ (r'/healthcheck', HealthcheckHandler) ] if options.ALLOW_UNSAFE_URL: handlers.append( (Url.regex(), MainHandler, handler_context), ) handlers.append( (r'/(?P<crypto>[^/]+)/(?P<image>(.+))', EncryptedHandler, handler_context) ) super(ThumborServiceApp, self).__init__(handlers)
def load(context, url, callback): url = quote_url(url) match = S3_RE.match(url) def callback_wrapper(result): if result.successful: callback(result) else: logger.info('s3 {0}'.format( os.path.join(match.group('bucket').rstrip('/'), match.group('path').lstrip('/'))) ) # If not on efs, try s3 S3Loader.load(context, os.path.join(match.group('bucket').rstrip('/'), match.group('path').lstrip('/')), callback) # If melody s3 file, first try to load from efs if match: logger.info('BOTO {0}'.format(match.group('path'))) # TEMP try s3 direct S3Loader.load(context, os.path.join(match.group('bucket').rstrip('/'), match.group('path').lstrip('/')), callback) # FileLoader.load(context, match.group('path'), callback_wrapper) # else get from the internet elif HTTP_RE.match(url): logger.info('WEB {0}'.format(url)) HttpLoader.load(context, url, callback) else: logger.info('FILE {0}'.format(url)) result = LoaderResult() result.successful = False result.error = LoaderResult.ERROR_NOT_FOUND # callback(result) # TEMP enable file loader FileLoader.load(context, url, callback)
def cleanup(self): if self.pool: logger.info("Shutting down threads") self.pool.shutdown()
def callback(key): logger.info("INSIDE SPACES PUT")
async def icc_profile_apply(self, profile=None): """ Optain an RGB(A) image by applying the given profile. """ # Check whether input image has color management. if not self.engine.icc_profile: logger.info("ICC: Image has no embedded profile. " "Skipping this image.") return # Sanitize profile parameter. if profile is not None: profile = os.path.basename(profile).lstrip(".") if not profile: logger.warning("ICC: Invalid profile name.") return # Find output profile. outprofile = self._find_profile(profile) if not outprofile: logger.warning("ICC: Failed to load profile: {:s}".format(profile)) return try: ext = self.engine.extension fmt = Image.EXTENSION[ext.lower()] except Exception: logger.exception("ICC: Failed to determine image format and " "extension before attempting to apply " "profile: {:s}".format(profile)) return try: inmode = self.engine.get_image_mode() insize = self.engine.size indata = self.engine.get_image_data() inimg = Image.frombytes(inmode, insize, indata) inprofile = BytesIO(self.engine.icc_profile) outmode = "RGBA" if "A" in inmode else "RGB" except Exception: logger.exception("ICC: Failed to determine image " "properties before attempting to apply " "profile: {:s}".format(profile)) return logger.info("ICC: Attempting to apply profile: {:s}, inmode: {:s}, " "outmode: {:s}".format(profile, inmode, outmode)) try: outimg = ImageCms.profileToProfile(inimg, inprofile, outprofile, outputMode=outmode) except Exception: logger.exception("ICC: Failed to apply profile: {:s}, " "inmode: {:s}, " "outmode: {:s}".format(profile, inmode, outmode)) return # Reload the image into the engine. outbuf = BytesIO() try: outimg.save(outbuf, fmt) self.engine.load(outbuf.getvalue(), ext) except Exception: logger.exception("ICC: Failed load the image with an applied " "profile: {:s}, inmode: {:s}, " "outmode: {:s}".format(profile, inmode, outmode)) return finally: outbuf.close()
def get_image(self): """ This function is called after the PRE_LOAD filters have been applied. It applies the AFTER_LOAD filters on the result, then crops the image. """ logger.info('Loading image for request: %s', self.context.request.url) try: result = yield self._fetch( self.context.request.image_url ) if not result.successful: if result.loader_error == LoaderResult.ERROR_NOT_FOUND: self._error(404) return elif result.loader_error == LoaderResult.ERROR_UPSTREAM: # Return a Bad Gateway status if the error came from upstream self._error(502) return elif result.loader_error == LoaderResult.ERROR_TIMEOUT: # Return a Gateway Timeout status if upstream timed out (i.e. 599) self._error(504) return elif isinstance(result.loader_error, int): self._error(result.loader_error) return elif hasattr(result, 'engine_error') and result.engine_error == EngineResult.COULD_NOT_LOAD_IMAGE: self._error(400) return else: self._error(500) return except Exception as e: msg = '[BaseHandler] get_image failed for url `{url}`. error: `{error}`'.format( url=self.context.request.image_url, error=e ) self.log_exception(*sys.exc_info()) if 'cannot identify image file' in e.message: logger.warning(msg) self._error(400) else: logger.error(msg) self._error(500) return normalized = result.normalized buffer = result.buffer engine = result.engine req = self.context.request if engine is None: if buffer is None: self._error(504) return engine = self.context.request.engine try: engine.load(buffer, self.context.request.extension) except Exception: self._error(504) return self.context.transformer = Transformer(self.context) def transform(): self.normalize_crops(normalized, req, engine) if req.meta: self.context.transformer.engine = \ self.context.request.engine = \ JSONEngine(engine, req.image_url, req.meta_callback) self.context.transformer.transform(self.after_transform) self.filters_runner.apply_filters(thumbor.filters.PHASE_AFTER_LOAD, transform)