def _connect(self):
if hasattr(self.context.config, 'HBASE_STORAGE_SERVER_HOSTS'):
host = self.context.config.HBASE_STORAGE_SERVER_HOSTS[
(self.context.server.port + self.hbase_server_offset) %
len(self.context.config.HBASE_STORAGE_SERVER_HOSTS)]
else:
host = self.context.config.HBASE_STORAGE_SERVER_HOST
transport = TBufferedTransport(
TSocket(host=host,
port=self.context.config.HBASE_STORAGE_SERVER_PORT))
socket = TSocket(host=host,
port=self.context.config.HBASE_STORAGE_SERVER_PORT)
# Timeout is sum of HTTP timeouts, plus a bit.
try:
timeout = 5
socket.setTimeout(timeout * 1000)
except:
pass
try:
transport = TBufferedTransport(socket)
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
self.storage = Hbase.Client(protocol)
logger.info("Connected to HBase server " + host + ":" +
str(self.context.config.HBASE_STORAGE_SERVER_PORT))
except:
logger.error("Error connecting to HBase server " + host + ":" +
str(self.context.config.HBASE_STORAGE_SERVER_PORT))
self.hbase_server_offset = self.hbase_server_offset + 1
def icc_profile_apply(self, profile=None):
# Check whether input image has color management.
if not self.engine.icc_profile:
logger.info('ICC: Image has no embedded profile. Skipping this image.')
return
# Sanitize profile parameter.
if profile != None:
profile = os.path.basename(profile).lstrip('.')
if len(profile) == 0:
logger.warning('ICC: Invalid profile name.')
return
# Find output profile.
outprofile = self._find_profile(profile)
if not outprofile:
logger.warning('ICC: Failed to load profile: {:s}'.format(profile))
return
try:
ext = self.engine.extension
fmt = Image.EXTENSION[ext.lower()]
except:
logger.exception('ICC: Failed to determine image format and extension before attempting to apply profile: {:s}'.format(profile))
return
try:
inmode = self.engine.get_image_mode()
insize = self.engine.size
inimg = Image.frombytes(inmode, insize, self.engine.get_image_data())
# In PIL>=3.0.0 / Thumbor 6, icc_profile is sometimes a tuple :/
# https://github.com/python-pillow/Pillow/issues/1462
profile_data = self.engine.icc_profile
if type(profile_data) == tuple:
profile_data = profile_data[0]
inprofile = StringIO(profile_data)
outmode = 'RGBA' if 'A' in inmode else 'RGB'
except:
logger.exception('ICC: Failed to determine image properties before attempting to apply profile: {:s}'.format(profile))
return
logger.info('ICC: Attempting to apply profile: {:s}, inmode: {:s}, outmode: {:s}'.format(profile, inmode, outmode))
try:
outimg = ImageCms.profileToProfile(inimg, inprofile, outprofile, outputMode=outmode)
except:
logger.exception('ICC: Failed to apply profile: {:s}, inmode: {:s}, outmode: {:s}'.format(profile, inmode, outmode))
return
# Reload the image into the engine.
outbuf = StringIO()
try:
outimg.save(outbuf, fmt)
self.engine.load(outbuf.getvalue(), ext)
except:
logger.exception('ICC: Failed load the image with an applied profile: {:s}, inmode: {:s}, outmode: {:s}'.format(profile, inmode, outmode))
return
finally:
outbuf.close()
def callback_wrapper(result):
if result.successful:
logger.info("Image " + path + " found on C5")
callback(result)
else:
# If file_loader failed try http_loader
logger.info("Image " + path + " NOT found on C5")
http_loader.load(context, path, callback)
def __init__(self, context):
super(Engine, self).__init__(context)
self.subsampling = None
self.qtables = None
self.original_mode = None
try:
if self.context.config.MAX_PIXELS is None or int(self.context.config.MAX_PIXELS):
Image.MAX_IMAGE_PIXELS = self.context.config.MAX_PIXELS
except (AttributeError, TypeError, ValueError): # invalid type
logger.info('MAX_PIXELS config variable set to invalid type. Has to be int on None')
def __init__(self, context):
super(Engine, self).__init__(context)
self.subsampling = None
self.qtables = None
self.original_mode = None
try:
if self.context.config.MAX_PIXELS is None or int(self.context.config.MAX_PIXELS):
Image.MAX_IMAGE_PIXELS = self.context.config.MAX_PIXELS
except (AttributeError, TypeError, ValueError): # invalid type
logger.info('MAX_PIXELS config variable set to invalid type. Has to be int on None')
def callback_wrapper(result):
if result.successful:
callback(result)
else:
logger.info('s3 {0}'.format(
os.path.join(match.group('bucket').rstrip('/'), match.group('path').lstrip('/')))
)
# If not on efs, try s3
S3Loader.load(context,
os.path.join(match.group('bucket').rstrip('/'),
match.group('path').lstrip('/')),
callback)
def run_optimizer(self, image_extension, buffer):
img = Image.open(cStringIO.StringIO(buffer))
logger.info('Image format is %s', img.format)
if img.format.upper() not in ('JPEG', 'JPG', 'PNG'):
return buffer
out = cStringIO.StringIO()
if img.mode == 'P' or img.mode == 'RGBA':
img = img.convert('RGB')
img.save(out, format='JPEG', optimize=True)
out.seek(0)
return out.getvalue()
def __init__(self, conf_file=None, security_key=None, custom_handlers=None):
if conf_file is None:
conf_file = ThumborServiceApp.get_conf_file(conf_file)
logger.info('Config file: %s' % conf_file)
parse_config_file(conf_file)
self.loader = real_import(conf.LOADER)
self.storage = real_import(conf.STORAGE)
self.engine = real_import(conf.ENGINE)
self.detectors = [real_import(detector_name).Detector for detector_name in conf.DETECTORS]
self.filters = [real_import(filter_name).Filter for filter_name in conf.FILTERS]
filters.compile_filters(self.filters)
# run again to overwrite the default settings on the
# imported modules with the ones defined into the config file
parse_config_file(conf_file)
#storage = storage.Storage()
#self.engine = self.engine.Engine()
if security_key:
options.SECURITY_KEY = security_key
handler_context = {
'loader': self.loader,
'storage': self.storage,
'engine': self.engine,
'detectors': self.detectors,
'filters': self.filters
}
handlers = [
(r'/healthcheck', HealthcheckHandler)
]
if conf.ALLOW_UNSAFE_URL:
handlers.append(
(Url.regex(), MainHandler, handler_context),
)
if custom_handlers:
for handler in custom_handlers:
handlers.append((handler[0], handler[1], handler_context))
else:
handlers.append(
(r'/(?P<crypto>[^/]+)/(?P<image>(.+))', CryptoHandler, handler_context)
)
super(ThumborServiceApp, self).__init__(handlers)
def __init__(self, conf_file=None, custom_handlers=None):
if conf_file is None:
conf_file = ThumborServiceApp.get_conf_file(conf_file)
logger.info('Config file: %s' % conf_file)
parse_config_file(conf_file)
loader = real_import(options.LOADER)
storage = real_import(options.STORAGE)
engine = real_import(options.ENGINE)
detectors = [
real_import(detector_name).Detector
for detector_name in options.DETECTORS
]
filters = [
real_import(filter_name).Filter for filter_name in options.FILTERS
]
# run again to overwrite the default settings on the
# imported modules with the ones defined into the config file
parse_config_file(conf_file)
storage = storage.Storage()
engine = engine.Engine()
handler_context = {
'loader': loader,
'storage': storage,
'engine': engine,
'detectors': detectors,
'filters': filters
}
handlers = [(r'/healthcheck', HealthcheckHandler)]
if options.ALLOW_UNSAFE_URL:
handlers.append((Url.regex(), MainHandler, handler_context), )
if custom_handlers:
for handler in custom_handlers:
handlers.append((handler[0], handler[1], handler_context))
else:
handlers.append((r'/(?P<crypto>[^/]+)/(?P<image>(.+))',
EncryptedHandler, handler_context))
super(ThumborServiceApp, self).__init__(handlers)
def import_community_modules(cls, instance):
for module in cls._community_modules:
setattr(instance, module['config_key'].lower(), None)
# Autoload
for module in cls._community_modules:
config_key = module['config_key']
if hasattr(instance.config, config_key):
instance.import_item(config_key, module['class_name'])
else:
logger.info(
"Configuration `{config_key}` not found for module (using default value: `{class_name}`)".format(
config_key=config_key,
class_name=module['class_name']
)
)
def __init__(self, conf_file=None):
if conf_file is None:
conf_file = ThumborServiceApp.get_conf_file(conf_file)
logger.info('Config file: %s' % conf_file)
parse_config_file(conf_file)
loader = real_import(options.LOADER)
storage = real_import(options.STORAGE)
engine = real_import(options.ENGINE)
detectors = []
for detector_name in options.DETECTORS:
detectors.append(real_import(detector_name).Detector)
# run again to overwrite the default settings on the
# imported modules with the ones defined into the config file
parse_config_file(conf_file)
storage = storage.Storage()
engine = engine.Engine()
handler_context = {
'loader': loader,
'storage': storage,
'engine': engine,
'detectors': detectors
}
handlers = [
(r'/healthcheck', HealthcheckHandler)
]
if options.ALLOW_UNSAFE_URL:
handlers.append(
(Url.regex(), MainHandler, handler_context),
)
handlers.append(
(r'/(?P<crypto>[^/]+)/(?P<image>(.+))', EncryptedHandler, handler_context)
)
super(ThumborServiceApp, self).__init__(handlers)
def load(context, url, callback):
url = quote_url(url)
match = S3_RE.match(url)
def callback_wrapper(result):
if result.successful:
callback(result)
else:
logger.info('s3 {0}'.format(
os.path.join(match.group('bucket').rstrip('/'), match.group('path').lstrip('/')))
)
# If not on efs, try s3
S3Loader.load(context,
os.path.join(match.group('bucket').rstrip('/'),
match.group('path').lstrip('/')),
callback)
# If melody s3 file, first try to load from efs
if match:
logger.info('BOTO {0}'.format(match.group('path')))
# TEMP try s3 direct
S3Loader.load(context,
os.path.join(match.group('bucket').rstrip('/'),
match.group('path').lstrip('/')),
callback)
# FileLoader.load(context, match.group('path'), callback_wrapper)
# else get from the internet
elif HTTP_RE.match(url):
logger.info('WEB {0}'.format(url))
HttpLoader.load(context, url, callback)
else:
logger.info('FILE {0}'.format(url))
result = LoaderResult()
result.successful = False
result.error = LoaderResult.ERROR_NOT_FOUND
# callback(result)
# TEMP enable file loader
FileLoader.load(context, url, callback)
def cleanup(self):
if self.pool:
logger.info("Shutting down threads")
self.pool.shutdown()
def callback(key):
logger.info("INSIDE SPACES PUT")
async def icc_profile_apply(self, profile=None):
"""
Optain an RGB(A) image by applying the given profile.
"""
# Check whether input image has color management.
if not self.engine.icc_profile:
logger.info("ICC: Image has no embedded profile. "
"Skipping this image.")
return
# Sanitize profile parameter.
if profile is not None:
profile = os.path.basename(profile).lstrip(".")
if not profile:
logger.warning("ICC: Invalid profile name.")
return
# Find output profile.
outprofile = self._find_profile(profile)
if not outprofile:
logger.warning("ICC: Failed to load profile: {:s}".format(profile))
return
try:
ext = self.engine.extension
fmt = Image.EXTENSION[ext.lower()]
except Exception:
logger.exception("ICC: Failed to determine image format and "
"extension before attempting to apply "
"profile: {:s}".format(profile))
return
try:
inmode = self.engine.get_image_mode()
insize = self.engine.size
indata = self.engine.get_image_data()
inimg = Image.frombytes(inmode, insize, indata)
inprofile = BytesIO(self.engine.icc_profile)
outmode = "RGBA" if "A" in inmode else "RGB"
except Exception:
logger.exception("ICC: Failed to determine image "
"properties before attempting to apply "
"profile: {:s}".format(profile))
return
logger.info("ICC: Attempting to apply profile: {:s}, inmode: {:s}, "
"outmode: {:s}".format(profile, inmode, outmode))
try:
outimg = ImageCms.profileToProfile(inimg, inprofile, outprofile,
outputMode=outmode)
except Exception:
logger.exception("ICC: Failed to apply profile: {:s}, "
"inmode: {:s}, "
"outmode: {:s}".format(profile, inmode, outmode))
return
# Reload the image into the engine.
outbuf = BytesIO()
try:
outimg.save(outbuf, fmt)
self.engine.load(outbuf.getvalue(), ext)
except Exception:
logger.exception("ICC: Failed load the image with an applied "
"profile: {:s}, inmode: {:s}, "
"outmode: {:s}".format(profile, inmode, outmode))
return
finally:
outbuf.close()
def get_image(self):
"""
This function is called after the PRE_LOAD filters have been applied.
It applies the AFTER_LOAD filters on the result, then crops the image.
"""
logger.info('Loading image for request: %s', self.context.request.url)
try:
result = yield self._fetch(
self.context.request.image_url
)
if not result.successful:
if result.loader_error == LoaderResult.ERROR_NOT_FOUND:
self._error(404)
return
elif result.loader_error == LoaderResult.ERROR_UPSTREAM:
# Return a Bad Gateway status if the error came from upstream
self._error(502)
return
elif result.loader_error == LoaderResult.ERROR_TIMEOUT:
# Return a Gateway Timeout status if upstream timed out (i.e. 599)
self._error(504)
return
elif isinstance(result.loader_error, int):
self._error(result.loader_error)
return
elif hasattr(result, 'engine_error') and result.engine_error == EngineResult.COULD_NOT_LOAD_IMAGE:
self._error(400)
return
else:
self._error(500)
return
except Exception as e:
msg = '[BaseHandler] get_image failed for url `{url}`. error: `{error}`'.format(
url=self.context.request.image_url,
error=e
)
self.log_exception(*sys.exc_info())
if 'cannot identify image file' in e.message:
logger.warning(msg)
self._error(400)
else:
logger.error(msg)
self._error(500)
return
normalized = result.normalized
buffer = result.buffer
engine = result.engine
req = self.context.request
if engine is None:
if buffer is None:
self._error(504)
return
engine = self.context.request.engine
try:
engine.load(buffer, self.context.request.extension)
except Exception:
self._error(504)
return
self.context.transformer = Transformer(self.context)
def transform():
self.normalize_crops(normalized, req, engine)
if req.meta:
self.context.transformer.engine = \
self.context.request.engine = \
JSONEngine(engine, req.image_url, req.meta_callback)
self.context.transformer.transform(self.after_transform)
self.filters_runner.apply_filters(thumbor.filters.PHASE_AFTER_LOAD, transform)
