def run_optimizer(self, image_extension, buffer):
if not self.should_run(image_extension, buffer):
return buffer
command = [
self.context.config.JPEGTRAN_PATH,
'-copy',
'comments',
'-optimize',
]
if self.context.config.PROGRESSIVE_JPEG:
command += [
'-progressive'
]
jpg_process = Popen(command, stdin=PIPE, stdout=PIPE, stderr=PIPE)
output_stdout, output_stderr = jpg_process.communicate(buffer)
if jpg_process.returncode != 0:
logger.warn('jpegtran finished with non-zero return code (%d): %s'
% (jpg_process.returncode, output_stderr))
return buffer
return output_stdout
def import_item(self, config_key=None, class_name=None, is_multiple=False, item_value=None, ignore_errors=False):
if item_value is None:
conf_value = getattr(self.config, config_key)
else:
conf_value = item_value
if is_multiple:
modules = []
if conf_value:
for module_name in conf_value:
try:
if class_name is not None:
module = self.import_class('%s.%s' % (module_name, class_name))
else:
module = self.import_class(module_name, get_module=True)
modules.append(module)
except ImportError:
if ignore_errors:
logger.warn('Module %s could not be imported.' % module_name)
else:
raise
setattr(self, config_key.lower(), tuple(modules))
else:
if class_name is not None:
module = self.import_class('%s.%s' % (conf_value, class_name))
else:
module = self.import_class(conf_value, get_module=True)
setattr(self, config_key.lower(), module)
def _process_result_from_storage(self, result):
if self.context.config.SEND_IF_MODIFIED_LAST_MODIFIED_HEADERS:
# Handle If-Modified-Since & Last-Modified header
try:
if isinstance(result, ResultStorageResult):
result_last_modified = result.last_modified
else:
result_last_modified = yield gen.maybe_future(
self.context.modules.result_storage.last_updated())
if result_last_modified:
if 'If-Modified-Since' in self.request.headers:
logger.debug('LASTMODIFIED HEADER CHECKED')
date_modified_since = datetime.datetime.strptime(
self.request.headers['If-Modified-Since'],
HTTP_DATE_FMT).replace(tzinfo=pytz.utc)
logger.debug(date_modified_since)
logger.debug(result_last_modified)
if result_last_modified <= date_modified_since:
logger.debug('LASTMODIFIED SET STATUS')
self.set_status(304)
self.finish()
return
self.set_header(
'Last-Modified',
result_last_modified.strftime(HTTP_DATE_FMT))
except NotImplementedError:
logger.warn(
'last_updated method is not supported by your result storage service, hence If-Modified-Since & '
'Last-Updated headers support is disabled.')
def get(self, callback):
path = self.context.request.url
file_abspath = self.normalize_path(path)
if not self.validate_path(file_abspath):
logger.warn(
"[RESULT_STORAGE] unable to read from outside root path: %s" %
file_abspath)
return None
logger.debug("[RESULT_STORAGE] getting from %s" % file_abspath)
if not exists(file_abspath) or self.is_expired(file_abspath):
logger.debug("[RESULT_STORAGE] image not found at %s" %
file_abspath)
callback(None)
else:
with open(file_abspath, 'r') as f:
buffer = f.read()
result = ResultStorageResult(
buffer=buffer,
metadata={
'LastModified':
datetime.fromtimestamp(
getmtime(file_abspath)).replace(tzinfo=pytz.utc),
'ContentLength':
len(buffer),
'ContentType':
BaseEngine.get_mimetype(buffer)
})
callback(result)
def load(context, url, callback):
enable_http_loader = context.config.get('AWS_ENABLE_HTTP_LOADER',
default=False)
if enable_http_loader and url.startswith('http'):
return http_loader.load_sync(context,
url,
callback,
normalize_url_func=_normalize_url)
url = urllib2.unquote(url)
bucket = context.config.get('S3_LOADER_BUCKET', default=None)
if not bucket:
bucket, url = _get_bucket(url,
root_path=context.config.S3_LOADER_ROOT_PATH)
if _validate_bucket(context, bucket):
bucket_loader = Bucket(connection=get_connection(context), name=bucket)
file_key = None
try:
file_key = bucket_loader.get_key(url)
except Exception, e:
logger.warn("ERROR retrieving image from S3 {0}: {1}".format(
url, str(e)))
if file_key:
callback(file_key.read())
return
def return_contents(response, url, callback, context, req_start=None):
if req_start:
finish = datetime.datetime.now()
res = urlparse(url)
context.metrics.timing(
'original_image.fetch.{0}.{1}'.format(response.code, res.netloc),
(finish - req_start).total_seconds() * 1000
)
result = LoaderResult()
context.metrics.incr('original_image.status.' + str(response.code))
if response.error:
result.successful = False
if response.code == 599:
# Return a Gateway Timeout status downstream if upstream times out
result.error = LoaderResult.ERROR_TIMEOUT
else:
result.error = LoaderResult.ERROR_NOT_FOUND
logger.warn(u"ERROR retrieving image {0}: {1}".format(url, str(response.error)))
elif response.body is None or len(response.body) == 0:
result.successful = False
result.error = LoaderResult.ERROR_UPSTREAM
logger.warn(u"ERROR retrieving image {0}: Empty response.".format(url))
else:
if response.time_info:
for x in response.time_info:
context.metrics.timing('original_image.time_info.' + x, response.time_info[x] * 1000)
context.metrics.timing('original_image.time_info.bytes_per_second', len(response.body) / response.time_info['total'])
result.buffer = response.body
context.metrics.incr('original_image.response_bytes', len(response.body))
callback(result)
def _process_result_from_storage(self, result):
if self.context.config.SEND_IF_MODIFIED_LAST_MODIFIED_HEADERS:
# Handle If-Modified-Since & Last-Modified header
try:
result_last_modified = self.context.modules.result_storage.last_updated(
)
except NotImplementedError:
logger.warn(
'last_updated method is not supported by your result storage service, hence If-Modified-Since & Last-Updated headers support is disabled.'
)
if result_last_modified:
if 'If-Modified-Since' in self.request.headers:
date_modified_since = datetime.datetime.strptime(
self.request.headers['If-Modified-Since'],
HTTP_DATE_FMT)
if result_last_modified <= date_modified_since:
self.set_status(304)
self.finish()
return
self.set_header('Last-Modified',
result_last_modified.strftime(HTTP_DATE_FMT))
return result
def return_contents(response, url, callback, context, req_start=None):
if req_start:
finish = datetime.datetime.now()
res = urlparse(url)
context.metrics.timing(
'original_image.fetch.{0}.{1}'.format(response.code, res.netloc),
(finish - req_start).total_seconds() * 1000
)
result = LoaderResult()
context.metrics.incr('original_image.status.' + str(response.code))
if response.error:
result.successful = False
if response.code == 599:
# Return a Gateway Timeout status downstream if upstream times out
result.error = LoaderResult.ERROR_TIMEOUT
else:
result.error = LoaderResult.ERROR_NOT_FOUND
logger.warn(u"ERROR retrieving image {0}: {1}".format(url, str(response.error)))
elif response.body is None or len(response.body) == 0:
result.successful = False
result.error = LoaderResult.ERROR_UPSTREAM
logger.warn(u"ERROR retrieving image {0}: Empty response.".format(url))
else:
if response.time_info:
for x in response.time_info:
context.metrics.timing('original_image.time_info.' + x, response.time_info[x] * 1000)
context.metrics.timing('original_image.time_info.bytes_per_second', len(response.body) / response.time_info['total'])
result.buffer = response.body
context.metrics.incr('original_image.response_bytes', len(response.body))
callback(result)
def reload_to_fit_in_kb(self, engine, initial_results, extension,
initial_quality, max_bytes):
logger.warn('This is the reload step')
if extension not in ['.webp', '.jpg', '.jpeg'
] or len(initial_results) <= max_bytes:
return initial_results
results = initial_results
quality = initial_quality
while len(results) > max_bytes:
quality = int(quality * 0.75)
if quality < 10:
logger.debug(
'Could not find any reduction that matches required size of %d bytes.'
% max_bytes)
return initial_results
logger.debug('Trying to downsize image with quality of %d...' %
quality)
results = engine.read(extension, quality)
prev_result = results
while len(results) <= max_bytes:
quality = int(quality * 1.1)
logger.debug('Trying to upsize image with quality of %d...' %
quality)
prev_result = results
results = engine.read(extension, quality)
return prev_result
def detect(self, callback):
engine = self.context.modules.engine
try:
img = np.array(
engine.convert_to_grayscale(update_image=False,
with_alpha=False))
except Exception as e:
logger.exception(e)
logger.warn(
'Error during feature detection; skipping to next detector')
self.next(callback)
return
points = cv2.goodFeaturesToTrack(
img,
maxCorners=20,
qualityLevel=0.04,
minDistance=1.0,
useHarrisDetector=False,
)
if points is not None:
for point in points:
x, y = point.ravel()
self.context.request.focal_points.append(
FocalPoint(x.item(), y.item(), 1))
callback()
else:
self.next(callback)
def return_contents(response, url, callback, context):
result = LoaderResult()
context.metrics.incr('original_image.status.' + str(response.code))
if response.error:
result.successful = False
if response.code == 599:
# Return a Gateway Timeout status downstream if upstream times out
result.error = LoaderResult.ERROR_TIMEOUT
else:
result.error = LoaderResult.ERROR_NOT_FOUND
logger.warn(u"ERROR retrieving image {0}: {1}".format(url, str(response.error)))
elif response.body is None or len(response.body) == 0:
result.successful = False
result.error = LoaderResult.ERROR_UPSTREAM
logger.warn(u"ERROR retrieving image {0}: Empty response.".format(url))
else:
if response.time_info:
for x in response.time_info:
context.metrics.timing('original_image.time_info.' + x, response.time_info[x] * 1000)
context.metrics.timing('original_image.time_info.bytes_per_second', len(response.body) / response.time_info['total'])
result.buffer = response.body
callback(result)
def return_data(file_key):
if not file_key or self._get_error(file_key) or self.is_expired(
file_key) or 'Body' not in file_key:
logger.warn("[AwsStorage] s3 key not found at %s" % path)
return None
else:
return loads(file_key['Body'].read())
def _handle_error(self, response):
"""
Logs error if necessary
:param dict response: AWS Response
"""
if self._get_error(response):
logger.warn("[STORAGE] error occured while storing data: %s" % self._get_error(response))
def _handle_error(self, response):
"""
Logs error if necessary
:param dict response: AWS Response
"""
if self._get_error(response):
logger.warn("[STORAGE] error occured while storing data: %s" % self._get_error(response))
def srgb(self):
try:
if not isinstance(self.engine, PILEngine):
logger.warn('Could not perform profileToProfile conversion: engine is not PIL engine')
return
if (ImageCms is None):
logger.warn('ImageCms is not installed. Could not perform profileToProfile conversion')
return
image = self.engine.image
embedded_profile = image.info.get('icc_profile')
if not embedded_profile:
logger.debug('Image does not have embedded profile. Assuming already in sRGB')
return
embedded_profile = BytesIO(embedded_profile)
srgb_profile = BytesIO(tiny_srgb)
output_mode = 'RGBA' if 'A' in image.mode else 'RGB'
image = ImageCms.profileToProfile(image, embedded_profile, srgb_profile, renderingIntent=0,
outputMode=output_mode)
self.engine.image = image
self.engine.icc_profile = image.info.get('icc_profile')
except Exception as err:
logger.exception(err)
def watermark(self,
callback,
url,
x,
y,
alpha,
w_ratio=False,
h_ratio=False):
self.url = url
self.x = x
self.y = y
self.alpha = alpha
self.w_ratio = float(
w_ratio) / 100.0 if w_ratio and w_ratio != 'none' else False
self.h_ratio = float(
h_ratio) / 100.0 if h_ratio and h_ratio != 'none' else False
self.callback = callback
self.watermark_engine = self.context.modules.engine.__class__(
self.context)
self.storage = self.context.modules.storage
try:
buffer = yield tornado.gen.maybe_future(self.storage.get(self.url))
if buffer is not None:
self.on_image_ready(buffer)
else:
self.context.modules.loader.load(self.context, self.url,
self.on_fetch_done)
except Exception as e:
logger.exception(e)
logger.warn("bad watermark")
raise tornado.web.HTTPError(500)
def detect(self, callback):
engine = self.context.modules.engine
try:
img = np.array(
engine.convert_to_grayscale(
update_image=False,
with_alpha=False
)
)
except Exception:
logger.warn('Error during feature detection; skipping to next detector')
self.next(callback)
return
points = cv2.goodFeaturesToTrack(
img,
maxCorners=20,
qualityLevel=0.04,
minDistance=1.0,
useHarrisDetector=False,
)
if points is not None:
for x, y in points.squeeze():
self.context.request.focal_points.append(FocalPoint(x.item(), y.item(), 1))
callback()
else:
self.next(callback)
def return_contents(response, url, callback, context):
result = LoaderResult()
context.metrics.incr('original_image.status.' + str(response.code))
if response.error:
result.successful = False
if response.code == 599:
# Return a Gateway Timeout status downstream if upstream times out
result.error = LoaderResult.ERROR_TIMEOUT
else:
result.error = LoaderResult.ERROR_NOT_FOUND
logger.warn("ERROR retrieving image {0}: {1}".format(
url, str(response.error)))
elif response.body is None or len(response.body) == 0:
result.successful = False
result.error = LoaderResult.ERROR_UPSTREAM
logger.warn("ERROR retrieving image {0}: Empty response.".format(url))
else:
if response.time_info:
for x in response.time_info:
context.metrics.timing('original_image.time_info.' + x,
response.time_info[x] * 1000)
context.metrics.timing(
'original_image.time_info.bytes_per_second',
len(response.body) / response.time_info['total'])
result.buffer = response.body
callback(result)
def return_data(file_key):
if not file_key or self._get_error(file_key) or self.is_expired(
file_key) or 'Body' not in file_key:
logger.warn("[STORAGE] s3 key not found at %s" % crypto_path)
return None
else:
return file_key['Body']
def _process_result_from_storage(self, result):
if self.context.config.SEND_IF_MODIFIED_LAST_MODIFIED_HEADERS:
# Handle If-Modified-Since & Last-Modified header
try:
if isinstance(result, ResultStorageResult):
result_last_modified = result.last_modified
else:
result_last_modified = yield gen.maybe_future(self.context.modules.result_storage.last_updated())
if result_last_modified:
if "If-Modified-Since" in self.request.headers:
date_modified_since = datetime.datetime.strptime(
self.request.headers["If-Modified-Since"], HTTP_DATE_FMT
)
if result_last_modified <= date_modified_since:
self.set_status(304)
self.finish()
return
self.set_header("Last-Modified", result_last_modified.strftime(HTTP_DATE_FMT))
except NotImplementedError:
logger.warn(
"last_updated method is not supported by your result storage service, hence If-Modified-Since & "
"Last-Updated headers support is disabled."
)
def run_optimizer(self, image_extension, buffer):
if not self.should_run(image_extension, buffer):
return buffer
if 'strip_icc' in self.context.request.filters:
copy_chunks = 'comments'
else:
# have to copy everything to preserve icc profile
copy_chunks = 'all'
command = [
self.context.config.JPEGTRAN_PATH,
'-copy',
copy_chunks,
'-optimize',
]
if self.context.config.PROGRESSIVE_JPEG:
command += ['-progressive']
jpg_process = Popen(command, stdin=PIPE, stdout=PIPE, stderr=PIPE)
output_stdout, output_stderr = jpg_process.communicate(buffer)
if jpg_process.returncode != 0:
logger.warn(
'jpegtran finished with non-zero return code (%d): %s' %
(jpg_process.returncode, output_stderr))
return buffer
return output_stdout
def get(self, callback):
path = self.context.request.url
file_abspath = self.normalize_path(path)
if not self.validate_path(file_abspath):
logger.warn("[RESULT_STORAGE] unable to read from outside root path: %s" % file_abspath)
return None
logger.debug("[RESULT_STORAGE] getting from %s" % file_abspath)
if not exists(file_abspath) or self.is_expired(file_abspath):
logger.debug("[RESULT_STORAGE] image not found at %s" % file_abspath)
callback(None)
else:
with open(file_abspath, 'r') as f:
buffer = f.read()
result = ResultStorageResult(
buffer=buffer,
metadata={
'LastModified': datetime.fromtimestamp(getmtime(file_abspath)).replace(tzinfo=pytz.utc),
'ContentLength': len(buffer),
'ContentType': BaseEngine.get_mimetype(buffer)
}
)
callback(result)
def trim(self):
is_gifsicle = (self.context.request.engine.extension == '.gif' and self.context.config.USE_GIFSICLE_ENGINE)
if self.context.request.trim is None or not trim_enabled or is_gifsicle:
return
mode, data = self.engine.image_data_as_rgb()
box = _bounding_box.apply(
mode,
self.engine.size[0],
self.engine.size[1],
self.context.request.trim_pos,
self.context.request.trim_tolerance,
data
)
if box[2] < box[0] or box[3] < box[1]:
logger.warn("Ignoring trim, there wouldn't be any image left, check the tolerance.")
return
self.engine.crop(box[0], box[1], box[2] + 1, box[3] + 1)
if self.context.request.should_crop:
self.context.request.crop['left'] -= box[0]
self.context.request.crop['top'] -= box[1]
self.context.request.crop['right'] -= box[0]
self.context.request.crop['bottom'] -= box[1]
def load(context, url, callback, normalize_url_func=_normalize_url):
result = LoaderResult()
start = time.perf_counter()
try:
result.buffer = ffmpeg(context, normalize_url_func(url))
except subprocess.CalledProcessError as err:
result.successful = False
result.error = err.stderr.decode('utf-8').strip()
logger.warn(f'ERROR retrieving image {url}: {result.error}')
if result.error.lower().endswith(
'Server returned 404 not found'.lower()):
result.error = LoaderResult.ERROR_NOT_FOUND
except Exception as err:
result.successful = False
result.error = str(err)
logger.warn(f'ERROR retrieving image {url}: {err}')
else:
total_time = (time.perf_counter() - start)
total_bytes = len(result.buffer)
result.metadata.update({
'size': total_bytes,
# 'updated_at': datetime.datetime.utcnow(),
})
context.metrics.incr('original_image.status.200')
context.metrics.incr('original_image.response_bytes', total_bytes)
context.metrics.timing(f'original_image.fetch.{url}',
total_time * 1000)
context.metrics.timing('original_image.time_info.bytes_per_second',
total_bytes / total_time)
return callback(result)
def dispatch(self, file_key):
""" Callback method for getObject from s3 """
if not file_key or 'Error' in file_key or 'Body' not in file_key:
logger.warn(
"ERROR retrieving image from S3 {0}: {1}".
format(self.key, str(file_key)))
# If we got here, there was a failure.
# We will return 404 if S3 returned a 404, otherwise 502.
result = LoaderResult()
result.successful = False
if not file_key:
result.error = LoaderResult.ERROR_UPSTREAM
self.callback(result)
return
response_metadata = file_key.get('ResponseMetadata', {})
status_code = response_metadata.get('HTTPStatusCode')
if status_code == 404:
result.error = LoaderResult.ERROR_NOT_FOUND
self.callback(result)
return
if self.max_retries_counter < self.limit_max_retries:
self.__increment_retry_counter()
self.bucket_loader.get(self.key,
callback=self.dispatch)
else:
result.error = LoaderResult.ERROR_UPSTREAM
self.callback(result)
else:
self.callback(file_key['Body'].read())
def load(context, url, callback):
enable_http_loader = context.config.get('AWS_ENABLE_HTTP_LOADER', default=False)
if enable_http_loader and url.startswith('http'):
return http_loader.load_sync(context, url, callback, normalize_url_func=_normalize_url)
url = urllib2.unquote(url)
bucket = context.config.get('S3_LOADER_BUCKET', default=None)
if not bucket:
bucket, url = _get_bucket(url)
if _validate_bucket(context, bucket):
bucket_loader = Bucket(
connection=thumbor_aws.connection.get_connection(context),
name=bucket
)
file_key = None
try:
file_key = bucket_loader.get_key(url)
except Exception, e:
logger.warn("ERROR retrieving image from S3 {0}: {1}".format(url, str(e)))
if file_key:
callback(file_key.read())
return
def import_item(self, config_key=None, class_name=None, is_multiple=False, item_value=None, ignore_errors=False):
if item_value is None:
conf_value = getattr(self.config, config_key)
else:
conf_value = item_value
if is_multiple:
modules = []
if conf_value:
for module_name in conf_value:
try:
if class_name is not None:
module = self.import_class('%s.%s' % (module_name, class_name))
else:
module = self.import_class(module_name, get_module=True)
modules.append(module)
except ImportError:
if ignore_errors:
logger.warn('Module %s could not be imported.' % module_name)
else:
raise
setattr(self, config_key.lower(), tuple(modules))
else:
if class_name is not None:
module = self.import_class('%s.%s' % (conf_value, class_name))
else:
module = self.import_class(conf_value, get_module=True)
setattr(self, config_key.lower(), module)
def detect(self, callback):
engine = self.context.modules.engine
try:
engine.image_data_as_rgb()
img = np.array(engine.image)
self.net.setInput(
cv2.dnn.blobFromImage(img, size=(300, 300), swapRB=True))
detections = self.net.forward()
except Exception as e:
logger.exception(e)
logger.warn(
'Error during feature detection; skipping to next detector')
self.next(callback)
return
confidence_threshold = 0.2
num_detections = 0
for detection in detections[0, 0, :, :]:
confidence = float(detection[2])
if confidence < confidence_threshold:
continue
num_detections += 1
class_id = int(detection[1]) - 1 # make it zero-indexed
class_name = coco_classes[class_id]
left = int(detection[3] * img.shape[1])
top = int(detection[4] * img.shape[0])
right = int(detection[5] * img.shape[1])
bottom = int(detection[6] * img.shape[0])
width = right - left
height = bottom - top
# If the detection is of a person,
# and the person is vertically oriented,
# this uses the upper 1/4 of the box to focus on the face.
# In the case the person is horizontal, perhaps reclining,
# then the focal point will remain at their center.
# In the case the person is upside down, this would focus on the feet instead of the face.
# But consider - whoever is publishing a picture of an upside down person
# might appreciate that it focuses on the feet.
if class_name == 'person' and height > width:
height = int(height * 0.25)
self.context.request.focal_points.append(
FocalPoint.from_dict({
'x':
left + (width / 2),
'y':
top + (height / 2),
'width':
width,
'height':
height,
'z':
confidence,
'origin':
'DNN Object Detection (class: {})'.format(class_name)
}))
if num_detections > 0:
callback()
else:
self.next(callback)
def should_run(self, image_extension, buffer):
if image_extension in ['.jpg', '.jpeg']:
if not exists(self.context.config.JPEGTRAN_PATH):
logger.warn(
'jpegtran optimizer enabled but binary JPEGTRAN_PATH does not exist')
return False
return True
return False
def on_file_fetched(file):
if not file or self._get_error(file) or self.is_expired(
file) or 'LastModified' not in file:
logger.warn("[AwsStorage] s3 key not found at %s" %
file_abspath)
return None
else:
return file['LastModified']
def __setattr__(self, name, value):
if name in Config.class_aliased_items:
logger.warn(
"Option %s is marked as deprecated please use %s instead." % (name, Config.class_aliased_items[name])
)
self.__setattr__(Config.class_aliased_items[name], value)
else:
super(Config, self).__setattr__(name, value)
def on_file_fetched(file):
if not file or self._get_error(file) or self.is_expired(
file) or 'LastModified' not in file:
logger.warn("[AwsStorage] s3 key not found at %s" %
file_abspath)
callback(None)
else:
callback(self._utc_to_local(file['LastModified']))
def format(self, format):
logger.warn('Setting format to %s' % format)
if format.lower() not in ALLOWED_FORMATS:
logger.debug('Format not allowed: %s' % format.lower())
self.context.request.format = None
else:
logger.debug('Format specified: %s' % format.lower())
self.context.request.format = format.lower()
def should_run(self, image_extension, buffer):
if 'gif' in image_extension and 'gifv' in self.context.request.filters:
if not exists(self.context.config.FFMPEG_PATH):
logger.warn(
'gifv optimizer enabled but binary FFMPEG_PATH does not exist')
return False
return True
return False
def optimize(self, context, image_extension, results):
logger.warn('This is the opt step')
for optimizer in context.modules.optimizers:
new_results = optimizer(context).run_optimizer(
image_extension, results)
if new_results is not None:
results = new_results
return results
def return_contents(response, url, callback):
if response.error:
logger.warn("ERROR retrieving image {0}: {1}".format(url, str(response.error)))
callback(None)
elif response.body is None or len(response.body) == 0:
logger.warn("ERROR retrieving image {0}: Empty response.".format(url))
callback(None)
else:
callback(response.body)
def return_contents(response, url, callback):
if response.error:
logger.warn("ERROR retrieving image {0}: {1}".format(url, str(response.error)))
callback(None)
elif response.body is None or len(response.body) == 0:
logger.warn("ERROR retrieving image {0}: Empty response.".format(url))
callback(None)
else:
callback(response.body)
def should_run(self, image_extension, buffer):
if 'gif' in image_extension and 'gifv' in self.context.request.filters:
if not exists(self.context.config.FFMPEG_PATH):
logger.warn(
'gifv optimizer enabled but binary FFMPEG_PATH does not exist'
)
return False
return True
return False
def should_run(self, image_extension, buffer):
if image_extension in ['.jpg', '.jpeg']:
if not exists(self.context.config.JPEGTRAN_PATH):
logger.warn(
'jpegtran optimizer enabled but binary JPEGTRAN_PATH does not exist'
)
return False
return True
return False
def validate(self, path):
if not hasattr(self.context.modules.loader, 'validate'):
return True
is_valid = self.context.modules.loader.validate(self.context, path)
if not is_valid:
logger.warn('Request denied because the specified path "%s" was not identified by the loader as a valid path' % path)
return is_valid
def validate(self, path):
if not hasattr(self.context.modules.loader, 'validate'):
return True
is_valid = self.context.modules.loader.validate(self.context, path)
if not is_valid:
logger.warn('Request denied because the specified path "%s" was not identified by the loader as a valid path' % path)
return is_valid
def last_updated(self):
path = self.context.request.url
file_abspath = self.normalize_path(path)
if not self.validate_path(file_abspath):
logger.warn("[RESULT_STORAGE] unable to read from outside root path: %s" % file_abspath)
return True
logger.debug("[RESULT_STORAGE] getting from %s" % file_abspath)
if not exists(file_abspath) or self.is_expired(file_abspath):
logger.debug("[RESULT_STORAGE] image not found at %s" % file_abspath)
return True
return datetime.fromtimestamp(getmtime(file_abspath)).replace(tzinfo=pytz.utc)
def handle_data(file_key):
if not file_key or 'Error' in file_key or 'Body' not in file_key:
logger.warn("ERROR retrieving image from S3 {0}: {1}".format(key, str(file_key)))
# If we got here, there was a failure. We will return 404 if S3 returned a 404, otherwise 502.
result = LoaderResult()
result.successful = False
if file_key and file_key.get('ResponseMetadata', {}).get('HTTPStatusCode') == 404:
result.error = LoaderResult.ERROR_NOT_FOUND
else:
result.error = LoaderResult.ERROR_UPSTREAM
callback(result)
else:
callback(file_key['Body'].read())
def get(self):
path = self.context.request.url
file_abspath = self.normalize_path(path)
if not self.validate_path(file_abspath):
logger.warn("[RESULT_STORAGE] unable to read from outside root path: %s" % file_abspath)
return None
logger.debug("[RESULT_STORAGE] getting from %s" % file_abspath)
if not exists(file_abspath) or self.is_expired(file_abspath):
logger.debug("[RESULT_STORAGE] image not found at %s" % file_abspath)
return None
with open(file_abspath, 'r') as f:
return f.read()
def get(self):
path = self.context.request.url
file_abspath = self.normalize_path(path)
if not self.validate_path(file_abspath):
logger.warn("[RESULT_STORAGE] unable to read from outside root path: %s" % file_abspath)
return None
logger.debug("[RESULT_STORAGE] getting from %s" % file_abspath)
if not exists(file_abspath) or self.is_expired(file_abspath):
logger.debug("[RESULT_STORAGE] image not found at %s" % file_abspath)
return None
with open(file_abspath, 'r') as f:
return f.read()
def last_updated(self):
path = self.context.request.url
file_abspath = self.normalize_path(path)
if not self.validate_path(file_abspath):
logger.warn("[RESULT_STORAGE] unable to read from outside root path: %s" % file_abspath)
return True
logger.debug("[RESULT_STORAGE] getting from %s" % file_abspath)
if not exists(file_abspath) or self.is_expired(file_abspath):
logger.debug("[RESULT_STORAGE] image not found at %s" % file_abspath)
return True
return datetime.fromtimestamp(getmtime(file_abspath)).replace(tzinfo=pytz.utc)
def get(self, callback):
path = self.context.request.url
file_abspath = self.normalize_path(path)
if not self.validate_path(file_abspath):
logger.warn(
"[RESULT_STORAGE] unable to read from outside root path: %s" %
file_abspath)
return callback(None)
logger.debug("[RESULT_STORAGE] getting from %s" % file_abspath)
if isdir(file_abspath):
logger.warn("[RESULT_STORAGE] cache location is a directory: %s" %
file_abspath)
return callback(None)
if not exists(file_abspath):
legacy_path = self.normalize_path_legacy(path)
if isfile(legacy_path):
logger.debug(
"[RESULT_STORAGE] migrating image from old location at %s"
% legacy_path)
self.ensure_dir(dirname(file_abspath))
move(legacy_path, file_abspath)
else:
logger.debug("[RESULT_STORAGE] image not found at %s" %
file_abspath)
return callback(None)
if self.is_expired(file_abspath):
logger.debug("[RESULT_STORAGE] cached image has expired")
return callback(None)
with open(file_abspath, 'r') as f:
buffer = f.read()
result = ResultStorageResult(
buffer=buffer,
metadata={
'LastModified':
datetime.fromtimestamp(
getmtime(file_abspath)).replace(tzinfo=pytz.utc),
'ContentLength':
len(buffer),
'ContentType':
BaseEngine.get_mimetype(buffer)
})
callback(result)
def trim(self):
if self.context.request.trim is None or not trim_enabled:
return
box = _bounding_box.apply(self.engine.get_image_mode(), self.engine.size[0], self.engine.size[1], self.context.request.trim_pos, self.context.request.trim_tolerance, self.engine.get_image_data())
if box[2] < box[0] or box[3] < box[1]:
logger.warn("Ignoring trim, there wouldn't be any image left, check the tolerance.")
return
self.engine.crop(box[0], box[1], box[2] + 1, box[3] + 1)
if self.context.request.should_crop:
self.context.request.crop['left'] -= box[0]
self.context.request.crop['top'] -= box[1]
self.context.request.crop['right'] -= box[0]
self.context.request.crop['bottom'] -= box[1]
def return_contents(response, url, callback, context):
context.statsd_client.incr('original_image.status.' + str(response.code))
if response.error:
logger.warn("ERROR retrieving image {0}: {1}".format(url, str(response.error)))
callback(None)
elif response.body is None or len(response.body) == 0:
logger.warn("ERROR retrieving image {0}: Empty response.".format(url))
callback(None)
else:
if response.time_info:
for x in response.time_info:
context.statsd_client.timing('original_image.time_info.' + x, response.time_info[x] * 1000)
context.statsd_client.timing('original_image.time_info.bytes_per_second', len(response.body) / response.time_info['total'])
callback(response.body)
def on_fetch_done(self, result):
if isinstance(result, LoaderResult) and not result.successful:
logger.warn('bad watermark result error=%s metadata=%s' %
(result.error, result.metadata))
raise tornado.web.HTTPError(400)
if isinstance(result, LoaderResult):
buffer = result.buffer
else:
buffer = result
self.storage.put(self.url, buffer)
self.storage.put_crypto(self.url)
self.on_image_ready(buffer)
def put(self, bytes):
file_abspath = self.normalize_path(self.context.request.url)
if not self.validate_path(file_abspath):
logger.warn("[RESULT_STORAGE] unable to write outside root path: %s" % file_abspath)
return
temp_abspath = "%s.%s" % (file_abspath, str(uuid4()).replace('-', ''))
file_dir_abspath = dirname(file_abspath)
logger.debug("[RESULT_STORAGE] putting at %s (%s)" % (file_abspath, file_dir_abspath))
self.ensure_dir(file_dir_abspath)
with open(temp_abspath, 'w') as _file:
_file.write(bytes)
move(temp_abspath, file_abspath)
def on_fetch_done(self, result):
if not result.successful:
logger.warn(
'bad watermark result error=%s metadata=%s' %
(result.error, result.metadata))
raise tornado.web.HTTPError(400)
if isinstance(result, LoaderResult):
buffer = result.buffer
else:
buffer = result
self.watermark_engine.load(buffer, self.extension)
self.storage.put(self.url, self.watermark_engine.read())
self.storage.put_crypto(self.url)
self.on_image_ready(buffer)
def poolcounter_throttle(self, filename, extension):
self.pc = None
if not self.context.config.get('POOLCOUNTER_SERVER', False):
raise tornado.gen.Return(False)
self.pc = PoolCounter(self.context)
cfg = self.context.config.get('POOLCOUNTER_CONFIG_PER_IP', False)
if cfg:
ff = self.request.headers.get('X-Forwarded-For', False)
if not ff:
logger.warn('[ImagesHandler] No X-Forwarded-For header in request, cannot throttle per IP')
else:
ff = ff.split(', ')[0]
throttled = yield self.poolcounter_throttle_key('thumbor-ip-%s' % ff, cfg)
if throttled:
raise tornado.gen.Return(True)
cfg = self.context.config.get('POOLCOUNTER_CONFIG_PER_ORIGINAL', False)
if cfg:
name_sha1 = hashlib.sha1(filename).hexdigest()
throttled = yield self.poolcounter_throttle_key('thumbor-render-%s' % name_sha1, cfg)
if throttled:
raise tornado.gen.Return(True)
cfg = self.context.config.get('POOLCOUNTER_CONFIG_EXPENSIVE', False)
if cfg and extension.lower() in cfg['extensions']:
throttled = yield self.poolcounter_throttle_key('thumbor-render-expensive', cfg)
if throttled:
raise tornado.gen.Return(True)
# This closes the PoolCounter connection in case it hasn't been closed normally.
# Which can happen if an exception occured while processing the file, for example.
release_timeout = self.context.config.get('POOLCOUNTER_RELEASE_TIMEOUT', False)
if release_timeout:
logger.debug('[ImagesHandler] Setting up PoolCounter cleanup callback')
tornado.ioloop.IOLoop.instance().call_later(
release_timeout,
partial(close_poolcounter, self.pc)
)
raise tornado.gen.Return(False)
def detect(self, callback):
try:
features = self.get_features()
except Exception:
logger.warn('Error during face detection; skipping to next detector')
self.next(callback)
return
if features:
for (left, top, width, height), neighbors in features:
top = self.__add_hair_offset(top, height)
self.context.request.focal_points.append(
FocalPoint.from_square(left, top, width, height, origin="Face Detection")
)
callback()
else:
self.next(callback)
def __getattr__(self, name):
if name in self.__dict__:
return self.__dict__[name]
if name in Config.class_aliased_items:
logger.warn(
"Option %s is marked as deprecated please use %s instead." % (name, Config.class_aliased_items[name])
)
return self.__getattr__(Config.class_aliased_items[name])
if "defaults" in self.__dict__ and name in self.__dict__["defaults"]:
return self.__dict__["defaults"][name]
if name in Config.class_defaults:
return Config.class_defaults[name]
raise AttributeError(name)
def _error(self, status, msg=None):
# If the error isn't due to throttling, we increase a
# failure counter for the given xkey, which will let us
# avoid re-trying thumbnails bound to fail too many times
xkey = self._headers.get('xkey', False)
mc = self.failure_memcache()
if status != 429 and status != 404 and xkey and mc:
key = self._mc_encode_key(xkey)
start = datetime.datetime.now()
counter = mc.get(key)
if not counter:
# We add randomness to the expiry to avoid stampedes
duration = self.context.config.get('FAILURE_THROTTLING_DURATION', 3600)
mc.set(key, '1', duration + random.randint(0, 300))
else:
mc.incr(key)
record_timing(self.context, datetime.datetime.now() - start, 'memcache.set', 'Thumbor-Memcache-Set-Time')
# Errors should explicitely not be cached
self.set_header('Cache-Control', 'no-cache')
self.clear_header('xkey')
self.clear_header('Content-Disposition')
self.clear_header('Thumbor-Wikimedia-Original-Container')
self.clear_header('Thumbor-Wikimedia-Thumbnail-Container')
self.clear_header('Thumbor-Wikimedia-Original-Path')
self.clear_header('Thumbor-Wikimedia-Thumbnail-Path')
self.clear_header('Thumbor-Parameters')
if status == 429:
# 429 is missing from httplib.responses
self.set_status(status, 'Too Many Requests')
else:
self.set_status(status)
if msg is not None:
try:
logger.warn(msg, extra=log_extra(self.context.request.url))
except AttributeError:
logger.warn(msg)
self.finish()
def run_optimizer(self, image_extension, buffer):
if not self.should_run(image_extension, buffer):
return buffer
if 'strip_icc' in self.context.request.filters:
copy_chunks = 'comments'
else:
# have to copy everything to preserve icc profile
copy_chunks = 'all'
command = [
self.context.config.JPEGTRAN_PATH,
'-copy',
copy_chunks,
'-optimize',
]
if self.context.config.PROGRESSIVE_JPEG:
command += [
'-progressive'
]
if self.context.config.JPEGTRAN_SCANS_FILE:
if exists(self.context.config.JPEGTRAN_SCANS_FILE):
command += [
'-scans',
self.context.config.JPEGTRAN_SCANS_FILE
]
else:
logger.warn('jpegtran optimizer scans file does not exist')
# close_fds would have a sane default on Python 3 but with Python 2.7 it is False
# per default but setting it to True + setting any of stdin, stdout, stderr will
# make this crash on Windows
# TODO remove close_fds=True if code was upgraded to Python 3
jpg_process = Popen(command, stdin=PIPE, stdout=PIPE,
stderr=PIPE, close_fds=True)
output_stdout, output_stderr = jpg_process.communicate(buffer)
if jpg_process.returncode != 0:
logger.warn('jpegtran finished with non-zero return code (%d): %s'
% (jpg_process.returncode, output_stderr))
return buffer
return output_stdout
def load_sync(context, url, callback):
if _use_http_loader(context, url):
return http_loader.load_sync(context, url, callback, normalize_url_func=_normalize_url)
bucket, key = _get_bucket_and_key(context, url)
if _validate_bucket(context, bucket):
bucket_loader = Bucket(
connection=get_connection(context),
name=bucket
)
file_key = None
try:
file_key = bucket_loader.get_key(url)
except Exception, e:
logger.warn("ERROR retrieving image from S3 {0}: {1}".format(url, str(e)))
if file_key:
callback(file_key.read())
return
def _process_result_from_storage(self, result):
if self.context.config.SEND_IF_MODIFIED_LAST_MODIFIED_HEADERS:
# Handle If-Modified-Since & Last-Modified header
try:
result_last_modified = self.context.modules.result_storage.last_updated()
except NotImplementedError:
logger.warn('last_updated method is not supported by your result storage service, hence If-Modified-Since & Last-Updated headers support is disabled.')
if result_last_modified:
if 'If-Modified-Since' in self.request.headers:
date_modified_since = datetime.datetime.strptime(self.request.headers['If-Modified-Since'], HTTP_DATE_FMT)
if result_last_modified <= date_modified_since:
self.set_status(304)
self.finish()
return
self.set_header('Last-Modified', result_last_modified.strftime(HTTP_DATE_FMT))
return result