def put(self, path, contents): key = self.key_for(path) try: self.storage.set(key, contents, time=self.context.config.STORAGE_EXPIRATION_SECONDS) except: logger.exception("[MEMCACHED] failed to set key '{0}'".format(key)); return path
async def smart_detect(self): is_gifsicle = ( self.context.request.engine.extension == ".gif" and self.context.config.USE_GIFSICLE_ENGINE ) if ( not (self.context.modules.detectors and self.context.request.smart) ) or is_gifsicle: return try: await self.do_smart_detection() except Exception: # pylint: disable=broad-except if not self.context.config.IGNORE_SMART_ERRORS: raise logger.exception("Ignored error during smart detection") if self.context.config.USE_CUSTOM_ERROR_HANDLING: self.context.modules.importer.error_handler.handle_error( context=self.context, handler=self.context.request_handler, exception=sys.exc_info(), ) self.context.request.prevent_result_storage = True self.context.request.detection_error = True
def put_detector_data(self, path, data): key = self.detector_key_for(path) try: self.storage.set(key, dumps(data)) except: logger.exception("[MEMCACHED] failed to set key '{0}'".format(key)); return key
def get_video_frame(context, file_path): """ A context manager that extracts a single frame out of a video file and stores it in a temporary file. Returns the path of the temporary file or None in case of failure. Depends on FFMPEG_PATH from Thumbor's configuration. """ import subprocess import tempfile import os # Fail nicely when ffmpeg cannot be found if not os.path.exists(context.config.FFMPEG_PATH): logger.error('%s does not exist, please configure FFMPEG_PATH', context.config.FFMPEG_PATH) yield None return # Prepare temporary file f, image_path = tempfile.mkstemp('.jpg') os.close(f) # Extract image try: cmd = [ context.config.FFMPEG_PATH, '-i', file_path, '-ss', '00:00:01.000', '-vframes', '1', '-y', '-nostats', '-loglevel', 'error', image_path ] subprocess.check_call(cmd) yield image_path except: logger.exception('Cannot extract image frame from %s', file_path) yield None finally: # Cleanup try_to_delete(image_path)
def smart_detect(self): if not (self.context.modules.detectors and self.context.request.smart): self.do_image_operations() return try: # Beware! Boolean hell ahead. # # The `running_smart_detection` flag is needed so we can know # whether `after_smart_detect()` is running synchronously or not. # # If we're running it in a sync fashion it will set # `should_run_image_operations` to True so we can avoid running # image operation inside the try block. self.should_run_image_operations = False self.running_smart_detection = True self.do_smart_detection() self.running_smart_detection = False except Exception: if not self.context.config.IGNORE_SMART_ERRORS: raise logger.exception("Ignored error during smart detection") if self.context.config.USE_CUSTOM_ERROR_HANDLING: self.context.modules.importer.error_handler.handle_error(context=self.context, handler=self.context.request_handler, exception=sys.exc_info()) self.context.request.prevent_result_storage = True self.context.request.detection_error = True self.do_image_operations() if self.should_run_image_operations: self.do_image_operations()
def detect(self, callback): engine = self.context.modules.engine try: img = np.array( engine.convert_to_grayscale(update_image=False, with_alpha=False)) except Exception as e: logger.exception(e) logger.warn( 'Error during feature detection; skipping to next detector') self.next(callback) return points = cv2.goodFeaturesToTrack( img, maxCorners=20, qualityLevel=0.04, minDistance=1.0, useHarrisDetector=False, ) if points is not None: for point in points: x, y = point.ravel() self.context.request.focal_points.append( FocalPoint(x.item(), y.item(), 1)) callback() else: self.next(callback)
def srgb(self): try: if not isinstance(self.engine, PILEngine): logger.warn('Could not perform profileToProfile conversion: engine is not PIL engine') return if (ImageCms is None): logger.warn('ImageCms is not installed. Could not perform profileToProfile conversion') return image = self.engine.image embedded_profile = image.info.get('icc_profile') if not embedded_profile: logger.debug('Image does not have embedded profile. Assuming already in sRGB') return embedded_profile = BytesIO(embedded_profile) srgb_profile = BytesIO(tiny_srgb) output_mode = 'RGBA' if 'A' in image.mode else 'RGB' image = ImageCms.profileToProfile(image, embedded_profile, srgb_profile, renderingIntent=0, outputMode=output_mode) self.engine.image = image self.engine.icc_profile = image.info.get('icc_profile') except Exception as err: logger.exception(err)
def put_detector_data(self, path, data): key = self.detector_key_for(path) try: self.storage.set(key, dumps(data)) except: logger.exception("[MEMCACHED] failed to set key '{0}'".format(key)) return key
def smart_detect(self): if not (self.context.modules.detectors and self.context.request.smart): self.do_image_operations() return try: # Beware! Boolean hell ahead. # # The `running_smart_detection` flag is needed so we can know # whether `after_smart_detect()` is running synchronously or not. # # If we're running it in a sync fashion it will set # `should_run_image_operations` to True so we can avoid running # image operation inside the try block. self.should_run_image_operations = False self.running_smart_detection = True self.do_smart_detection() self.running_smart_detection = False except Exception: if not self.context.config.IGNORE_SMART_ERRORS: raise logger.exception("Ignored error during smart detection") if self.context.config.USE_CUSTOM_ERROR_HANDLING: self.context.modules.importer.error_handler.handle_error( context=self.context, handler=self.context.request_handler, exception=sys.exc_info()) self.context.request.prevent_result_storage = True self.context.request.detection_error = True self.do_image_operations() if self.should_run_image_operations: self.do_image_operations()
async def finish_request(self, result_from_storage=None): if result_from_storage is not None: await self._process_result_from_storage(result_from_storage) _, content_type = self.define_image_type(self.context, result_from_storage) await self._write_results_to_client(result_from_storage, content_type) return context = self.context result_storage = context.modules.result_storage metrics = context.metrics should_store = (result_storage and not context.request.prevent_result_storage and (context.config.RESULT_STORAGE_STORES_UNSAFE or not context.request.unsafe)) try: result = await self.context.thread_pool.queue( operation=functools.partial(self._load_results, context), ) except Exception as error: # pylint: disable=broad-except logger.exception("[BaseHander.finish_request] %s", error) self._error( 500, "Error while trying to fetch the image: {}".format(error)) return (results, content_type) = result await self._write_results_to_client(results, content_type) if should_store: await self._store_results(result_storage, metrics, results)
def watermark(self, callback, url, x, y, alpha, w_ratio=False, h_ratio=False): self.url = url self.x = x self.y = y self.alpha = alpha self.w_ratio = float( w_ratio) / 100.0 if w_ratio and w_ratio != 'none' else False self.h_ratio = float( h_ratio) / 100.0 if h_ratio and h_ratio != 'none' else False self.callback = callback self.watermark_engine = self.context.modules.engine.__class__( self.context) self.storage = self.context.modules.storage try: buffer = yield tornado.gen.maybe_future(self.storage.get(self.url)) if buffer is not None: self.on_image_ready(buffer) else: self.context.modules.loader.load(self.context, self.url, self.on_fetch_done) except Exception as e: logger.exception(e) logger.warn("bad watermark") raise tornado.web.HTTPError(500)
def request(self): try: self.thumbor_filter.context.modules.loader.load( self.thumbor_filter.context, self.url, self.on_fetch_done) except Exception as err: self.failed = True logger.exception(err)
async def detect(self): self.context.request.prevent_result_storage = True try: if not QueuedDetector.queue: redis = Redis( host=self.context.config.REDIS_QUEUE_SERVER_HOST, port=self.context.config.REDIS_QUEUE_SERVER_PORT, db=self.context.config.REDIS_QUEUE_SERVER_DB, password=self.context.config.REDIS_QUEUE_SERVER_PASSWORD, ) QueuedDetector.queue = UniqueQueue(server=redis) QueuedDetector.queue.enqueue_unique_from_string( "remotecv.pyres_tasks.DetectTask", "Detect", args=[self.detection_type, self.context.request.image_url], key=self.context.request.image_url, ) except RedisError: self.context.request.detection_error = True QueuedDetector.queue = None logger.exception("Redis Error") # Error or not we return an empty list as detection # will be done later return []
async def detect(self): engine = self.context.modules.engine try: img = np.array( engine.convert_to_grayscale(update_image=False, alpha=False)) except Exception as error: logger.exception(error) logger.warning( "Error during feature detection; skipping to next detector") return await self.next() # pylint: disable=not-callable points = cv2.goodFeaturesToTrack( # pylint: disable=no-member img, maxCorners=20, qualityLevel=0.04, minDistance=1.0, useHarrisDetector=False, ) if points is not None: for point in points: x_pos, y_pos = point.ravel() self.context.request.focal_points.append( FocalPoint(x_pos.item(), y_pos.item(), 1)) return await self.next() # pylint: disable=not-callable
async def detect(self): self.context.request.prevent_result_storage = True try: if not Detector.detect_task: celery_tasks = CeleryTasks( self.context.config.SQS_QUEUE_KEY_ID, self.context.config.SQS_QUEUE_KEY_SECRET, self.context.config.SQS_QUEUE_REGION, None, ) Detector.detect_task = celery_tasks.get_detect_task() Detector.detect_task.delay( "all", self.context.request.image_url, self.context.request.image_url, ) except RuntimeError: self.context.request.detection_error = True Detector.detect_task = None logger.exception("Celery Error") # Error or not we return an empty list as detection # will be done later return []
def detect(self, callback): engine = self.context.modules.engine try: img = np.array( engine.convert_to_grayscale( update_image=False, with_alpha=False ) ) except Exception as e: logger.exception(e) logger.warn('Error during feature detection; skipping to next detector') self.next(callback) return points = cv2.goodFeaturesToTrack( img, maxCorners=20, qualityLevel=0.04, minDistance=1.0, useHarrisDetector=False, ) if points is not None: for point in points: x, y = point.ravel() self.context.request.focal_points.append(FocalPoint(x.item(), y.item(), 1)) callback() else: self.next(callback)
def detect(self, callback): engine = self.context.modules.engine try: engine.image_data_as_rgb() img = np.array(engine.image) self.net.setInput( cv2.dnn.blobFromImage(img, size=(300, 300), swapRB=True)) detections = self.net.forward() except Exception as e: logger.exception(e) logger.warn( 'Error during feature detection; skipping to next detector') self.next(callback) return confidence_threshold = 0.2 num_detections = 0 for detection in detections[0, 0, :, :]: confidence = float(detection[2]) if confidence < confidence_threshold: continue num_detections += 1 class_id = int(detection[1]) - 1 # make it zero-indexed class_name = coco_classes[class_id] left = int(detection[3] * img.shape[1]) top = int(detection[4] * img.shape[0]) right = int(detection[5] * img.shape[1]) bottom = int(detection[6] * img.shape[0]) width = right - left height = bottom - top # If the detection is of a person, # and the person is vertically oriented, # this uses the upper 1/4 of the box to focus on the face. # In the case the person is horizontal, perhaps reclining, # then the focal point will remain at their center. # In the case the person is upside down, this would focus on the feet instead of the face. # But consider - whoever is publishing a picture of an upside down person # might appreciate that it focuses on the feet. if class_name == 'person' and height > width: height = int(height * 0.25) self.context.request.focal_points.append( FocalPoint.from_dict({ 'x': left + (width / 2), 'y': top + (height / 2), 'width': width, 'height': height, 'z': confidence, 'origin': 'DNN Object Detection (class: {})'.format(class_name) })) if num_detections > 0: callback() else: self.next(callback)
def _get_exif_segment(self): try: segment = ExifSegment(None, None, self.exif, 'ro') except Exception: logger.exception('Ignored error handling exif for reorientation') else: return segment return None
async def execute_image_operations(self): self.context.request.quality = None req = self.context.request conf = self.context.config should_store = (self.context.config.RESULT_STORAGE_STORES_UNSAFE or not self.context.request.unsafe) if self.context.modules.result_storage and should_store: start = datetime.datetime.now() try: result = await self.context.modules.result_storage.get() except Exception as error: logger.exception("[BaseHander.execute_image_operations] %s", error) self._error( 500, "Error while trying to get the image " "from the result storage: {}".format(error), ) return finish = datetime.datetime.now() self.context.metrics.timing( "result_storage.incoming_time", (finish - start).total_seconds() * 1000, ) if result is None: self.context.metrics.incr("result_storage.miss") else: self.context.metrics.incr("result_storage.hit") self.context.metrics.incr("result_storage.bytes_read", len(result)) logger.debug("[RESULT_STORAGE] IMAGE FOUND: %s", req.url) await self.finish_request(result) return if (conf.MAX_WIDTH and (not isinstance(req.width, str)) and req.width > conf.MAX_WIDTH): req.width = conf.MAX_WIDTH if (conf.MAX_HEIGHT and (not isinstance(req.height, str)) and req.height > conf.MAX_HEIGHT): req.height = conf.MAX_HEIGHT req.meta_callback = (conf.META_CALLBACK_NAME or self.request.arguments.get("callback", [None])[0]) self.filters_runner = self.context.filters_factory.create_instances( self.context, self.context.request.filters) # Apply all the filters from the PRE_LOAD phase # and call get_image() afterwards. await self.filters_runner.apply_filters( thumbor.filters.PHASE_PRE_LOAD, ) await self.get_image()
def read(self, extension=None, quality=None): #returns image buffer in byte format. img_buffer = BytesIO() ext = extension or self.extension options = { 'quality': quality } if ext == '.jpg' or ext == '.jpeg': options['optimize'] = True options['progressive'] = True if quality is None: options['quality'] = 'keep' if options['quality'] is None: options['quality'] = self.context.config.QUALITY if self.icc_profile is not None: options['icc_profile'] = self.icc_profile if self.context.config.PRESERVE_EXIF_INFO: exif = self.image.info.get('exif', None) if exif is not None: options['exif'] = exif if self.image.mode == 'P' and self.transparency: options['transparency'] = self.transparency try: if ext == '.webp': if self.image.mode not in ['RGB', 'RGBA']: mode = None if self.image.mode != 'P': mode = 'RGBA' if self.image.mode[-1] == 'A' else 'RGB' self.image = self.image.convert(mode) self.image.save(img_buffer, FORMATS[ext], **options) except IOError: logger.exception('Could not save as improved image, consider to increase ImageFile.MAXBLOCK') self.image.save(img_buffer, FORMATS[ext]) except KeyError: logger.exception('Image format not found in PIL: %s' % ext) #extension is not present or could not help determine format => force JPEG if self.image.mode in ['P', 'RGBA', 'LA']: self.image.format = FORMATS['.png'] self.image.save(img_buffer, FORMATS['.png']) else: self.image.format = FORMATS['.jpg'] self.image.save(img_buffer, FORMATS['.jpg']) results = img_buffer.getvalue() img_buffer.close() return results
def _load_results(self, context): try: results, content_type = BaseHandler._load_results(self, context) except Exception: logger.exception('[ImagesHandler] Exception during _load_results', extra=log_extra(context)) self._error(500) return None, None return results, content_type
def put(self, path, contents): key = self.key_for(path) try: self.storage.set( key, contents, time=self.context.config.STORAGE_EXPIRATION_SECONDS) except: logger.exception("[MEMCACHED] failed to set key '{0}'".format(key)) return path
def detect_faces(self, raw_bytes): try: resp = self.rekognition_client().detect_faces( Image={'Bytes': raw_bytes}, Attributes=['DEFAULT']) return [Face(fd) for fd in resp.get('FaceDetails', [])] except Exception as e: logger.exception(e) return []
def _get_exif_object(self): if (not hasattr(self, "exif")) or self.exif is None: return None try: return ExifOrientationEditor(self.exif) except Exception as error: logger.exception("[exif] %s", error) return None
def _execute_in_foreground(self, operation, callback): result = Future() returned = None try: returned = operation() except Exception as e: # just log exception and release ioloop returned = e logger.exception(e) result.set_result(returned) callback(result)
def _get_exif_object(self): if (not hasattr(self, 'exif')) or self.exif is None: return None try: return ExifOrientationEditor(self.exif) except Exception as e: msg = """[exif] %s""" % e logger.exception(msg) return None
def _get_exif_segment(self): if (not hasattr(self, 'exif')) or self.exif is None: return None try: segment = ExifSegment(None, None, self.exif, 'ro') except Exception: logger.exception('Ignored error handling exif for reorientation') else: return segment return None
def _get_exif_segment(self): if (not hasattr(self, 'exif')) or self.exif is None: return None try: exif_dict = piexif.load(self.exif) except Exception: logger.exception('Ignored error handling exif for reorientation') else: return exif_dict return None
def _execute_in_foreground(self, operation, callback): result = Future() try: returned = operation() except Exception as e: logger.exception('[ThreadPool] %s', e) result.set_exception(e) else: result.set_result(returned) callback(result)
def execute_image_operations(self): self.context.request.quality = None req = self.context.request conf = self.context.config should_store = self.context.config.RESULT_STORAGE_STORES_UNSAFE or not self.context.request.unsafe if self.context.modules.result_storage and should_store: start = datetime.datetime.now() try: result = yield gen.maybe_future( self.context.modules.result_storage.get()) except Exception as e: logger.exception('[BaseHander.execute_image_operations] %s', e) self._error( 500, 'Error while trying to get the image from the result storage: {}' .format(e)) return finish = datetime.datetime.now() self.context.metrics.timing('result_storage.incoming_time', (finish - start).total_seconds() * 1000) if result is None: self.context.metrics.incr('result_storage.miss') else: self.context.metrics.incr('result_storage.hit') self.context.metrics.incr('result_storage.bytes_read', len(result)) logger.debug('[RESULT_STORAGE] IMAGE FOUND: %s' % req.url) self.finish_request(result) return if conf.MAX_WIDTH and (not isinstance( req.width, basestring)) and req.width > conf.MAX_WIDTH: req.width = conf.MAX_WIDTH if conf.MAX_HEIGHT and (not isinstance( req.height, basestring)) and req.height > conf.MAX_HEIGHT: req.height = conf.MAX_HEIGHT req.meta_callback = conf.META_CALLBACK_NAME or self.request.arguments.get( 'callback', [None])[0] self.filters_runner = self.context.filters_factory.create_instances( self.context, self.context.request.filters) # Apply all the filters from the PRE_LOAD phase and call get_image() afterwards. self.filters_runner.apply_filters(thumbor.filters.PHASE_PRE_LOAD, self.get_image)
def inner(future): try: future_result = future.result() except Exception as e: logger.exception('[BaseHander.finish_request] %s', e) self._error(500, 'Error while trying to fetch the image: {}'.format(e)) return results, content_type = future_result self._write_results_to_client(context, results, content_type) if should_store: self._store_results(context, results)
def put_crypto(self, path): if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE: return if not self.context.server.security_key: raise RuntimeError("STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified") key = self.crypto_key_for(path) try: self.storage.set(key, self.context.server.security_key) except: logger.exception("[MEMCACHED] failed to set key '{0}'".format(key)); return key
def inner(future): try: future_result = future.result() except Exception as e: logger.exception('[BaseHander.finish_request] %s', e) self._error(500, 'Error while trying to fetch the image: {}'.format(e)) return results, content_type = future_result self._write_results_to_client(results, content_type) if should_store: tornado.ioloop.IOLoop.instance().add_callback(self._store_results, result_storage, metrics, results)
def icc_profile_apply(self, profile=None): # Check whether input image has color management. if not self.engine.icc_profile: logger.info('ICC: Image has no embedded profile. Skipping this image.') return # Sanitize profile parameter. if profile != None: profile = os.path.basename(profile).lstrip('.') if len(profile) == 0: logger.warning('ICC: Invalid profile name.') return # Find output profile. outprofile = self._find_profile(profile) if not outprofile: logger.warning('ICC: Failed to load profile: {:s}'.format(profile)) return try: ext = self.engine.extension fmt = Image.EXTENSION[ext.lower()] except: logger.exception('ICC: Failed to determine image format and extension before attempting to apply profile: {:s}'.format(profile)) return try: inmode = self.engine.get_image_mode() insize = self.engine.size inimg = Image.frombytes(inmode, insize, self.engine.get_image_data()) # In PIL>=3.0.0 / Thumbor 6, icc_profile is sometimes a tuple :/ # https://github.com/python-pillow/Pillow/issues/1462 profile_data = self.engine.icc_profile if type(profile_data) == tuple: profile_data = profile_data[0] inprofile = StringIO(profile_data) outmode = 'RGBA' if 'A' in inmode else 'RGB' except: logger.exception('ICC: Failed to determine image properties before attempting to apply profile: {:s}'.format(profile)) return logger.info('ICC: Attempting to apply profile: {:s}, inmode: {:s}, outmode: {:s}'.format(profile, inmode, outmode)) try: outimg = ImageCms.profileToProfile(inimg, inprofile, outprofile, outputMode=outmode) except: logger.exception('ICC: Failed to apply profile: {:s}, inmode: {:s}, outmode: {:s}'.format(profile, inmode, outmode)) return # Reload the image into the engine. outbuf = StringIO() try: outimg.save(outbuf, fmt) self.engine.load(outbuf.getvalue(), ext) except: logger.exception('ICC: Failed load the image with an applied profile: {:s}, inmode: {:s}, outmode: {:s}'.format(profile, inmode, outmode)) return finally: outbuf.close()
def read(self, extension=None, quality=None): if quality is None: quality = self.context.request.quality #returns image buffer in byte format. img_buffer = BytesIO() ext = extension or self.extension options = { 'quality': quality } if ext == '.jpg' or ext == '.jpeg': options['optimize'] = True options['progressive'] = True if self.icc_profile is not None: options['icc_profile'] = self.icc_profile if self.context.config.PRESERVE_EXIF_INFO: exif = self.image.info.get('exif', None) if exif is not None: options['exif'] = exif image_format = self.context.request.format if image_format is None: image_format = FORMATS[ext] image_format = str(image_format).upper() try: if image_format == 'WEBP' and self.image.mode in ['L', 'LA', 'P', 'RGBA']: self.image = self.image.convert('RGB') self.image.save(img_buffer, image_format, **options) except IOError: logger.exception('Could not save as improved image, consider to increase ImageFile.MAXBLOCK') self.image.save(img_buffer, FORMATS[ext]) except KeyError: logger.exception('Image format not found in PIL: %s' % image_format) #extension is not present or could not help determine format => force JPEG #TODO : guess format by image headers maybe if self.image.mode in ['P', 'RGBA', 'LA']: self.image.format = FORMATS['.png'] self.image.save(img_buffer, FORMATS['.png']) else: self.image.format = FORMATS['.jpg'] self.image.save(img_buffer, FORMATS['.jpg']) results = img_buffer.getvalue() img_buffer.close() return results
def put_crypto(self, path): if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE: return if not self.context.server.security_key: raise RuntimeError( "STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified" ) key = self.crypto_key_for(path) try: self.storage.set(key, self.context.server.security_key) except: logger.exception("[MEMCACHED] failed to set key '{0}'".format(key)) return key
def inner(future): try: future_result = future.result() except Exception as e: logger.exception('[BaseHander.finish_request] %s', e) self._error(500, 'Error while trying to fetch the image: {}'.format(e)) return results, content_type = future_result self._write_results_to_client(context, results, content_type) if should_store: self._store_results(context, results) schedule.run_pending()
def process(self, canvas_width, canvas_height, size): try: self.engine.load(self.buffer, self.extension) width, height = self.engine.size new_width, new_height = calc_new_size_by_height(width, height, canvas_height) focal_points = StandaloneFaceDetector.features_to_focal_points( StandaloneFaceDetector.get_features(self.thumbor_filter.context, self.engine)) if focal_points: self.resize_focal_points(focal_points, float(new_width) / width) else: focal_points.append(FocalPoint.from_alignment('center', 'top', new_width, new_height)) self.engine.resize(new_width, new_height) self.engine.focus(focal_points) StandaloneFaceDetector.auto_crop(self.engine, focal_points, size, canvas_height) except Exception as err: logger.exception(err)
def save_on_disc(self): if self.fetched: try: self.engine.load(self.buffer, self.extension) except Exception as err: self.failed = True logger.exception(err) try: self.thumbor_filter.storage.put(self.url, self.engine.read()) self.thumbor_filter.storage.put_crypto(self.url) except Exception as err: self.failed = True logger.exception(err) else: self.failed = True logger.error("filters.distributed_collage: Can't save unfetched image")
def detect(self, callback): self.context.request.prevent_result_storage = True try: if not Detector.detect_task: celery_tasks = CeleryTasks( self.context.config.SQS_QUEUE_KEY_ID, self.context.config.SQS_QUEUE_KEY_SECRET, self.context.config.SQS_QUEUE_REGION, None ) Detector.detect_task = celery_tasks.get_detect_task() Detector.detect_task.delay('all', self.context.request.image_url, self.context.request.image_url) except RuntimeError: self.context.request.detection_error = True Detector.detect_task = None logger.exception('Celery Error') finally: callback([])
def detect(self, callback): try: features = self.get_features() except Exception as e: logger.exception(e) logger.warn('Error during face detection; skipping to next detector') self.next(callback) return if features: for (left, top, width, height), neighbors in features: top = self.__add_hair_offset(top, height) self.context.request.focal_points.append( FocalPoint.from_square(left, top, width, height, origin="Face Detection") ) callback() else: self.next(callback)
def save_on_disc(self): if self.fetched: try: self.engine.load(self.buffer, None) except Exception as err: self.failed = True logger.exception(err) try: self.thumbor_filter.storage.put(self.url, self.engine.read()) self.thumbor_filter.storage.put_crypto(self.url) except Exception as err: self.failed = True logger.exception(err) else: self.failed = True logger.error( "filters.distributed_collage: Can't save unfetched image")
def detect(self, callback): self.context.request.prevent_result_storage = True try: if not QueuedDetector.redis: QueuedDetector.redis = Redis(host=self.context.config.REDIS_QUEUE_SERVER_HOST, port=self.context.config.REDIS_QUEUE_SERVER_PORT, db=self.context.config.REDIS_QUEUE_SERVER_DB, password=self.context.config.REDIS_QUEUE_SERVER_PASSWORD) queue = UniqueQueue(server=QueuedDetector.redis) queue.enqueue_unique_from_string('remotecv.pyres_tasks.DetectTask', 'Detect', args=[self.detection_type, self.context.request.image_url], key=self.context.request.image_url) except RedisError: self.context.request.detection_error = True QueuedDetector.redis = None logger.exception('Redis Error') finally: callback([])
def get_orientation(self, override_exif=True): if (not hasattr(self, 'exif')) or self.exif is None: return orientation = None try: segment = ExifSegment(None, None, self.exif, 'ro') primary = segment.primary orientation = primary['Orientation'] if orientation: orientation = orientation[0] if orientation != 1 and override_exif: primary['Orientation'] = [1] self.exif = segment.get_data() except Exception: logger.exception('Ignored error handling exif for reorientation') finally: return orientation
def get_video_frame(context, file_path): """ A context manager that extracts a single frame out of a video file and stores it in a temporary file. Returns the path of the temporary file or None in case of failure. Depends on FFMPEG_PATH from Thumbor's configuration. """ import subprocess, tempfile, os # Fail nicely when ffmpeg cannot be found if not os.path.exists(context.config.FFMPEG_PATH): logger.error("%s does not exist, please configure FFMPEG_PATH", context.config.FFMPEG_PATH) yield None return # Prepare temporary file f, image_path = tempfile.mkstemp(".jpg") os.close(f) # Extract image try: cmd = [ context.config.FFMPEG_PATH, "-i", file_path, "-ss", "00:00:01.000", "-vframes", "1", "-y", "-nostats", "-loglevel", "error", image_path, ] subprocess.check_call(cmd) yield image_path except: logger.exception("Cannot extract image frame from %s", file_path) yield None finally: # Cleanup try_to_delete(image_path)
def watermark(self, callback, url, x, y, alpha, w_ratio=False, h_ratio=False): self.url = url self.x = x self.y = y self.alpha = alpha self.w_ratio = float(w_ratio) / 100.0 if w_ratio and w_ratio != 'none' else False self.h_ratio = float(h_ratio) / 100.0 if h_ratio and h_ratio != 'none' else False self.callback = callback self.watermark_engine = self.context.modules.engine.__class__(self.context) self.storage = self.context.modules.storage try: buffer = yield tornado.gen.maybe_future(self.storage.get(self.url)) if buffer is not None: self.on_image_ready(buffer) else: self.context.modules.loader.load(self.context, self.url, self.on_fetch_done) except Exception as e: logger.exception(e) logger.warn("bad watermark") raise tornado.web.HTTPError(500)
def assembly(self): logger.debug('filters.distributed_collage: assembly started') canvas = self.create_engine() canvas_width, canvas_height = self.engine.size canvas.image = canvas.gen_image((canvas_width, canvas_height), '#fff') slice_size, major_slice_size = self.divide_size(canvas_width, len(self.images)) for i, url in enumerate(self.urls): image = self.images[url] x, y = i * slice_size, 0 if i == len(self.images) - 1: slice_size = major_slice_size try: image.process(canvas_width, canvas_height, slice_size) canvas.paste(image.engine, (x, y), merge=True) except Exception as err: logger.exception(err) self.engine.image = canvas.image logger.debug('filters.distributed_collage: assembly finished') self.callback()
def execute_image_operations(self): self.context.request.quality = None req = self.context.request conf = self.context.config should_store = self.context.config.RESULT_STORAGE_STORES_UNSAFE or not self.context.request.unsafe if self.context.modules.result_storage and should_store: start = datetime.datetime.now() try: result = yield gen.maybe_future(self.context.modules.result_storage.get()) except Exception as e: logger.exception('[BaseHander.execute_image_operations] %s', e) self._error(500, 'Error while trying to get the image from the result storage: {}'.format(e)) return finish = datetime.datetime.now() self.context.metrics.timing('result_storage.incoming_time', (finish - start).total_seconds() * 1000) if result is None: self.context.metrics.incr('result_storage.miss') else: self.context.metrics.incr('result_storage.hit') self.context.metrics.incr('result_storage.bytes_read', len(result)) logger.debug('[RESULT_STORAGE] IMAGE FOUND: %s' % req.url) self.finish_request(result) return if conf.MAX_WIDTH and (not isinstance(req.width, basestring)) and req.width > conf.MAX_WIDTH: req.width = conf.MAX_WIDTH if conf.MAX_HEIGHT and (not isinstance(req.height, basestring)) and req.height > conf.MAX_HEIGHT: req.height = conf.MAX_HEIGHT req.meta_callback = conf.META_CALLBACK_NAME or self.request.arguments.get('callback', [None])[0] self.filters_runner = self.context.filters_factory.create_instances(self.context, self.context.request.filters) # Apply all the filters from the PRE_LOAD phase and call get_image() afterwards. self.filters_runner.apply_filters(thumbor.filters.PHASE_PRE_LOAD, self.get_image)
def reorientate(self): if (not hasattr(self, 'exif')) or self.exif is None: return orientation = None try: segment = ExifSegment(None, None, self.exif, 'ro') primary = segment.primary orientation = primary['Orientation'] if orientation: orientation = orientation[0] if orientation != 1: primary['Orientation'] = [1] self.exif = segment.get_data() except Exception: logger.exception('Ignored error handling exif for reorientation') if orientation == 2: self.flip_horizontally() elif orientation == 3: self.rotate(180) elif orientation == 4: self.flip_vertically() elif orientation == 5: # Horizontal Mirror + Rotation 270 self.flip_vertically() self.rotate(270) elif orientation == 6: self.rotate(270) elif orientation == 7: # Vertical Mirror + Rotation 270 self.flip_horizontally() self.rotate(270) elif orientation == 8: self.rotate(90)
def read(self, extension=None, quality=None): # NOQA # returns image buffer in byte format. img_buffer = BytesIO() ext = extension or self.extension or self.get_default_extension() options = { 'quality': quality } if ext == '.jpg' or ext == '.jpeg': options['optimize'] = True if self.context.config.PROGRESSIVE_JPEG: # Can't simply set options['progressive'] to the value # of self.context.config.PROGRESSIVE_JPEG because save # operates on the presence of the key in **options, not # the value of that setting. options['progressive'] = True if self.image.mode != 'RGB': self.image = self.image.convert('RGB') else: subsampling_config = self.context.config.PILLOW_JPEG_SUBSAMPLING qtables_config = self.context.config.PILLOW_JPEG_QTABLES if subsampling_config is not None or qtables_config is not None: options['quality'] = 0 # can't use 'keep' here as Pillow would try to extract qtables/subsampling and fail orig_subsampling = self.subsampling orig_qtables = self.qtables if (subsampling_config == 'keep' or subsampling_config is None) and (orig_subsampling is not None): options['subsampling'] = orig_subsampling else: options['subsampling'] = subsampling_config if (qtables_config == 'keep' or qtables_config is None) and (orig_qtables and 2 <= len(orig_qtables) <= 4): options['qtables'] = orig_qtables else: options['qtables'] = qtables_config if options['quality'] is None: options['quality'] = self.context.config.QUALITY if self.icc_profile is not None: options['icc_profile'] = self.icc_profile if self.context.config.PRESERVE_EXIF_INFO: if self.exif is not None: options['exif'] = self.exif if self.image.mode == 'P' and self.transparency: options['transparency'] = self.transparency try: if ext == '.webp': if self.image.mode not in ['RGB', 'RGBA']: mode = None if self.image.mode == 'P': mode = 'RGBA' else: mode = 'RGBA' if self.image.mode[-1] == 'A' else 'RGB' self.image = self.image.convert(mode) if ext in ['.png', '.gif'] and self.image.mode == 'CMYK': self.image = self.image.convert('RGBA') self.image.format = FORMATS[ext] self.image.save(img_buffer, FORMATS[ext], **options) except IOError: logger.exception('Could not save as improved image, consider to increase ImageFile.MAXBLOCK') self.image.save(img_buffer, FORMATS[ext]) except KeyError: logger.exception('Image format not found in PIL: %s' % ext) ext = self.get_default_extension() # extension could not help determine format => use default self.image.format = FORMATS[ext] self.image.save(img_buffer, FORMATS[ext]) results = img_buffer.getvalue() img_buffer.close() self.extension = ext return results
def read(self, extension=None, quality=None): #returns image buffer in byte format. img_buffer = BytesIO() ext = extension or self.extension options = { 'quality': quality } if ext == '.jpg' or ext == '.jpeg': options['optimize'] = True if self.context.config.PROGRESSIVE_JPEG: # Can't simply set options['progressive'] to the value # of self.context.config.PROGRESSIVE_JPEG because save # operates on the presence of the key in **options, not # the value of that setting. options['progressive'] = True if self.image.mode != 'RGB': self.image = self.image.convert('RGB') else: if self.extension == '.jpg': quantization = getattr(self.image, 'quantization', None) if quality is None and quantization and 2 <= len(quantization) <= 4: options['quality'] = 'keep' if options['quality'] is None: options['quality'] = self.context.config.QUALITY if self.icc_profile is not None: options['icc_profile'] = self.icc_profile if self.context.config.PRESERVE_EXIF_INFO: if self.exif is not None: options['exif'] = self.exif if self.image.mode == 'P' and self.transparency: options['transparency'] = self.transparency try: if ext == '.webp': if self.image.mode not in ['RGB', 'RGBA']: mode = None if self.image.mode != 'P': mode = 'RGBA' if self.image.mode[-1] == 'A' else 'RGB' self.image = self.image.convert(mode) if ext == '.png' and self.image.mode == 'CMYK': self.image = self.image.convert('RGBA') self.image.save(img_buffer, FORMATS[ext], **options) except IOError: logger.exception('Could not save as improved image, consider to increase ImageFile.MAXBLOCK') self.image.save(img_buffer, FORMATS[ext]) except KeyError: logger.exception('Image format not found in PIL: %s' % ext) #extension is not present or could not help determine format => force JPEG if self.image.mode in ['P', 'RGBA', 'LA']: self.image.format = FORMATS['.png'] self.image.save(img_buffer, FORMATS['.png']) else: self.image.format = FORMATS['.jpg'] self.image.save(img_buffer, FORMATS['.jpg']) results = img_buffer.getvalue() img_buffer.close() return results