def Clear( self ): with self._lock: self._data_cache.Clear() self._special_thumbs = {} names = [ 'hydrus', 'pdf', 'psd', 'audio', 'video', 'zip' ] bounding_dimensions = self._controller.options[ 'thumbnail_dimensions' ] for name in names: path = os.path.join( HC.STATIC_DIR, name + '.png' ) numpy_image = ClientImageHandling.GenerateNumPyImage( path, HC.IMAGE_PNG ) numpy_image_resolution = HydrusImageHandling.GetResolutionNumPy( numpy_image ) target_resolution = HydrusImageHandling.GetThumbnailResolution( numpy_image_resolution, bounding_dimensions ) numpy_image = HydrusImageHandling.ResizeNumPyImage( numpy_image, target_resolution ) hydrus_bitmap = ClientRendering.GenerateHydrusBitmapFromNumPyImage( numpy_image ) self._special_thumbs[ name ] = hydrus_bitmap self._controller.pub( 'notify_complete_thumbnail_reset' ) self._waterfall_queue_quick = set() self._delayed_regeneration_queue_quick = set() self._RecalcQueues()
def GenerateShapePerceptualHashes( path, mime ): if HG.phash_generation_report_mode: HydrusData.ShowText( 'phash generation: loading image' ) numpy_image = GenerateNumPyImage( path, mime ) if HG.phash_generation_report_mode: HydrusData.ShowText( 'phash generation: image shape: {}'.format( numpy_image.shape ) ) ( y, x, depth ) = numpy_image.shape if depth == 4: # doing this on 10000x10000 pngs eats ram like mad target_resolution = HydrusImageHandling.GetThumbnailResolution( ( x, y ), ( 1024, 1024 ) ) numpy_image = HydrusImageHandling.ResizeNumPyImage( numpy_image, target_resolution ) ( y, x, depth ) = numpy_image.shape # create weight and transform numpy_image to greyscale numpy_alpha = numpy_image[ :, :, 3 ] numpy_alpha_float = numpy_alpha / 255.0 numpy_image_bgr = numpy_image[ :, :, :3 ] numpy_image_gray_bare = cv2.cvtColor( numpy_image_bgr, cv2.COLOR_RGB2GRAY ) # create a white greyscale canvas white = numpy.ones( ( y, x ) ) * 255.0 # paste the grayscale image onto the white canvas using: pixel * alpha + white * ( 1 - alpha ) numpy_image_gray = numpy.uint8( ( numpy_image_gray_bare * numpy_alpha_float ) + ( white * ( numpy.ones( ( y, x ) ) - numpy_alpha_float ) ) ) else: numpy_image_gray = cv2.cvtColor( numpy_image, cv2.COLOR_RGB2GRAY ) if HG.phash_generation_report_mode: HydrusData.ShowText( 'phash generation: grey image shape: {}'.format( numpy_image_gray.shape ) ) numpy_image_tiny = cv2.resize( numpy_image_gray, ( 32, 32 ), interpolation = cv2.INTER_AREA ) if HG.phash_generation_report_mode: HydrusData.ShowText( 'phash generation: tiny image shape: {}'.format( numpy_image_tiny.shape ) ) # convert to float and calc dct numpy_image_tiny_float = numpy.float32( numpy_image_tiny ) if HG.phash_generation_report_mode: HydrusData.ShowText( 'phash generation: tiny float image shape: {}'.format( numpy_image_tiny_float.shape ) ) HydrusData.ShowText( 'phash generation: generating dct' ) dct = cv2.dct( numpy_image_tiny_float ) # take top left 8x8 of dct dct_88 = dct[:8,:8] # get median of dct # exclude [0,0], which represents flat colour # this [0,0] exclusion is apparently important for mean, but maybe it ain't so important for median--w/e # old mean code # mask = numpy.ones( ( 8, 8 ) ) # mask[0,0] = 0 # average = numpy.average( dct_88, weights = mask ) median = numpy.median( dct_88.reshape( 64 )[1:] ) if HG.phash_generation_report_mode: HydrusData.ShowText( 'phash generation: median: {}'.format( median ) ) # make a monochromatic, 64-bit hash of whether the entry is above or below the median dct_88_boolean = dct_88 > median if HG.phash_generation_report_mode: HydrusData.ShowText( 'phash generation: collapsing bytes' ) # convert TTTFTFTF to 11101010 by repeatedly shifting answer and adding 0 or 1 # you can even go ( a << 1 ) + b and leave out the initial param on the reduce call as bools act like ints for this # but let's not go crazy for another two nanoseconds def collapse_bools_to_binary_uint( a, b ): return ( a << 1 ) + int( b ) list_of_bytes = [] for i in range( 8 ): ''' # old way of doing it, which compared value to median every time byte = 0 for j in range( 8 ): byte <<= 1 # shift byte one left value = dct_88[i,j] if value > median: byte |= 1 ''' # this is a 0-255 int byte = reduce( collapse_bools_to_binary_uint, dct_88_boolean[i], 0 ) list_of_bytes.append( byte ) phash = bytes( list_of_bytes ) # this works! if HG.phash_generation_report_mode: HydrusData.ShowText( 'phash generation: phash: {}'.format( phash.hex() ) ) # now discard the blank hash, which is 1000000... and not useful phashes = set() phashes.add( phash ) phashes = DiscardBlankPerceptualHashes( phashes ) if HG.phash_generation_report_mode: HydrusData.ShowText( 'phash generation: final phashes: {}'.format( len( phashes ) ) ) # we good return phashes
def _GetThumbnailHydrusBitmap(self, display_media): bounding_dimensions = self._controller.options['thumbnail_dimensions'] hash = display_media.GetHash() mime = display_media.GetMime() locations_manager = display_media.GetLocationsManager() try: path = self._controller.client_files_manager.GetThumbnailPath( display_media) except HydrusExceptions.FileMissingException as e: if locations_manager.IsLocal(): summary = 'Unable to get thumbnail for file {}.'.format( hash.hex()) self._HandleThumbnailException(e, summary) return self._special_thumbs['hydrus'] try: numpy_image = ClientImageHandling.GenerateNumPyImage(path, mime) except Exception as e: try: # file is malformed, let's force a regen self._controller.files_maintenance_manager.RunJobImmediately( [display_media], ClientFiles.REGENERATE_FILE_DATA_JOB_FORCE_THUMBNAIL, pub_job_key=False) except Exception as e: summary = 'The thumbnail for file {} was not loadable. An attempt to regenerate it failed.'.format( hash.hex()) self._HandleThumbnailException(e, summary) return self._special_thumbs['hydrus'] try: numpy_image = ClientImageHandling.GenerateNumPyImage( path, mime) except Exception as e: summary = 'The thumbnail for file {} was not loadable. It was regenerated, but that file would not render either. Your image libraries or hard drive connection are unreliable. Please inform the hydrus developer what has happened.'.format( hash.hex()) self._HandleThumbnailException(e, summary) return self._special_thumbs['hydrus'] (current_width, current_height) = HydrusImageHandling.GetResolutionNumPy(numpy_image) (media_width, media_height) = display_media.GetResolution() (expected_width, expected_height) = HydrusImageHandling.GetThumbnailResolution( (media_width, media_height), bounding_dimensions) exactly_as_expected = current_width == expected_width and current_height == expected_height rotation_exception = current_width == expected_height and current_height == expected_width correct_size = exactly_as_expected or rotation_exception if not correct_size: it_is_definitely_too_big = current_width >= expected_width and current_height >= expected_height if it_is_definitely_too_big: if HG.file_report_mode: HydrusData.ShowText('Thumbnail {} too big.'.format( hash.hex())) # the thumb we have is larger than desired. we can use it to generate what we actually want without losing significant data # this is _resize_, not _thumbnail_, because we already know the dimensions we want # and in some edge cases, doing getthumbresolution on existing thumb dimensions results in float/int conversion imprecision and you get 90px/91px regen cycles that never get fixed numpy_image = HydrusImageHandling.ResizeNumPyImage( numpy_image, (expected_width, expected_height)) if locations_manager.IsLocal(): # we have the master file, so it is safe to save our resized thumb back to disk since we can regen from source if needed if HG.file_report_mode: HydrusData.ShowText( 'Thumbnail {} too big, saving back to disk.'. format(hash.hex())) try: try: thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesNumPy( numpy_image, mime) except HydrusExceptions.CantRenderWithCVException: thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesFromStaticImagePath( path, (expected_width, expected_height), mime) except: summary = 'The thumbnail for file {} was too large, but an attempt to shrink it failed.'.format( hash.hex()) self._HandleThumbnailException(e, summary) return self._special_thumbs['hydrus'] try: self._controller.client_files_manager.AddThumbnailFromBytes( hash, thumbnail_bytes, silent=True) self._controller.files_maintenance_manager.ClearJobs( {hash}, ClientFiles. REGENERATE_FILE_DATA_JOB_REFIT_THUMBNAIL) except: summary = 'The thumbnail for file {} was too large, but an attempt to save back the shrunk file failed.'.format( hash.hex()) self._HandleThumbnailException(e, summary) return self._special_thumbs['hydrus'] else: # the thumb we have is either too small or completely messed up due to a previous ratio misparse media_is_same_size_as_current_thumb = current_width == media_width and current_height == media_height if media_is_same_size_as_current_thumb: # the thumb is smaller than expected, but this is a 32x32 pixilart image or whatever, so no need to scale if HG.file_report_mode: HydrusData.ShowText( 'Thumbnail {} too small due to small source file.'. format(hash.hex())) else: numpy_image = HydrusImageHandling.ResizeNumPyImage( numpy_image, (expected_width, expected_height)) if locations_manager.IsLocal(): # we have the master file, so we should regen the thumb from source if HG.file_report_mode: HydrusData.ShowText( 'Thumbnail {} too small, scheduling regeneration from source.' .format(hash.hex())) delayed_item = display_media.GetMediaResult() with self._lock: if delayed_item not in self._delayed_regeneration_queue_quick: self._delayed_regeneration_queue_quick.add( delayed_item) self._delayed_regeneration_queue.append( delayed_item) else: # we do not have the master file, so we have to scale up from what we have if HG.file_report_mode: HydrusData.ShowText( 'Thumbnail {} was too small, only scaling up due to no local source.' .format(hash.hex())) hydrus_bitmap = ClientRendering.GenerateHydrusBitmapFromNumPyImage( numpy_image) return hydrus_bitmap
def ParseFileArguments(path, decompression_bombs_ok=False): HydrusImageHandling.ConvertToPNGIfBMP(path) hash = HydrusFileHandling.GetHashFromPath(path) try: mime = HydrusFileHandling.GetMime(path) if mime in HC.DECOMPRESSION_BOMB_IMAGES and not decompression_bombs_ok: if HydrusImageHandling.IsDecompressionBomb(path): raise HydrusExceptions.InsufficientCredentialsException( 'File seemed to be a Decompression Bomb, which you cannot upload!' ) (size, mime, width, height, duration, num_frames, has_audio, num_words) = HydrusFileHandling.GetFileInfo(path, mime) except Exception as e: raise HydrusExceptions.BadRequestException('File ' + hash.hex() + ' could not parse: ' + str(e)) args = ParsedRequestArguments() args['path'] = path args['hash'] = hash args['size'] = size args['mime'] = mime if width is not None: args['width'] = width if height is not None: args['height'] = height if duration is not None: args['duration'] = duration if num_frames is not None: args['num_frames'] = num_frames args['has_audio'] = has_audio if num_words is not None: args['num_words'] = num_words if mime in HC.MIMES_WITH_THUMBNAILS: try: bounding_dimensions = HC.SERVER_THUMBNAIL_DIMENSIONS target_resolution = HydrusImageHandling.GetThumbnailResolution( (width, height), bounding_dimensions) thumbnail_bytes = HydrusFileHandling.GenerateThumbnailBytes( path, target_resolution, mime, duration, num_frames) except Exception as e: tb = traceback.format_exc() raise HydrusExceptions.BadRequestException( 'Could not generate thumbnail from that file:' + os.linesep + tb) args['thumbnail'] = thumbnail_bytes return args
def GenerateInfo(self, status_hook=None): if self._pre_import_file_status.mime is None: if status_hook is not None: status_hook('generating filetype') mime = HydrusFileHandling.GetMime(self._temp_path) self._pre_import_file_status.mime = mime else: mime = self._pre_import_file_status.mime if HG.file_import_report_mode: HydrusData.ShowText('File import job mime: {}'.format( HC.mime_string_lookup[mime])) new_options = HG.client_controller.new_options if mime in HC.DECOMPRESSION_BOMB_IMAGES and not self._file_import_options.AllowsDecompressionBombs( ): if HG.file_import_report_mode: HydrusData.ShowText( 'File import job testing for decompression bomb') if HydrusImageHandling.IsDecompressionBomb(self._temp_path): if HG.file_import_report_mode: HydrusData.ShowText( 'File import job: it was a decompression bomb') raise HydrusExceptions.DecompressionBombException( 'Image seems to be a Decompression Bomb!') if status_hook is not None: status_hook('generating file metadata') self._file_info = HydrusFileHandling.GetFileInfo(self._temp_path, mime=mime) (size, mime, width, height, duration, num_frames, has_audio, num_words) = self._file_info if HG.file_import_report_mode: HydrusData.ShowText('File import job file info: {}'.format( self._file_info)) if mime in HC.MIMES_WITH_THUMBNAILS: if status_hook is not None: status_hook('generating thumbnail') if HG.file_import_report_mode: HydrusData.ShowText('File import job generating thumbnail') bounding_dimensions = HG.client_controller.options[ 'thumbnail_dimensions'] target_resolution = HydrusImageHandling.GetThumbnailResolution( (width, height), bounding_dimensions) percentage_in = HG.client_controller.new_options.GetInteger( 'video_thumbnail_percentage_in') try: self._thumbnail_bytes = HydrusFileHandling.GenerateThumbnailBytes( self._temp_path, target_resolution, mime, duration, num_frames, percentage_in=percentage_in) except Exception as e: raise HydrusExceptions.DamagedOrUnusualFileException( 'Could not render a thumbnail: {}'.format(str(e))) if mime in HC.FILES_THAT_HAVE_PERCEPTUAL_HASH: if status_hook is not None: status_hook('generating similar files metadata') if HG.file_import_report_mode: HydrusData.ShowText( 'File import job generating perceptual_hashes') self._perceptual_hashes = ClientImageHandling.GenerateShapePerceptualHashes( self._temp_path, mime) if HG.file_import_report_mode: HydrusData.ShowText( 'File import job generated {} perceptual_hashes: {}'. format(len(self._perceptual_hashes), [ perceptual_hash.hex() for perceptual_hash in self._perceptual_hashes ])) if HG.file_import_report_mode: HydrusData.ShowText('File import job generating other hashes') if status_hook is not None: status_hook('generating additional hashes') self._extra_hashes = HydrusFileHandling.GetExtraHashesFromPath( self._temp_path) has_icc_profile = False if mime in HC.FILES_THAT_CAN_HAVE_ICC_PROFILE: try: pil_image = HydrusImageHandling.RawOpenPILImage( self._temp_path) has_icc_profile = HydrusImageHandling.HasICCProfile(pil_image) except: pass self._has_icc_profile = has_icc_profile if mime in HC.FILES_THAT_CAN_HAVE_PIXEL_HASH and duration is None: try: self._pixel_hash = HydrusImageHandling.GetImagePixelHash( self._temp_path, mime) except: pass self._file_modified_timestamp = HydrusFileHandling.GetFileModifiedTimestamp( self._temp_path)