def ConvertToPngIfBmp( path ): with open( path, 'rb' ) as f: header = f.read( 2 ) if header == b'BM': ( os_file_handle, temp_path ) = HydrusPaths.GetTempPath() try: with open( path, 'rb' ) as f_source: with open( temp_path, 'wb' ) as f_dest: HydrusPaths.CopyFileLikeToFileLike( f_source, f_dest ) pil_image = GeneratePILImage( temp_path ) pil_image.save( path, 'PNG' ) finally: HydrusPaths.CleanUpTempPath( os_file_handle, temp_path )
def GenerateThumbnailBytes( path, target_resolution, mime, duration, num_frames, percentage_in = 35 ): if mime in ( HC.IMAGE_JPEG, HC.IMAGE_PNG, HC.IMAGE_GIF, HC.IMAGE_WEBP, HC.IMAGE_TIFF, HC.IMAGE_ICON ): # not apng atm thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesFromStaticImagePath( path, target_resolution, mime ) else: if mime == HC.APPLICATION_FLASH: ( os_file_handle, temp_path ) = HydrusPaths.GetTempPath() try: HydrusFlashHandling.RenderPageToFile( path, temp_path, 1 ) thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesFromStaticImagePath( temp_path, target_resolution, mime ) except: thumb_path = os.path.join( HC.STATIC_DIR, 'flash.png' ) thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesFromStaticImagePath( thumb_path, target_resolution, mime ) finally: HydrusPaths.CleanUpTempPath( os_file_handle, temp_path ) else: renderer = HydrusVideoHandling.VideoRendererFFMPEG( path, mime, duration, num_frames, target_resolution ) renderer.read_frame() # this initialises the renderer and loads the first frame as a fallback desired_thumb_frame = int( ( percentage_in / 100.0 ) * num_frames ) renderer.set_position( desired_thumb_frame ) numpy_image = renderer.read_frame() if numpy_image is None: raise Exception( 'Could not create a thumbnail from that video!' ) numpy_image = HydrusImageHandling.ResizeNumPyImage( numpy_image, target_resolution ) # just in case ffmpeg doesn't deliver right thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesNumPy( numpy_image, mime ) renderer.Stop() del renderer return thumbnail_bytes
def DumpToPNG(width, payload_bytes, title, payload_description, text, path): payload_bytes_length = len(payload_bytes) header_and_payload_bytes_length = payload_bytes_length + 4 payload_height = int(header_and_payload_bytes_length / width) if (header_and_payload_bytes_length / width) % 1.0 > 0: payload_height += 1 top_image = CreateTopImage(width, title, payload_description, text) payload_length_header = struct.pack('!I', payload_bytes_length) num_empty_bytes = payload_height * width - header_and_payload_bytes_length header_and_payload_bytes = payload_length_header + payload_bytes + b'\x00' * num_empty_bytes payload_image = numpy.fromstring(header_and_payload_bytes, dtype='uint8').reshape( (payload_height, width)) finished_image = numpy.concatenate((top_image, payload_image)) # this is to deal with unicode paths, which cv2 can't handle (os_file_handle, temp_path) = HydrusPaths.GetTempPath(suffix='.png') try: cv2.imwrite(temp_path, finished_image, [cv2.IMWRITE_PNG_COMPRESSION, 9]) HydrusPaths.MirrorFile(temp_path, path) except Exception as e: HydrusData.ShowException(e) raise Exception('Could not save the png!') finally: HydrusPaths.CleanUpTempPath(os_file_handle, temp_path)
def MainLoop(self): hashes_still_to_download_in_this_run = set() total_hashes_in_this_run = 0 total_successful_hashes_in_this_run = 0 while not (HydrusThreading.IsThreadShuttingDown() or self._shutting_down or HG.view_shutdown): with self._lock: if len(self._pending_hashes) > 0: if total_hashes_in_this_run == 0: job_key = ClientThreading.JobKey(cancellable=True) job_key.SetStatusTitle('downloading') job_key.SetVariable('popup_text_1', 'initialising downloader') job_key_pub_job = self._controller.CallLater( 2.0, self._controller.pub, 'message', job_key) num_before = len(hashes_still_to_download_in_this_run) hashes_still_to_download_in_this_run.update( self._pending_hashes) num_after = len(hashes_still_to_download_in_this_run) total_hashes_in_this_run += num_after - num_before self._pending_hashes = set() if len(hashes_still_to_download_in_this_run) == 0: total_hashes_in_this_run = 0 total_successful_hashes_in_this_run = 0 self._new_files_event.wait(5) self._new_files_event.clear() continue if job_key.IsCancelled(): hashes_still_to_download_in_this_run = set() continue hash = random.sample(hashes_still_to_download_in_this_run, 1)[0] hashes_still_to_download_in_this_run.discard(hash) total_done = total_hashes_in_this_run - len( hashes_still_to_download_in_this_run) job_key.SetVariable( 'popup_text_1', 'downloading files from remote services: {}'.format( HydrusData.ConvertValueRangeToPrettyString( total_done, total_hashes_in_this_run))) job_key.SetVariable('popup_gauge_1', (total_done, total_hashes_in_this_run)) try: errors_occured = [] file_successful = False media_result = self._controller.Read('media_result', hash) service_keys = list( media_result.GetLocationsManager().GetCurrent()) random.shuffle(service_keys) if CC.COMBINED_LOCAL_FILE_SERVICE_KEY in service_keys: total_successful_hashes_in_this_run += 1 continue for service_key in service_keys: try: service = self._controller.services_manager.GetService( service_key) except: continue try: if service.GetServiceType() == HC.FILE_REPOSITORY: file_repository = service if file_repository.IsFunctional(): (os_file_handle, temp_path) = HydrusPaths.GetTempPath() try: file_repository.Request( HC.GET, 'file', {'hash': hash}, temp_path=temp_path) exclude_deleted = False # this is the important part here do_not_check_known_urls_before_importing = False do_not_check_hashes_before_importing = False allow_decompression_bombs = True min_size = None max_size = None max_gif_size = None min_resolution = None max_resolution = None automatic_archive = False associate_source_urls = True file_import_options = ClientImportOptions.FileImportOptions( ) file_import_options.SetPreImportOptions( exclude_deleted, do_not_check_known_urls_before_importing, do_not_check_hashes_before_importing, allow_decompression_bombs, min_size, max_size, max_gif_size, min_resolution, max_resolution) file_import_options.SetPostImportOptions( automatic_archive, associate_source_urls) file_import_job = ClientImportFileSeeds.FileImportJob( temp_path, file_import_options) file_import_job.DoWork() file_successful = True break finally: HydrusPaths.CleanUpTempPath( os_file_handle, temp_path) elif service.GetServiceType() == HC.IPFS: multihashes = HG.client_controller.Read( 'service_filenames', service_key, {hash}) if len(multihashes) > 0: multihash = multihashes[0] service.ImportFile(multihash, silent=True) file_successful = True break except Exception as e: errors_occured.append(e) if file_successful: total_successful_hashes_in_this_run += 1 if len(errors_occured) > 0: if not file_successful: raise errors_occured[0] except Exception as e: HydrusData.ShowException(e) hashes_still_to_download_in_this_run = 0 finally: if len(hashes_still_to_download_in_this_run) == 0: job_key.DeleteVariable('popup_text_1') job_key.DeleteVariable('popup_gauge_1') if total_successful_hashes_in_this_run > 0: job_key.SetVariable( 'popup_text_1', HydrusData.ToHumanInt( total_successful_hashes_in_this_run) + ' files downloaded') job_key_pub_job.Cancel() job_key.Finish() job_key.Delete(1)
def LoadFromPNG( path ): # this is to deal with unicode paths, which cv2 can't handle ( os_file_handle, temp_path ) = HydrusPaths.GetTempPath() try: HydrusPaths.MirrorFile( path, temp_path ) numpy_image = cv2.imread( temp_path, flags = IMREAD_UNCHANGED ) except Exception as e: HydrusData.ShowException( e ) raise Exception( 'That did not appear to be a valid image!' ) finally: HydrusPaths.CleanUpTempPath( os_file_handle, temp_path ) try: try: ( height, width ) = numpy_image.shape except: raise Exception( 'The file did not appear to be monochrome!' ) try: complete_data = numpy_image.tostring() top_height_header = complete_data[:2] ( top_height, ) = struct.unpack( '!H', top_height_header ) payload_and_header_bytes = complete_data[ width * top_height : ] except: raise Exception( 'Header bytes were invalid!' ) try: payload_length_header = payload_and_header_bytes[:4] ( payload_bytes_length, ) = struct.unpack( '!I', payload_length_header ) payload_bytes = payload_and_header_bytes[ 4 : 4 + payload_bytes_length ] except: raise Exception( 'Payload bytes were invalid!' ) except Exception as e: message = 'The image loaded, but it did not seem to be a hydrus serialised png! The error was: {}'.format( str( e ) ) message += os.linesep * 2 message += 'If you believe this is a legit non-resized, non-converted hydrus serialised png, please send it to hydrus_dev.' raise Exception( message ) return payload_bytes
def GenerateThumbnailBytes(path, target_resolution, mime, duration, num_frames, percentage_in=35): if mime in (HC.IMAGE_JPEG, HC.IMAGE_PNG, HC.IMAGE_GIF, HC.IMAGE_WEBP, HC.IMAGE_TIFF, HC.IMAGE_ICON): # not apng atm thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesFromStaticImagePath( path, target_resolution, mime) elif mime in [HC.APPLICATION_ZIP]: temp_dir_path = HydrusPaths.GetTempDir() try: cmd = ["unzip", path, '-d', temp_dir_path] subprocess.call(cmd) cover = sorted(list(Path(temp_dir_path).rglob("*.jpg")) + list(Path(temp_dir_path).rglob("*.png")), key=lambda p: p.name)[0].as_posix() thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesFromStaticImagePath( cover, target_resolution, mime) except Exception as e: thumb_path = os.path.join(HC.STATIC_DIR, 'zip.png') thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesFromStaticImagePath( thumb_path, target_resolution, mime) elif mime in [HC.APPLICATION_RAR]: temp_dir_path = HydrusPaths.GetTempDir() try: cmd = ["unrar", 'x', path, temp_dir_path] subprocess.call(cmd) cover = sorted(list(Path(temp_dir_path).rglob("*.jpg")) + list(Path(temp_dir_path).rglob("*.png")), key=lambda p: p.name)[0].as_posix() thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesFromStaticImagePath( cover, target_resolution, mime) except Exception as e: thumb_path = os.path.join(HC.STATIC_DIR, 'rar.png') thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesFromStaticImagePath( thumb_path, target_resolution, mime) else: if mime == HC.APPLICATION_FLASH: (os_file_handle, temp_path) = HydrusPaths.GetTempPath() try: HydrusFlashHandling.RenderPageToFile(path, temp_path, 1) thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesFromStaticImagePath( temp_path, target_resolution, mime) except: thumb_path = os.path.join(HC.STATIC_DIR, 'flash.png') thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesFromStaticImagePath( thumb_path, target_resolution, mime) finally: HydrusPaths.CleanUpTempPath(os_file_handle, temp_path) else: renderer = HydrusVideoHandling.VideoRendererFFMPEG( path, mime, duration, num_frames, target_resolution) renderer.read_frame( ) # this initialises the renderer and loads the first frame as a fallback desired_thumb_frame = int((percentage_in / 100.0) * num_frames) renderer.set_position(desired_thumb_frame) numpy_image = renderer.read_frame() if numpy_image is None: raise Exception( 'Could not create a thumbnail from that video!') numpy_image = HydrusImageHandling.ResizeNumPyImage( numpy_image, target_resolution) # just in case ffmpeg doesn't deliver right thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesNumPy( numpy_image, mime) renderer.Stop() del renderer return thumbnail_bytes