def GeneratePreImportHashAndStatus(self, status_hook=None):
HydrusImageHandling.ConvertToPNGIfBMP(self._temp_path)
if status_hook is not None:
status_hook('calculating hash')
hash = HydrusFileHandling.GetHashFromPath(self._temp_path)
if HG.file_import_report_mode:
HydrusData.ShowText('File import job hash: {}'.format(hash.hex()))
if status_hook is not None:
status_hook('checking for file status')
self._pre_import_file_status = HG.client_controller.Read(
'hash_status', 'sha256', hash, prefix='file recognised')
# just in case
self._pre_import_file_status.hash = hash
self._pre_import_file_status = CheckFileImportStatus(
self._pre_import_file_status)
if HG.file_import_report_mode:
HydrusData.ShowText('File import job pre-import status: {}'.format(
self._pre_import_file_status.ToString()))
def __init__(self,
paths=None,
file_import_options=None,
paths_to_additional_service_keys_to_tags=None,
delete_after_success=None):
HydrusSerialisable.SerialisableBase.__init__(self)
if paths is None:
self._file_seed_cache = None
else:
self._file_seed_cache = ClientImportFileSeeds.FileSeedCache()
file_seeds = []
for path in paths:
file_seed = ClientImportFileSeeds.FileSeed(
ClientImportFileSeeds.FILE_SEED_TYPE_HDD, path)
try:
file_modified_time = HydrusFileHandling.GetFileModifiedTimestamp(
path)
file_seed.source_time = file_modified_time
except:
pass
if path in paths_to_additional_service_keys_to_tags:
file_seed.SetExternalAdditionalServiceKeysToTags(
paths_to_additional_service_keys_to_tags[path])
file_seeds.append(file_seed)
self._file_seed_cache.AddFileSeeds(file_seeds)
self._file_import_options = file_import_options
self._delete_after_success = delete_after_success
self._page_key = b'initialising page key'
self._files_status = ''
self._paused = False
self._lock = threading.Lock()
self._files_repeating_job = None
self._last_serialisable_change_timestamp = 0
HG.client_controller.sub(self, 'NotifyFileSeedsUpdated',
'file_seed_cache_file_seeds_updated')
def ParseFileArguments(path, decompression_bombs_ok=False):
HydrusImageHandling.ConvertToPNGIfBMP(path)
hash = HydrusFileHandling.GetHashFromPath(path)
try:
mime = HydrusFileHandling.GetMime(path)
if mime in HC.DECOMPRESSION_BOMB_IMAGES and not decompression_bombs_ok:
if HydrusImageHandling.IsDecompressionBomb(path):
raise HydrusExceptions.InsufficientCredentialsException(
'File seemed to be a Decompression Bomb, which you cannot upload!'
)
(size, mime, width, height, duration, num_frames, has_audio,
num_words) = HydrusFileHandling.GetFileInfo(path, mime)
except Exception as e:
raise HydrusExceptions.BadRequestException('File ' + hash.hex() +
' could not parse: ' +
str(e))
args = ParsedRequestArguments()
args['path'] = path
args['hash'] = hash
args['size'] = size
args['mime'] = mime
if width is not None: args['width'] = width
if height is not None: args['height'] = height
if duration is not None: args['duration'] = duration
if num_frames is not None: args['num_frames'] = num_frames
args['has_audio'] = has_audio
if num_words is not None: args['num_words'] = num_words
if mime in HC.MIMES_WITH_THUMBNAILS:
try:
bounding_dimensions = HC.SERVER_THUMBNAIL_DIMENSIONS
target_resolution = HydrusImageHandling.GetThumbnailResolution(
(width, height), bounding_dimensions)
thumbnail_bytes = HydrusFileHandling.GenerateThumbnailBytes(
path, target_resolution, mime, duration, num_frames)
except Exception as e:
tb = traceback.format_exc()
raise HydrusExceptions.BadRequestException(
'Could not generate thumbnail from that file:' + os.linesep +
tb)
args['thumbnail'] = thumbnail_bytes
return args
def GenerateInfo(self, status_hook=None):
if self._pre_import_file_status.mime is None:
if status_hook is not None:
status_hook('generating filetype')
mime = HydrusFileHandling.GetMime(self._temp_path)
self._pre_import_file_status.mime = mime
else:
mime = self._pre_import_file_status.mime
if HG.file_import_report_mode:
HydrusData.ShowText('File import job mime: {}'.format(
HC.mime_string_lookup[mime]))
new_options = HG.client_controller.new_options
if mime in HC.DECOMPRESSION_BOMB_IMAGES and not self._file_import_options.AllowsDecompressionBombs(
):
if HG.file_import_report_mode:
HydrusData.ShowText(
'File import job testing for decompression bomb')
if HydrusImageHandling.IsDecompressionBomb(self._temp_path):
if HG.file_import_report_mode:
HydrusData.ShowText(
'File import job: it was a decompression bomb')
raise HydrusExceptions.DecompressionBombException(
'Image seems to be a Decompression Bomb!')
if status_hook is not None:
status_hook('generating file metadata')
self._file_info = HydrusFileHandling.GetFileInfo(self._temp_path,
mime=mime)
(size, mime, width, height, duration, num_frames, has_audio,
num_words) = self._file_info
if HG.file_import_report_mode:
HydrusData.ShowText('File import job file info: {}'.format(
self._file_info))
if mime in HC.MIMES_WITH_THUMBNAILS:
if status_hook is not None:
status_hook('generating thumbnail')
if HG.file_import_report_mode:
HydrusData.ShowText('File import job generating thumbnail')
bounding_dimensions = HG.client_controller.options[
'thumbnail_dimensions']
thumbnail_scale_type = HG.client_controller.new_options.GetInteger(
'thumbnail_scale_type')
(clip_rect, target_resolution
) = HydrusImageHandling.GetThumbnailResolutionAndClipRegion(
(width, height), bounding_dimensions, thumbnail_scale_type)
percentage_in = HG.client_controller.new_options.GetInteger(
'video_thumbnail_percentage_in')
try:
self._thumbnail_bytes = HydrusFileHandling.GenerateThumbnailBytes(
self._temp_path,
target_resolution,
mime,
duration,
num_frames,
clip_rect=clip_rect,
percentage_in=percentage_in)
except Exception as e:
raise HydrusExceptions.DamagedOrUnusualFileException(
'Could not render a thumbnail: {}'.format(str(e)))
if mime in HC.FILES_THAT_HAVE_PERCEPTUAL_HASH:
if status_hook is not None:
status_hook('generating similar files metadata')
if HG.file_import_report_mode:
HydrusData.ShowText(
'File import job generating perceptual_hashes')
self._perceptual_hashes = ClientImageHandling.GenerateShapePerceptualHashes(
self._temp_path, mime)
if HG.file_import_report_mode:
HydrusData.ShowText(
'File import job generated {} perceptual_hashes: {}'.
format(len(self._perceptual_hashes), [
perceptual_hash.hex()
for perceptual_hash in self._perceptual_hashes
]))
if HG.file_import_report_mode:
HydrusData.ShowText('File import job generating other hashes')
if status_hook is not None:
status_hook('generating additional hashes')
self._extra_hashes = HydrusFileHandling.GetExtraHashesFromPath(
self._temp_path)
has_icc_profile = False
if mime in HC.FILES_THAT_CAN_HAVE_ICC_PROFILE:
try:
pil_image = HydrusImageHandling.RawOpenPILImage(
self._temp_path)
has_icc_profile = HydrusImageHandling.HasICCProfile(pil_image)
except:
pass
self._has_icc_profile = has_icc_profile
if mime in HC.FILES_THAT_CAN_HAVE_PIXEL_HASH and duration is None:
try:
self._pixel_hash = HydrusImageHandling.GetImagePixelHash(
self._temp_path, mime)
except:
pass
self._file_modified_timestamp = HydrusFileHandling.GetFileModifiedTimestamp(
self._temp_path)