def lookup_already_indexed_files(self, indexDB: IndexDB,
scanned_files: List[ScannedFile]):
IndexingHelper.__logger.info(
"BEGIN:: IndexDB lookup for indexed files")
total_scanned_files = len(scanned_files)
media_files_by_path = indexDB.get_all_media_files_by_path()
for scanned_file_num, scanned_file in enumerate(scanned_files,
start=1):
if self.__indexing_stop_event.is_set():
break
if scanned_file.file_path in media_files_by_path:
media_file = media_files_by_path[scanned_file.file_path]
scanned_file.already_indexed = True
if (scanned_file.creation_time != media_file.creation_time
or scanned_file.last_modification_time !=
media_file.last_modification_time):
scanned_file.hash = MiscUtils.generate_hash(
scanned_file.file_path)
if scanned_file.hash != media_file.original_file_hash:
scanned_file.needs_reindex = True
IndexingHelper.__logger.info(
"Searched Index %s/%s: %s (AlreadyIndexed = %s, NeedsReindex= %s)",
scanned_file_num, total_scanned_files, scanned_file.file_path,
scanned_file.already_indexed, scanned_file.needs_reindex)
IndexingHelper.__logger.info("END:: IndexDB lookup for indexed files")
def create_media_file(path_exiftool: str, index_time: datetime,
scanned_file: ScannedFile,
existing_media_file: MediaFile) -> MediaFile:
file_path = scanned_file.file_path
exif = ExifHelper.__get_exif_dict(path_exiftool, file_path)
error_str = ExifHelper.__get_exif(exif, "Error")
exif_file_type_str = ExifHelper.__get_exif(exif, "FileType")
if error_str:
ExifHelper.__logger.error("Error processing file. EXIF: %s", exif)
if error_str == 'File is empty' or error_str == 'File format error' or 'file is binary' in error_str:
return None
if exif_file_type_str == "TXT":
ExifHelper.__logger.error("Possibly corrupt file. EXIF: %s", exif)
return None
media_file = existing_media_file if existing_media_file else MediaFile(
)
media_file.parent_dir_path = scanned_file.parent_dir_path
media_file.file_path = file_path
media_file.extension = scanned_file.extension
media_file.file_type = scanned_file.file_type.name
media_file.is_raw = scanned_file.is_raw
media_file.mime = ExifHelper.__get_mime(file_path, exif)
media_file.original_size = os.path.getsize(file_path)
media_file.creation_time = scanned_file.creation_time
media_file.last_modification_time = scanned_file.last_modification_time
media_file.original_file_hash = scanned_file.hash if (
scanned_file.hash
is not None) else MiscUtils.generate_hash(file_path)
media_file.converted_file_hash = None
media_file.conversion_settings_hash = None
media_file.index_time = index_time
ExifHelper.__append_dimentions(media_file, exif)
media_file.capture_date = ExifHelper.__get_capture_date(
scanned_file, exif)
media_file.camera_make = ExifHelper.__get_exif(exif, "Make")
media_file.camera_model = ExifHelper.__get_exif(
exif, "CameraModelName", "Model")
media_file.lens_model = ExifHelper.__get_exif(exif, "LensModel",
"LensType", "LensInfo")
gps_info = ExifHelper.__get_gps_info(exif)
media_file.gps_alt = gps_info.get('altitude')
media_file.gps_lat = gps_info.get('latitude')
media_file.gps_long = gps_info.get('longitude')
exif_orientation = ExifHelper.__get_exif(exif, "Orientation",
"CameraOrientation")
media_file.view_rotation = ExifHelper.__get_view_rotation(
exif_orientation)
media_file.image_orientation = exif_orientation
media_file.video_duration = ExifHelper.__get_video_duration(exif)
ExifHelper.__append_video_rotation(media_file, exif)
return media_file
def conversion_process_exec(media_file_path: str, target_gpu: int,
save_file_path_computation_lock: Lock,
indexDB: IndexDB, task_id: str):
settings: Settings = indexDB.get_settings()
media_file: MediaFile = indexDB.get_by_file_path(media_file_path)
conversion_settings_hash: str = settings.generate_image_settings_hash(
) if (ScannedFileType.IMAGE.name == media_file.file_type
) else settings.generate_video_settings_hash()
processing_start_time = time.time()
original_file_path = media_file.file_path
save_file_path = "UNKNOWN"
try:
with save_file_path_computation_lock:
save_file_path = MediaProcessor.get_save_file_path(
indexDB, media_file, settings)
skip_conversion: bool = False
if (not media_file.capture_date and not settings.convert_unknown
): # No captureDate and conversion not requested for unknown
if os.path.exists(save_file_path):
os.remove(save_file_path)
logging.info(
"Deleted Previously Converted File %s: %s -> %s",
task_id, original_file_path, save_file_path)
skip_conversion = True
else:
os.makedirs(os.path.dirname(save_file_path), exist_ok=True)
if (not settings.overwrite_output_files and os.path.exists(
save_file_path
) and media_file.converted_file_hash == MiscUtils.generate_hash(
save_file_path
) # Converted file hash is None if the original file is re-indexed
and media_file.conversion_settings_hash
== conversion_settings_hash
): # Settings hash is None if the original file is re-indexed
skip_conversion = True
if not skip_conversion:
if ScannedFileType.IMAGE.name == media_file.file_type:
MediaProcessor.convert_image_file(settings, media_file,
original_file_path,
save_file_path)
if ScannedFileType.VIDEO.name == media_file.file_type:
MediaProcessor.convert_video_file(settings, media_file,
original_file_path,
save_file_path,
target_gpu)
MediaProcessor.copy_exif_to_file(settings, original_file_path,
save_file_path, media_file)
media_file.converted_file_hash = MiscUtils.generate_hash(
save_file_path)
media_file.conversion_settings_hash = conversion_settings_hash
with save_file_path_computation_lock:
indexDB.insert_media_file(media_file)
logging.info(
"Converted %s: %s -> %s (%s%%) (%ss)", task_id,
original_file_path, save_file_path,
round(
os.path.getsize(save_file_path) /
media_file.original_size * 100, 2),
round(time.time() - processing_start_time, 2))
else:
logging.info("Skipped Conversion %s: %s -> %s", task_id,
original_file_path, save_file_path)
except:
try:
if os.path.exists(save_file_path):
os.remove(
save_file_path) # Delete corrupt / invalid output file
except:
pass
logging.exception("Failed Processing %s: %s -> %s (%ss)", task_id,
original_file_path, save_file_path,
round(time.time() - processing_start_time, 2))
