def do_work(self):
if False:
# exiv2 pumps out a LOT to stderr - use cautiously!
context = show_errors()
self.error_stream = sys.stderr
else:
# Redirect stderr, hiding error output from exiv2
context = stdchannel_redirected(sys.stderr, os.devnull)
self.error_stream = sys.stdout
with context:
# In some situations, using a context manager for exiftool can
# result in exiftool processes not being terminated. So let's
# handle starting and terminating it manually.
self.exiftool_process = exiftool.ExifTool()
self.exiftool_process.start()
self.process_files()
self.exit()
def scan(
folder: str,
disk_cach_cleared: bool,
scan_types: List[str],
errors: bool,
outfile: str,
keep_file_names: bool,
analyze_previews: bool,
) -> Tuple[List[PhotoAttributes], List[VideoAttributes]]:
global stop
global kill
problematic_files = "RAW_LEICA_M8.DNG"
stop = kill = False
pbs = progress_bar_scanning()
pbs.start()
test_files = []
not_tested = []
# Phase 1
# Determine which files are safe to test i.e. are not cached
if analyze_previews:
disk_cach_cleared = True
for dir_name, subdirs, filenames in walk(folder):
for filename in filenames:
if filename not in problematic_files:
ext = extract_extension(filename)
if ext in scan_types:
full_file_name = os.path.join(dir_name, filename)
if disk_cach_cleared:
test_files.append((full_file_name, ext.upper()))
else:
bytes_cached, total, in_memory = vmtouch_output(
full_file_name)
if bytes_cached == 0:
test_files.append((full_file_name, ext.upper()))
else:
not_tested.append(full_file_name)
stop = True
pbs.join()
if not_tested:
print()
if len(not_tested) > 20:
for line in textwrap.wrap(
"WARNING: {:,} files will not be analyzed because they are already in the "
"kernel disk cache.".format(len(not_tested)),
width=80,
):
print(line)
else:
print(
"WARNING: these files will not be analyzed because they are already in the "
"kernel disk cache:")
for name in not_tested:
print(name)
print()
for line in textwrap.wrap(
"Run this script as super user and use command line option -c or --clear to safely "
"clear the disk cache.",
width=80,
):
print(line)
if confirm(prompt="\nDo you want to exit?", resp=True):
sys.exit(0)
photos = []
videos = []
if test_files:
print("\nAnalyzing {:,} files:".format(len(test_files)))
if have_progressbar and not errors:
bar = pyprind.ProgBar(iterations=len(test_files),
stream=1,
track_time=False,
width=80)
else:
print("\nNothing to analyze")
# Phase 2
# Get info from files
if errors:
context = show_errors()
else:
# Redirect stderr, hiding error output from exiv2
context = stdchannel_redirected(sys.stderr, os.devnull)
metadata_fail = []
with context:
with ExifTool() as exiftool_process:
for full_file_name, ext in test_files:
if ext.lower() in VIDEO_EXTENSIONS:
va = VideoAttributes(full_file_name, ext, exiftool_process)
videos.append(va)
else:
# TODO think about how to handle HEIF files!
if use_exiftool_on_photo(
ext.lower(), preview_extraction_irrelevant=False):
pa = ExifToolPhotoAttributes(full_file_name, ext,
exiftool_process,
analyze_previews)
pa.process(analyze_previews)
photos.append(pa)
else:
try:
metadata = mp.MetaData(
full_file_name=full_file_name,
et_process=exiftool_process,
)
except:
metadata_fail.append(full_file_name)
else:
pa = PhotoAttributes(full_file_name, ext,
exiftool_process,
analyze_previews)
pa.metadata = metadata
pa.process(analyze_previews)
photos.append(pa)
if have_progressbar and not errors:
bar.update()
if metadata_fail:
print()
for full_file_name in metadata_fail:
print("Could not read metadata from {}".format(full_file_name))
if outfile is not None:
if not keep_file_names:
for pa in photos:
pa.file_name = None
for va in videos:
va.file_name = None
with open(outfile, "wb") as save_to:
pickle.dump((photos, videos), save_to, pickle.HIGHEST_PROTOCOL)
return photos, videos
def run(self) -> None:
"""
Generate subfolder and filename, and attempt to move the file
from its temporary directory.
Move video THM and/or audio file if there is one.
If successful, increment sequence values.
Report any success or failure.
"""
i = 0
# Dict of filename keys and int values used to track ints to add as
# suffixes to duplicate files
self.duplicate_files = {}
self.initialise_downloads_today_stored_number()
self.sequences = gn.Sequences(
self.downloads_today_tracker, self.prefs.stored_sequence_no
)
with stdchannel_redirected(sys.stderr, os.devnull):
with exiftool.ExifTool() as self.exiftool_process:
while True:
if i:
logging.debug("Finished %s. Getting next task.", i)
# rename file and move to generated subfolder
directive, content = self.receiver.recv_multipart()
self.check_for_command(directive, content)
data = pickle.loads(content) # type: RenameAndMoveFileData
if data.message == RenameAndMoveStatus.download_started:
# reinitialize downloads today and stored sequence number
# in case the user has updated them via the user interface
self.initialise_downloads_today_stored_number()
self.sequences.downloads_today_tracker = (
self.downloads_today_tracker
)
self.sequences.stored_sequence_no = (
self.prefs.stored_sequence_no
)
dl_today = (
self.downloads_today_tracker.get_or_reset_downloads_today()
)
logging.debug("Completed downloads today: %s", dl_today)
self.initialise_sequence_number_usage()
self.must_synchronize_raw_jpg = (
self.prefs.must_synchronize_raw_jpg()
)
self.problems = RenamingProblems()
elif data.message == RenameAndMoveStatus.download_completed:
if len(self.problems):
self.content = pickle.dumps(
RenameAndMoveFileResults(problems=self.problems),
pickle.HIGHEST_PROTOCOL,
)
self.send_message_to_sink()
# Ask main application process to update prefs with stored
# sequence number and downloads today values. But first sync
# the prefs here, to write out the dirty values so they are not
# saved when a sync is done at download start, overwriting
# the values that may have been changed in the main process
logging.debug(
"Rename and move process syncing preferences to the file "
"system"
)
self.prefs.sync()
self.content = pickle.dumps(
RenameAndMoveFileResults(
stored_sequence_no=self.sequences.stored_sequence_no,
downloads_today=self.downloads_today_tracker.downloads_today,
),
pickle.HIGHEST_PROTOCOL,
)
dl_today = (
self.downloads_today_tracker.get_or_reset_downloads_today()
)
logging.debug("Downloads today: %s", dl_today)
self.send_message_to_sink()
else:
rpd_file = data.rpd_file
download_count = data.download_count
if data.download_succeeded:
move_succeeded = self.process_file(rpd_file, download_count)
if not move_succeeded:
self.process_rename_failure(rpd_file)
else:
# Record file as downloaded in SQLite database
try:
self.downloaded.add_downloaded_file(
name=rpd_file.name,
size=rpd_file.size,
modification_time=rpd_file.modification_time,
download_full_file_name=rpd_file.download_full_file_name,
)
except sqlite3.OperationalError as e:
# This should never happen because this is the only
# process writing to the database..... but just in
# case
logging.error(
"Database error adding download file %s: %s. "
"Will not retry.",
rpd_file.download_full_file_name,
e,
)
else:
move_succeeded = False
rpd_file.metadata = None
self.content = pickle.dumps(
RenameAndMoveFileResults(
move_succeeded=move_succeeded,
rpd_file=rpd_file,
download_count=download_count,
),
pickle.HIGHEST_PROTOCOL,
)
self.send_message_to_sink()
i += 1