def remove_duplicated(self):
g_logger.info('remove duplicated')
size_duplication = {}
for img_path in walk_in_folder(source_dir=self.source_dir, ignore_ext=self.ignore_extentions, ignore_dirs=self.ignore_dirs):
entity = FileEntity(img_path)
size_duplication.setdefault(entity.size, []).append(entity)
duplicated_files = {}
for duplicate_candidate in size_duplication.values():
if len(duplicate_candidate) == 1:
continue
for entity in duplicate_candidate:
duplicated_files.setdefault(entity.hexdigest, []).append(entity)
for _, v in duplicated_files.items():
if len(v) == 1:
continue
index = self.interactive.select_choice('> Choose file to keep', [str(f.filepath) for f in v])
if index == -1:
continue
v.pop(index)
for img in v:
self.policy.delete(self.source_dir, img.filepath)
def separate(self, path_pattern: str = '%Y/%m.%d'):
g_logger.info('separate')
for filepath in walk_in_folder(source_dir=self.source_dir, ignore_ext=self.ignore_extentions, ignore_dirs=self.ignore_dirs):
if is_ignored(filepath, whitelist=self.whitelist_extentions):
self.policy.blacklist_file(self.source_dir, filepath)
continue
entity = FileEntity(filepath)
date = entity.get_original_date()
if date is None:
self.policy.failed_to_detected_original_date(self.source_dir, filepath)
continue
pattern = time.strftime(path_pattern, date.timetuple())
dest_folder = self.target_dir.joinpath(pattern)
dest_img_path = dest_folder.joinpath(entity.filename)
is_duplicated = False
while dest_img_path.exists():
target_entity = FileEntity(dest_img_path)
if entity.is_equal(target_entity):
is_duplicated = True
break
dest_img_path = make_unique_path(dest_img_path)
if is_duplicated:
g_logger.warning("DUPLICATED: %s == %s", dest_img_path, filepath)
self.policy.delete(self.source_dir, filepath)
else:
self.policy.move(filepath, dest_img_path)
def main(context):
g_logger.setLevel(context.log_level.upper())
g_logger.info(context)
assert context.source is not None
source_dir: Path = Path(context.source).absolute()
if context.output is None:
target_dir = source_dir.parent.joinpath(f'{source_dir.name}_sorted')
else:
target_dir: Path = Path(context.output).absolute()
policy: FilePolicyBase = DefaultFilePolicy()
if context.transfer_policy == 'safe':
safe_dir = source_dir.parent.joinpath(f'{source_dir.name}_safe')
policy = SafeFilePolicy(safe_dir)
elif context.transfer_policy == 'debug':
policy = DebugFilePolicy()
g_logger.info('%s to %s', source_dir, target_dir)
storage = MemoryStorage(source_dir, target_dir,
policy, ConsolePolicy(),
whitelist_extentions={'.jpg', '.avi', '.png', '.webp', '.mp4', '.nef', '.mpg'},
ignore_dirs=context.ignore)
if context.action == 'remove_duplicated':
storage.remove_duplicated()
elif context.action == 'separate':
storage.separate(context.path_pattern)
elif context.action == 'video_compress':
storage.video_compress()
elif context.action == 'video_overview':
storage.video_overview()
else:
assert False, context.action
def video_compress(self):
g_logger.info('video compress')
for f in walk_in_folder(source_dir=self.source_dir, ignore_ext=self.ignore_extentions, ignore_dirs=self.ignore_dirs):
if not VideoEntity.is_video(f):
continue
# Move open action to policy
open_vlc(f)
choices = [
video.get_original_resolution(),
video.get_compressed_resolution(),
'delete',
]
index = self.interactive.select_choice(f'> Compress {video.filepath} size[{format_bytes(video.size)}]?', [str(c) for c in choices])
if index == -1:
continue
if choices[index] == 'delete':
self.policy.delete(self.source_dir, video.filepath)
continue
compressed_video = video.compress(choices[index])
if video.size > compressed_video.size:
g_logger.info('Keep compressed file')
self.policy.delete(self.source_dir, video.filepath)
else:
g_logger.info('Original file is smaller, keep original')
self.policy.delete(self.source_dir, compressed_video.filepath)
def video_overview(self):
g_logger.info('video overview')
for f in walk_in_folder(source_dir=self.source_dir, ignore_ext=self.ignore_extentions, ignore_dirs=self.ignore_dirs):
if not VideoEntity.is_video(f):
continue
# Move open action to policy
open_vlc(f)
video = VideoEntity(f)
choices = [
'keep',
'delete',
]
index = self.interactive.select_choice(f'> What to do {video.filepath} size[{format_bytes(video.size)}]?', choices)
if index == -1:
continue
if choices[index] == 'delete':
self.policy.delete(self.source_dir, video.filepath)
else:
continue