def _import(destination, source, file, album_from_folder, trash, paths, allow_duplicates):
"""Import files or directories by reading their EXIF and organizing them accordingly.
"""
result = Result()
destination = os.path.abspath(os.path.expanduser(destination))
files = set()
paths = set(paths)
if source:
paths.add(source)
if file:
paths.add(file)
for path in paths:
path = os.path.expanduser(path)
if os.path.isdir(path):
files.update(FILESYSTEM.get_all_files(path, None))
else:
files.add(path)
for current_file in files:
dest_path = import_file(current_file, destination, album_from_folder,
trash, allow_duplicates)
result.append((current_file, dest_path))
result.write()
def test_add_multiple_rows_with_success():
expected = """****** SUMMARY ******
Metric Count
-------- -------
Success 2
Error 0"""
result = Result()
result.append(('id1', '/some/path/1'))
result.append(('id2', '/some/path/2'))
call_result_and_assert(result, expected)
def test_add_multiple_rows_with_success():
expected = """****** SUMMARY ******
Metric Count
-------- -------
Success 2
Error 0"""
result = Result()
result.append(('id1', '/some/path/1'))
result.append(('id2', '/some/path/2'))
call_result_and_assert(result, expected)
def test_add_multiple_rows_with_failure_and_success():
expected = """****** ERROR DETAILS ******
File
------
id1
****** SUMMARY ******
Metric Count
-------- -------
Success 1
Error 1"""
result = Result()
result.append(('id1', False))
result.append(('id2', '/some/path'))
call_result_and_assert(result, expected)
def test_add_multiple_rows_with_failure():
expected = """****** ERROR DETAILS ******
File
------
id1
id2
****** SUMMARY ******
Metric Count
-------- -------
Success 0
Error 2"""
result = Result()
result.append(('id1', False))
result.append(('id2', False))
call_result_and_assert(result, expected)
def _generate_db(source):
"""Regenerate the hash.json database which contains all of the sha1 signatures of media files.
"""
result = Result()
source = os.path.abspath(os.path.expanduser(source))
extensions = set()
all_files = set()
valid_files = set()
if not os.path.isdir(source):
log.error('Source is not a valid directory %s' % source)
sys.exit(1)
subclasses = get_all_subclasses(Base)
for cls in subclasses:
extensions.update(cls.extensions)
all_files.update(FILESYSTEM.get_all_files(source, None))
db = Db()
db.backup_hash_db()
db.reset_hash_db()
for current_file in all_files:
if os.path.splitext(current_file)[1][1:].lower() not in extensions:
log.info('Skipping invalid file %s' % current_file)
result.append((current_file, False))
continue
result.append((current_file, True))
db.add_hash(db.checksum(current_file), current_file)
db.update_hash_db()
result.write()
def _verify():
result = Result()
db = Db()
for checksum, file_path in db.all():
if not os.path.isfile(file_path):
result.append((file_path, False))
continue
actual_checksum = db.checksum(file_path)
if checksum == actual_checksum:
result.append((file_path, True))
else:
result.append((file_path, False))
result.write()
def _verify():
result = Result()
db = Db()
for checksum, file_path in db.all():
if not os.path.isfile(file_path):
result.append((file_path, False))
log.progress('x')
continue
actual_checksum = db.checksum(file_path)
if checksum == actual_checksum:
result.append((file_path, True))
log.progress()
else:
result.append((file_path, False))
log.progress('x')
log.progress('', True)
result.write()
def _import(destination, source, file, album_from_folder, trash,
allow_duplicates, debug, paths, move):
"""Import files or directories by reading their EXIF and organizing them accordingly.
"""
constants.debug = debug
has_errors = False
result = Result()
move = move
destination = _decode(destination)
destination = os.path.abspath(os.path.expanduser(destination))
files = set()
paths = set(paths)
if source:
source = _decode(source)
paths.add(source)
if file:
paths.add(file)
for path in paths:
path = os.path.expanduser(path)
if os.path.isdir(path):
files.update(FILESYSTEM.get_all_files(path, None))
else:
files.add(path)
for current_file in files:
dest_path = import_file(current_file, destination, album_from_folder,
trash, allow_duplicates, move)
result.append((current_file, dest_path))
has_errors = has_errors is True or not dest_path
result.write()
if has_errors:
sys.exit(1)
def _import(destination, source, file, album_from_folder, trash, paths,
allow_duplicates):
"""Import files or directories by reading their EXIF and organizing them accordingly.
"""
result = Result()
destination = destination.decode(sys.getfilesystemencoding())
destination = os.path.abspath(os.path.expanduser(destination))
files = set()
paths = set(paths)
if source:
source = source.decode(sys.getfilesystemencoding())
paths.add(source)
if file:
paths.add(file)
for path in paths:
path = os.path.expanduser(path)
if os.path.isdir(path):
files.update(FILESYSTEM.get_all_files(path, None))
else:
files.add(path)
for current_file in files:
dest_path = import_file(current_file, destination, album_from_folder,
trash, allow_duplicates)
result.append((current_file, dest_path))
result.write()
def main(argv):
filesystem = FileSystem()
result = Result()
subclasses = get_all_subclasses()
paths = argv[1:]
for path in paths:
path = os.path.expanduser(path)
if os.path.isdir(path):
for source in filesystem.get_all_files(path, None):
status = add_original_name(source, subclasses)
result.append((_decode(source), status))
else:
status = add_original_name(path, subclasses)
result.append((_decode(path), status))
result.write()
def _import(destination, source, file, album_from_folder, trash,
allow_duplicates, debug, exclude_regex, paths):
"""Import files or directories by reading their EXIF and organizing them accordingly.
"""
constants.debug = debug
has_errors = False
result = Result()
destination = _decode(destination)
destination = os.path.abspath(os.path.expanduser(destination))
files = set()
paths = set(paths)
if source:
source = _decode(source)
paths.add(source)
if file:
paths.add(file)
# if no exclude list was passed in we check if there's a config
if len(exclude_regex) == 0:
config = load_config()
if 'Exclusions' in config:
exclude_regex = [
value for key, value in config.items('Exclusions')
]
exclude_regex_list = set(exclude_regex)
for path in paths:
path = os.path.expanduser(path)
if os.path.isdir(path):
files.update(
FILESYSTEM.get_all_files(path, None, exclude_regex_list))
else:
if not FILESYSTEM.should_exclude(path, exclude_regex_list, True):
files.add(path)
for current_file in files:
dest_path = import_file(current_file, destination, album_from_folder,
trash, allow_duplicates)
result.append((current_file, dest_path))
has_errors = has_errors is True or not dest_path
result.write()
if has_errors:
sys.exit(1)
def _generate_db(source):
"""Regenerate the hash.json database which contains all of the sha1 signatures of media files.
"""
result = Result()
source = os.path.abspath(os.path.expanduser(source))
if not os.path.isdir(source):
log.error('Source is not a valid directory %s' % source)
sys.exit(1)
db = Db()
db.backup_hash_db()
db.reset_hash_db()
for current_file in FILESYSTEM.get_all_files(source):
result.append((current_file, True))
db.add_hash(db.checksum(current_file), current_file)
log.progress()
db.update_hash_db()
log.progress('', True)
result.write()
def _import(source,
config_path,
manifest_path,
allow_duplicates,
dryrun,
debug,
move=False,
indent_manifest=False,
no_overwrite_manifest=False):
"""Import files or directories by reading their EXIF and organizing them accordingly.
"""
start_time = round(time.time())
constants.debug = debug
has_errors = False
result = Result()
# Load the configuration from the json file.
config = Config().load_from_file(config_path)
source = config["sources"][0] # For now, only one.
target = config["targets"][
0] # For now, only one target allowed...but data structure allows more
source_file_path = source["file_path"]
manifest = Manifest()
if manifest_path is not None:
manifest.load_from_file(manifest_path)
log_base_path, _ = os.path.split(manifest.file_path)
FILESYSTEM.create_directory(os.path.join(log_base_path, '.elodie'))
log_path = os.path.join(log_base_path, '.elodie',
'import_{}.log'.format(utility.timestamp_string()))
def signal_handler(sig, frame):
log.warn('[ ] Import cancelled')
log.write(log_path)
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
original_manifest_key_count = len(manifest)
# destination = _decode(destination)
# destination = os.path.abspath(os.path.expanduser(destination))
exiftool_addedargs = [
# '-overwrite_original',
u'-config',
u'"{}"'.format(constants.exiftool_config)
]
file_generator = FILESYSTEM.get_all_files(source_file_path, None)
source_file_count = 0
with ExifTool(addedargs=exiftool_addedargs) as et:
while True:
file_batch = list(
itertools.islice(file_generator,
constants.exiftool_batch_size))
if len(file_batch) == 0: break
# This will cause slight discrepancies in file counts: since elodie.json is counted but not imported,
# each one will set the count off by one.
source_file_count += len(file_batch)
metadata_list = et.get_metadata_batch(file_batch)
if not metadata_list:
raise Exception("Metadata scrape failed.")
# Key on the filename to make for easy access,
metadata_dict = dict((os.path.abspath(el["SourceFile"]), el)
for el in metadata_list)
for current_file in file_batch:
# Don't import localized config files.
if current_file.endswith(
"elodie.json"): # Faster than a os.path.split
continue
try:
result = import_file(current_file,
config,
manifest,
metadata_dict,
move=move,
dryrun=dryrun,
allow_duplicates=allow_duplicates)
except Exception as e:
log.warn("[!] Error importing {}: {}".format(
current_file, e))
result = False
has_errors = has_errors or not result
exiftool_waiting_time = et.waiting_time
manifest.write(indent=indent_manifest,
overwrite=(not no_overwrite_manifest))
manifest_key_count = len(manifest)
try:
total_time = round(time.time() - start_time)
log.info("Statistics:")
log.info("Source: File Count {}".format(source_file_count))
log.info("Manifest: New Hashes {}".format(manifest_key_count -
original_manifest_key_count))
log.info("Manifest: Total Hashes {}".format(manifest_key_count))
log.info("Time: Total {}s".format(total_time))
log.info("Time: Files/sec {}".format(
round(source_file_count / total_time)))
log.info("Time: Waiting on ExifTool {}s".format(
round(exiftool_waiting_time)))
except Exception as e:
log.error("[!] Error generating statistics: {}".format(e))
log.write(log_path)
if has_errors:
sys.exit(1)
def _update(album, location, time, title, files):
"""Update a file's EXIF. Automatically modifies the file's location and file name accordingly.
"""
result = Result()
for current_file in files:
if not os.path.exists(current_file):
if constants.debug:
print('Could not find %s' % current_file)
print('{"source":"%s", "error_msg":"Could not find %s"}' % \
(current_file, current_file))
continue
current_file = os.path.expanduser(current_file)
destination = os.path.expanduser(os.path.dirname(os.path.dirname(
os.path.dirname(current_file))))
media = Media.get_class_by_file(current_file, [Text, Audio, Photo, Video])
if not media:
continue
updated = False
if location:
update_location(media, current_file, location)
updated = True
if time:
update_time(media, current_file, time)
updated = True
if album:
media.set_album(album)
updated = True
# Updating a title can be problematic when doing it 2+ times on a file.
# You would end up with img_001.jpg -> img_001-first-title.jpg ->
# img_001-first-title-second-title.jpg.
# To resolve that we have to track the prior title (if there was one.
# Then we massage the updated_media's metadata['base_name'] to remove
# the old title.
# Since FileSystem.get_file_name() relies on base_name it will properly
# rename the file by updating the title instead of appending it.
remove_old_title_from_name = False
if title:
# We call get_metadata() to cache it before making any changes
metadata = media.get_metadata()
title_update_status = media.set_title(title)
original_title = metadata['title']
if title_update_status and original_title:
# @TODO: We should move this to a shared method since
# FileSystem.get_file_name() does it too.
original_title = re.sub(r'\W+', '-', original_title.lower())
original_base_name = metadata['base_name']
remove_old_title_from_name = True
updated = True
if updated:
updated_media = Media.get_class_by_file(current_file,
[Text, Audio, Photo, Video])
# See comments above on why we have to do this when titles
# get updated.
if remove_old_title_from_name and len(original_title) > 0:
updated_media.get_metadata()
updated_media.set_metadata_basename(
original_base_name.replace('-%s' % original_title, ''))
dest_path = FILESYSTEM.process_file(current_file, destination,
updated_media, move=True, allowDuplicate=True)
log.info(u'%s -> %s' % (current_file, dest_path))
print('{"source":"%s", "destination":"%s"}' % (current_file,
dest_path))
# If the folder we moved the file out of or its parent are empty
# we delete it.
FILESYSTEM.delete_directory_if_empty(os.path.dirname(current_file))
FILESYSTEM.delete_directory_if_empty(
os.path.dirname(os.path.dirname(current_file)))
result.append((current_file, dest_path))
else:
result.append((current_file, False))
result.write()
def _update(album, location, time, title, paths, debug):
"""Update a file's EXIF. Automatically modifies the file's location and file name accordingly.
"""
constants.debug = debug
has_errors = False
result = Result()
files = set()
for path in paths:
path = os.path.expanduser(path)
if os.path.isdir(path):
files.update(FILESYSTEM.get_all_files(path, None))
else:
files.add(path)
for current_file in files:
if not os.path.exists(current_file):
has_errors = True
result.append((current_file, False))
log.warn('Could not find %s' % current_file)
log.all('{"source":"%s", "error_msg":"Could not find %s"}' %
(current_file, current_file))
continue
current_file = os.path.expanduser(current_file)
# The destination folder structure could contain any number of levels
# So we calculate that and traverse up the tree.
# '/path/to/file/photo.jpg' -> '/path/to/file' ->
# ['path','to','file'] -> ['path','to'] -> '/path/to'
current_directory = os.path.dirname(current_file)
destination_depth = -1 * len(FILESYSTEM.get_folder_path_definition())
destination = os.sep.join(
os.path.normpath(
current_directory
).split(os.sep)[:destination_depth]
)
media = Media.get_class_by_file(current_file, get_all_subclasses())
if not media:
continue
updated = False
if location:
update_location(media, current_file, location)
updated = True
if time:
update_time(media, current_file, time)
updated = True
if album:
media.set_album(album)
updated = True
# Updating a title can be problematic when doing it 2+ times on a file.
# You would end up with img_001.jpg -> img_001-first-title.jpg ->
# img_001-first-title-second-title.jpg.
# To resolve that we have to track the prior title (if there was one.
# Then we massage the updated_media's metadata['base_name'] to remove
# the old title.
# Since FileSystem.get_file_name() relies on base_name it will properly
# rename the file by updating the title instead of appending it.
remove_old_title_from_name = False
if title:
# We call get_metadata() to cache it before making any changes
metadata = media.get_metadata()
title_update_status = media.set_title(title)
original_title = metadata['title']
if title_update_status and original_title:
# @TODO: We should move this to a shared method since
# FileSystem.get_file_name() does it too.
original_title = re.sub(r'\W+', '-', original_title.lower())
original_base_name = metadata['base_name']
remove_old_title_from_name = True
updated = True
if updated:
updated_media = Media.get_class_by_file(current_file,
get_all_subclasses())
# See comments above on why we have to do this when titles
# get updated.
if remove_old_title_from_name and len(original_title) > 0:
updated_media.get_metadata()
updated_media.set_metadata_basename(
original_base_name.replace('-%s' % original_title, ''))
dest_path = FILESYSTEM.process_file(current_file, destination,
updated_media, move=True, allowDuplicate=True)
log.info(u'%s -> %s' % (current_file, dest_path))
log.all('{"source":"%s", "destination":"%s"}' % (current_file,
dest_path))
# If the folder we moved the file out of or its parent are empty
# we delete it.
FILESYSTEM.delete_directory_if_empty(os.path.dirname(current_file))
FILESYSTEM.delete_directory_if_empty(
os.path.dirname(os.path.dirname(current_file)))
result.append((current_file, dest_path))
# Trip has_errors to False if it's already False or dest_path is.
has_errors = has_errors is True or not dest_path
else:
has_errors = False
result.append((current_file, False))
result.write()
if has_errors:
sys.exit(1)
def _update(album, location, time, title, files):
"""Update a file's EXIF. Automatically modifies the file's location and file name accordingly.
"""
result = Result()
for current_file in files:
if not os.path.exists(current_file):
if constants.debug:
print('Could not find %s' % current_file)
print('{"source":"%s", "error_msg":"Could not find %s"}' % \
(current_file, current_file))
continue
current_file = os.path.expanduser(current_file)
destination = os.path.expanduser(
os.path.dirname(os.path.dirname(os.path.dirname(current_file))))
media = Media.get_class_by_file(current_file,
[Text, Audio, Photo, Video])
if not media:
continue
updated = False
if location:
update_location(media, current_file, location)
updated = True
if time:
update_time(media, current_file, time)
updated = True
if album:
media.set_album(album)
updated = True
# Updating a title can be problematic when doing it 2+ times on a file.
# You would end up with img_001.jpg -> img_001-first-title.jpg ->
# img_001-first-title-second-title.jpg.
# To resolve that we have to track the prior title (if there was one.
# Then we massage the updated_media's metadata['base_name'] to remove
# the old title.
# Since FileSystem.get_file_name() relies on base_name it will properly
# rename the file by updating the title instead of appending it.
remove_old_title_from_name = False
if title:
# We call get_metadata() to cache it before making any changes
metadata = media.get_metadata()
title_update_status = media.set_title(title)
original_title = metadata['title']
if title_update_status and original_title:
# @TODO: We should move this to a shared method since
# FileSystem.get_file_name() does it too.
original_title = re.sub(r'\W+', '-', original_title.lower())
original_base_name = metadata['base_name']
remove_old_title_from_name = True
updated = True
if updated:
updated_media = Media.get_class_by_file(
current_file, [Text, Audio, Photo, Video])
# See comments above on why we have to do this when titles
# get updated.
if remove_old_title_from_name and len(original_title) > 0:
updated_media.get_metadata()
updated_media.set_metadata_basename(
original_base_name.replace('-%s' % original_title, ''))
dest_path = FILESYSTEM.process_file(current_file,
destination,
updated_media,
move=True,
allowDuplicate=True)
log.info(u'%s -> %s' % (current_file, dest_path))
print('{"source":"%s", "destination":"%s"}' %
(current_file, dest_path))
# If the folder we moved the file out of or its parent are empty
# we delete it.
FILESYSTEM.delete_directory_if_empty(os.path.dirname(current_file))
FILESYSTEM.delete_directory_if_empty(
os.path.dirname(os.path.dirname(current_file)))
result.append((current_file, dest_path))
else:
result.append((current_file, False))
result.write()
