def run_all_after(self, file_path, destination_folder, final_file_path,
metadata):
"""Process `before` methods of each plugin that was loaded.
"""
self.load()
pass_status = True
for cls in self.classes:
this_method = getattr(self.classes[cls], 'after')
# We try to call the plugin's `before()` method.
# If the method explicitly raises an ElodiePluginError we'll fail the import
# by setting pass_status to False.
# If any other error occurs we log the message and proceed as usual.
# By default, plugins don't change behavior.
try:
this_method(file_path, destination_folder, final_file_path,
metadata)
log.info('Called after() for {}'.format(cls))
except ElodiePluginError as err:
log.warn('Plugin {} raised an exception in run_all_before: {}'.
format(cls, err))
log.error(format_exc())
log.error('false')
pass_status = False
except:
log.error(format_exc())
return pass_status
def _generate_db(source):
"""Regenerate the hash.json database which contains all of the sha1 signatures of media files.
"""
result = Result()
source = os.path.abspath(os.path.expanduser(source))
extensions = set()
all_files = set()
valid_files = set()
if not os.path.isdir(source):
log.error('Source is not a valid directory %s' % source)
sys.exit(1)
subclasses = get_all_subclasses(Base)
for cls in subclasses:
extensions.update(cls.extensions)
all_files.update(FILESYSTEM.get_all_files(source, None))
db = Db()
db.backup_hash_db()
db.reset_hash_db()
for current_file in all_files:
if os.path.splitext(current_file)[1][1:].lower() not in extensions:
log.info('Skipping invalid file %s' % current_file)
result.append((current_file, False))
continue
result.append((current_file, True))
db.add_hash(db.checksum(current_file), current_file)
db.update_hash_db()
result.write()
def import_file(file_path,
config,
manifest,
metadata_dict,
move=False,
allow_duplicates=False,
dryrun=False):
"""Set file metadata and move it to destination.
"""
if not os.path.exists(file_path):
log.warn('Import_file: Could not find %s' % file_path)
return
target = config["targets"][0]
target_base_path = target["base_path"]
# Check if the source, _file, is a child folder within destination
# .... this is not the right time to be checking for that. Lots of unnecessary checks
# elif destination.startswith(os.path.abspath(os.path.dirname(_file))+os.sep):
# print('{"source": "%s", "destination": "%s", "error_msg": "Source cannot be in destination"}' % (_file, destination))
# return
# Creates an object of the right type, using the file extension ie .jpg -> photo
media = Media.get_class_by_file(file_path, get_all_subclasses())
if not media:
log.warn('Not a supported file (%s)' % file_path)
return
# if album_from_folder:
# media.set_album_from_folder()
checksum = manifest.checksum(file_path)
is_duplicate = (checksum in manifest.entries)
# Merge it into the manifest regardless of duplicate entries, to record all sources for a given file
manifest_entry = FILESYSTEM.generate_manifest(file_path, target,
metadata_dict, media)
manifest.merge({checksum: manifest_entry})
if (not allow_duplicates) and is_duplicate:
log.debug(
"[ ] File {} already present in manifest; allow_duplicates is false; skipping"
.format(file_path))
return True
if dryrun:
log.info("Generated manifest: {}".format(file_path))
return manifest_entry is not None
else:
result = FILESYSTEM.execute_manifest(file_path,
manifest_entry,
target_base_path,
move_not_copy=move)
# if dest_path:
# print('%s -> %s' % (_file, dest_path))
# if trash:
# send2trash(_file)
return result
def _merge(manifest_paths, output_path, debug):
constants.debug = debug
manifest = Manifest()
for manifest_path in manifest_paths:
manifest.load_from_file(manifest_path)
manifest.write(output_path, overwrite=False)
manifest_key_count = len(manifest)
log.info("Statistics:")
log.info("Merged Manifest: Total Hashes {}".format(manifest_key_count))
def parse_result(result):
log.info('MapQuest result "%s"' % result)
if ('error' in result):
return None
if ('results' in result and len(result['results']) > 0
and 'locations' in result['results'][0]
and len(result['results'][0]['locations']) > 0
and 'latLng' in result['results'][0]['locations'][0]):
latLng = result['results'][0]['locations'][0]['latLng']
if (latLng['lat'] == 39.78373 and latLng['lng'] == -100.445882):
return None
return result
def place_name(lat, lon):
lookup_place_name_default = {'default': __DEFAULT_LOCATION__}
if (lat is None or lon is None):
return lookup_place_name_default
# Convert lat/lon to floats
if (not isinstance(lat, float)):
lat = float(lat)
if (not isinstance(lon, float)):
lon = float(lon)
# Try to get cached location first
db = Db()
# 3km distace radius for a match
cached_place_name = db.get_location_name(lat, lon, 3000)
# We check that it's a dict to coerce an upgrade of the location
# db from a string location to a dictionary. See gh-160.
if (isinstance(cached_place_name, dict)):
return cached_place_name
lookup_place_name = {}
geolocation_info = lookup(lat=lat, lon=lon)
if (geolocation_info is not None and 'address' in geolocation_info):
address = geolocation_info['address']
log.info('Location: "%s"' % geolocation_info['display_name'])
# gh-386 adds support for town
# taking precedence after city for backwards compatibility
for loc in ['hamlet', 'village', 'city', 'town', 'state', 'country']:
if (loc in address):
lookup_place_name[loc] = address[loc]
# In many cases the desired key is not available so we
# set the most specific as the default.
if ('default' not in lookup_place_name):
lookup_place_name['default'] = address[loc]
if (lookup_place_name):
db.add_location(lat, lon, lookup_place_name)
# TODO: Maybe this should only be done on exit and not for every write.
db.update_location_db()
if ('default' not in lookup_place_name):
lookup_place_name = lookup_place_name_default
return lookup_place_name
def run_batch(self):
self.load()
pass_status = True
for cls in self.classes:
this_method = getattr(self.classes[cls], 'batch')
# We try to call the plugin's `before()` method.
# If the method explicitly raises an ElodiePluginError we'll fail the import
# by setting pass_status to False.
# If any other error occurs we log the message and proceed as usual.
# By default, plugins don't change behavior.
try:
this_method()
log.info('Called batch() for {}'.format(cls))
except ElodiePluginError as err:
log.warn('Plugin {} raised an exception in run_batch: {}'.format(cls, err))
log.error(format_exc())
pass_status = False
except:
log.error(format_exc())
return pass_status
def process_checksum(self, _file, allow_duplicate):
db = Db()
checksum = db.checksum(_file)
if(checksum is None):
log.info('Could not get checksum for %s.' % _file)
return None
# If duplicates are not allowed then we check if we've seen this file
# before via checksum. We also check that the file exists at the
# location we believe it to be.
# If we find a checksum match but the file doesn't exist where we
# believe it to be then we write a debug log and proceed to import.
checksum_file = db.get_hash(checksum)
if(allow_duplicate is False and checksum_file is not None):
if(os.path.isfile(checksum_file)):
log.info('%s already at %s.' % (
_file,
checksum_file
))
return None
else:
log.info('%s matched checksum but file not found at %s.' % ( # noqa
_file,
checksum_file
))
return checksum
def load_from_file(self, file_path):
self.file_path = file_path # To allow re-saving afterwards
if not os.path.isfile(file_path):
log.info("Specified manifest file {} does not exist, creating...".
format(file_path))
with open(file_path, 'a') as f:
json.dump({}, f)
os.utime(file_path, None)
log.info("[ ] Loading from {}...".format(file_path))
with open(file_path, 'r') as f:
self.merge(json.load(f))
log.info("[*] Load complete.".format(file_path))
return self # Allow chaining
def process_file(self, _file, destination, media, **kwargs):
move = False
if('move' in kwargs):
move = kwargs['move']
allow_duplicate = False
if('allowDuplicate' in kwargs):
allow_duplicate = kwargs['allowDuplicate']
if(not media.is_valid()):
print('%s is not a valid media file. Skipping...' % _file)
return
media.set_original_name()
metadata = media.get_metadata()
directory_name = self.destination_folder.get_folder_path(metadata)
dest_directory = os.path.join(destination, directory_name)
file_name = self.get_file_name(media)
dest_path = os.path.join(dest_directory, file_name)
db = Db()
checksum = db.checksum(_file)
if(checksum is None):
log.info('Could not get checksum for %s.' % _file)
return None
# If duplicates are not allowed then we check if we've seen this file
# before via checksum. We also check that the file exists at the
# location we believe it to be.
# If we find a checksum match but the file doesn't exist where we
# believe it to be then we write a debug log and proceed to import.
checksum_file = db.get_hash(checksum)
if(allow_duplicate is False and checksum_file is not None):
if(os.path.isfile(checksum_file)):
log.info('%s already at %s.' % (
_file,
checksum_file
))
return None
else:
log.info('%s matched checksum but file not found at %s.' % ( # noqa
_file,
checksum_file
))
return checksum
def _analyze(manifest_path, debug):
constants.debug = debug
manifest = Manifest()
manifest.load_from_file(manifest_path)
manifest_key_count = len(manifest)
duplicate_source_file_count = {}
# Could be made into a reduce, but I want more functionality here ( ie a list of the duplicated files )
for k, v in manifest.entries.items():
if len(v["sources"]) > 1:
length = len(v["sources"])
if length in duplicate_source_file_count:
duplicate_source_file_count[length] += 1
else:
duplicate_source_file_count[length] = 1
log.info("Statistics:")
log.info("Manifest: Total Hashes {}".format(manifest_key_count))
for k, v in duplicate_source_file_count.items():
log.info("Manifest: Duplicate (x{}) Source Files {}".format(k, v))
def write(self, write_path=None, indent=False, overwrite=True):
file_path, file_name = os.path.split(self.file_path)
name, ext = os.path.splitext(file_name)
if write_path is None:
filesystem.FileSystem().create_directory(
os.path.join(file_path, '.manifest_history'))
write_name = "{}{}".format(
'_'.join(
[name,
datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')]), ext)
# TODO: check to see if you're already in a manifest_history directory, so as to not nest another one
write_path = os.path.join(file_path, '.manifest_history',
write_name)
if overwrite is True and os.path.exists(self.file_path):
log.info("Writing manifest to {}".format(self.file_path))
with open(self.file_path, 'w') as f:
if indent:
json.dump(self.entries,
f,
indent=2,
separators=(',', ': '))
else:
json.dump(self.entries, f, separators=(',', ':'))
else:
log.warn("Not overwriting manifest at {}".format(
self.file_path))
log.info("Writing manifest to {}".format(write_path))
with open(write_path, 'w') as f:
if indent:
json.dump(self.entries, f, indent=2, separators=(',', ': '))
else:
json.dump(self.entries, f, separators=(',', ':'))
log.info("Manifest written.")
def process_file(self, _file, destination, media, **kwargs):
move = False
if ('move' in kwargs):
move = kwargs['move']
allow_duplicate = False
if ('allowDuplicate' in kwargs):
allow_duplicate = kwargs['allowDuplicate']
if (not media.is_valid()):
print('%s is not a valid media file. Skipping...' % _file)
return
metadata = media.get_metadata()
directory_name = self.get_folder_path(metadata)
dest_directory = os.path.join(destination, directory_name)
file_name = self.get_file_name(media)
dest_path = os.path.join(dest_directory, file_name)
db = Db()
checksum = db.checksum(_file)
if (checksum is None):
log.info('Could not get checksum for %s. Skipping...' % _file)
return
# If duplicates are not allowed then we check if we've seen this file
# before via checksum. We also check that the file exists at the
# location we believe it to be.
# If we find a checksum match but the file doesn't exist where we
# believe it to be then we write a debug log and proceed to import.
checksum_file = db.get_hash(checksum)
if (allow_duplicate is False and checksum_file is not None):
if (os.path.isfile(checksum_file)):
log.info('%s already exists at %s. Skipping...' %
(_file, checksum_file))
return
else:
log.info(
'%s matched checksum but file not found at %s. Importing again...'
% ( # noqa
_file, checksum_file))
self.create_directory(dest_directory)
if (move is True):
stat = os.stat(_file)
shutil.move(_file, dest_path)
os.utime(dest_path, (stat.st_atime, stat.st_mtime))
else:
# Do not use copy2(), will have an issue when copying to a
# network/mounted drive using copy and manual
# set_date_from_filename gets the job done
shutil.copy(_file, dest_path)
self.set_utime(media)
db.add_hash(checksum, dest_path)
db.update_hash_db()
return dest_path
def _update(album, location, time, title, paths, debug):
"""Update a file's EXIF. Automatically modifies the file's location and file name accordingly.
"""
constants.debug = debug
has_errors = False
result = Result()
files = set()
for path in paths:
path = os.path.expanduser(path)
if os.path.isdir(path):
files.update(FILESYSTEM.get_all_files(path, None))
else:
files.add(path)
for current_file in files:
if not os.path.exists(current_file):
has_errors = True
result.append((current_file, False))
log.warn('Could not find %s' % current_file)
log.all('{"source":"%s", "error_msg":"Could not find %s"}' %
(current_file, current_file))
continue
current_file = os.path.expanduser(current_file)
# The destination folder structure could contain any number of levels
# So we calculate that and traverse up the tree.
# '/path/to/file/photo.jpg' -> '/path/to/file' ->
# ['path','to','file'] -> ['path','to'] -> '/path/to'
current_directory = os.path.dirname(current_file)
destination_depth = -1 * len(FILESYSTEM.get_folder_path_definition())
destination = os.sep.join(
os.path.normpath(
current_directory
).split(os.sep)[:destination_depth]
)
media = Media.get_class_by_file(current_file, get_all_subclasses())
if not media:
continue
updated = False
if location:
update_location(media, current_file, location)
updated = True
if time:
update_time(media, current_file, time)
updated = True
if album:
media.set_album(album)
updated = True
# Updating a title can be problematic when doing it 2+ times on a file.
# You would end up with img_001.jpg -> img_001-first-title.jpg ->
# img_001-first-title-second-title.jpg.
# To resolve that we have to track the prior title (if there was one.
# Then we massage the updated_media's metadata['base_name'] to remove
# the old title.
# Since FileSystem.get_file_name() relies on base_name it will properly
# rename the file by updating the title instead of appending it.
remove_old_title_from_name = False
if title:
# We call get_metadata() to cache it before making any changes
metadata = media.get_metadata()
title_update_status = media.set_title(title)
original_title = metadata['title']
if title_update_status and original_title:
# @TODO: We should move this to a shared method since
# FileSystem.get_file_name() does it too.
original_title = re.sub(r'\W+', '-', original_title.lower())
original_base_name = metadata['base_name']
remove_old_title_from_name = True
updated = True
if updated:
updated_media = Media.get_class_by_file(current_file,
get_all_subclasses())
# See comments above on why we have to do this when titles
# get updated.
if remove_old_title_from_name and len(original_title) > 0:
updated_media.get_metadata()
updated_media.set_metadata_basename(
original_base_name.replace('-%s' % original_title, ''))
dest_path = FILESYSTEM.process_file(current_file, destination,
updated_media, move=True, allowDuplicate=True)
log.info(u'%s -> %s' % (current_file, dest_path))
log.all('{"source":"%s", "destination":"%s"}' % (current_file,
dest_path))
# If the folder we moved the file out of or its parent are empty
# we delete it.
FILESYSTEM.delete_directory_if_empty(os.path.dirname(current_file))
FILESYSTEM.delete_directory_if_empty(
os.path.dirname(os.path.dirname(current_file)))
result.append((current_file, dest_path))
# Trip has_errors to False if it's already False or dest_path is.
has_errors = has_errors is True or not dest_path
else:
has_errors = False
result.append((current_file, False))
result.write()
if has_errors:
sys.exit(1)
def process_file(self, _file, destination, media, **kwargs):
move = False
if ('move' in kwargs):
move = kwargs['move']
allow_duplicate = False
if ('allowDuplicate' in kwargs):
allow_duplicate = kwargs['allowDuplicate']
if (not media.is_valid()):
print('%s is not a valid media file. Skipping...' % _file)
return
media.set_original_name()
metadata = media.get_metadata()
directory_name = self.get_folder_path(metadata)
dest_directory = os.path.join(destination, directory_name)
file_name = self.get_file_name(media)
dest_path = os.path.join(dest_directory, file_name)
db = Db()
checksum = db.checksum(_file)
if (checksum is None):
log.info('Could not get checksum for %s. Skipping...' % _file)
return
# If duplicates are not allowed then we check if we've seen this file
# before via checksum. We also check that the file exists at the
# location we believe it to be.
# If we find a checksum match but the file doesn't exist where we
# believe it to be then we write a debug log and proceed to import.
checksum_file = db.get_hash(checksum)
if (allow_duplicate is False and checksum_file is not None):
if (os.path.isfile(checksum_file)):
log.info('%s already exists at %s. Skipping...' %
(_file, checksum_file))
return
else:
log.info(
'%s matched checksum but file not found at %s. Importing again...'
% ( # noqa
_file, checksum_file))
# If source and destination are identical then
# we should not write the file. gh-210
if (_file == dest_path):
print('Final source and destination path should not be identical')
return
self.create_directory(dest_directory)
if (move is True):
stat = os.stat(_file)
shutil.move(_file, dest_path)
os.utime(dest_path, (stat.st_atime, stat.st_mtime))
else:
compatability._copyfile(_file, dest_path)
self.set_utime(media)
db.add_hash(checksum, dest_path)
db.update_hash_db()
return dest_path
def log(self, msg):
# Writes an info log not shown unless being run in --debug mode.
log.info(dumps({self.__name__: msg}))
def _update(album, location, time, title, files):
"""Update a file's EXIF. Automatically modifies the file's location and file name accordingly.
"""
result = Result()
for current_file in files:
if not os.path.exists(current_file):
if constants.debug:
print('Could not find %s' % current_file)
print('{"source":"%s", "error_msg":"Could not find %s"}' % \
(current_file, current_file))
continue
current_file = os.path.expanduser(current_file)
destination = os.path.expanduser(
os.path.dirname(os.path.dirname(os.path.dirname(current_file))))
media = Media.get_class_by_file(current_file,
[Text, Audio, Photo, Video])
if not media:
continue
updated = False
if location:
update_location(media, current_file, location)
updated = True
if time:
update_time(media, current_file, time)
updated = True
if album:
media.set_album(album)
updated = True
# Updating a title can be problematic when doing it 2+ times on a file.
# You would end up with img_001.jpg -> img_001-first-title.jpg ->
# img_001-first-title-second-title.jpg.
# To resolve that we have to track the prior title (if there was one.
# Then we massage the updated_media's metadata['base_name'] to remove
# the old title.
# Since FileSystem.get_file_name() relies on base_name it will properly
# rename the file by updating the title instead of appending it.
remove_old_title_from_name = False
if title:
# We call get_metadata() to cache it before making any changes
metadata = media.get_metadata()
title_update_status = media.set_title(title)
original_title = metadata['title']
if title_update_status and original_title:
# @TODO: We should move this to a shared method since
# FileSystem.get_file_name() does it too.
original_title = re.sub(r'\W+', '-', original_title.lower())
original_base_name = metadata['base_name']
remove_old_title_from_name = True
updated = True
if updated:
updated_media = Media.get_class_by_file(
current_file, [Text, Audio, Photo, Video])
# See comments above on why we have to do this when titles
# get updated.
if remove_old_title_from_name and len(original_title) > 0:
updated_media.get_metadata()
updated_media.set_metadata_basename(
original_base_name.replace('-%s' % original_title, ''))
dest_path = FILESYSTEM.process_file(current_file,
destination,
updated_media,
move=True,
allowDuplicate=True)
log.info(u'%s -> %s' % (current_file, dest_path))
print('{"source":"%s", "destination":"%s"}' %
(current_file, dest_path))
# If the folder we moved the file out of or its parent are empty
# we delete it.
FILESYSTEM.delete_directory_if_empty(os.path.dirname(current_file))
FILESYSTEM.delete_directory_if_empty(
os.path.dirname(os.path.dirname(current_file)))
result.append((current_file, dest_path))
else:
result.append((current_file, False))
result.write()
def _update(album, location, time, title, files):
"""Update a file's EXIF. Automatically modifies the file's location and file name accordingly.
"""
result = Result()
for current_file in files:
if not os.path.exists(current_file):
if constants.debug:
print('Could not find %s' % current_file)
print('{"source":"%s", "error_msg":"Could not find %s"}' % \
(current_file, current_file))
continue
current_file = os.path.expanduser(current_file)
destination = os.path.expanduser(os.path.dirname(os.path.dirname(
os.path.dirname(current_file))))
media = Media.get_class_by_file(current_file, [Text, Audio, Photo, Video])
if not media:
continue
updated = False
if location:
update_location(media, current_file, location)
updated = True
if time:
update_time(media, current_file, time)
updated = True
if album:
media.set_album(album)
updated = True
# Updating a title can be problematic when doing it 2+ times on a file.
# You would end up with img_001.jpg -> img_001-first-title.jpg ->
# img_001-first-title-second-title.jpg.
# To resolve that we have to track the prior title (if there was one.
# Then we massage the updated_media's metadata['base_name'] to remove
# the old title.
# Since FileSystem.get_file_name() relies on base_name it will properly
# rename the file by updating the title instead of appending it.
remove_old_title_from_name = False
if title:
# We call get_metadata() to cache it before making any changes
metadata = media.get_metadata()
title_update_status = media.set_title(title)
original_title = metadata['title']
if title_update_status and original_title:
# @TODO: We should move this to a shared method since
# FileSystem.get_file_name() does it too.
original_title = re.sub(r'\W+', '-', original_title.lower())
original_base_name = metadata['base_name']
remove_old_title_from_name = True
updated = True
if updated:
updated_media = Media.get_class_by_file(current_file,
[Text, Audio, Photo, Video])
# See comments above on why we have to do this when titles
# get updated.
if remove_old_title_from_name and len(original_title) > 0:
updated_media.get_metadata()
updated_media.set_metadata_basename(
original_base_name.replace('-%s' % original_title, ''))
dest_path = FILESYSTEM.process_file(current_file, destination,
updated_media, move=True, allowDuplicate=True)
log.info(u'%s -> %s' % (current_file, dest_path))
print('{"source":"%s", "destination":"%s"}' % (current_file,
dest_path))
# If the folder we moved the file out of or its parent are empty
# we delete it.
FILESYSTEM.delete_directory_if_empty(os.path.dirname(current_file))
FILESYSTEM.delete_directory_if_empty(
os.path.dirname(os.path.dirname(current_file)))
result.append((current_file, dest_path))
else:
result.append((current_file, False))
result.write()
def _import(source,
config_path,
manifest_path,
allow_duplicates,
dryrun,
debug,
move=False,
indent_manifest=False,
no_overwrite_manifest=False):
"""Import files or directories by reading their EXIF and organizing them accordingly.
"""
start_time = round(time.time())
constants.debug = debug
has_errors = False
result = Result()
# Load the configuration from the json file.
config = Config().load_from_file(config_path)
source = config["sources"][0] # For now, only one.
target = config["targets"][
0] # For now, only one target allowed...but data structure allows more
source_file_path = source["file_path"]
manifest = Manifest()
if manifest_path is not None:
manifest.load_from_file(manifest_path)
log_base_path, _ = os.path.split(manifest.file_path)
FILESYSTEM.create_directory(os.path.join(log_base_path, '.elodie'))
log_path = os.path.join(log_base_path, '.elodie',
'import_{}.log'.format(utility.timestamp_string()))
def signal_handler(sig, frame):
log.warn('[ ] Import cancelled')
log.write(log_path)
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
original_manifest_key_count = len(manifest)
# destination = _decode(destination)
# destination = os.path.abspath(os.path.expanduser(destination))
exiftool_addedargs = [
# '-overwrite_original',
u'-config',
u'"{}"'.format(constants.exiftool_config)
]
file_generator = FILESYSTEM.get_all_files(source_file_path, None)
source_file_count = 0
with ExifTool(addedargs=exiftool_addedargs) as et:
while True:
file_batch = list(
itertools.islice(file_generator,
constants.exiftool_batch_size))
if len(file_batch) == 0: break
# This will cause slight discrepancies in file counts: since elodie.json is counted but not imported,
# each one will set the count off by one.
source_file_count += len(file_batch)
metadata_list = et.get_metadata_batch(file_batch)
if not metadata_list:
raise Exception("Metadata scrape failed.")
# Key on the filename to make for easy access,
metadata_dict = dict((os.path.abspath(el["SourceFile"]), el)
for el in metadata_list)
for current_file in file_batch:
# Don't import localized config files.
if current_file.endswith(
"elodie.json"): # Faster than a os.path.split
continue
try:
result = import_file(current_file,
config,
manifest,
metadata_dict,
move=move,
dryrun=dryrun,
allow_duplicates=allow_duplicates)
except Exception as e:
log.warn("[!] Error importing {}: {}".format(
current_file, e))
result = False
has_errors = has_errors or not result
exiftool_waiting_time = et.waiting_time
manifest.write(indent=indent_manifest,
overwrite=(not no_overwrite_manifest))
manifest_key_count = len(manifest)
try:
total_time = round(time.time() - start_time)
log.info("Statistics:")
log.info("Source: File Count {}".format(source_file_count))
log.info("Manifest: New Hashes {}".format(manifest_key_count -
original_manifest_key_count))
log.info("Manifest: Total Hashes {}".format(manifest_key_count))
log.info("Time: Total {}s".format(total_time))
log.info("Time: Files/sec {}".format(
round(source_file_count / total_time)))
log.info("Time: Waiting on ExifTool {}s".format(
round(exiftool_waiting_time)))
except Exception as e:
log.error("[!] Error generating statistics: {}".format(e))
log.write(log_path)
if has_errors:
sys.exit(1)
def process_file(self, _file, destination, media, **kwargs):
move = False
if('move' in kwargs):
move = kwargs['move']
allow_duplicate = False
if('allowDuplicate' in kwargs):
allow_duplicate = kwargs['allowDuplicate']
stat_info_original = os.stat(_file)
metadata = media.get_metadata()
if(not media.is_valid()):
print('%s is not a valid media file. Skipping...' % _file)
return
checksum = self.process_checksum(_file, allow_duplicate)
if(checksum is None):
log.info('Original checksum returned None for %s. Skipping...' %
_file)
return
# Run `before()` for every loaded plugin and if any of them raise an exception
# then we skip importing the file and log a message.
plugins_run_before_status = self.plugins.run_all_before(_file, destination)
if(plugins_run_before_status == False):
log.warn('At least one plugin pre-run failed for %s' % _file)
return
directory_name = self.get_folder_path(metadata)
dest_directory = os.path.join(destination, directory_name)
file_name = self.get_file_name(metadata)
dest_path = os.path.join(dest_directory, file_name)
media.set_original_name()
# If source and destination are identical then
# we should not write the file. gh-210
if(_file == dest_path):
print('Final source and destination path should not be identical')
return
self.create_directory(dest_directory)
# exiftool renames the original file by appending '_original' to the
# file name. A new file is written with new tags with the initial file
# name. See exiftool man page for more details.
exif_original_file = _file + '_original'
# Check if the source file was processed by exiftool and an _original
# file was created.
exif_original_file_exists = False
if(os.path.exists(exif_original_file)):
exif_original_file_exists = True
if(move is True):
stat = os.stat(_file)
# Move the processed file into the destination directory
shutil.move(_file, dest_path)
if(exif_original_file_exists is True):
# We can remove it as we don't need the initial file.
os.remove(exif_original_file)
os.utime(dest_path, (stat.st_atime, stat.st_mtime))
else:
if(exif_original_file_exists is True):
# Move the newly processed file with any updated tags to the
# destination directory
shutil.move(_file, dest_path)
# Move the exif _original back to the initial source file
shutil.move(exif_original_file, _file)
else:
compatability._copyfile(_file, dest_path)
# Set the utime based on what the original file contained
# before we made any changes.
# Then set the utime on the destination file based on metadata.
os.utime(_file, (stat_info_original.st_atime, stat_info_original.st_mtime))
self.set_utime_from_metadata(metadata, dest_path)
db = Db()
db.add_hash(checksum, dest_path)
db.update_hash_db()
# Run `after()` for every loaded plugin and if any of them raise an exception
# then we skip importing the file and log a message.
plugins_run_after_status = self.plugins.run_all_after(_file, destination, dest_path, metadata)
if(plugins_run_after_status == False):
log.warn('At least one plugin pre-run failed for %s' % _file)
return
return dest_path
def process_file(self, _file, destination, media, **kwargs):
move = False
if('move' in kwargs):
move = kwargs['move']
allow_duplicate = False
if('allowDuplicate' in kwargs):
allow_duplicate = kwargs['allowDuplicate']
if(not media.is_valid()):
print('%s is not a valid media file. Skipping...' % _file)
return
metadata = media.get_metadata()
directory_name = self.get_folder_path(metadata)
dest_directory = os.path.join(destination, directory_name)
file_name = self.get_file_name(media)
dest_path = os.path.join(dest_directory, file_name)
db = Db()
checksum = db.checksum(_file)
if(checksum is None):
log.info('Could not get checksum for %s. Skipping...' % _file)
return
# If duplicates are not allowed then we check if we've seen this file
# before via checksum. We also check that the file exists at the
# location we believe it to be.
# If we find a checksum match but the file doesn't exist where we
# believe it to be then we write a debug log and proceed to import.
checksum_file = db.get_hash(checksum)
if(allow_duplicate is False and checksum_file is not None):
if(os.path.isfile(checksum_file)):
log.info('%s already exists at %s. Skipping...' % (
_file,
checksum_file
))
return
else:
log.info('%s matched checksum but file not found at %s. Importing again...' % ( # noqa
_file,
checksum_file
))
self.create_directory(dest_directory)
if(move is True):
stat = os.stat(_file)
shutil.move(_file, dest_path)
os.utime(dest_path, (stat.st_atime, stat.st_mtime))
else:
# Do not use copy2(), will have an issue when copying to a
# network/mounted drive using copy and manual
# set_date_from_filename gets the job done
shutil.copy(_file, dest_path)
self.set_utime(media)
db.add_hash(checksum, dest_path)
db.update_hash_db()
return dest_path