def __init__(self):
# The default folder path is along the lines of 2017-06-17_01-04-14-dsc_1234-some-title.jpg
self.default_file_name_definition = {
'date': '%Y-%m-%d_%H-%M-%S',
'name': '%date-%original_name-%title.%extension',
}
# The default folder path is along the lines of 2015-01-Jan/Chicago
self.default_folder_path_definition = {
'date': '%Y-%m-%b',
'location': '%city',
'full_path': '%date/%album|%location|"{}"'.format(
geolocation.__DEFAULT_LOCATION__
),
}
self.cached_file_name_definition = None
self.cached_folder_path_definition = None
# Python3 treats the regex \s differently than Python2.
# It captures some additional characters like the unicode checkmark \u2713.
# See build failures in Python3 here.
# https://travis-ci.org/jmathai/elodie/builds/483012902
self.whitespace_regex = '[ \t\n\r\f\v]+'
# Instantiate a plugins object
self.plugins = Plugins()
#Initialize ExifTool Subprocess
exiftool_addedargs = [
u'-config',
u'"{}"'.format(constants.exiftool_config)
]
ExifTool(executable_=get_exiftool(), addedargs=exiftool_addedargs).start()
def __init__(self, source=None):
super(Media, self).__init__(source)
self.exif_map = {
'date_taken':
['EXIF:DateTimeOriginal', 'EXIF:CreateDate', 'EXIF:ModifyDate']
}
self.camera_make_keys = ['EXIF:Make', 'QuickTime:Make']
self.camera_model_keys = ['EXIF:Model', 'QuickTime:Model']
self.album_keys = ['XMP-xmpDM:Album', 'XMP:Album']
self.title_key = 'XMP:Title'
self.latitude_keys = ['EXIF:GPSLatitude']
self.longitude_keys = ['EXIF:GPSLongitude']
self.latitude_ref_key = 'EXIF:GPSLatitudeRef'
self.longitude_ref_key = 'EXIF:GPSLongitudeRef'
self.original_name_key = 'XMP:OriginalFileName'
self.set_gps_ref = True
self.exif_metadata = None
self.exiftool = ExifTool()
self.exiftool.start()
def __set_tags(self, tags):
if (not self.is_valid()):
return None
source = self.source
status = ''
status = ExifTool().set_tags(tags, source)
return status != ''
def __set_tags(self, tags):
if (not self.is_valid()):
return None
source = self.source
status = ''
with ExifTool(addedargs=self.exiftool_addedargs) as et:
status = et.set_tags(tags, source)
return status != ''
def __set_tags(self, tags):
if (not self.is_valid()):
return None
source = self.source
exiftool = get_exiftool()
if (exiftool is None):
return False
status = ''
with ExifTool(executable_=exiftool,
addedargs=self.exiftool_addedargs) as et:
status = et.set_tags(tags, source)
return status != ''
def get_exiftool_attributes(self):
"""Get attributes for the media object from exiftool.
:returns: dict, or False if exiftool was not available.
"""
source = self.source
#Cache exif metadata results and use if already exists for media
if (self.exif_metadata is None):
self.exif_metadata = ExifTool().get_metadata(source)
if not self.exif_metadata:
return False
return self.exif_metadata
def get_exiftool_attributes(self):
"""Get attributes for the media object from exiftool.
:returns: dict, or False if exiftool was not available.
"""
source = self.source
exiftool = get_exiftool()
if(exiftool is None):
return False
with ExifTool(addedargs=self.exiftool_addedargs) as et:
metadata = et.get_metadata(source)
if not metadata:
return False
return metadata
def get_exiftool_attributes(self):
# return self.get_exif_attributes()
"""Get attributes for the media object from exiftool.
:returns: dict, or False if exiftool was not available.
"""
if self.exif_metadata is not None:
return self.exif_metadata
source = self.source
exiftool = get_exiftool()
if (exiftool is None):
return False
with ExifTool(addedargs=self.exiftool_addedargs) as et:
metadata = et.get_metadata(source)
if not metadata:
return False
metadata["origin"] = self.get_origin()
self.exif_metadata = metadata
return metadata
def teardown_module():
ExifTool().terminate
def setup_module():
exiftool_addedargs = [
u'-config',
u'"{}"'.format(constants.exiftool_config)
]
ExifTool(executable_=get_exiftool(), addedargs=exiftool_addedargs).start()
class Media(Base):
"""The base class for all media objects.
:param str source: The fully qualified path to the video file.
"""
__name__ = 'Media'
d_coordinates = {'latitude': 'latitude_ref', 'longitude': 'longitude_ref'}
def __init__(self, source=None):
super(Media, self).__init__(source)
self.exif_map = {
'date_taken':
['EXIF:DateTimeOriginal', 'EXIF:CreateDate', 'EXIF:ModifyDate']
}
self.camera_make_keys = ['EXIF:Make', 'QuickTime:Make']
self.camera_model_keys = ['EXIF:Model', 'QuickTime:Model']
self.album_keys = ['XMP-xmpDM:Album', 'XMP:Album']
self.title_key = 'XMP:Title'
self.latitude_keys = ['EXIF:GPSLatitude']
self.longitude_keys = ['EXIF:GPSLongitude']
self.latitude_ref_key = 'EXIF:GPSLatitudeRef'
self.longitude_ref_key = 'EXIF:GPSLongitudeRef'
self.original_name_key = 'XMP:OriginalFileName'
self.set_gps_ref = True
self.exif_metadata = None
self.exiftool = ExifTool()
self.exiftool.start()
def __del__(self):
self.exiftool.terminate()
def get_album(self):
"""Get album from EXIF
:returns: None or string
"""
if (not self.is_valid()):
return None
exiftool_attributes = self.get_exiftool_attributes()
if exiftool_attributes is None:
return None
for album_key in self.album_keys:
if album_key in exiftool_attributes:
return exiftool_attributes[album_key]
return None
def get_coordinate(self, type='latitude'):
"""Get latitude or longitude of media from EXIF
:param str type: Type of coordinate to get. Either "latitude" or
"longitude".
:returns: float or None if not present in EXIF or a non-photo file
"""
exif = self.get_exiftool_attributes()
if not exif:
return None
# The lat/lon _keys array has an order of precedence.
# The first key is writable and we will give the writable
# key precence when reading.
direction_multiplier = 1.0
for key in self.latitude_keys + self.longitude_keys:
if key not in exif:
continue
if isinstance(exif[key], six.string_types) and len(exif[key]) == 0:
# If exiftool GPS output is empty, the data returned will be a str
# with 0 length.
# https://github.com/jmathai/elodie/issues/354
continue
# Cast coordinate to a float due to a bug in exiftool's
# -json output format.
# https://github.com/jmathai/elodie/issues/171
# http://u88.n24.queensu.ca/exiftool/forum/index.php/topic,7952.0.html # noqa
this_coordinate = float(exif[key])
# TODO: verify that we need to check ref key
# when self.set_gps_ref != True
if type == 'latitude' and key in self.latitude_keys:
if self.latitude_ref_key in exif and \
exif[self.latitude_ref_key] == 'S':
direction_multiplier = -1.0
return this_coordinate * direction_multiplier
elif type == 'longitude' and key in self.longitude_keys:
if self.longitude_ref_key in exif and \
exif[self.longitude_ref_key] == 'W':
direction_multiplier = -1.0
return this_coordinate * direction_multiplier
return None
def get_exiftool_attributes(self):
"""Get attributes for the media object from exiftool.
:returns: dict, or False if exiftool was not available.
"""
source = self.source
#Cache exif metadata results and use if already exists for media
if (self.exif_metadata is None):
self.exif_metadata = self.exiftool.get_metadata(source)
if not self.exif_metadata:
return False
return self.exif_metadata
def get_camera_make(self):
"""Get the camera make stored in EXIF.
:returns: str
"""
if (not self.is_valid()):
return None
exiftool_attributes = self.get_exiftool_attributes()
if exiftool_attributes is None:
return None
for camera_make_key in self.camera_make_keys:
if camera_make_key in exiftool_attributes:
return exiftool_attributes[camera_make_key]
return None
def get_camera_model(self):
"""Get the camera make stored in EXIF.
:returns: str
"""
if (not self.is_valid()):
return None
exiftool_attributes = self.get_exiftool_attributes()
if exiftool_attributes is None:
return None
for camera_model_key in self.camera_model_keys:
if camera_model_key in exiftool_attributes:
return exiftool_attributes[camera_model_key]
return None
def get_original_name(self):
"""Get the original name stored in EXIF.
:returns: str
"""
if (not self.is_valid()):
return None
exiftool_attributes = self.get_exiftool_attributes()
if exiftool_attributes is None:
return None
if (self.original_name_key not in exiftool_attributes):
return None
return exiftool_attributes[self.original_name_key]
def get_title(self):
"""Get the title for a photo of video
:returns: str or None if no title is set or not a valid media type
"""
if (not self.is_valid()):
return None
exiftool_attributes = self.get_exiftool_attributes()
if exiftool_attributes is None:
return None
if (self.title_key not in exiftool_attributes):
return None
return exiftool_attributes[self.title_key]
def reset_cache(self):
"""Resets any internal cache
"""
self.exiftool_attributes = None
self.exif_metadata = None
super(Media, self).reset_cache()
def set_album(self, album):
"""Set album for a photo
:param str name: Name of album
:returns: bool
"""
if (not self.is_valid()):
return None
tags = {self.album_keys[0]: album}
status = self.__set_tags(tags)
self.reset_cache()
return status
def set_date_taken(self, time):
"""Set the date/time a photo was taken.
:param datetime time: datetime object of when the photo was taken
:returns: bool
"""
if (time is None):
return False
tags = {}
formatted_time = time.strftime('%Y:%m:%d %H:%M:%S')
for key in self.exif_map['date_taken']:
tags[key] = formatted_time
status = self.__set_tags(tags)
self.reset_cache()
return status
def set_location(self, latitude, longitude):
if (not self.is_valid()):
return None
# The lat/lon _keys array has an order of precedence.
# The first key is writable and we will give the writable
# key precence when reading.
tags = {
self.latitude_keys[0]: latitude,
self.longitude_keys[0]: longitude,
}
# If self.set_gps_ref == True then it means we are writing an EXIF
# GPS tag which requires us to set the reference key.
# That's because the lat/lon are absolute values.
if self.set_gps_ref:
if latitude < 0:
tags[self.latitude_ref_key] = 'S'
if longitude < 0:
tags[self.longitude_ref_key] = 'W'
status = self.__set_tags(tags)
self.reset_cache()
return status
def set_original_name(self, name=None):
"""Sets the original name EXIF tag if not already set.
:returns: True, False, None
"""
if (not self.is_valid()):
return None
# If EXIF original name tag is set then we return.
if self.get_original_name() is not None:
return None
source = self.source
if not name:
name = os.path.basename(source)
tags = {self.original_name_key: name}
status = self.__set_tags(tags)
self.reset_cache()
return status
def set_title(self, title):
"""Set title for a photo.
:param str title: Title of the photo.
:returns: bool
"""
if (not self.is_valid()):
return None
if (title is None):
return None
tags = {self.title_key: title}
status = self.__set_tags(tags)
self.reset_cache()
return status
def __set_tags(self, tags):
if (not self.is_valid()):
return None
source = self.source
status = ''
status = self.exiftool.set_tags(tags, source)
return status != ''
def _import(source,
config_path,
manifest_path,
allow_duplicates,
dryrun,
debug,
move=False,
indent_manifest=False,
no_overwrite_manifest=False):
"""Import files or directories by reading their EXIF and organizing them accordingly.
"""
start_time = round(time.time())
constants.debug = debug
has_errors = False
result = Result()
# Load the configuration from the json file.
config = Config().load_from_file(config_path)
source = config["sources"][0] # For now, only one.
target = config["targets"][
0] # For now, only one target allowed...but data structure allows more
source_file_path = source["file_path"]
manifest = Manifest()
if manifest_path is not None:
manifest.load_from_file(manifest_path)
log_base_path, _ = os.path.split(manifest.file_path)
FILESYSTEM.create_directory(os.path.join(log_base_path, '.elodie'))
log_path = os.path.join(log_base_path, '.elodie',
'import_{}.log'.format(utility.timestamp_string()))
def signal_handler(sig, frame):
log.warn('[ ] Import cancelled')
log.write(log_path)
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
original_manifest_key_count = len(manifest)
# destination = _decode(destination)
# destination = os.path.abspath(os.path.expanduser(destination))
exiftool_addedargs = [
# '-overwrite_original',
u'-config',
u'"{}"'.format(constants.exiftool_config)
]
file_generator = FILESYSTEM.get_all_files(source_file_path, None)
source_file_count = 0
with ExifTool(addedargs=exiftool_addedargs) as et:
while True:
file_batch = list(
itertools.islice(file_generator,
constants.exiftool_batch_size))
if len(file_batch) == 0: break
# This will cause slight discrepancies in file counts: since elodie.json is counted but not imported,
# each one will set the count off by one.
source_file_count += len(file_batch)
metadata_list = et.get_metadata_batch(file_batch)
if not metadata_list:
raise Exception("Metadata scrape failed.")
# Key on the filename to make for easy access,
metadata_dict = dict((os.path.abspath(el["SourceFile"]), el)
for el in metadata_list)
for current_file in file_batch:
# Don't import localized config files.
if current_file.endswith(
"elodie.json"): # Faster than a os.path.split
continue
try:
result = import_file(current_file,
config,
manifest,
metadata_dict,
move=move,
dryrun=dryrun,
allow_duplicates=allow_duplicates)
except Exception as e:
log.warn("[!] Error importing {}: {}".format(
current_file, e))
result = False
has_errors = has_errors or not result
exiftool_waiting_time = et.waiting_time
manifest.write(indent=indent_manifest,
overwrite=(not no_overwrite_manifest))
manifest_key_count = len(manifest)
try:
total_time = round(time.time() - start_time)
log.info("Statistics:")
log.info("Source: File Count {}".format(source_file_count))
log.info("Manifest: New Hashes {}".format(manifest_key_count -
original_manifest_key_count))
log.info("Manifest: Total Hashes {}".format(manifest_key_count))
log.info("Time: Total {}s".format(total_time))
log.info("Time: Files/sec {}".format(
round(source_file_count / total_time)))
log.info("Time: Waiting on ExifTool {}s".format(
round(exiftool_waiting_time)))
except Exception as e:
log.error("[!] Error generating statistics: {}".format(e))
log.write(log_path)
if has_errors:
sys.exit(1)
def __del__(self):
ExifTool().terminate
else:
has_errors = False
result.append((current_file, False))
result.write()
if has_errors:
sys.exit(1)
@click.group()
def main():
pass
main.add_command(_import)
main.add_command(_update)
main.add_command(_generate_db)
main.add_command(_verify)
main.add_command(_batch)
if __name__ == '__main__':
#Initialize ExifTool Subprocess
exiftool_addedargs = [
u'-config',
u'"{}"'.format(constants.exiftool_config)
]
with ExifTool(executable_=get_exiftool(), addedargs=exiftool_addedargs) as et:
main()