def load(self):
"""Load plugins from config file.
"""
# If plugins have been loaded then return
if self.loaded == True:
return
plugin_list = load_plugin_config()
for plugin in plugin_list:
plugin_lower = plugin.lower()
try:
# We attempt to do the following.
# 1. Load the module of the plugin.
# 2. Instantiate an object of the plugin's class.
# 3. Add the plugin to the list of plugins.
#
# #3 should only happen if #2 doesn't throw an error
this_module = import_module('elodie.plugins.{}.{}'.format(
plugin_lower, plugin_lower))
self.classes[plugin] = getattr(this_module, plugin)()
# We only append to self.plugins if we're able to load the class
self.plugins.append(plugin)
except:
log.error(
'An error occurred initiating plugin {}'.format(plugin))
log.error(format_exc())
self.loaded = True
def _generate_db(source):
"""Regenerate the hash.json database which contains all of the sha1 signatures of media files.
"""
result = Result()
source = os.path.abspath(os.path.expanduser(source))
extensions = set()
all_files = set()
valid_files = set()
if not os.path.isdir(source):
log.error('Source is not a valid directory %s' % source)
sys.exit(1)
subclasses = get_all_subclasses(Base)
for cls in subclasses:
extensions.update(cls.extensions)
all_files.update(FILESYSTEM.get_all_files(source, None))
db = Db()
db.backup_hash_db()
db.reset_hash_db()
for current_file in all_files:
if os.path.splitext(current_file)[1][1:].lower() not in extensions:
log.info('Skipping invalid file %s' % current_file)
result.append((current_file, False))
continue
result.append((current_file, True))
db.add_hash(db.checksum(current_file), current_file)
db.update_hash_db()
result.write()
def update_location(media, file_path, location_name):
"""Update location exif metadata of media.
"""
location_coords = geolocation.coordinates_by_name(location_name)
if location_coords and 'latitude' in location_coords and \
'longitude' in location_coords:
location_status = media.set_location(location_coords[
'latitude'], location_coords['longitude'])
if not location_status:
log.error('Failed to update location')
log.all(('{"source":"%s",' % file_path,
'"error_msg":"Failed to update location"}'))
sys.exit(1)
return True
def update_time(media, file_path, time_string):
"""Update time exif metadata of media.
"""
time_format = '%Y-%m-%d %H:%M:%S'
if re.match(r'^\d{4}-\d{2}-\d{2}$', time_string):
time_string = '%s 00:00:00' % time_string
elif re.match(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}\d{2}$', time_string):
msg = ('Invalid time format. Use YYYY-mm-dd hh:ii:ss or YYYY-mm-dd')
log.error(msg)
log.all('{"source":"%s", "error_msg":"%s"}' % (file_path, msg))
sys.exit(1)
time = datetime.strptime(time_string, time_format)
media.set_date_taken(time)
return True
def update_time(media, file_path, time_string):
"""Update time exif metadata of media.
"""
time_format = '%Y-%m-%d %H:%M:%S'
if re.match(r'^\d{4}-\d{2}-\d{2}$', time_string):
time_string = '%s 00:00:00' % time_string
elif re.match(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}\d{2}$', time_string):
msg = ('Invalid time format. Use YYYY-mm-dd hh:ii:ss or YYYY-mm-dd')
log.error(msg)
print('{"source":"%s", "error_msg":"%s"}' % (file_path, msg))
sys.exit(1)
time = datetime.strptime(time_string, time_format)
media.set_date_taken(time)
return True
def update_location(media, file_path, location_name):
"""Update location exif metadata of media.
"""
location_coords = geolocation.coordinates_by_name(location_name)
if location_coords and 'latitude' in location_coords and \
'longitude' in location_coords:
location_status = media.set_location(location_coords[
'latitude'], location_coords['longitude'])
if not location_status:
log.error('Failed to update location')
print(('{"source":"%s",' % file_path,
'"error_msg":"Failed to update location"}'))
sys.exit(1)
return True
def get_date_taken(self):
"""Get the date which the photo was taken.
The date value returned is defined by the min() of mtime and ctime.
:returns: time object or None for non-photo files or 0 timestamp
"""
if (not self.is_valid()):
return None
source = self.source
seconds_since_epoch = min(os.path.getmtime(source),
os.path.getctime(source)) # noqa
exif = self.get_exiftool_attributes()
if not exif:
return None # seconds_since_epoch
# We need to parse a string from EXIF into a timestamp.
# EXIF DateTimeOriginal and EXIF DateTime are both stored
# in %Y:%m:%d %H:%M:%S format
# we split on a space and then r':|-' -> convert to int -> .timetuple()
# the conversion in the local timezone
# EXIF DateTime is already stored as a timestamp
# Sourced from https://github.com/photo/frontend/blob/master/src/libraries/models/Photo.php#L500 # noqa
for key in self.exif_map['date_taken']:
try:
if (key in exif):
if (re.match('\d{4}(-|:)\d{2}(-|:)\d{2}', exif[key])
is not None): # noqa
dt, tm = exif[key].split(' ')
dt_list = compile(r'-|:').split(dt)
dt_list = dt_list + compile(r'-|:').split(tm)
dt_list = map(int, dt_list)
time_tuple = datetime(*dt_list).timetuple()
seconds_since_epoch = time.mktime(time_tuple)
break
# None of the date-taken keys were present
return None
except BaseException as e:
log.error(e)
pass
if (seconds_since_epoch == 0):
return None
return time.gmtime(seconds_since_epoch)
def parse_metadata_line(self):
if isinstance(self.metadata_line, dict):
return self.metadata_line
source = self.source
if source is None:
return None
with open(source, "r") as f:
first_line = f.readline().strip()
try:
parsed_json = loads(first_line)
if isinstance(parsed_json, dict):
self.metadata_line = parsed_json
except ValueError:
log.error("Could not parse JSON from first line: %s" % first_line)
pass
def parse_metadata_line(self):
if isinstance(self.metadata_line, dict):
return self.metadata_line
source = self.source
if source is None:
return None
with open(source, 'r') as f:
first_line = f.readline().strip()
try:
parsed_json = loads(first_line)
if isinstance(parsed_json, dict):
self.metadata_line = parsed_json
except ValueError:
log.error('Could not parse JSON from first line: %s' % first_line)
pass
def get_date_taken(self):
"""Get the date which the photo was taken.
The date value returned is defined by the min() of mtime and ctime.
:returns: time object or None for non-photo files or 0 timestamp
"""
if(not self.is_valid()):
return None
source = self.source
seconds_since_epoch = min(os.path.getmtime(source), os.path.getctime(source)) # noqa
exif = self.get_exiftool_attributes()
if not exif:
return seconds_since_epoch
# We need to parse a string from EXIF into a timestamp.
# EXIF DateTimeOriginal and EXIF DateTime are both stored
# in %Y:%m:%d %H:%M:%S format
# we split on a space and then r':|-' -> convert to int -> .timetuple()
# the conversion in the local timezone
# EXIF DateTime is already stored as a timestamp
# Sourced from https://github.com/photo/frontend/blob/master/src/libraries/models/Photo.php#L500 # noqa
for key in self.exif_map['date_taken']:
try:
if(key in exif):
if(re.match('\d{4}(-|:)\d{2}(-|:)\d{2}', exif[key]) is not None): # noqa
dt, tm = exif[key].split(' ')
dt_list = compile(r'-|:').split(dt)
dt_list = dt_list + compile(r'-|:').split(tm)
dt_list = map(int, dt_list)
time_tuple = datetime(*dt_list).timetuple()
seconds_since_epoch = time.mktime(time_tuple)
break
except BaseException as e:
log.error(e)
pass
if(seconds_since_epoch == 0):
return None
return time.gmtime(seconds_since_epoch)
def run_batch(self):
self.load()
pass_status = True
for cls in self.classes:
this_method = getattr(self.classes[cls], 'batch')
# We try to call the plugin's `before()` method.
# If the method explicitly raises an ElodiePluginError we'll fail the import
# by setting pass_status to False.
# If any other error occurs we log the message and proceed as usual.
# By default, plugins don't change behavior.
try:
this_method()
log.info('Called batch() for {}'.format(cls))
except ElodiePluginError as err:
log.warn('Plugin {} raised an exception in run_batch: {}'.format(cls, err))
log.error(format_exc())
pass_status = False
except:
log.error(format_exc())
return pass_status
def lookup(**kwargs):
if ('location' not in kwargs and 'lat' not in kwargs
and 'lon' not in kwargs):
return None
key = get_key()
prefer_english_names = get_prefer_english_names()
if (key is None):
return None
try:
params = {'format': 'json', 'key': key}
params.update(kwargs)
path = '/geocoding/v1/address'
if ('lat' in kwargs and 'lon' in kwargs):
path = '/nominatim/v1/reverse.php'
url = '%s%s?%s' % (constants.mapquest_base_url, path,
urllib.parse.urlencode(params))
headers = {}
if (prefer_english_names):
headers = {'Accept-Language': 'en-EN,en;q=0.8'}
r = requests.get(url, headers=headers)
return parse_result(r.json())
except requests.exceptions.RequestException as e:
log.error(e)
return None
except ValueError as e:
log.error(r.text)
log.error(e)
return None
def lookup(**kwargs):
if(
'location' not in kwargs and
'lat' not in kwargs and
'lon' not in kwargs
):
return None
key = get_key()
if(key is None):
return None
try:
params = {'format': 'json', 'key': key}
params.update(kwargs)
path = '/geocoding/v1/address'
if('lat' in kwargs and 'lon' in kwargs):
path = '/nominatim/v1/reverse.php'
url = 'http://open.mapquestapi.com%s?%s' % (
path,
urllib.parse.urlencode(params)
)
r = requests.get(url)
return parse_result(r.json())
except requests.exceptions.RequestException as e:
log.error(e)
return None
except ValueError as e:
log.error(r.text)
log.error(e)
return None
def lookup(**kwargs):
if(
'location' not in kwargs and
'lat' not in kwargs and
'lon' not in kwargs
):
return None
key = get_key()
if(key is None):
return None
try:
params = {'format': 'json', 'accept-language': constants.accepted_language, 'key': key}
params.update(kwargs)
path = '/geocoding/v1/address'
if('lat' in kwargs and 'lon' in kwargs):
path = '/nominatim/v1/reverse.php'
url = 'http://open.mapquestapi.com%s?%s' % (
path,
urllib.parse.urlencode(params)
)
r = requests.get(url)
return parse_result(r.json())
except requests.exceptions.RequestException as e:
log.error(e)
return None
except ValueError as e:
log.error(r.text)
log.error(e)
return None
def run_all_after(self, file_path, destination_folder, final_file_path,
metadata):
"""Process `before` methods of each plugin that was loaded.
"""
self.load()
pass_status = True
for cls in self.classes:
this_method = getattr(self.classes[cls], 'after')
# We try to call the plugin's `before()` method.
# If the method explicitly raises an ElodiePluginError we'll fail the import
# by setting pass_status to False.
# If any other error occurs we log the message and proceed as usual.
# By default, plugins don't change behavior.
try:
this_method(file_path, destination_folder, final_file_path,
metadata)
log.info('Called after() for {}'.format(cls))
except ElodiePluginError as err:
log.warn('Plugin {} raised an exception in run_all_before: {}'.
format(cls, err))
log.error(format_exc())
log.error('false')
pass_status = False
except:
log.error(format_exc())
return pass_status
def _generate_db(source):
"""Regenerate the hash.json database which contains all of the sha1 signatures of media files.
"""
result = Result()
source = os.path.abspath(os.path.expanduser(source))
if not os.path.isdir(source):
log.error('Source is not a valid directory %s' % source)
sys.exit(1)
db = Db()
db.backup_hash_db()
db.reset_hash_db()
for current_file in FILESYSTEM.get_all_files(source):
result.append((current_file, True))
db.add_hash(db.checksum(current_file), current_file)
log.progress()
db.update_hash_db()
log.progress('', True)
result.write()
def _import(source,
config_path,
manifest_path,
allow_duplicates,
dryrun,
debug,
move=False,
indent_manifest=False,
no_overwrite_manifest=False):
"""Import files or directories by reading their EXIF and organizing them accordingly.
"""
start_time = round(time.time())
constants.debug = debug
has_errors = False
result = Result()
# Load the configuration from the json file.
config = Config().load_from_file(config_path)
source = config["sources"][0] # For now, only one.
target = config["targets"][
0] # For now, only one target allowed...but data structure allows more
source_file_path = source["file_path"]
manifest = Manifest()
if manifest_path is not None:
manifest.load_from_file(manifest_path)
log_base_path, _ = os.path.split(manifest.file_path)
FILESYSTEM.create_directory(os.path.join(log_base_path, '.elodie'))
log_path = os.path.join(log_base_path, '.elodie',
'import_{}.log'.format(utility.timestamp_string()))
def signal_handler(sig, frame):
log.warn('[ ] Import cancelled')
log.write(log_path)
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
original_manifest_key_count = len(manifest)
# destination = _decode(destination)
# destination = os.path.abspath(os.path.expanduser(destination))
exiftool_addedargs = [
# '-overwrite_original',
u'-config',
u'"{}"'.format(constants.exiftool_config)
]
file_generator = FILESYSTEM.get_all_files(source_file_path, None)
source_file_count = 0
with ExifTool(addedargs=exiftool_addedargs) as et:
while True:
file_batch = list(
itertools.islice(file_generator,
constants.exiftool_batch_size))
if len(file_batch) == 0: break
# This will cause slight discrepancies in file counts: since elodie.json is counted but not imported,
# each one will set the count off by one.
source_file_count += len(file_batch)
metadata_list = et.get_metadata_batch(file_batch)
if not metadata_list:
raise Exception("Metadata scrape failed.")
# Key on the filename to make for easy access,
metadata_dict = dict((os.path.abspath(el["SourceFile"]), el)
for el in metadata_list)
for current_file in file_batch:
# Don't import localized config files.
if current_file.endswith(
"elodie.json"): # Faster than a os.path.split
continue
try:
result = import_file(current_file,
config,
manifest,
metadata_dict,
move=move,
dryrun=dryrun,
allow_duplicates=allow_duplicates)
except Exception as e:
log.warn("[!] Error importing {}: {}".format(
current_file, e))
result = False
has_errors = has_errors or not result
exiftool_waiting_time = et.waiting_time
manifest.write(indent=indent_manifest,
overwrite=(not no_overwrite_manifest))
manifest_key_count = len(manifest)
try:
total_time = round(time.time() - start_time)
log.info("Statistics:")
log.info("Source: File Count {}".format(source_file_count))
log.info("Manifest: New Hashes {}".format(manifest_key_count -
original_manifest_key_count))
log.info("Manifest: Total Hashes {}".format(manifest_key_count))
log.info("Time: Total {}s".format(total_time))
log.info("Time: Files/sec {}".format(
round(source_file_count / total_time)))
log.info("Time: Waiting on ExifTool {}s".format(
round(exiftool_waiting_time)))
except Exception as e:
log.error("[!] Error generating statistics: {}".format(e))
log.write(log_path)
if has_errors:
sys.exit(1)
def _update(album, location, time, title, paths, debug):
"""Update a file's EXIF. Automatically modifies the file's location and file name accordingly.
"""
constants.debug = debug
has_errors = False
result = Result()
files = set()
for path in paths:
path = os.path.expanduser(path)
if os.path.isdir(path):
files.update(FILESYSTEM.get_all_files(path, None))
else:
files.add(path)
for current_file in files:
if not os.path.exists(current_file):
has_errors = True
result.append((current_file, False))
log.warn('Could not find %s' % current_file)
log.error('{"source":"%s", "error_msg":"Could not find %s"}' % \
(current_file, current_file))
continue
current_file = os.path.expanduser(current_file)
# The destination folder structure could contain any number of levels
# So we calculate that and traverse up the tree.
# '/path/to/file/photo.jpg' -> '/path/to/file' ->
# ['path','to','file'] -> ['path','to'] -> '/path/to'
current_directory = os.path.dirname(current_file)
destination_depth = -1 * len(FILESYSTEM.get_folder_path_definition())
destination = os.sep.join(
os.path.normpath(current_directory).split(
os.sep)[:destination_depth])
media = Media.get_class_by_file(current_file, get_all_subclasses())
if not media:
continue
updated = False
if location:
update_location(media, current_file, location)
updated = True
if time:
update_time(media, current_file, time)
updated = True
if album:
media.set_album(album)
updated = True
# Updating a title can be problematic when doing it 2+ times on a file.
# You would end up with img_001.jpg -> img_001-first-title.jpg ->
# img_001-first-title-second-title.jpg.
# To resolve that we have to track the prior title (if there was one.
# Then we massage the updated_media's metadata['base_name'] to remove
# the old title.
# Since FileSystem.get_file_name() relies on base_name it will properly
# rename the file by updating the title instead of appending it.
remove_old_title_from_name = False
if title:
# We call get_metadata() to cache it before making any changes
metadata = media.get_metadata()
title_update_status = media.set_title(title)
original_title = metadata['title']
if title_update_status and original_title:
# @TODO: We should move this to a shared method since
# FileSystem.get_file_name() does it too.
original_title = re.sub(r'\W+', '-', original_title.lower())
original_base_name = metadata['base_name']
remove_old_title_from_name = True
updated = True
if updated:
updated_media = Media.get_class_by_file(current_file,
get_all_subclasses())
# See comments above on why we have to do this when titles
# get updated.
if remove_old_title_from_name and len(original_title) > 0:
updated_media.get_metadata()
updated_media.set_metadata_basename(
original_base_name.replace('-%s' % original_title, ''))
dest_path = FILESYSTEM.process_file(current_file,
destination,
updated_media,
move=True,
allowDuplicate=True)
log.info(u'%s -> %s' % (current_file, dest_path))
log.info('{"source":"%s", "destination":"%s"}' %
(current_file, dest_path))
# If the folder we moved the file out of or its parent are empty
# we delete it.
FILESYSTEM.delete_directory_if_empty(os.path.dirname(current_file))
FILESYSTEM.delete_directory_if_empty(
os.path.dirname(os.path.dirname(current_file)))
result.append((current_file, dest_path))
# Trip has_errors to False if it's already False or dest_path is.
has_errors = has_errors is True or not dest_path
else:
has_errors = False
result.append((current_file, False))
result.write()
if has_errors:
sys.exit(1)