def test_get_folder_path_definition_multi_level_location_decode_unknown():
destination_folder = DestinationFolder()
media = Photo(helper.get_file('plain.jpg'))
_metadata = media.get_metadata()
# use multi_level_location_expected, no need to recalculate
path = destination_folder.get_folder_path(
_metadata, path_parts=multi_level_location_definition_CCCV)
assert path == os.path.join('Unknown Location', '2015', 'December'), path
def test_get_folder_path_definition_multi_level_location_decode_known_city():
destination_folder = DestinationFolder()
media = Photo(helper.get_file('with-location.jpg'))
_metadata = media.get_metadata()
_metadata['latitude'] = 54.9286804166667
_metadata['longitude'] = -2.94800427777778 # see mock_location_db_json_txt
# use multi_level_location_expected, no need to recalculate
path = destination_folder.get_folder_path(
_metadata, path_parts=multi_level_location_definition_CCCV)
assert path == os.path.join('UK', 'Carlisle', '2015', 'December'), path
def test_get_folder_path_definition_default():
if hasattr(load_config, 'config'):
del load_config.config
destination_folder = DestinationFolder()
path_definition = destination_folder.get_folder_path_definition()
if hasattr(load_config, 'config'):
del load_config.config
assert path_definition == [[('date', '%Y-%m-%b')],
[('album', ''), ('location', '%city'),
('"Unknown Location"', '')]], path_definition
def test_get_folder_path_definition_multi_level_location_decode_known_village(
):
destination_folder = DestinationFolder()
media = Photo(helper.get_file('with-location.jpg'))
_metadata = media.get_metadata()
_metadata['latitude'] = 55.92352675
_metadata['longitude'] = -5.15257691666667 # see mock_location_db_json_txt
# use multi_level_location_expected, no need to recalculate
path = destination_folder.get_folder_path(
_metadata, path_parts=multi_level_location_definition_CCCV)
assert path == os.path.join('UK', 'Colintraive', '2015', 'December'), path
def test_get_folder_path_definition_multi_level_location_decode_known_county():
destination_folder = DestinationFolder()
media = Photo(helper.get_file('with-location.jpg'))
_metadata = media.get_metadata()
_metadata['latitude'] = 55.4874000277778
_metadata['longitude'] = -3.290884
# use multi_level_location_expected, no need to recalculate
path = destination_folder.get_folder_path(
_metadata, path_parts=multi_level_location_definition_CCCV)
assert path == os.path.join('UK', 'Scottish Borders', '2015',
'December'), path
def test_get_folder_path_with_int_in_source_path():
# gh-239
destination_folder = DestinationFolder()
temporary_folder, folder = helper.create_working_folder('int')
origin = os.path.join(folder, 'plain.jpg')
shutil.copyfile(helper.get_file('plain.jpg'), origin)
media = Photo(origin)
path = destination_folder.get_folder_path(media.get_metadata())
assert path == os.path.join('2015-12-Dec', 'Unknown Location'), path
def test_get_folder_path_with_original_default_unknown_location():
with open(
'%s/config.ini-original-default-with-unknown-location' %
gettempdir(), 'w') as f:
f.write('')
if hasattr(load_config, 'config'):
del load_config.config
destination_folder = DestinationFolder()
media = Photo(helper.get_file('plain.jpg'))
path = destination_folder.get_folder_path(media.get_metadata())
if hasattr(load_config, 'config'):
del load_config.config
assert path == os.path.join('2015-12-Dec', 'Unknown Location'), path
def test_get_folder_path_definition_multi_level_location_definition():
with open('%s/config.ini-multi_level_location' % gettempdir(), 'w') as f:
f.write(multi_level_location_config_json)
mock_location_db(mock_location_db_json_txt)
if hasattr(load_config, 'config'):
del load_config.config
destination_folder = DestinationFolder()
path_definition = destination_folder.get_folder_path_definition()
if hasattr(load_config, 'config'):
del load_config.config
assert path_definition == multi_level_location_definition_CCCV, path_definition
def test_parse_folder_name_multiple_keys_not_found():
if hasattr(load_config, 'config'):
del load_config.config
destination_folder = DestinationFolder()
place_name = {
'default': u'United States of America',
'country': u'United States of America'
}
mask = '%city-%state'
location_parts = re.findall('(%[^%]+)', mask)
path = destination_folder.parse_mask_for_location(mask, location_parts,
place_name)
if hasattr(load_config, 'config'):
del load_config.config
assert path == 'United States of America', path
def test_get_folder_path_with_camera_make_and_model():
with open(
'%s/config.ini-original-with-camera-make-and-model' % gettempdir(),
'w') as f:
f.write("""
[Directory]
full_path=%camera_make/%camera_model
""")
if hasattr(load_config, 'config'):
del load_config.config
destination_folder = DestinationFolder()
media = Photo(helper.get_file('plain.jpg'))
path = destination_folder.get_folder_path(media.get_metadata())
if hasattr(load_config, 'config'):
del load_config.config
assert path == os.path.join('Canon', 'Canon EOS REBEL T2i'), path
def test_parse_folder_name_key_not_found_with_static_chars():
if hasattr(load_config, 'config'):
del load_config.config
destination_folder = DestinationFolder()
place_name = {
'default': u'California',
'country': u'United States of America',
'state': u'California'
}
mask = '%city-is-not-found'
location_parts = re.findall('(%[^%]+)', mask)
path = destination_folder.parse_mask_for_location(mask, location_parts,
place_name)
if hasattr(load_config, 'config'):
del load_config.config
assert path == 'California', path
def test_get_folder_path_with_camera_make_and_model_fallback():
with open(
'%s/config.ini-original-with-camera-make-and-model-fallback' %
gettempdir(), 'w') as f:
f.write("""
[Directory]
full_path=%camera_make|"nomake"/%camera_model|"nomodel"
""")
if hasattr(load_config, 'config'):
del load_config.config
destination_folder = DestinationFolder()
media = Photo(helper.get_file('no-exif.jpg'))
path = destination_folder.get_folder_path(media.get_metadata())
if hasattr(load_config, 'config'):
del load_config.config
assert path == os.path.join('nomake', 'nomodel'), path
def test_get_folder_path_definition_with_only_one_level():
with open('%s/config.ini-location-date' % gettempdir(), 'w') as f:
f.write("""
[Directory]
year=%Y
full_path=%year
""")
if hasattr(load_config, 'config'):
del load_config.config
destination_folder = DestinationFolder()
path_definition = destination_folder.get_folder_path_definition()
expected = [[('year', '%Y')]]
if hasattr(load_config, 'config'):
del load_config.config
assert path_definition == expected, path_definition
def test_get_folder_path_with_int_in_config_component():
# gh-239
with open('%s/config.ini-int-in-component-path' % gettempdir(), 'w') as f:
f.write("""
[Directory]
date=%Y
full_path=%date
""")
if hasattr(load_config, 'config'):
del load_config.config
destination_folder = DestinationFolder()
media = Photo(helper.get_file('plain.jpg'))
path = destination_folder.get_folder_path(media.get_metadata())
if hasattr(load_config, 'config'):
del load_config.config
assert path == os.path.join('2015'), path
def test_parse_folder_name_multiple():
if hasattr(load_config, 'config'):
del load_config.config
destination_folder = DestinationFolder()
place_name = {
'default': u'California',
'country': u'United States of America',
'state': u'California',
'city': u'Sunnyvale'
}
mask = '%city-%state-%country'
location_parts = re.findall('(%[^%]+)', mask)
path = destination_folder.parse_mask_for_location(mask, location_parts,
place_name)
if hasattr(load_config, 'config'):
del load_config.config
assert path == 'Sunnyvale-California-United States of America', path
def test_get_folder_path_with_with_only_one_level():
with open('%s/config.ini-location-date' % gettempdir(), 'w') as f:
f.write("""
[Directory]
year=%Y
full_path=%year
""")
if hasattr(load_config, 'config'):
del load_config.config
destination_folder = DestinationFolder()
media = Photo(helper.get_file('plain.jpg'))
path = destination_folder.get_folder_path(media.get_metadata())
if hasattr(load_config, 'config'):
del load_config.config
assert path == os.path.join('2015'), path
def test_get_folder_path_definition_location_date():
with open('%s/config.ini-location-date' % gettempdir(), 'w') as f:
f.write("""
[Directory]
date=%Y-%m-%d
location=%country
full_path=%location/%date
""")
if hasattr(load_config, 'config'):
del load_config.config
destination_folder = DestinationFolder()
path_definition = destination_folder.get_folder_path_definition()
expected = [[('location', '%country')], [('date', '%Y-%m-%d')]]
if hasattr(load_config, 'config'):
del load_config.config
assert path_definition == expected, path_definition
def test_get_folder_path_definition_with_more_than_two_levels():
with open('%s/config.ini-location-date' % gettempdir(), 'w') as f:
f.write("""
[Directory]
year=%Y
month=%m
day=%d
full_path=%year/%month/%day
""")
if hasattr(load_config, 'config'):
del load_config.config
destination_folder = DestinationFolder()
path_definition = destination_folder.get_folder_path_definition()
expected = [[('year', '%Y')], [('month', '%m')], [('day', '%d')]]
if hasattr(load_config, 'config'):
del load_config.config
assert path_definition == expected, path_definition
def test_get_folder_path_with_fallback_folder():
with open('%s/config.ini-fallback' % gettempdir(), 'w') as f:
f.write("""
[Directory]
year=%Y
month=%m
full_path=%year/%month/%album|%"No Album Fool"/%month
""")
#full_path=%year/%album|"No Album"
if hasattr(load_config, 'config'):
del load_config.config
destination_folder = DestinationFolder()
media = Photo(helper.get_file('plain.jpg'))
path = destination_folder.get_folder_path(media.get_metadata())
if hasattr(load_config, 'config'):
del load_config.config
assert path == os.path.join('2015', '12', 'No Album Fool', '12'), path
def test_get_folder_path_with_custom_path():
with open('%s/config.ini-custom-path' % gettempdir(), 'w') as f:
f.write("""
[MapQuest]
key=czjNKTtFjLydLteUBwdgKAIC8OAbGLUx
[Directory]
date=%Y-%m-%d
location=%country-%state-%city
full_path=%date/%location
""")
if hasattr(load_config, 'config'):
del load_config.config
destination_folder = DestinationFolder()
media = Photo(helper.get_file('with-location.jpg'))
path = destination_folder.get_folder_path(media.get_metadata())
if hasattr(load_config, 'config'):
del load_config.config
assert path == os.path.join(
'2015-12-05', 'United States of America-California-Sunnyvale'), path
def test_get_folder_path_definition_multi_level_custom():
with open('%s/config.ini-multi-level-custom' % gettempdir(), 'w') as f:
f.write("""
[Directory]
year=%Y
month=%M
full_path=%year/%album|%month|%"foo"/%month
""")
if hasattr(load_config, 'config'):
del load_config.config
destination_folder = DestinationFolder()
path_definition = destination_folder.get_folder_path_definition()
expected = [[('year', '%Y')],
[('album', ''), ('month', '%M'), ('"foo"', '')],
[('month', '%M')]]
if hasattr(load_config, 'config'):
del load_config.config
assert path_definition == expected, path_definition
def test_get_folder_path_with_with_more_than_two_levels():
with open('%s/config.ini-location-date' % gettempdir(), 'w') as f:
f.write("""
[MapQuest]
key=czjNKTtFjLydLteUBwdgKAIC8OAbGLUx
[Directory]
year=%Y
month=%m
location=%city, %state
full_path=%year/%month/%location
""")
if hasattr(load_config, 'config'):
del load_config.config
destination_folder = DestinationFolder()
media = Photo(helper.get_file('with-location.jpg'))
path = destination_folder.get_folder_path(media.get_metadata())
if hasattr(load_config, 'config'):
del load_config.config
assert path == os.path.join('2015', '12', 'Sunnyvale, California'), path
class FileSystem(object):
"""A class for interacting with the file system."""
def __init__(self):
self.destination_folder = DestinationFolder()
# Instantiate a plugins object
self.plugins = Plugins()
def create_directory(self, directory_path):
"""Create a directory if it does not already exist.
:param str directory_name: A fully qualified path of the
to create.
:returns: bool
"""
try:
if os.path.exists(directory_path):
return True
else:
os.makedirs(directory_path)
return True
except OSError:
# OSError is thrown for cases like no permission
pass
return False
def delete_directory_if_empty(self, directory_path):
"""Delete a directory only if it's empty.
Instead of checking first using `len([name for name in
os.listdir(directory_path)]) == 0`, we catch the OSError exception.
:param str directory_name: A fully qualified path of the directory
to delete.
"""
try:
os.rmdir(directory_path)
return True
except OSError:
pass
return False
def get_all_files(self, path, extensions=None, exclude_regex_list=set()):
"""Recursively get all files which match a path and extension.
:param str path string: Path to start recursive file listing
:param tuple(str) extensions: File extensions to include (whitelist)
:returns: generator
"""
# If extensions is None then we get all supported extensions
if not extensions:
extensions = set()
subclasses = get_all_subclasses(Base)
for cls in subclasses:
extensions.update(cls.extensions)
# Create a list of compiled regular expressions to match against the file path
compiled_regex_list = [re.compile(regex) for regex in exclude_regex_list]
for dirname, dirnames, filenames in os.walk(path):
for filename in filenames:
# If file extension is in `extensions`
# And if file path is not in exclude regexes
# Then append to the list
filename_path = os.path.join(dirname, filename)
if (
os.path.splitext(filename)[1][1:].lower() in extensions and
not self.should_exclude(filename_path, compiled_regex_list, False)
):
yield filename_path
def get_current_directory(self):
"""Get the current working directory.
:returns: str
"""
return os.getcwd()
def get_file_name(self, metadata):
"""Generate file name for a photo or video using its metadata.
Originally we hardcoded the file name to include an ISO date format.
We use an ISO8601-like format for the file name prefix. Instead of
colons as the separator for hours, minutes and seconds we use a hyphen.
https://en.wikipedia.org/wiki/ISO_8601#General_principles
PR #225 made the file name customizable and fixed issues #107 #110 #111.
https://github.com/jmathai/elodie/pull/225
:param media: A Photo or Video instance
:type media: :class:`~elodie.media.photo.Photo` or
:class:`~elodie.media.video.Video`
:returns: str or None for non-photo or non-videos
"""
if(metadata is None):
return None
# Get the name template and definition.
# Name template is in the form %date-%original_name-%title.%extension
# Definition is in the form
# [
# [('date', '%Y-%m-%d_%H-%M-%S')],
# [('original_name', '')], [('title', '')], // contains a fallback
# [('extension', '')]
# ]
name_template, definition = self.get_file_name_definition()
name = name_template
for parts in definition:
this_value = None
for this_part in parts:
part, mask = this_part
if part in ('date', 'day', 'month', 'year'):
this_value = time.strftime(mask, metadata['date_taken'])
break
elif part in ('location', 'city', 'state', 'country'):
place_name = geolocation.place_name(
metadata['latitude'],
metadata['longitude']
)
location_parts = re.findall('(%[^%]+)', mask)
this_value = self.parse_mask_for_location(
mask,
location_parts,
place_name,
)
break
elif part in ('album', 'extension', 'title'):
if metadata[part]:
this_value = re.sub(self.whitespace_regex, '-', metadata[part].strip())
break
elif part in ('original_name'):
# First we check if we have metadata['original_name'].
# We have to do this for backwards compatibility because
# we original did not store this back into EXIF.
if metadata[part]:
this_value = os.path.splitext(metadata['original_name'])[0]
else:
# We didn't always store original_name so this is
# for backwards compatibility.
# We want to remove the hardcoded date prefix we used
# to add to the name.
# This helps when re-running the program on file
# which were already processed.
this_value = re.sub(
'^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}-',
'',
metadata['base_name']
)
if(len(this_value) == 0):
this_value = metadata['base_name']
# Lastly we want to sanitize the name
this_value = re.sub(self.whitespace_regex, '-', this_value.strip())
elif part.startswith('"') and part.endswith('"'):
this_value = part[1:-1]
break
# Here we replace the placeholder with it's corresponding value.
# Check if this_value was not set so that the placeholder
# can be removed completely.
# For example, %title- will be replaced with ''
# Else replace the placeholder (i.e. %title) with the value.
if this_value is None:
name = re.sub(
#'[^a-z_]+%{}'.format(part),
'[^a-zA-Z0-9_]+%{}'.format(part),
'',
name,
)
else:
name = re.sub(
'%{}'.format(part),
this_value,
name,
)
config = load_config()
if('File' in config and 'capitalization' in config['File'] and config['File']['capitalization'] == 'upper'):
return name.upper()
else:
return name.lower()
def get_file_name_definition(self):
"""Returns a list of folder definitions.
Each element in the list represents a folder.
Fallback folders are supported and are nested lists.
Return values take the following form.
[
('date', '%Y-%m-%d'),
[
('location', '%city'),
('album', ''),
('"Unknown Location", '')
]
]
:returns: list
"""
# If we've done this already then return it immediately without
# incurring any extra work
if self.cached_file_name_definition is not None:
return self.cached_file_name_definition
config = load_config()
# If File is in the config we assume name and its
# corresponding values are also present
config_file = self.default_file_name_definition
if('File' in config):
config_file = config['File']
# Find all subpatterns of name that map to the components of the file's
# name.
# I.e. %date-%original_name-%title.%extension => ['date', 'original_name', 'title', 'extension'] #noqa
path_parts = re.findall(
'(\%[a-z_]+)',
config_file['name']
)
if not path_parts or len(path_parts) == 0:
return (config_file['name'], self.default_file_name_definition)
self.cached_file_name_definition = []
for part in path_parts:
if part in config_file:
part = part[1:]
self.cached_file_name_definition.append(
[(part, config_file[part])]
)
else:
this_part = []
for p in part.split('|'):
p = p[1:]
this_part.append(
(p, config_file[p] if p in config_file else '')
)
self.cached_file_name_definition.append(this_part)
# What if basename already has date_time information?
# e.g. 20160706_192934.jpg
# 20160706_192934-1.jpg
# strip it out
base_name = re.sub(
'^\d{4}\d{2}\d{2}_\d{2}\d{2}\d{2}',
'',
base_name
)
file_name = '%s-%s.%s' % (
time.strftime(
'%Y-%m-%d_%H-%M-%S',
metadata['date_taken']
),
base_name,
metadata['extension'])
return file_name.lower()
def process_file(self, _file, destination, media, **kwargs):
move = False
if('move' in kwargs):
move = kwargs['move']
allow_duplicate = False
if('allowDuplicate' in kwargs):
allow_duplicate = kwargs['allowDuplicate']
if(not media.is_valid()):
print('%s is not a valid media file. Skipping...' % _file)
return
media.set_original_name()
metadata = media.get_metadata()
directory_name = self.destination_folder.get_folder_path(metadata)
dest_directory = os.path.join(destination, directory_name)
file_name = self.get_file_name(media)
dest_path = os.path.join(dest_directory, file_name)
db = Db()
checksum = db.checksum(_file)
if(checksum is None):
log.info('Could not get checksum for %s.' % _file)
return None
# If duplicates are not allowed then we check if we've seen this file
# before via checksum. We also check that the file exists at the
# location we believe it to be.
# If we find a checksum match but the file doesn't exist where we
# believe it to be then we write a debug log and proceed to import.
checksum_file = db.get_hash(checksum)
if(allow_duplicate is False and checksum_file is not None):
if(os.path.isfile(checksum_file)):
log.info('%s already at %s.' % (
_file,
checksum_file
))
return None
else:
log.info('%s matched checksum but file not found at %s.' % ( # noqa
_file,
checksum_file
))
return checksum
def process_file(self, _file, destination, media, **kwargs):
move = False
if('move' in kwargs):
move = kwargs['move']
allow_duplicate = False
if('allowDuplicate' in kwargs):
allow_duplicate = kwargs['allowDuplicate']
stat_info_original = os.stat(_file)
metadata = media.get_metadata()
if(not media.is_valid()):
print('%s is not a valid media file. Skipping...' % _file)
return
checksum = self.process_checksum(_file, allow_duplicate)
if(checksum is None):
log.info('Original checksum returned None for %s. Skipping...' %
_file)
return
# Run `before()` for every loaded plugin and if any of them raise an exception
# then we skip importing the file and log a message.
plugins_run_before_status = self.plugins.run_all_before(_file, destination)
if(plugins_run_before_status == False):
log.warn('At least one plugin pre-run failed for %s' % _file)
return
directory_name = self.get_folder_path(metadata)
dest_directory = os.path.join(destination, directory_name)
file_name = self.get_file_name(metadata)
dest_path = os.path.join(dest_directory, file_name)
media.set_original_name()
# If source and destination are identical then
# we should not write the file. gh-210
if(_file == dest_path):
print('Final source and destination path should not be identical')
return
self.create_directory(dest_directory)
# exiftool renames the original file by appending '_original' to the
# file name. A new file is written with new tags with the initial file
# name. See exiftool man page for more details.
exif_original_file = _file + '_original'
# Check if the source file was processed by exiftool and an _original
# file was created.
exif_original_file_exists = False
if(os.path.exists(exif_original_file)):
exif_original_file_exists = True
if(move is True):
stat = os.stat(_file)
# Move the processed file into the destination directory
shutil.move(_file, dest_path)
if(exif_original_file_exists is True):
# We can remove it as we don't need the initial file.
os.remove(exif_original_file)
os.utime(dest_path, (stat.st_atime, stat.st_mtime))
else:
compatibility._copyfile(_file, dest_path)
self.set_utime_from_metadata(media.get_metadata(), dest_path)
db = Db()
db.add_hash(checksum, dest_path)
db.update_hash_db()
# Run `after()` for every loaded plugin and if any of them raise an exception
# then we skip importing the file and log a message.
plugins_run_after_status = self.plugins.run_all_after(_file, destination, dest_path, metadata)
if(plugins_run_after_status == False):
log.warn('At least one plugin pre-run failed for %s' % _file)
return
return dest_path
def set_utime_from_metadata(self, metadata, file_path):
""" Set the modification time on the file based on the file name.
"""
# Initialize date taken to what's returned from the metadata function.
# If the folder and file name follow a time format of
# YYYY-MM-DD_HH-MM-SS-IMG_0001.JPG then we override the date_taken
date_taken = metadata['date_taken']
base_name = metadata['base_name']
year_month_day_match = re.search(
'^(\d{4})-(\d{2})-(\d{2})_(\d{2})-(\d{2})-(\d{2})',
base_name
)
if(year_month_day_match is not None):
(year, month, day, hour, minute, second) = year_month_day_match.groups() # noqa
date_taken = time.strptime(
'{}-{}-{} {}:{}:{}'.format(year, month, day, hour, minute, second), # noqa
'%Y-%m-%d %H:%M:%S'
)
os.utime(file_path, (time.time(), time.mktime(date_taken)))
else:
# We don't make any assumptions about time zones and
# assume local time zone.
date_taken_in_seconds = time.mktime(date_taken)
os.utime(file_path, (time.time(), (date_taken_in_seconds)))
def should_exclude(self, path, regex_list=set(), needs_compiled=False):
if(len(regex_list) == 0):
return False
if(needs_compiled):
compiled_list = []
for regex in regex_list:
compiled_list.append(re.compile(regex))
regex_list = compiled_list
return any(regex.search(path) for regex in regex_list)
def test_get_folder_path_with_title():
destination_folder = DestinationFolder()
media = Photo(helper.get_file('with-title.jpg'))
path = destination_folder.get_folder_path(media.get_metadata())
assert path == os.path.join('2015-12-Dec', 'Unknown Location'), path
def __init__(self):
self.destination_folder = DestinationFolder()
# Instantiate a plugins object
self.plugins = Plugins()
def test_get_folder_path_with_location():
destination_folder = DestinationFolder()
media = Photo(helper.get_file('with-location.jpg'))
path = destination_folder.get_folder_path(media.get_metadata())
assert path == os.path.join('2015-12-Dec', 'Sunnyvale'), path