class LycheeSyncer:
"""
This class contains the logic behind this program
It consist mainly in filesystem operations
It relies on:
- LycheeDAO for dtabases operations
- LycheePhoto to store (and compute) photos propreties
"""
conf = {}
def __init__(self):
"""
Takes a dictionnary of conf as input
"""
borg = ConfBorg()
self.conf = borg.conf
def deleteFiles(self, filelist):
"""
Delete files in the Lychee file tree (uploads/big and uploads/thumbnails)
Give it the file name and it will delete relatives files and thumbnails
Parameters:
- filelist: a list of filenames
Returns nothing
"""
for url in filelist:
if isAPhoto(self, url):
thumbpath = os.path.join(self.conf["lycheepath"], "uploads",
"thumb", url)
filesplit = os.path.splitext(url)
thumb2path = ''.join([filesplit[0], "@2x",
filesplit[1]]).lower()
thumb2path = os.path.join(self.conf["lycheepath"], "uploads",
"thumb", thumb2path)
bigpath = os.path.join(self.conf["lycheepath"], "uploads",
"big", url)
remove_file(thumbpath)
remove_file(thumb2path)
remove_file(bigpath)
def deleteAllFiles(self):
"""
Deletes every photo file in Lychee
Returns nothing
"""
photopath = os.path.join(self.conf["lycheepath"], "uploads", "big")
filelist = [f for f in os.listdir(photopath)]
self.deleteFiles(filelist)
def sync(self):
"""
Program main loop
Scans files to add in the sourcedirectory and add them to Lychee
according to the conf file and given parameters
Returns nothing
"""
# Connect db
# and drop it if dropdb activated
self.dao = LycheeDAO(self.conf)
if self.conf['dropdb']:
self.deleteAllFiles()
# Load db
createdalbums = 0
discoveredphotos = 0
importedphotos = 0
album = {}
albums = []
album_name_max_width = self.dao.getAlbumNameDBWidth()
# walkthroug each file / dir of the srcdir
for root, dirs, files in os.walk(self.conf['srcdir']):
if sys.version_info.major == 2:
try:
root = root.decode('UTF-8')
except Exception as e:
logger.error(e)
# Init album data
album['id'] = None
album['name'] = None
album['path'] = None
album['relpath'] = None # path relative to srcdir
album['parent'] = "0"
album['photos'] = [] # path relative to srcdir
album = getAlbum(self, root)
# if a there is at least one photo in the files
album['path'] = root
# don't know what to do with theses photo
# and don't wan't to create a default album
if album['path'] == self.conf['srcdir']:
msg = "file at srcdir root won't be added to lychee, please move them in a subfolder: {}".format(
root)
logger.warn(msg)
continue
album['id'] = self.dao.albumExistsByNameAndParent(
album['name'], album['parent'])
if self.conf['replace'] and album['id']:
# drop album photos
filelist = self.dao.eraseAlbum(album['id'])
self.deleteFiles(filelist)
assert self.dao.dropAlbum(album['id'])
# Album should be recreated
album['id'] = False
if not (album['id']):
# create album
album['id'] = createAlbum(self, album)
if not (album['id']):
logger.error("didn't manage to create album for: " +
album['name'])
continue
else:
logger.info("############ Album created: %s",
album['name'])
createdalbums += 1
# Albums are created or emptied, now take care of photos
for f in sorted(files):
if isAPhoto(self, f):
try:
discoveredphotos += 1
error = False
logger.debug(
"**** Trying to add to lychee album %s: %s",
album['name'], os.path.join(root, f))
# corruption detected here by launching exception
photo = LycheePhoto(self.conf, f, album)
if not (self.dao.photoExists(photo)):
res = copyFileToLychee(self, photo)
adjustRotation(self, photo)
makeThumbnail(self, photo)
res = self.dao.addFileToAlbum(photo)
# increment counter
if res:
importedphotos += 1
album['photos'].append(photo)
else:
error = True
logger.error(
"while adding to album: %s photo: %s",
album['name'], photo.srcfullpath)
else:
logger.warn(
"photo already exists in this album with same name or same checksum: %s it won't be added to lychee",
photo.srcfullpath)
error = True
except Exception as e:
logger.exception(e)
logger.error("could not add %s to album %s", f,
album['name'])
error = True
finally:
if not (error):
logger.info(
"**** Successfully added %s to lychee album %s",
os.path.join(root, f), album['name'])
a = album.copy()
albums.append(a)
logger.info(
"~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
logger.info("Directory scanned:" + self.conf['srcdir'])
logger.info("Created albums: " + str(createdalbums))
if (importedphotos == discoveredphotos):
logger.info(
str(importedphotos) + " photos imported on " +
str(discoveredphotos) + " discovered")
else:
logger.error(
str(importedphotos) + " photos imported on " +
str(discoveredphotos) + " discovered")
logger.info(
"~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
updateAlbumsDate(self, albums)
if self.conf['sort']:
reorderalbumids(self, albums)
self.dao.reinitAlbumAutoIncrement()
if self.conf['sanity']:
logger.info("************ SANITY CHECK *************")
# get All Photos albums
photos = self.dao.get_all_photos()
albums = [p['album'] for p in photos]
albums = set(albums)
# for each album
for a_id in albums:
# check if it exists, if not remove photos
if not (self.dao.albumIdExists(a_id)):
to_delete = self.dao.get_all_photos(a_id)
self.dao.eraseAlbum(a_id)
file_list = [p['url'] for p in to_delete]
self.deleteFiles(file_list)
# get All Photos
photos = self.dao.get_all_photos()
to_delete = []
# for each photo
for p in photos:
delete_photo = False
# check if big exists
bigpath = os.path.join(self.conf["lycheepath"], "uploads",
"big", p['url'])
# if big is a link check if it's an orphan
# file does not exists
if not (os.path.lexists(bigpath)):
logger.error(
"File does not exists %s: will be delete in db",
bigpath)
delete_photo = True
# broken link
elif not (os.path.exists(bigpath)):
logger.error("Link is broken: %s will be delete in db",
bigpath)
delete_photo = True
if not (delete_photo):
# TODO: check if thumbnail exists
pass
else:
# if any of it is False remove and log
to_delete.append(p)
deletePhotos(self, to_delete)
# Detect broken symlinks / orphan files
for root, dirs, files in os.walk(
os.path.join(self.conf['lycheepath'], 'uploads', 'big')):
for f in files:
logger.debug("check orphan: %s", f)
file_name = os.path.basename(f)
# check if DB photo exists
if not self.dao.photoExistsByName(file_name):
# if not delete photo (or link)
self.deleteFiles([file_name])
logger.info("%s deleted. Wasn't existing in DB", f)
# if broken link
if os.path.lexists(f) and not (os.path.exists(f)):
id = self.dao.photoExistsByName(file_name)
# if exists in db
if id:
ps = {'id': id, 'url': file_name}
deletePhotos(self, [ps])
else:
self.deleteFiles([file_name])
logger.info("%s deleted. Was a broken link", f)
# drop empty albums
empty = self.dao.get_empty_albums()
if empty:
for e in empty:
self.dao.dropAlbum(e)
self.dao.close()
if self.conf['watch']:
event_handler = MyEventHandler()
observer = Observer()
observer.schedule(event_handler,
self.conf['srcdir'],
recursive=True)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
class LycheeSyncer:
"""
This class contains the logic behind this program
It consist mainly in filesystem operations
It relies on:
- LycheeDAO for dtabases operations
- LycheePhoto to store (and compute) photos propreties
"""
conf = {}
def __init__(self):
"""
Takes a dictionnary of conf as input
"""
borg = ConfBorg()
self.conf = borg.conf
def deleteFiles(self, filelist):
"""
Delete files in the Lychee file tree (uploads/big and uploads/thumbnails)
Give it the file name and it will delete relatives files and thumbnails
Parameters:
- filelist: a list of filenames
Returns nothing
"""
for url in filelist:
if isAPhoto(self, url):
thumbpath = os.path.join(self.conf["lycheepath"], "uploads", "thumb", url)
filesplit = os.path.splitext(url)
thumb2path = ''.join([filesplit[0], "@2x", filesplit[1]]).lower()
thumb2path = os.path.join(self.conf["lycheepath"], "uploads", "thumb", thumb2path)
bigpath = os.path.join(self.conf["lycheepath"], "uploads", "big", url)
remove_file(thumbpath)
remove_file(thumb2path)
remove_file(bigpath)
def deleteAllFiles(self):
"""
Deletes every photo file in Lychee
Returns nothing
"""
photopath = os.path.join(self.conf["lycheepath"], "uploads", "big")
filelist = [f for f in os.listdir(photopath)]
self.deleteFiles(filelist)
def sync(self):
"""
Program main loop
Scans files to add in the sourcedirectory and add them to Lychee
according to the conf file and given parameters
Returns nothing
"""
# Connect db
# and drop it if dropdb activated
self.dao = LycheeDAO(self.conf)
if self.conf['dropdb']:
self.deleteAllFiles()
# Load db
createdalbums = 0
discoveredphotos = 0
importedphotos = 0
album = {}
albums = []
album_name_max_width = self.dao.getAlbumNameDBWidth()
# walkthroug each file / dir of the srcdir
for root, dirs, files in os.walk(self.conf['srcdir']):
if sys.version_info.major == 2:
try:
root = root.decode('UTF-8')
except Exception as e:
logger.error(e)
# Init album data
album['id'] = None
album['name'] = None
album['path'] = None
album['relpath'] = None # path relative to srcdir
album['parent'] = "0"
album['photos'] = []# path relative to srcdir
album = getAlbum(self, root)
# if a there is at least one photo in the files
album['path'] = root
# don't know what to do with theses photo
# and don't wan't to create a default album
if album['path'] == self.conf['srcdir']:
msg = "file at srcdir root won't be added to lychee, please move them in a subfolder: {}".format(
root)
logger.warn(msg)
continue
album['id'] = self.dao.albumExistsByNameAndParent(album['name'], album['parent'])
if self.conf['replace'] and album['id']:
# drop album photos
filelist = self.dao.eraseAlbum(album['id'])
self.deleteFiles(filelist)
assert self.dao.dropAlbum(album['id'])
# Album should be recreated
album['id'] = False
if not (album['id']):
# create album
album['id'] = createAlbum(self, album)
if not (album['id']):
logger.error("didn't manage to create album for: " + album['name'])
continue
else:
logger.info("############ Album created: %s", album['name'])
createdalbums += 1
# Albums are created or emptied, now take care of photos
for f in sorted(files):
if isAPhoto(self, f):
try:
discoveredphotos += 1
error = False
logger.debug(
"**** Trying to add to lychee album %s: %s",
album['name'],
os.path.join(
root,
f))
# corruption detected here by launching exception
photo = LycheePhoto(self.conf, f, album)
if not (self.dao.photoExists(photo)):
res = copyFileToLychee(self, photo)
adjustRotation(self, photo)
makeThumbnail(self, photo)
res = self.dao.addFileToAlbum(photo)
# increment counter
if res:
importedphotos += 1
album['photos'].append(photo)
else:
error = True
logger.error(
"while adding to album: %s photo: %s",
album['name'],
photo.srcfullpath)
else:
logger.warn(
"photo already exists in this album with same name or same checksum: %s it won't be added to lychee",
photo.srcfullpath)
error = True
except Exception as e:
logger.exception(e)
logger.error("could not add %s to album %s", f, album['name'])
error = True
finally:
if not (error):
logger.info(
"**** Successfully added %s to lychee album %s",
os.path.join(
root,
f),
album['name'])
a = album.copy()
albums.append(a)
logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
logger.info("Directory scanned:" + self.conf['srcdir'])
logger.info("Created albums: " + str(createdalbums))
if (importedphotos == discoveredphotos):
logger.info(
str(importedphotos) + " photos imported on " + str(discoveredphotos) + " discovered")
else:
logger.error(
str(importedphotos) + " photos imported on " + str(discoveredphotos) + " discovered")
logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
updateAlbumsDate(self, albums)
if self.conf['sort']:
reorderalbumids(self, albums)
self.dao.reinitAlbumAutoIncrement()
if self.conf['sanity']:
logger.info("************ SANITY CHECK *************")
# get All Photos albums
photos = self.dao.get_all_photos()
albums = [p['album'] for p in photos]
albums = set(albums)
# for each album
for a_id in albums:
# check if it exists, if not remove photos
if not (self.dao.albumIdExists(a_id)):
to_delete = self.dao.get_all_photos(a_id)
self.dao.eraseAlbum(a_id)
file_list = [p['url'] for p in to_delete]
self.deleteFiles(file_list)
# get All Photos
photos = self.dao.get_all_photos()
to_delete = []
# for each photo
for p in photos:
delete_photo = False
# check if big exists
bigpath = os.path.join(self.conf["lycheepath"], "uploads", "big", p['url'])
# if big is a link check if it's an orphan
# file does not exists
if not (os.path.lexists(bigpath)):
logger.error("File does not exists %s: will be delete in db", bigpath)
delete_photo = True
# broken link
elif not (os.path.exists(bigpath)):
logger.error("Link is broken: %s will be delete in db", bigpath)
delete_photo = True
if not (delete_photo):
# TODO: check if thumbnail exists
pass
else:
# if any of it is False remove and log
to_delete.append(p)
deletePhotos(self, to_delete)
# Detect broken symlinks / orphan files
for root, dirs, files in os.walk(os.path.join(self.conf['lycheepath'], 'uploads', 'big')):
for f in files:
logger.debug("check orphan: %s", f)
file_name = os.path.basename(f)
# check if DB photo exists
if not self.dao.photoExistsByName(file_name):
# if not delete photo (or link)
self.deleteFiles([file_name])
logger.info("%s deleted. Wasn't existing in DB", f)
# if broken link
if os.path.lexists(f) and not (os.path.exists(f)):
id = self.dao.photoExistsByName(file_name)
# if exists in db
if id:
ps = {'id': id, 'url': file_name}
deletePhotos(self, [ps])
else:
self.deleteFiles([file_name])
logger.info("%s deleted. Was a broken link", f)
# drop empty albums
empty = self.dao.get_empty_albums()
if empty:
for e in empty:
self.dao.dropAlbum(e)
self.dao.close()
if self.conf['watch']:
event_handler = MyEventHandler()
observer = Observer()
observer.schedule(event_handler, self.conf['srcdir'], recursive=True)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
