Example #1
0
    def on_deleted(self, event):
        borg = ConfBorg()
        self.conf = borg.conf
        self.dao = LycheeDAO(self.conf)
        if event.is_directory:
            album = getAlbum(self, event.src_path)
            if album['id'] is not None:
                filelist = self.dao.eraseAlbum(album['id'])
                deleteFiles(self, filelist)
                logger.info("Deleted album: %s.", album['name'])
                assert self.dao.dropAlbum(album['id'])
            else:
                logger.error("Tried to delete album: %s, but it wasn't present in the DB", album['name'])

            return
        else:
            if match_path(event.src_path,
                          included_patterns=['*.jpg', '*.jpeg', '*.gif', '*.png'],
                          excluded_patterns=None,
                          case_sensitive=False):
                dirs = event.src_path.split(os.sep)
                albDir = os.sep.join(dirs[:-1])
                album = getAlbum(self, albDir)
                album['path'] = albDir
                dbPhoto = self.dao.get_photo_light(album['id'], os.sep.join(dirs[-1:]), "")
                if dbPhoto is not None:
                    delete = [dbPhoto]
                    deletePhotos(self, delete)
                    logger.info("Deleted Photo: %s.", os.sep.join(dirs[-1:]))
                else:
                    logger.info("Tried to delete Photo: %s, but it wasn't in the database.", os.sep.join(dirs[-1:]))
            return
Example #2
0
    def on_moved(self, event):
        borg = ConfBorg()
        self.conf = borg.conf
        self.dao = LycheeDAO(self.conf)

        if event.is_directory:

            albSrc = getAlbum(self, event.src_path)
            albDest = getAlbum(self, event.dest_path)
            logger.info("%s Album moved to %s. ", event.src_path, event.dest_path)
            self.dao.setAlbumParentAndTitle(albDest['name'], albDest['parent'], albSrc['id'])
            return
        else:
            if match_path(event.src_path,
                          included_patterns=['*.jpg', '*.jpeg', '*.gif', '*.png'],
                          excluded_patterns=None,
                          case_sensitive=False):
                dirs = event.src_path.split(os.sep)
                albDir = os.sep.join(dirs[:-1])
                dirs2 = event.dest_path.split(os.sep)
                albDir2 = os.sep.join(dirs2[:-1])
                album = getAlbum(self, albDir)
                if album['id'] == None:
                    album = getAlbum(self, albDir2)

                dbPhoto = self.dao.get_photo_light(album['id'], os.sep.join(dirs[-1:]), "")

                album2 = getAlbum(self, albDir2)
                logger.info("%s Photo moved to %s. ", event.src_path, event.dest_path)
                self.dao.setPhotoAlbumAndTitle(os.sep.join(dirs2[-1:]), album2['id'], dbPhoto['id'])



            return
Example #3
0
    def on_modified(self, event):
        borg = ConfBorg()
        self.conf = borg.conf
        self.dao = LycheeDAO(self.conf)
        if event.is_directory:
            return
        else:
            if match_path(
                    event.src_path,
                    included_patterns=['*.jpg', '*.jpeg', '*.gif', '*.png'],
                    excluded_patterns=None,
                    case_sensitive=False):
                dirs = event.src_path.split(os.sep)
                albDir = os.sep.join(dirs[:-1])
                album = getAlbum(self, albDir)
                album['path'] = albDir
                photo = LycheePhoto(self.conf, os.sep.join(dirs[-1:]), album)
                dbPhoto = self.dao.get_photo(photo)
                if dbPhoto is not None:
                    delete = [dbPhoto]
                    deletePhotos(self, delete)

                dirs = event.src_path.split(os.sep)
                albDir = os.sep.join(dirs[:-1])
                album = getAlbum(self, albDir)
                album['path'] = albDir
                photo = LycheePhoto(self.conf, os.sep.join(dirs[-1:]), album)
                if not (self.dao.photoExists(photo)):
                    res = copyFileToLychee(self, photo)

                    adjustRotation(self, photo)
                    makeThumbnail(self, photo)
                    res = self.dao.addFileToAlbum(photo)
                    logger.info("Modified Photo: %s.", photo.srcfullpath)
                    # increment counter
                    if not res:
                        logger.error("while adding to album: %s photo: %s",
                                     album['name'], photo.srcfullpath)
                else:
                    logger.error(
                        "photo already exists in this album with same name or same checksum: %s it won't be added to lychee",
                        photo.srcfullpath)
            return
Example #4
0
    def on_deleted(self, event):
        borg = ConfBorg()
        self.conf = borg.conf
        self.dao = LycheeDAO(self.conf)
        if event.is_directory:
            album = getAlbum(self, event.src_path)
            if album['id'] is not None:
                filelist = self.dao.eraseAlbum(album['id'])
                deleteFiles(self, filelist)
                logger.info("Deleted album: %s.", album['name'])
                assert self.dao.dropAlbum(album['id'])
            else:
                logger.error(
                    "Tried to delete album: %s, but it wasn't present in the DB",
                    album['name'])

            return
        else:
            if match_path(
                    event.src_path,
                    included_patterns=['*.jpg', '*.jpeg', '*.gif', '*.png'],
                    excluded_patterns=None,
                    case_sensitive=False):
                dirs = event.src_path.split(os.sep)
                albDir = os.sep.join(dirs[:-1])
                album = getAlbum(self, albDir)
                album['path'] = albDir
                dbPhoto = self.dao.get_photo_light(album['id'],
                                                   os.sep.join(dirs[-1:]), "")
                if dbPhoto is not None:
                    delete = [dbPhoto]
                    deletePhotos(self, delete)
                    logger.info("Deleted Photo: %s.", os.sep.join(dirs[-1:]))
                else:
                    logger.info(
                        "Tried to delete Photo: %s, but it wasn't in the database.",
                        os.sep.join(dirs[-1:]))
            return
Example #5
0
    def on_moved(self, event):
        borg = ConfBorg()
        self.conf = borg.conf
        self.dao = LycheeDAO(self.conf)

        if event.is_directory:

            albSrc = getAlbum(self, event.src_path)
            albDest = getAlbum(self, event.dest_path)
            logger.info("%s Album moved to %s. ", event.src_path,
                        event.dest_path)
            self.dao.setAlbumParentAndTitle(albDest['name'], albDest['parent'],
                                            albSrc['id'])
            return
        else:
            if match_path(
                    event.src_path,
                    included_patterns=['*.jpg', '*.jpeg', '*.gif', '*.png'],
                    excluded_patterns=None,
                    case_sensitive=False):
                dirs = event.src_path.split(os.sep)
                albDir = os.sep.join(dirs[:-1])
                dirs2 = event.dest_path.split(os.sep)
                albDir2 = os.sep.join(dirs2[:-1])
                album = getAlbum(self, albDir)
                if album['id'] == None:
                    album = getAlbum(self, albDir2)

                dbPhoto = self.dao.get_photo_light(album['id'],
                                                   os.sep.join(dirs[-1:]), "")

                album2 = getAlbum(self, albDir2)
                logger.info("%s Photo moved to %s. ", event.src_path,
                            event.dest_path)
                self.dao.setPhotoAlbumAndTitle(os.sep.join(dirs2[-1:]),
                                               album2['id'], dbPhoto['id'])

            return
Example #6
0
    def on_modified(self, event):
        borg = ConfBorg()
        self.conf = borg.conf
        self.dao = LycheeDAO(self.conf)
        if event.is_directory:
            return
        else:
            if match_path(event.src_path,
                          included_patterns=['*.jpg', '*.jpeg', '*.gif', '*.png'],
                          excluded_patterns=None,
                          case_sensitive=False):
                dirs = event.src_path.split(os.sep)
                albDir = os.sep.join(dirs[:-1])
                album = getAlbum(self, albDir)
                album['path'] = albDir
                photo = LycheePhoto(self.conf, os.sep.join(dirs[-1:]), album)
                dbPhoto = self.dao.get_photo(photo)
                if dbPhoto is not None:
                    delete = [dbPhoto]
                    deletePhotos(self, delete)

                dirs = event.src_path.split(os.sep)
                albDir = os.sep.join(dirs[:-1])
                album = getAlbum(self, albDir)
                album['path'] = albDir
                photo = LycheePhoto(self.conf, os.sep.join(dirs[-1:]), album)
                if not (self.dao.photoExists(photo)):
                    res = copyFileToLychee(self, photo)

                    adjustRotation(self, photo)
                    makeThumbnail(self, photo)
                    res = self.dao.addFileToAlbum(photo)
                    logger.info("Modified Photo: %s.", photo.srcfullpath)
                    # increment counter
                    if not res:
                        logger.error(
                            "while adding to album: %s photo: %s",
                            album['name'],
                            photo.srcfullpath)
                else:
                    logger.error(
                        "photo already exists in this album with same name or same checksum: %s it won't be added to lychee",
                        photo.srcfullpath)
            return
Example #7
0
    def sync(self):
        """
        Program main loop
        Scans files to add in the sourcedirectory and add them to Lychee
        according to the conf file and given parameters
        Returns nothing
        """

        # Connect db
        # and drop it if dropdb activated
        self.dao = LycheeDAO(self.conf)

        if self.conf['dropdb']:
            self.deleteAllFiles()

        # Load db

        createdalbums = 0
        discoveredphotos = 0
        importedphotos = 0
        album = {}
        albums = []

        album_name_max_width = self.dao.getAlbumNameDBWidth()

        # walkthroug each file / dir of the srcdir
        for root, dirs, files in os.walk(self.conf['srcdir']):

            if sys.version_info.major == 2:
                try:
                    root = root.decode('UTF-8')
                except Exception as e:
                    logger.error(e)
            # Init album data
            album['id'] = None
            album['name'] = None
            album['path'] = None
            album['relpath'] = None  # path relative to srcdir
            album['photos'] = []  # path relative to srcdir

            # if a there is at least one photo in the files
            if any([self.isAPhoto(f) for f in files]):
                album['path'] = root

                # Skip any albums that matches one of the exluded patterns
                if any([
                        True for pattern in self.conf['excludeAlbums']
                        if fnmatch.fnmatch(root, pattern)
                ]):
                    logger.info("Skipping excluded album {}".format(root))
                    continue

                # don't know what to do with theses photo
                # and don't wan't to create a default album
                if album['path'] == self.conf['srcdir']:
                    msg = "file at srcdir root won't be added to lychee, please move them in a subfolder: {}".format(
                        root)
                    logger.warn(msg)
                    continue

                # Fill in other album properties
                # albumnames start at srcdir (to avoid absolute path albumname)
                album['relpath'] = os.path.relpath(album['path'],
                                                   self.conf['srcdir'])
                album['name'] = self.getAlbumNameFromPath(album)

                if len(album['name']) > album_name_max_width:
                    logger.warn("album name too long, will be truncated " +
                                album['name'])
                    album['name'] = album['name'][0:album_name_max_width]
                    logger.warn("album name is now " + album['name'])

                album['id'] = self.dao.albumExists(album)

                if self.conf['replace'] and album['id']:
                    # drop album photos
                    filelist = self.dao.eraseAlbum(album['id'])
                    self.deleteFiles(filelist)
                    assert self.dao.dropAlbum(album['id'])
                    # Album should be recreated
                    album['id'] = False

                if not (album['id']):
                    # create album
                    album['id'] = self.createAlbum(album)

                    if not (album['id']):
                        logger.error("didn't manage to create album for: " +
                                     album['relpath'])
                        continue
                    else:
                        logger.info("############ Album created: %s",
                                    album['name'])

                    createdalbums += 1

                # Albums are created or emptied, now take care of photos
                for f in sorted(files):

                    if self.isAPhoto(f):
                        try:
                            discoveredphotos += 1
                            error = False
                            logger.debug(
                                "**** Trying to add to lychee album %s: %s",
                                album['name'], os.path.join(root, f))
                            # corruption detected here by launching exception
                            pid = self.dao.getUniqPhotoId()
                            photo = LycheePhoto(pid, self.conf, f, album)
                            if not (self.dao.photoExists(photo)):
                                res = self.copyFileToLychee(photo)
                                self.adjustRotation(photo)
                                self.makeThumbnail(photo)
                                res = self.dao.addFileToAlbum(photo)
                                # increment counter
                                if res:
                                    importedphotos += 1
                                    album['photos'].append(photo)
                                else:
                                    error = True
                                    logger.error(
                                        "while adding to album: %s photo: %s",
                                        album['name'], photo.srcfullpath)
                            else:
                                logger.error(
                                    "photo already exists in this album with same name or same checksum: %s it won't be added to lychee",
                                    photo.srcfullpath)
                                error = True
                        except Exception as e:

                            logger.exception(e)
                            logger.error("could not add %s to album %s", f,
                                         album['name'])
                            error = True
                        finally:
                            if not (error):
                                logger.info(
                                    "**** Successfully added %s to lychee album %s",
                                    os.path.join(root, f), album['name'])

                a = album.copy()
                albums.append(a)

        self.updateAlbumsDate(albums)
        if self.conf['sort']:
            self.reorderalbumids(albums)
            self.dao.reinitAlbumAutoIncrement()

        if self.conf['sanity']:

            logger.info("************ SANITY CHECK *************")
            # get All Photos albums
            photos = self.dao.get_all_photos()
            albums = [p['album'] for p in photos]
            albums = set(albums)

            # for each album
            for a_id in albums:
                # check if it exists, if not remove photos
                if not (self.dao.albumIdExists(a_id)):
                    to_delete = self.dao.get_all_photos(a_id)
                    self.dao.eraseAlbum(a_id)
                    file_list = [p['url'] for p in to_delete]
                    self.deleteFiles(file_list)

            # get All Photos
            photos = self.dao.get_all_photos()

            to_delete = []
            # for each photo
            for p in photos:
                delete_photo = False
                # check if big exists
                bigpath = os.path.join(self.conf["lycheepath"], "uploads",
                                       "big", p['url'])

                # if big is a link check if it's an orphan
                # file does not exists
                if not (os.path.lexists(bigpath)):
                    logger.error(
                        "File does not exists %s: will be delete in db",
                        bigpath)
                    delete_photo = True
                # broken link
                elif not (os.path.exists(bigpath)):
                    logger.error("Link is broken: %s will be delete in db",
                                 bigpath)
                    delete_photo = True

                if not (delete_photo):
                    # TODO: check if thumbnail exists
                    pass
                else:
                    # if any of it is False remove and log
                    to_delete.append(p)

            self.deletePhotos(to_delete)

            # Detect broken symlinks / orphan files
            for root, dirs, files in os.walk(
                    os.path.join(self.conf['lycheepath'], 'uploads', 'big')):

                for f in files:
                    logger.debug("check orphan: %s", f)
                    file_name = os.path.basename(f)
                    # check if DB photo exists
                    print(f, file_name)
                    if not self.dao.photoExistsByFileName(file_name):
                        # if not delete photo (or link)
                        self.deleteFiles([file_name])
                        logger.info("%s deleted. Wasn't existing in DB", f)

                    # if broken link
                    if os.path.lexists(f) and not (os.path.exists(f)):
                        id = self.dao.photoExistsByFileName(file_name)
                        # if exists in db
                        if id:
                            ps = {}
                            ps['id'] = id
                            ps['url'] = file_name
                            self.deletePhotos([ps])
                        else:
                            self.deleteFiles([file_name])
                        logger.info("%s deleted. Was a broken link", f)

            # drop empty albums
            empty = self.dao.get_empty_albums()
            if empty:
                for e in empty:
                    self.dao.dropAlbum(e)

        self.dao.close()

        # Final report
        logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
        logger.info("Directory scanned:" + self.conf['srcdir'])
        logger.info("Created albums: " + str(createdalbums))
        if (importedphotos == discoveredphotos):
            logger.info(
                str(importedphotos) + " photos imported on " +
                str(discoveredphotos) + " discovered")
        else:
            logger.error(
                str(importedphotos) + " photos imported on " +
                str(discoveredphotos) + " discovered")
        logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
Example #8
0
class LycheeSyncer:
    """
    This class contains the logic behind this program
    It consist mainly in filesystem operations
    It relies on:
    - LycheeDAO for dtabases operations
    - LycheePhoto to store (and compute) photos propreties
    """

    conf = {}

    def __init__(self):
        """
        Takes a dictionnary of conf as input
        """
        borg = ConfBorg()
        self.conf = borg.conf

    def getAlbumNameFromPath(self, album):
        """
        build a lychee compatible albumname from an albumpath (relative to the srcdir main argument)
        Takes an album properties list  as input. At least the path sould be specified (relative albumpath)
        Returns a string, the lychee album name
        """
        # make a list with directory and sub dirs
        alb_path_utf8 = album['relpath']  # .decode('UTF-8')

        path = alb_path_utf8.split(os.sep)

        # join the rest: no subfolders in lychee yet
        if len(path) > 1:
            album['name'] = "_".join(path)
        else:
            album['name'] = alb_path_utf8
        return album['name']

    def isAPhoto(self, file):
        """
        Determine if the filename passed is a photo or not based on the file extension
        Takes a string  as input (a file name)
        Returns a boolean
        """
        validimgext = ['.jpg', '.jpeg', '.gif', '.png']
        ext = os.path.splitext(file)[-1].lower()
        return (ext in validimgext)

    def albumExists(self, album):
        """
        Takes an album properties list  as input. At least the relpath sould be specified (relative albumpath)
        Returns an albumid or None if album does not exists
        """

    def createAlbum(self, album):
        """
        Creates an album
        Inputs:
        - album: an album properties list. at least path should be specified (relative albumpath)
        Returns an albumid or None if album does not exists
        """
        album['id'] = None
        if album['name'] != "":
            album['id'] = self.dao.createAlbum(album)
        return album['id']

    def thumbIt(self, res, photo, destinationpath, destfile):
        """
        Create the thumbnail of a given photo
        Parameters:
        - res: should be a set of h and v res (640, 480)
        - photo: a valid LycheePhoto object
        - destinationpath: a string the destination full path of the thumbnail (without filename)
        - destfile: the thumbnail filename
        Returns the fullpath of the thuumbnail
        """

        if photo.width > photo.height:
            delta = photo.width - photo.height
            left = int(delta / 2)
            upper = 0
            right = int(photo.height + left)
            lower = int(photo.height)
        else:
            delta = photo.height - photo.width
            left = 0
            upper = int(delta / 2)
            right = int(photo.width)
            lower = int(photo.width + upper)

        destimage = os.path.join(destinationpath, destfile)
        try:
            img = Image.open(photo.destfullpath)
        except Exception as e:
            logger.exception(e)
            logger.error("ioerror (corrupted file?): " + photo.srcfullpath)
            raise

        img = img.crop((left, upper, right, lower))
        img.thumbnail(res, Image.ANTIALIAS)
        img.save(destimage, quality=99)
        return destimage

    def makeThumbnail(self, photo):
        """
        Make the 2 thumbnails needed by Lychee for a given photo
        and store their path in the LycheePhoto object
        Parameters:
        - photo: a valid LycheePhoto object
        returns nothing
        """
        # set  thumbnail size
        sizes = [(200, 200), (400, 400)]
        # insert @2x in big thumbnail file name
        filesplit = os.path.splitext(photo.url)
        destfiles = [
            photo.url, ''.join([filesplit[0], "@2x", filesplit[1]]).lower()
        ]
        # compute destination path
        destpath = os.path.join(self.conf["lycheepath"], "uploads", "thumb")
        # make thumbnails
        photo.thumbnailfullpath = self.thumbIt(sizes[0], photo, destpath,
                                               destfiles[0])
        photo.thumbnailx2fullpath = self.thumbIt(sizes[1], photo, destpath,
                                                 destfiles[1])

    def copyFileToLychee(self, photo):
        """
        add a file to an album, the albumid must be previously stored in the LycheePhoto parameter
        Parameters:
        - photo: a valid LycheePhoto object
        Returns True if everything went ok
        """
        res = False

        try:
            # copy photo
            if self.conf['link']:
                os.symlink(photo.srcfullpath, photo.destfullpath)
            else:
                shutil.copy(photo.srcfullpath, photo.destfullpath)
            # adjust right (chmod/chown)
            try:
                os.lchown(photo.destfullpath, -1, self.conf['gid'])

                if not (self.conf['link']):
                    st = os.stat(photo.destfullpath)
                    os.chmod(photo.destfullpath,
                             st.st_mode | stat.S_IRWXU | stat.S_IRWXG)
                else:
                    st = os.stat(photo.srcfullpath)
                    os.chmod(photo.srcfullpath, st.st_mode | stat.S_IROTH)

            except Exception as e:
                if self.conf["verbose"]:
                    logger.warn(
                        "chgrp error,  check file permission for %s fix: eventually adjust source file permissions",
                        photo.destfullpath)
            res = True

        except Exception as e:
            logger.exception(e)
            res = False

        return res

    def deleteFiles(self, filelist):
        """
        Delete files in the Lychee file tree (uploads/big and uploads/thumbnails)
        Give it the file name and it will delete relatives files and thumbnails
        Parameters:
        - filelist: a list of filenames
        Returns nothing
        """

        for url in filelist:
            if self.isAPhoto(url):
                thumbpath = os.path.join(self.conf["lycheepath"], "uploads",
                                         "thumb", url)
                filesplit = os.path.splitext(url)
                thumb2path = ''.join([filesplit[0], "@2x",
                                      filesplit[1]]).lower()
                thumb2path = os.path.join(self.conf["lycheepath"], "uploads",
                                          "thumb", thumb2path)
                bigpath = os.path.join(self.conf["lycheepath"], "uploads",
                                       "big", url)
                remove_file(thumbpath)
                remove_file(thumb2path)
                remove_file(bigpath)

    def adjustRotation(self, photo):
        """
        Rotates photos according to the exif orientaion tag
        Returns nothing DOIT BEFORE THUMBNAILS !!!
        """

        if photo.exif.orientation != 1:

            img = Image.open(photo.destfullpath)
            if "exif" in img.info:
                exif_dict = piexif.load(img.info["exif"])

                if piexif.ImageIFD.Orientation in exif_dict["0th"]:
                    orientation = exif_dict["0th"][piexif.ImageIFD.Orientation]

                    if orientation == 2:
                        img = img.transpose(Image.FLIP_LEFT_RIGHT)
                    elif orientation == 3:
                        img = img.rotate(180)
                    elif orientation == 4:
                        img = img.rotate(180).transpose(Image.FLIP_LEFT_RIGHT)
                    elif orientation == 5:
                        img = img.rotate(-90, expand=True).transpose(
                            Image.FLIP_LEFT_RIGHT)
                    elif orientation == 6:
                        img = img.rotate(-90, expand=True)
                    elif orientation == 7:
                        img = img.rotate(90, expand=True).transpose(
                            Image.FLIP_LEFT_RIGHT)
                    elif orientation == 8:
                        img = img.rotate(90, expand=True)
                    else:
                        if orientation != 1:
                            logger.warn(
                                "Orientation not defined {} for photo {}".
                                format(orientation, photo.title))

                    if orientation in [5, 6, 7, 8]:
                        # invert width and height
                        h = photo.height
                        w = photo.width
                        photo.height = w
                        photo.width = h
                    exif_dict["0th"][piexif.ImageIFD.Orientation] = 1
                    exif_bytes = piexif.dump(exif_dict)
                    img.save(photo.destfullpath, exif=exif_bytes, quality=99)
            img.close()

    def reorderalbumids(self, albums):

        # sort albums by title
        def getName(album):
            return album['name']

        sortedalbums = sorted(albums, key=getName)

        # count albums
        nbalbum = len(albums)
        # get higher album id + 1 as a first new album id
        min, max = self.dao.getAlbumMinMaxIds()

        if min and max:

            if nbalbum + 1 < min:
                newid = 1
            else:
                newid = max + 1

            for a in sortedalbums:
                self.dao.changeAlbumId(a['id'], newid)
                newid = newid + 1

    def updateAlbumsDate(self, albums):
        now = datetime.datetime.now()
        last2min = now - datetime.timedelta(minutes=2)
        last2min_epoch = int(
            (last2min - datetime.datetime(1970, 1, 1)).total_seconds())

        for a in albums:
            try:
                # get photos with a real date (not just now)
                datelist = None

                if len(a['photos']) > 0:

                    datelist = [
                        photo.epoch_sysdate for photo in a['photos']
                        if photo.epoch_sysdate < last2min_epoch
                    ]

                    if datelist is not None and len(datelist) > 0:
                        newdate = max(datelist)
                        self.dao.updateAlbumDate(a['id'], newdate)
                        logger.debug(
                            "album %s sysstamp changed to: %s ", a['name'],
                            str(
                                time.strftime('%Y-%m-%d %H:%M:%S',
                                              time.localtime(newdate))))
            except Exception as e:
                logger.exception(e)
                logger.error("updating album date for album:" + a['name'], e)

    def deleteAllFiles(self):
        """
        Deletes every photo file in Lychee
        Returns nothing
        """
        filelist = []
        photopath = os.path.join(self.conf["lycheepath"], "uploads", "big")
        filelist = [f for f in os.listdir(photopath)]
        self.deleteFiles(filelist)

    def deletePhotos(self, photo_list):
        "photo_list: a list of dictionnary containing key url and id"
        if len(photo_list) > 0:
            url_list = [p['url'] for p in photo_list]
            self.deleteFiles(url_list)
            for p in photo_list:
                self.dao.dropPhoto(p['id'])

    def sync(self):
        """
        Program main loop
        Scans files to add in the sourcedirectory and add them to Lychee
        according to the conf file and given parameters
        Returns nothing
        """

        # Connect db
        # and drop it if dropdb activated
        self.dao = LycheeDAO(self.conf)

        if self.conf['dropdb']:
            self.deleteAllFiles()

        # Load db

        createdalbums = 0
        discoveredphotos = 0
        importedphotos = 0
        album = {}
        albums = []

        album_name_max_width = self.dao.getAlbumNameDBWidth()

        # walkthroug each file / dir of the srcdir
        for root, dirs, files in os.walk(self.conf['srcdir']):

            if sys.version_info.major == 2:
                try:
                    root = root.decode('UTF-8')
                except Exception as e:
                    logger.error(e)
            # Init album data
            album['id'] = None
            album['name'] = None
            album['path'] = None
            album['relpath'] = None  # path relative to srcdir
            album['photos'] = []  # path relative to srcdir

            # if a there is at least one photo in the files
            if any([self.isAPhoto(f) for f in files]):
                album['path'] = root

                # Skip any albums that matches one of the exluded patterns
                if any([
                        True for pattern in self.conf['excludeAlbums']
                        if fnmatch.fnmatch(root, pattern)
                ]):
                    logger.info("Skipping excluded album {}".format(root))
                    continue

                # don't know what to do with theses photo
                # and don't wan't to create a default album
                if album['path'] == self.conf['srcdir']:
                    msg = "file at srcdir root won't be added to lychee, please move them in a subfolder: {}".format(
                        root)
                    logger.warn(msg)
                    continue

                # Fill in other album properties
                # albumnames start at srcdir (to avoid absolute path albumname)
                album['relpath'] = os.path.relpath(album['path'],
                                                   self.conf['srcdir'])
                album['name'] = self.getAlbumNameFromPath(album)

                if len(album['name']) > album_name_max_width:
                    logger.warn("album name too long, will be truncated " +
                                album['name'])
                    album['name'] = album['name'][0:album_name_max_width]
                    logger.warn("album name is now " + album['name'])

                album['id'] = self.dao.albumExists(album)

                if self.conf['replace'] and album['id']:
                    # drop album photos
                    filelist = self.dao.eraseAlbum(album['id'])
                    self.deleteFiles(filelist)
                    assert self.dao.dropAlbum(album['id'])
                    # Album should be recreated
                    album['id'] = False

                if not (album['id']):
                    # create album
                    album['id'] = self.createAlbum(album)

                    if not (album['id']):
                        logger.error("didn't manage to create album for: " +
                                     album['relpath'])
                        continue
                    else:
                        logger.info("############ Album created: %s",
                                    album['name'])

                    createdalbums += 1

                # Albums are created or emptied, now take care of photos
                for f in sorted(files):

                    if self.isAPhoto(f):
                        try:
                            discoveredphotos += 1
                            error = False
                            logger.debug(
                                "**** Trying to add to lychee album %s: %s",
                                album['name'], os.path.join(root, f))
                            # corruption detected here by launching exception
                            pid = self.dao.getUniqPhotoId()
                            photo = LycheePhoto(pid, self.conf, f, album)
                            if not (self.dao.photoExists(photo)):
                                res = self.copyFileToLychee(photo)
                                self.adjustRotation(photo)
                                self.makeThumbnail(photo)
                                res = self.dao.addFileToAlbum(photo)
                                # increment counter
                                if res:
                                    importedphotos += 1
                                    album['photos'].append(photo)
                                else:
                                    error = True
                                    logger.error(
                                        "while adding to album: %s photo: %s",
                                        album['name'], photo.srcfullpath)
                            else:
                                logger.error(
                                    "photo already exists in this album with same name or same checksum: %s it won't be added to lychee",
                                    photo.srcfullpath)
                                error = True
                        except Exception as e:

                            logger.exception(e)
                            logger.error("could not add %s to album %s", f,
                                         album['name'])
                            error = True
                        finally:
                            if not (error):
                                logger.info(
                                    "**** Successfully added %s to lychee album %s",
                                    os.path.join(root, f), album['name'])

                a = album.copy()
                albums.append(a)

        self.updateAlbumsDate(albums)
        if self.conf['sort']:
            self.reorderalbumids(albums)
            self.dao.reinitAlbumAutoIncrement()

        if self.conf['sanity']:

            logger.info("************ SANITY CHECK *************")
            # get All Photos albums
            photos = self.dao.get_all_photos()
            albums = [p['album'] for p in photos]
            albums = set(albums)

            # for each album
            for a_id in albums:
                # check if it exists, if not remove photos
                if not (self.dao.albumIdExists(a_id)):
                    to_delete = self.dao.get_all_photos(a_id)
                    self.dao.eraseAlbum(a_id)
                    file_list = [p['url'] for p in to_delete]
                    self.deleteFiles(file_list)

            # get All Photos
            photos = self.dao.get_all_photos()

            to_delete = []
            # for each photo
            for p in photos:
                delete_photo = False
                # check if big exists
                bigpath = os.path.join(self.conf["lycheepath"], "uploads",
                                       "big", p['url'])

                # if big is a link check if it's an orphan
                # file does not exists
                if not (os.path.lexists(bigpath)):
                    logger.error(
                        "File does not exists %s: will be delete in db",
                        bigpath)
                    delete_photo = True
                # broken link
                elif not (os.path.exists(bigpath)):
                    logger.error("Link is broken: %s will be delete in db",
                                 bigpath)
                    delete_photo = True

                if not (delete_photo):
                    # TODO: check if thumbnail exists
                    pass
                else:
                    # if any of it is False remove and log
                    to_delete.append(p)

            self.deletePhotos(to_delete)

            # Detect broken symlinks / orphan files
            for root, dirs, files in os.walk(
                    os.path.join(self.conf['lycheepath'], 'uploads', 'big')):

                for f in files:
                    logger.debug("check orphan: %s", f)
                    file_name = os.path.basename(f)
                    # check if DB photo exists
                    print(f, file_name)
                    if not self.dao.photoExistsByFileName(file_name):
                        # if not delete photo (or link)
                        self.deleteFiles([file_name])
                        logger.info("%s deleted. Wasn't existing in DB", f)

                    # if broken link
                    if os.path.lexists(f) and not (os.path.exists(f)):
                        id = self.dao.photoExistsByFileName(file_name)
                        # if exists in db
                        if id:
                            ps = {}
                            ps['id'] = id
                            ps['url'] = file_name
                            self.deletePhotos([ps])
                        else:
                            self.deleteFiles([file_name])
                        logger.info("%s deleted. Was a broken link", f)

            # drop empty albums
            empty = self.dao.get_empty_albums()
            if empty:
                for e in empty:
                    self.dao.dropAlbum(e)

        self.dao.close()

        # Final report
        logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
        logger.info("Directory scanned:" + self.conf['srcdir'])
        logger.info("Created albums: " + str(createdalbums))
        if (importedphotos == discoveredphotos):
            logger.info(
                str(importedphotos) + " photos imported on " +
                str(discoveredphotos) + " discovered")
        else:
            logger.error(
                str(importedphotos) + " photos imported on " +
                str(discoveredphotos) + " discovered")
        logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
Example #9
0
    def sync(self):
        """
        Program main loop
        Scans files to add in the sourcedirectory and add them to Lychee
        according to the conf file and given parameters
        Returns nothing
        """

        # Connect db
        # and drop it if dropdb activated
        self.dao = LycheeDAO(self.conf)

        if self.conf['dropdb']:
            self.deleteAllFiles()

        # Load db

        createdalbums = 0
        discoveredphotos = 0
        importedphotos = 0
        album = {}
        albums = []

        album_name_max_width = self.dao.getAlbumNameDBWidth()

        # walkthroug each file / dir of the srcdir
        for root, dirs, files in os.walk(self.conf['srcdir']):

            if sys.version_info.major == 2:
                try:
                    root = root.decode('UTF-8')
                except Exception as e:
                    logger.error(e)
            # Init album data
            album['id'] = None
            album['name'] = None
            album['path'] = None
            album['relpath'] = None  # path relative to srcdir
            album['photos'] = []  # path relative to srcdir

            # if a there is at least one photo in the files
            if any([self.isAPhoto(f) for f in files]):
                album['path'] = root

                # don't know what to do with theses photo
                # and don't wan't to create a default album
                if album['path'] == self.conf['srcdir']:
                    msg = "file at srcdir root won't be added to lychee, please move them in a subfolder: {}".format(
                        root)
                    logger.warn(msg)
                    continue

                # Fill in other album properties
                # albumnames start at srcdir (to avoid absolute path albumname)
                album['relpath'] = os.path.relpath(album['path'], self.conf['srcdir'])
                album['name'] = self.getAlbumNameFromPath(album)

                if len(album['name']) > album_name_max_width:
                    logger.warn("album name too long, will be truncated " + album['name'])
                    album['name'] = album['name'][0:album_name_max_width]
                    logger.warn("album name is now " + album['name'])

                album['id'] = self.dao.albumExists(album)

                if self.conf['replace'] and album['id']:
                    # drop album photos
                    filelist = self.dao.eraseAlbum(album['id'])
                    self.deleteFiles(filelist)
                    assert self.dao.dropAlbum(album['id'])
                    # Album should be recreated
                    album['id'] = False

                if not(album['id']):
                    # create album
                    album['id'] = self.createAlbum(album)

                    if not(album['id']):
                        logger.error("didn't manage to create album for: " + album['relpath'])
                        continue
                    else:
                        logger.info("############ Album created: %s", album['name'])

                    createdalbums += 1

                # Albums are created or emptied, now take care of photos
                for f in sorted(files):

                    if self.isAPhoto(f):
                        try:
                            discoveredphotos += 1
                            error = False
                            logger.debug(
                                "**** Trying to add to lychee album %s: %s",
                                album['name'],
                                os.path.join(
                                    root,
                                    f))
                            # corruption detected here by launching exception
                            photo = LycheePhoto(self.conf, f, album)
                            if not(self.dao.photoExists(photo)):
                                res = self.copyFileToLychee(photo)
                                self.adjustRotation(photo)
                                self.makeThumbnail(photo)
                                res = self.dao.addFileToAlbum(photo)
                                # increment counter
                                if res:
                                    importedphotos += 1
                                    album['photos'].append(photo)
                                else:
                                    error = True
                                    logger.error(
                                        "while adding to album: %s photo: %s",
                                        album['name'],
                                        photo.srcfullpath)
                            else:
                                logger.error(
                                    "photo already exists in this album with same name or same checksum: %s it won't be added to lychee",
                                    photo.srcfullpath)
                                error = True
                        except Exception as e:

                            logger.exception(e)
                            logger.error("could not add %s to album %s", f, album['name'])
                            error = True
                        finally:
                            if not(error):
                                logger.info(
                                    "**** Successfully added %s to lychee album %s",
                                    os.path.join(
                                        root,
                                        f),
                                    album['name'])

                a = album.copy()
                albums.append(a)

        self.updateAlbumsDate(albums)
        if self.conf['sort']:
            self.reorderalbumids(albums)
            self.dao.reinitAlbumAutoIncrement()

        if self.conf['sanity']:

            logger.info("************ SANITY CHECK *************")
            # get All Photos albums
            photos = self.dao.get_all_photos()
            albums = [p['album'] for p in photos]
            albums = set(albums)

            # for each album
            for a_id in albums:
                # check if it exists, if not remove photos
                if not(self.dao.albumIdExists(a_id)):
                    to_delete = self.dao.get_all_photos(a_id)
                    self.dao.eraseAlbum(a_id)
                    file_list = [p['url'] for p in to_delete]
                    self.deleteFiles(file_list)

            # get All Photos
            photos = self.dao.get_all_photos()

            to_delete = []
            # for each photo
            for p in photos:
                delete_photo = False
                # check if big exists
                bigpath = os.path.join(self.conf["lycheepath"], "uploads", "big", p['url'])

                # if big is a link check if it's an orphan
                # file does not exists
                if not(os.path.lexists(bigpath)):
                    logger.error("File does not exists %s: will be delete in db", bigpath)
                    delete_photo = True
                # broken link
                elif not(os.path.exists(bigpath)):
                    logger.error("Link is broken: %s will be delete in db", bigpath)
                    delete_photo = True

                if not(delete_photo):
                    # TODO: check if thumbnail exists
                    pass
                else:
                    # if any of it is False remove and log
                    to_delete.append(p)

            self.deletePhotos(to_delete)

            # Detect broken symlinks / orphan files
            for root, dirs, files in os.walk(os.path.join(self.conf['lycheepath'], 'uploads', 'big')):

                for f in files:
                    logger.debug("check orphan: %s", f)
                    file_name = os.path.basename(f)
                    # check if DB photo exists
                    if not self.dao.photoExistsByName(file_name):
                        # if not delete photo (or link)
                        self.deleteFiles([file_name])
                        logger.info("%s deleted. Wasn't existing in DB", f)

                    # if broken link
                    if os.path.lexists(f) and not(os.path.exists(f)):
                        id = self.dao.photoExistsByName(file_name)
                        # if exists in db
                        if id:
                            ps = {}
                            ps['id'] = id
                            ps['url'] = file_name
                            self.deletePhotos([ps])
                        else:
                            self.deleteFiles([file_name])
                        logger.info("%s deleted. Was a broken link", f)

            # drop empty albums
            empty = self.dao.get_empty_albums()
            if empty:
                for e in empty:
                    self.dao.dropAlbum(e)

        self.dao.close()

        # Final report
        logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
        logger.info("Directory scanned:" + self.conf['srcdir'])
        logger.info("Created albums: " + str(createdalbums))
        if (importedphotos == discoveredphotos):
            logger.info(str(importedphotos) + " photos imported on " + str(discoveredphotos) + " discovered")
        else:
            logger.error(str(importedphotos) + " photos imported on " + str(discoveredphotos) + " discovered")
        logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
Example #10
0
class LycheeSyncer:

    """
    This class contains the logic behind this program
    It consist mainly in filesystem operations
    It relies on:
    - LycheeDAO for dtabases operations
    - LycheePhoto to store (and compute) photos propreties
    """

    conf = {}

    def __init__(self):
        """
        Takes a dictionnary of conf as input
        """
        borg = ConfBorg()
        self.conf = borg.conf

    def getAlbumNameFromPath(self, album):
        """
        build a lychee compatible albumname from an albumpath (relative to the srcdir main argument)
        Takes an album properties list  as input. At least the path sould be specified (relative albumpath)
        Returns a string, the lychee album name
        """
        # make a list with directory and sub dirs
        alb_path_utf8 = album['relpath']  # .decode('UTF-8')

        path = alb_path_utf8.split(os.sep)

        # join the rest: no subfolders in lychee yet
        if len(path) > 1:
            album['name'] = "_".join(path)
        else:
            album['name'] = alb_path_utf8
        return album['name']

    def isAPhoto(self, file):
        """
        Determine if the filename passed is a photo or not based on the file extension
        Takes a string  as input (a file name)
        Returns a boolean
        """
        validimgext = ['.jpg', '.jpeg', '.gif', '.png']
        ext = os.path.splitext(file)[-1].lower()
        return (ext in validimgext)

    def albumExists(self, album):
        """
        Takes an album properties list  as input. At least the relpath sould be specified (relative albumpath)
        Returns an albumid or None if album does not exists
        """

    def createAlbum(self, album):
        """
        Creates an album
        Inputs:
        - album: an album properties list. at least path should be specified (relative albumpath)
        Returns an albumid or None if album does not exists
        """
        album['id'] = None
        if album['name'] != "":
            album['id'] = self.dao.createAlbum(album)
        return album['id']

    def thumbIt(self, res, photo, destinationpath, destfile):
        """
        Create the thumbnail of a given photo
        Parameters:
        - res: should be a set of h and v res (640, 480)
        - photo: a valid LycheePhoto object
        - destinationpath: a string the destination full path of the thumbnail (without filename)
        - destfile: the thumbnail filename
        Returns the fullpath of the thuumbnail
        """

        if photo.width > photo.height:
            delta = photo.width - photo.height
            left = int(delta / 2)
            upper = 0
            right = int(photo.height + left)
            lower = int(photo.height)
        else:
            delta = photo.height - photo.width
            left = 0
            upper = int(delta / 2)
            right = int(photo.width)
            lower = int(photo.width + upper)

        destimage = os.path.join(destinationpath, destfile)
        try:
            img = Image.open(photo.destfullpath)
        except Exception as e:
            logger.exception(e)
            logger.error("ioerror (corrupted file?): " + photo.srcfullpath)
            raise

        img = img.crop((left, upper, right, lower))
        img.thumbnail(res, Image.ANTIALIAS)
        img.save(destimage, quality=99)
        return destimage

    def makeThumbnail(self, photo):
        """
        Make the 2 thumbnails needed by Lychee for a given photo
        and store their path in the LycheePhoto object
        Parameters:
        - photo: a valid LycheePhoto object
        returns nothing
        """
        # set  thumbnail size
        sizes = [(200, 200), (400, 400)]
        # insert @2x in big thumbnail file name
        filesplit = os.path.splitext(photo.url)
        destfiles = [photo.url, ''.join([filesplit[0], "@2x", filesplit[1]]).lower()]
        # compute destination path
        destpath = os.path.join(self.conf["lycheepath"], "uploads", "thumb")
        # make thumbnails
        photo.thumbnailfullpath = self.thumbIt(sizes[0], photo, destpath, destfiles[0])
        photo.thumbnailx2fullpath = self.thumbIt(sizes[1], photo, destpath, destfiles[1])

    def copyFileToLychee(self, photo):
        """
        add a file to an album, the albumid must be previously stored in the LycheePhoto parameter
        Parameters:
        - photo: a valid LycheePhoto object
        Returns True if everything went ok
        """
        res = False

        try:
            # copy photo
            if self.conf['link']:
                os.symlink(photo.srcfullpath, photo.destfullpath)
            else:
                shutil.copy(photo.srcfullpath, photo.destfullpath)
            # adjust right (chmod/chown)
            try:
                os.lchown(photo.destfullpath, -1, self.conf['gid'])

                if not(self.conf['link']):
                    st = os.stat(photo.destfullpath)
                    os.chmod(photo.destfullpath, st.st_mode | stat.S_IRWXU | stat.S_IRWXG)
                else:
                    st = os.stat(photo.srcfullpath)
                    os.chmod(photo.srcfullpath, st.st_mode | stat.S_IROTH)

            except Exception as e:
                if self.conf["verbose"]:
                    logger.warn(
                        "chgrp error,  check file permission for %s fix: eventually adjust source file permissions",
                        photo.destfullpath)
            res = True

        except Exception as e:
            logger.exception(e)
            res = False

        return res

    def deleteFiles(self, filelist):
        """
        Delete files in the Lychee file tree (uploads/big and uploads/thumbnails)
        Give it the file name and it will delete relatives files and thumbnails
        Parameters:
        - filelist: a list of filenames
        Returns nothing
        """

        for url in filelist:
            if self.isAPhoto(url):
                thumbpath = os.path.join(self.conf["lycheepath"], "uploads", "thumb", url)
                filesplit = os.path.splitext(url)
                thumb2path = ''.join([filesplit[0], "@2x", filesplit[1]]).lower()
                thumb2path = os.path.join(self.conf["lycheepath"], "uploads", "thumb", thumb2path)
                bigpath = os.path.join(self.conf["lycheepath"], "uploads", "big", url)
                remove_file(thumbpath)
                remove_file(thumb2path)
                remove_file(bigpath)

    def adjustRotation(self, photo):
        """
        Rotates photos according to the exif orientaion tag
        Returns nothing DOIT BEFORE THUMBNAILS !!!
        """

        if photo.exif.orientation != 1:

            img = Image.open(photo.destfullpath)
            if "exif" in img.info:
                exif_dict = piexif.load(img.info["exif"])

                if piexif.ImageIFD.Orientation in exif_dict["0th"]:
                    orientation = exif_dict["0th"][piexif.ImageIFD.Orientation]

                    if orientation == 2:
                        img = img.transpose(Image.FLIP_LEFT_RIGHT)
                    elif orientation == 3:
                        img = img.rotate(180)
                    elif orientation == 4:
                        img = img.rotate(180).transpose(Image.FLIP_LEFT_RIGHT)
                    elif orientation == 5:
                        img = img.rotate(-90, expand=True).transpose(Image.FLIP_LEFT_RIGHT)
                    elif orientation == 6:
                        img = img.rotate(-90, expand=True)
                    elif orientation == 7:
                        img = img.rotate(90, expand=True).transpose(Image.FLIP_LEFT_RIGHT)
                    elif orientation == 8:
                        img = img.rotate(90, expand=True)
                    else:
                        if orientation != 1:
                            logger.warn("Orientation not defined {} for photo {}".format(orientation, photo.title))

                    if orientation in [5, 6, 7, 8]:
                        # invert width and height
                        h = photo.height
                        w = photo.width
                        photo.height = w
                        photo.width = h
                    exif_dict["0th"][piexif.ImageIFD.Orientation] = 1
                    exif_bytes = piexif.dump(exif_dict)
                    img.save(photo.destfullpath, exif=exif_bytes, quality=99)
            img.close()

    def reorderalbumids(self, albums):

        # sort albums by title
        def getName(album):
            return album['name']

        sortedalbums = sorted(albums, key=getName)

        # count albums
        nbalbum = len(albums)
        # get higher album id + 1 as a first new album id
        min, max = self.dao.getAlbumMinMaxIds()

        if min and max:

            if nbalbum + 1 < min:
                newid = 1
            else:
                newid = max + 1

            for a in sortedalbums:
                self.dao.changeAlbumId(a['id'], newid)
                newid = newid + 1

    def updateAlbumsDate(self, albums):
        now = datetime.datetime.now()
        last2min = now - datetime.timedelta(minutes=2)
        last2min_epoch = int((last2min - datetime.datetime(1970, 1, 1)).total_seconds())

        for a in albums:
            try:
                # get photos with a real date (not just now)
                datelist = None

                if len(a['photos']) > 0:

                    datelist = [
                        photo.epoch_sysdate for photo in a['photos'] if photo.epoch_sysdate < last2min_epoch]

                    if datelist is not None and len(datelist) > 0:
                        newdate = max(datelist)
                        self.dao.updateAlbumDate(a['id'], newdate)
                        logger.debug(
                            "album %s sysstamp changed to: %s ", a['name'], str(
                                time.strftime(
                                    '%Y-%m-%d %H:%M:%S', time.localtime(newdate))))
            except Exception as e:
                logger.exception(e)
                logger.error("updating album date for album:" + a['name'], e)

    def deleteAllFiles(self):
        """
        Deletes every photo file in Lychee
        Returns nothing
        """
        filelist = []
        photopath = os.path.join(self.conf["lycheepath"], "uploads", "big")
        filelist = [f for f in os.listdir(photopath)]
        self.deleteFiles(filelist)

    def deletePhotos(self, photo_list):
        "photo_list: a list of dictionnary containing key url and id"
        if len(photo_list) > 0:
            url_list = [p['url'] for p in photo_list]
            self.deleteFiles(url_list)
            for p in photo_list:
                self.dao.dropPhoto(p['id'])

    def sync(self):
        """
        Program main loop
        Scans files to add in the sourcedirectory and add them to Lychee
        according to the conf file and given parameters
        Returns nothing
        """

        # Connect db
        # and drop it if dropdb activated
        self.dao = LycheeDAO(self.conf)

        if self.conf['dropdb']:
            self.deleteAllFiles()

        # Load db

        createdalbums = 0
        discoveredphotos = 0
        importedphotos = 0
        album = {}
        albums = []

        album_name_max_width = self.dao.getAlbumNameDBWidth()

        # walkthroug each file / dir of the srcdir
        for root, dirs, files in os.walk(self.conf['srcdir']):

            if sys.version_info.major == 2:
                try:
                    root = root.decode('UTF-8')
                except Exception as e:
                    logger.error(e)
            # Init album data
            album['id'] = None
            album['name'] = None
            album['path'] = None
            album['relpath'] = None  # path relative to srcdir
            album['photos'] = []  # path relative to srcdir

            # if a there is at least one photo in the files
            if any([self.isAPhoto(f) for f in files]):
                album['path'] = root

                # don't know what to do with theses photo
                # and don't wan't to create a default album
                if album['path'] == self.conf['srcdir']:
                    msg = "file at srcdir root won't be added to lychee, please move them in a subfolder: {}".format(
                        root)
                    logger.warn(msg)
                    continue

                # Fill in other album properties
                # albumnames start at srcdir (to avoid absolute path albumname)
                album['relpath'] = os.path.relpath(album['path'], self.conf['srcdir'])
                album['name'] = self.getAlbumNameFromPath(album)

                if len(album['name']) > album_name_max_width:
                    logger.warn("album name too long, will be truncated " + album['name'])
                    album['name'] = album['name'][0:album_name_max_width]
                    logger.warn("album name is now " + album['name'])

                album['id'] = self.dao.albumExists(album)

                if self.conf['replace'] and album['id']:
                    # drop album photos
                    filelist = self.dao.eraseAlbum(album['id'])
                    self.deleteFiles(filelist)
                    assert self.dao.dropAlbum(album['id'])
                    # Album should be recreated
                    album['id'] = False

                if not(album['id']):
                    # create album
                    album['id'] = self.createAlbum(album)

                    if not(album['id']):
                        logger.error("didn't manage to create album for: " + album['relpath'])
                        continue
                    else:
                        logger.info("############ Album created: %s", album['name'])

                    createdalbums += 1

                # Albums are created or emptied, now take care of photos
                for f in sorted(files):

                    if self.isAPhoto(f):
                        try:
                            discoveredphotos += 1
                            error = False
                            logger.debug(
                                "**** Trying to add to lychee album %s: %s",
                                album['name'],
                                os.path.join(
                                    root,
                                    f))
                            # corruption detected here by launching exception
                            photo = LycheePhoto(self.conf, f, album)
                            if not(self.dao.photoExists(photo)):
                                res = self.copyFileToLychee(photo)
                                self.adjustRotation(photo)
                                self.makeThumbnail(photo)
                                res = self.dao.addFileToAlbum(photo)
                                # increment counter
                                if res:
                                    importedphotos += 1
                                    album['photos'].append(photo)
                                else:
                                    error = True
                                    logger.error(
                                        "while adding to album: %s photo: %s",
                                        album['name'],
                                        photo.srcfullpath)
                            else:
                                logger.error(
                                    "photo already exists in this album with same name or same checksum: %s it won't be added to lychee",
                                    photo.srcfullpath)
                                error = True
                        except Exception as e:

                            logger.exception(e)
                            logger.error("could not add %s to album %s", f, album['name'])
                            error = True
                        finally:
                            if not(error):
                                logger.info(
                                    "**** Successfully added %s to lychee album %s",
                                    os.path.join(
                                        root,
                                        f),
                                    album['name'])

                a = album.copy()
                albums.append(a)

        self.updateAlbumsDate(albums)
        if self.conf['sort']:
            self.reorderalbumids(albums)
            self.dao.reinitAlbumAutoIncrement()

        if self.conf['sanity']:

            logger.info("************ SANITY CHECK *************")
            # get All Photos albums
            photos = self.dao.get_all_photos()
            albums = [p['album'] for p in photos]
            albums = set(albums)

            # for each album
            for a_id in albums:
                # check if it exists, if not remove photos
                if not(self.dao.albumIdExists(a_id)):
                    to_delete = self.dao.get_all_photos(a_id)
                    self.dao.eraseAlbum(a_id)
                    file_list = [p['url'] for p in to_delete]
                    self.deleteFiles(file_list)

            # get All Photos
            photos = self.dao.get_all_photos()

            to_delete = []
            # for each photo
            for p in photos:
                delete_photo = False
                # check if big exists
                bigpath = os.path.join(self.conf["lycheepath"], "uploads", "big", p['url'])

                # if big is a link check if it's an orphan
                # file does not exists
                if not(os.path.lexists(bigpath)):
                    logger.error("File does not exists %s: will be delete in db", bigpath)
                    delete_photo = True
                # broken link
                elif not(os.path.exists(bigpath)):
                    logger.error("Link is broken: %s will be delete in db", bigpath)
                    delete_photo = True

                if not(delete_photo):
                    # TODO: check if thumbnail exists
                    pass
                else:
                    # if any of it is False remove and log
                    to_delete.append(p)

            self.deletePhotos(to_delete)

            # Detect broken symlinks / orphan files
            for root, dirs, files in os.walk(os.path.join(self.conf['lycheepath'], 'uploads', 'big')):

                for f in files:
                    logger.debug("check orphan: %s", f)
                    file_name = os.path.basename(f)
                    # check if DB photo exists
                    if not self.dao.photoExistsByName(file_name):
                        # if not delete photo (or link)
                        self.deleteFiles([file_name])
                        logger.info("%s deleted. Wasn't existing in DB", f)

                    # if broken link
                    if os.path.lexists(f) and not(os.path.exists(f)):
                        id = self.dao.photoExistsByName(file_name)
                        # if exists in db
                        if id:
                            ps = {}
                            ps['id'] = id
                            ps['url'] = file_name
                            self.deletePhotos([ps])
                        else:
                            self.deleteFiles([file_name])
                        logger.info("%s deleted. Was a broken link", f)

            # drop empty albums
            empty = self.dao.get_empty_albums()
            if empty:
                for e in empty:
                    self.dao.dropAlbum(e)

        self.dao.close()

        # Final report
        logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
        logger.info("Directory scanned:" + self.conf['srcdir'])
        logger.info("Created albums: " + str(createdalbums))
        if (importedphotos == discoveredphotos):
            logger.info(str(importedphotos) + " photos imported on " + str(discoveredphotos) + " discovered")
        else:
            logger.error(str(importedphotos) + " photos imported on " + str(discoveredphotos) + " discovered")
        logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
Example #11
0
class MyEventHandler(FileSystemEventHandler):
    def catch_all_handler(self, event):
        return

    def on_moved(self, event):
        borg = ConfBorg()
        self.conf = borg.conf
        self.dao = LycheeDAO(self.conf)

        if event.is_directory:

            albSrc = getAlbum(self, event.src_path)
            albDest = getAlbum(self, event.dest_path)
            logger.info("%s Album moved to %s. ", event.src_path,
                        event.dest_path)
            self.dao.setAlbumParentAndTitle(albDest['name'], albDest['parent'],
                                            albSrc['id'])
            return
        else:
            if match_path(
                    event.src_path,
                    included_patterns=['*.jpg', '*.jpeg', '*.gif', '*.png'],
                    excluded_patterns=None,
                    case_sensitive=False):
                dirs = event.src_path.split(os.sep)
                albDir = os.sep.join(dirs[:-1])
                dirs2 = event.dest_path.split(os.sep)
                albDir2 = os.sep.join(dirs2[:-1])
                album = getAlbum(self, albDir)
                if album['id'] == None:
                    album = getAlbum(self, albDir2)

                dbPhoto = self.dao.get_photo_light(album['id'],
                                                   os.sep.join(dirs[-1:]), "")

                album2 = getAlbum(self, albDir2)
                logger.info("%s Photo moved to %s. ", event.src_path,
                            event.dest_path)
                self.dao.setPhotoAlbumAndTitle(os.sep.join(dirs2[-1:]),
                                               album2['id'], dbPhoto['id'])

            return

    def on_created(self, event):

        borg = ConfBorg()
        self.conf = borg.conf
        self.dao = LycheeDAO(self.conf)

        if event.is_directory:
            album = getAlbum(self, event.src_path)
            logger.info("Created album: %s.", album['name'])
            self.dao.createAlbum(album)
            return

        else:
            if match_path(
                    event.src_path,
                    included_patterns=['*.jpg', '*.jpeg', '*.gif', '*.png'],
                    excluded_patterns=None,
                    case_sensitive=False):
                dirs = event.src_path.split(os.sep)
                albDir = os.sep.join(dirs[:-1])
                album = getAlbum(self, albDir)
                album['path'] = albDir
                photo = LycheePhoto(self.conf, os.sep.join(dirs[-1:]), album)
                if not (self.dao.photoExists(photo)):
                    res = copyFileToLychee(self, photo)

                    adjustRotation(self, photo)
                    makeThumbnail(self, photo)
                    res = self.dao.addFileToAlbum(photo)
                    logger.info("Created Photo: %s.", photo.srcfullpath)
                    # increment counter
                    if not res:
                        logger.error("while adding to album: %s photo: %s",
                                     album['name'], photo.srcfullpath)
                else:
                    logger.error(
                        "photo already exists in this album with same name or same checksum: %s it won't be added to lychee",
                        photo.srcfullpath)
            return

    def on_deleted(self, event):
        borg = ConfBorg()
        self.conf = borg.conf
        self.dao = LycheeDAO(self.conf)
        if event.is_directory:
            album = getAlbum(self, event.src_path)
            if album['id'] is not None:
                filelist = self.dao.eraseAlbum(album['id'])
                deleteFiles(self, filelist)
                logger.info("Deleted album: %s.", album['name'])
                assert self.dao.dropAlbum(album['id'])
            else:
                logger.error(
                    "Tried to delete album: %s, but it wasn't present in the DB",
                    album['name'])

            return
        else:
            if match_path(
                    event.src_path,
                    included_patterns=['*.jpg', '*.jpeg', '*.gif', '*.png'],
                    excluded_patterns=None,
                    case_sensitive=False):
                dirs = event.src_path.split(os.sep)
                albDir = os.sep.join(dirs[:-1])
                album = getAlbum(self, albDir)
                album['path'] = albDir
                dbPhoto = self.dao.get_photo_light(album['id'],
                                                   os.sep.join(dirs[-1:]), "")
                if dbPhoto is not None:
                    delete = [dbPhoto]
                    deletePhotos(self, delete)
                    logger.info("Deleted Photo: %s.", os.sep.join(dirs[-1:]))
                else:
                    logger.info(
                        "Tried to delete Photo: %s, but it wasn't in the database.",
                        os.sep.join(dirs[-1:]))
            return

    def on_modified(self, event):
        borg = ConfBorg()
        self.conf = borg.conf
        self.dao = LycheeDAO(self.conf)
        if event.is_directory:
            return
        else:
            if match_path(
                    event.src_path,
                    included_patterns=['*.jpg', '*.jpeg', '*.gif', '*.png'],
                    excluded_patterns=None,
                    case_sensitive=False):
                dirs = event.src_path.split(os.sep)
                albDir = os.sep.join(dirs[:-1])
                album = getAlbum(self, albDir)
                album['path'] = albDir
                photo = LycheePhoto(self.conf, os.sep.join(dirs[-1:]), album)
                dbPhoto = self.dao.get_photo(photo)
                if dbPhoto is not None:
                    delete = [dbPhoto]
                    deletePhotos(self, delete)

                dirs = event.src_path.split(os.sep)
                albDir = os.sep.join(dirs[:-1])
                album = getAlbum(self, albDir)
                album['path'] = albDir
                photo = LycheePhoto(self.conf, os.sep.join(dirs[-1:]), album)
                if not (self.dao.photoExists(photo)):
                    res = copyFileToLychee(self, photo)

                    adjustRotation(self, photo)
                    makeThumbnail(self, photo)
                    res = self.dao.addFileToAlbum(photo)
                    logger.info("Modified Photo: %s.", photo.srcfullpath)
                    # increment counter
                    if not res:
                        logger.error("while adding to album: %s photo: %s",
                                     album['name'], photo.srcfullpath)
                else:
                    logger.error(
                        "photo already exists in this album with same name or same checksum: %s it won't be added to lychee",
                        photo.srcfullpath)
            return
Example #12
0
class LycheeSyncer:
    """
    This class contains the logic behind this program
    It consist mainly in filesystem operations
    It relies on:
    - LycheeDAO for dtabases operations
    - LycheePhoto to store (and compute) photos propreties
    """

    conf = {}

    def __init__(self):
        """
        Takes a dictionnary of conf as input
        """
        borg = ConfBorg()
        self.conf = borg.conf

    def deleteFiles(self, filelist):
        """
        Delete files in the Lychee file tree (uploads/big and uploads/thumbnails)
        Give it the file name and it will delete relatives files and thumbnails
        Parameters:
        - filelist: a list of filenames
        Returns nothing
        """

        for url in filelist:
            if isAPhoto(self, url):
                thumbpath = os.path.join(self.conf["lycheepath"], "uploads",
                                         "thumb", url)
                filesplit = os.path.splitext(url)
                thumb2path = ''.join([filesplit[0], "@2x",
                                      filesplit[1]]).lower()
                thumb2path = os.path.join(self.conf["lycheepath"], "uploads",
                                          "thumb", thumb2path)
                bigpath = os.path.join(self.conf["lycheepath"], "uploads",
                                       "big", url)
                remove_file(thumbpath)
                remove_file(thumb2path)
                remove_file(bigpath)

    def deleteAllFiles(self):
        """
        Deletes every photo file in Lychee
        Returns nothing
        """
        photopath = os.path.join(self.conf["lycheepath"], "uploads", "big")
        filelist = [f for f in os.listdir(photopath)]
        self.deleteFiles(filelist)

    def sync(self):
        """
        Program main loop
        Scans files to add in the sourcedirectory and add them to Lychee
        according to the conf file and given parameters
        Returns nothing
        """

        # Connect db
        # and drop it if dropdb activated
        self.dao = LycheeDAO(self.conf)
        if self.conf['dropdb']:
            self.deleteAllFiles()
            # Load db

            createdalbums = 0
            discoveredphotos = 0
            importedphotos = 0
            album = {}
            albums = []

            album_name_max_width = self.dao.getAlbumNameDBWidth()

            # walkthroug each file / dir of the srcdir
            for root, dirs, files in os.walk(self.conf['srcdir']):

                if sys.version_info.major == 2:
                    try:
                        root = root.decode('UTF-8')
                    except Exception as e:
                        logger.error(e)
                # Init album data
                album['id'] = None
                album['name'] = None
                album['path'] = None
                album['relpath'] = None  # path relative to srcdir
                album['parent'] = "0"
                album['photos'] = []  # path relative to srcdir
                album = getAlbum(self, root)
                # if a there is at least one photo in the files

                album['path'] = root
                # don't know what to do with theses photo
                # and don't wan't to create a default album
                if album['path'] == self.conf['srcdir']:
                    msg = "file at srcdir root won't be added to lychee, please move them in a subfolder: {}".format(
                        root)
                    logger.warn(msg)
                    continue

                album['id'] = self.dao.albumExistsByNameAndParent(
                    album['name'], album['parent'])

                if self.conf['replace'] and album['id']:
                    # drop album photos
                    filelist = self.dao.eraseAlbum(album['id'])
                    self.deleteFiles(filelist)
                    assert self.dao.dropAlbum(album['id'])
                    # Album should be recreated
                    album['id'] = False

                if not (album['id']):
                    # create album
                    album['id'] = createAlbum(self, album)

                    if not (album['id']):
                        logger.error("didn't manage to create album for: " +
                                     album['name'])
                        continue
                    else:
                        logger.info("############ Album created: %s",
                                    album['name'])

                    createdalbums += 1

                # Albums are created or emptied, now take care of photos
                for f in sorted(files):

                    if isAPhoto(self, f):
                        try:
                            discoveredphotos += 1
                            error = False
                            logger.debug(
                                "**** Trying to add to lychee album %s: %s",
                                album['name'], os.path.join(root, f))
                            # corruption detected here by launching exception
                            photo = LycheePhoto(self.conf, f, album)
                            if not (self.dao.photoExists(photo)):
                                res = copyFileToLychee(self, photo)
                                adjustRotation(self, photo)
                                makeThumbnail(self, photo)
                                res = self.dao.addFileToAlbum(photo)
                                # increment counter
                                if res:
                                    importedphotos += 1
                                    album['photos'].append(photo)
                                else:
                                    error = True
                                    logger.error(
                                        "while adding to album: %s photo: %s",
                                        album['name'], photo.srcfullpath)
                            else:
                                logger.warn(
                                    "photo already exists in this album with same name or same checksum: %s it won't be added to lychee",
                                    photo.srcfullpath)
                                error = True
                        except Exception as e:

                            logger.exception(e)
                            logger.error("could not add %s to album %s", f,
                                         album['name'])
                            error = True
                        finally:
                            if not (error):
                                logger.info(
                                    "**** Successfully added %s to lychee album %s",
                                    os.path.join(root, f), album['name'])

                a = album.copy()
                albums.append(a)
                logger.info(
                    "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
                logger.info("Directory scanned:" + self.conf['srcdir'])
                logger.info("Created albums: " + str(createdalbums))
                if (importedphotos == discoveredphotos):
                    logger.info(
                        str(importedphotos) + " photos imported on " +
                        str(discoveredphotos) + " discovered")
                else:
                    logger.error(
                        str(importedphotos) + " photos imported on " +
                        str(discoveredphotos) + " discovered")
                logger.info(
                    "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
            updateAlbumsDate(self, albums)
        if self.conf['sort']:
            reorderalbumids(self, albums)
            self.dao.reinitAlbumAutoIncrement()

        if self.conf['sanity']:

            logger.info("************ SANITY CHECK *************")
            # get All Photos albums
            photos = self.dao.get_all_photos()
            albums = [p['album'] for p in photos]
            albums = set(albums)

            # for each album
            for a_id in albums:
                # check if it exists, if not remove photos
                if not (self.dao.albumIdExists(a_id)):
                    to_delete = self.dao.get_all_photos(a_id)
                    self.dao.eraseAlbum(a_id)
                    file_list = [p['url'] for p in to_delete]
                    self.deleteFiles(file_list)

            # get All Photos
            photos = self.dao.get_all_photos()

            to_delete = []
            # for each photo
            for p in photos:
                delete_photo = False
                # check if big exists
                bigpath = os.path.join(self.conf["lycheepath"], "uploads",
                                       "big", p['url'])

                # if big is a link check if it's an orphan
                # file does not exists
                if not (os.path.lexists(bigpath)):
                    logger.error(
                        "File does not exists %s: will be delete in db",
                        bigpath)
                    delete_photo = True
                # broken link
                elif not (os.path.exists(bigpath)):
                    logger.error("Link is broken: %s will be delete in db",
                                 bigpath)
                    delete_photo = True

                if not (delete_photo):
                    # TODO: check if thumbnail exists
                    pass
                else:
                    # if any of it is False remove and log
                    to_delete.append(p)

            deletePhotos(self, to_delete)

            # Detect broken symlinks / orphan files
            for root, dirs, files in os.walk(
                    os.path.join(self.conf['lycheepath'], 'uploads', 'big')):

                for f in files:
                    logger.debug("check orphan: %s", f)
                    file_name = os.path.basename(f)
                    # check if DB photo exists
                    if not self.dao.photoExistsByName(file_name):
                        # if not delete photo (or link)
                        self.deleteFiles([file_name])
                        logger.info("%s deleted. Wasn't existing in DB", f)

                    # if broken link
                    if os.path.lexists(f) and not (os.path.exists(f)):
                        id = self.dao.photoExistsByName(file_name)
                        # if exists in db
                        if id:
                            ps = {'id': id, 'url': file_name}
                            deletePhotos(self, [ps])
                        else:
                            self.deleteFiles([file_name])
                        logger.info("%s deleted. Was a broken link", f)

            # drop empty albums
            empty = self.dao.get_empty_albums()
            if empty:
                for e in empty:
                    self.dao.dropAlbum(e)

        self.dao.close()
        if self.conf['watch']:
            event_handler = MyEventHandler()

            observer = Observer()
            observer.schedule(event_handler,
                              self.conf['srcdir'],
                              recursive=True)
            observer.start()
            try:
                while True:
                    time.sleep(1)
            except KeyboardInterrupt:
                observer.stop()
            observer.join()
Example #13
0
class MyEventHandler(FileSystemEventHandler):

    def catch_all_handler(self, event):
        return

    def on_moved(self, event):
        borg = ConfBorg()
        self.conf = borg.conf
        self.dao = LycheeDAO(self.conf)

        if event.is_directory:

            albSrc = getAlbum(self, event.src_path)
            albDest = getAlbum(self, event.dest_path)
            logger.info("%s Album moved to %s. ", event.src_path, event.dest_path)
            self.dao.setAlbumParentAndTitle(albDest['name'], albDest['parent'], albSrc['id'])
            return
        else:
            if match_path(event.src_path,
                          included_patterns=['*.jpg', '*.jpeg', '*.gif', '*.png'],
                          excluded_patterns=None,
                          case_sensitive=False):
                dirs = event.src_path.split(os.sep)
                albDir = os.sep.join(dirs[:-1])
                dirs2 = event.dest_path.split(os.sep)
                albDir2 = os.sep.join(dirs2[:-1])
                album = getAlbum(self, albDir)
                if album['id'] == None:
                    album = getAlbum(self, albDir2)

                dbPhoto = self.dao.get_photo_light(album['id'], os.sep.join(dirs[-1:]), "")

                album2 = getAlbum(self, albDir2)
                logger.info("%s Photo moved to %s. ", event.src_path, event.dest_path)
                self.dao.setPhotoAlbumAndTitle(os.sep.join(dirs2[-1:]), album2['id'], dbPhoto['id'])



            return

    def on_created(self, event):

        borg = ConfBorg()
        self.conf = borg.conf
        self.dao = LycheeDAO(self.conf)

        if event.is_directory:
            album = getAlbum(self, event.src_path)
            logger.info("Created album: %s.", album['name'])
            self.dao.createAlbum(album)
            return

        else:
            if match_path(event.src_path,
                          included_patterns=['*.jpg', '*.jpeg', '*.gif', '*.png'],
                          excluded_patterns=None,
                          case_sensitive=False):
                dirs = event.src_path.split(os.sep)
                albDir = os.sep.join(dirs[:-1])
                album = getAlbum(self, albDir)
                album['path'] = albDir
                photo = LycheePhoto(self.conf, os.sep.join(dirs[-1:]), album)
                if not (self.dao.photoExists(photo)):
                    res = copyFileToLychee(self, photo)

                    adjustRotation(self, photo)
                    makeThumbnail(self, photo)
                    res = self.dao.addFileToAlbum(photo)
                    logger.info("Created Photo: %s.", photo.srcfullpath)
                    # increment counter
                    if not res:
                        logger.error(
                            "while adding to album: %s photo: %s",
                            album['name'],
                            photo.srcfullpath)
                else:
                    logger.error(
                        "photo already exists in this album with same name or same checksum: %s it won't be added to lychee",
                        photo.srcfullpath)
            return

    def on_deleted(self, event):
        borg = ConfBorg()
        self.conf = borg.conf
        self.dao = LycheeDAO(self.conf)
        if event.is_directory:
            album = getAlbum(self, event.src_path)
            if album['id'] is not None:
                filelist = self.dao.eraseAlbum(album['id'])
                deleteFiles(self, filelist)
                logger.info("Deleted album: %s.", album['name'])
                assert self.dao.dropAlbum(album['id'])
            else:
                logger.error("Tried to delete album: %s, but it wasn't present in the DB", album['name'])

            return
        else:
            if match_path(event.src_path,
                          included_patterns=['*.jpg', '*.jpeg', '*.gif', '*.png'],
                          excluded_patterns=None,
                          case_sensitive=False):
                dirs = event.src_path.split(os.sep)
                albDir = os.sep.join(dirs[:-1])
                album = getAlbum(self, albDir)
                album['path'] = albDir
                dbPhoto = self.dao.get_photo_light(album['id'], os.sep.join(dirs[-1:]), "")
                if dbPhoto is not None:
                    delete = [dbPhoto]
                    deletePhotos(self, delete)
                    logger.info("Deleted Photo: %s.", os.sep.join(dirs[-1:]))
                else:
                    logger.info("Tried to delete Photo: %s, but it wasn't in the database.", os.sep.join(dirs[-1:]))
            return

    def on_modified(self, event):
        borg = ConfBorg()
        self.conf = borg.conf
        self.dao = LycheeDAO(self.conf)
        if event.is_directory:
            return
        else:
            if match_path(event.src_path,
                          included_patterns=['*.jpg', '*.jpeg', '*.gif', '*.png'],
                          excluded_patterns=None,
                          case_sensitive=False):
                dirs = event.src_path.split(os.sep)
                albDir = os.sep.join(dirs[:-1])
                album = getAlbum(self, albDir)
                album['path'] = albDir
                photo = LycheePhoto(self.conf, os.sep.join(dirs[-1:]), album)
                dbPhoto = self.dao.get_photo(photo)
                if dbPhoto is not None:
                    delete = [dbPhoto]
                    deletePhotos(self, delete)

                dirs = event.src_path.split(os.sep)
                albDir = os.sep.join(dirs[:-1])
                album = getAlbum(self, albDir)
                album['path'] = albDir
                photo = LycheePhoto(self.conf, os.sep.join(dirs[-1:]), album)
                if not (self.dao.photoExists(photo)):
                    res = copyFileToLychee(self, photo)

                    adjustRotation(self, photo)
                    makeThumbnail(self, photo)
                    res = self.dao.addFileToAlbum(photo)
                    logger.info("Modified Photo: %s.", photo.srcfullpath)
                    # increment counter
                    if not res:
                        logger.error(
                            "while adding to album: %s photo: %s",
                            album['name'],
                            photo.srcfullpath)
                else:
                    logger.error(
                        "photo already exists in this album with same name or same checksum: %s it won't be added to lychee",
                        photo.srcfullpath)
            return
Example #14
0
class LycheeSyncer:
    """
    This class contains the logic behind this program
    It consist mainly in filesystem operations
    It relies on:
    - LycheeDAO for dtabases operations
    - LycheePhoto to store (and compute) photos propreties
    """

    conf = {}

    def __init__(self):
        """
        Takes a dictionnary of conf as input
        """
        borg = ConfBorg()
        self.conf = borg.conf

    def deleteFiles(self, filelist):
        """
        Delete files in the Lychee file tree (uploads/big and uploads/thumbnails)
        Give it the file name and it will delete relatives files and thumbnails
        Parameters:
        - filelist: a list of filenames
        Returns nothing
        """

        for url in filelist:
            if isAPhoto(self, url):
                thumbpath = os.path.join(self.conf["lycheepath"], "uploads", "thumb", url)
                filesplit = os.path.splitext(url)
                thumb2path = ''.join([filesplit[0], "@2x", filesplit[1]]).lower()
                thumb2path = os.path.join(self.conf["lycheepath"], "uploads", "thumb", thumb2path)
                bigpath = os.path.join(self.conf["lycheepath"], "uploads", "big", url)
                remove_file(thumbpath)
                remove_file(thumb2path)
                remove_file(bigpath)

    def deleteAllFiles(self):
        """
        Deletes every photo file in Lychee
        Returns nothing
        """
        photopath = os.path.join(self.conf["lycheepath"], "uploads", "big")
        filelist = [f for f in os.listdir(photopath)]
        self.deleteFiles(filelist)

    def sync(self):
        """
        Program main loop
        Scans files to add in the sourcedirectory and add them to Lychee
        according to the conf file and given parameters
        Returns nothing
        """

        # Connect db
        # and drop it if dropdb activated
        self.dao = LycheeDAO(self.conf)
        if self.conf['dropdb']:
            self.deleteAllFiles()
            # Load db

            createdalbums = 0
            discoveredphotos = 0
            importedphotos = 0
            album = {}
            albums = []

            album_name_max_width = self.dao.getAlbumNameDBWidth()

            # walkthroug each file / dir of the srcdir
            for root, dirs, files in os.walk(self.conf['srcdir']):

                if sys.version_info.major == 2:
                    try:
                        root = root.decode('UTF-8')
                    except Exception as e:
                        logger.error(e)
                # Init album data
                album['id'] = None
                album['name'] = None
                album['path'] = None
                album['relpath'] = None  # path relative to srcdir
                album['parent'] = "0"
                album['photos'] = []# path relative to srcdir
                album = getAlbum(self, root)
                # if a there is at least one photo in the files

                album['path'] = root
                # don't know what to do with theses photo
                # and don't wan't to create a default album
                if album['path'] == self.conf['srcdir']:
                    msg = "file at srcdir root won't be added to lychee, please move them in a subfolder: {}".format(
                        root)
                    logger.warn(msg)
                    continue

                album['id'] = self.dao.albumExistsByNameAndParent(album['name'], album['parent'])

                if self.conf['replace'] and album['id']:
                    # drop album photos
                    filelist = self.dao.eraseAlbum(album['id'])
                    self.deleteFiles(filelist)
                    assert self.dao.dropAlbum(album['id'])
                    # Album should be recreated
                    album['id'] = False

                if not (album['id']):
                    # create album
                    album['id'] = createAlbum(self, album)

                    if not (album['id']):
                        logger.error("didn't manage to create album for: " + album['name'])
                        continue
                    else:
                        logger.info("############ Album created: %s", album['name'])

                    createdalbums += 1

                # Albums are created or emptied, now take care of photos
                for f in sorted(files):

                    if isAPhoto(self, f):
                        try:
                            discoveredphotos += 1
                            error = False
                            logger.debug(
                                "**** Trying to add to lychee album %s: %s",
                                album['name'],
                                os.path.join(
                                    root,
                                    f))
                            # corruption detected here by launching exception
                            photo = LycheePhoto(self.conf, f, album)
                            if not (self.dao.photoExists(photo)):
                                res = copyFileToLychee(self, photo)
                                adjustRotation(self, photo)
                                makeThumbnail(self, photo)
                                res = self.dao.addFileToAlbum(photo)
                                # increment counter
                                if res:
                                    importedphotos += 1
                                    album['photos'].append(photo)
                                else:
                                    error = True
                                    logger.error(
                                        "while adding to album: %s photo: %s",
                                        album['name'],
                                        photo.srcfullpath)
                            else:
                                logger.warn(
                                    "photo already exists in this album with same name or same checksum: %s it won't be added to lychee",
                                    photo.srcfullpath)
                                error = True
                        except Exception as e:

                            logger.exception(e)
                            logger.error("could not add %s to album %s", f, album['name'])
                            error = True
                        finally:
                            if not (error):
                                logger.info(
                                    "**** Successfully added %s to lychee album %s",
                                    os.path.join(
                                        root,
                                        f),
                                    album['name'])

                a = album.copy()
                albums.append(a)
                logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
                logger.info("Directory scanned:" + self.conf['srcdir'])
                logger.info("Created albums: " + str(createdalbums))
                if (importedphotos == discoveredphotos):
                    logger.info(
                        str(importedphotos) + " photos imported on " + str(discoveredphotos) + " discovered")
                else:
                    logger.error(
                        str(importedphotos) + " photos imported on " + str(discoveredphotos) + " discovered")
                logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
            updateAlbumsDate(self, albums)
        if self.conf['sort']:
            reorderalbumids(self, albums)
            self.dao.reinitAlbumAutoIncrement()

        if self.conf['sanity']:

            logger.info("************ SANITY CHECK *************")
            # get All Photos albums
            photos = self.dao.get_all_photos()
            albums = [p['album'] for p in photos]
            albums = set(albums)

            # for each album
            for a_id in albums:
                # check if it exists, if not remove photos
                if not (self.dao.albumIdExists(a_id)):
                    to_delete = self.dao.get_all_photos(a_id)
                    self.dao.eraseAlbum(a_id)
                    file_list = [p['url'] for p in to_delete]
                    self.deleteFiles(file_list)

            # get All Photos
            photos = self.dao.get_all_photos()

            to_delete = []
            # for each photo
            for p in photos:
                delete_photo = False
                # check if big exists
                bigpath = os.path.join(self.conf["lycheepath"], "uploads", "big", p['url'])

                # if big is a link check if it's an orphan
                # file does not exists
                if not (os.path.lexists(bigpath)):
                    logger.error("File does not exists %s: will be delete in db", bigpath)
                    delete_photo = True
                # broken link
                elif not (os.path.exists(bigpath)):
                    logger.error("Link is broken: %s will be delete in db", bigpath)
                    delete_photo = True

                if not (delete_photo):
                    # TODO: check if thumbnail exists
                    pass
                else:
                    # if any of it is False remove and log
                    to_delete.append(p)

            deletePhotos(self, to_delete)

            # Detect broken symlinks / orphan files
            for root, dirs, files in os.walk(os.path.join(self.conf['lycheepath'], 'uploads', 'big')):

                for f in files:
                    logger.debug("check orphan: %s", f)
                    file_name = os.path.basename(f)
                    # check if DB photo exists
                    if not self.dao.photoExistsByName(file_name):
                        # if not delete photo (or link)
                        self.deleteFiles([file_name])
                        logger.info("%s deleted. Wasn't existing in DB", f)

                    # if broken link
                    if os.path.lexists(f) and not (os.path.exists(f)):
                        id = self.dao.photoExistsByName(file_name)
                        # if exists in db
                        if id:
                            ps = {'id': id, 'url': file_name}
                            deletePhotos(self, [ps])
                        else:
                            self.deleteFiles([file_name])
                        logger.info("%s deleted. Was a broken link", f)

            # drop empty albums
            empty = self.dao.get_empty_albums()
            if empty:
                for e in empty:
                    self.dao.dropAlbum(e)

        self.dao.close()
        if self.conf['watch']:
            event_handler = MyEventHandler()

            observer = Observer()
            observer.schedule(event_handler, self.conf['srcdir'], recursive=True)
            observer.start()
            try:
                while True:
                    time.sleep(1)
            except KeyboardInterrupt:
                observer.stop()
            observer.join()