Ejemplo n.º 1
0
    def __init__(self, base_dir):
        self.base_dir = normpath(base_dir)
        self.fs = FileSystem(log.debug)
        self._cache = {}

        db_path = self._db_path_from_base_dir(self.base_dir)
        if exists(db_path):  # Otherwise `.create()` will set `self.db`.
            self.db = Database(db_path)
Ejemplo n.º 2
0
class WorkingCopy(object):
    """API for a pics working copy directory.

    Usage:
        # Create a new working copy directory (used by `pics co`).
        wc = WorkingCopy.create(...)

        # Or, for an existing working copy.
        wc = WorkingCopy(base_dir)
    """
    VERSION = "1.0.0"

    @staticmethod
    def _db_path_from_base_dir(base_dir):
        return join(base_dir, ".pics", "photos.sqlite3")

    def __init__(self, base_dir):
        self.base_dir = normpath(base_dir)
        self.fs = FileSystem(log.debug)
        self._cache = {}

        db_path = self._db_path_from_base_dir(self.base_dir)
        if exists(db_path):  # Otherwise `.create()` will set `self.db`.
            self.db = Database(db_path)

    @classmethod
    def create(cls, base_dir, ilk, user, base_date=None, size="original",
            permission="all"):
        """Create a working copy and return a `WorkingCopy` instance for it.

        @param base_dir {str} The base directory for the working copy.
        @param ilk {str} The type of the pics repo. Currently only
            "flickr" is supported.
        @param user {str} Username of the pics repo user.
        @param base_date {datetime.date} The date (UTC) from which to
            start getting photos. If not given, then all photos for
            that user are retrieved.
        @param size {str} The name of photos sizes to download. Supported
            values are: "small", "medium" and "original". The actual size
            that the former two mean depends on the pics repository. Default
            is "original".
            TODO: specify the sizes for flickr.
        @param permission {str} Photo permissions to retrieve.
            Valid values:
                'all' (default) all photos,
                'family' only photos that family would see,
                'friend' only photos that friends would see,
                'public' only public photos
        @returns {WorkingCopy} The working copy instance.
        """
        # Sanity checks.
        assert ilk == "flickr", "unknown pics repo ilk: %r" % ilk
        assert isinstance(base_date, (type(None), datetime.date))
        assert size in ("small", "medium", "original")
        assert permission in ("all", "family", "friend", "public")
        if exists(base_dir):
            raise PicsError("cannot create working copy: `%s' exists" % base_dir)

        self = cls(base_dir)

        # Create base structure.
        #TODO: assert dirname(base_dir) exists?
        self.fs.mkdir(self.base_dir, parents=True)
        d = join(self.base_dir, ".pics")
        self.fs.mkdir(d, hidden=True)
        open(join(d, "version"), 'w').write(self.VERSION+'\n')

        # Main working copy database.
        db_path = self._db_path_from_base_dir(self.base_dir)
        self.db = Database(db_path)
        with self.db.connect(True) as cu:
            self.db.set_meta("ilk", ilk)
            self.db.set_meta("user", user)
            self.db.set_meta("size", size)
            self.db.set_meta("permission", permission)
            if base_date:
                self.db.set_meta("base_date", base_date)

        return self

    @property
    def ilk(self):
        return self.db.get_meta("ilk")

    @property
    def user(self):
        return self.db.get_meta("user")

    @property
    def size(self):
        return self.db.get_meta("size")

    @property
    def base_date(self):
        """The base date (UTC) of this working copy. I.e. the first date
        for which photos are retrieved.
        """
        if "base_date" not in self._cache:
            s = self.db.get_meta("base_date")
            if s is None:
                self._cache["base_date"] = None
            else:
                t = datetime.datetime.strptime(s, "%Y-%m-%d")
                self._cache["base_date"] = datetime.date(t.year, t.month, t.day)
        return self._cache["base_date"]

    @property
    def version(self):
        if "version" not in self._cache:
            version_path = join(self.base_dir, ".pics", "version")
            self._cache["version"] = open(version_path, 'r').read().strip()
        return self._cache["version"]

    @property
    def version_info(self):
        return tuple(map(int, self.version.split('.')))

    def __repr__(self):
        p = utils.nicepath(self.base_dir, prefer_absolute=True)
        return "<WorkingCopy `%s'>" % p

    @property
    def api(self):
        if self._api_cache is None:
            self._api_cache = simpleflickrapi.SimpleFlickrAPI(
                utils.get_flickr_api_key(), utils.get_flickr_secret())
            #TODO: For now 'pics' is just read-only so this is good
            #      enough. However, eventually we'll want separate
            #      `self.read_api', `self.write_api' and
            #      `self.delete_api' or similar mechanism.
            #TODO: cache this auth token in the pics user data dir
            self._api_cache.get_auth_token("read")
        return self._api_cache
    _api_cache = None

    # Getter and setter for `last-update`, a `datetime.datetime` field for
    # the latest photo update sync'd to the working copy.
    _last_update_cache = None
    def get_last_update(self, cu=None):
        if self._last_update_cache is None:
            last_update_str = self.db.get_meta("last-update", cu=cu)
            if last_update_str is None:
                self._last_update_cache = None
            else:
                self._last_update_cache = datetime.datetime.strptime(
                    last_update_str, "%Y-%m-%d %H:%M:%S")
        return self._last_update_cache
    def set_last_update(self, value, cu=None):
        curr_last_update = self.get_last_update(cu=cu)
        if curr_last_update is None or value > curr_last_update:
            self.db.set_meta("last-update",
                value.strftime("%Y-%m-%d %H:%M:%S"), cu=cu)
            self._last_update_cache = value

    def _add_photo(self, id, dry_run=False):
        """Add the given photo to the working copy."""
        # Gather necessary info.
        info = self.api.photos_getInfo(photo_id=id)[0]  # <photo> elem
        datedir = info.find("dates").get("taken")[:7]
        dir = join(self.base_dir, datedir)
        url, filename = self._download_info_from_info(info, size=self.size)
        path = join(dir, filename)
        title = info.findtext("title")
        log.info("A  %s  [%s]", path,
                 utils.one_line_summary_from_text(title, 40))
        last_update = _photo_last_update_from_info(info)

        if not dry_run:
            # Create the dirs, as necessary.
            pics_dir = join(dir, ".pics")
            if not exists(dir):
                self.fs.mkdir(dir)
            if not exists(pics_dir):
                self.fs.mkdir(pics_dir, hidden=True)

            # Get the photo itself.
            #TODO: add a reporthook for progressbar (unless too quick to bother)
            #TODO: handle ContentTooShortError (py2.5)
            fname, headers = urllib.urlretrieve(url, path)
            mtime = utils.timestamp_from_datetime(last_update)
            os.utime(path, (mtime, mtime))

            # Gather and save all metadata.
            if _photo_num_comments_from_info(info):
                comments = self.api.photos_comments_getList(photo_id=id)[0]
            else:
                comments = None
            self._save_photo_data(dir, id, info, comments)
        return datedir, last_update

    def _fetch_info_from_photo_id(self, id):
        info = self.api.photos_getInfo(photo_id=id)[0]  # <photo> elem
        # Drop tail for canonicalization to allow diffing of the
        # serialized XML.
        info.tail = None
        return info

    def _update_photo(self, id, local_datedir, local_info, dry_run=False):
        """Update the given photo in the working copy."""
        info = self._fetch_info_from_photo_id(id)
        datedir = info.find("dates").get("taken")[:7]
        last_update = _photo_last_update_from_info(info)

        # Figure out what work needs to be done.
        # From *experimentation* it looks like the "secret" attribute
        # changes if the photo itself changes (i.e. is replaced or
        # "Edited" or rotated).
        todos = []
        if datedir != local_datedir:
            log.debug("update %s: datedir change: %r -> %r",
                      id, local_datedir, datedir)
            todos.append("remove-old")
            todos.append("photo")
        elif info.get("secret") != local_info.get("secret"):
            log.debug("update %s: photo secret change: %r -> %r",
                      id, local_info.get("secret"), info.get("secret"))
            todos.append("photo")
        todos.append("info")
        if _photo_num_comments_from_info(info):
            todos.append("comments")
        if not todos:
            return datedir, last_update

        # Do the necessary updates.
        XXX # Figure out ext for videos. _download_info_from_info is wrong
            # here b/c is isn't using  *local* info.
        url, filename = self._download_info_from_info(info, size=self.size)
        #XXX
        #size_ext = (self.size != "original" and "."+self.size or "")
        #ext = (self.size != "original" and ".jpg"
        #       or "."+local_info.get("originalformat"))

        # - Remove the old bits, if the datedir has changed.
        if "remove-old" in todos:
            d = join(self.base_dir, local_datedir)
            path = join(d, filename)
            log.info("D  %s  [%s]", path,
                utils.one_line_summary_from_text(local_info.findtext("title"), 40))
            if not dry_run:
                log.debug("rm `%s'", path)
                os.remove(path)
                self._remove_photo_data(d, id)
                remaining_paths = set(os.listdir(d))
                remaining_paths.difference_update(set([".pics"]))
                if not remaining_paths:
                    log.info("D  %s", d)
                    self.fs.rm(d)

        # - Add the new stuff.
        d = join(self.base_dir, datedir)
        action_str = ("photo" in todos and "U " or " u")
        path = join(d, filename)
        log.info("%s %s  [%s]", action_str, path,
            utils.one_line_summary_from_text(info.findtext("title"), 40))
        if not dry_run:
            if not exists(d):
                self.fs.mkdir(d)
                pics_dir = join(d, ".pics")
                if not exists(pics_dir):
                    self.fs.mkdir(pics_dir, hidden=True)
            if "photo" in todos:
                fname, headers = urllib.urlretrieve(url, path)
                mtime = utils.timestamp_from_datetime(last_update)
                os.utime(path, (mtime, mtime))
            if "comments" in todos:
                comments = self.api.photos_comments_getList(photo_id=id)[0]
            else:
                comments = None
            self._save_photo_data(d, id, info, comments=comments)

        #print "... %s" % id
        #print "originalsecret: %s <- %s" % (info.get("originalsecret"), local_info.get("originalsecret"))
        #print "secret: %s <- %s" % (info.get("secret"), local_info.get("secret"))
        #print "rotation: %s <- %s" % (info.get("rotation"), local_info.get("rotation"))
        return datedir, last_update

    def check_version(self):
        if self.version != self.VERSION:
            raise PicsError("out of date working copy (v%s != v%s): you must "
                            "first upgrade", self.version, self.VERSION)

    def _remove_photo_data(self, dir, id):
        #TODO: 'dir' correct here? need to use self.base_dir?
        paths = [join(dir, ".pics", "%s-%s.xml" % (id, name))
                 for name in ("info", "comments")]
        for path in paths:
            if exists(path):
                log.debug("remove photo data: `%s'", path)
                #TODO:XXX use self.fs.rm for this?
                os.remove(path)

    def _save_photo_data(self, dir, id, info, comments=None):
        """Save the given photo metadata.

        @param dir {str} The photo's dir in the working copy.
        @param id {int} The photo's id.
        @param info {xml.etree.Element} The <photo> element to save.
        @param comments {xml.etree.Element} Optional. The <comments> element
            to save, if any.
        """
        info_path = join(dir, ".pics", "%s-info.xml" % id)
        log.debug("save photo data: `%s'", info_path)
        f = open(info_path, 'wb')
        try:
            f.write(ET.tostring(info))
        finally:
            f.close()
        if comments:
            comments_path = join(dir, ".pics", "%s-comments.xml" % id)
            f = open(comments_path, 'wb')
            try:
                f.write(ET.tostring(comments))
            finally:
                f.close()

    def _get_photo_data(self, datedir, id, type):
        """Read and return the given photo data.

        Photo data is one or more XML files in the photos dirs ".pics" subdir.

        @param datedir {str} A datedir of the form YYYYMM in which the photo
            lives.
        @param id {int} The photo's id.
        @param type {str} The photo data type. One of "info" or "comments".
        @returns {xml.etree.Element} or None, if no such data file.
        """
        path = join(self.base_dir, datedir, ".pics", "%s-%s.xml" % (id, type))
        if exists(path):
            log.debug("load photo data: `%s'", path)
            f = open(path, 'rb')
            try:
                return ET.parse(f).getroot()
            except pyexpat.ParserError, ex:
                log.debug("corrupt photo data: XXX")
                #TODO: what to do with it?
                XXX
                return None
            finally: