def __init__(self, base_dir):
self.base_dir = normpath(base_dir)
self.fs = FileSystem(log.debug)
self._cache = {}
db_path = self._db_path_from_base_dir(self.base_dir)
if exists(db_path): # Otherwise `.create()` will set `self.db`.
self.db = Database(db_path)
class WorkingCopy(object):
"""API for a pics working copy directory.
Usage:
# Create a new working copy directory (used by `pics co`).
wc = WorkingCopy.create(...)
# Or, for an existing working copy.
wc = WorkingCopy(base_dir)
"""
VERSION = "1.0.0"
@staticmethod
def _db_path_from_base_dir(base_dir):
return join(base_dir, ".pics", "photos.sqlite3")
def __init__(self, base_dir):
self.base_dir = normpath(base_dir)
self.fs = FileSystem(log.debug)
self._cache = {}
db_path = self._db_path_from_base_dir(self.base_dir)
if exists(db_path): # Otherwise `.create()` will set `self.db`.
self.db = Database(db_path)
@classmethod
def create(cls, base_dir, ilk, user, base_date=None, size="original",
permission="all"):
"""Create a working copy and return a `WorkingCopy` instance for it.
@param base_dir {str} The base directory for the working copy.
@param ilk {str} The type of the pics repo. Currently only
"flickr" is supported.
@param user {str} Username of the pics repo user.
@param base_date {datetime.date} The date (UTC) from which to
start getting photos. If not given, then all photos for
that user are retrieved.
@param size {str} The name of photos sizes to download. Supported
values are: "small", "medium" and "original". The actual size
that the former two mean depends on the pics repository. Default
is "original".
TODO: specify the sizes for flickr.
@param permission {str} Photo permissions to retrieve.
Valid values:
'all' (default) all photos,
'family' only photos that family would see,
'friend' only photos that friends would see,
'public' only public photos
@returns {WorkingCopy} The working copy instance.
"""
# Sanity checks.
assert ilk == "flickr", "unknown pics repo ilk: %r" % ilk
assert isinstance(base_date, (type(None), datetime.date))
assert size in ("small", "medium", "original")
assert permission in ("all", "family", "friend", "public")
if exists(base_dir):
raise PicsError("cannot create working copy: `%s' exists" % base_dir)
self = cls(base_dir)
# Create base structure.
#TODO: assert dirname(base_dir) exists?
self.fs.mkdir(self.base_dir, parents=True)
d = join(self.base_dir, ".pics")
self.fs.mkdir(d, hidden=True)
open(join(d, "version"), 'w').write(self.VERSION+'\n')
# Main working copy database.
db_path = self._db_path_from_base_dir(self.base_dir)
self.db = Database(db_path)
with self.db.connect(True) as cu:
self.db.set_meta("ilk", ilk)
self.db.set_meta("user", user)
self.db.set_meta("size", size)
self.db.set_meta("permission", permission)
if base_date:
self.db.set_meta("base_date", base_date)
return self
@property
def ilk(self):
return self.db.get_meta("ilk")
@property
def user(self):
return self.db.get_meta("user")
@property
def size(self):
return self.db.get_meta("size")
@property
def base_date(self):
"""The base date (UTC) of this working copy. I.e. the first date
for which photos are retrieved.
"""
if "base_date" not in self._cache:
s = self.db.get_meta("base_date")
if s is None:
self._cache["base_date"] = None
else:
t = datetime.datetime.strptime(s, "%Y-%m-%d")
self._cache["base_date"] = datetime.date(t.year, t.month, t.day)
return self._cache["base_date"]
@property
def version(self):
if "version" not in self._cache:
version_path = join(self.base_dir, ".pics", "version")
self._cache["version"] = open(version_path, 'r').read().strip()
return self._cache["version"]
@property
def version_info(self):
return tuple(map(int, self.version.split('.')))
def __repr__(self):
p = utils.nicepath(self.base_dir, prefer_absolute=True)
return "<WorkingCopy `%s'>" % p
@property
def api(self):
if self._api_cache is None:
self._api_cache = simpleflickrapi.SimpleFlickrAPI(
utils.get_flickr_api_key(), utils.get_flickr_secret())
#TODO: For now 'pics' is just read-only so this is good
# enough. However, eventually we'll want separate
# `self.read_api', `self.write_api' and
# `self.delete_api' or similar mechanism.
#TODO: cache this auth token in the pics user data dir
self._api_cache.get_auth_token("read")
return self._api_cache
_api_cache = None
# Getter and setter for `last-update`, a `datetime.datetime` field for
# the latest photo update sync'd to the working copy.
_last_update_cache = None
def get_last_update(self, cu=None):
if self._last_update_cache is None:
last_update_str = self.db.get_meta("last-update", cu=cu)
if last_update_str is None:
self._last_update_cache = None
else:
self._last_update_cache = datetime.datetime.strptime(
last_update_str, "%Y-%m-%d %H:%M:%S")
return self._last_update_cache
def set_last_update(self, value, cu=None):
curr_last_update = self.get_last_update(cu=cu)
if curr_last_update is None or value > curr_last_update:
self.db.set_meta("last-update",
value.strftime("%Y-%m-%d %H:%M:%S"), cu=cu)
self._last_update_cache = value
def _add_photo(self, id, dry_run=False):
"""Add the given photo to the working copy."""
# Gather necessary info.
info = self.api.photos_getInfo(photo_id=id)[0] # <photo> elem
datedir = info.find("dates").get("taken")[:7]
dir = join(self.base_dir, datedir)
url, filename = self._download_info_from_info(info, size=self.size)
path = join(dir, filename)
title = info.findtext("title")
log.info("A %s [%s]", path,
utils.one_line_summary_from_text(title, 40))
last_update = _photo_last_update_from_info(info)
if not dry_run:
# Create the dirs, as necessary.
pics_dir = join(dir, ".pics")
if not exists(dir):
self.fs.mkdir(dir)
if not exists(pics_dir):
self.fs.mkdir(pics_dir, hidden=True)
# Get the photo itself.
#TODO: add a reporthook for progressbar (unless too quick to bother)
#TODO: handle ContentTooShortError (py2.5)
fname, headers = urllib.urlretrieve(url, path)
mtime = utils.timestamp_from_datetime(last_update)
os.utime(path, (mtime, mtime))
# Gather and save all metadata.
if _photo_num_comments_from_info(info):
comments = self.api.photos_comments_getList(photo_id=id)[0]
else:
comments = None
self._save_photo_data(dir, id, info, comments)
return datedir, last_update
def _fetch_info_from_photo_id(self, id):
info = self.api.photos_getInfo(photo_id=id)[0] # <photo> elem
# Drop tail for canonicalization to allow diffing of the
# serialized XML.
info.tail = None
return info
def _update_photo(self, id, local_datedir, local_info, dry_run=False):
"""Update the given photo in the working copy."""
info = self._fetch_info_from_photo_id(id)
datedir = info.find("dates").get("taken")[:7]
last_update = _photo_last_update_from_info(info)
# Figure out what work needs to be done.
# From *experimentation* it looks like the "secret" attribute
# changes if the photo itself changes (i.e. is replaced or
# "Edited" or rotated).
todos = []
if datedir != local_datedir:
log.debug("update %s: datedir change: %r -> %r",
id, local_datedir, datedir)
todos.append("remove-old")
todos.append("photo")
elif info.get("secret") != local_info.get("secret"):
log.debug("update %s: photo secret change: %r -> %r",
id, local_info.get("secret"), info.get("secret"))
todos.append("photo")
todos.append("info")
if _photo_num_comments_from_info(info):
todos.append("comments")
if not todos:
return datedir, last_update
# Do the necessary updates.
XXX # Figure out ext for videos. _download_info_from_info is wrong
# here b/c is isn't using *local* info.
url, filename = self._download_info_from_info(info, size=self.size)
#XXX
#size_ext = (self.size != "original" and "."+self.size or "")
#ext = (self.size != "original" and ".jpg"
# or "."+local_info.get("originalformat"))
# - Remove the old bits, if the datedir has changed.
if "remove-old" in todos:
d = join(self.base_dir, local_datedir)
path = join(d, filename)
log.info("D %s [%s]", path,
utils.one_line_summary_from_text(local_info.findtext("title"), 40))
if not dry_run:
log.debug("rm `%s'", path)
os.remove(path)
self._remove_photo_data(d, id)
remaining_paths = set(os.listdir(d))
remaining_paths.difference_update(set([".pics"]))
if not remaining_paths:
log.info("D %s", d)
self.fs.rm(d)
# - Add the new stuff.
d = join(self.base_dir, datedir)
action_str = ("photo" in todos and "U " or " u")
path = join(d, filename)
log.info("%s %s [%s]", action_str, path,
utils.one_line_summary_from_text(info.findtext("title"), 40))
if not dry_run:
if not exists(d):
self.fs.mkdir(d)
pics_dir = join(d, ".pics")
if not exists(pics_dir):
self.fs.mkdir(pics_dir, hidden=True)
if "photo" in todos:
fname, headers = urllib.urlretrieve(url, path)
mtime = utils.timestamp_from_datetime(last_update)
os.utime(path, (mtime, mtime))
if "comments" in todos:
comments = self.api.photos_comments_getList(photo_id=id)[0]
else:
comments = None
self._save_photo_data(d, id, info, comments=comments)
#print "... %s" % id
#print "originalsecret: %s <- %s" % (info.get("originalsecret"), local_info.get("originalsecret"))
#print "secret: %s <- %s" % (info.get("secret"), local_info.get("secret"))
#print "rotation: %s <- %s" % (info.get("rotation"), local_info.get("rotation"))
return datedir, last_update
def check_version(self):
if self.version != self.VERSION:
raise PicsError("out of date working copy (v%s != v%s): you must "
"first upgrade", self.version, self.VERSION)
def _remove_photo_data(self, dir, id):
#TODO: 'dir' correct here? need to use self.base_dir?
paths = [join(dir, ".pics", "%s-%s.xml" % (id, name))
for name in ("info", "comments")]
for path in paths:
if exists(path):
log.debug("remove photo data: `%s'", path)
#TODO:XXX use self.fs.rm for this?
os.remove(path)
def _save_photo_data(self, dir, id, info, comments=None):
"""Save the given photo metadata.
@param dir {str} The photo's dir in the working copy.
@param id {int} The photo's id.
@param info {xml.etree.Element} The <photo> element to save.
@param comments {xml.etree.Element} Optional. The <comments> element
to save, if any.
"""
info_path = join(dir, ".pics", "%s-info.xml" % id)
log.debug("save photo data: `%s'", info_path)
f = open(info_path, 'wb')
try:
f.write(ET.tostring(info))
finally:
f.close()
if comments:
comments_path = join(dir, ".pics", "%s-comments.xml" % id)
f = open(comments_path, 'wb')
try:
f.write(ET.tostring(comments))
finally:
f.close()
def _get_photo_data(self, datedir, id, type):
"""Read and return the given photo data.
Photo data is one or more XML files in the photos dirs ".pics" subdir.
@param datedir {str} A datedir of the form YYYYMM in which the photo
lives.
@param id {int} The photo's id.
@param type {str} The photo data type. One of "info" or "comments".
@returns {xml.etree.Element} or None, if no such data file.
"""
path = join(self.base_dir, datedir, ".pics", "%s-%s.xml" % (id, type))
if exists(path):
log.debug("load photo data: `%s'", path)
f = open(path, 'rb')
try:
return ET.parse(f).getroot()
except pyexpat.ParserError, ex:
log.debug("corrupt photo data: XXX")
#TODO: what to do with it?
XXX
return None
finally: