def update():
delicious = DeliciousClient(settings.DELICIOUS_USERNAME, settings.DELICIOUS_PASSWORD)
# Check to see if we need an update
last_update_date = Item.objects.get_last_update_of_model(Bookmark)
last_post_date = utils.parsedate(delicious.posts.update().get("time"))
if last_post_date <= last_update_date:
log.info("Skipping update: last update date: %s; last post date: %s", last_update_date, last_post_date)
return
for datenode in reversed(list(delicious.posts.dates().getiterator('date'))):
dt = utils.parsedate(datenode.get("date"))
if dt > last_update_date:
_update_bookmarks_from_date(delicious, dt)
def update():
delicious = DeliciousClient(settings.DELICIOUS_USERNAME, settings.DELICIOUS_PASSWORD)
# Check to see if we need an update
last_update_date = Item.objects.get_last_update_of_model(Bookmark)
last_post_date = utils.parsedate(delicious.posts.update().get("time"))
if last_post_date <= last_update_date:
log.info("Skipping update: last update date: %s; last post date: %s", last_update_date, last_post_date)
return
for datenode in reversed(list(delicious.posts.dates().getiterator('date'))):
dt = utils.parsedate(datenode.get("date"))
if dt > last_update_date:
log.debug("There is a record indicating bookmarks have been added after our last update")
_update_bookmarks_from_date(delicious, dt)
def _handle_bookmark(info):
"""
info.keys() ==> ['id', 'created', 'updated', 'rating', 'private', 'owner',
'title', 'url', 'description', 'screenshot', 'tags']
Current Bookmark model doesn't support all of these.
"""
if info['private'] == 'false':
b, created = Bookmark.objects.get_or_create(
url = info['url'],
defaults = dict(
description = info['title'],
extended = info.get('description', ''),
thumbnail_url = info.get('screenshot', ''),
)
)
if not created:
b.description = info['title']
b.extended = info.get('description', '')
b.thumbnail_url = info.get('screenshot', ''),
b.save()
return Item.objects.create_or_update(
instance = b,
timestamp = utils.parsedate(info['created']),
tags = info.get('tags', ''),
source = __name__,
source_id = info['id'],
)
def update():
flickr = FlickrClient(settings.FLICKR_API_KEY)
# Preload the list of licenses
licenses = licenses = flickr.photos.licenses.getInfo()
licenses = dict((l["id"], smart_unicode(l["url"])) for l in licenses["licenses"]["license"])
# Handle update by pages until we see photos we've already handled
last_update_date = Item.objects.get_last_update_of_model(Photo)
page = 1
while True:
log.debug("Fetching page %s of photos", page)
resp = flickr.people.getPublicPhotos(user_id=settings.FLICKR_USER_ID, extras="license,date_taken", per_page="500", page=str(page))
photos = resp["photos"]
if page > photos["pages"]:
log.debug("Ran out of photos; stopping.")
break
for photodict in photos["photo"]:
timestamp = utils.parsedate(str(photodict["datetaken"]))
if timestamp < last_update_date:
log.debug("Hit an old photo (taken %s; last update was %s); stopping.", timestamp, last_update_date)
break
photo_id = utils.safeint(photodict["id"])
license = licenses[photodict["license"]]
secret = smart_unicode(photodict["secret"])
_handle_photo(flickr, photo_id, secret, license, timestamp)
page += 1
def update_bookmark(self, delicious):
last_update_date = Item.objects.get_last_update_of_model(Bookmark)
bookmarks = self.incoming['bookmark'] = list()
last_post_date = utils.parsedate(delicious.posts.update().get("time"))
if last_post_date <= last_update_date:
log.info("Skipping update: last update date: %s; last post date: %s", last_update_date, last_post_date)
return
for datenode in reversed(list(delicious.posts.dates().getiterator('date'))):
dt = utils.parsedate(datenode.get("date"))
if dt > last_update_date:
xml = delicious.posts.get(dt=dt.strftime("%Y-%m-%d"))
for post in xml.getiterator('post'):
info = dict((k, smart_unicode(post.get(k))) for k in post.keys())
info['tags'] = info['tag']
info['url'] = info['href']
info['timestamp'] = utils.parsedate(info['time'])
bookmarks.append( info )
def update_photo(self, flickr):
last_update_date = Item.objects.get_last_update_of_model(Photo)
log.debug("Last update date: %s", last_update_date)
licenses = licenses = flickr.photos.licenses.getInfo()
licenses = dict((l["id"], smart_unicode(l["url"])) for l in licenses["licenses"]["license"])
page = 1
keep_working = True
photo_list = self.incoming["photo"] = list()
while True:
log.debug("Fetching page %s of photos", page)
resp = flickr.people.getPublicPhotos(user_id=settings.FLICKR_USER_ID, extras="license,date_taken",
per_page="500", page=str(page))
photos = resp["photos"]
if page > photos["pages"]:
log.debug("Ran out of photos; stopping.")
return
for photodict in photos["photo"]:
timestamp = utils.parsedate(str(photodict["datetaken"]))
if timestamp < last_update_date:
log.debug("Hit an old photo (taken %s; last update was %s); stopping.",
timestamp, last_update_date)
break
obj = {}
obj['photo_id'] = smart_unicode(photodict["id"])
obj['cc_license'] = licenses[photodict["license"]]
obj['secret'] = smart_unicode(photodict["secret"])
info = flickr.photos.getInfo(photo_id=obj['photo_id'], secret=obj['secret'])["photo"]
obj['server_id'] = utils.safeint(info["server"])
obj['farm_id'] = utils.safeint(info["farm"])
obj['taken_by'] = smart_unicode(info["owner"]["username"])
obj['title'] = smart_unicode(info["title"]["_content"])
obj['description'] = smart_unicode(info["description"]["_content"])
obj['comment_count'] = utils.safeint(info["comments"]["_content"])
obj['date_uploaded'] = datetime.datetime.fromtimestamp(utils.safeint(info["dates"]["posted"]))
obj['date_updated'] = datetime.datetime.fromtimestamp(utils.safeint(info["dates"]["lastupdate"]))
obj['tags'] = self.convert_tags(info["tags"])
obj['timestamp'] = timestamp
obj['photoset'] = None
photo_list.append( obj )
page += 1
def _handle_bookmark(info):
b, created = Bookmark.objects.get_or_create(
url = info['href'],
defaults = dict(
description = info['description'],
extended = info.get('extended', ''),
)
)
if not created:
b.description = info['description']
b.extended = info.get('extended', '')
b.save()
return Item.objects.create_or_update(
instance = b,
timestamp = utils.parsedate(info['time']),
tags = info.get('tag', ''),
source = __name__,
source_id = info['hash'],
)
def update():
flickr = FlickrClient(settings.FLICKR_API_KEY)
# Preload the list of licenses
licenses = licenses = flickr.photos.licenses.getInfo()
licenses = dict((l["id"], smart_unicode(l["url"]))
for l in licenses["licenses"]["license"])
# Handle update by pages until we see photos we've already handled
last_update_date = Item.objects.get_last_update_of_model(Photo)
page = 1
while True:
log.debug("Fetching page %s of photos", page)
resp = flickr.people.getPublicPhotos(user_id=settings.FLICKR_USER_ID,
extras="license,date_taken",
per_page="500",
page=str(page))
photos = resp["photos"]
if page > photos["pages"]:
log.debug("Ran out of photos; stopping.")
break
for photodict in photos["photo"]:
timestamp = utils.parsedate(str(photodict["datetaken"]))
if timestamp < last_update_date:
log.debug(
"Hit an old photo (taken %s; last update was %s); stopping.",
timestamp, last_update_date)
break
photo_id = utils.safeint(photodict["id"])
license = licenses[photodict["license"]]
secret = smart_unicode(photodict["secret"])
_handle_photo(flickr, photo_id, secret, license, timestamp)
page += 1
