def migrate_sync_record(sync_record, content_size):
if content_size:
sync_record.transfer_in = content_size
sync_record.transfer_out = content_size
Session.commit()
else:
print "Couldn't get content size for", sync_record.id
def migrate_sync_record(sync_record, content_size):
if content_size:
sync_record.transfer_in = content_size
sync_record.transfer_out = content_size
Session.commit()
else:
print "Couldn't get content size for", sync_record.id
def set_status(self, completed, total, data):
log.debug("%s %s Completed %s/%s: %r",
self.__class__.__name__, self.job.jid,
completed, total, data)
self.job.touch()
self.task.set_status(completed, total, data)
Session.commit()
def __call__(self, environ, start_response):
"""Invoke the Controller"""
# WSGIController.__call__ dispatches to the Controller method
# the request is routed to. This routing information is
# available in environ['pylons.routes_dict']
try:
return WSGIController.__call__(self, environ, start_response)
finally:
Session.remove()
def __call__(self, environ, start_response):
"""Invoke the Controller"""
# WSGIController.__call__ dispatches to the Controller method
# the request is routed to. This routing information is
# available in environ['pylons.routes_dict']
try:
return WSGIController.__call__(self, environ, start_response)
finally:
Session.remove()
def verify_user(self):
"""Check the access tokens"""
if self.fb_access_token:
fbuser = fb.GraphUser(access_token=self.fb_access_token)
try:
fbuser.load()
except fb.OAuthException, e:
self.fb_access_token = None
Session.commit()
return False
def index(self):
gd_client = gdata.photos.service.PhotosService()
#token = request.GET.getone('token')
parameters = cgi.FieldStorage()
token = parameters['token']
gd_client.auth_token = token
gd_client.UpgradeToSessionToken()
session['picasa_token'] = token
session.save()
if session.get('user_id'):
# a user is already logged in
user = Session.query(User).filter_by(id=session.get('user_id')).first()
else:
# the user is not already logged in, let's see if they have
# already created an account before
user = Session.query(User).filter_by(picasa_token=token).first()
if user:
user.picasa_token = token
Session.commit()
else:
# the user does not have an account. We need to create a new one
# for them.
user = User(picasa_token=token)
Session.add(user)
Session.commit()
user = Session.query(User).filter_by(picasa_token=picasa_token).first()
session['user_id'] = user.id
session['picasa_token'] = token
session.save()
log.info("Logged in user %s", user)
redirect(url('index'))
def index(self):
code = request.GET.getone('code')
nexturl = request.GET.get('nexturl')
if nexturl:
# we are acting only as an auth server.
# redirect to the server that wants the auth code
redirect(nexturl+'?code=%s' % code)
return
token = fb.get_access_token(code)
if not token:
#lame... this failed for some reason
h.flash("The login process failed :(")
redirect(url('index'))
return
fbuser = fb.GraphUser(access_token=token)
if session.get('user_id'):
# a user is already logged in
user = Session.query(User).filter_by(id=session.get('user_id')).first()
else:
# the user is not already logged in, let's see if they have
# already created an account before
user = Session.query(User).filter_by(fb_uid=fbuser.id).first()
if user:
# the user does have an account, let's update their auth token
user.fb_uid = fbuser.id
user.fb_access_token = token
Session.commit()
else:
# the user does not have an account. We need to create a new one
# for them.
for attempt in xrange(3):
user = User(fb_uid=fbuser.id,
fb_access_token=token)
Session.add(user)
Session.commit()
user = Session.query(User).filter_by(fb_uid=fbuser.id).first()
if user:
break
log.error("Failed to create user with fb_uid=%r attempt %r",
fbuser.id, attempt)
if not user:
log.error("Trying to log in, but couldn't get a user object. "
"user=%r code=%r token=%r fbuser=%r",
user, code, token, fbuser)
session['user_id'] = user.id
session['fb_access_token'] = token
session.save()
log.info("Logged in user %s %s: %s",
fbuser.first_name,
fbuser.last_name,
user)
redirect(url('index'))
def index(self):
code = request.GET.getone('code')
nexturl = request.GET.get('nexturl')
if nexturl:
# we are acting only as an auth server.
# redirect to the server that wants the auth code
redirect(nexturl + '?code=%s' % code)
return
token = fb.get_access_token(code)
if not token:
#lame... this failed for some reason
h.flash("The login process failed :(")
redirect(url('index'))
return
fbuser = fb.GraphUser(access_token=token)
if session.get('user_id'):
# a user is already logged in
user = Session.query(User).filter_by(
id=session.get('user_id')).first()
else:
# the user is not already logged in, let's see if they have
# already created an account before
user = Session.query(User).filter_by(fb_uid=fbuser.id).first()
if user:
# the user does have an account, let's update their auth token
user.fb_uid = fbuser.id
user.fb_access_token = token
Session.commit()
else:
# the user does not have an account. We need to create a new one
# for them.
for attempt in xrange(3):
user = User(fb_uid=fbuser.id, fb_access_token=token)
Session.add(user)
Session.commit()
user = Session.query(User).filter_by(fb_uid=fbuser.id).first()
if user:
break
log.error("Failed to create user with fb_uid=%r attempt %r",
fbuser.id, attempt)
if not user:
log.error(
"Trying to log in, but couldn't get a user object. "
"user=%r code=%r token=%r fbuser=%r", user, code, token,
fbuser)
session['user_id'] = user.id
session['fb_access_token'] = token
session.save()
log.info("Logged in user %s %s: %s", fbuser.first_name,
fbuser.last_name, user)
redirect(url('index'))
def __call__(self):
try:
if self.task.queue_id:
if self.task.queue_id != self.job.jid:
# this job is no longer supposed to exist...
return self.job.delete()
else:
self.task.queue_id = self.job.jid
Session.commit()
result = self.run(*self.__args, **self.__kwargs)
except Exception, e:
traceback.print_exc()
log.exception("Failed to run task %s", self.__class__.__name__)
raise
def __call__(self):
try:
if self.task.queue_id:
if self.task.queue_id != self.job.jid:
# this job is no longer supposed to exist...
return self.job.delete()
else:
self.task.queue_id = self.job.jid
Session.commit()
result = self.run(*self.__args, **self.__kwargs)
except Exception, e:
traceback.print_exc()
log.exception("Failed to run task %s", self.__class__.__name__)
raise
def load_transfer_data():
"""Loads bytes transferred data into the db.
You will want to run this sql first:
ALTER TABLE sync_records ADD COLUMN transfer_in int(11) DEFAULT 0;
ALTER TABLE sync_records ADD COLUMN transfer_out int(11) DEFAULT 0;
"""
records = Session.query(SyncRecord).filter(
SyncRecord.type==SyncRecord.TYPE_PHOTO).filter(
SyncRecord.transfer_in==0).all()
def progress(message):
def func(processed, total):
print "%i/%i\t%i\t%s" % (processed, total, float(processed)/total*100, message)
return func
def get_size_request(record):
return http.JsonRequest(flickr.get_url(
record.user.flickr_token,
method='flickr.photos.getSizes',
photo_id=record.flickrid))
def get_byte_size_request(request):
try:
url = request.read_response()['sizes']['size'][-1]['source']
return http.Request(url, method="HEAD")
except:
pass
def get_content_size(request):
return int(request.read_headers()['Content-Length']) if request else None
def migrate_sync_record(sync_record, content_size):
if content_size:
sync_record.transfer_in = content_size
sync_record.transfer_out = content_size
Session.commit()
else:
print "Couldn't get content size for", sync_record.id
fetcher = http.Fetcher(
progress_callback=progress("getting image urls"))
map(compose(fetcher.queue, get_size_request), records)
size_fetcher = http.Fetcher(
progress_callback=progress("getting byte sizes"))
map(compose(size_fetcher.queue, get_byte_size_request), fetcher.run())
list(starmap(migrate_sync_record,
izip(records, imap(get_content_size, size_fetcher.run()))))
def load_transfer_data():
"""Loads bytes transferred data into the db.
You will want to run this sql first:
ALTER TABLE sync_records ADD COLUMN transfer_in int(11) DEFAULT 0;
ALTER TABLE sync_records ADD COLUMN transfer_out int(11) DEFAULT 0;
"""
records = Session.query(SyncRecord).filter(
SyncRecord.type == SyncRecord.TYPE_PHOTO).filter(
SyncRecord.transfer_in == 0).all()
def progress(message):
def func(processed, total):
print "%i/%i\t%i\t%s" % (processed, total,
float(processed) / total * 100, message)
return func
def get_size_request(record):
return http.JsonRequest(
flickr.get_url(record.user.flickr_token,
method='flickr.photos.getSizes',
photo_id=record.flickrid))
def get_byte_size_request(request):
try:
url = request.read_response()['sizes']['size'][-1]['source']
return http.Request(url, method="HEAD")
except:
pass
def get_content_size(request):
return int(
request.read_headers()['Content-Length']) if request else None
def migrate_sync_record(sync_record, content_size):
if content_size:
sync_record.transfer_in = content_size
sync_record.transfer_out = content_size
Session.commit()
else:
print "Couldn't get content size for", sync_record.id
fetcher = http.Fetcher(progress_callback=progress("getting image urls"))
map(compose(fetcher.queue, get_size_request), records)
size_fetcher = http.Fetcher(
progress_callback=progress("getting byte sizes"))
map(compose(size_fetcher.queue, get_byte_size_request), fetcher.run())
list(
starmap(migrate_sync_record,
izip(records, imap(get_content_size, size_fetcher.run()))))
def multiget(user_id=None, settings=None):
user_id = user_id or session.get('user_id')
query = Session.query(UserSetting).filter(UserSetting.user_id==user_id)
if settings is not None:
query = query.filter(UserSetting.setting_id.in_(settings))
results = {}
for result in query:
results[result.setting_id] = result.value
return results
def get_for_flickrid(flickrid=None, limit=3, type=None):
query = Session.query(SyncRecord)\
.filter(SyncRecord.flickrid==flickrid)
if type:
query = query.filter(SyncRecord.type==type)
query = query.order_by(desc('timestamp'))\
.limit(limit)
return query
def get_for_user(user_id=None, limit=3, type=None):
if not user_id:
user_id = session.get('user_id')
query = Session.query(AsyncTask)\
.filter(AsyncTask.user_id==user_id)
if type:
query = query.filter(AsyncTask.type==type)
query = query.order_by(desc('created_time'))\
.limit(limit)
return query
def stats(self):
c.user_count = Session.query(User).count()
c.sync_count = Session.query(SyncRecord).count()
c.async_tasks_count = Session.query(AsyncTask).count()
data = Session.query(SyncRecord.user_id,
func.count(1),
func.sum(SyncRecord.transfer_in),
func.sum(SyncRecord.transfer_out))\
.filter(SyncRecord.type == SyncRecord.TYPE_PHOTO)\
.group_by(SyncRecord.user_id).all()
c.user_stats = []
c.total_count = 0
c.total_cost = 0
c.total_tin = 0
c.total_tout = 0
for user_id, count, tin, tout in data:
tin = tin or 0
tout = tout or 0
bandwidth_cost = cost.get_bandwidth_cost(tin,tout)
c.total_cost += bandwidth_cost
c.total_tout += int(tout)
c.total_tin += int(tin)
c.total_count += count
c.user_stats.append([User.get_by_id(user_id),
count,
round(tin/1024/1024, 2),
round(tout/1024/1024, 2),
bandwidth_cost])
c.total_tout = round(c.total_tout/1024./1024, 2)
c.total_tin = round(c.total_tin/1024./1024, 2)
order_by = int(request.GET.get('order_by', 4))
c.user_stats.sort(key=lambda d: d[order_by], reverse=True)
return render('/admin/stats.mako')
def stats(self):
c.user_count = Session.query(User).count()
c.sync_count = Session.query(SyncRecord).count()
c.async_tasks_count = Session.query(AsyncTask).count()
data = Session.query(SyncRecord.user_id,
func.count(1),
func.sum(SyncRecord.transfer_in),
func.sum(SyncRecord.transfer_out))\
.filter(SyncRecord.type == SyncRecord.TYPE_PHOTO)\
.group_by(SyncRecord.user_id).all()
c.user_stats = []
c.total_count = 0
c.total_cost = 0
c.total_tin = 0
c.total_tout = 0
for user_id, count, tin, tout in data:
tin = tin or 0
tout = tout or 0
bandwidth_cost = cost.get_bandwidth_cost(tin, tout)
c.total_cost += bandwidth_cost
c.total_tout += int(tout)
c.total_tin += int(tin)
c.total_count += count
c.user_stats.append([
User.get_by_id(user_id), count,
round(tin / 1024 / 1024, 2),
round(tout / 1024 / 1024, 2), bandwidth_cost
])
c.total_tout = round(c.total_tout / 1024. / 1024, 2)
c.total_tin = round(c.total_tin / 1024. / 1024, 2)
order_by = int(request.GET.get('order_by', 4))
c.user_stats.sort(key=lambda d: d[order_by], reverse=True)
return render('/admin/stats.mako')
def submit_advanced(cls, args, kwargs, delay=0, user_id=None):
task = AsyncTask(user_id=user_id)
task.set_status(*cls.get_initial_status())
task.type = cls.get_type()
Session.add(task)
Session.commit()
queue_id = job.submit_advanced(cls, (task.id, args, kwargs), {}, delay=delay)
task.queue_id = queue_id
Session.commit()
return task
def submit_advanced(cls, args, kwargs, delay=0, user_id=None):
task = AsyncTask(user_id=user_id)
task.set_status(*cls.get_initial_status())
task.type = cls.get_type()
Session.add(task)
Session.commit()
queue_id = job.submit_advanced(cls, (task.id, args, kwargs), {}, delay=delay)
task.queue_id = queue_id
Session.commit()
return task
def delete_empty_albums():
"""Delete empty albums that were accidentally created."""
records = Session.query(SyncRecord).filter(
SyncRecord.type == SyncRecord.TYPE_ALBUM).all()
for record in records:
user = record.user
if user.fb_access_token:
fbalbum = fb.GraphAlbum(id=record.fbid,
access_token=user.fb_access_token)
if len(fbalbum.photos) == 0:
#this is an empty album... let's delete it.
print "Deleting album %r for user %r" % (record.fbid, user)
fbalbum.delete()
#Session.delete(record)
print "All done"
def command(self):
"""Main command that starts the worker."""
super(KickCommand, self).command()
users = Session.query(User)\
.filter(User.fb_uid != None)\
.filter(User.flickr_nsid != None)
if self.options.simulate:
log.info("--SIMULATING--")
for user in users:
tasks = AsyncTask.get_for_user(user.id)
for task in tasks:
if not (task.is_completed or task.time_left) and task.is_buried:
log.info("Running task %r", task)
if not self.options.simulate:
task.run_now()
def index(self):
gd_client = gdata.photos.service.PhotosService()
#token = request.GET.getone('token')
parameters = cgi.FieldStorage()
token = parameters['token']
gd_client.auth_token = token
gd_client.UpgradeToSessionToken()
session['picasa_token'] = token
session.save()
if session.get('user_id'):
# a user is already logged in
user = Session.query(User).filter_by(
id=session.get('user_id')).first()
else:
# the user is not already logged in, let's see if they have
# already created an account before
user = Session.query(User).filter_by(picasa_token=token).first()
if user:
user.picasa_token = token
Session.commit()
else:
# the user does not have an account. We need to create a new one
# for them.
user = User(picasa_token=token)
Session.add(user)
Session.commit()
user = Session.query(User).filter_by(
picasa_token=picasa_token).first()
session['user_id'] = user.id
session['picasa_token'] = token
session.save()
log.info("Logged in user %s", user)
redirect(url('index'))
def delete_empty_albums():
"""Delete empty albums that were accidentally created."""
records = Session.query(SyncRecord).filter(
SyncRecord.type==SyncRecord.TYPE_ALBUM).all()
for record in records:
user = record.user
if user.fb_access_token:
fbalbum = fb.GraphAlbum(
id=record.fbid,
access_token=user.fb_access_token)
if len(fbalbum.photos) == 0:
#this is an empty album... let's delete it.
print "Deleting album %r for user %r" % (record.fbid, user)
fbalbum.delete()
#Session.delete(record)
print "All done"
def index(self):
flickr = FlickrAPI()
token = flickr.get_token(request.GET.getone('frob'))
session['flickr_token'] = token
session.save()
flickr = FlickrAPI(token)
result = flickr.auth_checkToken()
nsid = result[0][2].get('nsid')
if session.get('user_id'):
# a user is already logged in
user = Session.query(User).filter_by(
id=session.get('user_id')).first()
else:
# the user is not already logged in, let's see if they have
# already created an account before
user = Session.query(User).filter_by(flickr_nsid=nsid).first()
if user:
user.flickr_nsid = nsid
user.flickr_token = token
Session.commit()
else:
# the user does not have an account. We need to create a new one
# for them.
user = User(flickr_nsid=nsid, flickr_token=token)
Session.add(user)
Session.commit()
user = Session.query(User).filter_by(flickr_nsid=nsid).first()
session['user_id'] = user.id
session['flickr_token'] = token
session.save()
log.info("Logged in user %s", user)
redirect(url('index'))
def index(self):
flickr = FlickrAPI()
token = flickr.get_token(request.GET.getone('frob'))
session['flickr_token'] = token
session.save()
flickr = FlickrAPI(token)
result = flickr.auth_checkToken()
nsid = result[0][2].get('nsid')
if session.get('user_id'):
# a user is already logged in
user = Session.query(User).filter_by(id=session.get('user_id')).first()
else:
# the user is not already logged in, let's see if they have
# already created an account before
user = Session.query(User).filter_by(flickr_nsid=nsid).first()
if user:
user.flickr_nsid = nsid
user.flickr_token = token
Session.commit()
else:
# the user does not have an account. We need to create a new one
# for them.
user = User(flickr_nsid=nsid, flickr_token=token)
Session.add(user)
Session.commit()
user = Session.query(User).filter_by(flickr_nsid=nsid).first()
session['user_id'] = user.id
session['flickr_token'] = token
session.save()
log.info("Logged in user %s", user)
redirect(url('index'))
class User(Base):
__tablename__ = 'users'
id = Column('id', Integer, primary_key=True)
fb_uid = Column('fb_uid', Integer, index=True)
flickr_nsid = Column('flickr_nsid', String(100), index=True)
flickr_token = Column('flickr_token', UnicodeText)
fb_access_token = Column('fb_access_token', UnicodeText)
def __init__(self, fb_uid=None, fb_access_token=None, flickr_nsid=None, flickr_token=None):
self.fb_uid = fb_uid
self.flickr_nsid = flickr_nsid
self.fb_access_token = fb_access_token
self.flickr_token = flickr_token
def __unicode__(self):
return u'%s' % self.id
def __repr__(self):
return "<User id=%s fb_uid=%s flickr_nsid=%s>" % (
self.id, self.fb_uid, self.flickr_nsid)
def verify_user(self):
"""Check the access tokens"""
if self.fb_access_token:
fbuser = fb.GraphUser(access_token=self.fb_access_token)
try:
fbuser.load()
except fb.OAuthException, e:
self.fb_access_token = None
Session.commit()
return False
if self.flickr_token:
fluser = flickr.FlickrUser(token=self.flickr_token)
try:
fluser.auth_checkToken()
except flickr.FlickrError, e:
self.flickr_token = None
Session.commit()
return False
def sync_photoset(self, photoset):
log.debug("Syncing flickr photoset %s to facebook", photoset.get('id'))
sync = Session.query(SyncRecord).filter_by(
flickrid=photoset.get('id'), type=SyncRecord.TYPE_ALBUM).first()
album = None
if sync and sync.success:
# don't resync unless we failed before
log.debug("skipping... already synced")
album = fb.GraphAlbum(id=sync.fbid, access_token=self.user.fb_access_token)
if not album.data:
album = None
if not album:
sync = SyncRecord(SyncRecord.TYPE_ALBUM, self.task.user_id)
sync.flickrid = photoset.get('id')
title = photoset.find('title').text
description = photoset.find('description').text
album = self.fb_user.albums.add(
title.encode('utf-8') if title else '',
description.encode('utf-8') if description else '',
privacy=self.fb_privacy)
if album:
sync.fbid = album.id
sync.status = SyncRecord.STATUS_SUCCESS
else:
sync.status = SyncRecord.STATUS_FAILED
Session.add(sync)
Session.commit()
self.set_status(self.synced_photos, self.total_photos,
"Failed to sync %s" % title)
return
photos = self.fk.photosets_getPhotos(photoset_id=photoset.get('id'))[0]
# max out at 200 photos until we figure out what to do with bigger sets
photos = photos[:200]
photos_to_sync = []
for photo in photos:
log.debug("Syncing flickr photo %s to facebook", photo.get('id'))
sync = SyncRecord.get_for_flickrid(photo.get('id')).first()
fb_photo = None
if sync and sync.fbid and sync.success:
log.debug("Skipping... already synced")
fb_photo = fb.GraphPhoto(id=sync.fbid, access_token=self.user.fb_access_token)
#if not fb_photo.data:
# fb_photo = None
self.synced_photos += 1
if not fb_photo:
photos_to_sync.append(photo)
status = "%s photos from %s already synced" % (self.synced_photos,
photoset.find('title').text)
self.set_status(self.synced_photos, self.total_photos, status)
if not photos_to_sync:
return
def on_progress(processed, total):
self.set_status(
processed,
total,
"Found %s/%s photos in %s..." % (
processed, total, photoset.find('title').text))
fetcher = http.Fetcher(progress_callback=on_progress)
requests = []
for photo in photos_to_sync:
url = flickr.get_url(
self.user.flickr_token,
method='flickr.photos.getSizes',
photo_id=photo.get('id'))
request = http.JsonRequest(url)
requests.append((photo, request))
fetcher.queue(request)
fetcher.run()
def on_progress(processed, total):
self.set_status(
processed,
total,
"Downloaded %s/%s images from %s" % (
processed, total, photoset.find('title').text))
fetcher = http.Fetcher(progress_callback=on_progress)
img_requests = []
# TODO: make tmp directory path configurable
if not os.path.exists('/tmp/photosync'):
os.mkdir('/tmp/photosync')
os.chmod('/tmp/photosync', 0777)
for i, (photo, request) in enumerate(requests):
sync = SyncRecord(SyncRecord.TYPE_PHOTO, self.task.user_id)
sync.flickrid = photo.get("id")
Session.add(sync)
res = request.read_response()
try:
img_url = res['sizes']['size'][-1]['source']
except:
#import pdb; pdb.set_trace()
raise
filename = '/tmp/photosync/flickr-'+sync.flickrid+'.jpg'
log.debug("Downloading image %s to %s", img_url, filename)
img_request = None
if not os.path.exists(filename):
f = open(filename, 'wb')
f.write(urllib2.urlopen(img_url).read())
f.close()
on_progress(i, len(requests))
# TODO: Figure out why curl isn't working here
# for some reason when we use the code below,
# the complete file does not get downloaded.
#img_request = http.Request(img_url, filename=filename)
#fetcher.queue(img_request)
img_requests.append((photo, filename, sync, img_request))
Session.commit()
fetcher.run()
for photo, temp_filename, sync, img_request in img_requests:
sync.transfer_in = os.path.getsize(temp_filename)
log.debug("Uploading image %s to facebook", temp_filename)
graph_photo = None
title = photo.get('title')
try:
title = title.encode('utf-8')
except UnicodeEncodeError, e:
log.error("Failed to encode %s to utf-8", title)
log.exception(e)
# better a photo with no title than no photo at all.
title = ""
try:
graph_photo = album.photos.add(temp_filename, title)
except TypeError, e:
log.error("Error uploading image %s", temp_filename)
log.exception(e)
def run_now(self):
self.queue_id = None
Session.commit()
self.queue_id = self._job.resubmit()
self.__job = "SENTINAL"
Session.commit()
def set_status(self, completed, total, data):
log.debug("%s %s Completed %s/%s: %r", self.__class__.__name__, self.job.jid, completed, total, data)
self.job.touch()
self.task.set_status(completed, total, data)
Session.commit()
def task(self):
if not self.__task_fetched:
self.__task_fetched = True
self.__task = Session.query(AsyncTask).filter_by(id=self.__task_id).first()
return self.__task
def sync_photoset(self, photoset):
log.debug("Syncing flickr photoset %s to facebook", photoset.get("id"))
sync = Session.query(SyncRecord).filter_by(flickrid=photoset.get("id"), type=SyncRecord.TYPE_ALBUM).first()
album = None
if sync and sync.success:
# don't resync unless we failed before
log.debug("skipping... already synced")
album = fb.GraphAlbum(id=sync.fbid, access_token=self.user.fb_access_token)
if not album.data:
album = None
if not album:
sync = SyncRecord(SyncRecord.TYPE_ALBUM, self.task.user_id)
sync.flickrid = photoset.get("id")
title = photoset.find("title").text
description = photoset.find("description").text
album = self.fb_user.albums.add(
title.encode("utf-8") if title else "",
description.encode("utf-8") if description else "",
privacy=self.fb_privacy,
)
if album:
sync.fbid = album.id
sync.status = SyncRecord.STATUS_SUCCESS
else:
sync.status = SyncRecord.STATUS_FAILED
Session.add(sync)
Session.commit()
self.set_status(self.synced_photos, self.total_photos, "Failed to sync %s" % title)
return
photos = self.fk.photosets_getPhotos(photoset_id=photoset.get("id"))[0]
# max out at 200 photos until we figure out what to do with bigger sets
photos = photos[:200]
photos_to_sync = []
for photo in photos:
log.debug("Syncing flickr photo %s to facebook", photo.get("id"))
sync = SyncRecord.get_for_flickrid(photo.get("id")).first()
fb_photo = None
if sync and sync.fbid and sync.success:
log.debug("Skipping... already synced")
fb_photo = fb.GraphPhoto(id=sync.fbid, access_token=self.user.fb_access_token)
# if not fb_photo.data:
# fb_photo = None
self.synced_photos += 1
if not fb_photo:
photos_to_sync.append(photo)
status = "%s photos from %s already synced" % (self.synced_photos, photoset.find("title").text)
self.set_status(self.synced_photos, self.total_photos, status)
if not photos_to_sync:
return
def on_progress(processed, total):
self.set_status(
processed, total, "Found %s/%s photos in %s..." % (processed, total, photoset.find("title").text)
)
fetcher = http.Fetcher(progress_callback=on_progress)
requests = []
for photo in photos_to_sync:
url = flickr.get_url(self.user.flickr_token, method="flickr.photos.getSizes", photo_id=photo.get("id"))
request = http.JsonRequest(url)
requests.append((photo, request))
fetcher.queue(request)
fetcher.run()
def on_progress(processed, total):
self.set_status(
processed, total, "Downloaded %s/%s images from %s" % (processed, total, photoset.find("title").text)
)
fetcher = http.Fetcher(progress_callback=on_progress)
img_requests = []
# TODO: make tmp directory path configurable
if not os.path.exists("/tmp/photosync"):
os.mkdir("/tmp/photosync")
os.chmod("/tmp/photosync", 0777)
for i, (photo, request) in enumerate(requests):
sync = SyncRecord(SyncRecord.TYPE_PHOTO, self.task.user_id)
sync.flickrid = photo.get("id")
Session.add(sync)
res = request.read_response()
try:
img_url = res["sizes"]["size"][-1]["source"]
except:
# import pdb; pdb.set_trace()
raise
filename = "/tmp/photosync/flickr-" + sync.flickrid + ".jpg"
log.debug("Downloading image %s to %s", img_url, filename)
img_request = None
if not os.path.exists(filename):
f = open(filename, "wb")
f.write(urllib2.urlopen(img_url).read())
f.close()
on_progress(i, len(requests))
# TODO: Figure out why curl isn't working here
# for some reason when we use the code below,
# the complete file does not get downloaded.
# img_request = http.Request(img_url, filename=filename)
# fetcher.queue(img_request)
img_requests.append((photo, filename, sync, img_request))
Session.commit()
fetcher.run()
for photo, temp_filename, sync, img_request in img_requests:
sync.transfer_in = os.path.getsize(temp_filename)
log.debug("Uploading image %s to facebook", temp_filename)
graph_photo = None
title = photo.get("title")
try:
title = title.encode("utf-8")
except UnicodeEncodeError, e:
log.error("Failed to encode %s to utf-8", title)
log.exception(e)
# better a photo with no title than no photo at all.
title = ""
try:
graph_photo = album.photos.add(temp_filename, title)
except TypeError, e:
log.error("Error uploading image %s", temp_filename)
log.exception(e)
def init_model(engine):
"""Call me before using any of the tables or classes in the model"""
Session.configure(bind=engine)
def get_bytes_transfered_by_user(user_id):
bytes_in, bytes_out = Session.execute(
"SELECT SUM(transfer_in), SUM(transfer_out) "
"FROM sync_records WHERE user_id=:id",
{'id':user_id}).fetchone()
return int(bytes_in or 0), int(bytes_out or 0)
def get_by_id(id):
return Session.query(User).filter_by(id=id).first()
def set(setting_id, value, user_id=None):
user_id = user_id or session.get('user_id')
result = Session.merge(UserSetting(user_id, setting_id, value))
Session.commit()
return result
class FullSync(TaskHandler):
@classmethod
def run_for_user_now(cls, user_id):
last = AsyncTask.get_for_user(user_id=user_id,
type=cls.get_type(),
limit=1).first()
if not last or last.is_completed:
cls.submit_advanced((user_id,), {}, user_id=user_id)
else:
last.run_now()
@classmethod
def get_initial_status(cls):
return (None, None, "Waiting for syncer to become available...")
def run(self, user_id):
self.user = User.get_by_id(user_id)
if not self.user.verify_user():
# not really much we can do about this...
log.info("Missing tokens for user %r", self.user)
self.set_status(0, 0, "You need to log out and log back in again.")
return
self.fk = flickr.FlickrAPI(self.user.flickr_token)
self.fb_user = fb.GraphUser(access_token=self.user.fb_access_token)
photosets = self.fk.photosets_getList()[0]
self.synced_photos = 0
self.total_photos = sum(int(pset.get('photos')) for pset in photosets)
self.set_status(
self.synced_photos,
self.total_photos,
"syncing Flickr photos to Facebook")
settings = UserSetting.multiget(
user_id=user_id,
settings=[UserSettingConst.FB_PRIVACY,
UserSettingConst.FLICKR_SET_SYNCING])
self.fb_privacy = settings.get(UserSettingConst.FB_PRIVACY, 'FB_DEFAULT')
self.flickr_settings = settings.get(UserSettingConst.FLICKR_SET_SYNCING, {})
select_sets = self.flickr_settings.get('select_sets', False)
selected_sets = self.flickr_settings.get('selected_sets', [])
for photoset in photosets:
if not select_sets or photoset.get('id') in selected_sets:
self.sync_photoset(photoset)
log.info("Finished job %r for user %r", self.job.jid, self.user.id)
# once we are all done, let's submit the task to rerun in an hour.
self.resubmit(delay=60*60)
def sync_photoset(self, photoset):
log.debug("Syncing flickr photoset %s to facebook", photoset.get('id'))
sync = Session.query(SyncRecord).filter_by(
flickrid=photoset.get('id'), type=SyncRecord.TYPE_ALBUM).first()
album = None
if sync and sync.success:
# don't resync unless we failed before
log.debug("skipping... already synced")
album = fb.GraphAlbum(id=sync.fbid, access_token=self.user.fb_access_token)
if not album.data:
album = None
if not album:
sync = SyncRecord(SyncRecord.TYPE_ALBUM, self.task.user_id)
sync.flickrid = photoset.get('id')
title = photoset.find('title').text
description = photoset.find('description').text
album = self.fb_user.albums.add(
title.encode('utf-8') if title else '',
description.encode('utf-8') if description else '',
privacy=self.fb_privacy)
if album:
sync.fbid = album.id
sync.status = SyncRecord.STATUS_SUCCESS
else:
sync.status = SyncRecord.STATUS_FAILED
Session.add(sync)
Session.commit()
self.set_status(self.synced_photos, self.total_photos,
"Failed to sync %s" % title)
return
photos = self.fk.photosets_getPhotos(photoset_id=photoset.get('id'))[0]
# max out at 200 photos until we figure out what to do with bigger sets
photos = photos[:200]
photos_to_sync = []
for photo in photos:
log.debug("Syncing flickr photo %s to facebook", photo.get('id'))
sync = SyncRecord.get_for_flickrid(photo.get('id')).first()
fb_photo = None
if sync and sync.fbid and sync.success:
log.debug("Skipping... already synced")
fb_photo = fb.GraphPhoto(id=sync.fbid, access_token=self.user.fb_access_token)
#if not fb_photo.data:
# fb_photo = None
self.synced_photos += 1
if not fb_photo:
photos_to_sync.append(photo)
status = "%s photos from %s already synced" % (self.synced_photos,
photoset.find('title').text)
self.set_status(self.synced_photos, self.total_photos, status)
if not photos_to_sync:
return
def on_progress(processed, total):
self.set_status(
processed,
total,
"Found %s/%s photos in %s..." % (
processed, total, photoset.find('title').text))
fetcher = http.Fetcher(progress_callback=on_progress)
requests = []
for photo in photos_to_sync:
url = flickr.get_url(
self.user.flickr_token,
method='flickr.photos.getSizes',
photo_id=photo.get('id'))
request = http.JsonRequest(url)
requests.append((photo, request))
fetcher.queue(request)
fetcher.run()
def on_progress(processed, total):
self.set_status(
processed,
total,
"Downloaded %s/%s images from %s" % (
processed, total, photoset.find('title').text))
fetcher = http.Fetcher(progress_callback=on_progress)
img_requests = []
# TODO: make tmp directory path configurable
if not os.path.exists('/tmp/photosync'):
os.mkdir('/tmp/photosync')
os.chmod('/tmp/photosync', 0777)
for i, (photo, request) in enumerate(requests):
sync = SyncRecord(SyncRecord.TYPE_PHOTO, self.task.user_id)
sync.flickrid = photo.get("id")
Session.add(sync)
res = request.read_response()
try:
img_url = res['sizes']['size'][-1]['source']
except:
#import pdb; pdb.set_trace()
raise
filename = '/tmp/photosync/flickr-'+sync.flickrid+'.jpg'
log.debug("Downloading image %s to %s", img_url, filename)
img_request = None
if not os.path.exists(filename):
f = open(filename, 'wb')
f.write(urllib2.urlopen(img_url).read())
f.close()
on_progress(i, len(requests))
# TODO: Figure out why curl isn't working here
# for some reason when we use the code below,
# the complete file does not get downloaded.
#img_request = http.Request(img_url, filename=filename)
#fetcher.queue(img_request)
img_requests.append((photo, filename, sync, img_request))
Session.commit()
fetcher.run()
for photo, temp_filename, sync, img_request in img_requests:
sync.transfer_in = os.path.getsize(temp_filename)
log.debug("Uploading image %s to facebook", temp_filename)
graph_photo = None
title = photo.get('title')
try:
title = title.encode('utf-8')
except UnicodeEncodeError, e:
log.error("Failed to encode %s to utf-8", title)
log.exception(e)
# better a photo with no title than no photo at all.
title = ""
try:
graph_photo = album.photos.add(temp_filename, title)
except TypeError, e:
log.error("Error uploading image %s", temp_filename)
log.exception(e)
os.remove(temp_filename)
if graph_photo:
sync.fbid = graph_photo.id
sync.status = SyncRecord.STATUS_SUCCESS
sync.transfer_out = sync.transfer_in
else:
sync.status = SyncRecord.STATUS_FAILED
Session.commit()
self.synced_photos += 1
status = "Synced %s from %s to Facebook" % (photo.get('title'),
photoset.find('title').text)
self.set_status(self.synced_photos, self.total_photos, status)
def task(self):
if not self.__task_fetched:
self.__task_fetched = True
self.__task = Session.query(AsyncTask).filter_by(
id=self.__task_id).first()
return self.__task
