def migrate_sync_record(sync_record, content_size): if content_size: sync_record.transfer_in = content_size sync_record.transfer_out = content_size Session.commit() else: print "Couldn't get content size for", sync_record.id
def index(self): gd_client = gdata.photos.service.PhotosService() #token = request.GET.getone('token') parameters = cgi.FieldStorage() token = parameters['token'] gd_client.auth_token = token gd_client.UpgradeToSessionToken() session['picasa_token'] = token session.save() if session.get('user_id'): # a user is already logged in user = Session.query(User).filter_by(id=session.get('user_id')).first() else: # the user is not already logged in, let's see if they have # already created an account before user = Session.query(User).filter_by(picasa_token=token).first() if user: user.picasa_token = token Session.commit() else: # the user does not have an account. We need to create a new one # for them. user = User(picasa_token=token) Session.add(user) Session.commit() user = Session.query(User).filter_by(picasa_token=picasa_token).first() session['user_id'] = user.id session['picasa_token'] = token session.save() log.info("Logged in user %s", user) redirect(url('index'))
def migrate_sync_record(sync_record, content_size): if content_size: sync_record.transfer_in = content_size sync_record.transfer_out = content_size Session.commit() else: print "Couldn't get content size for", sync_record.id
def set_status(self, completed, total, data): log.debug("%s %s Completed %s/%s: %r", self.__class__.__name__, self.job.jid, completed, total, data) self.job.touch() self.task.set_status(completed, total, data) Session.commit()
def index(self): code = request.GET.getone('code') nexturl = request.GET.get('nexturl') if nexturl: # we are acting only as an auth server. # redirect to the server that wants the auth code redirect(nexturl+'?code=%s' % code) return token = fb.get_access_token(code) if not token: #lame... this failed for some reason h.flash("The login process failed :(") redirect(url('index')) return fbuser = fb.GraphUser(access_token=token) if session.get('user_id'): # a user is already logged in user = Session.query(User).filter_by(id=session.get('user_id')).first() else: # the user is not already logged in, let's see if they have # already created an account before user = Session.query(User).filter_by(fb_uid=fbuser.id).first() if user: # the user does have an account, let's update their auth token user.fb_uid = fbuser.id user.fb_access_token = token Session.commit() else: # the user does not have an account. We need to create a new one # for them. for attempt in xrange(3): user = User(fb_uid=fbuser.id, fb_access_token=token) Session.add(user) Session.commit() user = Session.query(User).filter_by(fb_uid=fbuser.id).first() if user: break log.error("Failed to create user with fb_uid=%r attempt %r", fbuser.id, attempt) if not user: log.error("Trying to log in, but couldn't get a user object. " "user=%r code=%r token=%r fbuser=%r", user, code, token, fbuser) session['user_id'] = user.id session['fb_access_token'] = token session.save() log.info("Logged in user %s %s: %s", fbuser.first_name, fbuser.last_name, user) redirect(url('index'))
def index(self): code = request.GET.getone('code') nexturl = request.GET.get('nexturl') if nexturl: # we are acting only as an auth server. # redirect to the server that wants the auth code redirect(nexturl + '?code=%s' % code) return token = fb.get_access_token(code) if not token: #lame... this failed for some reason h.flash("The login process failed :(") redirect(url('index')) return fbuser = fb.GraphUser(access_token=token) if session.get('user_id'): # a user is already logged in user = Session.query(User).filter_by( id=session.get('user_id')).first() else: # the user is not already logged in, let's see if they have # already created an account before user = Session.query(User).filter_by(fb_uid=fbuser.id).first() if user: # the user does have an account, let's update their auth token user.fb_uid = fbuser.id user.fb_access_token = token Session.commit() else: # the user does not have an account. We need to create a new one # for them. for attempt in xrange(3): user = User(fb_uid=fbuser.id, fb_access_token=token) Session.add(user) Session.commit() user = Session.query(User).filter_by(fb_uid=fbuser.id).first() if user: break log.error("Failed to create user with fb_uid=%r attempt %r", fbuser.id, attempt) if not user: log.error( "Trying to log in, but couldn't get a user object. " "user=%r code=%r token=%r fbuser=%r", user, code, token, fbuser) session['user_id'] = user.id session['fb_access_token'] = token session.save() log.info("Logged in user %s %s: %s", fbuser.first_name, fbuser.last_name, user) redirect(url('index'))
def submit_advanced(cls, args, kwargs, delay=0, user_id=None): task = AsyncTask(user_id=user_id) task.set_status(*cls.get_initial_status()) task.type = cls.get_type() Session.add(task) Session.commit() queue_id = job.submit_advanced(cls, (task.id, args, kwargs), {}, delay=delay) task.queue_id = queue_id Session.commit() return task
def submit_advanced(cls, args, kwargs, delay=0, user_id=None): task = AsyncTask(user_id=user_id) task.set_status(*cls.get_initial_status()) task.type = cls.get_type() Session.add(task) Session.commit() queue_id = job.submit_advanced(cls, (task.id, args, kwargs), {}, delay=delay) task.queue_id = queue_id Session.commit() return task
def verify_user(self): """Check the access tokens""" if self.fb_access_token: fbuser = fb.GraphUser(access_token=self.fb_access_token) try: fbuser.load() except fb.OAuthException, e: self.fb_access_token = None Session.commit() return False
def __call__(self): try: if self.task.queue_id: if self.task.queue_id != self.job.jid: # this job is no longer supposed to exist... return self.job.delete() else: self.task.queue_id = self.job.jid Session.commit() result = self.run(*self.__args, **self.__kwargs) except Exception, e: traceback.print_exc() log.exception("Failed to run task %s", self.__class__.__name__) raise
def __call__(self): try: if self.task.queue_id: if self.task.queue_id != self.job.jid: # this job is no longer supposed to exist... return self.job.delete() else: self.task.queue_id = self.job.jid Session.commit() result = self.run(*self.__args, **self.__kwargs) except Exception, e: traceback.print_exc() log.exception("Failed to run task %s", self.__class__.__name__) raise
def index(self): gd_client = gdata.photos.service.PhotosService() #token = request.GET.getone('token') parameters = cgi.FieldStorage() token = parameters['token'] gd_client.auth_token = token gd_client.UpgradeToSessionToken() session['picasa_token'] = token session.save() if session.get('user_id'): # a user is already logged in user = Session.query(User).filter_by( id=session.get('user_id')).first() else: # the user is not already logged in, let's see if they have # already created an account before user = Session.query(User).filter_by(picasa_token=token).first() if user: user.picasa_token = token Session.commit() else: # the user does not have an account. We need to create a new one # for them. user = User(picasa_token=token) Session.add(user) Session.commit() user = Session.query(User).filter_by( picasa_token=picasa_token).first() session['user_id'] = user.id session['picasa_token'] = token session.save() log.info("Logged in user %s", user) redirect(url('index'))
def index(self): flickr = FlickrAPI() token = flickr.get_token(request.GET.getone('frob')) session['flickr_token'] = token session.save() flickr = FlickrAPI(token) result = flickr.auth_checkToken() nsid = result[0][2].get('nsid') if session.get('user_id'): # a user is already logged in user = Session.query(User).filter_by( id=session.get('user_id')).first() else: # the user is not already logged in, let's see if they have # already created an account before user = Session.query(User).filter_by(flickr_nsid=nsid).first() if user: user.flickr_nsid = nsid user.flickr_token = token Session.commit() else: # the user does not have an account. We need to create a new one # for them. user = User(flickr_nsid=nsid, flickr_token=token) Session.add(user) Session.commit() user = Session.query(User).filter_by(flickr_nsid=nsid).first() session['user_id'] = user.id session['flickr_token'] = token session.save() log.info("Logged in user %s", user) redirect(url('index'))
def index(self): flickr = FlickrAPI() token = flickr.get_token(request.GET.getone('frob')) session['flickr_token'] = token session.save() flickr = FlickrAPI(token) result = flickr.auth_checkToken() nsid = result[0][2].get('nsid') if session.get('user_id'): # a user is already logged in user = Session.query(User).filter_by(id=session.get('user_id')).first() else: # the user is not already logged in, let's see if they have # already created an account before user = Session.query(User).filter_by(flickr_nsid=nsid).first() if user: user.flickr_nsid = nsid user.flickr_token = token Session.commit() else: # the user does not have an account. We need to create a new one # for them. user = User(flickr_nsid=nsid, flickr_token=token) Session.add(user) Session.commit() user = Session.query(User).filter_by(flickr_nsid=nsid).first() session['user_id'] = user.id session['flickr_token'] = token session.save() log.info("Logged in user %s", user) redirect(url('index'))
class User(Base): __tablename__ = 'users' id = Column('id', Integer, primary_key=True) fb_uid = Column('fb_uid', Integer, index=True) flickr_nsid = Column('flickr_nsid', String(100), index=True) flickr_token = Column('flickr_token', UnicodeText) fb_access_token = Column('fb_access_token', UnicodeText) def __init__(self, fb_uid=None, fb_access_token=None, flickr_nsid=None, flickr_token=None): self.fb_uid = fb_uid self.flickr_nsid = flickr_nsid self.fb_access_token = fb_access_token self.flickr_token = flickr_token def __unicode__(self): return u'%s' % self.id def __repr__(self): return "<User id=%s fb_uid=%s flickr_nsid=%s>" % ( self.id, self.fb_uid, self.flickr_nsid) def verify_user(self): """Check the access tokens""" if self.fb_access_token: fbuser = fb.GraphUser(access_token=self.fb_access_token) try: fbuser.load() except fb.OAuthException, e: self.fb_access_token = None Session.commit() return False if self.flickr_token: fluser = flickr.FlickrUser(token=self.flickr_token) try: fluser.auth_checkToken() except flickr.FlickrError, e: self.flickr_token = None Session.commit() return False
def set_status(self, completed, total, data): log.debug("%s %s Completed %s/%s: %r", self.__class__.__name__, self.job.jid, completed, total, data) self.job.touch() self.task.set_status(completed, total, data) Session.commit()
def sync_photoset(self, photoset): log.debug("Syncing flickr photoset %s to facebook", photoset.get("id")) sync = Session.query(SyncRecord).filter_by(flickrid=photoset.get("id"), type=SyncRecord.TYPE_ALBUM).first() album = None if sync and sync.success: # don't resync unless we failed before log.debug("skipping... already synced") album = fb.GraphAlbum(id=sync.fbid, access_token=self.user.fb_access_token) if not album.data: album = None if not album: sync = SyncRecord(SyncRecord.TYPE_ALBUM, self.task.user_id) sync.flickrid = photoset.get("id") title = photoset.find("title").text description = photoset.find("description").text album = self.fb_user.albums.add( title.encode("utf-8") if title else "", description.encode("utf-8") if description else "", privacy=self.fb_privacy, ) if album: sync.fbid = album.id sync.status = SyncRecord.STATUS_SUCCESS else: sync.status = SyncRecord.STATUS_FAILED Session.add(sync) Session.commit() self.set_status(self.synced_photos, self.total_photos, "Failed to sync %s" % title) return photos = self.fk.photosets_getPhotos(photoset_id=photoset.get("id"))[0] # max out at 200 photos until we figure out what to do with bigger sets photos = photos[:200] photos_to_sync = [] for photo in photos: log.debug("Syncing flickr photo %s to facebook", photo.get("id")) sync = SyncRecord.get_for_flickrid(photo.get("id")).first() fb_photo = None if sync and sync.fbid and sync.success: log.debug("Skipping... already synced") fb_photo = fb.GraphPhoto(id=sync.fbid, access_token=self.user.fb_access_token) # if not fb_photo.data: # fb_photo = None self.synced_photos += 1 if not fb_photo: photos_to_sync.append(photo) status = "%s photos from %s already synced" % (self.synced_photos, photoset.find("title").text) self.set_status(self.synced_photos, self.total_photos, status) if not photos_to_sync: return def on_progress(processed, total): self.set_status( processed, total, "Found %s/%s photos in %s..." % (processed, total, photoset.find("title").text) ) fetcher = http.Fetcher(progress_callback=on_progress) requests = [] for photo in photos_to_sync: url = flickr.get_url(self.user.flickr_token, method="flickr.photos.getSizes", photo_id=photo.get("id")) request = http.JsonRequest(url) requests.append((photo, request)) fetcher.queue(request) fetcher.run() def on_progress(processed, total): self.set_status( processed, total, "Downloaded %s/%s images from %s" % (processed, total, photoset.find("title").text) ) fetcher = http.Fetcher(progress_callback=on_progress) img_requests = [] # TODO: make tmp directory path configurable if not os.path.exists("/tmp/photosync"): os.mkdir("/tmp/photosync") os.chmod("/tmp/photosync", 0777) for i, (photo, request) in enumerate(requests): sync = SyncRecord(SyncRecord.TYPE_PHOTO, self.task.user_id) sync.flickrid = photo.get("id") Session.add(sync) res = request.read_response() try: img_url = res["sizes"]["size"][-1]["source"] except: # import pdb; pdb.set_trace() raise filename = "/tmp/photosync/flickr-" + sync.flickrid + ".jpg" log.debug("Downloading image %s to %s", img_url, filename) img_request = None if not os.path.exists(filename): f = open(filename, "wb") f.write(urllib2.urlopen(img_url).read()) f.close() on_progress(i, len(requests)) # TODO: Figure out why curl isn't working here # for some reason when we use the code below, # the complete file does not get downloaded. # img_request = http.Request(img_url, filename=filename) # fetcher.queue(img_request) img_requests.append((photo, filename, sync, img_request)) Session.commit() fetcher.run() for photo, temp_filename, sync, img_request in img_requests: sync.transfer_in = os.path.getsize(temp_filename) log.debug("Uploading image %s to facebook", temp_filename) graph_photo = None title = photo.get("title") try: title = title.encode("utf-8") except UnicodeEncodeError, e: log.error("Failed to encode %s to utf-8", title) log.exception(e) # better a photo with no title than no photo at all. title = "" try: graph_photo = album.photos.add(temp_filename, title) except TypeError, e: log.error("Error uploading image %s", temp_filename) log.exception(e)
def set(setting_id, value, user_id=None): user_id = user_id or session.get('user_id') result = Session.merge(UserSetting(user_id, setting_id, value)) Session.commit() return result
def sync_photoset(self, photoset): log.debug("Syncing flickr photoset %s to facebook", photoset.get('id')) sync = Session.query(SyncRecord).filter_by( flickrid=photoset.get('id'), type=SyncRecord.TYPE_ALBUM).first() album = None if sync and sync.success: # don't resync unless we failed before log.debug("skipping... already synced") album = fb.GraphAlbum(id=sync.fbid, access_token=self.user.fb_access_token) if not album.data: album = None if not album: sync = SyncRecord(SyncRecord.TYPE_ALBUM, self.task.user_id) sync.flickrid = photoset.get('id') title = photoset.find('title').text description = photoset.find('description').text album = self.fb_user.albums.add( title.encode('utf-8') if title else '', description.encode('utf-8') if description else '', privacy=self.fb_privacy) if album: sync.fbid = album.id sync.status = SyncRecord.STATUS_SUCCESS else: sync.status = SyncRecord.STATUS_FAILED Session.add(sync) Session.commit() self.set_status(self.synced_photos, self.total_photos, "Failed to sync %s" % title) return photos = self.fk.photosets_getPhotos(photoset_id=photoset.get('id'))[0] # max out at 200 photos until we figure out what to do with bigger sets photos = photos[:200] photos_to_sync = [] for photo in photos: log.debug("Syncing flickr photo %s to facebook", photo.get('id')) sync = SyncRecord.get_for_flickrid(photo.get('id')).first() fb_photo = None if sync and sync.fbid and sync.success: log.debug("Skipping... already synced") fb_photo = fb.GraphPhoto(id=sync.fbid, access_token=self.user.fb_access_token) #if not fb_photo.data: # fb_photo = None self.synced_photos += 1 if not fb_photo: photos_to_sync.append(photo) status = "%s photos from %s already synced" % (self.synced_photos, photoset.find('title').text) self.set_status(self.synced_photos, self.total_photos, status) if not photos_to_sync: return def on_progress(processed, total): self.set_status( processed, total, "Found %s/%s photos in %s..." % ( processed, total, photoset.find('title').text)) fetcher = http.Fetcher(progress_callback=on_progress) requests = [] for photo in photos_to_sync: url = flickr.get_url( self.user.flickr_token, method='flickr.photos.getSizes', photo_id=photo.get('id')) request = http.JsonRequest(url) requests.append((photo, request)) fetcher.queue(request) fetcher.run() def on_progress(processed, total): self.set_status( processed, total, "Downloaded %s/%s images from %s" % ( processed, total, photoset.find('title').text)) fetcher = http.Fetcher(progress_callback=on_progress) img_requests = [] # TODO: make tmp directory path configurable if not os.path.exists('/tmp/photosync'): os.mkdir('/tmp/photosync') os.chmod('/tmp/photosync', 0777) for i, (photo, request) in enumerate(requests): sync = SyncRecord(SyncRecord.TYPE_PHOTO, self.task.user_id) sync.flickrid = photo.get("id") Session.add(sync) res = request.read_response() try: img_url = res['sizes']['size'][-1]['source'] except: #import pdb; pdb.set_trace() raise filename = '/tmp/photosync/flickr-'+sync.flickrid+'.jpg' log.debug("Downloading image %s to %s", img_url, filename) img_request = None if not os.path.exists(filename): f = open(filename, 'wb') f.write(urllib2.urlopen(img_url).read()) f.close() on_progress(i, len(requests)) # TODO: Figure out why curl isn't working here # for some reason when we use the code below, # the complete file does not get downloaded. #img_request = http.Request(img_url, filename=filename) #fetcher.queue(img_request) img_requests.append((photo, filename, sync, img_request)) Session.commit() fetcher.run() for photo, temp_filename, sync, img_request in img_requests: sync.transfer_in = os.path.getsize(temp_filename) log.debug("Uploading image %s to facebook", temp_filename) graph_photo = None title = photo.get('title') try: title = title.encode('utf-8') except UnicodeEncodeError, e: log.error("Failed to encode %s to utf-8", title) log.exception(e) # better a photo with no title than no photo at all. title = "" try: graph_photo = album.photos.add(temp_filename, title) except TypeError, e: log.error("Error uploading image %s", temp_filename) log.exception(e)
class FullSync(TaskHandler): @classmethod def run_for_user_now(cls, user_id): last = AsyncTask.get_for_user(user_id=user_id, type=cls.get_type(), limit=1).first() if not last or last.is_completed: cls.submit_advanced((user_id,), {}, user_id=user_id) else: last.run_now() @classmethod def get_initial_status(cls): return (None, None, "Waiting for syncer to become available...") def run(self, user_id): self.user = User.get_by_id(user_id) if not self.user.verify_user(): # not really much we can do about this... log.info("Missing tokens for user %r", self.user) self.set_status(0, 0, "You need to log out and log back in again.") return self.fk = flickr.FlickrAPI(self.user.flickr_token) self.fb_user = fb.GraphUser(access_token=self.user.fb_access_token) photosets = self.fk.photosets_getList()[0] self.synced_photos = 0 self.total_photos = sum(int(pset.get('photos')) for pset in photosets) self.set_status( self.synced_photos, self.total_photos, "syncing Flickr photos to Facebook") settings = UserSetting.multiget( user_id=user_id, settings=[UserSettingConst.FB_PRIVACY, UserSettingConst.FLICKR_SET_SYNCING]) self.fb_privacy = settings.get(UserSettingConst.FB_PRIVACY, 'FB_DEFAULT') self.flickr_settings = settings.get(UserSettingConst.FLICKR_SET_SYNCING, {}) select_sets = self.flickr_settings.get('select_sets', False) selected_sets = self.flickr_settings.get('selected_sets', []) for photoset in photosets: if not select_sets or photoset.get('id') in selected_sets: self.sync_photoset(photoset) log.info("Finished job %r for user %r", self.job.jid, self.user.id) # once we are all done, let's submit the task to rerun in an hour. self.resubmit(delay=60*60) def sync_photoset(self, photoset): log.debug("Syncing flickr photoset %s to facebook", photoset.get('id')) sync = Session.query(SyncRecord).filter_by( flickrid=photoset.get('id'), type=SyncRecord.TYPE_ALBUM).first() album = None if sync and sync.success: # don't resync unless we failed before log.debug("skipping... already synced") album = fb.GraphAlbum(id=sync.fbid, access_token=self.user.fb_access_token) if not album.data: album = None if not album: sync = SyncRecord(SyncRecord.TYPE_ALBUM, self.task.user_id) sync.flickrid = photoset.get('id') title = photoset.find('title').text description = photoset.find('description').text album = self.fb_user.albums.add( title.encode('utf-8') if title else '', description.encode('utf-8') if description else '', privacy=self.fb_privacy) if album: sync.fbid = album.id sync.status = SyncRecord.STATUS_SUCCESS else: sync.status = SyncRecord.STATUS_FAILED Session.add(sync) Session.commit() self.set_status(self.synced_photos, self.total_photos, "Failed to sync %s" % title) return photos = self.fk.photosets_getPhotos(photoset_id=photoset.get('id'))[0] # max out at 200 photos until we figure out what to do with bigger sets photos = photos[:200] photos_to_sync = [] for photo in photos: log.debug("Syncing flickr photo %s to facebook", photo.get('id')) sync = SyncRecord.get_for_flickrid(photo.get('id')).first() fb_photo = None if sync and sync.fbid and sync.success: log.debug("Skipping... already synced") fb_photo = fb.GraphPhoto(id=sync.fbid, access_token=self.user.fb_access_token) #if not fb_photo.data: # fb_photo = None self.synced_photos += 1 if not fb_photo: photos_to_sync.append(photo) status = "%s photos from %s already synced" % (self.synced_photos, photoset.find('title').text) self.set_status(self.synced_photos, self.total_photos, status) if not photos_to_sync: return def on_progress(processed, total): self.set_status( processed, total, "Found %s/%s photos in %s..." % ( processed, total, photoset.find('title').text)) fetcher = http.Fetcher(progress_callback=on_progress) requests = [] for photo in photos_to_sync: url = flickr.get_url( self.user.flickr_token, method='flickr.photos.getSizes', photo_id=photo.get('id')) request = http.JsonRequest(url) requests.append((photo, request)) fetcher.queue(request) fetcher.run() def on_progress(processed, total): self.set_status( processed, total, "Downloaded %s/%s images from %s" % ( processed, total, photoset.find('title').text)) fetcher = http.Fetcher(progress_callback=on_progress) img_requests = [] # TODO: make tmp directory path configurable if not os.path.exists('/tmp/photosync'): os.mkdir('/tmp/photosync') os.chmod('/tmp/photosync', 0777) for i, (photo, request) in enumerate(requests): sync = SyncRecord(SyncRecord.TYPE_PHOTO, self.task.user_id) sync.flickrid = photo.get("id") Session.add(sync) res = request.read_response() try: img_url = res['sizes']['size'][-1]['source'] except: #import pdb; pdb.set_trace() raise filename = '/tmp/photosync/flickr-'+sync.flickrid+'.jpg' log.debug("Downloading image %s to %s", img_url, filename) img_request = None if not os.path.exists(filename): f = open(filename, 'wb') f.write(urllib2.urlopen(img_url).read()) f.close() on_progress(i, len(requests)) # TODO: Figure out why curl isn't working here # for some reason when we use the code below, # the complete file does not get downloaded. #img_request = http.Request(img_url, filename=filename) #fetcher.queue(img_request) img_requests.append((photo, filename, sync, img_request)) Session.commit() fetcher.run() for photo, temp_filename, sync, img_request in img_requests: sync.transfer_in = os.path.getsize(temp_filename) log.debug("Uploading image %s to facebook", temp_filename) graph_photo = None title = photo.get('title') try: title = title.encode('utf-8') except UnicodeEncodeError, e: log.error("Failed to encode %s to utf-8", title) log.exception(e) # better a photo with no title than no photo at all. title = "" try: graph_photo = album.photos.add(temp_filename, title) except TypeError, e: log.error("Error uploading image %s", temp_filename) log.exception(e) os.remove(temp_filename) if graph_photo: sync.fbid = graph_photo.id sync.status = SyncRecord.STATUS_SUCCESS sync.transfer_out = sync.transfer_in else: sync.status = SyncRecord.STATUS_FAILED Session.commit() self.synced_photos += 1 status = "Synced %s from %s to Facebook" % (photo.get('title'), photoset.find('title').text) self.set_status(self.synced_photos, self.total_photos, status)
def run_now(self): self.queue_id = None Session.commit() self.queue_id = self._job.resubmit() self.__job = "SENTINAL" Session.commit()