Beispiel #1
0
 def migrate_sync_record(sync_record, content_size):
     if content_size:
         sync_record.transfer_in = content_size
         sync_record.transfer_out = content_size
         Session.commit()
     else:
         print "Couldn't get content size for", sync_record.id
Beispiel #2
0
 def migrate_sync_record(sync_record, content_size):
     if content_size:
         sync_record.transfer_in = content_size
         sync_record.transfer_out = content_size
         Session.commit()
     else:
         print "Couldn't get content size for", sync_record.id
Beispiel #3
0
 def set_status(self, completed, total, data):
     log.debug("%s %s Completed %s/%s: %r",
              self.__class__.__name__, self.job.jid,
              completed, total, data)
     self.job.touch()
     self.task.set_status(completed, total, data)
     Session.commit()
Beispiel #4
0
 def __call__(self, environ, start_response):
     """Invoke the Controller"""
     # WSGIController.__call__ dispatches to the Controller method
     # the request is routed to. This routing information is
     # available in environ['pylons.routes_dict']
     try:
         return WSGIController.__call__(self, environ, start_response)
     finally:
         Session.remove()
Beispiel #5
0
 def __call__(self, environ, start_response):
     """Invoke the Controller"""
     # WSGIController.__call__ dispatches to the Controller method
     # the request is routed to. This routing information is
     # available in environ['pylons.routes_dict']
     try:
         return WSGIController.__call__(self, environ, start_response)
     finally:
         Session.remove()
Beispiel #6
0
 def verify_user(self):
     """Check the access tokens"""
     if self.fb_access_token:
         fbuser = fb.GraphUser(access_token=self.fb_access_token)
         try:
             fbuser.load()
         except fb.OAuthException, e:
             self.fb_access_token = None
             Session.commit()
             return False
Beispiel #7
0
    def index(self):
	gd_client = gdata.photos.service.PhotosService()
	#token = request.GET.getone('token')
	parameters = cgi.FieldStorage()
	token = parameters['token']
	gd_client.auth_token = token
	gd_client.UpgradeToSessionToken()

        session['picasa_token'] = token
        session.save()

        if session.get('user_id'):
            # a user is already logged in
            user = Session.query(User).filter_by(id=session.get('user_id')).first()
        else:
            # the user is not already logged in, let's see if they have
            # already created an account before
            user = Session.query(User).filter_by(picasa_token=token).first()

        if user:
            user.picasa_token = token
            Session.commit()
        else:
            # the user does not have an account.  We need to create a new one
            # for them.
            user = User(picasa_token=token)
            Session.add(user)
            Session.commit()
            user = Session.query(User).filter_by(picasa_token=picasa_token).first()

        session['user_id'] = user.id
        session['picasa_token'] = token
        session.save()
        log.info("Logged in user %s", user)
        redirect(url('index'))
Beispiel #8
0
    def index(self):
        code = request.GET.getone('code')
        nexturl = request.GET.get('nexturl')
        if nexturl:
            # we are acting only as an auth server.
            # redirect to the server that wants the auth code
            redirect(nexturl+'?code=%s' % code)
            return

        token = fb.get_access_token(code)
        if not token:
            #lame... this failed for some reason
            h.flash("The login process failed :(")
            redirect(url('index'))
            return
        fbuser = fb.GraphUser(access_token=token)

        if session.get('user_id'):
            # a user is already logged in
            user = Session.query(User).filter_by(id=session.get('user_id')).first()
        else:
            # the user is not already logged in, let's see if they have
            # already created an account before
            user = Session.query(User).filter_by(fb_uid=fbuser.id).first()

        if user:
            # the user does have an account, let's update their auth token
            user.fb_uid = fbuser.id
            user.fb_access_token = token
            Session.commit()
        else:
            # the user does not have an account.  We need to create a new one
            # for them.
            for attempt in xrange(3):
                user = User(fb_uid=fbuser.id,
                            fb_access_token=token)
                Session.add(user)
                Session.commit()
                user = Session.query(User).filter_by(fb_uid=fbuser.id).first()
                if user:
                    break
                log.error("Failed to create user with fb_uid=%r attempt %r",
                          fbuser.id, attempt)

        if not user:
            log.error("Trying to log in, but couldn't get a user object. "
                      "user=%r code=%r token=%r fbuser=%r",
                      user, code, token, fbuser)

        session['user_id'] = user.id
        session['fb_access_token'] = token
        session.save()
        log.info("Logged in user %s %s: %s",
                 fbuser.first_name,
                 fbuser.last_name,
                 user)
        redirect(url('index'))
Beispiel #9
0
    def index(self):
        code = request.GET.getone('code')
        nexturl = request.GET.get('nexturl')
        if nexturl:
            # we are acting only as an auth server.
            # redirect to the server that wants the auth code
            redirect(nexturl + '?code=%s' % code)
            return

        token = fb.get_access_token(code)
        if not token:
            #lame... this failed for some reason
            h.flash("The login process failed :(")
            redirect(url('index'))
            return
        fbuser = fb.GraphUser(access_token=token)

        if session.get('user_id'):
            # a user is already logged in
            user = Session.query(User).filter_by(
                id=session.get('user_id')).first()
        else:
            # the user is not already logged in, let's see if they have
            # already created an account before
            user = Session.query(User).filter_by(fb_uid=fbuser.id).first()

        if user:
            # the user does have an account, let's update their auth token
            user.fb_uid = fbuser.id
            user.fb_access_token = token
            Session.commit()
        else:
            # the user does not have an account.  We need to create a new one
            # for them.
            for attempt in xrange(3):
                user = User(fb_uid=fbuser.id, fb_access_token=token)
                Session.add(user)
                Session.commit()
                user = Session.query(User).filter_by(fb_uid=fbuser.id).first()
                if user:
                    break
                log.error("Failed to create user with fb_uid=%r attempt %r",
                          fbuser.id, attempt)

        if not user:
            log.error(
                "Trying to log in, but couldn't get a user object. "
                "user=%r code=%r token=%r fbuser=%r", user, code, token,
                fbuser)

        session['user_id'] = user.id
        session['fb_access_token'] = token
        session.save()
        log.info("Logged in user %s %s: %s", fbuser.first_name,
                 fbuser.last_name, user)
        redirect(url('index'))
Beispiel #10
0
 def __call__(self):
     try:
         if self.task.queue_id:
             if self.task.queue_id != self.job.jid:
                 # this job is no longer supposed to exist...
                 return self.job.delete()
         else:
             self.task.queue_id = self.job.jid
             Session.commit()
         result = self.run(*self.__args, **self.__kwargs)
     except Exception, e:
         traceback.print_exc()
         log.exception("Failed to run task %s", self.__class__.__name__)
         raise
Beispiel #11
0
 def __call__(self):
     try:
         if self.task.queue_id:
             if self.task.queue_id != self.job.jid:
                 # this job is no longer supposed to exist...
                 return self.job.delete()
         else:
             self.task.queue_id = self.job.jid
             Session.commit()
         result = self.run(*self.__args, **self.__kwargs)
     except Exception, e:
         traceback.print_exc()
         log.exception("Failed to run task %s", self.__class__.__name__)
         raise
Beispiel #12
0
def load_transfer_data():
    """Loads bytes transferred data into the db.

    You will want to run this sql first:

      ALTER TABLE sync_records ADD COLUMN transfer_in int(11) DEFAULT 0;
      ALTER TABLE sync_records ADD COLUMN transfer_out int(11) DEFAULT 0;

    """

    records = Session.query(SyncRecord).filter(
        SyncRecord.type==SyncRecord.TYPE_PHOTO).filter(
        SyncRecord.transfer_in==0).all()

    def progress(message):
        def func(processed, total):
            print "%i/%i\t%i\t%s" % (processed, total, float(processed)/total*100, message)
        return func

    def get_size_request(record):
        return http.JsonRequest(flickr.get_url(
                record.user.flickr_token,
                method='flickr.photos.getSizes',
                photo_id=record.flickrid))


    def get_byte_size_request(request):
        try:
            url = request.read_response()['sizes']['size'][-1]['source']
            return http.Request(url, method="HEAD")
        except:
            pass

    def get_content_size(request):
        return int(request.read_headers()['Content-Length']) if request else None


    def migrate_sync_record(sync_record, content_size):
        if content_size:
            sync_record.transfer_in = content_size
            sync_record.transfer_out = content_size
            Session.commit()
        else:
            print "Couldn't get content size for", sync_record.id


    fetcher = http.Fetcher(
        progress_callback=progress("getting image urls"))

    map(compose(fetcher.queue, get_size_request), records)

    size_fetcher = http.Fetcher(
        progress_callback=progress("getting byte sizes"))

    map(compose(size_fetcher.queue, get_byte_size_request), fetcher.run())

    list(starmap(migrate_sync_record,
                 izip(records, imap(get_content_size, size_fetcher.run()))))
Beispiel #13
0
def load_transfer_data():
    """Loads bytes transferred data into the db.

    You will want to run this sql first:

      ALTER TABLE sync_records ADD COLUMN transfer_in int(11) DEFAULT 0;
      ALTER TABLE sync_records ADD COLUMN transfer_out int(11) DEFAULT 0;

    """

    records = Session.query(SyncRecord).filter(
        SyncRecord.type == SyncRecord.TYPE_PHOTO).filter(
            SyncRecord.transfer_in == 0).all()

    def progress(message):
        def func(processed, total):
            print "%i/%i\t%i\t%s" % (processed, total,
                                     float(processed) / total * 100, message)

        return func

    def get_size_request(record):
        return http.JsonRequest(
            flickr.get_url(record.user.flickr_token,
                           method='flickr.photos.getSizes',
                           photo_id=record.flickrid))

    def get_byte_size_request(request):
        try:
            url = request.read_response()['sizes']['size'][-1]['source']
            return http.Request(url, method="HEAD")
        except:
            pass

    def get_content_size(request):
        return int(
            request.read_headers()['Content-Length']) if request else None

    def migrate_sync_record(sync_record, content_size):
        if content_size:
            sync_record.transfer_in = content_size
            sync_record.transfer_out = content_size
            Session.commit()
        else:
            print "Couldn't get content size for", sync_record.id

    fetcher = http.Fetcher(progress_callback=progress("getting image urls"))

    map(compose(fetcher.queue, get_size_request), records)

    size_fetcher = http.Fetcher(
        progress_callback=progress("getting byte sizes"))

    map(compose(size_fetcher.queue, get_byte_size_request), fetcher.run())

    list(
        starmap(migrate_sync_record,
                izip(records, imap(get_content_size, size_fetcher.run()))))
Beispiel #14
0
    def multiget(user_id=None, settings=None):
        user_id = user_id or session.get('user_id')
        query = Session.query(UserSetting).filter(UserSetting.user_id==user_id)
        if settings is not None:
            query = query.filter(UserSetting.setting_id.in_(settings))

        results = {}
        for result in query:
            results[result.setting_id] = result.value

        return results
Beispiel #15
0
    def get_for_flickrid(flickrid=None, limit=3, type=None):
        query = Session.query(SyncRecord)\
            .filter(SyncRecord.flickrid==flickrid)

        if type:
            query = query.filter(SyncRecord.type==type)

        query = query.order_by(desc('timestamp'))\
            .limit(limit)

        return query
Beispiel #16
0
    def get_for_user(user_id=None, limit=3, type=None):
        if not user_id:
            user_id = session.get('user_id')

        query = Session.query(AsyncTask)\
            .filter(AsyncTask.user_id==user_id)

        if type:
            query = query.filter(AsyncTask.type==type)

        query = query.order_by(desc('created_time'))\
            .limit(limit)

        return query
Beispiel #17
0
    def stats(self):
        c.user_count = Session.query(User).count()
        c.sync_count = Session.query(SyncRecord).count()
        c.async_tasks_count = Session.query(AsyncTask).count()


        data = Session.query(SyncRecord.user_id,
                             func.count(1),
                             func.sum(SyncRecord.transfer_in),
                             func.sum(SyncRecord.transfer_out))\
            .filter(SyncRecord.type == SyncRecord.TYPE_PHOTO)\
            .group_by(SyncRecord.user_id).all()

        c.user_stats = []
        c.total_count = 0
        c.total_cost = 0
        c.total_tin = 0
        c.total_tout = 0
        for user_id, count, tin, tout in data:
            tin = tin or 0
            tout = tout or 0
            bandwidth_cost = cost.get_bandwidth_cost(tin,tout)
            c.total_cost += bandwidth_cost
            c.total_tout += int(tout)
            c.total_tin += int(tin)
            c.total_count += count
            c.user_stats.append([User.get_by_id(user_id),
                                 count,
                                 round(tin/1024/1024, 2),
                                 round(tout/1024/1024, 2),
                                 bandwidth_cost])
        c.total_tout = round(c.total_tout/1024./1024, 2)
        c.total_tin = round(c.total_tin/1024./1024, 2)

        order_by = int(request.GET.get('order_by', 4))
        c.user_stats.sort(key=lambda d: d[order_by], reverse=True)
        return render('/admin/stats.mako')
Beispiel #18
0
    def stats(self):
        c.user_count = Session.query(User).count()
        c.sync_count = Session.query(SyncRecord).count()
        c.async_tasks_count = Session.query(AsyncTask).count()


        data = Session.query(SyncRecord.user_id,
                             func.count(1),
                             func.sum(SyncRecord.transfer_in),
                             func.sum(SyncRecord.transfer_out))\
            .filter(SyncRecord.type == SyncRecord.TYPE_PHOTO)\
            .group_by(SyncRecord.user_id).all()

        c.user_stats = []
        c.total_count = 0
        c.total_cost = 0
        c.total_tin = 0
        c.total_tout = 0
        for user_id, count, tin, tout in data:
            tin = tin or 0
            tout = tout or 0
            bandwidth_cost = cost.get_bandwidth_cost(tin, tout)
            c.total_cost += bandwidth_cost
            c.total_tout += int(tout)
            c.total_tin += int(tin)
            c.total_count += count
            c.user_stats.append([
                User.get_by_id(user_id), count,
                round(tin / 1024 / 1024, 2),
                round(tout / 1024 / 1024, 2), bandwidth_cost
            ])
        c.total_tout = round(c.total_tout / 1024. / 1024, 2)
        c.total_tin = round(c.total_tin / 1024. / 1024, 2)

        order_by = int(request.GET.get('order_by', 4))
        c.user_stats.sort(key=lambda d: d[order_by], reverse=True)
        return render('/admin/stats.mako')
Beispiel #19
0
 def submit_advanced(cls, args, kwargs, delay=0, user_id=None):
     task = AsyncTask(user_id=user_id)
     task.set_status(*cls.get_initial_status())
     task.type = cls.get_type()
     Session.add(task)
     Session.commit()
     queue_id = job.submit_advanced(cls, (task.id, args, kwargs), {}, delay=delay)
     task.queue_id = queue_id
     Session.commit()
     return task
Beispiel #20
0
 def submit_advanced(cls, args, kwargs, delay=0, user_id=None):
     task = AsyncTask(user_id=user_id)
     task.set_status(*cls.get_initial_status())
     task.type = cls.get_type()
     Session.add(task)
     Session.commit()
     queue_id = job.submit_advanced(cls, (task.id, args, kwargs), {}, delay=delay)
     task.queue_id = queue_id
     Session.commit()
     return task
Beispiel #21
0
def delete_empty_albums():
    """Delete empty albums that were accidentally created."""

    records = Session.query(SyncRecord).filter(
        SyncRecord.type == SyncRecord.TYPE_ALBUM).all()

    for record in records:
        user = record.user
        if user.fb_access_token:
            fbalbum = fb.GraphAlbum(id=record.fbid,
                                    access_token=user.fb_access_token)
            if len(fbalbum.photos) == 0:
                #this is an empty album... let's delete it.
                print "Deleting album %r for user %r" % (record.fbid, user)
                fbalbum.delete()
                #Session.delete(record)
    print "All done"
Beispiel #22
0
    def command(self):
        """Main command that starts the worker."""
        super(KickCommand, self).command()

        users = Session.query(User)\
            .filter(User.fb_uid != None)\
            .filter(User.flickr_nsid != None)

        if self.options.simulate:
            log.info("--SIMULATING--")
        for user in users:
            tasks = AsyncTask.get_for_user(user.id)
            for task in tasks:
                if not (task.is_completed or task.time_left) and task.is_buried:
                    log.info("Running task %r", task)
                    if not self.options.simulate:
                        task.run_now()
Beispiel #23
0
    def index(self):
        gd_client = gdata.photos.service.PhotosService()
        #token = request.GET.getone('token')
        parameters = cgi.FieldStorage()
        token = parameters['token']
        gd_client.auth_token = token
        gd_client.UpgradeToSessionToken()

        session['picasa_token'] = token
        session.save()

        if session.get('user_id'):
            # a user is already logged in
            user = Session.query(User).filter_by(
                id=session.get('user_id')).first()
        else:
            # the user is not already logged in, let's see if they have
            # already created an account before
            user = Session.query(User).filter_by(picasa_token=token).first()

        if user:
            user.picasa_token = token
            Session.commit()
        else:
            # the user does not have an account.  We need to create a new one
            # for them.
            user = User(picasa_token=token)
            Session.add(user)
            Session.commit()
            user = Session.query(User).filter_by(
                picasa_token=picasa_token).first()

        session['user_id'] = user.id
        session['picasa_token'] = token
        session.save()
        log.info("Logged in user %s", user)
        redirect(url('index'))
Beispiel #24
0
def delete_empty_albums():
    """Delete empty albums that were accidentally created."""

    records = Session.query(SyncRecord).filter(
        SyncRecord.type==SyncRecord.TYPE_ALBUM).all()

    for record in records:
        user = record.user
        if user.fb_access_token:
            fbalbum = fb.GraphAlbum(
                id=record.fbid,
                access_token=user.fb_access_token)
            if len(fbalbum.photos) == 0:
                #this is an empty album... let's delete it.
                print "Deleting album %r for user %r" % (record.fbid, user)
                fbalbum.delete()
                #Session.delete(record)
    print "All done"
Beispiel #25
0
    def index(self):
        flickr = FlickrAPI()
        token = flickr.get_token(request.GET.getone('frob'))

        session['flickr_token'] = token
        session.save()

        flickr = FlickrAPI(token)
        result = flickr.auth_checkToken()
        nsid = result[0][2].get('nsid')

        if session.get('user_id'):
            # a user is already logged in
            user = Session.query(User).filter_by(
                id=session.get('user_id')).first()
        else:
            # the user is not already logged in, let's see if they have
            # already created an account before
            user = Session.query(User).filter_by(flickr_nsid=nsid).first()

        if user:
            user.flickr_nsid = nsid
            user.flickr_token = token
            Session.commit()
        else:
            # the user does not have an account.  We need to create a new one
            # for them.
            user = User(flickr_nsid=nsid, flickr_token=token)
            Session.add(user)
            Session.commit()
            user = Session.query(User).filter_by(flickr_nsid=nsid).first()

        session['user_id'] = user.id
        session['flickr_token'] = token
        session.save()
        log.info("Logged in user %s", user)
        redirect(url('index'))
Beispiel #26
0
    def index(self):
        flickr = FlickrAPI()
        token = flickr.get_token(request.GET.getone('frob'))

        session['flickr_token'] = token
        session.save()

        flickr = FlickrAPI(token)
        result = flickr.auth_checkToken()
        nsid = result[0][2].get('nsid')

        if session.get('user_id'):
            # a user is already logged in
            user = Session.query(User).filter_by(id=session.get('user_id')).first()
        else:
            # the user is not already logged in, let's see if they have
            # already created an account before
            user = Session.query(User).filter_by(flickr_nsid=nsid).first()

        if user:
            user.flickr_nsid = nsid
            user.flickr_token = token
            Session.commit()
        else:
            # the user does not have an account.  We need to create a new one
            # for them.
            user = User(flickr_nsid=nsid, flickr_token=token)
            Session.add(user)
            Session.commit()
            user = Session.query(User).filter_by(flickr_nsid=nsid).first()

        session['user_id'] = user.id
        session['flickr_token'] = token
        session.save()
        log.info("Logged in user %s", user)
        redirect(url('index'))
Beispiel #27
0
class User(Base):
    __tablename__ = 'users'
    id = Column('id', Integer, primary_key=True)
    fb_uid = Column('fb_uid', Integer, index=True)
    flickr_nsid = Column('flickr_nsid', String(100), index=True)
    flickr_token = Column('flickr_token', UnicodeText)
    fb_access_token = Column('fb_access_token', UnicodeText)

    def __init__(self, fb_uid=None, fb_access_token=None, flickr_nsid=None, flickr_token=None):
        self.fb_uid = fb_uid
        self.flickr_nsid = flickr_nsid
        self.fb_access_token = fb_access_token
        self.flickr_token = flickr_token

    def __unicode__(self):
        return u'%s' % self.id

    def __repr__(self):
        return "<User id=%s fb_uid=%s flickr_nsid=%s>" % (
            self.id, self.fb_uid, self.flickr_nsid)

    def verify_user(self):
        """Check the access tokens"""
        if self.fb_access_token:
            fbuser = fb.GraphUser(access_token=self.fb_access_token)
            try:
                fbuser.load()
            except fb.OAuthException, e:
                self.fb_access_token = None
                Session.commit()
                return False

        if self.flickr_token:
            fluser = flickr.FlickrUser(token=self.flickr_token)
            try:
                fluser.auth_checkToken()
            except flickr.FlickrError, e:
                self.flickr_token = None
                Session.commit()
                return False
Beispiel #28
0
    def sync_photoset(self, photoset):
        log.debug("Syncing flickr photoset %s to facebook", photoset.get('id'))
        sync = Session.query(SyncRecord).filter_by(
            flickrid=photoset.get('id'), type=SyncRecord.TYPE_ALBUM).first()
        album = None
        if sync and sync.success:
            # don't resync unless we failed before
            log.debug("skipping... already synced")
            album = fb.GraphAlbum(id=sync.fbid, access_token=self.user.fb_access_token)
            if not album.data:
                album = None
        if not album:
            sync = SyncRecord(SyncRecord.TYPE_ALBUM, self.task.user_id)
            sync.flickrid = photoset.get('id')

            title = photoset.find('title').text
            description = photoset.find('description').text
            album = self.fb_user.albums.add(
                title.encode('utf-8') if title else '',
                description.encode('utf-8') if description else '',
                privacy=self.fb_privacy)

            if album:
                sync.fbid = album.id
                sync.status = SyncRecord.STATUS_SUCCESS
            else:
                sync.status = SyncRecord.STATUS_FAILED
            Session.add(sync)
            Session.commit()
            self.set_status(self.synced_photos, self.total_photos,
                            "Failed to sync %s" % title)
            return

        photos = self.fk.photosets_getPhotos(photoset_id=photoset.get('id'))[0]

        # max out at 200 photos until we figure out what to do with bigger sets
        photos = photos[:200]

        photos_to_sync = []
        for photo in photos:
            log.debug("Syncing flickr photo %s to facebook", photo.get('id'))
            sync = SyncRecord.get_for_flickrid(photo.get('id')).first()
            fb_photo = None
            if sync and sync.fbid and sync.success:
                log.debug("Skipping... already synced")
                fb_photo = fb.GraphPhoto(id=sync.fbid, access_token=self.user.fb_access_token)
                #if not fb_photo.data:
                #    fb_photo = None
                self.synced_photos += 1
            if not fb_photo:
                photos_to_sync.append(photo)

        status = "%s photos from %s already synced" % (self.synced_photos,
                                                       photoset.find('title').text)
        self.set_status(self.synced_photos, self.total_photos, status)

        if not photos_to_sync:
            return

        def on_progress(processed, total):
            self.set_status(
                processed,
                total,
                "Found %s/%s photos in %s..." % (
                    processed, total, photoset.find('title').text))

        fetcher = http.Fetcher(progress_callback=on_progress)
        requests = []
        for photo in photos_to_sync:
            url = flickr.get_url(
                self.user.flickr_token,
                method='flickr.photos.getSizes',
                photo_id=photo.get('id'))
            request = http.JsonRequest(url)
            requests.append((photo, request))
            fetcher.queue(request)
        fetcher.run()

        def on_progress(processed, total):
            self.set_status(
                processed,
                total,
                "Downloaded %s/%s images from %s" % (
                    processed, total, photoset.find('title').text))

        fetcher = http.Fetcher(progress_callback=on_progress)
        img_requests = []

        # TODO: make tmp directory path configurable
        if not os.path.exists('/tmp/photosync'):
            os.mkdir('/tmp/photosync')
            os.chmod('/tmp/photosync', 0777)

        for i, (photo, request) in enumerate(requests):
            sync = SyncRecord(SyncRecord.TYPE_PHOTO, self.task.user_id)
            sync.flickrid = photo.get("id")
            Session.add(sync)
            res = request.read_response()
            try:
                img_url = res['sizes']['size'][-1]['source']
            except:
                #import pdb; pdb.set_trace()
                raise
            filename = '/tmp/photosync/flickr-'+sync.flickrid+'.jpg'
            log.debug("Downloading image %s to %s", img_url, filename)
            img_request = None
            if not os.path.exists(filename):
                f = open(filename, 'wb')
                f.write(urllib2.urlopen(img_url).read())
                f.close()
                on_progress(i, len(requests))
                # TODO: Figure out why curl isn't working here
                # for some reason when we use the code below,
                # the complete file does not get downloaded.
                #img_request = http.Request(img_url, filename=filename)
                #fetcher.queue(img_request)
            img_requests.append((photo, filename, sync, img_request))
        Session.commit()
        fetcher.run()

        for photo, temp_filename, sync, img_request in img_requests:
            sync.transfer_in = os.path.getsize(temp_filename)
            log.debug("Uploading image %s to facebook", temp_filename)
            graph_photo = None
            title = photo.get('title')
            try:
                title = title.encode('utf-8')
            except UnicodeEncodeError, e:
                log.error("Failed to encode %s to utf-8", title)
                log.exception(e)
                # better a photo with no title than no photo at all.
                title = ""
            try:
                graph_photo = album.photos.add(temp_filename, title)
            except TypeError, e:
                log.error("Error uploading image %s", temp_filename)
                log.exception(e)
Beispiel #29
0
 def run_now(self):
     self.queue_id = None
     Session.commit()
     self.queue_id = self._job.resubmit()
     self.__job = "SENTINAL"
     Session.commit()
Beispiel #30
0
 def set_status(self, completed, total, data):
     log.debug("%s %s Completed %s/%s: %r", self.__class__.__name__, self.job.jid, completed, total, data)
     self.job.touch()
     self.task.set_status(completed, total, data)
     Session.commit()
Beispiel #31
0
 def task(self):
     if not self.__task_fetched:
         self.__task_fetched = True
         self.__task = Session.query(AsyncTask).filter_by(id=self.__task_id).first()
     return self.__task
Beispiel #32
0
    def sync_photoset(self, photoset):
        log.debug("Syncing flickr photoset %s to facebook", photoset.get("id"))
        sync = Session.query(SyncRecord).filter_by(flickrid=photoset.get("id"), type=SyncRecord.TYPE_ALBUM).first()
        album = None
        if sync and sync.success:
            # don't resync unless we failed before
            log.debug("skipping... already synced")
            album = fb.GraphAlbum(id=sync.fbid, access_token=self.user.fb_access_token)
            if not album.data:
                album = None
        if not album:
            sync = SyncRecord(SyncRecord.TYPE_ALBUM, self.task.user_id)
            sync.flickrid = photoset.get("id")

            title = photoset.find("title").text
            description = photoset.find("description").text
            album = self.fb_user.albums.add(
                title.encode("utf-8") if title else "",
                description.encode("utf-8") if description else "",
                privacy=self.fb_privacy,
            )

            if album:
                sync.fbid = album.id
                sync.status = SyncRecord.STATUS_SUCCESS
            else:
                sync.status = SyncRecord.STATUS_FAILED
            Session.add(sync)
            Session.commit()
            self.set_status(self.synced_photos, self.total_photos, "Failed to sync %s" % title)
            return

        photos = self.fk.photosets_getPhotos(photoset_id=photoset.get("id"))[0]

        # max out at 200 photos until we figure out what to do with bigger sets
        photos = photos[:200]

        photos_to_sync = []
        for photo in photos:
            log.debug("Syncing flickr photo %s to facebook", photo.get("id"))
            sync = SyncRecord.get_for_flickrid(photo.get("id")).first()
            fb_photo = None
            if sync and sync.fbid and sync.success:
                log.debug("Skipping... already synced")
                fb_photo = fb.GraphPhoto(id=sync.fbid, access_token=self.user.fb_access_token)
                # if not fb_photo.data:
                #    fb_photo = None
                self.synced_photos += 1
            if not fb_photo:
                photos_to_sync.append(photo)

        status = "%s photos from %s already synced" % (self.synced_photos, photoset.find("title").text)
        self.set_status(self.synced_photos, self.total_photos, status)

        if not photos_to_sync:
            return

        def on_progress(processed, total):
            self.set_status(
                processed, total, "Found %s/%s photos in %s..." % (processed, total, photoset.find("title").text)
            )

        fetcher = http.Fetcher(progress_callback=on_progress)
        requests = []
        for photo in photos_to_sync:
            url = flickr.get_url(self.user.flickr_token, method="flickr.photos.getSizes", photo_id=photo.get("id"))
            request = http.JsonRequest(url)
            requests.append((photo, request))
            fetcher.queue(request)
        fetcher.run()

        def on_progress(processed, total):
            self.set_status(
                processed, total, "Downloaded %s/%s images from %s" % (processed, total, photoset.find("title").text)
            )

        fetcher = http.Fetcher(progress_callback=on_progress)
        img_requests = []

        # TODO: make tmp directory path configurable
        if not os.path.exists("/tmp/photosync"):
            os.mkdir("/tmp/photosync")
            os.chmod("/tmp/photosync", 0777)

        for i, (photo, request) in enumerate(requests):
            sync = SyncRecord(SyncRecord.TYPE_PHOTO, self.task.user_id)
            sync.flickrid = photo.get("id")
            Session.add(sync)
            res = request.read_response()
            try:
                img_url = res["sizes"]["size"][-1]["source"]
            except:
                # import pdb; pdb.set_trace()
                raise
            filename = "/tmp/photosync/flickr-" + sync.flickrid + ".jpg"
            log.debug("Downloading image %s to %s", img_url, filename)
            img_request = None
            if not os.path.exists(filename):
                f = open(filename, "wb")
                f.write(urllib2.urlopen(img_url).read())
                f.close()
                on_progress(i, len(requests))
                # TODO: Figure out why curl isn't working here
                # for some reason when we use the code below,
                # the complete file does not get downloaded.
                # img_request = http.Request(img_url, filename=filename)
                # fetcher.queue(img_request)
            img_requests.append((photo, filename, sync, img_request))
        Session.commit()
        fetcher.run()

        for photo, temp_filename, sync, img_request in img_requests:
            sync.transfer_in = os.path.getsize(temp_filename)
            log.debug("Uploading image %s to facebook", temp_filename)
            graph_photo = None
            title = photo.get("title")
            try:
                title = title.encode("utf-8")
            except UnicodeEncodeError, e:
                log.error("Failed to encode %s to utf-8", title)
                log.exception(e)
                # better a photo with no title than no photo at all.
                title = ""
            try:
                graph_photo = album.photos.add(temp_filename, title)
            except TypeError, e:
                log.error("Error uploading image %s", temp_filename)
                log.exception(e)
Beispiel #33
0
def init_model(engine):
    """Call me before using any of the tables or classes in the model"""
    Session.configure(bind=engine)
Beispiel #34
0
 def get_bytes_transfered_by_user(user_id):
     bytes_in, bytes_out = Session.execute(
         "SELECT SUM(transfer_in), SUM(transfer_out) "
         "FROM sync_records WHERE user_id=:id",
         {'id':user_id}).fetchone()
     return int(bytes_in or 0), int(bytes_out or 0)
Beispiel #35
0
 def get_by_id(id):
     return Session.query(User).filter_by(id=id).first()
Beispiel #36
0
 def set(setting_id, value, user_id=None):
     user_id = user_id or session.get('user_id')
     result = Session.merge(UserSetting(user_id, setting_id, value))
     Session.commit()
     return result
Beispiel #37
0
class FullSync(TaskHandler):

    @classmethod
    def run_for_user_now(cls, user_id):
        last = AsyncTask.get_for_user(user_id=user_id,
                                      type=cls.get_type(),
                                      limit=1).first()
        if not last or last.is_completed:
            cls.submit_advanced((user_id,), {}, user_id=user_id)
        else:
            last.run_now()

    @classmethod
    def get_initial_status(cls):
        return (None, None, "Waiting for syncer to become available...")

    def run(self, user_id):
        self.user = User.get_by_id(user_id)
        if not self.user.verify_user():
            # not really much we can do about this...
            log.info("Missing tokens for user %r", self.user)
            self.set_status(0, 0, "You need to log out and log back in again.")
            return
        self.fk = flickr.FlickrAPI(self.user.flickr_token)
        self.fb_user = fb.GraphUser(access_token=self.user.fb_access_token)
        photosets = self.fk.photosets_getList()[0]

        self.synced_photos = 0
        self.total_photos = sum(int(pset.get('photos')) for pset in photosets)

        self.set_status(
            self.synced_photos,
            self.total_photos,
            "syncing Flickr photos to Facebook")

        settings = UserSetting.multiget(
            user_id=user_id,
            settings=[UserSettingConst.FB_PRIVACY,
                      UserSettingConst.FLICKR_SET_SYNCING])
        self.fb_privacy = settings.get(UserSettingConst.FB_PRIVACY, 'FB_DEFAULT')
        self.flickr_settings = settings.get(UserSettingConst.FLICKR_SET_SYNCING, {})

        select_sets = self.flickr_settings.get('select_sets', False)
        selected_sets = self.flickr_settings.get('selected_sets', [])
        for photoset in photosets:
            if not select_sets or photoset.get('id') in selected_sets:
                self.sync_photoset(photoset)

        log.info("Finished job %r for user %r", self.job.jid, self.user.id)
        # once we are all done, let's submit the task to rerun in an hour.
        self.resubmit(delay=60*60)

    def sync_photoset(self, photoset):
        log.debug("Syncing flickr photoset %s to facebook", photoset.get('id'))
        sync = Session.query(SyncRecord).filter_by(
            flickrid=photoset.get('id'), type=SyncRecord.TYPE_ALBUM).first()
        album = None
        if sync and sync.success:
            # don't resync unless we failed before
            log.debug("skipping... already synced")
            album = fb.GraphAlbum(id=sync.fbid, access_token=self.user.fb_access_token)
            if not album.data:
                album = None
        if not album:
            sync = SyncRecord(SyncRecord.TYPE_ALBUM, self.task.user_id)
            sync.flickrid = photoset.get('id')

            title = photoset.find('title').text
            description = photoset.find('description').text
            album = self.fb_user.albums.add(
                title.encode('utf-8') if title else '',
                description.encode('utf-8') if description else '',
                privacy=self.fb_privacy)

            if album:
                sync.fbid = album.id
                sync.status = SyncRecord.STATUS_SUCCESS
            else:
                sync.status = SyncRecord.STATUS_FAILED
            Session.add(sync)
            Session.commit()
            self.set_status(self.synced_photos, self.total_photos,
                            "Failed to sync %s" % title)
            return

        photos = self.fk.photosets_getPhotos(photoset_id=photoset.get('id'))[0]

        # max out at 200 photos until we figure out what to do with bigger sets
        photos = photos[:200]

        photos_to_sync = []
        for photo in photos:
            log.debug("Syncing flickr photo %s to facebook", photo.get('id'))
            sync = SyncRecord.get_for_flickrid(photo.get('id')).first()
            fb_photo = None
            if sync and sync.fbid and sync.success:
                log.debug("Skipping... already synced")
                fb_photo = fb.GraphPhoto(id=sync.fbid, access_token=self.user.fb_access_token)
                #if not fb_photo.data:
                #    fb_photo = None
                self.synced_photos += 1
            if not fb_photo:
                photos_to_sync.append(photo)

        status = "%s photos from %s already synced" % (self.synced_photos,
                                                       photoset.find('title').text)
        self.set_status(self.synced_photos, self.total_photos, status)

        if not photos_to_sync:
            return

        def on_progress(processed, total):
            self.set_status(
                processed,
                total,
                "Found %s/%s photos in %s..." % (
                    processed, total, photoset.find('title').text))

        fetcher = http.Fetcher(progress_callback=on_progress)
        requests = []
        for photo in photos_to_sync:
            url = flickr.get_url(
                self.user.flickr_token,
                method='flickr.photos.getSizes',
                photo_id=photo.get('id'))
            request = http.JsonRequest(url)
            requests.append((photo, request))
            fetcher.queue(request)
        fetcher.run()

        def on_progress(processed, total):
            self.set_status(
                processed,
                total,
                "Downloaded %s/%s images from %s" % (
                    processed, total, photoset.find('title').text))

        fetcher = http.Fetcher(progress_callback=on_progress)
        img_requests = []

        # TODO: make tmp directory path configurable
        if not os.path.exists('/tmp/photosync'):
            os.mkdir('/tmp/photosync')
            os.chmod('/tmp/photosync', 0777)

        for i, (photo, request) in enumerate(requests):
            sync = SyncRecord(SyncRecord.TYPE_PHOTO, self.task.user_id)
            sync.flickrid = photo.get("id")
            Session.add(sync)
            res = request.read_response()
            try:
                img_url = res['sizes']['size'][-1]['source']
            except:
                #import pdb; pdb.set_trace()
                raise
            filename = '/tmp/photosync/flickr-'+sync.flickrid+'.jpg'
            log.debug("Downloading image %s to %s", img_url, filename)
            img_request = None
            if not os.path.exists(filename):
                f = open(filename, 'wb')
                f.write(urllib2.urlopen(img_url).read())
                f.close()
                on_progress(i, len(requests))
                # TODO: Figure out why curl isn't working here
                # for some reason when we use the code below,
                # the complete file does not get downloaded.
                #img_request = http.Request(img_url, filename=filename)
                #fetcher.queue(img_request)
            img_requests.append((photo, filename, sync, img_request))
        Session.commit()
        fetcher.run()

        for photo, temp_filename, sync, img_request in img_requests:
            sync.transfer_in = os.path.getsize(temp_filename)
            log.debug("Uploading image %s to facebook", temp_filename)
            graph_photo = None
            title = photo.get('title')
            try:
                title = title.encode('utf-8')
            except UnicodeEncodeError, e:
                log.error("Failed to encode %s to utf-8", title)
                log.exception(e)
                # better a photo with no title than no photo at all.
                title = ""
            try:
                graph_photo = album.photos.add(temp_filename, title)
            except TypeError, e:
                log.error("Error uploading image %s", temp_filename)
                log.exception(e)
            os.remove(temp_filename)
            if graph_photo:
                sync.fbid = graph_photo.id
                sync.status = SyncRecord.STATUS_SUCCESS
                sync.transfer_out = sync.transfer_in
            else:
                sync.status = SyncRecord.STATUS_FAILED
            Session.commit()
            self.synced_photos += 1
            status = "Synced %s from %s to Facebook" % (photo.get('title'),
                                                        photoset.find('title').text)
            self.set_status(self.synced_photos, self.total_photos, status)
Beispiel #38
0
 def task(self):
     if not self.__task_fetched:
         self.__task_fetched = True
         self.__task = Session.query(AsyncTask).filter_by(
             id=self.__task_id).first()
     return self.__task