Beispiel #1
0
def getTweets():
    timeout = None
    api = tweepy.API(auth)
    # TODO: query the db for user & tweet max values and make sure we don't
    # query for anything we don't need to.
    cursor = tweepy.Cursor(api.list_members, 'twitter', 'team')
    timeout = datetime.datetime.now() + datetime.timedelta(minutes = 15)
    for page in cursor.pages():
        print cursor.iterator.count
        pageDone = False
        while not pageDone:
            try:
                for item in page:
                    try:
                        print datetime.datetime.now(),item.screen_name,item.id
                        dbop.addUser(item)
                        #timeline = api.user_timeline(count=100, user_id = item.id, trim_user=True, include_rts=True)
                        #for tweet in timeline:
                        #    dbop.addTweet(tweet)
                    # Some twitter profiles are private, skip these
                    except tweepy.error.TweepError, e:
                        if e.reason == "Not authorized":
                            continue
                        else:
                            raise e
                    dbop.commit()
                pageDone = True
            # Twitter has rate limits. Wait them out here.
            except tweepy.error.TweepError, e:
                print e.reason
                while datetime.datetime.now() < timeout:
                    time.sleep(1)
Beispiel #2
0
def getTweets():
    timeout = None
    api = tweepy.API(auth)
    cursor = tweepy.Cursor(api.list_members, list_owner, list_slug)
    timeout = datetime.datetime.now() + datetime.timedelta(minutes = 15)
    for page in cursor.pages():
        print 'page',cursor.iterator.count
        pageDone = False
        while not pageDone:
            try:
                for item in page:
                    try:
                        print item.screen_name
                        dbop.addUser(item)
                        timeline = api.user_timeline(count=tweets_per_user, user_id = item.id, trim_user=True, include_rts=include_rts)
                        for tweet in timeline:
                            dbop.addTweet(tweet)
                    # Some twitter profiles are private, skip these
                    except tweepy.error.TweepError, e:
                        if e.reason == "Not authorized":
                            continue
                        else:
                            raise e
                    dbop.commit()
                pageDone = True
            # Twitter has rate limits. Wait them out here, reset the timer when done
            except tweepy.error.TweepError, e:
                print e.reason
                while datetime.datetime.now() < timeout:
                    time.sleep(1)
                timeout = datetime.datetime.now() + datetime.timedelta(minutes = 15)