import MySQLdb import keys # SQL db config host = keys.getSetting('SQL','host') user = keys.getSetting('SQL','user') passwd = keys.getSetting('SQL','passwd') dbid = keys.getSetting('SQL','dbid') # Open the db connection, open a new cursor object. Commits must be made manually # by calling commit() below. db = MySQLdb.connect(host=host,user=user,passwd=passwd,db=dbid) cursor = db.cursor() # Adds a twitter user to the sql db. Requires id and screen_name attributes def addUser(user): sql = "INSERT IGNORE INTO users (id,screen_name) VALUES (%s,%s)" cursor.execute(sql,(user.id,user.screen_name)) # Get all id and screen_names from the database. Used for updating klout score def getUsers(): cursor.execute("SELECT id,screen_name FROM users") return cursor.fetchall() # Add tweet content to the db. Since tweepy gives us back the creator's info, we # can add the user.id to this table without requiring a user object. Also included are # Tweet id, tweet text, and retweet_count from the tweepy object. def addTweet(content): sql = "INSERT IGNORE INTO tweets (id,user_id,text,retweets) VALUES (%s,%s,%s,%s)" cursor.execute(sql,(content.id,content.user.id,content.text.encode('ascii', 'ignore'),content.retweet_count))
import dbop import keys import tweepy import datetime import time # Set authentication info for twitter OAuth auth = tweepy.OAuthHandler(keys.getSetting('Twitter','consumer_key'), keys.getSetting('Twitter','consumer_secret')) auth.set_access_token(keys.getSetting('Twitter','access_key'), keys.getSetting('Twitter','access_secret')) # Grab all members from the twitter team list, and the last 90 tweets from each. # Add the users and tweets to the db as we get them. # (80 tweets * 1250 users = 100k tweets. 90 tweets each accounts for users with # lower usage # TODO: I'd like this to be a little more flexible, allow external config for the list # we're accessing, how many tweets we grab from each user and separate user/tweet # processes, if practical def getTweets(): timeout = None api = tweepy.API(auth) # TODO: query the db for user & tweet max values and make sure we don't # query for anything we don't need to. cursor = tweepy.Cursor(api.list_members, 'twitter', 'team') timeout = datetime.datetime.now() + datetime.timedelta(minutes = 15) for page in cursor.pages(): print cursor.iterator.count pageDone = False while not pageDone: try: for item in page:
import keys from klout import * import dbop # Set the Klout auth key klout = Klout(keys.getSetting('Klout', 'key'),secure=True) # Grab scores for each klout user currently in the sql db. Sets -1 for # scores that are unavailable def updateScores(): users = dbop.getUsers() for user in users: try: id = klout.identity.klout(screenName=user[1]).get('id') score = klout.user.score(kloutId=id).get('score') dbop.updateKloutScore(user[0], score) print user[1] # TODO: expand the error handling a bit to register a difference between # unavailable klout score and other problems such as auth failure or # service interruption except: dbop.updateKloutScore(user[0], -1)