import MySQLdb
import keys
# SQL db config
host = keys.getSetting('SQL','host')
user = keys.getSetting('SQL','user')
passwd = keys.getSetting('SQL','passwd')
dbid = keys.getSetting('SQL','dbid')
# Open the db connection, open a new cursor object. Commits must be made manually
# by calling commit() below.
db = MySQLdb.connect(host=host,user=user,passwd=passwd,db=dbid)
cursor = db.cursor()
# Adds a twitter user to the sql db. Requires id and screen_name attributes
def addUser(user):
sql = "INSERT IGNORE INTO users (id,screen_name) VALUES (%s,%s)"
cursor.execute(sql,(user.id,user.screen_name))
# Get all id and screen_names from the database. Used for updating klout score
def getUsers():
cursor.execute("SELECT id,screen_name FROM users")
return cursor.fetchall()
# Add tweet content to the db. Since tweepy gives us back the creator's info, we
# can add the user.id to this table without requiring a user object. Also included are
# Tweet id, tweet text, and retweet_count from the tweepy object.
def addTweet(content):
sql = "INSERT IGNORE INTO tweets (id,user_id,text,retweets) VALUES (%s,%s,%s,%s)"
cursor.execute(sql,(content.id,content.user.id,content.text.encode('ascii', 'ignore'),content.retweet_count))
import dbop
import keys
import tweepy
import datetime
import time
# Set authentication info for twitter OAuth
auth = tweepy.OAuthHandler(keys.getSetting('Twitter','consumer_key'), keys.getSetting('Twitter','consumer_secret'))
auth.set_access_token(keys.getSetting('Twitter','access_key'), keys.getSetting('Twitter','access_secret'))
# Grab all members from the twitter team list, and the last 90 tweets from each.
# Add the users and tweets to the db as we get them.
# (80 tweets * 1250 users = 100k tweets. 90 tweets each accounts for users with
# lower usage
# TODO: I'd like this to be a little more flexible, allow external config for the list
# we're accessing, how many tweets we grab from each user and separate user/tweet
# processes, if practical
def getTweets():
timeout = None
api = tweepy.API(auth)
# TODO: query the db for user & tweet max values and make sure we don't
# query for anything we don't need to.
cursor = tweepy.Cursor(api.list_members, 'twitter', 'team')
timeout = datetime.datetime.now() + datetime.timedelta(minutes = 15)
for page in cursor.pages():
print cursor.iterator.count
pageDone = False
while not pageDone:
try:
for item in page:
import keys
from klout import *
import dbop
# Set the Klout auth key
klout = Klout(keys.getSetting('Klout', 'key'),secure=True)
# Grab scores for each klout user currently in the sql db. Sets -1 for
# scores that are unavailable
def updateScores():
users = dbop.getUsers()
for user in users:
try:
id = klout.identity.klout(screenName=user[1]).get('id')
score = klout.user.score(kloutId=id).get('score')
dbop.updateKloutScore(user[0], score)
print user[1]
# TODO: expand the error handling a bit to register a difference between
# unavailable klout score and other problems such as auth failure or
# service interruption
except:
dbop.updateKloutScore(user[0], -1)
