## constructor method

def __init__(self, proxyList):

## ProxyList object to retrieve new proxy addresses from

self.proxyList = ProxyList(proxyList, 100)

## number of attempted content lookups

self.lookupAttempts = 0

self.maxpage = 9999

## dictionary of month name to month number conversions

self.months = {}

self.months['Jan'] = 1

self.months['Feb'] = 2

self.months['Mar'] = 3

self.months['Apr'] = 4

self.months['May'] = 5

self.months['Jun'] = 6

self.months['Jul'] = 7

self.months['Aug'] = 8

self.months['Sep'] = 9

self.months['Oct'] = 10

self.months['Nov'] = 11

self.months['Dec'] = 12

## Splits into multiple threads, each getting the

## screename associated with an id in the given list.

## Returns the names found, as well as ids that could

## not be looked up

def getNameMulti(self, pageids):

foundNames = {}

badIds = []

nids = len(pageids)

pool = eventlet.GreenPool()

ngotten = 0

returned = False

for nametag in pool.imap(self.getNameMultiHelper, pageids):

if returned:

eventlet.kill()

ngotten += 1

name, tag = nametag

if tag:

id, username, popularity, importance, birthtime = name

foundNames[id] = (username, popularity, importance, birthtime)

else:

badIds.append(name)

if ngotten > nids * .8:

returned = True

return foundNames, badIds

return foundNames, badIds

## Tries to look up a users screename given their id,

## returning the name and True if successful,

## and the original id and False if not

def getNameMultiHelper(self, pageid):

## self.getName(pageid)

try:

iddata = self.getName(pageid)

return (iddata, True)

except:

return (pageid, False)

## Given a twitter page id as a string, retrieves and parses the

## user lookup .json file, returning as a string the user screen

## name corresponding to the id

def getName(self, pageid):

lookup = 'http://api.twitter.com/1/users/lookup.json?user_id=' \

+ pageid

content = self.getContent(lookup)

try:

name = ((content.split('\"screen_name\":\"'))[1]\

.split('\"'))[0]

nfollowing = ((content.split('\"friends_count\":'))[1]\

.split(','))[0]

nfollowing = int(nfollowing)

nfollowers = ((content.split('\"followers_count\":'))[1]\

.split(','))[0]

nfollowers = int(nfollowers)

birthtime = (((content.split('\"created_at\":\"'))[1])\

.split('\",\"'))[0]

birthtime = birthtime.split()

birthtime = birthtime[5] + ':' + str(self.months[birthtime[1]]) + ':' + birthtime[2]

## Can't find the user data in the file

except:

raise APIParseException("Name Parse Error!")

return pageid, name, nfollowers, log(float(nfollowing) / float(nfollowers), 10.), birthtime

## Splits into multiple threads to get the followers associated

## with each id in the given list.

## Found followers are returned as a list of lists, and ids

## that could not be looked up are returned in a list

def getFollowersMulti(self, pageids, maxpage=9999):

self.maxpage = maxpage

foundFollowers = []

badIds = []

pool = eventlet.GreenPool()

hasReturned = False

ngotten = 0

nids = len(pageids)

for idtag in pool.imap(self.getFollowersMultiHelper, pageids):

if hasReturned:

eventlet.kill()

ids, tag = idtag

if tag:

foundFollowers.append(ids)

ngotten += 1

else:

badIds.append(ids)

if ngotten > nids * .5:

hasReturned = True

return foundFollowers, badIds

return foundFollowers, badIds

## Tries to look up the followers associated with a given page,

## returning the list of followers and True if successful,

## and the original id and False if not

def getFollowersMultiHelper(self, pageid):

try:

ids = self.getFollowers(pageid)

return (ids, True)

except:

return (pageid, False)

## Given a twitter page id as a string, retrieves and parses the

## follwer .json file, returning as a list of strings the ids of

## the page

def getFollowers(self, pageid, verbose=False):

## looks up the first page

try:

nextcur, newids = self.getCursorFollowers(pageid, '-1')

except APICursorException:

return self.getFollowers(pageid)

page = 0

ids = newids

if self.maxpage == 0:

return ids

if verbose:

print len(ids), 'followers'

## looks up subsequent pages if necessary

while not nextcur == '0':

page += 1

## print 'page ', page, nextcur

gotPage = False

while not gotPage:

try:

oldcur = nextcur

oldids = newids

nextcur, newids = self.getCursorFollowers(pageid, nextcur)

gotPage = True

## didn't work, try again

except APICursorException:

nextcur = oldcur

newids = oldids

gotPage = False

ids.extend(newids)

if verbose:

print len(ids), 'followers'

return ids

## Helper function for getFollowers

## Retrieves a single page of a users followers

def getCursorFollowers(self, pageid, cursor):

lookup = 'http://api.twitter.com/1/followers/ids.json?cursor=' + \

cursor + '&user_id=' + pageid

content = self.getContent(lookup)

try:

ids = (((content.split('\"ids\":['))[1]).split(']'))[0]

ids = ids.split(',')

## Can't find followers to parse

except:

raise APICursorException("No followers!")

try:

nextcur = (((content.split('\"next_cursor_str\":\"'))[1])\

.split('\"'))[0]

## Can't find the cursor for the next page

except:

raise APICursorException("cursor error")

return nextcur, ids

## Splits into multiple threads to look up the tweets

## from each user in the given list.

## Returns found tweets as a list of lists, as well as

## a list of ids that could not be looked up

def getTweetsMulti(self, userids):

foundTweets = {}

badIds = []

nids = len(userids)

pool = eventlet.GreenPool()

ngotten = 0

notFound = copy(userids)

returned = False

for tweettag in pool.imap(self.getTweetsMultiHelper, userids):

if returned:

eventlet.kill()

ngotten += 1

tweets, tag, id = tweettag

if tag:

foundTweets[id] = tweets

else:

badIds.append(id)

notFound.remove(id)

if ngotten > nids * .8:

returned = True

return foundTweets, badIds, notFound

return foundTweets, badIds, notFound

## Tries to get the tweets for a given user,

## returning the tweets and True if successful,

## and the original screename and False if not,

## as well as the name

def getTweetsMultiHelper(self, id):

try:

tweets = self.getTweets(id)

return (tweets, True, id)

except APIParseException:

return (id, False, id)

## Given a twitter username as a string, retrieves and parses a

## .json file of the user's past week of tweets, returning the

## tweets as a list of tuples, with the julian time of the

## tweet as the first element and the tweet string as the second

def getTweets(self, userid):

tweets = []

lookup = 'http://api.twitter.com/1/statuses/user_timeline.json?count=200&user_id=' + userid

## lookup = 'http://search.twitter.com/search.json?q=from:' + username

content = self.getContent(lookup)

try:

tweetArr = content.split('},{')

tweetArr[-1] = (tweetArr[-1].split('}]'))[0]

for entry in tweetArr:

tweet = ((entry.split('\"text\":\"')[1]).split('\",\"'))[0]

tweettime = (((entry.split('\"created_at\":\"'))[1])\

.split('\",\"'))[0]

tweettime = tweettime.split()[1:]

tweet_t = tweettime[2].split(':')

tweettime = datetime(int(tweettime[4]), \

self.months[tweettime[0]], \

int(tweettime[1]), \

int(tweet_t[0]), \

int(tweet_t[1]), \

int(tweet_t[2]))

tweets.append((tweettime,tweet))

except:

raise APIParseException("No Tweets!")

return tweets

## Retrieves a given url using the TwitterCaller's proxy,

## returning the read page as a string

def getContent(self, lookup):

self.lookupAttempts += 1

try:

proxy = self.proxyList.getProxy()

except:

sleep(1)

return self.getContent(lookup)

## connect = HTTPConnection(proxy, timeout=20)

## connect.request("GET", lookup)

## content = connect.getresponse().read()

try:

connect = HTTPConnection(proxy, timeout=6)

connect.request("GET", lookup)

content = connect.getresponse().read()

## lookup failed, lets get a new proxy

except socket.timeout:

if self.lookupAttempts > 20:

return ''

## print 'timeout'

self.proxyList.reportBadProxy(proxy)

content = self.getContent(lookup)

except Exception,e:

## print e

self.proxyList.reportBadProxy(proxy)

return ''

self.lookupAttempts = 0