class TwitterStats():
def __init__(self):
# connect to twitter api
self.twitter = Twython(app_key=settings.consumer_key,
app_secret=settings.consumer_secret,
oauth_token=settings.oauth_token,
oauth_token_secret=settings.oauth_token_secret)
def init_storage(self):
storage = shelve.open('twitter_stats', writeback=True)
if not storage:
storage['followers'] = set()
storage['unfollowers'] = []
storage['unfollowers_since_last_check'] = None
storage['last_update'] = None
return storage
def get_followers(self):
follower_ids = self.twitter.getFollowersIDs()['ids']
return set(follower_ids)
def show_screen_name(self, user_id):
user = self.twitter.showUser(user_id=user_id)
screen_name = user['screen_name']
return screen_name
def update_unfollower_stats(self):
with closing(self.init_storage()) as storage:
previous_followers = storage['followers']
current_followers = self.get_followers()
new_unfollower_ids = previous_followers - current_followers
unfollowers_since_last_check = []
for follower_id in new_unfollower_ids:
unfollower = {
'id':
follower_id,
'screen_name':
self.show_screen_name(follower_id),
'timestamp':
datetime.datetime.now().strftime('%b %d %Y %H:%M:%S')
}
storage['unfollowers'].append(unfollower)
unfollowers_since_last_check.append(unfollower)
storage['followers'] = current_followers
storage[
'unfollowers_since_last_check'] = unfollowers_since_last_check
storage['last_update'] = datetime.datetime.now().strftime(
'%b %d %Y %H:%M:%S')
def index(request):
usuario = "@Jandrey15"#definimos una variable
twitter = Twython()#creamos un objeto la clase Twython
followers = twitter.getFollowersIDs( screen_name = usuario)#traemos los usuarios muy facil
#tweets = twitter.getPublicTimeline()
return HttpResponse(followers)
def index(request):
usuario = "@Jandrey15" #definimos una variable
twitter = Twython() #creamos un objeto la clase Twython
followers = twitter.getFollowersIDs(
screen_name=usuario) #traemos los usuarios muy facil
#tweets = twitter.getPublicTimeline()
return HttpResponse(followers)
class TwitterStats():
def __init__(self):
# connect to twitter api
self.twitter = Twython(
app_key=settings.consumer_key,
app_secret=settings.consumer_secret,
oauth_token=settings.oauth_token,
oauth_token_secret=settings.oauth_token_secret
)
def init_storage(self):
storage = shelve.open('twitter_stats', writeback=True)
if not storage:
storage['followers'] = set()
storage['unfollowers'] = []
storage['unfollowers_since_last_check'] = None
storage['last_update'] = None
return storage
def get_followers(self):
follower_ids = self.twitter.getFollowersIDs()['ids']
return set(follower_ids)
def show_screen_name(self, user_id):
user = self.twitter.showUser(user_id=user_id)
screen_name = user['screen_name']
return screen_name
def update_unfollower_stats(self):
with closing(self.init_storage()) as storage:
previous_followers = storage['followers']
current_followers = self.get_followers()
new_unfollower_ids = previous_followers - current_followers
unfollowers_since_last_check = []
for follower_id in new_unfollower_ids:
unfollower = {
'id': follower_id,
'screen_name': self.show_screen_name(follower_id),
'timestamp': datetime.datetime.now().strftime('%b %d %Y %H:%M:%S')
}
storage['unfollowers'].append(unfollower)
unfollowers_since_last_check.append(unfollower)
storage['followers'] = current_followers
storage['unfollowers_since_last_check'] = unfollowers_since_last_check
storage['last_update'] = datetime.datetime.now().strftime('%b %d %Y %H:%M:%S')
def getUserEgoNetwork (self, userName): #Returns the ego centric net for a user twitter = Twython() followers = twitter.getFollowersIDs(screen_name = userName) friends = twitter.getFriendsIDs(screen_name = userName) symmetricFriendship =set(followers).intersection(set(friends)) print "--------------FOLLOWERS: %d------------------" %(len(followers)) for follower_id in followers: print "User with ID %s, is following %s" % ( follower_id, userName ) print "--------------FRIENDSHIP: %d------------------" %(len(friends)) for friends_id in friends: print "%s is friend with user with ID %s" % ( userName, friends_id ) print "--------------SYMMETRIC: %d------------------" %(len(symmetricFriendship)) return symmetricFriendship
def search_with_tokens(tokens, queryType, params):
#print "params = ", params
#user = UserSocialAuth.objects.filter(user=request.user).get()
# Set up a new Twython object with the OAuth keys
#pprint(tokens)
print "app_key = ", settings.TWITTER_CONSUMER_KEY
print "app_secret = ", settings.TWITTER_CONSUMER_SECRET
print "oauth_token = ", tokens['oauth_token']
print "oauth_token_secret = ", tokens['oauth_token_secret']
t = Twython(app_key=settings.TWITTER_CONSUMER_KEY,
app_secret=settings.TWITTER_CONSUMER_SECRET,
oauth_token=tokens['oauth_token'],
oauth_token_secret=tokens['oauth_token_secret'])
# Obtain Twitter results using user's OAuth key
if queryType == "search":
params = params.copy()
pageID = params["pageID"]
del params["pageID"]
queryResults = t.search(**(params))
# Use helper to update the tweet list of the page
# Return an array of tweets to return to the client
helper = Helper()
# tweets = helper.updateWikiArticleTweet(pageID, queryResults) # Update the correct pageID
helper.updateWikiArticleTweet(pageID, queryResults) # Update the correct pageID
'''
tweetsSerialization = []
for tweet in tweets:
tweetsSerialization.append({
"id" : tweet.id,
"text" : tweet.text,
"source" : tweet.source,
"profileImageUrl" : tweet.profileImageUrl,
"createdAt" : tweet.createdAt
})
'''
# Get the correct logger based on convention:
# {APP}.{FILE}.{FUNCTION} = __name__.'.'.inspect.stack()[0][3]
# Need to evaulate performance of 'inspect' library as print function name
# is not supported by Python
#print __name__+"."+inspect.stack()[0][3]
logger = logging.getLogger(__name__+"."+inspect.stack()[0][3])
# Fake infomation, to be updated later
logger.info(json.dumps({
"pageID": pageID, # Pass the correct pageID here
"queryType": queryType,
"query": queryResults["search_metadata"]["query"],
"tweetsCount": queryResults["search_metadata"]["count"],
"source": 'TwitterAPI',
}))
return HttpResponse(json.dumps(queryResults), content_type="application/json")
## elif queryType == "articleSearch":
## params1 = dict(params)
## # TODO, swap this out later
## params1["q"] = unicode(params["articleId"])
## del params1["articleId"]
## queryResults = t.search(**(params1))
elif queryType == "followersID":
queryResults = t.getFollowersIDs(**(params))
elif queryType == "followersList":
params = params.copy()
screen_name = params["screen_name"]
cursor = params["cursor"]
queryResults = t.getFollowersList(**(params))
# Save results
helper = Helper()
helper.getUserFollower(screen_name, queryResults)
elif queryType == "friendsID":
queryResults = t.getFriendsIDs(**(params))
elif queryType == "friendshipLookup":
queryResults = t.lookupFriendships(**(params))
elif queryType == "stream":
queryResults = t.stream(params, on_results)
return HttpResponse(json.dumps(queryResults), content_type="application/json")
from twython import Twython
usuario = "ajamaica"
twitter = Twython()
followers = twitter.getFollowersIDs( screen_name = usuario )
for follower_id in followers :
print "Usuarios %d sigue a %s" % (follower_id, usuario)
tweets = twitter.getPublicTimeline()
for tweet in tweets :
print tweet['user']['name'].encode('utf-8')
print tweet['text'].encode('utf-8')
results = twitter.getDailyTrends()
for i in range(0,100) :
print i
for time, trend_list in results['trends'].iteritems() :
print time
for trend in trend_list :
print trend['query']
class DBTweetGetter(object):
"""
This class gets the tweets and stores them in the database.
query(query,numtweets=160000, usemaxid=True) is responsible for actually downloading the tweets. The IDs of the downloaded tweets are checked in the initialisation of the class, and query checks against this to know when it has downloaded all of the most recent, unseen tweets.
"""
def __init__(self, filename, tablename):
if filename!=None:
self.con = lite.connect(filename)
self.screened=[]
self.cur=self.con.cursor()
self.ucon = lite.connect("userdb.db")
self.ucur=self.ucon.cursor()
self.tablename=tablename
self.ucur.execute("SELECT ScreenName FROM usermap")
a=self.ucur.fetchall()
for item in a:
self.screened.append(item[0])
self.cur.execute("SELECT Id FROM "+tablename)
data=self.cur.fetchall()
l=[]
for item in data:
l.append(item[0])
self.idlist=l
try:
self.twython= Twython(app_key="674Getn4iR5ZonBvSZIE6w", app_secret="XIIhDdsCmByqLQG89ED9h7MILQbS4hsMV3ob6hlLYA", oauth_token="27203313-Oujk9Qu6sKe7tBBYYrheAb1r484PViz2w8GShZCeg", oauth_token_secret="cm95hSRUmkloWBVbYrTHuIYA7fSQqFJH5iYGIbw5ZKo")
except Exception as detail:
print "Error loading Twython object. Error message: " + str(detail)
raise Exception(detail)
def query(self,query,numtweets=160000, usemaxid=True):
if usemaxid==False:
self.maxid=None
self.query=query
ntweets=0
nmatch=0
while ntweets<numtweets:
try:
if self.maxid!=None:
mydict=self.twython.search(q=query, result_type="recent", count="100", max_id=str(self.maxid))
else:
mydict=self.twython.search(q=query, result_type="recent", count="100")
actualcount=len(mydict["statuses"])
if actualcount != 0:
self.maxid=int(mydict["statuses"][actualcount-1]["id_str"])-1 #int at the moment
for item in mydict["statuses"]:
if (not (int(item["id_str"]) in self.idlist)):
text=item["text"].replace(unichr(8220),'"')
text=text.replace(unichr(8221),'"')
ctime=str(self.convertTime(item["created_at"]))
isretweet="0"
source=u"-"
tweet=u"-"
if text[0:2] == "RT":
isretweet="1"
s=text
try:
atindex=s.index("@")
breakif=False
except:
breakif=True
if breakif==False:
keepgoing=True
i=1
while keepgoing==True:
try:
if not (s[atindex+i] in valid_characters):
keepgoing=False
endindex=i
else:
i+=1
except:
keepgoing=False
endindex=i
source=s[atindex+1:atindex+endindex]
tweet=s[atindex+endindex+1:]
if not (item["user"]["screen_name"] in self.screened):
self.ucur.execute("INSERT INTO usermap VALUES('" + item["user"]["screen_name"] + "'," + item["user"]["id_str"] + ")" )
self.screened.append(item["user"]["screen_name"])
self.con.execute("INSERT INTO "+self.tablename+" VALUES("+str(item["id_str"])+","+escapes(repr(item["user"]["screen_name"])[1:])+","+escapes(repr(item["user"]["name"])[1:])+","+escapes(repr(text)[1:])+",'"+item["created_at"]+"',"+str(item["retweet_count"]) +","+str(fixnone(item["in_reply_to_status_id_str"]))+","+str(fixnone(item["in_reply_to_user_id_str"])) +"," + str(convertbool(item["truncated"])) +","+str(convertbool(item["retweeted"])) +","+str(item["user"]["friends_count"]) +"," + str(item["user"]["followers_count"]) + "," + isretweet +","+ escapes(repr(source)[1:]) +","+ctime+"," +escapes(repr(tweet)[1:]) + ")" )
ntweets+=1
else:
nmatch+=1
if nmatch>3:
print "Grabbing old tweets, stopping."
ntweets=numtweets+1
else:
print "Cannot obtain more tweets"
ntweets=numtweets+1
except Exception as detail:
print "Some Twitter error: " + str(detail)
sleep(300)
print str(ntweets)
sleep(10)
self.con.commit()
self.con.close()
self.ucon.commit()
self.ucon.close()
def query2(self,query,numtweets=160000, usemaxid=True):
#Used to get tweets for new IPCC hashtags, to add in the userID which was omitted in the previous database by mistake
if usemaxid==False:
self.maxid=None
self.query=query
ntweets=0
nmatch=0
while ntweets<numtweets:
try:
if self.maxid!=None:
mydict=self.twython.search(q=query, result_type="recent", count="100", max_id=str(self.maxid))
else:
mydict=self.twython.search(q=query, result_type="recent", count="100")
actualcount=len(mydict["statuses"])
if actualcount != 0:
self.maxid=int(mydict["statuses"][actualcount-1]["id_str"])-1 #int at the moment
for item in mydict["statuses"]:
if (not (int(item["id_str"]) in self.idlist)):
text=item["text"].replace(unichr(8220),'"')
text=text.replace(unichr(8221),'"')
ctime=str(self.convertTime(item["created_at"]))
isretweet="0"
source=u"-"
tweet=u"-"
if text[0:2] == "RT":
isretweet="1"
s=text
try:
atindex=s.index("@")
breakif=False
except:
breakif=True
if breakif==False:
keepgoing=True
i=1
while keepgoing==True:
try:
if not (s[atindex+i] in valid_characters):
keepgoing=False
endindex=i
else:
i+=1
except:
keepgoing=False
endindex=i
source=s[atindex+1:atindex+endindex]
tweet=s[atindex+endindex+1:]
if not (item["user"]["screen_name"] in self.screened):
self.ucur.execute("INSERT INTO usermap VALUES('" + item["user"]["screen_name"] + "'," + item["user"]["id_str"] + ")" )
self.screened.append(item["user"]["screen_name"])
self.con.execute("INSERT INTO "+self.tablename+" VALUES("+str(item["id_str"])+","+escapes(repr(item["user"]["screen_name"])[1:])+","+str(item["user"]["id_str"])+","+escapes(repr(item["user"]["name"])[1:])+","+escapes(repr(text)[1:])+",'"+item["created_at"]+"',"+str(item["retweet_count"]) +","+str(fixnone(item["in_reply_to_status_id_str"]))+","+str(fixnone(item["in_reply_to_user_id_str"])) +"," + str(convertbool(item["truncated"])) +","+str(convertbool(item["retweeted"])) +","+str(item["user"]["friends_count"]) +"," + str(item["user"]["followers_count"]) + "," + isretweet +","+ escapes(repr(source)[1:]) +","+ctime+"," +escapes(repr(tweet)[1:]) + ")" )
ntweets+=1
else:
nmatch+=1
if nmatch>3:
print "Grabbing old tweets, stopping."
ntweets=numtweets+1
else:
print "Cannot obtain more tweets"
ntweets=numtweets+1
except Exception as detail:
print "Some Twitter error: " + str(detail)
sleep(300)
print str(ntweets)
sleep(10)
self.con.commit()
self.con.close()
self.ucon.commit()
self.ucon.close()
def convertTime(self, timestring):
#Fri Nov 16 22:31:39 +0000 2012
lts=timestring.split(" ")
tt=lts[3].split(":")
monthdict={"Jan":1, "Feb":2, "Mar":3, "Apr":4, "May":5, "Jun":6, "Jul":7, "Aug":8, "Sep":9, "Oct":10, "Nov":11, "Dec":12}
dt=datetime.datetime(int(lts[5]), int(monthdict[lts[1]]), int(lts[2]),int(tt[0]),int(tt[1]),int(tt[2]))
return int(time.mktime(dt.timetuple()))
def getIDfromUser(self,screen_name):
gotry=True
while gotry==True:
try:
d=self.twython.showUser(screen_name=screen_name, entities="false")
gotry=False
except Exception as detail:
if ("The URI requested is invalid" in str(detail)) or ("Unauthorized" in str(detail)):
print "User " + screen_name + " does not exist."
sleep(8)
return "FAIL"
elif "suspended" in str(detail):
print "Some Twitter error: " + str(detail)
gotry=True
sleep(3600)
else:
print "Some Twitter error: " + str(detail)
gotry=True
sleep(300)
sleep(8)
return d["id_str"]
def getUserDescription(self, screen_name):
gotry=True
while gotry==True:
try:
d=self.twython.showUser(screen_name=screen_name, entities="false")
gotry=False
except Exception as detail:
if ("The URI requested is invalid" in str(detail)) or ("Unauthorized" in str(detail)) or ("An error occurred processing your request" in str(detail)):
print "User " + screen_name + " does not exist."
sleep(5)
return "FAIL"
elif "suspended" in str(detail):
print "Some Twitter error: " + str(detail)
#sleep(300)
return "FAIL"
else:
print "Some Twitter error: " + str(detail)
gotry=True
sleep(5)
sleep(3)
return d["description"]
def getFriends(self, name, friendslist, cursor):
#Recursively get followers
sleep(8)
#print "Getting Friends:" + name
while True:
try:
d=self.twython.getFriendsIDs(screen_name=name, cursor=str(cursor))
cursor=d["next_cursor"]
#print str(cursor)
break
except Exception as detail:
print "Some Twitter error: " + str(detail)
print name
print str(cursor)
if ("The URI requested is invalid" in str(detail)) or ("Unauthorized" in str(detail)):
print name
cursor=-1
return "FAIL"
elif "suspended" in str(detail):
sleep(3600)
else:
sleep(300)
friendslist=friendslist+d["ids"]
if cursor>0:
sleep(7)
friendslist=self.getFriends(name, friendslist, cursor)
# else:
return friendslist
def getFollowers(self, name, followerslist, cursor):
#Recursively get followers
sleep(7)
#print "Getting Followers:" + name
while True:
try:
d=self.twython.getFollowersIDs(screen_name=name, cursor=str(cursor))
cursor=d["next_cursor"]
#print str(cursor)
break
except Exception as detail:
print "Some Twitter error: " + str(detail)
print name
print str(cursor)
if ("The URI requested is invalid" in str(detail)) or ("Unauthorized" in str(detail)):
cursor=-1
return "FAIL"
elif "suspended" in str(detail):
sleep(3600)
else:
sleep(300)
followerslist=followerslist+d["ids"]
# print name + ": " +str(len(followerslist))
if cursor>0:
sleep(7)
followerslist=self.getFollowers(name, followerslist, cursor)
# print len(followerslist)
return followerslist
from twython import Twython
import string
import urllib2
from lxml import etree
import StringIO
twitter = Twython()
data_raw = twitter.getFollowersIDs(screen_name = "ionainstitute")
followers = data_raw['ids']
dict = {'noloc': 0}
# print followers
for id in followers:
url = "http://twitter.com/account/redirect_by_id?id=" + str(id)
try:
htmldata = urllib2.urlopen(url)
html = htmldata.read()
parser = etree.HTMLParser()
tree = etree.parse(StringIO.StringIO(html), parser)
xpath = '/html/body/div/div[3]/div/div[2]/div/div[2]/p[2]/span/text()'
pre_result1 = ' '.join(tree.xpath(xpath))
pre_result = pre_result1.strip()
decode_pre_result = pre_result.decode('string_escape')
print str(id) + " : " + decode_pre_result
from twython import Twython
import string
import urllib2
from lxml import etree
import StringIO
twitter = Twython()
data_raw = twitter.getFollowersIDs(screen_name="ionainstitute")
followers = data_raw['ids']
dict = {'noloc': 0}
# print followers
for id in followers:
url = "http://twitter.com/account/redirect_by_id?id=" + str(id)
try:
htmldata = urllib2.urlopen(url)
html = htmldata.read()
parser = etree.HTMLParser()
tree = etree.parse(StringIO.StringIO(html), parser)
xpath = '/html/body/div/div[3]/div/div[2]/div/div[2]/p[2]/span/text()'
pre_result1 = ' '.join(tree.xpath(xpath))
pre_result = pre_result1.strip()
decode_pre_result = pre_result.decode('string_escape')
print str(id) + " : " + decode_pre_result
from twython import Twython
usuario = "andrey" #definimos una variable
twitter = Twython() #creamos un objeto la clase Twython
followers = twitter.getFollowersIDs(
screen_name=usuario) #traemos los usuarios muy facil
for follower_id in followers:
print "Usuario %d sigue a %s" % (follower_id, usuario)
tweets = twitter.getPublicTimeline() #traemos los twees
for tweet in tweets:
print tweet['user']['name'].encode('utf-8')
print tweet['text'].encode('utf-8')
results = twitter.getDailytrends() #usando el objeto usamos ese metodo
#esto se llama o se dice iterar objetos
for time, trend_list in results['trends'].iteritems():
print time
for trend in trend_list:
print trend['query']
class DBTweetGetter(object):
"""
This class gets the tweets and stores them in the database.
query(query,numtweets=160000, usemaxid=True) is responsible for actually downloading the tweets. The IDs of the downloaded tweets are checked in the initialisation of the class, and query checks against this to know when it has downloaded all of the most recent, unseen tweets.
"""
def __init__(self, filename, tablename):
if filename != None:
self.con = lite.connect(filename)
self.screened = []
self.cur = self.con.cursor()
self.ucon = lite.connect("userdb.db")
self.ucur = self.ucon.cursor()
self.tablename = tablename
self.ucur.execute("SELECT ScreenName FROM usermap")
a = self.ucur.fetchall()
for item in a:
self.screened.append(item[0])
self.cur.execute("SELECT Id FROM " + tablename)
data = self.cur.fetchall()
l = []
for item in data:
l.append(item[0])
self.idlist = l
try:
self.twython = Twython(
app_key="674Getn4iR5ZonBvSZIE6w",
app_secret="XIIhDdsCmByqLQG89ED9h7MILQbS4hsMV3ob6hlLYA",
oauth_token=
"27203313-Oujk9Qu6sKe7tBBYYrheAb1r484PViz2w8GShZCeg",
oauth_token_secret="cm95hSRUmkloWBVbYrTHuIYA7fSQqFJH5iYGIbw5ZKo"
)
except Exception as detail:
print "Error loading Twython object. Error message: " + str(detail)
raise Exception(detail)
def query(self, query, numtweets=160000, usemaxid=True):
if usemaxid == False:
self.maxid = None
self.query = query
ntweets = 0
nmatch = 0
while ntweets < numtweets:
try:
if self.maxid != None:
mydict = self.twython.search(q=query,
result_type="recent",
count="100",
max_id=str(self.maxid))
else:
mydict = self.twython.search(q=query,
result_type="recent",
count="100")
actualcount = len(mydict["statuses"])
if actualcount != 0:
self.maxid = int(mydict["statuses"][actualcount - 1]
["id_str"]) - 1 #int at the moment
for item in mydict["statuses"]:
if (not (int(item["id_str"]) in self.idlist)):
text = item["text"].replace(unichr(8220), '"')
text = text.replace(unichr(8221), '"')
ctime = str(self.convertTime(item["created_at"]))
isretweet = "0"
source = u"-"
tweet = u"-"
if text[0:2] == "RT":
isretweet = "1"
s = text
try:
atindex = s.index("@")
breakif = False
except:
breakif = True
if breakif == False:
keepgoing = True
i = 1
while keepgoing == True:
try:
if not (s[atindex + i]
in valid_characters):
keepgoing = False
endindex = i
else:
i += 1
except:
keepgoing = False
endindex = i
source = s[atindex + 1:atindex + endindex]
tweet = s[atindex + endindex + 1:]
if not (item["user"]["screen_name"]
in self.screened):
self.ucur.execute(
"INSERT INTO usermap VALUES('" +
item["user"]["screen_name"] + "'," +
item["user"]["id_str"] + ")")
self.screened.append(
item["user"]["screen_name"])
self.con.execute(
"INSERT INTO " + self.tablename + " VALUES(" +
str(item["id_str"]) + "," + escapes(
repr(item["user"]["screen_name"])[1:]) +
"," + escapes(repr(item["user"]["name"])[1:]) +
"," + escapes(repr(text)[1:]) + ",'" +
item["created_at"] + "'," +
str(item["retweet_count"]) + "," +
str(fixnone(
item["in_reply_to_status_id_str"])) + "," +
str(fixnone(item["in_reply_to_user_id_str"])) +
"," + str(convertbool(item["truncated"])) +
"," + str(convertbool(item["retweeted"])) +
"," + str(item["user"]["friends_count"]) +
"," + str(item["user"]["followers_count"]) +
"," + isretweet + "," +
escapes(repr(source)[1:]) + "," + ctime + "," +
escapes(repr(tweet)[1:]) + ")")
ntweets += 1
else:
nmatch += 1
if nmatch > 3:
print "Grabbing old tweets, stopping."
ntweets = numtweets + 1
else:
print "Cannot obtain more tweets"
ntweets = numtweets + 1
except Exception as detail:
print "Some Twitter error: " + str(detail)
sleep(300)
print str(ntweets)
sleep(10)
self.con.commit()
self.con.close()
self.ucon.commit()
self.ucon.close()
def query2(self, query, numtweets=160000, usemaxid=True):
#Used to get tweets for new IPCC hashtags, to add in the userID which was omitted in the previous database by mistake
if usemaxid == False:
self.maxid = None
self.query = query
ntweets = 0
nmatch = 0
while ntweets < numtweets:
try:
if self.maxid != None:
mydict = self.twython.search(q=query,
result_type="recent",
count="100",
max_id=str(self.maxid))
else:
mydict = self.twython.search(q=query,
result_type="recent",
count="100")
actualcount = len(mydict["statuses"])
if actualcount != 0:
self.maxid = int(mydict["statuses"][actualcount - 1]
["id_str"]) - 1 #int at the moment
for item in mydict["statuses"]:
if (not (int(item["id_str"]) in self.idlist)):
text = item["text"].replace(unichr(8220), '"')
text = text.replace(unichr(8221), '"')
ctime = str(self.convertTime(item["created_at"]))
isretweet = "0"
source = u"-"
tweet = u"-"
if text[0:2] == "RT":
isretweet = "1"
s = text
try:
atindex = s.index("@")
breakif = False
except:
breakif = True
if breakif == False:
keepgoing = True
i = 1
while keepgoing == True:
try:
if not (s[atindex + i]
in valid_characters):
keepgoing = False
endindex = i
else:
i += 1
except:
keepgoing = False
endindex = i
source = s[atindex + 1:atindex + endindex]
tweet = s[atindex + endindex + 1:]
if not (item["user"]["screen_name"]
in self.screened):
self.ucur.execute(
"INSERT INTO usermap VALUES('" +
item["user"]["screen_name"] + "'," +
item["user"]["id_str"] + ")")
self.screened.append(
item["user"]["screen_name"])
self.con.execute(
"INSERT INTO " + self.tablename + " VALUES(" +
str(item["id_str"]) + "," + escapes(
repr(item["user"]["screen_name"])[1:]) +
"," + str(item["user"]["id_str"]) + "," +
escapes(repr(item["user"]["name"])[1:]) + "," +
escapes(repr(text)[1:]) + ",'" +
item["created_at"] + "'," +
str(item["retweet_count"]) + "," +
str(fixnone(
item["in_reply_to_status_id_str"])) + "," +
str(fixnone(item["in_reply_to_user_id_str"])) +
"," + str(convertbool(item["truncated"])) +
"," + str(convertbool(item["retweeted"])) +
"," + str(item["user"]["friends_count"]) +
"," + str(item["user"]["followers_count"]) +
"," + isretweet + "," +
escapes(repr(source)[1:]) + "," + ctime + "," +
escapes(repr(tweet)[1:]) + ")")
ntweets += 1
else:
nmatch += 1
if nmatch > 3:
print "Grabbing old tweets, stopping."
ntweets = numtweets + 1
else:
print "Cannot obtain more tweets"
ntweets = numtweets + 1
except Exception as detail:
print "Some Twitter error: " + str(detail)
sleep(300)
print str(ntweets)
sleep(10)
self.con.commit()
self.con.close()
self.ucon.commit()
self.ucon.close()
def convertTime(self, timestring):
#Fri Nov 16 22:31:39 +0000 2012
lts = timestring.split(" ")
tt = lts[3].split(":")
monthdict = {
"Jan": 1,
"Feb": 2,
"Mar": 3,
"Apr": 4,
"May": 5,
"Jun": 6,
"Jul": 7,
"Aug": 8,
"Sep": 9,
"Oct": 10,
"Nov": 11,
"Dec": 12
}
dt = datetime.datetime(int(lts[5]), int(monthdict[lts[1]]),
int(lts[2]), int(tt[0]), int(tt[1]), int(tt[2]))
return int(time.mktime(dt.timetuple()))
def getIDfromUser(self, screen_name):
gotry = True
while gotry == True:
try:
d = self.twython.showUser(screen_name=screen_name,
entities="false")
gotry = False
except Exception as detail:
if ("The URI requested is invalid"
in str(detail)) or ("Unauthorized" in str(detail)):
print "User " + screen_name + " does not exist."
sleep(8)
return "FAIL"
elif "suspended" in str(detail):
print "Some Twitter error: " + str(detail)
gotry = True
sleep(3600)
else:
print "Some Twitter error: " + str(detail)
gotry = True
sleep(300)
sleep(8)
return d["id_str"]
def getUserDescription(self, screen_name):
gotry = True
while gotry == True:
try:
d = self.twython.showUser(screen_name=screen_name,
entities="false")
gotry = False
except Exception as detail:
if ("The URI requested is invalid"
in str(detail)) or ("Unauthorized" in str(detail)) or (
"An error occurred processing your request"
in str(detail)):
print "User " + screen_name + " does not exist."
sleep(5)
return "FAIL"
elif "suspended" in str(detail):
print "Some Twitter error: " + str(detail)
#sleep(300)
return "FAIL"
else:
print "Some Twitter error: " + str(detail)
gotry = True
sleep(5)
sleep(3)
return d["description"]
def getFriends(self, name, friendslist, cursor):
#Recursively get followers
sleep(8)
#print "Getting Friends:" + name
while True:
try:
d = self.twython.getFriendsIDs(screen_name=name,
cursor=str(cursor))
cursor = d["next_cursor"]
#print str(cursor)
break
except Exception as detail:
print "Some Twitter error: " + str(detail)
print name
print str(cursor)
if ("The URI requested is invalid"
in str(detail)) or ("Unauthorized" in str(detail)):
print name
cursor = -1
return "FAIL"
elif "suspended" in str(detail):
sleep(3600)
else:
sleep(300)
friendslist = friendslist + d["ids"]
if cursor > 0:
sleep(7)
friendslist = self.getFriends(name, friendslist, cursor)
# else:
return friendslist
def getFollowers(self, name, followerslist, cursor):
#Recursively get followers
sleep(7)
#print "Getting Followers:" + name
while True:
try:
d = self.twython.getFollowersIDs(screen_name=name,
cursor=str(cursor))
cursor = d["next_cursor"]
#print str(cursor)
break
except Exception as detail:
print "Some Twitter error: " + str(detail)
print name
print str(cursor)
if ("The URI requested is invalid"
in str(detail)) or ("Unauthorized" in str(detail)):
cursor = -1
return "FAIL"
elif "suspended" in str(detail):
sleep(3600)
else:
sleep(300)
followerslist = followerslist + d["ids"]
# print name + ": " +str(len(followerslist))
if cursor > 0:
sleep(7)
followerslist = self.getFollowers(name, followerslist, cursor)
# print len(followerslist)
return followerslist
class Bot:
""" The twitter bot class """
def __init__(self, bot):
self.twython_obj = Twython(app_key=bot.consumer_key,
app_secret=bot.consumer_secret,
oauth_token=bot.access_token,
oauth_token_secret=bot.access_token_secret)
self.keywords = bot.keywords
self.source_user_ids = bot.source_user_ids
self.source_followers_ids = bot.source_followers_ids
self.followers_ids = bot.followers_ids
def get_source_user_ids(self):
""" Get some users to initiate the twitter bot for the given keywords"""
source_user_ids = []
for keyword in self.keywords:
# get a list of user objects by searching for the keyword
userlist = self.twython_obj.searchUsers(q=keyword)
# select 3 random user objects
random_list = random.sample(userlist, 3)
# store the screen_name of the 5 users in a list
ids = [user['id'] for user in random_list]
#add screen_names to the list of our tweet sources
source_user_ids = source_user_ids + ids
for keyword in self.keywords:
# get a list of recent tweets
tweetlist = self.twython_obj.search(q=keyword, result_type="recent", lang='en')['results']
# select 7 random tweets
random_list = random.sample(tweetlist, 7)
# store screen_names of users who made the tweets
ids = [tweet['from_user_id'] for tweet in random_list]
#add screen_names to the list of our tweet sources
source_user_ids = source_user_ids + ids
self.source_user_ids = source_user_ids
return source_user_ids
def get_source_followers_ids(self):
""" Returns a list of all followers of the users in source_users"""
source_followers_ids = []
for source_user_id in self.source_user_ids:
follower_ids = self.twython_obj.getFollowersIDs(user_id=source_user_id)['ids']
source_followers_ids = source_followers_ids + follower_ids
self.source_followers_ids = source_followers_ids
return source_followers_ids
def find_and_follow(self):
""" Find users to follow and follow them """
# Randomly select one keyword from the list of keywords
keyword = random.choice(self.keywords)
# Seacrh for tweets with that keyword
tweets = self.twython_obj.search(q=keyword, result_type="recent", lang='en')['results']
done_following = False
while not done_following:
# Randomly select a tweet
tweet = random.choice(tweets)
user_id = tweet['from_user_id']
if user_id not in self.source_user_ids and user_id not in self.source_followers_ids:
self.twython_obj.createFriendship(user_id=user_id)
done_following = True
return done_following
def get_follwers(self):
""" Returns a list of ids of all followers of the bot """
followers_ids = self.twython_obj.getFollowersIDs(user_id=user_id)['ids']
self.followers_ids = followers
return followers_ids
def copy_and_tweet(self):
""" Copy a tweet and tweet it """
from twython import Twython
usuario = "ajamaica"
twitter = Twython()
followers = twitter.getFollowersIDs(screen_name=usuario)
for follower_id in followers:
print "Usuarios %d sigue a %s" % (follower_id, usuario)
tweets = twitter.getPublicTimeline()
for tweet in tweets:
print tweet['user']['name'].encode('utf-8')
print tweet['text'].encode('utf-8')
results = twitter.getDailyTrends()
for i in range(0, 100):
print i
for time, trend_list in results['trends'].iteritems():
print time
for trend in trend_list:
print trend['query']
from twython import Twython
usuario = "andrey"#definimos una variable
twitter = Twython()#creamos un objeto la clase Twython
followers = twitter.getFollowersIDs( screen_name = usuario)#traemos los usuarios muy facil
for follower_id in followers:
print "Usuario %d sigue a %s"%(follower_id,usuario)
tweets = twitter.getPublicTimeline()#traemos los twees
for tweet in tweets:
print tweet['user']['name'].encode('utf-8')
print tweet['text'].encode('utf-8')
results = twitter.getDailytrends()#usando el objeto usamos ese metodo
#esto se llama o se dice iterar objetos
for time, trend_list in results['trends'].iteritems():
print time
for trend in trend_list:
print trend['query']
