def userFriends(user=''):
twitter = Twython()
friends = twitter.getFriendsIDs(screen_name=user)["ids"]
friend_list=''
friends_bio=''
i=0
for friend in friends:
friend_list += str(friend) + ','
if((i+1)%100==0):
aux = twitter.lookupUser(user_id=friend_list[:-1])
for x in aux:
try:
friends_bio += x['description'].encode('ascii', 'replace')
except:
pass
print friends_bio
friend_list=''
sleep(10)
i+=1
#for i in range(len(friends)):
# friend_list = friend_list + str(friends[i]) + ','
# print i, friend_list
# if((i+1)%100==0):
# aux = twitter.lookupUser(user_id=friend_list[:-1])
# print friend_list
# for x in range(100):
# print x
# friends_bio += aux[x]['description'].encode('ascii', 'replace')
# friends_bio += ' '
# sleep(5)
# friend_list=''
#lookup = twitter.lookupUser(user_id=friend_list)
return friends_bio
def userFriends(user=''):
twitter = Twython()
friends = twitter.getFriendsIDs(screen_name=user)["ids"]
friend_list = ''
friends_bio = ''
i = 0
for friend in friends:
friend_list += str(friend) + ','
if ((i + 1) % 100 == 0):
aux = twitter.lookupUser(user_id=friend_list[:-1])
for x in aux:
try:
friends_bio += x['description'].encode('ascii', 'replace')
except:
pass
print friends_bio
friend_list = ''
sleep(10)
i += 1
#for i in range(len(friends)):
# friend_list = friend_list + str(friends[i]) + ','
# print i, friend_list
# if((i+1)%100==0):
# aux = twitter.lookupUser(user_id=friend_list[:-1])
# print friend_list
# for x in range(100):
# print x
# friends_bio += aux[x]['description'].encode('ascii', 'replace')
# friends_bio += ' '
# sleep(5)
# friend_list=''
#lookup = twitter.lookupUser(user_id=friend_list)
return friends_bio
def getUserEgoNetwork (self, userName): #Returns the ego centric net for a user twitter = Twython() followers = twitter.getFollowersIDs(screen_name = userName) friends = twitter.getFriendsIDs(screen_name = userName) symmetricFriendship =set(followers).intersection(set(friends)) print "--------------FOLLOWERS: %d------------------" %(len(followers)) for follower_id in followers: print "User with ID %s, is following %s" % ( follower_id, userName ) print "--------------FRIENDSHIP: %d------------------" %(len(friends)) for friends_id in friends: print "%s is friend with user with ID %s" % ( userName, friends_id ) print "--------------SYMMETRIC: %d------------------" %(len(symmetricFriendship)) return symmetricFriendship
def friends(request):
"""
An example view with Twython/OAuth hooks/calls to fetch data about the user
in question. Pretty self explanatory if you read through it...
"""
user = request.user.twitterprofile
twitter = Twython(
twitter_token=settings.TWITTER_KEY,
twitter_secret=settings.TWITTER_SECRET,
oauth_token=user.oauth_token,
oauth_token_secret=user.oauth_secret,
)
# user_tweets = twitter.getHomeTimeline()
user_friends_ids = twitter.getFriendsIDs()["ids"]
friends_details = twitter.lookupUser(user_id=user_friends_ids)
return render_to_response("friends.html", {"friends": friends_details})
def all_favorites(request):
user = request.user.twitterprofile
twitter = Twython(
twitter_token=settings.TWITTER_KEY,
twitter_secret=settings.TWITTER_SECRET,
oauth_token=user.oauth_token,
oauth_token_secret=user.oauth_secret,
)
user_friends_ids = twitter.getFriendsIDs()["ids"]
# friends_details = twitter.lookupUser(user_id=user_friends_ids)
all_favorites = []
for friend in user_friends_ids:
all_favorites.append(twitter.getFavorites(user_id=friend, count=200))
all_favorites = list(itertools.chain.from_iterable(all_favorites))
return render_to_response("all_favorites.html", {"all_favorites": all_favorites})
def search_with_tokens(tokens, queryType, params):
#print "params = ", params
#user = UserSocialAuth.objects.filter(user=request.user).get()
# Set up a new Twython object with the OAuth keys
#pprint(tokens)
print "app_key = ", settings.TWITTER_CONSUMER_KEY
print "app_secret = ", settings.TWITTER_CONSUMER_SECRET
print "oauth_token = ", tokens['oauth_token']
print "oauth_token_secret = ", tokens['oauth_token_secret']
t = Twython(app_key=settings.TWITTER_CONSUMER_KEY,
app_secret=settings.TWITTER_CONSUMER_SECRET,
oauth_token=tokens['oauth_token'],
oauth_token_secret=tokens['oauth_token_secret'])
# Obtain Twitter results using user's OAuth key
if queryType == "search":
params = params.copy()
pageID = params["pageID"]
del params["pageID"]
queryResults = t.search(**(params))
# Use helper to update the tweet list of the page
# Return an array of tweets to return to the client
helper = Helper()
# tweets = helper.updateWikiArticleTweet(pageID, queryResults) # Update the correct pageID
helper.updateWikiArticleTweet(pageID, queryResults) # Update the correct pageID
'''
tweetsSerialization = []
for tweet in tweets:
tweetsSerialization.append({
"id" : tweet.id,
"text" : tweet.text,
"source" : tweet.source,
"profileImageUrl" : tweet.profileImageUrl,
"createdAt" : tweet.createdAt
})
'''
# Get the correct logger based on convention:
# {APP}.{FILE}.{FUNCTION} = __name__.'.'.inspect.stack()[0][3]
# Need to evaulate performance of 'inspect' library as print function name
# is not supported by Python
#print __name__+"."+inspect.stack()[0][3]
logger = logging.getLogger(__name__+"."+inspect.stack()[0][3])
# Fake infomation, to be updated later
logger.info(json.dumps({
"pageID": pageID, # Pass the correct pageID here
"queryType": queryType,
"query": queryResults["search_metadata"]["query"],
"tweetsCount": queryResults["search_metadata"]["count"],
"source": 'TwitterAPI',
}))
return HttpResponse(json.dumps(queryResults), content_type="application/json")
## elif queryType == "articleSearch":
## params1 = dict(params)
## # TODO, swap this out later
## params1["q"] = unicode(params["articleId"])
## del params1["articleId"]
## queryResults = t.search(**(params1))
elif queryType == "followersID":
queryResults = t.getFollowersIDs(**(params))
elif queryType == "followersList":
params = params.copy()
screen_name = params["screen_name"]
cursor = params["cursor"]
queryResults = t.getFollowersList(**(params))
# Save results
helper = Helper()
helper.getUserFollower(screen_name, queryResults)
elif queryType == "friendsID":
queryResults = t.getFriendsIDs(**(params))
elif queryType == "friendshipLookup":
queryResults = t.lookupFriendships(**(params))
elif queryType == "stream":
queryResults = t.stream(params, on_results)
return HttpResponse(json.dumps(queryResults), content_type="application/json")
class DBTweetGetter(object):
"""
This class gets the tweets and stores them in the database.
query(query,numtweets=160000, usemaxid=True) is responsible for actually downloading the tweets. The IDs of the downloaded tweets are checked in the initialisation of the class, and query checks against this to know when it has downloaded all of the most recent, unseen tweets.
"""
def __init__(self, filename, tablename):
if filename!=None:
self.con = lite.connect(filename)
self.screened=[]
self.cur=self.con.cursor()
self.ucon = lite.connect("userdb.db")
self.ucur=self.ucon.cursor()
self.tablename=tablename
self.ucur.execute("SELECT ScreenName FROM usermap")
a=self.ucur.fetchall()
for item in a:
self.screened.append(item[0])
self.cur.execute("SELECT Id FROM "+tablename)
data=self.cur.fetchall()
l=[]
for item in data:
l.append(item[0])
self.idlist=l
try:
self.twython= Twython(app_key="674Getn4iR5ZonBvSZIE6w", app_secret="XIIhDdsCmByqLQG89ED9h7MILQbS4hsMV3ob6hlLYA", oauth_token="27203313-Oujk9Qu6sKe7tBBYYrheAb1r484PViz2w8GShZCeg", oauth_token_secret="cm95hSRUmkloWBVbYrTHuIYA7fSQqFJH5iYGIbw5ZKo")
except Exception as detail:
print "Error loading Twython object. Error message: " + str(detail)
raise Exception(detail)
def query(self,query,numtweets=160000, usemaxid=True):
if usemaxid==False:
self.maxid=None
self.query=query
ntweets=0
nmatch=0
while ntweets<numtweets:
try:
if self.maxid!=None:
mydict=self.twython.search(q=query, result_type="recent", count="100", max_id=str(self.maxid))
else:
mydict=self.twython.search(q=query, result_type="recent", count="100")
actualcount=len(mydict["statuses"])
if actualcount != 0:
self.maxid=int(mydict["statuses"][actualcount-1]["id_str"])-1 #int at the moment
for item in mydict["statuses"]:
if (not (int(item["id_str"]) in self.idlist)):
text=item["text"].replace(unichr(8220),'"')
text=text.replace(unichr(8221),'"')
ctime=str(self.convertTime(item["created_at"]))
isretweet="0"
source=u"-"
tweet=u"-"
if text[0:2] == "RT":
isretweet="1"
s=text
try:
atindex=s.index("@")
breakif=False
except:
breakif=True
if breakif==False:
keepgoing=True
i=1
while keepgoing==True:
try:
if not (s[atindex+i] in valid_characters):
keepgoing=False
endindex=i
else:
i+=1
except:
keepgoing=False
endindex=i
source=s[atindex+1:atindex+endindex]
tweet=s[atindex+endindex+1:]
if not (item["user"]["screen_name"] in self.screened):
self.ucur.execute("INSERT INTO usermap VALUES('" + item["user"]["screen_name"] + "'," + item["user"]["id_str"] + ")" )
self.screened.append(item["user"]["screen_name"])
self.con.execute("INSERT INTO "+self.tablename+" VALUES("+str(item["id_str"])+","+escapes(repr(item["user"]["screen_name"])[1:])+","+escapes(repr(item["user"]["name"])[1:])+","+escapes(repr(text)[1:])+",'"+item["created_at"]+"',"+str(item["retweet_count"]) +","+str(fixnone(item["in_reply_to_status_id_str"]))+","+str(fixnone(item["in_reply_to_user_id_str"])) +"," + str(convertbool(item["truncated"])) +","+str(convertbool(item["retweeted"])) +","+str(item["user"]["friends_count"]) +"," + str(item["user"]["followers_count"]) + "," + isretweet +","+ escapes(repr(source)[1:]) +","+ctime+"," +escapes(repr(tweet)[1:]) + ")" )
ntweets+=1
else:
nmatch+=1
if nmatch>3:
print "Grabbing old tweets, stopping."
ntweets=numtweets+1
else:
print "Cannot obtain more tweets"
ntweets=numtweets+1
except Exception as detail:
print "Some Twitter error: " + str(detail)
sleep(300)
print str(ntweets)
sleep(10)
self.con.commit()
self.con.close()
self.ucon.commit()
self.ucon.close()
def query2(self,query,numtweets=160000, usemaxid=True):
#Used to get tweets for new IPCC hashtags, to add in the userID which was omitted in the previous database by mistake
if usemaxid==False:
self.maxid=None
self.query=query
ntweets=0
nmatch=0
while ntweets<numtweets:
try:
if self.maxid!=None:
mydict=self.twython.search(q=query, result_type="recent", count="100", max_id=str(self.maxid))
else:
mydict=self.twython.search(q=query, result_type="recent", count="100")
actualcount=len(mydict["statuses"])
if actualcount != 0:
self.maxid=int(mydict["statuses"][actualcount-1]["id_str"])-1 #int at the moment
for item in mydict["statuses"]:
if (not (int(item["id_str"]) in self.idlist)):
text=item["text"].replace(unichr(8220),'"')
text=text.replace(unichr(8221),'"')
ctime=str(self.convertTime(item["created_at"]))
isretweet="0"
source=u"-"
tweet=u"-"
if text[0:2] == "RT":
isretweet="1"
s=text
try:
atindex=s.index("@")
breakif=False
except:
breakif=True
if breakif==False:
keepgoing=True
i=1
while keepgoing==True:
try:
if not (s[atindex+i] in valid_characters):
keepgoing=False
endindex=i
else:
i+=1
except:
keepgoing=False
endindex=i
source=s[atindex+1:atindex+endindex]
tweet=s[atindex+endindex+1:]
if not (item["user"]["screen_name"] in self.screened):
self.ucur.execute("INSERT INTO usermap VALUES('" + item["user"]["screen_name"] + "'," + item["user"]["id_str"] + ")" )
self.screened.append(item["user"]["screen_name"])
self.con.execute("INSERT INTO "+self.tablename+" VALUES("+str(item["id_str"])+","+escapes(repr(item["user"]["screen_name"])[1:])+","+str(item["user"]["id_str"])+","+escapes(repr(item["user"]["name"])[1:])+","+escapes(repr(text)[1:])+",'"+item["created_at"]+"',"+str(item["retweet_count"]) +","+str(fixnone(item["in_reply_to_status_id_str"]))+","+str(fixnone(item["in_reply_to_user_id_str"])) +"," + str(convertbool(item["truncated"])) +","+str(convertbool(item["retweeted"])) +","+str(item["user"]["friends_count"]) +"," + str(item["user"]["followers_count"]) + "," + isretweet +","+ escapes(repr(source)[1:]) +","+ctime+"," +escapes(repr(tweet)[1:]) + ")" )
ntweets+=1
else:
nmatch+=1
if nmatch>3:
print "Grabbing old tweets, stopping."
ntweets=numtweets+1
else:
print "Cannot obtain more tweets"
ntweets=numtweets+1
except Exception as detail:
print "Some Twitter error: " + str(detail)
sleep(300)
print str(ntweets)
sleep(10)
self.con.commit()
self.con.close()
self.ucon.commit()
self.ucon.close()
def convertTime(self, timestring):
#Fri Nov 16 22:31:39 +0000 2012
lts=timestring.split(" ")
tt=lts[3].split(":")
monthdict={"Jan":1, "Feb":2, "Mar":3, "Apr":4, "May":5, "Jun":6, "Jul":7, "Aug":8, "Sep":9, "Oct":10, "Nov":11, "Dec":12}
dt=datetime.datetime(int(lts[5]), int(monthdict[lts[1]]), int(lts[2]),int(tt[0]),int(tt[1]),int(tt[2]))
return int(time.mktime(dt.timetuple()))
def getIDfromUser(self,screen_name):
gotry=True
while gotry==True:
try:
d=self.twython.showUser(screen_name=screen_name, entities="false")
gotry=False
except Exception as detail:
if ("The URI requested is invalid" in str(detail)) or ("Unauthorized" in str(detail)):
print "User " + screen_name + " does not exist."
sleep(8)
return "FAIL"
elif "suspended" in str(detail):
print "Some Twitter error: " + str(detail)
gotry=True
sleep(3600)
else:
print "Some Twitter error: " + str(detail)
gotry=True
sleep(300)
sleep(8)
return d["id_str"]
def getUserDescription(self, screen_name):
gotry=True
while gotry==True:
try:
d=self.twython.showUser(screen_name=screen_name, entities="false")
gotry=False
except Exception as detail:
if ("The URI requested is invalid" in str(detail)) or ("Unauthorized" in str(detail)) or ("An error occurred processing your request" in str(detail)):
print "User " + screen_name + " does not exist."
sleep(5)
return "FAIL"
elif "suspended" in str(detail):
print "Some Twitter error: " + str(detail)
#sleep(300)
return "FAIL"
else:
print "Some Twitter error: " + str(detail)
gotry=True
sleep(5)
sleep(3)
return d["description"]
def getFriends(self, name, friendslist, cursor):
#Recursively get followers
sleep(8)
#print "Getting Friends:" + name
while True:
try:
d=self.twython.getFriendsIDs(screen_name=name, cursor=str(cursor))
cursor=d["next_cursor"]
#print str(cursor)
break
except Exception as detail:
print "Some Twitter error: " + str(detail)
print name
print str(cursor)
if ("The URI requested is invalid" in str(detail)) or ("Unauthorized" in str(detail)):
print name
cursor=-1
return "FAIL"
elif "suspended" in str(detail):
sleep(3600)
else:
sleep(300)
friendslist=friendslist+d["ids"]
if cursor>0:
sleep(7)
friendslist=self.getFriends(name, friendslist, cursor)
# else:
return friendslist
def getFollowers(self, name, followerslist, cursor):
#Recursively get followers
sleep(7)
#print "Getting Followers:" + name
while True:
try:
d=self.twython.getFollowersIDs(screen_name=name, cursor=str(cursor))
cursor=d["next_cursor"]
#print str(cursor)
break
except Exception as detail:
print "Some Twitter error: " + str(detail)
print name
print str(cursor)
if ("The URI requested is invalid" in str(detail)) or ("Unauthorized" in str(detail)):
cursor=-1
return "FAIL"
elif "suspended" in str(detail):
sleep(3600)
else:
sleep(300)
followerslist=followerslist+d["ids"]
# print name + ": " +str(len(followerslist))
if cursor>0:
sleep(7)
followerslist=self.getFollowers(name, followerslist, cursor)
# print len(followerslist)
return followerslist
class DBTweetGetter(object):
"""
This class gets the tweets and stores them in the database.
query(query,numtweets=160000, usemaxid=True) is responsible for actually downloading the tweets. The IDs of the downloaded tweets are checked in the initialisation of the class, and query checks against this to know when it has downloaded all of the most recent, unseen tweets.
"""
def __init__(self, filename, tablename):
if filename != None:
self.con = lite.connect(filename)
self.screened = []
self.cur = self.con.cursor()
self.ucon = lite.connect("userdb.db")
self.ucur = self.ucon.cursor()
self.tablename = tablename
self.ucur.execute("SELECT ScreenName FROM usermap")
a = self.ucur.fetchall()
for item in a:
self.screened.append(item[0])
self.cur.execute("SELECT Id FROM " + tablename)
data = self.cur.fetchall()
l = []
for item in data:
l.append(item[0])
self.idlist = l
try:
self.twython = Twython(
app_key="674Getn4iR5ZonBvSZIE6w",
app_secret="XIIhDdsCmByqLQG89ED9h7MILQbS4hsMV3ob6hlLYA",
oauth_token=
"27203313-Oujk9Qu6sKe7tBBYYrheAb1r484PViz2w8GShZCeg",
oauth_token_secret="cm95hSRUmkloWBVbYrTHuIYA7fSQqFJH5iYGIbw5ZKo"
)
except Exception as detail:
print "Error loading Twython object. Error message: " + str(detail)
raise Exception(detail)
def query(self, query, numtweets=160000, usemaxid=True):
if usemaxid == False:
self.maxid = None
self.query = query
ntweets = 0
nmatch = 0
while ntweets < numtweets:
try:
if self.maxid != None:
mydict = self.twython.search(q=query,
result_type="recent",
count="100",
max_id=str(self.maxid))
else:
mydict = self.twython.search(q=query,
result_type="recent",
count="100")
actualcount = len(mydict["statuses"])
if actualcount != 0:
self.maxid = int(mydict["statuses"][actualcount - 1]
["id_str"]) - 1 #int at the moment
for item in mydict["statuses"]:
if (not (int(item["id_str"]) in self.idlist)):
text = item["text"].replace(unichr(8220), '"')
text = text.replace(unichr(8221), '"')
ctime = str(self.convertTime(item["created_at"]))
isretweet = "0"
source = u"-"
tweet = u"-"
if text[0:2] == "RT":
isretweet = "1"
s = text
try:
atindex = s.index("@")
breakif = False
except:
breakif = True
if breakif == False:
keepgoing = True
i = 1
while keepgoing == True:
try:
if not (s[atindex + i]
in valid_characters):
keepgoing = False
endindex = i
else:
i += 1
except:
keepgoing = False
endindex = i
source = s[atindex + 1:atindex + endindex]
tweet = s[atindex + endindex + 1:]
if not (item["user"]["screen_name"]
in self.screened):
self.ucur.execute(
"INSERT INTO usermap VALUES('" +
item["user"]["screen_name"] + "'," +
item["user"]["id_str"] + ")")
self.screened.append(
item["user"]["screen_name"])
self.con.execute(
"INSERT INTO " + self.tablename + " VALUES(" +
str(item["id_str"]) + "," + escapes(
repr(item["user"]["screen_name"])[1:]) +
"," + escapes(repr(item["user"]["name"])[1:]) +
"," + escapes(repr(text)[1:]) + ",'" +
item["created_at"] + "'," +
str(item["retweet_count"]) + "," +
str(fixnone(
item["in_reply_to_status_id_str"])) + "," +
str(fixnone(item["in_reply_to_user_id_str"])) +
"," + str(convertbool(item["truncated"])) +
"," + str(convertbool(item["retweeted"])) +
"," + str(item["user"]["friends_count"]) +
"," + str(item["user"]["followers_count"]) +
"," + isretweet + "," +
escapes(repr(source)[1:]) + "," + ctime + "," +
escapes(repr(tweet)[1:]) + ")")
ntweets += 1
else:
nmatch += 1
if nmatch > 3:
print "Grabbing old tweets, stopping."
ntweets = numtweets + 1
else:
print "Cannot obtain more tweets"
ntweets = numtweets + 1
except Exception as detail:
print "Some Twitter error: " + str(detail)
sleep(300)
print str(ntweets)
sleep(10)
self.con.commit()
self.con.close()
self.ucon.commit()
self.ucon.close()
def query2(self, query, numtweets=160000, usemaxid=True):
#Used to get tweets for new IPCC hashtags, to add in the userID which was omitted in the previous database by mistake
if usemaxid == False:
self.maxid = None
self.query = query
ntweets = 0
nmatch = 0
while ntweets < numtweets:
try:
if self.maxid != None:
mydict = self.twython.search(q=query,
result_type="recent",
count="100",
max_id=str(self.maxid))
else:
mydict = self.twython.search(q=query,
result_type="recent",
count="100")
actualcount = len(mydict["statuses"])
if actualcount != 0:
self.maxid = int(mydict["statuses"][actualcount - 1]
["id_str"]) - 1 #int at the moment
for item in mydict["statuses"]:
if (not (int(item["id_str"]) in self.idlist)):
text = item["text"].replace(unichr(8220), '"')
text = text.replace(unichr(8221), '"')
ctime = str(self.convertTime(item["created_at"]))
isretweet = "0"
source = u"-"
tweet = u"-"
if text[0:2] == "RT":
isretweet = "1"
s = text
try:
atindex = s.index("@")
breakif = False
except:
breakif = True
if breakif == False:
keepgoing = True
i = 1
while keepgoing == True:
try:
if not (s[atindex + i]
in valid_characters):
keepgoing = False
endindex = i
else:
i += 1
except:
keepgoing = False
endindex = i
source = s[atindex + 1:atindex + endindex]
tweet = s[atindex + endindex + 1:]
if not (item["user"]["screen_name"]
in self.screened):
self.ucur.execute(
"INSERT INTO usermap VALUES('" +
item["user"]["screen_name"] + "'," +
item["user"]["id_str"] + ")")
self.screened.append(
item["user"]["screen_name"])
self.con.execute(
"INSERT INTO " + self.tablename + " VALUES(" +
str(item["id_str"]) + "," + escapes(
repr(item["user"]["screen_name"])[1:]) +
"," + str(item["user"]["id_str"]) + "," +
escapes(repr(item["user"]["name"])[1:]) + "," +
escapes(repr(text)[1:]) + ",'" +
item["created_at"] + "'," +
str(item["retweet_count"]) + "," +
str(fixnone(
item["in_reply_to_status_id_str"])) + "," +
str(fixnone(item["in_reply_to_user_id_str"])) +
"," + str(convertbool(item["truncated"])) +
"," + str(convertbool(item["retweeted"])) +
"," + str(item["user"]["friends_count"]) +
"," + str(item["user"]["followers_count"]) +
"," + isretweet + "," +
escapes(repr(source)[1:]) + "," + ctime + "," +
escapes(repr(tweet)[1:]) + ")")
ntweets += 1
else:
nmatch += 1
if nmatch > 3:
print "Grabbing old tweets, stopping."
ntweets = numtweets + 1
else:
print "Cannot obtain more tweets"
ntweets = numtweets + 1
except Exception as detail:
print "Some Twitter error: " + str(detail)
sleep(300)
print str(ntweets)
sleep(10)
self.con.commit()
self.con.close()
self.ucon.commit()
self.ucon.close()
def convertTime(self, timestring):
#Fri Nov 16 22:31:39 +0000 2012
lts = timestring.split(" ")
tt = lts[3].split(":")
monthdict = {
"Jan": 1,
"Feb": 2,
"Mar": 3,
"Apr": 4,
"May": 5,
"Jun": 6,
"Jul": 7,
"Aug": 8,
"Sep": 9,
"Oct": 10,
"Nov": 11,
"Dec": 12
}
dt = datetime.datetime(int(lts[5]), int(monthdict[lts[1]]),
int(lts[2]), int(tt[0]), int(tt[1]), int(tt[2]))
return int(time.mktime(dt.timetuple()))
def getIDfromUser(self, screen_name):
gotry = True
while gotry == True:
try:
d = self.twython.showUser(screen_name=screen_name,
entities="false")
gotry = False
except Exception as detail:
if ("The URI requested is invalid"
in str(detail)) or ("Unauthorized" in str(detail)):
print "User " + screen_name + " does not exist."
sleep(8)
return "FAIL"
elif "suspended" in str(detail):
print "Some Twitter error: " + str(detail)
gotry = True
sleep(3600)
else:
print "Some Twitter error: " + str(detail)
gotry = True
sleep(300)
sleep(8)
return d["id_str"]
def getUserDescription(self, screen_name):
gotry = True
while gotry == True:
try:
d = self.twython.showUser(screen_name=screen_name,
entities="false")
gotry = False
except Exception as detail:
if ("The URI requested is invalid"
in str(detail)) or ("Unauthorized" in str(detail)) or (
"An error occurred processing your request"
in str(detail)):
print "User " + screen_name + " does not exist."
sleep(5)
return "FAIL"
elif "suspended" in str(detail):
print "Some Twitter error: " + str(detail)
#sleep(300)
return "FAIL"
else:
print "Some Twitter error: " + str(detail)
gotry = True
sleep(5)
sleep(3)
return d["description"]
def getFriends(self, name, friendslist, cursor):
#Recursively get followers
sleep(8)
#print "Getting Friends:" + name
while True:
try:
d = self.twython.getFriendsIDs(screen_name=name,
cursor=str(cursor))
cursor = d["next_cursor"]
#print str(cursor)
break
except Exception as detail:
print "Some Twitter error: " + str(detail)
print name
print str(cursor)
if ("The URI requested is invalid"
in str(detail)) or ("Unauthorized" in str(detail)):
print name
cursor = -1
return "FAIL"
elif "suspended" in str(detail):
sleep(3600)
else:
sleep(300)
friendslist = friendslist + d["ids"]
if cursor > 0:
sleep(7)
friendslist = self.getFriends(name, friendslist, cursor)
# else:
return friendslist
def getFollowers(self, name, followerslist, cursor):
#Recursively get followers
sleep(7)
#print "Getting Followers:" + name
while True:
try:
d = self.twython.getFollowersIDs(screen_name=name,
cursor=str(cursor))
cursor = d["next_cursor"]
#print str(cursor)
break
except Exception as detail:
print "Some Twitter error: " + str(detail)
print name
print str(cursor)
if ("The URI requested is invalid"
in str(detail)) or ("Unauthorized" in str(detail)):
cursor = -1
return "FAIL"
elif "suspended" in str(detail):
sleep(3600)
else:
sleep(300)
followerslist = followerslist + d["ids"]
# print name + ": " +str(len(followerslist))
if cursor > 0:
sleep(7)
followerslist = self.getFollowers(name, followerslist, cursor)
# print len(followerslist)
return followerslist
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get('DATABASE_URL')
db = SQLAlchemy(app)
tavorite = Twython(app_key=os.environ['CONSUMER_KEY'],
app_secret=os.environ['CONSUMER_SECRET'],
oauth_token=os.environ['ACCESS_TOKEN'],
oauth_token_secret=os.environ['ACCESS_TOKEN_SECRET'])
following = tavorite.getFriendsIDs()['ids']
@app.route('/')
def home():
links = Tweet.query.filter_by(url_exists=True).order_by(Tweet.score_with_time.desc()).filter(Tweet.main_url != 'instagram.com', Tweet.main_url != 'www.instagram.com', Tweet.main_url != 'instagr.am', Tweet.main_url != 'youtube.com', Tweet.main_url != 'www.youtube.com', Tweet.main_url != 'www.vimeo.com', Tweet.main_url != 'twitpic.com', Tweet.main_url != 'www.twitpic.com', Tweet.main_url !='i.imgur.com', Tweet.main_url != 'www.yfrog.com',Tweet.main_url != 'twitter.yfrog.com', Tweet.main_url != 'twitter.com', Tweet.main_url != 'imgur.com').limit(30).all()
time = tweets_age_for_view(links)
#links = list_of_links(Nava_rank.rt_count)
### NEEDS A FILTER FOR MEDIA LINKS
return render_template('show_links.html', links=links, time=time)
@app.route('/best')
def best():
links = Tweet.query.filter_by(url_exists=True).order_by(Tweet.score.desc()).filter(Tweet.main_url != 'instagram.com', Tweet.main_url != 'www.instagram.com', Tweet.main_url != 'instagr.am', Tweet.main_url != 'youtube.com', Tweet.main_url != 'www.youtube.com', Tweet.main_url != 'www.vimeo.com', Tweet.main_url != 'twitpic.com', Tweet.main_url != 'www.twitpic.com', Tweet.main_url !='i.imgur.com', Tweet.main_url != 'www.yfrog.com').limit(50).all()
#links = list_of_links(Nava_rank.rt_count)
from datetime import datetime, timedelta
from BeautifulSoup import BeautifulSoup
from twython import Twython
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get('DATABASE_URL')
db = SQLAlchemy(app)
tavorite = Twython(app_key=os.environ['CONSUMER_KEY'],
app_secret=os.environ['CONSUMER_SECRET'],
oauth_token=os.environ['ACCESS_TOKEN'],
oauth_token_secret=os.environ['ACCESS_TOKEN_SECRET'])
following = tavorite.getFriendsIDs()['ids']
@app.route('/')
def home():
links = Tweet.query.filter_by(url_exists=True).order_by(
Tweet.score_with_time.desc()).filter(
Tweet.main_url != 'instagram.com',
Tweet.main_url != 'www.instagram.com',
Tweet.main_url != 'instagr.am', Tweet.main_url != 'youtube.com',
Tweet.main_url != 'www.youtube.com',
Tweet.main_url != 'www.vimeo.com', Tweet.main_url != 'twitpic.com',
Tweet.main_url != 'www.twitpic.com',
Tweet.main_url != 'i.imgur.com', Tweet.main_url != 'www.yfrog.com',
Tweet.main_url != 'twitter.yfrog.com',
Tweet.main_url != 'twitter.com',
app = Flask(__name__)
app.config["SQLALCHEMY_DATABASE_URI"] = os.environ.get("DATABASE_URL")
db = SQLAlchemy(app)
tavorite = Twython(
app_key=os.environ["CONSUMER_KEY"],
app_secret=os.environ["CONSUMER_SECRET"],
oauth_token=os.environ["ACCESS_TOKEN"],
oauth_token_secret=os.environ["ACCESS_TOKEN_SECRET"],
)
following = tavorite.getFriendsIDs()["ids"]
filter_out_media = [
"instagram.com",
"www.instagram.com",
"instagr.am",
"youtube.com",
"www.youtube.com",
"www.vimeo.com",
"vimeo.com",
"twitpic.com",
"www.twitpic.com",
"i.imgur.com",
"www.yfrog.com",
"twitter.yfrog.com",
"twitter.com",
