from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound from secrets import * TRACK = 'teaching' twitter_auth = twitter.oauth.OAuth(OAUTH_TOKEN, OAUTH_TOKEN_SECRET, CONSUMER_KEY, CONSUMER_SECRET) twitter_stream = twitter.TwitterStream(auth=twitter_auth) statuses = twitter_stream.statuses.filter(track=TRACK) for t in statuses: print(t['text']) try: u = db_session.query(User).filter_by(uid=str(t['user']['id'])).one() except NoResultFound: u = User(screen_name=t['user']['screen_name'], uid=t['user']['id']) db_session.add(u) db_session.commit() tw = Tweet(tweet=t['text'], tid=t['id'], user_id=u.id, created_at=t['created_at'], data=json.dumps(t)) try: words = tw.tweet.split() for w in words: try:
#find out who is a bot, and who is not from tweetsql.model import User from tweetsql.database import db_session from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC import time driver = webdriver.Firefox() driver.get("http://truthy.indiana.edu/botornot/") elem = driver.find_element_by_class_name("form-control") all_users = db_session.query(User).all() button = driver.find_element_by_id('btnGetTimeline') for user in all_users: screen_name = user.screen_name.encode('ascii', 'ignore') elem.send_keys(screen_name) button.click() # try: # print 'waiting' # element = WebDriverWait(driver, 30).until( # EC.presence_of_element_located((By.ID, 'span-power-readout')) # ) # finally: # driver.quit() # driver.implicitly_wait(10) time.sleep(10) print 'waiting some more? why...who knows?'
g.node[n]['weight'] = 1 g.node[n]['type'] = t def graph_add_edge(n1, n2, g): if g.has_edge(n1, n2): g[n1][n2]['weight']+=1 else: g.add_edge(n1,n2) g[n1][n2]['weight']=1 graph = nx.Graph() iterate through every tweet, storing each tweet in t print len(db_session.query(Friend.user_id, func.count(Friend.friend_id)).group_by(Friend.user_id).all()) add t to the graph graph_add_node(t.tweet, graph, 'tweet') # now iterate through all the words in t, storing each word in w for w in t.words: # looking for hashtags (there's a better way to do this) if w.word[0] == '#': graph_add_node(w.word, graph, 'hashtag') graph_add_edge(t.tweet, w.word, graph) # you probably want to change this string to something meaningful too q = 'example' # because it ends up in the file name for the GEXF output file
from tweetsql.database import db_session import pickle from collections import Counter def pickleIt(data, fname): output = open(fname, 'wb') pickle.dump(data, output) output.close() try: f = open('user_tweet.p', 'rb') f.close() except: tweets = db_session.query(User.screen_name, Tweet.id, Tweet.tweet).join(Tweet.user).all() pickleIt(tweets, 'user_tweet.p') try: f = open('user_hashtag.p', 'rb') except: hashtags = db_session.query(User.screen_name, Hashtag.hashtag).join(Hashtag.users).all() pickleIt(hashtags, 'user_hashtag.p') try: f = open('all_user_friends.p', 'rb') except: users = db_session.query(User.id, Tweet.id).join(Tweet.user).all() print 'users retrieved' tweet_counter = Counter()
#find out who is a bot, and who is not from tweetsql.model import User from tweetsql.database import db_session from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC import time driver = webdriver.Firefox() driver.get("http://truthy.indiana.edu/botornot/") elem = driver.find_element_by_class_name("form-control") all_users = db_session.query(User).all() button = driver.find_element_by_id('btnGetTimeline') for user in all_users: screen_name = user.screen_name.encode('ascii', 'ignore') elem.send_keys(screen_name) button.click() # try: # print 'waiting' # element = WebDriverWait(driver, 30).until( # EC.presence_of_element_located((By.ID, 'span-power-readout')) # ) # finally: # driver.quit() # driver.implicitly_wait(10) time.sleep(10) print 'waiting some more? why...who knows?'
g.node[n]['weight'] = 1 g.node[n]['type'] = t def graph_add_edge(n1, n2, g): if g.has_edge(n1, n2): g[n1][n2]['weight'] += 1 else: g.add_edge(n1, n2) g[n1][n2]['weight'] = 1 graph = nx.Graph() # iterate through every tweet, storing each tweet in t for t in db_session.query(Tweet).all(): # add t to the graph graph_add_node(t.tweet, graph, 'tweet') # now iterate through all the words in t, storing each word in w for w in t.words: # looking for hashtags (there's a better way to do this) if w.word[0] == '#': graph_add_node(w.word, graph, 'hashtag') graph_add_edge(t.tweet, w.word, graph) # you probably want to change this string to something meaningful too q = 'example' # because it ends up in the file name for the GEXF output file nx.write_gexf(graph, '{}_tweet_graph.gexf'.format(q)) print('{}_tweet_graph.gexf'.format(q))
#set up twitter api clean_key = get_clean_key() twitter_auth = twitter.oauth.OAuth(clean_key.OAUTH_TOKEN, clean_key.OAUTH_TOKEN_SECRET, clean_key.CONSUMER_KEY, clean_key.CONSUMER_SECRET) api = twitter.Twitter(auth=twitter_auth) if get_rate_limit(t=api, data='remaining') == 0: print 'rate limit hit' print 'try again at %d' % get_rate_limit(t=api, data='reset') sys.exit() #let's get 15 screen names that are not in the Friend table got_friends = db_session.query(distinct( Friend.user_id)).all() #first get all users in the friend table got_friends = [t[0] for t in got_friends] dead_users = db_session.query(NoUser.user_id).all() dead_users = [t[0] for t in dead_users] all_users = db_session.query(User.id, User.uid).all() all_users = [(pk, uid) for pk, uid in all_users if pk not in dead_users] #filter out dead users no_friends = [(pk, uid) for pk, uid in all_users if pk not in got_friends] #get rid of people with friends # rate_limit = 15 # requests = 0 print no_friends[:15] for pk, uid in no_friends: cursor = -1 friends = []
for k in keys: twitter_auth = twitter.oauth.OAuth(k.OAUTH_TOKEN, k.OAUTH_TOKEN_SECRET, k.CONSUMER_KEY, k.CONSUMER_SECRET) api = twitter.Twitter(auth=twitter_auth) remaining = get_rate_limit(t=api) if remaining > maxRemaining: maxRemaining = remaining winner = k return winner #set up twitter api clean_key = get_clean_key() twitter_auth = twitter.oauth.OAuth(clean_key.OAUTH_TOKEN, clean_key.OAUTH_TOKEN_SECRET, clean_key.CONSUMER_KEY, clean_key.CONSUMER_SECRET) api = twitter.Twitter(auth=twitter_auth) got_data = db_session.query(UserData.id).all() #first get all users in the UseData table got_data = [t[0] for t in got_data] # all_users = db_session.query(User.id, User.uid).all() all_users = db_session.query(Friend.id, Friend.friend_id).all() no_data = [(pk,uid) for pk,uid in all_users if pk not in got_data] #group no_data by 100 for our twitter call no_data_100s = [no_data[i:i+99] for i in range(0,len(no_data),100)] if get_rate_limit(t=api)==0: print 'rate limit hit' print 'try again at %d' %get_rate_limit(t=api, data='reset') for l in no_data_100s:
g.add_node(n) g.node[n]['label'] = n g.node[n]['weight'] = 1 g.node[n]['type'] = t def graph_add_edge(n1, n2, g): if g.has_edge(n1, n2): g[n1][n2]['weight']+=1 else: g.add_edge(n1,n2) g[n1][n2]['weight']=1 graph = nx.Graph() # iterate through every tweet, storing each tweet in t for t in db_session.query(Tweet).all(): # add t to the graph graph_add_node(t.tweet, graph, 'tweet') # now iterate through all the words in t, storing each word in w for w in t.words: # looking for hashtags (there's a better way to do this) if w.word[0] == '#': graph_add_node(w.word, graph, 'hashtag') graph_add_edge(t.tweet, w.word, graph) # you probably want to change this string to something meaningful too q = 'example' # because it ends up in the file name for the GEXF output file nx.write_gexf(graph, '{}_tweet_graph.gexf'.format(q)) print('{}_tweet_graph.gexf'.format(q))
import json import nltk nltk.data.path.append('./tweetEasy/nltk_data/') #this may need to change depending on when from nltk.corpus import stopwords from tweetEasy.tweetEasy import ParseStatus from tweetsql.database import db_session from tweetsql.model import Hashtag, Tweet, Word, User from sqlalchemy.exc import OperationalError from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound stop = stopwords.words('english') tweets_users = db_session.query(Tweet, User).join(Tweet.user).all() for t, u in tweets_users: data = json.loads(t.data) search = ParseStatus(data) if(len(t.hashtags)==0): hashtags = search.hashtags for h in hashtags: try: h_obj = db_session.query(Hashtag).filter(Hashtag.hashtag == h).one() except MultipleResultsFound: pass except NoResultFound: h_obj = Hashtag(hashtag=h) db_session.add(h_obj) except OperationalError: print 'error' db_session.rollback() #add the relationship between h_obj and tweets t.hashtags.append(h_obj)
import json import nltk from nltk.tokenize import wordpunct_tokenize nltk.data.path.append('./tweetEasy/nltk_data/') #this may need to change depending on when from nltk.corpus import stopwords from tweetEasy.tweetEasy import ParseStatus from tweetsql.database import db_session from tweetsql.model import Hashtag, Tweet, Word, User from sqlalchemy.exc import OperationalError from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound stop = stopwords.words('english') tweets = db_session.query(Tweet).all() tweet_words = [t.id for t in db_session.query(Tweet).join(Tweet.words).all()] tweets = [t for t in tweets if t.id not in tweet_words] words = db_session.query(Word).all() all_words = [] tweet_word_dict = {} count = 0 for t in tweets: data = json.loads(t.data) search = ParseStatus(data) words = wordpunct_tokenize(search.tweetText()) print len(words) count += 1 print count # for w in words: # if w.lower() not in stop:
#this is all about words import re, nltk from string import punctuation, ascii_lowercase from itertools import permutations from patterns import * from nltk.tokenize import wordpunct_tokenize nltk.data.path.append('./tweetEasy/nltk_data/') #this may need to change depending on when from nltk.corpus import stopwords from collections import Counter from tweetsql.model import Tweet, Hashtag from tweetsql.database import db_session import pickle tweets = db_session.query(Tweet.tweet).all() db_hashtags = db_session.query(Hashtag.hashtag).all() print 'db query done' word_set = set() word_count = Counter() wordco = {} #word co-occcurence dictionary hashtag_set = set([ht[0].lower() for ht in db_hashtags]) hashtag_count = Counter() hashtagco = {} #hashtag co-occurence dictionary cap_words_count = Counter() acronyms_count = Counter() #helpers def makeDict(d, k, v):
import json import nltk nltk.data.path.append( './tweetEasy/nltk_data/') #this may need to change depending on when from nltk.corpus import stopwords from tweetEasy.tweetEasy import ParseStatus from tweetsql.database import db_session from tweetsql.model import Hashtag, Tweet, Word, User from sqlalchemy.exc import OperationalError from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound stop = stopwords.words('english') tweets_users = db_session.query(Tweet, User).join(Tweet.user).all() for t, u in tweets_users: data = json.loads(t.data) search = ParseStatus(data) if (len(t.hashtags) == 0): hashtags = search.hashtags for h in hashtags: try: h_obj = db_session.query(Hashtag).filter( Hashtag.hashtag == h).one() except MultipleResultsFound: pass except NoResultFound: h_obj = Hashtag(hashtag=h) db_session.add(h_obj) except OperationalError: print 'error' db_session.rollback()
remaining = get_rate_limit(t=api) if remaining > maxRemaining: maxRemaining = remaining winner = k return winner #set up twitter api clean_key = get_clean_key() twitter_auth = twitter.oauth.OAuth(clean_key.OAUTH_TOKEN, clean_key.OAUTH_TOKEN_SECRET, clean_key.CONSUMER_KEY, clean_key.CONSUMER_SECRET) api = twitter.Twitter(auth=twitter_auth) got_data = db_session.query( UserData.id).all() #first get all users in the UseData table got_data = [t[0] for t in got_data] # all_users = db_session.query(User.id, User.uid).all() all_users = db_session.query(Friend.id, Friend.friend_id).all() no_data = [(pk, uid) for pk, uid in all_users if pk not in got_data] #group no_data by 100 for our twitter call no_data_100s = [no_data[i:i + 99] for i in range(0, len(no_data), 100)] if get_rate_limit(t=api) == 0: print 'rate limit hit' print 'try again at %d' % get_rate_limit(t=api, data='reset') for l in no_data_100s: if get_rate_limit(t=api) == 0:
twitter_auth = twitter.oauth.OAuth(keys.lu.OAUTH_TOKEN, keys.lu.OAUTH_TOKEN_SECRET, keys.lu.CONSUMER_KEY, keys.lu.CONSUMER_SECRET) twitter_stream = twitter.TwitterStream(auth=twitter_auth) statuses = twitter_stream.statuses.filter(track=TRACK) for t in statuses: #add user try: print t['text'] except: print 'no text found' try: u = db_session.query(User).filter_by(uid=str(t['user']['id'])).one() except NoResultFound: u = User(screen_name=t['user']['screen_name'], uid=t['user']['id']) db_session.add(u) db_session.commit() print 'user committed' #add tweet tw = Tweet(tweet=t['text'], tid=t['id'], user_id=u.id, created_at=t['created_at'], data=json.dumps(t)) db_session.add(tw) db_session.commit() print 'tweet commited' #add hashtag search = ParseStatus(t)
CONSUMER_SECRET = '' OAUTH_TOKEN = '-' OAUTH_TOKEN_SECRET = '' twitter_auth = twitter.oauth.OAuth(OAUTH_TOKEN, OAUTH_TOKEN_SECRET, CONSUMER_KEY, CONSUMER_SECRET) twitter_stream = twitter.TwitterStream(auth=twitter_auth) statuses = twitter_stream.statuses.filter(track='Temple Mount, Palestine') for t in statuses: # print(t['text']) try: u = db_session.query(User).filter_by(uid=str(t['user']['id'])).one() except MultipleResultsFound: u = User(screen_name=t['user']['screen_name'], uid=t['user']['id']) pass except NoResultFound: u = User(screen_name=t['user']['screen_name'], uid=t['user']['id']) db_session.add(u) db_session.commit() if t['coordinates']: tw = Tweet(tweet=t['text'], tid=t['id'], user_id=u.id, created_at=t['created_at'], coordinates=t['coordinates']['coordinates']) else: tw = Tweet(tweet=t['text'], tid=t['id'], user_id=u.id, created_at=t['created_at']) try: words = tw.tweet.split()
import json import nltk from nltk.tokenize import wordpunct_tokenize nltk.data.path.append( './tweetEasy/nltk_data/') #this may need to change depending on when from nltk.corpus import stopwords from tweetEasy.tweetEasy import ParseStatus from tweetsql.database import db_session from tweetsql.model import Hashtag, Tweet, Word, User from sqlalchemy.exc import OperationalError from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound stop = stopwords.words('english') tweets = db_session.query(Tweet).all() tweet_words = [t.id for t in db_session.query(Tweet).join(Tweet.words).all()] tweets = [t for t in tweets if t.id not in tweet_words] words = db_session.query(Word).all() all_words = [] tweet_word_dict = {} count = 0 for t in tweets: data = json.loads(t.data) search = ParseStatus(data) words = wordpunct_tokenize(search.tweetText()) print len(words) count += 1 print count # for w in words:
#!/usr/bin/env python import re from sqlalchemy.sql import text from tweetsql.database import Base, db_session, engine from tweetsql.model import Tweet, User, Word all_media = [] hashtags = [] REGEX = '#(\w+)' re_hash = re.compile(r'#[0-9a-zA-Z+_]*',re.IGNORECASE); for t in db_session.query(Tweet)[0:10]: #.filter(text('tweet ~ :reg')).params(reg=REGEX)[0:10]: all_media.append(t.tweet) print t.created_at # for w in t.tweet.split(): # hashtag = re_hash.match(w) # if(hashtag): # hashtags.append(hashtag.string) print len(all_media)
return winner #set up twitter api clean_key = get_clean_key() twitter_auth = twitter.oauth.OAuth(clean_key.OAUTH_TOKEN, clean_key.OAUTH_TOKEN_SECRET, clean_key.CONSUMER_KEY, clean_key.CONSUMER_SECRET) api = twitter.Twitter(auth=twitter_auth) if get_rate_limit(t=api, data='remaining')==0: print 'rate limit hit' print 'try again at %d' %get_rate_limit(t=api, data='reset') sys.exit() #let's get 15 screen names that are not in the Friend table got_friends = db_session.query(distinct(Friend.user_id)).all() #first get all users in the friend table got_friends = [t[0] for t in got_friends] dead_users = db_session.query(NoUser.user_id).all() dead_users = [t[0] for t in dead_users] all_users = db_session.query(User.id, User.uid).all() all_users = [(pk,uid) for pk,uid in all_users if pk not in dead_users] #filter out dead users no_friends = [(pk,uid) for pk,uid in all_users if pk not in got_friends] #get rid of people with friends # rate_limit = 15 # requests = 0 print no_friends[:15] for pk,uid in no_friends: cursor = -1 friends = [] if get_rate_limit(t=api, data='remaining')==0: print 'rate limit hit'