def create_multi_graph_of_list_memebers_and_followers(api, list):
"""Creates a multi directed graph where nodes are all members of a given list and their followers and edges(weight 1) are
added if a user follows another one on twitter.
Edges (weight 2) are added if one user replied to another one on twitter.
:parameter api
:parameter list
:return mDG - multi directed graph
"""
debug_print("EXEC - create_multi_graph_of_list_memebers_and_followers method :")
criteria = {"in_reply_to_user_id": {"$ne": None}, "$where": "this.user.id != this.in_reply_to_user_id"}
projection = {"id": 1, "user.screen_name": 1, "in_reply_to_screen_name": 1, "in_reply_to_user_id": 1, "text": 1}
tweets = load_from_mongo(mongo_db="twitter", mongo_db_coll=list, criteria=criteria, projection=projection)
#create a multi directed graph - one directions for reply / one for following
mdG = nx.MultiDiGraph()
db_coll_name = "%s_%s" % (list, "members")
members = load_from_mongo(mongo_db="twitter", mongo_db_coll=db_coll_name)
for member in members[:5]: # for member in members[:10]: can be used for testing
followers = twitter_get_followers(api, screen_name=member['screen_name'],followers_limit=400)
print member['screen_name'],len(followers)
#create network of following
mdG = create_keyplayers_graph(graph=mdG, user=member, followers=followers, weight=1)
#create network of conversation/replying
i = 0
for tweet in tweets:
if mdG.has_node(tweet['id']) and mdG.has_node(tweet['in_reply_to_user_id']):
debug_print("%d) user: %s \n in reply to: %s \n text: %s" % (i,tweet['user']['screen_name'],tweet['in_reply_to_screen_name'],tweet['text']))
i += 1
mdG.add_edge(tweet['id'], tweet['in_reply_to_user_id'],weight = 2)
return mdG
def create_directed_graph_of_list_members(list):
"""Creates a directed graph where nodes are all members of a given list and edges are
added if a member has replied to another one on twitter
:parameter list
:return dG - directed graph
"""
debug_print("EXEC - create_directed_graph_of_list_members method :")
criteria = {"in_reply_to_user_id": {"$ne": None}, "$where": "this.user.id != this.in_reply_to_user_id"}
projection = {"id": 1, "user.screen_name": 1, "in_reply_to_screen_name": 1, "in_reply_to_user_id": 1, "text": 1}
# tweets that aren't a reply to the same user's tweets
results = load_from_mongo(mongo_db="twitter", mongo_db_coll=list, criteria=criteria, projection=projection)
debug_print(" num of edges/results: %d" % len(results))
dG = nx.DiGraph()
db_coll_name = "%s_%s" % (list, "members") #name of collection for members of list
members = load_from_mongo(mongo_db="twitter", mongo_db_coll=db_coll_name)
debug_print(" num of members: %d" % len(members))
for member in members:
dG.add_node(member['id'], screen_name=member['screen_name'], location=member['location'],
followers_count=member['followers_count'], statuses_count=member['followers_count'],
friends_count=member['friends_count'], created_at=member['created_at'])
i = 0
for result in results:
if dG.has_node(result['id']) and dG.has_node(result['in_reply_to_user_id']):
debug_print("%d) user: %s \n in reply to: %s \n text: %s" % (i,result['user']['screen_name'],result['in_reply_to_screen_name'],result['text']))
i += 1
dG.add_edge(result['id'], result['in_reply_to_user_id'])
return dG
def twitter_search(twitter_api, q, max_results=1000, **kw):
"""
Retrieves tweets from the api for a given query and saves them into mongo database
description of parametars: https://dev.twitter.com/docs/api/1.1/get/search/tweets
Warning: OAuth users can "only" make 180 search queries per 15-minute interval.
For preventing HTTP errors a robust API wrapper is added
:param twitter_api
:param q
:param max_results
"""
debug_print('EXEC twitter_search method : ')
#Get a collection of relevant Tweets matching a specified query
try:
twitter_search_api_tweets = partial(twitter_api.search.tweets, q=q, count=180, **kw)
search_results = twitter_make_robust_request(twitter_search_api_tweets)
#Handle rate limit
except urllib2.HTTPError, e:
if e.e.code == 429 : #rate limit reached TODO: handle this error inside methods
#find the highest since_id from database to continue if a rate limitation is reached
since_id = load_from_mongo('twitter', q, return_cursor=False, find_since_id=True)
if since_id:
debug_print(" since_id: %i" % since_id)
kw = {'since_id': since_id}
else:
debug_print(" No since_id")
debug_print(" Rate limit reached. Start: %s . Retrying in 15 min ...zZz..." % str(time.ctime()))
sys.stderr.flush()
time.sleep(60*15 + 10)
debug_print(" Woke up ... End: %s " % str(time.ctime()))
twitter_search(twitter_api, q, **kw)
def get_users_for_hashtag_list(hashtags):
"""
Counts the number of users that used the hashtags from the list in their posts
:param hashtags - list of hashtags
:return: users_per_hashtag - dict where key the hastag and value number of users that have included it in their posts
"""
debug_print("EXEC get_users_for_hashtag_list method :")
users_per_hashtag = {}
for hashtag in hashtags:
criteria = {"entities.hashtags.text": "%s" % hashtag}
projection = {"user.screen_name": 1, "_id": 0}
results = load_from_mongo(mongo_db="twitter",
mongo_db_coll="community-councils",
criteria=criteria,
projection=projection)
unique_users = set()
for user in results:
#print user["user"]["screen_name"]
unique_users.add(user["user"]["screen_name"])
#print hashtag
#print len(unique_users)
users_per_hashtag.update({hashtag: unique_users})
return users_per_hashtag
def create_directed_graph_of_list_members_and_followers(api, list):
"""Creates a directed graph where nodes are all members of a given list and edges are
added if a member who follows another one on twitter.
:parameter api
:parameter list
:return dG - directed graph
"""
debug_print("EXEC - create_directed_graph_of_list_members_and_followers method :")
criteria = {"in_reply_to_user_id": {"$ne": None}, "$where": "this.user.id != this.in_reply_to_user_id"}
projection = {"id": 1, "user.screen_name": 1, "in_reply_to_screen_name": 1, "in_reply_to_user_id": 1, "text": 1}
tweets = load_from_mongo(mongo_db="twitter", mongo_db_coll=list, criteria=criteria, projection=projection)
#create a multi directed graph - one directions for reply / one for following
dG = nx.DiGraph()
db_coll_name = "%s_%s" % (list, "members")
members = load_from_mongo(mongo_db="twitter", mongo_db_coll=db_coll_name)
for member in members:
followers = twitter_get_followers(api, screen_name=member['screen_name'],followers_limit=400)
#print member['screen_name'],len(followers)
#create network of following
dG = create_keyplayers_graph(graph=dG, user=member, followers=followers, weight=1)
return dG
def twitter_get_list_members_tweets(twitter_api,
max_results=1000,
owner_screen_name="",
slug=""):
"""Gets all tweets from twitter api posted by list members and stores them into database
:parameter: twitter_api
:parameter: max_results
:parameter: owner_screen_name - name of the creator of the list
:parameter: slug - the name of the list
"""
debug_print("EXEC twitter_get_list_members method :")
#get id of last tweet from mongo
since_id = load_from_mongo(mongo_db="twitter",
mongo_db_coll=slug,
find_since_id=True)
if since_id is None:
since_id = 1
debug_print(" Getting since_id from db: " + str(since_id))
kw = {'count': 100, 'since_id': since_id, 'include_rts': False}
#get all statuses from list members
response = twitter_api.lists.statuses(owner_screen_name=owner_screen_name,
slug=slug,
**kw)
#401 (Not Authorized) - Need to bail out on loop entry
if response is None:
response = []
debug_print(" First response len : " + str(len(response)))
results = response
page_num = 1
max_pages = 16
if max_results == kw['count']:
page_num = max_pages # Prevent loop entry
#find all new tweets after the first response
while page_num < max_pages and len(response) > 0 and len(
results) < max_results:
kw['max_id'] = min([tweet['id'] for tweet in response]) - 1
response = twitter_api.lists.statuses(
owner_screen_name=owner_screen_name, slug=slug, **kw)
results += response
#debug_print(" All results len : "+str(len(results)) + " | New response len : "+str(len(response)))
page_num += 1
#Saving all results to database
twitter_save_to_mongo(results, "twitter", slug)
def facebook_get_likes_count(page_name):
"""
Reads from database the number of likes for a page
:param page_name:
:return: number of likes
"""
debug_print("EXEC facebook_get_likes_count method :")
if page_name is not None:
criteria = {"name": "%s" % page_name}
projection = {"likes": 1, "_id": 0}
result = load_from_mongo(mongo_db="facebook",mongo_db_coll="pages_info", criteria=criteria,projection=projection)
return result[0]["likes"]
def facebook_get_talkingabout_count(page_name):
"""
Reads from database the number of the parameter talking about(facebook) for a page
:param page_name:
:return: talking_about_count - number of people talking about your page
"""
debug_print("EXEC facebook_get_talkingabout_count method :")
if page_name is not None:
criteria = {"name": "%s" % page_name}
projection = {"talking_about_count": 1, "_id": 0}
result = load_from_mongo(mongo_db="facebook",mongo_db_coll="pages_info",criteria=criteria,projection=projection)
talking_about_count = result[0]["talking_about_count"]
return talking_about_count
def facebook_get_likes_count(page_name):
"""
Reads from database the number of likes for a page
:param page_name:
:return: number of likes
"""
debug_print("EXEC facebook_get_likes_count method :")
if page_name is not None:
criteria = {"name": "%s" % page_name}
projection = {"likes": 1, "_id": 0}
result = load_from_mongo(mongo_db="facebook",
mongo_db_coll="pages_info",
criteria=criteria,
projection=projection)
return result[0]["likes"]
def facebook_get_talkingabout_count(page_name):
"""
Reads from database the number of the parameter talking about(facebook) for a page
:param page_name:
:return: talking_about_count - number of people talking about your page
"""
debug_print("EXEC facebook_get_talkingabout_count method :")
if page_name is not None:
criteria = {"name": "%s" % page_name}
projection = {"talking_about_count": 1, "_id": 0}
result = load_from_mongo(mongo_db="facebook",
mongo_db_coll="pages_info",
criteria=criteria,
projection=projection)
talking_about_count = result[0]["talking_about_count"]
return talking_about_count
def get_popular_hashtags(slug):
"""
Returns popular hastags for a certain list of users
:parameter: slug - slug for the list from twitter
:returns: hashtags - dictionary where key is the hashtag and value number of occurrences
"""
debug_print("EXEC get_popular_hashtags method :")
mongo_db = "twitter"
mongo_db_coll = slug
results = load_from_mongo(mongo_db=mongo_db, mongo_db_coll=mongo_db_coll)
screen_names, hashtags, urls, media, symbols = get_popular_tweet_entities_list(
results, 25)
#hashtags = get_popular_tweet_entities_list(entity_threshold= 25)
hashtags_dict = {}
for (k, v) in hashtags:
print k, " : ", v
hashtags_dict.update({k: v})
return hashtags_dict
since_id = load_from_mongo('twitter', q, return_cursor=False, find_since_id=True)
if since_id:
debug_print(" since_id: %i" % since_id)
kw = {'since_id': since_id}
else:
debug_print(" No since_id")
debug_print(" Rate limit reached. Start: %s . Retrying in 15 min ...zZz..." % str(time.ctime()))
sys.stderr.flush()
time.sleep(60*15 + 10)
debug_print(" Woke up ... End: %s " % str(time.ctime()))
twitter_search(twitter_api, q, **kw)
#handle socket error
except SocketError, se:
logger.error(se)
debug_print(" " + se.message)
since_id = load_from_mongo('twitter', q, return_cursor=False, find_since_id=True)
debug_print(" SocketError occurred. Start:" + str(time.ctime()) + " . Retrying in 0.05 sec ...zZz...")
time.sleep(0.05)
debug_print(" Woke up ... End: " + str(time.ctime()))
if since_id:
kw = {'since_id': since_id} # continue where you left off
twitter_search(twitter_api, q, **kw)
statuses = search_results['statuses']
debug_print(" number of statuses: %i max_limit: %i" % (len(statuses), max_results))
# reach the desired number of results, keeping in mind that OAuth users
# can "only" make 180 search queries per 15-minute interval. See
# https://dev.twitter.com/docs/rate-limiting/1.1/limits
# for details. A reasonable number of results is ~1000, although
# that number of results may not exist for all queries.
def facebook_menu():
access_token = facebook_authorize()
action = None
while not action:
print "Type the number of the action you want to be executed: "
print "0. Save posts for each facebook page read from excel file"
print "1. Print sorted pages by number of likes and talking about parameter "
print "2. Save data for each page from list "
print "3. Get likes per page"
print "4. Get talking about count per page"
print "5. Get facebook posts sorted"
print "6. Save posts for one page"
print "7. Get date from most recent post of a page"
print "8. Create excel file with statistcs for facebook pages"
print "9. Print Facebook insights for a page"
print "10.Get number of posts for July "
action = raw_input('Enter the number of the action: ').strip()
if action == '0' or action == '1' :
pages = facebook_read_pages_from_excel(access_token) # get pages and basic data for pages
pages_in_database = getCollections("facebook")
if action == '0':
for page in pages:
# check if page info has already been saved in database
page_data = facebook_get_page_data(access_token, page)
if page_data:
if page_data['name'] not in pages_in_database: #store posts only for new pages
results = facebook_get_page_posts(access_token, page_data["id"]) #send id of page and access token to get posts
save_to_mongo(mongo_db="facebook", mongo_db_coll=page_data['name'], data=results)
if action == '1':
sorted_pages = facebook_sort_pages(pages, "DESC")
facebook_print_page_data(sorted_pages)
# if action == '10':
# for page in pages:
# page_data = facebook_get_page_data(access_token, page)
# if page_data:
# results = facebook_get_page_posts(access_token, page_data["id"]) #send id of page and access token to get posts
# facebook_save_to_mongo(mongo_db="facebook", mongo_db_coll=page_data['name'], data=results)
elif action == '2':
pages = facebook_read_pages_from_excel(access_token)
#get all pages from database
pages_data_in_database = load_from_mongo(mongo_db="facebook", mongo_db_coll="pages_info",projection={"_id":0,'name':1})
print "database: ",len(pages_data_in_database)
print pages_data_in_database[0]
i=0
for page in pages:
data = facebook_get_page_data(access_token, page)
if data and data['name'] not in pages_data_in_database: # check if page is in database
save_to_mongo(mongo_db="facebook", mongo_db_coll="pages_info", data=data)
elif action == '3':
page = "Connel Community Council" #sample page
print page, facebook_get_likes_count(page)
elif action == '4':
page = "Connel Community Council" #sample page
print page, facebook_get_talkingabout_count(page)
elif action == '5':
posts = facebook_get_posts_from_database(from_user="******", page_name="Connel Community Council")
for post in posts:
print post
print "\n \n _________________________________________________________________________ \n"
#break
elif action == '6':
page = "187281984718895" # sample id
data = facebook_get_page_posts(access_token, page, limit=500)
debug_print("posts from api :%i" % len(data))
save_to_mongo(mongo_db="facebook", mongo_db_coll="Connel Community Council", data=data)
elif action == '7':
page = "187281984718895" # sample id
result = facebook_get_posts_from_database(from_user="******", page_name="Connel Community Council", limit=1)
print result[0]['created_time']
elif action == '8':
#print(facebook_path_to_EXPORT_FILE)
workbook = xlwt.Workbook(encoding="utf-8") # XLWT - ONLY EXPORTS .XLS files
sheet = workbook.add_sheet("sheet1")
sheet.write(0, 0, 'FB Page ID ')
sheet.write(0, 1, 'CC Name')
sheet.write(0, 2, 'Count Likes')
sheet.write(0, 3, 'Count Talking about')
sheet.write(0, 4, 'Count Posts for July')
sheet.write(0, 5, 'Count Posts TOTAL')
sheet.write(0, 6, 'Date Most recent post')
sheet.write(0, 7, 'Date 5th Most recent post')
pages_data = load_from_mongo(mongo_db="facebook", mongo_db_coll="pages_info") # collection where all
# the data for each page is stored
pages_data = facebook_sort_pages(pages_data, "DESC")
i = 1 # index for rows in sheet
for page in pages_data:
sheet.write(i, 0, page['id'])
sheet.write(i, 1, page['name'])
sheet.write(i, 2, page['likes'])
sheet.write(i, 3, page['talking_about_count'])
most_recent_post = 0
fifth_most_recent_post = 0
result = facebook_get_posts_from_database(from_user=page['name'], page_name=page['name'], limit=5)
if result:
most_recent_post = result[0]['created_time']
fifth_most_recent_post = result[len(result)-1]['created_time'] # some pages have less than 5 posts
sheet.write(i, 6, most_recent_post)
sheet.write(i, 7, fifth_most_recent_post)
monthly_posts = len(facebook_get_posts_for_month(page['name'], month="07"))
if monthly_posts is None:
monthly_posts = 0
sheet.write(i, 4, monthly_posts)
total_posts = 0
result = facebook_get_posts_from_database(from_user=page['name'], page_name=page['name'])
if result:
total_posts = len(result)
sheet.write(i, 5, total_posts)
i += 1
workbook.save(facebook_path_to_EXPORT_FILE)
elif action == '9':
page_id = "173685876111989" # sample id
facebook_print_page_insights(access_token, page_id)
elif action == '10':
page_id = "173685876111989"
page_data = facebook_get_page_data(access_token, page_id)
if page_data:
monthly_posts = facebook_get_posts_for_month(page_data['name'], month="07")
print page_data['name'], " : ", len(monthly_posts)
for post in monthly_posts:
print post
def twitter_menu():
api = twitter_authorize()
if api:
debug_print("Successfully authenticated and authorized")
action = None
while not action:
print "Type the number of the action you want to be executed: "
print "0. Find users who have tweeted a hashtag , for a list of popular hashtags"
print "1. Get only hashtags from results"
print "2. Create & export graph of members of twitter list and their followers"
print "3. Get statuses of list members"
print "4. Find the trending topics in the world"
print "5. Call function for saving tweets from list members on interval( predefined for community-council)"
print "6. Get info about list members from api (json)"
print "7. Search & save tweets for a specific query"
print "8. Search & save tweets from the streaming api"
print "9. Get all tweets from a user's timeline"
print "10. Print common entities"
print "11. Get list members"
print "12. MULTIGRAPH"
print "13. find popular tweets from list of tweets"
print "14. Get twitter info about a specific user"
print "15. Get date for last tweet"
print "16. Get date for fifth tweet"
print "17. Create excel file with statistcs for members of twitter list"
action = raw_input('Enter the number of the action: ').strip()
WORLD_WOE_ID = 1 # for searching trends
if action == '0': # find users who have tweeted a hashtag , for a list of popular hashtags
hashtags_dict = get_popular_hashtags(slug="community-councils")
hashtags = hashtags_dict.keys()
for hashtag in hashtags:
print hashtag
dict = get_users_for_hashtag_list(hashtags_dict.keys())
elif action == '1': # get only hashtags from results
results = load_from_mongo(mongo_db="twitter", mongo_db_coll="community-councils")
hashtags = twitter_get_hashtags_from_tweets(results)
print "num of hashtags: ", len(hashtags)
for hashtag in hashtags:
print "#", hashtag
elif action == '2': # create & export graph of members of twitter list and their followers
#create directed graph of members of a list and their followers
#save list of followers for each member into a separate collection named after the slug ending with _member
slug="community-councils"
graph = create_directed_graph_of_list_members_and_followers(api, slug)
export_graph_to_gml(graph,path_to_graph_file)
elif action == '3': # get statuses of list members
twitter_get_list_members_tweets(api, owner_screen_name="spartakan", slug="community-councils")
elif action == '4': # Get trending topics
print "INFO: Getting World Trends ..."
world_trends = twitter_trends(api, WORLD_WOE_ID)
#for checking the structure of uncomment : #print json.dumps(world_trends[0]['trends'], indent=1)
if world_trends:
world_trends = world_trends[0]['trends']
for w in world_trends:
print "trend: ", w['name']
elif action == '5': # find trending topics on a time interval
#making a partial class from twitter_search to later add it as an argument in twitter_call_function_on_interval
tweets_from_list_members = partial(twitter_get_list_members_tweets, api, owner_screen_name="spartakan", slug="community-councils")
#get and save the trending topics on time intervals
twitter_call_function_on_interval(tweets_from_list_members)
elif action == '6': # get members of a list
twitter_get_list_members(api,owner_screen_name="spartakan", slug="community-councils")
elif action == '7' or action == '8':
q = None
print "Read How to build a query first ! ( https://dev.twitter.com/docs/using-search ) "
q = raw_input('Enter a query: ').strip()
while not q:
q = raw_input('Enter a query:').strip()
if action == '7': # Fetch tweets from the search/rest api
debug_print("Searching tweets for the query:" + q)
#twitter_call_function_on_interval(search_tweets, 'twitter', q)
results = twitter_search(api, q, 1000)
debug_print("Tweets saved into database")
if action == '8': # Fetch tweets from streaming api
debug_print("Searching and saving tweets from the streaming api for query: " +q+ "...")
twitter_stream_api(api, q)
elif action == '9': # Get all tweets from a user's timeline
# screen_name = raw_input('Enter the screen name: ').strip()
# while not screen_name:
# screen_name = raw_input('Enter a query:').strip()
# debug_print("Getting tweets from user: "******" ... ")
screen_name = "spartakan"
tweets = twitter_user_timeline(api, screen_name=screen_name)
elif action == '10':
results = load_from_mongo("twitter", "#indyref")
common_entities = get_common_tweet_entities(results)
print_prettytable(common_entities)
elif action == '11': # get list members
twitter_get_list_members(api,owner_screen_name="spartakan", slug="community-councils")
elif action == '12':
print "MULTIGRAPH"
slug = "community-councils"
mdG = create_multi_graph_of_list_memebers_and_followers(api,slug)
name_of_file = "twitter_multigraph.gml"
export_graph_to_gml(mdG, path_to_graph_file + name_of_file)
elif action == '13': # find popular tweets from list of tweets
statuses = load_from_mongo("twitter", "community-councils")
find_popular_tweets(statuses=statuses)
elif action == '14': # get info for user
screen_name = "spartakan"
user = twitter_get_user_info(api, screen_name)
debug_print(json.dumps(user, indent=1))
elif action == '15':
screen_name = "spartakan"
print twitter_date_of_last_tweet(screen_name)
elif action == '16':
screen_name = "spartakan"
print twitter_date_of_fifth_tweet(screen_name)
elif action == '17':
#print(facebook_path_to_EXPORT_FILE)
workbook = xlwt.Workbook(encoding="utf-8") # XLWT - ONLY EXPORTS .XLS files
sheet = workbook.add_sheet("sheet1")
sheet.write(0, 0, 'ID ')
sheet.write(0, 1, 'CC screen_name')
sheet.write(0, 2, 'Count Followers')
sheet.write(0, 3, 'Count Friends')
sheet.write(0, 4, 'Count Posts TOTAL')
sheet.write(0, 5, 'Date Most recent post')
sheet.write(0, 6, 'Date 5th Most recent post')
members = load_from_mongo(mongo_db="twitter", mongo_db_coll="community-councils_members") # collection where all
# the data for each page is stored
pages_data = facebook_sort_pages(members, "DESC")
i = 1 # index for rows in sheet
for member in members:
sheet.write(i, 0, member['id'])
sheet.write(i, 1, member['screen_name'])
sheet.write(i, 2, member['followers_count'])
sheet.write(i, 3, member['friends_count'])
most_recent_post = twitter_date_of_last_tweet(member['screen_name'],"community-councils")
fifth_most_recent_post = twitter_date_of_fifth_tweet(member['screen_name'],"community-councils")
sheet.write(i, 5, most_recent_post)
sheet.write(i, 6, fifth_most_recent_post)
#total
sheet.write(i, 4, member["statuses_count"])
i += 1
workbook.save(twitter_path_to_EXPORT_FILE)
else:
print "WRONG ACTION!!!"
else:
print "You are not authorized!!!"