def create_multi_graph_of_list_memebers_and_followers(api, list): """Creates a multi directed graph where nodes are all members of a given list and their followers and edges(weight 1) are added if a user follows another one on twitter. Edges (weight 2) are added if one user replied to another one on twitter. :parameter api :parameter list :return mDG - multi directed graph """ debug_print("EXEC - create_multi_graph_of_list_memebers_and_followers method :") criteria = {"in_reply_to_user_id": {"$ne": None}, "$where": "this.user.id != this.in_reply_to_user_id"} projection = {"id": 1, "user.screen_name": 1, "in_reply_to_screen_name": 1, "in_reply_to_user_id": 1, "text": 1} tweets = load_from_mongo(mongo_db="twitter", mongo_db_coll=list, criteria=criteria, projection=projection) #create a multi directed graph - one directions for reply / one for following mdG = nx.MultiDiGraph() db_coll_name = "%s_%s" % (list, "members") members = load_from_mongo(mongo_db="twitter", mongo_db_coll=db_coll_name) for member in members[:5]: # for member in members[:10]: can be used for testing followers = twitter_get_followers(api, screen_name=member['screen_name'],followers_limit=400) print member['screen_name'],len(followers) #create network of following mdG = create_keyplayers_graph(graph=mdG, user=member, followers=followers, weight=1) #create network of conversation/replying i = 0 for tweet in tweets: if mdG.has_node(tweet['id']) and mdG.has_node(tweet['in_reply_to_user_id']): debug_print("%d) user: %s \n in reply to: %s \n text: %s" % (i,tweet['user']['screen_name'],tweet['in_reply_to_screen_name'],tweet['text'])) i += 1 mdG.add_edge(tweet['id'], tweet['in_reply_to_user_id'],weight = 2) return mdG
def create_directed_graph_of_list_members(list): """Creates a directed graph where nodes are all members of a given list and edges are added if a member has replied to another one on twitter :parameter list :return dG - directed graph """ debug_print("EXEC - create_directed_graph_of_list_members method :") criteria = {"in_reply_to_user_id": {"$ne": None}, "$where": "this.user.id != this.in_reply_to_user_id"} projection = {"id": 1, "user.screen_name": 1, "in_reply_to_screen_name": 1, "in_reply_to_user_id": 1, "text": 1} # tweets that aren't a reply to the same user's tweets results = load_from_mongo(mongo_db="twitter", mongo_db_coll=list, criteria=criteria, projection=projection) debug_print(" num of edges/results: %d" % len(results)) dG = nx.DiGraph() db_coll_name = "%s_%s" % (list, "members") #name of collection for members of list members = load_from_mongo(mongo_db="twitter", mongo_db_coll=db_coll_name) debug_print(" num of members: %d" % len(members)) for member in members: dG.add_node(member['id'], screen_name=member['screen_name'], location=member['location'], followers_count=member['followers_count'], statuses_count=member['followers_count'], friends_count=member['friends_count'], created_at=member['created_at']) i = 0 for result in results: if dG.has_node(result['id']) and dG.has_node(result['in_reply_to_user_id']): debug_print("%d) user: %s \n in reply to: %s \n text: %s" % (i,result['user']['screen_name'],result['in_reply_to_screen_name'],result['text'])) i += 1 dG.add_edge(result['id'], result['in_reply_to_user_id']) return dG
def twitter_search(twitter_api, q, max_results=1000, **kw): """ Retrieves tweets from the api for a given query and saves them into mongo database description of parametars: https://dev.twitter.com/docs/api/1.1/get/search/tweets Warning: OAuth users can "only" make 180 search queries per 15-minute interval. For preventing HTTP errors a robust API wrapper is added :param twitter_api :param q :param max_results """ debug_print('EXEC twitter_search method : ') #Get a collection of relevant Tweets matching a specified query try: twitter_search_api_tweets = partial(twitter_api.search.tweets, q=q, count=180, **kw) search_results = twitter_make_robust_request(twitter_search_api_tweets) #Handle rate limit except urllib2.HTTPError, e: if e.e.code == 429 : #rate limit reached TODO: handle this error inside methods #find the highest since_id from database to continue if a rate limitation is reached since_id = load_from_mongo('twitter', q, return_cursor=False, find_since_id=True) if since_id: debug_print(" since_id: %i" % since_id) kw = {'since_id': since_id} else: debug_print(" No since_id") debug_print(" Rate limit reached. Start: %s . Retrying in 15 min ...zZz..." % str(time.ctime())) sys.stderr.flush() time.sleep(60*15 + 10) debug_print(" Woke up ... End: %s " % str(time.ctime())) twitter_search(twitter_api, q, **kw)
def get_users_for_hashtag_list(hashtags): """ Counts the number of users that used the hashtags from the list in their posts :param hashtags - list of hashtags :return: users_per_hashtag - dict where key the hastag and value number of users that have included it in their posts """ debug_print("EXEC get_users_for_hashtag_list method :") users_per_hashtag = {} for hashtag in hashtags: criteria = {"entities.hashtags.text": "%s" % hashtag} projection = {"user.screen_name": 1, "_id": 0} results = load_from_mongo(mongo_db="twitter", mongo_db_coll="community-councils", criteria=criteria, projection=projection) unique_users = set() for user in results: #print user["user"]["screen_name"] unique_users.add(user["user"]["screen_name"]) #print hashtag #print len(unique_users) users_per_hashtag.update({hashtag: unique_users}) return users_per_hashtag
def create_directed_graph_of_list_members_and_followers(api, list): """Creates a directed graph where nodes are all members of a given list and edges are added if a member who follows another one on twitter. :parameter api :parameter list :return dG - directed graph """ debug_print("EXEC - create_directed_graph_of_list_members_and_followers method :") criteria = {"in_reply_to_user_id": {"$ne": None}, "$where": "this.user.id != this.in_reply_to_user_id"} projection = {"id": 1, "user.screen_name": 1, "in_reply_to_screen_name": 1, "in_reply_to_user_id": 1, "text": 1} tweets = load_from_mongo(mongo_db="twitter", mongo_db_coll=list, criteria=criteria, projection=projection) #create a multi directed graph - one directions for reply / one for following dG = nx.DiGraph() db_coll_name = "%s_%s" % (list, "members") members = load_from_mongo(mongo_db="twitter", mongo_db_coll=db_coll_name) for member in members: followers = twitter_get_followers(api, screen_name=member['screen_name'],followers_limit=400) #print member['screen_name'],len(followers) #create network of following dG = create_keyplayers_graph(graph=dG, user=member, followers=followers, weight=1) return dG
def twitter_get_list_members_tweets(twitter_api, max_results=1000, owner_screen_name="", slug=""): """Gets all tweets from twitter api posted by list members and stores them into database :parameter: twitter_api :parameter: max_results :parameter: owner_screen_name - name of the creator of the list :parameter: slug - the name of the list """ debug_print("EXEC twitter_get_list_members method :") #get id of last tweet from mongo since_id = load_from_mongo(mongo_db="twitter", mongo_db_coll=slug, find_since_id=True) if since_id is None: since_id = 1 debug_print(" Getting since_id from db: " + str(since_id)) kw = {'count': 100, 'since_id': since_id, 'include_rts': False} #get all statuses from list members response = twitter_api.lists.statuses(owner_screen_name=owner_screen_name, slug=slug, **kw) #401 (Not Authorized) - Need to bail out on loop entry if response is None: response = [] debug_print(" First response len : " + str(len(response))) results = response page_num = 1 max_pages = 16 if max_results == kw['count']: page_num = max_pages # Prevent loop entry #find all new tweets after the first response while page_num < max_pages and len(response) > 0 and len( results) < max_results: kw['max_id'] = min([tweet['id'] for tweet in response]) - 1 response = twitter_api.lists.statuses( owner_screen_name=owner_screen_name, slug=slug, **kw) results += response #debug_print(" All results len : "+str(len(results)) + " | New response len : "+str(len(response))) page_num += 1 #Saving all results to database twitter_save_to_mongo(results, "twitter", slug)
def facebook_get_likes_count(page_name): """ Reads from database the number of likes for a page :param page_name: :return: number of likes """ debug_print("EXEC facebook_get_likes_count method :") if page_name is not None: criteria = {"name": "%s" % page_name} projection = {"likes": 1, "_id": 0} result = load_from_mongo(mongo_db="facebook",mongo_db_coll="pages_info", criteria=criteria,projection=projection) return result[0]["likes"]
def facebook_get_talkingabout_count(page_name): """ Reads from database the number of the parameter talking about(facebook) for a page :param page_name: :return: talking_about_count - number of people talking about your page """ debug_print("EXEC facebook_get_talkingabout_count method :") if page_name is not None: criteria = {"name": "%s" % page_name} projection = {"talking_about_count": 1, "_id": 0} result = load_from_mongo(mongo_db="facebook",mongo_db_coll="pages_info",criteria=criteria,projection=projection) talking_about_count = result[0]["talking_about_count"] return talking_about_count
def facebook_get_likes_count(page_name): """ Reads from database the number of likes for a page :param page_name: :return: number of likes """ debug_print("EXEC facebook_get_likes_count method :") if page_name is not None: criteria = {"name": "%s" % page_name} projection = {"likes": 1, "_id": 0} result = load_from_mongo(mongo_db="facebook", mongo_db_coll="pages_info", criteria=criteria, projection=projection) return result[0]["likes"]
def facebook_get_talkingabout_count(page_name): """ Reads from database the number of the parameter talking about(facebook) for a page :param page_name: :return: talking_about_count - number of people talking about your page """ debug_print("EXEC facebook_get_talkingabout_count method :") if page_name is not None: criteria = {"name": "%s" % page_name} projection = {"talking_about_count": 1, "_id": 0} result = load_from_mongo(mongo_db="facebook", mongo_db_coll="pages_info", criteria=criteria, projection=projection) talking_about_count = result[0]["talking_about_count"] return talking_about_count
def get_popular_hashtags(slug): """ Returns popular hastags for a certain list of users :parameter: slug - slug for the list from twitter :returns: hashtags - dictionary where key is the hashtag and value number of occurrences """ debug_print("EXEC get_popular_hashtags method :") mongo_db = "twitter" mongo_db_coll = slug results = load_from_mongo(mongo_db=mongo_db, mongo_db_coll=mongo_db_coll) screen_names, hashtags, urls, media, symbols = get_popular_tweet_entities_list( results, 25) #hashtags = get_popular_tweet_entities_list(entity_threshold= 25) hashtags_dict = {} for (k, v) in hashtags: print k, " : ", v hashtags_dict.update({k: v}) return hashtags_dict
since_id = load_from_mongo('twitter', q, return_cursor=False, find_since_id=True) if since_id: debug_print(" since_id: %i" % since_id) kw = {'since_id': since_id} else: debug_print(" No since_id") debug_print(" Rate limit reached. Start: %s . Retrying in 15 min ...zZz..." % str(time.ctime())) sys.stderr.flush() time.sleep(60*15 + 10) debug_print(" Woke up ... End: %s " % str(time.ctime())) twitter_search(twitter_api, q, **kw) #handle socket error except SocketError, se: logger.error(se) debug_print(" " + se.message) since_id = load_from_mongo('twitter', q, return_cursor=False, find_since_id=True) debug_print(" SocketError occurred. Start:" + str(time.ctime()) + " . Retrying in 0.05 sec ...zZz...") time.sleep(0.05) debug_print(" Woke up ... End: " + str(time.ctime())) if since_id: kw = {'since_id': since_id} # continue where you left off twitter_search(twitter_api, q, **kw) statuses = search_results['statuses'] debug_print(" number of statuses: %i max_limit: %i" % (len(statuses), max_results)) # reach the desired number of results, keeping in mind that OAuth users # can "only" make 180 search queries per 15-minute interval. See # https://dev.twitter.com/docs/rate-limiting/1.1/limits # for details. A reasonable number of results is ~1000, although # that number of results may not exist for all queries.
def facebook_menu(): access_token = facebook_authorize() action = None while not action: print "Type the number of the action you want to be executed: " print "0. Save posts for each facebook page read from excel file" print "1. Print sorted pages by number of likes and talking about parameter " print "2. Save data for each page from list " print "3. Get likes per page" print "4. Get talking about count per page" print "5. Get facebook posts sorted" print "6. Save posts for one page" print "7. Get date from most recent post of a page" print "8. Create excel file with statistcs for facebook pages" print "9. Print Facebook insights for a page" print "10.Get number of posts for July " action = raw_input('Enter the number of the action: ').strip() if action == '0' or action == '1' : pages = facebook_read_pages_from_excel(access_token) # get pages and basic data for pages pages_in_database = getCollections("facebook") if action == '0': for page in pages: # check if page info has already been saved in database page_data = facebook_get_page_data(access_token, page) if page_data: if page_data['name'] not in pages_in_database: #store posts only for new pages results = facebook_get_page_posts(access_token, page_data["id"]) #send id of page and access token to get posts save_to_mongo(mongo_db="facebook", mongo_db_coll=page_data['name'], data=results) if action == '1': sorted_pages = facebook_sort_pages(pages, "DESC") facebook_print_page_data(sorted_pages) # if action == '10': # for page in pages: # page_data = facebook_get_page_data(access_token, page) # if page_data: # results = facebook_get_page_posts(access_token, page_data["id"]) #send id of page and access token to get posts # facebook_save_to_mongo(mongo_db="facebook", mongo_db_coll=page_data['name'], data=results) elif action == '2': pages = facebook_read_pages_from_excel(access_token) #get all pages from database pages_data_in_database = load_from_mongo(mongo_db="facebook", mongo_db_coll="pages_info",projection={"_id":0,'name':1}) print "database: ",len(pages_data_in_database) print pages_data_in_database[0] i=0 for page in pages: data = facebook_get_page_data(access_token, page) if data and data['name'] not in pages_data_in_database: # check if page is in database save_to_mongo(mongo_db="facebook", mongo_db_coll="pages_info", data=data) elif action == '3': page = "Connel Community Council" #sample page print page, facebook_get_likes_count(page) elif action == '4': page = "Connel Community Council" #sample page print page, facebook_get_talkingabout_count(page) elif action == '5': posts = facebook_get_posts_from_database(from_user="******", page_name="Connel Community Council") for post in posts: print post print "\n \n _________________________________________________________________________ \n" #break elif action == '6': page = "187281984718895" # sample id data = facebook_get_page_posts(access_token, page, limit=500) debug_print("posts from api :%i" % len(data)) save_to_mongo(mongo_db="facebook", mongo_db_coll="Connel Community Council", data=data) elif action == '7': page = "187281984718895" # sample id result = facebook_get_posts_from_database(from_user="******", page_name="Connel Community Council", limit=1) print result[0]['created_time'] elif action == '8': #print(facebook_path_to_EXPORT_FILE) workbook = xlwt.Workbook(encoding="utf-8") # XLWT - ONLY EXPORTS .XLS files sheet = workbook.add_sheet("sheet1") sheet.write(0, 0, 'FB Page ID ') sheet.write(0, 1, 'CC Name') sheet.write(0, 2, 'Count Likes') sheet.write(0, 3, 'Count Talking about') sheet.write(0, 4, 'Count Posts for July') sheet.write(0, 5, 'Count Posts TOTAL') sheet.write(0, 6, 'Date Most recent post') sheet.write(0, 7, 'Date 5th Most recent post') pages_data = load_from_mongo(mongo_db="facebook", mongo_db_coll="pages_info") # collection where all # the data for each page is stored pages_data = facebook_sort_pages(pages_data, "DESC") i = 1 # index for rows in sheet for page in pages_data: sheet.write(i, 0, page['id']) sheet.write(i, 1, page['name']) sheet.write(i, 2, page['likes']) sheet.write(i, 3, page['talking_about_count']) most_recent_post = 0 fifth_most_recent_post = 0 result = facebook_get_posts_from_database(from_user=page['name'], page_name=page['name'], limit=5) if result: most_recent_post = result[0]['created_time'] fifth_most_recent_post = result[len(result)-1]['created_time'] # some pages have less than 5 posts sheet.write(i, 6, most_recent_post) sheet.write(i, 7, fifth_most_recent_post) monthly_posts = len(facebook_get_posts_for_month(page['name'], month="07")) if monthly_posts is None: monthly_posts = 0 sheet.write(i, 4, monthly_posts) total_posts = 0 result = facebook_get_posts_from_database(from_user=page['name'], page_name=page['name']) if result: total_posts = len(result) sheet.write(i, 5, total_posts) i += 1 workbook.save(facebook_path_to_EXPORT_FILE) elif action == '9': page_id = "173685876111989" # sample id facebook_print_page_insights(access_token, page_id) elif action == '10': page_id = "173685876111989" page_data = facebook_get_page_data(access_token, page_id) if page_data: monthly_posts = facebook_get_posts_for_month(page_data['name'], month="07") print page_data['name'], " : ", len(monthly_posts) for post in monthly_posts: print post
def twitter_menu(): api = twitter_authorize() if api: debug_print("Successfully authenticated and authorized") action = None while not action: print "Type the number of the action you want to be executed: " print "0. Find users who have tweeted a hashtag , for a list of popular hashtags" print "1. Get only hashtags from results" print "2. Create & export graph of members of twitter list and their followers" print "3. Get statuses of list members" print "4. Find the trending topics in the world" print "5. Call function for saving tweets from list members on interval( predefined for community-council)" print "6. Get info about list members from api (json)" print "7. Search & save tweets for a specific query" print "8. Search & save tweets from the streaming api" print "9. Get all tweets from a user's timeline" print "10. Print common entities" print "11. Get list members" print "12. MULTIGRAPH" print "13. find popular tweets from list of tweets" print "14. Get twitter info about a specific user" print "15. Get date for last tweet" print "16. Get date for fifth tweet" print "17. Create excel file with statistcs for members of twitter list" action = raw_input('Enter the number of the action: ').strip() WORLD_WOE_ID = 1 # for searching trends if action == '0': # find users who have tweeted a hashtag , for a list of popular hashtags hashtags_dict = get_popular_hashtags(slug="community-councils") hashtags = hashtags_dict.keys() for hashtag in hashtags: print hashtag dict = get_users_for_hashtag_list(hashtags_dict.keys()) elif action == '1': # get only hashtags from results results = load_from_mongo(mongo_db="twitter", mongo_db_coll="community-councils") hashtags = twitter_get_hashtags_from_tweets(results) print "num of hashtags: ", len(hashtags) for hashtag in hashtags: print "#", hashtag elif action == '2': # create & export graph of members of twitter list and their followers #create directed graph of members of a list and their followers #save list of followers for each member into a separate collection named after the slug ending with _member slug="community-councils" graph = create_directed_graph_of_list_members_and_followers(api, slug) export_graph_to_gml(graph,path_to_graph_file) elif action == '3': # get statuses of list members twitter_get_list_members_tweets(api, owner_screen_name="spartakan", slug="community-councils") elif action == '4': # Get trending topics print "INFO: Getting World Trends ..." world_trends = twitter_trends(api, WORLD_WOE_ID) #for checking the structure of uncomment : #print json.dumps(world_trends[0]['trends'], indent=1) if world_trends: world_trends = world_trends[0]['trends'] for w in world_trends: print "trend: ", w['name'] elif action == '5': # find trending topics on a time interval #making a partial class from twitter_search to later add it as an argument in twitter_call_function_on_interval tweets_from_list_members = partial(twitter_get_list_members_tweets, api, owner_screen_name="spartakan", slug="community-councils") #get and save the trending topics on time intervals twitter_call_function_on_interval(tweets_from_list_members) elif action == '6': # get members of a list twitter_get_list_members(api,owner_screen_name="spartakan", slug="community-councils") elif action == '7' or action == '8': q = None print "Read How to build a query first ! ( https://dev.twitter.com/docs/using-search ) " q = raw_input('Enter a query: ').strip() while not q: q = raw_input('Enter a query:').strip() if action == '7': # Fetch tweets from the search/rest api debug_print("Searching tweets for the query:" + q) #twitter_call_function_on_interval(search_tweets, 'twitter', q) results = twitter_search(api, q, 1000) debug_print("Tweets saved into database") if action == '8': # Fetch tweets from streaming api debug_print("Searching and saving tweets from the streaming api for query: " +q+ "...") twitter_stream_api(api, q) elif action == '9': # Get all tweets from a user's timeline # screen_name = raw_input('Enter the screen name: ').strip() # while not screen_name: # screen_name = raw_input('Enter a query:').strip() # debug_print("Getting tweets from user: "******" ... ") screen_name = "spartakan" tweets = twitter_user_timeline(api, screen_name=screen_name) elif action == '10': results = load_from_mongo("twitter", "#indyref") common_entities = get_common_tweet_entities(results) print_prettytable(common_entities) elif action == '11': # get list members twitter_get_list_members(api,owner_screen_name="spartakan", slug="community-councils") elif action == '12': print "MULTIGRAPH" slug = "community-councils" mdG = create_multi_graph_of_list_memebers_and_followers(api,slug) name_of_file = "twitter_multigraph.gml" export_graph_to_gml(mdG, path_to_graph_file + name_of_file) elif action == '13': # find popular tweets from list of tweets statuses = load_from_mongo("twitter", "community-councils") find_popular_tweets(statuses=statuses) elif action == '14': # get info for user screen_name = "spartakan" user = twitter_get_user_info(api, screen_name) debug_print(json.dumps(user, indent=1)) elif action == '15': screen_name = "spartakan" print twitter_date_of_last_tweet(screen_name) elif action == '16': screen_name = "spartakan" print twitter_date_of_fifth_tweet(screen_name) elif action == '17': #print(facebook_path_to_EXPORT_FILE) workbook = xlwt.Workbook(encoding="utf-8") # XLWT - ONLY EXPORTS .XLS files sheet = workbook.add_sheet("sheet1") sheet.write(0, 0, 'ID ') sheet.write(0, 1, 'CC screen_name') sheet.write(0, 2, 'Count Followers') sheet.write(0, 3, 'Count Friends') sheet.write(0, 4, 'Count Posts TOTAL') sheet.write(0, 5, 'Date Most recent post') sheet.write(0, 6, 'Date 5th Most recent post') members = load_from_mongo(mongo_db="twitter", mongo_db_coll="community-councils_members") # collection where all # the data for each page is stored pages_data = facebook_sort_pages(members, "DESC") i = 1 # index for rows in sheet for member in members: sheet.write(i, 0, member['id']) sheet.write(i, 1, member['screen_name']) sheet.write(i, 2, member['followers_count']) sheet.write(i, 3, member['friends_count']) most_recent_post = twitter_date_of_last_tweet(member['screen_name'],"community-councils") fifth_most_recent_post = twitter_date_of_fifth_tweet(member['screen_name'],"community-councils") sheet.write(i, 5, most_recent_post) sheet.write(i, 6, fifth_most_recent_post) #total sheet.write(i, 4, member["statuses_count"]) i += 1 workbook.save(twitter_path_to_EXPORT_FILE) else: print "WRONG ACTION!!!" else: print "You are not authorized!!!"