def get_following(api, centre, max_depth=3, curr_depth=0): if curr_depth > max_depth: return userfname = os.path.join(USER_DIR, str(centre) + ".json") if os.path.exists(userfname): return logging.info("User being mined is %s" % str(centre)) print("User being mined %s" % str(centre)) following = [] while True: try: i = 1 for friends in tweepy.Cursor(api.friends, screen_name=centre, count=200).items(): # print i,friends.screen_name i += 1 following.append(friends.screen_name) if i % 200 == 0: time.sleep(60) for name in following: get_following(api, name, max_depth, curr_depth + 1) UserDB.store_friends(api, centre, following) mentions.get_mentions(api, centre) break except tweepy.TweepError as error: print(str(error)) if str(error) == 'Not authorized.': print('Can' 't access user data - not authorized.') break if str(error) == 'User has been suspended.': print('User suspended.') break if str( error ) == "[{'message': 'Rate limit exceeded', 'code': 88}]" or str( error ) == "[{u'message': u'Rate limit exceeded', u'code': 88}]": print('Rate limited. Sleeping for 15 minutes.') logging.info('Rate limit exceeded at %s' % str(datetime.now())) time.sleep(15 * 60 + 15) continue return following
def get_followers(api, centre, max_depth=1, current_depth=0): if current_depth >= max_depth: return userfname = os.path.join(USER_DIR, str(centre) + '.json') if not os.path.exists(userfname): # mine follower list for user print('Retrieving details for user %s' % str(centre)) logging.info('Retrieving details for uer %s' % str(centre)) while True: try: user = api.get_user(centre) followers = tweepy.Cursor(api.followers, screen_name=centre).items() print(user.name) follower_names = [] for id in followers: follower_names.append(id.screen_name) d = { 'name': user.name, 'screen_name': user.screen_name, 'id': user.id, 'friends_count': user.friends_count, 'followers_count': user.followers_count, 'followers_ids': follower_names } with open(userfname, 'w') as outf: outf.write(json.dumps(d, indent=1)) outf.close() for follower in follower_names: get_followers(api, follower, max_depth, current_depth + 1) if current_depth < max_depth - 1: mentions.get_mentions(api, follower) break except tweepy.TweepError as error: print(str(error)) if str(error) == 'Not authorized.': print('Can' 't access user data - not authorized.') break if str(error) == 'User has been suspended.': print('User suspended.') break if str( error ) == "[{'message': 'Rate limit exceeded', 'code': 88}]" or str( error ) == "[{'code': 88, 'message': 'Rate limit exceeded'}]": print('Rate limited. Sleeping for 15 minutes.') logging.debug('Rate limit exceeded at %s' % str(datetime.now())) time.sleep(15 * 60 + 15) continue else: print("User %s already mined" % str(centre))
from maps import get_folium_map, get_altair_map from streamlit_folium import folium_static from scipy import stats import random ''' # Hi, I'm Vlad. ### Let's [work together](mailto:[email protected]). ___ ''' ''' ## Let's start with the most important thing... ''' from mentions import get_mentions dogs = get_mentions('dogs') cats = get_mentions('cats') if not dogs and cats: show_headlines = False dogs_mentions = dogs['mentions'] cats_mentions = cats['mentions'] dates = [x[0] for x in dogs_mentions] dates = [x[0] for x in dogs_mentions] import plotly.graph_objects as go fig = go.Figure() fig.add_trace(
def start_collection(depth): for account in seed: follower.get_followers(api,account,depth,0) mentions.get_mentions(api,account)