def get_following(api, centre, max_depth=3, curr_depth=0):
if curr_depth > max_depth:
return
userfname = os.path.join(USER_DIR, str(centre) + ".json")
if os.path.exists(userfname):
return
logging.info("User being mined is %s" % str(centre))
print("User being mined %s" % str(centre))
following = []
while True:
try:
i = 1
for friends in tweepy.Cursor(api.friends,
screen_name=centre,
count=200).items():
# print i,friends.screen_name
i += 1
following.append(friends.screen_name)
if i % 200 == 0:
time.sleep(60)
for name in following:
get_following(api, name, max_depth, curr_depth + 1)
UserDB.store_friends(api, centre, following)
mentions.get_mentions(api, centre)
break
except tweepy.TweepError as error:
print(str(error))
if str(error) == 'Not authorized.':
print('Can' 't access user data - not authorized.')
break
if str(error) == 'User has been suspended.':
print('User suspended.')
break
if str(
error
) == "[{'message': 'Rate limit exceeded', 'code': 88}]" or str(
error
) == "[{u'message': u'Rate limit exceeded', u'code': 88}]":
print('Rate limited. Sleeping for 15 minutes.')
logging.info('Rate limit exceeded at %s' % str(datetime.now()))
time.sleep(15 * 60 + 15)
continue
return following
def get_followers(api, centre, max_depth=1, current_depth=0):
if current_depth >= max_depth:
return
userfname = os.path.join(USER_DIR, str(centre) + '.json')
if not os.path.exists(userfname):
# mine follower list for user
print('Retrieving details for user %s' % str(centre))
logging.info('Retrieving details for uer %s' % str(centre))
while True:
try:
user = api.get_user(centre)
followers = tweepy.Cursor(api.followers,
screen_name=centre).items()
print(user.name)
follower_names = []
for id in followers:
follower_names.append(id.screen_name)
d = {
'name': user.name,
'screen_name': user.screen_name,
'id': user.id,
'friends_count': user.friends_count,
'followers_count': user.followers_count,
'followers_ids': follower_names
}
with open(userfname, 'w') as outf:
outf.write(json.dumps(d, indent=1))
outf.close()
for follower in follower_names:
get_followers(api, follower, max_depth, current_depth + 1)
if current_depth < max_depth - 1:
mentions.get_mentions(api, follower)
break
except tweepy.TweepError as error:
print(str(error))
if str(error) == 'Not authorized.':
print('Can' 't access user data - not authorized.')
break
if str(error) == 'User has been suspended.':
print('User suspended.')
break
if str(
error
) == "[{'message': 'Rate limit exceeded', 'code': 88}]" or str(
error
) == "[{'code': 88, 'message': 'Rate limit exceeded'}]":
print('Rate limited. Sleeping for 15 minutes.')
logging.debug('Rate limit exceeded at %s' %
str(datetime.now()))
time.sleep(15 * 60 + 15)
continue
else:
print("User %s already mined" % str(centre))
from maps import get_folium_map, get_altair_map from streamlit_folium import folium_static from scipy import stats import random ''' # Hi, I'm Vlad. ### Let's [work together](mailto:[email protected]). ___ ''' ''' ## Let's start with the most important thing... ''' from mentions import get_mentions dogs = get_mentions('dogs') cats = get_mentions('cats') if not dogs and cats: show_headlines = False dogs_mentions = dogs['mentions'] cats_mentions = cats['mentions'] dates = [x[0] for x in dogs_mentions] dates = [x[0] for x in dogs_mentions] import plotly.graph_objects as go fig = go.Figure() fig.add_trace(
def start_collection(depth):
for account in seed:
follower.get_followers(api,account,depth,0)
mentions.get_mentions(api,account)