def fetch_profiles(client, screen_names, storage):
"""
Fetch twitter profile information for screen_names and add them to storage
Can request 100 profiles per request and 180 profiles per 15mins,
hence the swapping of request sizes for 100, 80
"""
lookup_uri = client.twitter_uri('users', 'lookup')
rate_limit = 180
while screen_names:
if rate_limit > 100:
size_limit, rate_limit = 100, rate_limit - 100
else:
size_limit = rate_limit
clump = screen_names[:size_limit]
response = client.get(lookup_uri,
params={'screen_name': ",".join(clump)})
if ok(response):
del screen_names[:size_limit]
for profile in response.json:
storage.store_profile(profile)
logger.debug("fetched {} profiles, {} left".format(len(clump),
len(screen_names)))
elif not_found(response):
# none of the screen names were valid
del screen_names[:size_limit]
elif rate_limited(response):
# rate limiting, need to sleep
client.wait_for(lookup_uri)
rate_limit = 180
else:
raise UnexpectedError(response.status_code, response.text)
client.enhance_my_calm()
def fetch_followers_for(screen_name, client, storage):
"""
Fetch followers for a twitter profile
"""
followers_uri = client.twitter_uri('followers', 'ids', version='1')
fetch_cursored_collection(client, screen_name, followers_uri,
storage.store_followers)
