def fetch_profiles(client, screen_names, storage):
"""
Fetch twitter profile information for screen_names and add them to storage
Can request 100 profiles per request and 180 profiles per 15mins,
hence the swapping of request sizes for 100, 80
"""
lookup_uri = client.twitter_uri('users', 'lookup')
rate_limit = 180
while screen_names:
if rate_limit > 100:
size_limit, rate_limit = 100, rate_limit - 100
else:
size_limit = rate_limit
clump = screen_names[:size_limit]
response = client.get(lookup_uri,
params={'screen_name': ",".join(clump)})
if ok(response):
del screen_names[:size_limit]
for profile in response.json:
storage.store_profile(profile)
logger.debug("fetched {} profiles, {} left".format(len(clump),
len(screen_names)))
elif not_found(response):
# none of the screen names were valid
del screen_names[:size_limit]
elif rate_limited(response):
# rate limiting, need to sleep
client.wait_for(lookup_uri)
rate_limit = 180
else:
raise UnexpectedError(response.status_code, response.text)
client.enhance_my_calm()
def fetch_cursored_collection(client, screen_name, resource_uri, storage_func):
"""
Fetch each page of friends/followers collections for a screen name
adding the resulting list by calling storage_func(screen_name, result)
"""
cursor = -1
result = []
while cursor != 0:
response = client.get(resource_uri, params={'screen_name': screen_name,
'cursor': cursor})
client.enhance_my_calm()
if ok(response):
logger.debug('fetched {} ids from {}'.format(
len(response.json['ids']), resource_uri))
result.extend(response.json['ids'])
cursor = response.json['next_cursor']
if cursor == 0:
break
logger.debug('next cursor {}'.format(cursor))
elif not_found(response):
return
elif rate_limited(response):
client.wait_for(resource_uri)
else:
raise UnexpectedError(response.status_code, response.text)
storage_func(screen_name, result)