示例#1
0
def fetch_profiles(client, screen_names, storage):
    """
    Fetch twitter profile information for screen_names and add them to storage

    Can request 100 profiles per request and 180 profiles per 15mins,
    hence the swapping of request sizes for 100, 80
    """

    lookup_uri = client.twitter_uri('users', 'lookup')
    rate_limit = 180

    while screen_names:
        if rate_limit > 100:
            size_limit, rate_limit = 100, rate_limit - 100
        else:
            size_limit = rate_limit
        clump = screen_names[:size_limit]
        response = client.get(lookup_uri,
            params={'screen_name': ",".join(clump)})
        if ok(response):
            del screen_names[:size_limit]
            for profile in response.json:
                storage.store_profile(profile)
            logger.debug("fetched {} profiles, {} left".format(len(clump),
                len(screen_names)))
        elif not_found(response):
            # none of the screen names were valid
            del screen_names[:size_limit]
        elif rate_limited(response):
            # rate limiting, need to sleep
            client.wait_for(lookup_uri)
            rate_limit = 180
        else:
            raise UnexpectedError(response.status_code, response.text)
        client.enhance_my_calm()
示例#2
0
def fetch_cursored_collection(client, screen_name, resource_uri, storage_func):
    """
    Fetch each page of friends/followers collections for a screen name
    adding the resulting list by calling storage_func(screen_name, result)
    """

    cursor = -1
    result = []
    while cursor != 0:
        response = client.get(resource_uri, params={'screen_name': screen_name,
            'cursor': cursor})
        client.enhance_my_calm()
        if ok(response):
            logger.debug('fetched {} ids from {}'.format(
                len(response.json['ids']), resource_uri))
            result.extend(response.json['ids'])
            cursor = response.json['next_cursor']
            if cursor == 0:
                break
            logger.debug('next cursor {}'.format(cursor))
        elif not_found(response):
            return
        elif rate_limited(response):
            client.wait_for(resource_uri)
        else:
            raise UnexpectedError(response.status_code, response.text)
    storage_func(screen_name, result)