def test_parallel_client_sleep(mock_sleep, mock_time, mock_twitter):
mock_time.return_value = 1000
mock_twitter.return_value = MockSingleBlockedApi(1001, ['kanyewest'])
blocked_once = twitter.Api()
p = ParallelTwitterClient(apis=[blocked_once])
# The call should prompt a sleep then succeed.
p.get_friend_ids(screen_name='jack')
assert mock_sleep.called
def test_parallel_client_valid_and_blocked(mock_sleep, mock_time, mock_twitter):
mock_time.return_value = 1000
mock_twitter.return_value = MockBlockedApi(1001)
blocked = twitter.Api()
mock_twitter.return_value = MockValidApi(['kanyewest'])
valid = twitter.Api()
p = ParallelTwitterClient(apis=[blocked, valid])
p.get_friend_ids(screen_name='jack')
assert not mock_sleep.called
def test_parallel_client_staggers_requests(mock_sleep, mock_time, mock_twitter):
mock_time.return_value = 1000
mock_twitter.return_value = MockValidApi(['kanyewest'])
valid = twitter.Api()
p = ParallelTwitterClient(apis=[valid])
p.get_friend_ids(screen_name='jack')
assert not mock_sleep.called
# Not enough time has elapsed, so parallel client should sleep
mock_time.return_value = 1001
p.get_friend_ids(screen_name='jack')
assert mock_sleep.called
def calculate_industry_group(seed: List[int],
depth: int,
apis: List[twitter.Api],
max_count: int = 200) -> Dict[int, int]:
"""
Run a breadth-first search on a set of users' friends on Twitter.
Return the number of followers each user has within the group of nth-degree
connections.
A depth of `k` means that we will not visit the friends of anyone who is
more than `k` hops away from a seed user. Users who are exactly `k + 1`
hops away will still be included in the output dictionary (since we know
that at least one user follows them), but we will not count the users
who they are following.
Parameters
----------
seed : List[int]
List of Twitter user IDs to initialize the BFS
depth : int
The depth of the BFS. Defaults to 2.
apis : List[twitter.Api]
A list of twitter.Api objects, which can be obtained from
`parallel_client.oauth_dicts_to_apis`
max_count : int
The maximum number of friends to pull for a given user
"""
n_followers: Dict[int, int] = defaultdict(int)
client = ParallelTwitterClient(apis=apis)
LOGGER.info('Pulled {} valid keys'.format(
len(client.operators[GetFriendIDs])))
users_queue: deque = deque()
for u in seed:
users_queue.append((u, 0))
depth_flags: Set[int] = set()
while len(users_queue) > 0:
u, n = users_queue.popleft()
if n not in depth_flags:
depth_flags.add(n)
LOGGER.info('Reached depth: {}'.format(n))
LOGGER.info('Size of queue: {}'.format(len(users_queue)))
user_friends = client.get_friend_ids(user_id=u, max_count=max_count)
for f in user_friends:
if n < depth and f not in n_followers:
users_queue.append((f, n + 1))
n_followers[f] += 1
return n_followers
def test_parallel_client_single_req_blocked(mock_time, mock_twitter):
mock_time.return_value = 1000
mock_twitter.return_value = MockBlockedApi(1001)
p = ParallelTwitterClient(apis=[twitter.Api()])
with pytest.raises(OutOfKeysError):
p.get_friend_ids(screen_name='jack')