def test_add_member(self):
group = Group('testgroup')
group.add_member('membername', client_id='clientid', client_host='host1', metadata=b'\x00\x32', assignment=b'\x01\x34')
assert len(group.members) == 1
assert group.members[0].name == 'membername'
assert group.members[0].client_id == 'clientid'
assert group.members[0].client_host == 'host1'
assert group.members[0].metadata == b'\x00\x32'
assert group.members[0].assignment_data == b'\x01\x34'
def test_get_group_error(self):
group = Group('badgroup')
group.coordinator = self.client.cluster.brokers[1]
self.client.cluster.add_group(group)
self.client._send_group_aware_request = MagicMock()
self.client._send_group_aware_request.return_value = self.describe_groups_error
self.assertRaises(GroupError, self.client.get_group, 'badgroup', cache=False)
self.client._send_group_aware_request.assert_called_once()
def test_set_assignment(self):
group = Group('testgroup')
group.protocol_type = 'consumer'
member = GroupMember('membername', assignment=b'\x00\x00\x00\x00\x00\x01\x00\x06topic1\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x02\x65\xbd')
member.group = group
member.set_assignment()
assert member.assignment_version == 0
assert member.assignment_data == b'\x00\x00\x00\x00\x00\x01\x00\x06topic1\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x02\x65\xbd'
assert member.user_data == b'\x65\xbd'
assert member.topics == {'topic1': [0]}
def test_add_member(self):
group = Group('testgroup')
group.add_member('membername',
client_id='clientid',
client_host='host1',
metadata=b'\x00\x32',
assignment=b'\x01\x34')
assert len(group.members) == 1
assert group.members[0].name == 'membername'
assert group.members[0].client_id == 'clientid'
assert group.members[0].client_host == 'host1'
assert group.members[0].metadata == b'\x00\x32'
assert group.members[0].assignment_data == b'\x01\x34'
def test_subscribed_topics(self):
group = Group('testgroup')
group.protocol_type = 'consumer'
member = GroupMember('member1')
member.group = group
member.topics = {'topic1': [0]}
group.members.append(member)
member = GroupMember('member2')
member.group = group
member.topics = {'topic2': [0], 'topic1': [1]}
group.members.append(member)
topics = group.subscribed_topics()
print(topics)
assert set(['topic1', 'topic2']) == set(topics)
def test_set_assignment(self):
group = Group('testgroup')
group.protocol_type = 'consumer'
member = GroupMember(
'membername',
assignment=
b'\x00\x00\x00\x00\x00\x01\x00\x06topic1\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x02\x65\xbd'
)
member.group = group
member.set_assignment()
assert member.assignment_version == 0
assert member.assignment_data == b'\x00\x00\x00\x00\x00\x01\x00\x06topic1\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x02\x65\xbd'
assert member.user_data == b'\x65\xbd'
assert member.topics == {'topic1': [0]}
def test_subscribed_topics(self):
group = Group('testgroup')
group.protocol_type = 'consumer'
member = GroupMember('member1')
member.group = group
member.topics = {'topic1': [0]}
group.members.append(member)
member = GroupMember('member2')
member.group = group
member.topics = {'topic2': [0], 'topic1': [1]}
group.members.append(member)
topics = group.subscribed_topics()
print(topics)
assert set(['topic1', 'topic2']) == set(topics)
def _add_or_update_group(self, group_info, coordinator):
"""
Given group information from a ListGroups response, assure that the group exists in the cluster as specified
Args:
group_info (dict): A group from a ListGroups response, which contains group_id and protocol_type keys
coordinator (int): The ID of the group coordinator broker
"""
group_name = group_info['group_id']
try:
group = self.cluster.groups[group_name]
except KeyError:
group = Group(group_name)
self.cluster.add_group(group)
group.coordinator = self.cluster.brokers[coordinator]
group.protocol_type = group_info['protocol_type']
def _add_or_update_group(self, group_info, coordinator):
"""
Given group information from a ListGroups response, assure that the group exists in the cluster as specified
Args:
group_info (dict): A group from a ListGroups response, which contains group_id and protocol_type keys
coordinator (int): The ID of the group coordinator broker
"""
group_name = group_info['group_id']
try:
group = self.cluster.groups[group_name]
except KeyError:
group = Group(group_name)
self.cluster.add_group(group)
group.coordinator = self.cluster.brokers[coordinator]
group.protocol_type = group_info['protocol_type']
def test_get_group_existing_cached(self):
group = Group('testgroup')
group.coordinator = self.client.cluster.brokers[1]
group._last_updated = time.time()
self.client.cluster.add_group(group)
self.client._send_group_aware_request = MagicMock()
self.client._send_group_aware_request.return_value = self.describe_groups
self.client._update_groups_from_describe = MagicMock()
val = self.client.get_group('testgroup')
self.client._send_group_aware_request.assert_not_called()
self.client._update_groups_from_describe.assert_not_called()
assert isinstance(val, Group)
assert val.name == 'testgroup'
assert val.coordinator == self.client.cluster.brokers[1]
def setUp(self):
# Dummy client for testing - we're not going to connect that bootstrap broker
self.client = Client()
self.client._connected = True
# Get the broker and topic from a metadata update
self.client._update_from_metadata(topic_metadata())
self.group = Group('testgroup')
self.offset_fetch = offset_fetch()
def test_list_groups(self):
self.client.cluster.add_group(Group('group2'))
self.client._maybe_update_groups_list = MagicMock()
self.client._maybe_update_groups_list.return_value = 312
groups, errs = self.client.list_groups()
self.client._maybe_update_groups_list.assert_called_once_with(True)
assert errs == 312
assert groups == ['group2']
def test_add_or_update_group_update(self):
broker = Broker('host1.example.com', id=1, port=8031)
self.client.cluster.add_broker(broker)
group = Group('group1')
self.client.cluster.add_group(group)
list_group = list_groups()
self.client._add_or_update_group(list_group['groups'][0], 1)
assert 'group1' in self.client.cluster.groups
assert self.client.cluster.groups['group1'].coordinator == broker
assert self.client.cluster.groups['group1'].protocol_type == 'protocol1'
def _send_group_aware_request(self, group_name, request):
"""
Sends a request to the broker currently serving as the group coordinator for the specified
group name. The request is not introspected for whether or not this is the correct broker,
and the response is not checked as to whether or not there was an error.
As a side effect of this call, the group object is created if it exists, and the coordinator
attribute is set to the broker that is currently the coordinator for that group.
Args:
group_name (string): The name of the group to find the coordinator for
request (BaseRequest): A request instance, inherited from BaseRequest, to send to the
coordinator broker
Returns:
BaseResponse: The response instance, appropriate for the request, that is returned from
the broker
Raises:
ConnectionError: If there is a failure to send the request to the coordinator broker,
or a failure to retrieve the coordinator information
GroupError: If an error is returned when fetching coordinator information
"""
response = self._send_any_broker(
GroupCoordinatorV0Request({'group_id': group_name}))
raise_if_error(GroupError, response['error'])
if group_name not in self.cluster.groups:
self.cluster.add_group(Group(group_name))
try:
self.cluster.groups[group_name].coordinator = self.cluster.brokers[
response['node_id']]
except KeyError:
broker = self._make_broker(response)
self.cluster.add_broker(broker)
self.cluster.groups[group_name].coordinator = broker
response = self._send_to_broker(
self.cluster.groups[group_name].coordinator.id, request)
return response
def _update_groups_from_describe(self, response):
"""
Given a DescribeGroupsV0 response, update the group information for all groups in the response
in this cluster. This does not delete any groups not in the response, as we do not fetch all
describe groups data like that at this time
Args:
response (DescribeGroupsV0Response): A response to create or update groups for
"""
for g in response['groups']:
if g['group_id'] not in self.cluster.groups:
self.cluster.add_group(Group(g['group_id']))
group = self.cluster.groups[g['group_id']]
group.state = g['state']
group.protocol_type = g['protocol_type']
group.protocol = g['protocol']
group.clear_members()
for m in g['members']:
group.add_member(m['member_id'],
client_id=m['client_id'],
client_host=m['client_host'],
metadata=m['member_metadata'],
assignment=m['member_assignment'])
group._last_updated = time.time()
def test_updated_since(self):
group = Group('testgroup')
group._last_updated = 100
assert group.updated_since(99)
def test_clear_members(self):
group = Group('testgroup')
group.members = 'baddata'
group.clear_members()
assert group.members == []
def test_group_create(self):
group = Group('testgroup')
assert group.name == 'testgroup'
def test_clear_members(self):
group = Group('testgroup')
group.members = 'baddata'
group.clear_members()
assert group.members == []
def test_updated_since(self):
group = Group('testgroup')
group._last_updated = 100
assert group.updated_since(99)
def test_update_groups_from_describe_clear_members(self):
self.client.cluster.add_group(Group('testgroup'))
self.client.cluster.groups['testgroup'].members = [GroupMember('badmember')]
self.client._update_groups_from_describe(self.describe_groups)
assert_cluster_has_groups(self.client.cluster, self.describe_groups)
def test_update_groups_from_describe_update(self):
self.client.cluster.add_group(Group('testgroup'))
self.client._update_groups_from_describe(self.describe_groups)
assert_cluster_has_groups(self.client.cluster, self.describe_groups)
def add_group_to_cluster(group_name, request):
# _send_group_aware_request has a side effect of creating the group and setting the coordinator
self.client.cluster.add_group(Group(group_name))
self.client.cluster.groups[group_name].coordinator = self.client.cluster.brokers[1]
return self.describe_groups