def get_good_partition(key, all_partitions):
if isinstance(key, str):
key = bytes(key, 'utf-8')
idx = murmur2(key)
idx &= 0x7fffffff
idx %= len(all_partitions)
return all_partitions[idx]
def _on_span_generation_known(self, span: opentracing.Span) -> None:
if self._consumer:
coordinator = self._consumer._coordinator
coordinator_id = coordinator.coordinator_id
app_id = self.app.conf.id
generation = coordinator.generation
member_id = coordinator.member_id
try:
op_name = span.operation_name
set_tag = span.set_tag
except AttributeError: # pragma: no cover
pass # not a real span
else:
trace_id_str = f'reb-{app_id}-{generation}'
trace_id = murmur2(trace_id_str.encode())
span.context.trace_id = trace_id
if op_name.endswith('.REPLACE_WITH_MEMBER_ID'):
span.set_operation_name(f'rebalancing node {member_id}')
set_tag('kafka_generation', generation)
set_tag('kafka_member_id', member_id)
set_tag('kafka_coordinator_id', coordinator_id)
self.app._span_add_default_tags(span)
span._real_finish()
def __call__(self, key: Union[str, bytes], all_partitions: List[int], available_partitions: Optional[List[int]]) \
-> int:
"""
Returns a partition to be used for the message
Args:
key (Union[str, bytes]): the key to use for partitioning.
all_partitions (List[int]): a list of the topic's partitions.
available_partitions (Optional[List[int]]): a list of the broker's currently available partitions(optional).
Raises:
BadKeyType: If key is not bytes serializable
Returns:
int: Partition number
"""
if key is None:
return random.choice(all_partitions)
if isinstance(key, str):
key = bytes(key, 'utf-8')
if isinstance(key, bytes):
idx = murmur2(key)
idx &= 0x7fffffff
idx %= len(all_partitions)
return all_partitions[idx]
raise BadKeyType
def make_table(quotes=True):
d = defaultdict(set)
for i in range(10):
s = f"hellohello-{i}"
if quotes:
b = bytes(s, 'ascii')
else:
b = s.encode('ascii')
d[murmur2(b) % 8].add(b)
return d
def __call__(self, key, all_partitions, available):
logger.debug('KeyPartitioner')
if key is None:
return random.choice(all_partitions)
if isinstance(key, str):
key = bytes(key, 'utf-8')
if isinstance(key, bytes):
idx = murmur2(key)
idx &= 0x7fffffff
idx %= len(all_partitions)
return all_partitions[idx]
raise BadKeyType
def __call__(cls, key, all_partitions, available):
"""
Get the partition corresponding to key
:param key: partitioning key
:param all_partitions: list of all partitions sorted by partition ID
:param available: list of available partitions in no particular order
:return: one of the values from all_partitions or available
"""
if key is None:
if available:
return random.choice(available)
return random.choice(all_partitions)
idx = murmur2(key)
idx &= 0x7fffffff
idx %= len(all_partitions)
return all_partitions[idx]
def partioner(key, all_partitions, available):
idx = (murmur2(key) & 0x7fffffff) % len(all_partitions)
return idx
m = part_key.match(line)
d[m.group(1)].add(m.group(2))
return d
get_table(hh)
get_table(hcc)
# get_table(hh)
# defaultdict(<class 'set'>, {'0': {'hellohello-4'}, '2': {'hellohello-5'}, '3': {'hellohello-0'}, '4': {'hellohello-8', 'hellohello-9'}, '5': {'hellohello-1'}, '6': {'hellohello-2'}, '7': {'hellohello-7', 'hellohello-3', 'hellohello-6', 'hellohello-10'}})
# get_table(hcc)
# defaultdict(<class 'set'>, {'0': {'"hellohello-4"'}, '2': {'"hellohello-5"'}, '3': {'"hellohello-0"'}, '4': {'"hellohello-9"', '"hellohello-8"'}, '5': {'"hellohello-1"'}, '6': {'"hellohello-2"'}, '7': {'"hellohello-3"', '"hellohello-10"', '"hellohello-6"', '"hellohello-7"'}})
from kafka.partitioner.hashed import murmur2
murmur2(b"\"hellohello-5\"") % 8
murmur2(b"hellohello-5") % 8
def make_table(quotes=True):
d = defaultdict(set)
for i in range(10):
s = f"hellohello-{i}"
if quotes:
b = bytes(s, 'ascii')
else:
b = s.encode('ascii')
d[murmur2(b) % 8].add(b)
return d
def test_murmur2_not_ascii():
# Verify no regression of murmur2() bug encoding py2 bytes that don't ascii encode
murmur2(b'\xa4')
murmur2(b'\x81' * 1000)
