def get_good_partition(key, all_partitions): if isinstance(key, str): key = bytes(key, 'utf-8') idx = murmur2(key) idx &= 0x7fffffff idx %= len(all_partitions) return all_partitions[idx]
def _on_span_generation_known(self, span: opentracing.Span) -> None: if self._consumer: coordinator = self._consumer._coordinator coordinator_id = coordinator.coordinator_id app_id = self.app.conf.id generation = coordinator.generation member_id = coordinator.member_id try: op_name = span.operation_name set_tag = span.set_tag except AttributeError: # pragma: no cover pass # not a real span else: trace_id_str = f'reb-{app_id}-{generation}' trace_id = murmur2(trace_id_str.encode()) span.context.trace_id = trace_id if op_name.endswith('.REPLACE_WITH_MEMBER_ID'): span.set_operation_name(f'rebalancing node {member_id}') set_tag('kafka_generation', generation) set_tag('kafka_member_id', member_id) set_tag('kafka_coordinator_id', coordinator_id) self.app._span_add_default_tags(span) span._real_finish()
def __call__(self, key: Union[str, bytes], all_partitions: List[int], available_partitions: Optional[List[int]]) \ -> int: """ Returns a partition to be used for the message Args: key (Union[str, bytes]): the key to use for partitioning. all_partitions (List[int]): a list of the topic's partitions. available_partitions (Optional[List[int]]): a list of the broker's currently available partitions(optional). Raises: BadKeyType: If key is not bytes serializable Returns: int: Partition number """ if key is None: return random.choice(all_partitions) if isinstance(key, str): key = bytes(key, 'utf-8') if isinstance(key, bytes): idx = murmur2(key) idx &= 0x7fffffff idx %= len(all_partitions) return all_partitions[idx] raise BadKeyType
def make_table(quotes=True): d = defaultdict(set) for i in range(10): s = f"hellohello-{i}" if quotes: b = bytes(s, 'ascii') else: b = s.encode('ascii') d[murmur2(b) % 8].add(b) return d
def __call__(self, key, all_partitions, available): logger.debug('KeyPartitioner') if key is None: return random.choice(all_partitions) if isinstance(key, str): key = bytes(key, 'utf-8') if isinstance(key, bytes): idx = murmur2(key) idx &= 0x7fffffff idx %= len(all_partitions) return all_partitions[idx] raise BadKeyType
def __call__(cls, key, all_partitions, available): """ Get the partition corresponding to key :param key: partitioning key :param all_partitions: list of all partitions sorted by partition ID :param available: list of available partitions in no particular order :return: one of the values from all_partitions or available """ if key is None: if available: return random.choice(available) return random.choice(all_partitions) idx = murmur2(key) idx &= 0x7fffffff idx %= len(all_partitions) return all_partitions[idx]
def partioner(key, all_partitions, available): idx = (murmur2(key) & 0x7fffffff) % len(all_partitions) return idx
m = part_key.match(line) d[m.group(1)].add(m.group(2)) return d get_table(hh) get_table(hcc) # get_table(hh) # defaultdict(<class 'set'>, {'0': {'hellohello-4'}, '2': {'hellohello-5'}, '3': {'hellohello-0'}, '4': {'hellohello-8', 'hellohello-9'}, '5': {'hellohello-1'}, '6': {'hellohello-2'}, '7': {'hellohello-7', 'hellohello-3', 'hellohello-6', 'hellohello-10'}}) # get_table(hcc) # defaultdict(<class 'set'>, {'0': {'"hellohello-4"'}, '2': {'"hellohello-5"'}, '3': {'"hellohello-0"'}, '4': {'"hellohello-9"', '"hellohello-8"'}, '5': {'"hellohello-1"'}, '6': {'"hellohello-2"'}, '7': {'"hellohello-3"', '"hellohello-10"', '"hellohello-6"', '"hellohello-7"'}}) from kafka.partitioner.hashed import murmur2 murmur2(b"\"hellohello-5\"") % 8 murmur2(b"hellohello-5") % 8 def make_table(quotes=True): d = defaultdict(set) for i in range(10): s = f"hellohello-{i}" if quotes: b = bytes(s, 'ascii') else: b = s.encode('ascii') d[murmur2(b) % 8].add(b) return d
def test_murmur2_not_ascii(): # Verify no regression of murmur2() bug encoding py2 bytes that don't ascii encode murmur2(b'\xa4') murmur2(b'\x81' * 1000)