def _build_consumer( self, consumer_group, commit_log_topic, synchronize_commit_group, commit_batch_size=100, commit_batch_timeout_ms=5000, initial_offset_reset="latest", ): cluster_name = settings.KAFKA_TOPICS[settings.KAFKA_EVENTS]["cluster"] synchronized_consumer = SynchronizedConsumer( cluster_name=cluster_name, consumer_group=consumer_group, commit_log_topic=commit_log_topic, synchronize_commit_group=synchronize_commit_group, initial_offset_reset=initial_offset_reset, ) concurrency = options.get(_CONCURRENCY_OPTION) worker = PostProcessForwarderWorker(concurrency=concurrency) consumer = BatchingKafkaConsumer( topics=self.topic, worker=worker, max_batch_size=commit_batch_size, max_batch_time=commit_batch_timeout_ms, consumer=synchronized_consumer, commit_on_shutdown=True, ) return consumer
def relay(self, consumer_group, commit_log_topic, synchronize_commit_group, commit_batch_size=100, initial_offset_reset='latest'): consumer = SynchronizedConsumer( bootstrap_servers=self.producer_configuration['bootstrap.servers'], consumer_group=consumer_group, commit_log_topic=commit_log_topic, synchronize_commit_group=synchronize_commit_group, initial_offset_reset=initial_offset_reset, ) consumer.subscribe([self.publish_topic]) offsets = {} def commit_offsets(): consumer.commit(offsets=[ TopicPartition(topic, partition, offset) for (topic, partition), offset in offsets.items() ], asynchronous=False) try: i = 0 while True: message = consumer.poll(0.1) if message is None: continue error = message.error() if error is not None: raise Exception(error) i = i + 1 offsets[(message.topic(), message.partition())] = message.offset() + 1 payload = parse_event_message(message.value()) if payload is not None: post_process_group.delay(**payload) if i % commit_batch_size == 0: commit_offsets() except KeyboardInterrupt: pass logger.info('Committing offsets and closing consumer...') if offsets: commit_offsets() consumer.close()
def _build_consumer( self, entity, consumer_group, commit_log_topic, synchronize_commit_group, commit_batch_size=100, commit_batch_timeout_ms=5000, initial_offset_reset="latest", ): cluster_name = settings.KAFKA_TOPICS[settings.KAFKA_EVENTS]["cluster"] synchronized_consumer = SynchronizedConsumer( cluster_name=cluster_name, consumer_group=consumer_group, commit_log_topic=commit_log_topic, synchronize_commit_group=synchronize_commit_group, initial_offset_reset=initial_offset_reset, ) concurrency = options.get(_CONCURRENCY_OPTION) logger.info( f"Starting post process forwrader to consume {entity} messages") if entity == PostProcessForwarderType.TRANSACTIONS: worker = TransactionsPostProcessForwarderWorker( concurrency=concurrency) elif entity == PostProcessForwarderType.ERRORS: worker = ErrorsPostProcessForwarderWorker(concurrency=concurrency) else: # Default implementation which processes both errors and transactions # irrespective of values in the header. This would most likely be the case # for development environments. worker = PostProcessForwarderWorker(concurrency=concurrency) consumer = BatchingKafkaConsumer( topics=self.topic, worker=worker, max_batch_size=commit_batch_size, max_batch_time=commit_batch_timeout_ms, consumer=synchronized_consumer, commit_on_shutdown=True, ) return consumer
def test_consumer_rebalance_from_uncommitted_offset(requires_kafka): consumer_group = "consumer-{}".format(uuid.uuid1().hex) synchronize_commit_group = "consumer-{}".format(uuid.uuid1().hex) messages_delivered = defaultdict(list) def record_message_delivered(error, message): assert error is None messages_delivered[message.topic()].append(message) producer = Producer({ "bootstrap.servers": os.environ["SENTRY_KAFKA_HOSTS"], "on_delivery": record_message_delivered, }) with create_topic( partitions=2) as topic, create_topic() as commit_log_topic: # Produce some messages into the topic. for i in range(4): producer.produce(topic, "{}".format(i).encode("utf8"), partition=i % 2) assert producer.flush( 5) == 0, "producer did not successfully flush queue" for (topic, partition), offset in { (message.topic(), message.partition()): message.offset() for message in messages_delivered[topic] }.items(): producer.produce( commit_log_topic, key="{}:{}:{}".format(topic, partition, synchronize_commit_group).encode("utf8"), value="{}".format(offset + 1).encode("utf8"), ) assert producer.flush( 5) == 0, "producer did not successfully flush queue" consumer_a = SynchronizedConsumer( cluster_name="default", consumer_group=consumer_group, commit_log_topic=commit_log_topic, synchronize_commit_group=synchronize_commit_group, initial_offset_reset="earliest", ) assignments_received = defaultdict(list) def on_assign(consumer, assignment): assignments_received[consumer].append(assignment) consumer_a.subscribe([topic], on_assign=on_assign) consume_until_constraints_met( consumer_a, [ lambda message: assignments_received[consumer_a], collect_messages_received(4) ], 10, ) assert (len(assignments_received[consumer_a]) == 1 ), "expected to receive partition assignment" assert set( (i.topic, i.partition) for i in assignments_received[consumer_a][0]) == set([(topic, 0), (topic, 1)]) assignments_received[consumer_a].pop() message = consumer_a.poll(1) assert (message is None or message.error() is KafkaError._PARTITION_EOF ), "there should be no more messages to receive" consumer_b = SynchronizedConsumer( cluster_name="default", consumer_group=consumer_group, commit_log_topic=commit_log_topic, synchronize_commit_group=synchronize_commit_group, initial_offset_reset="earliest", ) consumer_b.subscribe([topic], on_assign=on_assign) consume_until_constraints_met( consumer_a, [lambda message: assignments_received[consumer_a]], 10) consume_until_constraints_met( consumer_b, [ lambda message: assignments_received[consumer_b], collect_messages_received(2) ], 10, ) for consumer in [consumer_a, consumer_b]: assert len(assignments_received[consumer][0]) == 1 message = consumer_a.poll(1) assert (message is None or message.error() is KafkaError._PARTITION_EOF ), "there should be no more messages to receive" message = consumer_b.poll(1) assert (message is None or message.error() is KafkaError._PARTITION_EOF ), "there should be no more messages to receive"
def test_consumer_start_from_partition_start(requires_kafka): synchronize_commit_group = "consumer-{}".format(uuid.uuid1().hex) messages_delivered = defaultdict(list) def record_message_delivered(error, message): assert error is None messages_delivered[message.topic()].append(message) producer = Producer({ "bootstrap.servers": os.environ["SENTRY_KAFKA_HOSTS"], "on_delivery": record_message_delivered, }) with create_topic() as topic, create_topic() as commit_log_topic: # Produce some messages into the topic. for i in range(3): producer.produce(topic, "{}".format(i).encode("utf8")) assert producer.flush( 5) == 0, "producer did not successfully flush queue" # Create the synchronized consumer. consumer = SynchronizedConsumer( cluster_name="default", consumer_group="consumer-{}".format(uuid.uuid1().hex), commit_log_topic=commit_log_topic, synchronize_commit_group=synchronize_commit_group, initial_offset_reset="earliest", ) assignments_received = [] def on_assign(c, assignment): assert c is consumer assignments_received.append(assignment) consumer.subscribe([topic], on_assign=on_assign) # Wait until we have received our assignments. for i in xrange(10): # this takes a while assert consumer.poll(1) is None if assignments_received: break assert len(assignments_received ) == 1, "expected to receive partition assignment" assert set((i.topic, i.partition) for i in assignments_received[0]) == set([(topic, 0)]) # TODO: Make sure that all partitions remain paused. # Make sure that there are no messages ready to consume. assert consumer.poll(1) is None # Move the committed offset forward for our synchronizing group. message = messages_delivered[topic][0] producer.produce( commit_log_topic, key="{}:{}:{}".format(message.topic(), message.partition(), synchronize_commit_group).encode("utf8"), value="{}".format(message.offset() + 1).encode("utf8"), ) assert producer.flush( 5) == 0, "producer did not successfully flush queue" # We should have received a single message. # TODO: Can we also assert that the position is unpaused?) for i in xrange(5): message = consumer.poll(1) if message is not None: break assert message is not None, "no message received" expected_message = messages_delivered[topic][0] assert message.topic() == expected_message.topic() assert message.partition() == expected_message.partition() assert message.offset() == expected_message.offset() # We should not be able to continue reading into the topic. # TODO: Can we assert that the position is paused? assert consumer.poll(1) is None
def test_consumer_rebalance_from_committed_offset(requires_kafka): consumer_group = "consumer-{}".format(uuid.uuid1().hex) synchronize_commit_group = "consumer-{}".format(uuid.uuid1().hex) messages_delivered = defaultdict(list) def record_message_delivered(error, message): assert error is None messages_delivered[message.topic()].append(message) producer = Producer({ "bootstrap.servers": os.environ["SENTRY_KAFKA_HOSTS"], "on_delivery": record_message_delivered, }) with create_topic( partitions=2) as topic, create_topic() as commit_log_topic: # Produce some messages into the topic. for i in range(4): producer.produce(topic, "{}".format(i).encode("utf8"), partition=i % 2) assert producer.flush( 5) == 0, "producer did not successfully flush queue" Consumer({ "bootstrap.servers": os.environ["SENTRY_KAFKA_HOSTS"], "group.id": consumer_group }).commit( offsets=[ TopicPartition(message.topic(), message.partition(), message.offset() + 1) for message in messages_delivered[topic][:2] ], asynchronous=False, ) consumer_a = SynchronizedConsumer( cluster_name="default", consumer_group=consumer_group, commit_log_topic=commit_log_topic, synchronize_commit_group=synchronize_commit_group, initial_offset_reset="earliest", ) assignments_received = defaultdict(list) def on_assign(consumer, assignment): assignments_received[consumer].append(assignment) consumer_a.subscribe([topic], on_assign=on_assign) # Wait until the first consumer has received its assignments. for i in xrange(10): # this takes a while assert consumer_a.poll(1) is None if assignments_received[consumer_a]: break assert (len(assignments_received[consumer_a]) == 1 ), "expected to receive partition assignment" assert set( (i.topic, i.partition) for i in assignments_received[consumer_a][0]) == set([(topic, 0), (topic, 1)]) assignments_received[consumer_a].pop() consumer_b = SynchronizedConsumer( cluster_name="default", consumer_group=consumer_group, commit_log_topic=commit_log_topic, synchronize_commit_group=synchronize_commit_group, initial_offset_reset="earliest", ) consumer_b.subscribe([topic], on_assign=on_assign) assignments = {} # Wait until *both* consumers have received updated assignments. for consumer in [consumer_a, consumer_b]: for i in xrange(10): # this takes a while assert consumer.poll(1) is None if assignments_received[consumer]: break assert (len(assignments_received[consumer]) == 1 ), "expected to receive partition assignment" assert (len(assignments_received[consumer][0]) == 1 ), "expected to have a single partition assignment" i = assignments_received[consumer][0][0] assignments[(i.topic, i.partition)] = consumer assert set(assignments.keys()) == set([(topic, 0), (topic, 1)]) for expected_message in messages_delivered[topic][2:]: consumer = assignments[(expected_message.topic(), expected_message.partition())] # Make sure that there are no messages ready to consume. assert consumer.poll(1) is None # Move the committed offset forward for our synchronizing group. producer.produce( commit_log_topic, key="{}:{}:{}".format(expected_message.topic(), expected_message.partition(), synchronize_commit_group).encode("utf8"), value="{}".format(expected_message.offset() + 1).encode("utf8"), ) assert producer.flush( 5) == 0, "producer did not successfully flush queue" # We should have received a single message. # TODO: Can we also assert that the position is unpaused?) for i in xrange(5): received_message = consumer.poll(1) if received_message is not None: break assert received_message is not None, "no message received" assert received_message.topic() == expected_message.topic() assert received_message.partition() == expected_message.partition() assert received_message.offset() == expected_message.offset() # We should not be able to continue reading into the topic. # TODO: Can we assert that the position is paused? assert consumer.poll(1) is None
def run_post_process_forwarder(self, consumer_group, commit_log_topic, synchronize_commit_group, commit_batch_size=100, initial_offset_reset='latest'): logger.debug('Starting post-process forwarder...') cluster_name = settings.KAFKA_TOPICS[settings.KAFKA_EVENTS]['cluster'] bootstrap_servers = settings.KAFKA_CLUSTERS[cluster_name][ 'bootstrap.servers'] consumer = SynchronizedConsumer( bootstrap_servers=bootstrap_servers, consumer_group=consumer_group, commit_log_topic=commit_log_topic, synchronize_commit_group=synchronize_commit_group, initial_offset_reset=initial_offset_reset, ) owned_partition_offsets = {} def commit(partitions): results = consumer.commit(offsets=partitions, asynchronous=False) errors = filter(lambda i: i.error is not None, results) if errors: raise Exception('Failed to commit %s/%s partitions: %r' % (len(errors), len(partitions), errors)) return results def on_assign(consumer, partitions): logger.debug('Received partition assignment: %r', partitions) for i in partitions: if i.offset == OFFSET_INVALID: updated_offset = None elif i.offset < 0: raise Exception( 'Received unexpected negative offset during partition assignment: %r' % (i, )) else: updated_offset = i.offset key = (i.topic, i.partition) previous_offset = owned_partition_offsets.get(key, None) if previous_offset is not None and previous_offset != updated_offset: logger.warning( 'Received new offset for owned partition %r, will overwrite previous stored offset %r with %r.', key, previous_offset, updated_offset) owned_partition_offsets[key] = updated_offset def on_revoke(consumer, partitions): logger.debug('Revoked partition assignment: %r', partitions) offsets_to_commit = [] for i in partitions: key = (i.topic, i.partition) try: offset = owned_partition_offsets.pop(key) except KeyError: logger.warning( 'Received unexpected partition revocation for unowned partition: %r', i, exc_info=True) continue if offset is None: logger.debug( 'Skipping commit of unprocessed partition: %r', i) continue offsets_to_commit.append( TopicPartition(i.topic, i.partition, offset)) if offsets_to_commit: logger.debug( 'Committing offset(s) for %s revoked partition(s): %r', len(offsets_to_commit), offsets_to_commit) commit(offsets_to_commit) consumer.subscribe( [self.topic], on_assign=on_assign, on_revoke=on_revoke, ) def commit_offsets(): offsets_to_commit = [] for (topic, partition), offset in owned_partition_offsets.items(): if offset is None: logger.debug( 'Skipping commit of unprocessed partition: %r', (topic, partition)) continue offsets_to_commit.append( TopicPartition(topic, partition, offset)) if offsets_to_commit: logger.debug( 'Committing offset(s) for %s owned partition(s): %r', len(offsets_to_commit), offsets_to_commit) commit(offsets_to_commit) try: i = 0 while True: message = consumer.poll(0.1) if message is None: continue error = message.error() if error is not None: raise Exception(error) key = (message.topic(), message.partition()) if key not in owned_partition_offsets: logger.warning( 'Skipping message for unowned partition: %r', key) continue i = i + 1 owned_partition_offsets[key] = message.offset() + 1 with metrics.timer('eventstream.duration', instance='get_task_kwargs_for_message'): task_kwargs = get_task_kwargs_for_message(message.value()) if task_kwargs is not None: with metrics.timer( 'eventstream.duration', instance='dispatch_post_process_group_task'): self._dispatch_post_process_group_task(**task_kwargs) if i % commit_batch_size == 0: commit_offsets() except KeyboardInterrupt: pass logger.debug('Committing offsets and closing consumer...') commit_offsets() consumer.close()
def test_consumer_rebalance_from_uncommitted_offset(): consumer_group = 'consumer-{}'.format(uuid.uuid1().hex) synchronize_commit_group = 'consumer-{}'.format(uuid.uuid1().hex) messages_delivered = defaultdict(list) def record_message_delivered(error, message): assert error is None messages_delivered[message.topic()].append(message) producer = Producer({ 'bootstrap.servers': os.environ['SENTRY_KAFKA_HOSTS'], 'on_delivery': record_message_delivered, }) with create_topic( partitions=2) as topic, create_topic() as commit_log_topic: # Produce some messages into the topic. for i in range(4): producer.produce(topic, '{}'.format(i).encode('utf8'), partition=i % 2) assert producer.flush( 5) == 0, 'producer did not successfully flush queue' for (topic, partition), offset in { (message.topic(), message.partition()): message.offset() for message in messages_delivered[topic] }.items(): producer.produce( commit_log_topic, key='{}:{}:{}'.format( topic, partition, synchronize_commit_group, ).encode('utf8'), value='{}'.format(offset + 1, ).encode('utf8'), ) assert producer.flush( 5) == 0, 'producer did not successfully flush queue' consumer_a = SynchronizedConsumer( bootstrap_servers=os.environ['SENTRY_KAFKA_HOSTS'], consumer_group=consumer_group, commit_log_topic=commit_log_topic, synchronize_commit_group=synchronize_commit_group, initial_offset_reset='earliest', ) assignments_received = defaultdict(list) def on_assign(consumer, assignment): assignments_received[consumer].append(assignment) consumer_a.subscribe([topic], on_assign=on_assign) consume_until_constraints_met(consumer_a, [ lambda message: assignments_received[consumer_a], collect_messages_recieved(4), ], 10) assert len(assignments_received[consumer_a] ) == 1, 'expected to receive partition assignment' assert set( (i.topic, i.partition) for i in assignments_received[consumer_a][0]) == set([(topic, 0), (topic, 1)]) assignments_received[consumer_a].pop() message = consumer_a.poll(1) assert message is None or message.error( ) is KafkaError._PARTITION_EOF, 'there should be no more messages to recieve' consumer_b = SynchronizedConsumer( bootstrap_servers=os.environ['SENTRY_KAFKA_HOSTS'], consumer_group=consumer_group, commit_log_topic=commit_log_topic, synchronize_commit_group=synchronize_commit_group, initial_offset_reset='earliest', ) consumer_b.subscribe([topic], on_assign=on_assign) consume_until_constraints_met(consumer_a, [ lambda message: assignments_received[consumer_a], ], 10) consume_until_constraints_met(consumer_b, [ lambda message: assignments_received[consumer_b], collect_messages_recieved(2), ], 10) for consumer in [consumer_a, consumer_b]: assert len(assignments_received[consumer][0]) == 1 message = consumer_a.poll(1) assert message is None or message.error( ) is KafkaError._PARTITION_EOF, 'there should be no more messages to recieve' message = consumer_b.poll(1) assert message is None or message.error( ) is KafkaError._PARTITION_EOF, 'there should be no more messages to recieve'
def test_consumer_rebalance_from_partition_start(): consumer_group = 'consumer-{}'.format(uuid.uuid1().hex) synchronize_commit_group = 'consumer-{}'.format(uuid.uuid1().hex) messages_delivered = defaultdict(list) def record_message_delivered(error, message): assert error is None messages_delivered[message.topic()].append(message) producer = Producer({ 'bootstrap.servers': os.environ['SENTRY_KAFKA_HOSTS'], 'on_delivery': record_message_delivered, }) with create_topic( partitions=2) as topic, create_topic() as commit_log_topic: # Produce some messages into the topic. for i in range(4): producer.produce(topic, '{}'.format(i).encode('utf8'), partition=i % 2) assert producer.flush( 5) == 0, 'producer did not successfully flush queue' consumer_a = SynchronizedConsumer( bootstrap_servers=os.environ['SENTRY_KAFKA_HOSTS'], consumer_group=consumer_group, commit_log_topic=commit_log_topic, synchronize_commit_group=synchronize_commit_group, initial_offset_reset='earliest', ) assignments_received = defaultdict(list) def on_assign(consumer, assignment): assignments_received[consumer].append(assignment) consumer_a.subscribe([topic], on_assign=on_assign) # Wait until the first consumer has received its assignments. for i in xrange(10): # this takes a while assert consumer_a.poll(1) is None if assignments_received[consumer_a]: break assert len(assignments_received[consumer_a] ) == 1, 'expected to receive partition assignment' assert set( (i.topic, i.partition) for i in assignments_received[consumer_a][0]) == set([(topic, 0), (topic, 1)]) assignments_received[consumer_a].pop() consumer_b = SynchronizedConsumer( bootstrap_servers=os.environ['SENTRY_KAFKA_HOSTS'], consumer_group=consumer_group, commit_log_topic=commit_log_topic, synchronize_commit_group=synchronize_commit_group, initial_offset_reset='earliest', ) consumer_b.subscribe([topic], on_assign=on_assign) assignments = {} # Wait until *both* consumers have received updated assignments. for consumer in [consumer_a, consumer_b]: for i in xrange(10): # this takes a while assert consumer.poll(1) is None if assignments_received[consumer]: break assert len(assignments_received[consumer] ) == 1, 'expected to receive partition assignment' assert len( assignments_received[consumer] [0]) == 1, 'expected to have a single partition assignment' i = assignments_received[consumer][0][0] assignments[(i.topic, i.partition)] = consumer assert set(assignments.keys()) == set([(topic, 0), (topic, 1)]) for expected_message in messages_delivered[topic]: consumer = assignments[(expected_message.topic(), expected_message.partition())] # Make sure that there are no messages ready to consume. assert consumer.poll(1) is None # Move the committed offset forward for our synchronizing group. producer.produce( commit_log_topic, key='{}:{}:{}'.format( expected_message.topic(), expected_message.partition(), synchronize_commit_group, ).encode('utf8'), value='{}'.format(expected_message.offset() + 1, ).encode('utf8'), ) assert producer.flush( 5) == 0, 'producer did not successfully flush queue' # We should have received a single message. # TODO: Can we also assert that the position is unpaused?) for i in xrange(5): received_message = consumer.poll(1) if received_message is not None: break assert received_message is not None, 'no message received' assert received_message.topic() == expected_message.topic() assert received_message.partition() == expected_message.partition() assert received_message.offset() == expected_message.offset() # We should not be able to continue reading into the topic. # TODO: Can we assert that the position is paused? assert consumer.poll(1) is None
def test_consumer_start_from_committed_offset(): consumer_group = 'consumer-{}'.format(uuid.uuid1().hex) synchronize_commit_group = 'consumer-{}'.format(uuid.uuid1().hex) messages_delivered = defaultdict(list) def record_message_delivered(error, message): assert error is None messages_delivered[message.topic()].append(message) producer = Producer({ 'bootstrap.servers': os.environ['SENTRY_KAFKA_HOSTS'], 'on_delivery': record_message_delivered, }) with create_topic() as topic, create_topic() as commit_log_topic: # Produce some messages into the topic. for i in range(3): producer.produce(topic, '{}'.format(i).encode('utf8')) assert producer.flush( 5) == 0, 'producer did not successfully flush queue' Consumer({ 'bootstrap.servers': os.environ['SENTRY_KAFKA_HOSTS'], 'group.id': consumer_group, }).commit( message=messages_delivered[topic][0], asynchronous=False, ) # Create the synchronized consumer. consumer = SynchronizedConsumer( bootstrap_servers=os.environ['SENTRY_KAFKA_HOSTS'], consumer_group=consumer_group, commit_log_topic=commit_log_topic, synchronize_commit_group=synchronize_commit_group, initial_offset_reset='earliest', ) assignments_received = [] def on_assign(c, assignment): assert c is consumer assignments_received.append(assignment) consumer.subscribe([topic], on_assign=on_assign) # Wait until we have received our assignments. for i in xrange(10): # this takes a while assert consumer.poll(1) is None if assignments_received: break assert len(assignments_received ) == 1, 'expected to receive partition assignment' assert set((i.topic, i.partition) for i in assignments_received[0]) == set([(topic, 0)]) # TODO: Make sure that all partitions are paused on assignment. # Move the committed offset forward for our synchronizing group. message = messages_delivered[topic][0] producer.produce( commit_log_topic, key='{}:{}:{}'.format( message.topic(), message.partition(), synchronize_commit_group, ).encode('utf8'), value='{}'.format(message.offset() + 1, ).encode('utf8'), ) # Make sure that there are no messages ready to consume. assert consumer.poll(1) is None # Move the committed offset forward for our synchronizing group. message = messages_delivered[topic][0 + 1] # second message producer.produce( commit_log_topic, key='{}:{}:{}'.format( message.topic(), message.partition(), synchronize_commit_group, ).encode('utf8'), value='{}'.format(message.offset() + 1, ).encode('utf8'), ) assert producer.flush( 5) == 0, 'producer did not successfully flush queue' # We should have received a single message. # TODO: Can we also assert that the position is unpaused?) for i in xrange(5): message = consumer.poll(1) if message is not None: break assert message is not None, 'no message received' expected_message = messages_delivered[topic][0 + 1] # second message assert message.topic() == expected_message.topic() assert message.partition() == expected_message.partition() assert message.offset() == expected_message.offset() # We should not be able to continue reading into the topic. # TODO: Can we assert that the position is paused? assert consumer.poll(1) is None
def run_streaming_consumer( self, consumer_group, commit_log_topic, synchronize_commit_group, commit_batch_size=100, initial_offset_reset="latest", ): cluster_name = settings.KAFKA_TOPICS[settings.KAFKA_EVENTS]["cluster"] consumer = SynchronizedConsumer( cluster_name=cluster_name, consumer_group=consumer_group, commit_log_topic=commit_log_topic, synchronize_commit_group=synchronize_commit_group, initial_offset_reset=initial_offset_reset, ) owned_partition_offsets = {} def commit(partitions): results = consumer.commit(offsets=partitions, asynchronous=False) errors = [i for i in results if i.error is not None] if errors: raise Exception( "Failed to commit {}/{} partitions: {!r}".format( len(errors), len(partitions), errors)) return results def on_assign(consumer, partitions): logger.info("Received partition assignment: %r", partitions) for i in partitions: if i.offset == OFFSET_INVALID: updated_offset = None elif i.offset < 0: raise Exception( f"Received unexpected negative offset during partition assignment: {i!r}" ) else: updated_offset = i.offset key = (i.topic, i.partition) previous_offset = owned_partition_offsets.get(key, None) if previous_offset is not None and previous_offset != updated_offset: logger.warning( "Received new offset for owned partition %r, will overwrite previous stored offset %r with %r.", key, previous_offset, updated_offset, ) owned_partition_offsets[key] = updated_offset def on_revoke(consumer, partitions): logger.info("Revoked partition assignment: %r", partitions) offsets_to_commit = [] for i in partitions: key = (i.topic, i.partition) try: offset = owned_partition_offsets.pop(key) except KeyError: logger.warning( "Received unexpected partition revocation for unowned partition: %r", i, exc_info=True, ) continue if offset is None: logger.debug( "Skipping commit of unprocessed partition: %r", i) continue offsets_to_commit.append( TopicPartition(i.topic, i.partition, offset)) if offsets_to_commit: logger.debug( "Committing offset(s) for %s revoked partition(s): %r", len(offsets_to_commit), offsets_to_commit, ) commit(offsets_to_commit) consumer.subscribe([self.topic], on_assign=on_assign, on_revoke=on_revoke) def commit_offsets(): offsets_to_commit = [] for (topic, partition), offset in owned_partition_offsets.items(): if offset is None: logger.debug( "Skipping commit of unprocessed partition: %r", (topic, partition)) continue offsets_to_commit.append( TopicPartition(topic, partition, offset)) if offsets_to_commit: logger.debug( "Committing offset(s) for %s owned partition(s): %r", len(offsets_to_commit), offsets_to_commit, ) commit(offsets_to_commit) shutdown_requested = False def handle_shutdown_request(signum: int, frame: Any) -> None: nonlocal shutdown_requested logger.debug("Received signal %r, requesting shutdown...", signum) shutdown_requested = True signal.signal(signal.SIGINT, handle_shutdown_request) signal.signal(signal.SIGTERM, handle_shutdown_request) i = 0 while not shutdown_requested: message = consumer.poll(0.1) if message is None: continue error = message.error() if error is not None: raise Exception(error) key = (message.topic(), message.partition()) if key not in owned_partition_offsets: logger.warning("Skipping message for unowned partition: %r", key) continue i = i + 1 owned_partition_offsets[key] = message.offset() + 1 use_kafka_headers = options.get( "post-process-forwarder:kafka-headers") if use_kafka_headers is True: try: with _sampled_eventstream_timer( instance="get_task_kwargs_for_message_from_headers" ): task_kwargs = get_task_kwargs_for_message_from_headers( message.headers()) if task_kwargs is not None: with _sampled_eventstream_timer( instance="dispatch_post_process_group_task"): if task_kwargs["group_id"] is None: metrics.incr( "eventstream.messages", tags={ "partition": message.partition(), "type": "transactions" }, ) else: metrics.incr( "eventstream.messages", tags={ "partition": message.partition(), "type": "errors" }, ) self._dispatch_post_process_group_task( **task_kwargs) except Exception as error: logger.error("Could not forward message: %s", error, exc_info=True) self._get_task_kwargs_and_dispatch(message) else: self._get_task_kwargs_and_dispatch(message) if i % commit_batch_size == 0: commit_offsets() logger.debug("Committing offsets and closing consumer...") commit_offsets() consumer.close()
def run_post_process_forwarder(self, consumer_group, commit_log_topic, synchronize_commit_group, commit_batch_size=100, initial_offset_reset='latest'): logger.debug('Starting post-process forwarder...') cluster_name = settings.KAFKA_TOPICS[settings.KAFKA_EVENTS]['cluster'] bootstrap_servers = settings.KAFKA_CLUSTERS[cluster_name]['bootstrap.servers'] consumer = SynchronizedConsumer( bootstrap_servers=bootstrap_servers, consumer_group=consumer_group, commit_log_topic=commit_log_topic, synchronize_commit_group=synchronize_commit_group, initial_offset_reset=initial_offset_reset, ) owned_partition_offsets = {} def commit(partitions): results = consumer.commit(offsets=partitions, asynchronous=False) errors = filter(lambda i: i.error is not None, results) if errors: raise Exception( 'Failed to commit %s/%s partitions: %r' % (len(errors), len(partitions), errors)) return results def on_assign(consumer, partitions): logger.debug('Received partition assignment: %r', partitions) for i in partitions: if i.offset == OFFSET_INVALID: updated_offset = None elif i.offset < 0: raise Exception( 'Received unexpected negative offset during partition assignment: %r' % (i,)) else: updated_offset = i.offset key = (i.topic, i.partition) previous_offset = owned_partition_offsets.get(key, None) if previous_offset is not None and previous_offset != updated_offset: logger.warning( 'Received new offset for owned partition %r, will overwrite previous stored offset %r with %r.', key, previous_offset, updated_offset) owned_partition_offsets[key] = updated_offset def on_revoke(consumer, partitions): logger.debug('Revoked partition assignment: %r', partitions) offsets_to_commit = [] for i in partitions: key = (i.topic, i.partition) try: offset = owned_partition_offsets.pop(key) except KeyError: logger.warning( 'Received unexpected partition revocation for unowned partition: %r', i, exc_info=True) continue if offset is None: logger.debug('Skipping commit of unprocessed partition: %r', i) continue offsets_to_commit.append(TopicPartition(i.topic, i.partition, offset)) if offsets_to_commit: logger.debug( 'Committing offset(s) for %s revoked partition(s): %r', len(offsets_to_commit), offsets_to_commit) commit(offsets_to_commit) consumer.subscribe( [self.topic], on_assign=on_assign, on_revoke=on_revoke, ) def commit_offsets(): offsets_to_commit = [] for (topic, partition), offset in owned_partition_offsets.items(): if offset is None: logger.debug('Skipping commit of unprocessed partition: %r', (topic, partition)) continue offsets_to_commit.append(TopicPartition(topic, partition, offset)) if offsets_to_commit: logger.debug( 'Committing offset(s) for %s owned partition(s): %r', len(offsets_to_commit), offsets_to_commit) commit(offsets_to_commit) try: i = 0 while True: message = consumer.poll(0.1) if message is None: continue error = message.error() if error is not None: raise Exception(error) key = (message.topic(), message.partition()) if key not in owned_partition_offsets: logger.warning('Skipping message for unowned partition: %r', key) continue i = i + 1 owned_partition_offsets[key] = message.offset() + 1 with metrics.timer('eventstream.duration', instance='get_task_kwargs_for_message'): task_kwargs = get_task_kwargs_for_message(message.value()) if task_kwargs is not None: with metrics.timer('eventstream.duration', instance='dispatch_post_process_group_task'): self._dispatch_post_process_group_task(**task_kwargs) if i % commit_batch_size == 0: commit_offsets() except KeyboardInterrupt: pass logger.debug('Committing offsets and closing consumer...') commit_offsets() consumer.close()