def error_cb(self, err): print('error_cb --------> {}'.format(err)) if err.code() == KafkaError._ALL_BROKERS_DOWN: raise KafkaException('ERROR: all brokers down...') else: print(err.code()) raise KafkaException(err.code())
def base_callback(self, err, msg): self._producer._log.debug( "Entering base callback with err=%s and msg=%s.", str(err), str(msg)) if err is not None: self._producer._log.debug("Callback failed (non-None err).") super().failure(KafkaException(err)) elif msg is None: self._producer._log.debug("Callback failed (None msg).") super().failure(KafkaException(KafkaError.UNKNOWN, 'null msg')) elif msg.error() != None: self._producer._log.debug("Callback failed (non-None msg.error).") super().failure(KafkaException(msg.error())) else: # success self._producer._log.debug("Callback success.") metadata = RecordMetadata( msg.topic(), msg.partition(), TopicPartition(msg.topic(), msg.partition()), msg.offset(), msg.timestamp()[1] if msg.timestamp()[0] != TIMESTAMP_NOT_AVAILABLE else int(time() * 1000), None, self.key_len, self.value_len, -1) super().success(metadata) self._producer._log.debug( "Finished with base callback with err=%s and msg=%s.", str(err), str(msg))
def _fail_fast_consume(err: Any, partitions: Any) -> None: if err is not None: print(f"Kafka consumer commit error: {err}", file=sys.stderr) raise KafkaException(err) for p in partitions: # check for partition-specific commit errors if p.error: print( f"Kafka consumer commit error: {p.error}", file=sys.stderr, ) raise KafkaException(p.error)
def fail_fast(err, partitions): if err is not None: print("Kafka consumer commit error: {}".format(err)) print("Bailing out...") # TODO: should it be sys.exit(-1)? raise KafkaException(err) for p in partitions: # check for partition-specific commit errors if p.error: print("Kafka consumer commit error: {}".format(p.error)) print("Bailing out...") # TODO: should it be sys.exit(-1)? raise KafkaException(err) #print("Kafka consumer commit successful") pass
def produce(self, topic, partition, key, value, timeout_s): self.last_msg = None self.producer.produce(topic, key=key.encode('utf-8'), value=value.encode('utf-8'), partition=partition, callback=lambda e, m: self.on_delivery(e, m)) self.producer.flush(timeout_s) msg = self.last_msg if msg == None: raise KafkaException(KafkaError(KafkaError._MSG_TIMED_OUT)) if msg.error() != None: raise KafkaException(msg.error()) assert msg.offset() != None return msg.offset()
def _receive_event(self, timeout=1) -> typing.Optional[bytes]: try: logger.debug("Start polling the message from queue...") msg = self.consumer.poll(timeout=timeout) logger.debug("Pooling was ended, checking returned data...") except Exception as e: logger.warning(f"Error polling: {e}") return None if msg is None: logger.info( "Polling timed out after %s sec. No queue message was received.", timeout) return None if msg.error(): # Error or event if msg.error().code() == KafkaError._PARTITION_EOF: # End of partition event logger.info( f"{msg.topic()} [msg.partition()] reached end at offset {msg.offset()}" ) else: # Error kafka_error = msg.error() ke = KafkaException(kafka_error) if kafka_error.code() in [ KafkaError._TIMED_OUT_QUEUE, KafkaError._TIMED_OUT ]: # ordinary error messages when no message was received. logger.warning( f"[TIMED_OUT errors] no messages were consumed throughout timeout - {timeout} sec: {ke}" ) elif kafka_error.code() == KafkaError._TRANSPORT: # when 'transport' was broken. Usually it'll be renewed automatically, but not all the time. logger.error( f"[Broker transport failure] Calling to re-subscribe. Error: {ke}" ) self.subscribe_to_topic( ) # This is not 100% needed but it's better to have it here for now. else: ke = KafkaException(kafka_error) logger.error(f"Error occurred: {ke}") return None else: # Proper message return msg.value()
def on_rebalance(consumer: Any, partitions: List[Any]) -> None: for p in partitions: if p.error: raise KafkaException(p.error) print("Kafka partitions rebalanced: {} / {}".format( consumer, partitions), file=sys.stderr)
def consume(conf, topic, proto_name): consumer = Consumer(conf) running = True cnt = 0 try: consumer.subscribe([topic]) while running: msg: Message = consumer.poll(timeout=1.0) if msg is None: continue if msg.error(): if msg.error().code() == KafkaError._PARTITION_EOF: # End of partition event sys.stderr.write('%% %s [%d] reached end at offset %d\n' % (msg.topic(), msg.partition(), msg.offset())) elif msg.error(): raise KafkaException(msg.error()) else: cnt += 1 if proto_name == "": print(msg.value()) else: decode_proto(msg, proto_name) print("Consumed ", cnt) finally: # Close down consumer to commit final offsets. consumer.close()
def consume_loop(consumerObj, topics): try: consumerObj.subscribe(topics) while running: msg = consumerObj.poll(timeout=1.0) if msg is None: continue if msg.error(): if msg.error().code() == KafkaError._PARTITION_EOF: # End of partition event sys.stderr.write( '%% %s [%d] reached end at offset %d\n' % (msg.topic(), msg.partition(), msg.offset())) elif msg.error(): raise KafkaException(msg.error()) else: consumerObj.commit(asynchronous=False) # msg_process(msg) print("Start processing message - ", str(msg), msg.key(), msg.value()) finally: # Close down consumer to commit final offsets. consumerObj.close()
def read(self, timeout: int = 0) -> Dict[Tuple[str, int], List[Message]]: msg_count = 0 msg_dict: Dict[Tuple[str, int], List[Any]] = defaultdict(list) try: counter = 0 start_time = datetime.now() while counter < 2: msg = self.consumer.poll(timeout=5.0) if msg is None: counter += 1 continue counter = 0 if msg.error(): raise KafkaException(msg.error()) message = Message(msg.offset(), msg.key(), msg.value(), msg.timestamp()[1], msg.headers()) msg_dict[(msg.topic(), msg.partition())].append(message) msg_count += 1 if not msg_count % 100000: logger.debug( f"So far read {msg_count} messages from kafka") if datetime.now() - start_time > timedelta(seconds=timeout): logger.info( f'Reached timeout: {timeout}s for reading messages.') break logger.info("Done with reading") except KeyboardInterrupt: logger.info("%% Aborted by user\n") finally: self.consumer.close() return msg_dict
def run(self): LOG.info(f"{self.prefix} - Processor is running. Awaiting msgs.") for msg in iter(self): if msg.error(): LOG.error("%s - Consumer error: %s", self.prefix, msg.error()) kafka_failures.labels(reporter=self.reporter).inc() raise KafkaException(msg.error()) try: msg = json.loads(msg.value().decode("utf-8")) self.handle_msg(msg) except json.decoder.JSONDecodeError: kafka_failures.labels(reporter=self.reporter).inc() LOG.error( 'Unable to decode kafka message: %s - %s', msg.value(), self.prefix ) except Exception as err: processor_requests_failures.labels( reporter=self.reporter, account_number=msg['input']['host']['account'] ).inc() LOG.error( 'An error occurred during message processing: %s - %s', repr(err), self.prefix ) finally: self.consumer.commit()
def fail_fast(err, msg): if err is not None: print("Kafka producer delivery error: {}".format(err), file=sys.stderr) print("Bailing out...", file=sys.stderr) # TODO: should it be sys.exit(-1)? raise KafkaException(err)
def basic_consume_loop(consumer, topics): try: consumer.subscribe(topics) while running: msg = consumer.poll(timeout=1.0) if msg is None: continue if msg.error(): if msg.error().code() == KafkaError._PARTITION_EOF: # End of partition event sys.stderr.write( '%% %s [%d] reached end at offset %d\n' % (msg.topic(), msg.partition(), msg.offset())) elif msg.error(): raise KafkaException(msg.error()) else: # msg_process(msg) # print(dir(msg)) print(msg.value()) print(msg.partition()) finally: # Close down consumer to commit final offsets. consumer.close()
def read_summary(self): print('SummaryDataReader is ready to consume the data from %s' % self.summary_topic_name) self.kafka_client.subscribe([self.summary_topic_name]) try: while True: msg = self.kafka_client.consumer.poll(1.0) if msg is None: continue if msg.error(): # Error or event if msg.error().code() == KafkaError._PARTITION_EOF: # End of partition event print('%% %s [%d] reached end at offset %d\n' % (msg.topic(), msg.partition(), msg.offset())) elif msg.error(): # Error raise KafkaException(msg.error()) else: # Proper message print('%% %s [%d] at offset %d with key %s:\n' % (msg.topic(), msg.partition(), msg.offset(), str(msg.key()))) print(msg.value()) except Exception as err: logging.error(err) # Close down consumer to commit final offsets. finally: self.kafka_client.close_consumer()
def _consume(group_id, topic, n, max_messages): config = { "bootstrap.servers": "localhost:9094", "group.id": group_id, "auto.offset.reset": "beginning", "enable.partition.eof": "true", "enable.auto.commit": "false", } consumer = Consumer(config) consumer.subscribe(topics=[topic]) messages = 0 while True: if messages == max_messages: return msg = consumer.consume(num_messages=n, timeout=5) if len(msg) == 0: continue for m in msg: if m.error(): if m.error().code() == KafkaError._PARTITION_EOF: return elif m.error(): raise KafkaException(m.error()) else: messages += 1 if messages == max_messages: break consumer.commit(asynchronous=False)
def consume(self): consumer_kafka_conf = { "bootstrap.servers": self.bootstrap_servers, 'group.id': self.group_id, 'auto.offset.reset': 'smallest' } consumer = Consumer(consumer_kafka_conf) try: consumer.assign( [TopicPartition(topic=self.topic, partition=self.partition)]) while self.stop is False: msg = consumer.poll(timeout=1.0) if msg is None: sleep(0.2) continue if msg.error(): if msg.error().code() == KafkaError._PARTITION_EOF: # End of partition event sys.stderr.write( '%% %s [%d] reached end at offset %d\n' % (msg.topic(), msg.partition(), msg.offset())) elif msg.error(): raise KafkaException(msg.error()) else: self.model.train(msg.value().strip(), self.group_id) if self.model.count == self.end: self.stop = True finally: # Close down consumer to commit final offsets. consumer.close()
def main() -> None: from notifications.worker import execute_message consumer = Consumer( { "bootstrap.servers": "kafka:29092", "group.id": "notifications", "auto.offset.reset": "earliest", } ) consumer.subscribe(["notifications"]) try: while True: msg = consumer.poll(timeout=1.0) if msg is None: continue if msg.error(): raise KafkaException(msg.error()) else: try: data = json.loads(msg.value()) execute_message(data) except Exception as e: print(msg.value()) print("Failed to handle message " + str(e)) except KeyboardInterrupt: sys.stderr.write("%% Aborted by user\n") finally: # Close down consumer to commit final offsets. consumer.close()
def read(self) -> Iterator[Message]: # sometimes it is better to make constant call to KAFKA # to keep the connection alive. timedout = False last_message_time = time.time() while not timedout: msg = self.consumer.poll(timeout=self.settings.poll_timeout) if msg: if msg.error(): if msg.error().code() == KafkaError._PARTITION_EOF: self.logger.warning("Reaching EOF") break else: raise KafkaException(msg.error()) else: self.logger.info("Read {}, {}".format( msg.topic(), msg.offset())) self.last_msg = msg yield Message.deserialize(msg.value()) last_message_time = time.time() if self.settings.timeout > 0: time_since_last_message = time.time() - last_message_time timedout = time_since_last_message > self.settings.timeout
def run(self): global producer_total, consumer_total, consumer_topics, consumer_timeout self.consumer.subscribe(consumer_topics) msg = None while self.runflag: try: msg = self.consumer.poll(timeout=consumer_timeout) if msg is None: raise AttributeError if msg.error(): raise KafkaException(msg.error()) consumer_total += 1 value = msg.value().decode('utf-8') socketio.emit('newMessage', {'msg': value}, namespace='/consumer') print("-> consumer recv: ", value) smsg = "EPD Simulator Totals --> Produced: {} Consumed: {}".format( producer_total, consumer_total) socketio.emit('newMessage', {'msg': smsg}, namespace='/epd') #print(msg.key().decode('utf-8'), msg.value().decode('utf-8')) gevent.sleep(0) except KafkaException: gevent.sleep(0.5) if msg.error().code() != KafkaError._PARTITION_EOF: print("KafkaException: ", msg.error()) # else: # print("KafkaException: ", msg.error()) except AttributeError: gevent.sleep(0.5)
def consume(self) -> Any: """ Consume messages from a give list of topic :return: """ records = [] start = datetime.now() try: while True: msg = self.consumer.poll(timeout=self.consumer_poll_timeout) end = datetime.now() # The consumer exceeds consume timeout if (end - start) > timedelta( seconds=self.consumer_total_timeout): # Exceed the consume timeout break if msg is None: continue if msg.error(): # Hit the EOF of partition if msg.error().code() == KafkaError._PARTITION_EOF: continue else: raise KafkaException(msg.error()) else: records.append(msg.value()) except Exception as e: LOGGER.exception(e) finally: return records
def consume(self): try: while True: msg = self.poll(timeout=1.0) if msg is None: continue if msg.error(): # Error or event if msg.error().code() == KafkaError._PARTITION_EOF: # End of partition event sys.stderr.write( '%% %s [%d] reached end at offset %d\n' % (msg.topic(), msg.partition(), msg.offset())) elif msg.error(): # Error raise KafkaException(msg.error()) else: # Proper message sys.stderr.write('%% %s [%d] at offset %d with key %s:\n' % (msg.topic(), msg.partition(), msg.offset(), str(msg.key()))) print(msg.value()) except KeyboardInterrupt: sys.stderr.write('%% Aborted by user\n') # Close down consumer to commit final offsets. self.close()
def receive_event(self, timeout=5) -> typing.Optional[bytes]: msg = None try: msg = self.consumer.poll(timeout=timeout) except Exception as e: logger.warning("Error polling: {e}") return None if msg is None: logger.warning(f"Polling timed out after {timeout} sec") return None if msg.error(): # Error or event if msg.error().code() == KafkaError._PARTITION_EOF: # End of partition event logger.info( f"{msg.topic()} [msg.partition()] reached end at offset {msg.offset()}" ) else: # Error ke = KafkaException(msg.error()) # raise e logger.error(f"Error occurred: {ke}") return None else: # Proper message return msg.value()
def test_submit_upstream_alert_failure(self): """Test that a KafkaException results in an AlertSubmissionException.""" t = Target.objects.create(name='test name', ra=1, dec=2) with patch('tom_scimma.scimma.Stream.open', mock_open(read_data='data')) as mock_stream: mock_stream().write.side_effect = KafkaException() with self.assertRaises(AlertSubmissionException): SCIMMABroker().submit_upstream_alert(target=t, observation_record=None)
def consumeFromKafka(self, topic): def print_assignment(consumer, partitions): for partition in partitions: print('Assignment:', partition) # Subscribe to topic self.consumer.subscribe(topic, on_assign=print_assignment) try: while True: # Consume form topic msg = self.consumer.poll(timeout=1.0) if msg is None: continue if msg.error(): if msg.error().code() == KafkaError._PARTITION_EOF: # End of partition event pass # sys.stderr.write('%s partition[%d] reached end at offset %d\n' %(msg.topic(), msg.partition(), msg.offset())) elif msg.error(): raise KafkaException(msg.error()) else: yield msg.value().decode('utf-8') except KeyboardInterrupt: sys.stderr.write('%% Aborted by user KeyboardInterrupt \n') self.consumer.close()
def test_notify_sources_application_availability_kafka_exception( self, mock_kafka_producer, ): """Test notify source application availability handling KafkaException.""" kafka_producer = mock_kafka_producer(self.sources_kafka_config) kafka_producer.produce.side_effect = KafkaException(KafkaError(5)) with override_settings( LISTENER_SERVER=self.listener_server, LISTENER_PORT=self.listener_port, SOURCES_STATUS_TOPIC=self.sources_kafka_topic, SOURCES_RESOURCE_TYPE=self.sources_resource_type, SOURCES_AVAILABILITY_EVENT_TYPE=self.sources_availability_event_type, SOURCES_ENABLE_DATA_MANAGEMENT_FROM_KAFKA=True, ): with self.assertRaises(KafkaProducerException): sources.notify_application_availability( self.account_number, self.application_id, availability_status=self.available_status, ) kafka_producer.produce.assert_called_with( topic=self.sources_kafka_topic, value=json.dumps(self.kafka_payload), headers={ "x-rh-identity": self.headers["X-RH-IDENTITY"], "event_type": self.sources_availability_event_type, }, callback=_check_response, ) kafka_producer.flush.assert_not_called()
async def read_stream(self): # 监听kafka try: while True: loop = asyncio.get_event_loop() message = await loop.run_in_executor(None, self.consumer.poll) if message is None: continue if message.error(): logger.exception(message.error()) raise KafkaException(message.error()) else: data = json.loads(message.value().decode("utf-8")) try: finish = await self.handler(data) except Exception as e: logger.info(f"handler解析出现异常[{e}]") time.sleep(1) continue if finish: self.consumer.commit(asynchronus=True) # 不需要等待到触发回调函数之后.(?) except Exception as e: logger.error(f"kafka error.[{e}]") return
def fetch_results(self, num=1): self.establishCon(self.r_tpoic) rs = [] attempts = 5 while (num > 0): msg = self.consumer.poll(timeout=1.0) if msg is None: # stop consumer when reaching end attempts -= 1 print('attemtps left %d', attempts) if (attempts <= 0): break continue if (msg and msg.error()): print("raising error!") raise KafkaException(msg.error()) else: sys.stderr.write( '%% Fetching Topic %s [%d] at offset %d with key %s:\n' % (msg.topic(), msg.partition(), msg.offset(), str( msg.key()))) # load from partitions, returned jd is a list rs.append(json.loads(msg.value().decode('utf-8'))) num -= 1 # self.consumer.commit(async=False) self.closeCon() return rs
def test_consumer(self): """ 消息消费者 """ conf = { 'bootstrap.servers': self.broker, 'group.id': self.group_id, 'default.topic.config': { 'auto.offset.reset': 'smallest' } } c = Consumer(conf) c.subscribe(self.topic_con) try: while True: msg = c.poll(timeout=1.0) if msg is None: continue if msg.error(): if msg.error().code() == KafkaError._PARTITION_EOF: print(msg.topic(), msg.partition(), msg.offset()) elif msg.error(): raise KafkaException(msg.error()) else: print('%% %s [%d] at offset %d with key %s:\n' % (msg.topic(), msg.partition(), msg.offset(), str(msg.key()))) print(msg.value()) except KeyboardInterrupt: print('%% Aborted by user\n') finally: c.close()
def run(self): try: partition_ends = 0 total_parts, partitions = self._partitions() self.consumer.assign(partitions) while True: msg = self.consumer.poll(timeout=0.5) if msg is None: continue if msg.error(): if msg.error().code() == KafkaError._PARTITION_EOF: eprint( f'{msg.topic()} reached end of partition [{msg.partition()}] at offset {msg.offset()}' ) partition_ends += 1 if partition_ends == total_parts: break elif msg.error(): raise KafkaException(msg.error()) else: record = { 'key': self._decode(self.key_decoder, msg.key()), 'payload': self._decode(self.value_decoder, msg.value()), 'topic': msg.topic(), 'partition': msg.partition(), 'offset': msg.offset(), 'timestamp': msg.timestamp()[1] } print(json.dumps(record)) finally: self.consumer.close()
def perform_consume(reset_offset_beginning: bool = False): """Perform consumer.""" conf = { 'bootstrap.servers': '127.0.0.1:9092', 'group.id': 'local-consumer', 'session.timeout.ms': 6000, 'auto.offset.reset': 'earliest' } topic_name = 'local.test.topic' c = Consumer(conf) if reset_offset_beginning: c.subscribe([topic_name], on_assign=on_assign) else: c.subscribe([topic_name]) try: while True: msg = c.poll(timeout=1.0) if msg is None: print('Waiting for message or event/error in poll()') elif msg.error(): raise KafkaException(msg.error()) else: print( f'{msg.topic()}[{msg.partition()}] at offset {msg.offset()}, key: {msg.key()}' ) # noqa message = json.loads(msg.value()) print(message) except KeyboardInterrupt: print('aborted by user.') finally: c.close()