class NotificationEngine(object): def __init__(self, config): self._topics = {} self._topics['notification_topic'] = config['kafka']['notification_topic'] self._topics['retry_topic'] = config['kafka']['notification_retry_topic'] self._statsd = monascastatsd.Client(name='monasca', dimensions=BaseProcessor.dimensions) self._consumer = KafkaConsumer(config['kafka']['url'], config['zookeeper']['url'], config['zookeeper']['notification_path'], config['kafka']['group'], config['kafka']['alarm_topic']) self._producer = KafkaProducer(config['kafka']['url']) self._alarm_ttl = config['processors']['alarm']['ttl'] self._alarms = AlarmProcessor(self._alarm_ttl, config) self._notifier = NotificationProcessor(config['notification_types']) def run(self): finished_count = self._statsd.get_counter(name='alarms_finished_count') for alarm in self._consumer: log.debug('Received alarm >|%s|<', str(alarm)) notifications, partition, offset = self._alarms.to_notification(alarm) if notifications: sent, failed = self._notifier.send(notifications) self._producer.publish(self._topics['notification_topic'], [i.to_json() for i in sent]) self._producer.publish(self._topics['retry_topic'], [i.to_json() for i in failed]) self._consumer.commit() finished_count.increment()
class NotificationEngine(object): def __init__(self, config): self._topics = {} self._topics['notification_topic'] = config['kafka'][ 'notification_topic'] self._topics['retry_topic'] = config['kafka'][ 'notification_retry_topic'] self._statsd = monascastatsd.Client( name='monasca', dimensions=BaseProcessor.dimensions) self._consumer = KafkaConsumer( config['kafka']['url'], config['zookeeper']['url'], config['zookeeper']['notification_path'], config['kafka']['group'], config['kafka']['alarm_topic']) self._producer = KafkaProducer(config['kafka']['url']) self._alarm_ttl = config['processors']['alarm']['ttl'] self._alarms = AlarmProcessor(self._alarm_ttl, config) self._notifier = NotificationProcessor(config['notification_types']) def run(self): finished_count = self._statsd.get_counter(name='alarms_finished_count') for alarm in self._consumer: log.debug('Received alarm >|%s|<', str(alarm)) notifications, partition, offset = self._alarms.to_notification( alarm) if notifications: sent, failed = self._notifier.send(notifications) self._producer.publish(self._topics['notification_topic'], [i.to_json() for i in sent]) self._producer.publish(self._topics['retry_topic'], [i.to_json() for i in failed]) self._consumer.commit() finished_count.increment()
class Persister(object): def __init__(self, kafka_conf, zookeeper_conf, repository): self._data_points = [] self._kafka_topic = kafka_conf.topic self._database_batch_size = kafka_conf.database_batch_size self._consumer = KafkaConsumer( kafka_conf.uri, zookeeper_conf.uri, kafka_conf.zookeeper_path, kafka_conf.group_id, kafka_conf.topic, repartition_callback=self._flush, commit_callback=self._flush, commit_timeout=kafka_conf.max_wait_time_seconds) self.repository = repository() def _flush(self): if not self._data_points: return try: self.repository.write_batch(self._data_points) LOG.info("Processed {} messages from topic '{}'".format( len(self._data_points), self._kafka_topic)) self._data_points = [] self._consumer.commit() except Exception: LOG.exception("Error writing to database: {}".format( self._data_points)) raise def run(self): try: for raw_message in self._consumer: try: message = raw_message[1] data_point = self.repository.process_message(message) self._data_points.append(data_point) except Exception: LOG.exception('Error processing message. Message is ' 'being dropped. {}'.format(message)) if len(self._data_points) >= self._database_batch_size: self._flush() except: LOG.exception('Persister encountered fatal exception processing ' 'messages. ' 'Shutting down all threads and exiting') os._exit(1)
class Persister(object): def __init__(self, kafka_conf, zookeeper_conf, repository): self._data_points = [] self._kafka_topic = kafka_conf.topic self._database_batch_size = kafka_conf.database_batch_size self._consumer = KafkaConsumer( kafka_conf.uri, zookeeper_conf.uri, kafka_conf.zookeeper_path, kafka_conf.group_id, kafka_conf.topic, repartition_callback=self._flush, commit_callback=self._flush, commit_timeout=kafka_conf.max_wait_time_seconds, ) self.repository = repository() def _flush(self): if not self._data_points: return try: self.repository.write_batch(self._data_points) LOG.info("Processed {} messages from topic '{}'".format(len(self._data_points), self._kafka_topic)) self._data_points = [] self._consumer.commit() except Exception: LOG.exception("Error writing to database: {}".format(self._data_points)) raise def run(self): try: for raw_message in self._consumer: try: message = raw_message[1] data_point = self.repository.process_message(message) self._data_points.append(data_point) except Exception: LOG.exception("Error processing message. Message is " "being dropped. {}".format(message)) if len(self._data_points) >= self._database_batch_size: self._flush() except: LOG.exception( "Persister encountered fatal exception processing " "messages. " "Shutting down all threads and exiting" ) os._exit(1)
def __init__(self, config): self._topics = {} self._topics['notification_topic'] = config['kafka'][ 'notification_topic'] self._topics['retry_topic'] = config['kafka'][ 'notification_retry_topic'] self._statsd = monascastatsd.Client( name='monasca', dimensions=BaseProcessor.dimensions) self._consumer = KafkaConsumer( config['kafka']['url'], config['zookeeper']['url'], config['zookeeper']['notification_path'], config['kafka']['group'], config['kafka']['alarm_topic']) self._producer = KafkaProducer(config['kafka']['url']) self._alarm_ttl = config['processors']['alarm']['ttl'] self._alarms = AlarmProcessor(self._alarm_ttl, config) self._notifier = NotificationProcessor(config['notification_types'])
def __init__(self, kafka_conf, zookeeper_conf, repository): self._data_points = [] self._kafka_topic = kafka_conf.topic self._database_batch_size = kafka_conf.database_batch_size self._consumer = KafkaConsumer( kafka_conf.uri, zookeeper_conf.uri, kafka_conf.zookeeper_path, kafka_conf.group_id, kafka_conf.topic, repartition_callback=self._flush, commit_callback=self._flush, commit_timeout=kafka_conf.max_wait_time_seconds, ) self.repository = repository() self.statsd_msg_count = STATSD_CLIENT.get_counter(MESSAGES_CONSUMED, dimensions={"type": self._kafka_topic}) self.statsd_msg_dropped_count = STATSD_CLIENT.get_counter( MESSAGES_DROPPED, dimensions={"type": self._kafka_topic} ) self.statsd_flush_error_count = STATSD_CLIENT.get_counter(FLUSH_ERRORS) self.statsd_kafka_consumer_error_count = STATSD_CLIENT.get_counter( KAFKA_CONSUMER_ERRORS, dimensions={"topic": self._kafka_topic} )
def __init__(self, config, interval): self._topic_name = config['kafka']['periodic'][interval] self._statsd = monascastatsd.Client( name='monasca', dimensions=BaseProcessor.dimensions) zookeeper_path = config['zookeeper']['periodic_path'][interval] self._consumer = KafkaConsumer(config['kafka']['url'], config['zookeeper']['url'], zookeeper_path, config['kafka']['group'], self._topic_name) self._producer = KafkaProducer(config['kafka']['url']) self._notifier = NotificationProcessor(config['notification_types']) self._db_repo = get_db_repo(config)
def setUp(self): self.kafka_client_patcher = mock.patch('kafka.client') self.kafka_common_patcher = mock.patch('kafka.common') self.kafka_consumer_patcher = mock.patch('kafka.consumer') self.kazoo_patcher = mock.patch( 'monasca_common.kafka.consumer.KazooClient') self.mock_kafka_client = self.kafka_client_patcher.start() self.mock_kafka_common = self.kafka_common_patcher.start() self.mock_kafka_consumer = self.kafka_consumer_patcher.start() self.kazoo_patcher.start() self.client = self.mock_kafka_client.KafkaClient.return_value self.consumer = self.mock_kafka_consumer.SimpleConsumer.return_value self.monasca_kafka_consumer = KafkaConsumer( FAKE_KAFKA_URL, FAKE_ZOOKEEPER_URL, FAKE_ZOOKEEPER_PATH, FAKE_KAFKA_CONSUMER_GROUP, FAKE_KAFKA_TOPIC)
def __init__(self, kafka_conf, zookeeper_conf, repository): self._data_points = [] self._kafka_topic = kafka_conf.topic self._database_batch_size = kafka_conf.database_batch_size self._consumer = KafkaConsumer( kafka_conf.uri, zookeeper_conf.uri, kafka_conf.zookeeper_path, kafka_conf.group_id, kafka_conf.topic, repartition_callback=self._flush, commit_callback=self._flush, commit_timeout=kafka_conf.max_wait_time_seconds) self.repository = repository()
def __init__(self, config): self._retry_interval = config['retry']['interval'] self._retry_max = config['retry']['max_attempts'] self._topics = {} self._topics['notification_topic'] = config['kafka'][ 'notification_topic'] self._topics['retry_topic'] = config['kafka'][ 'notification_retry_topic'] self._statsd = monascastatsd.Client( name='monasca', dimensions=BaseProcessor.dimensions) self._consumer = KafkaConsumer( config['kafka']['url'], config['zookeeper']['url'], config['zookeeper']['notification_retry_path'], config['kafka']['group'], config['kafka']['notification_retry_topic']) self._producer = KafkaProducer(config['kafka']['url']) self._notifier = NotificationProcessor(config['notification_types'])
def __init__(self, config): self._topics = {} self._topics['notification_topic'] = config['kafka']['notification_topic'] self._topics['retry_topic'] = config['kafka']['notification_retry_topic'] self._statsd = monascastatsd.Client(name='monasca', dimensions=BaseProcessor.dimensions) self._consumer = KafkaConsumer(config['kafka']['url'], config['zookeeper']['url'], config['zookeeper']['notification_path'], config['kafka']['group'], config['kafka']['alarm_topic']) self._producer = KafkaProducer(config['kafka']['url']) self._alarm_ttl = config['processors']['alarm']['ttl'] self._alarms = AlarmProcessor(self._alarm_ttl, config) self._notifier = NotificationProcessor(config['notification_types'])
def __init__(self, config, interval): self._topic_name = config['kafka']['periodic'][interval] self._statsd = monascastatsd.Client(name='monasca', dimensions=BaseProcessor.dimensions) zookeeper_path = config['zookeeper']['periodic_path'][interval] self._consumer = KafkaConsumer(config['kafka']['url'], config['zookeeper']['url'], zookeeper_path, config['kafka']['group'], self._topic_name) self._producer = KafkaProducer(config['kafka']['url']) self._notifier = NotificationProcessor(config['notification_types']) self._db_repo = get_db_repo(config)
def __init__(self, kafka_conf, zookeeper_conf, repository): self._data_points = [] self._kafka_topic = kafka_conf.topic self._database_batch_size = kafka_conf.database_batch_size self._consumer = KafkaConsumer( kafka_conf.uri, zookeeper_conf.uri, kafka_conf.zookeeper_path, kafka_conf.group_id, kafka_conf.topic, repartition_callback=self._flush, commit_callback=self._flush, commit_timeout=kafka_conf.max_wait_time_seconds, ) self.repository = repository()
def __init__(self, config): self._retry_interval = config['retry']['interval'] self._retry_max = config['retry']['max_attempts'] self._topics = {} self._topics['notification_topic'] = config['kafka']['notification_topic'] self._topics['retry_topic'] = config['kafka']['notification_retry_topic'] self._statsd = monascastatsd.Client(name='monasca', dimensions=BaseProcessor.dimensions) self._consumer = KafkaConsumer(config['kafka']['url'], config['zookeeper']['url'], config['zookeeper']['notification_retry_path'], config['kafka']['group'], config['kafka']['notification_retry_topic']) self._producer = KafkaProducer(config['kafka']['url']) self._notifier = NotificationProcessor(config['notification_types'])
class PeriodicEngine(object): def __init__(self, config, interval): self._topic_name = config['kafka']['periodic'][interval] self._statsd = monascastatsd.Client(name='monasca', dimensions=BaseProcessor.dimensions) zookeeper_path = config['zookeeper']['periodic_path'][interval] self._consumer = KafkaConsumer(config['kafka']['url'], config['zookeeper']['url'], zookeeper_path, config['kafka']['group'], self._topic_name) self._producer = KafkaProducer(config['kafka']['url']) self._notifier = NotificationProcessor(config['notification_types']) self._db_repo = get_db_repo(config) def _keep_sending(self, alarm_id, original_state): # Go to DB and check alarm state try: current_state = self._db_repo.get_alarm_current_state(alarm_id) except exceptions.DatabaseException: log.debug('Database Error. Attempting reconnect') current_state = self._db_repo.get_alarm_current_state(alarm_id) # Alarm was deleted if current_state is None: return False # Alarm state changed if current_state != original_state: return False return True def run(self): for raw_notification in self._consumer: partition = raw_notification[0] offset = raw_notification[1].offset message = raw_notification[1].message.value notification_data = json.loads(message) ntype = notification_data['type'] name = notification_data['name'] addr = notification_data['address'] period = notification_data['period'] notification = Notification(ntype, partition, offset, name, addr, period, notification_data['retry_count'], notification_data['raw_alarm']) if self._keep_sending(notification.alarm_id, notification.state): wait_duration = period - ( time.time() - notification_data['notification_timestamp']) if wait_duration > 0: time.sleep(wait_duration) notification.notification_timestamp = time.time() self._notifier.send([notification]) self._producer.publish(self._topic_name, [notification.to_json()]) self._consumer.commit()
class PeriodicEngine(object): def __init__(self, config, interval): self._topic_name = config['kafka']['periodic'][interval] self._statsd = monascastatsd.Client( name='monasca', dimensions=BaseProcessor.dimensions) zookeeper_path = config['zookeeper']['periodic_path'][interval] self._consumer = KafkaConsumer(config['kafka']['url'], config['zookeeper']['url'], zookeeper_path, config['kafka']['group'], self._topic_name) self._producer = KafkaProducer(config['kafka']['url']) self._notifier = NotificationProcessor(config['notification_types']) self._db_repo = get_db_repo(config) def _keep_sending(self, alarm_id, original_state): # Go to DB and check alarm state try: current_state = self._db_repo.get_alarm_current_state(alarm_id) except exceptions.DatabaseException: log.debug('Database Error. Attempting reconnect') current_state = self._db_repo.get_alarm_current_state(alarm_id) # Alarm was deleted if current_state is None: return False # Alarm state changed if current_state != original_state: return False return True def run(self): for raw_notification in self._consumer: partition = raw_notification[0] offset = raw_notification[1].offset message = raw_notification[1].message.value notification_data = json.loads(message) ntype = notification_data['type'] name = notification_data['name'] addr = notification_data['address'] period = notification_data['period'] notification = Notification(ntype, partition, offset, name, addr, period, notification_data['retry_count'], notification_data['raw_alarm']) if self._keep_sending(notification.alarm_id, notification.state): wait_duration = period - ( time.time() - notification_data['notification_timestamp']) if wait_duration > 0: time.sleep(wait_duration) notification.notification_timestamp = time.time() self._notifier.send([notification]) self._producer.publish(self._topic_name, [notification.to_json()]) self._consumer.commit()
class PeriodicEngine(object): def __init__(self, config, period): self._topic_name = config['kafka']['periodic'][period] self._statsd = monascastatsd.Client( name='monasca', dimensions=BaseProcessor.dimensions) zookeeper_path = config['zookeeper']['periodic_path'][period] self._consumer = KafkaConsumer(config['kafka']['url'], config['zookeeper']['url'], zookeeper_path, config['kafka']['group'], self._topic_name) self._producer = KafkaProducer(config['kafka']['url']) self._notifier = NotificationProcessor(config) self._db_repo = get_db_repo(config) self._period = period def _keep_sending(self, alarm_id, original_state, type, period): try: current_state = self._db_repo.get_alarm_current_state(alarm_id) except exceptions.DatabaseException: log.debug('Database Error. Attempting reconnect') current_state = self._db_repo.get_alarm_current_state(alarm_id) # Alarm was deleted if current_state is None: return False # Alarm state changed if current_state != original_state: return False # Period changed if period != self._period: return False if type != "webhook": return False return True def run(self): for raw_notification in self._consumer: message = raw_notification[1].message.value notification_data = json.loads(message) notification = construct_notification_object( self._db_repo, notification_data) if notification is None: self._consumer.commit() continue if self._keep_sending(notification.alarm_id, notification.state, notification.type, notification.period): wait_duration = notification.period - ( time.time() - notification_data['notification_timestamp']) if wait_duration > 0: time.sleep(wait_duration) notification.notification_timestamp = time.time() self._notifier.send([notification]) self._producer.publish(self._topic_name, [notification.to_json()]) self._consumer.commit()
class RetryEngine(object): def __init__(self, config): self._retry_interval = config['retry']['interval'] self._retry_max = config['retry']['max_attempts'] self._topics = {} self._topics['notification_topic'] = config['kafka'][ 'notification_topic'] self._topics['retry_topic'] = config['kafka'][ 'notification_retry_topic'] self._statsd = monascastatsd.Client( name='monasca', dimensions=BaseProcessor.dimensions) self._consumer = KafkaConsumer( config['kafka']['url'], config['zookeeper']['url'], config['zookeeper']['notification_retry_path'], config['kafka']['group'], config['kafka']['notification_retry_topic']) self._producer = KafkaProducer(config['kafka']['url']) self._notifier = NotificationProcessor(config) self._db_repo = get_db_repo(config) def run(self): for raw_notification in self._consumer: message = raw_notification[1].message.value notification_data = json.loads(message) notification = construct_notification_object( self._db_repo, notification_data) if notification is None: self._consumer.commit() continue wait_duration = self._retry_interval - ( time.time() - notification_data['notification_timestamp']) if wait_duration > 0: time.sleep(wait_duration) sent, failed = self._notifier.send([notification]) if sent: self._producer.publish(self._topics['notification_topic'], [notification.to_json()]) if failed: notification.retry_count += 1 notification.notification_timestamp = time.time() if notification.retry_count < self._retry_max: log.error(u"retry failed for {} with name {} " u"at {}. " u"Saving for later retry.".format( notification.type, notification.name, notification.address)) self._producer.publish(self._topics['retry_topic'], [notification.to_json()]) else: log.error(u"retry failed for {} with name {} " u"at {} after {} retries. " u"Giving up on retry.".format( notification.type, notification.name, notification.address, self._retry_max)) self._consumer.commit()
class TestKafkaConsumer(unittest.TestCase): def setUp(self): self.kafka_client_patcher = mock.patch('kafka.client') self.kafka_common_patcher = mock.patch('kafka.common') self.kafka_consumer_patcher = mock.patch('kafka.consumer') self.kazoo_patcher = mock.patch( 'monasca_common.kafka.consumer.KazooClient') self.mock_kafka_client = self.kafka_client_patcher.start() self.mock_kafka_common = self.kafka_common_patcher.start() self.mock_kafka_consumer = self.kafka_consumer_patcher.start() self.kazoo_patcher.start() self.client = self.mock_kafka_client.KafkaClient.return_value self.consumer = self.mock_kafka_consumer.SimpleConsumer.return_value self.monasca_kafka_consumer = KafkaConsumer( FAKE_KAFKA_URL, FAKE_ZOOKEEPER_URL, FAKE_ZOOKEEPER_PATH, FAKE_KAFKA_CONSUMER_GROUP, FAKE_KAFKA_TOPIC) def tearDown(self): self.kafka_client_patcher.stop() self.kafka_common_patcher.stop() self.kafka_consumer_patcher.stop() self.kazoo_patcher.stop() def test_kafka_consumer_init(self): self.assertTrue(self.mock_kafka_client.KafkaClient.called) self.assertTrue(self.mock_kafka_consumer.SimpleConsumer.called) @mock.patch('monasca_common.kafka.consumer.SetPartitioner') def test_kafka_consumer_process_messages(self, mock_set_partitioner): messages = [] for i in range(5): messages.append("message{}".format(i)) self.consumer.get_message.side_effect = messages mock_set_partitioner.return_value.failed = False mock_set_partitioner.return_value.release = False mock_set_partitioner.return_value.acquired = True mock_set_partitioner.return_value.__iter__.return_value = [1] for index, message in enumerate(self.monasca_kafka_consumer): self.assertEqual(message, messages[index]) @mock.patch('monasca_common.kafka.consumer.datetime') def test_commit(self, mock_datetime): self.monasca_kafka_consumer.commit() self.assertTrue(mock_datetime.datetime.now.called) self.consumer.commit.assert_called_once_with( partitions=self.monasca_kafka_consumer._partitions) @mock.patch('monasca_common.kafka.consumer.SetPartitioner') def test_iteration_failed_to_acquire_partition(self, mock_set_partitioner): mock_set_partitioner.return_value.failed = True try: list(self.monasca_kafka_consumer) except Exception as e: self.assertEqual(e.message, "Failed to acquire partition") @mock.patch('monasca_common.kafka.consumer.SetPartitioner') def test_kafka_consumer_reset_when_offset_out_of_range( self, mock_set_partitioner): class OffsetOutOfRangeError(Exception): pass self.mock_kafka_common.OffsetOutOfRangeError = OffsetOutOfRangeError self.consumer.get_message.side_effect = [OffsetOutOfRangeError, "message"] mock_set_partitioner.return_value.failed = False mock_set_partitioner.return_value.release = False mock_set_partitioner.return_value.acquired = True mock_set_partitioner.return_value.__iter__.return_value = [1] list(self.monasca_kafka_consumer) self.consumer.seek.assert_called_once_with(0, 0)
class RetryEngine(object): def __init__(self, config): self._retry_interval = config['retry']['interval'] self._retry_max = config['retry']['max_attempts'] self._topics = {} self._topics['notification_topic'] = config['kafka']['notification_topic'] self._topics['retry_topic'] = config['kafka']['notification_retry_topic'] self._statsd = monascastatsd.Client(name='monasca', dimensions=BaseProcessor.dimensions) self._consumer = KafkaConsumer(config['kafka']['url'], config['zookeeper']['url'], config['zookeeper']['notification_retry_path'], config['kafka']['group'], config['kafka']['notification_retry_topic']) self._producer = KafkaProducer(config['kafka']['url']) self._notifier = NotificationProcessor(config['notification_types']) def run(self): for raw_notification in self._consumer: partition = raw_notification[0] offset = raw_notification[1].offset message = raw_notification[1].message.value notification_data = json.loads(message) ntype = notification_data['type'] name = notification_data['name'] addr = notification_data['address'] notification = Notification(ntype, partition, offset, name, addr, notification_data['retry_count'], notification_data['raw_alarm']) wait_duration = self._retry_interval - ( time.time() - notification_data['notification_timestamp']) if wait_duration > 0: time.sleep(wait_duration) sent, failed = self._notifier.send([notification]) if sent: self._producer.publish(self._topics['notification_topic'], sent) if failed: notification.retry_count += 1 notification.notification_timestamp = time.time() if notification.retry_count < self._retry_max: log.error(u"retry failed for {} with name {} " u"at {}. " u"Saving for later retry.".format(ntype, name, addr)) self._producer.publish(self._topics['retry_topic'], [notification.to_json()]) else: log.error(u"retry failed for {} with name {} " u"at {} after {} retries. " u"Giving up on retry." .format(ntype, name, addr, self._retry_max)) self._consumer.commit()
class Persister(object): def __init__(self, kafka_conf, zookeeper_conf, repository): self._data_points = [] self._kafka_topic = kafka_conf.topic self._database_batch_size = kafka_conf.database_batch_size self._consumer = KafkaConsumer( kafka_conf.uri, zookeeper_conf.uri, kafka_conf.zookeeper_path, kafka_conf.group_id, kafka_conf.topic, repartition_callback=self._flush, commit_callback=self._flush, commit_timeout=kafka_conf.max_wait_time_seconds, ) self.repository = repository() self.statsd_msg_count = STATSD_CLIENT.get_counter(MESSAGES_CONSUMED, dimensions={"type": self._kafka_topic}) self.statsd_msg_dropped_count = STATSD_CLIENT.get_counter( MESSAGES_DROPPED, dimensions={"type": self._kafka_topic} ) self.statsd_flush_error_count = STATSD_CLIENT.get_counter(FLUSH_ERRORS) self.statsd_kafka_consumer_error_count = STATSD_CLIENT.get_counter( KAFKA_CONSUMER_ERRORS, dimensions={"topic": self._kafka_topic} ) def _write_batch(self, data_points): try: self.repository.write_batch(data_points) self.statsd_msg_count.increment(len(data_points)) LOG.info("Processed %d messages from topic %s", len(data_points), self._kafka_topic) self.statsd_msg_dropped_count.increment(0) # make metric avail self.statsd_flush_error_count.increment(0) # make metric avail except InvalidUpdateException: l = len(data_points) if l > 1: piv = int(l / 2) self._write_batch(data_points[0:piv]) if piv < l: self._write_batch(data_points[piv:]) else: LOG.error("Error storing message. Message is being dropped: %s", data_points) self.statsd_msg_dropped_count.increment(1, sample_rate=1.0) def _flush(self): if not self._data_points: return try: self._write_batch(self._data_points) self._data_points = [] self._consumer.commit() except Exception: LOG.exception("Error writing to database") self.statsd_flush_error_count.increment(1, sample_rate=1) raise def run(self): try: for raw_message in self._consumer: self.statsd_kafka_consumer_error_count.increment(0, sample_rate=0.001) # make metric avail message = None try: message = raw_message[1] data_point = self.repository.process_message(message) self._data_points.append(data_point) except Exception: LOG.exception("Error processing message. Message is " "being dropped. %s", message) self.statsd_msg_dropped_count.increment(1, sample_rate=1.0) if len(self._data_points) >= self._database_batch_size: self._flush() else: LOG.debug("buffering %d of %d", len(self._data_points), self._database_batch_size) except Exception: LOG.exception( "Persister encountered fatal exception processing " "messages. " "Shutting down all threads and exiting" ) self.statsd_kafka_consumer_error_count.increment(1, sample_rate=1) os._exit(1)