class NotificationEngine(object):
def __init__(self, config):
self._topics = {}
self._topics['notification_topic'] = config['kafka']['notification_topic']
self._topics['retry_topic'] = config['kafka']['notification_retry_topic']
self._statsd = monascastatsd.Client(name='monasca',
dimensions=BaseProcessor.dimensions)
self._consumer = KafkaConsumer(config['kafka']['url'],
config['zookeeper']['url'],
config['zookeeper']['notification_path'],
config['kafka']['group'],
config['kafka']['alarm_topic'])
self._producer = KafkaProducer(config['kafka']['url'])
self._alarm_ttl = config['processors']['alarm']['ttl']
self._alarms = AlarmProcessor(self._alarm_ttl, config)
self._notifier = NotificationProcessor(config['notification_types'])
def run(self):
finished_count = self._statsd.get_counter(name='alarms_finished_count')
for alarm in self._consumer:
log.debug('Received alarm >|%s|<', str(alarm))
notifications, partition, offset = self._alarms.to_notification(alarm)
if notifications:
sent, failed = self._notifier.send(notifications)
self._producer.publish(self._topics['notification_topic'],
[i.to_json() for i in sent])
self._producer.publish(self._topics['retry_topic'],
[i.to_json() for i in failed])
self._consumer.commit()
finished_count.increment()
class NotificationEngine(object):
def __init__(self, config):
self._topics = {}
self._topics['notification_topic'] = config['kafka'][
'notification_topic']
self._topics['retry_topic'] = config['kafka'][
'notification_retry_topic']
self._statsd = monascastatsd.Client(
name='monasca', dimensions=BaseProcessor.dimensions)
self._consumer = KafkaConsumer(
config['kafka']['url'], config['zookeeper']['url'],
config['zookeeper']['notification_path'], config['kafka']['group'],
config['kafka']['alarm_topic'])
self._producer = KafkaProducer(config['kafka']['url'])
self._alarm_ttl = config['processors']['alarm']['ttl']
self._alarms = AlarmProcessor(self._alarm_ttl, config)
self._notifier = NotificationProcessor(config['notification_types'])
def run(self):
finished_count = self._statsd.get_counter(name='alarms_finished_count')
for alarm in self._consumer:
log.debug('Received alarm >|%s|<', str(alarm))
notifications, partition, offset = self._alarms.to_notification(
alarm)
if notifications:
sent, failed = self._notifier.send(notifications)
self._producer.publish(self._topics['notification_topic'],
[i.to_json() for i in sent])
self._producer.publish(self._topics['retry_topic'],
[i.to_json() for i in failed])
self._consumer.commit()
finished_count.increment()
class Persister(object):
def __init__(self, kafka_conf, zookeeper_conf, repository):
self._data_points = []
self._kafka_topic = kafka_conf.topic
self._database_batch_size = kafka_conf.database_batch_size
self._consumer = KafkaConsumer(
kafka_conf.uri,
zookeeper_conf.uri,
kafka_conf.zookeeper_path,
kafka_conf.group_id,
kafka_conf.topic,
repartition_callback=self._flush,
commit_callback=self._flush,
commit_timeout=kafka_conf.max_wait_time_seconds)
self.repository = repository()
def _flush(self):
if not self._data_points:
return
try:
self.repository.write_batch(self._data_points)
LOG.info("Processed {} messages from topic '{}'".format(
len(self._data_points), self._kafka_topic))
self._data_points = []
self._consumer.commit()
except Exception:
LOG.exception("Error writing to database: {}".format(
self._data_points))
raise
def run(self):
try:
for raw_message in self._consumer:
try:
message = raw_message[1]
data_point = self.repository.process_message(message)
self._data_points.append(data_point)
except Exception:
LOG.exception('Error processing message. Message is '
'being dropped. {}'.format(message))
if len(self._data_points) >= self._database_batch_size:
self._flush()
except:
LOG.exception('Persister encountered fatal exception processing '
'messages. '
'Shutting down all threads and exiting')
os._exit(1)
class Persister(object):
def __init__(self, kafka_conf, zookeeper_conf, repository):
self._data_points = []
self._kafka_topic = kafka_conf.topic
self._database_batch_size = kafka_conf.database_batch_size
self._consumer = KafkaConsumer(
kafka_conf.uri,
zookeeper_conf.uri,
kafka_conf.zookeeper_path,
kafka_conf.group_id,
kafka_conf.topic,
repartition_callback=self._flush,
commit_callback=self._flush,
commit_timeout=kafka_conf.max_wait_time_seconds,
)
self.repository = repository()
def _flush(self):
if not self._data_points:
return
try:
self.repository.write_batch(self._data_points)
LOG.info("Processed {} messages from topic '{}'".format(len(self._data_points), self._kafka_topic))
self._data_points = []
self._consumer.commit()
except Exception:
LOG.exception("Error writing to database: {}".format(self._data_points))
raise
def run(self):
try:
for raw_message in self._consumer:
try:
message = raw_message[1]
data_point = self.repository.process_message(message)
self._data_points.append(data_point)
except Exception:
LOG.exception("Error processing message. Message is " "being dropped. {}".format(message))
if len(self._data_points) >= self._database_batch_size:
self._flush()
except:
LOG.exception(
"Persister encountered fatal exception processing " "messages. " "Shutting down all threads and exiting"
)
os._exit(1)
def __init__(self, config):
self._topics = {}
self._topics['notification_topic'] = config['kafka'][
'notification_topic']
self._topics['retry_topic'] = config['kafka'][
'notification_retry_topic']
self._statsd = monascastatsd.Client(
name='monasca', dimensions=BaseProcessor.dimensions)
self._consumer = KafkaConsumer(
config['kafka']['url'], config['zookeeper']['url'],
config['zookeeper']['notification_path'], config['kafka']['group'],
config['kafka']['alarm_topic'])
self._producer = KafkaProducer(config['kafka']['url'])
self._alarm_ttl = config['processors']['alarm']['ttl']
self._alarms = AlarmProcessor(self._alarm_ttl, config)
self._notifier = NotificationProcessor(config['notification_types'])
def __init__(self, kafka_conf, zookeeper_conf, repository):
self._data_points = []
self._kafka_topic = kafka_conf.topic
self._database_batch_size = kafka_conf.database_batch_size
self._consumer = KafkaConsumer(
kafka_conf.uri,
zookeeper_conf.uri,
kafka_conf.zookeeper_path,
kafka_conf.group_id,
kafka_conf.topic,
repartition_callback=self._flush,
commit_callback=self._flush,
commit_timeout=kafka_conf.max_wait_time_seconds,
)
self.repository = repository()
self.statsd_msg_count = STATSD_CLIENT.get_counter(MESSAGES_CONSUMED, dimensions={"type": self._kafka_topic})
self.statsd_msg_dropped_count = STATSD_CLIENT.get_counter(
MESSAGES_DROPPED, dimensions={"type": self._kafka_topic}
)
self.statsd_flush_error_count = STATSD_CLIENT.get_counter(FLUSH_ERRORS)
self.statsd_kafka_consumer_error_count = STATSD_CLIENT.get_counter(
KAFKA_CONSUMER_ERRORS, dimensions={"topic": self._kafka_topic}
)
def __init__(self, config, interval):
self._topic_name = config['kafka']['periodic'][interval]
self._statsd = monascastatsd.Client(
name='monasca', dimensions=BaseProcessor.dimensions)
zookeeper_path = config['zookeeper']['periodic_path'][interval]
self._consumer = KafkaConsumer(config['kafka']['url'],
config['zookeeper']['url'],
zookeeper_path,
config['kafka']['group'],
self._topic_name)
self._producer = KafkaProducer(config['kafka']['url'])
self._notifier = NotificationProcessor(config['notification_types'])
self._db_repo = get_db_repo(config)
def setUp(self):
self.kafka_client_patcher = mock.patch('kafka.client')
self.kafka_common_patcher = mock.patch('kafka.common')
self.kafka_consumer_patcher = mock.patch('kafka.consumer')
self.kazoo_patcher = mock.patch(
'monasca_common.kafka.consumer.KazooClient')
self.mock_kafka_client = self.kafka_client_patcher.start()
self.mock_kafka_common = self.kafka_common_patcher.start()
self.mock_kafka_consumer = self.kafka_consumer_patcher.start()
self.kazoo_patcher.start()
self.client = self.mock_kafka_client.KafkaClient.return_value
self.consumer = self.mock_kafka_consumer.SimpleConsumer.return_value
self.monasca_kafka_consumer = KafkaConsumer(
FAKE_KAFKA_URL, FAKE_ZOOKEEPER_URL, FAKE_ZOOKEEPER_PATH,
FAKE_KAFKA_CONSUMER_GROUP, FAKE_KAFKA_TOPIC)
def __init__(self, kafka_conf, zookeeper_conf, repository):
self._data_points = []
self._kafka_topic = kafka_conf.topic
self._database_batch_size = kafka_conf.database_batch_size
self._consumer = KafkaConsumer(
kafka_conf.uri,
zookeeper_conf.uri,
kafka_conf.zookeeper_path,
kafka_conf.group_id,
kafka_conf.topic,
repartition_callback=self._flush,
commit_callback=self._flush,
commit_timeout=kafka_conf.max_wait_time_seconds)
self.repository = repository()
def __init__(self, config):
self._retry_interval = config['retry']['interval']
self._retry_max = config['retry']['max_attempts']
self._topics = {}
self._topics['notification_topic'] = config['kafka'][
'notification_topic']
self._topics['retry_topic'] = config['kafka'][
'notification_retry_topic']
self._statsd = monascastatsd.Client(
name='monasca', dimensions=BaseProcessor.dimensions)
self._consumer = KafkaConsumer(
config['kafka']['url'], config['zookeeper']['url'],
config['zookeeper']['notification_retry_path'],
config['kafka']['group'],
config['kafka']['notification_retry_topic'])
self._producer = KafkaProducer(config['kafka']['url'])
self._notifier = NotificationProcessor(config['notification_types'])
def __init__(self, config):
self._topics = {}
self._topics['notification_topic'] = config['kafka']['notification_topic']
self._topics['retry_topic'] = config['kafka']['notification_retry_topic']
self._statsd = monascastatsd.Client(name='monasca',
dimensions=BaseProcessor.dimensions)
self._consumer = KafkaConsumer(config['kafka']['url'],
config['zookeeper']['url'],
config['zookeeper']['notification_path'],
config['kafka']['group'],
config['kafka']['alarm_topic'])
self._producer = KafkaProducer(config['kafka']['url'])
self._alarm_ttl = config['processors']['alarm']['ttl']
self._alarms = AlarmProcessor(self._alarm_ttl, config)
self._notifier = NotificationProcessor(config['notification_types'])
def __init__(self, config, interval):
self._topic_name = config['kafka']['periodic'][interval]
self._statsd = monascastatsd.Client(name='monasca',
dimensions=BaseProcessor.dimensions)
zookeeper_path = config['zookeeper']['periodic_path'][interval]
self._consumer = KafkaConsumer(config['kafka']['url'],
config['zookeeper']['url'],
zookeeper_path,
config['kafka']['group'],
self._topic_name)
self._producer = KafkaProducer(config['kafka']['url'])
self._notifier = NotificationProcessor(config['notification_types'])
self._db_repo = get_db_repo(config)
def __init__(self, kafka_conf, zookeeper_conf, repository):
self._data_points = []
self._kafka_topic = kafka_conf.topic
self._database_batch_size = kafka_conf.database_batch_size
self._consumer = KafkaConsumer(
kafka_conf.uri,
zookeeper_conf.uri,
kafka_conf.zookeeper_path,
kafka_conf.group_id,
kafka_conf.topic,
repartition_callback=self._flush,
commit_callback=self._flush,
commit_timeout=kafka_conf.max_wait_time_seconds,
)
self.repository = repository()
def __init__(self, config):
self._retry_interval = config['retry']['interval']
self._retry_max = config['retry']['max_attempts']
self._topics = {}
self._topics['notification_topic'] = config['kafka']['notification_topic']
self._topics['retry_topic'] = config['kafka']['notification_retry_topic']
self._statsd = monascastatsd.Client(name='monasca',
dimensions=BaseProcessor.dimensions)
self._consumer = KafkaConsumer(config['kafka']['url'],
config['zookeeper']['url'],
config['zookeeper']['notification_retry_path'],
config['kafka']['group'],
config['kafka']['notification_retry_topic'])
self._producer = KafkaProducer(config['kafka']['url'])
self._notifier = NotificationProcessor(config['notification_types'])
class PeriodicEngine(object):
def __init__(self, config, interval):
self._topic_name = config['kafka']['periodic'][interval]
self._statsd = monascastatsd.Client(name='monasca',
dimensions=BaseProcessor.dimensions)
zookeeper_path = config['zookeeper']['periodic_path'][interval]
self._consumer = KafkaConsumer(config['kafka']['url'],
config['zookeeper']['url'],
zookeeper_path,
config['kafka']['group'],
self._topic_name)
self._producer = KafkaProducer(config['kafka']['url'])
self._notifier = NotificationProcessor(config['notification_types'])
self._db_repo = get_db_repo(config)
def _keep_sending(self, alarm_id, original_state):
# Go to DB and check alarm state
try:
current_state = self._db_repo.get_alarm_current_state(alarm_id)
except exceptions.DatabaseException:
log.debug('Database Error. Attempting reconnect')
current_state = self._db_repo.get_alarm_current_state(alarm_id)
# Alarm was deleted
if current_state is None:
return False
# Alarm state changed
if current_state != original_state:
return False
return True
def run(self):
for raw_notification in self._consumer:
partition = raw_notification[0]
offset = raw_notification[1].offset
message = raw_notification[1].message.value
notification_data = json.loads(message)
ntype = notification_data['type']
name = notification_data['name']
addr = notification_data['address']
period = notification_data['period']
notification = Notification(ntype,
partition,
offset,
name,
addr,
period,
notification_data['retry_count'],
notification_data['raw_alarm'])
if self._keep_sending(notification.alarm_id, notification.state):
wait_duration = period - (
time.time() - notification_data['notification_timestamp'])
if wait_duration > 0:
time.sleep(wait_duration)
notification.notification_timestamp = time.time()
self._notifier.send([notification])
self._producer.publish(self._topic_name,
[notification.to_json()])
self._consumer.commit()
class PeriodicEngine(object):
def __init__(self, config, interval):
self._topic_name = config['kafka']['periodic'][interval]
self._statsd = monascastatsd.Client(
name='monasca', dimensions=BaseProcessor.dimensions)
zookeeper_path = config['zookeeper']['periodic_path'][interval]
self._consumer = KafkaConsumer(config['kafka']['url'],
config['zookeeper']['url'],
zookeeper_path,
config['kafka']['group'],
self._topic_name)
self._producer = KafkaProducer(config['kafka']['url'])
self._notifier = NotificationProcessor(config['notification_types'])
self._db_repo = get_db_repo(config)
def _keep_sending(self, alarm_id, original_state):
# Go to DB and check alarm state
try:
current_state = self._db_repo.get_alarm_current_state(alarm_id)
except exceptions.DatabaseException:
log.debug('Database Error. Attempting reconnect')
current_state = self._db_repo.get_alarm_current_state(alarm_id)
# Alarm was deleted
if current_state is None:
return False
# Alarm state changed
if current_state != original_state:
return False
return True
def run(self):
for raw_notification in self._consumer:
partition = raw_notification[0]
offset = raw_notification[1].offset
message = raw_notification[1].message.value
notification_data = json.loads(message)
ntype = notification_data['type']
name = notification_data['name']
addr = notification_data['address']
period = notification_data['period']
notification = Notification(ntype, partition, offset, name, addr,
period,
notification_data['retry_count'],
notification_data['raw_alarm'])
if self._keep_sending(notification.alarm_id, notification.state):
wait_duration = period - (
time.time() - notification_data['notification_timestamp'])
if wait_duration > 0:
time.sleep(wait_duration)
notification.notification_timestamp = time.time()
self._notifier.send([notification])
self._producer.publish(self._topic_name,
[notification.to_json()])
self._consumer.commit()
class PeriodicEngine(object):
def __init__(self, config, period):
self._topic_name = config['kafka']['periodic'][period]
self._statsd = monascastatsd.Client(
name='monasca', dimensions=BaseProcessor.dimensions)
zookeeper_path = config['zookeeper']['periodic_path'][period]
self._consumer = KafkaConsumer(config['kafka']['url'],
config['zookeeper']['url'],
zookeeper_path,
config['kafka']['group'],
self._topic_name)
self._producer = KafkaProducer(config['kafka']['url'])
self._notifier = NotificationProcessor(config)
self._db_repo = get_db_repo(config)
self._period = period
def _keep_sending(self, alarm_id, original_state, type, period):
try:
current_state = self._db_repo.get_alarm_current_state(alarm_id)
except exceptions.DatabaseException:
log.debug('Database Error. Attempting reconnect')
current_state = self._db_repo.get_alarm_current_state(alarm_id)
# Alarm was deleted
if current_state is None:
return False
# Alarm state changed
if current_state != original_state:
return False
# Period changed
if period != self._period:
return False
if type != "webhook":
return False
return True
def run(self):
for raw_notification in self._consumer:
message = raw_notification[1].message.value
notification_data = json.loads(message)
notification = construct_notification_object(
self._db_repo, notification_data)
if notification is None:
self._consumer.commit()
continue
if self._keep_sending(notification.alarm_id, notification.state,
notification.type, notification.period):
wait_duration = notification.period - (
time.time() - notification_data['notification_timestamp'])
if wait_duration > 0:
time.sleep(wait_duration)
notification.notification_timestamp = time.time()
self._notifier.send([notification])
self._producer.publish(self._topic_name,
[notification.to_json()])
self._consumer.commit()
class RetryEngine(object):
def __init__(self, config):
self._retry_interval = config['retry']['interval']
self._retry_max = config['retry']['max_attempts']
self._topics = {}
self._topics['notification_topic'] = config['kafka'][
'notification_topic']
self._topics['retry_topic'] = config['kafka'][
'notification_retry_topic']
self._statsd = monascastatsd.Client(
name='monasca', dimensions=BaseProcessor.dimensions)
self._consumer = KafkaConsumer(
config['kafka']['url'], config['zookeeper']['url'],
config['zookeeper']['notification_retry_path'],
config['kafka']['group'],
config['kafka']['notification_retry_topic'])
self._producer = KafkaProducer(config['kafka']['url'])
self._notifier = NotificationProcessor(config)
self._db_repo = get_db_repo(config)
def run(self):
for raw_notification in self._consumer:
message = raw_notification[1].message.value
notification_data = json.loads(message)
notification = construct_notification_object(
self._db_repo, notification_data)
if notification is None:
self._consumer.commit()
continue
wait_duration = self._retry_interval - (
time.time() - notification_data['notification_timestamp'])
if wait_duration > 0:
time.sleep(wait_duration)
sent, failed = self._notifier.send([notification])
if sent:
self._producer.publish(self._topics['notification_topic'],
[notification.to_json()])
if failed:
notification.retry_count += 1
notification.notification_timestamp = time.time()
if notification.retry_count < self._retry_max:
log.error(u"retry failed for {} with name {} "
u"at {}. "
u"Saving for later retry.".format(
notification.type, notification.name,
notification.address))
self._producer.publish(self._topics['retry_topic'],
[notification.to_json()])
else:
log.error(u"retry failed for {} with name {} "
u"at {} after {} retries. "
u"Giving up on retry.".format(
notification.type, notification.name,
notification.address, self._retry_max))
self._consumer.commit()
class TestKafkaConsumer(unittest.TestCase):
def setUp(self):
self.kafka_client_patcher = mock.patch('kafka.client')
self.kafka_common_patcher = mock.patch('kafka.common')
self.kafka_consumer_patcher = mock.patch('kafka.consumer')
self.kazoo_patcher = mock.patch(
'monasca_common.kafka.consumer.KazooClient')
self.mock_kafka_client = self.kafka_client_patcher.start()
self.mock_kafka_common = self.kafka_common_patcher.start()
self.mock_kafka_consumer = self.kafka_consumer_patcher.start()
self.kazoo_patcher.start()
self.client = self.mock_kafka_client.KafkaClient.return_value
self.consumer = self.mock_kafka_consumer.SimpleConsumer.return_value
self.monasca_kafka_consumer = KafkaConsumer(
FAKE_KAFKA_URL, FAKE_ZOOKEEPER_URL, FAKE_ZOOKEEPER_PATH,
FAKE_KAFKA_CONSUMER_GROUP, FAKE_KAFKA_TOPIC)
def tearDown(self):
self.kafka_client_patcher.stop()
self.kafka_common_patcher.stop()
self.kafka_consumer_patcher.stop()
self.kazoo_patcher.stop()
def test_kafka_consumer_init(self):
self.assertTrue(self.mock_kafka_client.KafkaClient.called)
self.assertTrue(self.mock_kafka_consumer.SimpleConsumer.called)
@mock.patch('monasca_common.kafka.consumer.SetPartitioner')
def test_kafka_consumer_process_messages(self, mock_set_partitioner):
messages = []
for i in range(5):
messages.append("message{}".format(i))
self.consumer.get_message.side_effect = messages
mock_set_partitioner.return_value.failed = False
mock_set_partitioner.return_value.release = False
mock_set_partitioner.return_value.acquired = True
mock_set_partitioner.return_value.__iter__.return_value = [1]
for index, message in enumerate(self.monasca_kafka_consumer):
self.assertEqual(message, messages[index])
@mock.patch('monasca_common.kafka.consumer.datetime')
def test_commit(self, mock_datetime):
self.monasca_kafka_consumer.commit()
self.assertTrue(mock_datetime.datetime.now.called)
self.consumer.commit.assert_called_once_with(
partitions=self.monasca_kafka_consumer._partitions)
@mock.patch('monasca_common.kafka.consumer.SetPartitioner')
def test_iteration_failed_to_acquire_partition(self, mock_set_partitioner):
mock_set_partitioner.return_value.failed = True
try:
list(self.monasca_kafka_consumer)
except Exception as e:
self.assertEqual(e.message, "Failed to acquire partition")
@mock.patch('monasca_common.kafka.consumer.SetPartitioner')
def test_kafka_consumer_reset_when_offset_out_of_range(
self, mock_set_partitioner):
class OffsetOutOfRangeError(Exception):
pass
self.mock_kafka_common.OffsetOutOfRangeError = OffsetOutOfRangeError
self.consumer.get_message.side_effect = [OffsetOutOfRangeError,
"message"]
mock_set_partitioner.return_value.failed = False
mock_set_partitioner.return_value.release = False
mock_set_partitioner.return_value.acquired = True
mock_set_partitioner.return_value.__iter__.return_value = [1]
list(self.monasca_kafka_consumer)
self.consumer.seek.assert_called_once_with(0, 0)
class RetryEngine(object):
def __init__(self, config):
self._retry_interval = config['retry']['interval']
self._retry_max = config['retry']['max_attempts']
self._topics = {}
self._topics['notification_topic'] = config['kafka']['notification_topic']
self._topics['retry_topic'] = config['kafka']['notification_retry_topic']
self._statsd = monascastatsd.Client(name='monasca',
dimensions=BaseProcessor.dimensions)
self._consumer = KafkaConsumer(config['kafka']['url'],
config['zookeeper']['url'],
config['zookeeper']['notification_retry_path'],
config['kafka']['group'],
config['kafka']['notification_retry_topic'])
self._producer = KafkaProducer(config['kafka']['url'])
self._notifier = NotificationProcessor(config['notification_types'])
def run(self):
for raw_notification in self._consumer:
partition = raw_notification[0]
offset = raw_notification[1].offset
message = raw_notification[1].message.value
notification_data = json.loads(message)
ntype = notification_data['type']
name = notification_data['name']
addr = notification_data['address']
notification = Notification(ntype,
partition,
offset,
name,
addr,
notification_data['retry_count'],
notification_data['raw_alarm'])
wait_duration = self._retry_interval - (
time.time() - notification_data['notification_timestamp'])
if wait_duration > 0:
time.sleep(wait_duration)
sent, failed = self._notifier.send([notification])
if sent:
self._producer.publish(self._topics['notification_topic'], sent)
if failed:
notification.retry_count += 1
notification.notification_timestamp = time.time()
if notification.retry_count < self._retry_max:
log.error(u"retry failed for {} with name {} "
u"at {}. "
u"Saving for later retry.".format(ntype, name, addr))
self._producer.publish(self._topics['retry_topic'],
[notification.to_json()])
else:
log.error(u"retry failed for {} with name {} "
u"at {} after {} retries. "
u"Giving up on retry."
.format(ntype, name, addr, self._retry_max))
self._consumer.commit()
class Persister(object):
def __init__(self, kafka_conf, zookeeper_conf, repository):
self._data_points = []
self._kafka_topic = kafka_conf.topic
self._database_batch_size = kafka_conf.database_batch_size
self._consumer = KafkaConsumer(
kafka_conf.uri,
zookeeper_conf.uri,
kafka_conf.zookeeper_path,
kafka_conf.group_id,
kafka_conf.topic,
repartition_callback=self._flush,
commit_callback=self._flush,
commit_timeout=kafka_conf.max_wait_time_seconds,
)
self.repository = repository()
self.statsd_msg_count = STATSD_CLIENT.get_counter(MESSAGES_CONSUMED, dimensions={"type": self._kafka_topic})
self.statsd_msg_dropped_count = STATSD_CLIENT.get_counter(
MESSAGES_DROPPED, dimensions={"type": self._kafka_topic}
)
self.statsd_flush_error_count = STATSD_CLIENT.get_counter(FLUSH_ERRORS)
self.statsd_kafka_consumer_error_count = STATSD_CLIENT.get_counter(
KAFKA_CONSUMER_ERRORS, dimensions={"topic": self._kafka_topic}
)
def _write_batch(self, data_points):
try:
self.repository.write_batch(data_points)
self.statsd_msg_count.increment(len(data_points))
LOG.info("Processed %d messages from topic %s", len(data_points), self._kafka_topic)
self.statsd_msg_dropped_count.increment(0) # make metric avail
self.statsd_flush_error_count.increment(0) # make metric avail
except InvalidUpdateException:
l = len(data_points)
if l > 1:
piv = int(l / 2)
self._write_batch(data_points[0:piv])
if piv < l:
self._write_batch(data_points[piv:])
else:
LOG.error("Error storing message. Message is being dropped: %s", data_points)
self.statsd_msg_dropped_count.increment(1, sample_rate=1.0)
def _flush(self):
if not self._data_points:
return
try:
self._write_batch(self._data_points)
self._data_points = []
self._consumer.commit()
except Exception:
LOG.exception("Error writing to database")
self.statsd_flush_error_count.increment(1, sample_rate=1)
raise
def run(self):
try:
for raw_message in self._consumer:
self.statsd_kafka_consumer_error_count.increment(0, sample_rate=0.001) # make metric avail
message = None
try:
message = raw_message[1]
data_point = self.repository.process_message(message)
self._data_points.append(data_point)
except Exception:
LOG.exception("Error processing message. Message is " "being dropped. %s", message)
self.statsd_msg_dropped_count.increment(1, sample_rate=1.0)
if len(self._data_points) >= self._database_batch_size:
self._flush()
else:
LOG.debug("buffering %d of %d", len(self._data_points), self._database_batch_size)
except Exception:
LOG.exception(
"Persister encountered fatal exception processing " "messages. " "Shutting down all threads and exiting"
)
self.statsd_kafka_consumer_error_count.increment(1, sample_rate=1)
os._exit(1)