def __init__(self, config): self._topics = {} self._topics['notification_topic'] = config['kafka']['notification_topic'] self._topics['retry_topic'] = config['kafka']['notification_retry_topic'] self._statsd = monascastatsd.Client(name='monasca', dimensions=BaseProcessor.dimensions) self._consumer = KafkaConsumer(config['kafka']['url'], config['zookeeper']['url'], config['zookeeper']['notification_path'], config['kafka']['group'], config['kafka']['alarm_topic']) self._producer = KafkaProducer(config['kafka']['url']) if 'ssl' in config['mysql']: ssl_config = config['mysql']['ssl'] else: ssl_config = None self._alarms = AlarmProcessor(config['processors']['alarm']['ttl'], config['mysql']['host'], config['mysql']['user'], config['mysql']['passwd'], config['mysql']['db'], ssl_config) self._notifier = NotificationProcessor(config['notification_types'])
class NotificationEngine(object): def __init__(self, config): self._topics = {} self._topics['notification_topic'] = config['kafka']['notification_topic'] self._topics['retry_topic'] = config['kafka']['notification_retry_topic'] self._statsd = monascastatsd.Client(name='monasca', dimensions=BaseProcessor.dimensions) self._consumer = KafkaConsumer(config['kafka']['url'], config['zookeeper']['url'], config['zookeeper']['notification_path'], config['kafka']['group'], config['kafka']['alarm_topic']) self._producer = KafkaProducer(config['kafka']['url']) self._alarm_ttl = config['processors']['alarm']['ttl'] self._alarms = AlarmProcessor(self._alarm_ttl, config) self._notifier = NotificationProcessor(config['notification_types']) def run(self): finished_count = self._statsd.get_counter(name='alarms_finished_count') for alarm in self._consumer: log.debug('Received alarm >|%s|<', str(alarm)) notifications, partition, offset = self._alarms.to_notification(alarm) if notifications: sent, failed = self._notifier.send(notifications) self._producer.publish(self._topics['notification_topic'], [i.to_json() for i in sent]) self._producer.publish(self._topics['retry_topic'], [i.to_json() for i in failed]) self._consumer.commit() finished_count.increment()
def __init__(self, config): self._topics = {} self._topics['notification_topic'] = config['kafka'][ 'notification_topic'] self._topics['retry_topic'] = config['kafka'][ 'notification_retry_topic'] self._statsd = monascastatsd.Client( name='monasca', dimensions=BaseProcessor.dimensions) self._consumer = KafkaConsumer( config['kafka']['url'], config['zookeeper']['url'], config['zookeeper']['notification_path'], config['kafka']['group'], config['kafka']['alarm_topic']) self._producer = KafkaProducer(config['kafka']['url']) self._alarm_ttl = config['processors']['alarm']['ttl'] self._alarms = AlarmProcessor(self._alarm_ttl, config) self._notifier = NotificationProcessor(config['notification_types'])
class NotificationEngine(BaseEngine): def __init__(self, config): super(NotificationEngine, self).__init__(config, config['kafka']['alarm_topic'], config['zookeeper']['notification_path']) self._topics = {} self._topics['notification_topic'] = config['kafka']['notification_topic'] self._topics['retry_topic'] = config['kafka']['notification_retry_topic'] self._alarm_ttl = config['processors']['alarm']['ttl'] self._alarms = AlarmProcessor(self._alarm_ttl, config) self._finished_count = self._statsd.get_counter(name=ALARMS_FINISHED_COUNT) self._notifier = NotificationProcessor(config) def _add_periodic_notifications(self, notifications): for notification in notifications: topic = notification.periodic_topic if topic in self._config['kafka']['periodic'] and notification.type == "webhook": notification.notification_timestamp = time.time() self._producer.publish(self._config['kafka']['periodic'][topic], [notification.to_json()]) def do_message(self, alarm): log.debug('Received alarm >|%s|<', str(alarm)) notifications, partition, offset = self._alarms.to_notification(alarm) if notifications: self._add_periodic_notifications(notifications) sent, failed = self._notifier.send(notifications) self.publish_messages(sent, self._topics['notification_topic']) self.publish_messages(failed, self._topics['retry_topic']) self._consumer.commit() self._finished_count.increment()
class NotificationEngine(object): def __init__(self, config): self._topics = {} self._topics['notification_topic'] = config['kafka'][ 'notification_topic'] self._topics['retry_topic'] = config['kafka'][ 'notification_retry_topic'] self._statsd = monascastatsd.Client( name='monasca', dimensions=BaseProcessor.dimensions) self._consumer = KafkaConsumer( config['kafka']['url'], config['zookeeper']['url'], config['zookeeper']['notification_path'], config['kafka']['group'], config['kafka']['alarm_topic']) self._producer = KafkaProducer(config['kafka']['url']) self._alarm_ttl = config['processors']['alarm']['ttl'] self._alarms = AlarmProcessor(self._alarm_ttl, config) self._notifier = NotificationProcessor(config['notification_types']) def run(self): finished_count = self._statsd.get_counter(name='alarms_finished_count') for alarm in self._consumer: log.debug('Received alarm >|%s|<', str(alarm)) notifications, partition, offset = self._alarms.to_notification( alarm) if notifications: sent, failed = self._notifier.send(notifications) self._producer.publish(self._topics['notification_topic'], [i.to_json() for i in sent]) self._producer.publish(self._topics['retry_topic'], [i.to_json() for i in failed]) self._consumer.commit() finished_count.increment()
def __init__(self, config, interval): self._topic_name = config['kafka']['periodic'][interval] self._statsd = monascastatsd.Client( name='monasca', dimensions=BaseProcessor.dimensions) zookeeper_path = config['zookeeper']['periodic_path'][interval] self._consumer = KafkaConsumer(config['kafka']['url'], config['zookeeper']['url'], zookeeper_path, config['kafka']['group'], self._topic_name) self._producer = KafkaProducer(config['kafka']['url']) self._notifier = NotificationProcessor(config['notification_types']) self._db_repo = get_db_repo(config)
def __init__(self, config): super(NotificationEngine, self).__init__(config, config['kafka']['alarm_topic'], config['zookeeper']['notification_path']) self._topics = {} self._topics['notification_topic'] = config['kafka']['notification_topic'] self._topics['retry_topic'] = config['kafka']['notification_retry_topic'] self._alarm_ttl = config['processors']['alarm']['ttl'] self._alarms = AlarmProcessor(self._alarm_ttl, config) self._finished_count = self._statsd.get_counter(name=ALARMS_FINISHED_COUNT) self._notifier = NotificationProcessor(config)
def __init__(self, config): self._retry_interval = config['retry']['interval'] self._retry_max = config['retry']['max_attempts'] self._topics = {} self._topics['notification_topic'] = config['kafka'][ 'notification_topic'] self._topics['retry_topic'] = config['kafka'][ 'notification_retry_topic'] self._statsd = monascastatsd.Client( name='monasca', dimensions=BaseProcessor.dimensions) self._consumer = KafkaConsumer( config['kafka']['url'], config['zookeeper']['url'], config['zookeeper']['notification_retry_path'], config['kafka']['group'], config['kafka']['notification_retry_topic']) self._producer = KafkaProducer(config['kafka']['url']) self._notifier = NotificationProcessor(config['notification_types'])
def __init__(self, config): self._topics = {} self._topics['notification_topic'] = config['kafka']['notification_topic'] self._topics['retry_topic'] = config['kafka']['notification_retry_topic'] self._statsd = monascastatsd.Client(name='monasca', dimensions=BaseProcessor.dimensions) self._consumer = KafkaConsumer(config['kafka']['url'], config['zookeeper']['url'], config['zookeeper']['notification_path'], config['kafka']['group'], config['kafka']['alarm_topic']) self._producer = KafkaProducer(config['kafka']['url']) self._alarm_ttl = config['processors']['alarm']['ttl'] self._alarms = AlarmProcessor(self._alarm_ttl, config) self._notifier = NotificationProcessor(config['notification_types'])
class NotificationEngine(object): def __init__(self, config): self._topics = {} self._topics['notification_topic'] = config['kafka']['notification_topic'] self._topics['retry_topic'] = config['kafka']['notification_retry_topic'] self._statsd = monascastatsd.Client(name='monasca', dimensions=BaseProcessor.dimensions) self._consumer = KafkaConsumer(config['kafka']['url'], config['zookeeper']['url'], config['zookeeper']['notification_path'], config['kafka']['group'], config['kafka']['alarm_topic']) self._producer = KafkaProducer(config['kafka']['url']) if 'ssl' in config['mysql']: ssl_config = config['mysql']['ssl'] else: ssl_config = None self._alarms = AlarmProcessor(config['processors']['alarm']['ttl'], config['mysql']['host'], config['mysql']['user'], config['mysql']['passwd'], config['mysql']['db'], ssl_config) self._notifier = NotificationProcessor(config['notification_types']) def run(self): finished_count = self._statsd.get_counter(name='alarms_finished_count') for alarm in self._consumer: notifications, partition, offset = self._alarms.to_notification(alarm) if notifications: sent, failed = self._notifier.send(notifications) self._producer.publish(self._topics['notification_topic'], sent) self._producer.publish(self._topics['retry_topic'], failed) self._consumer.commit([partition]) finished_count.increment()
def __init__(self, config, interval): self._topic_name = config['kafka']['periodic'][interval] self._statsd = monascastatsd.Client(name='monasca', dimensions=BaseProcessor.dimensions) zookeeper_path = config['zookeeper']['periodic_path'][interval] self._consumer = KafkaConsumer(config['kafka']['url'], config['zookeeper']['url'], zookeeper_path, config['kafka']['group'], self._topic_name) self._producer = KafkaProducer(config['kafka']['url']) self._notifier = NotificationProcessor(config['notification_types']) self._db_repo = get_db_repo(config)
def __init__(self, config): self._retry_interval = config['retry']['interval'] self._retry_max = config['retry']['max_attempts'] self._topics = {} self._topics['notification_topic'] = config['kafka']['notification_topic'] self._topics['retry_topic'] = config['kafka']['notification_retry_topic'] self._statsd = monascastatsd.Client(name='monasca', dimensions=BaseProcessor.dimensions) self._consumer = KafkaConsumer(config['kafka']['url'], config['zookeeper']['url'], config['zookeeper']['notification_retry_path'], config['kafka']['group'], config['kafka']['notification_retry_topic']) self._producer = KafkaProducer(config['kafka']['url']) self._notifier = NotificationProcessor(config['notification_types'])
class RetryEngine(object): def __init__(self, config): self._retry_interval = config['retry']['interval'] self._retry_max = config['retry']['max_attempts'] self._topics = {} self._topics['notification_topic'] = config['kafka']['notification_topic'] self._topics['retry_topic'] = config['kafka']['notification_retry_topic'] self._statsd = monascastatsd.Client(name='monasca', dimensions=BaseProcessor.dimensions) self._consumer = KafkaConsumer(config['kafka']['url'], config['zookeeper']['url'], config['zookeeper']['notification_retry_path'], config['kafka']['group'], config['kafka']['notification_retry_topic']) self._producer = KafkaProducer(config['kafka']['url']) self._notifier = NotificationProcessor(config['notification_types']) def run(self): for raw_notification in self._consumer: partition = raw_notification[0] offset = raw_notification[1].offset message = raw_notification[1].message.value notification_data = json.loads(message) ntype = notification_data['type'] name = notification_data['name'] addr = notification_data['address'] notification = Notification(ntype, partition, offset, name, addr, notification_data['retry_count'], notification_data['raw_alarm']) wait_duration = self._retry_interval - ( time.time() - notification_data['notification_timestamp']) if wait_duration > 0: time.sleep(wait_duration) sent, failed = self._notifier.send([notification]) if sent: self._producer.publish(self._topics['notification_topic'], sent) if failed: notification.retry_count += 1 notification.notification_timestamp = time.time() if notification.retry_count < self._retry_max: log.error(u"retry failed for {} with name {} " u"at {}. " u"Saving for later retry.".format(ntype, name, addr)) self._producer.publish(self._topics['retry_topic'], [notification.to_json()]) else: log.error(u"retry failed for {} with name {} " u"at {} after {} retries. " u"Giving up on retry." .format(ntype, name, addr, self._retry_max)) self._consumer.commit()
class RetryEngine(object): def __init__(self, config): self._retry_interval = config['retry']['interval'] self._retry_max = config['retry']['max_attempts'] self._topics = {} self._topics['notification_topic'] = config['kafka'][ 'notification_topic'] self._topics['retry_topic'] = config['kafka'][ 'notification_retry_topic'] self._statsd = monascastatsd.Client( name='monasca', dimensions=BaseProcessor.dimensions) self._consumer = KafkaConsumer( config['kafka']['url'], config['zookeeper']['url'], config['zookeeper']['notification_retry_path'], config['kafka']['group'], config['kafka']['notification_retry_topic']) self._producer = KafkaProducer(config['kafka']['url']) self._notifier = NotificationProcessor(config['notification_types']) def run(self): for raw_notification in self._consumer: partition = raw_notification[0] offset = raw_notification[1].offset message = raw_notification[1].message.value notification_data = json.loads(message) ntype = notification_data['type'] name = notification_data['name'] addr = notification_data['address'] notification = Notification(ntype, partition, offset, name, addr, notification_data['retry_count'], notification_data['raw_alarm']) wait_duration = self._retry_interval - ( time.time() - notification_data['notification_timestamp']) if wait_duration > 0: time.sleep(wait_duration) sent, failed = self._notifier.send([notification]) if sent: self._producer.publish(self._topics['notification_topic'], sent) if failed: notification.retry_count += 1 notification.notification_timestamp = time.time() if notification.retry_count < self._retry_max: log.error("retry failed for {} with name {} " "at {}. " "Saving for later retry.".format( ntype, name, addr)) self._producer.publish(self._topics['retry_topic'], [notification]) else: log.error("retry failed for {} with name {} " "at {} after {} retries. " "Giving up on retry.".format( ntype, name, addr, self._retry_max)) self._consumer.commit([partition])
class PeriodicEngine(object): def __init__(self, config, interval): self._topic_name = config['kafka']['periodic'][interval] self._statsd = monascastatsd.Client( name='monasca', dimensions=BaseProcessor.dimensions) zookeeper_path = config['zookeeper']['periodic_path'][interval] self._consumer = KafkaConsumer(config['kafka']['url'], config['zookeeper']['url'], zookeeper_path, config['kafka']['group'], self._topic_name) self._producer = KafkaProducer(config['kafka']['url']) self._notifier = NotificationProcessor(config['notification_types']) self._db_repo = get_db_repo(config) def _keep_sending(self, alarm_id, original_state): # Go to DB and check alarm state try: current_state = self._db_repo.get_alarm_current_state(alarm_id) except exceptions.DatabaseException: log.debug('Database Error. Attempting reconnect') current_state = self._db_repo.get_alarm_current_state(alarm_id) # Alarm was deleted if current_state is None: return False # Alarm state changed if current_state != original_state: return False return True def run(self): for raw_notification in self._consumer: partition = raw_notification[0] offset = raw_notification[1].offset message = raw_notification[1].message.value notification_data = json.loads(message) ntype = notification_data['type'] name = notification_data['name'] addr = notification_data['address'] period = notification_data['period'] notification = Notification(ntype, partition, offset, name, addr, period, notification_data['retry_count'], notification_data['raw_alarm']) if self._keep_sending(notification.alarm_id, notification.state): wait_duration = period - ( time.time() - notification_data['notification_timestamp']) if wait_duration > 0: time.sleep(wait_duration) notification.notification_timestamp = time.time() self._notifier.send([notification]) self._producer.publish(self._topic_name, [notification.to_json()]) self._consumer.commit()
class RetryEngine(object): def __init__(self, config): self._retry_interval = config['retry']['interval'] self._retry_max = config['retry']['max_attempts'] self._topics = {} self._topics['notification_topic'] = config['kafka'][ 'notification_topic'] self._topics['retry_topic'] = config['kafka'][ 'notification_retry_topic'] self._statsd = monascastatsd.Client( name='monasca', dimensions=BaseProcessor.dimensions) self._consumer = KafkaConsumer( config['kafka']['url'], config['zookeeper']['url'], config['zookeeper']['notification_retry_path'], config['kafka']['group'], config['kafka']['notification_retry_topic']) self._producer = KafkaProducer(config['kafka']['url']) self._notifier = NotificationProcessor(config) self._db_repo = get_db_repo(config) def run(self): for raw_notification in self._consumer: message = raw_notification[1].message.value notification_data = json.loads(message) notification = construct_notification_object( self._db_repo, notification_data) if notification is None: self._consumer.commit() continue wait_duration = self._retry_interval - ( time.time() - notification_data['notification_timestamp']) if wait_duration > 0: time.sleep(wait_duration) sent, failed = self._notifier.send([notification]) if sent: self._producer.publish(self._topics['notification_topic'], [notification.to_json()]) if failed: notification.retry_count += 1 notification.notification_timestamp = time.time() if notification.retry_count < self._retry_max: log.error(u"retry failed for {} with name {} " u"at {}. " u"Saving for later retry.".format( notification.type, notification.name, notification.address)) self._producer.publish(self._topics['retry_topic'], [notification.to_json()]) else: log.error(u"retry failed for {} with name {} " u"at {} after {} retries. " u"Giving up on retry.".format( notification.type, notification.name, notification.address, self._retry_max)) self._consumer.commit()
class PeriodicEngine(object): def __init__(self, config, period): self._topic_name = config['kafka']['periodic'][period] self._statsd = monascastatsd.Client( name='monasca', dimensions=BaseProcessor.dimensions) zookeeper_path = config['zookeeper']['periodic_path'][period] self._consumer = KafkaConsumer(config['kafka']['url'], config['zookeeper']['url'], zookeeper_path, config['kafka']['group'], self._topic_name) self._producer = KafkaProducer(config['kafka']['url']) self._notifier = NotificationProcessor(config) self._db_repo = get_db_repo(config) self._period = period def _keep_sending(self, alarm_id, original_state, type, period): try: current_state = self._db_repo.get_alarm_current_state(alarm_id) except exceptions.DatabaseException: log.debug('Database Error. Attempting reconnect') current_state = self._db_repo.get_alarm_current_state(alarm_id) # Alarm was deleted if current_state is None: return False # Alarm state changed if current_state != original_state: return False # Period changed if period != self._period: return False if type != "webhook": return False return True def run(self): for raw_notification in self._consumer: message = raw_notification[1].message.value notification_data = json.loads(message) notification = construct_notification_object( self._db_repo, notification_data) if notification is None: self._consumer.commit() continue if self._keep_sending(notification.alarm_id, notification.state, notification.type, notification.period): wait_duration = notification.period - ( time.time() - notification_data['notification_timestamp']) if wait_duration > 0: time.sleep(wait_duration) notification.notification_timestamp = time.time() self._notifier.send([notification]) self._producer.publish(self._topic_name, [notification.to_json()]) self._consumer.commit()
class PeriodicEngine(object): def __init__(self, config, interval): self._topic_name = config['kafka']['periodic'][interval] self._statsd = monascastatsd.Client(name='monasca', dimensions=BaseProcessor.dimensions) zookeeper_path = config['zookeeper']['periodic_path'][interval] self._consumer = KafkaConsumer(config['kafka']['url'], config['zookeeper']['url'], zookeeper_path, config['kafka']['group'], self._topic_name) self._producer = KafkaProducer(config['kafka']['url']) self._notifier = NotificationProcessor(config['notification_types']) self._db_repo = get_db_repo(config) def _keep_sending(self, alarm_id, original_state): # Go to DB and check alarm state try: current_state = self._db_repo.get_alarm_current_state(alarm_id) except exceptions.DatabaseException: log.debug('Database Error. Attempting reconnect') current_state = self._db_repo.get_alarm_current_state(alarm_id) # Alarm was deleted if current_state is None: return False # Alarm state changed if current_state != original_state: return False return True def run(self): for raw_notification in self._consumer: partition = raw_notification[0] offset = raw_notification[1].offset message = raw_notification[1].message.value notification_data = json.loads(message) ntype = notification_data['type'] name = notification_data['name'] addr = notification_data['address'] period = notification_data['period'] notification = Notification(ntype, partition, offset, name, addr, period, notification_data['retry_count'], notification_data['raw_alarm']) if self._keep_sending(notification.alarm_id, notification.state): wait_duration = period - ( time.time() - notification_data['notification_timestamp']) if wait_duration > 0: time.sleep(wait_duration) notification.notification_timestamp = time.time() self._notifier.send([notification]) self._producer.publish(self._topic_name, [notification.to_json()]) self._consumer.commit()