def stream_defs_from_database(self): slist = list() try: stream_definition_rows = \ self._streams_repo.get_all_stream_definitions() for row in stream_definition_rows: row['fire_criteria'] = json.loads(row['fire_criteria']) row['select_by'] = json.loads(row['select_by']) row['group_by'] = json.loads(row['group_by']) w_stream = stream_helpers.stream_def_to_winchester_format( row) slist.append(w_stream) except exceptions.RepositoryException as e: log.error(e) return slist
def stream_definition_consumer(self, conf, lock, group, manager): '''Stream Definition Consumer Stream definition consumer runs as a thread for the event processor and pipeline processor processes. :param CONF.cfg conf: the conf object. (clarity, CONF.cfg is global) :param thread.Lock() lock: Used to lock manager access. :param string group: The Kafka group for stream definitions. :param object manager: trigger_manager, or pipeline_manager object. ''' kafka_url = conf.kafka.url group = conf.kafka.stream_def_group topic = conf.kafka.stream_def_topic kafka = KafkaClient(kafka_url) consumer = SimpleConsumer( kafka, group, topic, auto_commit=True) consumer.seek(0, 2) statsd = monascastatsd.Client(name='monasca', dimensions=self.dimensions) stream_definitions_created = \ statsd.get_counter('stream_definitions_created') stream_definitions_deleted = \ statsd.get_counter('stream_definitions_deleted') for s in consumer: offset, message = s stream_def = json.loads(message.value) if 'stream-definition-created' in stream_def: log.debug('Received a stream definition created event') stream_create = stream_helpers.stream_def_to_winchester_format( stream_def['stream-definition-created']) slist = list() slist.append(stream_create) lock.acquire() try: manager.add_trigger_definition(slist) stream_definitions_created.increment() except Exception as e: log.exception(e) finally: lock.release() elif 'stream-definition-deleted' in stream_def: log.debug('Received a stream-definition-deleted event') name = stream_helpers.stream_unique_name( stream_def['stream-definition-deleted']) lock.acquire() try: manager.delete_trigger_definition(name) stream_definitions_deleted.increment() except Exception as e: log.exception(e) finally: lock.release() else: log.error('Unknown event received on stream_def_topic')