def _configure_pipeline_listener(self): ev_pipes = self.event_pipeline_manager.pipelines pipelines = self.pipeline_manager.pipelines + ev_pipes transport = messaging.get_transport(self.conf) partitioned = six.moves.range( self.conf.notification.pipeline_processing_queues ) if self.partition_coordinator: partitioned = list(filter( self.hashring.belongs_to_self, partitioned)) endpoints = [] targets = [] for pipe in pipelines: if isinstance(pipe, pipeline.EventPipeline): endpoints.append(pipeline.EventPipelineEndpoint(pipe)) else: endpoints.append(pipeline.SamplePipelineEndpoint(pipe)) for pipe_set, pipe in itertools.product(partitioned, pipelines): LOG.debug('Pipeline endpoint: %s from set: %s', pipe.name, pipe_set) topic = '%s-%s-%s' % (self.NOTIFICATION_IPC, pipe.name, pipe_set) targets.append(oslo_messaging.Target(topic=topic)) if self.pipeline_listener: self.pipeline_listener.stop() self.pipeline_listener.wait() self.pipeline_listener = messaging.get_batch_notification_listener( transport, targets, endpoints, batch_size=self.conf.notification.batch_size, batch_timeout=self.conf.notification.batch_timeout) # NOTE(gordc): set single thread to process data sequentially # if batching enabled. batch = (1 if self.conf.notification.batch_size > 1 else self.conf.max_parallel_requests) self.pipeline_listener.start(override_pool_size=batch)
def _configure_pipeline_listener(self): with self.coord_lock: ev_pipes = [] if cfg.CONF.notification.store_events: ev_pipes = self.event_pipeline_manager.pipelines pipelines = self.pipeline_manager.pipelines + ev_pipes transport = messaging.get_transport() partitioned = self.partition_coordinator.extract_my_subset( self.group_id, range(cfg.CONF.notification.pipeline_processing_queues)) endpoints = [] targets = [] for pipe in pipelines: if isinstance(pipe, pipeline.EventPipeline): endpoints.append( pipeline.EventPipelineEndpoint(self.ctxt, pipe)) else: endpoints.append( pipeline.SamplePipelineEndpoint(self.ctxt, pipe)) for pipe_set, pipe in itertools.product(partitioned, pipelines): LOG.debug('Pipeline endpoint: %s from set: %s', pipe.name, pipe_set) topic = '%s-%s-%s' % (self.NOTIFICATION_IPC, pipe.name, pipe_set) targets.append(oslo_messaging.Target(topic=topic)) if self.pipeline_listener: self.pipeline_listener.stop() self.pipeline_listener.wait() self.pipeline_listener = messaging.get_batch_notification_listener( transport, targets, endpoints, batch_size=cfg.CONF.notification.batch_size, batch_timeout=cfg.CONF.notification.batch_timeout) self.pipeline_listener.start()