def run(self): if self.conf.collector.udp_address: self.udp_thread = utils.spawn_thread(self.start_udp) transport = messaging.get_transport(self.conf, optional=True) if transport: if list(self.meter_manager): sample_target = oslo_messaging.Target( topic=self.conf.publisher_notifier.metering_topic) self.sample_listener = ( messaging.get_batch_notification_listener( transport, [sample_target], [ SampleEndpoint( self.conf.publisher.telemetry_secret, self.meter_manager) ], allow_requeue=True, batch_size=self.conf.collector.batch_size, batch_timeout=self.conf.collector.batch_timeout)) self.sample_listener.start() if list(self.event_manager): event_target = oslo_messaging.Target( topic=self.conf.publisher_notifier.event_topic) self.event_listener = ( messaging.get_batch_notification_listener( transport, [event_target], [ EventEndpoint(self.conf.publisher.telemetry_secret, self.event_manager) ], allow_requeue=True, batch_size=self.conf.collector.batch_size, batch_timeout=self.conf.collector.batch_timeout)) self.event_listener.start()
def run(self): if self.conf.collector.udp_address: self.udp_thread = utils.spawn_thread(self.start_udp) transport = messaging.get_transport(self.conf, optional=True) if transport: if list(self.meter_manager): sample_target = oslo_messaging.Target( topic=self.conf.publisher_notifier.metering_topic) self.sample_listener = ( messaging.get_batch_notification_listener( transport, [sample_target], [SampleEndpoint(self.conf.publisher.telemetry_secret, self.meter_manager)], allow_requeue=True, batch_size=self.conf.collector.batch_size, batch_timeout=self.conf.collector.batch_timeout)) self.sample_listener.start() if list(self.event_manager): event_target = oslo_messaging.Target( topic=self.conf.publisher_notifier.event_topic) self.event_listener = ( messaging.get_batch_notification_listener( transport, [event_target], [EventEndpoint(self.conf.publisher.telemetry_secret, self.event_manager)], allow_requeue=True, batch_size=self.conf.collector.batch_size, batch_timeout=self.conf.collector.batch_timeout)) self.event_listener.start()
def _configure_main_queue_listeners(self, pipe_manager, event_pipe_manager): notification_manager = self._get_notifications_manager(pipe_manager) if not list(notification_manager): LOG.warning(_("Failed to load any notification handlers for %s"), self.NOTIFICATION_NAMESPACE) ack_on_error = self.conf.notification.ack_on_event_error endpoints = [] endpoints.append(event_endpoint.EventsNotificationEndpoint(event_pipe_manager)) targets = [] for ext in notification_manager: handler = ext.obj if self.conf.notification.disable_non_metric_meters and isinstance(handler, base.NonMetricNotificationBase): continue LOG.debug( "Event types from %(name)s: %(type)s" " (ack_on_error=%(error)s)", {"name": ext.name, "type": ", ".join(handler.event_types), "error": ack_on_error}, ) # NOTE(gordc): this could be a set check but oslo_messaging issue # https://bugs.launchpad.net/oslo.messaging/+bug/1398511 # This ensures we don't create multiple duplicate consumers. for new_tar in handler.get_targets(self.conf): if new_tar not in targets: targets.append(new_tar) endpoints.append(handler) urls = self.conf.notification.messaging_urls or [None] for url in urls: transport = messaging.get_transport(self.conf, url) # NOTE(gordc): ignore batching as we want pull # to maintain sequencing as much as possible. listener = messaging.get_batch_notification_listener(transport, targets, endpoints) listener.start() self.listeners.append(listener)
def _configure_pipeline_listener(self): partitioned = list( filter(self.hashring.belongs_to_self, self.partition_set)) endpoints = [] for pipe_mgr in self.managers: endpoints.extend(pipe_mgr.get_interim_endpoints()) targets = [] for mgr, hash_id in itertools.product(self.managers, partitioned): topic = '-'.join([mgr.NOTIFICATION_IPC, mgr.pm_type, str(hash_id)]) LOG.debug('Listening to queue: %s', topic) targets.append(oslo_messaging.Target(topic=topic)) if self.pipeline_listener: self.pipeline_listener.stop() self.pipeline_listener.wait() self.pipeline_listener = messaging.get_batch_notification_listener( self.transport, targets, endpoints, batch_size=self.conf.notification.batch_size, batch_timeout=self.conf.notification.batch_timeout) # NOTE(gordc): set single thread to process data sequentially # if batching enabled. batch = (1 if self.conf.notification.batch_size > 1 else self.conf.max_parallel_requests) self.pipeline_listener.start(override_pool_size=batch)
def run(self): # Delay startup so workers are jittered time.sleep(self.startup_delay) super(NotificationService, self).run() self.managers = [ext.obj for ext in named.NamedExtensionManager( namespace='ceilometer.notification.pipeline', names=self.conf.notification.pipelines, invoke_on_load=True, on_missing_entrypoints_callback=self._log_missing_pipeline, invoke_args=(self.conf,))] # FIXME(sileht): endpoint uses the notification_topics option # and it should not because this is an oslo_messaging option # not a ceilometer. Until we have something to get the # notification_topics in another way, we must create a transport # to ensure the option has been registered by oslo_messaging. messaging.get_notifier(messaging.get_transport(self.conf), '') endpoints = [] for pipe_mgr in self.managers: endpoints.extend(pipe_mgr.get_main_endpoints()) targets = self.get_targets() urls = self.conf.notification.messaging_urls or [None] for url in urls: transport = messaging.get_transport(self.conf, url) # NOTE(gordc): ignore batching as we want pull # to maintain sequencing as much as possible. listener = messaging.get_batch_notification_listener( transport, targets, endpoints, allow_requeue=True) listener.start( override_pool_size=self.conf.max_parallel_requests ) self.listeners.append(listener)
def start(self): """Bind the UDP socket and handle incoming data.""" # ensure dispatcher is configured before starting other services dispatcher_managers = dispatcher.load_dispatcher_manager() (self.meter_manager, self.event_manager) = dispatcher_managers self.sample_listener = None self.event_listener = None self.udp_thread = None super(CollectorService, self).start() if cfg.CONF.collector.udp_address: self.udp_thread = threading.Thread(target=self.start_udp) self.udp_thread.daemon = True self.udp_thread.start() transport = messaging.get_transport(optional=True) if transport: if list(self.meter_manager): sample_target = oslo_messaging.Target( topic=cfg.CONF.publisher_notifier.metering_topic) self.sample_listener = ( messaging.get_batch_notification_listener( transport, [sample_target], [SampleEndpoint(self.meter_manager)], allow_requeue=True, batch_size=cfg.CONF.collector.batch_size, batch_timeout=cfg.CONF.collector.batch_timeout)) self.sample_listener.start() if cfg.CONF.notification.store_events and list(self.event_manager): event_target = oslo_messaging.Target( topic=cfg.CONF.publisher_notifier.event_topic) self.event_listener = ( messaging.get_batch_notification_listener( transport, [event_target], [EventEndpoint(self.event_manager)], allow_requeue=True, batch_size=cfg.CONF.collector.batch_size, batch_timeout=cfg.CONF.collector.batch_timeout)) self.event_listener.start() if not cfg.CONF.collector.udp_address: # NOTE(sileht): We have to drop oslo.service to remove this # last eventlet thread # Add a dummy thread to have wait() working self.tg.add_timer(604800, lambda: None)
def run(self): """Bind the UDP socket and handle incoming data.""" super(CollectorService, self).run() # ensure dispatcher is configured before starting other services dispatcher_managers = dispatcher.load_dispatcher_manager() (self.meter_manager, self.event_manager) = dispatcher_managers self.sample_listener = None self.event_listener = None self.udp_thread = None if cfg.CONF.collector.udp_address: self.udp_thread = utils.spawn_thread(self.start_udp) transport = messaging.get_transport(optional=True) if transport: if list(self.meter_manager): sample_target = oslo_messaging.Target( topic=cfg.CONF.publisher_notifier.metering_topic) self.sample_listener = ( messaging.get_batch_notification_listener( transport, [sample_target], [ SampleEndpoint(cfg.CONF.publisher.telemetry_secret, self.meter_manager) ], allow_requeue=True, batch_size=cfg.CONF.collector.batch_size, batch_timeout=cfg.CONF.collector.batch_timeout)) self.sample_listener.start() if cfg.CONF.notification.store_events and list(self.event_manager): event_target = oslo_messaging.Target( topic=cfg.CONF.publisher_notifier.event_topic) self.event_listener = ( messaging.get_batch_notification_listener( transport, [event_target], [ EventEndpoint(cfg.CONF.publisher.telemetry_secret, self.event_manager) ], allow_requeue=True, batch_size=cfg.CONF.collector.batch_size, batch_timeout=cfg.CONF.collector.batch_timeout)) self.event_listener.start()
def start(self): """Bind the UDP socket and handle incoming data.""" # ensure dispatcher is configured before starting other services dispatcher_managers = dispatcher.load_dispatcher_manager() (self.meter_manager, self.event_manager) = dispatcher_managers self.sample_listener = None self.event_listener = None self.udp_thread = None super(CollectorService, self).start() if cfg.CONF.collector.udp_address: self.udp_thread = utils.spawn_thread(self.start_udp) transport = messaging.get_transport(optional=True) if transport: if list(self.meter_manager): sample_target = oslo_messaging.Target( topic=cfg.CONF.publisher_notifier.metering_topic) self.sample_listener = ( messaging.get_batch_notification_listener( transport, [sample_target], [SampleEndpoint(cfg.CONF.publisher.telemetry_secret, self.meter_manager)], allow_requeue=True, batch_size=cfg.CONF.collector.batch_size, batch_timeout=cfg.CONF.collector.batch_timeout)) self.sample_listener.start() if cfg.CONF.notification.store_events and list(self.event_manager): event_target = oslo_messaging.Target( topic=cfg.CONF.publisher_notifier.event_topic) self.event_listener = ( messaging.get_batch_notification_listener( transport, [event_target], [EventEndpoint(cfg.CONF.publisher.telemetry_secret, self.event_manager)], allow_requeue=True, batch_size=cfg.CONF.collector.batch_size, batch_timeout=cfg.CONF.collector.batch_timeout)) self.event_listener.start()
def _configure_main_queue_listeners(self): endpoints = [] for pipe_mgr in self.managers: endpoints.extend(pipe_mgr.get_main_endpoints()) targets = self.get_targets() urls = self.conf.notification.messaging_urls or [None] for url in urls: transport = messaging.get_transport(self.conf, url) # NOTE(gordc): ignore batching as we want pull # to maintain sequencing as much as possible. listener = messaging.get_batch_notification_listener( transport, targets, endpoints) listener.start(override_pool_size=self.conf.max_parallel_requests) self.listeners.append(listener)
def _configure_main_queue_listeners(self, pipe_manager, event_pipe_manager): notification_manager = self._get_notifications_manager(pipe_manager) if not list(notification_manager): LOG.warning(_('Failed to load any notification handlers for %s'), self.NOTIFICATION_NAMESPACE) ack_on_error = cfg.CONF.notification.ack_on_event_error endpoints = [] if cfg.CONF.notification.store_events: endpoints.append( event_endpoint.EventsNotificationEndpoint(event_pipe_manager)) targets = [] for ext in notification_manager: handler = ext.obj if (cfg.CONF.notification.disable_non_metric_meters and isinstance(handler, base.NonMetricNotificationBase)): continue LOG.debug( 'Event types from %(name)s: %(type)s' ' (ack_on_error=%(error)s)', { 'name': ext.name, 'type': ', '.join(handler.event_types), 'error': ack_on_error }) # NOTE(gordc): this could be a set check but oslo_messaging issue # https://bugs.launchpad.net/oslo.messaging/+bug/1398511 # This ensures we don't create multiple duplicate consumers. for new_tar in handler.get_targets(cfg.CONF): if new_tar not in targets: targets.append(new_tar) endpoints.append(handler) urls = cfg.CONF.notification.messaging_urls or [None] for url in urls: transport = messaging.get_transport(url) listener = messaging.get_batch_notification_listener( transport, targets, endpoints, batch_size=cfg.CONF.notification.batch_size, batch_timeout=cfg.CONF.notification.batch_timeout) listener.start() self.listeners.append(listener)
def _configure_pipeline_listeners(self, reuse_listeners=False): with self.coord_lock: ev_pipes = [] if cfg.CONF.notification.store_events: ev_pipes = self.event_pipeline_manager.pipelines pipelines = self.pipeline_manager.pipelines + ev_pipes transport = messaging.get_transport() partitioned = self.partition_coordinator.extract_my_subset( self.group_id, range(cfg.CONF.notification.pipeline_processing_queues)) queue_set = {} for pipe_set, pipe in itertools.product(partitioned, pipelines): queue_set['%s-%s-%s' % (self.NOTIFICATION_IPC, pipe.name, pipe_set)] = pipe if reuse_listeners: topics = queue_set.keys() kill_list = [] for listener in self.pipeline_listeners: if listener.dispatcher.targets[0].topic in topics: queue_set.pop(listener.dispatcher.targets[0].topic) else: kill_list.append(listener) for listener in kill_list: utils.kill_listeners([listener]) self.pipeline_listeners.remove(listener) else: utils.kill_listeners(self.pipeline_listeners) self.pipeline_listeners = [] for topic, pipe in queue_set.items(): LOG.debug('Pipeline endpoint: %s from set: %s', pipe.name, pipe_set) pipe_endpoint = (pipeline.EventPipelineEndpoint if isinstance(pipe, pipeline.EventPipeline) else pipeline.SamplePipelineEndpoint) listener = messaging.get_batch_notification_listener( transport, [oslo_messaging.Target(topic=topic)], [pipe_endpoint(self.ctxt, pipe)], batch_size=cfg.CONF.notification.batch_size, batch_timeout=cfg.CONF.notification.batch_timeout) listener.start() self.pipeline_listeners.append(listener)
def _configure_pipeline_listeners(self, reuse_listeners=False): with self.coord_lock: ev_pipes = [] if cfg.CONF.notification.store_events: ev_pipes = self.event_pipeline_manager.pipelines pipelines = self.pipeline_manager.pipelines + ev_pipes transport = messaging.get_transport() partitioned = self.partition_coordinator.extract_my_subset( self.group_id, range(cfg.CONF.notification.pipeline_processing_queues)) queue_set = {} for pipe_set, pipe in itertools.product(partitioned, pipelines): queue_set['%s-%s-%s' % (self.NOTIFICATION_IPC, pipe.name, pipe_set)] = pipe if reuse_listeners: topics = queue_set.keys() kill_list = [] for listener in self.pipeline_listeners: if listener.dispatcher.targets[0].topic in topics: queue_set.pop(listener.dispatcher.targets[0].topic) else: kill_list.append(listener) for listener in kill_list: utils.kill_listeners([listener]) self.pipeline_listeners.remove(listener) else: utils.kill_listeners(self.pipeline_listeners) self.pipeline_listeners = [] for topic, pipe in queue_set.items(): LOG.debug('Pipeline endpoint: %s from set: %s', pipe.name, pipe_set) pipe_endpoint = (pipeline.EventPipelineEndpoint if isinstance( pipe, pipeline.EventPipeline) else pipeline.SamplePipelineEndpoint) listener = messaging.get_batch_notification_listener( transport, [oslo_messaging.Target(topic=topic)], [pipe_endpoint(self.ctxt, pipe)], batch_size=cfg.CONF.notification.batch_size, batch_timeout=cfg.CONF.notification.batch_timeout) listener.start() self.pipeline_listeners.append(listener)
def _configure_pipeline_listener(self): ev_pipes = self.event_pipeline_manager.pipelines pipelines = self.pipeline_manager.pipelines + ev_pipes transport = messaging.get_transport(self.conf) partitioned = six.moves.range( self.conf.notification.pipeline_processing_queues ) if self.partition_coordinator: partitioned = list(filter( self.hashring.belongs_to_self, partitioned)) endpoints = [] targets = [] for pipe in pipelines: if isinstance(pipe, pipeline.EventPipeline): endpoints.append(pipeline.EventPipelineEndpoint(pipe)) else: endpoints.append(pipeline.SamplePipelineEndpoint(pipe)) for pipe_set, pipe in itertools.product(partitioned, pipelines): LOG.debug('Pipeline endpoint: %s from set: %s', pipe.name, pipe_set) topic = '%s-%s-%s' % (self.NOTIFICATION_IPC, pipe.name, pipe_set) targets.append(oslo_messaging.Target(topic=topic)) if self.pipeline_listener: self.pipeline_listener.stop() self.pipeline_listener.wait() self.pipeline_listener = messaging.get_batch_notification_listener( transport, targets, endpoints, batch_size=self.conf.notification.batch_size, batch_timeout=self.conf.notification.batch_timeout) # NOTE(gordc): set single thread to process data sequentially # if batching enabled. batch = (1 if self.conf.notification.batch_size > 1 else self.conf.max_parallel_requests) self.pipeline_listener.start(override_pool_size=batch)
def _configure_main_queue_listeners(self, pipe_manager, event_pipe_manager): notification_manager = self._get_notifications_manager(pipe_manager) if not list(notification_manager): LOG.warning(_('Failed to load any notification handlers for %s'), self.NOTIFICATION_NAMESPACE) ack_on_error = cfg.CONF.notification.ack_on_event_error endpoints = [] if cfg.CONF.notification.store_events: endpoints.append( event_endpoint.EventsNotificationEndpoint(event_pipe_manager)) targets = [] for ext in notification_manager: handler = ext.obj if (cfg.CONF.notification.disable_non_metric_meters and isinstance(handler, base.NonMetricNotificationBase)): continue LOG.debug('Event types from %(name)s: %(type)s' ' (ack_on_error=%(error)s)', {'name': ext.name, 'type': ', '.join(handler.event_types), 'error': ack_on_error}) # NOTE(gordc): this could be a set check but oslo_messaging issue # https://bugs.launchpad.net/oslo.messaging/+bug/1398511 # This ensures we don't create multiple duplicate consumers. for new_tar in handler.get_targets(cfg.CONF): if new_tar not in targets: targets.append(new_tar) endpoints.append(handler) urls = cfg.CONF.notification.messaging_urls or [None] for url in urls: transport = messaging.get_transport(url) listener = messaging.get_batch_notification_listener( transport, targets, endpoints, batch_size=cfg.CONF.notification.batch_size, batch_timeout=cfg.CONF.notification.batch_timeout) listener.start() self.listeners.append(listener)
def _configure_main_queue_listeners(self, pipe_manager, event_pipe_manager): notification_manager = self._get_notifications_manager(pipe_manager) if not list(notification_manager): LOG.warning(_('Failed to load any notification handlers for %s'), self.NOTIFICATION_NAMESPACE) ack_on_error = self.conf.notification.ack_on_event_error endpoints = [] endpoints.append( event_endpoint.EventsNotificationEndpoint(event_pipe_manager)) targets = [] for ext in notification_manager: handler = ext.obj LOG.debug( 'Event types from %(name)s: %(type)s' ' (ack_on_error=%(error)s)', { 'name': ext.name, 'type': ', '.join(handler.event_types), 'error': ack_on_error }) # NOTE(gordc): this could be a set check but oslo_messaging issue # https://bugs.launchpad.net/oslo.messaging/+bug/1398511 # This ensures we don't create multiple duplicate consumers. for new_tar in handler.get_targets(self.conf): if new_tar not in targets: targets.append(new_tar) endpoints.append(handler) urls = self.conf.notification.messaging_urls or [None] for url in urls: transport = messaging.get_transport(self.conf, url) # NOTE(gordc): ignore batching as we want pull # to maintain sequencing as much as possible. listener = messaging.get_batch_notification_listener( transport, targets, endpoints) listener.start() self.listeners.append(listener)
def _configure_pipeline_listener(self): with self.coord_lock: ev_pipes = [] if cfg.CONF.notification.store_events: ev_pipes = self.event_pipeline_manager.pipelines pipelines = self.pipeline_manager.pipelines + ev_pipes transport = messaging.get_transport() partitioned = self.partition_coordinator.extract_my_subset( self.group_id, range(cfg.CONF.notification.pipeline_processing_queues)) endpoints = [] targets = [] for pipe in pipelines: if isinstance(pipe, pipeline.EventPipeline): endpoints.append(pipeline.EventPipelineEndpoint(self.ctxt, pipe)) else: endpoints.append(pipeline.SamplePipelineEndpoint(self.ctxt, pipe)) for pipe_set, pipe in itertools.product(partitioned, pipelines): LOG.debug('Pipeline endpoint: %s from set: %s', pipe.name, pipe_set) topic = '%s-%s-%s' % (self.NOTIFICATION_IPC, pipe.name, pipe_set) targets.append(oslo_messaging.Target(topic=topic)) if self.pipeline_listener: self.pipeline_listener.stop() self.pipeline_listener.wait() self.pipeline_listener = messaging.get_batch_notification_listener( transport, targets, endpoints, batch_size=cfg.CONF.notification.batch_size, batch_timeout=cfg.CONF.notification.batch_timeout) self.pipeline_listener.start()
def _configure_pipeline_listener(self): with self.coord_lock: ev_pipes = [] if cfg.CONF.notification.store_events: ev_pipes = self.event_pipeline_manager.pipelines pipelines = self.pipeline_manager.pipelines + ev_pipes transport = messaging.get_transport() partitioned = self.partition_coordinator.extract_my_subset( self.group_id, range(cfg.CONF.notification.pipeline_processing_queues)) endpoints = [] targets = [] for pipe in pipelines: if isinstance(pipe, pipeline.EventPipeline): endpoints.append( pipeline.EventPipelineEndpoint(self.ctxt, pipe)) else: endpoints.append( pipeline.SamplePipelineEndpoint(self.ctxt, pipe)) for pipe_set, pipe in itertools.product(partitioned, pipelines): LOG.debug('Pipeline endpoint: %s from set: %s', pipe.name, pipe_set) topic = '%s-%s-%s' % (self.NOTIFICATION_IPC, pipe.name, pipe_set) targets.append(oslo_messaging.Target(topic=topic)) if self.pipeline_listener: self.pipeline_listener.stop() self.pipeline_listener.wait() self.pipeline_listener = messaging.get_batch_notification_listener( transport, targets, endpoints, batch_size=cfg.CONF.notification.batch_size, batch_timeout=cfg.CONF.notification.batch_timeout) self.pipeline_listener.start()
def _configure_pipeline_listener(self): ev_pipes = self.event_pipeline_manager.pipelines pipelines = self.pipeline_manager.pipelines + ev_pipes transport = messaging.get_transport(self.conf) partitioned = self.partition_coordinator.extract_my_subset( self.group_id, range(self.conf.notification.pipeline_processing_queues)) endpoints = [] targets = [] for pipe in pipelines: if isinstance(pipe, pipeline.EventPipeline): endpoints.append(pipeline.EventPipelineEndpoint(pipe)) else: endpoints.append(pipeline.SamplePipelineEndpoint(pipe)) for pipe_set, pipe in itertools.product(partitioned, pipelines): LOG.debug('Pipeline endpoint: %s from set: %s', pipe.name, pipe_set) topic = '%s-%s-%s' % (self.NOTIFICATION_IPC, pipe.name, pipe_set) targets.append(oslo_messaging.Target(topic=topic)) if self.pipeline_listener: self.pipeline_listener.stop() self.pipeline_listener.wait() self.pipeline_listener = messaging.get_batch_notification_listener( transport, targets, endpoints, batch_size=self.conf.notification.batch_size, batch_timeout=self.conf.notification.batch_timeout) # NOTE(gordc): set single thread to process data sequentially # if batching enabled. batch = (1 if self.conf.notification.batch_size > 1 else None) self.pipeline_listener.start(override_pool_size=batch)