def run(self):
if self.conf.collector.udp_address:
self.udp_thread = utils.spawn_thread(self.start_udp)
transport = messaging.get_transport(self.conf, optional=True)
if transport:
if list(self.meter_manager):
sample_target = oslo_messaging.Target(
topic=self.conf.publisher_notifier.metering_topic)
self.sample_listener = (
messaging.get_batch_notification_listener(
transport, [sample_target], [
SampleEndpoint(
self.conf.publisher.telemetry_secret,
self.meter_manager)
],
allow_requeue=True,
batch_size=self.conf.collector.batch_size,
batch_timeout=self.conf.collector.batch_timeout))
self.sample_listener.start()
if list(self.event_manager):
event_target = oslo_messaging.Target(
topic=self.conf.publisher_notifier.event_topic)
self.event_listener = (
messaging.get_batch_notification_listener(
transport, [event_target], [
EventEndpoint(self.conf.publisher.telemetry_secret,
self.event_manager)
],
allow_requeue=True,
batch_size=self.conf.collector.batch_size,
batch_timeout=self.conf.collector.batch_timeout))
self.event_listener.start()
def run(self):
if self.conf.collector.udp_address:
self.udp_thread = utils.spawn_thread(self.start_udp)
transport = messaging.get_transport(self.conf, optional=True)
if transport:
if list(self.meter_manager):
sample_target = oslo_messaging.Target(
topic=self.conf.publisher_notifier.metering_topic)
self.sample_listener = (
messaging.get_batch_notification_listener(
transport, [sample_target],
[SampleEndpoint(self.conf.publisher.telemetry_secret,
self.meter_manager)],
allow_requeue=True,
batch_size=self.conf.collector.batch_size,
batch_timeout=self.conf.collector.batch_timeout))
self.sample_listener.start()
if list(self.event_manager):
event_target = oslo_messaging.Target(
topic=self.conf.publisher_notifier.event_topic)
self.event_listener = (
messaging.get_batch_notification_listener(
transport, [event_target],
[EventEndpoint(self.conf.publisher.telemetry_secret,
self.event_manager)],
allow_requeue=True,
batch_size=self.conf.collector.batch_size,
batch_timeout=self.conf.collector.batch_timeout))
self.event_listener.start()
def _configure_main_queue_listeners(self, pipe_manager, event_pipe_manager):
notification_manager = self._get_notifications_manager(pipe_manager)
if not list(notification_manager):
LOG.warning(_("Failed to load any notification handlers for %s"), self.NOTIFICATION_NAMESPACE)
ack_on_error = self.conf.notification.ack_on_event_error
endpoints = []
endpoints.append(event_endpoint.EventsNotificationEndpoint(event_pipe_manager))
targets = []
for ext in notification_manager:
handler = ext.obj
if self.conf.notification.disable_non_metric_meters and isinstance(handler, base.NonMetricNotificationBase):
continue
LOG.debug(
"Event types from %(name)s: %(type)s" " (ack_on_error=%(error)s)",
{"name": ext.name, "type": ", ".join(handler.event_types), "error": ack_on_error},
)
# NOTE(gordc): this could be a set check but oslo_messaging issue
# https://bugs.launchpad.net/oslo.messaging/+bug/1398511
# This ensures we don't create multiple duplicate consumers.
for new_tar in handler.get_targets(self.conf):
if new_tar not in targets:
targets.append(new_tar)
endpoints.append(handler)
urls = self.conf.notification.messaging_urls or [None]
for url in urls:
transport = messaging.get_transport(self.conf, url)
# NOTE(gordc): ignore batching as we want pull
# to maintain sequencing as much as possible.
listener = messaging.get_batch_notification_listener(transport, targets, endpoints)
listener.start()
self.listeners.append(listener)
def _configure_pipeline_listener(self):
partitioned = list(
filter(self.hashring.belongs_to_self, self.partition_set))
endpoints = []
for pipe_mgr in self.managers:
endpoints.extend(pipe_mgr.get_interim_endpoints())
targets = []
for mgr, hash_id in itertools.product(self.managers, partitioned):
topic = '-'.join([mgr.NOTIFICATION_IPC, mgr.pm_type, str(hash_id)])
LOG.debug('Listening to queue: %s', topic)
targets.append(oslo_messaging.Target(topic=topic))
if self.pipeline_listener:
self.pipeline_listener.stop()
self.pipeline_listener.wait()
self.pipeline_listener = messaging.get_batch_notification_listener(
self.transport,
targets,
endpoints,
batch_size=self.conf.notification.batch_size,
batch_timeout=self.conf.notification.batch_timeout)
# NOTE(gordc): set single thread to process data sequentially
# if batching enabled.
batch = (1 if self.conf.notification.batch_size > 1 else
self.conf.max_parallel_requests)
self.pipeline_listener.start(override_pool_size=batch)
def run(self):
# Delay startup so workers are jittered
time.sleep(self.startup_delay)
super(NotificationService, self).run()
self.managers = [ext.obj for ext in named.NamedExtensionManager(
namespace='ceilometer.notification.pipeline',
names=self.conf.notification.pipelines, invoke_on_load=True,
on_missing_entrypoints_callback=self._log_missing_pipeline,
invoke_args=(self.conf,))]
# FIXME(sileht): endpoint uses the notification_topics option
# and it should not because this is an oslo_messaging option
# not a ceilometer. Until we have something to get the
# notification_topics in another way, we must create a transport
# to ensure the option has been registered by oslo_messaging.
messaging.get_notifier(messaging.get_transport(self.conf), '')
endpoints = []
for pipe_mgr in self.managers:
endpoints.extend(pipe_mgr.get_main_endpoints())
targets = self.get_targets()
urls = self.conf.notification.messaging_urls or [None]
for url in urls:
transport = messaging.get_transport(self.conf, url)
# NOTE(gordc): ignore batching as we want pull
# to maintain sequencing as much as possible.
listener = messaging.get_batch_notification_listener(
transport, targets, endpoints, allow_requeue=True)
listener.start(
override_pool_size=self.conf.max_parallel_requests
)
self.listeners.append(listener)
def start(self):
"""Bind the UDP socket and handle incoming data."""
# ensure dispatcher is configured before starting other services
dispatcher_managers = dispatcher.load_dispatcher_manager()
(self.meter_manager, self.event_manager) = dispatcher_managers
self.sample_listener = None
self.event_listener = None
self.udp_thread = None
super(CollectorService, self).start()
if cfg.CONF.collector.udp_address:
self.udp_thread = threading.Thread(target=self.start_udp)
self.udp_thread.daemon = True
self.udp_thread.start()
transport = messaging.get_transport(optional=True)
if transport:
if list(self.meter_manager):
sample_target = oslo_messaging.Target(
topic=cfg.CONF.publisher_notifier.metering_topic)
self.sample_listener = (
messaging.get_batch_notification_listener(
transport, [sample_target],
[SampleEndpoint(self.meter_manager)],
allow_requeue=True,
batch_size=cfg.CONF.collector.batch_size,
batch_timeout=cfg.CONF.collector.batch_timeout))
self.sample_listener.start()
if cfg.CONF.notification.store_events and list(self.event_manager):
event_target = oslo_messaging.Target(
topic=cfg.CONF.publisher_notifier.event_topic)
self.event_listener = (
messaging.get_batch_notification_listener(
transport, [event_target],
[EventEndpoint(self.event_manager)],
allow_requeue=True,
batch_size=cfg.CONF.collector.batch_size,
batch_timeout=cfg.CONF.collector.batch_timeout))
self.event_listener.start()
if not cfg.CONF.collector.udp_address:
# NOTE(sileht): We have to drop oslo.service to remove this
# last eventlet thread
# Add a dummy thread to have wait() working
self.tg.add_timer(604800, lambda: None)
def run(self):
"""Bind the UDP socket and handle incoming data."""
super(CollectorService, self).run()
# ensure dispatcher is configured before starting other services
dispatcher_managers = dispatcher.load_dispatcher_manager()
(self.meter_manager, self.event_manager) = dispatcher_managers
self.sample_listener = None
self.event_listener = None
self.udp_thread = None
if cfg.CONF.collector.udp_address:
self.udp_thread = utils.spawn_thread(self.start_udp)
transport = messaging.get_transport(optional=True)
if transport:
if list(self.meter_manager):
sample_target = oslo_messaging.Target(
topic=cfg.CONF.publisher_notifier.metering_topic)
self.sample_listener = (
messaging.get_batch_notification_listener(
transport, [sample_target], [
SampleEndpoint(cfg.CONF.publisher.telemetry_secret,
self.meter_manager)
],
allow_requeue=True,
batch_size=cfg.CONF.collector.batch_size,
batch_timeout=cfg.CONF.collector.batch_timeout))
self.sample_listener.start()
if cfg.CONF.notification.store_events and list(self.event_manager):
event_target = oslo_messaging.Target(
topic=cfg.CONF.publisher_notifier.event_topic)
self.event_listener = (
messaging.get_batch_notification_listener(
transport, [event_target], [
EventEndpoint(cfg.CONF.publisher.telemetry_secret,
self.event_manager)
],
allow_requeue=True,
batch_size=cfg.CONF.collector.batch_size,
batch_timeout=cfg.CONF.collector.batch_timeout))
self.event_listener.start()
def start(self):
"""Bind the UDP socket and handle incoming data."""
# ensure dispatcher is configured before starting other services
dispatcher_managers = dispatcher.load_dispatcher_manager()
(self.meter_manager, self.event_manager) = dispatcher_managers
self.sample_listener = None
self.event_listener = None
self.udp_thread = None
super(CollectorService, self).start()
if cfg.CONF.collector.udp_address:
self.udp_thread = utils.spawn_thread(self.start_udp)
transport = messaging.get_transport(optional=True)
if transport:
if list(self.meter_manager):
sample_target = oslo_messaging.Target(
topic=cfg.CONF.publisher_notifier.metering_topic)
self.sample_listener = (
messaging.get_batch_notification_listener(
transport, [sample_target],
[SampleEndpoint(cfg.CONF.publisher.telemetry_secret,
self.meter_manager)],
allow_requeue=True,
batch_size=cfg.CONF.collector.batch_size,
batch_timeout=cfg.CONF.collector.batch_timeout))
self.sample_listener.start()
if cfg.CONF.notification.store_events and list(self.event_manager):
event_target = oslo_messaging.Target(
topic=cfg.CONF.publisher_notifier.event_topic)
self.event_listener = (
messaging.get_batch_notification_listener(
transport, [event_target],
[EventEndpoint(cfg.CONF.publisher.telemetry_secret,
self.event_manager)],
allow_requeue=True,
batch_size=cfg.CONF.collector.batch_size,
batch_timeout=cfg.CONF.collector.batch_timeout))
self.event_listener.start()
def _configure_main_queue_listeners(self):
endpoints = []
for pipe_mgr in self.managers:
endpoints.extend(pipe_mgr.get_main_endpoints())
targets = self.get_targets()
urls = self.conf.notification.messaging_urls or [None]
for url in urls:
transport = messaging.get_transport(self.conf, url)
# NOTE(gordc): ignore batching as we want pull
# to maintain sequencing as much as possible.
listener = messaging.get_batch_notification_listener(
transport, targets, endpoints)
listener.start(override_pool_size=self.conf.max_parallel_requests)
self.listeners.append(listener)
def _configure_main_queue_listeners(self, pipe_manager,
event_pipe_manager):
notification_manager = self._get_notifications_manager(pipe_manager)
if not list(notification_manager):
LOG.warning(_('Failed to load any notification handlers for %s'),
self.NOTIFICATION_NAMESPACE)
ack_on_error = cfg.CONF.notification.ack_on_event_error
endpoints = []
if cfg.CONF.notification.store_events:
endpoints.append(
event_endpoint.EventsNotificationEndpoint(event_pipe_manager))
targets = []
for ext in notification_manager:
handler = ext.obj
if (cfg.CONF.notification.disable_non_metric_meters
and isinstance(handler, base.NonMetricNotificationBase)):
continue
LOG.debug(
'Event types from %(name)s: %(type)s'
' (ack_on_error=%(error)s)', {
'name': ext.name,
'type': ', '.join(handler.event_types),
'error': ack_on_error
})
# NOTE(gordc): this could be a set check but oslo_messaging issue
# https://bugs.launchpad.net/oslo.messaging/+bug/1398511
# This ensures we don't create multiple duplicate consumers.
for new_tar in handler.get_targets(cfg.CONF):
if new_tar not in targets:
targets.append(new_tar)
endpoints.append(handler)
urls = cfg.CONF.notification.messaging_urls or [None]
for url in urls:
transport = messaging.get_transport(url)
listener = messaging.get_batch_notification_listener(
transport,
targets,
endpoints,
batch_size=cfg.CONF.notification.batch_size,
batch_timeout=cfg.CONF.notification.batch_timeout)
listener.start()
self.listeners.append(listener)
def _configure_pipeline_listeners(self, reuse_listeners=False):
with self.coord_lock:
ev_pipes = []
if cfg.CONF.notification.store_events:
ev_pipes = self.event_pipeline_manager.pipelines
pipelines = self.pipeline_manager.pipelines + ev_pipes
transport = messaging.get_transport()
partitioned = self.partition_coordinator.extract_my_subset(
self.group_id,
range(cfg.CONF.notification.pipeline_processing_queues))
queue_set = {}
for pipe_set, pipe in itertools.product(partitioned, pipelines):
queue_set['%s-%s-%s' %
(self.NOTIFICATION_IPC, pipe.name, pipe_set)] = pipe
if reuse_listeners:
topics = queue_set.keys()
kill_list = []
for listener in self.pipeline_listeners:
if listener.dispatcher.targets[0].topic in topics:
queue_set.pop(listener.dispatcher.targets[0].topic)
else:
kill_list.append(listener)
for listener in kill_list:
utils.kill_listeners([listener])
self.pipeline_listeners.remove(listener)
else:
utils.kill_listeners(self.pipeline_listeners)
self.pipeline_listeners = []
for topic, pipe in queue_set.items():
LOG.debug('Pipeline endpoint: %s from set: %s', pipe.name,
pipe_set)
pipe_endpoint = (pipeline.EventPipelineEndpoint
if isinstance(pipe, pipeline.EventPipeline)
else pipeline.SamplePipelineEndpoint)
listener = messaging.get_batch_notification_listener(
transport,
[oslo_messaging.Target(topic=topic)],
[pipe_endpoint(self.ctxt, pipe)],
batch_size=cfg.CONF.notification.batch_size,
batch_timeout=cfg.CONF.notification.batch_timeout)
listener.start()
self.pipeline_listeners.append(listener)
def _configure_pipeline_listeners(self, reuse_listeners=False):
with self.coord_lock:
ev_pipes = []
if cfg.CONF.notification.store_events:
ev_pipes = self.event_pipeline_manager.pipelines
pipelines = self.pipeline_manager.pipelines + ev_pipes
transport = messaging.get_transport()
partitioned = self.partition_coordinator.extract_my_subset(
self.group_id,
range(cfg.CONF.notification.pipeline_processing_queues))
queue_set = {}
for pipe_set, pipe in itertools.product(partitioned, pipelines):
queue_set['%s-%s-%s' %
(self.NOTIFICATION_IPC, pipe.name, pipe_set)] = pipe
if reuse_listeners:
topics = queue_set.keys()
kill_list = []
for listener in self.pipeline_listeners:
if listener.dispatcher.targets[0].topic in topics:
queue_set.pop(listener.dispatcher.targets[0].topic)
else:
kill_list.append(listener)
for listener in kill_list:
utils.kill_listeners([listener])
self.pipeline_listeners.remove(listener)
else:
utils.kill_listeners(self.pipeline_listeners)
self.pipeline_listeners = []
for topic, pipe in queue_set.items():
LOG.debug('Pipeline endpoint: %s from set: %s', pipe.name,
pipe_set)
pipe_endpoint = (pipeline.EventPipelineEndpoint if isinstance(
pipe, pipeline.EventPipeline) else
pipeline.SamplePipelineEndpoint)
listener = messaging.get_batch_notification_listener(
transport, [oslo_messaging.Target(topic=topic)],
[pipe_endpoint(self.ctxt, pipe)],
batch_size=cfg.CONF.notification.batch_size,
batch_timeout=cfg.CONF.notification.batch_timeout)
listener.start()
self.pipeline_listeners.append(listener)
def _configure_pipeline_listener(self):
ev_pipes = self.event_pipeline_manager.pipelines
pipelines = self.pipeline_manager.pipelines + ev_pipes
transport = messaging.get_transport(self.conf)
partitioned = six.moves.range(
self.conf.notification.pipeline_processing_queues
)
if self.partition_coordinator:
partitioned = list(filter(
self.hashring.belongs_to_self, partitioned))
endpoints = []
targets = []
for pipe in pipelines:
if isinstance(pipe, pipeline.EventPipeline):
endpoints.append(pipeline.EventPipelineEndpoint(pipe))
else:
endpoints.append(pipeline.SamplePipelineEndpoint(pipe))
for pipe_set, pipe in itertools.product(partitioned, pipelines):
LOG.debug('Pipeline endpoint: %s from set: %s',
pipe.name, pipe_set)
topic = '%s-%s-%s' % (self.NOTIFICATION_IPC,
pipe.name, pipe_set)
targets.append(oslo_messaging.Target(topic=topic))
if self.pipeline_listener:
self.pipeline_listener.stop()
self.pipeline_listener.wait()
self.pipeline_listener = messaging.get_batch_notification_listener(
transport,
targets,
endpoints,
batch_size=self.conf.notification.batch_size,
batch_timeout=self.conf.notification.batch_timeout)
# NOTE(gordc): set single thread to process data sequentially
# if batching enabled.
batch = (1 if self.conf.notification.batch_size > 1
else self.conf.max_parallel_requests)
self.pipeline_listener.start(override_pool_size=batch)
def _configure_main_queue_listeners(self, pipe_manager,
event_pipe_manager):
notification_manager = self._get_notifications_manager(pipe_manager)
if not list(notification_manager):
LOG.warning(_('Failed to load any notification handlers for %s'),
self.NOTIFICATION_NAMESPACE)
ack_on_error = cfg.CONF.notification.ack_on_event_error
endpoints = []
if cfg.CONF.notification.store_events:
endpoints.append(
event_endpoint.EventsNotificationEndpoint(event_pipe_manager))
targets = []
for ext in notification_manager:
handler = ext.obj
if (cfg.CONF.notification.disable_non_metric_meters and
isinstance(handler, base.NonMetricNotificationBase)):
continue
LOG.debug('Event types from %(name)s: %(type)s'
' (ack_on_error=%(error)s)',
{'name': ext.name,
'type': ', '.join(handler.event_types),
'error': ack_on_error})
# NOTE(gordc): this could be a set check but oslo_messaging issue
# https://bugs.launchpad.net/oslo.messaging/+bug/1398511
# This ensures we don't create multiple duplicate consumers.
for new_tar in handler.get_targets(cfg.CONF):
if new_tar not in targets:
targets.append(new_tar)
endpoints.append(handler)
urls = cfg.CONF.notification.messaging_urls or [None]
for url in urls:
transport = messaging.get_transport(url)
listener = messaging.get_batch_notification_listener(
transport, targets, endpoints,
batch_size=cfg.CONF.notification.batch_size,
batch_timeout=cfg.CONF.notification.batch_timeout)
listener.start()
self.listeners.append(listener)
def _configure_main_queue_listeners(self, pipe_manager,
event_pipe_manager):
notification_manager = self._get_notifications_manager(pipe_manager)
if not list(notification_manager):
LOG.warning(_('Failed to load any notification handlers for %s'),
self.NOTIFICATION_NAMESPACE)
ack_on_error = self.conf.notification.ack_on_event_error
endpoints = []
endpoints.append(
event_endpoint.EventsNotificationEndpoint(event_pipe_manager))
targets = []
for ext in notification_manager:
handler = ext.obj
LOG.debug(
'Event types from %(name)s: %(type)s'
' (ack_on_error=%(error)s)', {
'name': ext.name,
'type': ', '.join(handler.event_types),
'error': ack_on_error
})
# NOTE(gordc): this could be a set check but oslo_messaging issue
# https://bugs.launchpad.net/oslo.messaging/+bug/1398511
# This ensures we don't create multiple duplicate consumers.
for new_tar in handler.get_targets(self.conf):
if new_tar not in targets:
targets.append(new_tar)
endpoints.append(handler)
urls = self.conf.notification.messaging_urls or [None]
for url in urls:
transport = messaging.get_transport(self.conf, url)
# NOTE(gordc): ignore batching as we want pull
# to maintain sequencing as much as possible.
listener = messaging.get_batch_notification_listener(
transport, targets, endpoints)
listener.start()
self.listeners.append(listener)
def _configure_pipeline_listener(self):
with self.coord_lock:
ev_pipes = []
if cfg.CONF.notification.store_events:
ev_pipes = self.event_pipeline_manager.pipelines
pipelines = self.pipeline_manager.pipelines + ev_pipes
transport = messaging.get_transport()
partitioned = self.partition_coordinator.extract_my_subset(
self.group_id,
range(cfg.CONF.notification.pipeline_processing_queues))
endpoints = []
targets = []
for pipe in pipelines:
if isinstance(pipe, pipeline.EventPipeline):
endpoints.append(pipeline.EventPipelineEndpoint(self.ctxt,
pipe))
else:
endpoints.append(pipeline.SamplePipelineEndpoint(self.ctxt,
pipe))
for pipe_set, pipe in itertools.product(partitioned, pipelines):
LOG.debug('Pipeline endpoint: %s from set: %s',
pipe.name, pipe_set)
topic = '%s-%s-%s' % (self.NOTIFICATION_IPC,
pipe.name, pipe_set)
targets.append(oslo_messaging.Target(topic=topic))
if self.pipeline_listener:
self.pipeline_listener.stop()
self.pipeline_listener.wait()
self.pipeline_listener = messaging.get_batch_notification_listener(
transport,
targets,
endpoints,
batch_size=cfg.CONF.notification.batch_size,
batch_timeout=cfg.CONF.notification.batch_timeout)
self.pipeline_listener.start()
def _configure_pipeline_listener(self):
with self.coord_lock:
ev_pipes = []
if cfg.CONF.notification.store_events:
ev_pipes = self.event_pipeline_manager.pipelines
pipelines = self.pipeline_manager.pipelines + ev_pipes
transport = messaging.get_transport()
partitioned = self.partition_coordinator.extract_my_subset(
self.group_id,
range(cfg.CONF.notification.pipeline_processing_queues))
endpoints = []
targets = []
for pipe in pipelines:
if isinstance(pipe, pipeline.EventPipeline):
endpoints.append(
pipeline.EventPipelineEndpoint(self.ctxt, pipe))
else:
endpoints.append(
pipeline.SamplePipelineEndpoint(self.ctxt, pipe))
for pipe_set, pipe in itertools.product(partitioned, pipelines):
LOG.debug('Pipeline endpoint: %s from set: %s', pipe.name,
pipe_set)
topic = '%s-%s-%s' % (self.NOTIFICATION_IPC, pipe.name,
pipe_set)
targets.append(oslo_messaging.Target(topic=topic))
if self.pipeline_listener:
self.pipeline_listener.stop()
self.pipeline_listener.wait()
self.pipeline_listener = messaging.get_batch_notification_listener(
transport,
targets,
endpoints,
batch_size=cfg.CONF.notification.batch_size,
batch_timeout=cfg.CONF.notification.batch_timeout)
self.pipeline_listener.start()
def _configure_pipeline_listener(self):
ev_pipes = self.event_pipeline_manager.pipelines
pipelines = self.pipeline_manager.pipelines + ev_pipes
transport = messaging.get_transport(self.conf)
partitioned = self.partition_coordinator.extract_my_subset(
self.group_id,
range(self.conf.notification.pipeline_processing_queues))
endpoints = []
targets = []
for pipe in pipelines:
if isinstance(pipe, pipeline.EventPipeline):
endpoints.append(pipeline.EventPipelineEndpoint(pipe))
else:
endpoints.append(pipeline.SamplePipelineEndpoint(pipe))
for pipe_set, pipe in itertools.product(partitioned, pipelines):
LOG.debug('Pipeline endpoint: %s from set: %s',
pipe.name, pipe_set)
topic = '%s-%s-%s' % (self.NOTIFICATION_IPC,
pipe.name, pipe_set)
targets.append(oslo_messaging.Target(topic=topic))
if self.pipeline_listener:
self.pipeline_listener.stop()
self.pipeline_listener.wait()
self.pipeline_listener = messaging.get_batch_notification_listener(
transport,
targets,
endpoints,
batch_size=self.conf.notification.batch_size,
batch_timeout=self.conf.notification.batch_timeout)
# NOTE(gordc): set single thread to process data sequentially
# if batching enabled.
batch = (1 if self.conf.notification.batch_size > 1 else None)
self.pipeline_listener.start(override_pool_size=batch)
