class AsyncQueuePublisher(object):
"""
Sends the protobuf to an exchange in a non-blocking manner
"""
implements(IQueuePublisher)
def __init__(self):
self.reconnect()
def reconnect(self):
connectionInfo = getUtility(IAMQPConnectionInfo)
queueSchema = getUtility(IQueueSchema)
self._amqpClient = AMQPFactory(connectionInfo, queueSchema)
@defer.inlineCallbacks
def publish(self, exchange, routing_key, message, createQueues=None,
mandatory=False, immediate=False, headers=None,
declareExchange=True):
if createQueues:
for queue in createQueues:
yield self._amqpClient.createQueue(queue)
result = yield self._amqpClient.send(exchange, routing_key, message,
mandatory=mandatory,
immediate=immediate,
headers=headers,
declareExchange=declareExchange)
defer.returnValue(result)
@property
def channel(self):
return self._amqpClient.channel
def close(self):
return self._amqpClient.shutdown()
class AsyncQueuePublisher(object):
"""
Sends the protobuf to an exchange in a non-blocking manner
"""
implements(IQueuePublisher)
def __init__(self):
self.reconnect()
def reconnect(self):
connectionInfo = getUtility(IAMQPConnectionInfo)
queueSchema = getUtility(IQueueSchema)
self._amqpClient = AMQPFactory(connectionInfo, queueSchema)
@defer.inlineCallbacks
def publish(self, exchange, routing_key, message, createQueues=None,
mandatory=False, headers=None,
declareExchange=True):
if createQueues:
for queue in createQueues:
yield self._amqpClient.createQueue(queue)
result = yield self._amqpClient.send(exchange, routing_key, message,
mandatory=mandatory,
headers=headers,
declareExchange=declareExchange)
defer.returnValue(result)
@property
def channel(self):
return self._amqpClient.channel
def close(self):
return self._amqpClient.shutdown()
def __init__(self, task, dmd, amqpConnectionInfo=None, queueSchema=None):
self.dmd = dmd
if not amqpConnectionInfo:
amqpConnectionInfo = getUtility(IAMQPConnectionInfo)
if not queueSchema:
queueSchema = getUtility(IQueueSchema)
self.consumer = AMQPFactory(amqpConnectionInfo, queueSchema)
self.onReady = self._ready()
if not IQueueConsumerTask.providedBy(task):
raise AssertionError("%s does not implement IQueueConsumerTask" %
task)
self.task = task
self.task.dmd = self.dmd
# give a reference to the consumer to the task
self.task.queueConsumer = self
self.shuttingDown = False
def collect(self, config):
log.debug("Collect for OpenStack AMQP (%s)" % config.id)
# During the first collect run, we spin up the AMQP listener. After
# that, no active collecting is done in the collect() method.
#
# Instead, as each message arives over the AMQP listener, it goes through
# processMessage(), and is placed into a cache where it can be processed
# by the onSuccess method.
if config.id not in amqp_client:
# Spin up the AMQP queue listener
zcml.load_config('configure.zcml', zope.component)
zcml.load_config('configure.zcml', Products.ZenMessaging.queuemessaging)
self._amqpConnectionInfo = getUtility(IAMQPConnectionInfo)
self._queueSchema = getUtility(IQueueSchema)
amqp = AMQPFactory(self._amqpConnectionInfo, self._queueSchema)
queue = self._queueSchema.getQueue('$OpenStackInboundPerf', replacements={'device': config.id})
log.debug("Listening on queue: %s with binding to routing key %s" % (queue.name, queue.bindings['$OpenStackInbound'].routing_key))
yield amqp.listen(queue, callback=partial(self.processMessage, amqp, config.id))
amqp_client[config.id] = amqp
# Give time for some of the existing messages to be processed during
# this initial collection cycle
yield sleep(10)
data = self.new_data()
device_id = config.configId
for ds in config.datasources:
for entry in cache[device_id].get_perf(ds.params['resourceId'], ds.params['meter']):
log.debug("perf %s/%s=%s @ %s" % (ds.params['resourceId'], ds.params['meter'], entry.value, entry.timestamp))
data['values'][ds.component][ds.datasource] = (entry.value, entry.timestamp)
if len(data['values']):
data['events'].append({
'device': config.id,
'component': ds.component,
'summary': 'OpenStack Ceilometer AMQP: successful collection',
'severity': ZenEventClasses.Clear,
'eventKey': 'openstackCeilometerAMQPCollection',
'eventClassKey': 'PerfSuccess',
})
defer.returnValue(data)
def collect(self, config):
log.debug("Collect for OpenStack AMQP (%s)" % config.id)
# During the first collect run, we spin up the AMQP listener. After
# that, no active collecting is done in the collect() method.
#
# Instead, as each message arives over the AMQP listener, it goes through
# processMessage(), and is placed into a cache where it can be processed
# by the onSuccess method.
queue_key = "%s_%s" % (self.queue_name, config.id)
if queue_key not in amqp_client:
# Spin up the AMQP queue listener
zcml.load_config('configure.zcml', zope.component)
zcml.load_config('configure.zcml', Products.ZenMessaging.queuemessaging)
self._amqpConnectionInfo = getUtility(IAMQPConnectionInfo)
self._amqpConnectionInfo_collector = AMQPConfig(
amqphost='localhost',
amqpport=55672,
amqpvhost=self._amqpConnectionInfo.vhost,
amqpuser=self._amqpConnectionInfo.user,
amqppassword=self._amqpConnectionInfo.password,
amqpusessl=self._amqpConnectionInfo.usessl,
amqpconnectionheartbeat=self._amqpConnectionInfo.amqpconnectionheartbeat)
self._queueSchema = getUtility(IQueueSchema)
amqp = AMQPFactory(self._amqpConnectionInfo, self._queueSchema)
amqp_collector = AMQPFactory(self._amqpConnectionInfo_collector, self._queueSchema)
queue = self._queueSchema.getQueue(self.queue_name, replacements={'device': config.id})
log.debug("Listening on queue: %s with binding to routing key %s" % (queue.name, queue.bindings[self.exchange_name].routing_key))
yield amqp.listen(queue, callback=partial(self._processMessage, amqp, config.id))
yield amqp_collector.listen(queue, callback=partial(self._processMessage, amqp_collector, config.id))
amqp_client[queue_key] = amqp
amqp_client[queue_key + "_collector"] = amqp_collector
# Give time for some of the existing messages to be processed during
# this initial collection cycle
yield sleep(10)
data = self.new_data()
defer.returnValue(data)
def connect_to_amqp(self):
zcml.load_config('configure.zcml', zope.component)
zcml.load_config('configure.zcml',
Products.ZenMessaging.queuemessaging)
self._amqpConnectionInfo = getUtility(IAMQPConnectionInfo)
self._queueSchema = getUtility(IQueueSchema)
self.amqp = AMQPFactory(self._amqpConnectionInfo, self._queueSchema)
def __init__(self, task, dmd, amqpConnectionInfo=None, queueSchema=None):
self.dmd = dmd
if not amqpConnectionInfo:
amqpConnectionInfo = getUtility(IAMQPConnectionInfo)
if not queueSchema:
queueSchema = getUtility(IQueueSchema)
self.consumer = AMQPFactory(amqpConnectionInfo, queueSchema)
self.onReady = self._ready()
if not IQueueConsumerTask.providedBy(task):
raise AssertionError("%s does not implement IQueueConsumerTask" % task)
self.task = task
self.task.dmd = self.dmd
# give a reference to the consumer to the task
self.task.queueConsumer = self
self.shuttingDown = False
def collect(self, config):
log.debug("Collect for OpenStack AMQP (%s)" % config.id)
# During the first collect run, we spin up the AMQP listener. After
# that, no active collecting is done in the collect() method.
#
# Instead, as each message arives over the AMQP listener, it goes through
# processMessage(), and is placed into a cache where it can be processed
# by the onSuccess method.
queue_key = "%s_%s" % (self.queue_name, config.id)
if queue_key not in amqp_client:
# Spin up the AMQP queue listener
zcml.load_config('configure.zcml', zope.component)
zcml.load_config('configure.zcml',
Products.ZenMessaging.queuemessaging)
self._amqpConnectionInfo = getUtility(IAMQPConnectionInfo)
self._amqpConnectionInfo_collector = AMQPConfig(
amqphost='localhost',
amqpport=55672,
amqpvhost=self._amqpConnectionInfo.vhost,
amqpuser=self._amqpConnectionInfo.user,
amqppassword=self._amqpConnectionInfo.password,
amqpusessl=self._amqpConnectionInfo.usessl,
amqpconnectionheartbeat=self._amqpConnectionInfo.
amqpconnectionheartbeat)
self._queueSchema = getUtility(IQueueSchema)
amqp = AMQPFactory(self._amqpConnectionInfo, self._queueSchema)
amqp_collector = AMQPFactory(self._amqpConnectionInfo_collector,
self._queueSchema)
queue = self._queueSchema.getQueue(
self.queue_name, replacements={'device': config.id})
log.debug(
"Listening on queue: %s with binding to routing key %s" %
(queue.name, queue.bindings[self.exchange_name].routing_key))
yield amqp.listen(queue,
callback=partial(self._processMessage, amqp,
config.id))
yield amqp_collector.listen(queue,
callback=partial(
self._processMessage,
amqp_collector, config.id))
amqp_client[queue_key] = amqp
amqp_client[queue_key + "_collector"] = amqp_collector
# Give time for some of the existing messages to be processed during
# this initial collection cycle
yield sleep(10)
data = self.new_data()
defer.returnValue(data)
def collect(self, config):
log.debug("Collect for OpenStack AMQP Queue Size (%s)" % config.id)
results = {}
self._amqpConnectionInfo = getUtility(IAMQPConnectionInfo)
self._queueSchema = getUtility(IQueueSchema)
# reuse existing connectio if there is one.
if config.id in amqp_client and amqp_client[config.id]:
amqp = amqp_client[config.id]
else:
amqp = AMQPFactory(self._amqpConnectionInfo, self._queueSchema)
yield amqp._onConnectionMade
amqp_client[config.id] = amqp
for queuename in (
'$OpenStackInboundPerf',
'$OpenStackInboundEvent',
):
queue = self._queueSchema.getQueue(
queuename, replacements={'device': config.id})
try:
info = yield amqp.channel.queue_declare(queue=queue.name,
passive=True)
results[queuename] = info.fields[1]
except txamqp.client.Closed, e:
log.info(
"Unable to determine queue size for %s (queue does not exist)"
% queue.name)
pass
except Exception, e:
log.info("Unable to determine queue size for %s (%s)" %
(queue.name, e))
pass
class QueueConsumer(object):
"""
Listens to the model change queue and translates the
events into graph protobufs
"""
MARKER = str(hash(object()))
def __init__(self, task, dmd, amqpConnectionInfo=None, queueSchema=None):
self.dmd = dmd
if not amqpConnectionInfo:
amqpConnectionInfo = getUtility(IAMQPConnectionInfo)
if not queueSchema:
queueSchema = getUtility(IQueueSchema)
self.consumer = AMQPFactory(amqpConnectionInfo, queueSchema)
self.onReady = self._ready()
if not IQueueConsumerTask.providedBy(task):
raise AssertionError("%s does not implement IQueueConsumerTask" % task)
self.task = task
self.task.dmd = self.dmd
# give a reference to the consumer to the task
self.task.queueConsumer = self
self.shuttingDown = False
def authenticated(self):
return self.consumer._onAuthenticated
def connectionLost(self):
return self.consumer._onConnectionLost
def connectionMade(self):
return self.consumer._onConnectionMade
def connectionFailed(self):
return self.consumer._onConnectionFailed
def _ready(self):
"""
Calls back once everything's ready and test message went through.
"""
df = self.consumer._onConnectionMade
def logCb(result):
log.info('Queue consumer ready.')
return result
df.addCallback(logCb)
return df
def setPrefetch(self, prefetch):
if hasattr(self.consumer, "setPrefetch"):
self.consumer.setPrefetch(prefetch)
def run(self):
"""
Tell all the services to start up. Begin listening for queue messages.
"""
task = self.task
log.debug("listening to %s queue", task.queue.name)
self.consumer.listen(task.queue, callback=task.processMessage)
return self.onReady
def shutdown(self, *args):
"""
Tell all the services to shut down.
"""
self.shuttingDown = True
return self.consumer.shutdown()
def acknowledge(self, message):
"""
Called from a task when it is done successfully processing
the message
"""
return self.consumer.acknowledge(message)
def reject(self, message, requeue=False):
"""
Called from a task when it wants to reject and optionally requeue
a message.
"""
return self.consumer.reject(message, requeue)
def publishMessage(self, exchange, routing_key, message, mandatory=False, headers=None,
declareExchange=True):
"""
Publishes a message to another queue. This is for tasks that are both
consumers and producers.
"""
return self.consumer.send(exchange, routing_key, message, mandatory, headers, declareExchange)
def syncdb(self):
self.dmd.getPhysicalRoot()._p_jar.sync()
def reconnect(self):
connectionInfo = getUtility(IAMQPConnectionInfo)
queueSchema = getUtility(IQueueSchema)
self._amqpClient = AMQPFactory(connectionInfo, queueSchema)
class QueueConsumer(object):
"""
Listens to the model change queue and translates the
events into graph protobufs
"""
MARKER = str(hash(object()))
def __init__(self, task, dmd, amqpConnectionInfo=None, queueSchema=None):
self.dmd = dmd
if not amqpConnectionInfo:
amqpConnectionInfo = getUtility(IAMQPConnectionInfo)
if not queueSchema:
queueSchema = getUtility(IQueueSchema)
self.consumer = AMQPFactory(amqpConnectionInfo, queueSchema)
self.onReady = self._ready()
if not IQueueConsumerTask.providedBy(task):
raise AssertionError("%s does not implement IQueueConsumerTask" %
task)
self.task = task
self.task.dmd = self.dmd
# give a reference to the consumer to the task
self.task.queueConsumer = self
self.shuttingDown = False
def authenticated(self):
return self.consumer._onAuthenticated
def connectionLost(self):
return self.consumer._onConnectionLost
def connectionMade(self):
return self.consumer._onConnectionMade
def connectionFailed(self):
return self.consumer._onConnectionFailed
def _ready(self):
"""
Calls back once everything's ready and test message went through.
"""
df = self.consumer._onConnectionMade
def logCb(result):
log.info('Queue consumer ready.')
return result
df.addCallback(logCb)
return df
def setPrefetch(self, prefetch):
if hasattr(self.consumer, "setPrefetch"):
self.consumer.setPrefetch(prefetch)
def run(self):
"""
Tell all the services to start up. Begin listening for queue messages.
"""
task = self.task
log.debug("listening to %s queue", task.queue.name)
self.consumer.listen(task.queue, callback=task.processMessage)
return self.onReady
def shutdown(self, *args):
"""
Tell all the services to shut down.
"""
self.shuttingDown = True
return self.consumer.shutdown()
def acknowledge(self, message):
"""
Called from a task when it is done successfully processing
the message
"""
return self.consumer.acknowledge(message)
def reject(self, message, requeue=False):
"""
Called from a task when it wants to reject and optionally requeue
a message.
"""
return self.consumer.reject(message, requeue)
def publishMessage(self,
exchange,
routing_key,
message,
mandatory=False,
headers=None,
declareExchange=True):
"""
Publishes a message to another queue. This is for tasks that are both
consumers and producers.
"""
return self.consumer.send(exchange, routing_key, message, mandatory,
headers, declareExchange)
def syncdb(self):
self.dmd.getPhysicalRoot()._p_jar.sync()
def collect(self, config):
log.debug("Collect for OpenStack AMQP Heartbeats (%s)" % config.id)
service = yield self.getService('ZenPacks.zenoss.OpenStackInfrastructure.services.OpenStackService')
expected_heartbeats = yield service.callRemote('expected_ceilometer_heartbeats', config.id)
if not expected_heartbeats:
return
# During the first collect run, we spin up the AMQP listener. After
# that, no active collecting is done in the collect() method.
#
# Instead, as each message arrives over the AMQP listener, it goes through
# processMessage(), and is placed into a list, per host, per process,
# where it can be processed by the onSuccess method.
if config.id not in amqp_client:
# Spin up the AMQP queue listener
zcml.load_config('configure.zcml', zope.component)
zcml.load_config('configure.zcml', Products.ZenMessaging.queuemessaging)
self._amqpConnectionInfo = getUtility(IAMQPConnectionInfo)
self._queueSchema = getUtility(IQueueSchema)
amqp = AMQPFactory(self._amqpConnectionInfo, self._queueSchema)
queue = self._queueSchema.getQueue('$OpenStackInboundHeartbeat',
replacements={'device': config.id})
log.info("Listening on queue: %s with binding to routing key %s" % (queue.name, queue.bindings['$OpenStackInboundHeartbeats'].routing_key))
yield amqp.listen(queue, callback=partial(self.processMessage, amqp))
amqp_client[config.id] = amqp
# Give time for some of the existing messages to be processed during
# this initial collection cycle
yield sleep(10)
data = self.new_data()
device_id = config.id
for host in expected_heartbeats:
hostname = host['hostnames'][0]
possible_hostnames = host['hostnames']
required_processes = host['processes']
heartbeat_hostname = None
for possible_hostname in possible_hostnames:
if possible_hostname in last_heard_heartbeats:
heartbeat_hostname = possible_hostname
break
process_heartbeats = {}
if heartbeat_hostname is None:
# We have not heard a heartbeat from this host under
# any of its possible hostnames (short name, fqdn)
#
# So there won't be any process heartbeats, and we will
# consider them all to be down.
process_heartbeats = {}
else:
process_heartbeats = last_heard_heartbeats[heartbeat_hostname]
for proc in required_processes:
if proc not in process_heartbeats:
evt = {
'device': device_id,
'severity': ZenEventClasses.Warning,
'eventKey': hostname + '_' + proc,
'summary': "No heartbeats received from '%s' on %s. Check the host and process status." % (proc, hostname),
'eventClassKey': 'openStackCeilometerHeartbeat',
}
log.error(pformat(evt))
else:
# diff > MAX_TIME_LAPSE?
time_diff = time() - \
process_heartbeats[proc]['lastheard']
if time_diff > MAX_TIME_LAPSE:
evt = {
'device': device_id,
'severity': ZenEventClasses.Warning,
'eventKey': hostname + '_' + proc,
'summary': "No heartbeats received from '%s' on %s for more than %d seconds. Check its status and restart it if necessary." % (proc, hostname, MAX_TIME_LAPSE),
'eventClassKey': 'openStackCeilometerHeartbeat',
}
log.error(pformat(evt))
else:
evt = {
'device': device_id,
'severity': ZenEventClasses.Clear,
'eventKey': hostname + '_' + proc,
'summary': "Process '%s' on %s is sending heartbeats normally." % (proc, hostname),
'eventClassKey': 'openStackCeilometerHeartbeat',
}
data['events'].append(evt)
if len(data['events']):
data['events'].append({
'device': device_id,
'summary': 'OpenStack Ceilometer AMQP Heartbeat: successful collection',
'severity': ZenEventClasses.Clear,
'eventKey': 'openstackCeilometerAMQPHeartbeatCollection',
'eventClassKey': 'EventsSuccess',
})
defer.returnValue(data)
def collect(self, config):
log.debug("Collect for OpenStack AMQP Events (%s)" % config.id)
# During the first collect run, we spin up the AMQP listener. After
# that, no active collecting is done in the collect() method.
#
# Instead, as each message arives over the AMQP listener, it goes through
# processMessage(), and is placed into a cache where it can be processed
# by the onSuccess method.
if config.id not in amqp_client:
# Spin up the AMQP queue listener
zcml.load_config('configure.zcml', zope.component)
zcml.load_config('configure.zcml', Products.ZenMessaging.queuemessaging)
self._amqpConnectionInfo = getUtility(IAMQPConnectionInfo)
self._queueSchema = getUtility(IQueueSchema)
amqp = AMQPFactory(self._amqpConnectionInfo, self._queueSchema)
queue = self._queueSchema.getQueue('$OpenStackInboundEvent', replacements={'device': config.id})
log.info("Listening on queue: %s with binding to routing key %s" % (queue.name, queue.bindings['$OpenStackInbound'].routing_key))
yield amqp.listen(queue, callback=partial(self.processMessage, amqp, config.id))
amqp_client[config.id] = amqp
# Give time for some of the existing messages to be processed during
# this initial collection cycle
yield sleep(10)
data = self.new_data()
device_id = config.configId
for entry in cache[device_id].get():
c_event = entry.value
evt = {
'device': device_id,
'severity': ZenEventClasses.Info,
'eventKey': '',
'summary': 'OpenStackInfrastructure: ' + c_event['event_type'],
'eventClassKey': 'openstack|' + c_event['event_type'],
}
traits = {}
for trait in c_event['traits']:
# liberty: [[name, dtype, value] ...]
# [[u'display_name', 1, u'demo-volume1-snap'], ...]
traits[trait[0]] = trait[2]
if 'priority' in traits:
if traits['priority'] == 'WARN':
evt['severity'] = ZenEventClasses.Warning
elif traits['priority'] == 'ERROR':
evt['severity'] = ZenEventClasses.Error
evt['eventKey'] = c_event['message_id']
for trait in traits:
evt['trait_' + trait] = traits[trait]
from pprint import pformat
log.debug(pformat(evt))
data['events'].append(evt)
if len(data['events']):
data['events'].append({
'device': config.id,
'summary': 'OpenStack Ceilometer AMQP: successful collection',
'severity': ZenEventClasses.Clear,
'eventKey': 'openstackCeilometerAMQPCollection',
'eventClassKey': 'EventsSuccess',
})
defer.returnValue(data)
def reconnect(self):
connectionInfo = getUtility(IAMQPConnectionInfo)
queueSchema = getUtility(IQueueSchema)
self._amqpClient = AMQPFactory(connectionInfo, queueSchema)
