def main():
from optparse import OptionParser
parser = OptionParser(usage=usage())
parser.add_option('-E', '--exchange', type='string', dest='exchange',
help="Exchange to push to", action='store')
parser.add_option('-T', '--type', type='string', dest='messageType',
help="Type of message to create", action='store')
parser.add_option('-R', '--routingkey', type='string', dest='routingKey',
help="Routing key for message", action='store')
parser.add_option('-D', '--data', type='string', dest='data',
help="Message data as JSON, use '-' to read from stdin", action='store')
parser.add_option('-M', '--mandatory', dest='mandatory',
help="Publish message with mandatory flag set.", action='store_true')
parser.add_option('-I', '--immediate', dest='immediate',
help="Publish message with immediate flag set.", action='store_true')
parser = AMQPConfig.addOptionsToParser(parser)
parser = addLoggingOptions(parser)
options, args = parser.parse_args()
if not options.data:
parser.error('You must supply input data.')
elif not options.exchange:
parser.error('You must supply an exchange.')
elif not options.messageType:
parser.error('You must supply a message type.')
elif not options.routingKey:
parser.error('You must supply a routing key.')
schemas = [SCHEMA]
schemas.extend(get_zenpack_schemas())
amqpConnectionInfo = AMQPConfig()
amqpConnectionInfo.update(options)
schema = Schema(*schemas)
publisher = Publisher(amqpConnectionInfo, schema)
initLogging(options)
pusher = Pusher(options.exchange, options.messageType, schema, publisher)
if options.data == '-':
data = loads(sys.stdin.read())
else:
data = loads(options.data)
published = pusher.push(data=data, routingKey=options.routingKey, mandatory=options.mandatory,
immediate=options.immediate)
if not published:
sys.exit(1)
def main():
from optparse import OptionParser
import sys
parser = OptionParser(usage=usage())
parser.add_option('-F',
'--format',
type='string',
dest='format',
default='protostream',
help='Format to dump the messages in (%s)' %
', '.join(_FORMATTERS.keys()))
parser.add_option(
'-c',
'--compression',
type='string',
dest='compression',
default='none',
help='Message compression algorithm (possible values: deflate, none)')
parser = AMQPConfig.addOptionsToParser(parser)
parser = addLoggingOptions(parser)
options, args = parser.parse_args()
schemas = [SCHEMA]
schemas.extend(get_zenpack_schemas())
try:
formatter = _FORMATTERS[options.format.lower()]
except KeyError:
parser.error('Invalid format "%s"' % options.format)
initLogging(options)
amqpConnectionInfo = AMQPConfig()
amqpConnectionInfo.update(options)
schema = Schema(*schemas)
schema.loadProperties(
{'exchange.default.compression': options.compression.lower()})
publisher = getProtobufPubSub(amqpConnectionInfo, schema, None)
loader = Loader(sys.stdin, formatter, schema, publisher)
try:
loader.load()
except KeyboardInterrupt:
pass
loader.publisher.shutdown()
def collect(self, config):
log.debug("Collect for OpenStack AMQP (%s)" % config.id)
# During the first collect run, we spin up the AMQP listener. After
# that, no active collecting is done in the collect() method.
#
# Instead, as each message arives over the AMQP listener, it goes through
# processMessage(), and is placed into a cache where it can be processed
# by the onSuccess method.
queue_key = "%s_%s" % (self.queue_name, config.id)
if queue_key not in amqp_client:
# Spin up the AMQP queue listener
zcml.load_config('configure.zcml', zope.component)
zcml.load_config('configure.zcml',
Products.ZenMessaging.queuemessaging)
self._amqpConnectionInfo = getUtility(IAMQPConnectionInfo)
self._amqpConnectionInfo_collector = AMQPConfig(
amqphost='localhost',
amqpport=55672,
amqpvhost=self._amqpConnectionInfo.vhost,
amqpuser=self._amqpConnectionInfo.user,
amqppassword=self._amqpConnectionInfo.password,
amqpusessl=self._amqpConnectionInfo.usessl,
amqpconnectionheartbeat=self._amqpConnectionInfo.
amqpconnectionheartbeat)
self._queueSchema = getUtility(IQueueSchema)
amqp = AMQPFactory(self._amqpConnectionInfo, self._queueSchema)
amqp_collector = AMQPFactory(self._amqpConnectionInfo_collector,
self._queueSchema)
queue = self._queueSchema.getQueue(
self.queue_name, replacements={'device': config.id})
log.debug(
"Listening on queue: %s with binding to routing key %s" %
(queue.name, queue.bindings[self.exchange_name].routing_key))
yield amqp.listen(queue,
callback=partial(self._processMessage, amqp,
config.id))
yield amqp_collector.listen(queue,
callback=partial(
self._processMessage,
amqp_collector, config.id))
amqp_client[queue_key] = amqp
amqp_client[queue_key + "_collector"] = amqp_collector
# Give time for some of the existing messages to be processed during
# this initial collection cycle
yield sleep(10)
data = self.new_data()
defer.returnValue(data)
def main():
from optparse import OptionParser
import sys
parser = OptionParser(usage=usage())
parser.add_option("-A", '--ack', action="store_true", dest="acknowledge",
help="Acknowledge the message, acknowledging a message will remove it from the queue")
parser.add_option('-F', '--format', type='string', dest='format', default='json',
help='Format to dump the messages in (%s)' % ', '.join(_FORMATTERS.keys()))
parser.add_option('-M', '--max', type='int', dest='max_items',
help='Maximum items to dump')
parser.add_option('-S', '--skip', action="store_true", dest="skip",
help="Skip processing messages on the queue - use with --ack to clear a queue.")
parser = AMQPConfig.addOptionsToParser(parser)
parser = addLoggingOptions(parser)
options, args = parser.parse_args()
if not args:
parser.error("Require one or more queues as arguments")
if options.skip and not options.acknowledge:
parser.error("Option --skip must be used with --ack")
schemas = [SCHEMA]
schemas.extend(get_zenpack_schemas())
amqpConnectionInfo = AMQPConfig()
amqpConnectionInfo.update(options)
schema = Schema(*schemas)
try:
formatter = _FORMATTERS[options.format.lower()]
except KeyError:
parser.error('Invalid format "%s"' % options.format)
initLogging(options)
publisher = Publisher(amqpConnectionInfo, schema)
dumper = Dumper(sys.stdout, formatter, publisher.getChannel(), schema, acknowledge=options.acknowledge,
skip=options.skip)
dumper.dumpQueues(args, limit=options.max_items)
def main():
from optparse import OptionParser
import sys
parser = OptionParser(usage=usage())
parser.add_option('-F', '--format', type='string', dest='format', default='protostream',
help='Format to dump the messages in (%s)' % ', '.join(_FORMATTERS.keys()))
parser.add_option('-c', '--compression', type='string', dest='compression', default='none',
help='Message compression algorithm (possible values: deflate, none)')
parser = AMQPConfig.addOptionsToParser(parser)
parser = addLoggingOptions(parser)
options, args = parser.parse_args()
schemas = [SCHEMA]
schemas.extend(get_zenpack_schemas())
try:
formatter = _FORMATTERS[options.format.lower()]
except KeyError:
parser.error('Invalid format "%s"' % options.format)
initLogging(options)
amqpConnectionInfo = AMQPConfig()
amqpConnectionInfo.update(options)
schema = Schema(*schemas)
schema.loadProperties({'exchange.default.compression': options.compression.lower()})
publisher = getProtobufPubSub(amqpConnectionInfo, schema, None)
loader = Loader(sys.stdin, formatter, schema, publisher)
try:
loader.load()
except KeyboardInterrupt:
pass
loader.publisher.shutdown()
def read_configuration(self):
"""
Build a Celery configuration dictionary using global.conf.
This configuration may be overwritten or augmented later by a daemon.
"""
globalCfg = getGlobalConfiguration()
amqpCfg = AMQPConfig()
amqpCfg.update(globalCfg)
config = {
#######
# APP #
#######
# Configure the value of the 'CELERYD_POOL_PUTLOCKS'. When
# True, it causes a temporary deadlock to occur during shutdown
# when all workers are busy and there exists at least one
# pending job. The deadlock is broken when a worker completes
# its job. Setting 'CELERYD_POOL_PUTLOCKS' to False allows
# shutdown to occur without waiting.
"CELERYD_POOL_PUTLOCKS": False,
#############
# TRANSPORT #
#############
constants.BROKER_HOST: amqpCfg.host,
constants.BROKER_PORT: amqpCfg.port,
constants.BROKER_USER: amqpCfg.user,
constants.BROKER_PASSWORD: amqpCfg.password,
constants.BROKER_VHOST: amqpCfg.vhost,
constants.BROKER_USE_SSL: amqpCfg.usessl,
constants.ACK_LATE: True,
################
# RESULT STORE #
################
constants.RESULT_BACKEND: 'zodb',
###########
# WORKERS #
###########
# Default to 1 job per worker process
constants.MAX_TASKS_PER_PROCESS: 1,
# 2 job workers
constants.NUM_WORKERS: 2,
###########
# LOGGING #
###########
# Handle logging ourselves
constants.USE_CELERY_LOGGING: False,
# Log file formats
constants.LOG_FORMAT: "%(asctime)s %(levelname)s %(name)s: %(message)s",
constants.TASK_LOG_FORMAT: "%(asctime)s %(levelname)s zen.Job: %(message)s",
# Level at which stdout should be logged
constants.STDOUT_LOG_LEVEL: 'INFO'
}
return config
def main():
from optparse import OptionParser
parser = OptionParser(usage=usage())
parser.add_option('-E',
'--exchange',
type='string',
dest='exchange',
help="Exchange to push to",
action='store')
parser.add_option('-T',
'--type',
type='string',
dest='messageType',
help="Type of message to create",
action='store')
parser.add_option('-R',
'--routingkey',
type='string',
dest='routingKey',
help="Routing key for message",
action='store')
parser.add_option('-D',
'--data',
type='string',
dest='data',
help="Message data as JSON, use '-' to read from stdin",
action='store')
parser.add_option('-M',
'--mandatory',
dest='mandatory',
help="Publish message with mandatory flag set.",
action='store_true')
parser = AMQPConfig.addOptionsToParser(parser)
parser = addLoggingOptions(parser)
options, args = parser.parse_args()
if not options.data:
parser.error('You must supply input data.')
elif not options.exchange:
parser.error('You must supply an exchange.')
elif not options.messageType:
parser.error('You must supply a message type.')
elif not options.routingKey:
parser.error('You must supply a routing key.')
schemas = [SCHEMA]
schemas.extend(get_zenpack_schemas())
amqpConnectionInfo = AMQPConfig()
amqpConnectionInfo.update(options)
schema = Schema(*schemas)
publisher = Publisher(amqpConnectionInfo, schema)
initLogging(options)
pusher = Pusher(options.exchange, options.messageType, schema, publisher)
if options.data == '-':
data = loads(sys.stdin.read())
else:
data = loads(options.data)
published = pusher.push(data=data,
routingKey=options.routingKey,
mandatory=options.mandatory)
if not published:
sys.exit(1)
def _loadAmqpConnectionInfo():
from Products.ZenUtils.GlobalConfig import getGlobalConfiguration
from zenoss.protocols.amqpconfig import AMQPConfig
amqpConnectionInfo = AMQPConfig()
amqpConnectionInfo.update(getGlobalConfiguration())
return amqpConnectionInfo
def main():
from optparse import OptionParser
import sys
parser = OptionParser(usage=usage())
parser.add_option(
"-A",
'--ack',
action="store_true",
dest="acknowledge",
help=
"Acknowledge the message, acknowledging a message will remove it from the queue"
)
parser.add_option('-F',
'--format',
type='string',
dest='format',
default='json',
help='Format to dump the messages in (%s)' %
', '.join(_FORMATTERS.keys()))
parser.add_option('-M',
'--max',
type='int',
dest='max_items',
help='Maximum items to dump')
parser.add_option(
'-S',
'--skip',
action="store_true",
dest="skip",
help=
"Skip processing messages on the queue - use with --ack to clear a queue."
)
parser = AMQPConfig.addOptionsToParser(parser)
parser = addLoggingOptions(parser)
options, args = parser.parse_args()
if not args:
parser.error("Require one or more queues as arguments")
if options.skip and not options.acknowledge:
parser.error("Option --skip must be used with --ack")
schemas = [SCHEMA]
schemas.extend(get_zenpack_schemas())
amqpConnectionInfo = AMQPConfig()
amqpConnectionInfo.update(options)
schema = Schema(*schemas)
try:
formatter = _FORMATTERS[options.format.lower()]
except KeyError:
parser.error('Invalid format "%s"' % options.format)
initLogging(options)
publisher = Publisher(amqpConnectionInfo, schema)
dumper = Dumper(sys.stdout,
formatter,
publisher.getChannel(),
schema,
acknowledge=options.acknowledge,
skip=options.skip)
dumper.dumpQueues(args, limit=options.max_items)
def _loadAmqpConnectionInfo():
from Products.ZenUtils.GlobalConfig import getGlobalConfiguration
from zenoss.protocols.amqpconfig import AMQPConfig
amqpConnectionInfo = AMQPConfig()
amqpConnectionInfo.update(getGlobalConfiguration())
return amqpConnectionInfo
def read_configuration(self):
"""
Build a Celery configuration dictionary using global.conf.
This configuration may be overwritten or augmented later by a daemon.
"""
globalCfg = getGlobalConfiguration()
amqpCfg = AMQPConfig()
amqpCfg.update(globalCfg)
config = {
#############
# TRANSPORT #
#############
constants.BROKER_HOST:
amqpCfg.host,
constants.BROKER_PORT:
amqpCfg.port,
constants.BROKER_USER:
amqpCfg.user,
constants.BROKER_PASSWORD:
amqpCfg.password,
constants.BROKER_VHOST:
amqpCfg.vhost,
constants.BROKER_USE_SSL:
amqpCfg.usessl,
constants.ACK_LATE:
True,
################
# RESULT STORE #
################
constants.RESULT_BACKEND:
'zodb',
###########
# WORKERS #
###########
# Default to 1 job per worker process
constants.MAX_TASKS_PER_PROCESS:
1,
# 2 job workers
constants.NUM_WORKERS:
2,
###########
# LOGGING #
###########
# Handle logging ourselves
constants.USE_CELERY_LOGGING:
False,
# Log file formats
constants.LOG_FORMAT:
"%(asctime)s %(levelname)s %(name)s: %(message)s",
constants.TASK_LOG_FORMAT:
"%(asctime)s %(levelname)s zen.Job: %(message)s",
# Level at which stdout should be logged
constants.STDOUT_LOG_LEVEL:
'INFO'
}
return config
