def main(): from optparse import OptionParser parser = OptionParser(usage=usage()) parser.add_option('-E', '--exchange', type='string', dest='exchange', help="Exchange to push to", action='store') parser.add_option('-T', '--type', type='string', dest='messageType', help="Type of message to create", action='store') parser.add_option('-R', '--routingkey', type='string', dest='routingKey', help="Routing key for message", action='store') parser.add_option('-D', '--data', type='string', dest='data', help="Message data as JSON, use '-' to read from stdin", action='store') parser.add_option('-M', '--mandatory', dest='mandatory', help="Publish message with mandatory flag set.", action='store_true') parser.add_option('-I', '--immediate', dest='immediate', help="Publish message with immediate flag set.", action='store_true') parser = AMQPConfig.addOptionsToParser(parser) parser = addLoggingOptions(parser) options, args = parser.parse_args() if not options.data: parser.error('You must supply input data.') elif not options.exchange: parser.error('You must supply an exchange.') elif not options.messageType: parser.error('You must supply a message type.') elif not options.routingKey: parser.error('You must supply a routing key.') schemas = [SCHEMA] schemas.extend(get_zenpack_schemas()) amqpConnectionInfo = AMQPConfig() amqpConnectionInfo.update(options) schema = Schema(*schemas) publisher = Publisher(amqpConnectionInfo, schema) initLogging(options) pusher = Pusher(options.exchange, options.messageType, schema, publisher) if options.data == '-': data = loads(sys.stdin.read()) else: data = loads(options.data) published = pusher.push(data=data, routingKey=options.routingKey, mandatory=options.mandatory, immediate=options.immediate) if not published: sys.exit(1)
def main(): from optparse import OptionParser import sys parser = OptionParser(usage=usage()) parser.add_option('-F', '--format', type='string', dest='format', default='protostream', help='Format to dump the messages in (%s)' % ', '.join(_FORMATTERS.keys())) parser.add_option( '-c', '--compression', type='string', dest='compression', default='none', help='Message compression algorithm (possible values: deflate, none)') parser = AMQPConfig.addOptionsToParser(parser) parser = addLoggingOptions(parser) options, args = parser.parse_args() schemas = [SCHEMA] schemas.extend(get_zenpack_schemas()) try: formatter = _FORMATTERS[options.format.lower()] except KeyError: parser.error('Invalid format "%s"' % options.format) initLogging(options) amqpConnectionInfo = AMQPConfig() amqpConnectionInfo.update(options) schema = Schema(*schemas) schema.loadProperties( {'exchange.default.compression': options.compression.lower()}) publisher = getProtobufPubSub(amqpConnectionInfo, schema, None) loader = Loader(sys.stdin, formatter, schema, publisher) try: loader.load() except KeyboardInterrupt: pass loader.publisher.shutdown()
def collect(self, config): log.debug("Collect for OpenStack AMQP (%s)" % config.id) # During the first collect run, we spin up the AMQP listener. After # that, no active collecting is done in the collect() method. # # Instead, as each message arives over the AMQP listener, it goes through # processMessage(), and is placed into a cache where it can be processed # by the onSuccess method. queue_key = "%s_%s" % (self.queue_name, config.id) if queue_key not in amqp_client: # Spin up the AMQP queue listener zcml.load_config('configure.zcml', zope.component) zcml.load_config('configure.zcml', Products.ZenMessaging.queuemessaging) self._amqpConnectionInfo = getUtility(IAMQPConnectionInfo) self._amqpConnectionInfo_collector = AMQPConfig( amqphost='localhost', amqpport=55672, amqpvhost=self._amqpConnectionInfo.vhost, amqpuser=self._amqpConnectionInfo.user, amqppassword=self._amqpConnectionInfo.password, amqpusessl=self._amqpConnectionInfo.usessl, amqpconnectionheartbeat=self._amqpConnectionInfo. amqpconnectionheartbeat) self._queueSchema = getUtility(IQueueSchema) amqp = AMQPFactory(self._amqpConnectionInfo, self._queueSchema) amqp_collector = AMQPFactory(self._amqpConnectionInfo_collector, self._queueSchema) queue = self._queueSchema.getQueue( self.queue_name, replacements={'device': config.id}) log.debug( "Listening on queue: %s with binding to routing key %s" % (queue.name, queue.bindings[self.exchange_name].routing_key)) yield amqp.listen(queue, callback=partial(self._processMessage, amqp, config.id)) yield amqp_collector.listen(queue, callback=partial( self._processMessage, amqp_collector, config.id)) amqp_client[queue_key] = amqp amqp_client[queue_key + "_collector"] = amqp_collector # Give time for some of the existing messages to be processed during # this initial collection cycle yield sleep(10) data = self.new_data() defer.returnValue(data)
def main(): from optparse import OptionParser import sys parser = OptionParser(usage=usage()) parser.add_option("-A", '--ack', action="store_true", dest="acknowledge", help="Acknowledge the message, acknowledging a message will remove it from the queue") parser.add_option('-F', '--format', type='string', dest='format', default='json', help='Format to dump the messages in (%s)' % ', '.join(_FORMATTERS.keys())) parser.add_option('-M', '--max', type='int', dest='max_items', help='Maximum items to dump') parser.add_option('-S', '--skip', action="store_true", dest="skip", help="Skip processing messages on the queue - use with --ack to clear a queue.") parser = AMQPConfig.addOptionsToParser(parser) parser = addLoggingOptions(parser) options, args = parser.parse_args() if not args: parser.error("Require one or more queues as arguments") if options.skip and not options.acknowledge: parser.error("Option --skip must be used with --ack") schemas = [SCHEMA] schemas.extend(get_zenpack_schemas()) amqpConnectionInfo = AMQPConfig() amqpConnectionInfo.update(options) schema = Schema(*schemas) try: formatter = _FORMATTERS[options.format.lower()] except KeyError: parser.error('Invalid format "%s"' % options.format) initLogging(options) publisher = Publisher(amqpConnectionInfo, schema) dumper = Dumper(sys.stdout, formatter, publisher.getChannel(), schema, acknowledge=options.acknowledge, skip=options.skip) dumper.dumpQueues(args, limit=options.max_items)
def main(): from optparse import OptionParser import sys parser = OptionParser(usage=usage()) parser.add_option('-F', '--format', type='string', dest='format', default='protostream', help='Format to dump the messages in (%s)' % ', '.join(_FORMATTERS.keys())) parser.add_option('-c', '--compression', type='string', dest='compression', default='none', help='Message compression algorithm (possible values: deflate, none)') parser = AMQPConfig.addOptionsToParser(parser) parser = addLoggingOptions(parser) options, args = parser.parse_args() schemas = [SCHEMA] schemas.extend(get_zenpack_schemas()) try: formatter = _FORMATTERS[options.format.lower()] except KeyError: parser.error('Invalid format "%s"' % options.format) initLogging(options) amqpConnectionInfo = AMQPConfig() amqpConnectionInfo.update(options) schema = Schema(*schemas) schema.loadProperties({'exchange.default.compression': options.compression.lower()}) publisher = getProtobufPubSub(amqpConnectionInfo, schema, None) loader = Loader(sys.stdin, formatter, schema, publisher) try: loader.load() except KeyboardInterrupt: pass loader.publisher.shutdown()
def read_configuration(self): """ Build a Celery configuration dictionary using global.conf. This configuration may be overwritten or augmented later by a daemon. """ globalCfg = getGlobalConfiguration() amqpCfg = AMQPConfig() amqpCfg.update(globalCfg) config = { ####### # APP # ####### # Configure the value of the 'CELERYD_POOL_PUTLOCKS'. When # True, it causes a temporary deadlock to occur during shutdown # when all workers are busy and there exists at least one # pending job. The deadlock is broken when a worker completes # its job. Setting 'CELERYD_POOL_PUTLOCKS' to False allows # shutdown to occur without waiting. "CELERYD_POOL_PUTLOCKS": False, ############# # TRANSPORT # ############# constants.BROKER_HOST: amqpCfg.host, constants.BROKER_PORT: amqpCfg.port, constants.BROKER_USER: amqpCfg.user, constants.BROKER_PASSWORD: amqpCfg.password, constants.BROKER_VHOST: amqpCfg.vhost, constants.BROKER_USE_SSL: amqpCfg.usessl, constants.ACK_LATE: True, ################ # RESULT STORE # ################ constants.RESULT_BACKEND: 'zodb', ########### # WORKERS # ########### # Default to 1 job per worker process constants.MAX_TASKS_PER_PROCESS: 1, # 2 job workers constants.NUM_WORKERS: 2, ########### # LOGGING # ########### # Handle logging ourselves constants.USE_CELERY_LOGGING: False, # Log file formats constants.LOG_FORMAT: "%(asctime)s %(levelname)s %(name)s: %(message)s", constants.TASK_LOG_FORMAT: "%(asctime)s %(levelname)s zen.Job: %(message)s", # Level at which stdout should be logged constants.STDOUT_LOG_LEVEL: 'INFO' } return config
def main(): from optparse import OptionParser parser = OptionParser(usage=usage()) parser.add_option('-E', '--exchange', type='string', dest='exchange', help="Exchange to push to", action='store') parser.add_option('-T', '--type', type='string', dest='messageType', help="Type of message to create", action='store') parser.add_option('-R', '--routingkey', type='string', dest='routingKey', help="Routing key for message", action='store') parser.add_option('-D', '--data', type='string', dest='data', help="Message data as JSON, use '-' to read from stdin", action='store') parser.add_option('-M', '--mandatory', dest='mandatory', help="Publish message with mandatory flag set.", action='store_true') parser = AMQPConfig.addOptionsToParser(parser) parser = addLoggingOptions(parser) options, args = parser.parse_args() if not options.data: parser.error('You must supply input data.') elif not options.exchange: parser.error('You must supply an exchange.') elif not options.messageType: parser.error('You must supply a message type.') elif not options.routingKey: parser.error('You must supply a routing key.') schemas = [SCHEMA] schemas.extend(get_zenpack_schemas()) amqpConnectionInfo = AMQPConfig() amqpConnectionInfo.update(options) schema = Schema(*schemas) publisher = Publisher(amqpConnectionInfo, schema) initLogging(options) pusher = Pusher(options.exchange, options.messageType, schema, publisher) if options.data == '-': data = loads(sys.stdin.read()) else: data = loads(options.data) published = pusher.push(data=data, routingKey=options.routingKey, mandatory=options.mandatory) if not published: sys.exit(1)
def _loadAmqpConnectionInfo(): from Products.ZenUtils.GlobalConfig import getGlobalConfiguration from zenoss.protocols.amqpconfig import AMQPConfig amqpConnectionInfo = AMQPConfig() amqpConnectionInfo.update(getGlobalConfiguration()) return amqpConnectionInfo
def main(): from optparse import OptionParser import sys parser = OptionParser(usage=usage()) parser.add_option( "-A", '--ack', action="store_true", dest="acknowledge", help= "Acknowledge the message, acknowledging a message will remove it from the queue" ) parser.add_option('-F', '--format', type='string', dest='format', default='json', help='Format to dump the messages in (%s)' % ', '.join(_FORMATTERS.keys())) parser.add_option('-M', '--max', type='int', dest='max_items', help='Maximum items to dump') parser.add_option( '-S', '--skip', action="store_true", dest="skip", help= "Skip processing messages on the queue - use with --ack to clear a queue." ) parser = AMQPConfig.addOptionsToParser(parser) parser = addLoggingOptions(parser) options, args = parser.parse_args() if not args: parser.error("Require one or more queues as arguments") if options.skip and not options.acknowledge: parser.error("Option --skip must be used with --ack") schemas = [SCHEMA] schemas.extend(get_zenpack_schemas()) amqpConnectionInfo = AMQPConfig() amqpConnectionInfo.update(options) schema = Schema(*schemas) try: formatter = _FORMATTERS[options.format.lower()] except KeyError: parser.error('Invalid format "%s"' % options.format) initLogging(options) publisher = Publisher(amqpConnectionInfo, schema) dumper = Dumper(sys.stdout, formatter, publisher.getChannel(), schema, acknowledge=options.acknowledge, skip=options.skip) dumper.dumpQueues(args, limit=options.max_items)
def _loadAmqpConnectionInfo(): from Products.ZenUtils.GlobalConfig import getGlobalConfiguration from zenoss.protocols.amqpconfig import AMQPConfig amqpConnectionInfo = AMQPConfig() amqpConnectionInfo.update(getGlobalConfiguration()) return amqpConnectionInfo
def read_configuration(self): """ Build a Celery configuration dictionary using global.conf. This configuration may be overwritten or augmented later by a daemon. """ globalCfg = getGlobalConfiguration() amqpCfg = AMQPConfig() amqpCfg.update(globalCfg) config = { ############# # TRANSPORT # ############# constants.BROKER_HOST: amqpCfg.host, constants.BROKER_PORT: amqpCfg.port, constants.BROKER_USER: amqpCfg.user, constants.BROKER_PASSWORD: amqpCfg.password, constants.BROKER_VHOST: amqpCfg.vhost, constants.BROKER_USE_SSL: amqpCfg.usessl, constants.ACK_LATE: True, ################ # RESULT STORE # ################ constants.RESULT_BACKEND: 'zodb', ########### # WORKERS # ########### # Default to 1 job per worker process constants.MAX_TASKS_PER_PROCESS: 1, # 2 job workers constants.NUM_WORKERS: 2, ########### # LOGGING # ########### # Handle logging ourselves constants.USE_CELERY_LOGGING: False, # Log file formats constants.LOG_FORMAT: "%(asctime)s %(levelname)s %(name)s: %(message)s", constants.TASK_LOG_FORMAT: "%(asctime)s %(levelname)s zen.Job: %(message)s", # Level at which stdout should be logged constants.STDOUT_LOG_LEVEL: 'INFO' } return config