def main(opts): # set up our channel conn_factory = ConnectionFactory() conn_factory.setUri(config['RABBITMQ_URI']) conn = conn_factory.newConnection() channel = conn.createChannel() channel.queueDeclare(opts.queue_name, False, False, False, None) channel.basicQos(1) # tells the channel we're only going to deliver one response before req acknowledgement workers = [PdfExtractor(channel, opts) for i in xrange(opts.workers)] log.info("creating pool with %d threads" % opts.workers) tpool = Executors.newFixedThreadPool(opts.workers) log.info("executing threads") futures = tpool.invokeAll(workers) log.info("shutting down thread pool") tpool.shutdown() try: if not tpool.awaitTermination(50, TimeUnit.SECONDS): log.info("thread pool not shutting down; trying again") tpool.shutdownNow() if not tpool.awaitTermination(50, TimeUnit.SECONDS): log.error("Pool did not terminate") except InterruptedException: log.info("exception during thread pool shutdown; trying again") tpool.shutdownNow() Thread.currentThread().interrupt()
def main(opts): # set up our channel conn_factory = ConnectionFactory() conn_factory.setUri(config['RABBITMQ_URI']) conn = conn_factory.newConnection() channel = conn.createChannel() channel.queueDeclare(opts.queue_name, False, False, False, None) channel.basicQos(1); # tells the channel we're only going to deliver one response before req acknowledgement workers = [PdfExtractor(channel, opts) for i in xrange(opts.workers)] log.info("creating pool with %d threads" % opts.workers) tpool = Executors.newFixedThreadPool(opts.workers) log.info("executing threads") futures = tpool.invokeAll(workers) log.info("shutting down thread pool") tpool.shutdown() try: if not tpool.awaitTermination(50, TimeUnit.SECONDS): log.info("thread pool not shutting down; trying again") tpool.shutdownNow() if not tpool.awaitTermination(50, TimeUnit.SECONDS): log.error("Pool did not terminate") except InterruptedException: log.info("exception during thread pool shutdown; trying again") tpool.shutdownNow() Thread.currentThread().interrupt()
class RabbitMQClient: def __init__(self): """ Connections and other settings here should match those set in publisher script """ self.stdout = Logger() self.exchangeName = "bungeni_serialization_output_queue" self.queueName = "bungeni_serialization_output_queue" self.factory = ConnectionFactory() self.factory.setHost("localhost") self.conn = self.factory.newConnection() self.channel = self.conn.createChannel() self.channel.exchangeDeclare(self.exchangeName,"direct",False) def consume_msgs(self, parliament_cache_info): try: declareOk = self.channel.queueDeclare(self.queueName, True, False, False, None) self.channel.queueBind(self.queueName, self.exchangeName, self.queueName) self.consumer = QueueingConsumer(self.channel) self.channel.basicConsume(self.queueName, False, self.consumer) count = 0 if declareOk.messageCount <= 0: self.stdout.write(time.asctime(time.localtime(time.time())) + " NO MESSAGES \n") else: self.stdout.write(time.asctime(time.localtime(time.time())) + " " + str(declareOk.messageCount) + " MESSAGES! \n") while (count < declareOk.messageCount): delivery = QueueingConsumer.Delivery delivery = self.consumer.nextDelivery() message = str(String(delivery.getBody())) obj_data = json.loads(message) file_status = main_queue(__config_file__, str(obj_data['location']), parliament_cache_info) count = count + 1 if file_status is None: print "No Parliament Information could be gathered" sys.exit(0) elif file_status is True: # Acknowledgements to RabbitMQ the successfully, processed files self.channel.basicAck(delivery.getEnvelope().getDeliveryTag(), False) else: # Reject file, requeue for investigation or future attempts self.channel.basicReject(delivery.getEnvelope().getDeliveryTag(), True) finally: try: if self.channel is not None: self.channel.close() except Exception, ex: LOG.error("Error while closing channel", ex) try: if self.conn is not None: self.conn.close() except Exception, ex: LOG.error("Error while closing connection", ex)
def initialize(self): self.factory = ConnectionFactory() if self.networkRecoveryInterval != None: self.factory.setNetworkRecoveryInterval(self.networkRecoveryInterval) if self.automaticRecoveryEnabled != None: self.factory.setAutomaticRecoveryEnabled(self.automaticRecoveryEnabled) if self.topologyRecoveryEnabled != None: self.factory.setTopologyRecoveryEnabled(self.topologyRecoveryEnabled) if self.uri != None: logger.info("Initializing RabbitMQ with uri: %s", self.uri) self.factory.setUri(self.uri) self.connection = self.factory.newConnection() else: logger.info("Initializing RabbitMQ") self.addresses.append(Address(self.host, self.port)) if (self.username != None): self.factory.setUsername(self.username) if (self.password != None): self.factory.setPassword(self.password) if (self.virtualhost != None): self.factory.setVirtualHost(self.virtualhost) self.connection = self.factory.newConnection(self.addresses) self.channel = self.connection.createChannel() if (self.queue_name != None): self.passive = False self.durable = False self.exclusive = False self.autodelete = False self.declareQueue = False if "passive" in self.config: self.passive = self.config["passive"] if "durable" in self.config: self.durable = self.config["durable"] if "exclusive" in self.config: self.exclusive = self.config["exclusive"] if "declareQueue" in self.config: self.declareQueue = self.config["declareQueue"] if self.declareQueue: logger.info("Declaring queue %s", self.queue_name) self.channel.queueDeclare(self.queue_name, self.passive, self.durable, self.exclusive, None) else: logger.info("Binding to queue %s, with exchange '%s' and routingKey '%s'", self.queue_name, self.exchange, self.routingKey) self.channel.queueBind(self.queue_name, self.exchange, self.routingKey)
def __init__(self): """ Connections and other settings here should match those set in publisher script """ self.stdout = Logger() self.exchangeName = "bungeni_serialization_output_queue" self.queueName = "bungeni_serialization_output_queue" self.factory = ConnectionFactory() self.factory.setHost("localhost") self.conn = self.factory.newConnection() self.channel = self.conn.createChannel() self.channel.exchangeDeclare(self.exchangeName,"direct",False)
def __init__(self, config_file): """ Connections and other settings here should match those set in publisher script """ self.stdout = Logger() self.exchangeName = "bungeni_serialization_output_queue" self.queueName = "bungeni_serialization_output_queue" self.factory = ConnectionFactory() # !+HARCODED host self.config_file = config_file self.config = rabbitmq_config(config_file) self.factory.setHost(self.config.get_hostname()) self.factory.setUsername(self.config.get_username()) self.factory.setPassword(self.config.get_password()) # !+HARDCODED the following need to be fixed #self.factory.setVirtualHost(self.config.get_vhost()) #self.factory.setPort(self.config.get_port()) self.conn = self.factory.newConnection() self.channel = self.conn.createChannel() self.channel.exchangeDeclare(self.exchangeName,"direct",False)
def createRabbitConnectionFactory(host): rabbitConnectionFactory = ConnectionFactory() rabbitConnectionFactory.setUsername(grinder.properties.getProperty("rabbit.username")) rabbitConnectionFactory.setPassword(grinder.properties.getProperty("rabbit.password")) rabbitConnectionFactory.setVirtualHost(grinder.properties.getProperty("rabbit.virtualHost")) rabbitConnectionFactory.setHost(host) rabbitConnectionFactory.setPort(5672) return rabbitConnectionFactory
def createRabbitConnectionFactory(host): rabbitConnectionFactory = ConnectionFactory() rabbitConnectionFactory.setUsername( grinder.properties.getProperty("rabbit.username")) rabbitConnectionFactory.setPassword( grinder.properties.getProperty("rabbit.password")) rabbitConnectionFactory.setVirtualHost( grinder.properties.getProperty("rabbit.virtualHost")) rabbitConnectionFactory.setHost(host) rabbitConnectionFactory.setPort(5672) return rabbitConnectionFactory
class RabbitMQ(): def __init__(self, config): #TODO: eliminate message count ASAP, check work with RabbitMQ features, remove spaghetti logic self.messagecount = 0 self.config = config if "queue_name" in self.config: self.queue_name = self.config["queue_name"]; else: self.queue_name = None if "queue" in self.config: self.queue_name = self.config["queue"]; else: self.queue_name = None if "routingKey" in self.config: self.routingKey = self.config["routingKey"]; else: self.routingKey = "" if "host" in self.config: self.host = self.config["host"] else: self.host = "localhost" if "port" in self.config: self.port = self.config["port"] else: self.port = 5672 if "username" in self.config: self.username = self.config["username"] else: self.username = None if "password" in self.config: self.password = self.config["password"] else: self.password = None if "virtualhost" in self.config: self.virtualhost = self.config["virtualhost"] else: self.virtualhost = None if "uri" in self.config: self.uri = self.config["uri"] else: self.uri = None if "networkRecoveryInterval" in self.config: self.networkRecoveryInterval = self.config["networkRecoveryInterval"] else: self.networkRecoveryInterval = None if "automaticRecoveryEnabled" in self.config: self.automaticRecoveryEnabled = self.config["automaticRecoveryEnabled"] else: self.automaticRecoveryEnabled = None if "topologyRecoveryEnabled" in self.config: self.topologyRecoveryEnabled = self.config["topologyRecoveryEnabled"] else: self.topologyRecoveryEnabled = None if "exchange" in self.config: self.exchange = self.config["exchange"] else: self.exchange = "" self.addresses = [] if "addresses" in self.config: for address in self.config["addresses"]: logger.info(address) self.addresses.append(Address.parseAddress(address)) self.initialize(); def initialize(self): self.factory = ConnectionFactory() if self.networkRecoveryInterval != None: self.factory.setNetworkRecoveryInterval(self.networkRecoveryInterval) if self.automaticRecoveryEnabled != None: self.factory.setAutomaticRecoveryEnabled(self.automaticRecoveryEnabled) if self.topologyRecoveryEnabled != None: self.factory.setTopologyRecoveryEnabled(self.topologyRecoveryEnabled) if self.uri != None: logger.info("Initializing RabbitMQ with uri: %s", self.uri) self.factory.setUri(self.uri) self.connection = self.factory.newConnection() else: logger.info("Initializing RabbitMQ") self.addresses.append(Address(self.host, self.port)) if (self.username != None): self.factory.setUsername(self.username) if (self.password != None): self.factory.setPassword(self.password) if (self.virtualhost != None): self.factory.setVirtualHost(self.virtualhost) self.connection = self.factory.newConnection(self.addresses) self.channel = self.connection.createChannel() if (self.queue_name != None): self.passive = False self.durable = False self.exclusive = False self.autodelete = False self.declareQueue = False if "passive" in self.config: self.passive = self.config["passive"] if "durable" in self.config: self.durable = self.config["durable"] if "exclusive" in self.config: self.exclusive = self.config["exclusive"] if "declareQueue" in self.config: self.declareQueue = self.config["declareQueue"] if self.declareQueue: logger.info("Declaring queue %s", self.queue_name) self.channel.queueDeclare(self.queue_name, self.passive, self.durable, self.exclusive, None) else: logger.info("Binding to queue %s, with exchange '%s' and routingKey '%s'", self.queue_name, self.exchange, self.routingKey) self.channel.queueBind(self.queue_name, self.exchange, self.routingKey) def writeDocument(self, data, force): self.messagecount += 1 if (isinstance(data,dict)): #TODO: this is complete chaos, rethink on how to deal with dictionaries, for now, will send only values, keys will be ignored for k, v in data.items(): if self.queue_name != None: logger.debug("Publishing to rabbit queue %s: %s", self.queue_name, v) self.channel.basicPublish(self.exchange, self.queue_name, None, str(v)) else: logger.debug("Writing to rabbit exchange %s: %s", self.exchange, v) self.channel.basicPublish(self.exchange, self.routingKey, None, str(v)) else: if self.queue_name != None: logger.debug("Publishing to rabbit queue %s: %s", self.queue_name, data) self.channel.basicPublish(self.exchange, self.queue_name, None, data) else: logger.debug("Writing to rabbit exchange %s: %s", self.exchange, data) self.channel.basicPublish(self.exchange, self.routingKey, None, data) def flush(self): logger.info("Flushing. Total messages: %d", self.messagecount) return True def cleanup(self): self.channel.close() self.connection.close()