def main(opts):
# set up our channel
conn_factory = ConnectionFactory()
conn_factory.setUri(config['RABBITMQ_URI'])
conn = conn_factory.newConnection()
channel = conn.createChannel()
channel.queueDeclare(opts.queue_name, False, False, False, None)
channel.basicQos(1)
# tells the channel we're only going to deliver one response before req acknowledgement
workers = [PdfExtractor(channel, opts) for i in xrange(opts.workers)]
log.info("creating pool with %d threads" % opts.workers)
tpool = Executors.newFixedThreadPool(opts.workers)
log.info("executing threads")
futures = tpool.invokeAll(workers)
log.info("shutting down thread pool")
tpool.shutdown()
try:
if not tpool.awaitTermination(50, TimeUnit.SECONDS):
log.info("thread pool not shutting down; trying again")
tpool.shutdownNow()
if not tpool.awaitTermination(50, TimeUnit.SECONDS):
log.error("Pool did not terminate")
except InterruptedException:
log.info("exception during thread pool shutdown; trying again")
tpool.shutdownNow()
Thread.currentThread().interrupt()
def main(opts):
# set up our channel
conn_factory = ConnectionFactory()
conn_factory.setUri(config['RABBITMQ_URI'])
conn = conn_factory.newConnection()
channel = conn.createChannel()
channel.queueDeclare(opts.queue_name, False, False, False, None)
channel.basicQos(1); # tells the channel we're only going to deliver one response before req acknowledgement
workers = [PdfExtractor(channel, opts) for i in xrange(opts.workers)]
log.info("creating pool with %d threads" % opts.workers)
tpool = Executors.newFixedThreadPool(opts.workers)
log.info("executing threads")
futures = tpool.invokeAll(workers)
log.info("shutting down thread pool")
tpool.shutdown()
try:
if not tpool.awaitTermination(50, TimeUnit.SECONDS):
log.info("thread pool not shutting down; trying again")
tpool.shutdownNow()
if not tpool.awaitTermination(50, TimeUnit.SECONDS):
log.error("Pool did not terminate")
except InterruptedException:
log.info("exception during thread pool shutdown; trying again")
tpool.shutdownNow()
Thread.currentThread().interrupt()
class RabbitMQClient:
def __init__(self):
"""
Connections and other settings here should match those set in publisher script
"""
self.stdout = Logger()
self.exchangeName = "bungeni_serialization_output_queue"
self.queueName = "bungeni_serialization_output_queue"
self.factory = ConnectionFactory()
self.factory.setHost("localhost")
self.conn = self.factory.newConnection()
self.channel = self.conn.createChannel()
self.channel.exchangeDeclare(self.exchangeName,"direct",False)
def consume_msgs(self, parliament_cache_info):
try:
declareOk = self.channel.queueDeclare(self.queueName, True, False, False, None)
self.channel.queueBind(self.queueName, self.exchangeName, self.queueName)
self.consumer = QueueingConsumer(self.channel)
self.channel.basicConsume(self.queueName, False, self.consumer)
count = 0
if declareOk.messageCount <= 0:
self.stdout.write(time.asctime(time.localtime(time.time())) + " NO MESSAGES \n")
else:
self.stdout.write(time.asctime(time.localtime(time.time())) + " " + str(declareOk.messageCount) + " MESSAGES! \n")
while (count < declareOk.messageCount):
delivery = QueueingConsumer.Delivery
delivery = self.consumer.nextDelivery()
message = str(String(delivery.getBody()))
obj_data = json.loads(message)
file_status = main_queue(__config_file__, str(obj_data['location']), parliament_cache_info)
count = count + 1
if file_status is None:
print "No Parliament Information could be gathered"
sys.exit(0)
elif file_status is True:
# Acknowledgements to RabbitMQ the successfully, processed files
self.channel.basicAck(delivery.getEnvelope().getDeliveryTag(), False)
else:
# Reject file, requeue for investigation or future attempts
self.channel.basicReject(delivery.getEnvelope().getDeliveryTag(), True)
finally:
try:
if self.channel is not None:
self.channel.close()
except Exception, ex:
LOG.error("Error while closing channel", ex)
try:
if self.conn is not None:
self.conn.close()
except Exception, ex:
LOG.error("Error while closing connection", ex)
def initialize(self):
self.factory = ConnectionFactory()
if self.networkRecoveryInterval != None:
self.factory.setNetworkRecoveryInterval(self.networkRecoveryInterval)
if self.automaticRecoveryEnabled != None:
self.factory.setAutomaticRecoveryEnabled(self.automaticRecoveryEnabled)
if self.topologyRecoveryEnabled != None:
self.factory.setTopologyRecoveryEnabled(self.topologyRecoveryEnabled)
if self.uri != None:
logger.info("Initializing RabbitMQ with uri: %s", self.uri)
self.factory.setUri(self.uri)
self.connection = self.factory.newConnection()
else:
logger.info("Initializing RabbitMQ")
self.addresses.append(Address(self.host, self.port))
if (self.username != None):
self.factory.setUsername(self.username)
if (self.password != None):
self.factory.setPassword(self.password)
if (self.virtualhost != None):
self.factory.setVirtualHost(self.virtualhost)
self.connection = self.factory.newConnection(self.addresses)
self.channel = self.connection.createChannel()
if (self.queue_name != None):
self.passive = False
self.durable = False
self.exclusive = False
self.autodelete = False
self.declareQueue = False
if "passive" in self.config:
self.passive = self.config["passive"]
if "durable" in self.config:
self.durable = self.config["durable"]
if "exclusive" in self.config:
self.exclusive = self.config["exclusive"]
if "declareQueue" in self.config:
self.declareQueue = self.config["declareQueue"]
if self.declareQueue:
logger.info("Declaring queue %s", self.queue_name)
self.channel.queueDeclare(self.queue_name, self.passive, self.durable, self.exclusive, None)
else:
logger.info("Binding to queue %s, with exchange '%s' and routingKey '%s'", self.queue_name, self.exchange, self.routingKey)
self.channel.queueBind(self.queue_name, self.exchange, self.routingKey)
def __init__(self):
"""
Connections and other settings here should match those set in publisher script
"""
self.stdout = Logger()
self.exchangeName = "bungeni_serialization_output_queue"
self.queueName = "bungeni_serialization_output_queue"
self.factory = ConnectionFactory()
self.factory.setHost("localhost")
self.conn = self.factory.newConnection()
self.channel = self.conn.createChannel()
self.channel.exchangeDeclare(self.exchangeName,"direct",False)
def __init__(self, config_file):
"""
Connections and other settings here should match those set in publisher script
"""
self.stdout = Logger()
self.exchangeName = "bungeni_serialization_output_queue"
self.queueName = "bungeni_serialization_output_queue"
self.factory = ConnectionFactory()
# !+HARCODED host
self.config_file = config_file
self.config = rabbitmq_config(config_file)
self.factory.setHost(self.config.get_hostname())
self.factory.setUsername(self.config.get_username())
self.factory.setPassword(self.config.get_password())
# !+HARDCODED the following need to be fixed
#self.factory.setVirtualHost(self.config.get_vhost())
#self.factory.setPort(self.config.get_port())
self.conn = self.factory.newConnection()
self.channel = self.conn.createChannel()
self.channel.exchangeDeclare(self.exchangeName,"direct",False)
def createRabbitConnectionFactory(host):
rabbitConnectionFactory = ConnectionFactory()
rabbitConnectionFactory.setUsername(grinder.properties.getProperty("rabbit.username"))
rabbitConnectionFactory.setPassword(grinder.properties.getProperty("rabbit.password"))
rabbitConnectionFactory.setVirtualHost(grinder.properties.getProperty("rabbit.virtualHost"))
rabbitConnectionFactory.setHost(host)
rabbitConnectionFactory.setPort(5672)
return rabbitConnectionFactory
def createRabbitConnectionFactory(host):
rabbitConnectionFactory = ConnectionFactory()
rabbitConnectionFactory.setUsername(
grinder.properties.getProperty("rabbit.username"))
rabbitConnectionFactory.setPassword(
grinder.properties.getProperty("rabbit.password"))
rabbitConnectionFactory.setVirtualHost(
grinder.properties.getProperty("rabbit.virtualHost"))
rabbitConnectionFactory.setHost(host)
rabbitConnectionFactory.setPort(5672)
return rabbitConnectionFactory
class RabbitMQ():
def __init__(self, config):
#TODO: eliminate message count ASAP, check work with RabbitMQ features, remove spaghetti logic
self.messagecount = 0
self.config = config
if "queue_name" in self.config:
self.queue_name = self.config["queue_name"];
else:
self.queue_name = None
if "queue" in self.config:
self.queue_name = self.config["queue"];
else:
self.queue_name = None
if "routingKey" in self.config:
self.routingKey = self.config["routingKey"];
else:
self.routingKey = ""
if "host" in self.config:
self.host = self.config["host"]
else:
self.host = "localhost"
if "port" in self.config:
self.port = self.config["port"]
else:
self.port = 5672
if "username" in self.config:
self.username = self.config["username"]
else:
self.username = None
if "password" in self.config:
self.password = self.config["password"]
else:
self.password = None
if "virtualhost" in self.config:
self.virtualhost = self.config["virtualhost"]
else:
self.virtualhost = None
if "uri" in self.config:
self.uri = self.config["uri"]
else:
self.uri = None
if "networkRecoveryInterval" in self.config:
self.networkRecoveryInterval = self.config["networkRecoveryInterval"]
else:
self.networkRecoveryInterval = None
if "automaticRecoveryEnabled" in self.config:
self.automaticRecoveryEnabled = self.config["automaticRecoveryEnabled"]
else:
self.automaticRecoveryEnabled = None
if "topologyRecoveryEnabled" in self.config:
self.topologyRecoveryEnabled = self.config["topologyRecoveryEnabled"]
else:
self.topologyRecoveryEnabled = None
if "exchange" in self.config:
self.exchange = self.config["exchange"]
else:
self.exchange = ""
self.addresses = []
if "addresses" in self.config:
for address in self.config["addresses"]:
logger.info(address)
self.addresses.append(Address.parseAddress(address))
self.initialize();
def initialize(self):
self.factory = ConnectionFactory()
if self.networkRecoveryInterval != None:
self.factory.setNetworkRecoveryInterval(self.networkRecoveryInterval)
if self.automaticRecoveryEnabled != None:
self.factory.setAutomaticRecoveryEnabled(self.automaticRecoveryEnabled)
if self.topologyRecoveryEnabled != None:
self.factory.setTopologyRecoveryEnabled(self.topologyRecoveryEnabled)
if self.uri != None:
logger.info("Initializing RabbitMQ with uri: %s", self.uri)
self.factory.setUri(self.uri)
self.connection = self.factory.newConnection()
else:
logger.info("Initializing RabbitMQ")
self.addresses.append(Address(self.host, self.port))
if (self.username != None):
self.factory.setUsername(self.username)
if (self.password != None):
self.factory.setPassword(self.password)
if (self.virtualhost != None):
self.factory.setVirtualHost(self.virtualhost)
self.connection = self.factory.newConnection(self.addresses)
self.channel = self.connection.createChannel()
if (self.queue_name != None):
self.passive = False
self.durable = False
self.exclusive = False
self.autodelete = False
self.declareQueue = False
if "passive" in self.config:
self.passive = self.config["passive"]
if "durable" in self.config:
self.durable = self.config["durable"]
if "exclusive" in self.config:
self.exclusive = self.config["exclusive"]
if "declareQueue" in self.config:
self.declareQueue = self.config["declareQueue"]
if self.declareQueue:
logger.info("Declaring queue %s", self.queue_name)
self.channel.queueDeclare(self.queue_name, self.passive, self.durable, self.exclusive, None)
else:
logger.info("Binding to queue %s, with exchange '%s' and routingKey '%s'", self.queue_name, self.exchange, self.routingKey)
self.channel.queueBind(self.queue_name, self.exchange, self.routingKey)
def writeDocument(self, data, force):
self.messagecount += 1
if (isinstance(data,dict)):
#TODO: this is complete chaos, rethink on how to deal with dictionaries, for now, will send only values, keys will be ignored
for k, v in data.items():
if self.queue_name != None:
logger.debug("Publishing to rabbit queue %s: %s", self.queue_name, v)
self.channel.basicPublish(self.exchange, self.queue_name, None, str(v))
else:
logger.debug("Writing to rabbit exchange %s: %s", self.exchange, v)
self.channel.basicPublish(self.exchange, self.routingKey, None, str(v))
else:
if self.queue_name != None:
logger.debug("Publishing to rabbit queue %s: %s", self.queue_name, data)
self.channel.basicPublish(self.exchange, self.queue_name, None, data)
else:
logger.debug("Writing to rabbit exchange %s: %s", self.exchange, data)
self.channel.basicPublish(self.exchange, self.routingKey, None, data)
def flush(self):
logger.info("Flushing. Total messages: %d", self.messagecount)
return True
def cleanup(self):
self.channel.close()
self.connection.close()
