class PdfExtractor(Callable):
def __init__(self, channel, opts):
self.channel = channel
self.opts = opts
self.consumer = QueueingConsumer(channel)
def call(self):
log = logging.getLogger()
self.channel.basicConsume(opts.queue_name, False, self.consumer)
log.info("Awaiting pdf extraction tasks on %s...", opts.queue_name)
while True:
delivery = self.consumer.nextDelivery()
props = delivery.getProperties()
task = String(delivery.getBody())
reply_props = BasicProperties.Builder().build()
try:
task_data = json.loads(str(task))
log.debug("got task: %s", str(task_data))
resp = self.extract(task_data['bibcode'], task_data['ft_source'])
except Exception, e:
msg = traceback.format_exc()
log.debug("returning error response: %s" % str(msg))
resp = { '_exception': str(msg) }
finally:
class PdfExtractor(Callable):
def __init__(self, channel, opts):
self.channel = channel
self.opts = opts
self.consumer = QueueingConsumer(channel)
def call(self):
log = logging.getLogger()
self.channel.basicConsume(opts.queue_name, False, self.consumer)
log.info("Awaiting pdf extraction tasks on %s...", opts.queue_name)
while True:
delivery = self.consumer.nextDelivery()
props = delivery.getProperties()
task = String(delivery.getBody())
reply_props = BasicProperties.Builder().build()
try:
task_data = json.loads(str(task))
log.debug("got task: %s", str(task_data))
resp = self.extract(task_data['bibcode'],
task_data['ft_source'])
except Exception, e:
msg = traceback.format_exc()
log.debug("returning error response: %s" % str(msg))
resp = {'_exception': str(msg)}
finally:
class RabbitMQClient:
def __init__(self):
"""
Connections and other settings here should match those set in publisher script
"""
self.stdout = Logger()
self.exchangeName = "bungeni_serialization_output_queue"
self.queueName = "bungeni_serialization_output_queue"
self.factory = ConnectionFactory()
self.factory.setHost("localhost")
self.conn = self.factory.newConnection()
self.channel = self.conn.createChannel()
self.channel.exchangeDeclare(self.exchangeName,"direct",False)
def consume_msgs(self, parliament_cache_info):
try:
declareOk = self.channel.queueDeclare(self.queueName, True, False, False, None)
self.channel.queueBind(self.queueName, self.exchangeName, self.queueName)
self.consumer = QueueingConsumer(self.channel)
self.channel.basicConsume(self.queueName, False, self.consumer)
count = 0
if declareOk.messageCount <= 0:
self.stdout.write(time.asctime(time.localtime(time.time())) + " NO MESSAGES \n")
else:
self.stdout.write(time.asctime(time.localtime(time.time())) + " " + str(declareOk.messageCount) + " MESSAGES! \n")
while (count < declareOk.messageCount):
delivery = QueueingConsumer.Delivery
delivery = self.consumer.nextDelivery()
message = str(String(delivery.getBody()))
obj_data = json.loads(message)
file_status = main_queue(__config_file__, str(obj_data['location']), parliament_cache_info)
count = count + 1
if file_status is None:
print "No Parliament Information could be gathered"
sys.exit(0)
elif file_status is True:
# Acknowledgements to RabbitMQ the successfully, processed files
self.channel.basicAck(delivery.getEnvelope().getDeliveryTag(), False)
else:
# Reject file, requeue for investigation or future attempts
self.channel.basicReject(delivery.getEnvelope().getDeliveryTag(), True)
finally:
try:
if self.channel is not None:
self.channel.close()
except Exception, ex:
LOG.error("Error while closing channel", ex)
try:
if self.conn is not None:
self.conn.close()
except Exception, ex:
LOG.error("Error while closing connection", ex)
def consume_msgs(self, parliament_cache_info):
try:
declareOk = self.channel.queueDeclare(self.queueName, True, False, False, None)
self.channel.queueBind(self.queueName, self.exchangeName, self.queueName)
self.consumer = QueueingConsumer(self.channel)
self.channel.basicConsume(self.queueName, False, self.consumer)
count = 0
if declareOk.messageCount <= 0:
self.stdout.write(
time.asctime(time.localtime(time.time())) + " NO MESSAGES \n"
)
else:
self.stdout.write(
time.asctime(time.localtime(time.time())) + " " + str(declareOk.messageCount) + " MESSAGES! \n"
)
while (count < declareOk.messageCount):
delivery = QueueingConsumer.Delivery
delivery = self.consumer.nextDelivery()
message = str(String(delivery.getBody()))
obj_data = json.loads(message)
file_to_process = str(obj_data["location"])
file_status = main_queue(self.config_file, file_to_process, parliament_cache_info)
count = count + 1
if file_status is None:
print "No Parliament Information could be gathered"
sys.exit(0)
elif file_status is True:
# Acknowledgements to RabbitMQ the successfully, processed files
self.channel.basicAck(delivery.getEnvelope().getDeliveryTag(), False)
# post process the file : archive / delete / noaction
post_process_action(self.config_file, file_to_process)
else:
# Reject file, requeue for investigation or future attempts
self.channel.basicReject(delivery.getEnvelope().getDeliveryTag(), True)
finally:
try:
if self.channel is not None:
self.channel.close()
except Exception, ex:
LOG.error("Error while closing channel", ex)
try:
if self.conn is not None:
self.conn.close()
except Exception, ex:
LOG.error("Error while closing connection", ex)
def connectToRabbit(playerId, tableId,
rabbitConnection):
if rabbitConnection:
rabbitConnection.close()
rabbitConnectionFactory = getRabbitConnectionFactory(
playerId)
rabbitConnection = rabbitConnectionFactory.newConnection(
)
channel = rabbitConnection.createChannel()
queueName = channel.queueDeclare().getQueue()
channel.queueBind(
queueName, "player-direct",
"PLAYERTABLE." + playerId + "." + tableId)
channel.queueBind(queueName, "player-direct",
"PLAYER." + playerId)
consumer = QueueingConsumer(channel)
channel.basicConsume(queueName, True, consumer)
return (rabbitConnection, consumer)
def __init__(self, channel, opts):
self.channel = channel
self.opts = opts
self.consumer = QueueingConsumer(channel)
HOME_FILE = "/var/netshell/users/admin"
permissions = Hashtable()
# Get name of queue to receive messages from from file.
uuidQueue = UUIDManager(HOME_FILE)
queueName = uuidQueue.checkUUID()
# Create connection and queue
factory = SSLConnection(broker).createConnection()
connection = factory.newConnection()
channel = connection.createChannel()
channel.queueDeclare(queueName, False, False, True, None)
try:
consumer = QueueingConsumer(channel)
# Declare as exclusive consumer of queue
channel.basicConsume(queueName, True, "consumer", False, True, None, consumer)
print " [*] Waiting for messages."
except IOError:
print "Unable to connect to queue."
connection.close()
sys.exit(1)
writer = WriteMsgToFile(HOME_FILE)
while True:
try:
delivery = consumer.nextDelivery()
message = delivery.getBody()
message = StringUtil.fromBytes(message)
HOME_FILE = "/var/netshell/users/admin"
permissions = Hashtable()
# Get name of queue to receive messages from from file.
uuidQueue = UUIDManager(HOME_FILE)
queueName = uuidQueue.checkUUID()
# Create connection and queue
factory = SSLConnection(broker).createConnection()
connection = factory.newConnection()
channel = connection.createChannel()
channel.queueDeclare(queueName, False, False, True, None)
try:
consumer = QueueingConsumer(channel)
# Declare as exclusive consumer of queue
channel.basicConsume(queueName, True, "consumer", False, True, None,
consumer)
print " [*] Waiting for messages."
except IOError:
print "Unable to connect to queue."
connection.close()
sys.exit(1)
writer = WriteMsgToFile(HOME_FILE)
while True:
try:
delivery = consumer.nextDelivery()
message = delivery.getBody()
def __init__(self, channel, opts):
self.channel = channel
self.opts = opts
self.consumer = QueueingConsumer(channel)
