class PdfExtractor(Callable):
def __init__(self, channel, opts):
self.channel = channel
self.opts = opts
self.consumer = QueueingConsumer(channel)
def call(self):
log = logging.getLogger()
self.channel.basicConsume(opts.queue_name, False, self.consumer)
log.info("Awaiting pdf extraction tasks on %s...", opts.queue_name)
while True:
delivery = self.consumer.nextDelivery()
props = delivery.getProperties()
task = String(delivery.getBody())
reply_props = BasicProperties.Builder().build()
try:
task_data = json.loads(str(task))
log.debug("got task: %s", str(task_data))
resp = self.extract(task_data['bibcode'], task_data['ft_source'])
except Exception, e:
msg = traceback.format_exc()
log.debug("returning error response: %s" % str(msg))
resp = { '_exception': str(msg) }
finally:
class PdfExtractor(Callable):
def __init__(self, channel, opts):
self.channel = channel
self.opts = opts
self.consumer = QueueingConsumer(channel)
def call(self):
log = logging.getLogger()
self.channel.basicConsume(opts.queue_name, False, self.consumer)
log.info("Awaiting pdf extraction tasks on %s...", opts.queue_name)
while True:
delivery = self.consumer.nextDelivery()
props = delivery.getProperties()
task = String(delivery.getBody())
reply_props = BasicProperties.Builder().build()
try:
task_data = json.loads(str(task))
log.debug("got task: %s", str(task_data))
resp = self.extract(task_data['bibcode'],
task_data['ft_source'])
except Exception, e:
msg = traceback.format_exc()
log.debug("returning error response: %s" % str(msg))
resp = {'_exception': str(msg)}
finally:
class RabbitMQClient:
def __init__(self):
"""
Connections and other settings here should match those set in publisher script
"""
self.stdout = Logger()
self.exchangeName = "bungeni_serialization_output_queue"
self.queueName = "bungeni_serialization_output_queue"
self.factory = ConnectionFactory()
self.factory.setHost("localhost")
self.conn = self.factory.newConnection()
self.channel = self.conn.createChannel()
self.channel.exchangeDeclare(self.exchangeName,"direct",False)
def consume_msgs(self, parliament_cache_info):
try:
declareOk = self.channel.queueDeclare(self.queueName, True, False, False, None)
self.channel.queueBind(self.queueName, self.exchangeName, self.queueName)
self.consumer = QueueingConsumer(self.channel)
self.channel.basicConsume(self.queueName, False, self.consumer)
count = 0
if declareOk.messageCount <= 0:
self.stdout.write(time.asctime(time.localtime(time.time())) + " NO MESSAGES \n")
else:
self.stdout.write(time.asctime(time.localtime(time.time())) + " " + str(declareOk.messageCount) + " MESSAGES! \n")
while (count < declareOk.messageCount):
delivery = QueueingConsumer.Delivery
delivery = self.consumer.nextDelivery()
message = str(String(delivery.getBody()))
obj_data = json.loads(message)
file_status = main_queue(__config_file__, str(obj_data['location']), parliament_cache_info)
count = count + 1
if file_status is None:
print "No Parliament Information could be gathered"
sys.exit(0)
elif file_status is True:
# Acknowledgements to RabbitMQ the successfully, processed files
self.channel.basicAck(delivery.getEnvelope().getDeliveryTag(), False)
else:
# Reject file, requeue for investigation or future attempts
self.channel.basicReject(delivery.getEnvelope().getDeliveryTag(), True)
finally:
try:
if self.channel is not None:
self.channel.close()
except Exception, ex:
LOG.error("Error while closing channel", ex)
try:
if self.conn is not None:
self.conn.close()
except Exception, ex:
LOG.error("Error while closing connection", ex)
try:
consumer = QueueingConsumer(channel)
# Declare as exclusive consumer of queue
channel.basicConsume(queueName, True, "consumer", False, True, None, consumer)
print " [*] Waiting for messages."
except IOError:
print "Unable to connect to queue."
connection.close()
sys.exit(1)
writer = WriteMsgToFile(HOME_FILE)
while True:
try:
delivery = consumer.nextDelivery()
message = delivery.getBody()
message = StringUtil.fromBytes(message)
token = message.split(":",1)[0]
rest = message.split(":",1)[1]
message = str(message)
# If message is a "token_request", process request and add new token to hashmap
if message[0:13] == ("TOKEN_REQUEST"):
makeToken = ProcessTokenRequest(message, channel)
tokenID = makeToken.sendToken()
permissions.put(tokenID[0], tokenID[1])
# Otherwise, check if message has valid token (token in hashmap).
# If so, accept message, then delete token from valid list.
else:
try:
consumer = QueueingConsumer(channel)
# Declare as exclusive consumer of queue
channel.basicConsume(queueName, True, "consumer", False, True, None,
consumer)
print " [*] Waiting for messages."
except IOError:
print "Unable to connect to queue."
connection.close()
sys.exit(1)
writer = WriteMsgToFile(HOME_FILE)
while True:
try:
delivery = consumer.nextDelivery()
message = delivery.getBody()
message = StringUtil.fromBytes(message)
token = message.split(":", 1)[0]
rest = message.split(":", 1)[1]
message = str(message)
# If message is a "token_request", process request and add new token to hashmap
if message[0:13] == ("TOKEN_REQUEST"):
makeToken = ProcessTokenRequest(message, channel)
tokenID = makeToken.sendToken()
permissions.put(tokenID[0], tokenID[1])
# Otherwise, check if message has valid token (token in hashmap).
# If so, accept message, then delete token from valid list.
else:
