def _enqueue(self, resource):
# Discovered URI - capped collection --db.discovered_uri
mongoConnection = Connection()
db = mongoConnection.feed_reading_discovery
# settings
max_content_length = config['discovery']['resource_enqueueing']['max_content_length']
# message queue
mq_client = message_queue_client_from_config(config['message_queue']['client'])
mq_client.connect()
mq_codec = JSONCodec()
queue = 'discovered_resources'
try:
#insert into capped
db.discovered_uri.insert({'_id':makeIdFromURI(resource.uri), 'uri':resource.uri})
#insert into queue
if len(resource.content) > max_content_length:
self._logger.warning('Skipped %s: Content length is %s.' %
(resource, len(resource.content)))
msg_body = mq_codec.encode(resource)
mq_client.put_message(queue, msg_body)
self._logger.debug("Enqueued: %s" % resource._id)
except DuplicateKeyError:
pass
# remove resource from collection
self._resources_collection.remove_model(resource)
mq_client.connect()
p_client.connect()
# begin processing
while True:
try:
# input
input_message = mq_client.get_message(input_queue)
discovered_resource = mq_codec.decode(input_message.body, DiscoveredResource)
logger.info('Dequeued discovered resource: %s ' % discovered_resource)
try:
# process
resource = p_client.process(discovered_resource)
# output
msg_body = mq_codec.encode(resource)
mq_client.put_message(output_queue, msg_body)
logger.info('Enqueued processed resource: %s ' % resource)
except ProcessingException, e:
logger.error(e)
# delete the input message
mq_client.delete_message(input_queue, input_message.id)
except EmptyQueueException:
logger.info('Empty queue. Sleeping...')
time.sleep(EMPTY_QUEUE_TIMEOUT)
# close service clients
mq_client.disconnect()
p_client.disconnect()
