'protocol': 'binary',
'host': 'localhost',
'port': 9091
}
mq_client = message_queue_client_from_config(mq_config)
mq_codec = JSONCodec()
processed_resource_queue = 'processed_resources'
# ElasticSearch
es = ES('localhost:9200', timeout=60)
es_index = 'topic_tracking'
# dequeue one resource
mq_client.connect()
message = mq_client.get_message(processed_resource_queue)
resource = mq_codec.decode(message.body, Resource)
mq_client.delete_message(processed_resource_queue, message.id)
mq_client.disconnect()
# save the resource to mongo
resource._id = makeIdFromURI(resource.uri)
resource_collection.insert_model(resource)
# index the resource
for boost in [1, 1000]:
es_doc = {}
es_doc['content'] = resource.content
es_doc['title'] = resource.title
es_doc['entities'] = build_payload_string(resource.entities, boost)
es_doc['terms'] = build_payload_string(resource.terms, boost)
id = '%s_%d' % (resource._id, boost)
input_queue = 'discovered_resources'
output_queue = 'processed_resources'
# processing
p_client = processing_client_from_config(config['processing']['client'])
# start service clients
mq_client.connect()
p_client.connect()
# begin processing
while True:
try:
# input
input_message = mq_client.get_message(input_queue)
discovered_resource = mq_codec.decode(input_message.body, DiscoveredResource)
logger.info('Dequeued discovered resource: %s ' % discovered_resource)
try:
# process
resource = p_client.process(discovered_resource)
# output
msg_body = mq_codec.encode(resource)
mq_client.put_message(output_queue, msg_body)
logger.info('Enqueued processed resource: %s ' % resource)
except ProcessingException, e:
logger.error(e)
# delete the input message
mq_client.delete_message(input_queue, input_message.id)
except EmptyQueueException: