'protocol': 'binary', 'host': 'localhost', 'port': 9091 } mq_client = message_queue_client_from_config(mq_config) mq_codec = JSONCodec() processed_resource_queue = 'processed_resources' # ElasticSearch es = ES('localhost:9200', timeout=60) es_index = 'topic_tracking' # dequeue one resource mq_client.connect() message = mq_client.get_message(processed_resource_queue) resource = mq_codec.decode(message.body, Resource) mq_client.delete_message(processed_resource_queue, message.id) mq_client.disconnect() # save the resource to mongo resource._id = makeIdFromURI(resource.uri) resource_collection.insert_model(resource) # index the resource for boost in [1, 1000]: es_doc = {} es_doc['content'] = resource.content es_doc['title'] = resource.title es_doc['entities'] = build_payload_string(resource.entities, boost) es_doc['terms'] = build_payload_string(resource.terms, boost) id = '%s_%d' % (resource._id, boost)
input_queue = 'discovered_resources' output_queue = 'processed_resources' # processing p_client = processing_client_from_config(config['processing']['client']) # start service clients mq_client.connect() p_client.connect() # begin processing while True: try: # input input_message = mq_client.get_message(input_queue) discovered_resource = mq_codec.decode(input_message.body, DiscoveredResource) logger.info('Dequeued discovered resource: %s ' % discovered_resource) try: # process resource = p_client.process(discovered_resource) # output msg_body = mq_codec.encode(resource) mq_client.put_message(output_queue, msg_body) logger.info('Enqueued processed resource: %s ' % resource) except ProcessingException, e: logger.error(e) # delete the input message mq_client.delete_message(input_queue, input_message.id) except EmptyQueueException: