def _enqueue(self, resource): # Discovered URI - capped collection --db.discovered_uri mongoConnection = Connection() db = mongoConnection.feed_reading_discovery # settings max_content_length = config['discovery']['resource_enqueueing']['max_content_length'] # message queue mq_client = message_queue_client_from_config(config['message_queue']['client']) mq_client.connect() mq_codec = JSONCodec() queue = 'discovered_resources' try: #insert into capped db.discovered_uri.insert({'_id':makeIdFromURI(resource.uri), 'uri':resource.uri}) #insert into queue if len(resource.content) > max_content_length: self._logger.warning('Skipped %s: Content length is %s.' % (resource, len(resource.content))) msg_body = mq_codec.encode(resource) mq_client.put_message(queue, msg_body) self._logger.debug("Enqueued: %s" % resource._id) except DuplicateKeyError: pass # remove resource from collection self._resources_collection.remove_model(resource)
mq_client.connect() p_client.connect() # begin processing while True: try: # input input_message = mq_client.get_message(input_queue) discovered_resource = mq_codec.decode(input_message.body, DiscoveredResource) logger.info('Dequeued discovered resource: %s ' % discovered_resource) try: # process resource = p_client.process(discovered_resource) # output msg_body = mq_codec.encode(resource) mq_client.put_message(output_queue, msg_body) logger.info('Enqueued processed resource: %s ' % resource) except ProcessingException, e: logger.error(e) # delete the input message mq_client.delete_message(input_queue, input_message.id) except EmptyQueueException: logger.info('Empty queue. Sleeping...') time.sleep(EMPTY_QUEUE_TIMEOUT) # close service clients mq_client.disconnect() p_client.disconnect()