def send_to_splash(message): try: broker_service = BrokerService("splash-kafka", KAFKA_HOST, KAFKA_PORT) url = "http://" + SPLASH_HOST + ":" + SPLASH_PORT + message["splash_url_path"] workspace = message["workspace"] collection = message["collection"] except: print "Got an invalid message, or couldn't connect the broker service" traceback.print_exc() return try: r = requests.get(url) r.raise_for_status() snapshot = r.text except KeyboardInterrupt: sys.exit() except: traceback.print_exc() snapshot = '{}' broker_service.output_message({"snapshot" : snapshot, "workspace" : workspace, "collection" : collection, "url" : message["url"] })
import sys def send_to_splash(message): try: broker_service = BrokerService("splash-kafka", KAFKA_HOST, KAFKA_PORT) url = "http://" + SPLASH_HOST + ":" + SPLASH_PORT + message["splash_url_path"] workspace = message["workspace"] collection = message["collection"] except: print "Got an invalid message, or couldn't connect the broker service" traceback.print_exc() return try: r = requests.get(url) r.raise_for_status() snapshot = r.text except KeyboardInterrupt: sys.exit() except: traceback.print_exc() snapshot = '{}' broker_service.output_message({"snapshot" : snapshot, "workspace" : workspace, "collection" : collection, "url" : message["url"] }) if __name__ == "__main__": broker_service = BrokerService("splash-kafka", KAFKA_HOST, KAFKA_PORT) broker_service.read_topic_from_splash_queue_blocking(send_to_splash)