def output_kafka(graph_db, registry,
kafka_url=None):
ldict = {"step": MODULEFILE + "/" + inspect.stack()[0][3],
"hostname": platform.node().split(".")[0]}
l = logging.LoggerAdapter(common.fetch_lg(), ldict)
kafka_topic = "cs"
if kafka_url is None:
kafka_url = registry.get_config("kafka_url",
"localhost:9092")
else:
l.info("Updating registry with kafka_url: {}".format(kafka_url))
registry.put_config("kafka_url",
kafka_url)
(nodes, rels) = out.output_json(graph_db, None, None, as_list=True)
l.info("Connecting to kafka_url {}".format(kafka_url))
kafka = KafkaClient(kafka_url)
# To send messages asynchronously
producer = SimpleProducer(kafka)
l.info("Sending nodes to kafka {}/{}".format(kafka_url, kafka_topic))
for n in nodes:
producer.send_messages(kafka_topic, n)
l.info("Sending rels to kafka {}/{}".format(kafka_url, kafka_topic))
for n in rels:
producer.send_messages(kafka_topic, n)
kafka.close()
etcdport=args.etcdport)
elif args.inputlocal:
g = Node.myGraph(cache=None,
etcdhost=args.etcdhost,
etcdport=args.etcdport)
else:
sys.exit(1)
#if not os.path.exists(args.inputjson[0]) or \
# not os.path.exists(args.inputjson[1]):
# pass
#else:
# l.error("input json files do not exist")
# sys.exit(1)
if args.outputjson:
from resources.output import out
out.output_json(Node.CONST_GRAPH_NODE.db,
args.outputjson[0],
args.outputjson[1])
elif args.outputneo4j:
from resources.output import outneo4j
outneo4j.output_neo4j(Node.CONST_GRAPH_NODE.db,
Node.CONST_GRAPH_NODE.registry,
args.outputneo4j)
elif args.outputkafka:
from resources.output import outkafka
outkafka.output_kafka(Node.CONST_GRAPH_NODE.db,
Node.CONST_GRAPH_NODE.registry,
args.outputkafka)
l.info("Have a good day! Bye!")