Ejemplo n.º 1
0
def output_kafka(graph_db, registry,
                 kafka_url=None):
    ldict = {"step": MODULEFILE + "/" + inspect.stack()[0][3],
             "hostname": platform.node().split(".")[0]}
    l = logging.LoggerAdapter(common.fetch_lg(), ldict)
    kafka_topic = "cs"
    if kafka_url is None:
        kafka_url = registry.get_config("kafka_url",
                                        "localhost:9092")
    else:
        l.info("Updating registry with kafka_url: {}".format(kafka_url))
        registry.put_config("kafka_url",
                            kafka_url)
    (nodes, rels) = out.output_json(graph_db, None, None, as_list=True)
    l.info("Connecting to kafka_url {}".format(kafka_url))
    kafka = KafkaClient(kafka_url)
    # To send messages asynchronously
    producer = SimpleProducer(kafka)
    l.info("Sending nodes to kafka {}/{}".format(kafka_url, kafka_topic))
    for n in nodes:
        producer.send_messages(kafka_topic, n)
    l.info("Sending rels to kafka {}/{}".format(kafka_url, kafka_topic))
    for n in rels:
        producer.send_messages(kafka_topic, n)
    kafka.close()
Ejemplo n.º 2
0
                     etcdport=args.etcdport)
elif args.inputlocal:
    g = Node.myGraph(cache=None,
                     etcdhost=args.etcdhost,
                     etcdport=args.etcdport)
else:
    sys.exit(1)
    #if not os.path.exists(args.inputjson[0]) or \
    #        not os.path.exists(args.inputjson[1]):
    #    pass
    #else:
    #    l.error("input json files do not exist")
    #    sys.exit(1)

if args.outputjson:
    from resources.output import out
    out.output_json(Node.CONST_GRAPH_NODE.db,
                    args.outputjson[0],
                    args.outputjson[1])
elif args.outputneo4j:
    from resources.output import outneo4j
    outneo4j.output_neo4j(Node.CONST_GRAPH_NODE.db,
                          Node.CONST_GRAPH_NODE.registry,
                          args.outputneo4j)
elif args.outputkafka:
    from resources.output import outkafka
    outkafka.output_kafka(Node.CONST_GRAPH_NODE.db,
                          Node.CONST_GRAPH_NODE.registry,
                          args.outputkafka)
l.info("Have a good day! Bye!")