def load_and_merge(yaml_file: str) -> Transformer:
"""Load and merge sources defined in the config YAML.
Args:
yaml_file: A string pointing to a KGX compatible config YAML.
Returns:
kgx.Transformer: The merged transformer that contains the merged graph.
"""
config = parse_load_config(yaml_file)
transformers: List = []
# read all the sources defined in the YAML
for key in config['target']:
target = config['target'][key]
logging.info("Loading {}".format(key))
if target['type'] in get_file_types():
# loading from a file
transformer = get_transformer(target['type'])()
for f in target['filename']:
transformer.parse(f, input_format='tsv')
transformers.append(transformer)
elif target['type'] == 'neo4j':
transformer = NeoTransformer(None, target['uri'],
target['username'],
target['password'])
transformer.load()
transformers.append(transformer)
else:
logging.error("type {} not yet supported".format(target['type']))
# merge all subgraphs into a single graph
merged_transformer = Transformer()
merged_transformer.merge_graphs([x.graph for x in transformers])
merged_transformer.report()
# write the merged graph
if 'destination' in config:
destination = config['destination']
if destination['type'] in ['csv', 'tsv', 'ttl', 'json', 'tar']:
destination_transformer = get_transformer(destination['type'])(
merged_transformer.graph)
destination_transformer.save(destination['filename'],
extension=destination['type'])
elif destination['type'] == 'neo4j':
destination_transformer = NeoTransformer(
merged_transformer.graph,
uri=destination['uri'],
username=destination['username'],
password=destination['password'])
destination_transformer.save_with_unwind()
else:
logging.error(
"type {} not yet supported for KGX load-and-merge operation.".
format(destination['type']))
return merged_transformer
def test_neo_to_graph_upload():
""" loads a neo4j graph from a json file
"""
jt = JsonTransformer()
jt.parse('resources/robodb2.json')
nt = NeoTransformer(jt.graph, host='localhost', port='7474', username='******', password='******')
nt.save_with_unwind()
nt.neo4j_report()
def test_csv_to_neo_load():
"""
load csv to neo4j test
"""
pt = PandasTransformer()
pt.parse(os.path.join(resource_dir, "cm_nodes.csv"))
pt.parse(os.path.join(resource_dir, "cm_edges.csv"))
nt = NeoTransformer(pt.graph, host='localhost', port='7474', username='******', password='******')
nt.save_with_unwind()
nt.neo4j_report()
def test_csv_to_neo_load():
"""
load csv to neo4j test
"""
pt = PandasTransformer()
pt.parse("resources/x1n.csv")
pt.parse("resources/x1e.csv")
nt = NeoTransformer(pt.graph,
host='http://localhost',
port='7474',
username='******',
password='******')
nt.save_with_unwind()
nt.neo4j_report()
def test_csv_to_neo_load():
"""
load csv to neo4j test
"""
return
pt = PandasTransformer()
pt.parse("resources/nodes.csv")
pt.parse("resources/edges.csv")
nt = NeoTransformer(pt.graph,
host='localhost',
port='7474',
username='',
password='')
nt.save_with_unwind()
nt.neo4j_report()
default='7474')
parser.add_argument('--username',
help='username (default: neo4j)',
default='neo4j')
parser.add_argument('--password',
help='password (default: demo)',
default='demo')
args = parser.parse_args()
if args.nodes is None or args.edges is None:
usage()
exit()
print(args)
# Initialize PandasTransformer
t = PandasTransformer()
# Load nodes and edges into graph
t.parse(args.nodes, error_bad_lines=False)
t.parse(args.edges, error_bad_lines=False)
# Initialize NeoTransformer
# TODO: eliminate bolt
n = NeoTransformer(t.graph, args.host, {'http': args.http_port}, args.username,
args.password)
# Save graph into Neo4j
n.save_with_unwind()
n.neo4j_report()
def load_and_merge(yaml_file: str) -> nx.MultiDiGraph:
"""Load and merge sources defined in the config YAML.
Args:
yaml_file: A string pointing to a KGX compatible config YAML.
Returns:
networkx.MultiDiGraph: The merged graph.
"""
gm = GraphMerge()
config = parse_load_config(yaml_file)
transformers: List = []
# make sure all files exist before we start load
for key in config['target']:
target = config['target'][key]
logging.info("Checking that file exist for {}".format(key))
if target['type'] in get_file_types():
for f in target['filename']:
if not os.path.exists(f) or not os.path.isfile(f):
raise FileNotFoundError(
"File {} for transform {} in yaml file {} "
"doesn't exist! Dying.", f, key, yaml_file)
# read all the sources defined in the YAML
for key in config['target']:
target = config['target'][key]
logging.info("Loading {}".format(key))
if target['type'] in get_file_types():
# loading from a file
transformer = get_transformer(target['type'])()
for f in target['filename']:
transformer.parse(f, input_format='tsv')
transformer.graph.name = key
transformers.append(transformer)
elif target['type'] == 'neo4j':
transformer = NeoTransformer(None, target['uri'],
target['username'],
target['password'])
transformer.load()
transformers.append(transformer)
transformer.graph.name = key
else:
logging.error("type {} not yet supported".format(target['type']))
stats_filename = f"{key}_stats.yaml"
generate_graph_stats(transformer.graph, key, stats_filename)
# merge all subgraphs into a single graph
merged_graph = gm.merge_all_graphs([x.graph for x in transformers])
merged_graph.name = 'merged_graph'
generate_graph_stats(merged_graph, merged_graph.name,
f"merged_graph_stats.yaml")
# write the merged graph
if 'destination' in config:
for _, destination in config['destination'].items():
if destination['type'] == 'neo4j':
destination_transformer = NeoTransformer(
merged_graph,
uri=destination['uri'],
username=destination['username'],
password=destination['password'])
destination_transformer.save_with_unwind()
elif destination['type'] in get_file_types():
destination_transformer = get_transformer(
destination['type'])(merged_graph)
destination_transformer.save(destination['filename'],
extension=destination['type'])
else:
logging.error(
"type {} not yet supported for KGX load-and-merge operation."
.format(destination['type']))
return merged_graph