def test_merge_no_preserve():
"""
Test for merging graphs, overwriting conflicting properties
"""
pt1 = PandasTransformer()
pt1.parse(os.path.join(resource_dir, 'merge', 'nodes1.tsv'),
input_format='tsv')
pt1.parse(os.path.join(resource_dir, 'merge', 'edges1.tsv'),
input_format='tsv')
pt2 = PandasTransformer()
pt2.parse(os.path.join(resource_dir, 'merge', 'nodes2.tsv'),
input_format='tsv')
pt2.parse(os.path.join(resource_dir, 'merge', 'edges2.tsv'),
input_format='tsv')
merged_graph = merge_all_graphs([pt1.graph, pt2.graph], preserve=False)
assert len(merged_graph.nodes()) == 6
assert len(merged_graph.edges()) == 8
x1 = merged_graph.nodes['x1']
print(x1)
assert x1['name'] == 'node x1'
assert isinstance(x1['category'], list)
assert list(pt1.graph.nodes['x1']['category'])[0] in x1['category']
assert list(pt2.graph.nodes['x1']['category'])[0] in x1['category']
assert x1['p1'] == 'a'
def test_merge():
"""
Test for merging graphs
"""
pt1 = PandasTransformer()
pt1.parse(os.path.join(resource_dir, 'merge', 'nodes1.tsv'),
input_format='tsv')
pt1.parse(os.path.join(resource_dir, 'merge', 'edges1.tsv'),
input_format='tsv')
pt2 = PandasTransformer()
pt2.parse(os.path.join(resource_dir, 'merge', 'nodes2.tsv'),
input_format='tsv')
pt2.parse(os.path.join(resource_dir, 'merge', 'edges2.tsv'),
input_format='tsv')
merged_graph = merge_all_graphs([pt1.graph, pt2.graph], preserve=True)
assert len(merged_graph.nodes()) == 6
assert len(merged_graph.edges()) == 8
x1 = merged_graph.nodes['x1']
assert x1['name'] == 'node x1'
assert isinstance(x1['category'], list)
assert 'a' in x1['p1']
assert '1' in x1['p1']
x10 = merged_graph.nodes['x10']
assert x10['id'] == 'x10'
assert x10['name'] == 'node x10'
def main(force_transform):
"""
Goes through a series of data files, transforms them to CSV format and saves
them. Then re-loads each CSV file and merges them into a single file named
clique_merged.csv, while performing a clique merge as well as categorizing
all nodes and edges.
"""
for filename, constructor in data.items():
if filename.endswith('.csv') or filename.endswith('.csv.tar'):
# Already in csv format, no need to transform it
continue
out = change_extention(filename, 'csv.tar')
if not force_transform and os.path.isfile(out):
# CSV has already been generated, no need to recreate it
continue
t = constructor()
t.parse(filename)
t = PandasTransformer(t)
t.save(out)
t = PandasTransformer()
for filename in data.keys():
# Load each CSV into a single transformer
filename = change_extention(filename, 'csv.tar')
t.parse(filename)
t.merge_cliques()
t.clean_categories()
t.save('results/clique_merged.csv')
def test_mapping():
"""
create a random graph and save it in different formats
"""
G = nx.MultiDiGraph()
N = 100
E = N * 3
mapping = {}
for i in range(0,N+1):
nid = curie(i)
mapping[nid] = mapped_curie(i)
G.add_node(nid, label="node {}".format(i))
for i in range(1,E):
s = random_curie(N)
o = random_curie(N)
G.add_edge(o,s,predicate='related_to')
print('Nodes={}'.format(len(G.nodes())))
mapper.map_graph(G, mapping)
print("Mapped..")
count = 0
for nid in G.nodes():
src = G.node[nid]['source_curie']
assert nid.startswith("Y:")
assert src.startswith("X:")
count += 1
if count > 5:
break
print("Saving tsv")
w = PandasTransformer(G)
w.save("target/maptest.tar")
w = ObanRdfTransformer(G)
w.save("target/maptest.ttl")
def test_clique_merge():
"""
Test for clique merge (lenient)
"""
t = PandasTransformer()
os.makedirs(target_dir, exist_ok=True)
t.parse(os.path.join(resource_dir, 'cm_nodes.csv'))
t.parse(os.path.join(resource_dir, 'cm_edges.csv'))
t.report()
cm = CliqueMerge(prefix_prioritization_map)
cm.build_cliques(t.graph)
cm.elect_leader()
updated_graph = cm.consolidate_edges()
leaders = nx.get_node_attributes(updated_graph, 'clique_leader')
leader_list = list(leaders.keys())
leader_list.sort()
assert len(leader_list) == 2
n1 = updated_graph.nodes[leader_list[0]]
assert n1['election_strategy'] == 'PREFIX_PRIORITIZATION'
assert 'NCBIGene:100302240' in n1['aliases']
assert 'ENSEMBL:ENSG00000284458' in n1['aliases']
n2 = updated_graph.nodes[leader_list[1]]
assert n2['election_strategy'] == 'PREFIX_PRIORITIZATION'
assert 'NCBIGene:8202' in n2['aliases']
assert 'OMIM:601937' in n2['aliases']
assert 'ENSEMBL:ENSG00000124151' not in n2['aliases']
def main(path, output, model):
if model is not None:
bmt.load(model)
t = JsonTransformer()
t.parse(path)
t = PandasTransformer(t.graph)
t.save(output)
def test_neo_to_graph_transform():
"""
load from neo4j and transform to nx graph
"""
nt = NeoTransformer(host='localhost', port='7474', username='******', password='******')
nt.load()
nt.report()
t = PandasTransformer(nt.graph)
t.save(os.path.join(target_dir, "neo_graph.csv"))
def test_csv_to_neo_load():
"""
load csv to neo4j test
"""
pt = PandasTransformer()
pt.parse(os.path.join(resource_dir, "cm_nodes.csv"))
pt.parse(os.path.join(resource_dir, "cm_edges.csv"))
nt = NeoTransformer(pt.graph, host='localhost', port='7474', username='******', password='******')
nt.save_with_unwind()
nt.neo4j_report()
def test_neo_to_graph_transform():
"""
load from neo4j and transform to nx graph
"""
n = NeoTransformer()
n.load()
n.report()
t = PandasTransformer(n)
t.save("target/neo_graph.csv")
def test_load():
"""
Test for loading data into PandasTransformer
"""
t = PandasTransformer()
os.makedirs(target_dir, exist_ok=True)
t.parse(os.path.join(resource_dir, "x1n.csv"))
t.parse(os.path.join(resource_dir, "x1e.csv"))
t.report()
t.save(os.path.join(target_dir, 'x1copy.csv'))
def test_neo_to_graph_transform():
"""
load from neo4j and transform to nx graph
"""
nt = NeoTransformer(uri=DEFAULT_NEO4J_URL,
username=DEFAULT_NEO4J_USERNAME,
password=DEFAULT_NEO4J_PASSWORD)
nt.load()
nt.report()
t = PandasTransformer(nt.graph)
t.save(os.path.join(target_dir, "neo_graph.csv"))
def test_load():
"""
load tests
"""
t = PandasTransformer()
t.parse("tests/resources/x1n.csv")
t.parse("tests/resources/x1e.csv")
t.report()
t.save('target/x1copy.csv')
w = GraphMLTransformer(t)
w.save("target/x1n.graphml")
def test_read_achive():
"""
Test reading of tar, tar.gz and tar.bz2 archives
"""
tar_file = os.path.join(target_dir, "semmeddb_test_export.tar")
tar_gz_file = os.path.join(target_dir, "semmeddb_test_export.tar.gz")
tar_bz_file = os.path.join(target_dir, "semmeddb_test_export.tar.bz2")
pt = PandasTransformer()
pt.parse(tar_file)
assert not pt.is_empty()
pt2 = PandasTransformer()
pt2.parse(tar_gz_file)
assert not pt2.is_empty()
pt3 = PandasTransformer()
pt3.parse(tar_bz_file)
assert not pt3.is_empty()
def test_load_tsv():
t = PandasTransformer()
t.parse(os.path.join(resource_dir, 'test_nodes.tsv'), input_format='tsv')
t.parse(os.path.join(resource_dir, 'test_edges.tsv'), input_format='tsv')
assert t.graph.number_of_nodes() == 3
assert t.graph.number_of_edges() == 1
assert t.graph.nodes['CURIE:123'][
'description'] == '"Node of type Gene, CURIE:123"'
assert t.graph.nodes['CURIE:456'][
'description'] == '"Node of type Disease, CURIE:456"'
def test_clique_generation():
"""
Test for generation of cliques
"""
t = PandasTransformer()
t.parse(os.path.join(resource_dir, 'cm_nodes.csv'))
t.parse(os.path.join(resource_dir, 'cm_edges.csv'))
t.report()
cm = CliqueMerge(prefix_prioritization_map)
cm.build_cliques(t.graph)
cliques = list(nx.connected_components(cm.clique_graph))
assert len(cliques) == 2
def test_graph_to_neo_load():
"""
load nx graph to neo4j test
"""
t = PandasTransformer()
t.parse("tests/resources/x1n.csv")
t.parse("tests/resources/x1e.csv")
t.report()
n = NeoTransformer(t)
n.save()
n.neo4j_report()
def test_csv_to_neo_load():
"""
load csv to neo4j test
"""
pt = PandasTransformer()
pt.parse(os.path.join(resource_dir, "cm_nodes.csv"))
pt.parse(os.path.join(resource_dir, "cm_edges.csv"))
nt = NeoTransformer(pt.graph,
uri=DEFAULT_NEO4J_URL,
username=DEFAULT_NEO4J_USERNAME,
password=DEFAULT_NEO4J_PASSWORD)
nt.save()
nt.neo4j_report()
def parse(self, name: str, data_file: str, source: str) -> None:
"""Processes the data_file.
Args:
name: Name of the ontology
data_file: data file to parse
source: Source name
Returns:
None.
"""
print(f"Parsing {data_file}")
transformer = ObographJsonTransformer()
transformer.parse(data_file, provided_by=source)
output_transformer = PandasTransformer(transformer.graph)
output_transformer.save(filename=os.path.join(self.output_dir, f'{name}'), output_format='tsv', mode=None)
def test_load():
"""
load tests
"""
t = ObanRdfTransformer()
t.parse("tests/resources/monarch/biogrid_test.ttl")
t.report()
w1 = PandasTransformer(t)
w1.save('target/biogrid-e.csv', type='e')
w1.save('target/biogrid-n.csv', type='n')
w2 = GraphMLTransformer(t)
w2.save("target/x1n.graphml")
w3 = JsonTransformer(t)
w3.save("target/x1n.json")
def test_csv_to_neo_load():
"""
load csv to neo4j test
"""
pt = PandasTransformer()
pt.parse("resources/x1n.csv")
pt.parse("resources/x1e.csv")
nt = NeoTransformer(pt.graph,
host='http://localhost',
port='7474',
username='******',
password='******')
nt.save_with_unwind()
nt.neo4j_report()
def test_neo_to_graph_transform():
"""
load from neo4j and transform to nx graph
"""
return
nt = NeoTransformer(host='localhost',
port='7474',
username='',
password='')
nt.load()
nt.report()
t = PandasTransformer(nt.graph)
t.save("target/neo_graph.csv")
def test_filters(query):
nodes = os.path.join(resource_dir, 'test_nodes2.tsv')
edges = os.path.join(resource_dir, 'test_edges2.tsv')
t = PandasTransformer()
for nf in query[0].keys():
t.set_node_filter(nf, query[0][nf])
for ef in query[1].keys():
t.set_edge_filter(ef, query[1][ef])
t.parse(nodes, input_format='tsv')
t.parse(edges, input_format='tsv')
assert t.graph.number_of_nodes() == query[2]
assert t.graph.number_of_edges() == query[3]
def test_semmeddb_csv_to_tsv():
"""
Read nodes and edges from CSV and export the resulting graph as an archive
"""
t = PandasTransformer()
nodes_file = os.path.join(resource_dir, "semmed/semmeddb_test_nodes.csv")
edges_file = os.path.join(resource_dir, "semmed/semmeddb_test_edges.csv")
output = os.path.join(target_dir, "semmeddb_test_tsv_export")
t.parse(nodes_file)
t.parse(edges_file)
# save output as TSV in a tar archive
t.save(output, extension='tsv')
def test_csv_to_neo_load():
"""
load csv to neo4j test
"""
return
pt = PandasTransformer()
pt.parse("resources/nodes.csv")
pt.parse("resources/edges.csv")
nt = NeoTransformer(pt.graph,
host='localhost',
port='7474',
username='',
password='')
nt.save_with_unwind()
nt.neo4j_report()
def test_load():
"""
load tests
"""
t = PandasTransformer()
cwd = os.path.abspath(os.path.dirname(__file__))
resdir = os.path.join(cwd, 'resources')
targetdir = os.path.join(cwd, 'target')
os.makedirs(targetdir, exist_ok=True)
t.parse(os.path.join(resdir, "x1n.csv"))
t.parse(os.path.join(resdir, "x1e.csv"))
t.report()
t.save(os.path.join(targetdir, 'x1copy.csv'))
w = GraphMLTransformer(t.graph)
w.save(os.path.join(targetdir, "x1n.graphml"))
def test_owl_load():
"""
Load a test OWL and export as JSON
"""
input_file = os.path.join(resource_dir, 'mody.ttl')
output_archive_file = os.path.join(target_dir, 'mondo_test')
output_json_file = os.path.join(target_dir, 'mondo_test.json')
t = RdfOwlTransformer()
t.parse(input_file, input_format='ttl')
t.report()
pt = PandasTransformer(t.graph)
pt.save(output_archive_file)
jt = JsonTransformer(t.graph)
jt.save(output_json_file)
def test_load():
"""
load and save tests
"""
cwd = os.path.abspath(os.path.dirname(__file__))
src_path = os.path.join(cwd, 'resources', 'monarch', 'biogrid_test.ttl')
tpath = os.path.join(cwd, 'target')
os.makedirs(tpath, exist_ok=True)
tg_path = os.path.join(tpath, "test_output.ttl")
# execute ObanRdfTransformer's parse and save function
t = ObanRdfTransformer()
t.parse(src_path, input_format="turtle")
t.save(tg_path, output_format="turtle")
t.report()
w1 = PandasTransformer(t.graph)
w1.save(os.path.join(tpath, 'biogrid-e.csv'), type='e')
w1.save(os.path.join(tpath, 'biogrid-n.csv'), type='n')
# read again the source, test graph
src_graph = rdflib.Graph()
src_graph.parse(src_path, format="turtle")
# read again the dumped target graph
tg_graph = rdflib.Graph()
tg_graph.parse(tg_path, format="turtle")
# compare subgraphs from the source and the target graph.
OBAN = Namespace('http://purl.org/oban/')
for a in src_graph.subjects(RDF.type, OBAN.association):
oban_src_graph = rdflib.Graph()
oban_src_graph += src_graph.triples((a, None, None))
oban_tg_graph = rdflib.Graph()
oban_tg_graph += tg_graph.triples((a, None, None))
# see they are indeed identical (isomorphic)
if not oban_src_graph.isomorphic(oban_tg_graph):
raise RuntimeError('The subgraphs whose subject is ' + str(a) +
' are not isomorphic ones.')
w2 = GraphMLTransformer(t.graph)
w2.save(os.path.join(tpath, "x1n.graphml"))
w3 = JsonTransformer(t.graph)
w3.save(os.path.join(tpath, "x1n.json"))
def test_export(query):
nodes = os.path.join(resource_dir, 'test_nodes.tsv')
edges = os.path.join(resource_dir, 'test_edges.tsv')
t = PandasTransformer()
t.parse(nodes, input_format='tsv')
t.parse(edges, input_format='tsv')
assert t.graph.number_of_nodes() == 3
assert t.graph.number_of_edges() == 1
output_filename = os.path.join(target_dir, query[0])
t.save(filename=output_filename, output_format=query[1], mode=query[2])
if isinstance(query[3], str):
assert os.path.exists(os.path.join(target_dir, query[3]))
else:
assert os.path.exists(os.path.join(target_dir, query[3][0]))
assert os.path.exists(os.path.join(target_dir, query[3][1]))
def test_load():
"""
load tests
"""
cwd = os.path.abspath(os.path.dirname(__file__))
resdir = os.path.join(cwd, 'resources')
tdir = os.path.join(cwd, 'target')
os.makedirs(tdir, exist_ok=True)
t = RdfOwlTransformer()
fn = os.path.join(resdir, "mody.ttl.gz")
f = gzip.open(fn, 'rb')
t.parse(f, input_format='ttl')
t.report()
w1 = PandasTransformer(t.graph)
w1.save(os.path.join(tdir, 'mondo-e.csv'), type='e')
w1.save(os.path.join(tdir, 'mondo-n.csv'), type='n')
w3 = JsonTransformer(t.graph)
w3.save(os.path.join(tdir, "mondo.json"))
def parse(self, name: str, data_file: str) -> None:
"""Processes the data_file.
Args:
name: Name of the ontology
data_file: data file to parse
Returns:
None.
"""
logging.info(f"Parsing {data_file}")
transformer = ObographJsonTransformer()
transformer.parse(data_file)
output_transformer = PandasTransformer(transformer.graph)
output_transformer.save(filename=os.path.join(self.output_dir,
f'{name}'),
extension='tsv',
mode=None)