def test_add_resource_node() -> None:
kg = KnowledgeGraph()
resource_id = kg.add_resource_node("Bitcoin desc",
resource_type=ResourceType.DESCRIPTION)
assert resource_id
resource_id2 = kg.add_resource_node("Bitcoin link",
link="http://fakelink.bitcoin",
resource_type=ResourceType.PAPER)
assert resource_id2
assert resource_id != resource_id2
def test_add_topic_node() -> None:
kg = KnowledgeGraph()
node_id = kg.add_topic_node("Bitcoin")
assert node_id
node_id2 = kg.add_topic_node("Bifcoin")
assert node_id2
assert node_id2 != node_id
# force a collision
node_id3 = kg.add_topic_node("Bitcoin CLONE", id=node_id)
assert node_id == node_id3
def run_scraper(link="", file="", out="", debug=False):
if link:
r = requests.get(link)
graph = r.json()
g = KnowledgeGraph(obj=graph, debug=debug)
else:
g = KnowledgeGraph(filename=file, debug=debug)
a = AwesomeClient(g)
a.build_map()
if out:
a.write_to_file(out)
else:
print(a.build_str())
def update_scopes(match: str, line: str, scopes: list,
graph: KnowledgeGraph) -> list:
"""
Update the scopes and add a new node to graph
"""
scopes_cpy = scopes.copy()
# build the node and add to graph
new_scope = Scope(match, line, graph)
id = new_scope.get_id()
# connect
if scopes_cpy:
graph.add_edge(id, scopes_cpy[-1].get_id())
scopes_cpy.append(new_scope)
return scopes_cpy
def test_write_to_json() -> None:
kg = KnowledgeGraph()
node_name = "bitcoin"
node_id = kg.add_topic_node(node_name)
node_name2 = "bitcoin2"
node_id2 = kg.add_topic_node(node_name2)
resource = "bitcoin description"
resource_id = kg.add_resource_node(resource)
edge_id = kg.add_edge(node_id, node_id2)
j = kg.write_to_json()
print(j) # if pytest fails, we'll see the output
assert len(j["nodes"]) == 3
assert len(j["edges"]) == 1
assert [
x for x in j["nodes"] if x["data"]["node_type"] == NodeType.TOPIC
and x["data"]["name"] == node_name and x["data"]["id"] == node_id
]
assert [
x for x in j["nodes"] if x["data"]["node_type"] == NodeType.TOPIC
and x["data"]["name"] == node_name2 and x["data"]["id"] == node_id2
]
assert [
x for x in j["nodes"] if x["data"]["node_type"] == NodeType.RESOURCE
]
assert [
x for x in j["edges"] if x["data"]["source"] == node_id
and x["data"]["target"] == node_id2 and x["data"]["id"] == edge_id
]
def test_scope() -> None:
kg = KnowledgeGraph()
scope = Scope(r"^###", "### Bitcoin", kg)
assert scope._get_name() == "Bitcoin"
test_text = "Script - Bitcoin Wiki"
test_link = "https://en.bitcoin.it/wiki/Script"
scope = Scope(r"^[*-]", f"* [{test_text}]({test_link})", kg)
data = scope._get_data()
assert data["text"] == test_text
assert data["link"] == test_link
def main(link: str) -> KnowledgeGraph:
f = requests.get(link)
text = f.text
textlines = text.splitlines()
scopes = []
graph = KnowledgeGraph(0)
for line in textlines:
match = match_hierarchy(line)
scopes = match_line(match, line, scopes, graph)
return graph
def run_scrape(link: str, out_file: str, debug: bool = False) -> None:
r = requests.get(link)
graph = r.json()
g = KnowledgeGraph(obj=graph, debug=debug)
a = AwesomeClient(g)
a.build_map()
if out_file:
a.write_to_file(out_file)
else:
print(a.build_str())
def main(link: str, path: str) -> KnowledgeGraph:
if link:
f = requests.get(link)
text = f.text
elif path:
with open(path, "r") as f:
text = f.read()
else:
raise Exception("Need either Link or Path")
textlines = text.splitlines()
scopes = []
graph = KnowledgeGraph(0)
for line in textlines:
match = match_hierarchy(line)
scopes = match_line(match, line, scopes, graph)
return graph
def test_init() -> None:
"""KnowledgeGraph loads empty by default"""
kg = KnowledgeGraph()
g = kg.get_graph()
assert len(g.nodes) == 0
assert len(g.edges) == 0