def parse(self, data):
"""Convert CSV topics into vertices and topics"""
vertex = Vertex.get_or_create(
name=data['institution'],
graph_id=self.graph.id)
for topic, keywords in data['topics']:
topic = Topic.get_or_create(name=topic)
tv = TopicVertex.get_or_create(
topic_id=topic.id, vertex_id=vertex.id)
for keyword in keywords:
kw = Keyword.get_or_create(name=keyword)
kt = KeywordTopic.get_or_create(
keyword_id=kw.id, topic_id=topic.id)
self.vertices.append(vertex)
def load(self, args):
"""Load all domains from scraper into new graph"""
domains = set(h.domain for h in HTML.query().all())
for domain in domains:
Vertex.get_or_create(domain=domain).save()
self.graph.vertices = Vertex.query().all()
