def import_all_graph_data(self, graph_data: GraphData, clear=True):
"""
import all data in one GraphData into neo4j and create index on node
:param graph_data:
:param clear: clear the graph content, default is not clear the graph contain
:return:
"""
index_accessor = IndexGraphAccessor(self.graph_accessor)
index_accessor.create_index(label=self.DEFAULT_LABEL, property_name=self.DEFAULT_PRIMARY_KEY)
if clear:
self.graph_accessor.delete_all_relations()
self.graph_accessor.delete_all_nodes()
# todo: this is slow, need to speed up, maybe not commit on every step
all_node_ids = graph_data.get_node_ids()
for node_id in all_node_ids:
## todo: fix this by not using 'properties','labels'
node_info_dict = graph_data.get_node_info_dict(node_id)
properties = node_info_dict['properties']
labels = node_info_dict['labels']
self.import_one_entity(node_id, properties, labels)
print("all entity imported")
relations = graph_data.get_relations()
for r in relations:
start_node_id, r_name, end_node_id = r
start_node = self.graph_accessor.find_node(primary_label=self.DEFAULT_LABEL,
primary_property=self.DEFAULT_PRIMARY_KEY,
primary_property_value=start_node_id)
end_node = self.graph_accessor.find_node(primary_label=self.DEFAULT_LABEL,
primary_property=self.DEFAULT_PRIMARY_KEY,
primary_property_value=end_node_id)
if start_node is not None and end_node is not None:
try:
self.graph_accessor.create_relation_without_duplicate(start_node, r_name, end_node)
except Exception as e:
traceback.print_exc()
else:
print("fail create relation because start node or end node is none.")
print("all relation imported")
print("all graph data import finish")
def test_get_graph(self):
graph_data = GraphData()
graph_data.add_node({"method"}, {"qualified_name": "ArrayList.add"})
graph_data.add_node({"override method"},
{"qualified_name": "ArrayList.pop"})
graph_data.add_node({"method"}, {"qualified_name": "ArrayList.remove"})
graph_data.add_node({"method"}, {"qualified_name": "ArrayList.clear"})
print(graph_data.get_node_ids())
print(graph_data.get_relation_pairs_with_type())
graph_data.add_relation(1, "related to", 2)
graph_data.add_relation(1, "related to", 3)
graph_data.add_relation(1, "related to", 4)
graph_data.add_relation(2, "related to", 3)
graph_data.add_relation(3, "related to", 4)
print(graph_data.get_relations(1, "related to"))
print("get relation by type")
print(graph_data.get_relations(relation_type="related to"))
def start_training(self, graph_data: GraphData, *properties):
"""
start train the kg name searcher model from a graph data object by specifying name properties.
:param properties: the properties that need to be searched on. e.g., "name","qualified_name","labels_en"
:param graph_data: the GraphData instance
:return:
"""
# todo: add some config arguments, to control whether lower the case, split the words.
self.clear()
for node_id in graph_data.get_node_ids():
node_properties = graph_data.get_properties_for_node(
node_id=node_id)
for property_name in properties:
property_value = node_properties.get(property_name, None)
if not property_value:
continue
if type(property_value) == list or type(property_value) == set:
iterable_property_values = property_value
for single_value in iterable_property_values:
self.add_from_property_value(single_value, node_id)
else:
single_value = property_value
self.add_from_property_value(single_value, node_id)
def graphdata2csv(csv_folder, graph: GraphData, node_file_id=GraphData.DEFAULT_KEY_NODE_ID,
csv_labels=GraphData.DEFAULT_KEY_NODE_LABELS,
relation_file_start_id=GraphData.DEFAULT_KEY_RELATION_START_ID,
relation_file_end_id=GraphData.DEFAULT_KEY_RELATION_END_ID,
relation_file_type=GraphData.DEFAULT_KEY_RELATION_TYPE,
only_one_relation_file=True):
'''
:param csv_folder: 存放生产csv文件的文件夹路径
:param graph: 将要导出的graphdata
:param csv_id: 生成的csv文件id列的列名，默认是id
:param csv_labels: 生成的csv文件labels列的列名，默认是labels
:return: 无返回
'''
csvfilename2label = {}
csvfilename2ids = {}
csvfilename2property_name = {}
ids = graph.get_node_ids()
for id in ids:
node = graph.get_node_info_dict(id)
labels = node.get(GraphData.DEFAULT_KEY_NODE_LABELS)
property_names = node.get(GraphData.DEFAULT_KEY_NODE_PROPERTIES).keys()
csvfilename = '_'.join(labels)
if labels not in csvfilename2label.values():
csvfilename2label[csvfilename] = labels
csvfilename2ids[csvfilename] = set([])
csvfilename2property_name[csvfilename] = set([])
csvfilename2ids[csvfilename].add(id)
for property_name in property_names:
csvfilename2property_name[csvfilename].add(property_name)
for k, v in csvfilename2property_name.items():
csvfilename2property_name[k] = list(v)
node_count = 0
for csvfilename, ids in csvfilename2ids.items():
with open(os.path.join(csv_folder, '{}.{}'.format(csvfilename.replace(' ',''), 'csv')), 'w', newline='',
encoding='utf-8') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
first_node = True
for id in ids:
node = graph.get_node_info_dict(id)
if node:
node_dic = {}
node_properties = node.get(GraphData.DEFAULT_KEY_NODE_PROPERTIES)
node_dic[node_file_id] = node.get(GraphData.DEFAULT_KEY_NODE_ID)
node_dic[csv_labels] = node.get(GraphData.DEFAULT_KEY_NODE_LABELS)
for property_name in csvfilename2property_name[csvfilename]:
node_dic[property_name] = node_properties.get(property_name)
if first_node:
writer.writerow(node_dic)
writer.writerow(node_dic.values())
node_count = node_count + 1
first_node = False
print("一共导入csv的节点个数: ", node_count)
relation_types = graph.get_all_relation_types()
relation_count = 0
if only_one_relation_file:
with open(os.path.join(csv_folder, '{}.{}'.format('relations', 'csv')), 'w', newline='',
encoding='utf-8') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
first_relation = True
for relation_type in relation_types:
relation_pairs = graph.get_relations(relation_type=relation_type)
for relation_pair in relation_pairs:
relation_dic = {}
relation_dic[relation_file_start_id] = int(relation_pair[0])
relation_dic[relation_file_type] = relation_pair[1]
relation_dic[relation_file_end_id] = int(relation_pair[2])
if first_relation:
writer.writerow(relation_dic)
first_relation = False
writer.writerow(relation_dic.values())
relation_count = relation_count + 1
print("一共导入csv的关系个数: ", relation_count)
print("一共生成", len(csvfilename2label), "个csv节点文件和", str(1), "个csv关系文件")
else:
for relation_type in relation_types:
relation_pairs = graph.get_relations(relation_type=relation_type)
with open(os.path.join(csv_folder, '{}.{}'.format(relation_type.replace(' ', ''), 'csv')), 'w', newline='',
encoding='utf-8') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
first_relation = True
for relation_pair in relation_pairs:
relation_dic = {}
relation_dic[GraphData.DEFAULT_KEY_RELATION_START_ID] = int(relation_pair[0])
relation_dic[GraphData.DEFAULT_KEY_RELATION_TYPE] = relation_pair[1]
relation_dic[GraphData.DEFAULT_KEY_RELATION_END_ID] = int(relation_pair[2])
if first_relation:
writer.writerow(relation_dic)
first_relation = False
writer.writerow(relation_dic.values())
relation_count = relation_count + 1
print("一共导入csv的关系个数: ", relation_count)
print("一共生成", len(csvfilename2label), "个csv节点文件和", len(relation_types), "个csv关系文件")