class TestNodeCleaner(TestCase): def test_clean_labels(self): self.graphClient = DefaultGraphAccessor(GraphClient(server_number=1)) node = self.graphClient.find_node_by_id(16) self.assertEqual(NodeCleaner.clean_labels(node), [u'software', u'background knowledge', u'WikiData']) node = self.graphClient.find_node_by_id(177777) print node self.assertEqual(NodeCleaner.clean_labels(node), [u'background knowledge', u'WikiData']) def test_construct_property_set(self): node_list = [{"aaa": 1, "ccc": 6}, {"aaa": 2, "bbb": 3, "ccc": 6}, {"ccc": 4, "ddd": 5}] result = {"aaa", "ccc"} self.assertEqual(construct_property_set(node_list), result) print construct_property_set(node_list) def test_rename_property(self): self.graphClient = DefaultGraphAccessor(GraphClient(server_number=0)) node_list = self.graphClient.find_by_name_property("awesome item", "acl9") result = rename_property(node_list) print result def test_public_labels_name(self): t = PUBLIC_LABELS self.assertIsNotNone(t) print(t)
def start_import_for_api_entity(self, linking_result_file): graph_client = GraphClient(server_number=4) default_graph_client = DefaultGraphAccessor(graph_client) api_entity_graph_client = DomainEntityAccessor(graph_client) api_entity_graph_client.delete_all_api_entity_to_wikipedia_relation() print("delete all old may link relation complete") with open(linking_result_file, 'r') as f: link_relation_list = json.load(f) for each in link_relation_list: api_entity_id = each['api_entity_id'] wikipedia_entity_id = each['wikipedia_entity_id'] if api_entity_id is None or wikipedia_entity_id is None: continue api_entity = api_entity_graph_client.find_api_entity_node_by_id(api_entity_id) if api_entity is None: continue wikipedia_entity = default_graph_client.find_node_by_id(wikipedia_entity_id) if wikipedia_entity is None: continue api_entity_graph_client.create_entity_to_general_concept_relation(api_entity, wikipedia_entity)
class TestGraphClient(TestCase): graphClient = None def setUp(self): self.graphClient = DefaultGraphAccessor(GraphClient()) self.nodeCleaner = NodeCleaner() def test_get_max_id_for_node(self): self.assertEqual(self.graphClient.get_max_id_for_node(), 697753) def test_get_adjacent_node_id_list(self): self.assertEqual(self.graphClient.get_adjacent_node_id_list(66666666), []) correct = [64289, 52628, 62565] self.assertEqual(self.graphClient.get_adjacent_node_id_list(7899), correct) def test_get_node_name_by_id(self): self.assertEqual(self.graphClient.get_node_name_by_id(66666666), None) self.assertEqual(self.graphClient.get_node_name_by_id(3444), "Adobe Device Central") def test_expand_node_for_directly_adjacent_nodes_to_subgraph(self): # self.assertEqual(self.graphClient.expand_node_for_adjacent_nodes_to_subgraph(3444), # "Adobe Device Central") pass def test_find_by_alias_name_property_exactly_match_from_label_limit_one( self): self.assertEqual( self.graphClient.find_one_by_alias_name_property( "entity", "Adobe Device Central"), None) interface = self.graphClient.find_one_by_alias_name_property( "api", "Interface PrintGraphics") self.assertEqual(93008, self.graphClient.get_id_for_node(interface)) def test_find_by_alias_name_property(self): self.assertEqual( self.graphClient.find_by_alias_name_property( "entity", "Adobe Device Central"), []) interfaces = self.graphClient.find_by_alias_name_property( "api", "Interface PrintGraphics") self.assertEqual(len(interfaces), 1) self.assertEqual(93008, self.graphClient.get_id_for_node(interfaces[0])) def test_get_relation_by_relation_id(self): relation = self.graphClient.get_relation_by_relation_id(470129) self.assertIsNone(relation) relation = self.graphClient.get_relation_by_relation_id(122211) self.assertEqual(122211, self.graphClient.get_id_for_node(relation)) self.assertEqual( 91, self.graphClient.get_id_for_node(relation.start_node())) self.assertEqual(29390, self.graphClient.get_id_for_node(relation.end_node())) subgraph = self.graphClient.get_relations_between_two_nodes_in_subgraph( 246029, 246030) relations_json = [] for r in subgraph.relationships(): r = { "id": self.graphClient.get_id_for_node(relation), "name": relation.type(), "start_id": self.graphClient.get_start_id_for_relation(relation), "end_id": self.graphClient.get_end_id_for_relation(relation) } print r subgraph = self.graphClient.get_relations_between_two_nodes_in_subgraph( 246029, 246033) self.assertEqual(subgraph, None) def test_find_node_by_id(self): node = self.graphClient.find_node_by_id(5444) self.assertEqual(5444, self.graphClient.get_id_for_node(node)) def test_search_nodes_by_name(self): nodes = self.graphClient.search_nodes_by_name("java") count = 0 for n in nodes: count = count + 1 self.assertEqual(10, count) nodes = self.graphClient.search_nodes_by_name("String buffer()") count = 0 for n in nodes: count = count + 1 self.assertEqual(10, count) def test_search_nodes_by_name_in_subgraph(self): subgraph = self.graphClient.search_nodes_by_name_in_subgraph("java") count = 0 for n in subgraph.nodes(): count = count + 1 self.assertEqual(10, count) subgraph = self.graphClient.search_nodes_by_name_in_subgraph( "String buffer()") count = 0 if subgraph is not None: for n in subgraph.nodes(): count = count + 1 self.assertEqual(10, count) def test_get_relations_between_two_nodes_in_subgraph(self): subgraph = self.graphClient.get_relations_between_two_nodes_in_subgraph( 48, 3600) self.assertEqual(None, subgraph) subgraph = self.graphClient.get_relations_between_two_nodes_in_subgraph( 48, 3643) self.assertEqual(2, len(subgraph.nodes())) self.assertEqual(1, len(subgraph.relationships())) def test_get_relations_between_two_nodes(self): record_list = self.graphClient.get_relations_between_two_nodes( 48, 3600) count = 0 for n in record_list: count = count + 1 self.assertEqual(0, count) record_list = self.graphClient.get_relations_between_two_nodes_in_subgraph( 48, 3643) count = 0 for n in record_list: count = count + 1 self.assertEqual(1, count) def test_cleaner(self): node = self.graphClient.find_node_by_id(444) self.assertEqual(self.nodeCleaner.get_clean_node_name(node), "fake news") node = self.graphClient.find_node_by_id(4444) self.assertEqual(self.nodeCleaner.get_clean_node_name(node), "") self.assertEqual(self.graphClient.get_id_for_node(Node("lll", a=3)), -1) self.assertEqual(self.graphClient.get_id_for_node(node), 4444) def test_get_shortest_path_to_name(self): name = self.graphClient.get_node_name_by_id(8000) subraph = self.graphClient.get_shortest_path_to_name_in_subgraph( 444, name) print subraph def test_get_shortest_path(self): record_list = self.graphClient.get_shortest_path(444, 8000, max_degree=2) self.assertEqual(0, count_record_list(record_list)) record_list = self.graphClient.get_shortest_path(444, 8000) self.assertNotEqual(None, record_list) self.assertEqual(1, count_record_list(record_list)) subgraph = self.graphClient.get_shortest_path_in_subgraph(444, 8000, max_degree=2) self.assertEqual(None, subgraph) subgraph = self.graphClient.get_shortest_path_in_subgraph(444, 8000, max_degree=6) self.assertNotEqual(None, subgraph) self.assertEqual(len(subgraph.nodes()), 7) self.assertEqual(len(subgraph.relationships()), 6) print subgraph def test_get_newest_nodes(self): node_list = self.graphClient.get_newest_nodes(10) self.assertEqual(10, len(node_list)) print(node_list) graphJsonParser = GraphJsonParser() returns = graphJsonParser.parse_node_list_to_json(node_list) print(returns)
class OldQuestionAnswerSystem: def __init__(self): self.question_preprossor = QuestionPreprossor() self.question_analyzer = QuestionAnalyzer() self.candidate_answer_generator = CandidateAnswerSetGenerator() # self.answer_generator = AnswerGenerator() self.answer_generator = None self.client = DefaultGraphAccessor(GraphClient()) def simple_answer(self, question_text): ''' get a simple answer string :param question_text: question text :return: a simple answer string ''' # todo complete this answer method return "I can't answer this question" def full_answer(self, question_text, top_number=10): ''' get a full answer set for the question :param question_text: question text :return: a full answer set ''' question = self.question_preprossor.preprosse_question( question_text=question_text) question = self.question_analyzer.analyze_question(question=question) question.print_to_console() candidate_answer_set = self.candidate_answer_generator.generate_candidate_answer_set( question=question) answer_set = self.answer_generator.generate_answer_set( question=question, candidate_answer_set=candidate_answer_set) return answer_set def fake_full_answer(self, question_text, top_number=10): ''' get a full answer set for the question :param question_text: question text :return: a full answer set ''' ## todo fix this fake ful answer answer_set = AnswerSet(answer_list=[]) return answer_set answer_list = [] if 'what is java' in question_text.lower(): node = self.client.find_node_by_id(678253) answer = Answer(self.get_entity_name(node), [node], 0.9) answer_list.append(answer) node = self.client.find_node_by_id(676793) answer = Answer(self.get_entity_name(node), [node], 0.7) answer_list.append(answer) node = self.client.find_node_by_id(7686) answer = Answer(self.get_entity_name(node), [node], 0.6) answer_list.append(answer) node = self.client.find_node_by_id(19204) answer = Answer(self.get_entity_name(node), [node], 0.4) answer_list.append(answer) node = self.client.find_node_by_id(821488) answer = Answer(self.get_entity_name(node), [node], 0.2) answer_list.append(answer) time.sleep(4) elif 'which library can process json' in question_text.lower(): node = self.client.find_node_by_id(1518) answer = Answer(self.get_entity_name(node), [node], 0.8) answer_list.append(answer) node = self.client.find_node_by_id(698753) answer = Answer(self.get_entity_name(node), [node], 0.7) answer_list.append(answer) node = self.client.find_node_by_id(699419) answer = Answer(self.get_entity_name(node), [node], 0.6) answer_list.append(answer) node = self.client.find_node_by_id(700229) answer = Answer(self.get_entity_name(node), [node], 0.5) answer_list.append(answer) time.sleep(5) elif 'what string building api is thread safe' in question_text.lower( ): node = self.client.find_node_by_id(659611) answer = Answer(self.get_entity_name(node), [node], 0.8) answer_list.append(answer) node = self.client.find_node_by_id(52741) answer = Answer(self.get_entity_name(node), [node], 0.6) answer_list.append(answer) node = self.client.find_node_by_id(659612) answer = Answer(self.get_entity_name(node), [node], 0.1) answer_list.append(answer) time.sleep(7) elif 'where can i find apache' in question_text.lower(): node = self.client.find_node_by_id(692324) answer = Answer(self.get_entity_name(node), [node], 0.8) answer_list.append(answer) node = self.client.find_node_by_id(1465147) answer = Answer(self.get_entity_name(node), [node], 0.6) answer_list.append(answer) node = self.client.find_node_by_id(2217) answer = Answer(self.get_entity_name(node), [node], 0.1) answer_list.append(answer) node = self.client.find_node_by_id(2805) answer = Answer(self.get_entity_name(node), [node], 0.1) answer_list.append(answer) time.sleep(3) answer_set = AnswerSet(answer_list=answer_list) return answer_set def get_entity_name(self, node): if node.has_key('name'): name = node.get('name') elif node.has_key('labels_en'): name = node.get('labels_en') else: name = '' return name
class WikiAliasDBImporter: def __init__(self): self.graphClient = None self.session = None def init(self): self.graphClient = DefaultGraphAccessor(GraphClient(server_number=4)) self.session = EngineFactory.create_session() print("init complete") def clean_table(self): WikipediaEntityName.delete_all(self.session) WikipediaEntityNameToWikipediaMapping.delete_all(self.session) print("delete all exist table") def start_import_wiki_aliases_to_db(self): label = "wikipedia" wiki_nodes = self.graphClient.get_all_nodes_by_label(label) for node in wiki_nodes: node_id = self.graphClient.get_id_for_node(node) # print ('node_id: %r', node_id) # name, site_enwiki, labels_ = '' name_set = set([]) if 'name' in dict(node): # print ("name: %r", node['name']) if isinstance(node['name'], list): for each in node['name']: name_set.add(each) else: name_set.add(node['name']) if 'site:enwiki' in dict(node): # print ('site_enwiki: %s', node['site:enwiki']) if isinstance(node['site:enwiki'], list): for each in node['site:enwiki']: title = URLUtil.parse_url_to_title(each) # print ('site_name: %r', title) name_set.add(title) else: title = URLUtil.parse_url_to_title(node["site:enwiki"]) # print ('site_name: %r', title) name_set.add(title) if 'labels_en' in dict(node): # print( "labels_en: ", node['labels_en']) if isinstance(node['labels_en'], list): for each in node['labels_en']: name_set.add(each) else: name_set.add(node['labels_en']) if 'aliases_en' in dict(node): # print("aliases_en: ", node['aliases_en']) for each in node['aliases_en']: name_set.add(each) # print (name_set) for name in name_set: try: wikipedia_entity_name = WikipediaEntityName( node_id, str(name)) wikipedia_entity_name.find_or_create(self.session, autocommit=True) except Exception: traceback.print_exc() # self.session.commit() self.session.commit() def start_generate_wiki_entity_text_map(self): wikipedia_entity_name_data = WikipediaEntityName.get_all_wikipedia_names( self.session) kg_id_list = set([]) for each in wikipedia_entity_name_data: if each is not None: kg_id_list.add(each.kg_id) # print kg_id_list for kg_id in kg_id_list: node = self.graphClient.find_node_by_id(kg_id) if node is not None: if "site:enwiki" in dict(node): title = URLUtil.parse_url_to_title(node["site:enwiki"]) wikipedia_doc = WikipediaDocument.get_document_by_wikipedia_title( self.session, title) if wikipedia_doc is not None: wikipedia_id = wikipedia_doc.id wiki_name_to_wikipedia_mapping = WikipediaEntityNameToWikipediaMapping( kg_id, wikipedia_id) wiki_name_to_wikipedia_mapping.find_or_create( self.session, autocommit=False) self.session.commit() def start_import(self): self.init() self.clean_table() self.start_import_wiki_aliases_to_db() self.start_generate_wiki_entity_text_map()