def __init__(self): self.question_preprossor = QuestionPreprossor() self.question_analyzer = QuestionAnalyzer() self.candidate_answer_generator = CandidateAnswerSetGenerator() # self.answer_generator = AnswerGenerator() self.answer_generator = None self.client = DefaultGraphAccessor(GraphClient())
def init(self, path="word2vec_api_software_wiki.txt", binary=True): self.session = EngineFactory.create_session() self.graphClient = DefaultGraphAccessor(GraphClient(server_number=4)) self.entity_vector_model = EntityVectorComputeModel() self.entity_vector_model.init_word2vec_model(path=path, binary=binary) print("init complete")
def test_expand_nodes_with_filter_nodes(self): graphClient = DefaultGraphAccessor(GraphClient()) # test_case = [(55730, True, 50, 50), (15555, True, 4, 4), (93008, True, 10, 11), (1708, True, 8, 7)] test_case = [ (55730, True, 50, 50), ] graphJsonParser = GraphJsonParser() for node_id, is_valid, node_num, relation_num in test_case: print("test case=", node_id, is_valid, node_num, relation_num) subgraph = graphClient.expand_node_for_adjacent_nodes_to_subgraph( node_id) subgraph_json = graphJsonParser.parse_subgraph_to_public_json( subgraph) print(subgraph_json) if is_valid: self.assertNotEqual(subgraph_json, { "nodes": [], "relations": [] }) else: self.assertEqual(subgraph_json, {"nodes": [], "relations": []}) continue self.assertEqual(node_num, len(subgraph_json["nodes"])) self.assertEqual(relation_num, len(subgraph_json["relations"])) for n in subgraph_json["nodes"]: print(n) for r in subgraph_json["relations"]: print(r)
def get_all_nodes(self, step=5000, labels=None): total_node_list = [] self.graph_accessor = DefaultGraphAccessor(self._graph) if labels is None: labels = [] if labels: max_id = self.graph_accessor.get_max_id_for_labels(*labels) min_id = self.graph_accessor.get_min_id_for_labels(*labels) else: max_id = self.graph_accessor.get_max_id_for_node() min_id = 0 iteration = range(min_id, max_id, step) for start_id in iteration: try: end_id = min(start_id + step, max_id) nodes_in_scope = self.get_nodes_in_scope(start_id, end_id, labels) _logger.info("start id=%s,end_id=%s", str(start_id), str(end_id)) if nodes_in_scope is not None: _logger.info("get nodes in scope successfully") total_node_list.extend(nodes_in_scope) else: _logger.info("get nodes in scope failed") except Exception, error: _logger.exception("")
def test_clean_labels(self): self.graphClient = DefaultGraphAccessor(GraphClient(server_number=1)) node = self.graphClient.find_node_by_id(16) self.assertEqual(NodeCleaner.clean_labels(node), [u'software', u'background knowledge', u'WikiData']) node = self.graphClient.find_node_by_id(177777) print node self.assertEqual(NodeCleaner.clean_labels(node), [u'background knowledge', u'WikiData'])
def test_sort_nodes_by_quality(self): graphClient = DefaultGraphAccessor(GraphClient(server_number=1)) graphJson = GraphJsonParser() keyword = "java" top_number = 10 subgraph = graphClient.search_nodes_by_name_in_subgraph( keyword, top_number) print subgraph nodes = graphJson.parse_nodes_in_subgraph_to_public_json(subgraph) print nodes
def build_aliases_for_domain_entity(self): EntityForQA.delete_names_by_source(session=self.session, source="domain entity") client = GraphClient(server_number=4) accessor = DomainEntityAccessor(client) default_accessor = DefaultGraphAccessor(client) domain_entity_list = accessor.get_all_domain_entity() for domain_entity in domain_entity_list: entity = EntityForQA(kg_id=default_accessor.get_id_for_node(node=domain_entity), entity_id=domain_entity['domain_entity_id'], source="domain entity", attr='domain_entity_id', attr_value=domain_entity['domain_entity:name']) self.session.add(entity) self.session.commit()
def init(self, vector_dir_path="./model/"): self.kg_models = KnowledgeGraphFeafureModels() self.kg_models.init(vector_dir_path=vector_dir_path) self._session = EngineFactory.create_session(echo=False) self._entity_extractor = EntityExtractor() # self._tf_idf_model = TFIDFModel() # self._tf_idf_model.load(dict_type=2) self.qa_searcher = QAEntitySearcher() client = GraphClient(server_number=4) self.semanticSearchAccessor = SemanticSearchAccessor(client) self.defaultAccessor = DefaultGraphAccessor(client) self._logger = Logger("QAResultSearch").get_log()
def init_graph_index(self): if self.graph_client is None: print("graphClient is None") return accessor = DefaultGraphAccessor(graph=self.graph_client) accessor.create_index("CREATE INDEX ON :`api`(`api_id`)") accessor.create_index("CREATE INDEX ON :`entity`(`api_id`)") accessor.create_index("CREATE INDEX ON :`wikidata`(`wd_item_id`)") accessor.create_index("CREATE INDEX ON :`entity`(`wd_item_id`)") accessor.create_index("CREATE INDEX ON :`entity`(`site:enwiki`)") accessor.create_index("CREATE INDEX ON :`wikipedia`(`site:enwiki`)") accessor.create_index("CREATE INDEX ON :`wikidata`(`site:enwiki`)") accessor.create_index("CREATE INDEX ON :`entity`(`sentence_id`)") accessor.create_index("CREATE INDEX ON :`sentence`(`sentence_id`)") accessor.create_index("CREATE INDEX ON :`entity`(`domain_entity_id`)") accessor.create_index( "CREATE INDEX ON :`domain entity`(`domain_entity_id`)")
def walk(self): ''' ''' file = open('log1.txt', 'w') client = DefaultGraphAccessor() iterator = 0 possibility_list = [0] * RandomWalk.max_id adjacent_node_list = [0] * RandomWalk.max_id file.writelines("begin: " + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) + '\n') file.flush() while iterator < self.generationNumber: file.writelines("iteration: " + str(iterator) + " " + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) + '\n') file.flush() # print str(iterator) + " " + time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())) current_node_index = 1 step_length_list = [0] * RandomWalk.max_id for i in range(0, self.stepNumber): if (current_node_index > 0 and current_node_index < 7520) or current_node_index > 7524: if adjacent_node_list[current_node_index - 1] == 0: adjacent_node = client.get_adjacent_node_id_list(current_node_index) adjacent_node_list[current_node_index - 1] = adjacent_node else: adjacent_node = adjacent_node_list[current_node_index - 1] next_node_index = adjacent_node[random.randint(0, len(adjacent_node) - 1)] # print str(iterator) + " " + str(i) + " " + str(len(adjacent_node)) + " " + str( # next_node_index) if (next_node_index > 1 and next_node_index < 7520) or next_node_index > 7524: current_node_index = next_node_index if step_length_list[next_node_index - 1] == 0 and next_node_index != 1: step_length_list[next_node_index - 1] = i + 1 possibility_list[next_node_index - 1] += 1 - ( step_length_list[next_node_index - 1] / float(self.stepNumber)) iterator += 1 file.writelines("end: " + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) + '\n') file.flush() file.close() for i in range(0, RandomWalk.max_id): possibility_list[i] /= self.generationNumber if possibility_list[i] > 0: print str(i) + " " + str(possibility_list[i])
def init(self, vector_dir_path="./model/", ): time_start = time.time() print("start init the model=%d" % time_start) client = GraphClient(server_number=4) self.defaultAccessor = DefaultGraphAccessor(client) self._api_wv = EntityVectorModel.load(vector_dir_path + self.WORD2VEC_FILE_LIST["api"], binary=True) self._domain_entity_wv = EntityVectorModel.load(vector_dir_path + self.WORD2VEC_FILE_LIST["domain entity"], binary=True) self._wiki_wv = EntityVectorModel.load(vector_dir_path + self.WORD2VEC_FILE_LIST["wikidata"], binary=True) self._sentence_wv = EntityVectorModel.load(vector_dir_path + self.WORD2VEC_FILE_LIST["sentence"], binary=True) self._graph_wv = EntityVectorModel.load(vector_dir_path + self.WORD2VEC_FILE_LIST["graph"], binary=True) self._entity_vector_compute_model = EntityVectorComputeModel() self._entity_vector_compute_model.init_word2vec_model(vector_dir_path + self.WORD2VEC_FILE_LIST["word2vec"], binary=True) self.NP_VECTOR_NOT_EXIST = np.zeros(128) self.NP_VECTOR_NOT_EXIST[1] = 1e-07 time_end = time.time() print("init complete in %d" % (time_end - time_start))
def start_import_for_api_entity(self, linking_result_file): graph_client = GraphClient(server_number=4) default_graph_client = DefaultGraphAccessor(graph_client) api_entity_graph_client = DomainEntityAccessor(graph_client) api_entity_graph_client.delete_all_api_entity_to_wikipedia_relation() print("delete all old may link relation complete") with open(linking_result_file, 'r') as f: link_relation_list = json.load(f) for each in link_relation_list: api_entity_id = each['api_entity_id'] wikipedia_entity_id = each['wikipedia_entity_id'] if api_entity_id is None or wikipedia_entity_id is None: continue api_entity = api_entity_graph_client.find_api_entity_node_by_id(api_entity_id) if api_entity is None: continue wikipedia_entity = default_graph_client.find_node_by_id(wikipedia_entity_id) if wikipedia_entity is None: continue api_entity_graph_client.create_entity_to_general_concept_relation(api_entity, wikipedia_entity)
from skgraph.graph.accessor.factory import NodeBuilder # Mysql connect from skgraph.graph.accessor.graph_accessor import DefaultGraphAccessor, GraphClient from skgraph.util.code_text_process import clean_html_text conn = MySQLdb.connect(host='10.141.221.73', port=3306, user='******', passwd='root', db='fdroid', charset="utf8") cur = conn.cursor() # neo4j connect connect_graph = DefaultGraphAccessor(GraphClient(server_number=1)).graph # read from mysql def mySQLReader(start, end): node1lib = connect_graph.find_one(label="schema", property_key="wd_item_id", property_value="Q21127166") # get all-version library cur.execute( "select * from jdk_library where library_id >= %s and library_id < %s", (start, end)) lib_sql_data_list = cur.fetchall() for lib_node_mysql_data_index in range(0, len(lib_sql_data_list)):
import codecs import json from py2neo import Relationship from skgraph.graph.accessor.graph_accessor import GraphClient, DefaultGraphAccessor from skgraph.graph.accessor.graph_client_for_awesome import AwesomeGraphAccessor from skgraph.graph.accessor.graph_client_for_wikipedia import WikipediaGraphAccessor from shared.logger_util import Logger _logger = Logger("AwesomeImporter").get_log() awesomeGraphAccessor = AwesomeGraphAccessor(GraphClient(server_number=0)) wikipediaGraphAccessor = WikipediaGraphAccessor(awesomeGraphAccessor) defaultGraphAccessor = DefaultGraphAccessor(awesomeGraphAccessor) baseGraphClient = awesomeGraphAccessor.graph file_name = "awesome_item_category_related_to_wikipedia_relation_list.json" with codecs.open(file_name, 'r', 'utf-8') as f: relation_list = json.load(f) for tag_relation in relation_list: start_entity_name = tag_relation["start_entity_name"] relation = tag_relation["relation"] end_url = tag_relation["end_url"] start_node = awesomeGraphAccessor.find_awesome_cate_by_name( start_entity_name) end_node = defaultGraphAccessor.get_node_by_wikipedia_link(end_url) if end_node is None: end_node = wikipediaGraphAccessor.create_wikipedia_item_entity_by_url( end_url)
def setUp(self): self.graphClient = DefaultGraphAccessor(GraphClient()) self.nodeCleaner = NodeCleaner()
def test_rename_property(self): self.graphClient = DefaultGraphAccessor(GraphClient(server_number=0)) node_list = self.graphClient.find_by_name_property("awesome item", "acl9") result = rename_property(node_list) print result
from skgraph.graph.node_cleaner import GraphJsonParser reload(sys) sys.setdefaultencoding("utf-8") app = Flask(__name__) CORS(app) db_handler = SQLAlchemyHandler() db_handler.setLevel(logging.WARN) # Only serious messages app.logger.addHandler(db_handler) logger = Logger("neo4jServer").get_log() logger.info("create logger") client = GraphClient(server_number=4) graphClient = DefaultGraphAccessor(client) logger.info("create graphClient") apiGraphClient = APIGraphAccessor(client) semanticSearchAccessor = SemanticSearchAccessor(client) # api_entity_linker = APIEntityLinking() # logger.info("create api_entity_linker object") questionAnswerSystem = QuestionAnswerSystem() logger.info("create questionAnswerSystem") dbSOPostSearcher = SOPostSearcher(EngineFactory.create_so_session(), logger=app.logger) logger.info("create SO POST Searcher")
# -*- coding:utf8 -*- import sys import nltk from skgraph.graph.accessor.graph_accessor import GraphClient from skgraph.graph.accessor.graph_accessor import DefaultGraphAccessor reload(sys) sys.setdefaultencoding('utf8') graph_client = GraphClient(server_number=0) graph_accessor = DefaultGraphAccessor(graph_client) ''' merge node and relation belong to extended knowledge and entity which name ''' class DuplicateCleaner: def merge_node_with_same_name(self): print 'begin merge node with same name' query = "match(a:`extended knowledge`:entity) return id(a) as id,a.name as name" try: nodeList = [] result = graph_accessor.graph.run(query) for n in result: node = (n['id'], n['name']) nodeList.append(node) except Exception: return [] nodeDict = {} a = 1 for n in nodeList:
def setUp(self): self.graphClient = DefaultGraphAccessor(GraphClient()) self.filter = NodeRelationFilter()
def setUp(self): graphClient = DefaultGraphAccessor(GraphClient(server_number=1)) relation_util = RelationUtil(graphClient) self.relation_util = relation_util
def __init__(self): #file_dir = os.path.split(os.path.realpath(__file__))[0] #self.path = os.path.join(file_dir, self.path) #self.stackoverflow_word2vec_model = Word2Vec.load(self.path) self.graph_operator = DefaultGraphAccessor(GraphClient()) print "done load word2vec"
import threading from Queue import Queue from py2neo import Node from skgraph.graph.accessor.graph_accessor import DefaultGraphAccessor, GraphClient from skgraph.graph.accessor.graph_client_for_rwr import RandomWalkGraphAccessor from skgraph.graph.operation.generateDataPpi import DataCreateUtil from skgraph.graph.operation.random_walk_restart import Walker log_file = open('log_single.txt', 'w') input_file_name = 'data.ppi' node_id = 53157 client_1 = DefaultGraphAccessor(GraphClient(server_number=1)) client_2 = DefaultGraphAccessor(GraphClient(server_number=2)) data_util = DataCreateUtil(client_1) end_status = 0 class Producer(threading.Thread): def __init__(self, queue): threading.Thread.__init__(self) self.data = queue def run(self): print 'begin produce' log_file.writelines('begin %d\n' % (node_id)) log_file.flush() if client_1.find_node_by_id(node_id) == None: return 0 if data_util.createData(node_id) == 1:
from py2neo import Relationship from skgraph.graph.accessor.graph_accessor import DefaultGraphAccessor, GraphClient from graph_operation import GraphOperation graphClient = DefaultGraphAccessor(GraphClient(server_number=1)) class JavaReturnValueTypeLinkerOperation(GraphOperation): name = 'JavaReturnValueTypeLinkerOperation' def operate(self, node): return_value_type = node["value type"] if return_value_type: type_node = graphClient.find_one_by_alias_name_property( "java class", return_value_type) if type_node is not None: relation = Relationship(node, "type of", type_node) graphClient.merge(relation) return node, node
def setUp(self): self.graphClient = DefaultGraphAccessor(GraphClient())
def __init__(self, graph_client, api_searcher, api_semantic_search): self.graph_accessor = DefaultGraphAccessor(graph_client) self.api_searcher = api_searcher self.api_semantic_search = api_semantic_search
def init(self): self.graphClient = DefaultGraphAccessor(GraphClient(server_number=4)) self.session = EngineFactory.create_session() print("init complete")