def import_jdk_parameter_relation(cur, session): parameter_data = read_parameter_data(cur) for each in parameter_data: if each is not None: name = each[0] method_id = each[1] type_string = each[2] description = each[3] type_class = each[4] full_declaration = type_string + " " + name description = clean_html_text(description) parameter_entity = APIEntity.find_by_full_declaration_and_description(session, full_declaration, description) jdk_method_knowledge_table = KnowledgeTableFactory.get_jdk_method_table(session) api_knowledge_table = KnowledgeTableFactory.get_api_entity_table(session) method_entity_id = KnowledgeTableRowMapRecord.get_end_row_id(session, jdk_method_knowledge_table, api_knowledge_table, method_id) if parameter_entity is not None and method_entity_id is not None: end_api_id = parameter_entity.id api_relation_has_parameter = APIRelation(method_entity_id, end_api_id, APIRelation.RELATION_TYPE_HAS_PARAMETER) print("------------------------") print(api_relation_has_parameter) api_relation_has_parameter.find_or_create(session, autocommit=False) if type_class == 0: qualified_class_name = type_string else: qualified_class_name = get_qualified_class_name(type_class, cur) if qualified_class_name is not None and parameter_entity is not None: class_entity = APIEntity.find_by_qualifier(session, qualified_class_name) if class_entity is not None: api_relation_has_type = APIRelation(parameter_entity.id, class_entity.id, APIRelation.RELATION_TYPE_PARAMETER_HAS_TYPE) print("=============================") print(api_relation_has_type) api_relation_has_type.find_or_create(session, autocommit=False) session.commit()
def import_all_jdk_version(): jdk_library_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session, "jdk_library") library_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session, LibraryEntity.__tablename__) # get all-version library cur.execute("select * from jdk_library") lib_sql_data_list = cur.fetchall() result_tuples = [] for i in range(0, len(lib_sql_data_list)): lib_node, old_id = create_library_node(lib_sql_data_list[i]) if KnowledgeTableRowMapRecord.exist_import_record( session, jdk_library_knowledge_table, library_knowledge_table, old_id): print old_id, " has been import to new table" else: lib_node = lib_node.find_or_create(session, autocommit=False) result_tuples.append((lib_node, old_id)) session.commit() for item in result_tuples: (lib_node, old_id) = item record = KnowledgeTableRowMapRecord(jdk_library_knowledge_table, library_knowledge_table, old_id, lib_node.id) record.create(session, autocommit=False) session.commit() "reading from mysql completed!"
def import_all_jdk_class(): jdk_class_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name(session, "jdk_class") api_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name(session, APIEntity.__tablename__) # get all-version library cur.execute("select * from jdk_class") data_list = cur.fetchall() result_tuples = [] for i in range(0, len(data_list)): class_node, old_id = create_class_node(data_list[i]) if class_node is None: print "unkonwn api type", data_list[i] continue if KnowledgeTableRowMapRecord.exist_import_record(session, jdk_class_knowledge_table, api_knowledge_table, old_id): print old_id, " has been import to new table" else: class_node = class_node.find_or_create(session, autocommit=False) result_tuples.append((class_node, old_id)) session.commit() for item in result_tuples: (class_node, old_id) = item record = KnowledgeTableRowMapRecord(jdk_class_knowledge_table, api_knowledge_table, old_id, class_node.id) record.create(session, autocommit=False) session.commit() "reading from mysql completed!"
def import_all_android_package(): jdk_package_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name(session, "androidAPI_support_package") api_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name(session, APIEntity.__tablename__) # get all-version library cur.execute("select * from androidAPI_support_package") package_data_list = cur.fetchall() result_tuples = [] for i in range(0, len(package_data_list)): package_node, old_id = create_package_node(package_data_list[i]) if KnowledgeTableRowMapRecord.exist_import_record(session, jdk_package_knowledge_table, api_knowledge_table, old_id): print old_id, " has been import to new table" else: package_node = package_node.find_or_create(session, autocommit=False) result_tuples.append((package_node, old_id)) session.commit() for item in result_tuples: (package_node, old_id) = item record = KnowledgeTableRowMapRecord(jdk_package_knowledge_table, api_knowledge_table, old_id, package_node.id) record.create(session, autocommit=False) session.commit() "reading from mysql completed!"
def import_parameter_has_parameter_relation(cur, session): parameter_data_from_knowledgegraph = read_parameter_data_from_knowledgegraph( cur) for each in parameter_data_from_knowledgegraph: method_id = each[0] full_declaration = each[2] simple_parameter_list = get_simple_parameter_list(full_declaration) if simple_parameter_list is not None: for i in range(0, len(simple_parameter_list)): simple_parameter_list[i] = simple_parameter_list[i].replace( "[]", "").replace("...", "").strip() parameter_entity = APIEntity.find_by_full_declaration_and_description( session, simple_parameter_list[i], None) api_knowledge_table = KnowledgeTableFactory.get_api_entity_table( session) android_method_table = KnowledgeTableFactory.get_android_method_table( session) method_entity_id = KnowledgeTableRowMapRecord.get_end_row_id( session, android_method_table, api_knowledge_table, method_id) if parameter_entity is not None and method_entity_id is not None: api_relation_has_parameter = APIRelation( method_entity_id, parameter_entity.id, APIRelation.RELATION_TYPE_HAS_PARAMETER) api_relation_has_parameter.find_or_create(session, autocommit=False) session.commit()
def test_exist_import_record(self): session = EngineFactory.create_session() jdk_method_knowledge_table = KnowledgeTableFactory.get_jdk_method_table(session) api_relation_table = KnowledgeTableFactory.get_api_relation_table(session) api_knowledge_table = KnowledgeTableFactory.get_api_entity_table(session) result= KnowledgeTableColumnMapRecord.exist_import_record(session, jdk_method_knowledge_table, api_relation_table, 1, "class_id") print result self.assertEqual(result,True)
def import_android_exception_relation(cur, session): total = 0 type1 = 0 type2 = 0 exception_data = read_exception_data(cur) for each in exception_data: total += 1 exception_name = each[0] exception_text = each[1].replace("\n", '').replace( " ", '').replace(" ", '').replace(" ", '').replace(" ", '') exception_entity = APIEntity.find_by_full_declaration_and_description( session, exception_name, exception_text) method_id = each[2] api_knowledge_table = KnowledgeTableFactory.get_api_entity_table( session) android_method_table = KnowledgeTableFactory.get_android_method_table( session) method_entity_id = KnowledgeTableRowMapRecord.get_end_row_id( session, android_method_table, api_knowledge_table, method_id) exception_url = each[3] qualified_name_list = exception_url.replace( "https://developer.android.google.cn/reference/", '').replace(".html", "").split("/") qualified_name = "" for i in range(0, len(qualified_name_list)): if i == 0: qualified_name += qualified_name_list[i] else: qualified_name += ("." + qualified_name_list[i]) print(qualified_name) exception_class = session.query(APIEntity).filter_by( qualified_name=qualified_name).first() if exception_entity is not None and method_entity_id is not None: api_relation_has_exception = APIRelation( method_entity_id, exception_entity.id, APIRelation.RELATION_TYPE_HAS_EXCEPTION) api_relation_has_exception.find_or_create(session, autocommit=False) type1 += 1 if exception_entity is not None and exception_class is not None: api_relation_has_type = APIRelation( exception_entity.id, exception_class.id, APIRelation.RELATION_TYPE_EXCEPTION_HAS_TYPE) api_relation_has_type.find_or_create(session, autocommit=False) type2 += 1 session.commit() print(total) print(type1) print(type2)
def import_jdk_return_value_relation(cur, session): return_value_data = read_return_value_data(cur) total = 0 type1 = 0 type2 = 0 for each in return_value_data: total += 1 method_id = each[0] full_declaration = each[1] return_type = each[2] return_string = each[3] return_type = process_return_type_string(return_type) qualified_name = construct_qualified_name_by_full_declaration(full_declaration, return_type) print("****************") print(method_id) print(return_type) print(full_declaration) print(qualified_name) if qualified_name is None: qualified_name = return_type return_string = clean_html_text(return_string) parameter_entity = APIEntity.find_by_full_declaration_and_description(session, return_type, return_string) jdk_method_knowledge_table = KnowledgeTableFactory.get_jdk_method_table(session) api_knowledge_table = KnowledgeTableFactory.get_api_entity_table(session) method_entity_id = KnowledgeTableRowMapRecord.get_end_row_id(session, jdk_method_knowledge_table, api_knowledge_table, method_id) if parameter_entity is not None and method_entity_id is not None: end_api_id = parameter_entity.id api_relation_has_return_value = APIRelation(method_entity_id, end_api_id, APIRelation.RELATION_TYPE_HAS_RETURN_VALUE) api_relation_has_return_value.find_or_create(session, autocommit=False) print("------------------------") print(api_relation_has_return_value) type1 += 1 if qualified_name is not None and parameter_entity is not None: print("+++++++++++++++++++++") print(qualified_name) class_entity = APIEntity.find_by_qualifier(session, qualified_name) if class_entity is not None: api_relation_has_type = APIRelation(parameter_entity.id, class_entity.id, APIRelation.RELATION_TYPE_RETURN_VALUE_HAS_TYPE) api_relation_has_type.find_or_create(session, autocommit=False) print("=============================") print(api_relation_has_type) type2 += 1 session.commit() print("total: " + str(total) + ", type1: " + str(type1) + ", type2: " + str(type2))
def import_doc_from_api_html_table(self): # step 2: api html->doc session = self.session api_html_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session, APIHTMLText.__tablename__) html_text_list = session.query(APIHTMLText).filter_by( html_type=APIHTMLText.HTML_TYPE_API_DETAIL_DESCRIPTION).all() result_tuples = [] for html_text in html_text_list: if DocumentSourceRecord.exist_import_record( session, api_html_knowledge_table.id, html_text.id): print("%d has been import to new table", (html_text.id, )) continue clean_text = html_text.clean_text if clean_text is None or clean_text.strip() == "": continue doc_text = DocumentText(html_text_id=html_text.id, text=clean_text) # text with no html tags doc_text.create(session, autocommit=False) result_tuples.append((doc_text, html_text.id)) session.commit() for doc_text, html_text_id in result_tuples: record = DocumentSourceRecord( doc_id=doc_text.id, kg_table_id=api_html_knowledge_table.id, kg_table_primary_key=html_text_id) record.create(session, autocommit=False) session.commit() print("import clean text from html table complete import")
def import_jdk_exception_relation(cur, session): jdk_exception_data = read_jdk_exception_data(cur) simple_exception_name_list = get_simple_exception_name_list(session) total = 0 type1 = 0 type2 = 0 for each in jdk_exception_data: total += 1 name = each[0] method_id = each[1] description = each[2] description = clean_html_text(description) exception_entity = APIEntity.find_by_full_declaration_and_description( session, name, description) jdk_method_knowledge_table = KnowledgeTableFactory.get_jdk_method_table( session) api_knowledge_table = KnowledgeTableFactory.get_api_entity_table( session) method_entity_id = KnowledgeTableRowMapRecord.get_end_row_id( session, jdk_method_knowledge_table, api_knowledge_table, method_id) if exception_entity is not None and method_entity_id is not None: type1 += 1 api_relation_has_exception = APIRelation( method_entity_id, exception_entity.id, APIRelation.RELATION_TYPE_HAS_EXCEPTION) api_relation_has_exception.find_or_create(session, autocommit=False) print("------------------------") print(api_relation_has_exception) if exception_entity is not None and name is not None: type2 += 1 exception_id = get_exception_id_by_simple_name( simple_exception_name_list, name) if exception_id is not None: api_relation_has_type = APIRelation( exception_entity.id, exception_id, APIRelation.RELATION_TYPE_EXCEPTION_HAS_TYPE) api_relation_has_type.find_or_create(session, autocommit=False) print("=============================") print(api_relation_has_type) session.commit() print("total: " + str(total) + ", type1: " + str(type1) + ", type2: " + str(type2))
def import_android_return_value_relation(cur, session): total = 0 type1 = 0 type2 = 0 return_value_data = read_android_return_value_data(cur) for each in return_value_data: total += 1 return_value_name = each[0] return_value_text = clean_html_text(each[1]) method_id = each[2] return_value_url = each[3] return_value_entity = APIEntity.find_by_full_declaration_and_description(session, return_value_name, return_value_text) api_knowledge_table = KnowledgeTableFactory.get_api_entity_table(session) android_method_table = KnowledgeTableFactory.get_android_method_table(session) method_entity_id = KnowledgeTableRowMapRecord.get_end_row_id(session, android_method_table, api_knowledge_table, method_id) qualified_name = "" if return_value_url != "": qualified_name_list = return_value_url.replace("https://developer.android.google.cn/reference/", "").replace(".html", "").split("/") for i in range(0, len(qualified_name_list)): if i == 0: qualified_name += qualified_name_list[i] else: qualified_name += ("." + qualified_name_list[i]) else: qualified_name = return_value_name return_value_class = session.query(APIEntity).filter_by(qualified_name=qualified_name).first() if return_value_entity is not None and method_entity_id is not None: type1 += 1 api_relation_has_return_value = APIRelation(method_entity_id, return_value_entity.id, APIRelation.RELATION_TYPE_HAS_RETURN_VALUE) api_relation_has_return_value.find_or_create(session, autocommit=False) if return_value_entity is not None and return_value_class is not None: type2 += 1 api_relation_has_type = APIRelation(return_value_entity.id, return_value_class.id, APIRelation.RELATION_TYPE_RETURN_VALUE_HAS_TYPE) api_relation_has_type.find_or_create(session, autocommit=False) session.commit() print(total) print(type1) print(type2)
import gc from db.engine_factory import ConnectionFactory, EngineFactory from db.model import KnowledgeTableRowMapRecord, APIEntity, APIDocumentWebsite from db.model_factory import KnowledgeTableFactory from shared.logger_util import Logger logger = Logger("import_doc_website_for_jdk_method").get_log() cur = ConnectionFactory.create_cursor_for_jdk_importer() session = EngineFactory.create_session() jdk_method_knowledge_table = KnowledgeTableFactory.get_jdk_method_table( session) api_knowledge_table = KnowledgeTableFactory.get_api_entity_table(session) def create_doc_website_relation(method_name, full_declaration, qualified_name, class_website): # print class_website if "http://docs.oracle.com/javase/7/docs/api/" in class_website: return None if full_declaration: if method_name[0] == "_": method_name = "Z:Z" + method_name if "(" in qualified_name and ")" in qualified_name: left_bracket_index = qualified_name.find("(") right_bracket_index = qualified_name.find(")") param_str = qualified_name[left_bracket_index + 1:right_bracket_index] if "," in param_str: param_list = param_str.split(",")
from db.engine_factory import ConnectionFactory, EngineFactory from db.model import KnowledgeTableRowMapRecord, APIRelation, KnowledgeTableColumnMapRecord from db.model_factory import KnowledgeTableFactory from shared.logger_util import Logger logger = Logger("import_belong_to_relation_for_jdk_method").get_log() cur = ConnectionFactory.create_cursor_for_jdk_importer() session = EngineFactory.create_session() jdk_method_knowledge_table = KnowledgeTableFactory.get_jdk_method_table( session) jdk_class_knowledge_table = KnowledgeTableFactory.get_jdk_class_table(session) api_knowledge_table = KnowledgeTableFactory.get_api_entity_table(session) api_relation_table = KnowledgeTableFactory.get_api_relation_table(session) COMMIT_STEP = 5000 def is_imported(row_id): if KnowledgeTableColumnMapRecord.exist_import_record( session, jdk_method_knowledge_table, api_relation_table, row_id, "class_id"): return True else: return False def create_method_belong_to_relation(old_method_id, old_class_id): logger.info("old_method_id=%d old_class_id=%d", old_method_id,
from bs4 import BeautifulSoup from db.engine_factory import EngineFactory from db.cursor_factory import ConnectionFactory from db.model import KnowledgeTableRowMapRecord, APIEntity from db.model_factory import KnowledgeTableFactory from db.util.code_text_process import parse_declaration_html_with_full_qualified_type, clean_declaration_html, \ clean_html_text_with_format from shared.logger_util import Logger session = EngineFactory.create_session() logger = Logger("import_android_method").get_log() IMPORT_DATA_SOURCE_TABLE_NAME = "androidAPI_method" class_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session=session, name="androidAPI_class") method_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session=session, name="androidAPI_method") api_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session=session, name=APIEntity.__tablename__) def get_api_entity_id_by_old_class_id(old_class_id): return KnowledgeTableRowMapRecord.get_end_row_id( session=session, start_knowledge_table=class_knowledge_table, end_knowledge_table=api_knowledge_table, start_row_id=old_class_id)
from db.engine_factory import EngineFactory from db.model import KnowledgeTableRowMapRecord, APIEntity, APIDocumentWebsite from db.model_factory import KnowledgeTableFactory session = EngineFactory.create_session() android_package_knowledge_table = KnowledgeTableFactory.get_android_package_table(session) api_entity_knowledge_table = KnowledgeTableFactory.get_api_entity_table(session) def get_android_package_data(): package_id_list = [] android_package_data = KnowledgeTableRowMapRecord.get_transformed_table_data(session, android_package_knowledge_table, api_entity_knowledge_table) if android_package_data is not None: for each in android_package_data: end_row_id = each.end_row_id print end_row_id package_id_list.append(end_row_id) # print android_package_data return package_id_list def construct_android_package_url(api_entity): url = "" url_prefix = "https://developer.android.com/reference/" url_suffix = "package-summary" if api_entity is not None: qualified_name = api_entity.qualified_name if "." in qualified_name: name_list = qualified_name.split(".") else: name_list = [qualified_name]
from db.engine_factory import ConnectionFactory, EngineFactory from db.model import KnowledgeTableRowMapRecord, APIEntity from db.model_factory import KnowledgeTableFactory from db.util.code_text_process import clean_html_text from shared.logger_util import Logger session = EngineFactory.create_session() logger = Logger("import_android_class").get_log() IMPORT_DATA_SOURCE_TABLE_NAME = "androidAPI_class" package_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session=session, name="androidAPI_package") class_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session=session, name=IMPORT_DATA_SOURCE_TABLE_NAME) api_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session=session, name=APIEntity.__tablename__) def get_package_full_name_by_old_package_id(package_id): return KnowledgeTableRowMapRecord.get_end_row_id( session=session, start_knowledge_table=package_knowledge_table, end_knowledge_table=api_knowledge_table, start_row_id=package_id) def get_qualified_name_of_package(package_id): api_entity_id = get_package_full_name_by_old_package_id( package_id=package_id) api_entity = APIEntity.find_by_id(session, api_entity_id) if api_entity:
from db.engine_factory import EngineFactory from db.cursor_factory import ConnectionFactory from db.model import KnowledgeTableRowMapRecord, APIDocumentWebsite, KnowledgeTableColumnMapRecord from db.model_factory import KnowledgeTableFactory from shared.logger_util import Logger logger = Logger("import_doc_website_for_jdk_package").get_log() cur = ConnectionFactory.create_cursor_for_jdk_importer() session = EngineFactory.create_session() jdk_package_knowledge_table = KnowledgeTableFactory.get_jdk_package_table( session) api_knowledge_table = KnowledgeTableFactory.get_api_entity_table(session) api_document_website_table = KnowledgeTableFactory.get_api_document_website_table( session) def create_doc_website_relation(old_package_id, doc_website): if doc_website is None: logger.error("no doc_website for %d", old_package_id) return None new_package_api_entity_id = KnowledgeTableRowMapRecord.get_end_row_id( session=session, start_knowledge_table=jdk_package_knowledge_table, end_knowledge_table=api_knowledge_table, start_row_id=old_package_id) if new_package_api_entity_id is None: logger.error("no new_package_api_entity_id for %d", old_package_id) return None
from db.engine_factory import ConnectionFactory, EngineFactory from db.model import KnowledgeTableRowMapRecord, APIEntity, parse_api_type_string_to_api_type_constant from db.model_factory import KnowledgeTableFactory from db.util.code_text_process import clean_declaration_html, clean_html_text_with_format, \ parse_declaration_html_with_full_qualified_type from shared.logger_util import Logger session = EngineFactory.create_session() IMPORT_DATA_SOURCE_TABLE_NAME = "jdk_method" logger = Logger("import_" + IMPORT_DATA_SOURCE_TABLE_NAME).get_log() class_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session=session, name="jdk_class") method_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session=session, name=IMPORT_DATA_SOURCE_TABLE_NAME) api_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session=session, name=APIEntity.__tablename__) def get_api_entity_id_by_old_class_id(old_class_id): return KnowledgeTableRowMapRecord.get_end_row_id( session=session, start_knowledge_table=class_knowledge_table, end_knowledge_table=api_knowledge_table, start_row_id=old_class_id) def get_qualified_name_of_class(old_class_id): api_entity_id = get_api_entity_id_by_old_class_id( old_class_id=old_class_id)
from db.engine_factory import ConnectionFactory, EngineFactory from db.model import KnowledgeTableRowMapRecord, APIRelation, KnowledgeTableColumnMapRecord, APIEntity from db.model_factory import KnowledgeTableFactory from shared.logger_util import Logger logger = Logger("import_belong_to_relation_for_android_class").get_log() cur = ConnectionFactory.create_cursor_for_android_importer() session = EngineFactory.create_session() android_package_knowledge_table = KnowledgeTableFactory.get_android_package_table(session) android_class_knowledge_table = KnowledgeTableFactory.get_android_class_table(session) api_knowledge_table = KnowledgeTableFactory.get_api_entity_table(session) api_relation_table = KnowledgeTableFactory.get_api_relation_table(session) def create_class_belong_to_relation(old_class_id, old_package_id): if old_package_id is None: logger.error("no old_package_id for %d", old_class_id) return None new_class_api_entity_id = KnowledgeTableRowMapRecord.get_end_row_id(session=session, start_knowledge_table=android_class_knowledge_table, end_knowledge_table=api_knowledge_table, start_row_id=old_class_id) if new_class_api_entity_id is None: logger.error("no new_class_api_entity_id for %d", old_class_id) return None new_package_api_entity_id = KnowledgeTableRowMapRecord.get_end_row_id(session=session, start_knowledge_table=android_package_knowledge_table, end_knowledge_table=api_knowledge_table, start_row_id=old_package_id)
from db.engine_factory import EngineFactory from db.cursor_factory import ConnectionFactory from db.model import KnowledgeTableRowMapRecord, APIBelongToLibraryRelation from db.model_factory import KnowledgeTableFactory from shared.logger_util import Logger logger = Logger("import_belong_to_relation_for_android_package").get_log() cur = ConnectionFactory.create_cursor_for_android_importer() session = EngineFactory.create_session() android_package_knowledge_table = KnowledgeTableFactory.get_android_package_table( session) library_entity_knowledge_table = KnowledgeTableFactory.get_library_entity_table( session) api_knowledge_table = KnowledgeTableFactory.get_api_entity_table(session) api_belong_to_table = KnowledgeTableFactory.get_api_belong_to_library_relation_table( session) COMMIT_STEP = 5000 def create_package_belong_to_relation(old_package_id, new_library_entity_id): if new_library_entity_id is None: logger.error("no new_library_entity_id for %d", new_library_entity_id) return None new_package_api_entity_id = KnowledgeTableRowMapRecord.get_end_row_id( session=session, start_knowledge_table=android_package_knowledge_table,