def import_all_android_package(): jdk_package_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name(session, "androidAPI_support_package") api_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name(session, APIEntity.__tablename__) # get all-version library cur.execute("select * from androidAPI_support_package") package_data_list = cur.fetchall() result_tuples = [] for i in range(0, len(package_data_list)): package_node, old_id = create_package_node(package_data_list[i]) if KnowledgeTableRowMapRecord.exist_import_record(session, jdk_package_knowledge_table, api_knowledge_table, old_id): print old_id, " has been import to new table" else: package_node = package_node.find_or_create(session, autocommit=False) result_tuples.append((package_node, old_id)) session.commit() for item in result_tuples: (package_node, old_id) = item record = KnowledgeTableRowMapRecord(jdk_package_knowledge_table, api_knowledge_table, old_id, package_node.id) record.create(session, autocommit=False) session.commit() "reading from mysql completed!"
def import_all_jdk_class(): jdk_class_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name(session, "jdk_class") api_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name(session, APIEntity.__tablename__) # get all-version library cur.execute("select * from jdk_class") data_list = cur.fetchall() result_tuples = [] for i in range(0, len(data_list)): class_node, old_id = create_class_node(data_list[i]) if class_node is None: print "unkonwn api type", data_list[i] continue if KnowledgeTableRowMapRecord.exist_import_record(session, jdk_class_knowledge_table, api_knowledge_table, old_id): print old_id, " has been import to new table" else: class_node = class_node.find_or_create(session, autocommit=False) result_tuples.append((class_node, old_id)) session.commit() for item in result_tuples: (class_node, old_id) = item record = KnowledgeTableRowMapRecord(jdk_class_knowledge_table, api_knowledge_table, old_id, class_node.id) record.create(session, autocommit=False) session.commit() "reading from mysql completed!"
def import_all_jdk_version(): jdk_library_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session, "jdk_library") library_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session, LibraryEntity.__tablename__) # get all-version library cur.execute("select * from jdk_library") lib_sql_data_list = cur.fetchall() result_tuples = [] for i in range(0, len(lib_sql_data_list)): lib_node, old_id = create_library_node(lib_sql_data_list[i]) if KnowledgeTableRowMapRecord.exist_import_record( session, jdk_library_knowledge_table, library_knowledge_table, old_id): print old_id, " has been import to new table" else: lib_node = lib_node.find_or_create(session, autocommit=False) result_tuples.append((lib_node, old_id)) session.commit() for item in result_tuples: (lib_node, old_id) = item record = KnowledgeTableRowMapRecord(jdk_library_knowledge_table, library_knowledge_table, old_id, lib_node.id) record.create(session, autocommit=False) session.commit() "reading from mysql completed!"
def import_doc_from_api_html_table(self): # step 2: api html->doc session = self.session api_html_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session, APIHTMLText.__tablename__) html_text_list = session.query(APIHTMLText).filter_by( html_type=APIHTMLText.HTML_TYPE_API_DETAIL_DESCRIPTION).all() result_tuples = [] for html_text in html_text_list: if DocumentSourceRecord.exist_import_record( session, api_html_knowledge_table.id, html_text.id): print("%d has been import to new table", (html_text.id, )) continue clean_text = html_text.clean_text if clean_text is None or clean_text.strip() == "": continue doc_text = DocumentText(html_text_id=html_text.id, text=clean_text) # text with no html tags doc_text.create(session, autocommit=False) result_tuples.append((doc_text, html_text.id)) session.commit() for doc_text, html_text_id in result_tuples: record = DocumentSourceRecord( doc_id=doc_text.id, kg_table_id=api_html_knowledge_table.id, kg_table_primary_key=html_text_id) record.create(session, autocommit=False) session.commit() print("import clean text from html table complete import")
from bs4 import BeautifulSoup from db.engine_factory import EngineFactory from db.cursor_factory import ConnectionFactory from db.model import KnowledgeTableRowMapRecord, APIEntity from db.model_factory import KnowledgeTableFactory from db.util.code_text_process import parse_declaration_html_with_full_qualified_type, clean_declaration_html, \ clean_html_text_with_format from shared.logger_util import Logger session = EngineFactory.create_session() logger = Logger("import_android_method").get_log() IMPORT_DATA_SOURCE_TABLE_NAME = "androidAPI_method" class_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session=session, name="androidAPI_class") method_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session=session, name="androidAPI_method") api_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session=session, name=APIEntity.__tablename__) def get_api_entity_id_by_old_class_id(old_class_id): return KnowledgeTableRowMapRecord.get_end_row_id( session=session, start_knowledge_table=class_knowledge_table, end_knowledge_table=api_knowledge_table, start_row_id=old_class_id)
from db.engine_factory import ConnectionFactory, EngineFactory from db.model import KnowledgeTableRowMapRecord, APIEntity from db.model_factory import KnowledgeTableFactory from db.util.code_text_process import clean_html_text from shared.logger_util import Logger session = EngineFactory.create_session() logger = Logger("import_android_class").get_log() IMPORT_DATA_SOURCE_TABLE_NAME = "androidAPI_class" package_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session=session, name="androidAPI_package") class_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session=session, name=IMPORT_DATA_SOURCE_TABLE_NAME) api_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session=session, name=APIEntity.__tablename__) def get_package_full_name_by_old_package_id(package_id): return KnowledgeTableRowMapRecord.get_end_row_id( session=session, start_knowledge_table=package_knowledge_table, end_knowledge_table=api_knowledge_table, start_row_id=package_id) def get_qualified_name_of_package(package_id): api_entity_id = get_package_full_name_by_old_package_id( package_id=package_id) api_entity = APIEntity.find_by_id(session, api_entity_id) if api_entity:
from db.engine_factory import ConnectionFactory, EngineFactory from db.model import KnowledgeTableRowMapRecord, APIEntity, parse_api_type_string_to_api_type_constant from db.model_factory import KnowledgeTableFactory from db.util.code_text_process import clean_declaration_html, clean_html_text_with_format, \ parse_declaration_html_with_full_qualified_type from shared.logger_util import Logger session = EngineFactory.create_session() IMPORT_DATA_SOURCE_TABLE_NAME = "jdk_method" logger = Logger("import_" + IMPORT_DATA_SOURCE_TABLE_NAME).get_log() class_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session=session, name="jdk_class") method_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session=session, name=IMPORT_DATA_SOURCE_TABLE_NAME) api_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session=session, name=APIEntity.__tablename__) def get_api_entity_id_by_old_class_id(old_class_id): return KnowledgeTableRowMapRecord.get_end_row_id( session=session, start_knowledge_table=class_knowledge_table, end_knowledge_table=api_knowledge_table, start_row_id=old_class_id) def get_qualified_name_of_class(old_class_id): api_entity_id = get_api_entity_id_by_old_class_id( old_class_id=old_class_id)