def create_package_belong_to_relation(old_package_id, old_library_id): if old_library_id is None: logger.error("no old_library_id for %d", old_package_id) return None new_library_entity_id = KnowledgeTableRowMapRecord.get_end_row_id( session=session, start_knowledge_table=jdk_library_knowledge_table, end_knowledge_table=library_entity_knowledge_table, start_row_id=old_library_id) if new_library_entity_id is None: logger.error("no new_library_entity_id for %d", old_library_id) return None new_package_api_entity_id = KnowledgeTableRowMapRecord.get_end_row_id( session=session, start_knowledge_table=jdk_package_knowledge_table, end_knowledge_table=api_knowledge_table, start_row_id=old_package_id) if new_package_api_entity_id is None: logger.error("no new_package_api_entity_id for %d", old_package_id) return None relation = APIBelongToLibraryRelation(api_id=new_package_api_entity_id, library_id=new_library_entity_id) logger.info("%d belong to %d", new_package_api_entity_id, new_library_entity_id) return relation
def create_class_extends_relation(old_class_id, old_extended_class_id): if old_extended_class_id is None: logger.error("no old_extends_parent_class_id for %d", old_class_id) return None new_class_api_entity_id = KnowledgeTableRowMapRecord.get_end_row_id( session=session, start_knowledge_table=jdk_class_knowledge_table, end_knowledge_table=api_knowledge_table, start_row_id=old_class_id) if new_class_api_entity_id is None: logger.error("no new_class_api_entity_id for %d", old_class_id) return None new_extended_class_api_entity_id = KnowledgeTableRowMapRecord.get_end_row_id( session=session, start_knowledge_table=jdk_class_knowledge_table, end_knowledge_table=api_knowledge_table, start_row_id=old_extended_class_id) if new_extended_class_api_entity_id is None: logger.error("no new_extended_class_api_entity_id for %d", old_extended_class_id) return None relation = APIRelation(new_class_api_entity_id, new_extended_class_api_entity_id, APIRelation.RELATION_TYPE_EXTENDS) logger.info("%d extends to %d", new_class_api_entity_id, new_extended_class_api_entity_id) return relation
def import_all_class(): # get all-version library cur.execute("select * from " + IMPORT_DATA_SOURCE_TABLE_NAME) data_list = cur.fetchall() result_tuples = [] for i in range(0, len(data_list)): class_node, old_id = create_class_node(data_list[i]) if class_node is None: logger.error(data_list[i]) logger.error("unkonwn api type") continue if KnowledgeTableRowMapRecord.exist_import_record( session, class_knowledge_table, api_knowledge_table, old_id): logger.info("%d has been import to new table", old_id) else: class_node = class_node.find_or_create(session, autocommit=False) result_tuples.append((class_node, old_id)) session.commit() for item in result_tuples: (class_node, old_id) = item record = KnowledgeTableRowMapRecord(class_knowledge_table, api_knowledge_table, old_id, class_node.id) record.create(session, autocommit=False) session.commit() "reading from mysql completed!"
def import_all_jdk_version(): jdk_library_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session, "jdk_library") library_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name( session, LibraryEntity.__tablename__) # get all-version library cur.execute("select * from jdk_library") lib_sql_data_list = cur.fetchall() result_tuples = [] for i in range(0, len(lib_sql_data_list)): lib_node, old_id = create_library_node(lib_sql_data_list[i]) if KnowledgeTableRowMapRecord.exist_import_record( session, jdk_library_knowledge_table, library_knowledge_table, old_id): print old_id, " has been import to new table" else: lib_node = lib_node.find_or_create(session, autocommit=False) result_tuples.append((lib_node, old_id)) session.commit() for item in result_tuples: (lib_node, old_id) = item record = KnowledgeTableRowMapRecord(jdk_library_knowledge_table, library_knowledge_table, old_id, lib_node.id) record.create(session, autocommit=False) session.commit() "reading from mysql completed!"
def import_all_android_method(): # get all-version library cur.execute("select * from " + IMPORT_DATA_SOURCE_TABLE_NAME) data_list = cur.fetchall() result_tuples = [] for i in range(0, len(data_list)): method_node, old_id = create_method_node(data_list[i]) if method_node is None: logger.error(data_list[i]) logger.error("unkonwn api type") continue if KnowledgeTableRowMapRecord.exist_import_record( session, method_knowledge_table, api_knowledge_table, old_id): print old_id, " has been import to new table" else: method_node = method_node.find_or_create(session, autocommit=False) result_tuples.append((method_node, old_id)) session.commit() for item in result_tuples: (method_node, old_id) = item record = KnowledgeTableRowMapRecord(method_knowledge_table, api_knowledge_table, old_id, method_node.id) record.create(session, autocommit=False) session.commit() "complete import method"
def import_all_android_package(): jdk_package_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name(session, "androidAPI_support_package") api_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name(session, APIEntity.__tablename__) # get all-version library cur.execute("select * from androidAPI_support_package") package_data_list = cur.fetchall() result_tuples = [] for i in range(0, len(package_data_list)): package_node, old_id = create_package_node(package_data_list[i]) if KnowledgeTableRowMapRecord.exist_import_record(session, jdk_package_knowledge_table, api_knowledge_table, old_id): print old_id, " has been import to new table" else: package_node = package_node.find_or_create(session, autocommit=False) result_tuples.append((package_node, old_id)) session.commit() for item in result_tuples: (package_node, old_id) = item record = KnowledgeTableRowMapRecord(jdk_package_knowledge_table, api_knowledge_table, old_id, package_node.id) record.create(session, autocommit=False) session.commit() "reading from mysql completed!"
def create_method_belong_to_relation(old_method_id, old_class_id): logger.info("old_method_id=%d old_class_id=%d", old_method_id, old_class_id) if old_class_id is None: logger.error("no old_class_id for %d", old_method_id) return None new_method_api_id = KnowledgeTableRowMapRecord.get_end_row_id( session=session, start_knowledge_table=jdk_method_knowledge_table, end_knowledge_table=api_knowledge_table, start_row_id=old_method_id) if new_method_api_id is None: logger.error("no new_method_api_id for %d", old_method_id) return None new_class_api_entity_id = KnowledgeTableRowMapRecord.get_end_row_id( session=session, start_knowledge_table=jdk_class_knowledge_table, end_knowledge_table=api_knowledge_table, start_row_id=old_class_id) if new_class_api_entity_id is None: logger.error("no new_class_api_entity_id for %d", old_class_id) return None relation = APIRelation(new_method_api_id, new_class_api_entity_id, APIRelation.RELATION_TYPE_BELONG_TO) logger.info("%d belong to %d", new_method_api_id, new_class_api_entity_id) return relation
def import_all_jdk_class(): jdk_class_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name(session, "jdk_class") api_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name(session, APIEntity.__tablename__) # get all-version library cur.execute("select * from jdk_class") data_list = cur.fetchall() result_tuples = [] for i in range(0, len(data_list)): class_node, old_id = create_class_node(data_list[i]) if class_node is None: print "unkonwn api type", data_list[i] continue if KnowledgeTableRowMapRecord.exist_import_record(session, jdk_class_knowledge_table, api_knowledge_table, old_id): print old_id, " has been import to new table" else: class_node = class_node.find_or_create(session, autocommit=False) result_tuples.append((class_node, old_id)) session.commit() for item in result_tuples: (class_node, old_id) = item record = KnowledgeTableRowMapRecord(jdk_class_knowledge_table, api_knowledge_table, old_id, class_node.id) record.create(session, autocommit=False) session.commit() "reading from mysql completed!"
def import_parameter_has_parameter_relation(cur, session): parameter_data_from_knowledgegraph = read_parameter_data_from_knowledgegraph( cur) for each in parameter_data_from_knowledgegraph: method_id = each[0] full_declaration = each[2] simple_parameter_list = get_simple_parameter_list(full_declaration) if simple_parameter_list is not None: for i in range(0, len(simple_parameter_list)): simple_parameter_list[i] = simple_parameter_list[i].replace( "[]", "").replace("...", "").strip() parameter_entity = APIEntity.find_by_full_declaration_and_description( session, simple_parameter_list[i], None) api_knowledge_table = KnowledgeTableFactory.get_api_entity_table( session) android_method_table = KnowledgeTableFactory.get_android_method_table( session) method_entity_id = KnowledgeTableRowMapRecord.get_end_row_id( session, android_method_table, api_knowledge_table, method_id) if parameter_entity is not None and method_entity_id is not None: api_relation_has_parameter = APIRelation( method_entity_id, parameter_entity.id, APIRelation.RELATION_TYPE_HAS_PARAMETER) api_relation_has_parameter.find_or_create(session, autocommit=False) session.commit()
def create_class_extends_relation(old_class_id, parent_class_name): if parent_class_name is None or parent_class_name == "null" or parent_class_name.strip() == "": logger.error("no old_extends_parent_class_id for %d", old_class_id) return None parent_class_name = parent_class_name.strip() parent_class_entity = APIEntity.find_by_qualifier(session=session, qualified_name=parent_class_name) if parent_class_entity is None: logger.error("no parent_class_entity for %s", parent_class_name) return None new_class_api_entity_id = KnowledgeTableRowMapRecord.get_end_row_id(session=session, start_knowledge_table=android_class_knowledge_table, end_knowledge_table=api_knowledge_table, start_row_id=old_class_id) if new_class_api_entity_id is None: logger.error("no new_class_api_entity_id for %d", old_class_id) return None new_extended_class_api_entity_id = parent_class_entity.id if new_extended_class_api_entity_id is None: logger.error("no new_extended_class_api_entity_id ") return None relation = APIRelation(new_class_api_entity_id, new_extended_class_api_entity_id, APIRelation.RELATION_TYPE_EXTENDS) logger.info("%d extends to %d", new_class_api_entity_id, new_extended_class_api_entity_id) return relation
def import_jdk_parameter_relation(cur, session): parameter_data = read_parameter_data(cur) for each in parameter_data: if each is not None: name = each[0] method_id = each[1] type_string = each[2] description = each[3] type_class = each[4] full_declaration = type_string + " " + name description = clean_html_text(description) parameter_entity = APIEntity.find_by_full_declaration_and_description(session, full_declaration, description) jdk_method_knowledge_table = KnowledgeTableFactory.get_jdk_method_table(session) api_knowledge_table = KnowledgeTableFactory.get_api_entity_table(session) method_entity_id = KnowledgeTableRowMapRecord.get_end_row_id(session, jdk_method_knowledge_table, api_knowledge_table, method_id) if parameter_entity is not None and method_entity_id is not None: end_api_id = parameter_entity.id api_relation_has_parameter = APIRelation(method_entity_id, end_api_id, APIRelation.RELATION_TYPE_HAS_PARAMETER) print("------------------------") print(api_relation_has_parameter) api_relation_has_parameter.find_or_create(session, autocommit=False) if type_class == 0: qualified_class_name = type_string else: qualified_class_name = get_qualified_class_name(type_class, cur) if qualified_class_name is not None and parameter_entity is not None: class_entity = APIEntity.find_by_qualifier(session, qualified_class_name) if class_entity is not None: api_relation_has_type = APIRelation(parameter_entity.id, class_entity.id, APIRelation.RELATION_TYPE_PARAMETER_HAS_TYPE) print("=============================") print(api_relation_has_type) api_relation_has_type.find_or_create(session, autocommit=False) session.commit()
def import_jdk_throw_exception_relation_for_method(): jdk_exception_data = get_jdk_exception_data() for each in jdk_exception_data: exception_name = each[0] original_method_id = each[1] print exception_name, " ", original_method_id start_api_id = KnowledgeTableRowMapRecord.get_end_row_id(session, jdk_method_knowledge_table, api_knowledge_table, original_method_id) print "start_api_id: ", start_api_id original_exception_id = get_class_id_by_exception_name(exception_name) if original_exception_id is not None: end_api_id = KnowledgeTableRowMapRecord.get_end_row_id(session, jdk_class_knowledge_table, api_knowledge_table, original_exception_id) if end_api_id is not None and start_api_id is not None: api_relation = APIRelation(start_api_id, end_api_id, APIRelation.RELATION_TYPE_THROW_EXCEPTION) print original_method_id, "------------------------------", api_relation api_relation.find_or_create(session, autocommit=False) session.commit()
def get_android_class_data(): class_id_list = [] android_class_data = KnowledgeTableRowMapRecord.get_transformed_table_data( session, android_class_knowledge_table, api_entity_knowledge_table) for each in android_class_data: # print each.end_row_id class_id_list.append(each.end_row_id) return class_id_list
def is_imported(row_data): old_id = row_data[0] if KnowledgeTableRowMapRecord.exist_import_record(session, method_knowledge_table, api_knowledge_table, old_id): return True else: return False
def get_android_package_data(): package_id_list = [] android_package_data = KnowledgeTableRowMapRecord.get_transformed_table_data(session, android_package_knowledge_table, api_entity_knowledge_table) if android_package_data is not None: for each in android_package_data: end_row_id = each.end_row_id print end_row_id package_id_list.append(end_row_id) # print android_package_data return package_id_list
def import_all_method(): # get all-version method cur.execute("select * from " + IMPORT_DATA_SOURCE_TABLE_NAME) data_list = cur.fetchall() result_tuples = [] for i in range(0, len(data_list)): row_data = data_list[i] try: import_one_row(result_tuples, row_data) except Exception: logger.error(row_data) logger.error("import one row fail") session.commit() for item in result_tuples: (method_node, old_id) = item record = KnowledgeTableRowMapRecord(method_knowledge_table, api_knowledge_table, old_id, method_node.id) record.create(session, autocommit=False) session.commit() logger.info("import from mysql completed!")
def import_android_exception_relation(cur, session): total = 0 type1 = 0 type2 = 0 exception_data = read_exception_data(cur) for each in exception_data: total += 1 exception_name = each[0] exception_text = each[1].replace("\n", '').replace( " ", '').replace(" ", '').replace(" ", '').replace(" ", '') exception_entity = APIEntity.find_by_full_declaration_and_description( session, exception_name, exception_text) method_id = each[2] api_knowledge_table = KnowledgeTableFactory.get_api_entity_table( session) android_method_table = KnowledgeTableFactory.get_android_method_table( session) method_entity_id = KnowledgeTableRowMapRecord.get_end_row_id( session, android_method_table, api_knowledge_table, method_id) exception_url = each[3] qualified_name_list = exception_url.replace( "https://developer.android.google.cn/reference/", '').replace(".html", "").split("/") qualified_name = "" for i in range(0, len(qualified_name_list)): if i == 0: qualified_name += qualified_name_list[i] else: qualified_name += ("." + qualified_name_list[i]) print(qualified_name) exception_class = session.query(APIEntity).filter_by( qualified_name=qualified_name).first() if exception_entity is not None and method_entity_id is not None: api_relation_has_exception = APIRelation( method_entity_id, exception_entity.id, APIRelation.RELATION_TYPE_HAS_EXCEPTION) api_relation_has_exception.find_or_create(session, autocommit=False) type1 += 1 if exception_entity is not None and exception_class is not None: api_relation_has_type = APIRelation( exception_entity.id, exception_class.id, APIRelation.RELATION_TYPE_EXCEPTION_HAS_TYPE) api_relation_has_type.find_or_create(session, autocommit=False) type2 += 1 session.commit() print(total) print(type1) print(type2)
def import_jdk_return_value_relation(cur, session): return_value_data = read_return_value_data(cur) total = 0 type1 = 0 type2 = 0 for each in return_value_data: total += 1 method_id = each[0] full_declaration = each[1] return_type = each[2] return_string = each[3] return_type = process_return_type_string(return_type) qualified_name = construct_qualified_name_by_full_declaration(full_declaration, return_type) print("****************") print(method_id) print(return_type) print(full_declaration) print(qualified_name) if qualified_name is None: qualified_name = return_type return_string = clean_html_text(return_string) parameter_entity = APIEntity.find_by_full_declaration_and_description(session, return_type, return_string) jdk_method_knowledge_table = KnowledgeTableFactory.get_jdk_method_table(session) api_knowledge_table = KnowledgeTableFactory.get_api_entity_table(session) method_entity_id = KnowledgeTableRowMapRecord.get_end_row_id(session, jdk_method_knowledge_table, api_knowledge_table, method_id) if parameter_entity is not None and method_entity_id is not None: end_api_id = parameter_entity.id api_relation_has_return_value = APIRelation(method_entity_id, end_api_id, APIRelation.RELATION_TYPE_HAS_RETURN_VALUE) api_relation_has_return_value.find_or_create(session, autocommit=False) print("------------------------") print(api_relation_has_return_value) type1 += 1 if qualified_name is not None and parameter_entity is not None: print("+++++++++++++++++++++") print(qualified_name) class_entity = APIEntity.find_by_qualifier(session, qualified_name) if class_entity is not None: api_relation_has_type = APIRelation(parameter_entity.id, class_entity.id, APIRelation.RELATION_TYPE_RETURN_VALUE_HAS_TYPE) api_relation_has_type.find_or_create(session, autocommit=False) print("=============================") print(api_relation_has_type) type2 += 1 session.commit() print("total: " + str(total) + ", type1: " + str(type1) + ", type2: " + str(type2))
def import_jdk_see_also_relation_for_method(): see_also_data = get_see_also_data() # print see_also_data for each in see_also_data: see_also_website = each[0] original_method_id = each[1] end_api_id = APIDocumentWebsite.get_api_id_by_website(session, see_also_website) # print see_also_website, " ", end_api_id start_api_id = KnowledgeTableRowMapRecord.get_end_row_id(session, jdk_method_knowledge_table, api_knowledge_table, original_method_id) # print start_api_id if end_api_id is not None and start_api_id is not None: api_relation = APIRelation(start_api_id, end_api_id, APIRelation.RELATION_TYPE_SEE_ALSO) api_relation.find_or_create(session, autocommit=False) print original_method_id, "-------------------------------", api_relation session.commit()
def create_doc_website_relation(old_class_id, doc_website): if doc_website is None: logger.error("no doc_website for %d", old_class_id) return None new_class_api_entity_id = KnowledgeTableRowMapRecord.get_end_row_id(session=session, start_knowledge_table=jdk_class_knowledge_table, end_knowledge_table=api_knowledge_table, start_row_id=old_class_id) if new_class_api_entity_id is None: logger.error("no new_class_api_entity_id for %d", old_class_id) return None relation = APIDocumentWebsite(new_class_api_entity_id, doc_website) logger.info("API %d : %s", new_class_api_entity_id, doc_website) return relation
def import_jdk_exception_relation(cur, session): jdk_exception_data = read_jdk_exception_data(cur) simple_exception_name_list = get_simple_exception_name_list(session) total = 0 type1 = 0 type2 = 0 for each in jdk_exception_data: total += 1 name = each[0] method_id = each[1] description = each[2] description = clean_html_text(description) exception_entity = APIEntity.find_by_full_declaration_and_description( session, name, description) jdk_method_knowledge_table = KnowledgeTableFactory.get_jdk_method_table( session) api_knowledge_table = KnowledgeTableFactory.get_api_entity_table( session) method_entity_id = KnowledgeTableRowMapRecord.get_end_row_id( session, jdk_method_knowledge_table, api_knowledge_table, method_id) if exception_entity is not None and method_entity_id is not None: type1 += 1 api_relation_has_exception = APIRelation( method_entity_id, exception_entity.id, APIRelation.RELATION_TYPE_HAS_EXCEPTION) api_relation_has_exception.find_or_create(session, autocommit=False) print("------------------------") print(api_relation_has_exception) if exception_entity is not None and name is not None: type2 += 1 exception_id = get_exception_id_by_simple_name( simple_exception_name_list, name) if exception_id is not None: api_relation_has_type = APIRelation( exception_entity.id, exception_id, APIRelation.RELATION_TYPE_EXCEPTION_HAS_TYPE) api_relation_has_type.find_or_create(session, autocommit=False) print("=============================") print(api_relation_has_type) session.commit() print("total: " + str(total) + ", type1: " + str(type1) + ", type2: " + str(type2))
def get_data_id_list(): id_list = [] print jdk_method_knowledge_table.id, " ", api_knowledge_table.id data_list = KnowledgeTableRowMapRecord.get_transformed_table_data( session=session, start_knowledge_table=jdk_method_knowledge_table, end_knowledge_table=api_knowledge_table) if data_list is None: logger.error("no data in %s", jdk_method_knowledge_table.table_name) return None for each in data_list: # print("end_row_id: " + str(each.end_row_id)) temp = [each.start_row_id, each.end_row_id] id_list.append(temp) del data_list gc.collect() return id_list
def import_jdk_relation_value_relation_for_method(): return_value_data = get_return_value_data() for each in return_value_data: original_method_id = each[0] full_declaration = each[1] return_type = each[2] start_api_id = KnowledgeTableRowMapRecord.get_end_row_id(session, jdk_method_knowledge_table, api_knowledge_table, original_method_id) qualified_name = construct_qualified_name_by_full_declaration(full_declaration, return_type) if start_api_id is not None and qualified_name is not None: print "-------------------------------------" print "start_api_id: ", start_api_id, ", qualified_name: ", qualified_name api_entity = APIEntity.find_by_qualifier(session, qualified_name) if api_entity is not None: end_api_id = api_entity.id print api_entity, " ", end_api_id api_relation = APIRelation(start_api_id, end_api_id, APIRelation.RELATION_TYPE_RETURN_VALUE) print api_relation api_relation.find_or_create(session, autocommit=False) session.commit()
def import_android_return_value_relation(cur, session): total = 0 type1 = 0 type2 = 0 return_value_data = read_android_return_value_data(cur) for each in return_value_data: total += 1 return_value_name = each[0] return_value_text = clean_html_text(each[1]) method_id = each[2] return_value_url = each[3] return_value_entity = APIEntity.find_by_full_declaration_and_description(session, return_value_name, return_value_text) api_knowledge_table = KnowledgeTableFactory.get_api_entity_table(session) android_method_table = KnowledgeTableFactory.get_android_method_table(session) method_entity_id = KnowledgeTableRowMapRecord.get_end_row_id(session, android_method_table, api_knowledge_table, method_id) qualified_name = "" if return_value_url != "": qualified_name_list = return_value_url.replace("https://developer.android.google.cn/reference/", "").replace(".html", "").split("/") for i in range(0, len(qualified_name_list)): if i == 0: qualified_name += qualified_name_list[i] else: qualified_name += ("." + qualified_name_list[i]) else: qualified_name = return_value_name return_value_class = session.query(APIEntity).filter_by(qualified_name=qualified_name).first() if return_value_entity is not None and method_entity_id is not None: type1 += 1 api_relation_has_return_value = APIRelation(method_entity_id, return_value_entity.id, APIRelation.RELATION_TYPE_HAS_RETURN_VALUE) api_relation_has_return_value.find_or_create(session, autocommit=False) if return_value_entity is not None and return_value_class is not None: type2 += 1 api_relation_has_type = APIRelation(return_value_entity.id, return_value_class.id, APIRelation.RELATION_TYPE_RETURN_VALUE_HAS_TYPE) api_relation_has_type.find_or_create(session, autocommit=False) session.commit() print(total) print(type1) print(type2)
def get_api_entity_id_by_old_class_id(old_class_id): return KnowledgeTableRowMapRecord.get_end_row_id( session=session, start_knowledge_table=class_knowledge_table, end_knowledge_table=api_knowledge_table, start_row_id=old_class_id)
def get_package_full_name_by_old_package_id(package_id): return KnowledgeTableRowMapRecord.get_end_row_id( session=session, start_knowledge_table=package_knowledge_table, end_knowledge_table=api_knowledge_table, start_row_id=package_id)
def get_android_method_data(): method_id_list = [] android_method_data = KnowledgeTableRowMapRecord.get_transformed_table_data(session, android_method_knowledge_table, api_entity_knowledge_table) for each in android_method_data: method_id_list.append(each.end_row_id) return method_id_list