def import_android_return_value(cur, session):
return_value_data = read_android_return_value_data(cur)
for each in return_value_data:
print(each)
return_value_name = each[0]
return_value_text = clean_html_text(each[1])
return_value_url = each[2]
# print(return_value_name)
# print(return_value_text)
# print(return_value_url)
qualified_name = ""
if return_value_url != "":
qualified_name_list = return_value_url.replace(
"https://developer.android.google.cn/reference/",
"").replace(".html", "").split("/")
for i in range(0, len(qualified_name_list)):
if i == 0:
qualified_name += qualified_name_list[i]
else:
qualified_name += ("." + qualified_name_list[i])
else:
qualified_name = return_value_name
qualified_name = qualified_name + " (R)"
print(qualified_name)
api_entity = APIEntity(qualified_name, APIEntity.API_TYPE_RETURN_VALUE,
return_value_name, return_value_text)
# print(api_entity)
api_entity.find_or_create(session, autocommit=False)
session.commit()
def import_jdk_parameter_relation(cur, session):
parameter_data = read_parameter_data(cur)
for each in parameter_data:
if each is not None:
name = each[0]
method_id = each[1]
type_string = each[2]
description = each[3]
type_class = each[4]
full_declaration = type_string + " " + name
description = clean_html_text(description)
parameter_entity = APIEntity.find_by_full_declaration_and_description(session, full_declaration, description)
jdk_method_knowledge_table = KnowledgeTableFactory.get_jdk_method_table(session)
api_knowledge_table = KnowledgeTableFactory.get_api_entity_table(session)
method_entity_id = KnowledgeTableRowMapRecord.get_end_row_id(session, jdk_method_knowledge_table, api_knowledge_table, method_id)
if parameter_entity is not None and method_entity_id is not None:
end_api_id = parameter_entity.id
api_relation_has_parameter = APIRelation(method_entity_id, end_api_id, APIRelation.RELATION_TYPE_HAS_PARAMETER)
print("------------------------")
print(api_relation_has_parameter)
api_relation_has_parameter.find_or_create(session, autocommit=False)
if type_class == 0:
qualified_class_name = type_string
else:
qualified_class_name = get_qualified_class_name(type_class, cur)
if qualified_class_name is not None and parameter_entity is not None:
class_entity = APIEntity.find_by_qualifier(session, qualified_class_name)
if class_entity is not None:
api_relation_has_type = APIRelation(parameter_entity.id, class_entity.id, APIRelation.RELATION_TYPE_PARAMETER_HAS_TYPE)
print("=============================")
print(api_relation_has_type)
api_relation_has_type.find_or_create(session, autocommit=False)
session.commit()
def transfer_belong_to_type(self, start_api_id):
if start_api_id is not None:
start_api_entity = APIEntity.find_by_id(self.session, start_api_id)
if start_api_entity is not None:
start_api_type = start_api_entity.api_type
type_str = APIEntity.get_simple_type_string(start_api_type)
type_str = type_str.replace("api", "")
relation_str = "has" + type_str
return relation_str
return None
def import_jdk_return_value_relation(cur, session):
return_value_data = read_return_value_data(cur)
total = 0
type1 = 0
type2 = 0
for each in return_value_data:
total += 1
method_id = each[0]
full_declaration = each[1]
return_type = each[2]
return_string = each[3]
return_type = process_return_type_string(return_type)
qualified_name = construct_qualified_name_by_full_declaration(full_declaration, return_type)
print("****************")
print(method_id)
print(return_type)
print(full_declaration)
print(qualified_name)
if qualified_name is None:
qualified_name = return_type
return_string = clean_html_text(return_string)
parameter_entity = APIEntity.find_by_full_declaration_and_description(session, return_type, return_string)
jdk_method_knowledge_table = KnowledgeTableFactory.get_jdk_method_table(session)
api_knowledge_table = KnowledgeTableFactory.get_api_entity_table(session)
method_entity_id = KnowledgeTableRowMapRecord.get_end_row_id(session, jdk_method_knowledge_table,
api_knowledge_table, method_id)
if parameter_entity is not None and method_entity_id is not None:
end_api_id = parameter_entity.id
api_relation_has_return_value = APIRelation(method_entity_id, end_api_id,
APIRelation.RELATION_TYPE_HAS_RETURN_VALUE)
api_relation_has_return_value.find_or_create(session, autocommit=False)
print("------------------------")
print(api_relation_has_return_value)
type1 += 1
if qualified_name is not None and parameter_entity is not None:
print("+++++++++++++++++++++")
print(qualified_name)
class_entity = APIEntity.find_by_qualifier(session, qualified_name)
if class_entity is not None:
api_relation_has_type = APIRelation(parameter_entity.id, class_entity.id,
APIRelation.RELATION_TYPE_RETURN_VALUE_HAS_TYPE)
api_relation_has_type.find_or_create(session, autocommit=False)
print("=============================")
print(api_relation_has_type)
type2 += 1
session.commit()
print("total: " + str(total) + ", type1: " + str(type1) + ", type2: " + str(type2))
def construct_android_method_url(api_entity):
url = ""
if api_entity is not None:
class_id = APIRelation.get_end_id_by_start_id_and_relation_type(session, api_entity.id,
APIRelation.RELATION_TYPE_BELONG_TO)[0]
qualified_method_name = api_entity.qualified_name
if class_id is not None and qualified_method_name is not None:
class_document_website_list = APIDocumentWebsite.get_document_website_list_by_api_id(session, class_id)
class_document_website = ""
if class_document_website_list is not None:
for each in class_document_website_list:
print "-----------------------------"
print each[0]
website = each[0]
if "https://developer.android.com/reference/" in website:
class_document_website = website
break
parent = APIEntity.find_by_id(session, class_id)
qualified_class_name = parent.qualified_name
method_name = qualified_method_name.replace(qualified_class_name, "", 1)
method_name = method_name[1:]
if "," in method_name:
method_name = method_name.replace(",", ",%20")
url = class_document_website + "#" + method_name
print url
return url
def create_class_extends_relation(old_class_id, parent_class_name):
if parent_class_name is None or parent_class_name == "null" or parent_class_name.strip() == "":
logger.error("no old_extends_parent_class_id for %d", old_class_id)
return None
parent_class_name = parent_class_name.strip()
parent_class_entity = APIEntity.find_by_qualifier(session=session, qualified_name=parent_class_name)
if parent_class_entity is None:
logger.error("no parent_class_entity for %s", parent_class_name)
return None
new_class_api_entity_id = KnowledgeTableRowMapRecord.get_end_row_id(session=session,
start_knowledge_table=android_class_knowledge_table,
end_knowledge_table=api_knowledge_table,
start_row_id=old_class_id)
if new_class_api_entity_id is None:
logger.error("no new_class_api_entity_id for %d", old_class_id)
return None
new_extended_class_api_entity_id = parent_class_entity.id
if new_extended_class_api_entity_id is None:
logger.error("no new_extended_class_api_entity_id ")
return None
relation = APIRelation(new_class_api_entity_id, new_extended_class_api_entity_id, APIRelation.RELATION_TYPE_EXTENDS)
logger.info("%d extends to %d", new_class_api_entity_id, new_extended_class_api_entity_id)
return relation
def import_parameter_has_parameter_relation(cur, session):
parameter_data_from_knowledgegraph = read_parameter_data_from_knowledgegraph(
cur)
for each in parameter_data_from_knowledgegraph:
method_id = each[0]
full_declaration = each[2]
simple_parameter_list = get_simple_parameter_list(full_declaration)
if simple_parameter_list is not None:
for i in range(0, len(simple_parameter_list)):
simple_parameter_list[i] = simple_parameter_list[i].replace(
"[]", "").replace("...", "").strip()
parameter_entity = APIEntity.find_by_full_declaration_and_description(
session, simple_parameter_list[i], None)
api_knowledge_table = KnowledgeTableFactory.get_api_entity_table(
session)
android_method_table = KnowledgeTableFactory.get_android_method_table(
session)
method_entity_id = KnowledgeTableRowMapRecord.get_end_row_id(
session, android_method_table, api_knowledge_table,
method_id)
if parameter_entity is not None and method_entity_id is not None:
api_relation_has_parameter = APIRelation(
method_entity_id, parameter_entity.id,
APIRelation.RELATION_TYPE_HAS_PARAMETER)
api_relation_has_parameter.find_or_create(session,
autocommit=False)
session.commit()
def construct_android_class_ur(api_entity):
url = ""
url_prefix = "https://developer.android.com/reference/"
qualified_class_name = api_entity.qualified_name
package_id = APIRelation.get_end_id_by_start_id_and_relation_type(
session, api_entity.id, APIRelation.RELATION_TYPE_BELONG_TO)[0]
if qualified_class_name is not None and package_id is not None:
print package_id
parent = APIEntity.find_by_id(session, package_id)
qualified_package_name = parent.qualified_name
if qualified_package_name is not None:
print "-----------------------------------"
print qualified_package_name, " ", qualified_class_name
class_name = qualified_class_name.replace(qualified_package_name,
"")
class_name = class_name[1:]
url += url_prefix
if "." in qualified_package_name:
name_list = qualified_package_name.split(".")
else:
name_list = [qualified_package_name]
for name in name_list:
url += (name + "/")
url += class_name
print url
return url
def create_class_node(row_data):
old_id = row_data[0]
qualified_name = row_data[1]
short_description = row_data[3]
first_version = row_data[6]
api_type_string = row_data[8]
api_type = APIEntity.type_string_to_api_type_constant(api_type_string)
if api_type == APIEntity.API_TYPE_UNKNOWN:
print "---------------xxxxxxx ", api_type, api_type_string, row_data
return None, 0
api_entity = APIEntity(qualified_name=qualified_name, api_type=api_type,
full_declaration=None, added_in_version=first_version,
short_description=short_description)
return api_entity, old_id
def get_property_from_api_entity(end_row_id):
api_entity_data = APIEntity.find_by_id(session=session,
api_entity_id=end_row_id)
if api_entity_data is not None:
return api_entity_data.qualified_name, api_entity_data.full_declaration
else:
return None, None
def import_jdk_exception_condition(cur, session):
exception_condition_data = read_exception_condition_data(cur)
for each in exception_condition_data:
name = each[0]
description = each[1]
qualified_name = get_qualified_name_by_simple_name(name, cur)
if qualified_name is not None:
qualified_name = qualified_name + ' (E)'
else:
qualified_name = name + ' (E)'
print(qualified_name)
description = clean_html_text(description)
api_entity = APIEntity(qualified_name,
APIEntity.API_TYPE_EXCEPTION_CONDITION, name,
description)
api_entity.find_or_create(session, autocommit=False)
session.commit()
def get_qualified_name_of_package(package_id):
api_entity_id = get_package_full_name_by_old_package_id(
package_id=package_id)
api_entity = APIEntity.find_by_id(session, api_entity_id)
if api_entity:
return api_entity.qualified_name
else:
return None
def import_jdk_return_value(cur, session):
return_value_data = get_return_value_data(cur)
for each in return_value_data:
full_declaration = each[0]
return_type = each[1]
return_string = each[2]
print(return_type)
return_type = process_return_type_string(return_type)
qualified_name = construct_qualified_name_by_full_declaration(full_declaration, return_type)
if qualified_name is not None:
qualified_name = qualified_name + " (R)"
else:
qualified_name = return_type + " (R)"
return_string = clean_html_text(return_string)
api_entity = APIEntity(qualified_name, APIEntity.API_TYPE_RETURN_VALUE, return_type, return_string)
api_entity.find_or_create(session, autocommit=False)
session.commit()
def get_qualified_name_of_class(old_class_id):
api_entity_id = get_api_entity_id_by_old_class_id(
old_class_id=old_class_id)
api_entity = APIEntity.find_by_id(session, api_entity_id)
if api_entity:
return api_entity.qualified_name
else:
return None
def create_method_node(row_data):
api_method_id = row_data[0]
api_method_simple_name = row_data[1]
api_method_first_version = row_data[8]
api_method_description_html = row_data[2]
api_method_declaration_html = get_declaration_html_of_method(
method_id=api_method_id)
class_id = row_data[10]
api_type_string_in_table = row_data[7]
api_type_string_in_table = api_type_string_in_table.split(" ")[-1]
api_type = get_api_type(api_type_string_in_table)
full_declaration = None
if api_method_declaration_html:
full_declaration = clean_declaration_html(api_method_declaration_html)
if full_declaration is None:
return None, 0
short_description = None
if api_method_description_html:
short_description = clean_html_text_with_format(
api_method_description_html)
short_description = short_description.strip()
if not short_description:
short_description = None
first_version = None
if api_method_first_version:
first_version = api_method_first_version
if api_type == APIEntity.API_TYPE_UNKNOWN:
return None, 0
qualified_name = generate_qualified_name(
api_type=api_type,
simple_method_name=api_method_simple_name,
api_full_declaration_html=api_method_declaration_html,
parent_class_id=class_id)
if qualified_name is None:
logger.error("qualified_name is None")
return None, 0
api_entity = APIEntity(qualified_name=qualified_name,
api_type=api_type,
full_declaration=full_declaration,
added_in_version=first_version,
short_description=short_description)
logger.info(row_data)
logger.info(qualified_name)
logger.info(api_type)
logger.info(full_declaration)
logger.info(first_version)
logger.info(short_description)
return api_entity, api_method_id
def one_name(self, generator, qualifier_name, right_answer, right_type):
aliases = generator.generate_aliases(
APIEntity(qualified_name=qualifier_name, api_type=APIEntity.API_TYPE_METHOD))
print aliases
if right_answer is not None:
self.assertEqual(aliases[0].alias,
right_answer)
self.assertEqual(aliases[0].type, right_type)
else:
self.assertEqual(aliases, [])
def create_package_node(row_data):
package_id = row_data[0]
package_full_name = row_data[1]
package_first_version = row_data[2]
package_short_description = row_data[3]
package_entity = APIEntity(qualified_name=package_full_name, api_type=APIEntity.API_TYPE_PACKAGE,
full_declaration=package_full_name, added_in_version=package_first_version,
short_description=package_short_description)
return package_entity, package_id
def test_get_api_relation(self):
session = EngineFactory.create_session()
api = APIEntity.find_by_id(session, 462)
api=APIEntity.find_by_qualifier(session,"java.lang.String")
print "api =", api
print api.document_websites
print "out_relation"
for r in api.out_relation:
print r
print "start_api=", r.start_api
print "end_api=", r.end_api
print "------------"
print "in_relation"
for r in api.in_relation:
print r
print "start_api=", r.start_api
print "end_api=", r.end_api
print "------------"
def import_android_doc_website_for_method():
method_id_list = get_android_method_data()
if method_id_list is not None and len(method_id_list) > 0:
for method_id in method_id_list:
api_entity = APIEntity.find_by_id(session, method_id)
if api_entity is not None:
url = construct_android_method_url(api_entity)
if url is not None and url is not "":
api_document_website = APIDocumentWebsite(api_entity.id, url)
api_document_website.find_or_create(session, autocommit=False)
session.commit()
def import_android_doc_website_for_package():
package_id_list = get_android_package_data()
if package_id_list is not None and len(package_id_list) > 0:
for package_id in package_id_list:
api_entity = APIEntity.find_by_id(session, package_id)
url = construct_android_package_url(api_entity)
print url
if api_entity is not None and url is not "":
api_document_website = APIDocumentWebsite(api_entity.id, url)
print api_document_website
api_document_website.find_or_create(session=session, autocommit=False)
session.commit()
def update_jdk_annotation_method():
id_list = get_data_id_list()
for each in id_list:
start_row_id = each[0]
end_row_id = each[1]
qualified_name = check_and_modify(start_row_id)
if qualified_name is not None:
print start_row_id, " ", qualified_name
api_entity = APIEntity.find_by_id(session=session,
api_entity_id=end_row_id)
api_entity.qualified_name = qualified_name
session.commit()
def import_android_doc_website_for_class():
class_id_list = get_android_class_data()
if class_id_list is not None and len(class_id_list) > 0:
for class_id in class_id_list:
api_entity = APIEntity.find_by_id(session, class_id)
if api_entity is not None:
url = construct_android_class_ur(api_entity)
if url is not None and url != "":
api_document_website = APIDocumentWebsite(
api_entity.id, url)
api_document_website.find_or_create(session,
autocommit=False)
session.commit()
def get_api_type_from_full_declaration(full_declaration,
api_type_string_in_table):
api_type = APIEntity.type_string_to_api_type_constant(
api_type_string_in_table)
if api_type != APIEntity.API_TYPE_UNKNOWN:
return api_type
if not full_declaration:
return APIEntity.API_TYPE_UNKNOWN
api_type = APIEntity.API_TYPE_UNKNOWN
return api_type
def import_jdk_parameter(cur, session):
jdk_parameter_data = read_jdk_parameter_data(cur)
for each in jdk_parameter_data:
print(each)
name = each[0]
type_class = each[1]
type_string = each[2]
description = each[3]
qualified_class_name = get_qualified_class_name(type_class, cur)
print(qualified_class_name)
if qualified_class_name is not None:
qualified_name = qualified_class_name[0] + " " + name
else:
qualified_name = type_string + " " + name
full_declaration = type_string + " " + name
print(qualified_name)
print(full_declaration)
description = clean_html_text(description)
api_entity = APIEntity(qualified_name, APIEntity.API_TYPE_PARAMETER,
full_declaration, description)
api_entity.find_or_create(session, autocommit=False)
session.commit()
def import_android_exception_relation(cur, session):
total = 0
type1 = 0
type2 = 0
exception_data = read_exception_data(cur)
for each in exception_data:
total += 1
exception_name = each[0]
exception_text = each[1].replace("\n", '').replace(
" ",
'').replace(" ", '').replace(" ", '').replace(" ", '')
exception_entity = APIEntity.find_by_full_declaration_and_description(
session, exception_name, exception_text)
method_id = each[2]
api_knowledge_table = KnowledgeTableFactory.get_api_entity_table(
session)
android_method_table = KnowledgeTableFactory.get_android_method_table(
session)
method_entity_id = KnowledgeTableRowMapRecord.get_end_row_id(
session, android_method_table, api_knowledge_table, method_id)
exception_url = each[3]
qualified_name_list = exception_url.replace(
"https://developer.android.google.cn/reference/",
'').replace(".html", "").split("/")
qualified_name = ""
for i in range(0, len(qualified_name_list)):
if i == 0:
qualified_name += qualified_name_list[i]
else:
qualified_name += ("." + qualified_name_list[i])
print(qualified_name)
exception_class = session.query(APIEntity).filter_by(
qualified_name=qualified_name).first()
if exception_entity is not None and method_entity_id is not None:
api_relation_has_exception = APIRelation(
method_entity_id, exception_entity.id,
APIRelation.RELATION_TYPE_HAS_EXCEPTION)
api_relation_has_exception.find_or_create(session,
autocommit=False)
type1 += 1
if exception_entity is not None and exception_class is not None:
api_relation_has_type = APIRelation(
exception_entity.id, exception_class.id,
APIRelation.RELATION_TYPE_EXCEPTION_HAS_TYPE)
api_relation_has_type.find_or_create(session, autocommit=False)
type2 += 1
session.commit()
print(total)
print(type1)
print(type2)
def import_parameter_has_type_relation(session):
parameter_data_from_codehub = read_parameter_data_from_codehub(session)
for each in parameter_data_from_codehub:
# print(each.qualified_name)
qualified_type = each.qualified_name.split(" ")[0]
print(qualified_type)
type_entity = APIEntity.find_by_qualifier(session, qualified_type)
if type_entity is not None:
# print(type_entity.id)
api_relation_has_type = APIRelation(
each.id, type_entity.id,
APIRelation.RELATION_TYPE_PARAMETER_HAS_TYPE)
api_relation_has_type.find_or_create(session, autocommit=False)
session.commit()
def import_android_parameter(cur, session):
android_parameter_data = read_parameter_data(cur)
for each in android_parameter_data:
# print(each)
method_id = each[0]
method_name = each[1]
qualified_type_list = get_qualified_type_list(method_id, method_name, cur)
full_declaration = each[2]
simple_parameter_list = get_simple_parameter_list(full_declaration)
if qualified_type_list is not None and simple_parameter_list is not None and len(qualified_type_list) == len(simple_parameter_list):
for i in range(0, len(qualified_type_list)):
parameter_var = simple_parameter_list[i].split(" ")[-1]
qualified_type_list[i] += (" " + parameter_var)
qualified_type_list[i] = qualified_type_list[i].replace("[]", "").replace("...", "").strip()
simple_parameter_list[i] = simple_parameter_list[i].replace("[]", "").replace("...", "").strip()
if qualified_type_list[i] != "" and simple_parameter_list[i] != "":
print(qualified_type_list[i])
print(simple_parameter_list[i])
api_entity = APIEntity(qualified_type_list[i], APIEntity.API_TYPE_PARAMETER, simple_parameter_list[i])
api_entity.find_or_create(session, autocommit=False)
session.commit()
def import_android_exception_condition(cur, session):
exception_data = read_android_exception_data(cur)
for each in exception_data:
exception_name = each[0]
exception_text = each[1].replace("\n", '').replace(
" ",
'').replace(" ", '').replace(" ", '').replace(" ", '')
exception_url = each[2]
qualified_name_list = exception_url.replace(
"https://developer.android.google.cn/reference/",
'').replace(".html", "").split("/")
qualified_name = ""
for i in range(0, len(qualified_name_list)):
if i == 0:
qualified_name += qualified_name_list[i]
else:
qualified_name += ("." + qualified_name_list[i])
qualified_name += " (E)"
print(qualified_name)
api_entity = APIEntity(qualified_name,
APIEntity.API_TYPE_EXCEPTION_CONDITION,
exception_name, exception_text)
api_entity.find_or_create(session, autocommit=False)
session.commit()
def start_import(self):
self.session = EngineFactory.create_session()
html_type = APIHTMLText.HTML_TYPE_API_DETAIL_DESCRIPTION
api_entity_list = APIEntity.get_all_value_instance_api(
session=self.session)
for api_entity in api_entity_list:
description = api_entity.short_description
if description is None or description == "":
continue
api_html_entity = APIHTMLText(api_id=api_entity.id,
html=description,
html_type=html_type)
api_html_entity.find_or_create(session=self.session,
autocommit=False)
self.session.commit()
def import_jdk_exception_relation(cur, session):
jdk_exception_data = read_jdk_exception_data(cur)
simple_exception_name_list = get_simple_exception_name_list(session)
total = 0
type1 = 0
type2 = 0
for each in jdk_exception_data:
total += 1
name = each[0]
method_id = each[1]
description = each[2]
description = clean_html_text(description)
exception_entity = APIEntity.find_by_full_declaration_and_description(
session, name, description)
jdk_method_knowledge_table = KnowledgeTableFactory.get_jdk_method_table(
session)
api_knowledge_table = KnowledgeTableFactory.get_api_entity_table(
session)
method_entity_id = KnowledgeTableRowMapRecord.get_end_row_id(
session, jdk_method_knowledge_table, api_knowledge_table,
method_id)
if exception_entity is not None and method_entity_id is not None:
type1 += 1
api_relation_has_exception = APIRelation(
method_entity_id, exception_entity.id,
APIRelation.RELATION_TYPE_HAS_EXCEPTION)
api_relation_has_exception.find_or_create(session,
autocommit=False)
print("------------------------")
print(api_relation_has_exception)
if exception_entity is not None and name is not None:
type2 += 1
exception_id = get_exception_id_by_simple_name(
simple_exception_name_list, name)
if exception_id is not None:
api_relation_has_type = APIRelation(
exception_entity.id, exception_id,
APIRelation.RELATION_TYPE_EXCEPTION_HAS_TYPE)
api_relation_has_type.find_or_create(session, autocommit=False)
print("=============================")
print(api_relation_has_type)
session.commit()
print("total: " + str(total) + ", type1: " + str(type1) + ", type2: " +
str(type2))