class APIAliasesTableFuller:
ALL_API = -1
def __init__(self,
table=APIEntity.__tablename__,
session=None,
commit_step=5000):
self.table = table
self.commit_step = commit_step
self.logger_file_name = "generate_api_aliases"
self.logger = None
self.session = session
self.api_html_table = None
self.api_knowledge_table = None
self.data_source_knowledge_table = None
def get_session(self):
if not self.session:
self.session = EngineFactory.create_session()
return self.session
def start_fix_exist_aliases_for_aliases_type(self, alias_type):
pass
self.logger = Logger(self.logger_file_name).get_log()
self.logger.info("-----------------------start-----------------------")
self.logger.info("generate api aliases for alias type=%d", alias_type)
session = self.get_session()
def start_generate_aliases_for_api_type(self,
alias_type,
api_type=ALL_API):
self.logger = Logger(self.logger_file_name).get_log()
self.logger.info("-----------------------start-----------------------")
self.logger.info("generate api aliases for api type=%d alias type=%d",
api_type, alias_type)
session = self.get_session()
api_alias_generator = APIAliasGeneratorFactory.create_generator(
alias_type)
if api_alias_generator is None:
self.logger.error("Not Implemented Generator for %d ", alias_type)
return
if api_type == APIAliasesTableFuller.ALL_API:
count_api = session.query(APIEntity).count()
api_list_query = session.query(APIEntity)
else:
count_api = session.query(APIEntity).filter(
APIEntity.api_type == api_type).count()
api_list_query = session.query(APIEntity).filter(
APIEntity.api_type == api_type)
start_index_list = range(0, count_api, self.commit_step)
for start_index in start_index_list:
end_index = min(start_index + self.commit_step, count_api)
for api in api_list_query.all()[start_index:end_index]:
try:
api_alias_list = api_alias_generator.generate_aliases(api)
self.add_aliases_to_api_entity(api, api_alias_list,
session)
except Exception:
traceback.print_exc()
self.logger.info("complete %d-%d", start_index, end_index)
session.commit()
self.logger.info("complete all")
def start_generate_aliases_for_multi_api_type(self, alias_type,
*api_type_list):
for api_type in api_type_list:
self.start_generate_aliases_for_api_type(alias_type=alias_type,
api_type=api_type)
def add_aliases_to_api_entity(self, api, api_alias_list, session):
for api_alias in api_alias_list:
api_alias = api_alias.find_or_create(session=session,
autocommit=False)
self.logger.info(api)
self.logger.info(api_alias)
if api_alias not in api.all_aliases:
api.all_aliases.append(api_alias)
def clean_table(self):
APIAlias.delete_all(session=self.session)
def start_add_all_api_aliases(self):
self.clean_table()
self.start_generate_aliases_for_api_type(
APIAlias.ALIAS_TYPE_QUALIFIER_NAME)
self.start_generate_aliases_for_api_type(
APIAlias.ALIAS_TYPE_SIMPLE_NAME)
# fuller.start_generate_aliases_for_api_type(APIAlias.ALIAS_TYPE_SIMPLE_NAME_WITH_TYPE)
self.start_generate_aliases_for_multi_api_type(
APIAlias.ALIAS_TYPE_SIMPLE_METHOD_WITH_QUALIFIER_PARAMETER_TYPE,
APIEntity.API_TYPE_METHOD, APIEntity.API_TYPE_CONSTRUCTOR)
self.start_generate_aliases_for_multi_api_type(
APIAlias.
ALIAS_TYPE_SIMPLE_CLASS_NAME_METHOD_WITH_QUALIFIER_PARAMETER_TYPE,
APIEntity.API_TYPE_CONSTRUCTOR, APIEntity.API_TYPE_METHOD)
self.start_generate_aliases_for_multi_api_type(
APIAlias.ALIAS_TYPE_SIMPLE_NAME_METHOD_WITH_SIMPLE_PARAMETER_TYPE,
APIEntity.API_TYPE_METHOD, APIEntity.API_TYPE_CONSTRUCTOR)
self.start_generate_aliases_for_multi_api_type(
APIAlias.
ALIAS_TYPE_SIMPLE_CLASS_NAME_METHOD_WITH_SIMPLE_PARAMETER_TYPE,
APIEntity.API_TYPE_METHOD, APIEntity.API_TYPE_CONSTRUCTOR)
self.start_generate_aliases_for_multi_api_type(
APIAlias.ALIAS_TYPE_SIMPLE_PARENT_API_NAME_WITH_SIMPLE_NAME,
APIEntity.API_TYPE_ENUM_CONSTANTS, APIEntity.API_TYPE_FIELD,
APIEntity.API_TYPE_INTERFACE, APIEntity.API_TYPE_EXCEPTION,
APIEntity.API_TYPE_PACKAGE, APIEntity.API_TYPE_ANNOTATION,
APIEntity.API_TYPE_ERROR)
self.start_generate_aliases_for_api_type(
APIAlias.ALIAS_TYPE_ANNOTATION_REFERENCE,
APIEntity.API_TYPE_ANNOTATION)
self.start_generate_aliases_for_multi_api_type(
APIAlias.ALIAS_TYPE_CAMEL_CASE_TO_SPACE, APIEntity.API_TYPE_CLASS,
APIEntity.API_TYPE_INTERFACE, APIEntity.API_TYPE_EXCEPTION,
APIEntity.API_TYPE_ERROR, APIEntity.API_TYPE_ENUM_CLASS,
APIEntity.API_TYPE_ANNOTATION, APIEntity.API_TYPE_METHOD,
APIEntity.API_TYPE_CONSTRUCTOR)
print("finished")
self.start_generate_aliases_for_multi_api_type(
APIAlias.ALIAS_TYPE_UNDERLINE_TO_SPACE, APIEntity.API_TYPE_CLASS,
APIEntity.API_TYPE_INTERFACE, APIEntity.API_TYPE_EXCEPTION,
APIEntity.API_TYPE_ERROR, APIEntity.API_TYPE_ENUM_CLASS,
APIEntity.API_TYPE_ANNOTATION, APIEntity.API_TYPE_FIELD,
APIEntity.API_TYPE_METHOD, APIEntity.API_TYPE_CONSTRUCTOR)
print("finished")
class APIAliasesTableFuller:
ALL_API = -1
def __init__(self,
table=APIEntity.__tablename__,
session=None,
commit_step=5000):
self.table = table
self.commit_step = commit_step
self.logger_file_name = "generate_api_aliases"
self.logger = None
self.session = session
self.api_html_table = None
self.api_knowledge_table = None
self.data_source_knowledge_table = None
def get_session(self):
if not self.session:
self.session = EngineFactory.create_session()
return self.session
def start_fix_exist_aliases_for_aliases_type(self, alias_type):
pass
self.logger = Logger(self.logger_file_name).get_log()
self.logger.info("-----------------------start-----------------------")
self.logger.info("generate api aliases for alias type=%d", alias_type)
session = self.get_session()
def start_generate_aliases_for_api_type(self,
alias_type,
api_type=ALL_API):
self.logger = Logger(self.logger_file_name).get_log()
self.logger.info("-----------------------start-----------------------")
self.logger.info("generate api aliases for api type=%d alias type=%d",
api_type, alias_type)
session = self.get_session()
api_alias_generator = APIAliasGeneratorFactory.create_generator(
alias_type)
if api_alias_generator is None:
self.logger.error("Not Implemented Generator for %d ", alias_type)
return
if api_type == APIAliasesTableFuller.ALL_API:
count_api = session.query(APIEntity).count()
api_list_query = session.query(APIEntity)
else:
count_api = session.query(APIEntity).filter(
APIEntity.api_type == api_type).count()
api_list_query = session.query(APIEntity).filter(
APIEntity.api_type == api_type)
start_index_list = range(0, count_api, self.commit_step)
for start_index in start_index_list:
end_index = min(start_index + self.commit_step, count_api)
for api in api_list_query.all()[start_index:end_index]:
api_alias_list = api_alias_generator.generate_aliases(api)
self.add_aliases_to_api_entity(api, api_alias_list, session)
self.logger.info("complete %d-%d", start_index, end_index)
session.commit()
self.logger.info("complete all")
def start_generate_aliases_for_multi_api_type(self, alias_type,
*api_type_list):
for api_type in api_type_list:
self.start_generate_aliases_for_api_type(alias_type=alias_type,
api_type=api_type)
def add_aliases_to_api_entity(self, api, api_alias_list, session):
for api_alias in api_alias_list:
api_alias = api_alias.find_or_create(session=session,
autocommit=False)
self.logger.info(api)
self.logger.info(api_alias)
if api_alias not in api.all_aliases:
api.all_aliases.append(api_alias)
class APIHTMLTextImport:
"""
import the API HTML from one column of one table
"""
def __init__(self,
table,
primary_key_name,
html_column,
html_text_type,
session=None,
commit_step=5000):
self.table = table
self.primary_key_name = primary_key_name
self.html_column = html_column
self.html_text_type = html_text_type
self.commit_step = commit_step
self.logger_file_name = "import_html_for_" + self.table
self.logger = None
self.session = session
self.api_html_table = None
self.api_knowledge_table = None
self.data_source_knowledge_table = None
def start_import(self):
self.logger = Logger(self.logger_file_name).get_log()
if not self.session:
self.session = EngineFactory.create_session()
self.init_knowledge_table()
cur = ConnectionFactory.create_cursor_by_knowledge_table(
self.data_source_knowledge_table)
select_sql = "select {primary_key_name},{html_column} from {table}".format(
primary_key_name=self.primary_key_name,
html_column=self.html_column,
table=self.table)
cur.execute(select_sql)
data_list = cur.fetchall()
result_tuples = []
for i in range(0, len(data_list)):
row_data = data_list[i]
primary_key = row_data[0]
html_text = row_data[1]
if KnowledgeTableColumnMapRecord.exist_import_record(
session=self.session,
start_knowledge_table=self.data_source_knowledge_table,
end_knowledge_table=self.api_html_table,
start_row_id=primary_key,
start_row_name=self.html_column):
self.logger.info("%d has been import to new table",
primary_key)
continue
api_html_text = self.create_from_one_row_data(
primary_key, html_text)
if api_html_text:
api_html_text = api_html_text.create(self.session,
autocommit=False)
result_tuples.append((api_html_text, primary_key))
else:
self.logger.warn("None api_html_text fot %s", str(row_data))
continue
if len(result_tuples) > self.commit_step:
self.commit_to_server_for_column_map(map_tuples=result_tuples)
result_tuples = []
self.commit_to_server_for_column_map(map_tuples=result_tuples)
self.logger.info("import api html completed!")
cur.close()
def init_knowledge_table(self):
self.api_html_table = KnowledgeTableFactory.get_api_html_table(
self.session)
self.api_knowledge_table = KnowledgeTableFactory.get_api_entity_table(
self.session)
self.data_source_knowledge_table = KnowledgeTableFactory.find_knowledge_table_by_name(
self.session, self.table)
def create_from_one_row_data(self, primary_key, html_text):
new_primary_id_for_api = self.get_api_new_primary_key_by_old_primary_key(
old_primary_id=primary_key)
api_html_text = self.create_api_html_text_html_entity(
api_id=new_primary_id_for_api, html_text=html_text)
return api_html_text
def get_api_new_primary_key_by_old_primary_key(self, old_primary_id):
new_primary_id_for_api = KnowledgeTableRowMapRecord.get_end_row_id(
session=self.session,
start_knowledge_table=self.data_source_knowledge_table,
end_knowledge_table=self.api_knowledge_table,
start_row_id=old_primary_id)
if new_primary_id_for_api is None:
self.logger.error(
"no new_primary_id_for_api for old_primary_id=%d",
old_primary_id)
return None
else:
self.logger.info("old_primary_id=%d -> new_primary_id_for_api =%d",
old_primary_id, new_primary_id_for_api)
return new_primary_id_for_api
def create_api_html_text_html_entity(self, api_id, html_text):
if not html_text or not html_text.strip() or html_text == "null":
self.logger.error("no html_text")
return None
if not api_id:
self.logger.error("no api_id ")
return None
api_html_entity = APIHTMLText(api_id=api_id,
html=html_text,
html_type=self.html_text_type)
return api_html_entity
def commit_to_server_for_column_map(self, map_tuples):
self.logger.info("success=%d", len(map_tuples))
try:
self.session.commit()
for (relation, old_id) in map_tuples:
record = KnowledgeTableColumnMapRecord(
self.data_source_knowledge_table, self.api_html_table,
old_id, relation.id, self.html_column)
record.create(self.session, autocommit=False)
self.session.commit()
except Exception:
self.logger.exception("commit_to_server_for_column_map")