def insert_batch(num, clean=True, average_iteration_num=1):
sum_time = 0.0
for i in range(0, average_iteration_num):
starttime = datetime.datetime.now()
old_session = EngineFactory.create_session_to_so_old(echo=False)
new_session = EngineFactory.create_session_to_new_so(echo=False)
old_post_list = old_session.query(PostsRecord).limit(num)
for post in old_post_list:
new_session.add(post.make_copy())
## 全部写入缓存再一次性commit写入数据库
new_session.commit()
endtime = datetime.datetime.now()
time = (endtime - starttime).total_seconds()
print("test_insert_batch num={num} time={time}".format(num=num,
time=time))
if clean:
PostsRecord.delete_all(new_session)
sum_time = sum_time + time
return {
"type": "insert batch",
"num": num,
"time": sum_time / average_iteration_num
}
def insert_separate(num, clean=True, average_iteration_num=1):
sum_time = 0.0
for i in range(0, average_iteration_num):
old_session = EngineFactory.create_session_to_databackup_so(echo=False)
test_session = EngineFactory.create_session_to_test_so(echo=False)
old_post_list = old_session.query(PostsRecord).limit(num)
starttime = datetime.datetime.now()
for post in old_post_list:
test_session.add(post.make_copy())
## 每插入一条就commit写入数据库
test_session.commit()
endtime = datetime.datetime.now()
time = (endtime - starttime).total_seconds()
print("test_insert_separate num={num} time={time}".format(num=num,
time=time))
if clean:
PostsRecord.delete_all(test_session)
sum_time = sum_time + time
return {
"type": "insert separate",
"num": num,
"time": sum_time / average_iteration_num
}
def insert_batch(num, clean=True, average_iteration_num=1):
sum_time = 0.0
for i in range(0, average_iteration_num):
## 接下来的三行代码从旧的总表中获取想要的数目的数据，作为之后插入的数据源，其实也可以读文件得到，但是那就太麻烦了
old_session = EngineFactory.create_session_to_databackup_so(echo=False)
test_session = EngineFactory.create_session_to_test_so(echo=False)
old_post_list = old_session.query(PostsRecord).limit(num)
starttime = datetime.datetime.now()
for post in old_post_list:
test_session.add(post.make_copy())
## 全部写入缓存再一次性commit写入数据库
test_session.commit()
endtime = datetime.datetime.now()
time = (endtime - starttime).total_seconds()
print("test_insert_batch num={num} time={time}".format(num=num,
time=time))
if clean:
PostsRecord.delete_all(test_session)
sum_time = sum_time + time
return {
"type": "insert batch",
"num": num,
"time": sum_time / average_iteration_num
}
def search_one_table_mul_filter(num, average_iteration_num=1):
sum_time = 0.0
for i in range(0, average_iteration_num):
starttime = datetime.datetime.now()
session = EngineFactory.create_session_to_test_so(echo=False)
res = session.query(PostsRecord).filter(
PostsRecord.owner_user_id < num,
PostsRecord.view_count > 1000).all()
if len(res) > 0:
print("search_one_table_mul_filter_result:", len(res), ":", res[0])
else:
print("search_one_table_mul_filter_result: null")
endtime = datetime.datetime.now()
time = (endtime - starttime).total_seconds()
print("test_search_one_table_mul_filter num={num} time={time}".format(
num=num, time=time))
sum_time = sum_time + time
return {
"type": "search_one_table_mul_filter",
"num": num,
"time": sum_time / average_iteration_num
}
def search_aggregate(num, average_iteration_num=1):
sum_time = 0.0
for i in range(0, average_iteration_num):
starttime = datetime.datetime.now()
session = EngineFactory.create_session_to_test_so(echo=False)
res = session.query(
func.sum(PostsRecord.favorite_count), UsersRecord.display_name,
UsersRecord.reputation).filter(
UsersRecord.id < num,
PostsRecord.owner_user_id == UsersRecord.id).all()
if len(res) > 0:
print("search_aggregate_result:", len(res), ":", res[0])
else:
print("search_aggregate_result: null")
endtime = datetime.datetime.now()
time = (endtime - starttime).total_seconds()
print("test_search_aggregate num={num} time={time}".format(num=num,
time=time))
sum_time = sum_time + time
return {
"type": "search_aggregate",
"num": num,
"time": sum_time / average_iteration_num
}
def get_game_data(doc_name, game_id: int):
vote_table_id = (game_id - 1) * 2
action_table_id = (game_id - 1) * 2 + 1
vote_df = pd.DataFrame(read_table(doc_name, vote_table_id))
action_df = pd.DataFrame(read_table(doc_name, action_table_id))
factory = EngineFactory()
name = get_game_name_by_id(doc_name, game_id)
winner = get_game_winner_by_id(doc_name, game_id)
cleaned_data = {"template": name, "winner": winner}
parser = factory.construct(name)
parser.read_data(action_df, vote_df)
cleaned_data["data"] = parser.parse(winner)
return cleaned_data
def show_index():
engine = EngineFactory.create_engine_to_test_so()
conn = engine.connect()
text_sql = 'show index from {table_name}'.format(
table_name=PostsRecord.__tablename__)
s = text(text_sql)
conn.execute(s)
conn.close()
def delete_post_test_data_in_test_db():
"""
删除测试服务器得帖子表数据
:param num: 要取得测试数据得数目
:return:
"""
test_session = EngineFactory.create_session_to_test_so(echo=False)
PostsRecord.delete_all(test_session)
def create_post_test_data_in_test_db(num):
"""
利用全表服务器在测试服务器上创建新的数据,这个创建得是用户表得测试数据
:param num: 要取得测试数据得数目
:return:
"""
## 接下来的三行代码从旧的总表中获取想要的数目的数据，作为之后插入的数据源，其实也可以读文件得到，但是那就太麻烦了
old_session = EngineFactory.create_session_to_databackup_so(echo=False)
new_session = EngineFactory.create_session_to_test_so(echo=False)
old_post_list = old_session.query(PostsRecord).limit(num)
count = 0
for post in old_post_list:
new_session.add(post.make_copy())
count = count + 1
if count >= 100000:
new_session.commit()
count = 0
new_session.commit()
def __init__(self, session=None):
if session is None:
session = EngineFactory.create_session()
self.session = session
self.candidate_generator = APICandidateGenerator(session)
self.scorer_list = []
self.scorer_list.append(APITypeScorer(weight=35, api_linker=None))
self.scorer_list.append(
MethodParametersScorer(weight=50, api_linker=None))
self.scorer_list.append(ParentAPIScorer(weight=100.0, api_linker=self))
self.scorer_list.append(DeclarationScorer(weight=60.0,
api_linker=self))
def add_score_index():
try:
engine = EngineFactory.create_engine_to_test_so()
conn = engine.connect()
index_name = "score_index"
text_sql = 'alter table {table_name} add index {index_name}(score)'.format(
table_name=PostsRecord.__tablename__, index_name=index_name)
s = text(text_sql)
conn.execute(s)
conn.close()
except:
traceback.print_exc()
def delete_score_view_count_index():
try:
engine = EngineFactory.create_engine_to_test_so()
conn = engine.connect()
index_name = "score_view_count_index"
text_sql = 'alter table `{table_name}` drop index `{index_name}`'.format(
table_name=PostsRecord.__tablename__, index_name=index_name)
s = text(text_sql)
conn.execute(s)
conn.close()
except:
traceback.print_exc()
def start_import(self):
self.logger = Logger(self.logger_file_name).get_log()
if not self.session:
self.session = EngineFactory.create_session()
self.init_knowledge_table()
cur = ConnectionFactory.create_cursor_by_knowledge_table(
self.data_source_knowledge_table)
select_sql = "select {primary_key_name},{html_column} from {table}".format(
primary_key_name=self.primary_key_name,
html_column=self.html_column,
table=self.table)
cur.execute(select_sql)
data_list = cur.fetchall()
result_tuples = []
for i in range(0, len(data_list)):
row_data = data_list[i]
primary_key = row_data[0]
html_text = row_data[1]
if KnowledgeTableColumnMapRecord.exist_import_record(
session=self.session,
start_knowledge_table=self.data_source_knowledge_table,
end_knowledge_table=self.api_html_table,
start_row_id=primary_key,
start_row_name=self.html_column):
self.logger.info("%d has been import to new table",
primary_key)
continue
api_html_text = self.create_from_one_row_data(
primary_key, html_text)
if api_html_text:
api_html_text = api_html_text.create(self.session,
autocommit=False)
result_tuples.append((api_html_text, primary_key))
else:
self.logger.warn("None api_html_text fot %s", str(row_data))
continue
if len(result_tuples) > self.commit_step:
self.commit_to_server_for_column_map(map_tuples=result_tuples)
result_tuples = []
self.commit_to_server_for_column_map(map_tuples=result_tuples)
self.logger.info("import api html completed!")
cur.close()
className = dic['className: ']
param = dic['param: ']
javadocComment = dic['javadocComment: ']
blockComment = dic['blockComment: ']
update_by_className_methodName_param(className, methodName, param,
code, javadocComment,
blockComment)
counter += 1
if counter > step:
counter = 0
session.commit()
session.commit()
if __name__ == "__main__":
schema_name = 'domainkg'
engine = EngineFactory.create_engine_by_schema_name(schema_name)
session = EngineFactory.create_session(engine=engine, autocommit=False)
read_json_by_line('traces.json')
# trace_list = session.query(Traces).filter_by()
# for trace in trace_list:
# try:
# className = trace.className
# methodName = trace.methodName
# param = trace.param
# trace.code = 'code'
# except Exception:
# print("error")
def __init__(self, session=None):
if session is None:
session = EngineFactory.create_session()
self.session = session
def get_so_session(self):
if not self.session:
self.session = EngineFactory.create_so_session()
return self.session
class Wikipedia(Base):
__tablename__ = 'wiki_pedia'
id = Column(Integer, primary_key=True) # auto incrementing
doc_id = Column(Integer, index=True, unique=True, nullable=False)
url = Column(String(128), index=True, nullable=False)
title = Column(String(64), index=True, nullable=False)
content = Column(LONGTEXT())
def __init__(self, doc_id=None, url=None, title=None, content=None):
self.doc_id = doc_id
self.url = url
self.title = title
self.content = content
def __unicode__(self):
return self.__repr__()
def __repr__(self):
return '<Wikipedia: %r: doc_id=%s >' % (self.title, self.doc_id)
if __name__ == "__main__":
engine = EngineFactory.create_engine_to_wiki()
metadata = MetaData(bind=engine)
# delete all table
# Base.metadata.drop_all(bind=engine)
# create the table
Base.metadata.create_all(bind=engine)
user.creation_date = self.creation_date
user.display_name = self.display_name
user.last_access_date = self.last_access_date
user.views = self.views
user.web_site_url = self.web_site_url
user.location = self.location
user.about_me = self.about_me
user.age = self.age
user.up_votes = self.up_votes
user.down_votes = self.down_votes
user.email_hash = self.email_hash
return user
if __name__ == "__main__":
engine = EngineFactory.create_engine_to_test_so()
metadata = MetaData(bind=engine)
# create all the table by model
Base.metadata.create_all(bind=engine)
if api_type_string == "Nested":
return APIEntity.API_TYPE_CLASS
if api_type_string == "Required":
return APIEntity.API_TYPE_FIELD
if api_type_string == "Optional":
return APIEntity.API_TYPE_FIELD
if api_type_string == "Field":
return APIEntity.API_TYPE_FIELD
if api_type_string == "Enum":
return APIEntity.API_TYPE_ENUM_CONSTANTS
api_type = APIEntity.API_TYPE_UNKNOWN
return api_type
# if __name__ == "__main__":
# # create table in 75
# engine = EngineFactory.create_graphdata_engine_to_center()
# metadata = MetaData(bind=engine)
# Base.metadata.create_all(bind=engine)
if __name__ == "__main__":
engine = EngineFactory.create_engine_to_center()
metadata = MetaData(bind=engine)
# delete all table
# Base.metadata.drop_all(bind=engine)
# create the table
Base.metadata.create_all(bind=engine)
try:
return session.query(POIMethod).filter_by(
package_name=self.package_name,
class_name=self.class_name,
return_type=self.return_type,
method_name=self.method_name,
description=self.description,
type=self.type).first()
except Exception:
traceback.print_exc()
session.rollback()
return None
def find_or_create(self, session, autocommit=True):
remote_instance = self.get_remote_object(session)
if not remote_instance:
session.add(self)
if autocommit:
session.commit()
return self
else:
return remote_instance
if __name__ == "__main__":
engine = EngineFactory.create_engine_by_schema_name('domainkg')
metadata = MetaData(bind=engine)
# create the table
Base.metadata.create_all(bind=engine)