def test():
HINRec_model = HINRec(model_name='IsRec_best',
semantic_mode='TF_IDF',
epoch_num=40,
neighbor_size=15,
topTopicNum=3,
cluster_mode='LDA',
cluster_mode_topic_num=50)
print(HINRec_model.get_true_candi_apis())
def bl_IsRec_best(a_dataset):
model_name = 'IsRec_best' # 'IsRec' 'IsRec_best_modified'
epoch_num = 20
neighbor_size = 15
topTopicNum = 3
cluster_mode = 'LDA'
cluster_mode_topic_nums = [50] # 10,25,75,,100,125,150
train_data, test_data = get_train_test_data(a_dataset.train_data,
a_dataset.test_data)
for cluster_mode_topic_num in cluster_mode_topic_nums:
HINRec_model = HINRec(model_name=model_name,
semantic_mode='TF_IDF',
epoch_num=epoch_num,
neighbor_size=neighbor_size,
topTopicNum=topTopicNum,
cluster_mode=cluster_mode,
cluster_mode_topic_num=cluster_mode_topic_num)
if os.path.exists(HINRec_model.weight_path):
print('have trained,return!')
else:
HINRec_model.train(test_data)
HINRec_model.save_model()
evalute_by_epoch(HINRec_model,
HINRec_model,
HINRec_model.model_name,
test_data,
evaluate_by_slt_apiNum=True) # )
def bl_IsRec(a_dataset):
model_name = 'IsRec' # ''
epoch_nums = [20] # 15,100,1000
neighbor_size = 15
topTopicNums = [3] # [3,4,5,6]
train_data, test_data = get_train_test_data(a_dataset.train_data,
a_dataset.test_data)
for epoch_num in epoch_nums:
for topTopicNum in topTopicNums:
HINRec_model = HINRec(model_name=model_name,
epoch_num=epoch_num,
neighbor_size=neighbor_size,
topTopicNum=topTopicNum)
if os.path.exists(HINRec_model.weight_path):
print('have trained,return!')
else:
HINRec_model.train(test_data)
# HINRec_model.test_model(test_data)
HINRec_model.save_model()
evalute_by_epoch(HINRec_model,
HINRec_model,
HINRec_model.model_name,
test_data,
evaluate_by_slt_apiNum=True
) # ,if_save_recommend_result=True)
def bl_PasRec():
model_name = 'PasRec_2path' # 'PasRec_2path'
epoch_num = 60 # 之前是40 40比20差点
neighbor_size = 15
topTopicNum = 3
args = data_repository.get_args()
train_data, test_data = data_repository.get_ds(
).train_data, data_repository.get_ds().test_data
HINRec_model = HINRec(args,
model_name=model_name,
epoch_num=epoch_num,
neighbor_size=neighbor_size,
topTopicNum=topTopicNum)
if os.path.exists(HINRec_model.weight_path):
print('have trained,return!')
else:
# 这里是每隔20epoch测试一下，所以train中输入test_data
HINRec_model.train(test_data)
HINRec_model.save_model()
evalute_by_epoch(
HINRec_model,
HINRec_model,
HINRec_model.model_name,
test_data,
evaluate_by_slt_apiNum=False) # ,if_save_recommend_result=True)
def NI_online(): # 可以用于CI,NI,topMLP,ft等
# HINRec_model = HINRec(model_name='IsRec_best',semantic_mode='TF_IDF', epoch_num=40, neighbor_size=15,topTopicNum=3,cluster_mode='LDA',cluster_mode_topic_num=50)
HINRec_model = HINRec(model_name='PasRec',
epoch_num=40,
neighbor_size=15,
topTopicNum=3)
CI_recommend_model = CI_Model(new_old) # 'old'
# CI_recommend_model.prepare()
# CI_model_obj = CI_recommend_model.get_model()
# CI_model_obj = train_model(CI_recommend_model, CI_model_obj,train_data,test_data,*new_Para.param.train_paras) # ,true_candidates_dict=HINRec_model.get_true_candi_apis() 'monitor loss&acc'
# evalute_by_epoch(CI_recommend_model, CI_model_obj, CI_recommend_model.model_name, test_data,
# if_save_recommend_result=True, evaluate_by_slt_apiNum=True)
# analyze_result(CI_recommend_model, new_Para.param.topKs)
# CI_recommend_model.show_slt_apis_tag_features(a_dataset.train_data) # 检查中间feature结果
# # CI_recommend_model.get_slt_apis_mid_features(train_data,test_data) # 存储所有样本的attention的中间结果,为deepFm准备
# # CI_recommend_model.save_for_deepFM()
#
# 调优NI的score
# for pruned_neighbor_baseScore in [0,0.2,0.3]: #
# NI_OL_recommend_model = NI_Model(new_old, if_implict=True, if_explict=False,
# if_correlation=False,
# pruned_neighbor_baseScore=pruned_neighbor_baseScore)
# sim_model = HINRec_model if new_Para.param.NI_OL_mode == 'IsRec_best_Sim' else None # CI_recommend_model
# NI_OL_recommend_model.prepare(sim_model, train_data, test_data)
# NI_OL_model_obj = NI_OL_recommend_model.get_model()
# NI_OL_model_obj = train_model(NI_OL_recommend_model, NI_OL_model_obj, train_data, test_data,
# *new_Para.param.train_paras,
# true_candidates_dict=HINRec_model.get_true_candi_apis())
NI_OL_recommend_model = NI_Model_online(
'new', if_implict=True, if_explict=False,
if_correlation=False) # 'new' ,pruned_neighbor_baseScore = 0
# 构建即可，读取之前训练好的相似度数据
# HINRec_model = HINRec(model_name='IsRec_best',semantic_mode='TF_IDF', epoch_num=40, neighbor_size=15,topTopicNum=3,cluster_mode='LDA',cluster_mode_topic_num=50)
# 'IsRec_best_Sim'
sim_model = CI_recommend_model if new_Para.param.NI_OL_mode == 'tagSim' else HINRec_model #
NI_OL_recommend_model.prepare(sim_model, train_data, test_data)
NI_OL_model_obj = NI_OL_recommend_model.get_model()
NI_OL_model_obj = train_model(
NI_OL_recommend_model, NI_OL_model_obj, train_data, test_data,
*new_Para.param.train_paras
) # ,true_candidates_dict=HINRec_model.get_true_candi_apis()
def newDeepFM():
CI_recommend_model = CI_Model(new_old) # 'old'
CI_recommend_model.prepare()
HINRec_model = HINRec(model_name='PasRec',
epoch_num=40,
neighbor_size=15,
topTopicNum=3)
sim_model = CI_recommend_model if new_Para.param.NI_OL_mode == 'tagSim' else HINRec_model #
NI_OL_recommend_model = NI_Model_online(
new_old, if_implict=True, if_explict=False,
if_correlation=False) # 'new' ,pruned_neighbor_baseScore = 0
NI_OL_recommend_model.prepare(sim_model, train_data, test_data)
mashup_NI_features = NI_OL_recommend_model.mid_sltAids_2NI_feas
api_NI_features = NI_OL_recommend_model.i_factors_matrix
NI_feas = mashup_NI_features, api_NI_features
if not os.path.exists(CI_recommend_model.ma_text_tag_feas_path):
# 如果特征的存储文件不存在，再加载模型，退出重新运行
CI_model_obj = CI_recommend_model.get_model()
CI_model_obj = train_model(
CI_recommend_model, CI_model_obj, train_data, test_data,
*new_Para.param.train_paras
) # ,true_candidates_dict=HINRec_model.get_true_candi_apis() 'monitor loss&acc'
CI_feas = CI_recommend_model.get_mashup_api_features(
CI_recommend_model.all_mashup_num,
CI_recommend_model.all_api_num + 1) # 最后一个是填充虚拟api的特征
print('re-run the program!')
else:
CI_feas = CI_recommend_model.get_mashup_api_features(
CI_recommend_model.all_mashup_num,
CI_recommend_model.all_api_num + 1)
run_new_deepFM(CI_feas,
NI_feas,
train_data,
test_data,
CI_recommend_model.all_api_num,
epoch_num=10)
def get_preTrain_CINI_model():
HINRec_model = HINRec(model_name='PasRec',
epoch_num=40,
neighbor_size=15,
topTopicNum=3)
CI_recommend_model = CI_Model(new_old) # 'old'
CI_recommend_model.prepare()
CI_model_obj = CI_recommend_model.get_model()
CI_model_obj = train_model(
CI_recommend_model, CI_model_obj, train_data, test_data,
*new_Para.param.train_paras
) # ,true_candidates_dict=HINRec_model.get_true_candi_apis() 'monitor loss&acc'
NI_OL_recommend_model = NI_Model_online(
new_old, if_implict=True, if_explict=False,
if_correlation=False) # 'new' ,pruned_neighbor_baseScore = 0
# 构建即可，读取之前训练好的相似度数据
# HINRec_model = HINRec(model_name='IsRec_best',semantic_mode='TF_IDF', epoch_num=40, neighbor_size=15,topTopicNum=3,cluster_mode='LDA',cluster_mode_topic_num=50)
# 'IsRec_best_Sim'
sim_model = CI_recommend_model if new_Para.param.NI_OL_mode == 'tagSim' else HINRec_model #
NI_OL_recommend_model.prepare(sim_model, train_data, test_data)
return CI_recommend_model, NI_OL_recommend_model
def test_simModes(a_dataset, new_old='new', if_few=False):
if if_few:
train_data, test_data = a_dataset.get_few_samples(128)
print(type(train_data))
print(type(test_data))
else:
train_data, test_data = a_dataset.train_data, a_dataset.test_data
HINRec_model = HINRec(model_name=new_Para.param.NI_OL_mode,
epoch_num=20,
neighbor_size=15,
topTopicNum=3)
# 'IsRec_best' 这个是预训练的相似度模型
CI_recommend_model = CI_Model(new_old) # 'old'
CI_recommend_model.prepare()
CI_model_obj = CI_recommend_model.get_model()
CI_model_obj = train_model(
CI_recommend_model, CI_model_obj, train_data, test_data,
*new_Para.param.train_paras
) # ,true_candidates_dict=HINRec_model.get_true_candi_apis() 'monitor loss&acc'
# evalute_by_epoch(CI_recommend_model, CI_model_obj, CI_recommend_model.simple_name, test_data,
# if_save_recommend_result=True, evaluate_by_slt_apiNum=True)
# 为no_slt CI设计
# CI_model_obj = train_model(CI_recommend_model, CI_model_obj, train_data, a_dataset.test_data_no_reduct,
# *new_Para.param.train_paras) #
# evalute_by_epoch(CI_recommend_model, CI_model_obj, CI_recommend_model.simple_name, a_dataset.test_data_no_reduct,
# if_save_recommend_result=True, evaluate_by_slt_apiNum=True)
sim_model = CI_recommend_model if new_Para.param.NI_OL_mode == 'tagSim' else HINRec_model #
NI_OL_recommend_model = NI_Model_online(
new_old,
if_implict=True,
if_explict=False,
if_correlation=False,
eachPath_topK=True) # 'new' ,pruned_neighbor_baseScore = 0
# 构建即可，读取之前训练好的相似度数据
NI_OL_recommend_model.prepare_sims(sim_model, train_data, test_data)
NI_OL_model_obj = NI_OL_recommend_model.get_model()
NI_OL_model_obj = train_model(NI_OL_recommend_model, NI_OL_model_obj,
train_data, test_data,
*new_Para.param.train_paras) #
# # # explicit
# explicit_NI_recommend_model = NI_Model(new_old,if_implict=False,if_explict=True,if_correlation=False)
# explicit_NI_recommend_model.prepare(sim_model,train_data, test_data) # NI的模型搭建需要CI模型生成所有mashup/api的feature
# explict_NI_model_obj = explicit_NI_recommend_model.get_model()
# explict_NI_model_obj = train_model(explicit_NI_recommend_model, explict_NI_model_obj,train_data, test_data, *new_Para.param.train_paras)
# evalute_by_epoch(NI_OL_recommend_model, NI_OL_model_obj, NI_OL_recommend_model.simple_name, a_dataset.test_data,
# if_save_recommend_result=True, evaluate_by_slt_apiNum=True)
# 专门为no_slt NI设计
# NI_OL_model_obj = train_model(NI_OL_recommend_model, NI_OL_model_obj, train_data, a_dataset.test_data_no_reduct,
# *new_Para.param.train_paras) #
# evalute_by_epoch(NI_OL_recommend_model, NI_OL_model_obj, NI_OL_recommend_model.simple_name, a_dataset.test_data_no_reduct,
# if_save_recommend_result=True, evaluate_by_slt_apiNum=True)
# #
# # # # # CI+ implict
top_MLP_recommend_model = top_MLP(
CI_recommend_model,
CI_model_obj,
NI_recommend_model1=NI_OL_recommend_model,
NI_model1=NI_OL_model_obj)
top_MLP_model = top_MLP_recommend_model.get_model()
top_MLP_model = train_model(top_MLP_recommend_model, top_MLP_model,
train_data, test_data,
*new_Para.param.train_paras)
top_MLP_recommend_model.save_sth() # 存储训练测试过程中使用的所有实例的中间结果