def CI_NI_fineTuning():
args = data_repository.get_args()
train_data, test_data = data_repository.get_ds(
).train_data, data_repository.get_ds().test_data
CI_recommend_model = CI_Model(args)
CI_model_obj = CI_recommend_model.get_model()
CI_model_obj = train_model(CI_recommend_model, CI_model_obj, train_data,
test_data, args.train_mode, args.train_new)
def deepFM():
CI_recommend_model = CI_Model(new_old) # 'old'
NI_OL_recommend_model = NI_Model_online(new_old,
if_implict=True,
if_explict=False,
if_correlation=False)
# 上面仅初始化，否则加载模型会占用太多现存
run_deepFM(CI_recommend_model,
NI_OL_recommend_model,
train_data,
test_data,
epoch_num=10)
def get_preTrain_CINI_model():
HINRec_model = HINRec(model_name='PasRec',
epoch_num=40,
neighbor_size=15,
topTopicNum=3)
CI_recommend_model = CI_Model(new_old) # 'old'
CI_recommend_model.prepare()
CI_model_obj = CI_recommend_model.get_model()
CI_model_obj = load_preTrained_model(
CI_recommend_model, CI_model_obj, train_data, test_data,
*new_Para.param.train_paras
) # ,true_candidates_dict=HINRec_model.get_true_candi_apis() 'monitor loss&acc'
NI_OL_recommend_model = NI_Model_online(
new_old, if_implict=True, if_explict=False,
if_correlation=False) # 'new' ,pruned_neighbor_baseScore = 0
# 构建即可，读取之前训练好的相似度数据
# HINRec_model = HINRec(model_name='IsRec_best',semantic_mode='TF_IDF', epoch_num=40, neighbor_size=15,topTopicNum=3,cluster_mode='LDA',cluster_mode_topic_num=50)
# 'IsRec_best_Sim'
sim_model = CI_recommend_model if new_Para.param.NI_OL_mode == 'tagSim' else HINRec_model #
NI_OL_recommend_model.prepare(sim_model, train_data, test_data)
return CI_recommend_model, NI_OL_recommend_model
def co_trainCINI(): # 参数完全随机化，联合训练CI和NI
CI_recommend_model = CI_Model(new_old)
CI_recommend_model.prepare()
CI_model_obj = CI_recommend_model.get_model()
NI_OL_recommend_model = NI_Model_online(new_old,
if_implict=True,
if_explict=False,
if_correlation=False)
# CI_recommend_model = CI_recommend_model if 'Sim' in new_Para.param.NI_OL_mode else None 只训练NI时，OL_GE不需要CI
NI_OL_recommend_model.prepare(CI_recommend_model)
NI_OL_model_obj = NI_OL_recommend_model.get_model()
co_trainCINI_recommend_model = fine_Tune(CI_recommend_model,
CI_model_obj,
NI_OL_recommend_model,
NI_OL_model_obj,
model_mode='co_train',
lr=0.001) # 0.0003
co_trainCINI_model = co_trainCINI_recommend_model.get_model()
co_trainCINI_model = train_model(co_trainCINI_recommend_model,
co_trainCINI_model, train_data, test_data,
*new_Para.param.train_paras)
def DINRec(a_dataset, new_old='new'):
train_data, test_data = a_dataset.train_data, a_dataset.test_data
CI_recommend_model = CI_Model(new_old) # 'old'
CI_recommend_model.prepare()
CI_model_obj = CI_recommend_model.get_model()
CI_model_obj = load_preTrained_model(
CI_recommend_model, CI_model_obj, train_data, test_data,
*new_Para.param.train_paras
) # ,true_candidates_dict=HINRec_model.get_true_candi_apis() 'monitor loss&acc'
DINRec_model = DIN_Rec(CI_recommend_model,
new_Para.param.predict_fc_unit_nums)
DINRec_model.prepare()
DINRec_model_obj = DINRec_model.get_model()
DINRec_model_obj = load_preTrained_model(
DINRec_model, DINRec_model_obj, train_data, test_data,
*new_Para.param.train_paras
) # ,true_candidates_dict=HINRec_model.get_true_candi_apis() 'monitor loss&acc'
evalute_by_epoch(DINRec_model,
DINRec_model_obj,
DINRec_model.simple_name,
test_data,
if_save_recommend_result=True,
evaluate_by_slt_apiNum=True)
def NI_online(): # 可以用于CI,NI,topMLP,ft等
# HINRec_model = HINRec(model_name='IsRec_best',semantic_mode='TF_IDF', epoch_num=40, neighbor_size=15,topTopicNum=3,cluster_mode='LDA',cluster_mode_topic_num=50)
HINRec_model = HINRec(model_name='PasRec',
epoch_num=40,
neighbor_size=15,
topTopicNum=3)
CI_recommend_model = CI_Model(new_old) # 'old'
# CI_recommend_model.prepare()
# CI_model_obj = CI_recommend_model.get_model()
# CI_model_obj = load_preTrained_model(CI_recommend_model, CI_model_obj,train_data,test_data,*new_Para.param.train_paras) # ,true_candidates_dict=HINRec_model.get_true_candi_apis() 'monitor loss&acc'
# evalute_by_epoch(CI_recommend_model, CI_model_obj, CI_recommend_model.model_name, test_data,
# if_save_recommend_result=True, evaluate_by_slt_apiNum=True)
# analyze_result(CI_recommend_model, new_Para.param.topKs)
# CI_recommend_model.show_slt_apis_tag_features(a_dataset.train_data) # 检查中间feature结果
# # CI_recommend_model.get_slt_apis_mid_features(train_data,test_data) # 存储所有样本的attention的中间结果,为deepFm准备
# # CI_recommend_model.save_for_deepFM()
#
# 调优NI的score
# for pruned_neighbor_baseScore in [0,0.2,0.3]: #
# NI_OL_recommend_model = NI_Model_online(new_old, if_implict=True, if_explict=False,
# if_correlation=False,
# pruned_neighbor_baseScore=pruned_neighbor_baseScore)
# sim_model = HINRec_model if new_Para.param.NI_OL_mode == 'IsRec_best_Sim' else None # CI_recommend_model
# NI_OL_recommend_model.prepare(sim_model, train_data, test_data)
# NI_OL_model_obj = NI_OL_recommend_model.get_model()
# NI_OL_model_obj = load_preTrained_model(NI_OL_recommend_model, NI_OL_model_obj, train_data, test_data,
# *new_Para.param.train_paras,
# true_candidates_dict=HINRec_model.get_true_candi_apis())
NI_OL_recommend_model = NI_Model_online(
'new', if_implict=True, if_explict=False,
if_correlation=False) # 'new' ,pruned_neighbor_baseScore = 0
# 构建即可，读取之前训练好的相似度数据
# HINRec_model = HINRec(model_name='IsRec_best',semantic_mode='TF_IDF', epoch_num=40, neighbor_size=15,topTopicNum=3,cluster_mode='LDA',cluster_mode_topic_num=50)
# 'IsRec_best_Sim'
sim_model = CI_recommend_model if new_Para.param.NI_OL_mode == 'tagSim' else HINRec_model #
NI_OL_recommend_model.prepare(sim_model, train_data, test_data)
NI_OL_model_obj = NI_OL_recommend_model.get_model()
NI_OL_model_obj = load_preTrained_model(
NI_OL_recommend_model, NI_OL_model_obj, train_data, test_data,
*new_Para.param.train_paras
) # ,true_candidates_dict=HINRec_model.get_true_candi_apis()
def CI_NI_fineTuning():
CI_recommend_model = CI_Model('new') # new_old
# CI_recommend_model.prepare()
# CI_model_obj = CI_recommend_model.get_model()
# CI_model_obj = load_preTrained_model(CI_recommend_model, CI_model_obj,train_data,test_data,*new_Para.param.train_paras) # 'monitor loss&acc'
# CI_recommend_model.show_slt_apis_tag_features(a_dataset.train_data) # 存储中间feature结果
# implict
implict_NI_recommend_model = NI_Model('new',
if_implict=True,
if_explict=False,
if_correlation=False) # new_old
implict_NI_recommend_model.prepare(
CI_recommend_model) # NI的模型搭建需要CI模型生成所有mashup/api的feature
implict_NI_model_obj = implict_NI_recommend_model.get_model()
implict_NI_model_obj = load_preTrained_model(implict_NI_recommend_model,
implict_NI_model_obj,
train_data, test_data,
*new_Para.param.train_paras)
def newDeepFM():
CI_recommend_model = CI_Model(new_old) # 'old'
CI_recommend_model.prepare()
HINRec_model = HINRec(model_name='PasRec',
epoch_num=40,
neighbor_size=15,
topTopicNum=3)
sim_model = CI_recommend_model if new_Para.param.NI_OL_mode == 'tagSim' else HINRec_model #
NI_OL_recommend_model = NI_Model_online(
new_old, if_implict=True, if_explict=False,
if_correlation=False) # 'new' ,pruned_neighbor_baseScore = 0
NI_OL_recommend_model.prepare(sim_model, train_data, test_data)
mashup_NI_features = NI_OL_recommend_model.mid_sltAids_2NI_feas
api_NI_features = NI_OL_recommend_model.i_factors_matrix
NI_feas = mashup_NI_features, api_NI_features
if not os.path.exists(CI_recommend_model.ma_text_tag_feas_path):
# 如果特征的存储文件不存在，再加载模型，退出重新运行
CI_model_obj = CI_recommend_model.get_model()
CI_model_obj = load_preTrained_model(
CI_recommend_model, CI_model_obj, train_data, test_data,
*new_Para.param.train_paras
) # ,true_candidates_dict=HINRec_model.get_true_candi_apis() 'monitor loss&acc'
CI_feas = CI_recommend_model.get_mashup_api_features(
CI_recommend_model.all_mashup_num,
CI_recommend_model.all_api_num + 1) # 最后一个是填充虚拟api的特征
print('re-run the program!')
else:
CI_feas = CI_recommend_model.get_mashup_api_features(
CI_recommend_model.all_mashup_num,
CI_recommend_model.all_api_num + 1)
run_new_deepFM(CI_feas,
NI_feas,
train_data,
test_data,
CI_recommend_model.all_api_num,
epoch_num=10)
def test_simModes(a_dataset, new_old='new', if_few=False):
if if_few:
train_data, test_data = a_dataset.get_few_samples(128)
print(type(train_data))
print(type(test_data))
else:
train_data, test_data = a_dataset.train_data, a_dataset.test_data
HINRec_model = HINRec_new(model_name=new_Para.param.NI_OL_mode,
epoch_num=20,
neighbor_size=15,
topTopicNum=3)
# 'IsRec_best' 这个是预训练的相似度模型
CI_recommend_model = CI_Model(new_old) # 'old'
CI_recommend_model.prepare()
CI_model_obj = CI_recommend_model.get_model()
CI_model_obj = load_preTrained_model(
CI_recommend_model, CI_model_obj, train_data, test_data,
*new_Para.param.train_paras
) # ,true_candidates_dict=HINRec_model.get_true_candi_apis() 'monitor loss&acc'
# evalute_by_epoch(CI_recommend_model, CI_model_obj, CI_recommend_model.simple_name, test_data,
# if_save_recommend_result=True, evaluate_by_slt_apiNum=True)
# 为no_slt CI设计
# CI_model_obj = load_preTrained_model(CI_recommend_model, CI_model_obj, train_data, a_dataset.test_data_no_reduct,
# *new_Para.param.train_paras) #
# evalute_by_epoch(CI_recommend_model, CI_model_obj, CI_recommend_model.simple_name, a_dataset.test_data_no_reduct,
# if_save_recommend_result=True, evaluate_by_slt_apiNum=True)
sim_model = CI_recommend_model if new_Para.param.NI_OL_mode == 'tagSim' else HINRec_model #
NI_OL_recommend_model = NI_Model_online(
new_old,
if_implict=True,
if_explict=False,
if_correlation=False,
eachPath_topK=True) # 'new' ,pruned_neighbor_baseScore = 0
# 构建即可，读取之前训练好的相似度数据
NI_OL_recommend_model.prepare_sims(sim_model, train_data, test_data)
NI_OL_model_obj = NI_OL_recommend_model.get_model()
NI_OL_model_obj = load_preTrained_model(NI_OL_recommend_model,
NI_OL_model_obj, train_data,
test_data,
*new_Para.param.train_paras) #
# # # explicit
# explicit_NI_recommend_model = NI_Model_online(new_old,if_implict=False,if_explict=True,if_correlation=False)
# explicit_NI_recommend_model.prepare(sim_model,train_data, test_data) # NI的模型搭建需要CI模型生成所有mashup/api的feature
# explict_NI_model_obj = explicit_NI_recommend_model.get_model()
# explict_NI_model_obj = load_preTrained_model(explicit_NI_recommend_model, explict_NI_model_obj,train_data, test_data, *new_Para.param.train_paras)
# evalute_by_epoch(NI_OL_recommend_model, NI_OL_model_obj, NI_OL_recommend_model.simple_name, a_dataset.test_data,
# if_save_recommend_result=True, evaluate_by_slt_apiNum=True)
# 专门为no_slt NI设计
# NI_OL_model_obj = load_preTrained_model(NI_OL_recommend_model, NI_OL_model_obj, train_data, a_dataset.test_data_no_reduct,
# *new_Para.param.train_paras) #
# evalute_by_epoch(NI_OL_recommend_model, NI_OL_model_obj, NI_OL_recommend_model.simple_name, a_dataset.test_data_no_reduct,
# if_save_recommend_result=True, evaluate_by_slt_apiNum=True)
# #
# # # # # CI+ implict
top_MLP_recommend_model = top_MLP(CI_recommend_model,
CI_model_obj,
NI_recommend_model=NI_OL_recommend_model,
NI_model=NI_OL_model_obj)
top_MLP_model = top_MLP_recommend_model.get_model()
top_MLP_model = load_preTrained_model(top_MLP_recommend_model,
top_MLP_model, train_data, test_data,
*new_Para.param.train_paras)
top_MLP_recommend_model.save_sth() # 存储训练测试过程中使用的所有实例的中间结果