def ncp_run(N1, N2, N3, gR, dR, time): # ncp test X = synthetic_data_cp([N1, N2, N3], gR, 0) data_provider = Provider() data_provider.full_tensor = lambda: X env = Environment(data_provider, summary_path='/tmp/ncp_' + str(N1)) ncp = NCP_BCU(env) args = NCP_BCU.NCP_Args(rank=dR, validation_internal=200) ncp.build_model(args) print('\n\nNCP with %dx%dx%d, gR=%d, dR=%d, time=%d' % (N1, N2, N3, gR, dR, time)) loss_hist = ncp.train(6000) scale = str(N1) + '_' + str(gR) + '_' + str(dR) out_path = '/root/tensorD_f/data_out_tmp/python_out/ncp_' + scale + '_' + str(time) + '.txt' with open(out_path, 'w') as out: for loss in loss_hist: out.write('%.6f\n' % loss)
def cp_run(N1, N2, N3, gR, dR, time): # cp test X = synthetic_data_cp([N1, N2, N3], gR, 0) data_provider = Provider() data_provider.full_tensor = lambda: X env = Environment(data_provider, summary_path='/tmp/cp_' + str(N1)) cp = CP_ALS(env) args = CP_ALS.CP_Args(rank=dR, validation_internal=50, tol=1.0e-4) cp.build_model(args) print('CP with %dx%dx%d, gR=%d, dR=%d, time=%d' % (N1, N2, N3, gR, dR, time)) hist = cp.train(600) scale = str(N1) + '_' + str(gR) + '_' + str(dR) out_path = '/root/tensorD_f/data_out_tmp/python_out/cp_' + scale + '_' + str(time) + '.txt' with open(out_path, 'w') as out: for iter in hist: loss = iter[0] rel_res = iter[1] out.write('%.10f, %.10f\n' % (loss, rel_res))
def tucker_run(N1, N2, N3, gR, dR, time): # tucker X = synthetic_data_tucker([N1, N2, N3], [gR, gR, gR]) data_provider = Provider() data_provider.full_tensor = lambda: X env = Environment(data_provider, summary_path='/tmp/tucker_' + str(N1)) hooi = HOOI(env) args = HOOI.HOOI_Args(ranks=[dR, dR, dR], validation_internal=200) hooi.build_model(args) print('\n\nTucker with %dx%dx%d, gR=%d, dR=%d, time=%d' % (N1, N2, N3, gR, dR, time)) loss_hist = hooi.train(6000) scale = str(N1) + '_' + str(gR) + '_' + str(dR) out_path = '/root/tensorD_f/data_out_tmp/python_out/tucker_' + scale + '_' + str( time) + '.txt' with open(out_path, 'w') as out: for loss in loss_hist: out.write('%.6f\n' % loss)
def ntucker_run(N1, N2, N3, gR, dR, time): # ntucker X = synthetic_data_tucker([N1, N2, N3], [gR, gR, gR], 0) data_provider = Provider() data_provider.full_tensor = lambda: X env = Environment(data_provider, summary_path='/tmp/ntucker_' + str(N1)) ntucker = NTUCKER_BCU(env) args = NTUCKER_BCU.NTUCKER_Args(ranks=[dR, dR, dR], validation_internal=500, tol=1.0e-4) ntucker.build_model(args) print('\n\nNTucker with %dx%dx%d, gR=%d, dR=%d, time=%d' % (N1, N2, N3, gR, dR, time)) loss_hist = ntucker.train(10000) scale = str(N1) + '_' + str(gR) + '_' + str(dR) out_path = '/root/tensorD_f/data_out_tmp/python_out/ntucker_' + scale + '_' + str( time) + '.txt' with open(out_path, 'w') as out: for loss in loss_hist: out.write('%.6f\n' % loss)
#!/usr/bin/env python # -*- coding: utf-8 -*- # @Time : 2017/10/4 PM8:41 # @Author : Shiloh Leung # @Site : # @File : ncp_demo.py # @Software: PyCharm Community Edition from tensorD.factorization.env import Environment from tensorD.dataproc.provider import Provider from tensorD.factorization.ncp import NCP_BCU from tensorD.demo.DataGenerator import * if __name__ == '__main__': print('=========Train=========') X = synthetic_data_cp([30, 30, 30], 10) data_provider = Provider() data_provider.full_tensor = lambda: X env = Environment(data_provider, summary_path='/tmp/ncp_demo_' + '30') ncp = NCP_BCU(env) args = NCP_BCU.NCP_Args(rank=10, validation_internal=1) ncp.build_model(args) ncp.train(100) factor_matrices = ncp.factors lambdas = ncp.lambdas print('Training ends.\n\n\n')
def main(): w2v = gensim.models.Word2Vec.load( '../data/skip_w2v_model_stemmed') # pre-trained word embedding idf = pickle.load( open('../data/my_idf', 'rb')) # pre-trained idf value of all words in the w2v dictionary records = pickle.load(open("../data/records_final.pkl", 'rb')) print(len(records)) #获取需要推荐的问题 experiments = util.get_class_experiments() print(len(experiments)) csvfile_path = os.path.join(args.output_path, "topclass_expand11-10.csv") #输出结果 csvfile = open(csvfile_path, 'w', newline="") writer = csv.writer(csvfile) writer.writerow( ["question_title", "top5", "ground_truth_intersection", "true_apis"]) #所有问题的api的集合,看这个集合里面是否有答案存在 #统计能进行推荐的问题个数,推荐出来的问题的个数 recommend_num = 0 recommend_success_num = 0 processnum = 0 #统计指标 mrr = 0.0 map = 0.0 precision = 0 recall = 0 ndcg = 0.0 rec_num = args.rec_num start = time.clock() for experiment in experiments: experiment_method_annotation = experiment.method_annotation # print(experiment_method_annotation) experiment_now_method_flat = experiment.now_method_flat experiment_true_api = experiment.true_api experiment_now_api = experiment.now_api # 求差,取出交集 experiment_true_api = set(experiment_true_api) - set( experiment_now_api) query = experiment_method_annotation query_words = WordPunctTokenizer().tokenize(query.lower()) query_words = [ SnowballStemmer('english').stem(word) for word in query_words ] query_matrix = similarity.init_doc_matrix(query_words, w2v) query_idf_vector = similarity.init_doc_idf_vector(query_words, idf) #获取相似的TOP-N问题 top_questions = similarity.get_topk_questions(query_words, query_matrix, query_idf_vector, records, 11, 0.0) #获取得到问题的长度 # print(top_questions) similar_questions_length = len(top_questions) # print("similar_questions_length:",similar_questions_length) #查看现有问题是否在相似问题中,如果不在则加入,否则直接根据相似问题构建张量 flag = False similar_records_list = list(top_questions.keys()) for record in similar_records_list: if (record.title_words == query_words): flag = True processnum += 1 #现有问题在相似问题里面 record_method_annotation_words = list() record_method_flat = list() record_api = list() for record in similar_records_list: if record.title_words not in record_method_annotation_words: record_method_annotation_words.append(record.title_words) if record.method_block_flat not in record_method_flat: record_method_flat.append(record.method_block_flat) for api in record.method_api_sequence: if api not in record_api: record_api.append(api) #加入编程环境中出现的api for now_api in experiment_now_api: if now_api not in record_api: record_api.append(now_api) api_rec_all = [] if flag == True: recommend_num += 1 #构建张量 print(len(record_method_annotation_words), len(record_method_flat), len(record_api)) record_method_annotation_words_dict = dict( zip(range(len(record_method_annotation_words)), record_method_annotation_words)) record_method_flat_dict = dict( zip(range(len(record_method_flat)), record_method_flat)) record_api_dict = dict(zip(range(len(record_api)), record_api)) tensor = np.zeros((len(record_method_annotation_words), len(record_method_flat), len(record_api)), dtype=int) for record in similar_records_list: for concrete_api in record.method_api_sequence: tensor[list(record_method_annotation_words_dict.keys( ))[list(record_method_annotation_words_dict.values()). index(record.title_words)], list(record_method_flat_dict.keys() )[list(record_method_flat_dict.values()). index(record.method_block_flat)], list(record_api_dict.keys( ))[list(record_api_dict.values()).index(concrete_api )]] = 1 for api in experiment_now_api: if api in record_api_dict.values(): tensor[list(record_method_annotation_words_dict.keys( ))[list(record_method_annotation_words_dict.values()). index(query_words)], :, list(record_api_dict.keys( ))[list(record_api_dict.values()).index(api)]] = 1 #处理不是张量的情况 one = query_words if len(record_api) == 0: continue if (len(record_method_annotation_words) == 1 or len(record_method_flat) == 1 or len(record_api) == 1): if (len(record_method_annotation_words) == 1 and len(record_method_flat) == 1 or len(record_method_flat) == 1 and len(record_api) == 1 or len(record_api) == 1 and len(record_method_annotation_words) == 1): api_rec_all = record_api for m in set(experiment_now_api): if m in api_rec_all: api_rec_all.remove(m) elif (len(record_api) == 1): api_rec_all = record_api for m in set(experiment_now_api): if m in api_rec_all: api_rec_all.remove(m) else: if (len(record_method_annotation_words) == 1): matrix = tl.unfold(tensor, mode=1) nmf = nimfa.Nmf(matrix, max_iter=200, rank=round(min(matrix.shape) / 2), update='euclidean', objective='fro') nmf_fit = nmf() W = nmf_fit.basis() H = nmf_fit.coef() matrix = np.dot(W, H) two = list( similarity.get_topk_method_flat( experiment_now_method_flat, list(record_method_flat_dict.values()), 1, 1, -1, 1).values())[0] rec_combine_api_key = np.argsort( -matrix[list(record_method_flat_dict.keys() )[list(record_method_flat_dict.values( )).index(two)], :]).tolist()[0] api_rec_all = [ record_api_dict[i] for i in rec_combine_api_key ] for m in set(experiment_now_api): if m in api_rec_all: api_rec_all.remove(m) elif (len(record_method_flat) == 1): matrix = tl.unfold(tensor, mode=0) nmf = nimfa.Nmf(matrix, max_iter=200, rank=round(min(matrix.shape) / 2), update='euclidean', objective='fro') nmf_fit = nmf() W = nmf_fit.basis() H = nmf_fit.coef() matrix = np.dot(W, H) rec_combine_api_key = np.argsort(-matrix[ list(record_method_annotation_words_dict.keys( ))[list(record_method_annotation_words_dict.values( )).index(one)], :]).tolist()[0] api_rec_all = [ record_api_dict[i] for i in rec_combine_api_key ] for m in set(experiment_now_api): if m in api_rec_all: api_rec_all.remove(m) else: #张量分解 tf.reset_default_graph() tensor = tl.tensor(tensor).astype(np.float32) data_provider = Provider() data_provider.full_tensor = lambda: tensor env = Environment(data_provider, summary_path='/tensor/ncp_ml') ncp = NCP_BCU(env) arg = NCP_BCU.NCP_Args(rank=round( min(len(record_method_annotation_words), len(record_method_flat), len(record_api)) / 2), validation_internal=1) ncp.build_model(arg) loss_hist = ncp.train(100) factor_matrices = ncp.factors full_tensor = tl.kruskal_to_tensor(factor_matrices) two = list( similarity.get_topk_method_flat( experiment_now_method_flat, list(record_method_flat_dict.values()), 1, 1, -1, 1).values())[0] rec_combine_api_key = np.argsort( -full_tensor[list(record_method_annotation_words_dict.keys( ))[list(record_method_annotation_words_dict.values()). index(one)], list(record_method_flat_dict.keys() )[list(record_method_flat_dict.values()). index(two)], :]).tolist() # 推荐的API列表,去除情境中已经含有的api api_rec_all = [record_api_dict[i] for i in rec_combine_api_key] for m in set(experiment_now_api): if m in api_rec_all: api_rec_all.remove(m) #现有问题不在相似问题里面 else: similar_questions_length += 1 #去除找不到相似问题的问题 if similar_questions_length == 1: continue recommend_num += 1 #添加新来的query record_method_annotation_words.append(query_words) print(len(record_method_annotation_words), len(record_method_flat), len(record_api)) #构建张量 record_method_annotation_words_dict = dict( zip(range(len(record_method_annotation_words)), record_method_annotation_words)) record_method_flat_dict = dict( zip(range(len(record_method_flat)), record_method_flat)) record_api_dict = dict(zip(range(len(record_api)), record_api)) tensor = np.zeros((len(record_method_annotation_words), len(record_method_flat), len(record_api)), dtype=int) for record in similar_records_list: for concrete_api in record.method_api_sequence: tensor[list(record_method_annotation_words_dict.keys( ))[list(record_method_annotation_words_dict.values()). index(record.title_words)], list(record_method_flat_dict.keys() )[list(record_method_flat_dict.values()). index(record.method_block_flat)], list(record_api_dict.keys( ))[list(record_api_dict.values()).index(concrete_api )]] = 1 for api in experiment_now_api: if api in record_api_dict.values(): tensor[list(record_method_annotation_words_dict.keys( ))[list(record_method_annotation_words_dict.values()). index(query_words)], :, list(record_api_dict.keys( ))[list(record_api_dict.values()).index(api)]] = 1 #处理不是张量分解 one = query_words if len(record_api) == 0: continue if (len(record_method_annotation_words) == 1 or len(record_method_flat) == 1 or len(record_api) == 1): if (len(record_method_annotation_words) == 1 and len(record_method_flat) == 1 or len(record_method_flat) == 1 and len(record_api) == 1 or len(record_api) == 1 and len(record_method_annotation_words) == 1): api_rec_all = record_api for m in set(experiment_now_api): if m in api_rec_all: api_rec_all.remove(m) elif (len(record_api) == 1): api_rec_all = record_api for m in set(experiment_now_api): if m in api_rec_all: api_rec_all.remove(m) else: if (len(record_method_annotation_words) == 1): matrix = tl.unfold(tensor, mode=1) nmf = nimfa.Nmf(matrix, max_iter=200, rank=round(min(matrix.shape) / 2), update='euclidean', objective='fro') nmf_fit = nmf() W = nmf_fit.basis() H = nmf_fit.coef() matrix = np.dot(W, H) two = list( similarity.get_topk_method_flat( experiment_now_method_flat, list(record_method_flat_dict.values()), 1, 1, -1, 1).values())[0] rec_combine_api_key = np.argsort( -matrix[list(record_method_flat_dict.keys() )[list(record_method_flat_dict.values( )).index(two)], :]).tolist()[0] api_rec_all = [ record_api_dict[i] for i in rec_combine_api_key ] for m in set(experiment_now_api): if m in api_rec_all: api_rec_all.remove(m) elif (len(record_method_flat) == 1): matrix = tl.unfold(tensor, mode=0) nmf = nimfa.Nmf(matrix, max_iter=200, rank=round(min(matrix.shape) / 2), update='euclidean', objective='fro') nmf_fit = nmf() W = nmf_fit.basis() H = nmf_fit.coef() matrix = np.dot(W, H) rec_combine_api_key = np.argsort(-matrix[ list(record_method_annotation_words_dict.keys( ))[list(record_method_annotation_words_dict.values( )).index(one)], :]).tolist()[0] api_rec_all = [ record_api_dict[i] for i in rec_combine_api_key ] for m in set(experiment_now_api): if m in api_rec_all: api_rec_all.remove(m) else: # 张量分解 tf.reset_default_graph() tensor = tl.tensor(tensor).astype(np.float32) data_provider = Provider() data_provider.full_tensor = lambda: tensor env = Environment(data_provider, summary_path='/tensor/ncp_ml') ncp = NCP_BCU(env) arg = NCP_BCU.NCP_Args(rank=round( min(len(record_method_annotation_words), len(record_method_flat), len(record_api)) / 2), validation_internal=1) ncp.build_model(arg) loss_hist = ncp.train(100) factor_matrices = ncp.factors full_tensor = tl.kruskal_to_tensor(factor_matrices) # one = query_words two = list( similarity.get_topk_method_flat( experiment_now_method_flat, list(record_method_flat_dict.values()), 1, 1, -1, 1).values())[0] rec_combine_api_key = np.argsort( -full_tensor[list(record_method_annotation_words_dict.keys( ))[list(record_method_annotation_words_dict.values()). index(one)], list(record_method_flat_dict.keys() )[list(record_method_flat_dict.values()). index(two)], :]).tolist() #推荐的API列表 api_rec_all = [record_api_dict[i] for i in rec_combine_api_key] for m in set(experiment_now_api): if m in api_rec_all: api_rec_all.remove(m) #判断结果在相似的问题中有没有出现 # print(experiment_true_api) # print('----------------------------------') experiment_true_api = [ true_api.split('.')[-2] for true_api in experiment_true_api ] experiment_true_api = removelist(experiment_true_api) experiment_now_api = [ true_api.split('.')[-2] for true_api in experiment_now_api ] experiment_now_api = removelist(experiment_now_api) #去除experiment_now_api experiment_true_api = set(experiment_true_api) - set( experiment_now_api) record_api = [true_api.split('.')[-2] for true_api in record_api] record_api = removelist(record_api) api_rec_all = [true_api.split('.')[-2] for true_api in api_rec_all] api_rec_all = removelist(api_rec_all) for m in set(experiment_now_api): if m in api_rec_all: api_rec_all.remove(m) api_rec = api_rec_all[:rec_num] pos = -1 tmp_map = 0.0 hits = 0.0 vector = list() for i, api in enumerate(api_rec_all[:rec_num]): if api in set(experiment_true_api) and pos == -1: pos = i + 1 if api in set(experiment_true_api): vector.append(1) hits += 1 tmp_map += hits / (i + 1) else: vector.append(0) tmp_map /= len(set(experiment_true_api)) tmp_mrr = 0.0 if pos != -1: tmp_mrr = 1.0 / pos map += tmp_map mrr += tmp_mrr ndcg += calculateNDCG.ndcg_at_k(vector[:rec_num], rec_num) ground_truth_intersection = set(api_rec).intersection( set(experiment_true_api)) if (len(ground_truth_intersection) > 0): recommend_success_num += 1 precision += len(ground_truth_intersection) / rec_num recall += len(ground_truth_intersection) / len( set(experiment_true_api)) writer.writerow([ experiment_method_annotation, api_rec, ground_truth_intersection, experiment_true_api ]) writer.writerow(["recommend_num", "recommend_success_num"]) writer.writerow([recommend_num, recommend_success_num]) writer.writerow([ "mrr/recommend_num", "recommend_num", "map/recommend_num", "success_rate@N", "precision@N/recommend_num", "recall@N/recommend_num", "ndcg/recommend_num" ]), writer.writerow([ mrr / recommend_num, recommend_num, map / recommend_num, recommend_success_num / recommend_num, precision / recommend_num, recall / recommend_num, ndcg / recommend_num ]) csvfile.close() end = time.clock() print('Running time: %s Seconds' % (end - start)) logging.info("Finish")
from tensorD.factorization.env import Environment from tensorD.factorization.pitf_numpy import PITF_np from tensorD.factorization.tucker import * from tensorD.dataproc.provider import Provider #from tensorD.dataproc.reader import TensorReader import tensorflow as tf import numpy as np if __name__ == '__main__': data_provider = Provider() data_provider.full_tensor = lambda: tf.constant( np.random.rand(50, 50, 8) * 10, dtype=tf.float32) pitf_np_env = Environment(data_provider, summary_path='/tmp/tensord') pitf_np = PITF_np(pitf_np_env) sess_t = pitf_np_env.sess init_op = tf.global_variables_initializer() sess_t.run(init_op) tensor = pitf_np_env.full_data().eval(session=sess_t) args = PITF_np.PITF_np_Args(rank=5, delt=0.8, tao=12, sample_num=100, validation_internal=1, verbose=False, steps=500) y, X_t, Y_t, Z_t, Ef_t, If_t, Rf_t = pitf_np.exact_recovery(args, tensor) y = tf.convert_to_tensor(y) X = tf.convert_to_tensor(X_t) Y = tf.convert_to_tensor(Y_t) Z = tf.convert_to_tensor(Z_t) Ef = tf.convert_to_tensor(Ef_t)