def kfold_results(y_true, y_pred, nfolds): kf = KFold(n_splits=nfolds, random_state=0) accuracy, precision, recall, f1, auc = list(), list(), list(), list( ), list() for train_index, test_index in kf.split(y_true): y_true_test_index, y_pred_test_index = get_items(items=y_true, indexes=test_index), \ get_items(items=y_pred, indexes=test_index) accuracy.append( accuracy_score(y_true=y_true_test_index, y_pred=y_pred_test_index)) precision.append( precision_score(y_true=y_true_test_index, y_pred=y_pred_test_index)) recall.append( recall_score(y_true=y_true_test_index, y_pred=y_pred_test_index)) f1.append(f1_score(y_true=y_true_test_index, y_pred=y_pred_test_index)) auc.append( auc_score(y_true=y_true_test_index, y_pred=y_pred_test_index)) algorithm = "" print "Accuracy and std of %s: %f %f" % ( algorithm, np.mean(np.array(accuracy)), np.std(np.array(accuracy))) print "Precision of %s: %f %f" % (algorithm, np.mean( np.array(precision)), np.std(np.array(precision))) print "Recall of %s: %f %f" % (algorithm, np.mean( np.array(recall)), np.std(np.array(recall))) print "F1 of %s: %f %f" % (algorithm, np.mean( np.array(f1)), np.std(np.array(f1))) print "AUC of %s: %f %f" % (algorithm, np.mean( np.array(auc)), np.std(np.array(auc)))
def get_predict(name, X, y, algorithm, folds): kf = KFold(n_splits=folds, random_state=0) kf.get_n_splits(X=X) auc, accuracy, precision, recall, f1 = list(), list(), list(), list(), list() pred_dict = dict() for train_index, test_index in kf.split(X): X_train, y_train = get_items(items=X, indexes=train_index), get_items(items=y, indexes=train_index) X_test, y_test = get_items(items=X, indexes=test_index), get_items(items=y, indexes=test_index) vectorizer = CountVectorizer() X_train = vectorizer.fit_transform(X_train) X_test = vectorizer.transform(X_test) if algorithm == "svm": clf = LinearSVC(random_state=0) elif algorithm == "lr": clf = LogisticRegression() elif algorithm == "dt": clf = DecisionTreeClassifier() else: print "Wrong algorithm name -- please retype again" exit() clf.fit(X=X_train, y=y_train) y_pred = clf.predict(X_test) pred_dict.update(make_dictionary(y_pred=y_pred, y_index=test_index)) # path_file = "./statistical_test/" + name + "_" + algorithm + ".txt" # path_file = "./statistical_test/3_mar7/" + name + "_" + algorithm + ".txt" # path_file = "./statistical_test_ver2/3_mar7" + name + "_" + algorithm + ".txt" # write_file(path_file, y_pred) accuracy.append(accuracy_score(y_true=y_test, y_pred=y_pred)) precision.append(precision_score(y_true=y_test, y_pred=y_pred)) recall.append(recall_score(y_true=y_test, y_pred=y_pred)) f1.append(f1_score(y_true=y_test, y_pred=y_pred)) auc.append(auc_score(y_true=y_test, y_pred=y_pred)) path_file = "./statistical_test_ver2/3_mar7/" + name + "_" + algorithm + ".txt" write_file(path_file=path_file, data=sorted_dict(dict=pred_dict)) print "Accuracy and std of %s: %f %f" % (algorithm, np.mean(np.array(accuracy)), np.std(np.array(accuracy))) print "Precision of %s: %f %f" % (algorithm, np.mean(np.array(precision)), np.std(np.array(precision))) print "Recall of %s: %f %f" % (algorithm, np.mean(np.array(recall)), np.std(np.array(recall))) print "F1 of %s: %f %f" % (algorithm, np.mean(np.array(f1)), np.std(np.array(f1))) print "AUC of %s: %f %f" % (algorithm, np.mean(np.array(auc)), np.std(np.array(auc)))
def loading_data_lstm(FLAGS): print FLAGS.model if "msg" in FLAGS.model: commits_ = extract_commit(path_file=FLAGS.path) filter_commits = filtering_commit(commits=commits_, num_file=FLAGS.code_file, num_hunk=FLAGS.code_hunk, num_loc=FLAGS.code_line, size_line=FLAGS.code_length) msgs_, codes_ = extract_msg(commits=filter_commits), extract_code(commits=filter_commits) dict_msg_, dict_code_ = dictionary(data=msgs_), dictionary(data=codes_) pad_msg = mapping_commit_msg(msgs=msgs_, max_length=FLAGS.msg_length, dict_msg=dict_msg_) pad_added_code = mapping_commit_code(type="added", commits=filter_commits, max_hunk=FLAGS.code_hunk, max_code_line=FLAGS.code_line, max_code_length=FLAGS.code_length, dict_code=dict_code_) pad_removed_code = mapping_commit_code(type="removed", commits=filter_commits, max_hunk=FLAGS.code_hunk, max_code_line=FLAGS.code_line, max_code_length=FLAGS.code_length, dict_code=dict_code_) labels = load_label_commits(commits=filter_commits) elif "all" in FLAGS.model: commits_ = extract_commit(path_file=FLAGS.path) filter_commits = filtering_commit(commits=commits_, num_file=FLAGS.code_file, num_hunk=FLAGS.code_hunk, num_loc=FLAGS.code_line, size_line=FLAGS.code_length) msgs_, codes_ = extract_msg(commits=filter_commits), extract_code(commits=filter_commits) all_lines = add_two_list(list1=msgs_, list2=codes_) msgs_ = all_lines dict_msg_, dict_code_ = dictionary(data=msgs_), dictionary(data=codes_) pad_msg = mapping_commit_msg(msgs=msgs_, max_length=FLAGS.msg_length, dict_msg=dict_msg_) pad_added_code = mapping_commit_code(type="added", commits=filter_commits, max_hunk=FLAGS.code_hunk, max_code_line=FLAGS.code_line, max_code_length=FLAGS.code_length, dict_code=dict_code_) pad_removed_code = mapping_commit_code(type="removed", commits=filter_commits, max_hunk=FLAGS.code_hunk, max_code_line=FLAGS.code_line, max_code_length=FLAGS.code_length, dict_code=dict_code_) labels = load_label_commits(commits=filter_commits) elif "code" in FLAGS.model: commits_ = extract_commit(path_file=FLAGS.path) filter_commits = filtering_commit(commits=commits_, num_file=FLAGS.code_file, num_hunk=FLAGS.code_hunk, num_loc=FLAGS.code_line, size_line=FLAGS.code_length) msgs_, codes_ = extract_msg(commits=filter_commits), extract_code(commits=filter_commits) msgs_ = codes_ dict_msg_, dict_code_ = dictionary(data=msgs_), dictionary(data=codes_) pad_msg = mapping_commit_msg(msgs=msgs_, max_length=FLAGS.msg_length, dict_msg=dict_msg_) pad_added_code = mapping_commit_code(type="added", commits=filter_commits, max_hunk=FLAGS.code_hunk, max_code_line=FLAGS.code_line, max_code_length=FLAGS.code_length, dict_code=dict_code_) pad_removed_code = mapping_commit_code(type="removed", commits=filter_commits, max_hunk=FLAGS.code_hunk, max_code_line=FLAGS.code_line, max_code_length=FLAGS.code_length, dict_code=dict_code_) labels = load_label_commits(commits=filter_commits) else: print "You need to type correct model" exit() kf = KFold(n_splits=FLAGS.folds, random_state=FLAGS.seed) for train_index, test_index in kf.split(filter_commits): X_train_msg, X_test_msg = np.array(get_items(items=pad_msg, indexes=train_index)), \ np.array(get_items(items=pad_msg, indexes=test_index)) X_train_added_code, X_test_added_code = np.array(get_items(items=pad_added_code, indexes=train_index)), \ np.array(get_items(items=pad_added_code, indexes=test_index)) X_train_removed_code, X_test_removed_code = np.array(get_items(items=pad_removed_code, indexes=train_index)), \ np.array(get_items(items=pad_removed_code, indexes=test_index)) y_train, y_test = np.array(get_items(items=labels, indexes=train_index)), \ np.array(get_items(items=labels, indexes=test_index)) return X_test_msg, X_test_added_code, X_test_removed_code, y_test
def eval_PatchNet_fold(tf, checkpoint_dir, fold_num, fold_index, pad_msg, pad_added_code, pad_removed_code, labels): FLAGS = tf.flags.FLAGS allow_soft_placement = True # "Allow device soft device placement" log_device_placement = False # "Log placement of ops on devices" train_index, test_index = fold_index["train"], fold_index["test"] X_train_msg, X_test_msg = np.array(get_items(items=pad_msg, indexes=train_index)), \ np.array(get_items(items=pad_msg, indexes=test_index)) X_train_added_code, X_test_added_code = np.array(get_items(items=pad_added_code, indexes=train_index)), \ np.array(get_items(items=pad_added_code, indexes=test_index)) X_train_removed_code, X_test_removed_code = np.array(get_items(items=pad_removed_code, indexes=train_index)), \ np.array(get_items(items=pad_removed_code, indexes=test_index)) y_train, y_test = np.array(get_items(items=labels, indexes=train_index)), \ np.array(get_items(items=labels, indexes=test_index)) print X_train_msg.shape, X_test_msg.shape print X_train_added_code.shape, X_test_added_code.shape print X_train_removed_code.shape, X_test_removed_code.shape print y_train.shape, y_test.shape dirs = get_all_checkpoints(checkpoint_dir=checkpoint_dir) graph = tf.Graph() for checkpoint_file in dirs: # checkpoint_file = checkpoint_file.replace("thonghoang", "jameshoang") # checkpoint_file = checkpoint_file.replace("runs", "runs_all_512_[1, 2]_32") # print checkpoint_file with graph.as_default(): session_conf = tf.ConfigProto( allow_soft_placement=allow_soft_placement, log_device_placement=log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): # Load the saved meta graph and restore variables saver = tf.train.import_meta_graph( "{}.meta".format(checkpoint_file)) saver.restore(sess, checkpoint_file) # Get the placeholders from the graph by name input_msg = graph.get_operation_by_name("input_msg").outputs[0] input_addedcode = graph.get_operation_by_name( "input_addedcode").outputs[0] input_removedcode = graph.get_operation_by_name( "input_removedcode").outputs[0] dropout_keep_prob = graph.get_operation_by_name( "dropout_keep_prob").outputs[0] # Tensors we want to evaluate predictions = graph.get_operation_by_name( "output/predictions").outputs[0] scores = graph.get_operation_by_name( "output/scores").outputs[0] # Generate batches for one epoch batches = mini_batches(X_msg=X_test_msg, X_added_code=X_test_added_code, X_removed_code=X_test_removed_code, Y=y_test, mini_batch_size=FLAGS.batch_size) # Collect the predictions here all_predictions, all_scores = [], [] for batch in batches: batch_input_msg, batch_input_added_code, batch_input_removed_code, batch_input_labels = batch batch_predictions = sess.run( predictions, { input_msg: batch_input_msg, input_addedcode: batch_input_added_code, input_removedcode: batch_input_removed_code, dropout_keep_prob: 1.0 }) # print batch_predictions.shape all_predictions = np.concatenate( [all_predictions, batch_predictions]) batch_scores = sess.run( scores, { input_msg: batch_input_msg, input_addedcode: batch_input_added_code, input_removedcode: batch_input_removed_code, dropout_keep_prob: 1.0 }) batch_scores = np.ravel(softmax(batch_scores)[:, [1]]) # print batch_scores.shape all_scores = np.concatenate([all_scores, batch_scores]) split_checkpoint_file = checkpoint_file.split("/") path_write = "./patchNet_results/%s_%s.txt" % ( split_checkpoint_file[-3], split_checkpoint_file[-1]) write_file(path_file=path_write, data=all_scores) print checkpoint_file, "Accuracy:", accuracy_score( y_true=convert_to_binary(y_test), y_pred=all_predictions) print checkpoint_file, "Precision:", precision_score( y_true=convert_to_binary(y_test), y_pred=all_predictions) print checkpoint_file, "Recall:", recall_score( y_true=convert_to_binary(y_test), y_pred=all_predictions) print checkpoint_file, "F1:", f1_score( y_true=convert_to_binary(y_test), y_pred=all_predictions) print checkpoint_file, "AUC:", auc_score( y_true=convert_to_binary(y_test), y_pred=all_predictions) print "\n"
commits_label_ = [c["stable"] for c in commits_] dl_labels = load_file("./statistical_test/lstm_cnn_all.txt") dl_labels = [float(l) for l in dl_labels] fold_index = split_train_test(data=commits_, folds=5, random_state=None) accuracy_, precision_, recall_, f1_, auc_ = list(), list(), list(), list( ), list() probs, true_pos, false_neg = list(), list(), list() threshold = 0.5 for f in files_path: print f test_fold = fold_index[check_fold(string=f) - 1]["test"] test_commit_id, test_commit_label = get_items( items=commits_id_, indexes=test_fold), get_items(items=commits_label_, indexes=test_fold) test_dl_labels = get_items(items=dl_labels, indexes=test_fold) print len(test_commit_id), len(test_commit_label), len(test_dl_labels) # acc, prc, rc, f1, auc, prob, tp, fn = load_results(id_gt=test_commit_id, label_gt=test_commit_label, # single_file=f) acc, prc, rc, f1, auc, prob, tp, fn = load_results( id_gt=test_commit_id, label_gt=test_commit_label, single_file=f, threshold=threshold) accuracy_.append(acc) precision_.append(prc) recall_.append(rc)
# path_pred = "./statistical_test_prob/lstm_cnn_all_fold_0.txt" # path_pred = "./statistical_test_prob/lstm_cnn_all_check_fold_0.txt" # path_pred = "./statistical_test_prob/lstm_cnn_all_checking.txt" # path_pred = "./statistical_test/lstm_cnn_all_ver2.txt" # path_pred = "./statistical_test_prob_ver3/PatchNet.txt" # path_pred = "./statistical_test_prob_ver3/LPU-SVM.txt" path_pred = "./statistical_test_prob_ver3/LS-CNN.txt" # path_pred, threshold = "./statistical_test_prob_ver3/sasha_results.txt", 50 y_pred = load_file(path_file=path_pred) if "sasha" in path_pred: y_pred = np.array([1 if float(y) > threshold else 0 for y in y_pred]) else: y_pred = np.array([1 if float(y) > 0.5 else 0 for y in y_pred]) # y_pred = np.array([float(y) for y in y_pred]) # y_pred[y_pred > 0.5] = 1 # y_pred[y_pred <= 0.5] = 0 split_data = split_train_test(data=y_true, folds=folds, random_state=None) for i in xrange(len(split_data)): train_index, test_index = split_data[i]["train"], split_data[i]["test"] y_true_, y_pred_ = get_items( items=y_true, indexes=test_index), get_items(items=y_pred, indexes=test_index) acc = accuracy_score(y_true=y_true_, y_pred=y_pred_) prc = precision_score(y_true=y_true_, y_pred=y_pred_) rc = recall_score(y_true=y_true_, y_pred=y_pred_) f1 = f1_score(y_true=y_true_, y_pred=y_pred_) auc = auc_score(y_true=y_true_, y_pred=y_pred_) print acc, prc, rc, f1, auc
def training_model(tf, timestamp, fold_num, fold_index, pad_msg, pad_added_code, pad_removed_code, labels, dict_msg, dict_code): FLAGS = tf.flags.FLAGS train_index, test_index = fold_index["train"], fold_index["test"] X_train_msg, X_test_msg = np.array(get_items(items=pad_msg, indexes=train_index)), \ np.array(get_items(items=pad_msg, indexes=test_index)) X_train_added_code, X_test_added_code = np.array(get_items(items=pad_added_code, indexes=train_index)), \ np.array(get_items(items=pad_added_code, indexes=test_index)) X_train_removed_code, X_test_removed_code = np.array(get_items(items=pad_removed_code, indexes=train_index)), \ np.array(get_items(items=pad_removed_code, indexes=test_index)) y_train, y_test = np.array(get_items(items=labels, indexes=train_index)), \ np.array(get_items(items=labels, indexes=test_index)) # y_train, y_test = convert_to_binary(labels), convert_to_binary(labels) # y_train, y_test = y_train.reshape((len(labels), 1)), y_test.reshape((len(labels), 1)) print X_train_msg.shape, X_test_msg.shape print X_train_added_code.shape, X_test_added_code.shape print X_train_removed_code.shape, X_test_removed_code.shape print y_train.shape, y_test.shape with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): model = PatchNet(max_msg_length=FLAGS.msg_length, max_code_length=FLAGS.code_length, max_code_line=FLAGS.code_line, max_code_hunk=FLAGS.code_hunk, vocab_size_text=len(dict_msg), vocab_size_code=len(dict_code), embedding_size_text=FLAGS.embedding_dim_text, filter_sizes=list( map(int, FLAGS.filter_sizes.split(","))), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda, num_classes=y_train.shape[1], hidden_units=FLAGS.hidden_units) model.build_graph(model=FLAGS.model) # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) test_step = 0 optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate) grads_and_vars = optimizer.compute_gradients(model.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # out_dir = os.path.abspath(os.path.join(os.path.curdir, "runs", "fold_" + str(fold_num) + "_" + timestamp)) out_dir = os.path.abspath( os.path.join(os.path.curdir, "runs", timestamp + "_" + "fold_" + str(fold_num))) print("Writing to {}\n".format(out_dir)) # Summaries for loss and accuracy loss_summary = tf.summary.scalar("loss", model.loss) acc_summary = tf.summary.scalar("accuracy", model.accuracy) # Train Summaries train_summary_op = tf.summary.merge([loss_summary, acc_summary]) train_summary_dir = os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.summary.FileWriter( train_summary_dir, sess.graph) # Dev Summaries dev_summary_op = tf.summary.merge([loss_summary, acc_summary]) dev_summary_dir = os.path.join(out_dir, "summaries", "dev") dev_summary_writer = tf.summary.FileWriter(dev_summary_dir, sess.graph) # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it checkpoint_dir = os.path.abspath( os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints) # Initialize all variables sess.run(tf.global_variables_initializer()) def train_step(input_msg, input_added_code, input_removed_code, input_labels): """ A training step """ feed_dict = { model.input_msg: input_msg, model.input_addedcode: input_added_code, model.input_removedcode: input_removed_code, model.input_y: input_labels, model.dropout_keep_prob: FLAGS.dropout_keep_prob } _, step, summaries, loss, accuracy = sess.run([ train_op, global_step, train_summary_op, model.loss, model.accuracy ], feed_dict) time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {:g}, acc {:g}".format( time_str, step, loss, accuracy)) train_summary_writer.add_summary(summaries, step) def dev_step(input_msg, input_added_code, input_removed_code, input_labels, step, len_train_batch): """ A testing step """ mini_batches = random_mini_batch( X_msg=input_msg, X_added_code=input_added_code, X_removed_code=input_removed_code, Y=input_labels, mini_batch_size=FLAGS.batch_size) slope = len_train_batch / float(len(mini_batches)) accs, losses = list(), list() for batch in mini_batches: test_input_msg, test_input_added_code, test_input_removed_code, test_input_labels = batch feed_dict = { model.input_msg: test_input_msg, model.input_addedcode: test_input_added_code, model.input_removedcode: test_input_removed_code, model.input_y: test_input_labels, model.dropout_keep_prob: 1.0 } summaries, loss, accuracy = sess.run( [dev_summary_op, model.loss, model.accuracy], feed_dict) accs.append(accuracy) losses.append(loss) if step * FLAGS.folds == 0: dev_summary_writer.add_summary(summaries, 1) # print "step {}".format(1) else: dev_summary_writer.add_summary(summaries, step * slope + 1) # print "step {}".format(step * slope) step += 1 time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {:g}, acc {:g}".format( time_str, step, sum(losses) / float(len(losses)), sum(accs) / float(len(accs)))) return step for i in xrange(0, FLAGS.num_epochs): # Generate batches mini_batches = random_mini_batch( X_msg=X_train_msg, X_added_code=X_train_added_code, X_removed_code=X_train_removed_code, Y=y_train, mini_batch_size=FLAGS.batch_size) saving_step = int(len(mini_batches) / 3) for j in xrange(len(mini_batches)): batch = mini_batches[j] input_msg, input_added_code, input_removed_code, input_labels = batch train_step(input_msg, input_added_code, input_removed_code, input_labels) current_step = tf.train.global_step(sess, global_step) if j == (len(mini_batches) - 1): print "\nEpoch:%i" % i print("\nEvaluation:") test_step = dev_step( input_msg=X_test_msg, input_added_code=X_test_added_code, input_removed_code=X_test_removed_code, input_labels=y_test, step=test_step, len_train_batch=len(mini_batches)) print("") path = saver.save(sess, checkpoint_prefix, global_step=current_step) print "Saved model checkpoint to {}\n".format(path) elif (j + 1) % saving_step == 0: path = saver.save(sess, checkpoint_prefix, global_step=current_step) print "Saved model checkpoint to {}\n".format(path)
def loading_baseline_july(tf, folds, random_state): FLAGS = tf.flags.FLAGS commits_ = extract_commit_july(path_file=FLAGS.path) filter_commits = commits_ print len(commits_) kf = KFold(n_splits=folds, random_state=random_state) idx_folds = list() for train_index, test_index in kf.split(filter_commits): idx = dict() idx["train"], idx["test"] = train_index, test_index idx_folds.append(idx) if "msg" in FLAGS.model: msgs_, codes_ = extract_msg(commits=filter_commits), extract_code( commits=filter_commits) elif "all" in FLAGS.model: msgs_, codes_ = extract_msg(commits=filter_commits), extract_code( commits=filter_commits) all_lines = add_two_list(list1=msgs_, list2=codes_) msgs_ = all_lines elif "code" in FLAGS.model: msgs_, codes_ = extract_msg(commits=filter_commits), extract_code( commits=filter_commits) msgs_ = codes_ else: print "You need to type correct model" exit() dict_msg_, dict_code_ = dictionary(data=msgs_), dictionary(data=codes_) pad_msg = mapping_commit_msg(msgs=msgs_, max_length=FLAGS.msg_length, dict_msg=dict_msg_) labels = load_label_commits(commits=filter_commits) labels = convert_to_binary(labels) # path_file = "./statistical_test_prob/true_label.txt" # write_file(path_file=path_file, data=labels) # exit() print pad_msg.shape, labels.shape, len(dict_msg_) cntfold = 0 pred_dict = dict() pred_dict_list = list() for i in xrange(cntfold, len(idx_folds)): idx = idx_folds[i] train_index, test_index = idx["train"], idx["test"] X_train_msg, X_test_msg = np.array(get_items(items=pad_msg, indexes=train_index)), \ np.array(get_items(items=pad_msg, indexes=test_index)) Y_train, Y_test = np.array(get_items(items=labels, indexes=train_index)), \ np.array(get_items(items=labels, indexes=test_index)) if FLAGS.model == "lstm_cnn_all" or FLAGS.model == "lstm_cnn_msg" \ or FLAGS.model == "lstm_cnn_code" or FLAGS.model == "cnn_all" \ or FLAGS.model == "cnn_msg" or FLAGS.model == "cnn_code": # path_model = "./keras_model/%s_%s.h5" % (FLAGS.model, str(cntfold)) path_model = "./keras_model/test_%s_%s.h5" % (FLAGS.model, str(cntfold)) # path_model = "./keras_model/%s_%s_testing.h5" % (FLAGS.model, str(cntfold)) model = load_model(path_model) else: print "You need to give correct model name" exit() y_pred = model.predict(X_test_msg, batch_size=FLAGS.batch_size) y_pred = np.ravel(y_pred) pred_dict.update(make_dictionary(y_pred=y_pred, y_index=test_index)) y_pred = y_pred.tolist() pred_dict_list += y_pred # print len(pred_dict_list) # exit() # path_file = "./statistical_test_prob/" + FLAGS.model + ".txt" # write_file(path_file=path_file, data=sorted_dict(dict=pred_dict)) path_file = "./statistical_test_prob/" + FLAGS.model + "_checking.txt" write_file(path_file=path_file, data=pred_dict_list)
dict_msg_, dict_code_ = dictionary(data=msgs_), dictionary(data=codes_) pad_msg = mapping_commit_msg(msgs=msgs_, max_length=FLAGS.msg_length, dict_msg=dict_msg_) labels = load_label_commits(commits=filter_commits) labels = convert_to_binary(labels) print pad_msg.shape, labels.shape, len(dict_msg_) folds = 10 kf = KFold(n_splits=folds, random_state=FLAGS.seed) cntfold = 0 timestamp = str(int(time.time())) accuracy, precision, recall, f1, auc = list(), list(), list(), list( ), list() pred_dict = dict() for train_index, test_index in kf.split(filter_commits): X_train_msg, X_test_msg = np.array(get_items(items=pad_msg, indexes=train_index)), \ np.array(get_items(items=pad_msg, indexes=test_index)) Y_train, Y_test = np.array(get_items(items=labels, indexes=train_index)), \ np.array(get_items(items=labels, indexes=test_index)) if FLAGS.model == "lstm_msg" or FLAGS.model == "lstm_code" or FLAGS.model == "lstm_all": model = lstm_model(x_train=X_train_msg, y_train=Y_train, x_test=X_test_msg, y_test=Y_test, dictionary_size=len(dict_msg_), FLAGS=FLAGS) elif FLAGS.model == "bi_lstm_msg" or FLAGS.model == "bi_lstm_code" or FLAGS.model == "bi_lstm_all": model = bi_lstm_model(x_train=X_train_msg, y_train=Y_train, x_test=X_test_msg,
def running_baseline_july(tf, folds, random_state): FLAGS = tf.flags.FLAGS commits_ = extract_commit_july(path_file=FLAGS.path) filter_commits = commits_ print len(commits_) kf = KFold(n_splits=folds, random_state=random_state) idx_folds = list() for train_index, test_index in kf.split(filter_commits): idx = dict() idx["train"], idx["test"] = train_index, test_index idx_folds.append(idx) if "msg" in FLAGS.model: msgs_, codes_ = extract_msg(commits=filter_commits), extract_code(commits=filter_commits) elif "all" in FLAGS.model: msgs_, codes_ = extract_msg(commits=filter_commits), extract_code(commits=filter_commits) all_lines = add_two_list(list1=msgs_, list2=codes_) msgs_ = all_lines elif "code" in FLAGS.model: msgs_, codes_ = extract_msg(commits=filter_commits), extract_code(commits=filter_commits) msgs_ = codes_ else: print "You need to type correct model" exit() dict_msg_, dict_code_ = dictionary(data=msgs_), dictionary(data=codes_) pad_msg = mapping_commit_msg(msgs=msgs_, max_length=FLAGS.msg_length, dict_msg=dict_msg_) labels = load_label_commits(commits=filter_commits) labels = convert_to_binary(labels) print pad_msg.shape, labels.shape, len(dict_msg_) # exit() timestamp = str(int(time.time())) accuracy, precision, recall, f1, auc = list(), list(), list(), list(), list() cntfold = 0 pred_dict, pred_dict_prob = dict(), dict() for i in xrange(cntfold, len(idx_folds)): idx = idx_folds[i] train_index, test_index = idx["train"], idx["test"] X_train_msg, X_test_msg = np.array(get_items(items=pad_msg, indexes=train_index)), \ np.array(get_items(items=pad_msg, indexes=test_index)) Y_train, Y_test = np.array(get_items(items=labels, indexes=train_index)), \ np.array(get_items(items=labels, indexes=test_index)) if FLAGS.model == "lstm_cnn_msg" or FLAGS.model == "lstm_cnn_code" or FLAGS.model == "lstm_cnn_all": model = lstm_cnn(x_train=X_train_msg, y_train=Y_train, x_test=X_test_msg, y_test=Y_test, dictionary_size=len(dict_msg_), FLAGS=FLAGS) elif FLAGS.model == "cnn_msg" or FLAGS.model == "cnn_code" or FLAGS.model == "cnn_all": model = cnn_model(x_train=X_train_msg, y_train=Y_train, x_test=X_test_msg, y_test=Y_test, dictionary_size=len(dict_msg_), FLAGS=FLAGS) else: print "You need to give correct model name" exit() # model.save("./keras_model/" + FLAGS.model + "_" + str(cntfold) + ".h5") # model.save("./keras_model/" + FLAGS.model + "_" + str(cntfold) + "_testing.h5") # model.save("./keras_model/test_" + FLAGS.model + "_" + str(cntfold) + ".h5") model.save("./keras_model/newres_funcalls_" + FLAGS.model + "_" + str(cntfold) + ".h5") y_pred = model.predict(X_test_msg, batch_size=FLAGS.batch_size) y_pred = np.ravel(y_pred) y_pred_tolist = y_pred.tolist() data_fold = [str(i) + "\t" + str(l) for i, l in zip(test_index, y_pred)] path_file = "./statistical_test/newres_funcalls_%s_fold_%s.txt" % (FLAGS.model, str(cntfold)) write_file(path_file=path_file, data=data_fold) y_pred[y_pred > 0.5] = 1 y_pred[y_pred <= 0.5] = 0 pred_dict.update(make_dictionary(y_pred=y_pred, y_index=test_index)) accuracy.append(accuracy_score(y_true=Y_test, y_pred=y_pred)) precision.append(precision_score(y_true=Y_test, y_pred=y_pred)) recall.append(recall_score(y_true=Y_test, y_pred=y_pred)) f1.append(f1_score(y_true=Y_test, y_pred=y_pred)) auc.append(auc_score(y_true=Y_test, y_pred=y_pred)) print "accuracy", accuracy_score(y_true=Y_test, y_pred=y_pred) print "precision", precision_score(y_true=Y_test, y_pred=y_pred) print "recall", recall_score(y_true=Y_test, y_pred=y_pred) print "f1", f1_score(y_true=Y_test, y_pred=y_pred) cntfold += 1 break
def cross_validation_ver2(id, X, y, algorithm, folds): kf = KFold(n_splits=folds, random_state=None) kf.get_n_splits(X=X) accuracy, precision, recall, f1 = list(), list(), list(), list() probs = list() for train_index, test_index in kf.split(X): X_train, y_train = get_items(items=X, indexes=train_index), get_items( items=y, indexes=train_index) X_test, y_test = get_items(items=X, indexes=test_index), get_items( items=y, indexes=test_index) id_train, id_test = get_items( items=id, indexes=train_index), get_items(items=id, indexes=test_index) vectorizer = CountVectorizer() X_train = vectorizer.fit_transform(X_train) X_test = vectorizer.transform(X_test) # X = vectorizer.transform(X) # eval_train, eval_labels = loading_data("./data/3_mar7/typeaddres.out") # eval_train = vectorizer.transform(eval_train) if algorithm == "svm": clf = LinearSVC(random_state=0) elif algorithm == "lr": clf = LogisticRegression() elif algorithm == "dt": clf = DecisionTreeClassifier() else: print "Wrong algorithm name -- please retype again" exit() clf.fit(X=X_train, y=y_train) accuracy.append( accuracy_score(y_true=y_test, y_pred=clf.predict(X_test))) precision.append( precision_score(y_true=y_test, y_pred=clf.predict(X_test))) recall.append(recall_score(y_true=y_test, y_pred=clf.predict(X_test))) f1.append(f1_score(y_true=y_test, y_pred=clf.predict(X_test))) # print accuracy, precision, recall, f1 # print X_test.shape # y_pred = clf.predict(X_test) # y_pred_proba = clf.predict_proba(X_test)[:, 1] # y_pred_log_proba = clf.predict_log_proba(X_test) # print clf.predict_proba(X_test).shape # print clf.predict_log_proba(X_test).shape # exit() # probs += clf.predict_proba(X_test)[:, 1] probs = np.concatenate((probs, clf.predict_proba(X_test)[:, 1]), axis=0) # accuracy.append(accuracy_score(y_true=eval_labels, y_pred=clf.predict(eval_train))) # precision.append(precision_score(y_true=eval_labels, y_pred=clf.predict(eval_train))) # recall.append(recall_score(y_true=eval_labels, y_pred=clf.predict(eval_train))) # f1.append(f1_score(y_true=eval_labels, y_pred=clf.predict(eval_train))) # break print accuracy, "Accuracy of %s: %f" % (algorithm, avg_list(accuracy)) print precision, "Precision of %s: %f" % (algorithm, avg_list(precision)) print recall, "Recall of %s: %f" % (algorithm, avg_list(recall)) print f1, "F1 of %s: %f" % (algorithm, avg_list(f1)) path_write = "./statistical_test_prob/%s.txt" % (algorithm) write_file(path_file=path_write, data=probs) print len(probs)