def loading_data_all(FLAGS): # load all data from FLAGS path # split data to training and testing, only load testing data commits_ = extract_commit(path_file=FLAGS.path) filter_commits = filtering_commit(commits=commits_, num_file=FLAGS.code_file, num_hunk=FLAGS.code_hunk, num_loc=FLAGS.code_line, size_line=FLAGS.code_length) msgs_, codes_ = extract_msg(commits=filter_commits), extract_code( commits=filter_commits) dict_msg_, dict_code_ = dictionary(data=msgs_), dictionary(data=codes_) pad_msg = mapping_commit_msg(msgs=msgs_, max_length=FLAGS.msg_length, dict_msg=dict_msg_) pad_added_code = mapping_commit_code(type="added", commits=filter_commits, max_hunk=FLAGS.code_hunk, max_code_line=FLAGS.code_line, max_code_length=FLAGS.code_length, dict_code=dict_code_) pad_removed_code = mapping_commit_code(type="removed", commits=filter_commits, max_hunk=FLAGS.code_hunk, max_code_line=FLAGS.code_line, max_code_length=FLAGS.code_length, dict_code=dict_code_) labels = load_label_commits(commits=filter_commits) return pad_msg, pad_added_code, pad_removed_code, labels
def loading_data(path_file): commits_ = extract_commit(path_file=path_data) nfile, nhunk, nline, nleng = 1, 8, 10, 120 filter_commits = filtering_commit(commits=commits_, num_file=nfile, num_hunk=nhunk, num_loc=nline, size_line=nleng) msgs = extract_msg(commits=filter_commits) labels = extract_label(commits=filter_commits) codes = extract_code(commits=filter_commits) all_lines = add_two_list(list1=msgs, list2=codes) return all_lines, labels
def loading_training_data(): path_ftr = "./data/3_mar7/new_features_ver1.txt" ids_, X_, y_ = load_data_ICSE(path=path_ftr) print len(ids_), X_.shape, y_.shape path_data = "./data/3_mar7/typediff.out" commits_ = extract_commit(path_file=path_data) nfile, nhunk, nline, nleng = 1, 8, 10, 120 commits_ = get_commits(commits=filtering_commit(commits=commits_, num_file=nfile, num_hunk=nhunk, num_loc=nline, size_line=nleng), ids=ids_) return commits_, ids_, X_, y_
def loading_testing_data(FLAGS, path_file, type): if type == "msg": commits_ = extract_commit(path_file=path_file) filter_commits = filtering_commit(commits=commits_, num_file=FLAGS.code_file, num_hunk=FLAGS.code_hunk, num_loc=FLAGS.code_line, size_line=FLAGS.code_length) msgs_, codes_ = extract_msg(commits=filter_commits), extract_code( commits=filter_commits) elif type == "all": commits_ = extract_commit(path_file=path_file) filter_commits = filtering_commit(commits=commits_, num_file=FLAGS.code_file, num_hunk=FLAGS.code_hunk, num_loc=FLAGS.code_line, size_line=FLAGS.code_length) msgs_, codes_ = extract_msg(commits=filter_commits), extract_code( commits=filter_commits) all_lines = add_two_list(list1=msgs_, list2=codes_) msgs_ = all_lines elif type == "code": commits_ = extract_commit(path_file=path_file) filter_commits = filtering_commit(commits=commits_, num_file=FLAGS.code_file, num_hunk=FLAGS.code_hunk, num_loc=FLAGS.code_line, size_line=FLAGS.code_length) msgs_, codes_ = extract_msg(commits=filter_commits), extract_code( commits=filter_commits) msgs_ = codes_ else: print "You need to type correct model" exit() return msgs_, codes_, filter_commits
def loading_data_lstm(FLAGS): print FLAGS.model if "msg" in FLAGS.model: commits_ = extract_commit(path_file=FLAGS.path) filter_commits = filtering_commit(commits=commits_, num_file=FLAGS.code_file, num_hunk=FLAGS.code_hunk, num_loc=FLAGS.code_line, size_line=FLAGS.code_length) msgs_, codes_ = extract_msg(commits=filter_commits), extract_code(commits=filter_commits) dict_msg_, dict_code_ = dictionary(data=msgs_), dictionary(data=codes_) pad_msg = mapping_commit_msg(msgs=msgs_, max_length=FLAGS.msg_length, dict_msg=dict_msg_) pad_added_code = mapping_commit_code(type="added", commits=filter_commits, max_hunk=FLAGS.code_hunk, max_code_line=FLAGS.code_line, max_code_length=FLAGS.code_length, dict_code=dict_code_) pad_removed_code = mapping_commit_code(type="removed", commits=filter_commits, max_hunk=FLAGS.code_hunk, max_code_line=FLAGS.code_line, max_code_length=FLAGS.code_length, dict_code=dict_code_) labels = load_label_commits(commits=filter_commits) elif "all" in FLAGS.model: commits_ = extract_commit(path_file=FLAGS.path) filter_commits = filtering_commit(commits=commits_, num_file=FLAGS.code_file, num_hunk=FLAGS.code_hunk, num_loc=FLAGS.code_line, size_line=FLAGS.code_length) msgs_, codes_ = extract_msg(commits=filter_commits), extract_code(commits=filter_commits) all_lines = add_two_list(list1=msgs_, list2=codes_) msgs_ = all_lines dict_msg_, dict_code_ = dictionary(data=msgs_), dictionary(data=codes_) pad_msg = mapping_commit_msg(msgs=msgs_, max_length=FLAGS.msg_length, dict_msg=dict_msg_) pad_added_code = mapping_commit_code(type="added", commits=filter_commits, max_hunk=FLAGS.code_hunk, max_code_line=FLAGS.code_line, max_code_length=FLAGS.code_length, dict_code=dict_code_) pad_removed_code = mapping_commit_code(type="removed", commits=filter_commits, max_hunk=FLAGS.code_hunk, max_code_line=FLAGS.code_line, max_code_length=FLAGS.code_length, dict_code=dict_code_) labels = load_label_commits(commits=filter_commits) elif "code" in FLAGS.model: commits_ = extract_commit(path_file=FLAGS.path) filter_commits = filtering_commit(commits=commits_, num_file=FLAGS.code_file, num_hunk=FLAGS.code_hunk, num_loc=FLAGS.code_line, size_line=FLAGS.code_length) msgs_, codes_ = extract_msg(commits=filter_commits), extract_code(commits=filter_commits) msgs_ = codes_ dict_msg_, dict_code_ = dictionary(data=msgs_), dictionary(data=codes_) pad_msg = mapping_commit_msg(msgs=msgs_, max_length=FLAGS.msg_length, dict_msg=dict_msg_) pad_added_code = mapping_commit_code(type="added", commits=filter_commits, max_hunk=FLAGS.code_hunk, max_code_line=FLAGS.code_line, max_code_length=FLAGS.code_length, dict_code=dict_code_) pad_removed_code = mapping_commit_code(type="removed", commits=filter_commits, max_hunk=FLAGS.code_hunk, max_code_line=FLAGS.code_line, max_code_length=FLAGS.code_length, dict_code=dict_code_) labels = load_label_commits(commits=filter_commits) else: print "You need to type correct model" exit() kf = KFold(n_splits=FLAGS.folds, random_state=FLAGS.seed) for train_index, test_index in kf.split(filter_commits): X_train_msg, X_test_msg = np.array(get_items(items=pad_msg, indexes=train_index)), \ np.array(get_items(items=pad_msg, indexes=test_index)) X_train_added_code, X_test_added_code = np.array(get_items(items=pad_added_code, indexes=train_index)), \ np.array(get_items(items=pad_added_code, indexes=test_index)) X_train_removed_code, X_test_removed_code = np.array(get_items(items=pad_removed_code, indexes=train_index)), \ np.array(get_items(items=pad_removed_code, indexes=test_index)) y_train, y_test = np.array(get_items(items=labels, indexes=train_index)), \ np.array(get_items(items=labels, indexes=test_index)) return X_test_msg, X_test_added_code, X_test_removed_code, y_test
parser.add_argument('--data_type', type=str, default='all', help='type of model for learning') parser.add_argument('--model', type=str, default='model', help='names of our model') return parser if __name__ == '__main__': input_option = read_args().parse_args() input_help = read_args().print_help() commits = extract_commit(path_file=input_option.data) commits = reformat_commit_code(commits=commits, num_file=1, num_hunk=input_option.code_hunk, num_loc=input_option.code_line, num_leng=input_option.code_length) if input_option.train is True: train_model(commits=commits, params=input_option) print '--------------------------------------------------------------------------------' print '--------------------------Finish the training process---------------------------' print '--------------------------------------------------------------------------------' exit() elif input_option.predict is True: predict_model(commits=commits, params=input_option) print '--------------------------------------------------------------------------------'
num_hunk=nhunk, num_loc=nline, size_line=nleng), ids=ids_) return commits_, ids_, X_, y_ def clean_merging_data(ids, ftrs): ftr_id = [f.split(",")[0] for f in ftrs] new_ftr = [ftrs[ftr_id.index(i)] for i in ids] return new_ftr if __name__ == "__main__": path_data = "./data/test_data/merging_markus_sasha.txt" commits_ = extract_commit(path_file=path_data) nfile, nhunk, nline, nleng = 1, 8, 10, 120 filter_commits = filtering_commit(commits=commits_, num_file=nfile, num_hunk=nhunk, num_loc=nline, size_line=nleng) ids_ = [c["id"] for c in filter_commits] labels_ = [1 if c["stable"] == "true" else 0 for c in filter_commits] path_ftr = "./data/test_data/features_merging_markus_sasha.txt" ftr = load_file(path_file=path_ftr) new_ftr = clean_merging_data(ids=ids_, ftrs=ftr) commits_test, ids_test, X_ftr_test, y_test = loading_testing_data( ftr_data=new_ftr, commit_data=filter_commits)
default=None, help='starting epoch of loading model') parser.add_argument('-end_epoch', type=int, default=None, help='ending epoch of loading model') # load model name parser.add_argument('-file_model', type=str, default=None, help='date of model [default: None]') return parser if __name__ == '__main__': input_option = read_args().parse_args() input_help = read_args().print_help() path_file = './data/newres_funcalls_jul28.out.sorted' commits = extract_commit(path_file=path_file) commits = reformat_commit_code(commits=commits, num_file=1, num_hunk=8, num_loc=10, num_leng=120) nfolds, random_state = 5, None train, test = training_testing_split(commits=commits, nfolds=5, random_state=None) print(len(commits))
clf.fit(X=X_train.toarray(), y=y_train) y_pred = clf.predict(X_test) path_write = "./data_test_data_pred_results/cnn_" + type + ".txt" write_file(path_file=path_write, data=y_pred) print "Accuracy: ", accuracy_score(y_true=y_test, y_pred=y_pred) print "Precision: ", precision_score(y_true=y_test, y_pred=y_pred) print "Recall: ", recall_score(y_true=y_test, y_pred=y_pred) print "F1: ", f1_score(y_true=y_test, y_pred=y_pred) print "AUC: ", auc_score(y_true=y_test, y_pred=y_pred) if __name__ == "__main__": nfile, nhunk, nline, nleng = 1, 8, 10, 120 path_data = "./data/3_mar7/typediff.out" commits_train = extract_commit(path_file=path_data) filter_commits_train = filtering_commit(commits=commits_train, num_file=nfile, num_hunk=nhunk, num_loc=nline, size_line=nleng) msgs_train = extract_msg(commits=filter_commits_train) labels_train = extract_label(commits=filter_commits_train) codes_train = extract_code(commits=filter_commits_train) all_lines_train = add_two_list(list1=msgs_train, list2=codes_train) # path_test = "./data/test_data/sasha_translated.out" path_test = "./data/test_data/merging_markus_sasha.txt" type = "all" # type = "msg" # type = "code" commits_test = extract_commit(path_file=path_test) filter_commits_test = filtering_commit(commits=commits_test,
print "hello" if __name__ == "__main__": tf = model_parameter_evaluation_keras() FLAGS = tf.flags.FLAGS print_params(tf) path_file_model = "./keras_model/" model_name = FLAGS.model # model_name = "lstm_code" model_name = "lstm_all" model = load_model(path_file_model + model_name + ".h5") if "msg" in FLAGS.model: commits_ = extract_commit(path_file=FLAGS.path) filter_commits = filtering_commit(commits=commits_, num_file=FLAGS.code_file, num_hunk=FLAGS.code_hunk, num_loc=FLAGS.code_line, size_line=FLAGS.code_length) msgs_, codes_ = extract_msg(commits=filter_commits), extract_code( commits=filter_commits) elif "all" in FLAGS.model: commits_ = extract_commit(path_file=FLAGS.path) filter_commits = filtering_commit(commits=commits_, num_file=FLAGS.code_file, num_hunk=FLAGS.code_hunk, num_loc=FLAGS.code_line, size_line=FLAGS.code_length) msgs_, codes_ = extract_msg(commits=filter_commits), extract_code(