def run_dcpc_processing(dcpc_folder, num_classes, method=0, logger=None):
logger.info('obj folder:' + dcpc_folder)
dcpc_list = list_files(dcpc_folder)
logger.info(dcpc_list)
score_folder = dcpc_folder[:-1] + "_score/"
score_folder = init_folder(score_folder)
for dcpc_obj in dcpc_list:
dcpc = load_obj(dcpc_folder + dcpc_obj)[0]
if method == 0:
out_label_array = []
out_label_dict = {}
for label in range(0, num_classes):
logger.info('class: ' + str(label))
label_dcpc = dcpc[label]
logger.info("dcpc shape: " + str(label_dcpc.shape))
attr_score = clever_rank(label_dcpc, logger)
logger.info(attr_score)
sorted_dict = sorted(attr_score.items(),
key=operator.itemgetter(1),
reverse=True)
sorted_attr = []
for item in sorted_dict:
sorted_attr.append(item[0])
#label_array = []
#for label in range(0, num_classes):
# class_array = sorted_attr
# label_array.append(class_array)
out_label_array.append(sorted_attr)
out_label_dict[label] = attr_score
logger.info(sorted_attr)
logger.info(attr_score)
save_obj([out_label_array, out_label_dict],
score_folder + dcpc_obj)
logger.info("score obj: " + score_folder + dcpc_obj)
return score_folder
def global_cnn_lda_feature_main(parameter_file, method):
data_keyword, data_folder, attr_num, attr_len, class_column, start_class, num_classes, pckl_folder, log_folder, log_postfix, out_obj_folder = read_global_feature_generation_parameter(
parameter_file)
log_file = log_folder + data_keyword + '_' + method + log_postfix
#log_file = '' # without write to file
logger = init_logging(log_file)
logger.info('METHOD: ' + method)
logger.info('DATA KEYWORD: ' + data_keyword)
logger.info('ATTRIBUTE NUMBER: ' + str(attr_num))
logger.info('ATTRIBUTE LENGTH: ' + str(attr_len))
logger.info('CLASS NUMBER: ' + str(num_classes))
logger.info('CLASS COLUMN: ' + str(class_column))
logger.info('START CLASS: ' + str(start_class))
logger.info('PCKL FOLDER: ' + pckl_folder)
logger.info('LOG FOLDER: ' + log_folder)
logger.info('LOG POSTFIX: ' + log_postfix)
logger.info('OUTPUT FOLDER: ' + out_obj_folder)
function_name = sys._getframe(1).f_code.co_name
logger = init_logging(log_file)
file_list = listFiles(pckl_folder)
overall_time = 0
ret_feature_array = []
file_count = 0
for train_file_pckl in file_list:
if "train" not in train_file_pckl:
continue
train_file = train_file_pckl[0:train_file_pckl.index('.txt')] + '.txt'
logger.info("PCKL FILE: " + train_file_pckl)
logger.info("DATA FILE: " + train_file)
train_x_matrix, train_y_vector = readFile(data_folder + train_file)
out_matrix, weight_matrix, bias_vector = load_obj(pckl_folder +
train_file_pckl)
out_matrix = np.squeeze(out_matrix)
row_num, attr_num, attr_len = out_matrix.shape
out_matrix = out_matrix.reshape(row_num, attr_num * attr_len)
if file_count == 0:
logger.info('layer out matrix shape: ' + str(out_matrix.shape))
logger.info('weight matrix shape: ' + str(weight_matrix.shape))
logger.info('bias vector shape: ' + str(bias_vector.shape))
feature_index_vector, run_time = gene_global_lda_feature(
out_matrix, train_y_vector, attr_num, logger)
overall_time = overall_time + run_time
logger.info(feature_index_vector.shape)
ret_feature_array.append(feature_index_vector)
file_count = file_count + 1
break
#if file_count > 1:
# break
ret_feature_array = np.matrix(ret_feature_array)
logger.info(ret_feature_array.shape)
logger.info("return feature array samples:")
logger.info("\n" + str(ret_feature_array[0:4, 0:6]))
start_time = time.time()
ret_feature_index, ret_feature_value = majority_vote_index(
ret_feature_array, -1)
overall_time = overall_time + time.time() - start_time
logger.info("\n" + str(ret_feature_index[0:6]))
logger.info(method + " global feature run time (sec): " +
str(overall_time))
obj_file = out_obj_folder + method + "_global_feature.pckl"
save_obj(ret_feature_index, obj_file)
return ret_feature_index, overall_time
def multi_proj_feature_classification(
parameter_file,
file_keyword,
function_keyword="multi_proj_feature_classification"):
data_keyword, data_folder, attr_num, attr_len, num_classes, start_class, class_column, class_id, obj_folder, top_k, method, log_folder, cnn_obj_folder, cnn_temp_folder, cnn_setting_file = read_feature_classification(
parameter_file, function_keyword)
log_folder = init_folder(log_folder)
if method == 'cnn':
return projected_cnn_classification_main(parameter_file, file_keyword)
else:
# Need to check the rest
return False
print data_keyword, data_folder, attr_num, attr_len, num_classes, start_class, class_column, class_id, obj_folder, top_k, method, log_folder, cnn_obj_folder, cnn_temp_folder, cnn_setting_file
data_stru = return_data_stru(num_classes, start_class, attr_num, attr_len,
class_column)
print obj_folder
file_list = list_files(data_folder)
obj_list = list_files(obj_folder)
class_column = 0
header = True
save_obj_folder = obj_folder[:-1] + "_" + method + "_out"
save_obj_folder = init_folder(save_obj_folder)
delimiter = ' '
loop_count = -1
for train_file in file_list:
if file_keyword not in train_file:
continue
loop_count = loop_count + 1
file_key = train_file.replace('.txt', '')
log_file = log_folder + data_keyword + '_' + file_key + '_' + function_keyword + '_class' + str(
class_id) + '_top' + str(top_k) + '_' + method + '.log'
print "log file: " + log_file
logger = setup_logger(log_file, 'logger_' + str(loop_count))
logger.info('\nlog file: ' + log_file)
logger.info(train_file)
logger.info('method: ' + method)
logger.info('============')
found_obj_file = ''
for obj_file in obj_list:
if file_key in obj_file:
found_obj_file = obj_file
break
if found_obj_file == '':
raise Exception('No obj file found')
print found_obj_file
found_obj_file = obj_folder + found_obj_file
feature_array = load_obj(found_obj_file)[0]
feature_array = np.array(feature_array)
logger.info("feature array shape: " + str(feature_array.shape))
test_file = train_file.replace('train', 'test')
train_x_matrix, train_y_vector, test_x_matrix, test_y_vector, attr_num = train_test_file_reading_with_attrnum(
data_folder + train_file, data_folder + test_file, class_column,
delimiter, header)
if loop_count == 0:
logger.info('train matrix shape: ' + str(train_x_matrix.shape))
logger.info('train label shape: ' + str(train_y_vector.shape))
logger.info('test matrix shape: ' + str(test_x_matrix.shape))
logger.info('test label shape: ' + str(test_y_vector.shape))
train_x_matrix = train_test_transpose(train_x_matrix, attr_num,
attr_len, False)
test_x_matrix = train_test_transpose(test_x_matrix, attr_num, attr_len,
False)
data_stru.attr_num = top_k
fold_accuracy, fold_f1_value, fold_predict_y, fold_train_time, fold_test_time, fold_predict_matrix = run_feature_projected_classification(
train_x_matrix, train_y_vector, test_x_matrix, test_y_vector,
feature_array, top_k, method, class_id, logger)
logger.info("Fold F1: " + str(fold_f1_value))
logger.info(method + ' fold training time (sec):' +
str(fold_train_time))
logger.info(method + ' fold testing time (sec):' + str(fold_test_time))
logger.info(method + ' fold accuracy: ' + str(fold_accuracy))
logger.info("save obj to " + save_obj_folder + file_key + "_" +
method + "_project_" + method + "_result.ckpt")
save_obj([
fold_accuracy, fold_f1_value, fold_predict_y, fold_train_time,
fold_test_time, fold_predict_matrix
], save_obj_folder + file_key + "_" + method + "_project_" + method +
"_result.ckpt")
def run_cnn_projected_feature_analysis(feature_folder,
class_id,
data_folder,
data_file_keyword,
method="rf_lda",
log_folder='./'):
data_file_list = list_files(data_folder)
feature_file_list = list_files(feature_folder)
out_obj_folder = feature_folder[:-1] + "_" + method
out_obj_folder = init_folder(out_obj_folder)
class_column = 0
for train_file in data_file_list:
if data_file_keyword not in train_file:
continue
data_key = train_file.replace('.txt', '')
data_matrix, attr_num = file_reading(data_folder + train_file)
train_x_matrix, train_y_vector = x_y_spliting(data_matrix,
class_column)
#train_y_vector = np.array([0, 0, 1, 1, 1, 1, 2, 2, 2, 3])
if class_id < 0:
min_class = min(train_y_vector)
max_class = max(train_y_vector) + 1
else:
min_class = class_id
max_class = min_class + 1
log_file = data_key + "_" + method + "_min" + str(
min_class) + "_max" + str(max_class) + ".log"
logger = setup_logger(log_folder + log_file)
logger.info('data file: ' + train_file)
out_obj_file = data_key + "_" + method + "_min" + str(
min_class) + "_max" + str(max_class) + ".obj"
out_obj_matrix = []
for label in range(min_class, max_class):
logger.info("class: " + str(label))
feature_key = "_class" + str(label) + "_"
for feature_file in feature_file_list:
if data_key not in feature_file or feature_key not in feature_file:
continue
logger.info("feature file: " + feature_file)
feature_obj = load_obj(feature_folder + feature_file)
train_feature = obj_processing(feature_obj[0])
logger.info("train feature shape: " + str(train_feature.shape))
class_train_y = np.where(train_y_vector == label, 1, 0)
logger.info("feature method: " + str(method))
if method == "rf_lda_sum":
class_attr_imp_matrix, class_run_time = project_cnn_feature_combined_rf_lda_analysis(
train_feature, class_train_y, logger)
elif method == "rf":
class_attr_imp_matrix, class_run_time = project_cnn_feature_combined_rf_analysis(
train_feature, class_train_y, logger)
elif method == "lda":
class_attr_imp_matrix, class_run_time = project_cnn_feature_combined_lda_analysis(
train_feature, class_train_y, logger)
elif method == "cpca":
class_attr_imp_matrix, class_run_time = project_cnn_feature_combined_cpca_analysis(
train_feature, class_train_y, logger)
if method == "cpca":
class_attr_list = class_attr_imp_matrix
else:
logger.info("class attr imp matrix shape: " +
str(class_attr_imp_matrix.shape))
class_attr_list = map_attr_imp_analysis(
class_attr_imp_matrix, logger)
logger.info(class_attr_list)
out_obj_matrix.append(class_attr_list)
out_obj_matrix = np.array(out_obj_matrix)
logger.info("out obj to: " + out_obj_folder + out_obj_file)
logger.info(out_obj_matrix.shape)
save_obj([out_obj_matrix], out_obj_folder + out_obj_file)
def multi_projected_cnn_classification_main(parameter_file, file_keyword, function_keyword="multi_proj_classification"):
data_keyword, data_folder, attr_num, attr_len, num_classes, start_class, class_column, class_id, obj_folder, top_k, method, log_folder, cnn_obj_folder, cnn_temp_folder, cnn_setting_file = read_feature_classification(parameter_file, function_keyword)
obj_keyword = obj_folder.split('/')[-2]
model_saved_folder = "../../object/" + data_keyword + "/projected_classification/" + obj_keyword + "_top" + str(top_k) + "_cnn_model_folder/"
print obj_keyword
print cnn_obj_folder
print model_saved_folder
top_keyword = "_top" + str(top_k) + "."
group_all = False
log_folder = init_folder(log_folder)
#cnn_obj_folder = init_folder(cnn_obj_folder)
#cnn_temp_folder = init_folder(cnn_temp_folder)
data_stru = return_data_stru(num_classes, start_class, attr_num, attr_len, class_column)
file_list = list_files(data_folder)
obj_list = list_files(obj_folder)
file_count = 0
class_column = 0
header = True
cnn_setting = return_cnn_setting_from_file(cnn_setting_file)
cnn_setting.save_obj_folder = cnn_obj_folder
cnn_setting.temp_obj_folder = cnn_temp_folder
cnn_setting.eval_method = 'f1'
#init_folder(cnn_obj_folder)
#init_folder(cnn_temp_folder)
save_obj_folder = "../../object/" + data_keyword + "/" + function_keyword + "/" + obj_keyword + "/"
save_obj_folder = init_folder(save_obj_folder)
delimiter = ' '
loop_count = -1
for train_file in file_list:
if file_keyword not in train_file:
continue
loop_count = loop_count + 1
file_key = train_file.replace('.txt', '')
log_file = log_folder + data_keyword + '_' + file_key + '_' + function_keyword + '_class' + str(class_id) + '_top' + str(top_k) + '_' + method + '.log'
print "log file: " + log_file
logger = setup_logger(log_file, 'logger_' + str(loop_count))
logger.info('\nlog file: ' + log_file)
logger.info(train_file)
logger.info('cnn setting:\n ' + cnn_setting.to_string())
logger.info('method: ' + method)
logger.info('============')
found_obj_file = ''
for obj_file in obj_list:
if file_key in obj_file:
found_obj_file = obj_file
break
if found_obj_file == '':
raise Exception('No obj file found')
#
found_obj_file = obj_folder + found_obj_file
feature_dict = load_obj(found_obj_file)[0]
feature_dict = np.array(feature_dict)
logger.info("feature array shape: " + str(feature_dict.shape))
test_file = train_file.replace('train', 'test')
train_x_matrix, train_y_vector, test_x_matrix, test_y_vector, attr_num = train_test_file_reading_with_attrnum(
data_folder + train_file, data_folder + test_file, class_column, delimiter, header)
train_x_matrix = train_test_transpose(train_x_matrix, attr_num, attr_len, False)
test_x_matrix = train_test_transpose(test_x_matrix, attr_num, attr_len, False)
if file_count == 0:
logger.info('train matrix shape: ' + str(train_x_matrix.shape))
logger.info('train label shape: ' + str(train_y_vector.shape))
logger.info('test matrix shape: ' + str(test_x_matrix.shape))
logger.info('test label shape: ' + str(test_y_vector.shape))
logger.info("topk: " + str(top_k) )
data_stru.attr_num = top_k
fold_accuracy, fold_f1_list, fold_load_time, fold_test_time = run_load_predict_cnn(file_key, model_saved_folder, feature_dict, top_k, test_x_matrix, test_y_vector, data_stru, cnn_setting, group_all, save_obj_folder, logger)
logger.info("Fold ACC: " + str(fold_accuracy))
logger.info("Fold F1 list: " + str(fold_f1_list))
logger.info(method + ' fold training time (sec):' + str(fold_load_time))
logger.info(method + ' fold testing time (sec):' + str(fold_test_time))
def mask_evaluation_main(log_folder,
obj_folder,
out_obj_folder,
obj_keyword,
shap_k=-1,
shap_min=-1,
shap_max=-1,
func_key="arxiv_mask_gene"):
log_folder = log_folder + func_key
log_folder = init_folder(log_folder)
log_file = obj_keyword + "_allclass_" + func_key + ".log"
#logger = setup_logger('')
logger = setup_logger(log_folder + log_file)
logger.info("log folder: " + log_folder)
logger.info("obj folder: " + obj_folder)
obj_file_list = list_files(obj_folder)
if shap_k != -1:
obj_sec_key = "shapNum" + str(shap_k) + "_shapMin" + str(
shap_min) + "_shapMax" + str(shap_max)
else:
obj_sec_key = ".obj"
min_class = 100
max_class = -1
output_array = []
for obj_file in obj_file_list:
if obj_keyword not in obj_file:
continue
if "_class" not in obj_file:
continue
if obj_sec_key not in obj_file:
continue
class_key = obj_file.split('_')[-1]
class_key = class_key.replace('class', '').replace('.obj', '')
logger.info("obj file:" + obj_file)
logger.info("class key: " + class_key)
class_key = int(class_key)
if min_class > class_key:
min_class = class_key
if max_class < class_key:
max_class = class_key
shap_mask = load_obj(obj_folder + obj_file)[0]
if len(shap_mask) == 0:
continue
shap_mask = numpy.array(shap_mask)
shap_mask = numpy.squeeze(shap_mask)
logger.info("shap_mask shape: " + str(shap_mask.shape))
#shap_num, attr_num = shap_mask.shape
shap_mask = numpy.absolute(shap_mask)
shap_mask = numpy.sum(shap_mask, axis=0)
logger.info(shap_mask)
sort_index = numpy.argsort(shap_mask)
imp_value = 0
norm_imp = numpy.zeros(len(shap_mask))
for index in sort_index:
norm_imp[index] = imp_value
imp_value = imp_value + 1
shap_mask_index = numpy.argsort(norm_imp)[::-1]
logger.info(shap_mask_index)
logger.info("====")
output_array.append(shap_mask_index)
logger.info("shap_mask final shape: " + str(shap_mask.shape))
output_array = numpy.array(output_array)
obj_file = obj_keyword + "_min" + str(min_class) + "_max" + str(
max_class) + "out.obj"
logger.info("final output obj shape: " + str(output_array.shape))
logger.info(output_array)
save_obj([output_array], out_obj_folder + obj_file)
def forward_multitime_main(parameter_file="../../parameters/",
file_keyword="train_"):
function_keyword = "forward_wrapper"
data_keyword, data_folder, attr_num, attr_len, num_classes, start_class, class_column, class_id, obj_folder, method, log_folder, out_obj_folder, out_model_folder, cnn_setting_file = read_all_feature_classification(
parameter_file, function_keyword)
print data_keyword, data_folder, attr_num, attr_len, num_classes, start_class, class_column, class_id, obj_folder, method, log_folder, out_obj_folder, out_model_folder, cnn_setting_file
if data_keyword == "dsa":
n_selected_features = 15
num_classes = 19
elif data_keyword == "rar":
n_selected_features = 30
num_classes = 33
elif data_keyword == "arc" or data_keyword == "fixed_arc":
n_selected_features = 30
num_classes = 18
elif data_keyword == "asl":
n_selected_features = 6
num_classes = 95
log_folder = init_folder(log_folder)
#out_obj_folder = init_folder(out_obj_folder)
#out_model_folder = init_folder(out_model_folder)
data_stru = return_data_stru(num_classes, start_class, attr_num, attr_len,
class_column)
file_list = list_files(data_folder)
file_count = 0
class_column = 0
header = True
delimiter = ' '
loop_count = -1
##########
###already remove later
already_obj_folder = "../../object/" + data_keyword + "/forward_wrapper/"
already_obj_list = list_files(already_obj_folder)
###end of already remove later
for train_file in file_list:
if file_keyword not in train_file:
continue
loop_count = loop_count + 1
file_key = train_file.replace('.txt', '')
already_obj_file = ""
already = False
for already_obj_file in already_obj_list:
if file_key in already_obj_file and method in already_obj_file:
already = True
break
##########
###already part
if already is True:
already_class_feature = load_obj(already_obj_folder +
already_obj_file)[0]
else:
log_file = log_folder + data_keyword + '_' + file_key + '_' + function_keyword + '_class' + str(
class_id) + '_' + method + '.log'
already_class_feature = None
###end of already part
log_file = log_folder + data_keyword + '_' + file_key + '_' + function_keyword + '_class' + str(
class_id) + '_' + method + "_top" + str(
n_selected_features) + '_already' + str(already) + '.log'
print "log file: " + log_file
logger = setup_logger(log_file, 'logger_' + str(loop_count))
logger.info('\nlog file: ' + log_file)
logger.info(train_file)
logger.info('method: ' + method)
logger.info('============')
test_file = train_file.replace('train', 'test')
train_x_matrix, train_y_vector, test_x_matrix, test_y_vector = train_test_file_reading(
data_folder + train_file, data_folder + test_file, class_column,
delimiter, header)
n_samples, n_col = train_x_matrix.shape
train_x_matrix = train_x_matrix.reshape(n_samples, attr_num, attr_len)
n_samples, n_col = test_x_matrix.shape
test_x_matrix = test_x_matrix.reshape(n_samples, attr_num, attr_len)
if file_count == 0:
logger.info('train matrix shape: ' + str(train_x_matrix.shape))
logger.info('train label shape: ' + str(train_y_vector.shape))
logger.info('test matrix shape: ' + str(test_x_matrix.shape))
logger.info('test label shape: ' + str(test_y_vector.shape))
min_class = min(train_y_vector)
max_class = max(train_y_vector) + 1
for c in range(min_class, max_class):
logger.info("Class: " + str(c))
already_feature = []
if already_class_feature is not None:
class_already = already_class_feature[c, :]
for already_f in class_already:
already_feature.append(already_f)
logger.info("already features: " + file_key + " with class " +
str(c) + ": " + str(already_feature))
temp_train_y_vector = np.where(train_y_vector == c, 1, 0)
temp_test_y_vector = np.where(test_y_vector == c, 1, 0)
print already_feature
top_features = forward_multitime(
train_x_matrix, temp_train_y_vector, test_x_matrix,
temp_test_y_vector, n_selected_features, data_keyword,
file_key, method, cnn_setting_file, logger, already_feature)
logger.info("Top Features For Class " + str(c) + ": " +
str(top_features))
logger.info("End Of Class: " + str(c))
def global_classification_main(parameter_file, file_keyword):
function_keyword = "global_classification"
data_keyword, data_folder, attr_num, attr_len, num_classes, start_class, class_column, class_id, obj_folder, top_k, method, log_folder, cnn_obj_folder, cnn_temp_folder, cnn_setting_file = read_feature_classification(
parameter_file, function_keyword)
data_stru = return_data_stru(num_classes, start_class, attr_num, attr_len,
class_column)
file_list = list_files(data_folder)
obj_list = list_files(obj_folder)
file_count = 0
class_column = 0
header = True
cnn_setting = return_cnn_setting_from_file(cnn_setting_file)
cnn_setting.save_obj_folder = cnn_obj_folder
cnn_setting.temp_obj_folder = cnn_temp_folder
cnn_setting.eval_method = 'f1'
init_folder(cnn_obj_folder)
init_folder(cnn_temp_folder)
all_result_matrix = np.zeros((10, num_classes))
train_file_vector = []
prediction_matrix = []
f1_value_matrix = []
accuracy_vector = []
delimiter = ' '
all_accuracy = 0
all_train_time = 0
all_test_time = 0
loop_count = -1
for train_file in file_list:
if file_keyword not in train_file:
continue
loop_count = loop_count + 1
file_key = train_file.replace('.txt', '')
log_file = log_folder + data_keyword + '_' + file_key + '_' + function_keyword + '_class' + str(
class_id) + '_top' + str(top_k) + '_' + method + '.log'
print "log file: " + log_file
logger = setup_logger(log_file, 'logger_' + str(loop_count))
logger.info('\nlog file: ' + log_file)
logger.info(train_file)
logger.info('cnn setting:\n ' + cnn_setting.to_string())
logger.info('method: ' + method)
logger.info('============')
continue
found_obj_file = ''
for obj_file in obj_list:
if file_key in obj_file:
found_obj_file = obj_file
break
if found_obj_file == '':
raise Exception('No obj file found')
print found_obj_file
print cnn_setting.save_obj_folder + file_key + "_" + method + "_projected_result.ckpt"
#
found_obj_file = obj_folder + found_obj_file
feature_dict = load_obj(found_obj_file)[0]
feature_dict = np.array(feature_dict)
logger.info("feature array shape: " + str(feature_dict.shape))
test_file = train_file.replace('train', 'test')
train_x_matrix, train_y_vector, test_x_matrix, test_y_vector, attr_num = train_test_file_reading_with_attrnum(
data_folder + train_file, data_folder + test_file, class_column,
delimiter, header)
if file_count == 0:
logger.info('train matrix shape: ' + str(train_x_matrix.shape))
logger.info('train label shape: ' + str(train_y_vector.shape))
logger.info('test matrix shape: ' + str(test_x_matrix.shape))
logger.info('test label shape: ' + str(test_y_vector.shape))
train_x_matrix = train_test_transpose(train_x_matrix, attr_num,
attr_len, False)
test_x_matrix = train_test_transpose(test_x_matrix, attr_num, attr_len,
False)
data_stru.attr_num = top_k
fold_accuracy, fold_avg_eval, fold_predict_y, fold_train_time, fold_test_time, fold_predict_matrix = run_feature_projected_cnn(
train_x_matrix, train_y_vector, test_x_matrix, test_y_vector,
data_stru, cnn_setting, feature_dict, top_k,
file_key + '_count' + str(file_count), class_id, logger)
prediction_matrix.append(fold_predict_y)
logger.info("Fold F1: " + str(fold_f1_value_list))
accuracy_vector.append(fold_accuracy)
all_accuracy = all_accuracy + fold_accuracy
all_train_time = all_train_time + fold_train_time
all_test_time = all_test_time + fold_test_time
logger.info(method + ' fold accuracy: ' + str(fold_accuracy))
logger.info(method + ' fold training time (sec):' +
str(fold_train_time))
logger.info(method + ' fold testing time (sec):' + str(fold_test_time))
save_obj([
fold_accuracy, fold_avg_eval, fold_predict_y, fold_train_time,
fold_test_time, fold_predict_matrix
], save_obj_folder + file_key + "_" + method +
"_global_cnn_result.ckpt")