def data_preprocessing(self):
if os.path.exists(self.parameters['dataset_dirname']) is True:
pass
else:
print ("Input_Data 폴더명과 위치를 다시 확인하고 프로그램을 수행하세요")
exit(-1)
self.set_answer_info()
self.set_docpath_info()
# Training Set
TF_IDF_Feature_Matrix, target_idx_list = self.read_student_feature(self.parameters['dataset_dirname'])
train_idx, train_target_idx = self.make_data(target_idx_list, TF_IDF_Feature_Matrix, False)
# A list of tuples consisting of (target label, tf_idf_feature vector)
self.train_data = data_helpers.batch_construction(train_target_idx, train_idx)
# Validation Set
Val_TF_IDF_Feature_Matrix, Val_target_idx_list = self.read_student_feature(self.parameters['dataset_validation'])
val_idx, val_target_idx = self.make_data(Val_target_idx_list, Val_TF_IDF_Feature_Matrix, True)
# A list of tuples consisting of (target label, tf_idf_feature vector)
self.valid_data = data_helpers.batch_construction(val_target_idx, val_idx)
print ("")
print (" >> Implementation1 is complete!!")
print (" >> Do the next task, Implementation2.")
print ("")
def test():
print(" >> Loading preprocessing data...", "\n")
parameters, data_info = load_preprocessing()
print(" >> Loading Test Dataset...", "\n")
TF_IDF_Feature_Matrix, target_idx_list = data_info.read_student_feature(parameters['dataset_testset'])
test_idx, test_target_idx = data_info.make_data(target_idx_list, TF_IDF_Feature_Matrix, is_test = True)
test_data = data_helpers.batch_construction(test_target_idx, test_idx)
session_conf = tf.ConfigProto()
session_conf.gpu_options.allow_growth = True
with tf.Session(config=session_conf) as sess:
Model = load_model(sess, parameters, data_info)
test_input_indices, test_target_indices, test_target_origin =\
data_helpers.get_minibatch(dataset=test_data,\
minibatch_seq=np.arange(len(test_data)),\
is_test = True)
feed_dict = {
Model.X: test_input_indices,\
Model.Y: test_target_indices
}
test_logits = sess.run([Model.softmax_output], feed_dict=feed_dict)
# Save the output of softmax layer
np.savetxt(fname=parameters['output_path'], X=test_logits[0], \
fmt='%.10f', delimiter = '\t')
np.savetxt(fname='answer.txt', X=test_target_indices, fmt='%d')
print(" >> End of Test...")
print(" >> Check 'output.txt' and 'answer.txt' file...")
print("")