running_stat_item = {}
start_time = time.time()
# generate feature for training set
#with open("../testing_set.json", "r") as rf_training_set:
with open("../training_set.json", "r") as rf_training_set:
data_training_set = json.load(rf_training_set)
X_training_set_Adv, Y_training_set_Adv, str_output = pf.generate_wordfeature_and_output(wordcount, data_training_set, b_with_wf, i_len_word_feature, False, 0, b_without_punctuation, False, False, False, False, b_without_stopwords)
#print(str_is_root, str_controversiality, str_children, str_popularity_score)
str_to_write = "\n\n=================!!!!======================" + "\n"
str_to_write += "X_training_set_Adv = \n" + str(X_training_set_Adv) + "\n" + str(X_training_set_Adv.shape) + "\n"
str_to_write += "Y_training_set_Adv = \n" + str(Y_training_set_Adv) + "\n" + str(Y_training_set_Adv.shape) + "\n"
str_to_write += str_output
fout_t2r.write(str_to_write)
print(str_to_write)
wc_index = 0
X_training_set_Adv = np.array([])
Y_training_set_Adv = np.array([])
X_validation_set_Adv = np.array([])
Y_validation_set_Adv = np.array([])
running_stat_item = {}
with open("../training_set.json", "r") as rf_training_set:
data = json.load(rf_training_set)
X_training_set_Adv, Y_training_set_Adv, str_output = pf.generate_wordfeature_and_output(wordcount, data, False, 0, False, 0)
with open("../validation_set.json", "r") as rf_validation_set:
data = json.load(rf_validation_set)
X_validation_set_Adv, Y_validation_set_Adv, str_output = pf.generate_wordfeature_and_output(wordcount, data, False, 0, False, 0)
#測試Closed form的性能
start_time = time.time()
W_closed, str_to_write = lr.least_squares_estimate_linear_regression_alg(X_validation_set_Adv, Y_validation_set_Adv)
closed_running_time = time.time() - start_time
est_Y = np.dot(X_training_set_Adv, W_closed)
Y_testing_set_Adv = np.array([])
start_time = time.time()
# generate feature for training set
#with open("../testing_set.json", "r") as rf_training_set:
with open("../training_set.json", "r") as rf_training_set:
data_training_set = json.load(rf_training_set)
running_stat_item = {}
X_training_set_Adv, Y_training_set_Adv, str_out = pf.generate_wordfeature_and_output(wordcount, data_training_set, b_with_word_count_feature, i_word_count_feature_count, b_with_Advanced_feature, i_Advanced_feature_count, b_without_punctuation, b_with_total_comment_word_number_feature, b_with_total_number_of_sentence_feature, b_with_average_word_per_sentence_feature, b_with_average_length_per_word_feature, b_without_stopwords, b_adv_feature_replace_original_feature, i_adv_feature_power)
#X_training_set_Adv, Y_training_set_Adv = pf.generate_wordfeature_and_output(wordcount, data_training_set, b_with_wf, i_len_word_feature, False, 0, b_without_punctuation)
#print(str_is_root, str_controversiality, str_children, str_popularity_score)
str_to_write = "\n\n=================!!!!======================" + "\n"
str_to_write += "X_training_set_Adv = \n" + str(X_training_set_Adv) + "\n" + str(X_training_set_Adv.shape) + "\n"
str_to_write += "Y_training_set_Adv = \n" + str(Y_training_set_Adv) + "\n" + str(Y_training_set_Adv.shape) + "\n"
str_to_write += str_out
fout_t2r.write(str_to_write)