def train_evaluate_linear_svm(C=1.0):
X_train, Y_train = prepare_structured_dataset('train_struct.txt')
X_test, Y_test = prepare_structured_dataset('test_struct.txt')
# linear svm takes data of shape (N, 128) as input
x_train = convert_word_to_character_dataset(X_train)
y_train = convert_word_to_character_dataset(Y_train)
x_test = convert_word_to_character_dataset(X_test)
# train the model
model = LinearSVC(C=C, verbose=10, random_state=0)
model.fit(x_train, y_train)
# evaluate the model
y_preds = model.predict(x_test)
# reshape the predictions into a list of words
y_preds = convert_character_to_word_dataset(
y_preds, Y_test) # Y_test represents the word indices
# compute accuracy
word_acc, char_acc = compute_word_char_accuracy_score(y_preds, Y_test)
CHAR_CV_SCORES.append(char_acc)
WORD_CV_SCORES.append(word_acc)
def process_line(packed_line):
line, i, X_test, y_test, verbose = packed_line
split = line.split()
params = np.array(split[1:]).astype(np.float)
w = matricize_W(params)
t = matricize_Tij(params)
predictions = decode_crf(X_test, w, t)
word_acc, char_acc = compute_word_char_accuracy_score(predictions, y_test)
if verbose:
print(str(i) + ": ", word_acc)
return (1. - word_acc)
def save_word_error(result_file, output_file, X_test, y_test):
f_vals = []
file = open(result_file, 'r')
lines = file.readlines()
file.close()
for i, line in enumerate(lines):
split = line.split()
print(i, ": ", end='')
params = np.array(split[1:]).astype(np.float)
w = matricize_W(params)
t = matricize_Tij(params)
predictions = decode_crf(X_test, w, t)
word_acc, char_acc = compute_word_char_accuracy_score(predictions, y_test)
print(word_acc)
# only word accuracy so just accuracy[0]
f_vals.append(str(1 - word_acc) + "\n")
file = open(output_file, 'w')
file.writelines(f_vals)
file.close()
if sum(abs(parameters - previous_params)) <= self.epsilon or iteration == self.iterations:
break
return parameters
if __name__ == '__main__':
LAMBDA = 1e-2
X_train, y_train = prepare_dataset("train_sgd.txt")
X_test, y_test = prepare_dataset("test_sgd.txt")
filepath = "%s_%s.txt" % ('GIBBS', LAMBDA)
callback = Callback(X_train, y_train, filepath, LAMBDA)
gibbs = SamplingOptimizer(LAMBDA, callback_fn=callback.callback_fn_return_vals)
opt_params = gibbs.train(X_train)
W = matricize_W(opt_params)
T = matricize_Tij(opt_params)
y_preds = decode_crf(X_train, W, T)
word_acc, char_acc = compute_word_char_accuracy_score(y_preds, y_train)
print("Final train accuracy :", "Word =", word_acc, "Char =", char_acc)
y_preds = decode_crf(X_test, W, T)
word_acc, char_acc = compute_word_char_accuracy_score(y_preds, y_test)
print("Final test accuracy :", "Word =", word_acc, "Char =", char_acc)
# plot_scores(limits, scale=None, xlabel='distortion count')
''' CRF '''
CHAR_CV_SCORES.clear()
WORD_CV_SCORES.clear()
X_test, Y_test = prepare_structured_dataset('test_struct.txt')
print("Computing scores for best model with no distortion")
# compute the test scores for best model first
params = get_trained_model_parameters('solution')
w = matricize_W(params)
t = matricize_Tij(params)
y_preds = decode_crf(X_test, w, t)
word_acc, char_acc = compute_word_char_accuracy_score(y_preds, Y_test)
CHAR_CV_SCORES.append(char_acc)
WORD_CV_SCORES.append(word_acc)
for limit in limits:
print("Beginning distortion of first %d ids" % (limit))
''' training is commented out since it takes a few hours '''
# X_train, Y_train = read_data_formatted('train_distorted_%d.txt' % (limit))
# train_crf_lbfgs(params, X_train, Y_train, C=1000, model_name='model_%d_distortion' % limit)
params = get_trained_model_parameters('model_%d_distortion' % limit)
w = matricize_W(params)
t = matricize_Tij(params)
prediction = decode_crf(X_test, w, t)
''' training '''
X_train, y_train = prepare_structured_dataset('train_struct.txt')
X_test, y_test = prepare_structured_dataset('test_struct.txt')
params = load_model_params()
'''
Run optimization. For C= 1000 it takes about an 56 minutes
'''
# optimize(params, X_train, y_train, C=1000, name='solution')
params = get_trained_model_parameters('solution')
w = matricize_W(params)
t = matricize_Tij(params)
print("Function value: ", optimization_function(params, X_train, y_train, C=1000))
''' accuracy '''
#x_test = convert_word_to_character_dataset(X_test)
y_preds = decode_crf(X_test, w, t)
#y_preds = convert_character_to_word_dataset(y_preds, y_test)
with open("result/prediction.txt", "w") as text_file:
for i, elt in enumerate(y_preds):
# convert to characters
for word in elt:
text_file.write(str(word + 1))
text_file.write("\n")
print("Test accuracy : ", compute_word_char_accuracy_score(y_preds, y_test))
