def HFTest(seq, targets, t_seq, t_targets, n_hidden=10, n_updates=250): """ Test RNN with hessian free optimization """ n_in = 2 n_out = 2 n_classes = 10 # SequenceDataset wants a list of sequences # this allows them to be different lengths, but here they're not seq = [i for i in seq] targets = [i for i in targets] gradient_dataset = SequenceDataset([seq, targets], batch_size=None, number_batches=500) cg_dataset = SequenceDataset([seq, targets], batch_size=None, number_batches=100) model = MetaRNN(n_in=n_in, n_hidden=n_hidden, n_out=n_out, activation='tanh', output_type='softmax', use_symbolic_softmax=True) # optimizes negative log likelihood # but also reports zero-one error opt = hf_optimizer(p=model.rnn.params, inputs=[model.x, model.y], s=model.rnn.y_pred, costs=[model.rnn.loss(model.y), model.rnn.errors(model.y)], h=model.rnn.h) mse_updates = [] for i in range(n_updates): opt.train(gradient_dataset, cg_dataset, num_updates=1) mse = 0 for t in range(len(t_seq)): guess = model.predict_proba(t_seq[t]) if guess != t_target: mse += 1 mse_updates.append(mse) print i return (mse_updates, model)
# optimizes negative log likelihood # but also reports zero-one error opt = hf_optimizer(p=model.rnn.params, inputs=[model.x, model.y], s=model.rnn.y_pred, n_in=n_in, word2vec=word2vec, word2label=word2label, costs=[model.rnn.loss(model.y), model.rnn.errors(model.y)], h=model.rnn.h) print("Training ...") opt.train(gradient_dataset, cg_dataset, num_updates=n_updates, save_progress='save_param') seqs = range(10) plt.close('all') for seq_num in seqs: fig = plt.figure() ax1 = plt.subplot(211) plt.plot(x_seq_list[seq_num]) ax1.set_title('input') ax2 = plt.subplot(212) # blue line will represent true classes true_targets = plt.step(range(n_steps), y_seq_list[seq_num], marker='o') # show probabilities (in b/w) output by model guess = model.predict_proba(x_seq_list[seq_num]) guessed_probs = plt.imshow(guess.T, interpolation='nearest', cmap='gray') ax2.set_title('blue: true class, grayscale: probs assigned by model') print("Elapsed time: %f" % (time.time() - t0))
def test_binary(multiple_out=False, n_updates=250): """ Test RNN with binary outputs. """ n_hidden = 10 n_in = 5 if multiple_out: n_out = 2 else: n_out = 1 n_steps = 10 n_seq = 100 np.random.seed(0) # simple lag test seq = np.random.randn(n_seq, n_steps, n_in) targets = np.zeros((n_seq, n_steps, n_out), dtype='int32') # whether lag 1 (dim 3) is greater than lag 2 (dim 0) targets[:, 2:, 0] = np.cast[np.int32](seq[:, 1:-1, 3] > seq[:, :-2, 0]) if multiple_out: # whether product of lag 1 (dim 4) and lag 1 (dim 2) # is less than lag 2 (dim 0) targets[:, 2:, 1] = np.cast[np.int32]( (seq[:, 1:-1, 4] * seq[:, 1:-1, 2]) > seq[:, :-2, 0]) # SequenceDataset wants a list of sequences # this allows them to be different lengths, but here they're not seq = [i for i in seq] targets = [i for i in targets] gradient_dataset = SequenceDataset([seq, targets], batch_size=None, number_batches=500) cg_dataset = SequenceDataset([seq, targets], batch_size=None, number_batches=100) model = MetaRNN(n_in=n_in, n_hidden=n_hidden, n_out=n_out, activation='tanh', output_type='binary') # optimizes negative log likelihood # but also reports zero-one error opt = hf_optimizer(p=model.rnn.params, inputs=[model.x, model.y], s=model.rnn.y_pred, costs=[model.rnn.loss(model.y), model.rnn.errors(model.y)], h=model.rnn.h) # using settings of initial_lambda and mu given in Nicolas' RNN example # seem to do a little worse than the default opt.train(gradient_dataset, cg_dataset, num_updates=n_updates) seqs = xrange(10) plt.close('all') for seq_num in seqs: fig = plt.figure() ax1 = plt.subplot(211) plt.plot(seq[seq_num]) ax1.set_title('input') ax2 = plt.subplot(212) true_targets = plt.step(xrange(n_steps), targets[seq_num], marker='o') guess = model.predict_proba(seq[seq_num]) guessed_targets = plt.step(xrange(n_steps), guess) plt.setp(guessed_targets, linestyle='--', marker='d') for i, x in enumerate(guessed_targets): x.set_color(true_targets[i].get_color()) ax2.set_ylim((-0.1, 1.1)) ax2.set_title('solid: true output, dashed: model output (prob)')
def test_softmax(n_updates=250): """ Test RNN with softmax outputs. """ n_hidden = 10 n_in = 5 n_steps = 10 n_seq = 100 n_classes = 3 n_out = n_classes # restricted to single softmax per time step np.random.seed(0) # simple lag test seq = np.random.randn(n_seq, n_steps, n_in) targets = np.zeros((n_seq, n_steps), dtype='int32') thresh = 0.5 # if lag 1 (dim 3) is greater than lag 2 (dim 0) + thresh # class 1 # if lag 1 (dim 3) is less than lag 2 (dim 0) - thresh # class 2 # if lag 2(dim0) - thresh <= lag 1 (dim 3) <= lag2(dim0) + thresh # class 0 targets[:, 2:][seq[:, 1:-1, 3] > seq[:, :-2, 0] + thresh] = 1 targets[:, 2:][seq[:, 1:-1, 3] < seq[:, :-2, 0] - thresh] = 2 #targets[:, 2:, 0] = np.cast[np.int](seq[:, 1:-1, 3] > seq[:, :-2, 0]) # SequenceDataset wants a list of sequences # this allows them to be different lengths, but here they're not seq = [i for i in seq] targets = [i for i in targets] gradient_dataset = SequenceDataset([seq, targets], batch_size=None, number_batches=500) cg_dataset = SequenceDataset([seq, targets], batch_size=None, number_batches=100) model = MetaRNN(n_in=n_in, n_hidden=n_hidden, n_out=n_out, activation='tanh', output_type='softmax', use_symbolic_softmax=True) # optimizes negative log likelihood # but also reports zero-one error opt = hf_optimizer(p=model.rnn.params, inputs=[model.x, model.y], s=model.rnn.y_pred, costs=[model.rnn.loss(model.y), model.rnn.errors(model.y)], h=model.rnn.h) # using settings of initial_lambda and mu given in Nicolas' RNN example # seem to do a little worse than the default opt.train(gradient_dataset, cg_dataset, num_updates=n_updates) seqs = xrange(10) plt.close('all') for seq_num in seqs: fig = plt.figure() ax1 = plt.subplot(211) plt.plot(seq[seq_num]) ax1.set_title('input') ax2 = plt.subplot(212) # blue line will represent true classes true_targets = plt.step(xrange(n_steps), targets[seq_num], marker='o') # show probabilities (in b/w) output by model guess = model.predict_proba(seq[seq_num]) guessed_probs = plt.imshow(guess.T, interpolation='nearest', cmap='gray') ax2.set_title('blue: true class, grayscale: probs assigned by model')
# optimizes negative log likelihood # but also reports zero-one error opt = hf_optimizer(p=model.rnn.params, inputs=[model.x, model.y], s=model.rnn.y_pred, n_in=n_in, costs=[model.rnn.loss(model.y), model.rnn.errors(model.y)], h=model.rnn.h) print("Training ...") opt.train(gradient_dataset, cg_dataset, initial_lambda=1.0, num_updates=n_updates, save_progress='save_param') seqs = range(10) plt.close('all') for seq_num in seqs: fig = plt.figure() ax1 = plt.subplot(211) plt.plot(x_seq_list[seq_num]) ax1.set_title('input') ax2 = plt.subplot(212) # blue line will represent true classes true_targets = plt.step(range(n_steps), y_seq_list[seq_num], marker='o') # show probabilities (in b/w) output by model guess = model.predict_proba(x_seq_list[seq_num]) guessed_probs = plt.imshow(guess.T, interpolation='nearest', cmap='gray') ax2.set_title('blue: true class, grayscale: probs assigned by model') print("Elapsed time: %f" % (time.time() - t0))
def RNN_k_fold_croos_validation(sample, lable, n_updates=20): X = sample y = lable kf = KFold(n_splits=5, shuffle=True) split_num = kf.get_n_splits(X) k = 1 G1, G2, S, Total = 0, 0, 0, 0 for train_index, test_index in kf.split(X): # print("TRAIN:", train_index, "TEST:", test_index) X_train, X_test = X[train_index], X[test_index] y_train, y_test = y[train_index], y[test_index] n_hidden = 10 n_in = 40 n_classes = 3 n_out = n_classes # restricted to single softmax per time step np.random.seed(0) train_sample = [i for i in X_train] train_lable = [i for i in y_train] gradient_dataset = SequenceDataset([train_sample, train_lable], batch_size=None, number_batches=500) cg_dataset = SequenceDataset([train_sample, train_lable], batch_size=None, number_batches=100) model = MetaRNN(n_in=n_in, n_hidden=n_hidden, n_out=n_out, activation='tanh', output_type='softmax', use_symbolic_softmax=True) opt = hf_optimizer( p=model.rnn.params, inputs=[model.x, model.y], s=model.rnn.y_pred, costs=[model.rnn.loss(model.y), model.rnn.errors(model.y)], h=model.rnn.h) opt.train(gradient_dataset, cg_dataset, num_updates=n_updates) right, first, second, third = 0, 0, 0, 0 first_sum, second_sum, third_sum = 0, 0, 0 for count in range(0, X_test.shape[0]): guess = model.predict_proba(X_test[count]) tmp_list = np.ndarray.tolist(guess.T) # print test_sample.shape if (y_test[count][0] == 0): first_sum += 1 if (tmp_list.index(max(tmp_list)) == y_test[count][0]): print True first += 1 right += 1 else: print k, False elif (y_test[count][0] == 1): second_sum += 1 if (tmp_list.index(max(tmp_list)) == y_test[count][0]): print True second += 1 right += 1 else: print False else: third_sum += 1 if (tmp_list.index(max(tmp_list)) == y_test[count][0]): print True third += 1 right += 1 else: print False print "...................................................................................................." print k, "interation" print "...................................................................................................." G1 = G1 + 1.0 * first / first_sum S = S + 1.0 * second / second_sum G2 = G2 + 1.0 * third / third_sum Total = Total + 1.0 * right / X_test.shape[0] print "class G1:", G1 / k print "class S:", S / k print "class G2:", G2 / k print "class total:", Total / k k += 1 print "...................................................................................................." print "Final Result:" print "...................................................................................................." print "class G1:", G1 / split_num print "class S:", S / split_num print "class G2:", G2 / split_num print "class total:", Total / split_num
def RNN_leave_one_cross_validation(seq, targets, n_updates=250, n_seq=182): """ Test RNN with softmax outputs. """ length = len(seq) right, first, second, third = 0, 0, 0, 0 false_list = [] for k in range(0, length): n_hidden = 10 n_in = 40 n_classes = 3 n_out = n_classes # restricted to single softmax per time step np.random.seed(0) train_sample = copy.deepcopy(seq) train_lable = copy.deepcopy(targets) test_sample = seq[k] train_sample = np.delete(train_sample, k, 0) train_lable = np.delete(train_lable, k, 0) train_sample = [i for i in train_sample] train_lable = [i for i in train_lable] gradient_dataset = SequenceDataset([train_sample, train_lable], batch_size=None, number_batches=500) cg_dataset = SequenceDataset([train_sample, train_lable], batch_size=None, number_batches=100) model = MetaRNN(n_in=n_in, n_hidden=n_hidden, n_out=n_out, activation='tanh', output_type='softmax', use_symbolic_softmax=True) opt = hf_optimizer( p=model.rnn.params, inputs=[model.x, model.y], s=model.rnn.y_pred, costs=[model.rnn.loss(model.y), model.rnn.errors(model.y)], h=model.rnn.h) opt.train(gradient_dataset, cg_dataset, num_updates=n_updates) guess = model.predict_proba(test_sample) tmp_list = np.ndarray.tolist(guess.T) # print tmp_list if (targets[k][0] == 0): if (tmp_list.index(max(tmp_list)) == targets[k][0]): print k, True first += 1 right += 1 else: print k, False false_list.append(k) elif (targets[k][0] == 1): if (tmp_list.index(max(tmp_list)) == targets[k][0]): print k, True second += 1 right += 1 else: print k, False false_list.append(k) else: if (tmp_list.index(max(tmp_list)) == targets[k][0]): print k, True third += 1 right += 1 else: print k, False false_list.append(k) print "class G1:", 1.0 * first / 59 print "class S:", 1.0 * second / 58 print "class G2:", 1.0 * third / 65 print "class total:", 1.0 * right / length print false_list
def test_binary(multiple_out=False, n_updates=250): """ Test RNN with binary outputs. """ n_hidden = 10 n_in = 5 if multiple_out: n_out = 2 else: n_out = 1 n_steps = 10 n_seq = 100 np.random.seed(0) # simple lag test seq = np.random.randn(n_seq, n_steps, n_in) targets = np.zeros((n_seq, n_steps, n_out), dtype='int32') # whether lag 1 (dim 3) is greater than lag 2 (dim 0) targets[:, 2:, 0] = np.cast[np.int32](seq[:, 1:-1, 3] > seq[:, :-2, 0]) if multiple_out: # whether product of lag 1 (dim 4) and lag 1 (dim 2) # is less than lag 2 (dim 0) targets[:, 2:, 1] = np.cast[np.int32]( (seq[:, 1:-1, 4] * seq[:, 1:-1, 2]) > seq[:, :-2, 0]) # SequenceDataset wants a list of sequences # this allows them to be different lengths, but here they're not seq = [i for i in seq] targets = [i for i in targets] gradient_dataset = SequenceDataset([seq, targets], batch_size=None, number_batches=500) cg_dataset = SequenceDataset([seq, targets], batch_size=None, number_batches=100) model = MetaRNN(n_in=n_in, n_hidden=n_hidden, n_out=n_out, activation='tanh', output_type='binary') # optimizes negative log likelihood # but also reports zero-one error opt = hf_optimizer(p=model.rnn.params, inputs=[model.x, model.y], s=model.rnn.y_pred, costs=[model.rnn.loss(model.y), model.rnn.errors(model.y)], h=model.rnn.h) # using settings of initial_lambda and mu given in Nicolas' RNN example # seem to do a little worse than the default opt.train(gradient_dataset, cg_dataset, num_updates=n_updates) seqs = range(10) plt.close('all') for seq_num in seqs: fig = plt.figure() ax1 = plt.subplot(211) plt.plot(seq[seq_num]) ax1.set_title('input') ax2 = plt.subplot(212) true_targets = plt.step(range(n_steps), targets[seq_num], marker='o') guess = model.predict_proba(seq[seq_num]) guessed_targets = plt.step(range(n_steps), guess) plt.setp(guessed_targets, linestyle='--', marker='d') for i, x in enumerate(guessed_targets): x.set_color(true_targets[i].get_color()) ax2.set_ylim((-0.1, 1.1)) ax2.set_title('solid: true output, dashed: model output (prob)')
def test_softmax(n_updates=250): """ Test RNN with softmax outputs. """ n_hidden = 10 n_in = 5 n_steps = 10 n_seq = 100 n_classes = 3 n_out = n_classes # restricted to single softmax per time step np.random.seed(0) # simple lag test seq = np.random.randn(n_seq, n_steps, n_in) targets = np.zeros((n_seq, n_steps), dtype='int32') thresh = 0.5 # if lag 1 (dim 3) is greater than lag 2 (dim 0) + thresh # class 1 # if lag 1 (dim 3) is less than lag 2 (dim 0) - thresh # class 2 # if lag 2(dim0) - thresh <= lag 1 (dim 3) <= lag2(dim0) + thresh # class 0 targets[:, 2:][seq[:, 1:-1, 3] > seq[:, :-2, 0] + thresh] = 1 targets[:, 2:][seq[:, 1:-1, 3] < seq[:, :-2, 0] - thresh] = 2 #targets[:, 2:, 0] = np.cast[np.int](seq[:, 1:-1, 3] > seq[:, :-2, 0]) # SequenceDataset wants a list of sequences # this allows them to be different lengths, but here they're not seq = [i for i in seq] targets = [i for i in targets] gradient_dataset = SequenceDataset([seq, targets], batch_size=None, number_batches=500) cg_dataset = SequenceDataset([seq, targets], batch_size=None, number_batches=100) model = MetaRNN(n_in=n_in, n_hidden=n_hidden, n_out=n_out, activation='tanh', output_type='softmax', use_symbolic_softmax=True) # optimizes negative log likelihood # but also reports zero-one error opt = hf_optimizer(p=model.rnn.params, inputs=[model.x, model.y], s=model.rnn.y_pred, costs=[model.rnn.loss(model.y), model.rnn.errors(model.y)], h=model.rnn.h) # using settings of initial_lambda and mu given in Nicolas' RNN example # seem to do a little worse than the default opt.train(gradient_dataset, cg_dataset, num_updates=n_updates) seqs = range(10) plt.close('all') for seq_num in seqs: fig = plt.figure() ax1 = plt.subplot(211) plt.plot(seq[seq_num]) ax1.set_title('input') ax2 = plt.subplot(212) # blue line will represent true classes true_targets = plt.step(range(n_steps), targets[seq_num], marker='o') # show probabilities (in b/w) output by model guess = model.predict_proba(seq[seq_num]) guessed_probs = plt.imshow(guess.T, interpolation='nearest', cmap='gray') ax2.set_title('blue: true class, grayscale: probs assigned by model')