def __init__(self, voca_size, input_size, bilstm_hidden_size, ffnn_hidden_size, ffnn_output_size, freeze=True): super(BOW_BiLSTM_RANDOM, self).__init__() # Get the dimension of embeddings as the input size of bilstm self.voca_size = voca_size self.input_size = input_size self.bilstm_hidden_size = bilstm_hidden_size self.ffnn_hidden_size = ffnn_hidden_size self.ffnn_output_size = ffnn_output_size self.freeze = freeze # BOW layer self.bow = nn.EmbeddingBag(voca_size, input_size, mode='mean') self.bow.weight.requires_grad = not self.freeze # Bilstm network self.bilstm = nn.LSTM(self.input_size, self.bilstm_hidden_size, bidirectional=True) # Feed forward neural network with one hidden layer self.ffnn = FFNN(self.bilstm_hidden_size * 2, self.ffnn_hidden_size, self.ffnn_output_size) # Softmax layer self.log_softmax = nn.LogSoftmax(dim=1)
def __init__(self, voca_size, input_size, bilstm_hidden_size, ffnn_hidden_size, ffnn_output_size, freeze=True): super(BiLSTM_FFNN_RANDOM, self).__init__() self.voca_size = voca_size self.input_size = input_size self.bilstm_hidden_size = bilstm_hidden_size self.ffnn_hidden_size = ffnn_hidden_size self.ffnn_output_size = ffnn_output_size self.freeze = freeze # Generate random word embeddings self.embeddingLayer = nn.Embedding(self.voca_size, self.input_size) # Freezed --> gradient not required ; Not freezed --> gradient required self.embeddingLayer.weight.requires_grad = not self.freeze # Bilstm network self.bilstm = nn.LSTM(self.input_size, self.bilstm_hidden_size, bidirectional=True) # Feed forward neural network with one hidden layer self.ffnn = FFNN(self.bilstm_hidden_size * 2, self.ffnn_hidden_size, self.ffnn_output_size) # Softmax layer self.log_softmax = nn.LogSoftmax(dim=1)
def __init__(self, embeddings, bilstm_hidden_size, ffnn_hidden_size, ffnn_output_size, freeze=True): super(BOW_BiLSTM_PRE, self).__init__() # Get the dimension of embeddings as the input size of bilstm _, self.input_size = embeddings.size() self.bilstm_hidden_size = bilstm_hidden_size self.ffnn_hidden_size = ffnn_hidden_size self.ffnn_output_size = ffnn_output_size self.freeze = freeze # Use pretrained embeddings, bag of words layer self.bow = nn.EmbeddingBag.\ from_pretrained( embeddings, freeze=self.freeze, mode='mean' ) # Bilstm network self.bilstm = nn.LSTM(self.input_size, self.bilstm_hidden_size, bidirectional=True) # Feed forward neural network with one hidden layer self.ffnn = FFNN(self.bilstm_hidden_size * 2, self.ffnn_hidden_size, self.ffnn_output_size) # Softmax layer self.log_softmax = nn.LogSoftmax(dim=1)
def __init__(self, embeddings, hidden_size, output_size, freeze=True): super(BOW_FFNN, self).__init__() self.voca_size, self.input_size = embeddings.size() self.hidden_size = hidden_size self.output_size = output_size self.freeze = freeze self.bow = nn.EmbeddingBag.from_pretrained(embeddings, self.freeze, mode='mean') self.ffnn = FFNN(self.input_size, self.hidden_size, self.output_size)
def __init__(self, embeddings, hidden_size, output_size, freeze = True): super(BOW_FFNN_PRE,self).__init__() self.voca_size,self.input_size = embeddings.size() self.hidden_size = hidden_size self.output_size = output_size self.freeze = freeze # BOW layer self.bow = nn.EmbeddingBag.from_pretrained(embeddings,self.freeze,mode='mean') # FFNN with single hidden layer self.ffnn = FFNN(self.input_size,self.hidden_size,self.output_size) self.log_softmax = nn.LogSoftmax(dim = 1)
def __init__(self, voca_size, input_size, bilstm_hidden_size, ffnn_hidden_size, ffnn_output_size, freeze = True): super(BiLSTM_FFNN_RANDOM, self).__init__() self.voca_size = voca_size self.input_size = input_size self.bilstm_hidden_size = bilstm_hidden_size self.ffnn_hidden_size = ffnn_hidden_size self.ffnn_output_size = ffnn_output_size self.freeze = freeze self.embeddingLayer = nn.Embedding(self.voca_size, self.input_size) self.embeddingLayer.weight.requires_grad = not self.freeze self.bilstm = nn.LSTM(self.input_size,self.bilstm_hidden_size, bidirectional=True) self.ffnn = FFNN(self.bilstm_hidden_size*2,self.ffnn_hidden_size, self.ffnn_output_size) self.log_softmax = nn.LogSoftmax(dim=1)
def __init__(self, voca_size, input_size, hidden_size, output_size, freeze=True): super(BOW_FFNN_RANDOM, self).__init__() self.input_size = input_size self.hidden_size = hidden_size self.output_size = output_size self.freeze = freeze self.bow = nn.EmbeddingBag(voca_size, input_size, mode='mean') self.bow.weight.requires_grad = not self.freeze self.ffnn = FFNN(self.input_size, self.hidden_size, self.output_size) self.log_softmax = nn.LogSoftmax(dim=1)
def __init__(self, embeddings, bilstm_hidden_size, ffnn_hidden_size, ffnn_output_size, freeze=True): super(BiLSTM_FFNN_PRE, self).__init__() _, self.input_size = embeddings.size() self.bilstm_hidden_size = bilstm_hidden_size self.ffnn_hidden_size = ffnn_hidden_size self.ffnn_output_size = ffnn_output_size self.freeze = freeze self.emb = nn.Embedding.from_pretrained(embeddings, self.freeze) self.bilstm = nn.LSTM(self.input_size, self.bilstm_hidden_size, bidirectional=True) self.ffnn = FFNN(self.bilstm_hidden_size * 2, self.ffnn_hidden_size, self.ffnn_output_size) self.log_softmax = nn.LogSoftmax(dim=1)
def test_ffnn(): params = { 'n_layers': 4, 'hidden_nodes': [512, 512, 512, 512], 'epochs': 10, 'use_dynamic_features': True, 'use_mspec': False, 'as_mat': False, 'speaker_norm': False, 'context_length': 17 } net = FFNN(params) model = net.train_model() net.set_model(model) y_true, yp = net.predict_on_test() print("FFNN RESULTS") print(get_f1_score(y_true, yp)) print(get_accuracy(y_true, yp)) print(classification_report(y_true, yp))
from ffnn import FFNN import numpy as np from sklearn.datasets import make_moons nn = FFNN(2, [5], 2, noise_scale=.1) I = np.identity(2) X_data, y = make_moons(200, .17) y_data = np.array([I[value] for value in y]) fit_score = nn.fit(X_data, y_data) for fit_line in fit_score: print(fit_line) print(nn.accuracy(X_data, y_data))
from ffnn import FFNN import numpy as np from sklearn.datasets import make_moons nn = FFNN(2,[5],2,noise_scale=.1) I = np.identity(2) X_data,y = make_moons(200,.17) y_data = np.array([I[value] for value in y]) fit_score = nn.fit(X_data,y_data) for fit_line in fit_score: print(fit_line) print(nn.accuracy(X_data,y_data))
y_test = test.pIC50.values tan_gp = TanimotoGP() tan_gp.fit(X_train, y_train) gp_pred, gp_var = tan_gp.predict(X_test) gp_r2 = r2_score(y_test, gp_pred) gp_rmse = rmse(y_test, gp_pred) print(gp_r2, gp_rmse) xgb = XGBRegressor() xgb.fit(X_train, y_train) xgb_pred = xgb.predict(X_test) xgb_r2 = r2_score(y_test, xgb_pred) xgb_rmse = rmse(y_test, xgb_pred) ff_nn = FFNN() ff_nn.fit(X_train, y_train) ff_pred = ff_nn.predict(X_test) ff_r2 = r2_score(y_test, ff_pred) ff_rmse = rmse(y_test, ff_pred) print([dataset, col, gp_r2, xgb_r2, ff_r2, gp_rmse, xgb_rmse, ff_rmse]) sys.stdout.flush() res.append( [dataset, col, gp_r2, xgb_r2, ff_r2, gp_rmse, xgb_rmse, ff_rmse]) res_df = pd.DataFrame(res, columns=[ "dataset", "split", 'gp_r2', 'xgb_r2', 'ffnn_r2', 'gp_rmse', 'xgb_rmse', 'ffnn_rmse' ]) res_df.to_csv("comparison.csv", index=False)
if __name__ == "__main__": feature_vector_dimensions = 300 questions_vector_dimensions = 500 kernel_size = 3 classifier_hidden_size_1 = 300 classifier_hidden_size_2 = 150 num_labels = 2 learning_rate = 1e-4 weight_decay = 1e-3 n_epochs = 4 batch_size = 16 encoder = CNN(feature_vector_dimensions, questions_vector_dimensions, kernel_size) classifier = FFNN(questions_vector_dimensions, classifier_hidden_size_1, classifier_hidden_size_2, num_labels) lamb_list = [1e-1] best_lamb = 0 best_score = 0 database = TransferLearningDatabase() for lamb in lamb_list: training_dataset = database.get_training_set() android_validation_dataset = database.get_validation_set() android_test_dataset = database.get_testing_set() optimizer_encoder = torch.optim.Adam(encoder.parameters(), lr=learning_rate, weight_decay=weight_decay) optimizer_domain = torch.optim.Adam(classifier.parameters(), lr=learning_rate, weight_decay=weight_decay) for epoch in xrange(n_epochs):
base = ('rnn_sgd_base.pt', 'ffnn_sgd_base.pt') hx2 = ('rnn_sgd_hx2.pt', 'ffnn_sgd_hx2.pt') lx2 = ('rnn_sgd_lx2.pt', 'ffnn_sgd_lx2.pt') files = (base, hx2, lx2) directory = 'models_part_a/' base_models = [] #RNN SGD BASE path = directory + base[0] model = RNN(32, 1, 64, True) model.load_state_dict(torch.load(path)) base_models.append(model) #FFNN SGD BASE path = directory + base[1] model = FFNN(97305, 32, 1) model.load_state_dict(torch.load(path)) base_models.append(model) hx2_models = [] #RNN SGD hx2 path = directory + hx2[0] model = RNN(64, 1, 64, True) model.load_state_dict(torch.load(path)) hx2_models.append(model) #FFNN SGD hx2 path = directory + hx2[1] model = FFNN(97305, 64, 1) model.load_state_dict(torch.load(path)) hx2_models.append(model)
def ffnn_instance(): ffnn = FFNN(RANDOM_INIT, ALL_WEIGHTS_TRAINABLE, 17, [40, 4], [['rbmw1', 'rbmhb1'], ['rbmw2', 'rbmhb2']], transfer_function=tf.nn.sigmoid) return ffnn
pickle_file = "network.pickle" with open("mnist_array_output.pickle", "rb") as f: X_train, X_test, y_train, y_test = pickle.load(f) load_pickle = input("Load pickled networks (y/n)? ") == "y" do_plot = input("Plot the predicted data? (y/n)? ") == "y" if load_pickle: with open(pickle_file, "rb") as f: nn_relu = pickle.load(f) else: do_pickle = input("Pickle the neural networks (y/n)? ") == "y" layer_structure = [X_train.shape[1], 200, y_train.shape[1]] training_data = list(zip(X_train, y_train)) nn_relu = FFNN(layer_structure, activation_function=ReLU) nn_relu.SGD_train(training_data) if do_pickle: with open(pickle_file, "wb") as f: pickle.dump(nn_relu, f) if do_plot: y_predict_relu_vector_form = nn_relu.predict(X_test) # Reshape from vector y values into scalars y_predict_relu = np.zeros(y_predict_relu_vector_form.shape[1]) for i, prediction in enumerate(y_predict_relu_vector_form.T): y_predict_relu[i] = np.argmax(prediction) with open("mnist_X_test_unraveled.pickle", "rb") as f:
z2[x][27] = 1.0 train_labels = np.concatenate((train_labels, z2), axis=0) train_in = train_in.reshape( (train_in.shape[0], train_in.shape[1] * train_in.shape[2])) train_in = np.concatenate((train_in, im_list)) train_in = train_in.astype(float) for x in range(len(train_in)): train_in[x] = abs(255 - train_in[x]) train_in[x] = train_in[x] / 255.0 print(train_in.shape) # Define the neural network structure # Here we have just one hidden layer with 2048 nodes network = FFNN([784, 2048, 28], post_function=tf.nn.softmax, session=1) # Train the model using gradient descent def cross_entropy_with_softmax(model_output, true_output): return tf.reduce_sum( tf.nn.softmax_cross_entropy_with_logits(labels=true_output, logits=model_output)) network.train( train_in, train_labels, epochs=5, # was 1100 batch_size=100, lc_interval=300,
from ffnn import FFNN out_dir = "../../doc/assets/" N = 100 deg = 8 x = np.linspace(0, 1, N) y = np.linspace(0, 1, N) x, y = np.meshgrid(x, y) x = np.ravel(x) y = np.ravel(y) X = design_matrix(x, y, deg) Y = franke(x, y, noise_sigma=0.1, noise=True) X_train, X_test, y_train, y_test = split_and_scale(X, Y, test_size=0.3) layer_structure = [X.shape[1], 50, 1] training_data = list(zip(X_train, y_train)) nn1 = FFNN(layer_structure, eta=0.1) eta01_MSE = nn1.SGD_train(training_data, report_convergence=True) nn2 = FFNN(layer_structure, eta=0.01) eta001_MSE = nn2.SGD_train(training_data, report_convergence=True) nn2 = FFNN(layer_structure, eta=0.001) eta0001_MSE = nn2.SGD_train(training_data, report_convergence=True) batches = list(range(len(eta01_MSE))) plt.plot(batches, eta01_MSE, label=r"$\eta = 0.1$") plt.plot(batches, eta001_MSE, label=r"$\eta = 0.01$") plt.plot(batches, eta0001_MSE, label=r"$\eta = 0.001$") plt.legend() plt.show()
x = np.ravel(x) y = np.ravel(y) X = design_matrix(x, y, deg) Y = franke(x, y, noise_sigma=0.1, noise=True) X_train, X_test, y_train, y_test = split_and_scale(X, Y, test_size=0.3) load_pickle = input("Load pickled networks (y/n)? ") == "y" if load_pickle: with open("network.pickle", "rb") as f: nn_sigmoid, nn_relu, nn_leaky = pickle.load(f) else: do_pickle = input("Pickle the neural networks (y/n)? ") == "y" layer_structure = [X.shape[1], 50, 1] training_data = list(zip(X_train, y_train)) nn_sigmoid = FFNN(layer_structure, epochs=20) nn_sigmoid.SGD_train(training_data) nn_relu = FFNN(layer_structure, epochs=20, activation_function=ReLU) nn_relu.SGD_train(training_data) nn_leaky = FFNN(layer_structure, epochs=20, activation_function=leaky_ReLU) nn_leaky.SGD_train(training_data) if do_pickle: with open("network.pickle", "wb") as f: pickle.dump((nn_sigmoid, nn_relu, nn_leaky), f) nn_sigmoid_predict = nn_sigmoid.predict(X_test) nn_relu_predict = nn_relu.predict(X_test) nn_leaky_predict = nn_leaky.predict(X_test)