def train(self, X, y, learning_rate=1e-3, reg=1e-4, decay_rate=1.00, opt='sgd', n_iters=1000, batch_size=200, verbose=True): lr = learning_rate self.reg = reg for i in range(n_iters): ids = np.random.choice(X.shape[0], batch_size) layer1, l1cache = layers.dense_forward(X[ids], self.W1, self.b1) layer2, l2cache = layers.non_linearity_forward(layer1, hiddenLayer='relu') layer3, l3cache = layers.dense_forward(layer2, self.W2, self.b2) layer4, l4cache = layers.non_linearity_forward(layer3, hiddenLayer='sigmoid') loss, l5cache = layers.binary_cross_entropy_loss_forward(layer4, y[ids]) # adding regularization loss loss += 0.5*self.reg*(np.sum(layer2*layer2))/(batch_size*batch_size) dlayer5 = 1.0 dlayer4 = layers.binary_cross_entropy_loss_backward(dlayer5, l5cache) dlayer3 = layers.non_linearity_backward(dlayer4, l4cache, hiddenLayer='sigmoid') dlayer2, dW2, db2 = layers.dense_backward(dlayer3, l3cache) dlayer2 += (self.reg*layer2)/batch_size dlayer1 = layers.non_linearity_backward(dlayer2, l2cache, hiddenLayer='relu') _, dW1, db1 = layers.dense_backward(dlayer1, l1cache) if i % 500 == 0: lr *= decay_rate if verbose: print "Iteration %d, loss = %g" % (i, loss) self.params, self.W1 = optimizers.optimize(self.params, self.W1, dW1, lr=lr, name='W1', opt=opt) self.params, self.b1 = optimizers.optimize(self.params, self.b1, db1, lr=lr, name='b1', opt=opt) self.params, self.W2 = optimizers.optimize(self.params, self.W2, dW2, lr=lr, name='W2', opt=opt) self.params, self.b2 = optimizers.optimize(self.params, self.b2, db2, lr=lr, name='b2', opt=opt) self.loss_history.append(loss)
def getloss(self, X, y): layer1, _ = layers.dense_forward(X, self.W1, self.b1) layer2, _ = layers.non_linearity_forward(layer1, hiddenLayer='relu') layer3, _ = layers.dense_forward(layer2, self.W2, self.b2) layer4, _ = layers.non_linearity_forward(layer3, hiddenLayer='sigmoid') loss, _ = layers.binary_cross_entropy_loss_forward(layer4, y) loss += 0.5 * self.reg * (np.sum(self.W1 * self.W1) + np.sum(self.W2 * self.W2)) return loss
def predict(self, X): # return the highest value for each row after a forward pass l1out, _ = layers.dense_forward(X, self.W1, self.b1) l2out, _ = layers.non_linearity_forward(l1out, self.hiddenLayer) l3out, _ = layers.dense_forward(l2out, self.W2, self.b2) l4out, _ = layers.non_linearity_forward(l3out,self.hiddenLayer) l5out, _ = layers.dense_forward(l4out, self.W3, self.b3) return np.argmax(l5out, axis=1)
def predict(self, X): W1, b1 = self.weights['W1'], self.weights['b1'] W2, b2 = self.weights['W2'], self.weights['b2'] W3, b3 = self.weights['W3'], self.weights['b3'] # return the highest value for each row after a forward pass l1out, _ = layers.dense_forward(X, W1, b1) l2out, _ = layers.non_linearity_forward(l1out, self.non_linearity) l3out, _ = layers.dense_forward(l2out, W2, b2) l4out, _ = layers.non_linearity_forward(l3out, self.non_linearity) l5out, _ = layers.dense_forward(l4out, W3, b3) return np.argmax(l5out, axis=1)
def train(self, X, y, X_val=None, y_val=None, learning_rate=1e-2, reg = 1e-4, decay_rate=0.95, opt='sgd', n_iters=5000, batch_size=200, verbose=1): lr = learning_rate for i in xrange(n_iters): # adding dense layer1 ids = np.random.choice(X.shape[0], batch_size) l1out, l1cache = layers.dense_forward(X[ids], self.W1, self.b1) # adding non-linearity layer2 l2out, l2cache = layers.non_linearity_forward(l1out,self.hiddenLayer) # adding dense layer3 l3out, l3cache = layers.dense_forward(l2out, self.W2, self.b2) # adding non-linearity layer4 l4out,l4cache = layers.non_linearity_forward(l3out, self.hiddenLayer) # adding dense layer5 l5out,l5cache = layers.dense_forward(l4out,self.W3, self.b3) # adding softmax layer loss, l6cache = layers.softmax_loss_forward(l5out, y[ids]) loss = loss + 0.5*reg*(np.sum(self.W1**2) + np.sum(self.W2**2) + np.sum(self.W3**2)) self.loss_history.append(loss) if verbose and i % 500 == 0: lr *= decay_rate print "Iteration %d, loss = %f" % (i, loss) if X_val is not None and y_val is not None: print "Validation Accuracy :%f" % (self.accuracy(X_val, y_val)) dlayer6 = 1.0 dlayer5 = layers.softmax_loss_backward(dlayer6, l6cache) dlayer4, dW3, db3 = layers.dense_backward(dlayer5, l5cache) dlayer3 = layers.non_linearity_backward(dlayer4, l4cache, self.hiddenLayer) dlayer2, dW2, db2 = layers.dense_backward(dlayer3, l3cache) dlayer1 = layers.non_linearity_backward(dlayer2, l2cache, self.hiddenLayer) _, dW1, db1 = layers.dense_backward(dlayer1, l1cache) self.gradientLayer1.append(np.mean(np.abs(dlayer1))) self.gradientLayer2.append(np.mean(np.abs(dlayer3))) self.params, self.W1 = optimizers.optimize(self.params, self.W1, dW1, lr=lr, name='W1', opt=opt) self.params, self.b1 = optimizers.optimize(self.params, self.b1, db1, lr=lr, name='b1', opt=opt) self.params, self.W2 = optimizers.optimize(self.params, self.W2, dW2, lr=lr, name='W2', opt=opt) self.params, self.b2 = optimizers.optimize(self.params, self.b2, db2, lr=lr, name='b2', opt=opt) self.params, self.W3 = optimizers.optimize(self.params, self.W3, dW3, lr=lr, name='W3', opt=opt) self.params, self.b3 = optimizers.optimize(self.params, self.b3, db3, lr=lr, name='b3', opt=opt) # gradients due to regularization self.W1 += reg * dW1 self.W2 += reg * dW2 self.W3 += reg * dW3
def predict(self, X): N, T, D = X.shape h0 = np.zeros((N, self.hidden_dim)) layer1, l1cache = rnn_layers.rnn_forward(X, h0, self.Wx, self.Wh, self.b, self.non_liniearity) final_layer = (layer1[:, T - 1, :]) layer2, _ = layers.dense_forward(final_layer, self.W1, self.b1) return np.argmax(layer2, axis=1)
def test_denselayer(): x = np.random.randn(10, 6) w = np.random.randn(6, 5) b = np.random.randn(5) dout = np.random.randn(10, 5) dx_num = eval_numerical_gradient_array(lambda x: layers.dense_forward(x, w, b)[0], x, dout) dw_num = eval_numerical_gradient_array(lambda w: layers.dense_forward(x, w, b)[0], w, dout) db_num = eval_numerical_gradient_array(lambda b: layers.dense_forward(x, w, b)[0], b, dout) _, cache = layers.dense_forward(x, w, b) dx, dw, db = layers.dense_backward(dout, cache) # The error should be around 1e-10 print 'Testing dense layers:' print 'dx error: ', rel_error(dx_num, dx) print 'dw error: ', rel_error(dw_num, dw) print 'db error: ', rel_error(db_num, db)
def test_denselayer(): x = np.random.randn(10, 6) w = np.random.randn(6, 5) b = np.random.randn(5) dout = np.random.randn(10, 5) dx_num = eval_numerical_gradient_array( lambda x: layers.dense_forward(x, w, b)[0], x, dout) dw_num = eval_numerical_gradient_array( lambda w: layers.dense_forward(x, w, b)[0], w, dout) db_num = eval_numerical_gradient_array( lambda b: layers.dense_forward(x, w, b)[0], b, dout) _, cache = layers.dense_forward(x, w, b) dx, dw, db = layers.dense_backward(dout, cache) # The error should be around 1e-10 print 'Testing dense layers:' print 'dx error: ', rel_error(dx_num, dx) print 'dw error: ', rel_error(dw_num, dw) print 'db error: ', rel_error(db_num, db)
def train(self, X, learning_rate=1e-2, batch_size=100, nb_epochs=1): """ Training based on CBOW model using negative sampling :param nb_epochs: number of iterations :param batch_size: the number of sentences trained upon in 1 iteration :param learning_rate: :param X: list of sentences used for training """ N = len(X) start_index = self.word_to_index[self.start_token] end_index = self.word_to_index[self.end_token] unknown_index = self.word_to_index[self.unknown_token] id_x = [] for i in xrange(N): sentence = nltk.word_tokenize(X[i]) if len(sentence) == 0: continue id_x.append(start_index) for word in sentence: if word in self.word_to_index: id_x.append(self.word_to_index[word]) else: id_x.append(unknown_index) id_x.append(end_index) corpus_size = len(id_x) print corpus_size n_iters = corpus_size//batch_size for epoch in xrange(nb_epochs): for itr in xrange(n_iters): batch = np.random.randint(corpus_size, size=batch_size) trX = np.zeros([batch_size, self.size]) trY = np.zeros([batch_size], dtype=np.int32) context = [] ids_to_update = np.zeros([batch_size], dtype=np.int32) for id, w_id in enumerate(batch): context_ids = id_x[max(0, w_id-self.window):w_id] + id_x[w_id+1:min(w_id+1+self.window, corpus_size)] context.append(context_ids) context_window = np.array(context_ids) trX[id] = np.mean(self.W_inp[context_window, :], axis=0) trY[id] = id ids_to_update[id] = id_x[w_id] context = np.array(context) trX, trY, ids_to_update, context = shuffle(trX, trY, ids_to_update, context, random_state=0) W = self.W_out[ids_to_update] # print trX, trY, ids_to_update, context, W b = np.zeros([batch_size]) layer1, l1cache = layers.dense_forward(trX, W.T, b) layer2, l2cache = layers.sigmoid_forward(layer1) loss, l3cache = layers.softmax_loss_forward(layer2, trY) self.loss_history.append(loss) dlayer3 = 1.0 dlayer2 = layers.softmax_loss_backward(dlayer3, l3cache) dlayer1 = layers.sigmoid_backward(dlayer2, l2cache) dx_inp, dW_tmp, db = layers.dense_backward(dlayer1, l1cache) dW = dW_tmp.T for i in xrange(batch_size): self.W_inp[context[i], :] -= (learning_rate * dx_inp[i])/len(context[i]) self.W_out[ids_to_update, :] -= learning_rate * dW
def train(self, X, y, X_val=None, y_val=None, learning_rate=1e-2, reg=1e-4, decay_rate=0.95, opt='sgd', n_iters=5000, batch_size=200, verbose=1): lr = learning_rate for i in xrange(n_iters): W1, b1 = self.weights['W1'], self.weights['b1'] W2, b2 = self.weights['W2'], self.weights['b2'] W3, b3 = self.weights['W3'], self.weights['b3'] # dense layer1 ids = np.random.choice(X.shape[0], batch_size) l1out, l1cache = layers.dense_forward(X[ids], W1, b1) # non-linearity layer2 l2out, l2cache = layers.non_linearity_forward( l1out, self.non_linearity) # dense layer3 l3out, l3cache = layers.dense_forward(l2out, W2, b2) # non-linearity layer4 l4out, l4cache = layers.non_linearity_forward( l3out, self.non_linearity) # dense layer5 l5out, l5cache = layers.dense_forward(l4out, W3, b3) # softmax layer loss, l6cache = layers.softmax_loss_forward(l5out, y[ids]) loss = loss + 0.5 * reg * (np.sum(W1**2) + np.sum(W2**2) + np.sum(W3**2)) self.loss_history.append(loss) if verbose and i % 500 == 0: lr *= decay_rate print "Iteration %d, loss = %f" % (i, loss) if X_val is not None and y_val is not None: print "Validation Accuracy :%f" % (self.accuracy( X_val, y_val)) dlayer6 = 1.0 dlayer5 = layers.softmax_loss_backward(dlayer6, l6cache) dlayer4, dW3, db3 = layers.dense_backward(dlayer5, l5cache) dlayer3 = layers.non_linearity_backward(dlayer4, l4cache, self.non_linearity) dlayer2, dW2, db2 = layers.dense_backward(dlayer3, l3cache) dlayer1 = layers.non_linearity_backward(dlayer2, l2cache, self.non_linearity) _, dW1, db1 = layers.dense_backward(dlayer1, l1cache) self.gradientLayer1.append(np.mean(np.abs(dlayer1))) self.gradientLayer2.append(np.mean(np.abs(dlayer3))) # gradients due to regularization dW1 += reg * W1 dW2 += reg * W2 dW3 += reg * W3 self.params, W1 = optimizers.optimize(self.params, W1, dW1, lr=lr, name='W1', opt=opt) self.params, b1 = optimizers.optimize(self.params, b1, db1, lr=lr, name='b1', opt=opt) self.params, W2 = optimizers.optimize(self.params, W2, dW2, lr=lr, name='W2', opt=opt) self.params, b2 = optimizers.optimize(self.params, b2, db2, lr=lr, name='b2', opt=opt) self.params, W3 = optimizers.optimize(self.params, W3, dW3, lr=lr, name='W3', opt=opt) self.params, b3 = optimizers.optimize(self.params, b3, db3, lr=lr, name='b3', opt=opt) self.weights['W1'], self.weights['b1'] = W1, b1 self.weights['W2'], self.weights['b2'] = W2, b2 self.weights['W3'], self.weights['b3'] = W3, b3
def train(self, X, y, learning_rate=1e-2, opt='sgd', n_iters=5000, batch_size=200, verbose=1): lr = learning_rate N, T, D = X.shape for i in xrange(n_iters): ids = np.random.choice(X.shape[0], batch_size) h0 = np.zeros((batch_size, self.hidden_dim)) layer1, l1cache = rnn_layers.rnn_forward(X[ids], h0, self.Wx, self.Wh, self.b, self.non_liniearity) final_layer = (layer1[:, T - 1, :]) layer2, l2cache = layers.dense_forward(final_layer, self.W1, self.b1) loss, l3cache = layers.softmax_loss_forward(layer2, y[ids]) self.loss_history.append(loss) if verbose == 1 and i % 500 == 0: print 'Iteration %d: loss %g' % (i, loss) dlayer3 = 1.0 dlayer2 = layers.softmax_loss_backward(dlayer3, l3cache) dlayer1, dW1, db1 = layers.dense_backward(dlayer2, l2cache) dh = np.zeros((batch_size, T, self.hidden_dim)) dh[:, T - 1, :] = dlayer1 _, _, dWx, dWh, db = rnn_layers.rnn_backward(dh, l1cache) self.params, self.Wx = optimizers.optimize(self.params, self.Wx, dWx, lr=lr, name='Wx', opt=opt) self.params, self.Wh = optimizers.optimize(self.params, self.Wh, dWh, lr=lr, name='Wh', opt=opt) self.params, self.b = optimizers.optimize(self.params, self.b, db, lr=lr, name='b', opt=opt) self.params, self.W1 = optimizers.optimize(self.params, self.W1, dW1, lr=lr, name='W1', opt=opt) self.params, self.b1 = optimizers.optimize(self.params, self.b1, db1, lr=lr, name='b1', opt=opt)
def predict(self, X): l1, _ = layers.dense_forward(X, self.W1, self.b1) l2, _ = layers.non_linearity_forward(l1, hiddenLayer='relu') l3, _ = layers.dense_forward(l2, self.W2, self.b2) l4, _ = layers.non_linearity_forward(l3, hiddenLayer='sigmoid') return l4