def __init__(self): super(SimplerCNN, self).__init__() self.dropout2d_input = nn.Dropout2d(rate=0.3) self.conv1 = nn.Conv2d(in_channels=3, out_channels=15, kernel_size=3, stride=3, padding=2) self.relu1 = nn.LeakyRelu() self.conv2 = nn.Conv2d(in_channels=15, out_channels=30, kernel_size=3, stride=3, padding=3) self.relu2 = nn.LeakyRelu() self.dropout2d_conv1 = nn.Dropout2d(rate=0.5) self.conv3 = nn.Conv2d(in_channels=30, out_channels=40, kernel_size=4) self.relu3 = nn.LeakyRelu() self.flatten = nn.Flatten() self.dropout2d_conv2 = nn.Dropout2d(rate=0.2) self.linear = nn.Linear(in_dimension=360, out_dimension=180) self.relu4 = nn.LeakyRelu() self.bn1 = nn.BatchNorm() self.dropout3 = nn.Dropout(rate=0.3) self.linear2 = nn.Linear(in_dimension=180, out_dimension=10) self.bn2 = nn.BatchNorm() self.softmax = nn.Softmax() self.set_forward()
def __init__(self): super(NN, self).__init__() self.linear1 = nn.Linear(in_dimension=3072, out_dimension=256) self.relu1 = nn.LeakyRelu() self.dropout1 = nn.Dropout(rate=0.3) self.linear2 = nn.Linear(in_dimension=256, out_dimension=10) self.softmax = nn.Softmax() self.set_forward()
def __init__(self): super(LinearNet, self).__init__([ lrp_module.Reshape(28, 28, 1), lrp_module.Linear(28 * 28, 1296), lrp_module.ReLU(), lrp_module.Linear(1296, 1296), lrp_module.ReLU(), lrp_module.Linear(1296, 1296), lrp_module.ReLU(), lrp_module.Linear(1296, 10) ]) self.outputLayers = [0, 3, 5, 7, 8]
def __init__(self, in_features, hidden_features, num_layers, num_classes): super(MLPClassifier, self).__init__() net = [] net.append(modules.Linear(in_features, hidden_features)) net.append(modules.BatchRegularization(hidden_features)) net.append(modules.CPReLU(hidden_features)) for i in range(num_layers - 1): net.append( modules.LinearResidueBlock(hidden_features, residue_ratio=1 / (i + 2), batch_reg=True)) net.append(modules.Linear(hidden_features, num_classes)) self.net = nn.Sequential(*net)
def testNN2(runs=1, width=3, data="iris.txt", iters=20000, std=True, trainPct=0.666): if data == 'gauss': X, y = multimodalData(numModes=4, numPerMode=30) # X, y = sklearn.datasets.make_classification() XY = np.asarray(np.hstack([X, y.reshape((X.shape[0], 1))])) else: XY = data_io.read(data) nclass = len(set(XY[:, -1])) # number of classes # y has nclass classes (0, ..., nclass-1) def unary(yi): return [(1 if i == yi else 0) for i in range(nclass)] # build a network u = width nn = modules.Sequential([ modules.Linear(XY.shape[1] - 1, u), modules.Tanh(), modules.Linear(u, u), modules.Tanh(), modules.Linear(u, nclass), modules.SoftMax() ]) results = {False: [], True: []} for run in range(runs): Xtrain, ytrain, Xtest, ytest = splitByClass(XY, trainPct) # Map into n softmax outputs Ytrain = np.array([unary(yi) for yi in ytrain]) for rms in (False, True): # train the network. nn.clean() nn.train2(np.asarray(Xtrain), np.asarray(Ytrain), batchsize=1, iters=iters, lrate_decay_step=1000, rms=rms, momentum=(0.9 if rms else None)) errors = predictionErrors(nn, Xtest, ytest) accuracy = 1.0 - (float(errors) / Xtest.shape[0]) print 'RMS', rms, 'Prediction accuracy', accuracy results[rms].append(accuracy) print 'Results', results print 'Average accuracy', 'rms=False', sum( results[False]) / runs, 'rms=True', sum(results[True]) / runs,
def test_updateWeights(self): in_features = 3 out_features = 2 x = FloatTensor([[1, 2, 3], [3, 2, -1]]) A = FloatTensor([[1, 2, 3], [4, 5, 6]]) b = FloatTensor([[1, 2]]) expected_output = FloatTensor([[15, 34], [5, 18]]) m = M.Linear(in_features, out_features) m.weights = A m.bias = b m.forward(x) grad = FloatTensor(([1, 4], [-1, -1])) output = m.backward(grad) expected_gradient = grad.transpose(0, 1).mm(x) eta = 0.1 m.updateWeights(eta) new_value = A - eta * expected_gradient if not areEqual(m.weights, new_value): return 1 new_value = b - eta * FloatTensor(np.sum(grad.numpy(), axis=0)) if not areEqual(m.bias, new_value): return 1 return 0
def __init__(self): super(ConvNet, self).__init__([ lrp_module.Conv2d(1, 6, 5), lrp_module.ReLU(), lrp_module.MaxPool2d(2, 2), lrp_module.Conv2d(6, 16, 5), lrp_module.ReLU(), lrp_module.MaxPool2d(2, 2), lrp_module.Reshape(4, 4, 16), lrp_module.Linear(4 * 4 * 16, 120), lrp_module.ReLU(), lrp_module.Linear(120, 100), lrp_module.ReLU(), lrp_module.Linear(100, 10) ]) self.outputLayers = [0, 2, 3, 5, 6, 9, 11, 12]
def conv_layer(x): """ the derivative check in the gradient checker relates to the input of the function hence, the input should be z - since the backward step computes @loss / @z """ conv1 = nn.Conv2d(in_channels=1, out_channels=2, kernel_size=2) relu1 = nn.Relu() conv2 = nn.Conv2d(in_channels=2, out_channels=4, kernel_size=2) relu2 = nn.Relu() flatten = nn.Flatten() linear = nn.Linear(4, 2) softmax = nn.Softmax() # forward pass a = relu1(conv1(x)) a = relu2(conv2(a)) a_flatten = flatten(a) dist = softmax(linear(a_flatten)) # backward labels = np.zeros(dist.shape) labels[:, 1] = 1 loss = -np.log(np.sum(dist * labels, axis=1)) softmax_grad = softmax.backward(labels) linear_grad = linear.backward(softmax_grad) flatten_grad = flatten.backward(linear_grad) relu2_grad = relu2.backward(flatten_grad) conv2_grad = conv2.backward(relu2_grad) relu1_grad = relu1.backward(conv2_grad) conv1_grad = conv1.backward(relu1_grad) return loss, conv1_grad
def conv(b): """ the derivative check in the gradient checker relates to the input of the function hence, the input should be z - since the backward step computes @loss / @z """ # simulate end of classification conv = nn.Conv2d(in_channels=1, out_channels=3, kernel_size=2) relu = nn.Relu() flatten = nn.Flatten() linear = nn.Linear(in_dimension=12, out_dimension=4) softmax = nn.Softmax() conv.set_biases(b.reshape(3, 1)) # forward a = flatten(relu(conv(x))) dist = softmax(linear(a)) # backward labels = np.zeros(dist.shape) labels[:, 1] = 1 loss = -np.log(np.sum(dist * labels, axis=1)) softmax_grad = softmax.backward(labels) linear_grad = linear.backward(softmax_grad) flatten_grad = flatten.backward(linear_grad) relu_grad = relu.backward(flatten_grad) conv_grad = conv.backward(relu_grad) b_grad = conv.b_grad return loss, b_grad
def test_resetGradient(self): in_features = 3 out_features = 2 x = FloatTensor([[1, 2, 3], [3, 2, -1]]) A = FloatTensor([[1, 2, 3], [4, 5, 6]]) b = FloatTensor([[1, 2]]) expected_output = FloatTensor([[15, 34], [5, 18]]) m = M.Linear(in_features, out_features) m.weights = A m.bias = b m.forward(x) grad = FloatTensor(([1, 4], [-1, -1])) output = m.backward(grad) m.resetGradient() if torch.max(torch.abs(m.weights_grad)) != 0 or \ torch.max(torch.abs(m.bias_grad)) != 0: return 1 return 0
def __init__(self, num_phn=hp.num_phn, encoder_dim=hp.encoder_dim, encoder_n_layer=hp.encoder_n_layer, kernel_size=hp.kernel_size, stride=hp.stride, padding=hp.padding, decoder_dim=hp.decoder_dim, decoder_n_layer=hp.decoder_n_layer, dropout=hp.dropout): super(DurIAN, self).__init__() self.n_step = hp.n_frames_per_step self.embedding = nn.Embedding(num_phn, encoder_dim) self.conv_banks = nn.ModuleList([ modules.BatchNormConv1d(encoder_dim, encoder_dim, kernel_size, stride, padding, activation=nn.ReLU(), w_init_gain="relu") for _ in range(3) ]) self.encoder = nn.GRU(encoder_dim, encoder_dim // 2, encoder_n_layer, batch_first=True, bidirectional=True) self.length_regulator = modules.LengthRegulator() self.prenet = modules.Prenet(hp.num_mels, hp.prenet_dim, hp.prenet_dim) self.decoder = nn.GRU(decoder_dim, decoder_dim, decoder_n_layer, batch_first=True, bidirectional=False) self.mel_linear = modules.Linear(decoder_dim, hp.num_mels * self.n_step) self.postnet = modules.CBHG(hp.num_mels, K=8, projections=[256, hp.num_mels]) self.last_linear = modules.Linear(hp.num_mels * 2, hp.num_mels)
def dense(layer): m = layer.input_shape[-1] n = layer.output_shape[-1] module = modules.Linear(m, n) W, B = layer.get_weights() module.W = W module.B = B activation_module = get_activation_lrpmodule(layer.activation) return module, activation_module
def test_backwardBeforeForward(self): in_features = 1 out_features = 10 m = M.Linear(in_features, out_features) try: m.backward(FloatTensor([[1]])) except ValueError: return 0 return 1
def test_forwardWrongInputDimension(self): in_features = 3 out_features = 10 m = M.Linear(in_features, out_features) try: m.forward(FloatTensor([[1, 2]])) except ValueError: return 0 return 1
def test_backwardWrongInputDimension(self): in_features = 1 out_features = 10 m = M.Linear(in_features, out_features) m.forward(FloatTensor([[1]])) try: m.backward(FloatTensor([[1, 2], [3, 4]]), FloatTensor([[1, 2, 3], [3, 4, 1]])) except ValueError: return 0 return 1
def test_linear_module_1(self): x = np.array([[1, 2, 33, 15]]) w = np.array([[0., 1., 0., 0.], [0., 2., 2., 0.]]) b = np.zeros(2) expected_res = np.array([[2., 70.]]) linear_layer = nn.Linear(4, 2) linear_layer.set_weights(w) linear_layer.set_biases(b) z = linear_layer(x) np.testing.assert_allclose(z, expected_res, atol=0.0001)
def __init__(self, w_in, h_in, num_features, blocks, adain_features, adain_blocks, num_classes=-1, in_features=4): super(ClassifierOrDiscriminator, self).__init__(w_in, h_in, num_features, blocks, adain_features, adain_blocks, batch_reg=True, in_features=in_features) self.num_classes = num_classes if num_classes > 0: self.cla = modules.Linear(self.out_features, num_classes) else: self.dis = modules.Linear(self.out_features, 1)
def test_linear_module_softmax_1(self): x = np.array([[1, 2, 0, -1], [1, 2, -1, -2]]) w = np.array([[0., 1., 0., 0.], [0., 2., 2., 0.]]) b = np.zeros(2) expected_res = np.array([[.119202, .88079], [.5, .5]]) linear = nn.Linear(4, 2) softmax = nn.Softmax() linear.set_weights(w) linear.set_biases(b) z = linear(x) a = softmax(z) np.testing.assert_allclose(a, expected_res, atol=0.0001)
def test_linear_module_relu_2(self): x = np.array([[1, 2, 0, -1], [1, 2, -1, -2]]) w = np.array([[0., 1., 0., 0.], [0., 2., 2., 0.]]) b = np.array([-5., -1.]) expected_res = np.array([[0., 3.], [0., 1.]]) linear_layer = nn.Linear(4, 2) linear_layer.set_weights(w) linear_layer.set_biases(b) relu_layer = nn.Relu() z = linear_layer(x) a = relu_layer(z) np.testing.assert_allclose(a, expected_res, atol=0.0001)
def test_forwardCorrectOutput(self): in_features = 3 out_features = 2 x = FloatTensor([[1, 2, 3], [3, 2, -1]]) A = FloatTensor([[1, 2, 3], [4, 5, 6]]) b = FloatTensor([[1, 2]]) expected_output = FloatTensor([[15, 34], [5, 18]]) m = M.Linear(in_features, out_features) m.weights = A m.bias = b output = m.forward(x) if not areEqual(expected_output, output): return 1 return 0
def flatten(x): flatten_ = nn.Flatten() linear = nn.Linear(in_dimension=48, out_dimension=4) softmax = nn.Softmax() # forward flatten_x = flatten_(x) dist = softmax(linear(flatten_x)) # backward labels = np.zeros(dist.shape) labels[:, 1] = 1 loss = -np.log(np.sum(dist * labels, axis=1)) softmax_grad = softmax.backward(labels) linear_grad = linear.backward(softmax_grad) flatten_grad = flatten_.backward(linear_grad) return loss, flatten_grad
def __init__(self): super(SimpleCNN, self).__init__() self.conv1 = nn.Conv2d(in_channels=3, out_channels=6, kernel_size=3, stride=3, padding=2) self.tanh1 = nn.Tanh() self.conv2 = nn.Conv2d(in_channels=6, out_channels=10, kernel_size=3, stride=3, padding=3) self.tanh2 = nn.Tanh() self.dropout2d = nn.Dropout2d(rate=0.5) self.flatten = nn.Flatten() self.linear = nn.Linear(in_dimension=360, out_dimension=10) self.softmax = nn.Softmax() self.set_forward()
def test_constructor(self): in_features = 4 out_features = 10 m = M.Linear(in_features, out_features) if m.weights.size() != m.weights_grad.size(): return 1 if m.bias.size() != m.bias_grad.size(): return 1 if m.weights.size() != (out_features, in_features): return 1 if m.bias.size() != (1, out_features): return 1 return 0
def linear_layer(z): """ the derivative check in the gradient checker relates to the input of the function hence, the input should be z - since the backward step computes @loss / @z """ # simulate end of classification relu_layer = nn.Relu() linear = nn.Linear(in_dimension=2, out_dimension=5) softmax = nn.Softmax() a_L_mins_1 = relu_layer(z) z_L = linear(a_L_mins_1) a_L = softmax(z_L) labels = np.zeros(a_L.shape) labels[:, 1] = 1 loss = -np.log(np.sum(a_L * labels, axis=1)) softmax_grad = softmax.backward(labels) layer_L_grad = linear.backward(softmax_grad) relu_grad = relu_layer.backward(layer_L_grad) return loss, relu_grad
def linear_layer(b): """ the derivative check in the gradient checker relates to the input of the function hence, the input should be z - since the backward step computes @loss / @z """ # simulate end of classification linear = nn.Linear(in_dimension=3, out_dimension=2) linear.set_biases(b) softmax = nn.Softmax() # forward dist = softmax(linear(z)) # backward labels = np.zeros(dist.shape) labels[:, 1] = 1 loss = -np.log(np.sum(dist * labels, axis=1)) softmax_grad = softmax.backward(labels) linear_grad = linear.backward(softmax_grad) b_grad = linear.b_grad return loss, b_grad
import tools.data_loader import tools.model_io import modules import shutil import settings # user init batchsize = 10 numbIters = 10000 # load data X, Y = tools.data_loader.load_data() # setup neural network nn = modules.Sequential([ modules.Linear(32**2, 2), modules.NegAbs(), modules.BinStep(), modules.Linear(2, 2), modules.SoftMax() ]) # train neural network nn.train(X['train'], Y['train'], Xval=X['valid'], Yval=Y['valid'], batchsize=batchsize, iters=numbIters) # determine training name of neural net
import tools.data_loader import tools.model_io import modules import shutil import settings # user init batchsize = 10 numbIters = 10000 # load data X, Y = tools.data_loader.load_data() # setup neural network nn = modules.Sequential([ modules.Linear(settings.nrOfPixels, 4), modules.Tanh(), modules.Linear(4, 4), modules.Tanh(), modules.Linear(4, 2), modules.SoftMax() ]) # train neural network nn.train(X['train'], Y['train'], Xval=X['valid'], Yval=Y['valid'], batchsize=batchsize, iters=numbIters)
def testNN(iters=10000, width=3, learning_rates=[0.005], std=True): # Test cases # X, y = xor() # X, y = xor_more() X, y = multimodalData(numModes=4) # X, y = sklearn.datasets.make_moons(noise=0.25) # X, y = sklearn.datasets.make_classification() # y has 2 classes (0, 1) # Map into 2 softmax outputs nclass = len(set(listify(y))) def unary(yi): return [(1 if i == yi else 0) for i in range(nclass)] Y = np.array([unary(yi) for yi in y]) #build a network nd = X.shape[1] # number of features u = width nn = modules.Sequential([ modules.Linear(nd, u), modules.Tanh(), modules.Linear(u, u), modules.Tanh(), modules.Linear(u, nclass), modules.SoftMax() ]) # train the network. errors = [] for lrate in learning_rates: nn.clean() # Default does not do learning rate decay or early stopping. #print ('X,Y,',X,Y) nn.train2(np.asarray(X), np.asarray(Y), batchsize=1, iters=iters, lrate=lrate, lrate_decay_step=100000) errors.append((lrate, predictionErrors(nn, X, y))) print 'Errors for learning rates', errors # Plot the last result if nd > 2: return # only plot two-feature cases eps = .1 xmin = np.min(X[:, 0]) - eps xmax = np.max(X[:, 0]) + eps ymin = np.min(X[:, 1]) - eps ymax = np.max(X[:, 1]) + eps ax = tidyPlot(xmin, xmax, ymin, ymax, xlabel='x', ylabel='y') def fizz(x1, x2): y = nn.forward(np.array([x1, x2])) return y[0, 1] # class 1 res = 30 # resolution of plot ima = np.array([[fizz(xi, yi) for xi in np.linspace(xmin, xmax, res)] \ for yi in np.linspace(ymin, ymax, res)]) im = ax.imshow(np.flipud(ima), interpolation='none', extent=[xmin, xmax, ymin, ymax], cmap='viridis' if std else 'jet') plt.colorbar(im) if std: colors = [('r' if l == 0 else 'g') for l in y] ax.scatter(X[:, 0], X[:, 1], c=colors, marker='o', s=80, edgecolors='none') else: pinds = np.where(y == 0) ninds = np.where(y == 1) plt.plot(X[pinds[0], 0], X[pinds[0], 1], 'ob') plt.plot(X[ninds[0], 0], X[ninds[0], 1], 'or')
import tools.data_loader import tools.model_io import modules import shutil import settings # user init batchsize = 10 numbIters = 10000 # load data X, Y = tools.data_loader.load_data() # setup neural network nn = modules.Sequential([ modules.Linear(32**2, 1), modules.NegAbs(), modules.Tanh(), modules.Linear(1, 2), modules.SoftMax() ]) # train neural network nn.train(X['train'], Y['train'], Xval=X['valid'], Yval=Y['valid'], batchsize=batchsize, iters=numbIters) # determine training name of neural net
import tools.data_loader import tools.model_io import modules import shutil import settings # user init batchsize = 10 numbIters = 10000 # load data X, Y = tools.data_loader.load_data() # setup neural network nn = modules.Sequential([ modules.Linear(settings.nrOfPixels, 16), modules.Tanh(), modules.Linear(16, 16), modules.Tanh(), modules.Linear(16, 3), modules.SoftMax() ]) # train neural network nn.train(X['train'], Y['train'], Xval=X['valid'], Yval=Y['valid'], batchsize=batchsize, iters=numbIters)