def __init__(self, args, device='cuda'): super().__init__() self.args = args self.device = device self.img_channels = 3 self.depths = [args.zdim, 256, 256, 256, 128, 128] self.didx = 0 self.alpha = 1. # init G self.G = nn.ModuleList() blk = nn.ModuleList() blk.append(ll.Conv2d(self.depths[0], self.depths[0], 4, padding=3)) # to 4x4 blk.append(ll.Conv2d(self.depths[0], self.depths[0], 3, padding=1)) self.G.append(blk) self.toRGB = nn.ModuleList() self.toRGB.append(ll.Conv2d(self.depths[0], self.img_channels, 1, lrelu=False, pnorm=False)) # toRGB # init D self.fromRGB = nn.ModuleList() self.fromRGB.append(ll.Conv2d(self.img_channels, self.depths[0], 1)) # fromRGB self.D = nn.ModuleList() blk = nn.ModuleList() blk.append(ll.MinibatchStddev()) blk.append(ll.Conv2d(self.depths[0]+1, self.depths[0], 3, padding=1)) blk.append(ll.Conv2d(self.depths[0], self.depths[0], 4, stride=4)) # to 1x1 blk.append(ll.Flatten()) blk.append(ll.Linear(self.depths[0], 1)) self.D.append(blk) self.doubling = nn.Upsample(scale_factor=2) self.halving = nn.AvgPool2d(2, 2) self.set_optimizer() # self.criterion = losses.GANLoss(loss_type=args.loss_type, device=device) self.loss_type = args.loss_type
def __init__(self, input_shape): super(LocalDiscriminator, self).__init__() self.input_shape = input_shape self.output_shape = (1024,) self.img_c = input_shape[0] self.img_h = input_shape[1] self.img_w = input_shape[2] self.conv1 = nn.Conv2d(self.img_c, 64, kernel_size=5, stride=2, padding=2) self.bn1 = nn.BatchNorm2d(64) self.act1 = nn.ReLU() self.conv2 = nn.Conv2d(64, 128, kernel_size=5, stride=2, padding=2) self.bn2 = nn.BatchNorm2d(128) self.act2 = nn.ReLU() self.conv3 = nn.Conv2d(128, 256, kernel_size=5, stride=2, padding=2) self.bn3 = nn.BatchNorm2d(256) self.act3 = nn.ReLU() self.conv4 = nn.Conv2d(256, 512, kernel_size=5, stride=2, padding=2) self.bn4 = nn.BatchNorm2d(512) self.act4 = nn.ReLU() self.conv5 = nn.Conv2d(512, 512, kernel_size=5, stride=2, padding=2) self.bn5 = nn.BatchNorm2d(512) self.act5 = nn.ReLU() in_features = 512 * (self.img_h//32) * (self.img_w//32) self.flatten6 = layers.Flatten() self.linear6 = nn.Linear(in_features, 1024) self.act6 = nn.ReLU()
def Encoder(img_size, in_channel, conv_channel, filter_size, latent_dim, dense_size, bn): inner_conv_channel = conv_channel // 2 if img_size % 4 != 0: print("WARNING: image size mod 4 != 0, may produce bug.") # total input number of the input of the last conv layer, new image size = old / 2 / 2 flatten_img_size = inner_conv_channel * img_size / 4 * img_size / 4 # explain: first two layer's padding = 2, because we set W/S = W/S + floor((-F+2P)/S+1), S=2,F=5,so P=2 if VERBOSE: print(img_size, in_channel, conv_channel, filter_size, latent_dim, bn) model = nn.Sequential( layers.ConvLayer(in_channel, conv_channel, filter_size, stride=2, padding=2, bn=bn), layers.ConvLayer(conv_channel, inner_conv_channel, filter_size, stride=2, padding=2, bn=bn), layers.ConvLayer(inner_conv_channel, inner_conv_channel, filter_size, stride=1, padding=2, bn=bn), layers.Flatten(), layers.Dense(flatten_img_size, dense_size), layers.Dense(dense_size, latent_dim)) model = model.to(device=device, dtype=dtype) model = torch.nn.DataParallel(model, device_ids=GPU_IDs) return model
def __init__(self, input_shape, arc='places2'): super(GlobalDiscriminator, self).__init__() self.arc = arc self.input_shape = input_shape self.output_shape = (1024, ) self.img_c = input_shape[0] self.img_h = input_shape[1] self.img_w = input_shape[2] self.conv1 = nn.Conv2d(self.img_c, 64, kernel_size=5, stride=2, padding=2) self.bn1 = nn.BatchNorm2d(64) self.act1 = nn.ReLU() self.conv2 = nn.Conv2d(64, 128, kernel_size=5, stride=2, padding=2) self.bn2 = nn.BatchNorm2d(128) self.act2 = nn.ReLU() self.conv3 = nn.Conv2d(128, 256, kernel_size=5, stride=2, padding=2) self.bn3 = nn.BatchNorm2d(256) self.act3 = nn.ReLU() self.conv4 = nn.Conv2d(256, 512, kernel_size=5, stride=2, padding=2) self.bn4 = nn.BatchNorm2d(512) self.act4 = nn.ReLU() self.conv5 = nn.Conv2d(512, 512, kernel_size=5, stride=2, padding=2) self.bn5 = nn.BatchNorm2d(512) self.act5 = nn.ReLU() if arc == 'celeba': in_features = 512 * (self.img_h // 32) * (self.img_w // 32) self.flatten6 = layers.Flatten() self.linear6 = nn.Linear(in_features, 1024) self.act6 = nn.ReLU() elif arc == 'places2': self.conv6 = nn.Conv2d(512, 512, kernel_size=5, stride=2, padding=2) self.bn6 = nn.BatchNorm2d(512) self.act6 = nn.ReLU() in_features = 512 * (self.img_h // 64) * (self.img_w // 64) self.flatten7 = layers.Flatten() self.linear7 = nn.Linear(in_features, 1024) self.act7 = nn.ReLU() else: raise ValueError('Unsupported architecture \'%s\'.' % self.arc)
def __init__(self, inputSize, outputSize): super().__init__() self.outputSize = outputSize self.Flatten = layers.Flatten() self.fc1 = layers.FullyConnected(inputSize, 10, activation=activate.SoftMax()) self.inputLayer = self.Flatten self.outputLayer = self.fc1 self.Flatten.addSon(self.fc1)
def test_layers(): x = torch.randn(3, 64, 224, 224) m = layers.ConvBatchnormRelu2d(64, 32, 3, padding=1) x = m(x) m = layers.Flatten() x = m(x) m = layers.SingSqrt() x = m(x) m = layers.L2Norm() x = m(x)
def __init__(self, output_size=5, hidden_size=64, layers=4): super().__init__() self.hidden_size = hidden_size self.base = ConvBase(output_size=hidden_size, hidden=hidden_size, channels=1, max_pool=False, layers=layers) self.features = M.Sequential( kl.Lambda(lambda x: F.reshape(x, (-1, 1, 28, 28))), self.base, kl.Lambda(lambda x: kf.mean(x, axis=[2, 3])), kl.Flatten(), ) self.classifier = M.Linear(hidden_size, output_size, bias=True)
def build_raw_branch(self , resnet_kwargs ): branch_layers = [] blocks = self.trunk.build_blocks( resnet_kwargs['block'] , resnet_kwargs['num_features'][-2] , resnet_kwargs['num_features'][-1] , resnet_kwargs['strides'][-1] , resnet_kwargs['num_blocks'][-1] ) branch_layers.append( blocks ) shape = ( resnet_kwargs['input_shape'][0] // prod( resnet_kwargs['strides']) , resnet_kwargs['input_shape'][1] // prod( resnet_kwargs['strides'] ) ) if resnet_kwargs['use_maxpool']: shape = ( shape[0] // 2 , shape[1] // 2 ) if resnet_kwargs['use_avgpool']: avgpool = nn.AvgPool2d( [*shape] , 1 ) branch_layers.append( avgpool ) shape = 1 * 1 if resnet_kwargs['feature_layer_dim'] is not None: fc1 = nn.Sequential( layers.Flatten() , layers.linear( resnet_kwargs['num_features'][-1] * shape , resnet_kwargs['feature_layer_dim'] , activation_fn = None , pre_activation = False , use_batchnorm = resnet_kwargs['use_batchnorm']) ) branch_layers.append( fc1 ) return nn.Sequential( *branch_layers )
def __init__(self, input_size, output_size, sizes=None): super().__init__() if sizes is None: sizes = [256, 128, 64, 64] layers = [ LinearBlock(input_size, sizes[0]), ] for s_i, s_o in zip(sizes[:-1], sizes[1:]): layers.append(LinearBlock(s_i, s_o)) layers = M.Sequential(*layers) self.features = M.Sequential( kl.Flatten(), layers, ) self.classifier = M.Linear(sizes[-1], output_size) fc_init_(self.classifier) self.input_size = input_size
def __init__(self, inputSize, outputSize): super().__init__() self.outputSize = outputSize self.conv1 = layers.Convolution(5, 1) self.Pool1 = layers.Pool(5, stride=2) self.Flatten = layers.Flatten() self.fc1 = layers.FullyConnected(14 * 14 * 5, 50, activation=activate.Relu()) self.fc2 = layers.FullyConnected(50, outputSize, activation=activate.SoftMax()) self.inputLayer = self.conv1 self.outputLayer = self.fc2 self.conv1.addSon(self.Pool1) self.Pool1.addSon(self.Flatten) self.Flatten.addSon(self.fc1) self.fc1.addSon(self.fc2)
def __init__(self, lossfunc, optimizer, batch_size): super().__init__(lossfunc, optimizer, batch_size) self.conv0 = L.Convolution_(n_filter=8, filter_size=(3, 3), stride=1) self.conv1 = L.Convolution_(n_filter=16, filter_size=(3, 3), stride=1) self.fc0 = L.Linear_(output_size=1024) self.fc1 = L.Linear_(output_size=10) self.bn0 = L.BatchNormalization_() self.bn1 = L.BatchNormalization_() self.bn4 = L.BatchNormalization_() self.acti0 = L.ELU() self.acti1 = L.ELU() self.acti4 = L.ELU() self.pool0 = L.MaxPooling(7, 7) self.pool1 = L.MaxPooling(5, 5) self.flat = L.Flatten() self.drop0 = L.Dropout(0.5) self.drop1 = L.Dropout(0.5) self.layers = [ self.conv0, self.acti0, self.pool0, self.bn0, #self.drop0, self.conv1, self.acti1, self.pool1, self.bn1, #self.drop1, self.flat, self.fc0, self.acti4, self.bn4, self.fc1, ]
def test_mnist_with_cov2d(): (train_x, train_y), (test_x, test_y) = mnist.load_data(flatten=False) val_x = train_x[50000:] val_y = train_y[50000:] train_x = train_x[:50000] train_y = train_y[:50000] batch_size = 200 modle = models.Sequential() modle.add( layers.Conv2D(4, (3, 3), stride=1, pad=1, input_shape=(None, 1, 28, 28))) modle.add(layers.ReLU()) modle.add(layers.MaxPool2D((2, 2), stride=2)) modle.add(layers.Flatten()) modle.add(layers.Linear(10)) modle.add(layers.Softmax()) acc = losses.categorical_accuracy.__name__ modle.compile(losses.CrossEntropy(), optimizers.SGD(lr=0.001), metrics=[losses.categorical_accuracy]) modle.summary() history = modle.train(train_x, train_y, batch_size, epochs=4, validation_data=(val_x, val_y)) epochs = range(1, len(history["loss"]) + 1) plt.plot(epochs, history["loss"], 'ro', label="Traning loss") plt.plot(epochs, history["val_loss"], 'go', label="Validating loss") plt.plot(epochs, history[acc], 'r', label="Traning accuracy") plt.plot(epochs, history["val_" + acc], 'g', label="Validating accuracy") plt.title('Training/Validating loss/accuracy') plt.xlabel('Epochs') plt.ylabel('Loss/Accuracy') plt.legend() plt.show(block=True)
sig = mf.sigmoid() # MaxPool layer 2x2 stride of 1 pool1 = layers.Maxpool(conv1.out_dim, size=2, stride=2) # Conv layer with 2 filters kernel size of 3x3 stride of 1 and no padding conv2 = layers.Conv(pool1.out_dim, n_filter=2, h_filter=3, w_filter=3, stride=1, padding=0) # activation for layer 2 rectified linear relu = mf.ReLU() # MaxPool layer 2x2 stride 1 pool2 = layers.Maxpool(conv2.out_dim, size=2, stride=1) # Flatten the matrix flat = layers.Flatten() # Fully connected layer with 50 neurons fc1 = layers.FullyConnected(np.prod(pool2.out_dim), 50) # Activation for fully connected layer of 50 neurons is tanh tanh = mf.TanH() # Fully connected layer with 10 neurons 'output layer' out = layers.FullyConnected(50, num_classes) cnn = layers.CNN([conv1, sig, pool1, conv2, relu, pool2, flat, fc1, tanh, out]) mf.model_summary(cnn, 'cnn_model_plot.png', f) e_nnet, e_accuracy, e_validate, e_loss, e_loss_val = mf.sgd(cnn, x_train, y_train,
layers.Convolution("conv5", kernel_size=3, kernel_channels=384, n_kernels=256, stride=1, same_padding=True, logging=False)) graph.add_layer(layers.Activation(activation="relu", logging=False)) graph.add_layer( layers.MaxPooling(window_size=3, window_channels=256, stride=2, logging=False)) graph.add_layer(layers.Flatten((9216, 1), logging=False)) graph.add_layer( layers.FullyConnected("fc1", 9216, 4096, activation="relu", logging=False)) graph.add_layer( layers.FullyConnected("fc2", 4096, 4096, activation="relu", logging=False)) graph.add_layer( layers.FullyConnected("fc3", 4096, n_classes, activation="softmax", logging=False)) try: print() print("[+] N_EXAMPLES:", n_examples) for e in range(epochs):
def get_accuracy(self, X, Y): # print("Expected: ", Y) Yhat = self.predict(X) # print("Predictions: ", Yhat) matches = (Yhat == Y.reshape(self.m)) # print(matches) return np.sum(matches) / len(matches) if __name__ == "__main__": nx = 10 m = 12 X = np.array(np.random.randn(nx, 12)) Y = np.array([[0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1]]) nn = NeuralNetwork() nn.add_layer(layers.Flatten(nx)) nn.add_layer( layers.Dense(20, activation=ActivationFunctions.relu, initialization="He")) nn.add_layer( layers.Dense(30, activation=ActivationFunctions.relu, initialization="He")) nn.add_layer( layers.Dense(1, activation=ActivationFunctions.sigmoid, initialization="He")) costs = nn.train(X, Y, alpha=0.01, num_epochs=1500, print_cost=True)
activation = layers.Activation(activation="relu") X = activation.forward(X) print("[+] ACTIVATION_shape:", X.shape) dActivation = activation.backward(np.ones_like(X)) print("dActivation_SHAPE:", dActivation.shape) print() maxpool = layers.MaxPooling(window_size=2, window_channels=10, stride=2) X = maxpool.forward(X) print("[+] MAXPOOLED_shape:", X.shape) dMaxpool = maxpool.backward(np.ones_like(X)) print("dMaxpool_SHAPE:", dMaxpool.shape) N, C, H, W = X.shape flatten = layers.Flatten((C * H * W, N)) X = flatten.forward(X) print(X.shape) I, N = X.shape fc1 = layers.FullyConnected("fc1", I, 1000, activation="relu") X = fc1.forward(X) print(X.shape) fc2 = layers.FullyConnected("fc2", 1000, 10, activation="softmax") Y = fc2.forward(X) print(Y.shape) cross_entropy = layers.CrossEntropy() loss = cross_entropy.forward(Y, T)
import layers import functions x = np.asarray([[1, 1, 1], [2, 2, 2], [3, 3, 3]]) a = x.shape // 2 w = x[0, :] x = np.sqrt(1e-8) / np.sqrt(1e-8) net = layers.Network(layers.Adam(0.01)) net.add(layers.Dense(2, 5, functions.LeakyReLu(), '1st hidden')) net.add(layers.Dense(5, 20, functions.LeakyReLu(), '1st hidden')) net.add(layers.Dense(20, 5, functions.LeakyReLu(), '2nd hidden')) net.add(layers.Flatten()) net.add(layers.Dense(5, 3, functions.LeakyReLu(), '3nd hidden')) net.add(layers.Dense(3, 1, functions.Sigmoid(), 'Out')) inp = np.asarray([[0, 0], [1, 0], [0, 1], [1, 1]]) shape = inp.shape x = np.reshape(inp, (8)) y = np.reshape(x, shape) print(inp) target = np.asarray([0, 1, 1, 0]) print(target) result = net.propagate(inp[0]) print(result)
weight_decay = config['weight_decay'] net = [] regularizers = [] inputs = np.random.randn(config['batch_size'], 1, 28, 28) net += [layers.Convolution(inputs, 16, 5, "conv1")] regularizers += [ layers.L2Regularizer(net[-1].weights, weight_decay, 'conv1_l2reg') ] net += [layers.MaxPooling(net[-1], "pool1")] net += [layers.ReLU(net[-1], "relu1")] net += [layers.Convolution(net[-1], 32, 5, "conv2")] regularizers += [ layers.L2Regularizer(net[-1].weights, weight_decay, 'conv2_l2reg') ] net += [layers.MaxPooling(net[-1], "pool2")] net += [layers.ReLU(net[-1], "relu2")] ## 7x7 net += [layers.Flatten(net[-1], "flatten3")] net += [layers.FC(net[-1], 512, "fc3")] regularizers += [ layers.L2Regularizer(net[-1].weights, weight_decay, 'fc3_l2reg') ] net += [layers.ReLU(net[-1], "relu3")] net += [layers.FC(net[-1], 10, "logits")] data_loss = layers.SoftmaxCrossEntropyWithLogits() loss = layers.RegularizedLoss(data_loss, regularizers) nn.train(train_x, train_y, valid_x, valid_y, net, loss, config) nn.evaluate("Test", test_x, test_y, net, loss, config)
def __init__( self, lossfunc, optimizer, batch_size=32, ): self.lossfunc = lossfunc self.optimizer = optimizer self.batch_size = batch_size input_size = 64 hidden_size = 3136 output_size = 10 # self.lr = 0.001 # self.alpha = 0.9 self.l1 = 1e-4 self.l2 = 1e-4 self.optimizer = optimizers.Adam(l1=self.l1, l2=self.l2) self.conv0 = L.Convolution_(n_filter=8, filter_size=(3, 3), stride=1) self.conv1 = L.Convolution_(n_filter=16, filter_size=(3, 3), stride=1) self.conv2 = L.Convolution_(n_filter=32, filter_size=(5, 5), stride=1) self.conv3 = L.Convolution_(n_filter=64, filter_size=(5, 5), stride=1) self.fc0 = L.Linear_(output_size=1024) self.fc1 = L.Linear_(output_size=10) self.bn0 = L.BatchNormalization_() self.bn1 = L.BatchNormalization_() self.bn2 = L.BatchNormalization_() self.bn3 = L.BatchNormalization_() self.bn4 = L.BatchNormalization_() self.acti0 = L.ELU() self.acti1 = L.ELU() self.acti2 = L.ELU() self.acti3 = L.ELU() self.acti4 = L.ELU() self.pool0 = L.MaxPooling(7, 7) self.pool1 = L.MaxPooling(5, 5) self.pool2 = L.MaxPooling(3, 3) self.pool3 = L.MaxPooling(3, 3) self.flat = L.Flatten() self.drop0 = L.Dropout(0.5) self.drop1 = L.Dropout(0.5) self.drop2 = L.Dropout(0.5) self.drop3 = L.Dropout(0.25) self.layers = [ self.conv0, self.acti0, self.pool0, self.bn0, #self.drop0, self.conv1, self.acti1, self.pool1, self.bn1, #self.drop1, #self.conv2, #self.acti2, #self.pool2, #self.bn2, #self.drop2, #self.conv3, #self.acti3, #self.pool3, #self.bn3, #self.drop3, self.flat, self.fc0, self.acti4, self.bn4, self.fc1, ]