def build_cnn(input_var=None, W_init=None): """ Builds a VGG style CNN network followed by a fully-connected layer and a softmax layer. input_var: Theano variable for input to the network outputs: pointer to the output of the last layer of network (softmax) :param input_var: theano variable as input to the network :param W_init: Initial weight values :return: a pointer to the output of last layer """ weights = [] # Keeps the weights for all layers count = 0 # If no initial weight is given, initialize with GlorotUniform if W_init is None: W_init = [lasagne.init.GlorotUniform()] * 7 # Input layer network = InputLayer(shape=(None, 3, imSize, imSize), input_var=input_var) # CNN Stack 1 network = Conv2DLayer(network, num_filters=32, filter_size=(3, 3), W=W_init[count], pad='same') count += 1 weights.append(network.W) network = Conv2DLayer(network, num_filters=32, filter_size=(3, 3), W=W_init[count], pad='same') count += 1 weights.append(network.W) network = Conv2DLayer(network, num_filters=32, filter_size=(3, 3), W=W_init[count], pad='same') count += 1 weights.append(network.W) network = Conv2DLayer(network, num_filters=32, filter_size=(3, 3), W=W_init[count], pad='same') count += 1 weights.append(network.W) network = MaxPool2DLayer(network, pool_size=(2, 2)) # CNN Stack 2 network = Conv2DLayer(network, num_filters=64, filter_size=(3, 3), W=W_init[count], pad='same') count += 1 weights.append(network.W) network = Conv2DLayer(network, num_filters=64, filter_size=(3, 3), W=W_init[count], pad='same') count += 1 weights.append(network.W) network = MaxPool2DLayer(network, pool_size=(2, 2)) # CNN Stack 3 network = Conv2DLayer(network, num_filters=128, filter_size=(3, 3), W=W_init[count], pad='same') count += 1 weights.append(network.W) network = MaxPool2DLayer(network, pool_size=(2, 2)) return network, weights
def SonoNet32(input_var, image_size, num_labels): net = {} net['input'] = InputLayer(shape=(None, 1, image_size[0], image_size[1]), input_var=input_var) net['conv1_1'] = batch_norm( Conv2DLayer(net['input'], 32, 3, pad=1, flip_filters=False)) net['conv1_2'] = batch_norm( Conv2DLayer(net['conv1_1'], 32, 3, pad=1, flip_filters=False)) net['pool1'] = MaxPool2DLayer(net['conv1_2'], 2) net['conv2_1'] = batch_norm( Conv2DLayer(net['pool1'], 64, 3, pad=1, flip_filters=False)) net['conv2_2'] = batch_norm( Conv2DLayer(net['conv2_1'], 64, 3, pad=1, flip_filters=False)) net['pool2'] = MaxPool2DLayer(net['conv2_2'], 2) net['conv3_1'] = batch_norm( Conv2DLayer(net['pool2'], 128, 3, pad=1, flip_filters=False)) net['conv3_2'] = batch_norm( Conv2DLayer(net['conv3_1'], 128, 3, pad=1, flip_filters=False)) net['conv3_3'] = batch_norm( Conv2DLayer(net['conv3_2'], 128, 3, pad=1, flip_filters=False)) net['pool3'] = MaxPool2DLayer(net['conv3_3'], 2) net['conv4_1'] = batch_norm( Conv2DLayer(net['pool3'], 256, 3, pad=1, flip_filters=False)) net['conv4_2'] = batch_norm( Conv2DLayer(net['conv4_1'], 256, 3, pad=1, flip_filters=False)) net['conv4_3'] = batch_norm( Conv2DLayer(net['conv4_2'], 256, 3, pad=1, flip_filters=False)) net['pool4'] = MaxPool2DLayer(net['conv4_3'], 2) net['conv5_1'] = batch_norm( Conv2DLayer(net['pool4'], 256, 3, pad=1, flip_filters=False)) net['conv5_2'] = batch_norm( Conv2DLayer(net['conv5_1'], 256, 3, pad=1, flip_filters=False)) net['conv5_3'] = batch_norm( Conv2DLayer(net['conv5_2'], 256, 3, pad=1, flip_filters=False)) net['conv5_p'] = batch_norm( Conv2DLayer(net['conv5_3'], num_filters=128, filter_size=(1, 1))) net['conv6_p'] = batch_norm( Conv2DLayer(net['conv5_p'], num_filters=num_labels, filter_size=(1, 1), nonlinearity=linear)) net['average_pool_p'] = GlobalPoolLayer(net['conv6_p'], pool_function=T.mean) net['softmax_p'] = NonlinearityLayer(net['average_pool_p'], nonlinearity=softmax) net['output'] = net['softmax_p'] net['feature_maps'] = net['conv6_p'] net['last_activation'] = net['average_pool_p'] return net
def simple_network4(input_size, output_size): l_in = InputLayer(shape=((None, ) + input_size), name='inputLayer') #print lasagne.layers.get_output_shape(l_in) network = lasagne.layers.Conv2DLayer( l_in, num_filters=32, filter_size=(8, 8), stride=4, pad=2, nonlinearity=lasagne.nonlinearities.rectify, W=lasagne.init.HeNormal(gain='relu')) network = MaxPool2DLayer(network, 2) network = lasagne.layers.Conv2DLayer( network, num_filters=64, filter_size=(4, 4), stride=2, pad=1, nonlinearity=lasagne.nonlinearities.rectify, W=lasagne.init.HeNormal(gain='relu')) network = MaxPool2DLayer(network, 2, pad=1) network = lasagne.layers.Conv2DLayer( network, num_filters=64, filter_size=(3, 3), stride=1, pad=1, nonlinearity=lasagne.nonlinearities.rectify, W=lasagne.init.HeNormal(gain='relu')) network = MaxPool2DLayer(network, 2, pad=1) network = lasagne.layers.DenseLayer( network, num_units=256, nonlinearity=lasagne.nonlinearities.rectify, W=lasagne.init.HeNormal(gain='relu')) network = lasagne.layers.DenseLayer( network, num_units=256, nonlinearity=lasagne.nonlinearities.rectify, W=lasagne.init.HeNormal(gain='relu')) network = lasagne.layers.DenseLayer(network, num_units=output_size, nonlinearity=None, b=lasagne.init.Constant(.1)) return network
def build_network(input_var, num_input_channels, num_classes): conv_defs = { 'W': lasagne.init.HeNormal('relu'), 'b': lasagne.init.Constant(0.0), 'filter_size': (3, 3), 'stride': (1, 1), 'nonlinearity': lasagne.nonlinearities.LeakyRectify(0.1) } nin_defs = { 'W': lasagne.init.HeNormal('relu'), 'b': lasagne.init.Constant(0.0), 'nonlinearity': lasagne.nonlinearities.LeakyRectify(0.1) } dense_defs = { 'W': lasagne.init.HeNormal(1.0), 'b': lasagne.init.Constant(0.0), 'nonlinearity': lasagne.nonlinearities.softmax } wn_defs = { 'momentum': config.batch_normalization_momentum } net = InputLayer ( name='input', shape=(None, num_input_channels, 28, 28), input_var=input_var) net = GaussianNoiseLayer(net, name='noise', sigma=config.augment_noise_stddev) net = WN(Conv2DLayer (net, name='conv1a', num_filters=32, pad='same', **conv_defs), **wn_defs) net = WN(Conv2DLayer (net, name='conv1b', num_filters=64, pad='same', **conv_defs), **wn_defs) # net = WN(Conv2DLayer (net, name='conv1c', num_filters=128, pad='same', **conv_defs), **wn_defs) net = MaxPool2DLayer (net, name='pool1', pool_size=(2, 2)) net = DropoutLayer (net, name='drop1', p=.5) net = WN(Conv2DLayer (net, name='conv2a', num_filters=32, pad='same', **conv_defs), **wn_defs) net = WN(Conv2DLayer (net, name='conv2b', num_filters=64, pad='same', **conv_defs), **wn_defs) # net = WN(Conv2DLayer (net, name='conv2c', num_filters=256, pad='same', **conv_defs), **wn_defs) net = MaxPool2DLayer (net, name='pool2', pool_size=(2, 2)) net = DropoutLayer (net, name='drop2', p=.5) net = WN(Conv2DLayer (net, name='conv3a', num_filters=32, pad=0, **conv_defs), **wn_defs) # net = WN(NINLayer (net, name='conv3b', num_units=256, **nin_defs), **wn_defs) net = WN(NINLayer (net, name='conv3c', num_units=256, **nin_defs), **wn_defs) net = GlobalPoolLayer (net, name='pool3') net = WN(DenseLayer (net, name='dense', num_units=num_classes, **dense_defs), **wn_defs) # net = GaussianNoiseLayer(net, name='noise', sigma=config.augment_noise_stddev) # net = WN(DenseLayer (net, name='dense1', num_units=256, **dense_defs), **wn_defs) # net = DropoutLayer (net, name='drop1', p=.5) # net = WN(DenseLayer (net, name='dense2', num_units=256, **dense_defs), **wn_defs) # net = DropoutLayer (net, name='drop2', p=.5) # net = WN(DenseLayer (net, name='dense3', num_units=256, **dense_defs), **wn_defs) # # net = WN(DenseLayer (net, name='dense4', num_units=num_classes, **dense_defs), **wn_defs) return net
def create_network(available_actions_num): # Creates the input variables s1 = tensor.tensor4("States") a = tensor.vector("Actions", dtype="int32") q2 = tensor.vector("Next State best Q-Value") r = tensor.vector("Rewards") nonterminal = tensor.vector("Nonterminal", dtype="int8") # Creates the input layer of the network. dqn = InputLayer(shape=[None, 1, downsampled_y, downsampled_x], input_var=s1) # Adds 3 convolutional layers, each followed by a max pooling layer. dqn = Conv2DLayer(dqn, num_filters=32, filter_size=[8, 8], nonlinearity=rectify, W=GlorotUniform("relu"), b=Constant(.1)) dqn = MaxPool2DLayer(dqn, pool_size=[2, 2]) dqn = Conv2DLayer(dqn, num_filters=64, filter_size=[4, 4], nonlinearity=rectify, W=GlorotUniform("relu"), b=Constant(.1)) dqn = MaxPool2DLayer(dqn, pool_size=[2, 2]) dqn = Conv2DLayer(dqn, num_filters=64, filter_size=[3, 3], nonlinearity=rectify, W=GlorotUniform("relu"), b=Constant(.1)) dqn = MaxPool2DLayer(dqn, pool_size=[2, 2]) # Adds a single fully connected layer. dqn = DenseLayer(dqn, num_units=512, nonlinearity=rectify, W=GlorotUniform("relu"), b=Constant(.1)) # Adds a single fully connected layer which is the output layer. # (no nonlinearity as it is for approximating an arbitrary real function) dqn = DenseLayer(dqn, num_units=available_actions_num, nonlinearity=None) # Theano stuff q = get_output(dqn) # Only q for the chosen actions is updated more or less according to following formula: # target Q(s,a,t) = r + gamma * max Q(s2,_,t+1) target_q = tensor.set_subtensor(q[tensor.arange(q.shape[0]), a], r + discount_factor * nonterminal * q2) loss = squared_error(q, target_q).mean() # Updates the parameters according to the computed gradient using rmsprop. params = get_all_params(dqn, trainable=True) updates = rmsprop(loss, params, learning_rate) # Compiles theano functions print "Compiling the network ..." function_learn = theano.function([s1, q2, a, r, nonterminal], loss, updates=updates, name="learn_fn") function_get_q_values = theano.function([s1], q, name="eval_fn") function_get_best_action = theano.function([s1], tensor.argmax(q), name="test_fn") print "Network compiled." # Returns Theano objects for the net and functions. # We wouldn't need the net anymore but it is nice to save your model. return dqn, function_learn, function_get_q_values, function_get_best_action
def build_architecture(input_shape, trained_weights=None): """ Build the Theano symbolic graph representing the CNN model. :param input_shape: A tuple representing the input shape (h,w) :param trained_weights: Pre-trained weights. If None, the network is initialized at random. :return: A dictionary containing all layers """ net = {} net['input'] = InputLayer(input_shape) net['conv1'] = batch_norm( Conv2DLayer(net['input'], num_filters=96, filter_size=11, stride=4, flip_filters=False)) net['pool1'] = MaxPool2DLayer(net['conv1'], pool_size=3, stride=2) net['conv2'] = batch_norm( Conv2DLayer(net['pool1'], num_filters=256, filter_size=5, pad=2, flip_filters=False)) net['pool2'] = MaxPool2DLayer(net['conv2'], pool_size=3, stride=2) net['conv3'] = batch_norm( Conv2DLayer(net['pool2'], num_filters=384, filter_size=3, pad=1, flip_filters=False)) net['conv4'] = batch_norm( Conv2DLayer(net['conv3'], num_filters=384, filter_size=3, pad=1, flip_filters=False)) net['conv5'] = batch_norm( Conv2DLayer(net['conv4'], num_filters=256, filter_size=3, pad=1, flip_filters=False)) net['pool5'] = MaxPool2DLayer(net['conv5'], pool_size=3, stride=2) net['fc1'] = batch_norm(DenseLayer(net['pool5'], num_units=2048)) net['fc2'] = batch_norm(DenseLayer(net['fc1'], num_units=2048)) if trained_weights: lasagne.layers.set_all_param_values(net['fc2'], trained_weights) return net
def build_architecture(input_shape, trained_weights=None): net = {} net['input'] = InputLayer((None, 1, None, None)) net['large_conv1'] = batch_norm( Conv2DLayer(net['input'], num_filters=32, filter_size=11, stride=3, pad=5, flip_filters=False)) net['large_pool1'] = MaxPool2DLayer(net['large_conv1'], pool_size=3, stride=2) net['large_conv2'] = batch_norm( Conv2DLayer(net['large_pool1'], num_filters=64, filter_size=5, pad=2, flip_filters=False)) net['large_pool2'] = MaxPool2DLayer(net['large_conv2'], pool_size=3) net['large_conv3'] = batch_norm( Conv2DLayer(net['large_pool2'], num_filters=128, filter_size=3, pad=1, flip_filters=False)) net['large_conv4'] = batch_norm( Conv2DLayer(net['large_conv3'], num_filters=128, filter_size=3, pad=1, flip_filters=False)) net['large_pool4'] = MaxPool2DLayer(net['large_conv4'], pool_size=2) net['large_conv5'] = batch_norm( Conv2DLayer(net['large_pool4'], num_filters=128, filter_size=3, pad=1, flip_filters=False)) net['large_pool5'] = SpatialPyramidPoolingLayer(net['large_conv5'], implementation='kaiming') net['fc1'] = batch_norm(DenseLayer(net['large_pool5'], num_units=2048)) net['fc2'] = batch_norm(DenseLayer(net['fc1'], num_units=2048)) if trained_weights: lasagne.layers.set_all_param_values(net['fc2'], trained_weights) return net
def build_neural_network(): net = {} net['input'] = InputLayer((None, 1, 28, 28)) net['conv1'] = Conv2DLayer(net['input'], num_filters=8, filter_size=5) net['pool1'] = MaxPool2DLayer(net['conv1'], pool_size=2) net['conv2'] = Conv2DLayer(net['pool1'], num_filters=16, filter_size=5) net['pool2'] = MaxPool2DLayer(net['conv2'], pool_size=3) net['hid1'] = DenseLayer(net['pool2'], num_units=100) net['hid2'] = DenseLayer(net['hid1'], num_units=100) net['hid3'] = DenseLayer(net['hid2'], num_units=100) net['out'] = DenseLayer(net['hid3'], num_units=10, nonlinearity=softmax) return net
def build_model(self): ''' Build Acoustic Event Net model :return: ''' # A architecture 41 classes nonlin = lasagne.nonlinearities.rectify net = {} # channel, time. frequency net['input'] = InputLayer( (None, feat_shape[0], feat_shape[1], feat_shape[2])) # ----------- 1st layer group --------------- net['conv1a'] = ConvLayer(net['input'], num_filters=64, filter_size=(3, 3), stride=1, nonlinearity=nonlin) net['conv1b'] = ConvLayer(net['conv1a'], num_filters=64, filter_size=(3, 3), stride=1, nonlinearity=nonlin) net['pool1'] = MaxPool2DLayer(net['conv1b'], pool_size=(1, 2)) # (time, freq) # ----------- 2nd layer group --------------- net['conv2a'] = ConvLayer(net['pool1'], num_filters=128, filter_size=(3, 3), stride=1, nonlinearity=nonlin) net['conv2b'] = ConvLayer(net['conv2a'], num_filters=128, filter_size=(3, 3), stride=1, nonlinearity=nonlin) net['pool2'] = MaxPool2DLayer(net['conv2b'], pool_size=(2, 2)) # (time, freq) # ----------- fully connected layer group --------------- net['fc5'] = DenseLayer(net['pool2'], num_units=1024, nonlinearity=nonlin) net['fc6'] = DenseLayer(net['fc5'], num_units=1024, nonlinearity=nonlin) net['prob'] = DenseLayer(net['fc6'], num_units=41, nonlinearity=lasagne.nonlinearities.softmax) return net
def build_overfeat(inp): net['inp'] = InputLayer((None, 3, 231, 231), input_var=inp) net['conv1'] = Conv2DLayer(net['inp'], 96, 11, stride=4, pad='valid') net['pool1'] = MaxPool2DLayer(net['conv1'], 2) net['conv2'] = Conv2DLayer(net['pool1'], 256, 5, pad='valid') net['pool2'] = MaxPool2DLayer(net['conv2'], 2) net['conv3'] = Conv2DLayer(net['pool2'], 512, 3, pad='same') net['conv4'] = Conv2DLayer(net['conv3'], 1024, 3, pad='same') net['conv5'] = Conv2DLayer(net['conv4'], 1024, 3, pad='same') net['pool3'] = MaxPool2DLayer(net['conv5'], 2) net['fc6'] = DenseLayer(net['pool3'], 3072) net['fc7'] = DenseLayer(net['fc6'], 4096) net['fc8'] = DenseLayer(net['fc7'], 1000, nonlinearity=None) return net
def simple_network2(input_size, output_size): l_in = InputLayer(shape=((None, ) + input_size), name='inputLayer') network = lasagne.layers.Conv2DLayer( l_in, num_filters=16, filter_size=(3, 3), nonlinearity=lasagne.nonlinearities.rectify, W=lasagne.init.HeNormal(gain='relu')) network = lasagne.layers.Conv2DLayer( l_in, num_filters=16, filter_size=(3, 3), nonlinearity=lasagne.nonlinearities.rectify, W=lasagne.init.HeNormal(gain='relu')) network = MaxPool2DLayer(network, 2) network = lasagne.layers.Conv2DLayer( l_in, num_filters=16, filter_size=(3, 3), nonlinearity=lasagne.nonlinearities.rectify, W=lasagne.init.HeNormal(gain='relu')) network = MaxPool2DLayer(network, 2) network = lasagne.layers.Conv2DLayer( l_in, num_filters=16, filter_size=(3, 3), nonlinearity=lasagne.nonlinearities.rectify, W=lasagne.init.HeNormal(gain='relu')) network = DropoutLayer(network, p=0.5) network = lasagne.layers.DenseLayer( network, num_units=128, nonlinearity=lasagne.nonlinearities.rectify, W=lasagne.init.HeNormal(gain='relu')) network = lasagne.layers.DenseLayer(network, num_units=output_size, nonlinearity=None, b=lasagne.init.Constant(.1)) return network
def build_architecture(input_shape, trained_weights=None): net = {} net['input'] = InputLayer(input_shape) net['conv1'] = batch_norm( Conv2DLayer(net['input'], num_filters=96, filter_size=11, stride=4, flip_filters=False)) net['pool1'] = MaxPool2DLayer(net['conv1'], pool_size=3, stride=2) net['conv2'] = batch_norm( Conv2DLayer(net['pool1'], num_filters=256, filter_size=5, pad=2, flip_filters=False)) net['pool2'] = MaxPool2DLayer(net['conv2'], pool_size=3, stride=2) net['conv3'] = batch_norm( Conv2DLayer(net['pool2'], num_filters=384, filter_size=3, pad=1, flip_filters=False)) net['conv4'] = batch_norm( Conv2DLayer(net['conv3'], num_filters=384, filter_size=3, pad=1, flip_filters=False)) net['conv5'] = batch_norm( Conv2DLayer(net['conv4'], num_filters=256, filter_size=3, pad=1, flip_filters=False)) net['pool5'] = MaxPool2DLayer(net['conv5'], pool_size=3, stride=2) net['fc1'] = batch_norm(DenseLayer(net['pool5'], num_units=2048)) net['fc2'] = batch_norm(DenseLayer(net['fc1'], num_units=2048)) if trained_weights: lasagne.layers.set_all_param_values(net['fc2'], trained_weights) return net
def _model_definition(self, net): """ Builds the architecture of the network """ he_norm = HeNormal(gain='relu') # Input filtering and downsampling with max pooling net = batch_norm( Conv2DLayer(net, num_filters=32, filter_size=7, pad='same', nonlinearity=rectify, W=he_norm)) net = MaxPool2DLayer(net, 2) net = batch_norm( Conv2DLayer(net, num_filters=64, filter_size=3, pad='same', nonlinearity=rectify, W=he_norm)) net = MaxPool2DLayer(net, 2) net = batch_norm( Conv2DLayer(net, num_filters=128, filter_size=3, pad='same', nonlinearity=rectify, W=he_norm)) net = batch_norm( Conv2DLayer(net, num_filters=128, filter_size=3, pad='same', nonlinearity=rectify, W=he_norm)) net = batch_norm( DenseLayer(net, num_units=1024, nonlinearity=rectify, W=he_norm)) net = batch_norm( DenseLayer(net, num_units=1024, nonlinearity=rectify, W=he_norm)) # Pooling #net = MaxPool1DLayer(net, 1) return net
def build_inception_module(name, input_layer, nfilters): """ See: https://github.com/Lasagne/Recipes/blob/master/modelzoo/googlenet.py TODO: the pooling size is wrong here - 3x3 doesn't really make sense on our images - consider recomputing the pooling -> also requires recomputing all the filter sizes, etc. """ # nfilters: (pool_proj, 1x1, 3x3_reduce, 3x3, 5x5_reduce, 5x5) net = {} net['pool'] = MaxPool2DLayer(input_layer, pool_size=3, stride=1, pad=1) net['pool_proj'] = Conv2DLayer(net['pool'], nfilters[0], 1) net['1x1'] = Conv2DLayer(input_layer, nfilters[1], 1) net['3x3_reduce'] = Conv2DLayer(input_layer, nfilters[2], 1) net['3x3'] = Conv2DLayer(net['3x3_reduce'], nfilters[3], 3, pad=1) net['5x5_reduce'] = Conv2DLayer(input_layer, nfilters[4], 1) net['5x5'] = Conv2DLayer(net['5x5_reduce'], nfilters[5], 5, pad=2) net['output'] = ConcatLayer([ net['1x1'], net['3x3'], net['5x5'], net['pool_proj'], ]) return {'{}/{}'.format(name, k): v for k, v in net.items()}
def build_cnn(input_var=None, w_init=None, n_layers=(4, 2, 1), n_filters_first=32, imsize=32, n_colors=3): weights = [] # Keeps the weights for all layers count = 0 # If no initial weight is given, initialize with GlorotUniform if w_init is None: w_init = [lasagne.init.GlorotUniform()] * sum(n_layers) # Input layer network = InputLayer(shape=(None, n_colors, imsize, imsize), input_var=input_var) for i, s in enumerate(n_layers): for l in range(s): network = Conv2DLayer(network, num_filters=n_filters_first * (2**i), filter_size=(3, 3), W=w_init[count], pad='same') count += 1 weights.append(network.W) network = MaxPool2DLayer(network, pool_size=(2, 2)) return network, weights
def contraction(depth, deepest): n_filters = filter_for_depth(depth) incoming = net['input'] if depth == 0 else net['pool{}'.format(depth - 1)] net['conv{}_1'.format(depth)] = Conv2DLayer(incoming, num_filters=n_filters, filter_size=3, pad='valid', W=HeNormal(gain='relu'), nonlinearity=nonlinearity) net['conv{}_2'.format(depth)] = Conv2DLayer( net['conv{}_1'.format(depth)], num_filters=n_filters, filter_size=3, pad='valid', W=HeNormal(gain='relu'), nonlinearity=nonlinearity) if P.BATCH_NORMALIZATION: net['conv{}_2'.format(depth)] = batch_norm( net['conv{}_2'.format(depth)], alpha=P.BATCH_NORMALIZATION_ALPHA) if not deepest: net['pool{}'.format(depth)] = MaxPool2DLayer( net['conv{}_2'.format(depth)], pool_size=2, stride=2)
def architecture(input_var, input_shape): layer = InputLayer(input_shape, input_var) kwargs = dict(nonlinearity=lasagne.nonlinearities.leaky_rectify, W=lasagne.init.Orthogonal()) layer = Conv2DLayer(layer, 64, 3, **kwargs) layer = Conv2DLayer(layer, 32, 3, **kwargs) layer = MaxPool2DLayer(layer, 3) layer = Conv2DLayer(layer, 128, 3, **kwargs) layer = Conv2DLayer(layer, 64, 3, **kwargs) layer = MaxPool2DLayer(layer, 3) layer = DenseLayer(dropout(layer, 0.5), 256, **kwargs) layer = DenseLayer(dropout(layer, 0.5), 64, **kwargs) layer = DenseLayer(dropout(layer, 0.5), 1, nonlinearity=lasagne.nonlinearities.sigmoid, W=lasagne.init.Orthogonal()) return layer
def build_cnn(input_var=None, w_init=None, n_layers=(4, 2, 1), n_filters_first=32, imsize=32, n_colors=3): """ Builds a VGG style CNN network followed by a fully-connected layer and a softmax layer. Stacks are separated by a maxpool layer. Number of kernels in each layer is twice the number in previous stack. input_var: Theano variable for input to the network outputs: pointer to the output of the last layer of network (softmax) :param input_var: theano variable as input to the network :param w_init: Initial weight values :param n_layers: number of layers in each stack. An array of integers with each value corresponding to the number of layers in each stack. (e.g. [4, 2, 1] == 3 stacks with 4, 2, and 1 layers in each. :param n_filters_first: number of filters in the first layer :param imSize: Size of the image :param n_colors: Number of color channels (depth) :return: a pointer to the output of last layer """ weights = [] # Keeps the weights for all layers count = 0 # If no initial weight is given, initialize with GlorotUniform if w_init is None: w_init = [lasagne.init.GlorotUniform()] * sum(n_layers) # Input layer network = InputLayer(shape=(None, n_colors, imsize, imsize), input_var=input_var) for i, s in enumerate(n_layers): for l in range(s): network = Conv2DLayer(network, num_filters=n_filters_first * (2 ** i), filter_size=(3, 3), W=w_init[count], pad='same') count += 1 weights.append(network.W) network = MaxPool2DLayer(network, pool_size=(2, 2)) return network, weights
def deep1(input_var): net = {} net['data'] = InputLayer(inshape, input_var=input_var) net['conv1'] = Conv2DLayer(net['data'], num_filters=20, filter_size=(1, 255), stride=(1, 16), pad='same', nonlinearity=rectify) net['conv2'] = Conv2DLayer(net['conv1'], num_filters=20, filter_size=(1, 255), stride=(1, 16), pad='same', nonlinearity=rectify) net['pool1'] = MaxPool2DLayer(net['conv2'], pool_size=(1, 2)) net['conv3'] = Conv2DLayer( net['pool1'], num_filters=30, filter_size=(1, 127), stride=(1, 16), pad='same', nonlinearity=rectify) # should have dims of (23 x 32 x ?) net['conv4'] = Conv2DLayer(net['conv3'], num_filters=30, filter_size=(1, 7), stride=1, pad='same', nonlinearity=rectify) # dims (23 x 32 x ?) #net['pool2'] = MaxPool2DLayer(net['conv4'], pool_size=(1,2)) net['fcl'] = DenseLayer(net['conv4'], num_units=100, nonlinearity=rectify) net['out'] = DenseLayer(net['fcl'], num_units=outshape, nonlinearity=sigmoid) return net
def build_DCNN_maxpool_softmax(input_var=None): from lasagne.layers import Conv2DLayer, MaxPool2DLayer print('Training the maxpool network!!') # Define the input variable which is 4 frames of IPW fields and 4 frames of # reflectivity fields l_in = lasagne.layers.InputLayer(shape=(None, 8, 33, 33), input_var=input_var) l_conv1 = Conv2DLayer(l_in, num_filters=32, filter_size=(5, 5), stride=(1, 1), nonlinearity=lasagne.nonlinearities.rectify, W=lasagne.init.HeUniform(), b=lasagne.init.Constant(.1), pad='full') l_maxpool = MaxPool2DLayer(l_conv1, (2, 2)) l_hidden1 = lasagne.layers.DenseLayer( lasagne.layers.dropout(l_maxpool, p=0.3), num_units=2000, nonlinearity=lasagne.nonlinearities.sigmoid, W=lasagne.init.HeUniform(), b=lasagne.init.Constant(.1)) network = lasagne.layers.DenseLayer( l_hidden1, num_units=2, nonlinearity=lasagne.nonlinearities.softmax) return network, l_hidden1
def net_color_non_square(NeuralNet): l = InputLayer(shape=(None, 3, 20, 28)) l = Conv2DLayer(l, name='conv1', filter_size=(5, 5), num_filters=8) l = MaxPool2DLayer(l, name='pool1', pool_size=(2, 2)) l = Conv2DLayer(l, name='conv2', filter_size=(5, 5), num_filters=8) l = MaxPool2DLayer(l, name='pool2', pool_size=(2, 2)) l = DenseLayer(l, name='hidden1', num_units=128) l = DenseLayer(l, name='output', nonlinearity=softmax, num_units=10) net = NeuralNet( layers=l, update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, max_epochs=1, ) return net
def build_resfuse_net(input_var=None, projection=True): """ Args: projection: If False, we are using pure ResNet Returns: """ # Building the network l_in = InputLayer(shape=(None, 3, 64, 64), input_var=input_var) # first layer, output is 64 x 64 x 64 l = batch_norm( ConvLayer(l_in, num_filters=64, filter_size=(3, 3), stride=(1, 1), nonlinearity=rectify, pad='same', W=lasagne.init.HeNormal(gain='relu'))) l = MaxPool2DLayer(l, 2) # 64 x 32 x 32 # first stack of residual blocks, output is 64 x 32 x 32 (2 residual blocks) (4 conv layers) l = resfuse_block(l, projection=projection) l = residual_block( l, increase_dim=True) # 128 x 16 x 16 (1 res block) (2 conv layers) l = resfuse_block(l, projection=projection ) # 128 x 16 x 16 (2 residual blocks) (4 conv layers) l = residual_block( l, increase_dim=True) # 256 x 8 x 8 (1 residual blocks) (2 conv layers) l = resfuse_block(l, projection=projection ) # 256 x 8 x 8 (2 residual blocks) (4 conv layers) # those are 25-layer addition (before is 19-layer config) l = resfuse_block(l, projection=projection ) # 256 x 8 x 8 (2 residual blocks) (4 conv layers) l = residual_block( l, increase_dim=True) # 512 x 4 x 4 (1 res block) (2 conv layers) # (4 + 2) * 2 + 4 * 2 + 2 + 3 = 25 layers # average pooling l = GlobalPoolLayer(l) # fully connected layer network = DenseLayer(l, num_units=100, W=lasagne.init.HeNormal(), nonlinearity=softmax) return network
def load_mnist_classifier(pkl_path): nl = lasagne.nonlinearities.LeakyRectify(0.1) net = InputLayer((None, 1, 32, 32)) net = Conv2DLayer(net, 32, (3, 3), pad='same', nonlinearity=nl) net = Conv2DLayer(net, 32, (3, 3), pad='same', nonlinearity=nl) net = MaxPool2DLayer(net, (2, 2)) net = Conv2DLayer(net, 55, (3, 3), pad='same', nonlinearity=nl) net = Conv2DLayer(net, 55, (3, 3), pad='same', nonlinearity=nl) net = MaxPool2DLayer(net, (2, 2)) net = Conv2DLayer(net, 96, (3, 3), pad=0, nonlinearity=nl) net = Conv2DLayer(net, 96, (3, 3), pad=0, nonlinearity=nl) net = MaxPool2DLayer(net, (2, 2)) net = DenseLayer(net, num_units=10, nonlinearity=lasagne.nonlinearities.softmax) with open(pkl_path, 'rb') as file: lasagne.layers.set_all_param_values(net, cPickle.load(file)) return net
def net(NeuralNet): l = InputLayer(shape=(None, 1, 28, 28)) l = Conv2DLayer(l, name='conv1', filter_size=(5, 5), num_filters=8) l = MaxPool2DLayer(l, name='pool1', pool_size=(2, 2)) l = Conv2DLayer(l, name='conv2', filter_size=(5, 5), num_filters=8) l = MaxPool2DLayer(l, name='pool2', pool_size=(2, 2)) l = DenseLayer(l, name='hidden1', num_units=128) l = DenseLayer(l, name='output', nonlinearity=softmax, num_units=10) return NeuralNet( layers=l, update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, max_epochs=5, on_epoch_finished=[_OnEpochFinished()], verbose=99, )
def build_network(): conv_defs = { 'W': lasagne.init.HeNormal('relu'), 'b': lasagne.init.Constant(0.0), 'filter_size': (3, 3), 'stride': (1, 1), 'nonlinearity': lasagne.nonlinearities.LeakyRectify(0.1) } nin_defs = { 'W': lasagne.init.HeNormal('relu'), 'b': lasagne.init.Constant(0.0), 'nonlinearity': lasagne.nonlinearities.LeakyRectify(0.1) } dense_defs = { 'W': lasagne.init.HeNormal(1.0), 'b': lasagne.init.Constant(0.0), 'nonlinearity': lasagne.nonlinearities.softmax } wn_defs = { 'momentum': .999 } net = InputLayer ( name='input', shape=(None, 3, 32, 32)) net = GaussianNoiseLayer(net, name='noise', sigma=.15) net = WN(Conv2DLayer (net, name='conv1a', num_filters=128, pad='same', **conv_defs), **wn_defs) net = WN(Conv2DLayer (net, name='conv1b', num_filters=128, pad='same', **conv_defs), **wn_defs) net = WN(Conv2DLayer (net, name='conv1c', num_filters=128, pad='same', **conv_defs), **wn_defs) net = MaxPool2DLayer (net, name='pool1', pool_size=(2, 2)) net = DropoutLayer (net, name='drop1', p=.5) net = WN(Conv2DLayer (net, name='conv2a', num_filters=256, pad='same', **conv_defs), **wn_defs) net = WN(Conv2DLayer (net, name='conv2b', num_filters=256, pad='same', **conv_defs), **wn_defs) net = WN(Conv2DLayer (net, name='conv2c', num_filters=256, pad='same', **conv_defs), **wn_defs) net = MaxPool2DLayer (net, name='pool2', pool_size=(2, 2)) net = DropoutLayer (net, name='drop2', p=.5) net = WN(Conv2DLayer (net, name='conv3a', num_filters=512, pad=0, **conv_defs), **wn_defs) net = WN(NINLayer (net, name='conv3b', num_units=256, **nin_defs), **wn_defs) net = WN(NINLayer (net, name='conv3c', num_units=128, **nin_defs), **wn_defs) net = GlobalPoolLayer (net, name='pool3') net = WN(DenseLayer (net, name='dense', num_units=10, **dense_defs), **wn_defs) return net
def create_model(incoming, options): input_p = 0.2 hidden_p = 0.5 conv_num_filters1 = int(100 / (1.0 - input_p)) conv_num_filters2 = int(150 / (1.0 - hidden_p)) conv_num_filters3 = int(200 / (1.0 - hidden_p)) filter_size1 = 5 filter_size2 = 5 filter_size3 = 3 pool_size = 2 encode_size = int(options['BOTTLENECK'] / 0.5) dense_mid_size = int(options['DENSE'] / 0.5) pad_in = 'valid' pad_out = 'full' scaled_tanh = create_scaled_tanh() dropout0 = DropoutLayer(incoming, p=0.2, name='dropout0') conv2d1 = Conv2DLayer(dropout0, num_filters=conv_num_filters1, filter_size=filter_size1, pad=pad_in, name='conv2d1', nonlinearity=scaled_tanh) maxpool2d2 = MaxPool2DLayer(conv2d1, pool_size=pool_size, name='maxpool2d2') dropout1 = DropoutLayer(maxpool2d2, name='dropout1') conv2d3 = Conv2DLayer(dropout1, num_filters=conv_num_filters2, filter_size=filter_size2, pad=pad_in, name='conv2d3', nonlinearity=scaled_tanh) maxpool2d4 = MaxPool2DLayer(conv2d3, pool_size=pool_size, name='maxpool2d4', pad=(1,0)) dropout2 = DropoutLayer(maxpool2d4, name='dropout2') conv2d5 = Conv2DLayer(dropout2, num_filters=conv_num_filters3, filter_size=filter_size3, pad=pad_in, name='conv2d5', nonlinearity=scaled_tanh) reshape6 = ReshapeLayer(conv2d5, shape=([0], -1), name='reshape6') # 3000 reshape6_output = reshape6.output_shape[1] dropout3 = DropoutLayer(reshape6, name='dropout3') dense7 = DenseLayer(dropout3, num_units=dense_mid_size, name='dense7', nonlinearity=scaled_tanh) dropout4 = DropoutLayer(dense7, name='dropout4') bottleneck = DenseLayer(dropout4, num_units=encode_size, name='bottleneck', nonlinearity=linear) # print_network(bottleneck) dense8 = DenseLayer(bottleneck, num_units=dense_mid_size, W=bottleneck.W.T, name='dense8', nonlinearity=linear) dense9 = DenseLayer(dense8, num_units=reshape6_output, W=dense7.W.T, nonlinearity=scaled_tanh, name='dense9') reshape10 = ReshapeLayer(dense9, shape=([0], conv_num_filters3, 3, 5), name='reshape10') # 32 x 4 x 7 deconv2d11 = Deconv2DLayer(reshape10, conv2d5.input_shape[1], conv2d5.filter_size, stride=conv2d5.stride, W=conv2d5.W, flip_filters=not conv2d5.flip_filters, name='deconv2d11', nonlinearity=scaled_tanh) upscale2d12 = Upscale2DLayer(deconv2d11, scale_factor=pool_size, name='upscale2d12') deconv2d13 = Deconv2DLayer(upscale2d12, conv2d3.input_shape[1], conv2d3.filter_size, stride=conv2d3.stride, W=conv2d3.W, flip_filters=not conv2d3.flip_filters, name='deconv2d13', nonlinearity=scaled_tanh) upscale2d14 = Upscale2DLayer(deconv2d13, scale_factor=pool_size, name='upscale2d14') deconv2d15 = Deconv2DLayer(upscale2d14, conv2d1.input_shape[1], conv2d1.filter_size, stride=conv2d1.stride, crop=(1, 0), W=conv2d1.W, flip_filters=not conv2d1.flip_filters, name='deconv2d14', nonlinearity=scaled_tanh) reshape16 = ReshapeLayer(deconv2d15, ([0], -1), name='reshape16') return reshape16, bottleneck
def build_model(batch_size=128): x = T.tensor4('input') layer = InputLayer((batch_size, 3, image_sz, image_sz), input_var=x) layer = Conv2DLayer(layer, 64, 11, stride=4, pad='valid') layer = MaxPool2DLayer(layer, 3, stride=2) layer = Conv2DLayer(layer, 192, 5, pad='same') layer = MaxPool2DLayer(layer, 3, stride=2) layer = Conv2DLayer(layer, 384, 3, pad='same') layer = Conv2DLayer(layer, 256, 3, pad='same') layer = Conv2DLayer(layer, 256, 3, pad='same') layer = MaxPool2DLayer(layer, 3, stride=2) layer = DenseLayer(layer, 4096) layer = DenseLayer(layer, 4096) layer = DenseLayer(layer, 1000, nonlinearity=None) return layer, x
def create_model(incoming, options): conv_num_filters1 = 100 conv_num_filters2 = 150 conv_num_filters3 = 200 filter_size1 = 5 filter_size2 = 5 filter_size3 = 3 pool_size = 2 encode_size = options['BOTTLENECK'] dense_mid_size = options['DENSE'] pad_in = 'valid' pad_out = 'full' scaled_tanh = create_scaled_tanh() conv2d1 = Conv2DLayer(incoming, num_filters=conv_num_filters1, filter_size=filter_size1, pad=pad_in, name='conv2d1', nonlinearity=scaled_tanh) maxpool2d3 = MaxPool2DLayer(conv2d1, pool_size=pool_size, name='maxpool2d3') bn2 = BatchNormLayer(maxpool2d3, name='batchnorm2') conv2d4 = Conv2DLayer(bn2, num_filters=conv_num_filters2, filter_size=filter_size2, pad=pad_in, name='conv2d4', nonlinearity=scaled_tanh) maxpool2d6 = MaxPool2DLayer(conv2d4, pool_size=pool_size, name='maxpool2d6', pad=(1,0)) bn3 = BatchNormLayer(maxpool2d6, name='batchnorm3') conv2d7 = Conv2DLayer(bn3, num_filters=conv_num_filters3, filter_size=filter_size3, pad=pad_in, name='conv2d7', nonlinearity=scaled_tanh) reshape9 = ReshapeLayer(conv2d7, shape=([0], -1), name='reshape9') # 3000 reshape9_output = reshape9.output_shape[1] bn8 = BatchNormLayer(reshape9, name='batchnorm8') dense10 = DenseLayer(bn8, num_units=dense_mid_size, name='dense10', nonlinearity=scaled_tanh) bn11 = BatchNormLayer(dense10, name='batchnorm11') bottleneck = DenseLayer(bn11, num_units=encode_size, name='bottleneck', nonlinearity=linear) # print_network(bottleneck) dense12 = DenseLayer(bottleneck, num_units=dense_mid_size, W=bottleneck.W.T, name='dense12', nonlinearity=linear) dense13 = DenseLayer(dense12, num_units=reshape9_output, W=dense10.W.T, nonlinearity=scaled_tanh, name='dense13') reshape14 = ReshapeLayer(dense13, shape=([0], conv_num_filters3, 3, 5), name='reshape14') # 32 x 4 x 7 deconv2d19 = Deconv2DLayer(reshape14, conv2d7.input_shape[1], conv2d7.filter_size, stride=conv2d7.stride, W=conv2d7.W, flip_filters=not conv2d7.flip_filters, name='deconv2d19', nonlinearity=scaled_tanh) upscale2d16 = Upscale2DLayer(deconv2d19, scale_factor=pool_size, name='upscale2d16') deconv2d17 = Deconv2DLayer(upscale2d16, conv2d4.input_shape[1], conv2d4.filter_size, stride=conv2d4.stride, W=conv2d4.W, flip_filters=not conv2d4.flip_filters, name='deconv2d17', nonlinearity=scaled_tanh) upscale2d18 = Upscale2DLayer(deconv2d17, scale_factor=pool_size, name='upscale2d18') deconv2d19 = Deconv2DLayer(upscale2d18, conv2d1.input_shape[1], conv2d1.filter_size, stride=conv2d1.stride, crop=(1, 0), W=conv2d1.W, flip_filters=not conv2d1.flip_filters, name='deconv2d14', nonlinearity=scaled_tanh) reshape20 = ReshapeLayer(deconv2d19, ([0], -1), name='reshape20') return reshape20, bottleneck
def SmallNet(input_var, image_size, num_labels): net = {} net['input'] = InputLayer(shape=(None, 1, image_size[0], image_size[1]), input_var=input_var) net['conv1'] = Conv2DLayer(net['input'], num_filters=32, filter_size=(7, 7), stride=(2, 2)) net['pool1'] = MaxPool2DLayer(net['conv1'], pool_size=(2, 2)) net['conv2'] = Conv2DLayer(net['pool1'], num_filters=64, filter_size=(5, 5), stride=(2, 2)) net['pool2'] = MaxPool2DLayer(net['conv2'], pool_size=(2, 2)) net['conv3'] = Conv2DLayer(net['pool2'], num_filters=128, filter_size=(3, 3), pad=(1, 1)) net['conv4'] = Conv2DLayer(net['conv3'], num_filters=128, filter_size=(3, 3), pad=(1, 1)) net['conv5_p'] = Conv2DLayer(net['conv4'], num_filters=64, filter_size=(1, 1)) net['conv6_p'] = Conv2DLayer(net['conv5_p'], num_filters=num_labels, filter_size=(1, 1), nonlinearity=linear) net['average_pool_p'] = GlobalPoolLayer(net['conv6_p'], pool_function=T.mean) net['softmax_p'] = NonlinearityLayer(net['average_pool_p'], nonlinearity=softmax) net['output'] = net['softmax_p'] net['feature_maps'] = net['conv6_p'] net['last_activation'] = net['average_pool_p'] return net
def build_model(batch_size=128): x = T.tensor4('input') layer = InputLayer((batch_size, ) + featureDim, input_var=x) layer = Conv2DLayer(layer, 96, 11, stride=4, pad=2) layer = MaxPool2DLayer(layer, 3, stride=2) layer = Conv2DLayer(layer, 256, 5, pad='same') layer = MaxPool2DLayer(layer, 3, stride=2) layer = Conv2DLayer(layer, 384, 3, pad='same') layer = Conv2DLayer(layer, 384, 3, pad='same') layer = Conv2DLayer(layer, 256, 3, pad='same') layer = MaxPool2DLayer(layer, 3, stride=2) layer = ReshapeLayer(layer, (-1, 256 * 6 * 6)) layer = DenseLayer(layer, 4096) layer = DenseLayer(layer, 4096) layer = DenseLayer(layer, labelDim, nonlinearity=None) return layer, x