def __init__(self): self.conv1 = ConvLayer(32, (9, 9), strides=(1, 1), padding='same', name="conv_1") self.conv2 = ConvLayer(64, (3, 3), strides=(2, 2), padding='same', name="conv_2") self.conv3 = ConvLayer(128, (3, 3), strides=(2, 2), padding='same', name="conv_3") self.res1 = ResBlock(128, prefix="res_1") self.res2 = ResBlock(128, prefix="res_2") self.res3 = ResBlock(128, prefix="res_3") self.res4 = ResBlock(128, prefix="res_4") self.res5 = ResBlock(128, prefix="res_5") self.convt1 = ConvTLayer(64, (3, 3), strides=(2, 2), padding='same', name="conv_t_1") self.convt2 = ConvTLayer(32, (3, 3), strides=(2, 2), padding='same', name="conv_t_2") self.conv4 = ConvLayer(3, (9, 9), strides=(1, 1), padding='same', activate=False, name="conv_4") self.tanh = Activation('tanh') self.model = self._get_model()
class cnn(object): class simple_cnn_model(object): def __init__(self, epochs, batch_size, lr): self.epochs = epochs self.batch_size = batch_size self.lr = lr def load_data(self): # load data from cifar100 folder (x_train, y_train), (x_test, y_test) = cifar100(1211506319) return x_train, y_train, x_test, y_test def train_model(self, layers, loss_metrics, x_train, y_train): # build model self.model = Sequential(layers, loss_metrics) # train the model loss = self.model.fit(x_train, y_train, self.epochs, self.lr, self.batch_size, print_output=True) avg_loss = np.mean(np.reshape(loss, (self.epochs, -1)), axis=1) return avg_loss def test_model(self, x_test, y_test): # make a prediction pred_result = self.model.predict(x_test) accuracy = np.mean(pred_result == y_test) return accuracy if __name__ == '__main__': # define model parameters epochs = 15 batch_size = 128 lr = [.1] # define layers layers = (ConvLayer(3, 16, 3), ReluLayer(), MaxPoolLayer(), ConvLayer(16, 32, 3), ReluLayer(), MaxPoolLayer(), FlattenLayer(), FullLayer(2048, 4), SoftMaxLayer()) loss_matrics = CrossEntropyLayer() # build and train model model = simple_cnn_model(epochs, batch_size, lr) x_train, y_train, x_test, y_test = model.load_data() loss = model.train_model(layers, loss_matrics, x_train, y_train) accuracy = model.test_model(x_test, y_test) print("loss: %s" % loss) print("The accuracy of the model is %s" % accuracy)
def __init__(self, width_stages, n_cell_stages, stride_stages, dropout=0): super(NASNet, self).__init__() self.width_stages = width_stages self.n_cell_stages = n_cell_stages self.stride_stages = stride_stages in_channels = 32 first_cell_width = 16 # first conv layer self.first_conv = ConvLayer(3, in_channels, 3, 2, 1, 1, False, False, True, 'relu6', 0, 'weight_bn_act') # first block first_block_config = { "name": "MobileInvertedResidualBlock", "mobile_inverted_conv": { "name": "MBInvertedConvLayer", "in_channels": in_channels, "out_channels": first_cell_width, "kernel_size": 3, "stride": 1, "expand_ratio": 1 }, "shortcut": None } self.first_block = MobileInvertedResidualBlock.build_from_config( first_block_config) in_channels = first_cell_width # blocks self.blocks = nn.ModuleList() for width, n_cell, s in zip(self.width_stages, self.n_cell_stages, self.stride_stages): for i in range(n_cell): if i == 0: stride = s else: stride = 1 block = WSMobileInvertedResidualBlock(in_channels, width, stride) in_channels = width self.blocks.append(block) self.feature_mix_layer = ConvLayer(in_channels, 1280, 1, 1, 1, 1, False, False, True, 'relu6', 0, 'weight_bn_act') self.global_avg_pooling = nn.AdaptiveAvgPool2d(1) self.classifier = LinearLayer(1280, 1000, True, False, None, dropout, 'weight_bn_act')
def mnist(output_size=10): conv_config = [{ 'channels': 1, 'kernel': (3, 3), 'stride': (1, 1), 'activation': 'relu' }] config = [{'out': 20, 'activation': 'relu'}] input_shape = [28, 28, 1] output_size = output_size depth = 3 width = 10 max_modules_pr_layer = 3 min_modules_pr_layer = 1 learning_rate = 0.0001 optimizer_type = Adam loss = 'categorical_crossentropy' flatten_in_unique = True layers = [] #layers.append(DenseLayer(width, 'L0', config, flatten=not flatten_in_unique)) #layers.append(DenseLayer(width, 'L1', config)) #layers.append(DenseLayer(width, 'L2', config)) layers.append(ConvLayer(width, 'L0', conv_config)) layers.append(ConvLayer(width, 'L1', conv_config)) layers.append(ConvLayer(width, 'L2', conv_config, maxpool=True)) Layer.initialize_whole_network(layers, input_shape) task = TaskContainer(input_shape, output_size, flatten_in_unique, name='unique_mnist', optimizer=optimizer_type, loss=loss, lr=learning_rate) pathnet = PathNet(input_shape=input_shape, width=width, depth=depth, max_active_modules=20) pathnet._layers = layers pathnet._tasks = [task] pathnet.max_modules_pr_layer = max_modules_pr_layer pathnet.min_modules_pr_layer = min_modules_pr_layer for layer in pathnet._layers: layer.save_initialized_weights() return pathnet, task
def make_encoder_layers(input_size: int, latent_dim: int = 100, hidden_channel: int = 128, last_act: str = None) -> list: # mnist, fmnist if input_size == 28: layers = [ ConvLayer(1, hidden_channel, 4, 2, 1, True, "leakyrelu"), ConvLayer(hidden_channel, hidden_channel * 2, 4, 2, 1, True, "leakyrelu"), ConvLayer(hidden_channel * 2, hidden_channel * 4, 4, 2, 1, True, "leakyrelu"), ConvLayer(hidden_channel * 4, latent_dim, 3, 1, 0, False, last_act), ] # cifar10 elif input_size == 32: layers = [ ConvLayer(3, hidden_channel, 4, 2, 1, True, "leakyrelu"), ConvLayer(hidden_channel, hidden_channel * 2, 4, 2, 1, True, "leakyrelu"), ConvLayer(hidden_channel * 2, hidden_channel * 4, 4, 2, 1, True, "leakyrelu"), ConvLayer(hidden_channel * 4, latent_dim, 4, 1, 0, False, last_act), ] else: raise ValueError return layers
def make_discriminator_layers(input_size: int, hidden_channel: int = 128) -> list: # mnist, fmnist if input_size == 28: layers = [ ConvLayer(1, hidden_channel, 4, 2, 1, True, "leakyrelu"), ConvLayer(hidden_channel, hidden_channel * 2, 4, 2, 1, True, "leakyrelu"), ConvLayer(hidden_channel * 2, hidden_channel * 4, 4, 2, 1, True, "leakyrelu"), ConvLayer(hidden_channel * 4, 1, 3, 1, 0, False), ] # cifar10 elif input_size == 32: layers = [ ConvLayer(3, hidden_channel, 4, 2, 1, True, "leakyrelu"), ConvLayer(hidden_channel, hidden_channel * 2, 4, 2, 1, True, "leakyrelu"), ConvLayer(hidden_channel * 2, hidden_channel * 4, 4, 2, 1, True, "leakyrelu"), ConvLayer(hidden_channel * 4, 1, 4, 1, 0, False), ] else: raise ValueError return layers
def get_active_subnet(self, in_channel, preserve_weight=True): sub_layer = ConvLayer( in_channel, self.active_out_channel, self.kernel_size, self.stride, self.dilation, use_bn=self.use_bn, act_func=self.act_func ) sub_layer = sub_layer.to(get_net_device(self)) if not preserve_weight: return sub_layer sub_layer.conv.weight.data.copy_(self.conv.conv.weight.data[:self.active_out_channel, :in_channel, :, :]) if self.use_bn: copy_bn(sub_layer.bn, self.bn.bn) return sub_layer
def load_pathnet(filename): log = None with open(filename, 'rb') as f: log = pickle.load(f) layers = [] for layer_log in log['layer_logs']: if layer_log['layer_type'] == 'dense': layers.append(DenseLayer.build_from_log(layer_log)) if layer_log['layer_type'] == 'conv': layers.append(ConvLayer.build_from_log(layer_log)) Layer.initialize_whole_network(layers, log['in_shape']) for layer, layer_log in zip(layers, log['layer_logs']): layer.load_layer_log(layer_log) pathnet = PathNet(input_shape=log['in_shape'], width=log['width'], depth=log['depth']) pathnet._layers = layers pathnet.training_counter = log['training_counter'] pathnet.max_modules_pr_layer = log['max_modules_pr_layer'] pathnet.min_modules_pr_layer = log['min_modules_pr_layer'] tasks = [] for task_log in log['task_logs']: task = TaskContainer.build_from_log(task_log) pathnet.path2model(pathnet.random_path(), task) task.layer.set_weights(task_log['layer_weights']) tasks.append(task) pathnet._tasks = tasks return pathnet
def build_model(self): layers = [] input_shape = np.array( [self.batch_size, self.x_dim, self.x_dim, self.c_dim]) # layer_1: input_layer ==> [n, 28, 28, 1] x = InputLayer(input_shape) layers.append(x) # layer_2: conv_layer [n, 28, 28, 1] ==> [n, 28, 28, 32] x = ConvLayer(x, output_nums=20, kernel=5, strides=1, padding='SAME', name='conv1') layers.append(x) # layer_4: avgpool_layer [n, 28, 28, 32] ==> [n, 14, 14, 32] x = MaxPoolLayer(x, kernel=2, strides=2, paddind='SAME', name='pool1') layers.append(x) # layer_5: conv_layer [n, 14, 14, 32] ==> [n, 14, 14, 64] x = ConvLayer(x, output_nums=50, kernel=5, strides=1, padding='SAME', name='conv2') layers.append(x) # layer_7: avgpool_layer [n, 14, 14, 64] ==> [n, 7, 7, 64] x = MaxPoolLayer(x, kernel=2, strides=2, padding='SAME', name='pool2') layers.append(x) # layer_8: flatten_layer [n, 7, 7, 64] ==> [n, 7*7*64] x = FlattenLayer(x, name='flatten') layers.append(x) # layer_9: fullconnected_layer [n, 3136] ==> [n, 500] x = DenseLayer(x, output_nums=500, name='dense1') layers.append(x) # layer_10: relu_layer [n, 500] ==> [n, 500] x = ReLULayer(x, name='relu1') layers.append(x) # layer_11: fullconnected_layer [n, 500] ==> [n, 10] x = DenseLayer(x, output_nums=10, name='dense2') layers.append(x) # layer_12: softmax_layer [n, 10] ==> [n, 10] x = SoftMaxLayer(x, name='softmax') layers.append(x) self.layers = layers
def compile_make_fully_convolutional(nnet): # for naming convenience nnet.dense3_layer = nnet.svm_layer pad = 'valid' nnet.dense1_conv_layer = ConvLayer(nnet.maxpool5_layer, num_filters=4096, filter_size=(7, 7), pad=pad, flip_filters=False) relu_ = ReLU(nnet.dense1_conv_layer) nnet.dense2_conv_layer = ConvLayer(relu_, num_filters=4096, filter_size=(1, 1), pad=pad, flip_filters=False) relu_ = ReLU(nnet.dense2_conv_layer) nnet.dense3_conv_layer = ConvLayer(relu_, num_filters=1000, filter_size=(1, 1), pad=pad, flip_filters=False) W_dense1_reshaped = \ nnet.dense1_layer.W.T.reshape(nnet.dense1_conv_layer.W.shape) W_dense2_reshaped = \ nnet.dense2_layer.W.T.reshape(nnet.dense2_conv_layer.W.shape) W_dense3_reshaped = \ nnet.dense3_layer.W.T.reshape(nnet.dense3_conv_layer.W.shape) updates = ((nnet.dense1_conv_layer.W, W_dense1_reshaped), (nnet.dense2_conv_layer.W, W_dense2_reshaped), (nnet.dense3_conv_layer.W, W_dense3_reshaped), (nnet.dense1_conv_layer.b, nnet.dense1_layer.b), (nnet.dense2_conv_layer.b, nnet.dense2_layer.b), (nnet.dense3_conv_layer.b, nnet.dense3_layer.b)) return theano.function([], updates=updates)
def discriminator(self): disc = [ ConvLayer(num_filters=64, kernel_size=4, stride=2, padding="SAME", weights_init=tf.truncated_normal_initializer( stddev=self.weight_stdev), activation=leakyrelu), ConvLayer(num_filters=128, kernel_size=4, stride=2, padding="SAME", weights_init=tf.truncated_normal_initializer( stddev=self.weight_stdev), normalizer=slim.batch_norm, activation=leakyrelu), ConvLayer(num_filters=256, kernel_size=4, stride=2, padding="SAME", weights_init=tf.truncated_normal_initializer( stddev=self.weight_stdev), normalizer=slim.batch_norm, activation=leakyrelu), ConvLayer(num_filters=512, kernel_size=4, stride=2, padding="SAME", weights_init=tf.truncated_normal_initializer( stddev=self.weight_stdev), normalizer=slim.batch_norm, activation=leakyrelu), ConvLayer(num_filters=1, kernel_size=4, stride=1, padding="SAME", weights_init=tf.truncated_normal_initializer( stddev=self.weight_stdev), normalizer=slim.batch_norm, activation=leakyrelu) ] return forward(disc)
def __init__(self, reshaped_input, name='unnamed'): self.name = name self.conv_layer1 = ConvLayer( reshaped_input, filter_shape=(2, 1, 4, 1), #num outs, num ins, size image_shape=(None, 1, None, 1), stride=(1, 1), name=self.name + '_conv1', border_mode=(2, 0), act_fn='relu') self.conv_layer2 = ConvLayer(self.conv_layer1, filter_shape=(4, 2, 2, 1), image_shape=(None, 2, None, 1), stride=(2, 1), name=self.name + '_conv2', border_mode=(0, 0), act_fn='relu') self.conv_layer3 = ConvLayer(self.conv_layer2, filter_shape=(4, 4, 1, 1), image_shape=(None, 4, None, 1), stride=(1, 1), name=self.name + '_conv3', border_mode=(0, 0), act_fn='relu') self.conv_layer4 = ConvLayer(self.conv_layer3, filter_shape=(1, 4, 1, 1), image_shape=(None, 4, None, 1), stride=(1, 1), name=self.name + '_conv4', border_mode=(0, 0), act_fn='tanh') self.output = self.conv_layer4.output self.layers = [ self.conv_layer1, self.conv_layer2, self.conv_layer3, self.conv_layer4 ] self.params = [] for l in self.layers: self.params += l.params
def add_a_random_conv_layer(self): s1 = self.init_feature_size() filter_size = s1, s1 feature_map_size = self.init_feature_map_size() mean = self.init_mean() std = self.init_std() conv_layer = ConvLayer(filter_size=filter_size, feature_map_size=feature_map_size, weight_matrix=[mean, std]) return conv_layer
def encoder(self, inputs): # convolutional layer conv1 = ConvLayer(input_filters=tf.cast(inputs.shape[3], tf.int32), output_filters=8, act=tf.nn.relu, kernel_size=3, kernel_stride=1, kernel_padding="SAME") conv1_act = conv1.__call__(inputs) print(conv1_act.shape) # convolutional and pooling layer conv_pool1 = ConvPoolLayer(input_filters=8, output_filters=8, act=tf.nn.relu, kernel_size=3, kernel_stride=1, kernel_padding="SAME", pool_size=3, pool_stride=2, pool_padding="SAME") conv_pool1_act = conv_pool1.__call__(conv1_act) print(conv_pool1_act.shape) # convolutional layer conv2 = ConvLayer(input_filters=8, output_filters=16, act=tf.nn.relu, kernel_size=3, kernel_stride=1, kernel_padding="SAME") conv2_act = conv2.__call__(conv_pool1_act) print(conv2_act.shape) # convolutional and pooling layer conv_pool2 = ConvPoolLayer(input_filters=16, output_filters=16, act=tf.nn.relu, kernel_size=3, kernel_stride=1, kernel_padding="SAME", pool_size=3, pool_stride=2, pool_padding="SAME") conv_pool2_act = conv_pool2.__call__(conv2_act) print(conv_pool2_act.shape) conv3 = ConvLayer(input_filters=16, output_filters=32, act=tf.nn.relu, kernel_size=3, kernel_stride=1, kernel_padding="SAME") conv3_act = conv3.__call__(conv_pool2_act) print(conv3_act.shape) conv_pool3 = ConvPoolLayer(input_filters=32, output_filters=32, act=tf.nn.relu, kernel_size=3, kernel_stride=1, kernel_padding="SAME", pool_size=3, pool_stride=2, pool_padding="SAME") conv_pool3_act = conv_pool3.__call__(conv3_act) print(conv_pool3_act.shape) last_conv_dims = conv_pool3_act.shape[1:] # make output of pooling flatten flatten = tf.reshape(conv_pool3_act, [-1,last_conv_dims[0]*last_conv_dims[1]*last_conv_dims[2]]) print(flatten.shape) weights_encoder = normal_initializer((tf.cast(flatten.shape[1], tf.int32), FLAGS.code_size)) bias_encoder = zero_initializer((FLAGS.code_size)) # apply fully connected layer dense = tf.matmul(flatten, weights_encoder) + bias_encoder print(dense.shape) return dense, last_conv_dims
def cifar10(): conv_config = [{ 'channels': 3, 'kernel': (3, 3), 'stride': (1, 1), 'activation': 'relu' }] dense_config = [{'out': 20, 'activation': 'relu'}] input_shape = [32, 32, 3] output_size = 10 depth = 3 width = 10 max_modules_pr_layer = 3 learning_rate = 0.001 optimizer_type = Adam loss = 'categorical_crossentropy' layers = [] layers.append(ConvLayer(width, 'L0', conv_config)) layers.append(ConvLayer(width, 'L1', conv_config)) layers.append(ConvLayer(width, 'L2', conv_config, maxpool=True)) #layers.append(DenseLayer(width, 'L2', dense_config, flatten=True)) Layer.initialize_whole_network(layers, input_shape) task = TaskContainer(input_shape, output_size, True, name='unique_cifar10', optimizer=optimizer_type, loss=loss, lr=learning_rate) pathnet = PathNet(input_shape=input_shape, width=width, depth=depth) pathnet._layers = layers pathnet._tasks = [task] pathnet.max_modules_pr_layer = max_modules_pr_layer for layer in pathnet._layers: layer.save_initialized_weights() return pathnet, task
def __init__(self, iterations=1, learning_rate=0.5, topo=[('c', 3, 4), ('p', 2), ('c', 3, 4), ('p', 9), ('mlp', 4, 4, 2)], activation_func=(np.tanh, nputils.tanh_deriv)): """ Creates a new convolutional neural network with the given topology (architecture), learning rate and number of iterations. :param iterations: number of iterations for training. :param learning_rate: rate for updating the weights :param topo: defines the architecture of the net. It is a list of tuples. Each tuple represents a layer, where the first element is a character that specifies the type of layer. E.g. 'c' convolutional layer, 'p' pooling layer, 'mlp' fully connected conventional neural network. The next elements in the tuple are layer specific. Convolutional: 2nd element defines the kernel size, e.g. 3 for a 3x3 kernel. 3rd element specifies the number of maps in the layer. Pooling: 2nd element defines the pool patch size, e.g. 2 for a pool patch size of 2x2. MLP: each element defines the layer size for the network. A complete example looks like this: [('c', 3, 4), ('p', 2), ('c', 3, 4), ('p', 9), ('mlp', 4, 4, 2)] """ self.split_ratio = 0.8 self.iterations = iterations self.learning_rate = learning_rate self.layers = [] self.activ_func = activation_func[0] self.deriv_acitv_func = activation_func[1] num_prev_maps = 1 self.topo = topo # parse topology for layer in topo: # convolutional layer if layer[0] == 'c': conv_layer = ConvLayer(num_prev_maps=num_prev_maps, kernel_size=layer[1], num_maps=layer[2]) self.add_layer(conv_layer) num_prev_maps = layer[2] # pooling layer elif layer[0] == 'p': self.add_layer(MaxPoolLayer(layer[1], num_prev_maps)) # multilayer perceptron elif layer[0] == 'mlp': self.mlp = MultilayerPerceptron( list(layer[1:]), do_classification=True, update_method=SimpleUpdate(self.learning_rate), activ_func=(self.activ_func, self.deriv_acitv_func))
def mutate_conv_unit(self, unit, eta): # feature map size, feature map number, mean std fms = unit.filter_width fmn = unit.feature_map_size mean = unit.weight_matrix_mean std = unit.weight_matrix_std new_fms = int( self.pm(self.filter_size_range[0], self.filter_size_range[-1], fms, eta)) new_fmn = int( self.pm(self.featur_map_size_range[0], self.featur_map_size_range[1], fmn, eta)) new_mean = self.pm(self.mean_range[0], self.mean_range[1], mean, eta) new_std = self.pm(self.std_range[0], self.std_range[1], std, eta) conv_layer = ConvLayer(filter_size=[new_fms, new_fms], feature_map_size=new_fmn, weight_matrix=[new_mean, new_std]) return conv_layer
def addConvLayer(self, use_batch_norm=False, **kwargs): """ Add convolutional layer. If batch norm flag is True, the convolutional layer will be followed by a batch-normalization layer """ input_layer = self.input_layer if not self.all_layers \ else self.all_layers[-1] self.n_conv_layers += 1 name = "conv%i" % self.n_conv_layers new_layer = ConvLayer(input_layer, name=name, **kwargs) self.all_layers += (new_layer, ) self.trainable_layers += (new_layer, ) if use_batch_norm: self.n_bn_layers += 1 name = "bn%i" % self.n_bn_layers self.all_layers += (BatchNorm(new_layer, name=name), )
def __init__(self, layers): self._network = [] for layer in layers: layer_type = layer.pop("type") if layer_type == "data": # this is a data layer new_layer = DataLayer(**layer) elif layer_type == "conv": new_layer = ConvLayer(**layer) elif layer_type == "pool": new_layer = PoolLayer(**layer) elif layer_type == "dense": new_layer = DenseLayer(**layer) elif layer_type == "relu": new_layer = ReLULayer() elif layer_type == "loss": new_layer = LossLayer(**layer) else: raise NotImplementedError( "Layer type: {0} not found".format(layer_type)) self._network.append(new_layer) self.initialize()
def __init__(self, n_classe): super(MultiView, self).__init__() # Layer 1 self.layer1 = nn.Sequential(OrderedDict([ ('conv1', ConvLayer(3, out_channels=32, filter_size=(3,3), stride=(2,2))) ])) # Layer 2 self.layer2 = nn.Sequential(OrderedDict([ ('conv2a', ConvLayer(32, out_channels=64, filter_size=(3, 3), stride=(2, 2))), ('conv2b', ConvLayer(64, out_channels=64, filter_size=(3, 3), stride=(1, 1))), ('conv2c', ConvLayer(64, out_channels=64, filter_size=(3, 3), stride=(1, 1))), ])) # Layer 3 self.layer3 = nn.Sequential(OrderedDict([ ('conv3a', ConvLayer(64, out_channels=128, filter_size=(3, 3), stride=(1, 1))), ('conv3b', ConvLayer(128, out_channels=128, filter_size=(3, 3), stride=(1, 1))), ('conv3c', ConvLayer(128, out_channels=128, filter_size=(3, 3), stride=(1, 1))), ])) # Layer 4 self.layer4 = nn.Sequential(OrderedDict([ ('conv4a', ConvLayer(128, out_channels=256, filter_size=(3, 3), stride=(1, 1))), ('conv4b', ConvLayer(256, out_channels=256, filter_size=(3, 3), stride=(1, 1))), ('conv4c', ConvLayer(256, out_channels=256, filter_size=(3, 3), stride=(1, 1))), ])) # Layer 5 self.layer5 = nn.Sequential(OrderedDict([ ('conv5a', ConvLayer(256, out_channels=512, filter_size=(3, 3), stride=(1, 1))), ('conv5b', ConvLayer(512, out_channels=512, filter_size=(3, 3), stride=(1, 1))), ('conv5c', ConvLayer(512, out_channels=512, filter_size=(3, 3), stride=(1, 1))), ])) # FC self.max_pool = MaxPooling() self.avg_pool = AvgPooling() self.dropout = nn.Dropout(p=0.5) self.fc1 = nn.Linear(512 * 2, 512 * 2) self.fc2 = nn.Linear(512 * 2, n_classe)
def encoder(self, inputs): # Build Convolutional Part of Encoder # Put sequential layers: # ConvLayer1 ==> ConvPoolLayer1 ==> ConvLayer2 ==> ConvPoolLayer2 ==> ConvLayer3 ==> ConvPoolLayer3 # Settings of layers: # For all ConvLayers: filter size = 3, filter stride = 1, padding type = SAME # For all ConvPoolLayers: # Conv : filter size = 3, filter stride = 1, padding type = SAME # Pooling : pool size = 3, pool stride = 2, padding type = SAME # Number of Filters: # num_channel defined in FLAGS (input) ==> 8 ==> 8 ==> 16 ==> 16 ==> 32 ==> 32 # convolutional layer conv1_class = ConvLayer(input_filters=FLAGS.num_channel, output_filters=8, act=tf.nn.relu, kernel_size=3, kernel_stride=1, kernel_padding='SAME') conv1 = conv1_class(inputs=inputs) print(conv1.shape) # convolutional and pooling layer conv_pool1_class = ConvPoolLayer(input_filters=8, output_filters=8, act=tf.nn.relu, kernel_size=3, kernel_stride=1, kernel_padding='SAME', pool_size=3, pool_stride=2, pool_padding='SAME') conv_pool1 = conv_pool1_class(inputs=conv1) print(conv_pool1.shape) # convolutional layer conv2_class = ConvLayer(input_filters=8, output_filters=16, act=tf.nn.relu, kernel_size=3, kernel_stride=1, kernel_padding='SAME') conv2 = conv2_class(inputs=conv_pool1) print(conv2.shape) # convolutional and pooling layer conv_pool2_class = ConvPoolLayer(input_filters=16, output_filters=16, act=tf.nn.relu, kernel_size=3, kernel_stride=1, kernel_padding='SAME', pool_size=3, pool_stride=2, pool_padding='SAME') conv_pool2 = conv_pool2_class(inputs=conv2) print(conv_pool2.shape) conv3_class = ConvLayer(input_filters=16, output_filters=32, act=tf.nn.relu, kernel_size=3, kernel_stride=1, kernel_padding='SAME') conv3 = conv3_class(inputs=conv_pool2) print(conv3.shape) conv_pool3_class = ConvPoolLayer(input_filters=32, output_filters=32, act=tf.nn.relu, kernel_size=3, kernel_stride=1, kernel_padding='SAME', pool_size=3, pool_stride=2, pool_padding='SAME') conv_pool3 = conv_pool3_class(inputs=conv3) print(conv_pool3.shape) # Make Output Flatten and Apply Transformation # Num of features for dense is defined by code_size in FLAG # make output of pooling flatten WholeShape = tf.shape(conv_pool3) NumSamples = WholeShape[0] last_conv_dims = tf.constant(value=(4, 4, 32), dtype=tf.int32, shape=(3, )) #WholeShape[1:] FlattedShape = tf.reduce_prod(last_conv_dims) flatten = tf.reshape(conv_pool3, shape=[NumSamples, FlattedShape]) print(flatten.shape) # apply fully connected layer W_Trans = normal_initializer(shape=[FlattedShape, FLAGS.code_size]) B_Trans = zero_initializer(shape=[FLAGS.code_size]) dense = tf.nn.xw_plus_b(flatten, W_Trans, B_Trans) print(dense.shape) return dense, last_conv_dims
def conv_layer(self, dtype, N, C, K, D=1, H=1, W=1, T=1, R=1, S=1, pad_d=0, pad_h=0, pad_w=0, str_d=1, str_h=1, str_w=1, grid_P=0, grid_Q=0, update_size=None): """ Create a new ConvLayer parameter object. This then is passed as an argument to all the convolution operations. N: Number of images in mini-batch C: Number of input feature maps K: Number of output feature maps D: Depth of input image H: Height of input image W: Width of input image T: Depth of filter kernel R: Height of filter kernel S: Width of filter kernel padding: amount of zero-padding around the given edge strides: factor to step the filters by in a given direction grid_P, grid_Q: For the update operation define the size of the grid to distribute the work accross SMs. The smaller the grid, the deeper the MM and hence more accumulation is done in fp32. The bigger the grid, the more the work can be evenly spanned accross the SMs, at the cost of needing more fp16 accumuation operations and increased error. Set to 1,1 for full fp32 accuracy Set to P,Q for maximal distribution of work acrross SMs Set to 0,0 for automactially calculated optimal balance (recommened). Tweaking these params can have a large impact on performance as the L2 cache utilization is greatly effected by them. update_size: override kernel size selection for update. "C64_K64" (fp16 only) "C128_K64" (fp32 only) "C128_K128" (both) dtype: need to know dtype to setup proper kernels and params. Maximum utilization is achieved when N, K and C*R*S*T is a multiple of 64 """ return ConvLayer(self, dtype, N, C, K, D, H, W, T, R, S, pad_d, pad_h, pad_w, str_d, str_h, str_w, grid_P, grid_Q, update_size)
def encoder(self, inputs): ############################################################################################################# # TODO: Build Convolutional Part of Encoder # # Put sequential layers: # # ConvLayer1 ==> ConvPoolLayer1 ==> ConvLayer2 ==> ConvPoolLayer2 ==> ConvLayer3 ==> ConvPoolLayer3 # # Settings of layers: # # For all ConvLayers: filter size = 3, filter stride = 1, padding type = SAME # # For all ConvPoolLayers: # # Conv : filter size = 3, filter stride = 1, padding type = SAME # # Pooling : pool size = 3, pool stride = 2, padding type = SAME # # Number of Filters: # # num_channel defined in FLAGS (input) ==> 8 ==> 8 ==> 16 ==> 16 ==> 32 ==> 32 # ############################################################################################################# relu = tf.nn.relu # convolutional layer cl1 = ConvLayer(FLAGS.num_channel, 8, relu, 3, 1, 'SAME') conv1 = cl1(inputs) print(conv1.shape) # convolutional and pooling layer cl2 = ConvPoolLayer(8, 8, relu, 3, 1, 'SAME', 3, 2, 'SAME') conv_pool1 = cl2(conv1) print(conv_pool1.shape) # convolutional layer cl3 = ConvLayer(8, 16, relu, 3, 1, 'SAME') conv2 = cl3(conv_pool1) print(conv2.shape) # convolutional and pooling layer cl4 = ConvPoolLayer(16, 16, relu, 3, 1, 'SAME', 3, 2, 'SAME') conv_pool2 = cl4(conv2) print(conv_pool2.shape) cl5 = ConvLayer(16, 32, relu, 3, 1, 'SAME') conv3 = cl5(conv_pool2) print(conv3.shape) cl6 = ConvPoolLayer(32, 32, tf.nn.relu, 3, 1, 'SAME', 3, 2, 'SAME') conv_pool3 = cl6(conv3) print(conv_pool3.shape) ########################################################################## # END OF YOUR CODE # ########################################################################## ########################################################################## # TODO: Make Output Flatten and Apply Transformation # # Please save the last three dimensions of output of the above code # # Save these numbers in a variable called last_conv_dims # # Multiply all these dimensions to find num of features if flatten # # Use tf.reshape to make a tensor flat # # Define some weights and bias and apply linear transformation # # Use normal and zero initializer for weights and bias respectively # # Please store output of transformation in a variable called dense # # Num of features for dense is defined by code_size in FLAG # # Note that there is no need apply any kind of activation function # ########################################################################## # make output of pooling flatten dim = np.prod(conv_pool3.shape[1:]) flatten = tf.reshape(conv_pool3, [-1, dim]) print(flatten.shape) # apply fully connected layer W_fc = normal_initializer(shape=(dim.__int__(), FLAGS.code_size)) B_fc = zero_initializer(shape=FLAGS.code_size) dense = tf.matmul(flatten, W_fc) + B_fc print(dense.shape) ########################################################################## # END OF YOUR CODE # ########################################################################## last_conv_dims = conv_pool3.shape[1:] return dense, last_conv_dims
def __init__(self, rng, params, cost_function='mse', optimizer=RMSprop): lr = params["lr"] batch_size = params["batch_size"] sequence_length = params["seq_length"] # minibatch) X = T.matrix(name="input", dtype=dtype) # batch of sequence of vector Y = T.matrix(name="output", dtype=dtype) # batch of sequence of vector is_train = T.iscalar( 'is_train' ) # pseudo boolean for switching between training and prediction #CNN global parameters. subsample = (1, 1) p_1 = 0.5 border_mode = "same" cnn_batch_size = batch_size pool_size = (2, 2) #Layer1: conv2+pool+drop filter_shape = (128, 1, 10, 10) input_shape = (cnn_batch_size, 1, 144, 176 ) #input_shape= (samples, channels, rows, cols) input = X.reshape(input_shape) c1 = ConvLayer(rng, input, filter_shape, input_shape, border_mode, subsample, activation=nn.relu) p1 = PoolLayer(c1.output, pool_size=pool_size, input_shape=c1.output_shape) dl1 = DropoutLayer(rng, input=p1.output, prob=p_1, is_train=is_train) #Layer2: conv2+pool subsample = (1, 1) filter_shape = (256, p1.output_shape[1], 3, 3) c2 = ConvLayer(rng, dl1.output, filter_shape, p1.output_shape, border_mode, subsample, activation=nn.relu) p2 = PoolLayer(c2.output, pool_size=pool_size, input_shape=c2.output_shape) #Layer3: conv2+pool filter_shape = (256, p2.output_shape[1], 3, 3) c3 = ConvLayer(rng, p2.output, filter_shape, p2.output_shape, border_mode, subsample, activation=nn.relu) p3 = PoolLayer(c3.output, pool_size=pool_size, input_shape=c3.output_shape) #Layer4: conv2+pool filter_shape = (128, p3.output_shape[1], 3, 3) c4 = ConvLayer(rng, p3.output, filter_shape, p3.output_shape, border_mode, subsample, activation=nn.relu) p4 = PoolLayer(c4.output, pool_size=pool_size, input_shape=c4.output_shape) #Layer5: hidden n_in = reduce(lambda x, y: x * y, p4.output_shape[1:]) x_flat = p4.output.flatten(2) h1 = HiddenLayer(rng, x_flat, n_in, 1024, activation=nn.relu) #Layer6: hidden lreg = LogisticRegression(rng, h1.output, 1024, params['n_output']) self.output = lreg.y_pred self.params = c1.params + c2.params + c3.params + c4.params + h1.params + lreg.params cost = get_err_fn(self, cost_function, Y) L2_reg = 0.0001 L2_sqr = theano.shared(0.) for param in self.params: L2_sqr += (T.sum(param[0]**2) + T.sum(param[1]**2)) cost += L2_reg * L2_sqr _optimizer = optimizer(cost, self.params, lr=lr) self.train = theano.function(inputs=[X, Y, is_train], outputs=cost, updates=_optimizer.getUpdates(), allow_input_downcast=True) self.predictions = theano.function(inputs=[X, is_train], outputs=self.output, allow_input_downcast=True) self.n_param = count_params(self.params)
def encoder(self, inputs): ############################################################################################################# # TODO: Build Convolutional Part of Encoder # # Put sequential layers: # # ConvLayer1 ==> ConvPoolLayer1 ==> ConvLayer2 ==> ConvPoolLayer2 ==> ConvLayer3 ==> ConvPoolLayer3 # # Settings of layers: # # For all ConvLayers: filter size = 3, filter stride = 1, padding type = SAME # # For all ConvPoolLayers: # # Conv : filter size = 3, filter stride = 1, padding type = SAME # # Pooling : pool size = 3, pool stride = 2, padding type = SAME # # Number of Filters: # # num_channel defined in FLAGS (input) ==> 8 ==> 8 ==> 16 ==> 16 ==> 32 ==> 32 # ############################################################################################################# # convolutional layer relu = tf.nn.relu conv1 = ConvLayer(input_filters=FLAGS.num_channel, output_filters=8, act=relu, kernel_size=3, kernel_stride=1, kernel_padding='SAME')(inputs) print(conv1.shape) # convolutional and pooling layer conv_pool1 = ConvPoolLayer(input_filters=8, output_filters=8, act=relu, kernel_size=3, kernel_stride=1, kernel_padding='SAME', pool_size=3, pool_stride=2, pool_padding='SAME')(conv1) print(conv_pool1.shape) # convolutional layer conv2 = ConvLayer(input_filters=8, output_filters=16, act=relu, kernel_size=3, kernel_stride=1, kernel_padding='SAME')(conv_pool1) print(conv2.shape) # convolutional and pooling layer conv_pool2 = ConvPoolLayer(input_filters=16, output_filters=16, act=relu, kernel_size=3, kernel_stride=1, kernel_padding='SAME', pool_size=3, pool_stride=2, pool_padding='SAME')(conv2) print(conv_pool2.shape) conv3 = ConvLayer(input_filters=16, output_filters=32, act=relu, kernel_size=3, kernel_stride=1, kernel_padding='SAME')(conv_pool2) print(conv3.shape) conv_pool3 = ConvPoolLayer(input_filters=32, output_filters=32, act=relu, kernel_size=3, kernel_stride=1, kernel_padding='SAME', pool_size=3, pool_stride=2, pool_padding='SAME')(conv3) print(conv_pool3.shape) ########################################################################## # END OF YOUR CODE # ########################################################################## ########################################################################## # TODO: Make Output Flatten and Apply Transformation # # Please save the last three dimensions of output of the above code # # Save these numbers in a variable called last_conv_dims # # Multiply all these dimensions to find num of features if flatten # # Use tf.reshape to make a tensor flat # # Define some weights and bias and apply linear transformation # # Use normal and zero initializer for weights and bias respectively # # Please store output of transformation in a variable called dense # # Num of features for dense is defined by code_size in FLAG # # Note that there is no need apply any kind of activation function # ########################################################################## # make output of pooling flatten last_conv_dims = conv_pool3.shape[1:] flatten_dim = np.prod(last_conv_dims) flatten = tf.reshape(conv_pool3, [tf.shape(conv_pool3)[0], flatten_dim]) print(flatten.shape) # apply fully connected layer dense = tf.matmul(flatten, normal_initializer([ flatten_dim, FLAGS.code_size ])) + zero_initializer([FLAGS.code_size]) print(dense.shape) ########################################################################## # END OF YOUR CODE # ########################################################################## return dense, last_conv_dims
def _conv_layer(self, NHWC_X, M, feature_map, filter_size, stride, layer_params=None): if layer_params is None: layer_params = {} NHWC = NHWC_X.shape view = FullView(input_size=NHWC[1:3], filter_size=filter_size, feature_maps=NHWC[3], stride=stride) if self.flags.identity_mean: conv_mean = Conv2dMean(filter_size, NHWC[3], feature_map, stride=stride) else: conv_mean = gpflow.mean_functions.Zero() conv_mean.set_trainable(False) output_shape = image_HW(view.patch_count) + [feature_map] H_X = identity_conv(NHWC_X, filter_size, NHWC[3], feature_map, stride) if len(layer_params) == 0: conv_features = PatchInducingFeatures.from_images( NHWC_X, M, filter_size) else: conv_features = PatchInducingFeatures(layer_params.get('Z')) patch_length = filter_size**2 * NHWC[3] if self.flags.base_kernel == 'rbf': lengthscales = layer_params.get('base_kernel/lengthscales', 5.0) variance = layer_params.get('base_kernel/variance', 5.0) base_kernel = kernels.RBF(patch_length, variance=variance, lengthscales=lengthscales) elif self.flags.base_kernel == 'acos': base_kernel = kernels.ArcCosine(patch_length, order=0) else: raise ValueError("Not a valid base-kernel value") q_mu = layer_params.get('q_mu') q_sqrt = layer_params.get('q_sqrt') conv_layer = ConvLayer(base_kernel=base_kernel, mean_function=conv_mean, feature=conv_features, view=view, white=self.flags.white, gp_count=feature_map, q_mu=q_mu, q_sqrt=q_sqrt) if q_sqrt is None: # Start with low variance. conv_layer.q_sqrt = conv_layer.q_sqrt.value * 1e-5 return conv_layer, H_X
def __init__(self, n_classes=1000, bn_param=(0.1, 1e-5), dropout_rate=0.1, base_stage_width=None, width_mult_list=1.0, ks_list=3, expand_ratio_list=6, depth_list=4): self.width_mult_list = int2list(width_mult_list, 1) self.ks_list = int2list(ks_list, 1) self.expand_ratio_list = int2list(expand_ratio_list, 1) self.depth_list = int2list(depth_list, 1) self.base_stage_width = base_stage_width self.width_mult_list.sort() self.ks_list.sort() self.expand_ratio_list.sort() self.depth_list.sort() base_stage_width = [16, 24, 40, 80, 112, 160, 960, 1280] final_expand_width = [ make_divisible(base_stage_width[-2] * max(self.width_mult_list), 8) for _ in self.width_mult_list ] self.final_expand_width = final_expand_width last_channel = [ make_divisible(base_stage_width[-1] * max(self.width_mult_list), 8) for _ in self.width_mult_list ] self.last_channel = last_channel # stride_stages = [1, 2, 2, 2, 1, 2] stride_stages = [1, 2, 2, 2, 1, 1] act_stages = ['relu', 'relu', 'relu', 'h_swish', 'h_swish', 'h_swish'] se_stages = [False, False, True, False, True, True] if depth_list is None: n_block_list = [1, 2, 3, 4, 2, 3] self.depth_list = [4, 4] print('Use MobileNetV3 Depth Setting') else: n_block_list = [1] + [max(self.depth_list)] * 5 width_list = [] for base_width in base_stage_width[:-2]: width = [ make_divisible(base_width * width_mult, 8) for width_mult in self.width_mult_list ] width_list.append(width) input_channel = width_list[0] # first conv layer # if width_mult_list has only one elem if len(set(input_channel)) == 1: first_conv = ConvLayer(3, max(input_channel), kernel_size=3, stride=2, act_func='h_swish') first_block_conv = MBInvertedConvLayer( in_channels=max(input_channel), out_channels=max(input_channel), kernel_size=3, stride=stride_stages[0], expand_ratio=1, act_func=act_stages[0], use_se=se_stages[0], ) else: first_conv = DynamicConvLayer( in_channel_list=int2list(3, len(input_channel)), out_channel_list=input_channel, kernel_size=3, stride=2, act_func='h_swish', ) first_block_conv = DynamicMBConvLayer( in_channel_list=input_channel, out_channel_list=input_channel, kernel_size_list=3, expand_ratio_list=1, stride=stride_stages[0], act_func=act_stages[0], use_se=se_stages[0], ) first_block = MobileInvertedResidualBlock( first_block_conv, IdentityLayer(input_channel, input_channel)) # inverted residual blocks self.block_group_info = [] blocks = [first_block] _block_index = 1 feature_dim = input_channel for width, n_block, s, act_func, use_se in zip(width_list[1:], n_block_list[1:], stride_stages[1:], act_stages[1:], se_stages[1:]): self.block_group_info.append( [_block_index + i for i in range(n_block)]) _block_index += n_block output_channel = width for i in range(n_block): if i == 0: stride = s else: stride = 1 mobile_inverted_conv = DynamicMBConvLayer( in_channel_list=feature_dim, out_channel_list=output_channel, kernel_size_list=ks_list, expand_ratio_list=expand_ratio_list, stride=stride, act_func=act_func, use_se=use_se, ) if stride == 1 and feature_dim == output_channel: shortcut = IdentityLayer(feature_dim, feature_dim) else: shortcut = None blocks.append( MobileInvertedResidualBlock(mobile_inverted_conv, shortcut)) feature_dim = output_channel # final expand layer, feature mix layer & classifier if len(final_expand_width) == 1: final_expand_layer = ConvLayer(max(feature_dim), max(final_expand_width), kernel_size=1, act_func='h_swish') feature_mix_layer = ConvLayer( max(final_expand_width), max(last_channel), kernel_size=1, bias=False, use_bn=False, act_func='h_swish', ) else: final_expand_layer = DynamicConvLayer( in_channel_list=feature_dim, out_channel_list=final_expand_width, kernel_size=1, act_func='h_swish') feature_mix_layer = DynamicConvLayer( in_channel_list=final_expand_width, out_channel_list=last_channel, kernel_size=1, use_bn=False, act_func='h_swish', ) if len(set(last_channel)) == 1: classifier = LinearLayer(max(last_channel), n_classes, dropout_rate=dropout_rate) else: classifier = DynamicLinearLayer(in_features_list=last_channel, out_features=n_classes, bias=True, dropout_rate=dropout_rate) super(OFAMobileNetV3, self).__init__(first_conv, blocks, final_expand_layer, feature_mix_layer, classifier) # set bn param self.set_bn_param(momentum=bn_param[0], eps=bn_param[1]) # runtime_depth self.runtime_depth = [ len(block_idx) for block_idx in self.block_group_info ]
def __init__(self, mode=None, n_bits=16, ch_in=1, ch_init=128, n_convs=4, n_classes=2, attention="sqex", conv_type="stdz"): super().__init__() mode = mode.lower() if mode == "bsn": self.layer0 = BitplaneSeparation(ch_in=ch_in, n_bits=n_bits, no_weight=True, inv_weight=False) self.layer1_fe = FeatureExtractionLayer(n_bits, ch_init, n_convs) elif mode == "bsn-hpf": self.layer0 = HPF(ch_in, 4, 5, padding=2) self.layer1_fe = FeatureExtractionLayer(4, ch_init, n_convs) elif mode == "bsn-hpf-tlu": self.layer0 = nn.Sequential(HPF(ch_in, 4, 5, padding=2), nn.Conv1d(4, 8, 1), TLU(3.0)) self.layer1_fe = FeatureExtractionLayer(8, ch_init, n_convs) elif mode == "bsn-nobs": self.layer0 = nn.Conv1d(1, 8, 1) self.layer1_fe = FeatureExtractionLayer(8, ch_init, n_convs) else: raise ValueError("mode should be designated: %s" % (mode)) self._name = mode self.layer2_conv = ConvLayer(ch_init, 2 * ch_init, conv_type=conv_type) self.layer3_conv = ConvLayer(2 * ch_init, 2 * ch_init, conv_type=conv_type) self.layer4_conv = ConvLayer(2 * ch_init, 2 * ch_init, conv_type=conv_type, attention=attention) self.layer5_conv = ConvLayer(2 * ch_init, 4 * ch_init, stride=2, conv_type=conv_type) self.layer6_conv = ConvLayer(4 * ch_init, 4 * ch_init, conv_type=conv_type) self.layer7_conv = ConvLayer(4 * ch_init, 4 * ch_init, conv_type=conv_type, attention=attention) self.layer8_conv = ConvLayer(4 * ch_init, 8 * ch_init, stride=2, conv_type=conv_type) self.layer9_conv = ConvLayer(8 * ch_init, 8 * ch_init, conv_type=conv_type) self.layer10_conv = ConvLayer(8 * ch_init, 8 * ch_init, conv_type=conv_type, attention=attention) self.layer11_conv = ConvLayer(8 * ch_init, 16 * ch_init, stride=2, conv_type=conv_type) self.layer12_conv = ConvLayer(16 * ch_init, 16 * ch_init, conv_type=conv_type) self.layer13_conv = ConvLayer(16 * ch_init, 16 * ch_init, conv_type=conv_type, attention=attention) self.layer14_cls = ClassificationLayer(16 * ch_init, n_classes=n_classes) self.initialize_parameters()
class NASNet(BasicUnit): def __init__(self, width_stages, n_cell_stages, stride_stages, dropout=0): super(NASNet, self).__init__() self.width_stages = width_stages self.n_cell_stages = n_cell_stages self.stride_stages = stride_stages in_channels = 32 first_cell_width = 16 # first conv layer self.first_conv = ConvLayer(3, in_channels, 3, 2, 1, 1, False, False, True, 'relu6', 0, 'weight_bn_act') # first block first_block_config = { "name": "MobileInvertedResidualBlock", "mobile_inverted_conv": { "name": "MBInvertedConvLayer", "in_channels": in_channels, "out_channels": first_cell_width, "kernel_size": 3, "stride": 1, "expand_ratio": 1 }, "shortcut": None } self.first_block = MobileInvertedResidualBlock.build_from_config( first_block_config) in_channels = first_cell_width # blocks self.blocks = nn.ModuleList() for width, n_cell, s in zip(self.width_stages, self.n_cell_stages, self.stride_stages): for i in range(n_cell): if i == 0: stride = s else: stride = 1 block = WSMobileInvertedResidualBlock(in_channels, width, stride) in_channels = width self.blocks.append(block) self.feature_mix_layer = ConvLayer(in_channels, 1280, 1, 1, 1, 1, False, False, True, 'relu6', 0, 'weight_bn_act') self.global_avg_pooling = nn.AdaptiveAvgPool2d(1) self.classifier = LinearLayer(1280, 1000, True, False, None, dropout, 'weight_bn_act') def forward(self, x, arch, bn_train=False): if bn_train: for m in self.modules(): if isinstance(m, nn.BatchNorm1d): m.train() x = self.first_conv(x) x = self.first_block(x) for i, block in enumerate(self.blocks): x = block(x, arch[i]) #x = self.last_block(x) if self.feature_mix_layer: x = self.feature_mix_layer(x) x = self.global_avg_pooling(x) x = x.view(x.size(0), -1) # flatten x = self.classifier(x) return x def get_flops(self, x): flop, x = self.first_conv.get_flops(x) for block in self.blocks: delta_flop, x = block.get_flops(x) flop += delta_flop if self.feature_mix_layer: delta_flop, x = self.feature_mix_layer.get_flops(x) flop += delta_flop x = self.global_avg_pooling(x) x = x.view(x.size(0), -1) # flatten delta_flop, x = self.classifier.get_flops(x) flop += delta_flop return flop, x def set_bn_param(self, bn_momentum, bn_eps): for m in self.modules(): if isinstance(m, nn.BatchNorm2d): m.momentum = bn_momentum m.eps = bn_eps return def get_bn_param(self): for m in self.modules(): if isinstance(m, nn.BatchNorm2d): return { 'momentum': m.momentum, 'eps': m.eps, } return None def init_model(self, model_init, init_div_groups=True): for m in self.modules(): if isinstance(m, nn.Conv2d): if model_init == 'he_fout': n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels if init_div_groups: n /= m.groups m.weight.data.normal_(0, math.sqrt(2. / n)) elif model_init == 'he_fin': n = m.kernel_size[0] * m.kernel_size[1] * m.in_channels if init_div_groups: n /= m.groups m.weight.data.normal_(0, math.sqrt(2. / n)) else: raise NotImplementedError elif isinstance(m, nn.BatchNorm2d): m.weight.data.fill_(1) m.bias.data.zero_() elif isinstance(m, nn.Linear): if m.bias is not None: m.bias.data.zero_() elif isinstance(m, nn.BatchNorm1d): m.weight.data.fill_(1) m.bias.data.zero_() def weight_parameters(self): return self.parameters() @staticmethod def _make_divisible(v, divisor, min_val=None): """ This function is taken from the original tf repo. It ensures that all layers have a channel number that is divisible by 8 It can be seen here: https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py :param v: :param divisor: :param min_val: :return: """ if min_val is None: min_val = divisor new_v = max(min_val, int(v + divisor / 2) // divisor * divisor) # Make sure that round down does not go down by more than 10%. if new_v < 0.9 * v: new_v += divisor return new_v
def __init__(self,rng,params,cost_function='mse',optimizer = RMSprop): lr=params["lr"] n_lstm=params['n_hidden'] n_out=params['n_output'] batch_size=params["batch_size"] sequence_length=params["seq_length"] X = T.tensor3() # batch of sequence of vector Y = T.tensor3() # batch of sequence of vector is_train = T.iscalar('is_train') # pseudo boolean for switching between training and prediction #CNN global parameters. subsample=(1,1) p_1=0.5 border_mode="valid" cnn_batch_size=batch_size*sequence_length pool_size=(2,2) #Layer1: conv2+pool+drop filter_shape=(64,1,9,9) input_shape=(cnn_batch_size,1,120,60) #input_shape= (samples, channels, rows, cols) input= X.reshape(input_shape) c1=ConvLayer(rng, input,filter_shape, input_shape,border_mode,subsample, activation=nn.relu) p1=PoolLayer(c1.output,pool_size=pool_size,input_shape=c1.output_shape) dl1=DropoutLayer(rng,input=p1.output,prob=p_1,is_train=is_train) #Layer2: conv2+pool filter_shape=(128,p1.output_shape[1],3,3) c2=ConvLayer(rng, dl1.output, filter_shape,p1.output_shape,border_mode,subsample, activation=nn.relu) p2=PoolLayer(c2.output,pool_size=pool_size,input_shape=c2.output_shape) #Layer3: conv2+pool filter_shape=(128,p2.output_shape[1],3,3) c3=ConvLayer(rng, p2.output,filter_shape,p2.output_shape,border_mode,subsample, activation=nn.relu) p3=PoolLayer(c3.output,pool_size=pool_size,input_shape=c3.output_shape) #Layer4: hidden n_in= reduce(lambda x, y: x*y, p3.output_shape[1:]) x_flat = p3.output.flatten(2) h1=HiddenLayer(rng,x_flat,n_in,1024,activation=nn.relu) n_in=1024 rnn_input = h1.output.reshape((batch_size,sequence_length, n_in)) #Layer5: LSTM self.n_in = n_in self.n_lstm = n_lstm self.n_out = n_out self.W_hy = init_weight((self.n_lstm, self.n_out), rng=rng,name='W_hy', sample= 'glorot') self.b_y = init_bias(self.n_out,rng=rng, sample='zero') layer1=LSTMLayer(rng,0,self.n_in,self.n_lstm) self.params = layer1.params self.params.append(self.W_hy) self.params.append(self.b_y) def step_lstm(x_t,h_tm1,c_tm1): [h_t,c_t,y_t]=layer1.run(x_t,h_tm1,c_tm1) y = T.dot(y_t, self.W_hy) + self.b_y return [h_t,c_t,y] H = T.matrix(name="H",dtype=dtype) # initial hidden state C = T.matrix(name="C",dtype=dtype) # initial hidden state [h_t,c_t,y_vals], _ = theano.scan(fn=step_lstm, sequences=[rnn_input.dimshuffle(1,0,2)], outputs_info=[H, C, None]) self.output = y_vals.dimshuffle(1,0,2) self.params =c1.params+c2.params+c3.params+h1.params+self.params cost=get_err_fn(self,cost_function,Y) L2_reg=0.0001 L2_sqr = theano.shared(0.) for param in self.params: L2_sqr += (T.sum(param ** 2)) cost += L2_reg*L2_sqr _optimizer = optimizer(cost, self.params, lr=lr) self.train = theano.function(inputs=[X,Y,is_train,H,C],outputs=[cost,h_t[-1],c_t[-1]],updates=_optimizer.getUpdates(),allow_input_downcast=True) self.predictions = theano.function(inputs = [X,is_train,H,C], outputs = [self.output,h_t[-1],c_t[-1]],allow_input_downcast=True) self.n_param=count_params(self.params)