def deeper_conv_block(conv_layer, kernel_size, weighted=True): filter_shape = (kernel_size, ) * 2 n_filters = conv_layer.filters weight = np.zeros((n_filters, n_filters) + filter_shape) center = tuple(map(lambda x: int((x - 1) / 2), filter_shape)) for i in range(n_filters): filter_weight = np.zeros((n_filters, ) + filter_shape) index = (i, ) + center filter_weight[index] = 1 weight[i, ...] = filter_weight bias = np.zeros(n_filters) new_conv_layer = StubConv(conv_layer.filters, n_filters, kernel_size=kernel_size) bn = StubBatchNormalization(n_filters) if weighted: new_conv_layer.set_weights( (add_noise(weight, np.array([0, 1])), add_noise(bias, np.array([0, 1])))) new_weights = [ add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])) ] bn.set_weights(new_weights) return [StubReLU(), new_conv_layer, bn]
def deeper_conv_block(conv_layer, kernel_size, weighted=True): filter_shape = (kernel_size,) * 2 n_filters = conv_layer.filters weight = np.zeros((n_filters, n_filters) + filter_shape) center = tuple(map(lambda x: int((x - 1) / 2), filter_shape)) for i in range(n_filters): filter_weight = np.zeros((n_filters,) + filter_shape) index = (i,) + center filter_weight[index] = 1 weight[i, ...] = filter_weight bias = np.zeros(n_filters) new_conv_layer = StubConv(conv_layer.filters, n_filters, kernel_size=kernel_size) bn = StubBatchNormalization(n_filters) if weighted: new_conv_layer.set_weights((add_noise(weight, np.array([0, 1])), add_noise(bias, np.array([0, 1])))) new_weights = [add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1]))] bn.set_weights(new_weights) return [StubReLU(), new_conv_layer, bn]
def deeper_conv_block(conv_layer, kernel_size, weighted=True): filter_shape = (kernel_size, ) * 2 n_filters = conv_layer.filters weight = np.zeros(filter_shape + (n_filters, n_filters)) center = tuple(map(lambda x: int((x - 1) / 2), filter_shape)) for i in range(n_filters): filter_weight = np.zeros(filter_shape + (n_filters, )) index = center + (i, ) filter_weight[index] = 1 weight[..., i] = filter_weight bias = np.zeros(n_filters) new_conv_layer = StubConv(n_filters, kernel_size=filter_shape, func=conv_layer.func) bn = StubBatchNormalization() if weighted: new_conv_layer.set_weights( (add_noise(weight, np.array([0, 1])), add_noise(bias, np.array([0, 1])))) new_weights = [ np.ones(n_filters, dtype=np.float32), np.zeros(n_filters, dtype=np.float32), np.zeros(n_filters, dtype=np.float32), np.ones(n_filters, dtype=np.float32) ] bn.set_weights(new_weights) return [ StubActivation('relu'), new_conv_layer, bn, StubDropout(constant.CONV_DROPOUT_RATE) ]
def wider_bn(layer, start_dim, total_dim, n_add, weighted=True): """Get new layer with wider batch normalization for current layer Args: weighted: layer: the layer from which we get new layer with wider batch normalization start_dim: the started dimension total_dim: the total dimension n_add: the output shape Returns: The new layer with wider batch normalization """ if not weighted: return StubBatchNormalization() weights = layer.get_weights() new_weights = [ np.ones(n_add, dtype=np.float32), np.zeros(n_add, dtype=np.float32), np.zeros(n_add, dtype=np.float32), np.ones(n_add, dtype=np.float32) ] student_w = tuple() for weight, new_weight in zip(weights, new_weights): temp_w = weight.copy() temp_w = np.concatenate( (temp_w[:start_dim], new_weight, temp_w[start_dim:total_dim])) student_w += (temp_w, ) new_layer = StubBatchNormalization() new_layer.set_weights(student_w) return new_layer
def to_add_skip_model(self, start_id, end_id): """Add a weighted add skip-connection from after start node to end node. Args: start_id: The convolutional layer ID, after which to start the skip-connection. end_id: The convolutional layer ID, after which to end the skip-connection. """ self.operation_history.append(('to_add_skip_model', start_id, end_id)) conv_block_input_id = self._conv_block_end_node(start_id) conv_block_input_id = self.adj_list[conv_block_input_id][0][0] block_last_layer_input_id = self._conv_block_end_node(end_id) # Add the pooling layer chain. layer_list = self._get_pooling_layers(conv_block_input_id, block_last_layer_input_id) skip_output_id = conv_block_input_id for index, layer_id in enumerate(layer_list): skip_output_id = self.add_layer(deepcopy(self.layer_list[layer_id]), skip_output_id) # Add the conv layer new_relu_layer = StubReLU() skip_output_id = self.add_layer(new_relu_layer, skip_output_id) new_conv_layer = StubConv(self.layer_list[start_id].filters, self.layer_list[end_id].filters, 1) skip_output_id = self.add_layer(new_conv_layer, skip_output_id) new_bn_layer = StubBatchNormalization(self.layer_list[end_id].filters) skip_output_id = self.add_layer(new_bn_layer, skip_output_id) # Add the add layer. block_last_layer_output_id = self.adj_list[block_last_layer_input_id][0][0] add_input_node_id = self._add_node(deepcopy(self.node_list[block_last_layer_output_id])) add_layer = StubAdd() self._redirect_edge(block_last_layer_input_id, block_last_layer_output_id, add_input_node_id) self._add_edge(add_layer, add_input_node_id, block_last_layer_output_id) self._add_edge(add_layer, skip_output_id, block_last_layer_output_id) add_layer.input = [self.node_list[add_input_node_id], self.node_list[skip_output_id]] add_layer.output = self.node_list[block_last_layer_output_id] self.node_list[block_last_layer_output_id].shape = add_layer.output_shape # Set weights to the additional conv layer. if self.weighted: filters_end = self.layer_list[end_id].filters filters_start = self.layer_list[start_id].filters filter_shape = (1,) * (len(self.layer_list[end_id].get_weights()[0].shape) - 2) weights = np.zeros((filters_end, filters_start) + filter_shape) bias = np.zeros(filters_end) new_conv_layer.set_weights((add_noise(weights, np.array([0, 1])), add_noise(bias, np.array([0, 1])))) n_filters = filters_end new_weights = [add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1]))] new_bn_layer.set_weights(new_weights)
def wider_bn(layer, start_dim, total_dim, n_add, weighted=True): if not weighted: return StubBatchNormalization(layer.num_features + n_add) weights = layer.get_weights() new_weights = [add_noise(np.ones(n_add, dtype=np.float32), np.array([0, 1])), add_noise(np.zeros(n_add, dtype=np.float32), np.array([0, 1])), add_noise(np.zeros(n_add, dtype=np.float32), np.array([0, 1])), add_noise(np.ones(n_add, dtype=np.float32), np.array([0, 1]))] student_w = tuple() for weight, new_weight in zip(weights, new_weights): temp_w = weight.copy() temp_w = np.concatenate((temp_w[:start_dim], new_weight, temp_w[start_dim:total_dim])) student_w += (temp_w,) new_layer = StubBatchNormalization(layer.num_features + n_add) new_layer.set_weights(student_w) return new_layer
def to_concat_skip_model(self, start_id, end_id): """Add a weighted add concatenate connection from after start node to end node. Args: start_id: The convolutional layer ID, after which to start the skip-connection. end_id: The convolutional layer ID, after which to end the skip-connection. """ self.operation_history.append( ('to_concat_skip_model', start_id, end_id)) conv_block_input_id = self._conv_block_end_node(start_id) conv_block_input_id = self.adj_list[conv_block_input_id][0][0] block_last_layer_input_id = self._conv_block_end_node(end_id) # Add the pooling layer chain. pooling_layer_list = self._get_pooling_layers( conv_block_input_id, block_last_layer_input_id) skip_output_id = conv_block_input_id for index, layer_id in enumerate(pooling_layer_list): skip_output_id = self.add_layer( deepcopy(self.layer_list[layer_id]), skip_output_id) block_last_layer_output_id = self.adj_list[block_last_layer_input_id][ 0][0] concat_input_node_id = self._add_node( deepcopy(self.node_list[block_last_layer_output_id])) self._redirect_edge(block_last_layer_input_id, block_last_layer_output_id, concat_input_node_id) concat_layer = StubConcatenate() concat_layer.input = [ self.node_list[concat_input_node_id], self.node_list[skip_output_id] ] concat_output_node_id = self._add_node(Node(concat_layer.output_shape)) self._add_edge(concat_layer, concat_input_node_id, concat_output_node_id) self._add_edge(concat_layer, skip_output_id, concat_output_node_id) concat_layer.output = self.node_list[concat_output_node_id] self.node_list[concat_output_node_id].shape = concat_layer.output_shape # Add the concatenate layer. new_relu_layer = StubReLU() concat_output_node_id = self.add_layer(new_relu_layer, concat_output_node_id) new_conv_layer = StubConv( self.layer_list[start_id].filters + self.layer_list[end_id].filters, self.layer_list[end_id].filters, 1) concat_output_node_id = self.add_layer(new_conv_layer, concat_output_node_id) new_bn_layer = StubBatchNormalization(self.layer_list[end_id].filters) self._add_edge(new_bn_layer, concat_output_node_id, block_last_layer_output_id) new_bn_layer.input = self.node_list[concat_output_node_id] new_bn_layer.output = self.node_list[block_last_layer_output_id] self.node_list[ block_last_layer_output_id].shape = new_bn_layer.output_shape if self.weighted: filters_end = self.layer_list[end_id].filters filters_start = self.layer_list[start_id].filters filter_shape = (1, ) * ( len(self.layer_list[end_id].get_weights()[0].shape) - 2) weights = np.zeros((filters_end, filters_end) + filter_shape) for i in range(filters_end): filter_weight = np.zeros((filters_end, ) + filter_shape) filter_weight[(i, 0, 0)] = 1 weights[i, ...] = filter_weight weights = np.concatenate( (weights, np.zeros((filters_end, filters_start) + filter_shape)), axis=1) bias = np.zeros(filters_end) new_conv_layer.set_weights((add_noise(weights, np.array([0, 1])), add_noise(bias, np.array([0, 1])))) n_filters = filters_end new_weights = [ add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])) ] new_bn_layer.set_weights(new_weights)
def to_add_skip_model(self, start_id, end_id): """Add a weighted add skip-connection from after start node to end node. Args: start_id: The convolutional layer ID, after which to start the skip-connection. end_id: The convolutional layer ID, after which to end the skip-connection. """ self.operation_history.append(('to_add_skip_model', start_id, end_id)) conv_block_input_id = self._conv_block_end_node(start_id) conv_block_input_id = self.adj_list[conv_block_input_id][0][0] block_last_layer_input_id = self._conv_block_end_node(end_id) # Add the pooling layer chain. layer_list = self._get_pooling_layers(conv_block_input_id, block_last_layer_input_id) skip_output_id = conv_block_input_id for index, layer_id in enumerate(layer_list): skip_output_id = self.add_layer( deepcopy(self.layer_list[layer_id]), skip_output_id) # Add the conv layer new_relu_layer = StubReLU() skip_output_id = self.add_layer(new_relu_layer, skip_output_id) new_conv_layer = StubConv(self.layer_list[start_id].filters, self.layer_list[end_id].filters, 1) skip_output_id = self.add_layer(new_conv_layer, skip_output_id) new_bn_layer = StubBatchNormalization(self.layer_list[end_id].filters) skip_output_id = self.add_layer(new_bn_layer, skip_output_id) # Add the add layer. block_last_layer_output_id = self.adj_list[block_last_layer_input_id][ 0][0] add_input_node_id = self._add_node( deepcopy(self.node_list[block_last_layer_output_id])) add_layer = StubAdd() self._redirect_edge(block_last_layer_input_id, block_last_layer_output_id, add_input_node_id) self._add_edge(add_layer, add_input_node_id, block_last_layer_output_id) self._add_edge(add_layer, skip_output_id, block_last_layer_output_id) add_layer.input = [ self.node_list[add_input_node_id], self.node_list[skip_output_id] ] add_layer.output = self.node_list[block_last_layer_output_id] self.node_list[ block_last_layer_output_id].shape = add_layer.output_shape # Set weights to the additional conv layer. if self.weighted: filters_end = self.layer_list[end_id].filters filters_start = self.layer_list[start_id].filters filter_shape = (1, ) * ( len(self.layer_list[end_id].get_weights()[0].shape) - 2) weights = np.zeros((filters_end, filters_start) + filter_shape) bias = np.zeros(filters_end) new_conv_layer.set_weights((add_noise(weights, np.array([0, 1])), add_noise(bias, np.array([0, 1])))) n_filters = filters_end new_weights = [ add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])) ] new_bn_layer.set_weights(new_weights)
def to_concat_skip_model(self, start_id, end_id): """Add a weighted add concatenate connection from after start node to end node. Args: start_id: The convolutional layer ID, after which to start the skip-connection. end_id: The convolutional layer ID, after which to end the skip-connection. """ self.operation_history.append(('to_concat_skip_model', start_id, end_id)) conv_block_input_id = self._conv_block_end_node(start_id) conv_block_input_id = self.adj_list[conv_block_input_id][0][0] block_last_layer_input_id = self._conv_block_end_node(end_id) # Add the pooling layer chain. pooling_layer_list = self._get_pooling_layers(conv_block_input_id, block_last_layer_input_id) skip_output_id = conv_block_input_id for index, layer_id in enumerate(pooling_layer_list): skip_output_id = self.add_layer(deepcopy(self.layer_list[layer_id]), skip_output_id) block_last_layer_output_id = self.adj_list[block_last_layer_input_id][0][0] concat_input_node_id = self._add_node(deepcopy(self.node_list[block_last_layer_output_id])) self._redirect_edge(block_last_layer_input_id, block_last_layer_output_id, concat_input_node_id) concat_layer = StubConcatenate() concat_layer.input = [self.node_list[concat_input_node_id], self.node_list[skip_output_id]] concat_output_node_id = self._add_node(Node(concat_layer.output_shape)) self._add_edge(concat_layer, concat_input_node_id, concat_output_node_id) self._add_edge(concat_layer, skip_output_id, concat_output_node_id) concat_layer.output = self.node_list[concat_output_node_id] self.node_list[concat_output_node_id].shape = concat_layer.output_shape # Add the concatenate layer. new_relu_layer = StubReLU() concat_output_node_id = self.add_layer(new_relu_layer, concat_output_node_id) new_conv_layer = StubConv(self.layer_list[start_id].filters + self.layer_list[end_id].filters, self.layer_list[end_id].filters, 1) concat_output_node_id = self.add_layer(new_conv_layer, concat_output_node_id) new_bn_layer = StubBatchNormalization(self.layer_list[end_id].filters) self._add_edge(new_bn_layer, concat_output_node_id, block_last_layer_output_id) new_bn_layer.input = self.node_list[concat_output_node_id] new_bn_layer.output = self.node_list[block_last_layer_output_id] self.node_list[block_last_layer_output_id].shape = new_bn_layer.output_shape if self.weighted: filters_end = self.layer_list[end_id].filters filters_start = self.layer_list[start_id].filters filter_shape = (1,) * (len(self.layer_list[end_id].get_weights()[0].shape) - 2) weights = np.zeros((filters_end, filters_end) + filter_shape) for i in range(filters_end): filter_weight = np.zeros((filters_end,) + filter_shape) filter_weight[(i, 0, 0)] = 1 weights[i, ...] = filter_weight weights = np.concatenate((weights, np.zeros((filters_end, filters_start) + filter_shape)), axis=1) bias = np.zeros(filters_end) new_conv_layer.set_weights((add_noise(weights, np.array([0, 1])), add_noise(bias, np.array([0, 1])))) n_filters = filters_end new_weights = [add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1]))] new_bn_layer.set_weights(new_weights)