def create_new_layer(input_shape, n_dim):
dense_deeper_classes = [StubDense, get_dropout_class(n_dim), StubReLU]
conv_deeper_classes = [
get_conv_class(n_dim),
get_batch_norm_class(n_dim), StubReLU
]
if len(input_shape) == 1:
# It is in the dense layer part.
layer_class = sample(dense_deeper_classes, 1)[0]
else:
# It is in the conv layer part.
layer_class = sample(conv_deeper_classes, 1)[0]
if layer_class == StubDense:
new_layer = StubDense(input_shape[0], input_shape[0])
elif layer_class == get_dropout_class(n_dim):
new_layer = layer_class(Constant.DENSE_DROPOUT_RATE)
elif layer_class == get_conv_class(n_dim):
new_layer = layer_class(input_shape[-1],
input_shape[-1],
sample((1, 3, 5), 1)[0],
stride=1)
elif layer_class == get_batch_norm_class(n_dim):
new_layer = layer_class(input_shape[-1])
elif layer_class == get_pooling_class(n_dim):
new_layer = layer_class(sample((1, 3, 5), 1)[0])
else:
new_layer = layer_class()
return new_layer
def wider_pre_conv(layer, n_add_filters, weighted=True):
n_dim = get_n_dim(layer)
if not weighted:
return get_conv_class(n_dim)(layer.input_channel,
layer.filters + n_add_filters,
kernel_size=layer.kernel_size,
stride=layer.stride)
n_pre_filters = layer.filters
rand = np.random.randint(n_pre_filters, size=n_add_filters)
teacher_w, teacher_b = layer.get_weights()
student_w = teacher_w.copy()
student_b = teacher_b.copy()
# target layer update (i)
for i in range(len(rand)):
teacher_index = rand[i]
new_weight = teacher_w[teacher_index, ...]
new_weight = new_weight[np.newaxis, ...]
student_w = np.concatenate((student_w, new_weight), axis=0)
student_b = np.append(student_b, teacher_b[teacher_index])
new_pre_layer = get_conv_class(n_dim)(layer.input_channel,
n_pre_filters + n_add_filters,
kernel_size=layer.kernel_size,
stride=layer.stride)
new_pre_layer.set_weights((add_noise(student_w, teacher_w), add_noise(student_b, teacher_b)))
return new_pre_layer
def wider_pre_conv(layer, n_add_filters, weighted=True):
n_dim = get_n_dim(layer)
if not weighted:
return get_conv_class(n_dim)(layer.input_channel,
layer.filters + n_add_filters,
kernel_size=layer.kernel_size,
stride=layer.stride,
padding =layer.padding,
groups=layer.groups)
n_pre_filters = layer.filters
rand = np.random.randint(n_pre_filters, size=n_add_filters)
teacher_w, teacher_b = layer.get_weights()
student_w = teacher_w.copy()
student_b = teacher_b.copy()
# target layer update (i)
for i in range(len(rand)):
teacher_index = rand[i]
new_weight = teacher_w[teacher_index, ...]
new_weight = new_weight[np.newaxis, ...]
student_w = np.concatenate((student_w, new_weight), axis=0)
student_b = np.append(student_b, teacher_b[teacher_index])
new_pre_layer = get_conv_class(n_dim)(layer.input_channel,
n_pre_filters + n_add_filters,
kernel_size=layer.kernel_size,
stride=layer.stride,
padding =layer.padding,
groups=layer.groups)
new_pre_layer.set_weights((add_noise(student_w, teacher_w), add_noise(student_b, teacher_b)))
return new_pre_layer
def wider_next_conv(layer, start_dim, total_dim, n_add, weighted=True):
n_dim = get_n_dim(layer)
groups = layer.groups + n_add if layer.groups is not None and layer.groups > 1 else 1
if not weighted:
return get_conv_class(n_dim)(layer.input_channel + n_add,
layer.filters,
kernel_size=layer.kernel_size,
stride=layer.stride,
padding =layer.padding,
groups =groups)
n_filters = layer.filters
new_layer = get_conv_class(n_dim)(layer.input_channel + n_add,
n_filters,
kernel_size=layer.kernel_size,
stride=layer.stride,
padding =layer.padding,
groups =groups)
teacher_w, teacher_b = layer.get_weights()
if layer.groups is not None and layer.groups > 1:
new_layer.set_weights((teacher_w, teacher_b))
else:
new_weight_shape = list(teacher_w.shape)
new_weight_shape[1] = n_add
new_weight = np.zeros(tuple(new_weight_shape))
student_w = np.concatenate((teacher_w[:, :start_dim, ...].copy(),
add_noise(new_weight, teacher_w),
teacher_w[:, start_dim:total_dim, ...].copy()), axis=1)
new_layer.set_weights((student_w, teacher_b))
return new_layer
def deeper_conv_block(conv_layer, kernel_size, weighted=True):
n_dim = get_n_dim(conv_layer)
filter_shape = (kernel_size, ) * 2
n_filters = conv_layer.filters
weight = np.zeros((n_filters, n_filters) + filter_shape)
center = tuple(map(lambda x: int((x - 1) / 2), filter_shape))
for i in range(n_filters):
filter_weight = np.zeros((n_filters, ) + filter_shape)
index = (i, ) + center
filter_weight[index] = 1
weight[i, ...] = filter_weight
bias = np.zeros(n_filters)
new_conv_layer = get_conv_class(n_dim)(conv_layer.filters,
n_filters,
kernel_size=kernel_size)
bn = get_batch_norm_class(n_dim)(n_filters)
if weighted:
new_conv_layer.set_weights(
(add_noise(weight,
np.array([0, 1])), add_noise(bias, np.array([0, 1]))))
new_weights = [
add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])),
add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])),
add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])),
add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1]))
]
bn.set_weights(new_weights)
return [StubReLU(), new_conv_layer, bn]
def __init__(self, input_shape, weighted=True):
"""Initializer for Graph.
Args:
input_shape: A tuple describing the input tensor shape, not including the number of instances.
weighted: A boolean marking if there are actual values in the weights of the layers.
Sometime we only need the neural architecture information with a graph. In that case,
we do not save the weights to save memory and time.
"""
self.input_shape = input_shape
self.weighted = weighted
self.node_list = []
self.layer_list = []
# node id start with 0
self.node_to_id = {}
self.layer_to_id = {}
self.layer_id_to_input_node_ids = {}
self.layer_id_to_output_node_ids = {}
self.adj_list = {}
self.reverse_adj_list = {}
self.operation_history = []
self.n_dim = len(input_shape) - 1
self.conv = get_conv_class(self.n_dim)
self.batch_norm = get_batch_norm_class(self.n_dim)
self.vis = None
self._add_node(Node(input_shape))
def __init__(self, n_output_node, input_shape):
super(MobileNetV2Generator, self).__init__(n_output_node, input_shape)
""" configuration for complete net:
self.cfg = [(1, 16, 1, 1),
(6, 24, 2, 1) , # NOTE: change stride 2 -> 1 for CIFAR10
(6, 32, 3, 2),
(6, 64, 4, 2),
(6, 96, 3, 1),
(6, 160, 3, 2),
(6, 320, 1, 1)]
"""
# we try smaller net configuration (so autokeras will be able to expand the net)
self.cfg = [(1, 16, 1, 1),
(6, 24, 2, 1)] # , # NOTE: change stride 2 -> 1 for CIFAR10
#(6, 32, 3, 2) ,
#(6, 64, 4, 2),
#(6, 96, 3, 1),
#(6, 160, 3, 2),
#(6, 320, 1, 1)]
self.in_planes = 32
self.block_expansion = 1
self.n_dim = len(self.input_shape) - 1
if len(self.input_shape) > 4:
raise ValueError('The input dimension is too high.')
elif len(self.input_shape) < 2:
raise ValueError('The input dimension is too low.')
self.conv = get_conv_class(self.n_dim)
self.dropout = get_dropout_class(self.n_dim)
self.global_avg_pooling = get_global_avg_pooling_class(self.n_dim)
self.adaptive_avg_pooling = get_global_avg_pooling_class(self.n_dim)
self.batch_norm = get_batch_norm_class(self.n_dim)
def to_concat_skip_model(self, start_id, end_id):
"""Add a weighted add concatenate connection from after start node to end node.
Args:
start_id: The convolutional layer ID, after which to start the skip-connection.
end_id: The convolutional layer ID, after which to end the skip-connection.
"""
self.operation_history.append(
('to_concat_skip_model', start_id, end_id))
filters_end = self.layer_list[end_id].output.shape[-1]
filters_start = self.layer_list[start_id].output.shape[-1]
start_node_id = self.layer_id_to_output_node_ids[start_id][0]
pre_end_node_id = self.layer_id_to_input_node_ids[end_id][0]
end_node_id = self.layer_id_to_output_node_ids[end_id][0]
skip_output_id = self._insert_pooling_layer_chain(
start_node_id, end_node_id)
concat_input_node_id = self._add_node(
deepcopy(self.node_list[end_node_id]))
self._redirect_edge(pre_end_node_id, end_node_id, concat_input_node_id)
concat_layer = StubConcatenate()
concat_layer.input = [
self.node_list[concat_input_node_id],
self.node_list[skip_output_id]
]
concat_output_node_id = self._add_node(Node(concat_layer.output_shape))
self._add_edge(concat_layer, concat_input_node_id,
concat_output_node_id)
self._add_edge(concat_layer, skip_output_id, concat_output_node_id)
concat_layer.output = self.node_list[concat_output_node_id]
self.node_list[concat_output_node_id].shape = concat_layer.output_shape
# Add the concatenate layer.
new_conv_layer = get_conv_class(self.n_dim)(
filters_start + filters_end, filters_end, 1)
self._add_edge(new_conv_layer, concat_output_node_id, end_node_id)
new_conv_layer.input = self.node_list[concat_output_node_id]
new_conv_layer.output = self.node_list[end_node_id]
self.node_list[end_node_id].shape = new_conv_layer.output_shape
if self.weighted:
filter_shape = (1, ) * self.n_dim
weights = np.zeros((filters_end, filters_end) + filter_shape)
for i in range(filters_end):
filter_weight = np.zeros((filters_end, ) + filter_shape)
center_index = (i, ) + (0, ) * self.n_dim
filter_weight[center_index] = 1
weights[i, ...] = filter_weight
weights = np.concatenate(
(weights,
np.zeros((filters_end, filters_start) + filter_shape)),
axis=1)
bias = np.zeros(filters_end)
new_conv_layer.set_weights((add_noise(weights, np.array([0, 1])),
add_noise(bias, np.array([0, 1]))))
def _insert_pooling_layer_chain(self, start_node_id, end_node_id):
skip_output_id = start_node_id
for layer in self._get_pooling_layers(start_node_id, end_node_id):
new_layer = deepcopy(layer)
if is_layer(new_layer, 'Conv'):
filters = self.node_list[start_node_id].shape[-1]
new_layer = get_conv_class(self.n_dim)(filters, filters, 1, layer.stride)
else:
new_layer = deepcopy(layer)
skip_output_id = self.add_layer(new_layer, skip_output_id)
skip_output_id = self.add_layer(StubReLU(), skip_output_id)
return skip_output_id
def __init__(self, n_output_node, input_shape):
super(CnnGenerator, self).__init__(n_output_node, input_shape)
self.n_dim = len(self.input_shape) - 1
if len(self.input_shape) > 4:
raise ValueError('The input dimension is too high.')
if len(self.input_shape) < 2:
raise ValueError('The input dimension is too low.')
self.conv = get_conv_class(self.n_dim)
self.dropout = get_dropout_class(self.n_dim)
self.global_avg_pooling = get_global_avg_pooling_class(self.n_dim)
self.pooling = get_pooling_class(self.n_dim)
self.batch_norm = get_batch_norm_class(self.n_dim)
def wider_next_conv(layer, start_dim, total_dim, n_add, weighted=True):
n_dim = get_n_dim(layer)
if not weighted:
return get_conv_class(n_dim)(layer.input_channel + n_add,
layer.filters,
kernel_size=layer.kernel_size)
n_filters = layer.filters
teacher_w, teacher_b = layer.get_weights()
new_weight_shape = list(teacher_w.shape)
new_weight_shape[1] = n_add
new_weight = np.zeros(tuple(new_weight_shape))
student_w = np.concatenate(
(teacher_w[:, :start_dim, ...].copy(), add_noise(
new_weight, teacher_w), teacher_w[:, start_dim:total_dim,
...].copy()),
axis=1)
new_layer = get_conv_class(n_dim)(layer.input_channel + n_add, n_filters,
layer.kernel_size)
new_layer.set_weights((student_w, teacher_b))
return new_layer
def create_new_layer(layer, n_dim):
input_shape = layer.output.shape
dense_deeper_classes = [StubDense, get_dropout_class(n_dim), StubReLU]
conv_deeper_classes = [get_conv_class(n_dim), get_batch_norm_class(n_dim), StubReLU]
if is_layer(layer, LayerType.RELU):
conv_deeper_classes = [get_conv_class(n_dim), get_batch_norm_class(n_dim)]
dense_deeper_classes = [StubDense, get_dropout_class(n_dim)]
elif is_layer(layer, LayerType.DROPOUT):
dense_deeper_classes = [StubDense, StubReLU]
elif is_layer(layer, LayerType.BATCH_NORM):
conv_deeper_classes = [get_conv_class(n_dim), StubReLU]
if len(input_shape) == 1:
# It is in the dense layer part.
layer_class = sample(dense_deeper_classes, 1)[0]
else:
# It is in the conv layer part.
layer_class = sample(conv_deeper_classes, 1)[0]
if layer_class == StubDense:
new_layer = StubDense(input_shape[0], input_shape[0])
elif layer_class == get_dropout_class(n_dim):
new_layer = layer_class(Constant.DENSE_DROPOUT_RATE)
elif layer_class == get_conv_class(n_dim):
new_layer = layer_class(input_shape[-1], input_shape[-1], sample((1, 3, 5), 1)[0], stride=1)
elif layer_class == get_batch_norm_class(n_dim):
new_layer = layer_class(input_shape[-1])
elif layer_class == get_pooling_class(n_dim):
new_layer = layer_class(sample((1, 3, 5), 1)[0])
else:
new_layer = layer_class()
return new_layer
def wider_next_conv(layer, start_dim, total_dim, n_add, weighted=True):
n_dim = get_n_dim(layer)
if not weighted:
return get_conv_class(n_dim)(layer.input_channel + n_add,
layer.filters,
kernel_size=layer.kernel_size,
stride=layer.stride)
n_filters = layer.filters
teacher_w, teacher_b = layer.get_weights()
new_weight_shape = list(teacher_w.shape)
new_weight_shape[1] = n_add
new_weight = np.zeros(tuple(new_weight_shape))
student_w = np.concatenate((teacher_w[:, :start_dim, ...].copy(),
add_noise(new_weight, teacher_w),
teacher_w[:, start_dim:total_dim, ...].copy()), axis=1)
new_layer = get_conv_class(n_dim)(layer.input_channel + n_add,
n_filters,
kernel_size=layer.kernel_size,
stride=layer.stride)
new_layer.set_weights((student_w, teacher_b))
return new_layer
def __init__(self, n_output_node, input_shape):
super(ResNetGenerator, self).__init__(n_output_node, input_shape)
# self.layers = [2, 2, 2, 2]
self.in_planes = 64
self.block_expansion = 1
self.n_dim = len(self.input_shape) - 1
if len(self.input_shape) > 4:
raise ValueError('The input dimension is too high.')
elif len(self.input_shape) < 2:
raise ValueError('The input dimension is too low.')
self.conv = get_conv_class(self.n_dim)
self.dropout = get_dropout_class(self.n_dim)
self.global_avg_pooling = get_global_avg_pooling_class(self.n_dim)
self.adaptive_avg_pooling = get_global_avg_pooling_class(self.n_dim)
self.batch_norm = get_batch_norm_class(self.n_dim)
def to_concat_skip_model(self, start_id, end_id):
"""Add a weighted add concatenate connection from after start node to end node.
Args:
start_id: The convolutional layer ID, after which to start the skip-connection.
end_id: The convolutional layer ID, after which to end the skip-connection.
"""
self.operation_history.append(('to_concat_skip_model', start_id, end_id))
filters_end = self.layer_list[end_id].output.shape[-1]
filters_start = self.layer_list[start_id].output.shape[-1]
start_node_id = self.layer_id_to_output_node_ids[start_id][0]
pre_end_node_id = self.layer_id_to_input_node_ids[end_id][0]
end_node_id = self.layer_id_to_output_node_ids[end_id][0]
skip_output_id = self._insert_pooling_layer_chain(start_node_id, end_node_id)
concat_input_node_id = self._add_node(deepcopy(self.node_list[end_node_id]))
self._redirect_edge(pre_end_node_id, end_node_id, concat_input_node_id)
concat_layer = StubConcatenate()
concat_layer.input = [self.node_list[concat_input_node_id], self.node_list[skip_output_id]]
concat_output_node_id = self._add_node(Node(concat_layer.output_shape))
self._add_edge(concat_layer, concat_input_node_id, concat_output_node_id)
self._add_edge(concat_layer, skip_output_id, concat_output_node_id)
concat_layer.output = self.node_list[concat_output_node_id]
self.node_list[concat_output_node_id].shape = concat_layer.output_shape
# Add the concatenate layer.
new_conv_layer = get_conv_class(self.n_dim)(filters_start + filters_end,
filters_end, 1)
self._add_edge(new_conv_layer, concat_output_node_id, end_node_id)
new_conv_layer.input = self.node_list[concat_output_node_id]
new_conv_layer.output = self.node_list[end_node_id]
self.node_list[end_node_id].shape = new_conv_layer.output_shape
if self.weighted:
filter_shape = (1,) * self.n_dim
weights = np.zeros((filters_end, filters_end) + filter_shape)
for i in range(filters_end):
filter_weight = np.zeros((filters_end,) + filter_shape)
center_index = (i,) + (0,) * self.n_dim
filter_weight[center_index] = 1
weights[i, ...] = filter_weight
weights = np.concatenate((weights,
np.zeros((filters_end, filters_start) + filter_shape)), axis=1)
bias = np.zeros(filters_end)
new_conv_layer.set_weights((add_noise(weights, np.array([0, 1])), add_noise(bias, np.array([0, 1]))))
def __init__(self, n_output_node, input_shape):
super().__init__(n_output_node, input_shape)
# DenseNet Constant
self.num_init_features = 64
self.growth_rate = 32
self.block_config = (6, 12, 24, 16)
self.bn_size = 4
self.drop_rate = 0
# Stub layers
self.n_dim = len(self.input_shape) - 1
self.conv = get_conv_class(self.n_dim)
self.dropout = get_dropout_class(self.n_dim)
self.global_avg_pooling = get_global_avg_pooling_class(self.n_dim)
self.adaptive_avg_pooling = get_global_avg_pooling_class(self.n_dim)
self.max_pooling = get_pooling_class(self.n_dim)
self.avg_pooling = get_avg_pooling_class(self.n_dim)
self.batch_norm = get_batch_norm_class(self.n_dim)
def _insert_pooling_layer_chain(self, start_node_id, end_node_id):
skip_output_id = start_node_id
for layer in self._get_pooling_layers(start_node_id, end_node_id):
new_layer = deepcopy(layer)
if is_layer(new_layer, 'Conv'):
filters = self.node_list[start_node_id].shape[-1]
kernel_size = layer.kernel_size if layer.padding != int(
layer.kernel_size / 2) or layer.stride != 1 else 1
new_layer = get_conv_class(self.n_dim)(filters, filters, kernel_size, layer.stride,
padding=layer.padding)
if self.weighted:
init_conv_weight(new_layer)
else:
new_layer = deepcopy(layer)
skip_output_id = self.add_layer(new_layer, skip_output_id)
skip_output_id = self.add_layer(StubReLU(), skip_output_id)
return skip_output_id
def _insert_pooling_layer_chain(self, start_node_id, end_node_id):
skip_output_id = start_node_id
for layer in self._get_pooling_layers(start_node_id, end_node_id):
new_layer = deepcopy(layer)
if is_layer(new_layer, LayerType.CONV):
filters = self.node_list[start_node_id].shape[-1]
kernel_size = layer.kernel_size if layer.padding != int(
layer.kernel_size / 2) or layer.stride != 1 else 1
new_layer = get_conv_class(self.n_dim)(filters, filters, kernel_size, layer.stride,
padding=layer.padding)
if self.weighted:
init_conv_weight(new_layer)
else:
new_layer = deepcopy(layer)
skip_output_id = self.add_layer(new_layer, skip_output_id)
skip_output_id = self.add_layer(StubReLU(), skip_output_id)
return skip_output_id
def __init__(self, input_shape, weighted=True):
self.weighted = weighted
self.node_list = []
self.layer_list = []
# node id start with 0
self.node_to_id = {}
self.layer_to_id = {}
self.layer_id_to_input_node_ids = {}
self.layer_id_to_output_node_ids = {}
self.adj_list = {}
self.reverse_adj_list = {}
self.operation_history = []
self.n_dim = len(input_shape) - 1
self.conv = get_conv_class(self.n_dim)
self.batch_norm = get_batch_norm_class(self.n_dim)
self.vis = None
self._add_node(Node(input_shape))
def __init__(self, n_output_node, input_shape):
"""Initialize the instance.
Args:
n_output_node: An integer. Number of output nodes in the network.
input_shape: A tuple. Input shape of the network.
"""
super(CnnGenerator, self).__init__(n_output_node, input_shape)
self.n_dim = len(self.input_shape) - 1
if len(self.input_shape) > 4:
raise ValueError('The input dimension is too high.')
if len(self.input_shape) < 2:
raise ValueError('The input dimension is too low.')
self.conv = get_conv_class(self.n_dim)
self.dropout = get_dropout_class(self.n_dim)
self.global_avg_pooling = get_global_avg_pooling_class(self.n_dim)
self.pooling = get_pooling_class(self.n_dim)
self.batch_norm = get_batch_norm_class(self.n_dim)
def __init__(self, n_output_node, input_shape):
"""Initialize the instance.
Args:
n_output_node: An integer. Number of output nodes in the network.
input_shape: A tuple. Input shape of the network.
"""
super(CnnGenerator, self).__init__(n_output_node, input_shape)
self.n_dim = len(self.input_shape) - 1
if len(self.input_shape) > 4:
raise ValueError('The input dimension is too high.')
if len(self.input_shape) < 2:
raise ValueError('The input dimension is too low.')
self.conv = get_conv_class(self.n_dim)
self.dropout = get_dropout_class(self.n_dim)
self.global_avg_pooling = get_global_avg_pooling_class(self.n_dim)
self.pooling = get_pooling_class(self.n_dim)
self.batch_norm = get_batch_norm_class(self.n_dim)
def to_add_skip_model(self, start_id, end_id):
"""Add a weighted add skip-connection from after start node to end node.
Args:
start_id: The convolutional layer ID, after which to start the skip-connection.
end_id: The convolutional layer ID, after which to end the skip-connection.
"""
self.operation_history.append(('to_add_skip_model', start_id, end_id))
filters_end = self.layer_list[end_id].output.shape[-1]
filters_start = self.layer_list[start_id].output.shape[-1]
start_node_id = self.layer_id_to_output_node_ids[start_id][0]
pre_end_node_id = self.layer_id_to_input_node_ids[end_id][0]
end_node_id = self.layer_id_to_output_node_ids[end_id][0]
skip_output_id = self._insert_pooling_layer_chain(
start_node_id, end_node_id)
# Add the conv layer
new_conv_layer = get_conv_class(self.n_dim)(filters_start, filters_end,
1)
skip_output_id = self.add_layer(new_conv_layer, skip_output_id)
# Add the add layer.
add_input_node_id = self._add_node(
deepcopy(self.node_list[end_node_id]))
add_layer = StubAdd()
self._redirect_edge(pre_end_node_id, end_node_id, add_input_node_id)
self._add_edge(add_layer, add_input_node_id, end_node_id)
self._add_edge(add_layer, skip_output_id, end_node_id)
add_layer.input = [
self.node_list[add_input_node_id], self.node_list[skip_output_id]
]
add_layer.output = self.node_list[end_node_id]
self.node_list[end_node_id].shape = add_layer.output_shape
# Set weights to the additional conv layer.
if self.weighted:
filter_shape = (1, ) * self.n_dim
weights = np.zeros((filters_end, filters_start) + filter_shape)
bias = np.zeros(filters_end)
new_conv_layer.set_weights((add_noise(weights, np.array([0, 1])),
add_noise(bias, np.array([0, 1]))))
def to_add_skip_model(self, start_id, end_id):
"""Add a weighted add skip-connection from after start node to end node.
Args:
start_id: The convolutional layer ID, after which to start the skip-connection.
end_id: The convolutional layer ID, after which to end the skip-connection.
"""
self.operation_history.append(('to_add_skip_model', start_id, end_id))
filters_end = self.layer_list[end_id].output.shape[-1]
filters_start = self.layer_list[start_id].output.shape[-1]
start_node_id = self.layer_id_to_output_node_ids[start_id][0]
pre_end_node_id = self.layer_id_to_input_node_ids[end_id][0]
end_node_id = self.layer_id_to_output_node_ids[end_id][0]
skip_output_id = self._insert_pooling_layer_chain(start_node_id, end_node_id)
# Add the conv layer
new_conv_layer = get_conv_class(self.n_dim)(filters_start,
filters_end,
1)
skip_output_id = self.add_layer(new_conv_layer, skip_output_id)
# Add the add layer.
add_input_node_id = self._add_node(deepcopy(self.node_list[end_node_id]))
add_layer = StubAdd()
self._redirect_edge(pre_end_node_id, end_node_id, add_input_node_id)
self._add_edge(add_layer, add_input_node_id, end_node_id)
self._add_edge(add_layer, skip_output_id, end_node_id)
add_layer.input = [self.node_list[add_input_node_id], self.node_list[skip_output_id]]
add_layer.output = self.node_list[end_node_id]
self.node_list[end_node_id].shape = add_layer.output_shape
# Set weights to the additional conv layer.
if self.weighted:
filter_shape = (1,) * self.n_dim
weights = np.zeros((filters_end, filters_start) + filter_shape)
bias = np.zeros(filters_end)
new_conv_layer.set_weights((add_noise(weights, np.array([0, 1])), add_noise(bias, np.array([0, 1]))))
def create_new_layer(layer, n_dim):
input_shape = layer.output.shape
dense_deeper_classes = [StubDense, get_dropout_class(n_dim), StubReLU]
conv_deeper_classes = [
get_conv_class(n_dim),
get_batch_norm_class(n_dim), StubReLU
]
if is_layer(layer, LayerType.RELU):
conv_deeper_classes = [
get_conv_class(n_dim),
get_batch_norm_class(n_dim)
]
dense_deeper_classes = [StubDense, get_dropout_class(n_dim)]
elif is_layer(layer, LayerType.DROPOUT):
dense_deeper_classes = [StubDense, StubReLU]
elif is_layer(layer, LayerType.BATCH_NORM):
conv_deeper_classes = [get_conv_class(n_dim)] #, StubReLU]
new_layers = []
if len(input_shape) == 1:
# It is in the dense layer part.
layer_class = sample(dense_deeper_classes, 1)[0]
else:
# It is in the conv layer part.
layer_class = sample(conv_deeper_classes, 1)[0]
if layer_class == StubDense:
new_layer = StubDense(input_shape[0], input_shape[0])
new_layers.append(new_layer)
elif layer_class == get_dropout_class(n_dim):
new_layer = layer_class(Constant.DENSE_DROPOUT_RATE)
new_layers.append(new_layer)
elif layer_class == get_conv_class(n_dim):
# add conv layer
# new_layer = layer_class(input_shape[-1],, input_shape[-1], sample((1, 3, 5), 1)[0], stride=1)
# add mobilenet block
in_planes = input_shape[-1]
expansion = sample((1, 6), 1)[0]
stride = sample((1, 2), 1)[0]
planes = expansion * in_planes
new_layer = layer_class(in_planes, planes, 1, stride=1, padding=0)
new_layers.append(new_layer)
new_layer = get_batch_norm_class(n_dim)(planes)
new_layers.append(new_layer)
new_layer = StubReLU()
new_layers.append(new_layer)
new_layer = layer_class(planes,
planes,
3,
stride=stride,
padding=1,
groups=planes)
new_layers.append(new_layer)
new_layer = get_batch_norm_class(n_dim)(planes)
new_layers.append(new_layer)
new_layer = StubReLU()
new_layers.append(new_layer)
new_layer = layer_class(planes, in_planes, 1, stride=1, padding=0)
new_layers.append(new_layer)
new_layer = get_batch_norm_class(n_dim)(in_planes)
new_layers.append(new_layer)
elif layer_class == get_batch_norm_class(n_dim):
new_layer = layer_class(input_shape[-1])
new_layers.append(new_layer)
elif layer_class == get_pooling_class(n_dim):
new_layer = layer_class(sample((1, 3, 5), 1)[0])
new_layers.append(new_layer)
else:
new_layer = layer_class()
new_layers.append(new_layer)
return new_layers
