def dense_to_deeper_block(dense_layer, weighted=True):
'''deeper dense layer.
'''
units = dense_layer.units
weight = np.eye(units)
bias = np.zeros(units)
new_dense_layer = StubDense(units, units)
if weighted:
new_dense_layer.set_weights(
(add_noise(weight,
np.array([0, 1])), add_noise(bias, np.array([0, 1]))))
return [StubReLU(), new_dense_layer]
def create_new_layer(layer, n_dim):
''' create new layer for the graph
'''
input_shape = layer.output.shape
dense_deeper_classes = [StubDense, get_dropout_class(n_dim), StubReLU]
conv_deeper_classes = [
get_conv_class(n_dim),
get_batch_norm_class(n_dim), StubReLU
]
if is_layer(layer, "ReLU"):
conv_deeper_classes = [
get_conv_class(n_dim),
get_batch_norm_class(n_dim)
]
dense_deeper_classes = [StubDense, get_dropout_class(n_dim)]
elif is_layer(layer, "Dropout"):
dense_deeper_classes = [StubDense, StubReLU]
elif is_layer(layer, "BatchNormalization"):
conv_deeper_classes = [get_conv_class(n_dim), StubReLU]
layer_class = None
if len(input_shape) == 1:
# It is in the dense layer part.
layer_class = sample(dense_deeper_classes, 1)[0]
else:
# It is in the conv layer part.
layer_class = sample(conv_deeper_classes, 1)[0]
if layer_class == StubDense:
new_layer = StubDense(input_shape[0], input_shape[0])
elif layer_class == get_dropout_class(n_dim):
new_layer = layer_class(Constant.DENSE_DROPOUT_RATE)
elif layer_class == get_conv_class(n_dim):
new_layer = layer_class(input_shape[-1],
input_shape[-1],
sample((1, 3, 5), 1)[0],
stride=1)
elif layer_class == get_batch_norm_class(n_dim):
new_layer = layer_class(input_shape[-1])
elif layer_class == get_pooling_class(n_dim):
new_layer = layer_class(sample((1, 3, 5), 1)[0])
else:
new_layer = layer_class()
return new_layer
def wider_next_dense(layer, start_dim, total_dim, n_add, weighted=True):
'''wider next dense layer.
'''
if not weighted:
return StubDense(layer.input_units + n_add, layer.units)
teacher_w, teacher_b = layer.get_weights()
student_w = teacher_w.copy()
n_units_each_channel = int(teacher_w.shape[1] / total_dim)
new_weight = np.zeros((teacher_w.shape[0], n_add * n_units_each_channel))
student_w = np.concatenate(
(
student_w[:, :start_dim * n_units_each_channel],
add_noise(new_weight, student_w),
student_w[:, start_dim * n_units_each_channel:total_dim *
n_units_each_channel],
),
axis=1,
)
new_layer = StubDense(layer.input_units + n_add, layer.units)
new_layer.set_weights((student_w, teacher_b))
return new_layer
def generate(self, model_len=None, model_width=None):
"""Generates a CNN.
Args:
model_len: An integer. Number of convolutional layers.
model_width: An integer. Number of filters for the convolutional layers.
Returns:
An instance of the class Graph. Represents the neural architecture graph of the generated model.
"""
if model_len is None:
model_len = Constant.MODEL_LEN
if model_width is None:
model_width = Constant.MODEL_WIDTH
pooling_len = int(model_len / 4)
graph = Graph(self.input_shape, False)
temp_input_channel = self.input_shape[-1]
output_node_id = 0
stride = 1
###
###
###conv_1 :7 2
output_node_id = graph.add_layer(
self.conv(temp_input_channel, 64, kernel_size=7, stride=2),
output_node_id)
output_node_id = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id].shape[-1]),
output_node_id)
output_node_id = graph.add_layer(StubReLU(), output_node_id)
output_node_id = graph.add_layer(self.pooling(kernel_size=3, stride=2),
output_node_id)
###
###
###conv_2
output_node_id1 = graph.add_layer(
self.conv(64, 64, kernel_size=1, stride=1), output_node_id)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(64, 64, kernel_size=3, stride=1), output_node_id1)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(64, 256, kernel_size=1, stride=1), output_node_id1)
output_node_id00 = graph.add_layer(
self.conv(64, 256, kernel_size=1, stride=1), output_node_id)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id00 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id00].shape[-1]),
output_node_id00)
output_node_id = graph.add_layer(StubAdd(),
[output_node_id1, output_node_id00])
output_node_id = graph.add_layer(StubReLU(), output_node_id)
###
output_node_id00 = output_node_id
output_node_id1 = graph.add_layer(
self.conv(256, 64, kernel_size=1, stride=1), output_node_id)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(64, 64, kernel_size=3, stride=1), output_node_id1)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(64, 256, kernel_size=1, stride=1), output_node_id1)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id = graph.add_layer(StubAdd(),
[output_node_id1, output_node_id00])
output_node_id = graph.add_layer(StubReLU(), output_node_id)
###
output_node_id00 = output_node_id
output_node_id1 = graph.add_layer(
self.conv(256, 64, kernel_size=1, stride=1), output_node_id)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(64, 64, kernel_size=3, stride=1), output_node_id1)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(64, 256, kernel_size=1, stride=1), output_node_id1)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id = graph.add_layer(StubAdd(),
[output_node_id1, output_node_id00])
output_node_id = graph.add_layer(StubReLU(), output_node_id)
###
###
###
###conv_3
output_node_id1 = graph.add_layer(
self.conv(256, 128, kernel_size=1, stride=2), output_node_id)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(128, 128, kernel_size=3, stride=1), output_node_id1)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(128, 512, kernel_size=1, stride=1), output_node_id1)
output_node_id00 = graph.add_layer(
self.conv(256, 512, kernel_size=1, stride=2), output_node_id)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id00 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id00].shape[-1]),
output_node_id00)
output_node_id = graph.add_layer(StubAdd(),
[output_node_id1, output_node_id00])
output_node_id = graph.add_layer(StubReLU(), output_node_id)
###
output_node_id00 = output_node_id
output_node_id1 = graph.add_layer(
self.conv(512, 128, kernel_size=1, stride=1), output_node_id)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(128, 128, kernel_size=3, stride=1), output_node_id1)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(128, 512, kernel_size=1, stride=1), output_node_id1)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id = graph.add_layer(StubAdd(),
[output_node_id1, output_node_id00])
output_node_id = graph.add_layer(StubReLU(), output_node_id)
###
output_node_id00 = output_node_id
output_node_id1 = graph.add_layer(
self.conv(512, 128, kernel_size=1, stride=1), output_node_id)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(128, 128, kernel_size=3, stride=1), output_node_id1)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(128, 512, kernel_size=1, stride=1), output_node_id1)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id = graph.add_layer(StubAdd(),
[output_node_id1, output_node_id00])
output_node_id = graph.add_layer(StubReLU(), output_node_id)
###
output_node_id00 = output_node_id
output_node_id1 = graph.add_layer(
self.conv(512, 128, kernel_size=1, stride=1), output_node_id)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(128, 128, kernel_size=3, stride=1), output_node_id1)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(128, 512, kernel_size=1, stride=1), output_node_id1)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id = graph.add_layer(StubAdd(),
[output_node_id1, output_node_id00])
output_node_id = graph.add_layer(StubReLU(), output_node_id)
###
###
###conv_4
output_node_id1 = graph.add_layer(
self.conv(512, 256, kernel_size=1, stride=2), output_node_id)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(256, 256, kernel_size=3, stride=1), output_node_id1)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(256, 1024, kernel_size=1, stride=1), output_node_id1)
output_node_id00 = graph.add_layer(
self.conv(512, 1024, kernel_size=1, stride=2), output_node_id)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id00 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id00].shape[-1]),
output_node_id00)
output_node_id = graph.add_layer(StubAdd(),
[output_node_id1, output_node_id00])
output_node_id = graph.add_layer(StubReLU(), output_node_id)
###
output_node_id00 = output_node_id
output_node_id1 = graph.add_layer(
self.conv(1024, 256, kernel_size=1, stride=1), output_node_id)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(256, 256, kernel_size=3, stride=1), output_node_id1)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(256, 1024, kernel_size=1, stride=1), output_node_id1)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id = graph.add_layer(StubAdd(),
[output_node_id1, output_node_id00])
output_node_id = graph.add_layer(StubReLU(), output_node_id)
###
output_node_id00 = output_node_id
output_node_id1 = graph.add_layer(
self.conv(1024, 256, kernel_size=1, stride=1), output_node_id)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(256, 256, kernel_size=3, stride=1), output_node_id1)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(256, 1024, kernel_size=1, stride=1),
output_node_id1,
)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id = graph.add_layer(StubAdd(),
[output_node_id1, output_node_id00])
output_node_id = graph.add_layer(StubReLU(), output_node_id)
###
output_node_id00 = output_node_id
output_node_id1 = graph.add_layer(
self.conv(1024, 256, kernel_size=1, stride=1), output_node_id)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(256, 256, kernel_size=3, stride=1),
output_node_id1,
)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(256, 1024, kernel_size=1, stride=1),
output_node_id1,
)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id = graph.add_layer(StubAdd(),
[output_node_id1, output_node_id00])
output_node_id = graph.add_layer(StubReLU(), output_node_id)
###
output_node_id00 = output_node_id
output_node_id1 = graph.add_layer(
self.conv(1024, 256, kernel_size=1, stride=1), output_node_id)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(256, 256, kernel_size=3, stride=1),
output_node_id1,
)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(256, 1024, kernel_size=1, stride=1),
output_node_id1,
)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id = graph.add_layer(StubAdd(),
[output_node_id1, output_node_id00])
output_node_id = graph.add_layer(StubReLU(), output_node_id)
###
output_node_id00 = output_node_id
output_node_id1 = graph.add_layer(
self.conv(1024, 256, kernel_size=1, stride=1), output_node_id)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(256, 256, kernel_size=3, stride=1),
output_node_id1,
)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(256, 1024, kernel_size=1, stride=1),
output_node_id1,
)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id = graph.add_layer(StubAdd(),
[output_node_id1, output_node_id00])
output_node_id = graph.add_layer(StubReLU(), output_node_id)
###
###
###conv_5
output_node_id1 = graph.add_layer(
self.conv(1024, 512, kernel_size=1, stride=2), output_node_id)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(512, 512, kernel_size=3, stride=1),
output_node_id1,
)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(512, 2048, kernel_size=1, stride=1),
output_node_id1,
)
output_node_id00 = graph.add_layer(
self.conv(1024, 2048, kernel_size=1, stride=2), output_node_id)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id00 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id00].shape[-1]),
output_node_id00)
output_node_id = graph.add_layer(StubAdd(),
[output_node_id1, output_node_id00])
output_node_id = graph.add_layer(StubReLU(), output_node_id)
###
output_node_id00 = output_node_id
output_node_id1 = graph.add_layer(
self.conv(2048, 512, kernel_size=1, stride=1), output_node_id)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(512, 512, kernel_size=3, stride=1),
output_node_id1,
)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(512, 2048, kernel_size=1, stride=1),
output_node_id1,
)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id = graph.add_layer(StubAdd(),
[output_node_id1, output_node_id00])
output_node_id = graph.add_layer(StubReLU(), output_node_id)
###
output_node_id00 = output_node_id
output_node_id1 = graph.add_layer(
self.conv(2048, 512, kernel_size=1, stride=1), output_node_id)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(512, 512, kernel_size=3, stride=1),
output_node_id1,
)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id1 = graph.add_layer(StubReLU(), output_node_id1)
output_node_id1 = graph.add_layer(
self.conv(512, 2048, kernel_size=1, stride=1),
output_node_id1,
)
output_node_id1 = graph.add_layer(
self.batch_norm(graph.node_list[output_node_id1].shape[-1]),
output_node_id1)
output_node_id = graph.add_layer(StubAdd(),
[output_node_id1, output_node_id00])
output_node_id = graph.add_layer(StubReLU(), output_node_id)
###
output_node_id = graph.add_layer(self.global_avg_pooling(),
output_node_id)
# output_node_id = graph.add_layer(
# self.dropout(Constant.CONV_DROPOUT_RATE), output_node_id
# )
# output_node_id = graph.add_layer(
# StubDense(graph.node_list[output_node_id].shape[0], model_width),
# output_node_id
# )
# output_node_id = graph.add_layer(StubReLU(), output_node_id)
graph.add_layer(StubDense(2048, self.n_output_node), output_node_id)
return graph