def generate(self,
model_len=Constant.MLP_MODEL_LEN,
model_width=Constant.MLP_MODEL_WIDTH):
"""Generates a Multi-Layer Perceptron.
Args:
model_len: An integer. Number of hidden layers.
model_width: An integer or a list of integers of length `model_len`. If it is a list, it represents the
number of nodes in each hidden layer. If it is an integer, all hidden layers have nodes equal to this
value.
Returns:
An instance of the class Graph. Represents the neural architecture graph of the generated model.
"""
if type(model_width) is list and not len(model_width) == model_len:
raise ValueError(
'The length of \'model_width\' does not match \'model_len\'')
elif type(model_width) is int:
model_width = [model_width] * model_len
graph = Graph(self.input_shape, False)
output_node_id = 0
n_nodes_prev_layer = self.input_shape[0]
for width in model_width:
output_node_id = graph.add_layer(
StubDense(n_nodes_prev_layer, width), output_node_id)
output_node_id = graph.add_layer(
StubDropout1d(Constant.MLP_DROPOUT_RATE), output_node_id)
output_node_id = graph.add_layer(StubReLU(), output_node_id)
n_nodes_prev_layer = width
graph.add_layer(StubDense(n_nodes_prev_layer, self.n_output_node),
output_node_id)
return graph
def generate(self,
model_len=Constant.MODEL_LEN,
model_width=Constant.MODEL_WIDTH):
pooling_len = int(model_len / 4)
graph = Graph(self.input_shape, False)
temp_input_channel = self.input_shape[-1]
output_node_id = 0
for i in range(model_len):
output_node_id = graph.add_layer(StubReLU(), output_node_id)
output_node_id = graph.add_layer(
StubConv(temp_input_channel, model_width, kernel_size=3),
output_node_id)
output_node_id = graph.add_layer(
StubBatchNormalization(model_width), output_node_id)
temp_input_channel = model_width
if pooling_len == 0 or ((i + 1) % pooling_len == 0
and i != model_len - 1):
output_node_id = graph.add_layer(StubPooling(), output_node_id)
output_node_id = graph.add_layer(StubGlobalPooling(), output_node_id)
output_node_id = graph.add_layer(
StubDropout(Constant.CONV_DROPOUT_RATE), output_node_id)
output_node_id = graph.add_layer(
StubDense(graph.node_list[output_node_id].shape[0], model_width),
output_node_id)
output_node_id = graph.add_layer(StubReLU(), output_node_id)
graph.add_layer(StubDense(model_width, self.n_output_node),
output_node_id)
return graph
def wider_pre_dense(layer, n_add, weighted=True):
if not weighted:
return StubDense(layer.input_units, layer.units + n_add)
n_units2 = layer.units
teacher_w, teacher_b = layer.get_weights()
rand = np.random.randint(n_units2, size=n_add)
student_w = teacher_w.copy()
student_b = teacher_b.copy()
# target layer update (i)
for i in range(n_add):
teacher_index = rand[i]
new_weight = teacher_w[teacher_index, :]
new_weight = new_weight[np.newaxis, :]
student_w = np.concatenate(
(student_w, add_noise(new_weight, student_w)), axis=0)
student_b = np.append(student_b,
add_noise(teacher_b[teacher_index], student_b))
new_pre_layer = StubDense(layer.input_units, n_units2 + n_add)
new_pre_layer.set_weights((student_w, student_b))
return new_pre_layer
def generate(self, model_len=Constant.MODEL_LEN, model_width=Constant.MODEL_WIDTH):
"""Generates a CNN.
Args:
model_len: An integer. Number of convolutional layers.
model_width: An integer. Number of filters for the convolutional layers.
Returns:
An instance of the class Graph. Represents the neural architecture graph of the generated model.
"""
pooling_len = int(model_len / 4)
graph = Graph(self.input_shape, False)
temp_input_channel = self.input_shape[-1]
output_node_id = 0
stride = 1
for i in range(model_len):
output_node_id = graph.add_layer(StubReLU(), output_node_id)
output_node_id = graph.add_layer(self.batch_norm(graph.node_list[output_node_id].shape[-1]), output_node_id)
output_node_id = graph.add_layer(self.conv(temp_input_channel,
model_width,
kernel_size=3,
stride=stride), output_node_id)
# if stride == 1:
# stride = 2
temp_input_channel = model_width
if pooling_len == 0 or ((i + 1) % pooling_len == 0 and i != model_len - 1):
output_node_id = graph.add_layer(self.pooling(), output_node_id)
output_node_id = graph.add_layer(self.global_avg_pooling(), output_node_id)
output_node_id = graph.add_layer(self.dropout(Constant.CONV_DROPOUT_RATE), output_node_id)
output_node_id = graph.add_layer(StubDense(graph.node_list[output_node_id].shape[0], model_width),
output_node_id)
output_node_id = graph.add_layer(StubReLU(), output_node_id)
graph.add_layer(StubDense(model_width, self.n_output_node), output_node_id)
return graph
def get_conv_dense_model():
graph = Graph((32, 32, 3), False)
output_node_id = 0
output_node_id = graph.add_layer(StubReLU(), output_node_id)
output_node_id = graph.add_layer(StubConv2d(3, 3, 3), output_node_id)
output_node_id = graph.add_layer(StubBatchNormalization2d(3),
output_node_id)
output_node_id = graph.add_layer(StubReLU(), output_node_id)
output_node_id = graph.add_layer(StubConv2d(3, 3, 3), output_node_id)
output_node_id = graph.add_layer(StubBatchNormalization2d(3),
output_node_id)
output_node_id = graph.add_layer(StubFlatten(), output_node_id)
output_node_id = graph.add_layer(
StubDropout2d(Constant.DENSE_DROPOUT_RATE), output_node_id)
output_node_id = graph.add_layer(StubReLU(), output_node_id)
output_node_id = graph.add_layer(
StubDense(graph.node_list[output_node_id].shape[0], 5), output_node_id)
output_node_id = graph.add_layer(StubReLU(), output_node_id)
output_node_id = graph.add_layer(StubDense(5, 5), output_node_id)
graph.add_layer(StubSoftmax(), output_node_id)
graph.produce_model().set_weight_to_graph()
return graph
def dense_to_deeper_block(dense_layer, weighted=True):
units = dense_layer.units
weight = np.eye(units)
bias = np.zeros(units)
new_dense_layer = StubDense(units, units)
if weighted:
new_dense_layer.set_weights((add_noise(weight, np.array([0, 1])), add_noise(bias, np.array([0, 1]))))
return [StubReLU(), new_dense_layer]
def get_concat_skip_model():
graph = Graph((32, 32, 3), False)
output_node_id = 0
output_node_id = graph.add_layer(StubReLU(), output_node_id)
output_node_id = graph.add_layer(StubConv(3, 3, 3), output_node_id)
output_node_id = graph.add_layer(StubBatchNormalization(3), output_node_id)
output_node_id = graph.add_layer(StubReLU(), output_node_id)
output_node_id = graph.add_layer(StubConv(3, 3, 3), output_node_id)
output_node_id = graph.add_layer(StubBatchNormalization(3), output_node_id)
temp_node_id = output_node_id
output_node_id = graph.add_layer(StubReLU(), output_node_id)
output_node_id = graph.add_layer(StubConv(3, 3, 3), output_node_id)
output_node_id = graph.add_layer(StubBatchNormalization(3), output_node_id)
output_node_id = graph.add_layer(StubConcatenate(),
[output_node_id, temp_node_id])
output_node_id = graph.add_layer(StubReLU(), output_node_id)
output_node_id = graph.add_layer(StubConv(6, 3, 1), output_node_id)
output_node_id = graph.add_layer(StubBatchNormalization(3), output_node_id)
temp_node_id = output_node_id
output_node_id = graph.add_layer(StubReLU(), output_node_id)
output_node_id = graph.add_layer(StubConv(3, 3, 3), output_node_id)
output_node_id = graph.add_layer(StubBatchNormalization(3), output_node_id)
output_node_id = graph.add_layer(StubConcatenate(),
[output_node_id, temp_node_id])
output_node_id = graph.add_layer(StubReLU(), output_node_id)
output_node_id = graph.add_layer(StubConv(6, 3, 1), output_node_id)
output_node_id = graph.add_layer(StubBatchNormalization(3), output_node_id)
output_node_id = graph.add_layer(StubReLU(), output_node_id)
output_node_id = graph.add_layer(StubConv(3, 3, 3), output_node_id)
output_node_id = graph.add_layer(StubBatchNormalization(3), output_node_id)
output_node_id = graph.add_layer(StubFlatten(), output_node_id)
output_node_id = graph.add_layer(StubDropout(Constant.CONV_DROPOUT_RATE),
output_node_id)
output_node_id = graph.add_layer(StubReLU(), output_node_id)
output_node_id = graph.add_layer(
StubDense(graph.node_list[output_node_id].shape[0], 5), output_node_id)
output_node_id = graph.add_layer(StubReLU(), output_node_id)
output_node_id = graph.add_layer(StubDense(5, 5), output_node_id)
graph.add_layer(StubSoftmax(), output_node_id)
graph.produce_model().set_weight_to_graph()
return graph
def generate(self, model_len=None, model_width=None):
graph = Graph(self.input_shape, False)
temp_input_channel = self.input_shape[-1]
output_node_id = 0
#out_planes = self.in_planes*self.n_output_node
out_planes = 24
output_node_id = graph.add_layer(self.conv(temp_input_channel,
self.in_planes,
kernel_size=3,
stride=1,
padding=1), output_node_id)
output_node_id = graph.add_layer(self.batch_norm(graph.node_list[output_node_id].shape[-1]), output_node_id)
output_node_id = graph.add_layer(StubReLU(), output_node_id)
output_node_id = self._make_layer(graph, output_node_id)
output_node_id = graph.add_layer(self.conv(out_planes,
out_planes*4,
kernel_size=1,
stride=1,
padding=0), output_node_id)
output_node_id = graph.add_layer(self.batch_norm(graph.node_list[output_node_id].shape[-1]), output_node_id)
output_node_id = graph.add_layer(StubReLU(), output_node_id)
output_node_id = graph.add_layer(self.global_avg_pooling(), output_node_id)
graph.add_layer(StubDense(out_planes*4, self.n_output_node), output_node_id)
return graph
def create_new_layer(input_shape, n_dim):
dense_deeper_classes = [StubDense, get_dropout_class(n_dim), StubReLU]
conv_deeper_classes = [
get_conv_class(n_dim),
get_batch_norm_class(n_dim), StubReLU
]
if len(input_shape) == 1:
# It is in the dense layer part.
layer_class = sample(dense_deeper_classes, 1)[0]
else:
# It is in the conv layer part.
layer_class = sample(conv_deeper_classes, 1)[0]
if layer_class == StubDense:
new_layer = StubDense(input_shape[0], input_shape[0])
elif layer_class == get_dropout_class(n_dim):
new_layer = layer_class(Constant.DENSE_DROPOUT_RATE)
elif layer_class == get_conv_class(n_dim):
new_layer = layer_class(input_shape[-1],
input_shape[-1],
sample((1, 3, 5), 1)[0],
stride=1)
elif layer_class == get_batch_norm_class(n_dim):
new_layer = layer_class(input_shape[-1])
elif layer_class == get_pooling_class(n_dim):
new_layer = layer_class(sample((1, 3, 5), 1)[0])
else:
new_layer = layer_class()
return new_layer
def wider_next_dense(layer, start_dim, total_dim, n_add, weighted=True):
if not weighted:
return StubDense(layer.input_units + n_add, layer.units)
teacher_w, teacher_b = layer.get_weights()
student_w = teacher_w.copy()
n_units_each_channel = int(teacher_w.shape[1] / total_dim)
new_weight = np.zeros((teacher_w.shape[0], n_add * n_units_each_channel))
student_w = np.concatenate((student_w[:, :start_dim * n_units_each_channel],
add_noise(new_weight, student_w),
student_w[:, start_dim * n_units_each_channel:total_dim * n_units_each_channel]),
axis=1)
new_layer = StubDense(layer.input_units + n_add, layer.units)
new_layer.set_weights((student_w, teacher_b))
return new_layer
def wider_next_dense(layer, start_dim, total_dim, n_add, weighted=True):
if not weighted:
return StubDense(layer.input_units + n_add, layer.units)
teacher_w, teacher_b = layer.get_weights()
student_w = teacher_w.copy()
n_units_each_channel = int(teacher_w.shape[1] / total_dim)
new_weight = np.zeros((teacher_w.shape[0], n_add * n_units_each_channel))
student_w = np.concatenate((student_w[:, :start_dim * n_units_each_channel],
add_noise(new_weight, student_w),
student_w[:, start_dim * n_units_each_channel:total_dim * n_units_each_channel]),
axis=1)
new_layer = StubDense(layer.input_units + n_add, layer.units)
new_layer.set_weights((student_w, teacher_b))
return new_layer
def generate(self, model_len=None, model_width=None):
if model_len is None:
model_len = Constant.MODEL_LEN
if model_width is None:
model_width = Constant.MODEL_WIDTH
graph = Graph(self.input_shape, False)
temp_input_channel = self.input_shape[-1]
# First convolution
output_node_id = 0
output_node_id = graph.add_layer(self.conv(temp_input_channel, model_width, kernel_size=7),
output_node_id)
output_node_id = graph.add_layer(self.batch_norm(num_features=self.num_init_features), output_node_id)
output_node_id = graph.add_layer(StubReLU(), output_node_id)
db_input_node_id = graph.add_layer(self.max_pooling(kernel_size=3, stride=2, padding=1), output_node_id)
# Each DensebLock
num_features = self.num_init_features
for i, num_layers in enumerate(self.block_config):
db_input_node_id = self._dense_block(num_layers=num_layers, num_input_features=num_features,
bn_size=self.bn_size, growth_rate=self.growth_rate,
drop_rate=self.drop_rate,
graph=graph, input_node_id=db_input_node_id)
num_features = num_features + num_layers * self.growth_rate
if i != len(self.block_config) - 1:
db_input_node_id = self._transition(num_input_features=num_features,
num_output_features=num_features // 2,
graph=graph, input_node_id=db_input_node_id)
num_features = num_features // 2
# Final batch norm
out = graph.add_layer(self.batch_norm(num_features), db_input_node_id)
out = graph.add_layer(StubReLU(), out)
out = graph.add_layer(self.adaptive_avg_pooling(), out)
# Linear layer
graph.add_layer(StubDense(num_features, self.n_output_node), out)
return graph
def generate(self, model_len=None, model_width=None):
if model_width is None:
model_width = Constant.MODEL_WIDTH
graph = Graph(self.input_shape, False)
temp_input_channel = self.input_shape[-1]
output_node_id = 0
# output_node_id = graph.add_layer(StubReLU(), output_node_id)
output_node_id = graph.add_layer(
self.conv(temp_input_channel, model_width, kernel_size=3),
output_node_id)
output_node_id = graph.add_layer(self.batch_norm(model_width),
output_node_id)
# output_node_id = graph.add_layer(self.pooling(kernel_size=3, stride=2, padding=1), output_node_id)
output_node_id = self._make_layer(graph, model_width, 2,
output_node_id, 1)
model_width *= 2
output_node_id = self._make_layer(graph, model_width, 2,
output_node_id, 2)
model_width *= 2
output_node_id = self._make_layer(graph, model_width, 2,
output_node_id, 2)
model_width *= 2
output_node_id = self._make_layer(graph, model_width, 2,
output_node_id, 2)
output_node_id = graph.add_layer(self.global_avg_pooling(),
output_node_id)
graph.add_layer(
StubDense(model_width * self.block_expansion, self.n_output_node),
output_node_id)
return graph
def generate(self, model_len=Constant.MLP_MODEL_LEN, model_width=Constant.MLP_MODEL_WIDTH):
if type(model_width) is list and not len(model_width) == model_len:
raise ValueError('The length of \'model_width\' does not match \'model_len\'')
elif type(model_width) is int:
model_width = [model_width] * model_len
graph = Graph(self.input_shape, False)
output_node_id = 0
n_nodes_prev_layer = self.input_shape[0]
for width in model_width:
output_node_id = graph.add_layer(StubDense(n_nodes_prev_layer, width), output_node_id)
output_node_id = graph.add_layer(StubDropout1d(Constant.MLP_DROPOUT_RATE), output_node_id)
output_node_id = graph.add_layer(StubReLU(), output_node_id)
n_nodes_prev_layer = width
graph.add_layer(StubDense(n_nodes_prev_layer, self.n_output_node), output_node_id)
return graph
def wider_pre_dense(layer, n_add, weighted=True):
if not weighted:
return StubDense(layer.input_units, layer.units + n_add)
n_units2 = layer.units
teacher_w, teacher_b = layer.get_weights()
rand = np.random.randint(n_units2, size=n_add)
student_w = teacher_w.copy()
student_b = teacher_b.copy()
# target layer update (i)
for i in range(n_add):
teacher_index = rand[i]
new_weight = teacher_w[teacher_index, :]
new_weight = new_weight[np.newaxis, :]
student_w = np.concatenate((student_w, add_noise(new_weight, student_w)), axis=0)
student_b = np.append(student_b, add_noise(teacher_b[teacher_index], student_b))
new_pre_layer = StubDense(layer.input_units, n_units2 + n_add)
new_pre_layer.set_weights((student_w, student_b))
return new_pre_layer
def generate(self, model_len, model_width):
graph = Graph(self.input_shape, False)
temp_input_channel = self.input_shape[-1]
output_node_id = 0
output_node_id = graph.add_layer(StubReLU(), output_node_id)
output_node_id = graph.add_layer(
self.conv(temp_input_channel, model_width, kernel_size=7),
output_node_id)
output_node_id = graph.add_layer(self.batch_norm(model_width),
output_node_id)
output_node_id = graph.add_layer(
self.pooling(kernel_size=3, stride=2, padding=1), output_node_id)
for layer in self.layers:
output_node_id = self._make_layer(graph, model_width, layer,
output_node_id)
model_width *= 2
output_node_id = graph.add_layer(self.global_avg_pooling(),
output_node_id)
graph.add_layer(
StubDense(
int(model_width / 2) * self.block_expansion,
self.n_output_node), output_node_id)
return graph
def create_new_layer(layer, n_dim):
input_shape = layer.output.shape
dense_deeper_classes = [StubDense, get_dropout_class(n_dim), StubReLU]
conv_deeper_classes = [
get_conv_class(n_dim),
get_batch_norm_class(n_dim), StubReLU
]
if is_layer(layer, LayerType.RELU):
conv_deeper_classes = [
get_conv_class(n_dim),
get_batch_norm_class(n_dim)
]
dense_deeper_classes = [StubDense, get_dropout_class(n_dim)]
elif is_layer(layer, LayerType.DROPOUT):
dense_deeper_classes = [StubDense, StubReLU]
elif is_layer(layer, LayerType.BATCH_NORM):
conv_deeper_classes = [get_conv_class(n_dim)] #, StubReLU]
new_layers = []
if len(input_shape) == 1:
# It is in the dense layer part.
layer_class = sample(dense_deeper_classes, 1)[0]
else:
# It is in the conv layer part.
layer_class = sample(conv_deeper_classes, 1)[0]
if layer_class == StubDense:
new_layer = StubDense(input_shape[0], input_shape[0])
new_layers.append(new_layer)
elif layer_class == get_dropout_class(n_dim):
new_layer = layer_class(Constant.DENSE_DROPOUT_RATE)
new_layers.append(new_layer)
elif layer_class == get_conv_class(n_dim):
# add conv layer
# new_layer = layer_class(input_shape[-1],, input_shape[-1], sample((1, 3, 5), 1)[0], stride=1)
# add mobilenet block
in_planes = input_shape[-1]
expansion = sample((1, 6), 1)[0]
stride = sample((1, 2), 1)[0]
planes = expansion * in_planes
new_layer = layer_class(in_planes, planes, 1, stride=1, padding=0)
new_layers.append(new_layer)
new_layer = get_batch_norm_class(n_dim)(planes)
new_layers.append(new_layer)
new_layer = StubReLU()
new_layers.append(new_layer)
new_layer = layer_class(planes,
planes,
3,
stride=stride,
padding=1,
groups=planes)
new_layers.append(new_layer)
new_layer = get_batch_norm_class(n_dim)(planes)
new_layers.append(new_layer)
new_layer = StubReLU()
new_layers.append(new_layer)
new_layer = layer_class(planes, in_planes, 1, stride=1, padding=0)
new_layers.append(new_layer)
new_layer = get_batch_norm_class(n_dim)(in_planes)
new_layers.append(new_layer)
elif layer_class == get_batch_norm_class(n_dim):
new_layer = layer_class(input_shape[-1])
new_layers.append(new_layer)
elif layer_class == get_pooling_class(n_dim):
new_layer = layer_class(sample((1, 3, 5), 1)[0])
new_layers.append(new_layer)
else:
new_layer = layer_class()
new_layers.append(new_layer)
return new_layers
