def deep_layer_ids(self):
ret = []
for layer_id in self.get_main_chain_layers():
layer = self.layer_list[layer_id]
if is_layer(layer, LayerType.GLOBAL_POOL):
break
if is_layer(layer, LayerType.ADD) or is_layer(layer, LayerType.CONCAT):
continue
ret.append(layer_id)
return ret
def deep_layer_ids(self):
ret = []
for layer_id in self.get_main_chain_layers():
layer = self.layer_list[layer_id]
if is_layer(layer, LayerType.GLOBAL_POOL):
break
if is_layer(layer, LayerType.ADD) or is_layer(layer, LayerType.CONCAT):
continue
ret.append(layer_id)
return ret
def deep_layer_ids(self):
ret = []
for layer_id in self.get_main_chain_layers():
layer = self.layer_list[layer_id]
if is_layer(layer, 'GlobalAveragePooling'):
break
if is_layer(layer, 'Add') or is_layer(layer, 'Concatenate'):
continue
ret.append(layer_id)
return ret
def _upper_layer_width(self, u):
for v, layer_id in self.reverse_adj_list[u]:
layer = self.layer_list[layer_id]
if is_layer(layer, LayerType.CONV) or is_layer(layer, LayerType.DENSE):
return layer_width(layer)
elif is_layer(layer, LayerType.CONCAT):
a = self.layer_id_to_input_node_ids[layer_id][0]
b = self.layer_id_to_input_node_ids[layer_id][1]
return self._upper_layer_width(a) + self._upper_layer_width(b)
else:
return self._upper_layer_width(v)
return self.node_list[0].shape[-1]
def _upper_layer_width(self, u):
for v, layer_id in self.reverse_adj_list[u]:
layer = self.layer_list[layer_id]
if is_layer(layer, LayerType.CONV) or is_layer(layer, LayerType.DENSE):
return layer_width(layer)
elif is_layer(layer, LayerType.CONCAT):
a = self.layer_id_to_input_node_ids[layer_id][0]
b = self.layer_id_to_input_node_ids[layer_id][1]
return self._upper_layer_width(a) + self._upper_layer_width(b)
else:
return self._upper_layer_width(v)
return self.node_list[0].shape[-1]
def _upper_layer_width(self, u):
for v, layer_id in self.reverse_adj_list[u]:
layer = self.layer_list[layer_id]
if is_layer(layer, 'Conv') or is_layer(layer, 'Dense'):
return layer_width(layer)
elif is_layer(layer, 'Concatenate'):
a = self.layer_id_to_input_node_ids[layer_id][0]
b = self.layer_id_to_input_node_ids[layer_id][1]
return self._upper_layer_width(a) + self._upper_layer_width(b)
else:
return self._upper_layer_width(v)
return self.node_list[0].shape[-1]
def _get_pooling_layers(self, start_node_id, end_node_id):
"""Given two node IDs, return all the pooling layers between them."""
layer_list = []
node_list = [start_node_id]
assert self._depth_first_search(end_node_id, layer_list, node_list)
ret = []
for layer_id in layer_list:
layer = self.layer_list[layer_id]
if is_layer(layer, 'Pooling'):
ret.append(layer)
elif is_layer(layer, 'Conv') and layer.stride != 1:
ret.append(layer)
return ret
def layer_distance(a, b):
"""The distance between two layers."""
if type(a) != type(b):
return 1.0
if is_layer(a, LayerType.CONV):
att_diff = [(a.filters, b.filters), (a.kernel_size, b.kernel_size),
(a.stride, b.stride)]
return attribute_difference(att_diff)
if is_layer(a, LayerType.POOL):
att_diff = [(a.padding, b.padding), (a.kernel_size, b.kernel_size),
(a.stride, b.stride)]
return attribute_difference(att_diff)
return 0.0
def _get_pooling_layers(self, start_node_id, end_node_id):
"""Given two node IDs, return all the pooling layers between them."""
layer_list = []
node_list = [start_node_id]
assert self._depth_first_search(end_node_id, layer_list, node_list)
ret = []
for layer_id in layer_list:
layer = self.layer_list[layer_id]
if is_layer(layer, LayerType.POOL):
ret.append(layer)
elif is_layer(layer, LayerType.CONV) and (layer.stride != 1 or layer.padding != int(layer.kernel_size / 2)):
ret.append(layer)
return ret
def _get_pooling_layers(self, start_node_id, end_node_id):
"""Given two node IDs, return all the pooling layers between them."""
layer_list = []
node_list = [start_node_id]
assert self._depth_first_search(end_node_id, layer_list, node_list)
ret = []
for layer_id in layer_list:
layer = self.layer_list[layer_id]
if is_layer(layer, LayerType.POOL):
ret.append(layer)
elif is_layer(layer, LayerType.CONV) and (layer.stride != 1 or layer.padding != int(layer.kernel_size / 2)):
ret.append(layer)
return ret
def layer_distance(a, b):
"""The distance between two layers."""
if type(a) != type(b):
return 1.0
if is_layer(a, LayerType.CONV):
att_diff = [(a.filters, b.filters),
(a.kernel_size, b.kernel_size),
(a.stride, b.stride)]
return attribute_difference(att_diff)
if is_layer(a, LayerType.POOL):
att_diff = [(a.padding, b.padding),
(a.kernel_size, b.kernel_size),
(a.stride, b.stride)]
return attribute_difference(att_diff)
return 0.0
def extract_descriptor(self):
"""Extract the the description of the Graph as an instance of NetworkDescriptor."""
main_chain = self.get_main_chain()
index_in_main_chain = {}
for index, u in enumerate(main_chain):
index_in_main_chain[u] = index
ret = NetworkDescriptor()
for u in main_chain:
for v, layer_id in self.adj_list[u]:
if v not in index_in_main_chain:
continue
layer = self.layer_list[layer_id]
copied_layer = copy(layer)
copied_layer.weights = None
ret.add_layer(deepcopy(copied_layer))
for u in index_in_main_chain:
for v, layer_id in self.adj_list[u]:
if v not in index_in_main_chain:
temp_u = u
temp_v = v
temp_layer_id = layer_id
skip_type = None
while not (temp_v in index_in_main_chain
and temp_u in index_in_main_chain):
if is_layer(self.layer_list[temp_layer_id],
LayerType.CONCAT):
skip_type = NetworkDescriptor.CONCAT_CONNECT
if is_layer(self.layer_list[temp_layer_id],
LayerType.ADD):
skip_type = NetworkDescriptor.ADD_CONNECT
temp_u = temp_v
temp_v, temp_layer_id = self.adj_list[temp_v][0]
ret.add_skip_connection(index_in_main_chain[u],
index_in_main_chain[temp_u],
skip_type)
elif index_in_main_chain[v] - index_in_main_chain[u] != 1:
skip_type = None
if is_layer(self.layer_list[layer_id], LayerType.CONCAT):
skip_type = NetworkDescriptor.CONCAT_CONNECT
if is_layer(self.layer_list[layer_id], LayerType.ADD):
skip_type = NetworkDescriptor.ADD_CONNECT
ret.add_skip_connection(index_in_main_chain[u],
index_in_main_chain[v], skip_type)
return ret
def _get_pooling_layers(self, start_node_id, end_node_id):
layer_list = []
node_list = [start_node_id]
self._depth_first_search(end_node_id, layer_list, node_list)
return filter(
lambda layer_id: is_layer(self.layer_list[layer_id], 'Pooling'),
layer_list)
def create_new_layer(layer, n_dim):
input_shape = layer.output.shape
dense_deeper_classes = [StubDense, get_dropout_class(n_dim), StubReLU]
conv_deeper_classes = [
get_conv_class(n_dim),
get_batch_norm_class(n_dim), StubReLU
]
if is_layer(layer, LayerType.RELU):
conv_deeper_classes = [
get_conv_class(n_dim),
get_batch_norm_class(n_dim)
]
dense_deeper_classes = [StubDense, get_dropout_class(n_dim)]
elif is_layer(layer, LayerType.DROPOUT):
dense_deeper_classes = [StubDense, StubReLU]
elif is_layer(layer, LayerType.BATCH_NORM):
conv_deeper_classes = [get_conv_class(n_dim), StubReLU]
if len(input_shape) == 1:
# It is in the dense layer part.
layer_class = sample(dense_deeper_classes, 1)[0]
else:
# It is in the conv layer part.
layer_class = sample(conv_deeper_classes, 1)[0]
if layer_class == StubDense:
new_layer = StubDense(input_shape[0], input_shape[0])
elif layer_class == get_dropout_class(n_dim):
new_layer = layer_class(Constant.DENSE_DROPOUT_RATE)
elif layer_class == get_conv_class(n_dim):
new_layer = layer_class(input_shape[-1],
input_shape[-1],
sample((1, 3, 5), 1)[0],
stride=1)
elif layer_class == get_batch_norm_class(n_dim):
new_layer = layer_class(input_shape[-1])
elif layer_class == get_pooling_class(n_dim):
new_layer = layer_class(sample((1, 3, 5), 1)[0])
else:
new_layer = layer_class()
return new_layer
def _get_pooling_layers(self, start_node_id, end_node_id):
"""Given two node IDs, return all the pooling layers between them."""
layer_list = []
node_list = [start_node_id]
self._depth_first_search(end_node_id, layer_list, node_list)
return filter(
lambda layer_id: is_layer(self.layer_list[layer_id], 'Pooling'),
layer_list)
def _search(self, u, start_dim, total_dim, n_add):
"""Search the graph for widening the layers.
Args:
u: The starting node identifier.
start_dim: The position to insert the additional dimensions.
total_dim: The total number of dimensions the layer has before widening.
n_add: The number of dimensions to add.
"""
if (u, start_dim, total_dim, n_add) in self.vis:
return
self.vis[(u, start_dim, total_dim, n_add)] = True
for v, layer_id in self.adj_list[u]:
layer = self.layer_list[layer_id]
if is_layer(layer, 'Conv'):
new_layer = wider_next_conv(layer, start_dim, total_dim, n_add,
self.weighted)
self._replace_layer(layer_id, new_layer)
elif is_layer(layer, 'Dense'):
new_layer = wider_next_dense(layer, start_dim, total_dim,
n_add, self.weighted)
self._replace_layer(layer_id, new_layer)
elif is_layer(layer, 'BatchNormalization'):
new_layer = wider_bn(layer, start_dim, total_dim, n_add,
self.weighted)
self._replace_layer(layer_id, new_layer)
self._search(v, start_dim, total_dim, n_add)
elif is_layer(layer, 'Concatenate'):
if self.layer_id_to_input_node_ids[layer_id][1] == u:
# u is on the right of the concat
# next_start_dim += next_total_dim - total_dim
left_dim = self._upper_layer_width(
self.layer_id_to_input_node_ids[layer_id][0])
next_start_dim = start_dim + left_dim
next_total_dim = total_dim + left_dim
else:
next_start_dim = start_dim
next_total_dim = total_dim + self._upper_layer_width(
self.layer_id_to_input_node_ids[layer_id][1])
self._search(v, next_start_dim, next_total_dim, n_add)
else:
self._search(v, start_dim, total_dim, n_add)
for v, layer_id in self.reverse_adj_list[u]:
layer = self.layer_list[layer_id]
if is_layer(layer, 'Conv'):
new_layer = wider_pre_conv(layer, n_add, self.weighted)
self._replace_layer(layer_id, new_layer)
elif is_layer(layer, 'Dense'):
new_layer = wider_pre_dense(layer, n_add, self.weighted)
self._replace_layer(layer_id, new_layer)
elif is_layer(layer, 'Concatenate'):
continue
else:
self._search(v, start_dim, total_dim, n_add)
def deep_layer_ids(self):
ret = []
pre_layer = None
pre_layer_id = None
for layer_id in self.get_main_chain_layers():
layer = self.layer_list[layer_id]
if is_layer(layer, LayerType.GLOBAL_POOL):
break
if is_layer(layer, LayerType.ADD) or is_layer(
layer, LayerType.CONCAT):
continue
if is_layer(layer, LayerType.CONV) and (pre_layer
is not None) and is_layer(
pre_layer,
LayerType.BATCH_NORM):
ret.append(pre_layer_id)
pre_layer = layer
pre_layer_id = layer_id
return ret
def _search(self, u, start_dim, total_dim, n_add):
"""Search the graph for all the layers to be widened caused by an operation.
It is an recursive function with duplication check to avoid deadlock.
It searches from a starting node u until the corresponding layers has been widened.
Args:
u: The starting node ID.
start_dim: The position to insert the additional dimensions.
total_dim: The total number of dimensions the layer has before widening.
n_add: The number of dimensions to add.
"""
if (u, start_dim, total_dim, n_add) in self.vis:
return
self.vis[(u, start_dim, total_dim, n_add)] = True
for v, layer_id in self.adj_list[u]:
layer = self.layer_list[layer_id]
if is_layer(layer, LayerType.CONV):
new_layer = wider_next_conv(layer, start_dim, total_dim, n_add, self.weighted)
self._replace_layer(layer_id, new_layer)
elif is_layer(layer, LayerType.DENSE):
new_layer = wider_next_dense(layer, start_dim, total_dim, n_add, self.weighted)
self._replace_layer(layer_id, new_layer)
elif is_layer(layer, LayerType.BATCH_NORM):
new_layer = wider_bn(layer, start_dim, total_dim, n_add, self.weighted)
self._replace_layer(layer_id, new_layer)
self._search(v, start_dim, total_dim, n_add)
elif is_layer(layer, LayerType.CONCAT):
if self.layer_id_to_input_node_ids[layer_id][1] == u:
# u is on the right of the concat
# next_start_dim += next_total_dim - total_dim
left_dim = self._upper_layer_width(self.layer_id_to_input_node_ids[layer_id][0])
next_start_dim = start_dim + left_dim
next_total_dim = total_dim + left_dim
else:
next_start_dim = start_dim
next_total_dim = total_dim + self._upper_layer_width(self.layer_id_to_input_node_ids[layer_id][1])
self._search(v, next_start_dim, next_total_dim, n_add)
else:
self._search(v, start_dim, total_dim, n_add)
for v, layer_id in self.reverse_adj_list[u]:
layer = self.layer_list[layer_id]
if is_layer(layer, LayerType.CONV):
new_layer = wider_pre_conv(layer, n_add, self.weighted)
self._replace_layer(layer_id, new_layer)
elif is_layer(layer, LayerType.DENSE):
new_layer = wider_pre_dense(layer, n_add, self.weighted)
self._replace_layer(layer_id, new_layer)
elif is_layer(layer, LayerType.CONCAT):
continue
else:
self._search(v, start_dim, total_dim, n_add)
def _search(self, u, start_dim, total_dim, n_add):
"""Search the graph for all the layers to be widened caused by an operation.
It is an recursive function with duplication check to avoid deadlock.
It searches from a starting node u until the corresponding layers has been widened.
Args:
u: The starting node ID.
start_dim: The position to insert the additional dimensions.
total_dim: The total number of dimensions the layer has before widening.
n_add: The number of dimensions to add.
"""
if (u, start_dim, total_dim, n_add) in self.vis:
return
self.vis[(u, start_dim, total_dim, n_add)] = True
for v, layer_id in self.adj_list[u]:
layer = self.layer_list[layer_id]
if is_layer(layer, 'Conv'):
new_layer = wider_next_conv(layer, start_dim, total_dim, n_add, self.weighted)
self._replace_layer(layer_id, new_layer)
elif is_layer(layer, 'Dense'):
new_layer = wider_next_dense(layer, start_dim, total_dim, n_add, self.weighted)
self._replace_layer(layer_id, new_layer)
elif is_layer(layer, 'BatchNormalization'):
new_layer = wider_bn(layer, start_dim, total_dim, n_add, self.weighted)
self._replace_layer(layer_id, new_layer)
self._search(v, start_dim, total_dim, n_add)
elif is_layer(layer, 'Concatenate'):
if self.layer_id_to_input_node_ids[layer_id][1] == u:
# u is on the right of the concat
# next_start_dim += next_total_dim - total_dim
left_dim = self._upper_layer_width(self.layer_id_to_input_node_ids[layer_id][0])
next_start_dim = start_dim + left_dim
next_total_dim = total_dim + left_dim
else:
next_start_dim = start_dim
next_total_dim = total_dim + self._upper_layer_width(self.layer_id_to_input_node_ids[layer_id][1])
self._search(v, next_start_dim, next_total_dim, n_add)
else:
self._search(v, start_dim, total_dim, n_add)
for v, layer_id in self.reverse_adj_list[u]:
layer = self.layer_list[layer_id]
if is_layer(layer, 'Conv'):
new_layer = wider_pre_conv(layer, n_add, self.weighted)
self._replace_layer(layer_id, new_layer)
elif is_layer(layer, 'Dense'):
new_layer = wider_pre_dense(layer, n_add, self.weighted)
self._replace_layer(layer_id, new_layer)
elif is_layer(layer, 'Concatenate'):
continue
else:
self._search(v, start_dim, total_dim, n_add)
def to_deeper_model(self, target_id, new_layer):
"""Insert a relu-conv-bn block after the target block.
Args:
target_id: A convolutional layer ID. The new block should be inserted after the block.
new_layer: An instance of StubLayer subclasses.
"""
self.operation_history.append(('to_deeper_model', target_id, new_layer))
input_id = self.layer_id_to_input_node_ids[target_id][0]
output_id = self.layer_id_to_output_node_ids[target_id][0]
if self.weighted:
if is_layer(new_layer, LayerType.DENSE):
init_dense_weight(new_layer)
elif is_layer(new_layer, LayerType.CONV):
init_conv_weight(new_layer)
elif is_layer(new_layer, LayerType.BATCH_NORM):
init_bn_weight(new_layer)
self._insert_new_layers([new_layer], input_id, output_id)
def to_deeper_model(self, target_id, new_layer):
"""Insert a relu-conv-bn block after the target block.
Args:
target_id: A convolutional layer ID. The new block should be inserted after the block.
new_layer: An instance of StubLayer subclasses.
"""
self.operation_history.append(('to_deeper_model', target_id, new_layer))
input_id = self.layer_id_to_input_node_ids[target_id][0]
output_id = self.layer_id_to_output_node_ids[target_id][0]
if self.weighted:
if is_layer(new_layer, 'Dense'):
init_dense_weight(new_layer)
elif is_layer(new_layer, 'Conv'):
init_conv_weight(new_layer)
elif is_layer(new_layer, 'BatchNormalization'):
init_bn_weight(new_layer)
self._insert_new_layers([new_layer], input_id, output_id)
def extract_descriptor(self):
"""Extract the the description of the Graph as an instance of NetworkDescriptor."""
main_chain = self.get_main_chain()
index_in_main_chain = {}
for index, u in enumerate(main_chain):
index_in_main_chain[u] = index
ret = NetworkDescriptor()
for u in main_chain:
for v, layer_id in self.adj_list[u]:
if v not in index_in_main_chain:
continue
layer = self.layer_list[layer_id]
copied_layer = copy(layer)
copied_layer.weights = None
ret.add_layer(deepcopy(copied_layer))
for u in index_in_main_chain:
for v, layer_id in self.adj_list[u]:
if v not in index_in_main_chain:
temp_u = u
temp_v = v
temp_layer_id = layer_id
skip_type = None
while not (temp_v in index_in_main_chain and temp_u in index_in_main_chain):
if is_layer(self.layer_list[temp_layer_id], LayerType.CONCAT):
skip_type = NetworkDescriptor.CONCAT_CONNECT
if is_layer(self.layer_list[temp_layer_id], LayerType.ADD):
skip_type = NetworkDescriptor.ADD_CONNECT
temp_u = temp_v
temp_v, temp_layer_id = self.adj_list[temp_v][0]
ret.add_skip_connection(index_in_main_chain[u], index_in_main_chain[temp_u], skip_type)
elif index_in_main_chain[v] - index_in_main_chain[u] != 1:
skip_type = None
if is_layer(self.layer_list[layer_id], LayerType.CONCAT):
skip_type = NetworkDescriptor.CONCAT_CONNECT
if is_layer(self.layer_list[layer_id], LayerType.ADD):
skip_type = NetworkDescriptor.ADD_CONNECT
ret.add_skip_connection(index_in_main_chain[u], index_in_main_chain[v], skip_type)
return ret
def _insert_pooling_layer_chain(self, start_node_id, end_node_id):
skip_output_id = start_node_id
for layer in self._get_pooling_layers(start_node_id, end_node_id):
new_layer = deepcopy(layer)
if is_layer(new_layer, 'Conv'):
filters = self.node_list[start_node_id].shape[-1]
new_layer = get_conv_class(self.n_dim)(filters, filters, 1, layer.stride)
else:
new_layer = deepcopy(layer)
skip_output_id = self.add_layer(new_layer, skip_output_id)
skip_output_id = self.add_layer(StubReLU(), skip_output_id)
return skip_output_id
def to_deeper_graph(graph):
weighted_layer_ids = graph.deep_layer_ids()
deeper_layer_ids = sample(weighted_layer_ids, 1)
for layer_id in deeper_layer_ids:
layer = graph.layer_list[layer_id]
if is_layer(layer, 'Conv'):
graph.to_conv_deeper_model(layer_id, 3)
else:
graph.to_dense_deeper_model(layer_id)
return graph
def extract_descriptor(self):
ret = NetworkDescriptor()
topological_node_list = self.topological_order
for u in topological_node_list:
for v, layer_id in self.adj_list[u]:
layer = self.layer_list[layer_id]
if is_layer(layer, 'Conv') and layer.kernel_size not in [1, (1,), (1, 1), (1, 1, 1)]:
ret.add_conv_width(layer_width(layer))
if is_layer(layer, 'Dense'):
ret.add_dense_width(layer_width(layer))
# The position of each node, how many Conv and Dense layers before it.
pos = [0] * len(topological_node_list)
for v in topological_node_list:
layer_count = 0
for u, layer_id in self.reverse_adj_list[v]:
layer = self.layer_list[layer_id]
weighted = 0
if (is_layer(layer, 'Conv') and layer.kernel_size not in [1, (1,), (1, 1), (1, 1, 1)]) \
or is_layer(layer, 'Dense'):
weighted = 1
layer_count = max(pos[u] + weighted, layer_count)
pos[v] = layer_count
for u in topological_node_list:
for v, layer_id in self.adj_list[u]:
if pos[u] == pos[v]:
continue
layer = self.layer_list[layer_id]
if is_layer(layer, 'Concatenate'):
ret.add_skip_connection(pos[u], pos[v], NetworkDescriptor.CONCAT_CONNECT)
if is_layer(layer, 'Add'):
ret.add_skip_connection(pos[u], pos[v], NetworkDescriptor.ADD_CONNECT)
return ret
def to_wider_graph(graph):
weighted_layer_ids = graph.wide_layer_ids()
wider_layers = sample(weighted_layer_ids, 1)
for layer_id in wider_layers:
layer = graph.layer_list[layer_id]
if is_layer(layer, 'Conv'):
n_add = layer.filters
else:
n_add = layer.units
graph.to_wider_model(layer_id, n_add)
return graph
def create_new_layer(layer, n_dim):
input_shape = layer.output.shape
dense_deeper_classes = [StubDense, get_dropout_class(n_dim), StubReLU]
conv_deeper_classes = [get_conv_class(n_dim), get_batch_norm_class(n_dim), StubReLU]
if is_layer(layer, LayerType.RELU):
conv_deeper_classes = [get_conv_class(n_dim), get_batch_norm_class(n_dim)]
dense_deeper_classes = [StubDense, get_dropout_class(n_dim)]
elif is_layer(layer, LayerType.DROPOUT):
dense_deeper_classes = [StubDense, StubReLU]
elif is_layer(layer, LayerType.BATCH_NORM):
conv_deeper_classes = [get_conv_class(n_dim), StubReLU]
if len(input_shape) == 1:
# It is in the dense layer part.
layer_class = sample(dense_deeper_classes, 1)[0]
else:
# It is in the conv layer part.
layer_class = sample(conv_deeper_classes, 1)[0]
if layer_class == StubDense:
new_layer = StubDense(input_shape[0], input_shape[0])
elif layer_class == get_dropout_class(n_dim):
new_layer = layer_class(Constant.DENSE_DROPOUT_RATE)
elif layer_class == get_conv_class(n_dim):
new_layer = layer_class(input_shape[-1], input_shape[-1], sample((1, 3, 5), 1)[0], stride=1)
elif layer_class == get_batch_norm_class(n_dim):
new_layer = layer_class(input_shape[-1])
elif layer_class == get_pooling_class(n_dim):
new_layer = layer_class(sample((1, 3, 5), 1)[0])
else:
new_layer = layer_class()
return new_layer
def to_wider_graph(graph):
weighted_layer_ids = graph.wide_layer_ids()
weighted_layer_ids = list(filter(lambda x: graph.layer_list[x].output.shape[-1], weighted_layer_ids))
wider_layers = sample(weighted_layer_ids, 1)
for layer_id in wider_layers:
layer = graph.layer_list[layer_id]
if is_layer(layer, LayerType.CONV):
n_add = layer.filters
else:
n_add = layer.units
graph.to_wider_model(layer_id, n_add)
return graph
def to_deeper_graph(graph):
weighted_layer_ids = graph.deep_layer_ids()
if len(weighted_layer_ids) >= Constant.MAX_LAYERS:
return None
deeper_layer_ids = sample(weighted_layer_ids, 1)
for layer_id in deeper_layer_ids:
layer = graph.layer_list[layer_id]
if is_layer(layer, 'Conv'):
graph.to_conv_deeper_model(layer_id, 3)
else:
graph.to_dense_deeper_model(layer_id)
return graph
def to_wider_graph(graph):
weighted_layer_ids = graph.wide_layer_ids()
weighted_layer_ids = list(
filter(lambda x: graph.layer_list[x].output.shape[-1],
weighted_layer_ids))
wider_layers = sample(weighted_layer_ids, 1)
for layer_id in wider_layers:
layer = graph.layer_list[layer_id]
if is_layer(layer, 'Conv'):
n_add = layer.filters
else:
n_add = layer.units
graph.to_wider_model(layer_id, n_add)
return graph
def _insert_pooling_layer_chain(self, start_node_id, end_node_id):
skip_output_id = start_node_id
for layer in self._get_pooling_layers(start_node_id, end_node_id):
new_layer = deepcopy(layer)
if is_layer(new_layer, LayerType.CONV):
filters = self.node_list[start_node_id].shape[-1]
kernel_size = layer.kernel_size if layer.padding != int(
layer.kernel_size / 2) or layer.stride != 1 else 1
new_layer = get_conv_class(self.n_dim)(filters, filters, kernel_size, layer.stride,
padding=layer.padding)
if self.weighted:
init_conv_weight(new_layer)
else:
new_layer = deepcopy(layer)
skip_output_id = self.add_layer(new_layer, skip_output_id)
skip_output_id = self.add_layer(StubReLU(), skip_output_id)
return skip_output_id
def _insert_pooling_layer_chain(self, start_node_id, end_node_id):
skip_output_id = start_node_id
for layer in self._get_pooling_layers(start_node_id, end_node_id):
new_layer = deepcopy(layer)
if is_layer(new_layer, 'Conv'):
filters = self.node_list[start_node_id].shape[-1]
kernel_size = layer.kernel_size if layer.padding != int(
layer.kernel_size / 2) or layer.stride != 1 else 1
new_layer = get_conv_class(self.n_dim)(filters, filters, kernel_size, layer.stride,
padding=layer.padding)
if self.weighted:
init_conv_weight(new_layer)
else:
new_layer = deepcopy(layer)
skip_output_id = self.add_layer(new_layer, skip_output_id)
skip_output_id = self.add_layer(StubReLU(), skip_output_id)
return skip_output_id
def _search(self, u, start_dim, total_dim, n_add):
"""Search the graph for all the layers to be widened caused by an operation.
It is an recursive function with duplication check to avoid deadlock.
It searches from a starting node u until the corresponding layers has been widened.
Args:
u: The starting node ID.
start_dim: The position to insert the additional dimensions.
total_dim: The total number of dimensions the layer has before widening.
n_add: The number of dimensions to add.
"""
is_exist = True if (u, start_dim, total_dim,
n_add) in self.vis else False
#with open("log.txt", "a") as resfile:
# resfile.write("start_node_id:{},start_dim:{},total_dim:{},n_add:{},is_exist:{}\n".format(u, start_dim, total_dim, n_add,is_exist))
if (u, start_dim, total_dim, n_add) in self.vis:
return
self.vis[(u, start_dim, total_dim, n_add)] = True
for v, layer_id in self.adj_list[u]:
layer = self.layer_list[layer_id]
if is_layer(layer, LayerType.CONV):
new_layer = wider_next_conv(layer, start_dim, total_dim, n_add,
self.weighted)
self._replace_layer(layer_id, new_layer)
if layer.groups > 1:
self._search(v, start_dim, total_dim, n_add)
elif is_layer(layer, LayerType.DENSE):
new_layer = wider_next_dense(layer, start_dim, total_dim,
n_add, self.weighted)
self._replace_layer(layer_id, new_layer)
elif is_layer(layer, LayerType.BATCH_NORM):
new_layer = wider_bn(layer, start_dim, total_dim, n_add,
self.weighted)
self._replace_layer(layer_id, new_layer)
self._search(v, start_dim, total_dim, n_add)
elif is_layer(layer, LayerType.CONCAT):
if self.layer_id_to_input_node_ids[layer_id][1] == u:
# u is on the right of the concat
# next_start_dim += next_total_dim - total_dim
left_dim = self._upper_layer_width(
self.layer_id_to_input_node_ids[layer_id][0])
next_start_dim = start_dim + left_dim
next_total_dim = total_dim + left_dim
else:
next_start_dim = start_dim
next_total_dim = total_dim + self._upper_layer_width(
self.layer_id_to_input_node_ids[layer_id][1])
self._search(v, next_start_dim, next_total_dim, n_add)
else:
self._search(v, start_dim, total_dim, n_add)
for v, layer_id in self.reverse_adj_list[u]:
layer = self.layer_list[layer_id]
if is_layer(layer, LayerType.CONV):
new_layer = wider_pre_conv(layer, n_add, self.weighted)
self._replace_layer(layer_id, new_layer)
if layer.groups > 1:
self._search(v, start_dim, total_dim, n_add)
elif is_layer(layer, LayerType.DENSE):
new_layer = wider_pre_dense(layer, n_add, self.weighted)
self._replace_layer(layer_id, new_layer)
elif is_layer(layer, LayerType.CONCAT):
continue
else:
self._search(v, start_dim, total_dim, n_add)
def _layer_ids_by_type(self, type_str):
return list(filter(lambda layer_id: is_layer(self.layer_list[layer_id], type_str), range(self.n_layers)))
def _conv_layer_ids_in_order(self):
return list(filter(lambda layer_id: is_layer(self.layer_list[layer_id], LayerType.CONV),
self.get_main_chain_layers()))
def _layer_ids_by_type(self, type_str):
return list(
filter(
lambda layer_id: is_layer(self.layer_list[layer_id], type_str),
range(self.n_layers)))
def _conv_layer_ids_in_order(self):
return list(
filter(
lambda layer_id: is_layer(self.layer_list[layer_id], LayerType.
CONV), self.get_main_chain_layers()))
def create_new_layer(layer, n_dim):
input_shape = layer.output.shape
dense_deeper_classes = [StubDense, get_dropout_class(n_dim), StubReLU]
conv_deeper_classes = [
get_conv_class(n_dim),
get_batch_norm_class(n_dim), StubReLU
]
if is_layer(layer, LayerType.RELU):
conv_deeper_classes = [
get_conv_class(n_dim),
get_batch_norm_class(n_dim)
]
dense_deeper_classes = [StubDense, get_dropout_class(n_dim)]
elif is_layer(layer, LayerType.DROPOUT):
dense_deeper_classes = [StubDense, StubReLU]
elif is_layer(layer, LayerType.BATCH_NORM):
conv_deeper_classes = [get_conv_class(n_dim)] #, StubReLU]
new_layers = []
if len(input_shape) == 1:
# It is in the dense layer part.
layer_class = sample(dense_deeper_classes, 1)[0]
else:
# It is in the conv layer part.
layer_class = sample(conv_deeper_classes, 1)[0]
if layer_class == StubDense:
new_layer = StubDense(input_shape[0], input_shape[0])
new_layers.append(new_layer)
elif layer_class == get_dropout_class(n_dim):
new_layer = layer_class(Constant.DENSE_DROPOUT_RATE)
new_layers.append(new_layer)
elif layer_class == get_conv_class(n_dim):
# add conv layer
# new_layer = layer_class(input_shape[-1],, input_shape[-1], sample((1, 3, 5), 1)[0], stride=1)
# add mobilenet block
in_planes = input_shape[-1]
expansion = sample((1, 6), 1)[0]
stride = sample((1, 2), 1)[0]
planes = expansion * in_planes
new_layer = layer_class(in_planes, planes, 1, stride=1, padding=0)
new_layers.append(new_layer)
new_layer = get_batch_norm_class(n_dim)(planes)
new_layers.append(new_layer)
new_layer = StubReLU()
new_layers.append(new_layer)
new_layer = layer_class(planes,
planes,
3,
stride=stride,
padding=1,
groups=planes)
new_layers.append(new_layer)
new_layer = get_batch_norm_class(n_dim)(planes)
new_layers.append(new_layer)
new_layer = StubReLU()
new_layers.append(new_layer)
new_layer = layer_class(planes, in_planes, 1, stride=1, padding=0)
new_layers.append(new_layer)
new_layer = get_batch_norm_class(n_dim)(in_planes)
new_layers.append(new_layer)
elif layer_class == get_batch_norm_class(n_dim):
new_layer = layer_class(input_shape[-1])
new_layers.append(new_layer)
elif layer_class == get_pooling_class(n_dim):
new_layer = layer_class(sample((1, 3, 5), 1)[0])
new_layers.append(new_layer)
else:
new_layer = layer_class()
new_layers.append(new_layer)
return new_layers
