def deep_layer_ids2(self):
ret = []
for layer_id in self.get_main_chain_layers():
layer = self.layer_list[layer_id]
if is_layer(layer, "GlobalAveragePooling"):
break
if is_layer(layer, "ReLU"):
ret.append(layer_id)
return ret
def deep_layer_ids(self):
ret = []
for layer_id in self.get_main_chain_layers():
layer = self.layer_list[layer_id]
if is_layer(layer, "GlobalAveragePooling"):
break
if is_layer(layer, "Add") or is_layer(layer, "Concatenate"):
continue
ret.append(layer_id)
return ret
def _upper_layer_width(self, u):
for v, layer_id in self.reverse_adj_list[u]:
layer = self.layer_list[layer_id]
if is_layer(layer, "Conv") or is_layer(layer, "Dense"):
return layer_width(layer)
elif is_layer(layer, "Concatenate"):
a = self.layer_id_to_input_node_ids[layer_id][0]
b = self.layer_id_to_input_node_ids[layer_id][1]
return self._upper_layer_width(a) + self._upper_layer_width(b)
else:
return self._upper_layer_width(v)
return self.node_list[0].shape[-1]
def _get_pooling_layers(self, start_node_id, end_node_id):
"""Given two node IDs, return all the pooling layers between them."""
layer_list = []
node_list = [start_node_id]
assert self._depth_first_search(end_node_id, layer_list, node_list)
ret = []
for layer_id in layer_list:
layer = self.layer_list[layer_id]
if is_layer(layer, "Pooling"):
ret.append(layer)
elif is_layer(layer, "Conv") and layer.stride != 1:
ret.append(layer)
return ret
def create_new_layer(layer, n_dim):
''' create new layer for the graph
'''
input_shape = layer.output.shape
dense_deeper_classes = [StubDense, get_dropout_class(n_dim), StubReLU]
conv_deeper_classes = [
get_conv_class(n_dim),
get_batch_norm_class(n_dim), StubReLU
]
if is_layer(layer, "ReLU"):
conv_deeper_classes = [
get_conv_class(n_dim),
get_batch_norm_class(n_dim)
]
dense_deeper_classes = [StubDense, get_dropout_class(n_dim)]
elif is_layer(layer, "Dropout"):
dense_deeper_classes = [StubDense, StubReLU]
elif is_layer(layer, "BatchNormalization"):
conv_deeper_classes = [get_conv_class(n_dim), StubReLU]
layer_class = None
if len(input_shape) == 1:
# It is in the dense layer part.
layer_class = sample(dense_deeper_classes, 1)[0]
else:
# It is in the conv layer part.
layer_class = sample(conv_deeper_classes, 1)[0]
if layer_class == StubDense:
new_layer = StubDense(input_shape[0], input_shape[0])
elif layer_class == get_dropout_class(n_dim):
new_layer = layer_class(Constant.DENSE_DROPOUT_RATE)
elif layer_class == get_conv_class(n_dim):
new_layer = layer_class(input_shape[-1],
input_shape[-1],
sample((1, 3, 5), 1)[0],
stride=1)
elif layer_class == get_batch_norm_class(n_dim):
new_layer = layer_class(input_shape[-1])
elif layer_class == get_pooling_class(n_dim):
new_layer = layer_class(sample((1, 3, 5), 1)[0])
else:
new_layer = layer_class()
return new_layer
def extract_descriptor(self):
"""Extract the the description of the Graph as an instance of NetworkDescriptor."""
main_chain = self.get_main_chain()
index_in_main_chain = {}
for index, u in enumerate(main_chain):
index_in_main_chain[u] = index
ret = NetworkDescriptor()
for u in main_chain:
for v, layer_id in self.adj_list[u]:
if v not in index_in_main_chain:
continue
layer = self.layer_list[layer_id]
copied_layer = copy(layer)
copied_layer.weights = None
ret.add_layer(deepcopy(copied_layer))
for u in index_in_main_chain:
for v, layer_id in self.adj_list[u]:
if v not in index_in_main_chain:
temp_u = u
temp_v = v
temp_layer_id = layer_id
skip_type = None
while not (
temp_v in index_in_main_chain and temp_u in index_in_main_chain):
if is_layer(
self.layer_list[temp_layer_id], "Concatenate"):
skip_type = NetworkDescriptor.CONCAT_CONNECT
if is_layer(self.layer_list[temp_layer_id], "Add"):
skip_type = NetworkDescriptor.ADD_CONNECT
temp_u = temp_v
temp_v, temp_layer_id = self.adj_list[temp_v][0]
ret.add_skip_connection(
index_in_main_chain[u], index_in_main_chain[temp_u], skip_type
)
elif index_in_main_chain[v] - index_in_main_chain[u] != 1:
skip_type = None
if is_layer(self.layer_list[layer_id], "Concatenate"):
skip_type = NetworkDescriptor.CONCAT_CONNECT
if is_layer(self.layer_list[layer_id], "Add"):
skip_type = NetworkDescriptor.ADD_CONNECT
ret.add_skip_connection(
index_in_main_chain[u], index_in_main_chain[v], skip_type
)
return ret
def _conv_layer_ids_in_order(self):
return list(
filter(
lambda layer_id: is_layer(self.layer_list[layer_id], "Conv"),
self.get_main_chain_layers(),
)
)
def _layer_ids_by_type(self, type_str):
return list(
filter(
lambda layer_id: is_layer(self.layer_list[layer_id], type_str),
range(self.n_layers),
)
)
def to_deeper_model(self, target_id, new_layer):
"""Insert a relu-conv-bn block after the target block.
Args:
target_id: A convolutional layer ID. The new block should be inserted after the block.
new_layer: An instance of StubLayer subclasses.
"""
self.operation_history.append(
("to_deeper_model", target_id, new_layer))
input_id = self.layer_id_to_input_node_ids[target_id][0]
output_id = self.layer_id_to_output_node_ids[target_id][0]
if self.weighted:
if is_layer(new_layer, "Dense"):
init_dense_weight(new_layer)
elif is_layer(new_layer, "Conv"):
init_conv_weight(new_layer)
elif is_layer(new_layer, "BatchNormalization"):
init_bn_weight(new_layer)
self._insert_new_layers([new_layer], input_id, output_id)
def layer_distance(a, b):
"""The distance between two layers."""
# pylint: disable=unidiomatic-typecheck
if not isinstance(a, type(b)):
return 1.0
if is_layer(a, "Conv"):
att_diff = [
(a.filters, b.filters),
(a.kernel_size, b.kernel_size),
(a.stride, b.stride),
]
return attribute_difference(att_diff)
if is_layer(a, "Pooling"):
att_diff = [
(a.padding, b.padding),
(a.kernel_size, b.kernel_size),
(a.stride, b.stride),
]
return attribute_difference(att_diff)
return 0.0
def to_skip_connection_graph2(graph):
''' skip connection graph
'''
# The last conv layer cannot be widen since wider operator cannot be done
# over the two sides of flatten.
weighted_layer_ids = graph.skip_connection_layer_ids()
valid_connection = []
for index_a in range(4, len(weighted_layer_ids)):
a_id = weighted_layer_ids[index_a]
layer = graph.layer_list[a_id]
if is_layer(layer, "ReLU"):
for index_b in range(len(weighted_layer_ids))[index_a + 1:]:
b_id = weighted_layer_ids[index_b]
layer = graph.layer_list[b_id]
if is_layer(layer, "BatchNormalization"):
valid_connection.append((index_a, index_b, NetworkDescriptor.ADD_CONNECT))
'''
for index_a in range(4, len(weighted_layer_ids)):
a_id = weighted_layer_ids[index_a]
layer = graph.layer_list[a_id]
if is_layer(layer, "ReLU"):
for index_b in range(len(weighted_layer_ids))[index_a + 1:]:
b_id = weighted_layer_ids[index_b]
layer = graph.layer_list[b_id]
if is_layer(layer, "ReLU"):
valid_connection.append((index_a, index_b, NetworkDescriptor.CONCAT_CONNECT))
'''
if not valid_connection:
return graph
for index_a, index_b, skip_type in sample(valid_connection, 1):
a_id = weighted_layer_ids[index_a]
b_id = weighted_layer_ids[index_b]
if skip_type == NetworkDescriptor.ADD_CONNECT:
graph.to_add_skip_model(a_id, b_id)
else:
graph.to_concat_skip_model(a_id, b_id)
return graph
def _insert_pooling_layer_chain(self, start_node_id, end_node_id):
skip_output_id = start_node_id
for layer in self._get_pooling_layers(start_node_id, end_node_id):
new_layer = deepcopy(layer)
if is_layer(new_layer, "Conv"):
filters = self.node_list[start_node_id].shape[-1]
new_layer = get_conv_class(self.n_dim)(filters, filters, 1,
layer.stride)
if self.weighted:
init_conv_weight(new_layer)
else:
new_layer = deepcopy(layer)
skip_output_id = self.add_layer(new_layer, skip_output_id)
skip_output_id = self.add_layer(StubReLU(), skip_output_id)
return skip_output_id
def to_wider_graph(graph):
''' wider graph
'''
weighted_layer_ids = graph.wide_layer_ids()
weighted_layer_ids = list(
filter(lambda x: graph.layer_list[x].output.shape[-1],
weighted_layer_ids))
wider_layers = sample(weighted_layer_ids, 1)
for layer_id in wider_layers:
layer = graph.layer_list[layer_id]
if is_layer(layer, "Conv"):
n_add = layer.filters
else:
n_add = layer.units
graph.to_wider_model(layer_id, n_add)
return graph
def _search(self, u, start_dim, total_dim, n_add):
"""Search the graph for all the layers to be widened caused by an operation.
It is an recursive function with duplication check to avoid deadlock.
It searches from a starting node u until the corresponding layers has been widened.
Args:
u: The starting node ID.
start_dim: The position to insert the additional dimensions.
total_dim: The total number of dimensions the layer has before widening.
n_add: The number of dimensions to add.
"""
if (u, start_dim, total_dim, n_add) in self.vis:
return
self.vis[(u, start_dim, total_dim, n_add)] = True
for v, layer_id in self.adj_list[u]:
layer = self.layer_list[layer_id]
if is_layer(layer, "Conv"):
new_layer = wider_next_conv(layer, start_dim, total_dim, n_add,
self.weighted)
self._replace_layer(layer_id, new_layer)
elif is_layer(layer, "Dense"):
new_layer = wider_next_dense(layer, start_dim, total_dim,
n_add, self.weighted)
self._replace_layer(layer_id, new_layer)
elif is_layer(layer, "BatchNormalization"):
new_layer = wider_bn(layer, start_dim, total_dim, n_add,
self.weighted)
self._replace_layer(layer_id, new_layer)
self._search(v, start_dim, total_dim, n_add)
elif is_layer(layer, "Concatenate"):
if self.layer_id_to_input_node_ids[layer_id][1] == u:
# u is on the right of the concat
# next_start_dim += next_total_dim - total_dim
left_dim = self._upper_layer_width(
self.layer_id_to_input_node_ids[layer_id][0])
next_start_dim = start_dim + left_dim
next_total_dim = total_dim + left_dim
else:
next_start_dim = start_dim
next_total_dim = total_dim + self._upper_layer_width(
self.layer_id_to_input_node_ids[layer_id][1])
self._search(v, next_start_dim, next_total_dim, n_add)
else:
self._search(v, start_dim, total_dim, n_add)
for v, layer_id in self.reverse_adj_list[u]:
layer = self.layer_list[layer_id]
if is_layer(layer, "Conv"):
new_layer = wider_pre_conv(layer, n_add, self.weighted)
self._replace_layer(layer_id, new_layer)
elif is_layer(layer, "Dense"):
new_layer = wider_pre_dense(layer, n_add, self.weighted)
self._replace_layer(layer_id, new_layer)
elif is_layer(layer, "Concatenate"):
continue
else:
self._search(v, start_dim, total_dim, n_add)