def __init__(self):
"""
Initialize a new instance of the DARTS search space.
Note:
__init__ cannot take any parameters due to the way networkx is implemented.
If we want to change the number of classes set a static attribute `NUM_CLASSES`
before initializing the class. Default is 10 as for cifar-10.
"""
super().__init__()
self.channels = [16, 32, 64]
self.compact = None
self.load_labeled = None
self.num_classes = self.NUM_CLASSES if hasattr(self,
'NUM_CLASSES') else 10
self.max_epoch = 97
self.space_name = 'darts'
"""
Build the search space with the parameters specified in __init__.
"""
#
# Cell definition
#
# Normal cell first
normal_cell = Graph()
normal_cell.name = "normal_cell" # Use the same name for all cells with shared attributes
# Input nodes
normal_cell.add_node(1)
normal_cell.add_node(2)
# Intermediate nodes
normal_cell.add_node(3)
normal_cell.add_node(4)
normal_cell.add_node(5)
normal_cell.add_node(6)
# Output node
normal_cell.add_node(7)
# Edges
normal_cell.add_edges_from([(1, i) for i in range(3, 7)]) # input 1
normal_cell.add_edges_from([(2, i) for i in range(3, 7)]) # input 2
normal_cell.add_edges_from([(3, 4), (3, 5), (3, 6)])
normal_cell.add_edges_from([(4, 5), (4, 6)])
normal_cell.add_edges_from([(5, 6)])
# Edges connecting to the output are always the identity
normal_cell.add_edges_from([(i, 7, EdgeData().finalize())
for i in range(3, 7)]) # output
# Reduction cell has the same topology
reduction_cell = deepcopy(normal_cell)
reduction_cell.name = "reduction_cell"
# set the cell name for all edges. This is necessary to convert a genotype to a naslib object
for _, _, edge_data in normal_cell.edges.data():
if not edge_data.is_final():
edge_data.set("cell_name", "normal_cell")
for _, _, edge_data in reduction_cell.edges.data():
if not edge_data.is_final():
edge_data.set("cell_name", "reduction_cell")
#
# Makrograph definition
#
self.name = "makrograph"
self.add_node(1) # input node
self.add_node(2) # preprocessing
self.add_node(3,
subgraph=normal_cell.set_scope("n_stage_1").set_input(
[2, 2]))
self.add_node(
4,
subgraph=normal_cell.copy().set_scope("n_stage_1").set_input(
[2, 3]))
self.add_node(5,
subgraph=reduction_cell.set_scope("r_stage_1").set_input(
[3, 4]))
self.add_node(
6,
subgraph=normal_cell.copy().set_scope("n_stage_2").set_input(
[4, 5]))
self.add_node(
7,
subgraph=normal_cell.copy().set_scope("n_stage_2").set_input(
[5, 6]))
self.add_node(
8,
subgraph=reduction_cell.copy().set_scope("r_stage_2").set_input(
[6, 7]))
self.add_node(
9,
subgraph=normal_cell.copy().set_scope("n_stage_3").set_input(
[7, 8]))
self.add_node(
10,
subgraph=normal_cell.copy().set_scope("n_stage_3").set_input(
[8, 9]))
self.add_node(11) # output
self.add_edges_from([(i, i + 1) for i in range(1, 11)])
self.add_edges_from([(i, i + 2) for i in range(2, 9)])
#
# Operations at the makrograph edges
#
self.num_in_edges = 4
reduction_cell_indices = [5, 8]
channel_map_from, channel_map_to = channel_maps(reduction_cell_indices,
max_index=11)
self._set_makrograph_ops(channel_map_from,
channel_map_to,
reduction_cell_indices,
max_index=11,
affine=False)
self._set_cell_ops(reduction_cell_indices)
def __init__(self):
super().__init__()
self.num_classes = self.NUM_CLASSES if hasattr(self,
'NUM_CLASSES') else 10
# creating a dummy graph!
channels = [128, 256, 512]
#
# Cell definition
#
node_pair = Graph()
node_pair.name = "node_pair" # Use the same name for all cells with shared attributes
node_pair.set_scope("node")
# need to add subgraphs on the nodes, each subgraph has option for 3 ops
# Input node
node_pair.add_node(1)
node_pair.add_node(2)
node_pair.add_edges_from([(1, 2)])
cell = Graph()
cell.name = 'cell'
node_pair.update_edges(
update_func=lambda edge: _set_node_ops(edge, C=channels[0]),
private_edge_data=True)
cell.add_node(1) # input node
cell.add_node(2, subgraph=node_pair.set_input([1]))
cell.add_node(3, subgraph=node_pair.copy())
cell.add_node(4, subgraph=node_pair.copy())
cell.add_node(5, subgraph=node_pair.copy())
cell.add_node(6, subgraph=node_pair.copy())
cell.add_node(7) # output
cell.set_scope('cell', recursively=False)
# Edges
cell.add_edges_densly()
cell.update_edges(
update_func=lambda edge: _set_cell_ops(edge, C=channels[0]),
scope="cell",
private_edge_data=True)
#
# dummy Makrograph definition for RE for benchmark queries
#
self.name = "makrograph"
total_num_nodes = 3
self.add_nodes_from(range(1, total_num_nodes + 1))
self.add_edges_from([(i, i + 1) for i in range(1, total_num_nodes)])
self.edges[1, 2].set('op', ops.Stem(channels[0]))
self.edges[2, 3].set('op', cell.copy().set_scope('cell'))
def __init__(self):
super().__init__()
# Define the motifs (6 level-2 motifs)
level2_motifs = []
for j in range(6):
motif = Graph()
motif.name = "motif{}".format(j)
motif.add_nodes_from([i for i in range(1, 5)])
motif.add_edges_densly()
level2_motifs.append(motif)
# cell (= one level-3 motif)
cell = Graph()
cell.name = "cell"
cell.add_nodes_from([i for i in range(1, 6)])
cell.add_edges_densly()
cells = []
channels = [16, 32, 64]
for scope, c in zip(self.OPTIMIZER_SCOPE, channels):
cell_i = cell.copy().set_scope(scope)
cell_i.update_edges(
update_func=lambda edge: _set_motifs(edge, motifs=level2_motifs, c=c),
private_edge_data=True
)
cell_i.set_scope(scope)
# set the level 1 motifs (i.e. primitives)
cell_i.update_edges(
update_func=lambda edge: _set_cell_ops(edge, c, stride=1),
scope=[scope],
private_edge_data=True
)
cells.append(cell_i)
self.name = "makrograph"
self.add_nodes_from([i for i in range(1, 9)])
self.add_edges_from([(i, i+1) for i in range(1, 8)])
self.edges[1, 2].set('op', ops.Stem(16))
self.edges[2, 3].set('op', cells[0])
self.edges[3, 4].set('op', ops.SepConv(16, 32, kernel_size=3, stride=2, padding=1))
self.edges[4, 5].set('op', cells[1])
self.edges[5, 6].set('op', ops.SepConv(32, 64, kernel_size=3, stride=2, padding=1))
self.edges[6, 7].set('op', cells[2])
self.edges[7, 8].set('op', ops.Sequential(
ops.SepConv(64, 64, kernel_size=3, stride=1, padding=1),
nn.AdaptiveAvgPool2d(1),
nn.Flatten(),
nn.Linear(channels[-1], 10))
)
def __init__(self):
logger.info("This is not a search space but the final model found by "
"Liu et al. For the search space use `HierarchicalSearchSpace()`")
# skip the init of HierarchicalSearchSpace
Graph.__init__(self)
separable_3x3 = {
'op': ops.SepConv,
'kernel_size': 3,
'stride': 1,
'padding': 1
}
depthwise_3x3 = {
'op': DepthwiseConv,
'kernel_size': 3,
'stride': 1,
'padding': 1
}
conv_1x1 = {
'op': ConvBNReLU,
'kernel_size': 1,
}
motif = Graph()
motif.add_nodes_from([1, 2, 3, 4])
motif.add_edges_densly()
# Motif 1
motif1 = motif.clone()
motif1.name = "motif1"
motif1.edges[1, 2].set('op', ops.MaxPool1x1(kernel_size=3, stride=1))
motif1.edges[1, 3].update(conv_1x1)
motif1.edges[1, 4].update(separable_3x3)
motif1.edges[2, 3].update(depthwise_3x3)
motif1.edges[2, 4].update(conv_1x1)
motif1.edges[3, 4].set('op', ops.MaxPool1x1(kernel_size=3, stride=1))
# Motif 2
motif2 = motif.clone()
motif2.name = "motif2"
motif2.edges[1, 2].set('op', ops.MaxPool1x1(kernel_size=3, stride=1))
motif2.edges[1, 3].update(depthwise_3x3)
motif2.edges[1, 4].update(conv_1x1)
motif2.edges[2, 3].update(depthwise_3x3)
motif2.edges[2, 4].update(separable_3x3)
motif2.edges[3, 4].update(separable_3x3)
# Motif 3
motif3 = motif.clone()
motif3.name = "motif3"
motif3.edges[1, 2].update(separable_3x3)
motif3.edges[1, 3].update(depthwise_3x3)
motif3.edges[1, 4].update(separable_3x3)
motif3.edges[2, 3].update(separable_3x3)
motif3.edges[2, 4].set('op', ops.Identity()) # assuming no label is identiy
motif3.edges[3, 4].set('op', ops.AvgPool1x1(kernel_size=3, stride=1))
# Motif 4
motif4 = motif.clone()
motif4.name = "motif4"
motif4.edges[1, 2].set('op', ops.AvgPool1x1(kernel_size=3, stride=1))
motif4.edges[1, 3].update(separable_3x3)
motif4.edges[1, 4].set('op', ops.Identity())
motif4.edges[2, 3].update(separable_3x3)
motif4.edges[2, 4].set('op', ops.MaxPool1x1(kernel_size=3, stride=1))
motif4.edges[3, 4].set('op', ops.AvgPool1x1(kernel_size=3, stride=1))
# Motif 5
motif5 = motif.clone()
motif5.name = "motif5"
motif5.edges[1, 2].set('op', ops.Identity()) # Unclear in paper
motif5.edges[1, 3].update(separable_3x3)
motif5.edges[1, 4].update(separable_3x3)
motif5.edges[2, 3].set('op', ops.Identity())
motif5.edges[2, 4].set('op', ops.Identity())
motif5.edges[3, 4].update(separable_3x3)
# Motif 6
motif6 = motif.clone()
motif6.name = "motif6"
motif6.edges[1, 2].update(depthwise_3x3)
motif6.edges[1, 3].set('op', ops.MaxPool1x1(kernel_size=3, stride=1))
motif6.edges[1, 4].update(separable_3x3)
motif6.edges[2, 3].set('op', ops.MaxPool1x1(kernel_size=3, stride=1))
motif6.edges[2, 4].set('op', ops.Identity())
motif6.edges[3, 4].set('op', ops.AvgPool1x1(kernel_size=3, stride=1))
# Cell
cell = Graph("cell")
cell.add_nodes_from([1, 2, 3, 4, 5])
cell.add_edges_densly()
cell.edges[1, 2].set('op', motif5.copy())
cell.edges[1, 3].set('op', motif3.copy())
cell.edges[1, 4].set('op', motif3.copy())
cell.edges[1, 5].set('op', motif4.copy())
cell.edges[2, 3].set('op', motif3.copy())
cell.edges[2, 4].set('op', motif1.copy())
cell.edges[2, 5].set('op', motif5.copy())
cell.edges[3, 4].set('op', motif3.copy())
cell.edges[3, 5].set('op', motif5.copy())
cell.edges[4, 5].set('op', motif5.copy())
cells = []
channels = [16, 32, 64]
for scope, c in zip(self.OPTIMIZER_SCOPE, channels):
cell_i = cell.copy().set_scope(scope)
# Specify channels
cell_i.update_edges(
update_func=lambda edge: edge.data.update({'C_in': c, 'C_out': c}),
private_edge_data=True
)
cells.append(cell_i)
self.name = "makrograph"
self.add_nodes_from([i for i in range(1, 9)])
self.add_edges_from([(i, i+1) for i in range(1, 8)])
self.edges[1, 2].set('op', ops.Stem(16))
self.edges[2, 3].set('op', cells[0])
self.edges[3, 4].set('op', ops.SepConv(16, 32, kernel_size=3, stride=2, padding=1))
self.edges[4, 5].set('op', cells[1])
self.edges[5, 6].set('op', ops.SepConv(32, 64, kernel_size=3, stride=2, padding=1))
self.edges[6, 7].set('op', cells[2])
self.edges[7, 8].set('op', ops.Sequential(
ops.SepConv(64, 64, kernel_size=3, stride=1, padding=1),
nn.AdaptiveAvgPool2d(1),
nn.Flatten(),
nn.Linear(channels[-1], 10))
)
self.compile()
def __init__(self):
super().__init__()
self.num_classes = self.NUM_CLASSES if hasattr(self, 'NUM_CLASSES') else 10
#
# Cell definition
#
cell = Graph()
cell.name = "cell" # Use the same name for all cells with shared attributes
# Input node
cell.add_node(1)
# Intermediate nodes
cell.add_node(2)
cell.add_node(3)
# Output node
cell.add_node(4)
# Edges
cell.add_edges_densly()
#
# Makrograph definition
#
self.name = "makrograph"
# Cell is on the edges
# 1-2: Preprocessing
# 2-3, ..., 6-7: cells stage 1
# 7-8: residual block stride 2
# 8-9, ..., 12-13: cells stage 2
# 13-14: residual block stride 2
# 14-15, ..., 18-19: cells stage 3
# 19-20: post-processing
total_num_nodes = 20
self.add_nodes_from(range(1, total_num_nodes+1))
self.add_edges_from([(i, i+1) for i in range(1, total_num_nodes)])
channels = [16, 32, 64]
#
# operations at the edges
#
# preprocessing
self.edges[1, 2].set('op', ops.Stem(channels[0]))
# stage 1
for i in range(2, 7):
self.edges[i, i+1].set('op', cell.copy().set_scope('stage_1'))
# stage 2
self.edges[7, 8].set('op', ResNetBasicblock(C_in=channels[0], C_out=channels[1], stride=2))
for i in range(8, 13):
self.edges[i, i+1].set('op', cell.copy().set_scope('stage_2'))
# stage 3
self.edges[13, 14].set('op', ResNetBasicblock(C_in=channels[1], C_out=channels[2], stride=2))
for i in range(14, 19):
self.edges[i, i+1].set('op', cell.copy().set_scope('stage_3'))
# post-processing
self.edges[19, 20].set('op', ops.Sequential(
nn.AdaptiveAvgPool2d(1),
nn.Flatten(),
nn.Linear(channels[-1], self.num_classes)
))
# set the ops at the cells (channel dependent)
for c, scope in zip(channels, self.OPTIMIZER_SCOPE):
self.update_edges(
update_func=lambda edge: _set_cell_ops(edge, C=c),
scope=scope,
private_edge_data=True
)
def plot_cells():
cell = Graph()
cell.add_nodes_from(range(1, 8))
cell.add_edge(1, 3, op="sep_conv_3x3")
cell.add_edge(1, 4, op="identity")
cell.add_edge(1, 5, op="identity")
cell.add_edge(1, 6, op="identity")
cell.add_edge(2, 3, op="sep_conv_3x3")
cell.add_edge(2, 6, op="sep_conv_3x3")
cell.add_edge(3, 4, op="identity")
cell.add_edge(2, 5, op="dil_conv_5x5")
cell.add_edges_from([(i, 7) for i in range(3, 7)])
redu = Graph()
redu.add_nodes_from(range(1, 8))
redu.add_edge(1, 3, op="max_pool_3x3")
redu.add_edge(1, 4, op="max_pool_3x3")
redu.add_edge(1, 5, op="max_pool_3x3")
redu.add_edge(2, 3, op="max_pool_3x3")
redu.add_edge(2, 4, op="max_pool_3x3")
redu.add_edge(2, 5, op="identity")
redu.add_edge(3, 6, op="identity")
redu.add_edge(4, 6, op="identity")
redu.add_edges_from([(i, 7) for i in range(3, 7)])
fig, (ax_top, ax_bot) = plt.subplots(nrows=2, ncols=1)
pos = {
1: [-1, .5],
2: [-1, -.5],
3: [0, .35],
4: [.6, .5],
5: [0, -.5],
6: [.5, 0],
7: [1, 0]
}
nx.draw_networkx_nodes(cell,
pos,
ax=ax_top,
node_color=['g', 'g', 'y', 'y', 'y', 'y', 'm'])
nx.draw_networkx_labels(cell,
pos, {k: str(k)
for k in cell.nodes()},
ax=ax_top)
nx.draw_networkx_edges(cell, pos, ax=ax_top)
nx.draw_networkx_edge_labels(cell,
pos, {(u, v): d.op
for u, v, d in cell.edges(data=True)
if not isinstance(d.op, Identity)},
label_pos=.68,
ax=ax_top,
bbox=dict(facecolor='white',
alpha=0.4,
edgecolor='white'),
font_size=10)
ax_top.set_title("Normal cell")
pos = {
1: [-1, 1],
2: [-1, -1],
3: [0, 1],
4: [0, -.2],
5: [0, -1],
6: [.55, .1],
7: [1, 0]
}
nx.draw_networkx_nodes(redu,
pos,
ax=ax_bot,
node_color=['g', 'g', 'y', 'y', 'y', 'y', 'm'])
nx.draw_networkx_labels(redu,
pos, {k: str(k)
for k in redu.nodes()},
ax=ax_bot)
nx.draw_networkx_edges(redu, pos, ax=ax_bot)
nx.draw_networkx_edge_labels(redu,
pos, {(u, v): d.op
for u, v, d in redu.edges(data=True)
if not isinstance(d.op, Identity)},
label_pos=.68,
ax=ax_bot,
bbox=dict(facecolor='white',
alpha=0.4,
edgecolor='white'),
font_size=10)
ax_bot.set_title("Reduction cell")
plt.tight_layout()
plt.savefig('darts_cells.pdf')
print()
def __init__(self,
classes: int = 10,
intermediate_nodes: int = 2,
cells_per_stage: int = 1,
channels: list = [16, 32]):
"""
Initializes the simple cell search space.
Args:
classes (int): Number of classes. Default: 10.
intermediate_nodes (int): Number of intermediate nodes for normal and
reduction cells. Default: 2.
cells_per_stage (int): Number of normal cells at each stage. Default: 1.
channels (list): Channels for each stage. Must have len 2. Default: [16, 32]
"""
assert len(channels) == len(self.OPTIMIZER_SCOPE), \
"Expecting a channel for each scope. Expected {}, got {}.".format(len(self.OPTIMIZER_SCOPE), len(channels))
super().__init__()
# Cell definition
normal_cell = Graph(
name="normal_cell"
) # Use the same name for all cells with shared attributes
# Nodes
out_node_idx = intermediate_nodes + 3
normal_cell.add_nodes_from(range(1, out_node_idx + 1))
# Edges
normal_cell.add_edges_from([(1, i) for i in range(3, out_node_idx)
]) # input 1
normal_cell.add_edges_from([(2, i) for i in range(3, out_node_idx)
]) # input 2
normal_cell.add_edges_from([(u, v)
for u, v in normal_cell.get_dense_edges()
if u not in [1, 2] and v != out_node_idx])
# Edges connecting to the output are always the identity
normal_cell.add_edges_from([(i, out_node_idx, EdgeData().finalize())
for i in range(3, out_node_idx)]) # output
# set the parameters for the ops at all edges (that are not final)
for k, v in edge_attributes.items():
normal_cell.set_at_edges(k, v)
# Reduction cell
reduction_cell = deepcopy(normal_cell)
reduction_cell.name = "reduction_cell"
reduction_cell.update_edges(update_func=lambda edge: edge.data.set(
'stride', 2) if edge.head in [1, 2] else None)
# Makrograph definition
self.name = "makrograph"
self.add_node(1) # input node
self.add_node(2) # preprocessing
self.add_edge(1, 2) # pre-processing (stem)
j = 3 # index of next node to add
for scope in self.OPTIMIZER_SCOPE:
# reduction cell (beginning of each stage but first)
if j > 3:
input = [j - 2, j - 1]
self.add_node(j,
subgraph=reduction_cell.copy().set_scope(
scope).set_input(input))
self.add_edges_from([(input[0], j), (input[1], j)])
j += 1
# normal cells
for _ in range(cells_per_stage):
# single (copied) input if first node or every normal node after reduction cell
input = [
j - 1, j - 1
] if j == 3 or (j - 3) % (cells_per_stage + 1) == 0 else [
j - 2, j - 1
]
self.add_node(j,
subgraph=normal_cell.copy().set_scope(
scope).set_input(input))
self.add_edges_from([(input[0], j), (input[1], j)])
j += 1
self.add_node(j) # output
self.add_edge(j - 1, j) # post-processing (pooling, classifier)
# Compile the ops
self.edges[1, 2].set(
'op', ops.Stem(channels[0])
) # we can also set a compiled op. Will be ignored by compile()
def set_channels(edge, C):
C_in = C if edge.data.stride == 1 else C // 2
edge.data.set('C_in', C_in)
edge.data.set('C_out', C)
for scope, c in zip(self.OPTIMIZER_SCOPE, channels):
self.update_edges(lambda edge: set_channels(edge, c),
scope,
private_edge_data=True)
self.edges[j - 1, j].set(
'op',
ops.Sequential(nn.AdaptiveAvgPool2d(1), nn.Flatten(),
nn.Linear(channels[-1], classes)))
self.compile()
# Combining operations are currently not considered by compile()
def set_comb_op(node, in_edges, out_edges, C):
if node[0] == out_node_idx:
node[1]['comb_op'] = ops.Concat1x1(
num_in_edges=intermediate_nodes, C_out=C)
for scope, c in zip(self.OPTIMIZER_SCOPE, channels):
self.update_nodes(update_func=lambda node, in_edges, out_edges:
set_comb_op(node, in_edges, out_edges, c),
scope=scope,
single_instances=False)