Esempio n. 1
0
    def __init__(self):
        """
        Initialize a new instance of the DARTS search space.
        Note:
            __init__ cannot take any parameters due to the way networkx is implemented.
            If we want to change the number of classes set a static attribute `NUM_CLASSES`
            before initializing the class. Default is 10 as for cifar-10.
        """
        super().__init__()

        self.channels = [16, 32, 64]
        self.compact = None
        self.load_labeled = None
        self.num_classes = self.NUM_CLASSES if hasattr(self,
                                                       'NUM_CLASSES') else 10
        self.max_epoch = 97
        self.space_name = 'darts'
        """
        Build the search space with the parameters specified in __init__.
        """
        #
        # Cell definition
        #

        # Normal cell first
        normal_cell = Graph()
        normal_cell.name = "normal_cell"  # Use the same name for all cells with shared attributes

        # Input nodes
        normal_cell.add_node(1)
        normal_cell.add_node(2)

        # Intermediate nodes
        normal_cell.add_node(3)
        normal_cell.add_node(4)
        normal_cell.add_node(5)
        normal_cell.add_node(6)

        # Output node
        normal_cell.add_node(7)

        # Edges
        normal_cell.add_edges_from([(1, i) for i in range(3, 7)])  # input 1
        normal_cell.add_edges_from([(2, i) for i in range(3, 7)])  # input 2
        normal_cell.add_edges_from([(3, 4), (3, 5), (3, 6)])
        normal_cell.add_edges_from([(4, 5), (4, 6)])
        normal_cell.add_edges_from([(5, 6)])

        # Edges connecting to the output are always the identity
        normal_cell.add_edges_from([(i, 7, EdgeData().finalize())
                                    for i in range(3, 7)])  # output

        # Reduction cell has the same topology
        reduction_cell = deepcopy(normal_cell)
        reduction_cell.name = "reduction_cell"

        # set the cell name for all edges. This is necessary to convert a genotype to a naslib object
        for _, _, edge_data in normal_cell.edges.data():
            if not edge_data.is_final():
                edge_data.set("cell_name", "normal_cell")

        for _, _, edge_data in reduction_cell.edges.data():
            if not edge_data.is_final():
                edge_data.set("cell_name", "reduction_cell")

        #
        # Makrograph definition
        #
        self.name = "makrograph"

        self.add_node(1)  # input node
        self.add_node(2)  # preprocessing
        self.add_node(3,
                      subgraph=normal_cell.set_scope("n_stage_1").set_input(
                          [2, 2]))
        self.add_node(
            4,
            subgraph=normal_cell.copy().set_scope("n_stage_1").set_input(
                [2, 3]))
        self.add_node(5,
                      subgraph=reduction_cell.set_scope("r_stage_1").set_input(
                          [3, 4]))
        self.add_node(
            6,
            subgraph=normal_cell.copy().set_scope("n_stage_2").set_input(
                [4, 5]))
        self.add_node(
            7,
            subgraph=normal_cell.copy().set_scope("n_stage_2").set_input(
                [5, 6]))
        self.add_node(
            8,
            subgraph=reduction_cell.copy().set_scope("r_stage_2").set_input(
                [6, 7]))
        self.add_node(
            9,
            subgraph=normal_cell.copy().set_scope("n_stage_3").set_input(
                [7, 8]))
        self.add_node(
            10,
            subgraph=normal_cell.copy().set_scope("n_stage_3").set_input(
                [8, 9]))
        self.add_node(11)  # output

        self.add_edges_from([(i, i + 1) for i in range(1, 11)])
        self.add_edges_from([(i, i + 2) for i in range(2, 9)])

        #
        # Operations at the makrograph edges
        #
        self.num_in_edges = 4
        reduction_cell_indices = [5, 8]

        channel_map_from, channel_map_to = channel_maps(reduction_cell_indices,
                                                        max_index=11)

        self._set_makrograph_ops(channel_map_from,
                                 channel_map_to,
                                 reduction_cell_indices,
                                 max_index=11,
                                 affine=False)

        self._set_cell_ops(reduction_cell_indices)
Esempio n. 2
0
    def __init__(self):
        super().__init__()
        self.num_classes = self.NUM_CLASSES if hasattr(self,
                                                       'NUM_CLASSES') else 10

        # creating a dummy graph!
        channels = [128, 256, 512]
        #
        # Cell definition
        #
        node_pair = Graph()
        node_pair.name = "node_pair"  # Use the same name for all cells with shared attributes
        node_pair.set_scope("node")

        # need to add subgraphs on the nodes, each subgraph has option for 3 ops
        # Input node
        node_pair.add_node(1)
        node_pair.add_node(2)
        node_pair.add_edges_from([(1, 2)])

        cell = Graph()
        cell.name = 'cell'

        node_pair.update_edges(
            update_func=lambda edge: _set_node_ops(edge, C=channels[0]),
            private_edge_data=True)

        cell.add_node(1)  # input node
        cell.add_node(2, subgraph=node_pair.set_input([1]))
        cell.add_node(3, subgraph=node_pair.copy())
        cell.add_node(4, subgraph=node_pair.copy())
        cell.add_node(5, subgraph=node_pair.copy())
        cell.add_node(6, subgraph=node_pair.copy())
        cell.add_node(7)  # output
        cell.set_scope('cell', recursively=False)

        # Edges
        cell.add_edges_densly()

        cell.update_edges(
            update_func=lambda edge: _set_cell_ops(edge, C=channels[0]),
            scope="cell",
            private_edge_data=True)
        #
        # dummy Makrograph definition for RE for benchmark queries
        #

        self.name = "makrograph"

        total_num_nodes = 3
        self.add_nodes_from(range(1, total_num_nodes + 1))
        self.add_edges_from([(i, i + 1) for i in range(1, total_num_nodes)])

        self.edges[1, 2].set('op', ops.Stem(channels[0]))
        self.edges[2, 3].set('op', cell.copy().set_scope('cell'))
Esempio n. 3
0
    def __init__(self):
        super().__init__()

        # Define the motifs (6 level-2 motifs)
        level2_motifs = []
        for j in range(6):
            motif = Graph()
            motif.name = "motif{}".format(j)
            motif.add_nodes_from([i for i in range(1, 5)])
            motif.add_edges_densly()

            level2_motifs.append(motif)
        
        # cell (= one level-3 motif)
        cell = Graph()
        cell.name = "cell"
        cell.add_nodes_from([i for i in range(1, 6)])
        cell.add_edges_densly()

        cells = []
        channels = [16, 32, 64]
        for scope, c in zip(self.OPTIMIZER_SCOPE, channels):
            cell_i = cell.copy().set_scope(scope)

            cell_i.update_edges(
                update_func=lambda edge: _set_motifs(edge, motifs=level2_motifs, c=c),
                private_edge_data=True
            )
            

            cell_i.set_scope(scope)

            # set the level 1 motifs (i.e. primitives)
            cell_i.update_edges(
                update_func=lambda edge: _set_cell_ops(edge, c, stride=1),
                scope=[scope],
                private_edge_data=True
            )
            cells.append(cell_i)


        self.name = "makrograph"

        self.add_nodes_from([i for i in range(1, 9)])
        self.add_edges_from([(i, i+1) for i in range(1, 8)])

        self.edges[1, 2].set('op', ops.Stem(16))
        self.edges[2, 3].set('op', cells[0])
        self.edges[3, 4].set('op', ops.SepConv(16, 32, kernel_size=3, stride=2, padding=1))
        self.edges[4, 5].set('op', cells[1])
        self.edges[5, 6].set('op', ops.SepConv(32, 64, kernel_size=3, stride=2, padding=1))
        self.edges[6, 7].set('op', cells[2])
        self.edges[7, 8].set('op', ops.Sequential(
            ops.SepConv(64, 64, kernel_size=3, stride=1, padding=1),
            nn.AdaptiveAvgPool2d(1),
            nn.Flatten(),
            nn.Linear(channels[-1], 10))
        )
Esempio n. 4
0
    def __init__(self):
        logger.info("This is not a search space but the final model found by "
        "Liu et al. For the search space use `HierarchicalSearchSpace()`")
        # skip the init of HierarchicalSearchSpace
        Graph.__init__(self)

        separable_3x3 = {
            'op': ops.SepConv,
            'kernel_size': 3,
            'stride': 1,
            'padding': 1
        }

        depthwise_3x3 = {
            'op': DepthwiseConv,
            'kernel_size': 3,
            'stride': 1,
            'padding': 1
        }

        conv_1x1 = {
            'op': ConvBNReLU,
            'kernel_size': 1,
        }

        motif = Graph()
        motif.add_nodes_from([1, 2, 3, 4])
        motif.add_edges_densly()

        # Motif 1
        motif1 = motif.clone()
        motif1.name = "motif1"
        motif1.edges[1, 2].set('op', ops.MaxPool1x1(kernel_size=3, stride=1))
        motif1.edges[1, 3].update(conv_1x1)
        motif1.edges[1, 4].update(separable_3x3)
        motif1.edges[2, 3].update(depthwise_3x3) 
        motif1.edges[2, 4].update(conv_1x1)
        motif1.edges[3, 4].set('op', ops.MaxPool1x1(kernel_size=3, stride=1))

        # Motif 2
        motif2 = motif.clone()
        motif2.name = "motif2"
        motif2.edges[1, 2].set('op', ops.MaxPool1x1(kernel_size=3, stride=1))
        motif2.edges[1, 3].update(depthwise_3x3)
        motif2.edges[1, 4].update(conv_1x1)
        motif2.edges[2, 3].update(depthwise_3x3) 
        motif2.edges[2, 4].update(separable_3x3)
        motif2.edges[3, 4].update(separable_3x3)

        # Motif 3
        motif3 = motif.clone()
        motif3.name = "motif3"
        motif3.edges[1, 2].update(separable_3x3)
        motif3.edges[1, 3].update(depthwise_3x3)
        motif3.edges[1, 4].update(separable_3x3)
        motif3.edges[2, 3].update(separable_3x3) 
        motif3.edges[2, 4].set('op', ops.Identity())  # assuming no label is identiy
        motif3.edges[3, 4].set('op', ops.AvgPool1x1(kernel_size=3, stride=1))

        # Motif 4
        motif4 = motif.clone()
        motif4.name = "motif4"
        motif4.edges[1, 2].set('op', ops.AvgPool1x1(kernel_size=3, stride=1))
        motif4.edges[1, 3].update(separable_3x3)
        motif4.edges[1, 4].set('op', ops.Identity())
        motif4.edges[2, 3].update(separable_3x3) 
        motif4.edges[2, 4].set('op', ops.MaxPool1x1(kernel_size=3, stride=1))
        motif4.edges[3, 4].set('op', ops.AvgPool1x1(kernel_size=3, stride=1))

        # Motif 5
        motif5 = motif.clone()
        motif5.name = "motif5"
        motif5.edges[1, 2].set('op', ops.Identity())  # Unclear in paper
        motif5.edges[1, 3].update(separable_3x3)
        motif5.edges[1, 4].update(separable_3x3)
        motif5.edges[2, 3].set('op', ops.Identity())
        motif5.edges[2, 4].set('op', ops.Identity())
        motif5.edges[3, 4].update(separable_3x3)

        # Motif 6
        motif6 = motif.clone()
        motif6.name = "motif6"
        motif6.edges[1, 2].update(depthwise_3x3) 
        motif6.edges[1, 3].set('op', ops.MaxPool1x1(kernel_size=3, stride=1))
        motif6.edges[1, 4].update(separable_3x3)
        motif6.edges[2, 3].set('op', ops.MaxPool1x1(kernel_size=3, stride=1))
        motif6.edges[2, 4].set('op', ops.Identity())
        motif6.edges[3, 4].set('op', ops.AvgPool1x1(kernel_size=3, stride=1))

        # Cell
        cell = Graph("cell")
        cell.add_nodes_from([1, 2, 3, 4, 5])
        cell.add_edges_densly()
        cell.edges[1, 2].set('op', motif5.copy())
        cell.edges[1, 3].set('op', motif3.copy())
        cell.edges[1, 4].set('op', motif3.copy())
        cell.edges[1, 5].set('op', motif4.copy())
        cell.edges[2, 3].set('op', motif3.copy())
        cell.edges[2, 4].set('op', motif1.copy())
        cell.edges[2, 5].set('op', motif5.copy())
        cell.edges[3, 4].set('op', motif3.copy())
        cell.edges[3, 5].set('op', motif5.copy())
        cell.edges[4, 5].set('op', motif5.copy())

        cells = []
        channels = [16, 32, 64]
        for scope, c in zip(self.OPTIMIZER_SCOPE, channels):
            cell_i = cell.copy().set_scope(scope)

            # Specify channels
            cell_i.update_edges(
                update_func=lambda edge: edge.data.update({'C_in': c, 'C_out': c}),
                private_edge_data=True
            )
            cells.append(cell_i)

        self.name = "makrograph"

        self.add_nodes_from([i for i in range(1, 9)])
        self.add_edges_from([(i, i+1) for i in range(1, 8)])

        self.edges[1, 2].set('op', ops.Stem(16))
        self.edges[2, 3].set('op', cells[0])
        self.edges[3, 4].set('op', ops.SepConv(16, 32, kernel_size=3, stride=2, padding=1))
        self.edges[4, 5].set('op', cells[1])
        self.edges[5, 6].set('op', ops.SepConv(32, 64, kernel_size=3, stride=2, padding=1))
        self.edges[6, 7].set('op', cells[2])
        self.edges[7, 8].set('op', ops.Sequential(
            ops.SepConv(64, 64, kernel_size=3, stride=1, padding=1),
            nn.AdaptiveAvgPool2d(1),
            nn.Flatten(),
            nn.Linear(channels[-1], 10))
        )

        self.compile()


        
Esempio n. 5
0
    def __init__(self):
        super().__init__()
        self.num_classes = self.NUM_CLASSES if hasattr(self, 'NUM_CLASSES') else 10

        #
        # Cell definition
        #
        cell = Graph()
        cell.name = "cell"    # Use the same name for all cells with shared attributes

        # Input node
        cell.add_node(1)

        # Intermediate nodes
        cell.add_node(2)
        cell.add_node(3)

        # Output node
        cell.add_node(4)

        # Edges
        cell.add_edges_densly()

        #
        # Makrograph definition
        #
        self.name = "makrograph"

        # Cell is on the edges
        # 1-2:               Preprocessing
        # 2-3, ..., 6-7:     cells stage 1
        # 7-8:               residual block stride 2
        # 8-9, ..., 12-13:   cells stage 2
        # 13-14:             residual block stride 2
        # 14-15, ..., 18-19: cells stage 3
        # 19-20:             post-processing

        total_num_nodes = 20
        self.add_nodes_from(range(1, total_num_nodes+1))
        self.add_edges_from([(i, i+1) for i in range(1, total_num_nodes)])

        channels = [16, 32, 64]

        #
        # operations at the edges
        #

        # preprocessing
        self.edges[1, 2].set('op', ops.Stem(channels[0]))
        
        # stage 1
        for i in range(2, 7):
            self.edges[i, i+1].set('op', cell.copy().set_scope('stage_1'))
        
        # stage 2
        self.edges[7, 8].set('op', ResNetBasicblock(C_in=channels[0], C_out=channels[1], stride=2))
        for i in range(8, 13):
            self.edges[i, i+1].set('op', cell.copy().set_scope('stage_2'))

        # stage 3
        self.edges[13, 14].set('op', ResNetBasicblock(C_in=channels[1], C_out=channels[2], stride=2))
        for i in range(14, 19):
            self.edges[i, i+1].set('op', cell.copy().set_scope('stage_3'))

        # post-processing
        self.edges[19, 20].set('op', ops.Sequential(
            nn.AdaptiveAvgPool2d(1),
            nn.Flatten(),
            nn.Linear(channels[-1], self.num_classes)
        ))
        
        # set the ops at the cells (channel dependent)
        for c, scope in zip(channels, self.OPTIMIZER_SCOPE):
            self.update_edges(
                update_func=lambda edge: _set_cell_ops(edge, C=c),
                scope=scope,
                private_edge_data=True
            )
Esempio n. 6
0
def plot_cells():
    cell = Graph()
    cell.add_nodes_from(range(1, 8))
    cell.add_edge(1, 3, op="sep_conv_3x3")
    cell.add_edge(1, 4, op="identity")
    cell.add_edge(1, 5, op="identity")
    cell.add_edge(1, 6, op="identity")
    cell.add_edge(2, 3, op="sep_conv_3x3")
    cell.add_edge(2, 6, op="sep_conv_3x3")
    cell.add_edge(3, 4, op="identity")
    cell.add_edge(2, 5, op="dil_conv_5x5")
    cell.add_edges_from([(i, 7) for i in range(3, 7)])

    redu = Graph()
    redu.add_nodes_from(range(1, 8))
    redu.add_edge(1, 3, op="max_pool_3x3")
    redu.add_edge(1, 4, op="max_pool_3x3")
    redu.add_edge(1, 5, op="max_pool_3x3")
    redu.add_edge(2, 3, op="max_pool_3x3")
    redu.add_edge(2, 4, op="max_pool_3x3")
    redu.add_edge(2, 5, op="identity")
    redu.add_edge(3, 6, op="identity")
    redu.add_edge(4, 6, op="identity")
    redu.add_edges_from([(i, 7) for i in range(3, 7)])

    fig, (ax_top, ax_bot) = plt.subplots(nrows=2, ncols=1)

    pos = {
        1: [-1, .5],
        2: [-1, -.5],
        3: [0, .35],
        4: [.6, .5],
        5: [0, -.5],
        6: [.5, 0],
        7: [1, 0]
    }
    nx.draw_networkx_nodes(cell,
                           pos,
                           ax=ax_top,
                           node_color=['g', 'g', 'y', 'y', 'y', 'y', 'm'])

    nx.draw_networkx_labels(cell,
                            pos, {k: str(k)
                                  for k in cell.nodes()},
                            ax=ax_top)
    nx.draw_networkx_edges(cell, pos, ax=ax_top)
    nx.draw_networkx_edge_labels(cell,
                                 pos, {(u, v): d.op
                                       for u, v, d in cell.edges(data=True)
                                       if not isinstance(d.op, Identity)},
                                 label_pos=.68,
                                 ax=ax_top,
                                 bbox=dict(facecolor='white',
                                           alpha=0.4,
                                           edgecolor='white'),
                                 font_size=10)
    ax_top.set_title("Normal cell")

    pos = {
        1: [-1, 1],
        2: [-1, -1],
        3: [0, 1],
        4: [0, -.2],
        5: [0, -1],
        6: [.55, .1],
        7: [1, 0]
    }
    nx.draw_networkx_nodes(redu,
                           pos,
                           ax=ax_bot,
                           node_color=['g', 'g', 'y', 'y', 'y', 'y', 'm'])

    nx.draw_networkx_labels(redu,
                            pos, {k: str(k)
                                  for k in redu.nodes()},
                            ax=ax_bot)
    nx.draw_networkx_edges(redu, pos, ax=ax_bot)
    nx.draw_networkx_edge_labels(redu,
                                 pos, {(u, v): d.op
                                       for u, v, d in redu.edges(data=True)
                                       if not isinstance(d.op, Identity)},
                                 label_pos=.68,
                                 ax=ax_bot,
                                 bbox=dict(facecolor='white',
                                           alpha=0.4,
                                           edgecolor='white'),
                                 font_size=10)
    ax_bot.set_title("Reduction cell")
    plt.tight_layout()
    plt.savefig('darts_cells.pdf')

    print()
Esempio n. 7
0
    def __init__(self,
                 classes: int = 10,
                 intermediate_nodes: int = 2,
                 cells_per_stage: int = 1,
                 channels: list = [16, 32]):
        """
        Initializes the simple cell search space.

        Args:
            classes (int): Number of classes. Default: 10.
            intermediate_nodes (int): Number of intermediate nodes for normal and
                reduction cells. Default: 2.
            cells_per_stage (int): Number of normal cells at each stage. Default: 1.
            channels (list): Channels for each stage. Must have len 2. Default: [16, 32]
        """
        assert len(channels) == len(self.OPTIMIZER_SCOPE), \
            "Expecting a channel for each scope. Expected {}, got {}.".format(len(self.OPTIMIZER_SCOPE), len(channels))
        super().__init__()

        # Cell definition
        normal_cell = Graph(
            name="normal_cell"
        )  # Use the same name for all cells with shared attributes

        # Nodes
        out_node_idx = intermediate_nodes + 3
        normal_cell.add_nodes_from(range(1, out_node_idx + 1))

        # Edges
        normal_cell.add_edges_from([(1, i) for i in range(3, out_node_idx)
                                    ])  # input 1
        normal_cell.add_edges_from([(2, i) for i in range(3, out_node_idx)
                                    ])  # input 2
        normal_cell.add_edges_from([(u, v)
                                    for u, v in normal_cell.get_dense_edges()
                                    if u not in [1, 2] and v != out_node_idx])
        # Edges connecting to the output are always the identity
        normal_cell.add_edges_from([(i, out_node_idx, EdgeData().finalize())
                                    for i in range(3, out_node_idx)])  # output

        # set the parameters for the ops at all edges (that are not final)
        for k, v in edge_attributes.items():
            normal_cell.set_at_edges(k, v)

        # Reduction cell
        reduction_cell = deepcopy(normal_cell)
        reduction_cell.name = "reduction_cell"

        reduction_cell.update_edges(update_func=lambda edge: edge.data.set(
            'stride', 2) if edge.head in [1, 2] else None)

        # Makrograph definition
        self.name = "makrograph"

        self.add_node(1)  # input node
        self.add_node(2)  # preprocessing
        self.add_edge(1, 2)  # pre-processing (stem)

        j = 3  # index of next node to add
        for scope in self.OPTIMIZER_SCOPE:
            # reduction cell (beginning of each stage but first)
            if j > 3:
                input = [j - 2, j - 1]
                self.add_node(j,
                              subgraph=reduction_cell.copy().set_scope(
                                  scope).set_input(input))
                self.add_edges_from([(input[0], j), (input[1], j)])
                j += 1

            # normal cells
            for _ in range(cells_per_stage):
                # single (copied) input if first node or every normal node after reduction cell
                input = [
                    j - 1, j - 1
                ] if j == 3 or (j - 3) % (cells_per_stage + 1) == 0 else [
                    j - 2, j - 1
                ]
                self.add_node(j,
                              subgraph=normal_cell.copy().set_scope(
                                  scope).set_input(input))
                self.add_edges_from([(input[0], j), (input[1], j)])
                j += 1

        self.add_node(j)  # output
        self.add_edge(j - 1, j)  # post-processing (pooling, classifier)

        # Compile the ops
        self.edges[1, 2].set(
            'op', ops.Stem(channels[0])
        )  # we can also set a compiled op. Will be ignored by compile()

        def set_channels(edge, C):
            C_in = C if edge.data.stride == 1 else C // 2
            edge.data.set('C_in', C_in)
            edge.data.set('C_out', C)

        for scope, c in zip(self.OPTIMIZER_SCOPE, channels):
            self.update_edges(lambda edge: set_channels(edge, c),
                              scope,
                              private_edge_data=True)

        self.edges[j - 1, j].set(
            'op',
            ops.Sequential(nn.AdaptiveAvgPool2d(1), nn.Flatten(),
                           nn.Linear(channels[-1], classes)))

        self.compile()

        # Combining operations are currently not considered by compile()
        def set_comb_op(node, in_edges, out_edges, C):
            if node[0] == out_node_idx:
                node[1]['comb_op'] = ops.Concat1x1(
                    num_in_edges=intermediate_nodes, C_out=C)

        for scope, c in zip(self.OPTIMIZER_SCOPE, channels):
            self.update_nodes(update_func=lambda node, in_edges, out_edges:
                              set_comb_op(node, in_edges, out_edges, c),
                              scope=scope,
                              single_instances=False)