def __init__(self, key, prev_keys, hidden_units, choose_from_k, cnn_keep_prob, lstm_keep_prob, att_keep_prob, att_mask): super(Layer, self).__init__(key) def conv_shortcut(kernel_size): return ConvBN(kernel_size, hidden_units, hidden_units, cnn_keep_prob, False, True) self.n_candidates = len(prev_keys) if self.n_candidates: self.prec = mutables.InputChoice( choose_from=prev_keys[-choose_from_k:], n_chosen=1) else: # first layer, skip input choice self.prec = None self.op = mutables.LayerChoice([ conv_shortcut(1), conv_shortcut(3), conv_shortcut(5), conv_shortcut(7), AvgPool(3, False, True), MaxPool(3, False, True), RNN(hidden_units, lstm_keep_prob), Attention(hidden_units, 4, att_keep_prob, att_mask) ]) if self.n_candidates: self.skipconnect = mutables.InputChoice(choose_from=prev_keys) else: self.skipconnect = None self.bn = BatchNorm(hidden_units, False, True)
def __init__(self, node_id, num_prev_nodes, channels, num_downsample_connect): super().__init__() self.ops = nn.ModuleList() choice_keys = [] for i in range(num_prev_nodes): stride = 2 if i < num_downsample_connect else 1 choice_keys.append("{}_p{}".format(node_id, i)) self.ops.append( mutables.LayerChoice([ ops.PoolBN('max', channels, 3, stride, 1, affine=False), ops.PoolBN('avg', channels, 3, stride, 1, affine=False), nn.Identity() if stride == 1 else ops.FactorizedReduce( channels, channels, affine=False), ops.SepConv(channels, channels, 3, stride, 1, affine=False), ops.SepConv(channels, channels, 5, stride, 2, affine=False), ops.DilConv( channels, channels, 3, stride, 2, 2, affine=False), ops.DilConv( channels, channels, 5, stride, 4, 2, affine=False) ], key=choice_keys[-1])) self.drop_path = ops.DropPath() self.input_switch = mutables.InputChoice( choose_from=choice_keys, n_chosen=2, key="{}_switch".format(node_id))
def __init__(self, key, prev_labels, in_filters, out_filters): super().__init__(key) self.in_filters = in_filters self.out_filters = out_filters self.mutable = mutables.LayerChoice([ # ConvBranch(in_filters, out_filters, kernel_size=3, stride=1, separable=False), # ConvBranch(in_filters, out_filters, kernel_size=3, stride=1, separable=True), ConvBranch(in_filters, out_filters, kernel_size=5, stride=1, separable=False), # ConvBranch(in_filters, out_filters, kernel_size=5, stride=1, separable=True), # ConvBranch(in_filters, out_filters, kernel_size=7, stride=1, separable=False), # ConvBranch(in_filters, out_filters, kernel_size=7, stride=1, separable=True), # ConvBranch(in_filters, out_filters, kernel_size=9, stride=1, separable=False), ConvBranch(in_filters, out_filters, kernel_size=41, stride=1, separable=True), # ResidualConvBranch(in_filters, out_filters, kernel_size=3, stride=1, separable=False), # ResidualConvBranch(in_filters, out_filters, kernel_size=3, stride=1, separable=True), # ResidualConvBranch(in_filters, out_filters, kernel_size=5, stride=1, separable=False), # ResidualConvBranch(in_filters, out_filters, kernel_size=5, stride=1, separable=True), # ResidualConvBranch(in_filters, out_filters, kernel_size=7, stride=1, separable=False), # ResidualConvBranch(in_filters, out_filters, kernel_size=7, stride=1, separable=True), # ResidualConvBranch(in_filters, out_filters, kernel_size=9, stride=1, separable=False), # ResidualConvBranch(in_filters, out_filters, kernel_size=9, stride=1, separable=True), # PoolBranch('avg', in_filters, out_filters, kernel_size=3, stride=1), # PoolBranch('max', in_filters, out_filters, kernel_size=3, stride=1), ]) if len(prev_labels) > 0: self.skipconnect = mutables.InputChoice(choose_from=prev_labels, n_chosen=None) else: self.skipconnect = None self.batch_norm = nn.BatchNorm1d(out_filters, affine=False)
def __init__(self, node_id, num_prev_nodes, channels, num_downsample_connect): super().__init__() self.ops = nn.ModuleList() choice_keys = [] for i in range(num_prev_nodes): stride = 2 if i < num_downsample_connect else 1 choice_keys.append("{}_p{}".format(node_id, i)) self.ops.append(mutables.LayerChoice([ops.OPS[k](channels, stride, False) for k in ops.PRIMITIVES], key=choice_keys[-1])) self.drop_path = ops.DropPath() self.input_switch = mutables.InputChoice(choose_from=choice_keys, n_chosen=2, key="{}_switch".format(node_id))
def __init__(self, cell_name, prev_labels, channels): super().__init__() self.input_choice = mutables.InputChoice(choose_from=prev_labels, n_chosen=1, return_mask=True, key=cell_name + "_input") self.op_choice = mutables.LayerChoice([ SepConvBN(channels, channels, 3, 1), SepConvBN(channels, channels, 5, 2), Pool("avg", 3, 1, 1), Pool("max", 3, 1, 1), nn.Identity() ], key=cell_name + "_op")
def __init__(self, key, prev_labels, out_shape): super().__init__(key) self.mutable = mutables.LayerChoice([ block1(out_shape), block2(out_shape), block3(out_shape), block4(out_shape), block5(out_shape), ]) if len(prev_labels) > 0: self.skipconnect = mutables.InputChoice(choose_from=prev_labels, n_chosen=None) else: self.skipconnect = None self.batch_norm = nn.BatchNorm1d(out_shape)
def __init__(self, node_id, num_prev_nodes, out_shape): super().__init__() self.ops = nn.ModuleList() choice_keys = [] for i in range(num_prev_nodes): choice_keys.append("{}_p{}".format(node_id, i)) self.ops.append( mutables.LayerChoice(OrderedDict([ ("block1", block1(out_shape)), ("block2", block2(out_shape)), ("block3", block3(out_shape)), ("block4", block4(out_shape)), ("block5", block5(out_shape)) ]), key=choice_keys[-1])) self.input_switch = mutables.InputChoice(choose_from=choice_keys, n_chosen=2, key="{}_switch".format(node_id))
def _setup_forward_callback_submodule( deepcv_module: 'deepcv.meta.base_module.DeepcvModule', subm_name: str, submodule_params: Dict[str, Any], forward_callback_module: 'deepcv.submodule_creators.ForwardCallbackSubmodule' ) -> Tuple[str, Optional[torch.nn.Module]]: """ Specfic model definition logic for submodules based on forward pass callbacks (`deepcv.meta.submodule_creators.ForwardCallbackSubmodule` submodule instances are handled sperately allowing output tensor (residual/dense) references and NNI NAS Mutable InputChoice support). Allows referencing other submodule(s) output tensor(s) (`yaml_tokens.FROM` usage) and NNI NAS Mutable InputChoice (`yaml_tokens.FROM_NAS_INPUT_CHOICE` usage). """ deepcv_module._uses_forward_callback_submodules = True # yaml_tokens.FROM_NAS_INPUT_CHOICE occurences in `submodule_params` are handled like yaml_tokens.FROM entries: nni_mutables.InputChoice(references) + optional parameters 'n_chosen' (None by default, should be an integer between 1 and number of candidates) if yaml_tokens.FROM_NAS_INPUT_CHOICE in submodule_params: deepcv_module._uses_nni_nas_mutables = True n_chosen = submodule_params[ 'n_chosen'] if yaml_tokens.FROM_NAS_INPUT_N_CHOSEN in submodule_params else None n_candidates = len(submodule_params[yaml_tokens.FROM_NAS_INPUT_CHOICE]) mask = submodule_params[ yaml_tokens. NAS_MUTABLE_RETURN_MASK] if yaml_tokens.NAS_MUTABLE_RETURN_MASK in submodule_params else False forward_callback_module.mutable_input_choice = nni_mutables.InputChoice( n_candidates=n_candidates, n_chosen=n_chosen, return_mask=mask, key=subm_name, reduction='none') if yaml_tokens.FROM in submodule_params: raise ValueError( f'Error: Cant both specify "{yaml_tokens.FROM}" and "{yaml_tokens.FROM_NAS_INPUT_CHOICE}" in the same submodule ' '(You should either choose to use NNI NAS Mutable InputChoice candidate reference(s) or regular tensor reference(s)).' ) elif yaml_tokens.NAS_MUTABLE_RETURN_MASK in submodule_params or yaml_tokens.FROM_NAS_INPUT_N_CHOSEN: raise ValueError( f'Error: Cannot specify "{yaml_tokens.NAS_MUTABLE_RETURN_MASK}" nor "{yaml_tokens.FROM_NAS_INPUT_N_CHOSEN}" without using "{yaml_tokens.FROM_NAS_INPUT_CHOICE}".' f'("{yaml_tokens.NAS_MUTABLE_RETURN_MASK}" and "{yaml_tokens.FROM_NAS_INPUT_N_CHOSEN}" is an optional parameter reserved for NNI NAS InputChoice usage).' ) # Store any sub-module name/label references (used to store referenced submodule's output features during model's forward pass in order to reuse these features later in a forward callback (e.g. for residual links)) if yaml_tokens.FROM in submodule_params or yaml_tokens.FROM_NAS_INPUT_CHOICE in submodule_params: # Allow multiple referenced sub-module(s) (`yaml_tokens.FROM` entry can either be a list/tuple of referenced sub-modules name/label or a single sub-module name/label) tensor_references = submodule_params[ yaml_tokens. FROM] if yaml_tokens.FROM in submodule_params else submodule_params[ yaml_tokens.FROM_NAS_INPUT_CHOICE] forward_callback_module.referenced_submodules = [ tensor_references, ] if isinstance(tensor_references, str) else list(tensor_references)
def __init__(self, node_id, num_prev_nodes, channels, num_downsample_connect): """ builtin Darts Node structure Parameters --- node_id: str num_prev_nodes: int the number of previous nodes in this cell channels: int output channels num_downsample_connect: int downsample the input node if this cell is reduction cell """ super().__init__() self.ops = nn.ModuleList() choice_keys = [] for i in range(num_prev_nodes): stride = 2 if i < num_downsample_connect else 1 choice_keys.append("{}_p{}".format(node_id, i)) self.ops.append( mutables.LayerChoice(OrderedDict([ ("maxpool", PoolBN('max', channels, 3, stride, 1, affine=False)), ("avgpool", PoolBN('avg', channels, 3, stride, 1, affine=False)), ("skipconnect", nn.Identity() if stride == 1 else FactorizedReduce(channels, channels, affine=False)), ("sepconv3x3", SepConv(channels, channels, 3, stride, 1, affine=False)), ("sepconv5x5", SepConv(channels, channels, 5, stride, 2, affine=False)), ("dilconv3x3", DilConv(channels, channels, 3, stride, 2, 2, affine=False)), ("dilconv5x5", DilConv(channels, channels, 5, stride, 4, 2, affine=False)) ]), key=choice_keys[-1])) self.drop_path = DropPath() self.input_switch = mutables.InputChoice( choose_from=choice_keys, n_chosen=2, key="{}_switch".format(node_id))
def __init__(self, key, prev_labels, in_filters, out_filters): super().__init__(key) self.in_filters = in_filters self.out_filters = out_filters self.mutable = mutables.LayerChoice([ ConvBranch(in_filters, out_filters, 3, 1, 1, separable=False), ConvBranch(in_filters, out_filters, 3, 1, 1, separable=True), ConvBranch(in_filters, out_filters, 5, 1, 2, separable=False), ConvBranch(in_filters, out_filters, 5, 1, 2, separable=True), PoolBranch('avg', in_filters, out_filters, 3, 1, 1), PoolBranch('max', in_filters, out_filters, 3, 1, 1) ]) if prev_labels > 0: self.skipconnect = mutables.InputChoice(choose_from=prev_labels, n_chosen=None) else: self.skipconnect = None self.batch_norm = nn.BatchNorm2d(out_filters, affine=False)
def __init__(self, node_id, num_prev_nodes, channels, num_downsample_connect): ''' Node("{}_n{}".format("reduce" if reduction else "normal", depth), depth, channels, 2 if reduction else 0) num_prev_nodes: 之前的节点个数 ''' super().__init__() self.ops = nn.ModuleList() choice_keys = [] # 记录 节点+边 组合的名称 for i in range(num_prev_nodes): # 枚举之前的节点 stride = 2 if i < num_downsample_connect else 1 # 统一设置stride # 如果是reduction cell, stride=2, # 如果是normal cell, stride=1 choice_keys.append("{}_p{}".format(node_id, i)) self.ops.append( mutables.LayerChoice(OrderedDict([ ("maxpool", ops.PoolBN('max', channels, 3, stride, 1, affine=False)), ("avgpool", ops.PoolBN('avg', channels, 3, stride, 1, affine=False)), ("skipconnect", nn.Identity() if stride == 1 else ops.FactorizedReduce( channels, channels, affine=False)), ("sepconv3x3", ops.SepConv(channels, channels, 3, stride, 1, affine=False)), ("sepconv5x5", ops.SepConv(channels, channels, 5, stride, 2, affine=False)), ("dilconv3x3", ops.DilConv(channels, channels, 3, stride, 2, 2, affine=False)), ("dilconv5x5", ops.DilConv(channels, channels, 5, stride, 4, 2, affine=False)) ]), key=choice_keys[-1])) self.drop_path = ops.DropPath() # 以0.2的概率drop path self.input_switch = mutables.InputChoice( # 控制连接方式, 维护choice_key就是为了这个使用 choose_from=choice_keys, n_chosen=2, key="{}_switch".format(node_id))