def fc_with_initialize(input_channels, out_channels): return nn.Dense(input_channels, out_channels)
def __init__(self): super(Encoder, self).__init__() self.fc1 = nn.Dense(6, 3) self.relu = nn.ReLU()
def __init__(self): super(Decoder, self).__init__() self.fc1 = nn.Dense(3, 6) self.sigmoid = nn.Sigmoid()
def _fc(in_channel, out_channel): weight = np.random.normal(loc=0, scale=0.01, size=out_channel * in_channel) weight = Tensor(np.reshape(weight, (out_channel, in_channel)), dtype=mstype.float32) return nn.Dense(in_channel, out_channel, has_bias=True, weight_init=weight, bias_init=0)
def __init__(self): super(LinearNet, self).__init__() self.fc = nn.Dense(1, 1, Normal(0.02), Normal(0.02))
def fc_with_initialize(input_channels, out_channels, has_bias=True): return nn.Dense(input_channels, out_channels, has_bias=has_bias)
def test_dense_str_activation(): dense = nn.Dense(1, 1, activation='relu') assert isinstance(dense.activation, nn.ReLU) input_data = Tensor(np.random.randint(0, 255, [1, 1]).astype(np.float32)) dense(input_data)
def __init__(self, n_features, n_classes): super(LogisticRegression, self).__init__() self.model = nn.Dense(n_features, n_classes, TruncatedNormal(0.02), TruncatedNormal(0.02))
def __init__(self): super(Net, self).__init__() self.dense = nn.Dense(10, 10)
def __init__(self, hidden_size): super(Net, self).__init__() self.hidden_size = hidden_size self.fc1 = nn.Dense(2, hidden_size) self.fc2 = nn.Dense(hidden_size, 1) self.sig = ops.Sigmoid()
def __init__(self, has_bias=True, activation='relu'): super(DenseNet1, self).__init__() self.fc1 = nn.Dense(128, 128, has_bias=has_bias, activation=activation) self.fc2 = nn.Dense(128, 128, has_bias=has_bias, activation=activation) self.fc3 = nn.Dense(128, 128, has_bias=has_bias, activation=activation) self.fc4 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)
def _fc(in_channel, out_channel): weight_shape = (out_channel, in_channel) weight = _weight_variable(weight_shape) return nn.Dense(in_channel, out_channel, has_bias=True, weight_init=weight, bias_init=0)
def fc_with_initialize(input_channels, out_channels): return nn.Dense(input_channels, out_channels).add_flags_recursive(fp16=True)
def __init__(self, num_classes, is_training=True, stem_filters=32, penultimate_filters=1056, filters_multiplier=2): super(NASNetAMobile, self).__init__() self.is_training = is_training self.stem_filters = stem_filters self.penultimate_filters = penultimate_filters self.filters_multiplier = filters_multiplier filters = self.penultimate_filters//24 # 24 is default value for the architecture self.conv0 = nn.SequentialCell([ nn.Conv2d(in_channels=3, out_channels=self.stem_filters, kernel_size=3, stride=2, pad_mode='pad', padding=0, has_bias=False), nn.BatchNorm2d(num_features=self.stem_filters, eps=0.001, momentum=0.9, affine=True) ]) self.cell_stem_0 = CellStem0( self.stem_filters, num_filters=filters//(filters_multiplier**2) ) self.cell_stem_1 = CellStem1( self.stem_filters, num_filters=filters//filters_multiplier ) self.cell_0 = FirstCell( in_channels_left=filters, out_channels_left=filters//2, # 1, 0.5 in_channels_right=2*filters, out_channels_right=filters ) # 2, 1 self.cell_1 = NormalCell( in_channels_left=2*filters, out_channels_left=filters, # 2, 1 in_channels_right=6*filters, out_channels_right=filters ) # 6, 1 self.cell_2 = NormalCell( in_channels_left=6*filters, out_channels_left=filters, # 6, 1 in_channels_right=6*filters, out_channels_right=filters ) # 6, 1 self.cell_3 = NormalCell( in_channels_left=6*filters, out_channels_left=filters, # 6, 1 in_channels_right=6*filters, out_channels_right=filters ) # 6, 1 self.reduction_cell_0 = ReductionCell0( in_channels_left=6*filters, out_channels_left=2*filters, # 6, 2 in_channels_right=6*filters, out_channels_right=2*filters ) # 6, 2 self.cell_6 = FirstCell( in_channels_left=6*filters, out_channels_left=filters, # 6, 1 in_channels_right=8*filters, out_channels_right=2*filters ) # 8, 2 self.cell_7 = NormalCell( in_channels_left=8*filters, out_channels_left=2*filters, # 8, 2 in_channels_right=12*filters, out_channels_right=2*filters ) # 12, 2 self.cell_8 = NormalCell( in_channels_left=12*filters, out_channels_left=2*filters, # 12, 2 in_channels_right=12*filters, out_channels_right=2*filters ) # 12, 2 self.cell_9 = NormalCell( in_channels_left=12*filters, out_channels_left=2*filters, # 12, 2 in_channels_right=12*filters, out_channels_right=2*filters ) # 12, 2 if is_training: self.aux_logits = AuxLogits(in_channels=12*filters, out_channels=num_classes) self.reduction_cell_1 = ReductionCell1( in_channels_left=12*filters, out_channels_left=4*filters, # 12, 4 in_channels_right=12*filters, out_channels_right=4*filters ) # 12, 4 self.cell_12 = FirstCell( in_channels_left=12*filters, out_channels_left=2*filters, # 12, 2 in_channels_right=16*filters, out_channels_right=4*filters ) # 16, 4 self.cell_13 = NormalCell( in_channels_left=16*filters, out_channels_left=4*filters, # 16, 4 in_channels_right=24*filters, out_channels_right=4*filters ) # 24, 4 self.cell_14 = NormalCell( in_channels_left=24*filters, out_channels_left=4*filters, # 24, 4 in_channels_right=24*filters, out_channels_right=4*filters ) # 24, 4 self.cell_15 = NormalCell( in_channels_left=24*filters, out_channels_left=4*filters, # 24, 4 in_channels_right=24*filters, out_channels_right=4*filters ) # 24, 4 self.relu = nn.ReLU() self.dropout = nn.Dropout(keep_prob=0.5) self.classifier = nn.Dense(in_channels=24*filters, out_channels=num_classes) self.shape = P.Shape() self.reshape = P.Reshape() self.avg_pool = nn.AvgPool2d(kernel_size=7, stride=1) self._initialize_weights()
def fc_with_initialize(input_channels, out_channels): return nn.Dense(input_channels, out_channels, weight_init='XavierUniform', bias_init='Uniform')
def fc_with_initialize(input_channels, out_channels): """Fc layer weight initial.""" weight = weight_variable() bias = weight_variable() return nn.Dense(input_channels, out_channels, weight, bias)
def __init__(self, from_tensor_width, to_tensor_width, from_seq_length, to_seq_length, num_attention_heads=1, size_per_head=512, query_act=None, key_act=None, value_act=None, has_attention_mask=False, attention_probs_dropout_prob=0.0, use_one_hot_embeddings=False, initializer_range=0.02, do_return_2d_tensor=False, use_relative_positions=False, compute_type=mstype.float32): super(BertAttention, self).__init__() self.from_seq_length = from_seq_length self.to_seq_length = to_seq_length self.num_attention_heads = num_attention_heads self.size_per_head = size_per_head self.has_attention_mask = has_attention_mask self.use_relative_positions = use_relative_positions self.scores_mul = Tensor([1.0 / math.sqrt(float(self.size_per_head))], dtype=compute_type) self.reshape = P.Reshape() self.shape_from_2d = (-1, from_tensor_width) self.shape_to_2d = (-1, to_tensor_width) weight = TruncatedNormal(initializer_range) units = num_attention_heads * size_per_head self.query_layer = nn.Dense(from_tensor_width, units, activation=query_act, weight_init=weight).to_float(compute_type) self.key_layer = nn.Dense(to_tensor_width, units, activation=key_act, weight_init=weight).to_float(compute_type) self.value_layer = nn.Dense(to_tensor_width, units, activation=value_act, weight_init=weight).to_float(compute_type) self.shape_from = (-1, from_seq_length, num_attention_heads, size_per_head) self.shape_to = (-1, to_seq_length, num_attention_heads, size_per_head) self.matmul_trans_b = P.BatchMatMul(transpose_b=True) self.multiply = P.Mul() self.transpose = P.Transpose() self.trans_shape = (0, 2, 1, 3) self.trans_shape_relative = (2, 0, 1, 3) self.trans_shape_position = (1, 2, 0, 3) self.multiply_data = Tensor([ -10000.0, ], dtype=compute_type) self.matmul = P.BatchMatMul() self.softmax = nn.Softmax() self.dropout = nn.Dropout(1 - attention_probs_dropout_prob) if self.has_attention_mask: self.expand_dims = P.ExpandDims() self.sub = P.Sub() self.add = P.TensorAdd() self.cast = P.Cast() self.get_dtype = P.DType() if do_return_2d_tensor: self.shape_return = (-1, num_attention_heads * size_per_head) else: self.shape_return = (-1, from_seq_length, num_attention_heads * size_per_head) self.cast_compute_type = SaturateCast(dst_type=compute_type) if self.use_relative_positions: self._generate_relative_positions_embeddings = \ RelaPosEmbeddingsGenerator(length=to_seq_length, depth=size_per_head, max_relative_position=16, initializer_range=initializer_range, use_one_hot_embeddings=use_one_hot_embeddings)
def _fc(in_channels, out_channels): '''full connection layer''' return nn.Dense(in_channels, out_channels)
def test_dense_none(): with pytest.raises(TypeError): nn.Dense(3, 2, None, None)
def __init__(self, config, is_training, use_one_hot_embeddings=False): super(BertModel, self).__init__() config = copy.deepcopy(config) if not is_training: config.hidden_dropout_prob = 0.0 config.attention_probs_dropout_prob = 0.0 self.input_mask_from_dataset = config.input_mask_from_dataset self.token_type_ids_from_dataset = config.token_type_ids_from_dataset self.batch_size = config.batch_size self.seq_length = config.seq_length self.hidden_size = config.hidden_size self.num_hidden_layers = config.num_hidden_layers self.embedding_size = config.hidden_size self.token_type_ids = None self.last_idx = self.num_hidden_layers - 1 output_embedding_shape = [ self.batch_size, self.seq_length, self.embedding_size ] if not self.token_type_ids_from_dataset: self.token_type_ids = initializer( "zeros", [self.batch_size, self.seq_length], mstype.int32) self.bert_embedding_lookup = EmbeddingLookup( vocab_size=config.vocab_size, embedding_size=self.embedding_size, embedding_shape=output_embedding_shape, use_one_hot_embeddings=use_one_hot_embeddings, initializer_range=config.initializer_range) self.bert_embedding_postprocessor = EmbeddingPostprocessor( embedding_size=self.embedding_size, embedding_shape=output_embedding_shape, use_token_type=True, token_type_vocab_size=config.type_vocab_size, use_one_hot_embeddings=use_one_hot_embeddings, initializer_range=0.02, max_position_embeddings=config.max_position_embeddings, dropout_prob=config.hidden_dropout_prob) self.bert_encoder = BertTransformer( batch_size=self.batch_size, hidden_size=self.hidden_size, seq_length=self.seq_length, num_attention_heads=config.num_attention_heads, num_hidden_layers=self.num_hidden_layers, intermediate_size=config.intermediate_size, attention_probs_dropout_prob=config.attention_probs_dropout_prob, use_one_hot_embeddings=use_one_hot_embeddings, initializer_range=config.initializer_range, hidden_dropout_prob=config.hidden_dropout_prob, use_relative_positions=config.use_relative_positions, hidden_act=config.hidden_act, compute_type=config.compute_type, return_all_encoders=True) self.cast = P.Cast() self.dtype = config.dtype self.cast_compute_type = SaturateCast(dst_type=config.compute_type) self.slice = P.StridedSlice() self.squeeze_1 = P.Squeeze(axis=1) self.dense = nn.Dense(self.hidden_size, self.hidden_size, activation="tanh", weight_init=TruncatedNormal( config.initializer_range)).to_float( config.compute_type) self._create_attention_mask_from_input_mask = CreateAttentionMaskFromInputMask( config)
def test_dense_channels_error(): with pytest.raises(ValueError): nn.Dense(3, 0) with pytest.raises(ValueError): nn.Dense(-1, 2)
def __init__(self): super(DenseNet, self).__init__() w = np.array([[0.1, 0.8, 0.1, 0.1], [1, 1, 1, 1]]).astype(np.float32) b = np.array([0.3, 0.6]).astype(np.float32) self.dense = nn.Dense(4, 2, weight_init=Tensor(w), bias_init=Tensor(b))
def __init__(self, num_classes, out_channels): super(CommonHead, self).__init__() self.avgpool = GlobalAvgPooling() self.fc = nn.Dense(out_channels, num_classes, has_bias=True).add_flags_recursive(fp16=True)
def __init__(self, input_size=224, n_class=1000, model_size='1.0x'): super(ShuffleNetV2, self).__init__() print('model size is ', model_size) self.stage_repeats = [4, 8, 4] self.model_size = model_size if model_size == '0.5x': self.stage_out_channels = [-1, 24, 48, 96, 192, 1024] elif model_size == '1.0x': self.stage_out_channels = [-1, 24, 116, 232, 464, 1024] elif model_size == '1.5x': self.stage_out_channels = [-1, 24, 176, 352, 704, 1024] elif model_size == '2.0x': self.stage_out_channels = [-1, 24, 244, 488, 976, 2048] else: raise NotImplementedError # building first layer input_channel = self.stage_out_channels[1] self.first_conv = nn.SequentialCell([ nn.Conv2d(in_channels=3, out_channels=input_channel, kernel_size=3, stride=2, pad_mode='pad', padding=1, has_bias=False), nn.BatchNorm2d(num_features=input_channel, momentum=0.9), nn.ReLU(), ]) self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode='same') self.features = [] for idxstage in range(len(self.stage_repeats)): numrepeat = self.stage_repeats[idxstage] output_channel = self.stage_out_channels[idxstage + 2] for i in range(numrepeat): if i == 0: self.features.append( ShuffleV2Block(input_channel, output_channel, mid_channels=output_channel // 2, ksize=3, stride=2)) else: self.features.append( ShuffleV2Block(input_channel // 2, output_channel, mid_channels=output_channel // 2, ksize=3, stride=1)) input_channel = output_channel self.features = nn.SequentialCell([*self.features]) self.conv_last = nn.SequentialCell([ nn.Conv2d(in_channels=input_channel, out_channels=self.stage_out_channels[-1], kernel_size=1, stride=1, pad_mode='pad', padding=0, has_bias=False), nn.BatchNorm2d(num_features=self.stage_out_channels[-1], momentum=0.9), nn.ReLU() ]) self.globalpool = nn.AvgPool2d(kernel_size=7, stride=7, pad_mode='valid') if self.model_size == '2.0x': self.dropout = nn.Dropout(keep_prob=0.8) self.classifier = nn.SequentialCell([ nn.Dense(in_channels=self.stage_out_channels[-1], out_channels=n_class, has_bias=False) ]) ##TODO init weights self._initialize_weights()