def fc_with_initialize(input_channels, out_channels):
return nn.Dense(input_channels, out_channels)
def __init__(self):
super(Encoder, self).__init__()
self.fc1 = nn.Dense(6, 3)
self.relu = nn.ReLU()
def __init__(self):
super(Decoder, self).__init__()
self.fc1 = nn.Dense(3, 6)
self.sigmoid = nn.Sigmoid()
def _fc(in_channel, out_channel):
weight = np.random.normal(loc=0, scale=0.01, size=out_channel * in_channel)
weight = Tensor(np.reshape(weight, (out_channel, in_channel)), dtype=mstype.float32)
return nn.Dense(in_channel, out_channel, has_bias=True, weight_init=weight, bias_init=0)
def __init__(self):
super(LinearNet, self).__init__()
self.fc = nn.Dense(1, 1, Normal(0.02), Normal(0.02))
def fc_with_initialize(input_channels, out_channels, has_bias=True):
return nn.Dense(input_channels, out_channels, has_bias=has_bias)
def test_dense_str_activation():
dense = nn.Dense(1, 1, activation='relu')
assert isinstance(dense.activation, nn.ReLU)
input_data = Tensor(np.random.randint(0, 255, [1, 1]).astype(np.float32))
dense(input_data)
def __init__(self, n_features, n_classes):
super(LogisticRegression, self).__init__()
self.model = nn.Dense(n_features, n_classes, TruncatedNormal(0.02),
TruncatedNormal(0.02))
def __init__(self):
super(Net, self).__init__()
self.dense = nn.Dense(10, 10)
def __init__(self, hidden_size):
super(Net, self).__init__()
self.hidden_size = hidden_size
self.fc1 = nn.Dense(2, hidden_size)
self.fc2 = nn.Dense(hidden_size, 1)
self.sig = ops.Sigmoid()
def __init__(self, has_bias=True, activation='relu'):
super(DenseNet1, self).__init__()
self.fc1 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)
self.fc2 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)
self.fc3 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)
self.fc4 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)
def _fc(in_channel, out_channel):
weight_shape = (out_channel, in_channel)
weight = _weight_variable(weight_shape)
return nn.Dense(in_channel, out_channel, has_bias=True, weight_init=weight, bias_init=0)
def fc_with_initialize(input_channels, out_channels):
return nn.Dense(input_channels,
out_channels).add_flags_recursive(fp16=True)
def __init__(self, num_classes, is_training=True,
stem_filters=32, penultimate_filters=1056, filters_multiplier=2):
super(NASNetAMobile, self).__init__()
self.is_training = is_training
self.stem_filters = stem_filters
self.penultimate_filters = penultimate_filters
self.filters_multiplier = filters_multiplier
filters = self.penultimate_filters//24
# 24 is default value for the architecture
self.conv0 = nn.SequentialCell([
nn.Conv2d(in_channels=3, out_channels=self.stem_filters, kernel_size=3, stride=2, pad_mode='pad', padding=0,
has_bias=False),
nn.BatchNorm2d(num_features=self.stem_filters, eps=0.001, momentum=0.9, affine=True)
])
self.cell_stem_0 = CellStem0(
self.stem_filters, num_filters=filters//(filters_multiplier**2)
)
self.cell_stem_1 = CellStem1(
self.stem_filters, num_filters=filters//filters_multiplier
)
self.cell_0 = FirstCell(
in_channels_left=filters,
out_channels_left=filters//2, # 1, 0.5
in_channels_right=2*filters,
out_channels_right=filters
) # 2, 1
self.cell_1 = NormalCell(
in_channels_left=2*filters,
out_channels_left=filters, # 2, 1
in_channels_right=6*filters,
out_channels_right=filters
) # 6, 1
self.cell_2 = NormalCell(
in_channels_left=6*filters,
out_channels_left=filters, # 6, 1
in_channels_right=6*filters,
out_channels_right=filters
) # 6, 1
self.cell_3 = NormalCell(
in_channels_left=6*filters,
out_channels_left=filters, # 6, 1
in_channels_right=6*filters,
out_channels_right=filters
) # 6, 1
self.reduction_cell_0 = ReductionCell0(
in_channels_left=6*filters,
out_channels_left=2*filters, # 6, 2
in_channels_right=6*filters,
out_channels_right=2*filters
) # 6, 2
self.cell_6 = FirstCell(
in_channels_left=6*filters,
out_channels_left=filters, # 6, 1
in_channels_right=8*filters,
out_channels_right=2*filters
) # 8, 2
self.cell_7 = NormalCell(
in_channels_left=8*filters,
out_channels_left=2*filters, # 8, 2
in_channels_right=12*filters,
out_channels_right=2*filters
) # 12, 2
self.cell_8 = NormalCell(
in_channels_left=12*filters,
out_channels_left=2*filters, # 12, 2
in_channels_right=12*filters,
out_channels_right=2*filters
) # 12, 2
self.cell_9 = NormalCell(
in_channels_left=12*filters,
out_channels_left=2*filters, # 12, 2
in_channels_right=12*filters,
out_channels_right=2*filters
) # 12, 2
if is_training:
self.aux_logits = AuxLogits(in_channels=12*filters, out_channels=num_classes)
self.reduction_cell_1 = ReductionCell1(
in_channels_left=12*filters,
out_channels_left=4*filters, # 12, 4
in_channels_right=12*filters,
out_channels_right=4*filters
) # 12, 4
self.cell_12 = FirstCell(
in_channels_left=12*filters,
out_channels_left=2*filters, # 12, 2
in_channels_right=16*filters,
out_channels_right=4*filters
) # 16, 4
self.cell_13 = NormalCell(
in_channels_left=16*filters,
out_channels_left=4*filters, # 16, 4
in_channels_right=24*filters,
out_channels_right=4*filters
) # 24, 4
self.cell_14 = NormalCell(
in_channels_left=24*filters,
out_channels_left=4*filters, # 24, 4
in_channels_right=24*filters,
out_channels_right=4*filters
) # 24, 4
self.cell_15 = NormalCell(
in_channels_left=24*filters,
out_channels_left=4*filters, # 24, 4
in_channels_right=24*filters,
out_channels_right=4*filters
) # 24, 4
self.relu = nn.ReLU()
self.dropout = nn.Dropout(keep_prob=0.5)
self.classifier = nn.Dense(in_channels=24*filters, out_channels=num_classes)
self.shape = P.Shape()
self.reshape = P.Reshape()
self.avg_pool = nn.AvgPool2d(kernel_size=7, stride=1)
self._initialize_weights()
def fc_with_initialize(input_channels, out_channels):
return nn.Dense(input_channels, out_channels, weight_init='XavierUniform', bias_init='Uniform')
def fc_with_initialize(input_channels, out_channels):
"""Fc layer weight initial."""
weight = weight_variable()
bias = weight_variable()
return nn.Dense(input_channels, out_channels, weight, bias)
def __init__(self,
from_tensor_width,
to_tensor_width,
from_seq_length,
to_seq_length,
num_attention_heads=1,
size_per_head=512,
query_act=None,
key_act=None,
value_act=None,
has_attention_mask=False,
attention_probs_dropout_prob=0.0,
use_one_hot_embeddings=False,
initializer_range=0.02,
do_return_2d_tensor=False,
use_relative_positions=False,
compute_type=mstype.float32):
super(BertAttention, self).__init__()
self.from_seq_length = from_seq_length
self.to_seq_length = to_seq_length
self.num_attention_heads = num_attention_heads
self.size_per_head = size_per_head
self.has_attention_mask = has_attention_mask
self.use_relative_positions = use_relative_positions
self.scores_mul = Tensor([1.0 / math.sqrt(float(self.size_per_head))],
dtype=compute_type)
self.reshape = P.Reshape()
self.shape_from_2d = (-1, from_tensor_width)
self.shape_to_2d = (-1, to_tensor_width)
weight = TruncatedNormal(initializer_range)
units = num_attention_heads * size_per_head
self.query_layer = nn.Dense(from_tensor_width,
units,
activation=query_act,
weight_init=weight).to_float(compute_type)
self.key_layer = nn.Dense(to_tensor_width,
units,
activation=key_act,
weight_init=weight).to_float(compute_type)
self.value_layer = nn.Dense(to_tensor_width,
units,
activation=value_act,
weight_init=weight).to_float(compute_type)
self.shape_from = (-1, from_seq_length, num_attention_heads,
size_per_head)
self.shape_to = (-1, to_seq_length, num_attention_heads, size_per_head)
self.matmul_trans_b = P.BatchMatMul(transpose_b=True)
self.multiply = P.Mul()
self.transpose = P.Transpose()
self.trans_shape = (0, 2, 1, 3)
self.trans_shape_relative = (2, 0, 1, 3)
self.trans_shape_position = (1, 2, 0, 3)
self.multiply_data = Tensor([
-10000.0,
], dtype=compute_type)
self.matmul = P.BatchMatMul()
self.softmax = nn.Softmax()
self.dropout = nn.Dropout(1 - attention_probs_dropout_prob)
if self.has_attention_mask:
self.expand_dims = P.ExpandDims()
self.sub = P.Sub()
self.add = P.TensorAdd()
self.cast = P.Cast()
self.get_dtype = P.DType()
if do_return_2d_tensor:
self.shape_return = (-1, num_attention_heads * size_per_head)
else:
self.shape_return = (-1, from_seq_length,
num_attention_heads * size_per_head)
self.cast_compute_type = SaturateCast(dst_type=compute_type)
if self.use_relative_positions:
self._generate_relative_positions_embeddings = \
RelaPosEmbeddingsGenerator(length=to_seq_length,
depth=size_per_head,
max_relative_position=16,
initializer_range=initializer_range,
use_one_hot_embeddings=use_one_hot_embeddings)
def _fc(in_channels, out_channels):
'''full connection layer'''
return nn.Dense(in_channels, out_channels)
def test_dense_none():
with pytest.raises(TypeError):
nn.Dense(3, 2, None, None)
def __init__(self, config, is_training, use_one_hot_embeddings=False):
super(BertModel, self).__init__()
config = copy.deepcopy(config)
if not is_training:
config.hidden_dropout_prob = 0.0
config.attention_probs_dropout_prob = 0.0
self.input_mask_from_dataset = config.input_mask_from_dataset
self.token_type_ids_from_dataset = config.token_type_ids_from_dataset
self.batch_size = config.batch_size
self.seq_length = config.seq_length
self.hidden_size = config.hidden_size
self.num_hidden_layers = config.num_hidden_layers
self.embedding_size = config.hidden_size
self.token_type_ids = None
self.last_idx = self.num_hidden_layers - 1
output_embedding_shape = [
self.batch_size, self.seq_length, self.embedding_size
]
if not self.token_type_ids_from_dataset:
self.token_type_ids = initializer(
"zeros", [self.batch_size, self.seq_length], mstype.int32)
self.bert_embedding_lookup = EmbeddingLookup(
vocab_size=config.vocab_size,
embedding_size=self.embedding_size,
embedding_shape=output_embedding_shape,
use_one_hot_embeddings=use_one_hot_embeddings,
initializer_range=config.initializer_range)
self.bert_embedding_postprocessor = EmbeddingPostprocessor(
embedding_size=self.embedding_size,
embedding_shape=output_embedding_shape,
use_token_type=True,
token_type_vocab_size=config.type_vocab_size,
use_one_hot_embeddings=use_one_hot_embeddings,
initializer_range=0.02,
max_position_embeddings=config.max_position_embeddings,
dropout_prob=config.hidden_dropout_prob)
self.bert_encoder = BertTransformer(
batch_size=self.batch_size,
hidden_size=self.hidden_size,
seq_length=self.seq_length,
num_attention_heads=config.num_attention_heads,
num_hidden_layers=self.num_hidden_layers,
intermediate_size=config.intermediate_size,
attention_probs_dropout_prob=config.attention_probs_dropout_prob,
use_one_hot_embeddings=use_one_hot_embeddings,
initializer_range=config.initializer_range,
hidden_dropout_prob=config.hidden_dropout_prob,
use_relative_positions=config.use_relative_positions,
hidden_act=config.hidden_act,
compute_type=config.compute_type,
return_all_encoders=True)
self.cast = P.Cast()
self.dtype = config.dtype
self.cast_compute_type = SaturateCast(dst_type=config.compute_type)
self.slice = P.StridedSlice()
self.squeeze_1 = P.Squeeze(axis=1)
self.dense = nn.Dense(self.hidden_size,
self.hidden_size,
activation="tanh",
weight_init=TruncatedNormal(
config.initializer_range)).to_float(
config.compute_type)
self._create_attention_mask_from_input_mask = CreateAttentionMaskFromInputMask(
config)
def test_dense_channels_error():
with pytest.raises(ValueError):
nn.Dense(3, 0)
with pytest.raises(ValueError):
nn.Dense(-1, 2)
def __init__(self):
super(DenseNet, self).__init__()
w = np.array([[0.1, 0.8, 0.1, 0.1], [1, 1, 1, 1]]).astype(np.float32)
b = np.array([0.3, 0.6]).astype(np.float32)
self.dense = nn.Dense(4, 2, weight_init=Tensor(w), bias_init=Tensor(b))
def __init__(self, num_classes, out_channels):
super(CommonHead, self).__init__()
self.avgpool = GlobalAvgPooling()
self.fc = nn.Dense(out_channels, num_classes,
has_bias=True).add_flags_recursive(fp16=True)
def __init__(self, input_size=224, n_class=1000, model_size='1.0x'):
super(ShuffleNetV2, self).__init__()
print('model size is ', model_size)
self.stage_repeats = [4, 8, 4]
self.model_size = model_size
if model_size == '0.5x':
self.stage_out_channels = [-1, 24, 48, 96, 192, 1024]
elif model_size == '1.0x':
self.stage_out_channels = [-1, 24, 116, 232, 464, 1024]
elif model_size == '1.5x':
self.stage_out_channels = [-1, 24, 176, 352, 704, 1024]
elif model_size == '2.0x':
self.stage_out_channels = [-1, 24, 244, 488, 976, 2048]
else:
raise NotImplementedError
# building first layer
input_channel = self.stage_out_channels[1]
self.first_conv = nn.SequentialCell([
nn.Conv2d(in_channels=3,
out_channels=input_channel,
kernel_size=3,
stride=2,
pad_mode='pad',
padding=1,
has_bias=False),
nn.BatchNorm2d(num_features=input_channel, momentum=0.9),
nn.ReLU(),
])
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode='same')
self.features = []
for idxstage in range(len(self.stage_repeats)):
numrepeat = self.stage_repeats[idxstage]
output_channel = self.stage_out_channels[idxstage + 2]
for i in range(numrepeat):
if i == 0:
self.features.append(
ShuffleV2Block(input_channel,
output_channel,
mid_channels=output_channel // 2,
ksize=3,
stride=2))
else:
self.features.append(
ShuffleV2Block(input_channel // 2,
output_channel,
mid_channels=output_channel // 2,
ksize=3,
stride=1))
input_channel = output_channel
self.features = nn.SequentialCell([*self.features])
self.conv_last = nn.SequentialCell([
nn.Conv2d(in_channels=input_channel,
out_channels=self.stage_out_channels[-1],
kernel_size=1,
stride=1,
pad_mode='pad',
padding=0,
has_bias=False),
nn.BatchNorm2d(num_features=self.stage_out_channels[-1],
momentum=0.9),
nn.ReLU()
])
self.globalpool = nn.AvgPool2d(kernel_size=7,
stride=7,
pad_mode='valid')
if self.model_size == '2.0x':
self.dropout = nn.Dropout(keep_prob=0.8)
self.classifier = nn.SequentialCell([
nn.Dense(in_channels=self.stage_out_channels[-1],
out_channels=n_class,
has_bias=False)
])
##TODO init weights
self._initialize_weights()
