def train_mindspore_impl(self):
input_ = Tensor(np.random.randn(self.batch_num, self.input_channels).astype(np.float32))
weight_np = Tensor(np.random.randn(self.output_channels, self.input_channels).astype(np.float32))
bias = Tensor(np.random.randn(self.output_channels).astype(np.float32))
label_np = np.random.randint(self.output_channels, size=self.batch_num)
label_np_onehot = np.zeros(shape=(self.batch_num, self.output_channels)).astype(np.float32)
label_np_onehot[np.arange(self.batch_num), label_np] = 1.0
label = Tensor(label_np_onehot)
ms_dense = Dense(in_channels=self.input_channels,
out_channels=self.output_channels,
weight_init=weight_np,
bias_init=bias, has_bias=True)
criterion = SoftmaxCrossEntropyWithLogits()
optimizer = nn.Adam(ms_dense.trainable_params(),
learning_rate=1e-3,
beta1=0.9, beta2=0.999, eps=self.epsilon,
use_locking=False,
use_nesterov=False, weight_decay=0.0,
loss_scale=1.0)
net_with_criterion = WithLossCell(ms_dense, criterion)
train_network = TrainOneStepCell(net_with_criterion, optimizer)
train_network.set_train()
print('MS Initialized!')
for _ in range(self.epoch):
train_network(input_, label)
output = ms_dense(input_)
print("===============output=================", output)
return output.asnumpy()
def __init__(self):
super(LeNet, self).__init__()
self.relu = P.ReLU()
self.batch_size = 1
weight1 = Tensor(np.ones([6, 3, 5, 5]).astype(np.float32) * 0.01)
weight2 = Tensor(np.ones([16, 6, 5, 5]).astype(np.float32) * 0.01)
self.conv1 = nn.Conv2d(3,
6, (5, 5),
weight_init=weight1,
stride=1,
padding=0,
pad_mode='valid')
self.conv2 = nn.Conv2d(6,
16, (5, 5),
weight_init=weight2,
pad_mode='valid',
stride=1,
padding=0)
self.pool = nn.MaxPool2d(kernel_size=2, stride=2, pad_mode="valid")
self.reshape = P.Reshape()
self.reshape1 = P.Reshape()
self.fc1 = Dense(400, 120)
self.fc2 = Dense(120, 84)
self.fc3 = Dense(84, 10)
def __init__(self, strategy_dict=None):
super().__init__()
shared_np = np.full((16, 1, 32, 32), 0.5, dtype=np.float32)
self.shared_weight = Parameter(Tensor(shared_np), name='shared_weight')
self.fc1 = Dense(in_channels=1024,
out_channels=116,
weight_init='ones',
bias_init='ones',
has_bias=True)
self.relu = ReLU()
self.sigmoid = P.Sigmoid()
self.add1 = P.TensorAdd()
self.add2 = P.TensorAdd()
self.mul1 = P.Mul().add_prim_attr('primitive_target', 'CPU')
self.mul2 = P.Mul()
self.mul3 = P.Mul()
self.flatten = Flatten()
mul2_weight_np = np.full((16, 116), 1, dtype=np.float32)
self.mul2_weight = Parameter(Tensor(mul2_weight_np),
name='mul2_weight')
mul3_weight_np = np.full((16, 116), 1, dtype=np.float32)
self.mul3_weight = Parameter(Tensor(mul3_weight_np),
name='mul3_weight')
if strategy_dict is not None:
self.add1.shard(strategy_dict['add1'])
self.mul1.shard(strategy_dict['mul1'])
self.fc1.matmul.shard(strategy_dict['fc1_matmul'])
self.fc1.bias_add.shard(strategy_dict['fc1_bias_add'])
self.mul2.shard(strategy_dict['mul2'])
self.mul3.shard(strategy_dict['mul3'])
def __init__(self):
super(MomentumNet, self).__init__()
self.batch_size = 1
self.reshape = P.Reshape()
weight = Tensor(np.ones([10, 16]).astype(np.float32) * 0.01)
self.fc1 = Dense(16, 10, weight_init=weight)
def test_dense_nobias():
input_data = Tensor(np.ones([16, 8]).astype(np.float32))
kernel = Tensor(np.ones([8, 8]).astype(np.float32))
fc = Dense(8, 8, kernel, has_bias=False)
output = fc(input_data)
output_np = output.asnumpy()
print(output_np)
def __init__(self, in_dim, out_dim):
super(GCNAggregator, self).__init__()
self.add = ops.add()
self.div = ops.TensorDiv()
self.spmm = ops.SparseDenseMatmul()
self.fc = Dense(in_dim, out_dim)
self.relu = ReLU()
def fc_with_initialize(input_channels, out_channels):
weight_shape = (out_channels, input_channels)
bias_shape = (out_channels)
weight = random_normal_init(weight_shape)
bias = weight_variable_0(bias_shape)
return Dense(input_channels, out_channels, weight, bias)
def __init__(self, input_channel, out_channel):
super(Net, self).__init__()
weight_init1 = np.ones([64, 128]).astype(np.float32)
weight_init2 = np.ones([32, 64]).astype(np.float32)
self.weight1 = Parameter(Tensor(weight_init1), "loss_weight1", layerwise_parallel=True)
self.weight2 = Parameter(Tensor(weight_init2), "loss_weight2", layerwise_parallel=True)
self.fc = P.MatMul(transpose_b=True)
self.dense = Dense(input_channel, out_channel)
def __init__(self, input_channel, out_channel):
super(AllGatherNet, self).__init__()
self.dense = Dense(input_channel, out_channel)
if GlobalComm.BACKEND is Backend.HCCL:
self.allgather = AllGather(group=HCCL_WORLD_COMM_GROUP)
elif GlobalComm.BACKEND is Backend.NCCL:
self.allgather = AllGather(group=NCCL_WORLD_COMM_GROUP)
else:
self.allgather = AllGather()
self.relu = ReLU()
def __init__(self, batch_size, input_channel, out_channel):
super(AllSwapNet, self).__init__()
self.dense = Dense(input_channel, out_channel)
self.allswap = AllSwap()
self.relu = ReLU()
self.reduce = ReduceSum()
part_slice = batch_size / 2
self.send_size = Tensor(
[0, part_slice * out_channel, part_slice * out_channel],
mindspore.int64)
self.recv_size = Tensor(
[part_slice * out_channel, part_slice * out_channel, 0],
mindspore.int64)
def __init__(self, batch_size, input_channel, out_channel):
super(AllSwapNet, self).__init__()
self.dense = Dense(input_channel, out_channel)
self.allswap = AllSwap()
self.relu = ReLU()
part_slice = batch_size / 2
self.send_size = Tensor(
[0, part_slice * out_channel, part_slice * out_channel],
mindspore.int64)
self.recv_size = Tensor(
[part_slice * out_channel, part_slice * out_channel, 0],
mindspore.int64)
self.gatherv2 = GatherV2()
self.input = Tensor(np.ones([1]), mindspore.int32)
def __init__(self, in_channel, out_channel, axis, input_shape, mul_size,
test_size, prelu_size, transpose_b, matmul_size, num_class):
super().__init__()
mul_np = np.full(mul_size, 0.5, dtype=np.float32)
self.mul_weight = Parameter(Tensor(mul_np), name="mul_weight")
bias_np = np.full((12, ), 7.1, dtype=np.float32)
self.bias = Parameter(Tensor(bias_np), name="bias")
prelu_np = np.full(prelu_size, 0.8, dtype=np.float32)
self.prelu_weight = Parameter(Tensor(prelu_np), name="prelu_weight")
matmul_np = np.full(matmul_size, 1.1, dtype=np.float32)
self.matmul_weight = Parameter(Tensor(matmul_np), name="matmul_weight")
self.mul = P.Mul()
self.conv = Conv2d(in_channels=in_channel,
out_channels=out_channel,
kernel_size=5,
has_bias=True,
weight_init='ones',
bias_init='ones',
pad_mode='valid')
self.scalar = 0.5
self.parameter = Parameter(initializer(0.5,
test_size,
dtype=mstype.float32),
name='parameter')
self.tensor = Tensor(np.full(test_size, 0.05, dtype=np.float32))
self.softmax = Softmax(axis=axis)
self.relu = ReLU()
self.relu.relu.add_prim_attr("primitive_target", "CPU")
self.reshape = P.Reshape()
self.input_shape = input_shape
self.equal = P.Equal()
self.cast = P.Cast()
self.concat = P.Concat(axis=1)
self.reduce_sum = P.ReduceSum()
self.bias_add = P.BiasAdd()
self.cos = P.Cos()
self.prelu = P.PReLU()
self.matmul = P.MatMul(transpose_b=transpose_b)
self.l2norm = P.L2Normalize(axis=(1 - axis))
self.tensoradd = P.TensorAdd()
self.strided_slice = P.StridedSlice()
self.dense = Dense(in_channels=6,
out_channels=num_class,
weight_init='ones',
bias_init='ones',
has_bias=True)
def __init__(self, in_channel, out_channel):
super().__init__()
self.relu = PReLU(channel=in_channel, w=0.25)
self.bn = BatchNorm2d(num_features=in_channel)
self.conv = Conv2d(in_channels=in_channel,
out_channels=out_channel,
kernel_size=2,
stride=1,
has_bias=False,
weight_init='ones',
pad_mode='same')
self.mean = P.ReduceMean(keep_dims=False)
self.fc = Dense(in_channels=out_channel,
out_channels=out_channel,
weight_init='ones',
bias_init='zeros',
has_bias=True)
def __init__(self,
dense_in_channel,
dense_out_channel,
axis=0,
shape=None,
strategy=None):
super().__init__()
weight_np = np.full((dense_out_channel, dense_in_channel),
0.01,
dtype=np.float32)
bias_np = np.full((dense_out_channel), 0.01, dtype=np.float32)
self.pack_con = Tensor(np.full(shape, 0.01, dtype=np.float32))
self.flat = Flatten()
self.dense = Dense(in_channels=dense_in_channel,
out_channels=dense_out_channel,
weight_init=Tensor(weight_np),
bias_init=Tensor(bias_np),
has_bias=True)
self.mul = P.Mul()
self.pack = P.Stack(axis)
if strategy is not None:
self.pack.shard(strategy)
def __init__(self, input_channel, out_channel, op):
super(ReduceScatterNet, self).__init__()
self.dense = Dense(input_channel, out_channel)
self.reducescatter = ReduceScatter(op)
self.relu = ReLU()
def __init__(self, input_channel, output_channel):
super(HostAllGatherNet, self).__init__()
self.dense = Dense(input_channel, output_channel)
self.hostallgather = HostAllGather((0, 1))
self.relu = ReLU()
def __init__(self, input_channel, out_channel):
super(AlltoAllNet, self).__init__()
self.dense = Dense(input_channel, out_channel)
self.alltoall = _AlltoAll(1, 0, 1)
self.relu = ReLU()
def __init__(self, in_dim, out_dim, num_classes):
super(SingleLayerGCN, self).__init__()
self.aggregator = GCNAggregator(in_dim, out_dim)
self.output_layer = Dense(out_dim, num_classes)
def __init__(self, input_channel, out_channel):
super(BroadCastNet, self).__init__()
self.dense = Dense(input_channel, out_channel)
self.broadcast = Broadcast(0)
def __init__(self, input_channel, out_channel, op):
super(AllReduceNet, self).__init__()
self.dense = Dense(input_channel, out_channel)
self.reduce = AllReduce(op)
self.relu = ReLU()
def __init__(self, kernel, bias, in_channel, num_class):
super().__init__()
self.relu = P.ReLU()
self.mean = P.ReduceMean(keep_dims=False)
self.dense = Dense(in_channel, num_class, kernel, bias)
def fc_with_initialize(input_channels, out_channels):
return Dense(input_channels, out_channels)
def __init__(self, input_channel, out_channel, op):
super(HostReduceScatterNet, self).__init__()
self.dense = Dense(input_channel, out_channel)
self.hostreducescatter = HostReduceScatter(op, (0, 1))
self.relu = ReLU()
