def train_mindspore_impl(self): input_ = Tensor(np.random.randn(self.batch_num, self.input_channels).astype(np.float32)) weight_np = Tensor(np.random.randn(self.output_channels, self.input_channels).astype(np.float32)) bias = Tensor(np.random.randn(self.output_channels).astype(np.float32)) label_np = np.random.randint(self.output_channels, size=self.batch_num) label_np_onehot = np.zeros(shape=(self.batch_num, self.output_channels)).astype(np.float32) label_np_onehot[np.arange(self.batch_num), label_np] = 1.0 label = Tensor(label_np_onehot) ms_dense = Dense(in_channels=self.input_channels, out_channels=self.output_channels, weight_init=weight_np, bias_init=bias, has_bias=True) criterion = SoftmaxCrossEntropyWithLogits() optimizer = nn.Adam(ms_dense.trainable_params(), learning_rate=1e-3, beta1=0.9, beta2=0.999, eps=self.epsilon, use_locking=False, use_nesterov=False, weight_decay=0.0, loss_scale=1.0) net_with_criterion = WithLossCell(ms_dense, criterion) train_network = TrainOneStepCell(net_with_criterion, optimizer) train_network.set_train() print('MS Initialized!') for _ in range(self.epoch): train_network(input_, label) output = ms_dense(input_) print("===============output=================", output) return output.asnumpy()
def __init__(self): super(LeNet, self).__init__() self.relu = P.ReLU() self.batch_size = 1 weight1 = Tensor(np.ones([6, 3, 5, 5]).astype(np.float32) * 0.01) weight2 = Tensor(np.ones([16, 6, 5, 5]).astype(np.float32) * 0.01) self.conv1 = nn.Conv2d(3, 6, (5, 5), weight_init=weight1, stride=1, padding=0, pad_mode='valid') self.conv2 = nn.Conv2d(6, 16, (5, 5), weight_init=weight2, pad_mode='valid', stride=1, padding=0) self.pool = nn.MaxPool2d(kernel_size=2, stride=2, pad_mode="valid") self.reshape = P.Reshape() self.reshape1 = P.Reshape() self.fc1 = Dense(400, 120) self.fc2 = Dense(120, 84) self.fc3 = Dense(84, 10)
def __init__(self, strategy_dict=None): super().__init__() shared_np = np.full((16, 1, 32, 32), 0.5, dtype=np.float32) self.shared_weight = Parameter(Tensor(shared_np), name='shared_weight') self.fc1 = Dense(in_channels=1024, out_channels=116, weight_init='ones', bias_init='ones', has_bias=True) self.relu = ReLU() self.sigmoid = P.Sigmoid() self.add1 = P.TensorAdd() self.add2 = P.TensorAdd() self.mul1 = P.Mul().add_prim_attr('primitive_target', 'CPU') self.mul2 = P.Mul() self.mul3 = P.Mul() self.flatten = Flatten() mul2_weight_np = np.full((16, 116), 1, dtype=np.float32) self.mul2_weight = Parameter(Tensor(mul2_weight_np), name='mul2_weight') mul3_weight_np = np.full((16, 116), 1, dtype=np.float32) self.mul3_weight = Parameter(Tensor(mul3_weight_np), name='mul3_weight') if strategy_dict is not None: self.add1.shard(strategy_dict['add1']) self.mul1.shard(strategy_dict['mul1']) self.fc1.matmul.shard(strategy_dict['fc1_matmul']) self.fc1.bias_add.shard(strategy_dict['fc1_bias_add']) self.mul2.shard(strategy_dict['mul2']) self.mul3.shard(strategy_dict['mul3'])
def __init__(self): super(MomentumNet, self).__init__() self.batch_size = 1 self.reshape = P.Reshape() weight = Tensor(np.ones([10, 16]).astype(np.float32) * 0.01) self.fc1 = Dense(16, 10, weight_init=weight)
def test_dense_nobias(): input_data = Tensor(np.ones([16, 8]).astype(np.float32)) kernel = Tensor(np.ones([8, 8]).astype(np.float32)) fc = Dense(8, 8, kernel, has_bias=False) output = fc(input_data) output_np = output.asnumpy() print(output_np)
def __init__(self, in_dim, out_dim): super(GCNAggregator, self).__init__() self.add = ops.add() self.div = ops.TensorDiv() self.spmm = ops.SparseDenseMatmul() self.fc = Dense(in_dim, out_dim) self.relu = ReLU()
def fc_with_initialize(input_channels, out_channels): weight_shape = (out_channels, input_channels) bias_shape = (out_channels) weight = random_normal_init(weight_shape) bias = weight_variable_0(bias_shape) return Dense(input_channels, out_channels, weight, bias)
def __init__(self, input_channel, out_channel): super(Net, self).__init__() weight_init1 = np.ones([64, 128]).astype(np.float32) weight_init2 = np.ones([32, 64]).astype(np.float32) self.weight1 = Parameter(Tensor(weight_init1), "loss_weight1", layerwise_parallel=True) self.weight2 = Parameter(Tensor(weight_init2), "loss_weight2", layerwise_parallel=True) self.fc = P.MatMul(transpose_b=True) self.dense = Dense(input_channel, out_channel)
def __init__(self, input_channel, out_channel): super(AllGatherNet, self).__init__() self.dense = Dense(input_channel, out_channel) if GlobalComm.BACKEND is Backend.HCCL: self.allgather = AllGather(group=HCCL_WORLD_COMM_GROUP) elif GlobalComm.BACKEND is Backend.NCCL: self.allgather = AllGather(group=NCCL_WORLD_COMM_GROUP) else: self.allgather = AllGather() self.relu = ReLU()
def __init__(self, batch_size, input_channel, out_channel): super(AllSwapNet, self).__init__() self.dense = Dense(input_channel, out_channel) self.allswap = AllSwap() self.relu = ReLU() self.reduce = ReduceSum() part_slice = batch_size / 2 self.send_size = Tensor( [0, part_slice * out_channel, part_slice * out_channel], mindspore.int64) self.recv_size = Tensor( [part_slice * out_channel, part_slice * out_channel, 0], mindspore.int64)
def __init__(self, batch_size, input_channel, out_channel): super(AllSwapNet, self).__init__() self.dense = Dense(input_channel, out_channel) self.allswap = AllSwap() self.relu = ReLU() part_slice = batch_size / 2 self.send_size = Tensor( [0, part_slice * out_channel, part_slice * out_channel], mindspore.int64) self.recv_size = Tensor( [part_slice * out_channel, part_slice * out_channel, 0], mindspore.int64) self.gatherv2 = GatherV2() self.input = Tensor(np.ones([1]), mindspore.int32)
def __init__(self, in_channel, out_channel, axis, input_shape, mul_size, test_size, prelu_size, transpose_b, matmul_size, num_class): super().__init__() mul_np = np.full(mul_size, 0.5, dtype=np.float32) self.mul_weight = Parameter(Tensor(mul_np), name="mul_weight") bias_np = np.full((12, ), 7.1, dtype=np.float32) self.bias = Parameter(Tensor(bias_np), name="bias") prelu_np = np.full(prelu_size, 0.8, dtype=np.float32) self.prelu_weight = Parameter(Tensor(prelu_np), name="prelu_weight") matmul_np = np.full(matmul_size, 1.1, dtype=np.float32) self.matmul_weight = Parameter(Tensor(matmul_np), name="matmul_weight") self.mul = P.Mul() self.conv = Conv2d(in_channels=in_channel, out_channels=out_channel, kernel_size=5, has_bias=True, weight_init='ones', bias_init='ones', pad_mode='valid') self.scalar = 0.5 self.parameter = Parameter(initializer(0.5, test_size, dtype=mstype.float32), name='parameter') self.tensor = Tensor(np.full(test_size, 0.05, dtype=np.float32)) self.softmax = Softmax(axis=axis) self.relu = ReLU() self.relu.relu.add_prim_attr("primitive_target", "CPU") self.reshape = P.Reshape() self.input_shape = input_shape self.equal = P.Equal() self.cast = P.Cast() self.concat = P.Concat(axis=1) self.reduce_sum = P.ReduceSum() self.bias_add = P.BiasAdd() self.cos = P.Cos() self.prelu = P.PReLU() self.matmul = P.MatMul(transpose_b=transpose_b) self.l2norm = P.L2Normalize(axis=(1 - axis)) self.tensoradd = P.TensorAdd() self.strided_slice = P.StridedSlice() self.dense = Dense(in_channels=6, out_channels=num_class, weight_init='ones', bias_init='ones', has_bias=True)
def __init__(self, in_channel, out_channel): super().__init__() self.relu = PReLU(channel=in_channel, w=0.25) self.bn = BatchNorm2d(num_features=in_channel) self.conv = Conv2d(in_channels=in_channel, out_channels=out_channel, kernel_size=2, stride=1, has_bias=False, weight_init='ones', pad_mode='same') self.mean = P.ReduceMean(keep_dims=False) self.fc = Dense(in_channels=out_channel, out_channels=out_channel, weight_init='ones', bias_init='zeros', has_bias=True)
def __init__(self, dense_in_channel, dense_out_channel, axis=0, shape=None, strategy=None): super().__init__() weight_np = np.full((dense_out_channel, dense_in_channel), 0.01, dtype=np.float32) bias_np = np.full((dense_out_channel), 0.01, dtype=np.float32) self.pack_con = Tensor(np.full(shape, 0.01, dtype=np.float32)) self.flat = Flatten() self.dense = Dense(in_channels=dense_in_channel, out_channels=dense_out_channel, weight_init=Tensor(weight_np), bias_init=Tensor(bias_np), has_bias=True) self.mul = P.Mul() self.pack = P.Stack(axis) if strategy is not None: self.pack.shard(strategy)
def __init__(self, input_channel, out_channel, op): super(ReduceScatterNet, self).__init__() self.dense = Dense(input_channel, out_channel) self.reducescatter = ReduceScatter(op) self.relu = ReLU()
def __init__(self, input_channel, output_channel): super(HostAllGatherNet, self).__init__() self.dense = Dense(input_channel, output_channel) self.hostallgather = HostAllGather((0, 1)) self.relu = ReLU()
def __init__(self, input_channel, out_channel): super(AlltoAllNet, self).__init__() self.dense = Dense(input_channel, out_channel) self.alltoall = _AlltoAll(1, 0, 1) self.relu = ReLU()
def __init__(self, in_dim, out_dim, num_classes): super(SingleLayerGCN, self).__init__() self.aggregator = GCNAggregator(in_dim, out_dim) self.output_layer = Dense(out_dim, num_classes)
def __init__(self, input_channel, out_channel): super(BroadCastNet, self).__init__() self.dense = Dense(input_channel, out_channel) self.broadcast = Broadcast(0)
def __init__(self, input_channel, out_channel, op): super(AllReduceNet, self).__init__() self.dense = Dense(input_channel, out_channel) self.reduce = AllReduce(op) self.relu = ReLU()
def __init__(self, kernel, bias, in_channel, num_class): super().__init__() self.relu = P.ReLU() self.mean = P.ReduceMean(keep_dims=False) self.dense = Dense(in_channel, num_class, kernel, bias)
def fc_with_initialize(input_channels, out_channels): return Dense(input_channels, out_channels)
def __init__(self, input_channel, out_channel, op): super(HostReduceScatterNet, self).__init__() self.dense = Dense(input_channel, out_channel) self.hostreducescatter = HostReduceScatter(op, (0, 1)) self.relu = ReLU()