def __init__(self):
super(MatMulNet, self).__init__()
self.matmul = P.MatMul()
self.biasAdd = P.BiasAdd()
def __init__(self, strategy1, strategy2, strategy3):
super().__init__()
self.matmul = P.MatMul().set_strategy(strategy1)
self.logicalnot = P.LogicalNot().set_strategy(strategy2)
self.equal = P.Equal().set_strategy(strategy3)
def __init__(self, strategy1, strategy2):
super().__init__()
self.matmul = P.MatMul().set_strategy(strategy1)
self.pow = P.Pow().set_strategy(strategy2)
self.matmul2 = P.MatMul().set_strategy(strategy1)
def __init__(self, strategy0, strategy1, strategy2, strategy3):
super().__init__()
self.virtual_dataset = _VirtualDataset().set_strategy(strategy0)
self.matmul1 = P.MatMul().set_strategy(strategy1)
self.matmul2 = P.MatMul().set_strategy(strategy2)
self.gelu = P.Gelu().set_strategy(strategy3)
def __init__(self, strategy1, strategy2):
super().__init__()
self.matmul = P.MatMul().set_strategy(strategy1)
self.prelu = P.PReLU().set_strategy(strategy2)
def __init__(self):
super(Net, self).__init__()
self.weight = Parameter(Tensor(np.ones([64, 10]).astype(np.float32)), name='weight')
self.bias = Parameter(Tensor(np.ones([10]).astype(np.float32)), name='bias')
self.matmul = P.MatMul()
self.biasAdd = P.BiasAdd()
def __init__(self, strategy0, strategy1):
super().__init__()
self.fc_nobias = P.MatMul(transpose_b=True).shard(strategy0)
self.reduce_sum = P.ReduceSum(keep_dims=False).shard(strategy1)
def __init__(self, strategy1=None, strategy2=None):
super().__init__()
self.dropout = P.Dropout(keep_prob=0.6).shard(strategy1)
self.matmul = P.MatMul().shard(strategy2)
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
import numpy as np
import mindspore.context as context
import mindspore.nn as nn
import mindspore.ops.functional as F
from mindspore.common import dtype as mstype
from mindspore.common.tensor import Tensor
from mindspore.ops import composite as C
from mindspore.ops import operations as P
context.set_context(mode=context.GRAPH_MODE)
add1 = P.TensorAdd()
mul1 = P.MatMul()
add2 = P.TensorAdd()
def add(x, y):
return add1(x, y)
class Func(nn.Cell):
def __init__(self):
super(Func, self).__init__()
self.alloc_status = P.NPUAllocFloatStatus()
self.get_status = P.NPUGetFloatStatus()
self.clear_status = P.NPUClearFloatStatus()
def construct(self, x, y):
def __init__(self):
super().__init__()
self.matmul1 = P.MatMul(transpose_b=True)
self.matmul2 = P.MatMul()
def __init__(self):
super().__init__()
self.matmul1 = P.MatMul()
self.matmul2 = P.MatMul()
self.transpose1 = P.Transpose()
self.transpose2 = P.Transpose()
def __init__(self, strategy1, strategy2, weight, weight2):
super().__init__()
self.weight = Parameter(weight, "w1", requires_grad=True)
self.weight2 = Parameter(weight2, "w2", requires_grad=True)
self.matmul = P.MatMul().set_strategy(strategy1)
self.matmul2 = P.MatMul().set_strategy(strategy2)
def __init__(self, args, strategy):
super(SemiAutoOneHotNet, self).__init__()
self.a = args.a
self.b = args.b
self.c = args.c
self.d = args.d
self.e = args.e
self.cast = P.Cast()
self.cast.set_strategy(strategy=strategy.twod_strategy)
self.cast1 = P.Cast()
self.cast1.set_strategy(strategy=strategy.twod_strategy)
self.cast2 = P.Cast()
self.cast2.set_strategy(strategy=strategy.twod_strategy)
self.cast3 = P.Cast()
self.cast3.set_strategy(strategy=strategy.scalar_strategy)
self.cast4 = P.Cast()
self.cast4.set_strategy(strategy=strategy.scalar_strategy)
self.a_const = Tensor(self.a, dtype=mstype.float32)
self.b_const = Tensor(self.b, dtype=mstype.float32)
self.c_const = Tensor(self.c, dtype=mstype.float32)
self.d_const = Tensor(self.d, dtype=mstype.float32)
self.e_const = Tensor(self.e, dtype=mstype.float32)
self.m_const_zero = Tensor(0, dtype=mstype.float32)
self.a_const_one = Tensor(1, dtype=mstype.float32)
self.onehot = P.OneHot()
self.onehot.set_strategy(strategy=strategy.onehot_strategy)
self.exp = P.Exp()
self.exp.set_strategy(strategy=strategy.twod_strategy)
self.exp2 = P.Exp()
self.exp2.set_strategy(strategy=strategy.twod_strategy)
self.exp3 = P.Exp()
self.exp3.set_strategy(strategy=strategy.twod_strategy)
self.mul_const = P.Mul()
self.mul_const.set_strategy(strategy=strategy.scalar_twod_strategy)
self.mul_const2 = P.TensorAdd()
self.mul_const2.set_strategy(strategy=strategy.scalar_twod_strategy)
self.mul_const3 = P.Sub()
self.mul_const3.set_strategy(strategy=strategy.twod_scalar_strategy)
self.mul_const4 = P.Sub()
self.mul_const4.set_strategy(strategy=strategy.scalar_twod_strategy)
self.mul_const5 = P.Mul()
self.mul_const5.set_strategy(strategy=strategy.twod_scalar_strategy)
self.mul = P.Mul()
self.mul.set_strategy(strategy=strategy.twod_twod_strategy)
self.mul2 = P.Mul()
self.mul2.set_strategy(strategy=strategy.twod_twod_strategy)
self.mul3 = P.TensorAdd()
self.mul3.set_strategy(strategy=strategy.twod_twod_strategy)
self.mul4 = P.Sub()
self.mul4.set_strategy(strategy=strategy.twod_twodbc_strategy)
self.mul5 = P.RealDiv()
self.mul5.set_strategy(strategy=strategy.twod_twodbc_strategy)
self.mul6 = P.Mul()
self.mul6.set_strategy(strategy=strategy.twod_twod_strategy)
self.mul7 = P.Mul()
self.mul7.set_strategy(strategy=strategy.twod_scalar_strategy)
self.mul8 = P.RealDiv()
self.mul8.set_strategy(strategy=strategy.scalar_scalar_strategy)
self.mul9 = P.TensorAdd()
self.mul9.set_strategy(strategy=strategy.twod_scalar_strategy)
self.reduce_max = P.ReduceMax(keep_dims=True)
self.reduce_max.set_strategy(strategy=strategy.twod_strategy)
self.reduce_sum = P.ReduceSum(keep_dims=False)
self.reduce_sum.set_strategy(strategy=strategy.twod_strategy)
self.reduce_sum_2 = P.ReduceSum(keep_dims=False)
self.reduce_sum_2.set_strategy(strategy=strategy.twod_strategy)
self.reduce_sum_3 = P.ReduceSum(keep_dims=False)
self.reduce_sum_3.set_strategy(strategy=strategy.oned_strategy)
self.reshape = P.Reshape()
self.log = P.Log()
self.log.set_strategy(strategy=strategy.twod_strategy)
self.on_value = Tensor(1.0, mstype.float32)
self.off_value = Tensor(0.0, mstype.float32)
self.normalize = P.L2Normalize(axis=1)
self.normalize.set_strategy(strategy=strategy.twod_strategy_m)
self.normalize2 = P.L2Normalize(axis=1)
self.normalize2.set_strategy(strategy=strategy.twod_strategy_m)
self.fc = P.MatMul(transpose_b=True)
self.fc.set_strategy(strategy=strategy.twodbc_twod_strategy)
weight_shape = [args.num_classes, args.emb_size]
weight_np = np.zeros(weight_shape, np.float32)
self.weight = Parameter(Tensor(weight_np),
name='model_parallel_weight')
def __init__(self):
super(NetWithoutWeight, self).__init__()
self.matmul = P.MatMul()
def __init__(self, strategy1, strategy2):
super().__init__()
self.matmul = P.MatMul(transpose_b=True).shard(strategy1)
self.gelu = P.Gelu().shard(strategy2)
def __init__(self,
params,
learning_rate,
momentum,
matrix_A,
matrix_G,
A_inv_max,
G_inv_max,
weight_decay=0.0,
loss_scale=1.0,
use_nesterov=False,
decay_filter=lambda x: x.name not in []):
super(THOR_GPU, self).__init__(learning_rate, params, weight_decay,
loss_scale)
validator.check_value_type("momentum", momentum, [float],
self.cls_name)
if isinstance(momentum, float) and momentum < 0.0:
raise ValueError(
"momentum should be at least 0.0, but got momentum {}".format(
momentum))
self.momentum = Parameter(Tensor(momentum, mstype.float32),
name="momentum")
self.params = self.parameters
self.use_nesterov = check_bool(use_nesterov)
self.moments = self.params.clone(prefix="moments", init='zeros')
self.hyper_map = C.HyperMap()
self.opt = _selected_ops.ApplyMomentum(use_nesterov=self.use_nesterov)
self.feature_map = [
1.0 / 12544, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136,
1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136,
1.0 / 3136, 1.0 / 3136, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784,
1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784,
1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 196, 1.0 / 196, 1.0 / 196,
1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196,
1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196,
1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 49, 1.0 / 49,
1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49,
1.0 / 49, 1.0
]
self.feature_map_new = [x**0.5 for x in self.feature_map]
self.transpose = P.Transpose()
self.shape = P.Shape()
self.reshape = P.Reshape()
self.matmul = P.MatMul()
self.matrix_A = ParameterTuple(matrix_A)
self.matrix_G = ParameterTuple(matrix_G)
self.A_inv_max = ParameterTuple(A_inv_max)
self.G_inv_max = ParameterTuple(G_inv_max)
self.assign = P.Assign()
self.mul = P.Mul()
mean = _get_gradients_mean()
degree = _get_device_num()
parameter_length = len(self.feature_map)
self.grad_reducer_thorA = DistributedGradReducerThor(
parameter_length, ((parameter_length, ), 0), mean, degree)
self.grad_reducer_thorG = DistributedGradReducerThor(
parameter_length, ((parameter_length, ), 0), mean, degree)
self.weight_decay = weight_decay
self.decay_flags = tuple(decay_filter(x) for x in self.parameters)
self.update_gradient = P.UpdateThorGradient(split_dim=128)
def __init__(self, strategy1):
super().__init__()
self.matmul1 = P.MatMul().set_strategy(strategy1)
self.weight = Parameter(Tensor(np.ones([512, 256]).astype(np.float32) * 0.01), "w", requires_grad=True)
self.matmul2 = P.MatMul()
def __init__(self, strategy1, strategy2, weight):
super().__init__()
self.weight = Parameter(weight, "w1")
self.matmul = P.MatMul(transpose_a=False,
transpose_b=True).set_strategy(strategy1)
self.relu = P.ReLU().set_strategy(strategy2)
def __init__(self):
super().__init__()
self.matmul1 = P.MatMul()
self.matmul2 = P.MatMul()
self.matmul3 = P.MatMul()
def __init__(self, strategy0, strategy1, strategy2):
super(ReshapeNet, self).__init__()
self.relu = P.ReLU().shard(strategy0)
self.reshape = P.Reshape().shard(strategy1)
self.matmul = P.MatMul().shard(strategy2)
self.matmul_weight = Parameter(Tensor(np.ones([25088, 256]), dtype=ms.float32), name="weight")
def __init__(self, strategy1, strategy2):
super().__init__()
self.matmul1 = P.MatMul().shard(strategy1)
self.matmul2 = P.MatMul().shard(strategy2)
def __init__(self):
super(MEGeluLargeIn, self).__init__()
self.matmul = P.MatMul()
self.fast_gelu = P.GeLU()
('FloorDiv', {
'block':
P.FloorDiv(),
'desc_inputs': [
Tensor(np.random.rand(4).astype(np.float16)),
Tensor(np.random.rand(4).astype(np.float16))
],
'skip': ['backward']
}),
('identity', {
'block': ops.functional.identity,
'desc_inputs': [[2, 2]],
'skip': ['backward']
}),
('MatMul_1', {
'block': P.MatMul(transpose_a=False, transpose_b=False),
'desc_inputs': [[1024, 160], [160, 1024]],
'desc_bprop': [[1024, 1024]]
}),
('MatMul_2', {
'block': P.MatMul(transpose_a=True, transpose_b=True),
'desc_inputs': [[160, 1024], [1024, 160]],
'desc_bprop': [[1024, 1024]]
}),
('Sub', {
'block': P.Sub(),
'desc_inputs': [[3], [3]],
'desc_bprop': [[3]]
}),
('TruncatedNormal', {
'block': P.TruncatedNormal(),
def __init__(self):
super().__init__()
self.matmul1 = P.MatMul()
self.dropout = nn.Dropout()
self.matmul2 = P.MatMul()
def __init__(self, strategy1, strategy2):
super().__init__()
self.matmul = P.MatMul().set_strategy(strategy1)
self.log = P.Log().set_strategy(strategy2)
self.matmul2 = P.MatMul().set_strategy(strategy1)
def __init__(self, transpose_a=False, transpose_b=False):
super(CustomMatMul, self).__init__()
self.fc = P.MatMul(transpose_a=transpose_a, transpose_b=transpose_b)
def __init__(self, strategy1):
super().__init__()
self.matmul = P.MatMul().set_strategy(strategy1)
self.cast = P.Cast()
def __init__(self):
super().__init__()
self.matmul = P.MatMul(transpose_b=True)
self.gelu = P.Gelu()
def __init__(self, strategy1, strategy2):
super().__init__()
self.matmul = P.MatMul().set_strategy(strategy1)
self.exp = P.Exp().set_strategy(strategy2)
self.matmul2 = P.MatMul().set_strategy(strategy1)
def __init__(self):
super(AllToAllNet, self).__init__()
self.matmul = P.MatMul()
self.matmul_weight = Parameter(Tensor(np.ones([128, 32]), dtype=ms.float32), name="weight")
self.transpose1 = P.Transpose()
