def construct(self, x):
if self.update:
min_up, max_up = self.ema_update(x, self.minq, self.maxq)
out = self.fake_quant(x, min_up, max_up)
P.Assign()(self.minq, min_up)
P.Assign()(self.maxq, max_up)
else:
out = self.fake_quant(x, self.minq, self.maxq)
return out
def construct(self, x):
if self.is_ascend and self.training:
min_up, max_up = self.ema_update(x, self.minq, self.maxq)
out = self.fake_quant(x, min_up, max_up)
P.Assign()(self.minq, min_up)
P.Assign()(self.maxq, max_up)
else:
out = self.fake_quant(x, self.minq, self.maxq)
return out
def __init__(self, input_shape):
super().__init__()
self.addn = op.AddN()
self.assign = op.Assign()
self.input_data = Parameter(initializer(1, input_shape,
mstype.float32),
name="var")
def __init__(self, params, learning_rate, momentum, matrix_A, matrix_G, A_inv_max, G_inv_max, weight_decay=0.0,
loss_scale=1.0,
decay_filter=lambda x: x.name not in []):
super(THOR, self).__init__(learning_rate, params, weight_decay, loss_scale)
if isinstance(momentum, float) and momentum < 0.0:
raise ValueError("momentum should be at least 0.0, but got momentum {}".format(momentum))
self.momentum = Parameter(Tensor(momentum, mstype.float32))
self.params = self.parameters
self.moments = self.params.clone(prefix="moments", init='zeros')
self.hyper_map = C.HyperMap()
self.opt = P.ApplyMomentum()
self.matrix_A = ParameterTuple(matrix_A)
self.matrix_G = ParameterTuple(matrix_G)
self.A_inv_max = ParameterTuple(A_inv_max)
self.G_inv_max = ParameterTuple(G_inv_max)
self.cube_matmul_left = P.CusMatMulCubeFraczLeftCast()
self.cube_matmul_left_fc = P.CusMatMulCubeDenseLeft()
self.cube_matmul_right_fc = P.CusMatMulCubeDenseRight()
self.cube_matmul_right_mul = P.CusMatMulCubeFraczRightMul()
self.transpose = P.Transpose()
self.shape = P.Shape()
self.reshape = P.Reshape()
self.mul = P.Mul()
self.weight_idx = []
for i in range(len(self.params)):
if "conv" in self.params[i].name or "end_point" in self.params[i].name:
self.weight_idx.append(i)
self.weight_idx.append(len(self.params))
self.feature_map = [1.0 / 12544, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136,
1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136,
1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784,
1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784,
1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196,
1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196,
1.0 / 196, 1.0 / 196, 1.0 / 196,
1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49,
1.0]
mean = _get_gradients_mean()
degree = _get_device_num()
parameter_length = len(self.feature_map)
self.grad_reducer_Amax = DistributedGradReducerThor(parameter_length, ((27,), 2), mean, degree)
self.grad_reducer_Gmax = DistributedGradReducerThor(parameter_length, ((27,), 4), mean, degree)
self.grad_reducer_A = DistributedGradReducerThor(parameter_length, ((27,), 6), mean, degree)
self.grad_reducer_G = DistributedGradReducerThor(parameter_length, ((27,), 8), mean, degree)
self.matrix_A_inv = ()
self.matrix_G_inv = ()
self.matrix_max_inv = ()
for i in range(54):
self.matrix_max_inv = self.matrix_max_inv + (
Parameter(initializer(1, [1], mstype.float32), name="matrix_max" + str(i), requires_grad=False),)
self.log = P.Log()
self.exp = P.Exp()
self.sqrt = P.Sqrt()
self.matrix_max_inv = ParameterTuple(self.matrix_max_inv)
self.assign = P.Assign()
self.cast = P.Cast()
self.thor = True
self.weight_decay = weight_decay * loss_scale
self.decay_flags = tuple(decay_filter(x) for x in self.parameters)
def __init__(self, in_channel, x):
super().__init__()
#self._save_graphs(save_graph_flag=True, save_graph_path=".")
self.biasadd = P.BiasAdd()
self.equal = P.Equal()
self.addn = P.AddN()
self.conv = Conv2d(in_channels=in_channel,
out_channels=in_channel,
kernel_size=1,
stride=1,
has_bias=False,
weight_init='ones',
pad_mode='same')
self.bn = BatchNorm2d(num_features=in_channel)
self.assignadd = P.AssignAdd()
self.assign = P.Assign()
self.relu = ReLU()
self.mean = P.ReduceMean(keep_dims=False)
self.bias = Parameter(Tensor(
np.random.randint(2, size=(3, )).astype((np.float32))),
name="bias")
self.bias2 = Parameter(Tensor(np.ones([3]).astype(np.float32)),
name="bias2")
self.parameterupdate = ParameterUpdate(self.bias)
self.value = Tensor(np.random.randn(*(3, )), ms.float32)
self.x = x
def __init__(self):
super().__init__()
self.parameter1 = Parameter(
Tensor([1.0], ms.float32), name="parameter1")
self.assign = P.Assign()
self.addN = P.AddN()
self.relu = P.ReLU()
def __init__(self):
super(Assign_WAR, self).__init__()
self.assign = P.Assign()
self.sub = P.Sub()
self.add = P.Add()
self.para = Parameter(Tensor(1, dtype=ms.int32), name='para')
self.weight = Parameter(Tensor(5, dtype=ms.int32), name='weight')
def __init__(self):
super().__init__()
self.add = P.Add()
self.sub = P.Sub()
self.assign = P.Assign()
self.param_a = Parameter(Tensor(5, mstype.int32), name='a')
self.param_b = Parameter(Tensor(11, mstype.int32), name='b')
def __init__(self):
super().__init__()
self.parameter1 = Parameter(Tensor([199.0], ms.float32), name="parameter1")
self.assign = P.Assign()
self.assignadd = P.AssignAdd()
self.addn = P.AddN()
self.depend = P.Depend()
def __init__(self):
super().__init__()
self.mul = P.Mul()
self.addn = P.AddN()
self.assign = P.Assign()
self.assign_sub = P.AssignSub()
self.para = Parameter(Tensor(1.0, dtype=ms.float32), name='para')
def __init__(self, params, learning_rate, momentum, matrix_A, matrix_G, weight_decay=0.0,
loss_scale=1.0, num_hidden_layers=24, batch_size=12, damping=0.03,
decay_filter=lambda x: 'layernorm' not in x.name.lower() and 'bias' not in x.name.lower()):
super(THOR, self).__init__(learning_rate, params, weight_decay, loss_scale)
if isinstance(momentum, float) and momentum < 0.0:
raise ValueError("momentum should be at least 0.0, but got momentum {}".format(momentum))
self.momentum = Parameter(Tensor(momentum, mstype.float32), name="momentum")
self.params = self.parameters
self.moments = self.params.clone(prefix="moments", init='zeros')
self.hyper_map = C.HyperMap()
self.opt = P.ApplyMomentum()
self.matrix_A = ParameterTuple(matrix_A)
self.matrix_G = ParameterTuple(matrix_G)
self.matmul = P.MatMul()
self.transpose = P.Transpose()
self.shape = P.Shape()
self.reshape = P.Reshape()
self.mul = P.Mul()
self.gather = P.GatherV2()
self.matrix_A_inv = ()
self.matrix_G_inv = ()
self.num_hidden_layers = num_hidden_layers
self.sqrt = P.Sqrt()
self.assign = P.Assign()
self.cast = P.Cast()
self.thor = True
self.weight_decay = weight_decay * loss_scale
self.decay_flags = tuple(decay_filter(x) for x in self.parameters)
self.expand = P.ExpandDims()
self.square = P.Square()
self.inv = P.Inv()
self.batch_size = batch_size
self.damping = damping
self.one = Tensor(1, mstype.int32)
self.cov_step = Parameter(initializer(0, [1], mstype.int32), name="cov_step", requires_grad=False)
def __init__(self):
super().__init__()
self.pow = P.Pow()
self.print = P.Print()
self.assign = P.Assign()
self.exponent = Tensor([2.0], ms.float32)
self.para1 = Parameter(Tensor(1.0, dtype=ms.float32), name='para1')
self.para2 = Parameter(Tensor(3.0, dtype=ms.float32), name='para2')
def __init__(self):
super(StateNet, self).__init__()
weight = Tensor(np.ones([2, 1, 2, 2], np.float32))
self.s1 = Parameter(weight, name="s1")
self.s2 = Parameter(weight, name="s2")
self.sub = P.Sub()
self.loss = nn.SoftmaxCrossEntropyWithLogits()
self.assign = P.Assign()
def __init__(self):
super().__init__()
self.mul = P.Mul()
self.add = P.Add()
self.sub = P.Sub()
self.assign = P.Assign()
self.param_a = Parameter(Tensor(np.array(5), mstype.int32),
name='a')
self.param_b = Parameter(Tensor(np.array(2), mstype.int32),
name='b')
def __init__(self):
super().__init__()
self.mul = P.Mul()
self.add = P.Add()
self.sub = P.Sub()
self.assign = P.Assign()
param_a = np.full((1, ), 5, dtype=np.float32)
self.param_a = Parameter(Tensor(param_a), name='a')
param_b = np.full((1, ), 2, dtype=np.float32)
self.param_b = Parameter(Tensor(param_b), name='b')
def __init__(self):
super().__init__()
self.cast = P.Cast()
self.dtype = ms.float16
np.random.seed(5)
inputs1 = np.random.randn(5, 5)
inputs2 = np.random.randn(5, 5)
self.parameter_a = Parameter(Tensor(inputs1, ms.float32), name="a")
self.parameter_b = Parameter(Tensor(inputs2, ms.float32), name="b")
self.assign = P.Assign()
def __init__(self):
super().__init__()
self.parameter1 = Parameter(
Tensor([1.0], ms.float32), name="parameter1")
self.parameter2 = Parameter(
Tensor([3.0], ms.float32), name="parameter2")
self.assign = P.Assign()
self.addn = P.AddN()
self.mul = P.Mul()
self.print = P.Print()
def __init__(self):
super().__init__()
self.mul = P.Mul()
self.add = P.Add()
self.sub = P.Sub()
self.div = P.Div()
self.assign = P.Assign()
self.param_a = Parameter(Tensor(5, mstype.int32), name='a')
self.param_b = Parameter(Tensor(2, mstype.int32), name='b')
self.param_c = Parameter(Tensor(20, mstype.int32), name='c')
def __init__(self):
super().__init__()
self.assign_sub = P.Assign()
self.mul = P.Mul()
self.mul_weight = Parameter(Tensor(
np.full([128, 32], 0.5, dtype=np.float32)),
name="mul_weight")
self.assignsub_weight = Parameter(Tensor(
np.full([128, 32], 1.1, dtype=np.float32)),
name="assignsub_weight")
def __init__(self):
super().__init__()
inputs = np.array([[1, 1], [1, 1]])
self.parameter1 = Parameter(Tensor(inputs, ms.float32),
name="parameter1")
biasadd = np.array([0, -1])
self.parameter2 = Parameter(Tensor(biasadd, ms.float32),
name="biasadd")
self.assign = P.Assign()
self.matmul = P.MatMul()
self.biasadd = P.BiasAdd()
def construct(self, x, mean, variance, global_step):
if self.is_gpu:
if self.training:
batch_mean, batch_std, running_mean, running_std = self.bn_train(
x, mean, variance, global_step)
else:
batch_mean, batch_std, running_mean, running_std = self.bn_infer(
x, mean, variance, global_step)
else:
if self.training:
x_sum, x_square_sum = self.bn_reduce(x)
_, batch_mean, batch_std, running_mean, running_std, mean_updated, variance_updated = \
self.bn_update(x, x_sum, x_square_sum, mean, variance)
P.Assign()(mean, mean_updated)
P.Assign()(variance, variance_updated)
else:
batch_mean = P.ZerosLike()(variance)
batch_std = P.OnesLike()(variance)
running_mean = P.TensorAdd()(mean, 0.)
running_std = P.Sqrt()(P.TensorAdd()(variance, self.epsilon))
return batch_mean, batch_std, running_mean, running_std
def __init__(self, decay_policy, decay_rate, cur_noise_multiplier, init_noise_multiplier):
super(_MechanismsParamsUpdater, self).__init__()
self._decay_policy = decay_policy
self._decay_rate = decay_rate
self._cur_noise_multiplier = cur_noise_multiplier
self._init_noise_multiplier = init_noise_multiplier
self._div = P.Div()
self._add = P.TensorAdd()
self._assign = P.Assign()
self._sub = P.Sub()
self._one = Tensor(1, mstype.float32)
self._mul = P.Mul()
self._exp = P.Exp()
def __init__(self):
super().__init__()
self.mul = P.Mul()
self.add = P.Add()
self.sub = P.Sub()
self.div = P.Div()
self.relu = nn.ReLU()
self.assign = P.Assign()
param_a = np.full((1, ), 5, dtype=np.int32)
self.param_a = Parameter(Tensor(param_a), name='a')
param_b = np.full((1, ), 2, dtype=np.int32)
self.param_b = Parameter(Tensor(param_b), name='b')
param_c = np.full((1, ), 30, dtype=np.int32)
self.param_c = Parameter(Tensor(param_c), name='c')
def __init__(self, channel=1, w=0.25):
super(PReLU, self).__init__()
if isinstance(w, (np.float32, float)):
tmp = np.empty((channel, ), dtype=np.float32)
tmp.fill(w)
w = Tensor(tmp)
elif isinstance(w, list):
w = Tensor(w)
if not isinstance(w, Tensor):
raise TypeError("w only support np.float32, float or Tensor type.")
self.w = Parameter(initializer(w, [channel]), name='a')
self.prelu = P.PReLU()
self.relu = P.ReLU()
self.assign = P.Assign()
def __init__(self, params, learning_rate, momentum, matrix_A, matrix_G, A_inv_max, G_inv_max,
weight_decay=0.0, loss_scale=1.0, use_nesterov=False, decay_filter=lambda x: x.name not in []):
super(THOR_GPU, self).__init__(learning_rate, params, weight_decay, loss_scale)
Validator.check_value_type("momentum", momentum, [float], self.cls_name)
if isinstance(momentum, float) and momentum < 0.0:
raise ValueError("momentum should be at least 0.0, but got momentum {}".format(momentum))
self.momentum = Parameter(Tensor(momentum, mstype.float32))
self.params = self.parameters
self.use_nesterov = Validator.check_bool(use_nesterov)
self.moments = self.params.clone(prefix="moments", init='zeros')
self.hyper_map = C.HyperMap()
self.opt = P.ApplyMomentum(use_nesterov=self.use_nesterov)
self.feature_map = [1.0 / 12544, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136,
1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136,
1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784,
1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784,
1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196,
1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196,
1.0 / 196, 1.0 / 196, 1.0 / 196,
1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49,
1.0]
self.feature_map_new = [x ** 0.5 for x in self.feature_map]
self.transpose = P.Transpose()
self.shape = P.Shape()
self.reshape = P.Reshape()
self.matmul = P.MatMul()
self.matrix_A = ParameterTuple(matrix_A)
self.matrix_G = ParameterTuple(matrix_G)
self.A_inv_max = ParameterTuple(A_inv_max)
self.G_inv_max = ParameterTuple(G_inv_max)
self.assign = P.Assign()
self.mul = P.Mul()
mean = _get_gradients_mean()
degree = _get_device_num()
parameter_length = len(self.feature_map)
self.grad_reducer_thorA = DistributedGradReducerThor(parameter_length, ((parameter_length,), 0), mean, degree)
self.grad_reducer_thorG = DistributedGradReducerThor(parameter_length, ((parameter_length,), 0), mean, degree)
self.weight_decay = weight_decay
self.decay_flags = tuple(decay_filter(x) for x in self.parameters)
self.update_gradient = P.UpdateThorGradient(split_dim=128)
def __init__(self,
params,
learning_rate,
momentum,
matrix_A,
matrix_G,
A_inv_max,
G_inv_max,
weight_decay=0.0,
loss_scale=1.0,
use_nesterov=False,
decay_filter=lambda x: x.name not in []):
super(SKFAC_GPU, self).__init__(learning_rate, params, weight_decay,
loss_scale)
Validator.check_value_type("momentum", momentum, [float],
self.cls_name)
if isinstance(momentum, float) and momentum < 0.0:
raise ValueError(
"momentum should be at least 0.0, but got momentum {}".format(
momentum))
self.momentum = Parameter(Tensor(momentum, mstype.float32))
self.params = self.parameters
self.use_nesterov = Validator.check_bool(use_nesterov)
self.moments = self.params.clone(prefix="moments", init='zeros')
self.hyper_map = C.HyperMap()
self.opt = P.ApplyMomentum(use_nesterov=self.use_nesterov)
self.transpose = P.Transpose()
self.shape = P.Shape()
self.reshape = P.Reshape()
self.matmul = P.MatMul()
self.matrix_A = ParameterTuple(matrix_A)
self.matrix_G = ParameterTuple(matrix_G)
self.A_inv_max = ParameterTuple(A_inv_max)
self.G_inv_max = ParameterTuple(G_inv_max)
self.assign = P.Assign()
self.mul = P.Mul()
self.weight_decay = weight_decay
self.decay_flags = tuple(decay_filter(x) for x in self.parameters)
def __init__(self,
norm_bound=1.0,
initial_noise_multiplier=1.5,
noise_decay_rate=6e-4,
decay_policy='Time',
seed=0):
super(AdaGaussianRandom, self).__init__()
norm_bound = check_value_positive('norm_bound', norm_bound)
initial_noise_multiplier = check_value_positive(
'initial_noise_multiplier', initial_noise_multiplier)
self._norm_bound = Tensor(norm_bound, mstype.float32)
initial_noise_multiplier = Tensor(initial_noise_multiplier,
mstype.float32)
self._initial_noise_multiplier = Parameter(
initial_noise_multiplier, name='initial_noise_multiplier')
self._noise_multiplier = Parameter(initial_noise_multiplier,
name='noise_multiplier')
self._mean = Tensor(0, mstype.float32)
noise_decay_rate = check_param_type('noise_decay_rate',
noise_decay_rate, float)
check_param_in_range('noise_decay_rate', noise_decay_rate, 0.0, 1.0)
self._noise_decay_rate = Tensor(noise_decay_rate, mstype.float32)
if decay_policy not in ['Time', 'Step']:
raise NameError(
"The decay_policy must be in ['Time', 'Step'], but "
"get {}".format(decay_policy))
self._decay_policy = decay_policy
self._sub = P.Sub()
self._mul = P.Mul()
self._add = P.TensorAdd()
self._div = P.Div()
self._dtype = mstype.float32
self._normal = P.Normal(seed=seed)
self._assign = P.Assign()
self._one = Tensor(1, self._dtype)
def __init__(self):
super().__init__()
self.assign = op.Assign()
self.var = Parameter(initializer(1, (1), mstype.float32), name="var")
def __init__(self):
super().__init__()
self.para = Parameter(Tensor([1.0], ms.float32), name="para")
self.assign = P.Assign()
self.addn = P.AddN()
self.addn1 = AddnCell()
def __init__(self, input_shape):
super().__init__()
self.addn = P.AddN()
self.assign = P.Assign()
self.inputdata = Parameter(initializer(
1, input_shape, ms.float32), name="global_step")