# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
""" test_lr_schedule """
import numpy as np
from mindspore import Parameter, ParameterTuple, Tensor
from mindspore.nn import Cell
from mindspore.nn.optim import Optimizer
from mindspore.ops.operations import BiasAdd, MatMul
import mindspore.ops.composite as C
grad_by_list = C.GradOperation('get_by_list', get_by_list=True)
class Net(Cell):
""" Net definition """
def __init__(self):
super(Net, self).__init__()
self.weight = Parameter(Tensor(np.ones([64, 10])), name="weight")
self.bias = Parameter(Tensor(np.ones([10])), name="bias")
self.matmul = MatMul()
self.biasAdd = BiasAdd()
def construct(self, x):
x = self.biasAdd(self.matmul(x, self.weight), self.bias)
return x
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
""" test implicit conversion """
import numpy as np
import pytest
from mindspore import Tensor, nn, context, Parameter
from mindspore import dtype as mstype
from mindspore.ops import composite as C
grad_all_with_sens = C.GradOperation(get_all=True, sens_param=True)
def test_user_define_bprop_check_ok():
class Net(nn.Cell):
def __init__(self):
super(Net, self).__init__()
self.grad = Tensor(
np.array([[1.1, 2.2, 3.3], [2.0, 3.0, 4.0]], dtype=np.float32))
def construct(self, x):
ret = x * 2
return ret
def bprop(self, x, out, dout):
return (self.grad * 3, )
def __init__(self, net):
super(NetGrad, self).__init__()
self.grad_op = C.GradOperation(get_by_list=True, sens_param=False)
self.net = net
self.weights = ParameterTuple(self.net.trainable_params())
from mindspore.ops import operations as P
from mindspore.ops import prim_attr_register, PrimitiveWithInfer
from ..ut_filter import non_graph_engine
from ....mindspore_test_framework.mindspore_test import mindspore_test
from ....mindspore_test_framework.pipeline.forward.compile_forward \
import pipeline_for_compile_forward_ge_graph_for_case_by_case_config
from ....mindspore_test_framework.pipeline.forward.verify_exception \
import pipeline_for_verify_exception_for_case_by_case_config
context.set_context(mode=context.GRAPH_MODE)
# pylint: disable=W0613
# pylint: disable=W0231
# W0613: unused-argument
# W0231: super-init-not-called
grad = C.GradOperation('grad')
def test_multiply():
""" test_multiply """
input_x = Tensor(np.array([[-0.1, 0.3, 3.6], [0.4, 0.5, -3.2]]))
input_y = Tensor(np.array([[0.1, 0.3, -3.6], [0.4, 0.5, -3.2]]))
mul = P.Mul()
result = mul(input_x, input_y)
expect = np.array([[-0.01, 0.09, -12.96], [0.16, 0.25, 10.24]])
diff = result.asnumpy() - expect
error = np.ones(shape=[2, 3]) * 1.0e-6
assert np.all(diff < error)
assert np.all(-diff < error)
def __init__(self, network):
super(Grad, self).__init__()
self.grad = C.GradOperation(get_all=True, sens_param=True)
self.network = network
def step_end(self, run_context):
step_msseconds = (time.time() - self.step_time) * 1000
if step_msseconds < 275:
self.total = self.total + 1
print(f"step time:{step_msseconds}", flush=True)
def good_step(self):
return self.total
random.seed(1)
np.random.seed(1)
ds.config.set_seed(1)
grad_by_list = CP.GradOperation(get_by_list=True)
def weight_variable_0(shape):
zeros = np.zeros(shape).astype(np.float32)
return Tensor(zeros)
def weight_variable_1(shape):
ones = np.ones(shape).astype(np.float32)
return Tensor(ones)
def conv3x3(in_channels, out_channels, stride=1, padding=0):
"""3x3 convolution """
return nn.Conv2d(in_channels,
import mindspore as ms
import mindspore.common.dtype as mstype
import mindspore.nn as nn
from mindspore import Parameter, ParameterTuple
from mindspore import Tensor
from mindspore import context
from mindspore.common.api import ms_function
from mindspore.ops import composite as C
from mindspore.ops import operations as P
from mindspore.ops.functional import stop_gradient
from mindspore.ops.primitive import prim_attr_register, PrimitiveWithInfer
from ..ut_filter import non_graph_engine
from ....mindspore_test_framework.utils.bprop_util import bprop
grad_by_list = C.GradOperation('get_by_list', get_by_list=True)
grad_all = C.GradOperation('get_all', get_all=True)
def setup_module(module):
context.set_context(mode=context.PYNATIVE_MODE)
def stop_func(x, y):
""" stop_func"""
c = x * y
c_s = x + y
return c_s, c
def stop_test1(x, y):
def __init__(self, net):
super(GradNet, self).__init__()
self.forward_net = net
self.sens = Tensor(np.ones((2, 2), np.float32) * 5)
self.grad_all = C.GradOperation(get_all=True)
def __init__(self, network, sens_param=True, real_inputs_count=None):
super().__init__(grad=C.GradOperation(get_all=True,
sens_param=sens_param),
network=network,
real_inputs_count=real_inputs_count)
import numpy as np
import pytest
import mindspore as ms
import mindspore.ops.composite as C
from mindspore import context
import mindspore.nn as nn
from mindspore.ops import operations as P
from mindspore.ops import functional as F
from mindspore import Tensor
from mindspore.common.parameter import Parameter, ParameterTuple
grad_all_list = C.GradOperation(get_all=True, get_by_list=True)
grad_by_list = C.GradOperation(get_by_list=True)
context.set_context(mode=context.GRAPH_MODE, save_graphs=False)
def test_load_grad():
class LoadNet(nn.Cell):
def __init__(self):
super().__init__()
self.z = Parameter(Tensor(np.array([1.0], np.float32)), name='z')
def construct(self, x, y):
x = x * y * self.z
return x
x = Tensor(np.array([2.0], np.float32))
y = Tensor(np.array([3.0], np.float32))
load_net = LoadNet()
def __init__(self, network):
super(GetParamGrad, self).__init__(auto_prefix=False)
self.network = network
self.weights = ParameterTuple(network.trainable_params())
self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True)
def __init__(self, forward_net):
super(NetSqrtGradGrad, self).__init__()
self.forward_net = forward_net
self.gradOps = C.GradOperation(get_all=True, sens_param=True)
def __init__(self, net):
super(BackwardNet, self).__init__(auto_prefix=False)
self.forward_net = net
self.grad = C.GradOperation(get_all=True)
# See the License for the specific language governing permissions and
# limitations under the License.
import numpy as np
import mindspore as ms
import mindspore.nn as nn
from mindspore import Tensor
from mindspore import context
from mindspore.common.api import _executor
from mindspore.ops import composite as C
from mindspore.ops import operations as P
from tests.ut.python.ops.test_math_ops import VirtualLoss
grad_all = C.GradOperation('get_all', get_all=True)
class NetWithLoss(nn.Cell):
def __init__(self, network):
super(NetWithLoss, self).__init__()
self.loss = VirtualLoss()
self.network = network
def construct(self, x, y, b):
predict = self.network(x, y, b)
return self.loss(predict)
class GradWrap(nn.Cell):
def __init__(self, network):
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
import numpy as np
import mindspore.context as context
from mindspore import Tensor
from mindspore.nn import Cell
from mindspore.ops import composite as C
from mindspore.ops.operations import Minimum
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
grad = C.GradOperation(get_all=True, sens_param=True)
class MinNetMe(Cell):
def __init__(self):
super(MinNetMe, self).__init__()
self.min = Minimum()
def construct(self, inputA, inputB):
x = self.min(inputA, inputB)
return x
class GradWrap(Cell):
def __init__(self, network):
super(GradWrap, self).__init__()
def __init__(self, network):
super(Grad, self).__init__()
self.network = network
self.weights = ParameterTuple(network.trainable_params())
self.grad = C.GradOperation(get_by_list=True, sens_param=True)
from mindspore.ops import operations as P
from mindspore.ops import prim_attr_register, PrimitiveWithInfer
from ..ut_filter import non_graph_engine
from ....mindspore_test_framework.mindspore_test import mindspore_test
from ....mindspore_test_framework.pipeline.forward.compile_forward \
import pipeline_for_compile_forward_ge_graph_for_case_by_case_config
from ....mindspore_test_framework.pipeline.forward.verify_exception \
import pipeline_for_verify_exception_for_case_by_case_config
context.set_context(mode=context.GRAPH_MODE)
# pylint: disable=W0613
# pylint: disable=W0231
# W0613: unused-argument
# W0231: super-init-not-called
grad = C.GradOperation()
def test_multiply():
""" test_multiply """
input_x = Tensor(np.array([[-0.1, 0.3, 3.6], [0.4, 0.5, -3.2]]))
input_y = Tensor(np.array([[0.1, 0.3, -3.6], [0.4, 0.5, -3.2]]))
mul = P.Mul()
result = mul(input_x, input_y)
expect = np.array([[-0.01, 0.09, -12.96], [0.16, 0.25, 10.24]])
diff = result.asnumpy() - expect
error = np.ones(shape=[2, 3]) * 1.0e-6
assert np.all(diff < error)
assert np.all(-diff < error)
def __init__(self, network):
super(FusedBatchNormGrad, self).__init__()
self.grad = C.GradOperation(name="get_all", get_all=True, sens_param=True)
self.network = network
def __init__(self, network):
super(GradData, self).__init__()
self.grad = C.GradOperation(name="get_all",
get_all=True,
sens_param=False)
self.network = network
def __init__(self, net):
super(BackwardNetReplaceBreak, self).__init__(auto_prefix=False)
self.forward_net = net
self.grad = C.GradOperation()
def construct(self, x, label):
weights = self.weights
return C.GradOperation('get_by_list', get_by_list=True)(self.network,
weights)(x,
label)
import pytest
import mindspore as ms
from mindspore import Tensor
from mindspore import context
from mindspore import nn
from mindspore.common import dtype as mstype
from mindspore.ops import composite as C
from mindspore.ops import functional as F
from mindspore.ops import operations as P
from mindspore.common.parameter import Parameter, ParameterTuple
from mindspore.common import ms_function
context.set_context(mode=context.GRAPH_MODE)
grad_by_list = C.GradOperation(get_by_list=True)
grad_all = C.GradOperation(get_all=True)
grad_all_with_sens = C.GradOperation(get_all=True, sens_param=True)
def cond_data_test(x_init, y_init):
class Net(nn.Cell):
def __init__(self):
""""""
super(Net, self).__init__()
self.square = P.Square()
self.add = P.TensorAdd()
self.value = Tensor(3, dtype=ms.float32)
self.switch = P.GeSwitch()
self.merge = P.Merge()
self.less = P.Less()
def __init__(self,
network,
sens=1024.0,
host_device_mix=False,
parameter_server=False,
sparse=False):
super(TrainStepWrap, self).__init__()
parallel_mode = context.get_auto_parallel_context("parallel_mode")
is_auto_parallel = parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL,
ParallelMode.AUTO_PARALLEL)
self.network = network
self.network.set_train()
self.trainable_params = network.trainable_params()
weights_w = []
weights_d = []
for params in self.trainable_params:
if 'wide' in params.name:
weights_w.append(params)
else:
weights_d.append(params)
self.weights_w = ParameterTuple(weights_w)
self.weights_d = ParameterTuple(weights_d)
if (sparse and is_auto_parallel) or parameter_server:
self.optimizer_d = LazyAdam(self.weights_d,
learning_rate=3.5e-4,
eps=1e-8,
loss_scale=sens)
self.optimizer_w = FTRL(learning_rate=5e-2,
params=self.weights_w,
l1=1e-8,
l2=1e-8,
initial_accum=1.0,
loss_scale=sens)
if host_device_mix or parameter_server:
self.optimizer_w.target = "CPU"
self.optimizer_d.target = "CPU"
else:
self.optimizer_d = Adam(self.weights_d,
learning_rate=3.5e-4,
eps=1e-8,
loss_scale=sens)
self.optimizer_w = FTRL(learning_rate=5e-2,
params=self.weights_w,
l1=1e-8,
l2=1e-8,
initial_accum=1.0,
loss_scale=sens)
self.hyper_map = C.HyperMap()
self.grad_w = C.GradOperation(get_by_list=True, sens_param=True)
self.grad_d = C.GradOperation(get_by_list=True, sens_param=True)
self.sens = sens
self.loss_net_w = IthOutputCell(network, output_index=0)
self.loss_net_d = IthOutputCell(network, output_index=1)
self.loss_net_w.set_grad()
self.loss_net_d.set_grad()
self.reducer_flag = False
self.grad_reducer_w = None
self.grad_reducer_d = None
self.reducer_flag = parallel_mode in (ParallelMode.DATA_PARALLEL,
ParallelMode.HYBRID_PARALLEL)
if self.reducer_flag:
mean = context.get_auto_parallel_context("gradients_mean")
degree = context.get_auto_parallel_context("device_num")
self.grad_reducer_w = DistributedGradReducer(
self.optimizer_w.parameters, mean, degree)
self.grad_reducer_d = DistributedGradReducer(
self.optimizer_d.parameters, mean, degree)
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import numpy as np
import mindspore as ms
import mindspore.nn as nn
from mindspore import Tensor, Parameter
from mindspore import context
from mindspore.common.api import _executor
from mindspore.ops import composite as C
from mindspore.ops import operations as P
from tests.ut.python.ops.test_math_ops import VirtualLoss
grad_all = C.GradOperation(get_all=True)
class NetWithLoss(nn.Cell):
def __init__(self, network):
super(NetWithLoss, self).__init__()
self.loss = VirtualLoss()
self.network = network
def construct(self, x, y):
predict = self.network(x, y)
return self.loss(predict)
class GradWrap(nn.Cell):
def __init__(self, network):
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
""" test_tensor_slice """
import numpy as np
import pytest
from mindspore import Tensor, Parameter
from mindspore import context
from mindspore import dtype as mstype
from mindspore.nn import Cell
from mindspore.common.parameter import ParameterTuple
from mindspore.ops import composite as C
grad_by_list_with_sens = C.GradOperation('grad_by_list_with_sens',
get_by_list=True,
sens_param=True)
def setup_module():
context.set_context(mode=context.PYNATIVE_MODE, device_target="Ascend")
class NetWorkSlicePositive(Cell):
def __init__(self):
super(NetWorkSlicePositive, self).__init__()
self.tensor_ret0 = Tensor(np.ones([1, 2, 3], np.int32))
self.tensor_ret1 = Tensor(np.ones([4, 8, 10], np.int32))
self.tensor_ret2 = Tensor(np.ones([6, 8, 10], np.int32))
self.tensor_ret3 = Tensor(np.ones([3, 8, 10], np.int32))
