def test_allreduce_fusion1():
_set_has_initializer(False)
cost_model_context.set_cost_model_context(
costmodel_allreduce_fusion_algorithm=1)
cost_model_context.set_cost_model_context(
costmodel_allreduce_fusion_times=2)
cost_model_context.set_cost_model_context(
costmodel_allreduce_fusion_tail_percent=0.5)
context.reset_auto_parallel_context()
context.set_auto_parallel_context(
parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL)
net = SimpleDMLNet(DenseNet1(has_bias=False, activation=None),
DenseNet2(has_bias=False, activation=None))
allreduce_fusion_dict = train_common(net)
expect_dict = {
'backbone2.fc8.weight': 2,
'backbone2.fc7.weight': 2,
'backbone2.fc6.weight': 2,
'backbone1.fc4.weight': 2,
'backbone1.fc3.weight': 2,
'backbone1.fc2.weight': 2,
'backbone2.fc5.weight': 1,
'backbone2.fc4.weight': 1,
'backbone2.fc3.weight': 1,
'backbone2.fc2.weight': 1,
'backbone2.fc1.weight': 1,
'backbone1.fc1.weight': 1
}
assert allreduce_fusion_dict == expect_dict
cost_model_context.reset_cost_model_context()
def test_add_relu_all_gather():
_set_has_initializer(False)
context.set_auto_parallel_context(device_num=8, global_rank=7)
context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")
strategy0 = ((8, 1), (8, 1))
strategy1 = ((1, 1),)
net = Grad(NetWithLoss(AddRelu(strategy0, strategy1)))
x = Tensor(np.ones([128, 32]), dtype=ms.float32)
y = Tensor(np.ones([128, 32]), dtype=ms.float32)
compile_net(net, x, y)
def test_using_diffserent_seed_for_initializer():
np.random.seed(0)
net1 = ParameterNet()
net1.init_parameters_data()
np.random.seed(1)
net2 = ParameterNet()
net2.init_parameters_data()
_set_has_initializer(False)
for key in net1.parameters_dict():
if key not in net2.parameters_dict():
assert False
else:
assert not allclose(net1.parameters_dict()[key].data.asnumpy(),
net2.parameters_dict()[key].data.asnumpy())
def test_parameter_lazy_init():
_set_has_initializer(False)
# support lazy init in SEMI_AUTO_PARALLEL mode
context.reset_auto_parallel_context()
context.set_auto_parallel_context(parallel_mode="semi_auto_parallel",
device_num=8)
# Call init_data() without set default_input.
para = Parameter(initializer('ones', [1, 2, 3], mstype.float32), 'test1')
assert not isinstance(para.default_input, Tensor)
para = para.init_data()
assert isinstance(para.default_input, Tensor)
assert np.array_equal(para.default_input.asnumpy(), np.ones((1, 2, 3)))
# Call init_data() after default_input is set.
para = Parameter(initializer('ones', [1, 2, 3], mstype.float32), 'test2')
assert not isinstance(para.default_input, Tensor)
# expect type error when not init
with pytest.raises(TypeError):
para.default_input = Tensor(np.zeros((1, 2, 3)))
# init then assign
para = para.init_data()
# check the type
with pytest.raises(TypeError):
para.default_input = Tensor(np.zeros((1, 2, 3)))
# check the shape
with pytest.raises(ValueError):
para.default_input = Tensor(np.zeros((1, 2)))
# expect change ok
para.default_input = Tensor(np.zeros((1, 2, 3)).astype(np.float32))
assert np.array_equal(para.default_input.asnumpy(), np.zeros((1, 2, 3)))
para.default_input = initializer('ones', [1, 2, 3], mstype.float32)
assert isinstance(para.default_input, Tensor)
# same object and has inited
assert np.array_equal(para.default_input.asnumpy(), np.ones((1, 2, 3)))
# expect no effect.
para.init_data()
assert np.array_equal(para.default_input.asnumpy(), np.ones((1, 2, 3)))
para.set_parameter_data(Tensor(np.zeros((1, 2)).astype(np.float32)),
slice_shape=True)
assert np.array_equal(para.default_input.asnumpy(), np.zeros((1, 2)))
para.set_parameter_data(initializer('ones', [1, 2], mstype.float32),
slice_shape=True)
assert np.array_equal(para.default_input.asnumpy(), np.ones((1, 2)))
context.reset_auto_parallel_context()
def test_lenet5_train_step_training_pynative():
"""test_lenet5_train_step_training_pynative"""
context.set_context(mode=context.PYNATIVE_MODE)
context.reset_auto_parallel_context()
context.set_auto_parallel_context(parallel_mode=ParallelMode.DATA_PARALLEL,
device_num=8,
mirror_mean=True)
predict = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01)
label = Tensor(np.zeros([1, 10]).astype(np.float32))
DatasetLenet(predict, label, 2)
network = LeNet5()
loss_fn = nn.SoftmaxCrossEntropyWithLogits()
optimizer = Momentum(network.get_parameters(),
learning_rate=0.1,
momentum=0.9)
Model(network=network, loss_fn=loss_fn, optimizer=optimizer)
context.set_context(mode=context.GRAPH_MODE)
context.reset_auto_parallel_context()
_set_has_initializer(False)
def test_all_to_all():
_set_has_initializer(False)
strategy1 = ((8, 1), )
context.set_context(mode=context.GRAPH_MODE, save_graphs=False)
_reset_op_id()
strategys = all_to_all_common(strategy1)
print(strategys)
expect_dict = {
'Default/network-_VirtualDatasetCell/_backbone-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits'
'/SoftmaxCrossEntropyWithLogits-op3': [[8, 1], [8, 1]],
'Default/network-_VirtualDatasetCell/_backbone-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits/'
'OneHot-op4': [[8, 1], [], []],
'Default/network-_VirtualDatasetCell/_backbone-WithLossCell/_backbone-AllToAllNet/Transpose-op1':
[[8, 1]],
'Default/network-_VirtualDatasetCell/_backbone-WithLossCell/_backbone-AllToAllNet/MatMul-op0':
[[1, 1], [1, 8]]
}
assert strategys == expect_dict
context.set_context(save_graphs=False)
def test_matmul_sub():
class Net(nn.Cell):
def __init__(self, strategy1, strategy2):
super().__init__()
self.matmul = P.MatMul().set_strategy(strategy1)
self.sub = P.Sub().set_strategy(strategy2)
def construct(self, x, y, b):
out = self.matmul(x, y)
out = self.sub(out, b)
return out
_set_has_initializer(False)
context.set_auto_parallel_context(device_num=8, global_rank=0)
context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")
strategy1 = ((2, 2), (2, 2))
strategy2 = ((4, 2), (4, 2))
net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))
x = Tensor(np.ones([64, 32]), dtype=ms.float32)
y = Tensor(np.ones([32, 64]), dtype=ms.float32)
b = Tensor(np.ones([64, 64]), dtype=ms.float32)
compile_net(net, x, y, b)