def test_group_dynamic_2():
inputs = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01)
label = Tensor(np.ones([1, 10]).astype(np.float32))
net = LeNet5()
conv_lr = (0.1, 0.2, 0.3)
default_lr = 0.8
conv_params = list(
filter(lambda x: 'conv' in x.name, net.trainable_params()))
no_conv_params = list(
filter(lambda x: 'conv' not in x.name, net.trainable_params()))
group_params = [{
'params': conv_params,
'lr': conv_lr
}, {
'params': no_conv_params
}]
net.set_train()
loss = nn.SoftmaxCrossEntropyWithLogits()
opt = RMSProp(group_params, learning_rate=default_lr)
assert opt.is_group is True
assert opt.dynamic_lr is True
for lr, param in zip(opt.learning_rate, opt.parameters):
if 'conv' in param.name:
assert np.all(lr.learning_rate.data.asnumpy() == \
Tensor(np.array(list(conv_lr)).astype(np.float32)).asnumpy())
else:
assert np.all(lr.learning_rate.data.asnumpy() == \
Tensor(np.array([default_lr] * 3).astype(np.float32)).asnumpy())
net_with_loss = WithLossCell(net, loss)
train_network = TrainOneStepCell(net_with_loss, opt)
_executor.compile(train_network, inputs, label)
def test_checkpoint_save_ckpt_seconds():
"""Test checkpoint save ckpt seconds."""
train_config = CheckpointConfig(save_checkpoint_steps=16,
save_checkpoint_seconds=100,
keep_checkpoint_max=0,
keep_checkpoint_per_n_minutes=1)
ckpt_cb = ModelCheckpoint(config=train_config)
cb_params = _InternalCallbackParam()
net = Net()
loss = nn.SoftmaxCrossEntropyWithLogits()
optim = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9)
network_ = WithLossCell(net, loss)
_train_network = TrainOneStepCell(network_, optim)
cb_params.train_network = _train_network
cb_params.epoch_num = 10
cb_params.cur_epoch_num = 4
cb_params.cur_step_num = 128
cb_params.batch_num = 32
run_context = RunContext(cb_params)
ckpt_cb.begin(run_context)
ckpt_cb.step_end(run_context)
ckpt_cb2 = ModelCheckpoint(config=train_config)
cb_params.cur_epoch_num = 1
cb_params.cur_step_num = 16
ckpt_cb2.begin(run_context)
ckpt_cb2.step_end(run_context)
def test_inference_phase():
context.set_auto_parallel_context(device_num=8, global_rank=0)
context.set_auto_parallel_context(parallel_mode="auto_parallel")
set_cost_model_context(run_phase=1)
net = Net(512, 128)
predict = Tensor(np.ones([64, 512]).astype(np.float32) * 0.001)
label = Tensor(np.ones([64, 128]).astype(np.float32))
loss = nn.SoftmaxCrossEntropyWithLogits()
optimizer = Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9)
net_with_loss = WithLossCell(net, loss)
train_network = TrainOneStepCell(net_with_loss, optimizer)
train_network.set_train()
output = train_network(predict, label)
def test_adam_group1():
""" test_adam_group_lr_and_weight_decay """
inputs = Tensor(np.ones([1, 64]).astype(np.float32))
label = Tensor(np.zeros([1, 10]).astype(np.float32))
net = Net()
net.set_train()
loss = nn.SoftmaxCrossEntropyWithLogits()
net_with_loss = WithLossCell(net, loss)
all_params = net.trainable_params()
poly_decay_lr = polynomial_decay_lr(0.01,
0.0001,
total_step=10,
step_per_epoch=1,
decay_epoch=3,
power=1.0)
group_params = [{
'params': [all_params[0]],
'lr': poly_decay_lr,
'weight_decay': 0.9
}, {
'params': [all_params[1]]
}]
optimizer = nn.Adam(group_params, learning_rate=0.1)
train_network = TrainOneStepCell(net_with_loss, optimizer)
_executor.compile(train_network, inputs, label)
def test_AdamWeightDecaySparse():
""" test_AdamWeightDecaySparse """
context.set_context(mode=context.GRAPH_MODE)
class Loss(nn.Cell):
def __init__(self):
super(Loss, self).__init__()
def construct(self, base, target):
return base
class NetWithSparseGatherV2(nn.Cell):
def __init__(self):
super(NetWithSparseGatherV2, self).__init__()
self.w1 = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)),
name="w1",
sparse_grad=True)
self.w2 = Parameter(Tensor(np.ones([2, 1, 2]).astype(np.float32)),
name="w2")
self.gatherv2 = P.SparseGatherV2()
self.axis = 0
def construct(self, indices):
return self.gatherv2(self.w1, indices, self.axis) * self.w2
inputs = Tensor(np.array([0, 1]).astype(np.int32))
label = Tensor(np.zeros([2, 1, 2]).astype(np.float32))
net = NetWithSparseGatherV2()
net.set_train()
loss = Loss()
optimizer = AdamWeightDecaySparse(net.trainable_params())
net_with_loss = WithLossCell(net, loss)
train_network = TrainOneStepCell(net_with_loss, optimizer)
_executor.compile(train_network, inputs, label)
def compile_net(net):
optimizer = Adam(net.trainable_params(),
learning_rate=0.1,
loss_scale=1024.0,
weight_decay=0.9)
train_net = TrainOneStepCell(net, optimizer)
_executor.compile(train_net, _x, _b)
def test_lars():
class Net(nn.Cell):
def __init__(self, strategy1, strategy2, weight):
super().__init__()
self.weight = Parameter(weight, "w1")
self.matmul = P.MatMul(transpose_a=False, transpose_b=True).shard(strategy1)
self.relu = P.ReLU().shard(strategy2)
def construct(self, x):
out = self.matmul(x, self.weight)
out = self.relu(out)
return out
context.set_auto_parallel_context(device_num=4, global_rank=0)
context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")
strategy1 = ((2, 1), (2, 1))
strategy2 = ((4, 1),)
strategy3 = ((4, 1), (4, 1))
x = Tensor(np.ones([64, 32]), dtype=ms.float32)
weight = Tensor(np.ones([64, 32]), dtype=ms.float32)
b = Tensor(np.ones([64, 64]), dtype=ms.float32)
net = Net(strategy1, strategy2, weight)
lr = Tensor(np.ones([6]), dtype=ms.float32)
sgd = Momentum(net.trainable_params(), lr, 0.9)
optimizer = LARS(sgd, epsilon=1e-08, coefficient=0.02,
lars_filter=lambda x: 'bn' not in x.name)
net_with_loss = NetWithLoss(net, strategy3)
train_net = TrainOneStepCell(net_with_loss, optimizer)
compile_net(train_net, x, b)
def test_group_lr():
inputs = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01)
label = Tensor(np.ones([1, 10]).astype(np.float32))
net = LeNet5()
conv_lr = 0.8
default_lr = 0.1
conv_params = list(filter(lambda x: 'conv' in x.name, net.trainable_params()))
no_conv_params = list(filter(lambda x: 'conv' not in x.name, net.trainable_params()))
group_params = [{'params': no_conv_params},
{'params': conv_params, 'lr': conv_lr},
{'order_params': net.trainable_params()}]
net.set_train()
loss = nn.SoftmaxCrossEntropyWithLogits()
opt = Momentum(group_params, learning_rate=default_lr, momentum=0.9)
assert opt.is_group is True
assert opt.is_group_lr is True
assert opt.dynamic_lr is False
assert opt.is_group_params_ordered is True
for lr, param, order_param in zip(opt.learning_rate, opt.parameters, net.trainable_params()):
if param in conv_params:
assert np.all(lr.data.asnumpy() == Tensor(conv_lr, mstype.float32).asnumpy())
else:
assert np.all(lr.data.asnumpy() == Tensor(default_lr, mstype.float32).asnumpy())
assert param.name == order_param.name
net_with_loss = WithLossCell(net, loss)
train_network = TrainOneStepCell(net_with_loss, opt)
_executor.compile(train_network, inputs, label)
def test_checkpoint_save_ckpt_with_encryption():
"""Test checkpoint save ckpt with encryption."""
train_config = CheckpointConfig(
save_checkpoint_steps=16,
save_checkpoint_seconds=0,
keep_checkpoint_max=5,
keep_checkpoint_per_n_minutes=0,
enc_key=os.urandom(16),
enc_mode="AES-GCM")
ckpt_cb = ModelCheckpoint(config=train_config)
cb_params = _InternalCallbackParam()
net = Net()
loss = nn.SoftmaxCrossEntropyWithLogits()
optim = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9)
network_ = WithLossCell(net, loss)
_train_network = TrainOneStepCell(network_, optim)
cb_params.train_network = _train_network
cb_params.epoch_num = 10
cb_params.cur_epoch_num = 5
cb_params.cur_step_num = 160
cb_params.batch_num = 32
run_context = RunContext(cb_params)
ckpt_cb.begin(run_context)
ckpt_cb.step_end(run_context)
ckpt_cb2 = ModelCheckpoint(config=train_config)
cb_params.cur_epoch_num = 1
cb_params.cur_step_num = 15
if platform.system().lower() == "windows":
with pytest.raises(NotImplementedError):
ckpt_cb2.begin(run_context)
ckpt_cb2.step_end(run_context)
else:
ckpt_cb2.begin(run_context)
ckpt_cb2.step_end(run_context)
def test_row_tensor_env_get():
class Loss(nn.Cell):
def __init__(self):
super(Loss, self).__init__()
def construct(self, base, target):
return base
class NetWithSparseGatherV2(nn.Cell):
def __init__(self):
super(NetWithSparseGatherV2, self).__init__()
self.w1 = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)),
name="w1")
self.w2 = Parameter(Tensor(np.ones([2, 1, 2]).astype(np.float32)),
name="w2")
self.gatherv2 = MySparseGatherV2()
self.axis = 0
def construct(self, indices):
return self.gatherv2(self.w1, indices, self.axis) * self.w2
inputs = Tensor(np.array([0, 1]).astype(np.int32))
label = Tensor(np.zeros([2, 1, 2]).astype(np.float32))
net = NetWithSparseGatherV2()
net.set_train()
loss = Loss()
optimizer = AdamWeightDecaySparse(net.trainable_params())
net_with_loss = WithLossCell(net, loss)
train_network = TrainOneStepCell(net_with_loss, optimizer)
train_network(inputs, label)
def test_weight_decay():
inputs = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01)
label = Tensor(np.ones([1, 10]).astype(np.float32))
net = LeNet5()
conv_weight_decay = 0.8
default_weight_decay = 0.0
conv_params = list(filter(lambda x: 'conv' in x.name, net.trainable_params()))
no_conv_params = list(filter(lambda x: 'conv' not in x.name, net.trainable_params()))
group_params = [{'params': no_conv_params},
{'params': conv_params, 'weight_decay': conv_weight_decay},
{'order_params': net.trainable_params()}]
net.set_train()
loss = nn.SoftmaxCrossEntropyWithLogits()
opt = SGD(group_params, learning_rate=0.1, weight_decay=default_weight_decay)
assert opt.is_group is True
assert opt.is_group_lr is False
assert opt.is_group_params_ordered is True
for weight_decay, decay_flags, param, order_param in zip(
opt.weight_decay, opt.decay_flags, opt.parameters, net.trainable_params()):
if param in conv_params:
assert weight_decay == conv_weight_decay
assert decay_flags is True
else:
assert weight_decay == default_weight_decay
assert decay_flags is False
assert param.name == order_param.name
net_with_loss = WithLossCell(net, loss)
train_network = TrainOneStepCell(net_with_loss, opt)
_executor.compile(train_network, inputs, label)
def test_trainTensor(num_classes=10, epoch=15, batch_size=32):
net = AlexNet(num_classes)
lr = 0.1
momentum = 0.9
optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, momentum, weight_decay=0.0001)
criterion = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
net_with_criterion = WithLossCell(net, criterion)
train_network = TrainOneStepCell(net_with_criterion, optimizer)
train_network.set_train()
losses = []
for i in range(0, epoch):
data = Tensor(np.ones([batch_size, 3, 227, 227]).astype(np.float32) * 0.01)
label = Tensor(np.ones([batch_size]).astype(np.int32))
loss = train_network(data, label).asnumpy()
losses.append(loss)
assert losses[-1] < 0.01
def compile(net):
optimizer = Momentum(net.trainable_params(),
learning_rate=0.1,
momentum=0.9)
train_net = TrainOneStepCell(net, optimizer)
_executor.compile(train_net, _x, _b)
context.reset_auto_parallel_context()
def test_save_checkpoint():
"""Test save checkpoint."""
train_config = CheckpointConfig(
save_checkpoint_steps=16,
save_checkpoint_seconds=0,
keep_checkpoint_max=5,
keep_checkpoint_per_n_minutes=0)
cb_params = _InternalCallbackParam()
net = Net()
loss = nn.SoftmaxCrossEntropyWithLogits()
optim = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9)
network_ = WithLossCell(net, loss)
_train_network = TrainOneStepCell(network_, optim)
cb_params.train_network = _train_network
cb_params.epoch_num = 10
cb_params.cur_epoch_num = 5
cb_params.cur_step_num = 0
cb_params.batch_num = 32
ckpoint_cb = ModelCheckpoint(prefix="test_ckpt", directory='./test_files', config=train_config)
run_context = RunContext(cb_params)
ckpoint_cb.begin(run_context)
ckpoint_cb.step_end(run_context)
if os.path.exists('./test_files/test_ckpt-model.pkl'):
os.chmod('./test_files/test_ckpt-model.pkl', stat.S_IWRITE)
os.remove('./test_files/test_ckpt-model.pkl')
def test_momentum():
class Net(nn.Cell):
def __init__(self, strategy1, strategy2, weight):
super().__init__()
self.weight = Parameter(weight, "w1")
self.matmul = P.MatMul(transpose_a=False,
transpose_b=True).set_strategy(strategy1)
self.relu = P.ReLU().set_strategy(strategy2)
def construct(self, x):
out = self.matmul(x, self.weight)
out = self.relu(out)
return out
context.set_auto_parallel_context(device_num=4, global_rank=0)
strategy1 = ((2, 1), (2, 1))
strategy2 = ((4, 1), )
strategy3 = ((4, 1), (4, 1))
x = Tensor(np.ones([64, 32]), dtype=ms.float32)
weight = Tensor(np.ones([64, 32]), dtype=ms.float32)
b = Tensor(np.ones([64, 64]), dtype=ms.float32)
net = Net(strategy1, strategy2, weight)
optimizer = Momentum(net.trainable_params(),
learning_rate=0.1,
momentum=0.9)
net_with_loss = NetWithLoss(net, strategy3)
train_net = TrainOneStepCell(net_with_loss, optimizer)
context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")
compile_net(train_net, x, b)
def test_trainTensor(num_classes=10, epoch=8, batch_size=1):
context.set_context(mode=context.GRAPH_MODE, device_target="GPU")
net = resnet50(num_classes)
lr = 0.1
momentum = 0.9
optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, momentum)
criterion = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True)
net_with_criterion = WithLossCell(net, criterion)
train_network = TrainOneStepCell(net_with_criterion, optimizer) # optimizer
train_network.set_train()
losses = []
for i in range(0, epoch):
data = Tensor(np.ones([batch_size, 3, 224, 224]).astype(np.float32) * 0.01)
label = Tensor(np.ones([batch_size]).astype(np.int32))
loss = train_network(data, label)
losses.append(loss)
assert(losses[-1].asnumpy() < 1)
def train(net, data, label):
learning_rate = 0.01
momentum = 0.9
optimizer = Momentum(
filter(lambda x: x.requires_grad, net.get_parameters()), learning_rate,
momentum)
criterion = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True)
net_with_criterion = WithLossCell(net, criterion)
train_network = TrainOneStepCell(net_with_criterion,
optimizer) # optimizer
train_network.set_train()
res = train_network(data, label)
print("+++++++++Loss+++++++++++++")
print(res)
print("+++++++++++++++++++++++++++")
assert res
def test_async_dump_net_multi_layer_mode1():
test_name = "test_async_dump_net_multi_layer_mode1"
json_file = os.path.join(os.getcwd(), "{}.json".format(test_name))
device_id = context.get_context("device_id")
dump_full_path = os.path.join("/tmp/async_dump/",
"{}_{}".format(test_name, device_id))
os.system("rm -rf {}/*".format(dump_full_path))
os.environ["MINDSPORE_DUMP_CONFIG"] = json_file
weight = Tensor(np.ones((1000, 2048)).astype(np.float32))
bias = Tensor(np.ones((1000, )).astype(np.float32))
net = ReluReduceMeanDenseRelu(weight, bias, 2048, 1000)
criterion = SoftmaxCrossEntropyWithLogits(sparse=False)
optimizer = Momentum(learning_rate=0.1,
momentum=0.1,
params=filter(lambda x: x.requires_grad,
net.get_parameters()))
net_with_criterion = WithLossCell(net, criterion)
train_network = TrainOneStepCell(net_with_criterion, optimizer)
train_network.set_train()
inputs = Tensor(np.random.randn(32, 2048, 7, 7).astype(np.float32))
label = Tensor(np.zeros(shape=(32, 1000)).astype(np.float32))
net_dict = train_network(inputs, label)
dump_path = "/tmp/async_dump/{}/device_{}/test_graph_0/0/0/".format(
test_name, device_id)
dump_file = os.listdir(dump_path)
dump_file_name = ""
for file in dump_file:
if "SoftmaxCrossEntropyWithLogits" in file:
dump_file_name = file
dump_file_full_path = os.path.join(dump_path, dump_file_name)
npy_path = os.path.join(os.getcwd(), "./{}".format(test_name))
if os.path.exists(npy_path):
shutil.rmtree(npy_path)
os.mkdir(npy_path)
cmd = "python /usr/local/Ascend/toolkit/tools/operator_cmp/compare/msaccucmp.pyc " \
"convert -d {0} -out {1}".format(dump_file_full_path, npy_path)
os.system(cmd)
npy_file_list = os.listdir(npy_path)
dump_result = {}
for file in npy_file_list:
if "output.0.npy" in file:
dump_result["output0"] = np.load(os.path.join(npy_path, file))
for index, value in enumerate(net_dict):
assert value.asnumpy() == dump_result["output0"][index]
def train(net, data, label):
learning_rate = 0.01
momentum = 0.9
optimizer = Momentum(
filter(lambda x: x.requires_grad, net.get_parameters()), learning_rate,
momentum)
criterion = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
net_with_criterion = WithLossCell(net, criterion)
train_network = TrainOneStepCell(net_with_criterion,
optimizer) # optimizer
train_network.set_train()
res = train_network(data, label)
print("+++++++++Loss+++++++++++++")
print(res)
print("+++++++++++++++++++++++++++")
diff = res.asnumpy() - 2.302585
assert np.all(diff < 1.e-6)
def test_train_lenet():
epoch = 100
net = LeNet()
momentum = 0.9
learning_rate = multisteplr(epoch, 30)
optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), learning_rate, momentum)
criterion = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True)
net_with_criterion = WithLossCell(net, criterion)
train_network = TrainOneStepCell(net_with_criterion, optimizer) # optimizer
train_network.set_train()
losses = []
for i in range(epoch):
data = Tensor(np.ones([net.batch_size, 3, 32, 32]).astype(np.float32) * 0.01)
label = Tensor(np.ones([net.batch_size]).astype(np.int32))
loss = train_network(data, label).asnumpy()
losses.append(loss)
assert losses[-1] < 0.01
def test_spares_ftrl_compile():
""" test sparse ftrl compile """
indices = Tensor(np.array([0, 1]).astype(np.int32))
label = Tensor(np.zeros([2, 1, 2]).astype(np.float32))
net = NetWithSparseGatherV2()
net.set_train()
optimizer = FTRL(net.trainable_params(), loss_scale=2.0)
train_network = TrainOneStepCell(net, optimizer)
_executor.compile(train_network, indices, label)
def test_exec_save_checkpoint():
net = Net()
loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True)
opt = Momentum(net.trainable_params(), 0.0, 0.9, 0.0001, 1024)
loss_net = WithLossCell(net, loss)
train_network = TrainOneStepCell(loss_net, opt)
_exec_save_checkpoint(train_network, ckpoint_file_name="./new_ckpt.ckpt")
load_checkpoint("new_ckpt.ckpt")
def test_LSTM():
num_epochs = 5
embed_size = 100
num_hiddens = 100
num_layers = 2
bidirectional = True
labels = 2
vocab_size = 252193
max_len = 500
weight = np.ones((vocab_size + 1, embed_size)).astype(np.float32)
net = SentimentNet(vocab_size=(vocab_size + 1),
embed_size=embed_size,
num_hiddens=num_hiddens,
num_layers=num_layers,
bidirectional=bidirectional,
weight=weight,
labels=labels,
batch_size=batch_size)
learning_rate = 0.1
momentum = 0.9
optimizer = Momentum(
filter(lambda x: x.requires_grad, net.get_parameters()), learning_rate,
momentum)
criterion = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True)
net_with_criterion = WithLossCell(net, criterion)
train_network = TrainOneStepCell(net_with_criterion,
optimizer) # optimizer
train_network.set_train()
train_features = Tensor(np.ones([64, max_len]).astype(np.int32))
train_labels = Tensor(np.ones([
64,
]).astype(np.int32)[0:64])
losses = []
for epoch in range(num_epochs):
loss = train_network(train_features, train_labels)
losses.append(loss)
print("loss:", loss.asnumpy())
assert (losses[-1].asnumpy() < 0.01)
def test_spares_proximal_ada_grad():
""" test sparse proximal_ada_grad """
indices = Tensor(np.array([0, 1]).astype(np.int32))
label = Tensor(np.zeros([2, 1, 2]).astype(np.float32))
net = NetWithSparseGatherV2()
net.set_train()
optimizer = ProximalAdagrad(net.trainable_params(), weight_decay=0.9, loss_scale=1024.0)
train_network = TrainOneStepCell(net, optimizer)
_executor.compile(train_network, indices, label)
def test_spares_lazy_adam_compile():
""" test sparse adam compile """
indices = Tensor(np.array([0, 1]).astype(np.int32))
label = Tensor(np.zeros([2, 1, 2]).astype(np.float32))
net = NetWithSparseGatherV2()
net.set_train()
optimizer = LazyAdam(net.trainable_params(), learning_rate=0.1, weight_decay=0.9, loss_scale=2.0)
train_network = TrainOneStepCell(net, optimizer)
_executor.compile(train_network, indices, label)
def train_mindspore_impl(self):
input = Tensor(
np.random.randn(self.batch_num,
self.input_channels).astype(np.float32))
weight_np = Tensor(
np.random.randn(self.output_channels,
self.input_channels).astype(np.float32))
bias = Tensor(np.random.randn(self.output_channels).astype(np.float32))
label_np = np.random.randint(self.output_channels, size=self.batch_num)
label_np_onehot = np.zeros(shape=(self.batch_num,
self.output_channels)).astype(
np.float32)
label_np_onehot[np.arange(self.batch_num), label_np] = 1.0
label = Tensor(label_np_onehot)
ms_dense = Dense(in_channels=self.input_channels,
out_channels=self.output_channels,
weight_init=weight_np,
bias_init=bias,
has_bias=True)
criterion = SoftmaxCrossEntropyWithLogits()
optimizer = nn.Adam(ms_dense.trainable_params(),
learning_rate=1e-3,
beta1=0.9,
beta2=0.999,
eps=self.epsilon,
use_locking=False,
use_nesterov=False,
weight_decay=0.0,
loss_scale=1.0)
net_with_criterion = WithLossCell(ms_dense, criterion)
train_network = TrainOneStepCell(net_with_criterion, optimizer)
train_network.set_train()
print('MS Initialized!')
for i in range(self.epoch):
train_network(input, label)
output = ms_dense(input)
print("===============output=================", output)
return output.asnumpy()
def test_adam_weight_decay():
epoch = 3
net = NetAdamWeightDecay()
optimizer = AdamWeightDecayOp(filter(lambda x: x.requires_grad,
net.get_parameters()), learning_rate=0.01)
criterion = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
net_with_criterion = WithLossCell(net, criterion)
train_network = TrainOneStepCell(
net_with_criterion, optimizer)
train_network.set_train()
losses1 = []
for _ in range(epoch):
data = Tensor(np.arange(0, 16).reshape(
1, 1, 4, 4).astype(np.float32) * 0.01)
label = Tensor(np.array([0]).astype(np.int32))
loss = train_network(data, label)
losses1.append(loss.asnumpy())
assert losses1[0] > losses1[1]
assert losses1[1] > losses1[2]
def test_ftrl():
""" test_ftrl """
inputs = Tensor(np.ones([1, 64]).astype(np.float32))
label = Tensor(np.zeros([1, 10]).astype(np.float32))
net = Net()
net.set_train()
loss = nn.SoftmaxCrossEntropyWithLogits()
optimizer = FTRL(net.trainable_params(), weight_decay=0.9, loss_scale=2.0)
net_with_loss = WithLossCell(net, loss)
train_network = TrainOneStepCell(net_with_loss, optimizer)
_executor.compile(train_network, inputs, label)
def test_save_checkpoint_for_network():
""" test save_checkpoint for network"""
net = Net()
loss = SoftmaxCrossEntropyWithLogits(sparse=True)
opt = Momentum(net.trainable_params(), 0.0, 0.9, 0.0001, 1024)
loss_net = WithLossCell(net, loss)
train_network = TrainOneStepCell(loss_net, opt)
save_checkpoint(train_network, ckpt_file_name="./new_ckpt.ckpt")
load_checkpoint("new_ckpt.ckpt")
def test_proximal_ada_grad():
""" test_proximal_ada_grad """
inputs = Tensor(np.ones([1, 64]).astype(np.float32))
label = Tensor(np.zeros([1, 10]).astype(np.float32))
net = Net()
net.set_train()
loss = nn.SoftmaxCrossEntropyWithLogits()
optimizer = ProximalAdagrad(net.trainable_params())
net_with_loss = WithLossCell(net, loss)
train_network = TrainOneStepCell(net_with_loss, optimizer)
_executor.compile(train_network, inputs, label)
