def test_pynative_lenet_with_new_interface(): context.set_context(mode=context.PYNATIVE_MODE) epoch_size = 20 batch_size = 32 inputs = Tensor(np.ones([batch_size, 1, 32, 32]).astype(np.float32)) labels = Tensor(np.ones([batch_size]).astype(np.int32)) net = LeNet() criterion = CrossEntropyLoss() net_with_criterion = WithLossCell(net, criterion) net_with_criterion.set_train() weights = ParameterTuple( filter(lambda x: x.requires_grad, net.get_parameters())) optimizer = Momentum(weights, 0.1, 0.9) forward_value_and_grad = nn.ForwardValueAndGrad(network=net_with_criterion, weights=weights, get_by_list=True) total_time = 0 for epoch in range(0, epoch_size): start_time = time.time() loss_output, grads = forward_value_and_grad(inputs, labels) optimizer(grads) end_time = time.time() cost_time = end_time - start_time total_time = total_time + cost_time print("======epoch: ", epoch, " loss: ", loss_output.asnumpy(), " cost time: ", cost_time) assert loss_output.asnumpy() < 0.005 assert loss_output.asnumpy() > 0.003
def test_pynative_resnet50(): context.set_context(mode=context.PYNATIVE_MODE, device_target="Ascend") batch_size = 32 num_classes = 10 net = resnet50(batch_size, num_classes) criterion = CrossEntropyLoss() optimizer = Momentum(learning_rate=0.01, momentum=0.9, params=filter(lambda x: x.requires_grad, net.get_parameters())) net_with_criterion = WithLossCell(net, criterion) net_with_criterion.set_grad() train_network = GradWrap(net_with_criterion) train_network.set_train() step = 0 max_step = 21 exceed_num = 0 data_set = create_dataset(repeat_num=1, training=True, batch_size=batch_size) for element in data_set.create_dict_iterator(num_epochs=1): step = step + 1 if step > max_step: break start_time = time.time() input_data = element["image"] input_label = element["label"] loss_output = net_with_criterion(input_data, input_label) grads = train_network(input_data, input_label) optimizer(grads) end_time = time.time() cost_time = end_time - start_time print("======step: ", step, " loss: ", loss_output.asnumpy(), " cost time: ", cost_time) if step > 1 and cost_time > 0.25: exceed_num = exceed_num + 1 assert exceed_num < 20
def test_ascend_pynative_lenet(): context.set_context(mode=context.PYNATIVE_MODE, device_target="Ascend") epoch_size = 20 batch_size = 32 inputs = Tensor(np.ones([batch_size, 1, 32, 32]).astype(np.float32)) labels = Tensor(np.ones([batch_size]).astype(np.int32)) net = LeNet() criterion = CrossEntropyLoss() optimizer = Momentum( filter(lambda x: x.requires_grad, net.get_parameters()), 0.1, 0.9) net_with_criterion = WithLossCell(net, criterion) train_network = GradWrap(net_with_criterion) train_network.set_train() total_time = 0 for epoch in range(0, epoch_size): start_time = time.time() fw_output = net(inputs) loss_output = criterion(fw_output, labels) grads = train_network(inputs, labels) success = optimizer(grads) end_time = time.time() cost_time = end_time - start_time total_time = total_time + cost_time print("======epoch: ", epoch, " loss: ", loss_output.asnumpy(), " cost time: ", cost_time) assert (loss_output.asnumpy() < 0.1)
def test_loss_scale_fp16_lr_overflow_set_sense_scale(): inputs = Tensor(np.ones([16, 16]).astype(np.float32)) label = Tensor(np.zeros([16, 16]).astype(np.float32)) lr = Tensor(np.ones([1], np.float32) * 0.1) net = NetFP16(16, 16) net.set_train() loss = MSELoss() optimizer = Momentum(net.trainable_params(), learning_rate=lr, momentum=0.9) net_with_loss = WithLossCell(net, loss) train_network = TrainOneStepWithLossScaleCell( net_with_loss, optimizer, scale_sense=Tensor(np.full((1), np.finfo(np.float32).max), dtype=mstype.float32)) output_1 = train_network(inputs, label) train_network.set_sense_scale( Tensor(np.full((1), np.finfo(np.float32).max), dtype=mstype.float32)) output_2 = train_network(inputs, label) assert output_1[0].asnumpy() == output_2[0].asnumpy() assert output_1[1].asnumpy() == output_2[1].asnumpy() == True
def adam_compile(loss_scale=1.0): inputs = Tensor(np.ones([15, 1]).astype(np.float32)) label = Tensor(np.zeros([15, 1]).astype(np.float32)) net = Net(1, 1) loss = MSELoss() optimizer = Adam(net.trainable_params(), learning_rate=1e-3, beta1=0.9, beta2=0.999, eps=1e-8, use_locking=False, use_nesterov=False, weight_decay=0.0, loss_scale=loss_scale) net_with_loss = WithLossCell(net, loss) train_network = TrainOneStepWithLossScaleCell(net_with_loss, optimizer, scale_sense=Tensor( np.full((1), 1.0), dtype=mstype.float32)) train_network.set_train() output = train_network(inputs, label) print("the result is ", output)
def test_loss_scale_fp16_overflow(): inputs = Tensor(np.ones([16, 16]).astype(np.float32)) label = Tensor(np.zeros([16, 16]).astype(np.float32)) scaling_sens = Tensor(np.full((1), np.finfo(np.float32).max), dtype=mstype.float32) net = NetFP16(16, 16) net.set_train() loss = MSELoss() optimizer = Lamb(net.trainable_params(), learning_rate=0.01) net_with_loss = WithLossCell(net, loss) net_with_loss.set_grad() train_network = TrainOneStepWithLossScaleCell(net_with_loss, optimizer) output_1 = train_network(inputs, label, scaling_sens) output_2 = train_network(inputs, label, scaling_sens) assert output_1[0].asnumpy() == output_2[0].asnumpy() assert output_1[1].asnumpy() == output_2[1].asnumpy() == True
def test_compile_fp16_overflow(): inputs = Tensor(np.ones([16, 16]).astype(np.float32)) label = Tensor(np.zeros([16, 16]).astype(np.float32)) scaling_sens = Tensor(np.full((1), np.finfo(np.float32).max), dtype=mstype.float32) net = NetFP16(16, 16) loss = MSELoss() optimizer = Lamb(net.trainable_params(), decay_steps=10, warmup_steps=5) net_with_loss = WithLossCell(net, loss) train_network = TrainOneStepWithLossScaleCell(net_with_loss, optimizer) train_network.set_train() output = train_network(inputs, label, scaling_sens) print("the result is ", output)
def test_momentum_compile(): inputs = Tensor(np.ones([15, 1]).astype(np.float32)) label = Tensor(np.zeros([15, 1]).astype(np.float32)) scaling_sens = Tensor(np.full((1), 1.0), dtype=mstype.float32) net = Net(1, 1) loss = MSELoss() optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) net_with_loss = WithLossCell(net, loss) train_network = TrainOneStepWithLossScaleCell(net_with_loss, optimizer) train_network.set_train() output = train_network(inputs, label, scaling_sens) print("the result is ", output)
def test_compile_fp16_lr_overflow(): inputs = Tensor(np.ones([16, 16]).astype(np.float32)) label = Tensor(np.zeros([16, 16]).astype(np.float32)) scaling_sens = Tensor(np.full((1), np.finfo(np.float32).max), dtype=mstype.float32) lr = Tensor(np.ones([1], np.float32) * 0.1) net = NetFP16(16, 16) loss = MSELoss() optimizer = Momentum(net.trainable_params(), learning_rate=lr, momentum=0.9) net_with_loss = WithLossCell(net, loss) train_network = TrainOneStepWithLossScaleCell(net_with_loss, optimizer) train_network.set_train() output = train_network(inputs, label, scaling_sens) print("the result is ", output)
def test_compile_fp16_lr_overflow_dynamic_graph(): inputs = Tensor(np.ones([16, 16]).astype(np.float32)) label = Tensor(np.zeros([16, 16]).astype(np.float32)) lr = Tensor(np.ones([1], np.float32) * 0.1) net = NetFP16(16, 16) loss = MSELoss() optimizer = Momentum(net.trainable_params(), learning_rate=lr, momentum=0.9) net_with_loss = WithLossCell(net, loss) scale_manager = DynamicLossScaleManager() update_cell = scale_manager.get_update_cell() train_network = TrainOneStepWithLossScaleCell(net_with_loss, optimizer, scale_update_cell=update_cell) train_network.set_train() output = train_network(inputs, label) print("the result is ", output)
def test_compile_grad_error(): inputs = Tensor(np.ones([16, 16]).astype(np.float32)) label = Tensor(np.zeros([16, 16]).astype(np.float32)) lr = Tensor(np.ones([1], np.float32) * 0.1) net = NetFP16(16, 16) loss = MSELoss() optimizer = Momentum(net.trainable_params(), learning_rate=lr, momentum=0.9) net_with_loss = WithLossCell(net, loss) scale_manager = DynamicLossScaleManager() update_cell = scale_manager.get_update_cell() train_network = TrainOneStepWithLossScaleCell(net_with_loss, optimizer, scale_update_cell=update_cell) train_network.set_train() with pytest.raises(TypeError) as e: train_network(inputs, label) print(e)