def test_sit_auto_mix_precision_train_o3():
input_data = np.random.randn(32, 3, 224, 224).astype(np.float64)
label_data = np.random.randn(32, 10).astype(np.float32)
# graph mode
context.set_context(mode=context.GRAPH_MODE)
net = Net(3, 10)
opt = nn.Momentum(params=net.trainable_params(), learning_rate=0.001, momentum=0.0009, weight_decay=0.001,
loss_scale=0.0001)
loss = nn.SoftmaxCrossEntropyWithLogits(sparse=False)
train_network = amp.build_train_network(net, opt, loss, level="O3",
loss_scale_manager=FixedLossScaleManager(drop_overflow_update=False))
out = train_network(Tensor(input_data), Tensor(label_data))
# pynative mode
context.set_context(mode=context.PYNATIVE_MODE)
net_pynative = Net(3, 10)
opt_pynative = nn.Momentum(params=net_pynative.trainable_params(), learning_rate=0.001, momentum=0.0009,
weight_decay=0.001,
loss_scale=0.0001)
loss_pynative = nn.SoftmaxCrossEntropyWithLogits(sparse=False)
train_network_pynative = amp.build_train_network(net_pynative, opt_pynative, loss_pynative, level="O3",
loss_scale_manager=FixedLossScaleManager(
drop_overflow_update=False))
out_pynative = train_network_pynative(Tensor(input_data), Tensor(label_data))
assert np.allclose(out.asnumpy(), out_pynative.asnumpy(), 0.001, 0.001)
def test_sit_auto_mix_precision_model_o2():
input_data = np.random.randn(32, 3, 224, 224).astype(np.float32)
dataset1 = FakeData(size=32,
batch_size=32,
image_size=(3, 224, 224),
num_classes=10,
fakedata_mode=FakeDataInitMode.OnesInit)
dataset2 = FakeData(size=32,
batch_size=32,
image_size=(3, 224, 224),
num_classes=10,
fakedata_mode=FakeDataInitMode.OnesInit)
# graph mode
context.set_context(mode=context.GRAPH_MODE)
context.set_context(save_graphs=True, save_graphs_path='./test_amp_o2')
net = Net(3, 10)
opt = nn.Momentum(params=net.trainable_params(), learning_rate=0.001, momentum=0.0009)
loss = nn.SoftmaxCrossEntropyWithLogits(sparse=False)
model = Model(net, loss, opt, amp_level="O2")
model.train(1, dataset1, dataset_sink_mode=False)
contend = read_validateir_file('./test_amp_o2')
castnum = re.findall("Cast", contend)
assert len(castnum) == 14
clean_all_ir_files('./test_amp_o2')
out_graph = model.predict(Tensor(input_data))
# pynative mode
context.set_context(mode=context.PYNATIVE_MODE)
net_pynative = Net(3, 10)
opt_pynative = nn.Momentum(params=net_pynative.trainable_params(), learning_rate=0.001, momentum=0.0009)
loss_pynative = nn.SoftmaxCrossEntropyWithLogits(sparse=False)
model_pynative = Model(net_pynative, loss_pynative, opt_pynative, amp_level="O2")
model_pynative.train(1, dataset2, dataset_sink_mode=False)
out_pynative = model_pynative.predict(Tensor(input_data))
allclose_nparray(out_graph.asnumpy(), out_pynative.asnumpy(), 0.001, 0.001)
def test_train_lenet_with_new_interface(num_classes=10,
epoch=20,
batch_size=32):
context.set_context(mode=context.GRAPH_MODE, device_target="GPU")
network = LeNet5(num_classes)
criterion = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")
net_with_criterion = WithLossCell(network, criterion)
net_with_criterion.set_train()
weights = ParameterTuple(network.trainable_params())
optimizer = nn.Momentum(weights, 0.1, 0.9)
train_network = ForwardValueAndGrad(network=net_with_criterion,
weights=weights,
get_by_list=True,
sens_param=True)
losses = []
for i in range(0, epoch):
data = Tensor(
np.ones([batch_size, 1, 32, 32]).astype(np.float32) * 0.01)
label = Tensor(np.ones([batch_size]).astype(np.int32))
sens = Tensor(np.ones([1]).astype(np.float32))
loss, grads = train_network(data, label, sens)
grads = F.identity(grads)
optimizer(grads)
losses.append(loss)
assert losses[-1].asnumpy() < 0.01
assert losses[-1].asnumpy() > 0.001
def test_mobilenetv2_quant():
set_seed(1)
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
config = config_ascend_quant
print("training configure: {}".format(config))
epoch_size = config.epoch_size
# define network
network = mobilenetV2(num_classes=config.num_classes)
# define loss
if config.label_smooth > 0:
loss = CrossEntropyWithLabelSmooth(
smooth_factor=config.label_smooth, num_classes=config.num_classes)
else:
loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
# define dataset
dataset = create_dataset(dataset_path=dataset_path,
config=config,
repeat_num=1,
batch_size=config.batch_size)
step_size = dataset.get_dataset_size()
# convert fusion network to quantization aware network
quantizer = QuantizationAwareTraining(bn_fold=True,
per_channel=[True, False],
symmetric=[True, False])
network = quantizer.quantize(network)
# get learning rate
lr = Tensor(get_lr(global_step=config.start_epoch * step_size,
lr_init=0,
lr_end=0,
lr_max=config.lr,
warmup_epochs=config.warmup_epochs,
total_epochs=epoch_size + config.start_epoch,
steps_per_epoch=step_size))
# define optimization
opt = nn.Momentum(filter(lambda x: x.requires_grad, network.get_parameters()), lr, config.momentum,
config.weight_decay)
# define model
model = Model(network, loss_fn=loss, optimizer=opt)
print("============== Starting Training ==============")
monitor = Monitor(lr_init=lr.asnumpy(),
step_threshold=config.step_threshold)
callback = [monitor]
model.train(epoch_size, dataset, callbacks=callback,
dataset_sink_mode=False)
print("============== End Training ==============")
export_time_used = 650
train_time = monitor.step_mseconds
print('train_time_used:{}'.format(train_time))
assert train_time < export_time_used
expect_avg_step_loss = 2.32
avg_step_loss = np.mean(np.array(monitor.losses))
print("average step loss:{}".format(avg_step_loss))
assert avg_step_loss < expect_avg_step_loss
def main():
from mindspore import nn
# dataset
if os.path.exists(args.data_path):
dataset = ems.create_mnist_dataset(args.data_path,
is_train=(not parser.do_eval),
batch_size=args.batch_size)
else:
dataset = ems.create_mocker_dataset()
ems.print_ds_info(dataset)
# network
network = ems.LeNet5()
# loss & opt
loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")
optimizer = nn.Momentum(network.trainable_params(), 0.01, 0.9)
# train
ems.Trainer('lenet', parser) \
.set_dataset(dataset) \
.set_network(network) \
.set_loss_fn(loss_fn) \
.set_optimizer(optimizer) \
.set_callbacks([ems.TimeMonitor()]) \
.run()
def train_on_gpu():
config = config_gpu_quant if args_opt.quantization_aware else config_gpu
print("training args: {}".format(args_opt))
print("training configure: {}".format(config))
# define network
network = mobilenetV2(num_classes=config.num_classes)
# define loss
if config.label_smooth > 0:
loss = CrossEntropyWithLabelSmooth(smooth_factor=config.label_smooth,
num_classes=config.num_classes)
else:
loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction='mean')
# define dataset
epoch_size = config.epoch_size
dataset = create_dataset(dataset_path=args_opt.dataset_path,
do_train=True,
config=config,
device_target=args_opt.device_target,
repeat_num=1,
batch_size=config.batch_size)
step_size = dataset.get_dataset_size()
# resume
if args_opt.pre_trained:
param_dict = load_checkpoint(args_opt.pre_trained)
load_param_into_net(network, param_dict)
# convert fusion network to quantization aware network
if config.quantization_aware:
network = quant.convert_quant_network(network,
bn_fold=True,
per_channel=[True, False],
symmetric=[True, True])
# get learning rate
loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False)
lr = Tensor(get_lr(global_step=config.start_epoch * step_size,
lr_init=0,
lr_end=0,
lr_max=config.lr,
warmup_epochs=config.warmup_epochs,
total_epochs=epoch_size + config.start_epoch,
steps_per_epoch=step_size))
# define optimization
opt = nn.Momentum(filter(lambda x: x.requires_grad, network.get_parameters()), lr, config.momentum,
config.weight_decay, config.loss_scale)
# define model
model = Model(network, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale)
print("============== Starting Training ==============")
callback = [Monitor(lr_init=lr.asnumpy())]
ckpt_save_dir = config.save_checkpoint_path + "ckpt_" + str(get_rank()) + "/"
if config.save_checkpoint:
config_ck = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_epochs * step_size,
keep_checkpoint_max=config.keep_checkpoint_max)
ckpt_cb = ModelCheckpoint(prefix="mobilenetV2", directory=ckpt_save_dir, config=config_ck)
callback += [ckpt_cb]
model.train(epoch_size, dataset, callbacks=callback)
print("============== End Training ==============")
def mnist_train(epoch_size, batch_size, lr, momentum):
mnist_path = "./MNIST_unzip/"
ds = generate_mnist_dataset(os.path.join(mnist_path, "train"),
batch_size=batch_size,
repeat_size=1)
network = LeNet5()
net_loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False,
sparse=True,
reduction="mean")
net_opt = nn.Momentum(network.trainable_params(), lr, momentum)
config_ck = CheckpointConfig(save_checkpoint_steps=1875,
keep_checkpoint_max=10)
ckpoint_cb = ModelCheckpoint(prefix="checkpoint_lenet",
directory="./trained_ckpt_file/",
config=config_ck)
model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()})
LOGGER.info(TAG, "============== Starting Training ==============")
model.train(epoch_size,
ds,
callbacks=[ckpoint_cb, LossMonitor()],
dataset_sink_mode=False)
LOGGER.info(TAG, "============== Starting Testing ==============")
ckpt_file_name = "trained_ckpt_file/checkpoint_lenet-10_1875.ckpt"
param_dict = load_checkpoint(ckpt_file_name)
load_param_into_net(network, param_dict)
ds_eval = generate_mnist_dataset(os.path.join(mnist_path, "test"),
batch_size=batch_size)
acc = model.eval(ds_eval, dataset_sink_mode=False)
LOGGER.info(TAG, "============== Accuracy: %s ==============", acc)
def test_dp_model_with_graph_mode_ada_gaussian():
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
norm_bound = 1.0
initial_noise_multiplier = 0.01
network = LeNet5()
batch_size = 32
batches = 128
epochs = 1
alpha = 0.8
loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True)
noise_mech = NoiseMechanismsFactory().create(
'AdaGaussian',
norm_bound=norm_bound,
initial_noise_multiplier=initial_noise_multiplier,
noise_decay_rate=alpha,
decay_policy='Exp')
clip_mech = None
net_opt = nn.Momentum(network.trainable_params(),
learning_rate=0.1,
momentum=0.9)
model = DPModel(micro_batches=2,
clip_mech=clip_mech,
norm_bound=norm_bound,
noise_mech=noise_mech,
network=network,
loss_fn=loss,
optimizer=net_opt,
metrics=None)
ms_ds = ds.GeneratorDataset(dataset_generator(batch_size, batches),
['data', 'label'])
ms_ds.set_dataset_size(batch_size * batches)
model.train(epochs, ms_ds, dataset_sink_mode=False)
def main(data_path,
device_target='Ascend',
summary_dir='./summary_dir',
learning_rate=0.01):
context.set_context(mode=context.GRAPH_MODE, device_target=device_target)
momentum = 0.9
epoch_size = 1
batch_size = 32
network = LeNet5()
network.set_train()
net_loss = CrossEntropyLoss()
net_opt = nn.Momentum(network.trainable_params(), learning_rate, momentum)
model = Model(network, net_loss, net_opt)
# Init SummaryCollector callback to record summary data in model.train or model.eval
summary_collector = SummaryCollector(summary_dir=summary_dir,
collect_freq=10)
ds = create_dataset(os.path.join(data_path, "train"),
batch_size=batch_size)
print("============== Starting Training ==============")
model.train(epoch_size,
ds,
callbacks=[summary_collector],
dataset_sink_mode=False)
print("============== Train End =====================")
def test_sit_auto_mix_precision_model_o0():
input_data = np.random.randn(32, 3, 224, 224).astype(np.float32)
dataset1 = FakeData(size=32,
batch_size=32,
image_size=(3, 224, 224),
num_classes=10,
fakedata_mode=FakeDataInitMode.OnesInit)
dataset1.set_label_data_type(np.float16)
# graph mode
context.set_context(mode=context.GRAPH_MODE)
context.set_context(save_graphs=True, save_graphs_path='./test_amp_o0')
net = Net(3, 10)
net.to_float(dtype.float16)
opt = nn.Momentum(params=net.trainable_params(),
learning_rate=0.001,
momentum=0.0009)
loss = nn.SoftmaxCrossEntropyWithLogits(sparse=False)
model = Model(net, loss, opt, amp_level="O0")
model.train(1, dataset1, dataset_sink_mode=False)
contend = read_validateir_file('./test_amp_o0')
castnum = re.findall("Cast", contend)
assert len(castnum) == 17
model.predict(Tensor(input_data))
contend = read_validateir_file('./test_amp_o0')
castnum = re.findall("Cast", contend)
assert len(castnum) == 11
def train(data_dir, lr=0.01, momentum=0.9, num_epochs=2, ckpt_name="lenet"):
dataset_sink = context.get_context('device_target') == 'Ascend'
repeat = num_epochs if dataset_sink else 1
ds_train = create_dataset(data_dir, repeat=repeat)
ds_eval = create_dataset(data_dir, training=False)
steps_per_epoch = ds_train.get_dataset_size()
net = LeNet5()
loss = nn.loss.SoftmaxCrossEntropyWithLogits(is_grad=False,
sparse=True,
reduction='mean')
opt = nn.Momentum(net.trainable_params(), lr, momentum)
ckpt_cfg = CheckpointConfig(save_checkpoint_steps=steps_per_epoch,
keep_checkpoint_max=5)
ckpt_cb = ModelCheckpoint(prefix=ckpt_name,
directory='ckpt',
config=ckpt_cfg)
loss_cb = LossMonitor(steps_per_epoch)
model = Model(net, loss, opt, metrics={'acc', 'loss'})
model.train(num_epochs,
ds_train,
callbacks=[ckpt_cb, loss_cb],
dataset_sink_mode=dataset_sink)
metrics = model.eval(ds_eval, dataset_sink_mode=dataset_sink)
print('Metrics:', metrics)
def test_dp_model_with_graph_mode():
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
norm_bound = 1.0
initial_noise_multiplier = 0.01
network = Net()
epochs = 1
loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
noise_mech = NoiseMechanismsFactory().create(
'Gaussian',
norm_bound=norm_bound,
initial_noise_multiplier=initial_noise_multiplier)
clip_mech = ClipMechanismsFactory().create('Gaussian',
decay_policy='Linear',
learning_rate=0.01,
target_unclipped_quantile=0.9,
fraction_stddev=0.01)
net_opt = nn.Momentum(network.trainable_params(),
learning_rate=0.1,
momentum=0.9)
model = DPModel(micro_batches=2,
clip_mech=clip_mech,
norm_bound=norm_bound,
noise_mech=noise_mech,
network=network,
loss_fn=loss,
optimizer=net_opt,
metrics=None)
ms_ds = ds.GeneratorDataset(dataset_generator, ['data', 'label'])
model.train(epochs, ms_ds, dataset_sink_mode=False)
def __init__(self, model=None):
"""Initializing the trainer with the provided model.
Arguments:
client_id: The ID of the client using this trainer (optional).
model: The model to train.
"""
super().__init__()
if hasattr(Config().trainer, 'cpuonly') and Config().trainer.cpuonly:
mindspore.context.set_context(mode=mindspore.context.PYNATIVE_MODE,
device_target='CPU')
else:
mindspore.context.set_context(mode=mindspore.context.PYNATIVE_MODE,
device_target='GPU')
if model is None:
self.model = models_registry.get()
# Initializing the loss criterion
loss_criterion = SoftmaxCrossEntropyWithLogits(sparse=True,
reduction='mean')
# Initializing the optimizer
optimizer = nn.Momentum(self.model.trainable_params(),
Config().trainer.learning_rate,
Config().trainer.momentum)
self.mindspore_model = mindspore.Model(
self.model,
loss_criterion,
optimizer,
metrics={"Accuracy": Accuracy()})
def train_lenet_quant():
context.set_context(mode=context.GRAPH_MODE, device_target=device_target)
cfg = quant_cfg
ckpt_path = './ckpt_lenet_noquant-10_1875.ckpt'
ds_train = create_dataset(os.path.join(data_path, "train"), cfg.batch_size, 1)
step_size = ds_train.get_dataset_size()
# define fusion network
network = LeNet5Fusion(cfg.num_classes)
# load quantization aware network checkpoint
param_dict = load_checkpoint(ckpt_path)
load_nonquant_param_into_quant_net(network, param_dict)
# convert fusion network to quantization aware network
network = quant.convert_quant_network(network, quant_delay=900, bn_fold=False, per_channel=[True, False],
symmetric=[False, False])
# define network loss
net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")
# define network optimization
net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum)
# call back and monitor
config_ckpt = CheckpointConfig(save_checkpoint_steps=cfg.epoch_size * step_size,
keep_checkpoint_max=cfg.keep_checkpoint_max)
ckpt_callback = ModelCheckpoint(prefix="ckpt_lenet_quant", config=config_ckpt)
# define model
model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()})
print("============== Starting Training ==============")
model.train(cfg['epoch_size'], ds_train, callbacks=[ckpt_callback, LossMonitor()],
dataset_sink_mode=True)
print("============== End Training ==============")
def test_get_membership_inference_object():
net = Net()
loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
opt = nn.Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9)
model = Model(network=net, loss_fn=loss, optimizer=opt)
inference_model = MembershipInference(model, -1)
assert isinstance(inference_model, MembershipInference)
def test_dp_monitor_gpu():
context.set_context(mode=context.GRAPH_MODE, device_target="GPU")
batch_size = 16
batches = 128
epochs = 1
rdp = PrivacyMonitorFactory.create(policy='rdp',
num_samples=60000,
batch_size=batch_size,
initial_noise_multiplier=0.4,
noise_decay_rate=6e-5)
suggest_epoch = rdp.max_epoch_suggest()
LOGGER.info(TAG, 'The recommended maximum training epochs is: %s',
suggest_epoch)
network = LeNet5()
net_loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False,
sparse=True,
reduction="mean")
net_opt = nn.Momentum(network.trainable_params(), 0.01, 0.9)
model = Model(network, net_loss, net_opt)
LOGGER.info(TAG, "============== Starting Training ==============")
ds1 = ds.GeneratorDataset(dataset_generator(batch_size, batches),
["data", "label"])
ds1.set_dataset_size(batch_size * batches)
model.train(epochs, ds1, callbacks=[rdp], dataset_sink_mode=False)
def eval_quant():
context.set_context(mode=context.GRAPH_MODE, device_target=device_target)
cfg = quant_cfg
ds_eval = create_dataset(os.path.join(data_path, "test"), cfg.batch_size,
1)
ckpt_path = './ckpt_lenet_quant-10_937.ckpt'
# define fusion network
network = LeNet5Fusion(cfg.num_classes)
# convert fusion network to quantization aware network
quantizer = QuantizationAwareTraining(quant_delay=0,
bn_fold=False,
freeze_bn=10000,
per_channel=[True, False],
symmetric=[True, False])
network = quantizer.quantize(network)
# define loss
net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")
# define network optimization
net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum)
# call back and monitor
model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()})
# load quantization aware network checkpoint
param_dict = load_checkpoint(ckpt_path)
not_load_param = load_param_into_net(network, param_dict)
if not_load_param:
raise ValueError("Load param into net fail!")
print("============== Starting Testing ==============")
acc = model.eval(ds_eval, dataset_sink_mode=True)
print("============== {} ==============".format(acc))
assert acc['Accuracy'] > 0.98
def test_amp_o2():
inputs = Tensor(np.ones([16, 16]).astype(np.float32))
label = Tensor(np.zeros([16, 16]).astype(np.float32))
net = Net(16, 16)
optimizer = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9)
train_network = amp.build_train_network(net, optimizer, level="O2")
_ = train_network(inputs, label)
def test_amp_resnet50_loss():
inputs = Tensor(np.ones([2, 3, 224, 224]).astype(np.float32))
label = Tensor(np.zeros([2, 10]).astype(np.float32))
net = resnet50()
loss = nn.SoftmaxCrossEntropyWithLogits(reduction='mean')
optimizer = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9)
train_network = amp.build_train_network(net, optimizer, loss, level="O2")
train_network(inputs, label)
def test_amp_o0_loss():
inputs = Tensor(np.ones([16, 16]).astype(np.float32))
label = Tensor(np.zeros([16, 16]).astype(np.float32))
net = NetNoLoss(16, 16)
loss = nn.MSELoss()
optimizer = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9)
train_network = amp.build_train_network(net, optimizer, loss)
output = train_network(inputs, label)
def __init__(self, network):
super(TrainStepWrap, self).__init__()
self.network = network
self.network.set_train()
self.weights = ParameterTuple(network.trainable_params())
self.optimizer = nn.Momentum(self.weights, 0.1, 0.9)
self.hyper_map = C.HyperMap()
self.grad = C.GradOperation(get_by_list=True)
def __init__(self, network, sens):
super(TrainStepWrap2, self).__init__()
self.network = network
self.network.set_train()
self.weights = ParameterTuple(network.get_parameters())
self.optimizer = nn.Momentum(self.weights, 0.1, 0.9)
self.hyper_map = C.HyperMap()
self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True)
self.sens = sens
def test_bias_add():
context.set_context(mode=context.GRAPH_MODE)
context.set_auto_parallel_context(parallel_mode="auto_parallel", device_num=8)
input_np = np.ones([16, 3, 32, 32]).astype(np.float32)
label_np = np.zeros([16, 2048]).astype(np.float32)
dataset = DatasetLenet(Tensor(input_np), Tensor(label_np), 1)
net = Net()
loss = CrossEntropyLoss()
opt = nn.Momentum(learning_rate=0.01, momentum=0.9, params=net.get_parameters())
model = Model(network=net, loss_fn=loss, optimizer=opt)
model.train(epoch=1, train_dataset=dataset, dataset_sink_mode=False)
def test_gradient_accumulation():
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
ds_train = create_dataset(
os.path.join("/home/workspace/mindspore_dataset/mnist", "train"), 32)
network = LeNet5(10)
net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")
net_opt = nn.Momentum(network.trainable_params(), 0.01, 0.9)
model = GradientAccumulation(network, net_loss, net_opt)
print("============== Starting Training ==============")
model.train_process(2, ds_train, mini_steps=4)
def test_heterogeneous_excutor():
input_data = np.random.randn(32, 3, 224, 224).astype(np.float64)
label_data = np.random.randn(32, 10).astype(np.float32)
# graph mode
context.set_context(mode=context.GRAPH_MODE)
net = Net(3, 10)
opt = nn.Momentum(params=net.trainable_params(),
learning_rate=0.001,
momentum=0.0009,
weight_decay=0.001,
loss_scale=0.0001)
loss = nn.SoftmaxCrossEntropyWithLogits(sparse=False)
train_network = amp.build_train_network(
net,
opt,
loss,
level="O3",
loss_scale_manager=FixedLossScaleManager(drop_overflow_update=False))
out = train_network(Tensor(input_data), Tensor(label_data))
# heterogeneous_excutor
net_heter = Net(3, 10)
net_heter.relu.relu.add_prim_attr("primitive_target", "CPU")
net_heter.conv.conv2d.add_prim_attr("primitive_target", "CPU")
opt_heter = nn.Momentum(params=net_heter.trainable_params(),
learning_rate=0.001,
momentum=0.0009,
weight_decay=0.001,
loss_scale=0.0001)
loss_heter = nn.SoftmaxCrossEntropyWithLogits(sparse=False)
train_network_heter = amp.build_train_network(
net_heter,
opt_heter,
loss_heter,
level="O3",
loss_scale_manager=FixedLossScaleManager(drop_overflow_update=False))
out_heter = train_network_heter(Tensor(input_data), Tensor(label_data))
assert np.allclose(out.asnumpy(), out_heter.asnumpy(), 0.001, 0.001)
def me_train_tensor(net, input_np, label_np, epoch_size=2):
loss = SoftmaxCrossEntropyWithLogits(sparse=True)
opt = nn.Momentum(Tensor(np.array([0.1])), Tensor(np.array([0.9])),
filter(lambda x: x.requires_grad, net.get_parameters()))
context.set_context(mode=context.GRAPH_MODE)
Model(net, loss, opt)
_network = nn.WithLossCell(net, loss)
_train_net = MsWrapper(nn.TrainOneStepCell(_network, opt))
_train_net.set_train()
for epoch in range(0, epoch_size):
print(f"epoch %d" % (epoch))
output = _train_net(Tensor(input_np), Tensor(label_np))
print(output.asnumpy())
def eval_alexnet():
print("============== Starting Testing ==============")
device_num = get_device_num()
if device_num > 1:
# context.set_context(mode=context.GRAPH_MODE, device_target=config.device_target)
context.set_context(mode=context.GRAPH_MODE,
device_target='Davinci',
save_graphs=False)
if config.device_target == "Ascend":
context.set_context(device_id=get_device_id())
init()
elif config.device_target == "GPU":
init()
if config.dataset_name == 'cifar10':
network = AlexNet(config.num_classes, phase='test')
loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")
opt = nn.Momentum(network.trainable_params(), config.learning_rate,
config.momentum)
ds_eval = create_dataset_cifar10(config.data_path, config.batch_size, status="test", \
target=config.device_target)
param_dict = load_checkpoint(load_path)
print("load checkpoint from [{}].".format(load_path))
load_param_into_net(network, param_dict)
network.set_train(False)
model = Model(network, loss, opt, metrics={"Accuracy": Accuracy()})
elif config.dataset_name == 'imagenet':
network = AlexNet(config.num_classes, phase='test')
loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")
ds_eval = create_dataset_imagenet(config.data_path,
config.batch_size,
training=False)
param_dict = load_checkpoint(load_path)
print("load checkpoint from [{}].".format(load_path))
load_param_into_net(network, param_dict)
network.set_train(False)
model = Model(network,
loss_fn=loss,
metrics={'top_1_accuracy', 'top_5_accuracy'})
else:
raise ValueError("Unsupported dataset.")
if ds_eval.get_dataset_size() == 0:
raise ValueError(
"Please check dataset size > 0 and batch_size <= dataset size")
result = model.eval(ds_eval, dataset_sink_mode=config.dataset_sink_mode)
print("result : {}".format(result))
def test_all_trains():
ds_train = create_dataset(
os.path.join('/home/workspace/mindspore_dataset/mnist', "train"), 32,
1)
network = LeNet5(10)
net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")
net_opt = nn.Momentum(network.trainable_params(), 0.01, 0.9)
time_cb = TimeMonitor(data_size=ds_train.get_dataset_size())
model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()})
print("============== Starting Training ==============")
model.train(1, ds_train, callbacks=[time_cb, LossMonitor()])
def train(data_dir, lr=0.01, momentum=0.9, num_epochs=3):
ds_train = create_dataset(data_dir)
ds_eval = create_dataset(data_dir, training=False)
net = LeNet5()
loss = nn.loss.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
opt = nn.Momentum(net.trainable_params(), lr, momentum)
loss_cb = LossMonitor(per_print_times=ds_train.get_dataset_size())
model = Model(net, loss, opt, metrics={'acc', 'loss'})
# dataset_sink_mode can be True when using Ascend
model.train(num_epochs, ds_train, callbacks=[loss_cb], dataset_sink_mode=False)
metrics = model.eval(ds_eval, dataset_sink_mode=False)
print('Metrics:', metrics)
def train_lenet():
context.set_context(mode=context.GRAPH_MODE, save_graphs=True, device_target="CPU")
dataset_sink_mode = False
# download mnist dataset
download_dataset()
# learning rate setting
lr = 0.01
momentum = 0.9
epoch_size = 1
mnist_path = "../MNIST_Data"
# define the loss function
net_loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction="mean")
repeat_size = epoch_size
# create the network
network = LeNet5()
# define the optimizer
net_opt = nn.Momentum(network.trainable_params(), lr, momentum)
config_ck = CheckpointConfig(save_checkpoint_steps=1875, keep_checkpoint_max=10)
# save the network model and parameters for subsequence fine-tuning
ckpoint_cb = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ck)
# group layers into an object with training and evaluation features
model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()})
summary_writer = SummaryRecord(log_dir="../../summary", network=network)
summary_callback = SummaryStep(summary_writer, flush_step=10)
# Init TrainLineage to record the training information
train_callback = TrainLineage(summary_writer)
train_net(
model,
epoch_size,
mnist_path,
repeat_size,
ckpoint_cb,
dataset_sink_mode,
callbacks=[summary_callback, train_callback],
)
test_net(network, model, mnist_path)
summary_writer.close()
