def _build_train_forward_backward_network(self):
"""Build forward and backward network"""
network = self._network
network = nn.WithLossCell(network, self._loss_fn)
loss_scale = 1.0
network = TrainForwardBackward(network, self._optimizer, self._grad_sum, loss_scale).set_train()
return network
def test_svi_vae():
# define the encoder and decoder
encoder = Encoder()
decoder = Decoder()
# define the vae model
vae = VAE(encoder, decoder, hidden_size=400, latent_size=20)
# define the loss function
net_loss = ELBO(latent_prior='Normal', output_prior='Normal')
# define the optimizer
optimizer = nn.Adam(params=vae.trainable_params(), learning_rate=0.001)
# define the training dataset
ds_train = create_dataset(image_path, 128, 1)
net_with_loss = nn.WithLossCell(vae, net_loss)
# define the variational inference
vi = SVI(net_with_loss=net_with_loss, optimizer=optimizer)
# run the vi to return the trained network.
vae = vi.run(train_dataset=ds_train, epochs=5)
# get the trained loss
trained_loss = vi.get_train_loss()
# test function: generate_sample
generated_sample = vae.generate_sample(64, IMAGE_SHAPE)
# test function: reconstruct_sample
for sample in ds_train.create_dict_iterator():
sample_x = Tensor(sample['image'], dtype=mstype.float32)
reconstructed_sample = vae.reconstruct_sample(sample_x)
print('The loss of the trained network is ', trained_loss)
print('The hape of the generated sample is ', generated_sample.shape)
print('The shape of the reconstructed sample is ',
reconstructed_sample.shape)
def _build_train_network(self):
"""Build train network"""
network = self._network
if self._optimizer:
if self._loss_scale_manager_set:
network = amp.build_train_network(
network,
self._optimizer,
self._loss_fn,
level=self._amp_level,
loss_scale_manager=self._loss_scale_manager,
keep_batchnorm_fp32=self._keep_bn_fp32)
else:
network = amp.build_train_network(
network,
self._optimizer,
self._loss_fn,
level=self._amp_level,
keep_batchnorm_fp32=self._keep_bn_fp32)
elif self._loss_fn:
network = nn.WithLossCell(network, self._loss_fn)
# If need to check if loss_fn is not None, but optimizer is None
if self._parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL,
ParallelMode.AUTO_PARALLEL):
network.set_auto_parallel()
return network
def test_vae_gan():
vae_gan = VaeGan()
net_loss = VaeGanLoss()
optimizer = nn.Adam(params=vae_gan.trainable_params(), learning_rate=0.001)
ds_train = create_dataset(image_path, 128, 1)
net_with_loss = nn.WithLossCell(vae_gan, net_loss)
vi = SVI(net_with_loss=net_with_loss, optimizer=optimizer)
vae_gan = vi.run(train_dataset=ds_train, epochs=5)
def build_fn(batchnorm, dropout, l2):
# initilize the net, Loss, optimizer
net = CosineNet(batchnorm=batchnorm, dropout=dropout)
loss = nn.loss.MSELoss()
opt = nn.optim.Adam(net.trainable_params(),
learning_rate=LEARNING_RATE,
weight_decay=WEIGHT_DECAY if l2 else 0.0)
# Build a net with loss and optimizer.
with_loss = nn.WithLossCell(net, loss)
train_step = nn.TrainOneStepCell(with_loss, opt).set_train()
return train_step, with_loss, net
def me_train_tensor(net, input_np, label_np, epoch_size=2):
loss = SoftmaxCrossEntropyWithLogits(sparse=True)
opt = nn.Momentum(Tensor(np.array([0.1])), Tensor(np.array([0.9])),
filter(lambda x: x.requires_grad, net.get_parameters()))
context.set_context(mode=context.GRAPH_MODE)
Model(net, loss, opt)
_network = nn.WithLossCell(net, loss)
_train_net = MsWrapper(nn.TrainOneStepCell(_network, opt))
_train_net.set_train()
for epoch in range(0, epoch_size):
print(f"epoch %d" % (epoch))
output = _train_net(Tensor(input_np), Tensor(label_np))
print(output.asnumpy())
def me_train_tensor(net, input_np, label_np, epoch_size=2):
"""me_train_tensor"""
loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")
opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr_gen(lambda i: 0.1, epoch_size), 0.9,
0.01, 1024)
Model(net, loss, opt)
_network = nn.WithLossCell(net, loss)
_train_net = nn.TrainOneStepCell(_network, opt)
_train_net.set_train()
label_np = np.argmax(label_np, axis=-1).astype(np.int32)
for epoch in range(0, epoch_size):
print(f"epoch %d" % (epoch))
_train_net(Tensor(input_np), Tensor(label_np))
def me_train_tensor(net, input_np, label_np, epoch_size=2):
"""me_train_tensor"""
loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True)
# reorder the net parameters , leave the parameters that need to be passed into lars to the end part
opt = Momentum(
get_net_trainable_reordered_params(net)[2],
lr_gen(lambda i: 0.1, epoch_size), 0.9, 0.01, 1024)
Model(net, loss, opt)
_network = nn.WithLossCell(net, loss)
TrainOneStepWithLarsCell(_network, opt)
data = Tensor(input_np)
label = Tensor(label_np)
net(data, label)
def TrainWrap(net, loss_fn=None, optimizer=None, weights=None):
"""
TrainWrap
"""
if loss_fn is None:
loss_fn = nn.SoftmaxCrossEntropyWithLogits(reduction='mean', sparse=True)
loss_net = nn.WithLossCell(net, loss_fn)
loss_net.set_train()
if weights is None:
weights = ParameterTuple(net.trainable_params())
if optimizer is None:
optimizer = nn.Adam(weights, learning_rate=0.003, beta1=0.9, beta2=0.999, eps=1e-5, use_locking=False,
use_nesterov=False, weight_decay=4e-5, loss_scale=1.0)
train_net = nn.TrainOneStepCell(loss_net, optimizer)
return train_net
def me_train_tensor(net, input_np, label_np, epoch_size=2):
context.set_context(mode=context.GRAPH_MODE)
loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True)
opt = ApplyMomentum(Tensor(np.array([0.1])), Tensor(np.array([0.9])),
filter(lambda x: x.requires_grad, net.get_parameters()))
Model(net, loss, opt)
_network = wrap.WithLossCell(net, loss)
_train_net = MsWrapper(wrap.TrainOneStepCell(_network, opt))
_train_net.set_train()
with SummaryRecord(SUMMARY_DIR, file_suffix="_MS_GRAPH", network=_train_net) as summary_writer:
for epoch in range(0, epoch_size):
print(f"epoch %d" % (epoch))
output = _train_net(Tensor(input_np), Tensor(label_np))
summary_writer.record(i)
print("********output***********")
print(output.asnumpy())
def test_qat_mobile_train():
net = MobileNetV2(num_class=10)
img = Tensor(np.ones((1, 3, 224, 224)).astype(np.float32))
label = Tensor(np.ones((1, 10)).astype(np.float32))
net = qat.convert_quant_network(net,
quant_delay=0,
bn_fold=False,
freeze_bn=10000,
weight_bits=8,
act_bits=8)
loss = nn.SoftmaxCrossEntropyWithLogits(reduction='mean')
optimizer = nn.Momentum(net.trainable_params(),
learning_rate=0.1,
momentum=0.9)
net = nn.WithLossCell(net, loss)
net = nn.TrainOneStepCell(net, optimizer)
net(img, label)
def test_train_cifar(epoch_size=10):
"""train net"""
context.set_auto_parallel_context(parallel_mode=ParallelMode.AUTO_PARALLEL,
gradients_mean=True)
context.set_auto_parallel_context(pipeline_stages=2, full_batch=True)
loss_cb = LossMonitor()
data_path = os.getenv('DATA_PATH')
dataset = create_dataset(data_path)
batch_size = 32
num_classes = 10
net = resnet50(batch_size, num_classes)
loss = SoftmaxCrossEntropyExpand(sparse=True)
net_with_grads = nn.PipelineCell(nn.WithLossCell(net, loss), 4)
opt = Momentum(net.infer_param_pipeline_stage(), 0.01, 0.9)
model = Model(net_with_grads, optimizer=opt)
model.train(epoch_size,
dataset,
callbacks=[loss_cb],
dataset_sink_mode=True)
def test_train_cifar(epoch_size=10): # pylint: disable=missing-docstring
context.set_auto_parallel_context(parallel_mode=ParallelMode.AUTO_PARALLEL,\
gradients_mean=True, grad_accumulation_step=6)
loss_cb = LossMonitor()
data_path = os.getenv('DATA_PATH')
batch_size = 32
dataset = create_dataset(data_path, batch_size=batch_size)
num_classes = 10
net = resnet50(batch_size, num_classes)
loss = SoftmaxCrossEntropyExpand(sparse=True)
opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()),
0.01, 0.9)
net_with_loss = nn.WithLossCell(net, loss)
net_with_loss = VirtualDatasetCell(net_with_loss)
wrap_net = TrainAccuStepsCell(net_with_loss, opt)
model = Model_ACCU(wrap_net)
model.train(epoch_size,
dataset,
callbacks=[loss_cb],
dataset_sink_mode=True)
def _add_loss_network(network, loss_fn, cast_model_type):
"""Add loss network."""
class WithLossCell(nn.Cell):
"Wrap loss for amp. Cast network output back to float32"
def __init__(self, backbone, loss_fn):
super(WithLossCell, self).__init__(auto_prefix=False)
self._backbone = backbone
self._loss_fn = loss_fn
def construct(self, data, label):
out = self._backbone(data)
label = F.mixed_precision_cast(mstype.float32, label)
return self._loss_fn(F.mixed_precision_cast(mstype.float32, out),
label)
if cast_model_type == mstype.float16:
network = WithLossCell(network, loss_fn)
else:
network = nn.WithLossCell(network, loss_fn)
return network
def _build_train_network(self):
"""Build train network"""
network = self._network
if self._optimizer:
if self._loss_scale_manager_set:
network = amp.build_train_network(
network,
self._optimizer,
self._loss_fn,
level=self._amp_level,
loss_scale_manager=self._loss_scale_manager,
keep_batchnorm_fp32=self._keep_bn_fp32)
else:
network = amp.build_train_network(
network,
self._optimizer,
self._loss_fn,
level=self._amp_level,
keep_batchnorm_fp32=self._keep_bn_fp32)
elif self._loss_fn:
network = nn.WithLossCell(network, self._loss_fn)
# If need to check if loss_fn is not None, but optimizer is None
return network
def __init__(self, net, loss, opt):
super(Linear_Train, self).__init__()
self.netwithloss = nn.WithLossCell(net, loss)
self.train_net = nn.TrainOneStepCell(self.netwithloss, opt)
self.train_net.set_train()
resize_op = CV.Resize((resize_height, resize_width)) # Bilinear mode
rescale_op = CV.Rescale(rescale, shift)
hwc2chw_op = CV.HWC2CHW()
# apply map operations on images
mnist_ds = mnist_ds.map(input_columns="image",
operations=resize_op,
num_parallel_workers=num_parallel_workers)
mnist_ds = mnist_ds.map(input_columns="image",
operations=rescale_op,
num_parallel_workers=num_parallel_workers)
mnist_ds = mnist_ds.map(input_columns="image",
operations=hwc2chw_op,
num_parallel_workers=num_parallel_workers)
# apply DatasetOps
mnist_ds = mnist_ds.batch(batch_size)
mnist_ds = mnist_ds.repeat(repeat_size)
return mnist_ds
if __name__ == "__main__":
vae_gan = VaeGan()
net_loss = VaeGanLoss()
optimizer = nn.Adam(params=vae_gan.trainable_params(), learning_rate=0.001)
ds_train = create_dataset(image_path, 128, 1)
net_with_loss = nn.WithLossCell(vae_gan, net_loss)
vi = SVI(net_with_loss=net_with_loss, optimizer=optimizer)
vae_gan = vi.run(train_dataset=ds_train, epochs=10)
def construct(self, x):
x = self.fc1(x)
x = self.sig(x)
x = self.fc2(x)
return x
m = Net(HIDDEN_SIZE)
# create your optimizer
optim = nn.Momentum(m.trainable_params(), learning_rate=0.15, momentum=0.9)
loss_fn = nn.MSELoss()
loss_net = nn.WithLossCell(m, loss_fn)
train_net = nn.TrainOneStepCell(loss_net, optim)
train_net.set_train(True)
for e in range(ITERATIONS):
mloss = 0.0
for mi in range(4):
x1 = mi % 2
x2 = (mi // 2) % 2
data = Tensor([[1. if x1 else 0., 1. if x2 else 0.]],
mindspore.float32)
label = Tensor([[1. if x1 != x2 else 0.]], mindspore.float32)
#import pdb;pdb.set_trace()
loss = train_net(data, label)
print(f"data: {data}, label: {label}, pred: {m(data)}, loss: %0.9f" %
loss.asnumpy())
def train():
"""
对训练函数进行定义
"""
# 可设置训练结点个数，后续可把训练参数加入
parser = argparse.ArgumentParser(description='Graphsage')
parser.add_argument('--data_dir', type=str, default='../data_mr/cora', help='Dataset directory')
parser.add_argument('--train_nodes_num', type=int, default=1208, help='Nodes numbers for training')
parser.add_argument('--eval_nodes_num', type=int, default=500, help='Nodes numbers for evaluation')
parser.add_argument('--test_nodes_num', type=int, default=1000, help='Nodes numbers for test')
args = parser.parse_args()
# 创建文件，保存最优训练模型
if not os.path.exists("ckpts_graphsage"):
os.mkdir("ckpts_graphsage")
# 对模式、环境进行定义
context.set_context(mode=context.PYNATIVE_MODE,
device_target="CPU",
save_graphs=False)
# 读取训练、验证、测试数据
features, labels, train_mask, test_mask, eval_mask = load_and_process(args.data_dir,
args.train_nodes_num,
args.eval_nodes_num,
args.test_nodes_num)
rand_incides = np.random.permutation(features.shape[0])
test_nodes = rand_incides[args.train_nodes_num+args.eval_nodes_num:]
val_nodes = rand_incides[args.train_nodes_num:args.train_nodes_num+args.eval_nodes_num]
train_nodes = rand_incides[:args.train_nodes_num]
feature_size = features.shape[2]
num_nodes = features.shape[0]
num_class = labels.max() + 1
print("feature size: ", feature_size)
print("nodes number: ", num_nodes)
print("node classes: ", num_class)
# 定义训练参数、损失函数、优化器、训练过程
early_stopping = 15
eval_acc_max = 0.8
net_original = Graphsage(input_dim=1433, hidden_dim=128, output_dim=7, hops=[10, 10])
loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
opt_Adam = nn.Adam(net_original.trainable_params())
net_with_loss = nn.WithLossCell(net_original, loss_fn=loss)
net_train_step = nn.TrainOneStepCell(net_with_loss, opt_Adam)
for epoch in range(10):
net_train_step.set_train(mode=True)
for batch in range(20):
# 取每一个batch的训练数据
batch_src_index = np.random.choice(train_nodes, size=(16,))
features_sampled = []
for node in batch_src_index:
features_sampled.append((features[node]))
batch_train_mask = train_mask[batch_src_index]
label_source = labels[batch_src_index]
train_step_loss = net_train_step(Tensor(features_sampled, mindspore.float32),
Tensor(label_source[:, 0], mindspore.int32))
step_loss = P.ReduceSum()(train_step_loss).asnumpy()
# 取每一个batch的验证数据
batch_eval_index = val_nodes
eval_fea_sampled = []
for node in batch_eval_index:
eval_fea_sampled.append((features[node]))
batch_eval_mask = eval_mask[batch_eval_index]
eval_label_source = labels[batch_eval_index]
eval_lable = Tensor(eval_label_source[:, 0], mindspore.int32)
eval_soln = net_original(Tensor(eval_fea_sampled, mindspore.float32))
eval_logits = P.Argmax()(eval_soln)
eval_acc = P.ReduceMean()(P.Cast()((P.Equal()(eval_lable, eval_logits)), mindspore.float32))
print("Epoch:", epoch + 1, " Batch: ", batch + 1, "'s train loss =", step_loss,
" val accuracy =", eval_acc)
# 保存最优模型
if eval_acc.asnumpy() > eval_acc_max:
eval_acc_max = eval_acc
print("a more accurate model!")
if os.path.exists("ckpts_graphsage/graphsage.ckpt"):
os.remove("ckpts_graphsage/graphsage.ckpt")
save_checkpoint(net_train_step, "ckpts_graphsage/graphsage.ckpt")
# 取测试数据
batch_test_index = test_nodes
test_fea_sampled = []
for node in batch_test_index:
test_fea_sampled.append((features[node]))
batch_test_mask = eval_mask[batch_test_index]
test_label_source = labels[batch_test_index]
test_lable = Tensor(test_label_source[:, 0], mindspore.int32)
# 读取最优模型，进行测试集上的预测
test_net = Graphsage(input_dim=1433, hidden_dim=128, output_dim=7, hops=[10, 10])
test_net.set_train(mode=False)
load_checkpoint("ckpts_graphsage/graphsage.ckpt", net=test_net)
loss_test = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
test_soln = test_net(Tensor(test_fea_sampled, mindspore.float32))
test_logits = P.Argmax()(test_soln)
print("test accuracy:", P.ReduceMean()(P.Cast()((P.Equal()(test_lable, test_logits)), mindspore.float32)))
test_with_loss = nn.WithLossCell(test_net, loss_fn=loss_test)
test_loss = test_with_loss(Tensor(test_fea_sampled, mindspore.float32),
Tensor(test_label_source[:, 0], mindspore.int32))
print("test loss:", P.ReduceSum()(test_loss).asnumpy())
def __init__(self, network, loss_fn, optimizer):
self.optimizer = optimizer
self.step = nn.TrainOneStepCell(nn.WithLossCell(network, loss_fn),
self.optimizer)
def _train(self, train_data, test_data):
"""
Applying augmented Lagrangian to solve the continuous constrained problem.
Parameters
----------
train_data: NormalizationData object
train samples
test_data: NormalizationData object
test samples for validation
"""
# Initialize stuff for learning loop
aug_lagrangians_val = []
hs = []
not_nlls = [] # Augmented Lagrangian minus (pseudo) NLL
trainable_para_list = self.model.get_trainable_params()
if self.optimizer == "sgd":
optimizer = nn.optim.SGD(trainable_para_list,
learning_rate=self.lr)
elif self.optimizer == "rmsprop":
optimizer = nn.optim.RMSProp(trainable_para_list,
learning_rate=self.lr)
else:
raise ValueError("optimizer should be in {'sgd', 'rmsprop'}")
# Package training information
net_loss = GranLoss()
net = nn.WithLossCell(self.model, net_loss)
net = nn.TrainOneStepCell(net, optimizer)
# Learning loop:
for iter_num in tqdm(range(self.iterations),
desc='Training Iterations'):
x, _ = train_data.sample(self.batch_size)
ds_data = self._create_dataset(x.asnumpy(),
batch_size=self.batch_size)
w_adj = self.model.get_w_adj()
expm_input = expm(w_adj.asnumpy())
h = np.trace(expm_input) - self.input_dim
# model train
self.model.set_train(True)
net(*list(ds_data)[0])
self.model.set_train(False)
# clamp edges, thresholding
if self.edge_clamp_range != 0:
to_keep = (w_adj > self.edge_clamp_range) * 1
self.model.adjacency *= to_keep
# logging
not_nlls.append(0.5 * self.model.mu * h**2 + self.model.lamb * h)
# compute loss on whole validation set
if iter_num % self.stop_crit_win == 0:
x, _ = test_data.sample(test_data.n_samples)
loss_val = -ops.reduce_mean(
self.model.compute_log_likelihood(x))
aug_lagrangians_val.append(
[iter_num,
loss_val.asnumpy().item() + not_nlls[-1]])
# compute delta for lambda
if iter_num >= 2 * self.stop_crit_win \
and iter_num % (2 * self.stop_crit_win) == 0:
t0 = aug_lagrangians_val[-3][1]
t_half = aug_lagrangians_val[-2][1]
t1 = aug_lagrangians_val[-1][1]
# if the validation loss went up and down,
# do not update lagrangian and penalty coefficients.
if not (min(t0, t1) < t_half < max(t0, t1)):
delta_lambda = -np.inf
else:
delta_lambda = (t1 - t0) / self.stop_crit_win
else:
delta_lambda = -np.inf # do not update lambda nor mu
# Does the augmented lagrangian converged?
# if h value less than equal self.h_threshold value,
# means augmented lagrangian has converged, stop model training
if h > self.h_threshold:
# if we have found a stationary point of the augmented loss
if abs(delta_lambda) < self.omega_lambda or delta_lambda > 0:
self.model.lamb += self.model.mu * h
# Did the constraint improve sufficiently?
hs.append(h)
if len(hs) >= 2:
if hs[-1] > hs[-2] * self.omega_mu:
self.model.mu *= 10
# little hack to make sure the moving average is going down.
gap_in_not_nll = 0.5 * self.model.mu * h ** 2 +\
self.model.lamb * h - not_nlls[-1]
aug_lagrangians_val[-1][1] += gap_in_not_nll
trainable_para_list = self.model.get_trainable_params()
if self.optimizer == "rmsprop":
optimizer = nn.optim.RMSProp(trainable_para_list,
learning_rate=self.lr)
else:
optimizer = nn.optim.SGD(trainable_para_list,
learning_rate=self.lr)
net_loss = GranLoss()
net = nn.WithLossCell(self.model, net_loss)
net = nn.TrainOneStepCell(net, optimizer)
else:
# Final clamping of all edges == 0
to_keep = (w_adj > 0).astype(mstype.float32)
self.model.adjacency *= to_keep
return self.model
# os.environ['DEVICE_ID'] = '0'
import numpy as np
import mindspore as ms
from mindspore import nn
from mindspore import context
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
x = np.arange(-5, 5, 0.3)[:32].reshape((32, 1))
y = -5 * x + 0.1 * np.random.normal(loc=0.0, scale=20.0, size=x.shape)
net = nn.Dense(1, 1)
loss_fn = nn.loss.MSELoss()
opt = nn.optim.SGD(net.trainable_params(), learning_rate=0.01)
with_loss = nn.WithLossCell(net, loss_fn)
train_step = nn.TrainOneStepCell(with_loss, opt).set_train()
for epoch in range(20):
loss = train_step(ms.Tensor(x, ms.float32), ms.Tensor(y, ms.float32))
print('epoch: {0}, loss is {1}'.format(epoch, loss))
wb = [x.data.asnumpy() for x in net.trainable_params()]
w, b = np.squeeze(wb[0]), np.squeeze(wb[1])
print('The true linear function is y = -5 * x + 0.1')
# works in MindSpore0.3.0 or later.
print('The trained linear model is y = {0} * x + {1}'.format(w, b))
for i in range(-10, 11, 5):
print('x = {0}, predicted y = {1}'.format(
i, net(ms.Tensor([[i]], ms.float32))))
