def _build_train_forward_backward_network(self): """Build forward and backward network""" network = self._network network = nn.WithLossCell(network, self._loss_fn) loss_scale = 1.0 network = TrainForwardBackward(network, self._optimizer, self._grad_sum, loss_scale).set_train() return network
def test_svi_vae(): # define the encoder and decoder encoder = Encoder() decoder = Decoder() # define the vae model vae = VAE(encoder, decoder, hidden_size=400, latent_size=20) # define the loss function net_loss = ELBO(latent_prior='Normal', output_prior='Normal') # define the optimizer optimizer = nn.Adam(params=vae.trainable_params(), learning_rate=0.001) # define the training dataset ds_train = create_dataset(image_path, 128, 1) net_with_loss = nn.WithLossCell(vae, net_loss) # define the variational inference vi = SVI(net_with_loss=net_with_loss, optimizer=optimizer) # run the vi to return the trained network. vae = vi.run(train_dataset=ds_train, epochs=5) # get the trained loss trained_loss = vi.get_train_loss() # test function: generate_sample generated_sample = vae.generate_sample(64, IMAGE_SHAPE) # test function: reconstruct_sample for sample in ds_train.create_dict_iterator(): sample_x = Tensor(sample['image'], dtype=mstype.float32) reconstructed_sample = vae.reconstruct_sample(sample_x) print('The loss of the trained network is ', trained_loss) print('The hape of the generated sample is ', generated_sample.shape) print('The shape of the reconstructed sample is ', reconstructed_sample.shape)
def _build_train_network(self): """Build train network""" network = self._network if self._optimizer: if self._loss_scale_manager_set: network = amp.build_train_network( network, self._optimizer, self._loss_fn, level=self._amp_level, loss_scale_manager=self._loss_scale_manager, keep_batchnorm_fp32=self._keep_bn_fp32) else: network = amp.build_train_network( network, self._optimizer, self._loss_fn, level=self._amp_level, keep_batchnorm_fp32=self._keep_bn_fp32) elif self._loss_fn: network = nn.WithLossCell(network, self._loss_fn) # If need to check if loss_fn is not None, but optimizer is None if self._parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): network.set_auto_parallel() return network
def test_vae_gan(): vae_gan = VaeGan() net_loss = VaeGanLoss() optimizer = nn.Adam(params=vae_gan.trainable_params(), learning_rate=0.001) ds_train = create_dataset(image_path, 128, 1) net_with_loss = nn.WithLossCell(vae_gan, net_loss) vi = SVI(net_with_loss=net_with_loss, optimizer=optimizer) vae_gan = vi.run(train_dataset=ds_train, epochs=5)
def build_fn(batchnorm, dropout, l2): # initilize the net, Loss, optimizer net = CosineNet(batchnorm=batchnorm, dropout=dropout) loss = nn.loss.MSELoss() opt = nn.optim.Adam(net.trainable_params(), learning_rate=LEARNING_RATE, weight_decay=WEIGHT_DECAY if l2 else 0.0) # Build a net with loss and optimizer. with_loss = nn.WithLossCell(net, loss) train_step = nn.TrainOneStepCell(with_loss, opt).set_train() return train_step, with_loss, net
def me_train_tensor(net, input_np, label_np, epoch_size=2): loss = SoftmaxCrossEntropyWithLogits(sparse=True) opt = nn.Momentum(Tensor(np.array([0.1])), Tensor(np.array([0.9])), filter(lambda x: x.requires_grad, net.get_parameters())) context.set_context(mode=context.GRAPH_MODE) Model(net, loss, opt) _network = nn.WithLossCell(net, loss) _train_net = MsWrapper(nn.TrainOneStepCell(_network, opt)) _train_net.set_train() for epoch in range(0, epoch_size): print(f"epoch %d" % (epoch)) output = _train_net(Tensor(input_np), Tensor(label_np)) print(output.asnumpy())
def me_train_tensor(net, input_np, label_np, epoch_size=2): """me_train_tensor""" loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr_gen(lambda i: 0.1, epoch_size), 0.9, 0.01, 1024) Model(net, loss, opt) _network = nn.WithLossCell(net, loss) _train_net = nn.TrainOneStepCell(_network, opt) _train_net.set_train() label_np = np.argmax(label_np, axis=-1).astype(np.int32) for epoch in range(0, epoch_size): print(f"epoch %d" % (epoch)) _train_net(Tensor(input_np), Tensor(label_np))
def me_train_tensor(net, input_np, label_np, epoch_size=2): """me_train_tensor""" loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) # reorder the net parameters , leave the parameters that need to be passed into lars to the end part opt = Momentum( get_net_trainable_reordered_params(net)[2], lr_gen(lambda i: 0.1, epoch_size), 0.9, 0.01, 1024) Model(net, loss, opt) _network = nn.WithLossCell(net, loss) TrainOneStepWithLarsCell(_network, opt) data = Tensor(input_np) label = Tensor(label_np) net(data, label)
def TrainWrap(net, loss_fn=None, optimizer=None, weights=None): """ TrainWrap """ if loss_fn is None: loss_fn = nn.SoftmaxCrossEntropyWithLogits(reduction='mean', sparse=True) loss_net = nn.WithLossCell(net, loss_fn) loss_net.set_train() if weights is None: weights = ParameterTuple(net.trainable_params()) if optimizer is None: optimizer = nn.Adam(weights, learning_rate=0.003, beta1=0.9, beta2=0.999, eps=1e-5, use_locking=False, use_nesterov=False, weight_decay=4e-5, loss_scale=1.0) train_net = nn.TrainOneStepCell(loss_net, optimizer) return train_net
def me_train_tensor(net, input_np, label_np, epoch_size=2): context.set_context(mode=context.GRAPH_MODE) loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) opt = ApplyMomentum(Tensor(np.array([0.1])), Tensor(np.array([0.9])), filter(lambda x: x.requires_grad, net.get_parameters())) Model(net, loss, opt) _network = wrap.WithLossCell(net, loss) _train_net = MsWrapper(wrap.TrainOneStepCell(_network, opt)) _train_net.set_train() with SummaryRecord(SUMMARY_DIR, file_suffix="_MS_GRAPH", network=_train_net) as summary_writer: for epoch in range(0, epoch_size): print(f"epoch %d" % (epoch)) output = _train_net(Tensor(input_np), Tensor(label_np)) summary_writer.record(i) print("********output***********") print(output.asnumpy())
def test_qat_mobile_train(): net = MobileNetV2(num_class=10) img = Tensor(np.ones((1, 3, 224, 224)).astype(np.float32)) label = Tensor(np.ones((1, 10)).astype(np.float32)) net = qat.convert_quant_network(net, quant_delay=0, bn_fold=False, freeze_bn=10000, weight_bits=8, act_bits=8) loss = nn.SoftmaxCrossEntropyWithLogits(reduction='mean') optimizer = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) net = nn.WithLossCell(net, loss) net = nn.TrainOneStepCell(net, optimizer) net(img, label)
def test_train_cifar(epoch_size=10): """train net""" context.set_auto_parallel_context(parallel_mode=ParallelMode.AUTO_PARALLEL, gradients_mean=True) context.set_auto_parallel_context(pipeline_stages=2, full_batch=True) loss_cb = LossMonitor() data_path = os.getenv('DATA_PATH') dataset = create_dataset(data_path) batch_size = 32 num_classes = 10 net = resnet50(batch_size, num_classes) loss = SoftmaxCrossEntropyExpand(sparse=True) net_with_grads = nn.PipelineCell(nn.WithLossCell(net, loss), 4) opt = Momentum(net.infer_param_pipeline_stage(), 0.01, 0.9) model = Model(net_with_grads, optimizer=opt) model.train(epoch_size, dataset, callbacks=[loss_cb], dataset_sink_mode=True)
def test_train_cifar(epoch_size=10): # pylint: disable=missing-docstring context.set_auto_parallel_context(parallel_mode=ParallelMode.AUTO_PARALLEL,\ gradients_mean=True, grad_accumulation_step=6) loss_cb = LossMonitor() data_path = os.getenv('DATA_PATH') batch_size = 32 dataset = create_dataset(data_path, batch_size=batch_size) num_classes = 10 net = resnet50(batch_size, num_classes) loss = SoftmaxCrossEntropyExpand(sparse=True) opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), 0.01, 0.9) net_with_loss = nn.WithLossCell(net, loss) net_with_loss = VirtualDatasetCell(net_with_loss) wrap_net = TrainAccuStepsCell(net_with_loss, opt) model = Model_ACCU(wrap_net) model.train(epoch_size, dataset, callbacks=[loss_cb], dataset_sink_mode=True)
def _add_loss_network(network, loss_fn, cast_model_type): """Add loss network.""" class WithLossCell(nn.Cell): "Wrap loss for amp. Cast network output back to float32" def __init__(self, backbone, loss_fn): super(WithLossCell, self).__init__(auto_prefix=False) self._backbone = backbone self._loss_fn = loss_fn def construct(self, data, label): out = self._backbone(data) label = F.mixed_precision_cast(mstype.float32, label) return self._loss_fn(F.mixed_precision_cast(mstype.float32, out), label) if cast_model_type == mstype.float16: network = WithLossCell(network, loss_fn) else: network = nn.WithLossCell(network, loss_fn) return network
def _build_train_network(self): """Build train network""" network = self._network if self._optimizer: if self._loss_scale_manager_set: network = amp.build_train_network( network, self._optimizer, self._loss_fn, level=self._amp_level, loss_scale_manager=self._loss_scale_manager, keep_batchnorm_fp32=self._keep_bn_fp32) else: network = amp.build_train_network( network, self._optimizer, self._loss_fn, level=self._amp_level, keep_batchnorm_fp32=self._keep_bn_fp32) elif self._loss_fn: network = nn.WithLossCell(network, self._loss_fn) # If need to check if loss_fn is not None, but optimizer is None return network
def __init__(self, net, loss, opt): super(Linear_Train, self).__init__() self.netwithloss = nn.WithLossCell(net, loss) self.train_net = nn.TrainOneStepCell(self.netwithloss, opt) self.train_net.set_train()
resize_op = CV.Resize((resize_height, resize_width)) # Bilinear mode rescale_op = CV.Rescale(rescale, shift) hwc2chw_op = CV.HWC2CHW() # apply map operations on images mnist_ds = mnist_ds.map(input_columns="image", operations=resize_op, num_parallel_workers=num_parallel_workers) mnist_ds = mnist_ds.map(input_columns="image", operations=rescale_op, num_parallel_workers=num_parallel_workers) mnist_ds = mnist_ds.map(input_columns="image", operations=hwc2chw_op, num_parallel_workers=num_parallel_workers) # apply DatasetOps mnist_ds = mnist_ds.batch(batch_size) mnist_ds = mnist_ds.repeat(repeat_size) return mnist_ds if __name__ == "__main__": vae_gan = VaeGan() net_loss = VaeGanLoss() optimizer = nn.Adam(params=vae_gan.trainable_params(), learning_rate=0.001) ds_train = create_dataset(image_path, 128, 1) net_with_loss = nn.WithLossCell(vae_gan, net_loss) vi = SVI(net_with_loss=net_with_loss, optimizer=optimizer) vae_gan = vi.run(train_dataset=ds_train, epochs=10)
def construct(self, x): x = self.fc1(x) x = self.sig(x) x = self.fc2(x) return x m = Net(HIDDEN_SIZE) # create your optimizer optim = nn.Momentum(m.trainable_params(), learning_rate=0.15, momentum=0.9) loss_fn = nn.MSELoss() loss_net = nn.WithLossCell(m, loss_fn) train_net = nn.TrainOneStepCell(loss_net, optim) train_net.set_train(True) for e in range(ITERATIONS): mloss = 0.0 for mi in range(4): x1 = mi % 2 x2 = (mi // 2) % 2 data = Tensor([[1. if x1 else 0., 1. if x2 else 0.]], mindspore.float32) label = Tensor([[1. if x1 != x2 else 0.]], mindspore.float32) #import pdb;pdb.set_trace() loss = train_net(data, label) print(f"data: {data}, label: {label}, pred: {m(data)}, loss: %0.9f" % loss.asnumpy())
def train(): """ 对训练函数进行定义 """ # 可设置训练结点个数,后续可把训练参数加入 parser = argparse.ArgumentParser(description='Graphsage') parser.add_argument('--data_dir', type=str, default='../data_mr/cora', help='Dataset directory') parser.add_argument('--train_nodes_num', type=int, default=1208, help='Nodes numbers for training') parser.add_argument('--eval_nodes_num', type=int, default=500, help='Nodes numbers for evaluation') parser.add_argument('--test_nodes_num', type=int, default=1000, help='Nodes numbers for test') args = parser.parse_args() # 创建文件,保存最优训练模型 if not os.path.exists("ckpts_graphsage"): os.mkdir("ckpts_graphsage") # 对模式、环境进行定义 context.set_context(mode=context.PYNATIVE_MODE, device_target="CPU", save_graphs=False) # 读取训练、验证、测试数据 features, labels, train_mask, test_mask, eval_mask = load_and_process(args.data_dir, args.train_nodes_num, args.eval_nodes_num, args.test_nodes_num) rand_incides = np.random.permutation(features.shape[0]) test_nodes = rand_incides[args.train_nodes_num+args.eval_nodes_num:] val_nodes = rand_incides[args.train_nodes_num:args.train_nodes_num+args.eval_nodes_num] train_nodes = rand_incides[:args.train_nodes_num] feature_size = features.shape[2] num_nodes = features.shape[0] num_class = labels.max() + 1 print("feature size: ", feature_size) print("nodes number: ", num_nodes) print("node classes: ", num_class) # 定义训练参数、损失函数、优化器、训练过程 early_stopping = 15 eval_acc_max = 0.8 net_original = Graphsage(input_dim=1433, hidden_dim=128, output_dim=7, hops=[10, 10]) loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True) opt_Adam = nn.Adam(net_original.trainable_params()) net_with_loss = nn.WithLossCell(net_original, loss_fn=loss) net_train_step = nn.TrainOneStepCell(net_with_loss, opt_Adam) for epoch in range(10): net_train_step.set_train(mode=True) for batch in range(20): # 取每一个batch的训练数据 batch_src_index = np.random.choice(train_nodes, size=(16,)) features_sampled = [] for node in batch_src_index: features_sampled.append((features[node])) batch_train_mask = train_mask[batch_src_index] label_source = labels[batch_src_index] train_step_loss = net_train_step(Tensor(features_sampled, mindspore.float32), Tensor(label_source[:, 0], mindspore.int32)) step_loss = P.ReduceSum()(train_step_loss).asnumpy() # 取每一个batch的验证数据 batch_eval_index = val_nodes eval_fea_sampled = [] for node in batch_eval_index: eval_fea_sampled.append((features[node])) batch_eval_mask = eval_mask[batch_eval_index] eval_label_source = labels[batch_eval_index] eval_lable = Tensor(eval_label_source[:, 0], mindspore.int32) eval_soln = net_original(Tensor(eval_fea_sampled, mindspore.float32)) eval_logits = P.Argmax()(eval_soln) eval_acc = P.ReduceMean()(P.Cast()((P.Equal()(eval_lable, eval_logits)), mindspore.float32)) print("Epoch:", epoch + 1, " Batch: ", batch + 1, "'s train loss =", step_loss, " val accuracy =", eval_acc) # 保存最优模型 if eval_acc.asnumpy() > eval_acc_max: eval_acc_max = eval_acc print("a more accurate model!") if os.path.exists("ckpts_graphsage/graphsage.ckpt"): os.remove("ckpts_graphsage/graphsage.ckpt") save_checkpoint(net_train_step, "ckpts_graphsage/graphsage.ckpt") # 取测试数据 batch_test_index = test_nodes test_fea_sampled = [] for node in batch_test_index: test_fea_sampled.append((features[node])) batch_test_mask = eval_mask[batch_test_index] test_label_source = labels[batch_test_index] test_lable = Tensor(test_label_source[:, 0], mindspore.int32) # 读取最优模型,进行测试集上的预测 test_net = Graphsage(input_dim=1433, hidden_dim=128, output_dim=7, hops=[10, 10]) test_net.set_train(mode=False) load_checkpoint("ckpts_graphsage/graphsage.ckpt", net=test_net) loss_test = nn.SoftmaxCrossEntropyWithLogits(sparse=True) test_soln = test_net(Tensor(test_fea_sampled, mindspore.float32)) test_logits = P.Argmax()(test_soln) print("test accuracy:", P.ReduceMean()(P.Cast()((P.Equal()(test_lable, test_logits)), mindspore.float32))) test_with_loss = nn.WithLossCell(test_net, loss_fn=loss_test) test_loss = test_with_loss(Tensor(test_fea_sampled, mindspore.float32), Tensor(test_label_source[:, 0], mindspore.int32)) print("test loss:", P.ReduceSum()(test_loss).asnumpy())
def __init__(self, network, loss_fn, optimizer): self.optimizer = optimizer self.step = nn.TrainOneStepCell(nn.WithLossCell(network, loss_fn), self.optimizer)
def _train(self, train_data, test_data): """ Applying augmented Lagrangian to solve the continuous constrained problem. Parameters ---------- train_data: NormalizationData object train samples test_data: NormalizationData object test samples for validation """ # Initialize stuff for learning loop aug_lagrangians_val = [] hs = [] not_nlls = [] # Augmented Lagrangian minus (pseudo) NLL trainable_para_list = self.model.get_trainable_params() if self.optimizer == "sgd": optimizer = nn.optim.SGD(trainable_para_list, learning_rate=self.lr) elif self.optimizer == "rmsprop": optimizer = nn.optim.RMSProp(trainable_para_list, learning_rate=self.lr) else: raise ValueError("optimizer should be in {'sgd', 'rmsprop'}") # Package training information net_loss = GranLoss() net = nn.WithLossCell(self.model, net_loss) net = nn.TrainOneStepCell(net, optimizer) # Learning loop: for iter_num in tqdm(range(self.iterations), desc='Training Iterations'): x, _ = train_data.sample(self.batch_size) ds_data = self._create_dataset(x.asnumpy(), batch_size=self.batch_size) w_adj = self.model.get_w_adj() expm_input = expm(w_adj.asnumpy()) h = np.trace(expm_input) - self.input_dim # model train self.model.set_train(True) net(*list(ds_data)[0]) self.model.set_train(False) # clamp edges, thresholding if self.edge_clamp_range != 0: to_keep = (w_adj > self.edge_clamp_range) * 1 self.model.adjacency *= to_keep # logging not_nlls.append(0.5 * self.model.mu * h**2 + self.model.lamb * h) # compute loss on whole validation set if iter_num % self.stop_crit_win == 0: x, _ = test_data.sample(test_data.n_samples) loss_val = -ops.reduce_mean( self.model.compute_log_likelihood(x)) aug_lagrangians_val.append( [iter_num, loss_val.asnumpy().item() + not_nlls[-1]]) # compute delta for lambda if iter_num >= 2 * self.stop_crit_win \ and iter_num % (2 * self.stop_crit_win) == 0: t0 = aug_lagrangians_val[-3][1] t_half = aug_lagrangians_val[-2][1] t1 = aug_lagrangians_val[-1][1] # if the validation loss went up and down, # do not update lagrangian and penalty coefficients. if not (min(t0, t1) < t_half < max(t0, t1)): delta_lambda = -np.inf else: delta_lambda = (t1 - t0) / self.stop_crit_win else: delta_lambda = -np.inf # do not update lambda nor mu # Does the augmented lagrangian converged? # if h value less than equal self.h_threshold value, # means augmented lagrangian has converged, stop model training if h > self.h_threshold: # if we have found a stationary point of the augmented loss if abs(delta_lambda) < self.omega_lambda or delta_lambda > 0: self.model.lamb += self.model.mu * h # Did the constraint improve sufficiently? hs.append(h) if len(hs) >= 2: if hs[-1] > hs[-2] * self.omega_mu: self.model.mu *= 10 # little hack to make sure the moving average is going down. gap_in_not_nll = 0.5 * self.model.mu * h ** 2 +\ self.model.lamb * h - not_nlls[-1] aug_lagrangians_val[-1][1] += gap_in_not_nll trainable_para_list = self.model.get_trainable_params() if self.optimizer == "rmsprop": optimizer = nn.optim.RMSProp(trainable_para_list, learning_rate=self.lr) else: optimizer = nn.optim.SGD(trainable_para_list, learning_rate=self.lr) net_loss = GranLoss() net = nn.WithLossCell(self.model, net_loss) net = nn.TrainOneStepCell(net, optimizer) else: # Final clamping of all edges == 0 to_keep = (w_adj > 0).astype(mstype.float32) self.model.adjacency *= to_keep return self.model
# os.environ['DEVICE_ID'] = '0' import numpy as np import mindspore as ms from mindspore import nn from mindspore import context context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") x = np.arange(-5, 5, 0.3)[:32].reshape((32, 1)) y = -5 * x + 0.1 * np.random.normal(loc=0.0, scale=20.0, size=x.shape) net = nn.Dense(1, 1) loss_fn = nn.loss.MSELoss() opt = nn.optim.SGD(net.trainable_params(), learning_rate=0.01) with_loss = nn.WithLossCell(net, loss_fn) train_step = nn.TrainOneStepCell(with_loss, opt).set_train() for epoch in range(20): loss = train_step(ms.Tensor(x, ms.float32), ms.Tensor(y, ms.float32)) print('epoch: {0}, loss is {1}'.format(epoch, loss)) wb = [x.data.asnumpy() for x in net.trainable_params()] w, b = np.squeeze(wb[0]), np.squeeze(wb[1]) print('The true linear function is y = -5 * x + 0.1') # works in MindSpore0.3.0 or later. print('The trained linear model is y = {0} * x + {1}'.format(w, b)) for i in range(-10, 11, 5): print('x = {0}, predicted y = {1}'.format( i, net(ms.Tensor([[i]], ms.float32))))