def lstm_forward(): hs, _, _ = rnn(inputs, (h0, c0)) loss = autograd.softmax_cross_entropy(hs[0], target[0]) for i in range(1, len(hs)): l = autograd.softmax_cross_entropy(hs[i], target[i]) loss = autograd.add(loss, l) return loss
def lstm_forward(): hs, _, _ = rnn(inputs, (h0, c0)) loss = autograd.softmax_cross_entropy(hs[0], target[0]) for i in range(1, len(hs)): l = autograd.softmax_cross_entropy(hs[i], target[i]) loss = autograd.add(loss, l) return loss
def valinna_rnn_forward(): hs, _ = rnn(inputs, h0) loss = autograd.softmax_cross_entropy(hs[0], target[0]) for i in range(1, len(hs)): l = autograd.softmax_cross_entropy(hs[i], target[i]) loss = autograd.add(loss, l) #grads = autograd.gradients(loss) return loss
def valinna_rnn_forward(): hs, _ = rnn(inputs, h0) loss = autograd.softmax_cross_entropy(hs[0], target[0]) for i in range(1, len(hs)): l = autograd.softmax_cross_entropy(hs[i], target[i]) loss = autograd.add(loss, l) #grads = autograd.gradients(loss) return loss
def test_lstm_model(self, dev=gpu_dev): hidden_size = 3 seq_length = 2 batch_size = 4 feature_size = 3 bidirectional = False directions = 2 if bidirectional else 1 num_layers = 2 out_size = hidden_size return_sequences = False batch_first = True rnn_mode = "lstm" # manual test case x_data = np.array( [[[0, 0, 1], [0, 1, 0]], [[0, 1, 0], [1, 0, 0]], [[0, 0, 1], [0, 1, 0]], [[1, 0, 0], [0, 0, 1]]], dtype=np.float32).reshape(batch_size, seq_length, hidden_size) # bs, seq, fea if return_sequences: y_data = np.array([[[0, 1, 0], [1, 0, 0]], [[1, 0, 0], [0, 0, 1]], [[0, 1, 0], [1, 0, 0]], [[0, 0, 1], [0, 1, 0]]], dtype=np.float32).reshape( batch_size, seq_length, hidden_size) # bs, hidden y_data.reshape(batch_size, -1) else: y_data = np.array([[1, 0, 0], [0, 0, 1], [1, 0, 0], [0, 1, 0]], dtype=np.float32).reshape( batch_size, hidden_size) # bs, hidden x = tensor.Tensor(device=dev, data=x_data) y_t = tensor.Tensor(device=dev, data=y_data) m = LSTMModel(hidden_size, seq_length, batch_size, bidirectional, num_layers, return_sequences, rnn_mode, batch_first) m.compile([x], is_train=True, use_graph=False, sequential=False) m.train() for i in range(1000): y = m.forward(x) assert y.shape == y_t.shape loss = autograd.softmax_cross_entropy(y, y_t) if i % 100 == 0: print("loss", loss) m.optimizer(loss) m.eval() y = m.forward(x) loss = autograd.softmax_cross_entropy(y, y_t) print("eval loss", loss)
def test_vanillaRNN_gpu_tiny_ops_shape_check(self): # gradients shape check. inputs, target, h0 = prepare_inputs_targets_for_rnn_test() rnn = autograd.RNN(3, 2) hs, _ = rnn(inputs, h0) loss = autograd.softmax_cross_entropy(hs[0], target[0]) for i in range(1, len(hs)): l = autograd.softmax_cross_entropy(hs[i], target[i]) loss = autograd.add(loss, l) # d=autograd.infer_dependency(loss.creator) # print(d) for t, dt in autograd.backward(loss): self.check_shape(t.shape, dt.shape)
def test_vanillaRNN_gpu_tiny_ops_shape_check(self): # gradients shape check. inputs, target, h0 = prepare_inputs_targets_for_rnn_test() rnn = autograd.RNN(3, 2) hs, _ = rnn(inputs, h0) loss = autograd.softmax_cross_entropy(hs[0], target[0]) for i in range(1, len(hs)): l = autograd.softmax_cross_entropy(hs[i], target[i]) loss = autograd.add(loss, l) # d=autograd.infer_dependency(loss.creator) # print(d) for t, dt in autograd.backward(loss): self.check_shape(t.shape, dt.shape)
def transfer_learning(sg_ir, x, y, epochs=1, batch_size=64, dev=device.get_default_device()): batch_number = x.shape[0] // batch_size trans_model = Trans(sg_ir, -1) for i in range(epochs): for b in range(batch_number): l_idx = b * batch_size r_idx = (b + 1) * batch_size x_batch = tensor.Tensor(device=dev, data=x[l_idx:r_idx]) target_batch = tensor.Tensor(device=dev, data=y[l_idx:r_idx]) output_batch = trans_model.forward(x_batch) loss = autograd.softmax_cross_entropy(output_batch, target_batch) accuracy_rate = accuracy(tensor.to_numpy(output_batch), tensor.to_numpy(target_batch)) sgd = opt.SGD(lr=0.07) for p, gp in autograd.backward(loss): sgd.update(p, gp) sgd.step() if b % 1e2 == 0: print("acc %6.2f loss, %6.2f" % (accuracy_rate, tensor.to_numpy(loss)[0])) print("transfer-learning completed") return trans_model
def onnx_to_singa(niter, use_cpu=False): if use_cpu: print("Using CPU") dev = device.get_default_device() else: print("Using GPU") dev = device.create_cuda_gpu() model = sonnx.load("mlp.onnx") backend = sonnx.prepare(model, device=dev) sgd = opt.SGD(0.1) inputs = Tensor( data=data, device=dev, requires_grad=False, stores_grad=False, name="input", ) target = Tensor( data=label, device=dev, requires_grad=False, stores_grad=False, name="target", ) for i in range(100): y = backend.run([inputs])[0] loss = autograd.softmax_cross_entropy(y, target) for p, gp in autograd.backward(loss): sgd.update(p, gp) loss_rate = tensor.to_numpy(loss)[0] accuracy_rate = accuracy(tensor.to_numpy(y), label) print("Iter {}, accurate={}, loss={}".format(i, accuracy_rate, loss_rate))
def train(model, x, y, epochs=1, batch_size=64, dev=device.get_default_device()): batch_number = x.shape[0] // batch_size for i in range(epochs): for b in range(batch_number): l_idx = b * batch_size r_idx = (b + 1) * batch_size x_batch = tensor.Tensor(device=dev, data=x[l_idx:r_idx]) target_batch = tensor.Tensor(device=dev, data=y[l_idx:r_idx]) output_batch = model.forward(x_batch) # onnx_model = sonnx.to_onnx([x_batch], [y]) # print('The model is:\n{}'.format(onnx_model)) loss = autograd.softmax_cross_entropy(output_batch, target_batch) accuracy_rate = accuracy(tensor.to_numpy(output_batch), tensor.to_numpy(target_batch)) sgd = opt.SGD(lr=0.001) for p, gp in autograd.backward(loss): sgd.update(p, gp) sgd.step() if b % 1e2 == 0: print("acc %6.2f loss, %6.2f" % (accuracy_rate, tensor.to_numpy(loss)[0])) print("training completed") return x_batch, output_batch
def test_LSTM_gpu_tiny_ops_shape_check(self): # gradients shape check. inputs, target, h0 = prepare_inputs_targets_for_rnn_test() c_0 = np.random.random((2, 1)).astype(np.float32) c0 = tensor.Tensor(device=gpu_dev, data=c_0) rnn = autograd.LSTM(3, 2) hs, _, _ = rnn(inputs, (h0, c0)) loss = autograd.softmax_cross_entropy(hs[0], target[0]) for i in range(1, len(hs)): l = autograd.softmax_cross_entropy(hs[i], target[i]) loss = autograd.add(loss, l) # d=autograd.infer_dependency(loss.creator) # print(d) for t, dt in autograd.backward(loss): self.check_shape(t.shape, dt.shape)
def test_LSTM_gpu_tiny_ops_shape_check(self): # gradients shape check. inputs, target, h0 = prepare_inputs_targets_for_rnn_test() c_0 = np.random.random((2, 1)).astype(np.float32) c0 = tensor.Tensor(device=gpu_dev, data=c_0) rnn = autograd.LSTM(3, 2) hs, _, _ = rnn(inputs, (h0, c0)) loss = autograd.softmax_cross_entropy(hs[0], target[0]) for i in range(1, len(hs)): l = autograd.softmax_cross_entropy(hs[i], target[i]) loss = autograd.add(loss, l) # d=autograd.infer_dependency(loss.creator) # print(d) for t, dt in autograd.backward(loss): self.check_shape(t.shape, dt.shape)
def forward(x, t): y = conv1(x) y = autograd.relu(y) y = conv2(y) y = autograd.relu(y) y = pooling(y) y = autograd.flatten(y) y = linear(y) loss = autograd.softmax_cross_entropy(y, t) return loss, y
def forward(x, t): y = conv1(x) y = autograd.relu(y) y1 = conv21(y) y2 = conv22(y) y = autograd.cat((y1, y2), 1) y = autograd.relu(y) y = autograd.flatten(y) y = linear(y) loss = autograd.softmax_cross_entropy(y, t) return loss, y
def evaluate(model, data, batch_size, seq_length, dev, inputs, labels): model.eval() val_loss = 0.0 for b in range(data.num_test_batch): batch = data.val_dat[b * batch_size:(b + 1) * batch_size] inputs, labels = convert(batch, batch_size, seq_length, data.vocab_size, dev, inputs, labels) model.reset_states(dev) y = model(inputs) loss = autograd.softmax_cross_entropy(y, labels)[0] val_loss += tensor.to_numpy(loss)[0] print(' validation loss is %f' % (val_loss / data.num_test_batch / seq_length))
def forward(x, t): y = conv1(x) y = autograd.relu(y) y = bn1(y) y = pooling1(y) y1 = conv21(y) y2 = conv22(y) y = autograd.cat((y1, y2), 1) y = bn2(y) y = autograd.relu(y) y = bn2(y) y = pooling2(y) y = autograd.flatten(y) y = linear(y) loss = autograd.softmax_cross_entropy(y, t) return loss, y
def singa_to_onnx(niter, use_cpu=False): if use_cpu: print("Using CPU") dev = device.get_default_device() else: print("Using GPU") dev = device.create_cuda_gpu() inputs = Tensor( data=data, device=dev, requires_grad=False, stores_grad=False, name="input", ) target = Tensor( data=label, device=dev, requires_grad=False, stores_grad=False, name="target", ) w0 = Tensor(shape=(2, 3), device=dev, requires_grad=True, stores_grad=True) w0.gaussian(0.0, 0.1) b0 = Tensor(shape=(3,), device=dev, requires_grad=True, stores_grad=True) b0.set_value(0.0) w1 = Tensor(shape=(3, 2), device=dev, requires_grad=True, stores_grad=True) w1.gaussian(0.0, 0.1) b1 = Tensor(shape=(2,), device=dev, requires_grad=True, stores_grad=True) b1.set_value(0.0) sgd = opt.SGD(0.1) # training process for i in range(100): x = autograd.matmul(inputs, w0) x = autograd.add_bias(x, b0) x = autograd.relu(x) x = autograd.matmul(x, w1) x = autograd.add_bias(x, b1) loss = autograd.softmax_cross_entropy(x, target) for p, gp in autograd.backward(loss): sgd.update(p, gp) print("training loss = ", tensor.to_numpy(loss)[0]) sonnx.export([inputs], [x], file_path="mlp.onnx")
def train_one_batch(self, x, y, dist_option, spars): out = self.forward(x) loss = autograd.softmax_cross_entropy(out, y) if dist_option == 'fp32': self.optimizer.backward_and_update(loss) elif dist_option == 'fp16': self.optimizer.backward_and_update_half(loss) elif dist_option == 'partialUpdate': self.optimizer.backward_and_partial_update(loss) elif dist_option == 'sparseTopK': self.optimizer.backward_and_sparse_update(loss, topK=True, spars=spars) elif dist_option == 'sparseThreshold': self.optimizer.backward_and_sparse_update(loss, topK=False, spars=spars) return out, loss
def forward(x, t): y = conv1(x) y = autograd.tanh(y) y1 = conv21(y) y2 = conv22(y) y = autograd.cat((y1, y2), 1) y = autograd.sigmoid(y) y = bn(y) y = autograd.relu(y) y = autograd.mul(y, y) y = pooling1(y) y = autograd.sigmoid(y) y = pooling2(y) print(tensor.to_numpy(y).shape) y = autograd.flatten(y) y = linear(y) print(tensor.to_numpy(y).shape) loss = autograd.softmax_cross_entropy(y, t) return loss, y
def onnx_to_singa(epochs, use_cpu=False, batchsize=32): (x_train, y_train), (x_test, y_test), dev = common(use_cpu) model = sonnx.load("cnn.onnx") backend = sonnx.prepare(model, dev) autograd.training = True sgd = opt.SGD(lr=0.01) niter = x_train.shape[0] // batchsize for epoch in range(epochs): accuracy_rate = 0.0 loss_rate = 0.0 for i in range(niter): inputs = tensor.Tensor( device=dev, data=x_train[i * batchsize : (i + 1) * batchsize], stores_grad=False, name="input", ) targets = tensor.Tensor( device=dev, data=y_train[i * batchsize : (i + 1) * batchsize], requires_grad=False, stores_grad=False, name="target", ) y = backend.run([inputs])[0] loss = autograd.softmax_cross_entropy(y, targets) accuracy_rate += accuracy( tensor.to_numpy(y), y_train[i * batchsize : (i + 1) * batchsize] ) loss_rate += tensor.to_numpy(loss)[0] for p, gp in autograd.backward(loss): sgd.update(p, gp) print("accuracy is {}, loss is {}".format(accuracy_rate / niter, loss_rate / niter))
import time dev.Sync() start = time.time() fd = 0 softmax = 0 update = 0 with trange(niters) as t: for _ in t: dev.Sync() tick = time.time() x = model(tx) dev.Sync() fd += time.time() - tick tick = time.time() loss = autograd.softmax_cross_entropy(x, ty) dev.Sync() softmax += time.time() - tick plist = [] for p, g in autograd.backward(loss): #dev.Sync() # this Sync affects the concurrency and hence omitted tick = time.time() sgd.all_reduce(g) #dev.Sync() # this Sync affects the concurrency and hence omitted update += time.time() - tick plist.append((p, g)) sgd.wait() for p, g in plist: sgd.update(p, g) dev.Sync()
def train_one_batch(self, x, y): out = self.forward(x) loss = autograd.softmax_cross_entropy(out, y) self.optimizer(loss) return out, loss
def train_mnist_cnn(DIST=False, local_rank=None, world_size=None, nccl_id=None, spars=0, topK=False, corr=True): # Define the hypermeters good for the mnist_cnn max_epoch = 10 batch_size = 64 sgd = opt.SGD(lr=0.005, momentum=0.9, weight_decay=1e-5) # Prepare training and valadiation data train_x, train_y, test_x, test_y = load_dataset() IMG_SIZE = 28 num_classes = 10 train_y = to_categorical(train_y, num_classes) test_y = to_categorical(test_y, num_classes) # Normalization train_x = train_x / 255 test_x = test_x / 255 if DIST: # For Distributed GPU Training sgd = opt.DistOpt(sgd, nccl_id=nccl_id, local_rank=local_rank, world_size=world_size) dev = device.create_cuda_gpu_on(sgd.local_rank) # Dataset partition for distributed training train_x, train_y = data_partition(train_x, train_y, sgd.global_rank, sgd.world_size) test_x, test_y = data_partition(test_x, test_y, sgd.global_rank, sgd.world_size) world_size = sgd.world_size else: # For Single GPU dev = device.create_cuda_gpu() world_size = 1 # create model model = CNN() tx = tensor.Tensor((batch_size, 1, IMG_SIZE, IMG_SIZE), dev, tensor.float32) ty = tensor.Tensor((batch_size, num_classes), dev, tensor.int32) num_train_batch = train_x.shape[0] // batch_size num_test_batch = test_x.shape[0] // batch_size idx = np.arange(train_x.shape[0], dtype=np.int32) if DIST: #Sychronize the initial parameters autograd.training = True x = np.random.randn(batch_size, 1, IMG_SIZE, IMG_SIZE).astype(np.float32) y = np.zeros(shape=(batch_size, num_classes), dtype=np.int32) tx.copy_from_numpy(x) ty.copy_from_numpy(y) out = model.forward(tx) loss = autograd.softmax_cross_entropy(out, ty) for p, g in autograd.backward(loss): synchronize(p, sgd) # Training and Evaulation Loop for epoch in range(max_epoch): start_time = time.time() np.random.shuffle(idx) if ((DIST == False) or (sgd.global_rank == 0)): print('Starting Epoch %d:' % (epoch)) # Training Phase autograd.training = True train_correct = np.zeros(shape=[1], dtype=np.float32) test_correct = np.zeros(shape=[1], dtype=np.float32) train_loss = np.zeros(shape=[1], dtype=np.float32) for b in range(num_train_batch): x = train_x[idx[b * batch_size:(b + 1) * batch_size]] x = augmentation(x, batch_size) y = train_y[idx[b * batch_size:(b + 1) * batch_size]] tx.copy_from_numpy(x) ty.copy_from_numpy(y) out = model.forward(tx) loss = autograd.softmax_cross_entropy(out, ty) train_correct += accuracy(tensor.to_numpy(out), y) train_loss += tensor.to_numpy(loss)[0] if DIST: if (spars == 0): sgd.backward_and_update(loss, threshold=50000) else: sgd.backward_and_sparse_update(loss, spars=spars, topK=topK, corr=corr) else: sgd.backward_and_update(loss) if DIST: # Reduce the Evaluation Accuracy and Loss from Multiple Devices reducer = tensor.Tensor((1,), dev, tensor.float32) train_correct = reduce_variable(train_correct, sgd, reducer) train_loss = reduce_variable(train_loss, sgd, reducer) # Output the Training Loss and Accuracy if ((DIST == False) or (sgd.global_rank == 0)): print('Training loss = %f, training accuracy = %f' % (train_loss, train_correct / (num_train_batch * batch_size * world_size)), flush=True) # Evaluation Phase autograd.training = False for b in range(num_test_batch): x = test_x[b * batch_size:(b + 1) * batch_size] y = test_y[b * batch_size:(b + 1) * batch_size] tx.copy_from_numpy(x) ty.copy_from_numpy(y) out_test = model.forward(tx) test_correct += accuracy(tensor.to_numpy(out_test), y) if DIST: # Reduce the Evaulation Accuracy from Multiple Devices test_correct = reduce_variable(test_correct, sgd, reducer) # Output the Evaluation Accuracy if ((DIST == False) or (sgd.global_rank == 0)): print('Evaluation accuracy = %f, Elapsed Time = %fs' % (test_correct / (num_test_batch * batch_size * world_size), time.time() - start_time), flush=True)
def train_mnist_cnn(DIST=False, local_rank=None, world_size=None, nccl_id=None, spars=0, topK=False, corr=True): # Define the hypermeters good for the mnist_cnn max_epoch = 10 batch_size = 128 sgd = opt.SGD(lr=0.005, momentum=0.9, weight_decay=1e-5) # Prepare training and valadiation data train_x, train_y, test_x, test_y = load_dataset() IMG_SIZE = 28 num_classes = 10 train_y = to_categorical(train_y, num_classes) test_y = to_categorical(test_y, num_classes) # Normalization train_x = train_x / 255 test_x = test_x / 255 if DIST: # For Distributed GPU Training ''' sgd = opt.DistOpt(sgd, nccl_id=nccl_id, local_rank=local_rank, world_size=world_size) ''' dev = device.get_default_device() # create kvstore kv_type = 'dist_sync' #set synchronization mode lr = 0.005 kv = singa_kvstore.create_kvstore(kv_type, 'SingaSGD', lr=0.005, momentum=0.9, weight_decay=1e-5) global_rank = kv.rank world_size = kv.num_workers # Dataset partition for distributed training train_x, train_y = data_partition(train_x, train_y, global_rank, world_size) test_x, test_y = data_partition(test_x, test_y, global_rank, world_size) # create model model = CNN() ''' num_channels = train_x.shape[1] image_size = train_x.shape[2] data_size = np.prod(train_x.shape[1:train_x.ndim]).item() num_classes = (np.max(train_y) + 1).item() model = resnet.resnet18(num_channels=1, num_classes=num_classes) ''' tx = tensor.Tensor((batch_size, 1, IMG_SIZE, IMG_SIZE), dev, tensor.float32) ty = tensor.Tensor((batch_size, num_classes), dev, tensor.int32) num_train_batch = train_x.shape[0] // batch_size num_test_batch = test_x.shape[0] // batch_size idx = np.arange(train_x.shape[0], dtype=np.int32) ''' if DIST: #Initial a batch to help obtain model parameters autograd.training = True x = np.random.randn(batch_size, 1, IMG_SIZE, IMG_SIZE).astype(np.float32) y = np.zeros(shape=(batch_size, num_classes), dtype=np.int32) tx.copy_from_numpy(x) ty.copy_from_numpy(y) out = model.forward(tx) loss = autograd.softmax_cross_entropy(out, ty) #Initial kv store for workers of ps-architecture key = 0 for p, g in autograd.backward(loss): kv.init(key, mx.nd.array(tensor.to_numpy(p))) key += 1 ''' # Training and Evaulation Loop for epoch in range(max_epoch): start_time = time.time() np.random.shuffle(idx) if (DIST == True): print('^_^Starting Epoch %d:' % (epoch)) # Training Phase autograd.training = True train_correct = np.zeros(shape=[1], dtype=np.float32) test_correct = np.zeros(shape=[1], dtype=np.float32) train_loss = np.zeros(shape=[1], dtype=np.float32) time_start = time.time() for b in range(num_train_batch): x = train_x[idx[b * batch_size:(b + 1) * batch_size]] x = augmentation(x, batch_size) y = train_y[idx[b * batch_size:(b + 1) * batch_size]] tx.copy_from_numpy(x) ty.copy_from_numpy(y) out = model.forward(tx) loss = autograd.softmax_cross_entropy(out, ty) train_correct += accuracy(tensor.to_numpy(out), y) train_loss += tensor.to_numpy(loss)[0] singa_kvstore.backward_and_update(kv, loss) ''' if DIST: #push kv_pairs = [] key = 0 for p, g in autograd.backward(loss): kv.push(key,mx.nd.array(tensor.to_numpy(g))) kv_pairs.append((key,p,g)) key += 1 #pull for key,p,g in kv_pairs: out_buf = mx.nd.zeros(p.shape) kv.pull(key,out=out_buf) p.copy_from_numpy(out_buf.asnumpy()) ''' # Evaluation Phase if b % 20 != 0: continue autograd.training = False num_test_batch_inside = 20 test_correct = 0 for b in range(num_test_batch_inside): x = test_x[b * batch_size:(b + 1) * batch_size] y = test_y[b * batch_size:(b + 1) * batch_size] tx.copy_from_numpy(x) ty.copy_from_numpy(y) out_test = model.forward(tx) test_correct += accuracy(tensor.to_numpy(out_test), y) print('Evaluation accuracy = %f' % (test_correct / (batch_size * num_test_batch_inside)), flush=True) autograd.training = True print('epoch time is %f' % (time.time() - time_start))
def train_mnist_cnn(sgd, max_epoch, batch_size, DIST=False, data_partition=None, gpu_num=None, gpu_per_node=None, nccl_id=None): # Prepare training and valadiation data train_x, train_y, test_x, test_y = load_dataset() IMG_SIZE = 28 num_classes = 10 train_y = to_categorical(train_y, num_classes) test_y = to_categorical(test_y, num_classes) # Normalization train_x = train_x / 255 test_x = test_x / 255 if DIST: # For Distributed GPU Training sgd = opt.DistOpt(sgd, nccl_id=nccl_id, gpu_num=gpu_num, gpu_per_node=gpu_per_node) dev = device.create_cuda_gpu_on(sgd.rank_in_local) # Dataset partition for distributed training train_x, train_y = data_partition(train_x, train_y, sgd.rank_in_global, sgd.world_size) test_x, test_y = data_partition(test_x, test_y, sgd.rank_in_global, sgd.world_size) world_size = sgd.world_size else: # For Single GPU dev = device.create_cuda_gpu() world_size = 1 # create model model = CNN() tx = tensor.Tensor((batch_size, 1, IMG_SIZE, IMG_SIZE), dev, tensor.float32) ty = tensor.Tensor((batch_size, num_classes), dev, tensor.int32) num_train_batch = train_x.shape[0] // batch_size num_test_batch = test_x.shape[0] // batch_size idx = np.arange(train_x.shape[0], dtype=np.int32) if DIST: #Sychronize the initial parameters autograd.training = True x = np.random.randn(batch_size, 1, IMG_SIZE, IMG_SIZE).astype(np.float32) y = np.zeros(shape=(batch_size, num_classes), dtype=np.int32) tx.copy_from_numpy(x) ty.copy_from_numpy(y) out = model.forward(tx) loss = autograd.softmax_cross_entropy(out, ty) for p, g in autograd.backward(loss): sychronize(p, sgd) # Training and Evaulation Loop for epoch in range(max_epoch): start_time = time.time() np.random.shuffle(idx) if ((DIST == False) or (sgd.rank_in_global == 0)): print('Starting Epoch %d:' % (epoch)) # Training Phase autograd.training = True train_correct = np.zeros(shape=[1], dtype=np.float32) test_correct = np.zeros(shape=[1], dtype=np.float32) train_loss = np.zeros(shape=[1], dtype=np.float32) for b in range(num_train_batch): x = train_x[idx[b * batch_size:(b + 1) * batch_size]] x = augmentation(x, batch_size) y = train_y[idx[b * batch_size:(b + 1) * batch_size]] tx.copy_from_numpy(x) ty.copy_from_numpy(y) out = model.forward(tx) loss = autograd.softmax_cross_entropy(out, ty) train_correct += accuracy(tensor.to_numpy(out), y) train_loss += tensor.to_numpy(loss)[0] plist = [] for p, g in autograd.backward(loss): if DIST: sgd.all_reduce(g) plist.append((p, g)) if DIST: sgd.wait() for p, g in plist: sgd.update(p, g) if DIST: # Reduce the Evaluation Accuracy and Loss from Multiple Devices reducer = tensor.Tensor((1, ), dev, tensor.float32) train_correct = reduce_variable(train_correct, sgd, reducer) train_loss = reduce_variable(train_loss, sgd, reducer) # Output the Training Loss and Accuracy if ((DIST == False) or (sgd.rank_in_global == 0)): print('Training loss = %f, training accuracy = %f' % (train_loss, train_correct / (num_train_batch * batch_size * world_size)), flush=True) # Evaluation Phase autograd.training = False for b in range(num_test_batch): x = test_x[b * batch_size:(b + 1) * batch_size] y = test_y[b * batch_size:(b + 1) * batch_size] tx.copy_from_numpy(x) ty.copy_from_numpy(y) out_test = model.forward(tx) test_correct += accuracy(tensor.to_numpy(out_test), y) if DIST: # Reduce the Evaulation Accuracy from Multiple Devices test_correct = reduce_variable(test_correct, sgd, reducer) # Output the Evaluation Accuracy if ((DIST == False) or (sgd.rank_in_global == 0)): print('Evaluation accuracy = %f, Elapsed Time = %fs' % (test_correct / (num_test_batch * batch_size * world_size), time.time() - start_time), flush=True)
epochs = 1 sgd = opt.SGD(lr=0.00) x_train = preprocess(train[0]) y_train = to_categorical(train[1], num_classes) x_test = preprocess(test[0]) y_test = to_categorical(test[1], num_classes) print('the shape of training data is', x_train.shape) print('the shape of training label is', y_train.shape) print('the shape of testing data is', x_test.shape) print('the shape of testing label is', y_test.shape) model = onnx.load('cnn.onnx') rep = sonnx.prepare(model, dev) print('finish init') autograd.training = True # training process for epoch in range(1): inputs = tensor.Tensor(device=dev, data=x_train[0:100], stores_grad=False) targets = tensor.Tensor(device=dev, data=y_train[0:100], requires_grad=False, stores_grad=False) y0 = rep.run([inputs])[0] loss = autograd.softmax_cross_entropy(y0, targets) print('outputs', tensor.to_numpy(loss)[0])
def loss(out, y): return autograd.softmax_cross_entropy(out, y)
return x if __name__ == '__main__': model = Xception(num_classes=1000) print('Start intialization............') dev = device.create_cuda_gpu_on(0) #dev = device.create_cuda_gpu() niters = 20 batch_size = 16 IMG_SIZE = 299 sgd = opt.SGD(lr=0.1, momentum=0.9, weight_decay=1e-5) tx = tensor.Tensor((batch_size, 3, IMG_SIZE, IMG_SIZE), dev) ty = tensor.Tensor((batch_size,), dev, tensor.int32) autograd.training = True x = np.random.randn(batch_size, 3, IMG_SIZE, IMG_SIZE).astype(np.float32) y = np.random.randint(0, 1000, batch_size, dtype=np.int32) tx.copy_from_numpy(x) ty.copy_from_numpy(y) with trange(niters) as t: for b in t: x = model(tx) loss = autograd.softmax_cross_entropy(x, ty) for p, g in autograd.backward(loss): # print(p.shape, g.shape) sgd.update(p, g) # pass
def loss(self, out, ty): return autograd.softmax_cross_entropy(out, ty)
def loss(self, out, ty): ty = autograd.reshape(ty, (-1, 1)) return autograd.softmax_cross_entropy(out, ty)
def train_cifar10(sgd, max_epoch, batch_size, DIST=False, data_partition=None, gpu_num=None, gpu_per_node=None, nccl_id=None, partial_update=False): train_x, train_y = load_train_data() test_x, test_y = load_test_data() train_x, test_x = normalize_for_resnet(train_x, test_x) IMG_SIZE = 224 num_classes=10 if DIST: # For Distributed GPU Training sgd = opt.DistOpt(sgd, nccl_id=nccl_id, gpu_num=gpu_num, gpu_per_node=gpu_per_node) dev = device.create_cuda_gpu_on(sgd.rank_in_local) # Dataset partition for distributed training train_x, train_y = data_partition(train_x, train_y, sgd.rank_in_global, sgd.world_size) test_x, test_y = data_partition(test_x, test_y, sgd.rank_in_global, sgd.world_size) world_size = sgd.world_size else: # For Single GPU dev = device.create_cuda_gpu() world_size = 1 from resnet import resnet50 model = resnet50(num_classes=num_classes) tx = tensor.Tensor((batch_size, 3, IMG_SIZE, IMG_SIZE), dev, tensor.float32) ty = tensor.Tensor((batch_size,), dev, tensor.int32) num_train_batch = train_x.shape[0] // batch_size num_test_batch = test_x.shape[0] // batch_size idx = np.arange(train_x.shape[0], dtype=np.int32) if DIST: #Sychronize the initial parameters autograd.training = True x = np.random.randn(batch_size, 3, IMG_SIZE, IMG_SIZE).astype(np.float32) y = np.zeros( shape=(batch_size,), dtype=np.int32) tx.copy_from_numpy(x) ty.copy_from_numpy(y) out = model(tx) loss = autograd.softmax_cross_entropy(out, ty) param = [] for p, _ in autograd.backward(loss): sychronize(p, sgd) param.append(p) for epoch in range(max_epoch): start_time = time.time() np.random.shuffle(idx) if ((DIST == False) or (sgd.rank_in_global == 0)): print('Starting Epoch %d:' % (epoch)) #Training Phase autograd.training = True train_correct = np.zeros(shape=[1],dtype=np.float32) test_correct = np.zeros(shape=[1],dtype=np.float32) train_loss = np.zeros(shape=[1],dtype=np.float32) for b in range(num_train_batch): x = train_x[idx[b * batch_size: (b + 1) * batch_size]] x = augmentation(x, batch_size) x = resize_dataset(x,IMG_SIZE) y = train_y[idx[b * batch_size: (b + 1) * batch_size]] tx.copy_from_numpy(x) ty.copy_from_numpy(y) out = model(tx) loss = autograd.softmax_cross_entropy(out, ty) train_correct += accuracy(tensor.to_numpy(out), to_categorical(y, num_classes)).astype(np.float32) train_loss += tensor.to_numpy(loss)[0] if not partial_update: sgd.backward_and_update(loss) else: sgd.backward_and_partial_update(loss) if DIST: # Reduce the Evaluation Accuracy and Loss from Multiple Devices reducer = tensor.Tensor((1,), dev, tensor.float32) train_correct = reduce_variable(train_correct, sgd, reducer) train_loss = reduce_variable(train_loss, sgd, reducer) # Output the Training Loss and Accuracy if ((DIST == False) or (sgd.rank_in_global == 0)): print('Training loss = %f, training accuracy = %f' % (train_loss, train_correct / (num_train_batch*batch_size*world_size)), flush=True) if partial_update: # sychronize parameters before evaluation phase for p in param: sychronize(p, sgd) #Evaulation Phase autograd.training = False for b in range(num_test_batch): x = test_x[b * batch_size: (b + 1) * batch_size] x = resize_dataset(x,IMG_SIZE) y = test_y[b * batch_size: (b + 1) * batch_size] tx.copy_from_numpy(x) ty.copy_from_numpy(y) out_test = model(tx) test_correct += accuracy(tensor.to_numpy(out_test), to_categorical(y, num_classes)) if DIST: # Reduce the Evaulation Accuracy from Multiple Devices test_correct = reduce_variable(test_correct, sgd, reducer) # Output the Evaluation Accuracy if ((DIST == False) or (sgd.rank_in_global == 0)): print('Evaluation accuracy = %f, Elapsed Time = %fs' % (test_correct / (num_test_batch*batch_size*world_size), time.time() - start_time ), flush=True)