def get_loss(self, target, pred, context=None, ignore_index=None): loss_pred = nn.cross_entropy_loss(pred, target, ignore_index=ignore_index) if context is None: return loss_pred loss_context = nn.cross_entropy_loss(context, target, ignore_index=ignore_index) loss = loss_pred + 0.1 * loss_context return loss
def _forward_train(self,features,img_size,boxes,labels): N = features.shape[0] rpn_locs, rpn_scores, rois, roi_indices, anchor = self.rpn(features, img_size) sample_rois = [] gt_roi_locs = [] gt_roi_labels = [] sample_roi_indexs = [] gt_rpn_locs = [] gt_rpn_labels = [] for i in range(N): index = jt.where(roi_indices == i)[0] roi = rois[index,:] box = boxes[i] label = labels[i] sample_roi, gt_roi_loc, gt_roi_label = self.proposal_target_creator(roi,box,label) sample_roi_index = i*jt.ones((sample_roi.shape[0],)) sample_rois.append(sample_roi) gt_roi_labels.append(gt_roi_label) gt_roi_locs.append(gt_roi_loc) sample_roi_indexs.append(sample_roi_index) gt_rpn_loc, gt_rpn_label = self.anchor_target_creator(box,anchor,img_size) gt_rpn_locs.append(gt_rpn_loc) gt_rpn_labels.append(gt_rpn_label) sample_roi_indexs = jt.contrib.concat(sample_roi_indexs,dim=0) sample_rois = jt.contrib.concat(sample_rois,dim=0) roi_cls_loc, roi_score = self.head(features,sample_rois,sample_roi_indexs) # ------------------ RPN losses -------------------# rpn_locs = rpn_locs.reshape(-1,4) rpn_scores = rpn_scores.reshape(-1,2) gt_rpn_labels = jt.contrib.concat(gt_rpn_labels,dim=0) gt_rpn_locs = jt.contrib.concat(gt_rpn_locs,dim=0) rpn_loc_loss = _fast_rcnn_loc_loss(rpn_locs,gt_rpn_locs,gt_rpn_labels,self.rpn_sigma) rpn_cls_loss = nn.cross_entropy_loss(rpn_scores[gt_rpn_labels>=0,:],gt_rpn_labels[gt_rpn_labels>=0]) # ------------------ ROI losses (fast rcnn loss) -------------------# gt_roi_locs = jt.contrib.concat(gt_roi_locs,dim=0) gt_roi_labels = jt.contrib.concat(gt_roi_labels,dim=0) n_sample = roi_cls_loc.shape[0] roi_cls_loc = roi_cls_loc.view(n_sample, np.prod(roi_cls_loc.shape[1:]).item()//4, 4) roi_loc = roi_cls_loc[jt.arange(0, n_sample).int32(), gt_roi_labels] roi_loc_loss = _fast_rcnn_loc_loss(roi_loc,gt_roi_locs,gt_roi_labels,self.roi_sigma) roi_cls_loss = nn.cross_entropy_loss(roi_score, gt_roi_labels) losses = [rpn_loc_loss, rpn_cls_loss, roi_loc_loss, roi_cls_loss] losses = losses + [sum(losses)] return losses
def focal_conf_objectness_loss(self, conf_data, conf_t): """ Instead of using softmax, use class[0] to be the objectness score and do sigmoid focal loss on that. Then for the rest of the classes, softmax them and apply CE for only the positive examples. If class[0] = 1 implies forground and class[0] = 0 implies background then you achieve something similar during test-time to softmax by setting class[1:] = softmax(class[1:]) * class[0] and invert class[0]. """ conf_t = conf_t.view(-1) # [batch_size*num_priors] conf_data = conf_data.view(-1, conf_data.shape[-1]) # [batch_size*num_priors, num_classes] # Ignore neutral samples (class < 0) keep = (conf_t >= 0).float() conf_t[conf_t < 0] = 0 # so that gather doesn't drum up a fuss background = (conf_t == 0).float() at = (1 - cfg.focal_loss_alpha) * background + cfg.focal_loss_alpha * (1 - background) logpt = nn.log_sigmoid(conf_data[:, 0]) * (1 - background) + nn.log_sigmoid(-conf_data[:, 0]) * background pt = logpt.exp() obj_loss = -at * (1 - pt) ** cfg.focal_loss_gamma * logpt # All that was the objectiveness loss--now time for the class confidence loss pos_mask = conf_t > 0 conf_data_pos = (conf_data[:, 1:])[pos_mask] # Now this has just 80 classes conf_t_pos = conf_t[pos_mask] - 1 # So subtract 1 here class_loss = nn.cross_entropy_loss(conf_data_pos, conf_t_pos, size_average=False) return cfg.conf_alpha * (class_loss + (obj_loss * keep).sum())
def __call__(self, proposals, keypoint_logits): heatmaps = [] valid = [] for proposals_per_image in proposals: kp = proposals_per_image.get_field("keypoints") heatmaps_per_image, valid_per_image = project_keypoints_to_heatmap( kp, proposals_per_image, self.discretization_size) heatmaps.append(heatmaps_per_image.reshape(-1)) valid.append(valid_per_image.reshape(-1)) keypoint_targets = cat(heatmaps, dim=0) valid = cat(valid, dim=0).bool() valid = jt.nonzero(valid).squeeze(1) # torch.mean (in binary_cross_entropy_with_logits) does'nt # accept empty tensors, so handle it sepaartely if keypoint_targets.numel() == 0 or len(valid) == 0: return keypoint_logits.sum() * 0 N, K, H, W = keypoint_logits.shape keypoint_logits = keypoint_logits.reshape(N * K, H * W) keypoint_loss = nn.cross_entropy_loss(keypoint_logits[valid], keypoint_targets[valid]) return keypoint_loss
def train(model, train_loader, optimizer, epoch): model.train() for batch_idx, (inputs, targets) in enumerate(train_loader): outputs = model(inputs) loss = nn.cross_entropy_loss(outputs, targets) optimizer.step(loss) if batch_idx % 10 == 0: print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( epoch, batch_idx, len(train_loader), 100. * batch_idx / len(train_loader), loss.data[0]))
def main(): mnist = dataset.MNIST() model = Model(mnist[0][0].size) sgd = jt.optim.SGD(model.parameters(), 1e-3) fork_with_mpi() for data, label in mnist: pred = model(data.reshape(data.shape[0], -1)) # print(data.shape, label.shape, pred.shape) loss = nn.cross_entropy_loss(pred, label) sgd.step(loss) print(jt.rank, mnist.epoch_id, mnist.batch_id, loss)
def train(model, train_loader, optimizer, epoch, init_lr, writer): model.train() max_iter = len(train_loader) for idx, (image, target) in enumerate(train_loader): poly_lr_scheduler(optimizer, init_lr, idx, epoch, max_iter, 50) image = image.float32() pred = model(image) loss = nn.cross_entropy_loss(pred, target, ignore_index=255) # fix a bug writer.add_scalar('train/total_loss_iter', loss.data, idx + max_iter * epoch) optimizer.step (loss) print ('Training in epoch {} iteration {} loss = {}'.format(epoch, idx, loss.data[0]))
def train(net, optimizer, epoch, dataloader): net.train() pbar = tqdm(dataloader, desc=f'Epoch {epoch} [TRAIN]') for pts, normals, labels in pbar: output = net(pts, normals) loss = nn.cross_entropy_loss(output, labels) optimizer.step(loss) pred = np.argmax(output.data, axis=1) acc = np.mean(pred == labels.data) * 100 pbar.set_description(f'Epoch {epoch} [TRAIN] loss = {loss.data[0]:.2f}, acc = {acc:.2f}')
def __call__(self, class_logits, box_regression): """ Computes the loss for Faster R-CNN. This requires that the subsample method has been called beforehand. Arguments: class_logits (list[Tensor]) box_regression (list[Tensor]) Returns: classification_loss (Tensor) box_loss (Tensor) """ class_logits = cat(class_logits, dim=0) box_regression = cat(box_regression, dim=0) device = class_logits.device if not hasattr(self, "_proposals"): raise RuntimeError("subsample needs to be called before") proposals = self._proposals labels = cat([proposal.get_field("labels") for proposal in proposals], dim=0) regression_targets = cat([ proposal.get_field("regression_targets") for proposal in proposals ], dim=0) classification_loss = nn.cross_entropy_loss(class_logits, labels) # get indices that correspond to the regression targets for # the corresponding ground truth labels, to be used with # advanced indexing sampled_pos_inds_subset = jt.nonzero(labels > 0).squeeze(1) labels_pos = labels[sampled_pos_inds_subset] if self.cls_agnostic_bbox_reg: map_inds = jt.array([4, 5, 6, 7]) else: map_inds = 4 * labels_pos[:, None] + jt.array([0, 1, 2, 3]) box_loss = smooth_l1_loss( box_regression[sampled_pos_inds_subset[:, None], map_inds], regression_targets[sampled_pos_inds_subset], size_average=False, beta=1, ) box_loss = box_loss / labels.numel() return classification_loss, box_loss
def test_densenet(self): self.setup_seed(1) loss_list = [] acc_list = [] mnist_net = MnistNet() global prev prev = time.time() SGD = nn.SGD(mnist_net.parameters(), self.learning_rate, self.momentum, self.weight_decay) # SGD = jt.optim.Adam(mnist_net.parameters(), lr=0.0001) for batch_idx, (data, target) in enumerate(self.train_loader): output = mnist_net(data) loss = nn.cross_entropy_loss(output, target) SGD.step(loss) def callback(batch_idx, loss, output, target): # print train info global prev pred = np.argmax(output, axis=1) acc = np.mean(target == pred) loss_list.append(loss[0]) acc_list.append(acc) print( 'Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}\tAcc: {:.6f} \tTime:{:.3f}' .format(0, batch_idx, 600, 1. * batch_idx / 6.0, loss[0], acc, time.time() - prev)) # prev = time.time() jt.fetch(batch_idx, loss, output, target, callback) # Train Epoch: 0 [0/600 (0%)] Loss: 2.402650 Acc: 0.060000 # Train Epoch: 0 [1/600 (0%)] Loss: 2.770145 Acc: 0.100000 # Train Epoch: 0 [2/600 (0%)] Loss: 3.528072 Acc: 0.100000 # Train Epoch: 0 [3/600 (0%)] Loss: 2.992042 Acc: 0.100000 # Train Epoch: 0 [4/600 (1%)] Loss: 4.672772 Acc: 0.060000 # Train Epoch: 0 [5/600 (1%)] Loss: 5.003410 Acc: 0.080000 # Train Epoch: 0 [6/600 (1%)] Loss: 5.417546 Acc: 0.100000 # Train Epoch: 0 [7/600 (1%)] Loss: 5.137665 Acc: 0.100000 # Train Epoch: 0 [8/600 (1%)] Loss: 5.241075 Acc: 0.070000 # Train Epoch: 0 [9/600 (2%)] Loss: 4.515363 Acc: 0.100000 # Train Epoch: 0 [10/600 (2%)] Loss: 3.357187 Acc: 0.170000 # Train Epoch: 0 [20/600 (3%)] Loss: 2.265879 Acc: 0.100000 # Train Epoch: 0 [30/600 (5%)] Loss: 2.107000 Acc: 0.250000 # Train Epoch: 0 [40/600 (7%)] Loss: 1.918214 Acc: 0.290000 # Train Epoch: 0 [50/600 (8%)] Loss: 1.645694 Acc: 0.400000 jt.sync_all(True) assert np.mean(loss_list[-50:]) < 0.3 assert np.mean(acc_list[-50:]) > 0.9
def test_vgg(self): self.setup_seed(1) loss_list = [] acc_list = [] mnist_net = MnistNet() SGD = nn.SGD(mnist_net.parameters(), self.learning_rate, self.momentum, self.weight_decay) for batch_idx, (data, target) in enumerate(self.train_loader): output = mnist_net(data) loss = nn.cross_entropy_loss(output, target) # train step with jt.log_capture_scope( log_silent=1, log_v=1, log_vprefix="op.cc=100,exe=10", ) as logs: SGD.step(loss) def callback(loss, output, target, batch_idx): # print train info pred = np.argmax(output, axis=1) acc = np.sum(target == pred) / self.batch_size loss_list.append(loss[0]) acc_list.append(acc) print( 'Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}\tAcc: {:.6f}' .format(0, batch_idx, 100, 1. * batch_idx, loss[0], acc)) jt.fetch(batch_idx, loss, output, target, callback) log_conv = find_log_with_re( logs, "Jit op key (not )?found: ((mkl)|(cudnn))_conv.*") log_matmul = find_log_with_re( logs, "Jit op key (not )?found: ((mkl)|(cublas))_matmul.*") if batch_idx: assert len(log_conv) == 38 and len(log_matmul) == 12, ( len(log_conv), len(log_matmul)) mem_used = jt.flags.stat_allocator_total_alloc_byte \ -jt.flags.stat_allocator_total_free_byte assert mem_used < 11e9, mem_used assert jt.core.number_of_lived_vars() < 3500 if (np.mean(loss_list[-50:]) < 0.2): break assert np.mean(loss_list[-50:]) < 0.2
def train(model, train_loader, optimizer, epoch): model.train() total_acc = 0 total_num = 0 losses = 0.0 pbar = tqdm(train_loader, desc=f'Epoch {epoch} [TRAIN]') for images, labels in pbar: output = model(images) loss = nn.cross_entropy_loss(output, labels) optimizer.step(loss) pred = np.argmax(output.data, axis=1) acc = np.mean(pred == labels.data) * 100 total_acc += acc total_num += labels.shape[0] losses += loss pbar.set_description(f'Epoch {epoch} [TRAIN] loss = {loss.data[0]:.2f}, acc = {acc:.2f}')
def conf_objectness_loss(self, conf_data, conf_t, batch_size, loc_p, loc_t, priors): """ Instead of using softmax, use class[0] to be p(obj) * p(IoU) as in YOLO. Then for the rest of the classes, softmax them and apply CE for only the positive examples. """ conf_t = conf_t.view(-1) # [batch_size*num_priors] conf_data = conf_data.view( -1, conf_data.shape[-1]) # [batch_size*num_priors, num_classes] pos_mask = (conf_t > 0) neg_mask = (conf_t == 0) obj_data = conf_data[:, 0] obj_data_pos = obj_data[pos_mask] obj_data_neg = obj_data[neg_mask] # Don't be confused, this is just binary cross entropy similified obj_neg_loss = -nn.log_sigmoid(-obj_data_neg).sum() with jt.no_grad(): pos_priors = priors.unsqueeze(0).expand(batch_size, -1, -1).reshape(-1, 4)[pos_mask] boxes_pred = decode(loc_p, pos_priors, cfg.use_yolo_regressors) boxes_targ = decode(loc_t, pos_priors, cfg.use_yolo_regressors) iou_targets = elemwise_box_iou(boxes_pred, boxes_targ) obj_pos_loss = -iou_targets * nn.log_sigmoid(obj_data_pos) - ( 1 - iou_targets) * nn.log_sigmoid(-obj_data_pos) obj_pos_loss = obj_pos_loss.sum() # All that was the objectiveness loss--now time for the class confidence loss conf_data_pos = ( conf_data[:, 1:])[pos_mask] # Now this has just 80 classes conf_t_pos = conf_t[pos_mask] - 1 # So subtract 1 here class_loss = nn.cross_entropy_loss(conf_data_pos, conf_t_pos, size_average=False) return cfg.conf_alpha * (class_loss + obj_pos_loss + obj_neg_loss)
def train(epoch, model, trainloader, optimizer): model.train() loss_ = 0 correct = 0 total = 0 for batch_idx, (inputs, targets) in tqdm( enumerate(trainloader), total=math.ceil(len(trainloader) / trainloader.batch_size)): outputs = model(inputs) predicts = np.argmax(outputs.data, axis=1) correct += np.sum(targets.data == predicts) total += targets.shape[0] loss = nn.cross_entropy_loss(outputs, targets) optimizer.step(loss) loss_ += loss.data[0] loss_ /= len(trainloader) accuracy = correct / total print("Train Epoch: {} | Train Loss: {} | Train Acc: {}".format( epoch, loss_, accuracy)) return loss_, accuracy
def test(epoch, model, testloader): model.eval() loss_ = 0 total, correct = 0, 0 for batch_idx, (inputs, targets) in tqdm( enumerate(testloader), total=math.ceil(len(testloader) / testloader.batch_size)): outputs = model(inputs) predicts = np.argmax(outputs.data, axis=1) correct += np.sum(targets.data == predicts) total += targets.shape[0] loss = nn.cross_entropy_loss(outputs, targets) loss_ += loss.data[0] loss_ /= len(testloader) accuracy = correct / total print("Test Epoch : {} | Test Loss: {} | Test Acc: {}".format( epoch, loss_, accuracy)) return loss_, accuracy
def soft_cross_entropy_loss(output, target, smoothing=True): ''' Calculate cross entropy loss, apply label smoothing if needed. ''' target = target.view(-1) softmax = nn.Softmax(dim=1) if smoothing: eps = 0.2 b, n_class = output.shape one_hot = jt.zeros(output.shape) for i in range(b): one_hot[i, target[i].data] = 1 one_hot = one_hot * (1 - eps) + (1 - one_hot) * eps / (n_class - 1) # print (one_hot[0].data) log_prb = jt.log(softmax(output)) loss = -(one_hot * log_prb).sum(dim=1).mean() else: loss = nn.cross_entropy_loss(output, target) return loss
def test_resnet(self): self.setup_seed(1) loss_list=[] acc_list=[] mnist_net = MnistNet() global prev prev = time.time() SGD = nn.SGD(mnist_net.parameters(), self.learning_rate, self.momentum, self.weight_decay) iters = 10 for batch_idx, (data, target) in enumerate(self.train_loader): if (batch_idx > iters): break jt.display_memory_info() output = mnist_net(data) loss = nn.cross_entropy_loss(output, target) SGD.step(loss) def callback(batch_idx, loss, output, target): global prev pred = np.argmax(output, axis=1) acc = np.mean(target==pred) loss_list.append(loss[0]) acc_list.append(acc) print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}\tAcc: {:.6f} \tTime:{:.3f}' .format(0, batch_idx, iters,1. * batch_idx / 6.0, loss[0], acc, time.time()-prev)) jt.fetch(batch_idx, loss, output, target, callback) jt.sync_all(True) jt.display_max_memory_info() _, out = jt.get_max_memory_treemap() out_ = out.split('\n') assert(out_[0] == 'root()') assert(out_[3].endswith('(_run_module_as_main)')) assert(out_[7].endswith('(_run_code)')) _, out = jt.get_max_memory_treemap(build_by=1) out_ = out.split('\n') assert(out_[0] == 'root()') assert(out_[4].endswith('(_run_module_as_main)')) assert(out_[8].endswith('(_run_code)'))
def get_loss(self, target, pred, context=None, ignore_index=None): loss_pred = nn.cross_entropy_loss(pred, target, ignore_index=ignore_index) return loss_pred
def train(model): batch_size = 16 train_loader = ShapeNetPart(partition='trainval', num_points=2048, class_choice=None, batch_size=batch_size, shuffle=True) test_loader = ShapeNetPart(partition='test', num_points=2048, class_choice=None, batch_size=batch_size, shuffle=False) seg_num_all = 50 seg_start_index = 0 print(str(model)) base_lr = 0.01 optimizer = nn.SGD(model.parameters(), lr=base_lr, momentum=0.9, weight_decay=1e-4) lr_scheduler = LRScheduler(optimizer, base_lr) # criterion = nn.cross_entropy_loss() # here best_test_iou = 0 for epoch in range(200): #################### # Train #################### lr_scheduler.step(len(train_loader) * batch_size) train_loss = 0.0 count = 0.0 model.train() train_true_cls = [] train_pred_cls = [] train_true_seg = [] train_pred_seg = [] train_label_seg = [] # debug = 0 for data, label, seg in train_loader: # with jt.profile_scope() as report: seg = seg - seg_start_index label_one_hot = np.zeros((label.shape[0], 16)) # print (label.size()) for idx in range(label.shape[0]): label_one_hot[idx, label.numpy()[idx, 0]] = 1 label_one_hot = jt.array(label_one_hot.astype(np.float32)) data = data.permute(0, 2, 1) # for pointnet it should not be committed batch_size = data.size()[0] # print ('input data shape') # print (data.shape, label_one_hot.shape) # for pointnet b c n for pointnet2 b n c seg_pred = model(data, label_one_hot) seg_pred = seg_pred.permute(0, 2, 1) # print (seg_pred.size()) # print (seg_pred.size(), seg.size()) loss = nn.cross_entropy_loss(seg_pred.view(-1, seg_num_all), seg.view(-1)) # print (loss.data) optimizer.step(loss) pred = jt.argmax(seg_pred, dim=2)[0] # (batch_size, num_points) # print ('pred size =', pred.size(), seg.size()) count += batch_size train_loss += loss.numpy() * batch_size seg_np = seg.numpy() # (batch_size, num_points) pred_np = pred.numpy() # (batch_size, num_points) # print (type(label)) label = label.numpy() # added train_true_cls.append( seg_np.reshape(-1)) # (batch_size * num_points) train_pred_cls.append( pred_np.reshape(-1)) # (batch_size * num_points) train_true_seg.append(seg_np) train_pred_seg.append(pred_np) temp_label = label.reshape(-1, 1) train_label_seg.append(temp_label) # print(report) train_true_cls = np.concatenate(train_true_cls) train_pred_cls = np.concatenate(train_pred_cls) # print (train_true_cls.shape ,train_pred_cls.shape) train_acc = metrics.accuracy_score(train_true_cls, train_pred_cls) avg_per_class_acc = metrics.balanced_accuracy_score( train_true_cls.data, train_pred_cls.data) # print ('train acc =',train_acc, 'avg_per_class_acc', avg_per_class_acc) train_true_seg = np.concatenate(train_true_seg, axis=0) # print (len(train_pred_seg), train_pred_seg[0].shape) train_pred_seg = np.concatenate(train_pred_seg, axis=0) # print (len(train_label_seg), train_label_seg[0].size()) # print (train_label_seg[0]) train_label_seg = np.concatenate(train_label_seg, axis=0) # print (train_pred_seg.shape, train_true_seg.shape, train_label_seg.shape) train_ious = calculate_shape_IoU(train_pred_seg, train_true_seg, train_label_seg, None) outstr = 'Train %d, loss: %.6f, train acc: %.6f, train avg acc: %.6f, train iou: %.6f' % ( epoch, train_loss * 1.0 / count, train_acc, avg_per_class_acc, np.mean(train_ious)) # io.cprint(outstr) print(outstr) #################### # Test #################### test_loss = 0.0 count = 0.0 model.eval() test_true_cls = [] test_pred_cls = [] test_true_seg = [] test_pred_seg = [] test_label_seg = [] for data, label, seg in test_loader: seg = seg - seg_start_index label_one_hot = np.zeros((label.shape[0], 16)) for idx in range(label.shape[0]): label_one_hot[idx, label.numpy()[idx, 0]] = 1 label_one_hot = jt.array(label_one_hot.astype(np.float32)) data = data.permute(0, 2, 1) # for pointnet should not be commit batch_size = data.size()[0] seg_pred = model(data, label_one_hot) seg_pred = seg_pred.permute(0, 2, 1) loss = nn.cross_entropy_loss(seg_pred.view(-1, seg_num_all), seg.view(-1, 1).squeeze(-1)) pred = jt.argmax(seg_pred, dim=2)[0] count += batch_size test_loss += loss.numpy() * batch_size seg_np = seg.numpy() pred_np = pred.numpy() label = label.numpy() # added test_true_cls.append(seg_np.reshape(-1)) test_pred_cls.append(pred_np.reshape(-1)) test_true_seg.append(seg_np) test_pred_seg.append(pred_np) test_label_seg.append(label.reshape(-1, 1)) test_true_cls = np.concatenate(test_true_cls) test_pred_cls = np.concatenate(test_pred_cls) test_acc = metrics.accuracy_score(test_true_cls, test_pred_cls) avg_per_class_acc = metrics.balanced_accuracy_score( test_true_cls, test_pred_cls) test_true_seg = np.concatenate(test_true_seg, axis=0) test_pred_seg = np.concatenate(test_pred_seg, axis=0) test_label_seg = np.concatenate(test_label_seg) test_ious = calculate_shape_IoU(test_pred_seg, test_true_seg, test_label_seg, None) outstr = 'Test %d, loss: %.6f, test acc: %.6f, test avg acc: %.6f, test iou: %.6f' % ( epoch, test_loss * 1.0 / count, test_acc, avg_per_class_acc, np.mean(test_ious)) print(outstr)
def test_resnet(self): self.setup_seed(1) loss_list = [] acc_list = [] mnist_net = MnistNet() global prev prev = time.time() SGD = nn.SGD(mnist_net.parameters(), self.learning_rate, self.momentum, self.weight_decay) self.train_loader.endless = True for data, target in self.train_loader: batch_id = self.train_loader.batch_id epoch_id = self.train_loader.epoch_id # train step # with jt.log_capture_scope( # log_silent=1, # log_v=1, log_vprefix="op.cc=100,exe=10", # ) as logs: output = mnist_net(data) loss = nn.cross_entropy_loss(output, target) SGD.step(loss) def callback(epoch_id, batch_id, loss, output, target): # print train info global prev pred = np.argmax(output, axis=1) acc = np.mean(target == pred) loss_list.append(loss[0]) acc_list.append(acc) print( 'Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}\tAcc: {:.6f} \tTime:{:.3f}' .format(epoch_id, batch_id, 600, 1. * batch_id / 6.0, loss[0], acc, time.time() - prev)) # prev = time.time() jt.fetch(epoch_id, batch_id, loss, output, target, callback) # log_conv = find_log_with_re(logs, # "Jit op key (not )?found: ((mkl)|(cudnn))_conv.*") # log_matmul = find_log_with_re(logs, # "Jit op key (not )?found: ((mkl)|(cublas))_matmul.*") # if batch_id > 2: # assert len(log_conv)==59 and len(log_matmul)==6, (len(log_conv), len(log_matmul)) mem_used = jt.flags.stat_allocator_total_alloc_byte \ -jt.flags.stat_allocator_total_free_byte # assert mem_used < 4e9, mem_used # TODO: why bigger? assert mem_used < 5.6e9, mem_used # example log: # Train Epoch: 0 [0/100 (0%)] Loss: 2.352903 Acc: 0.110000 # Train Epoch: 0 [1/100 (1%)] Loss: 2.840830 Acc: 0.080000 # Train Epoch: 0 [2/100 (2%)] Loss: 3.473594 Acc: 0.100000 # Train Epoch: 0 [3/100 (3%)] Loss: 3.131615 Acc: 0.200000 # Train Epoch: 0 [4/100 (4%)] Loss: 2.524094 Acc: 0.230000 # Train Epoch: 0 [5/100 (5%)] Loss: 7.780025 Acc: 0.080000 # Train Epoch: 0 [6/100 (6%)] Loss: 3.890721 Acc: 0.160000 # Train Epoch: 0 [7/100 (7%)] Loss: 6.370137 Acc: 0.140000 # Train Epoch: 0 [8/100 (8%)] Loss: 11.390827 Acc: 0.150000 # Train Epoch: 0 [9/100 (9%)] Loss: 21.598564 Acc: 0.080000 # Train Epoch: 0 [10/100 (10%)] Loss: 23.369165 Acc: 0.130000 # Train Epoch: 0 [20/100 (20%)] Loss: 4.804510 Acc: 0.100000 # Train Epoch: 0 [30/100 (30%)] Loss: 3.393924 Acc: 0.110000 # Train Epoch: 0 [40/100 (40%)] Loss: 2.286762 Acc: 0.130000 # Train Epoch: 0 [50/100 (50%)] Loss: 2.055014 Acc: 0.290000 if jt.in_mpi: assert jt.core.number_of_lived_vars( ) < 8100, jt.core.number_of_lived_vars() else: assert jt.core.number_of_lived_vars( ) < 7000, jt.core.number_of_lived_vars() if self.train_loader.epoch_id >= 2: break jt.sync_all(True) assert np.mean(loss_list[-50:]) < 0.5 assert np.mean(acc_list[-50:]) > 0.8
def execute(self, output, target): return cross_entropy_loss(output, target, self.ignore_index)