class Visualier(): """Visulization, plot the logs during training process""" def __init__(self, num_classes=10): port = 8097 self.loss_logger = VisdomPlotLogger('line', port=port, win="Loss", opts={'title': 'Loss Logger'}) self.acc_logger = VisdomPlotLogger('line', port=port, win="acc", opts={'title': 'Accuracy Logger'}) self.confusion_logger = VisdomLogger('heatmap', port=port, win="confusion", opts={ 'title': 'Confusion matrix', 'columnnames': list(range(num_classes)), 'rownames': list(range(num_classes)) }) def plot(self, train_acc, train_err, val_acc, val_err, confusion, epoch): self.loss_logger.log(epoch, train_err, name="train") self.acc_logger.log(epoch, train_acc, name="train") self.loss_logger.log(epoch, val_err, name="val") self.acc_logger.log(epoch, val_acc, name="val") self.confusion_logger.log(confusion) print("epoch: [%d/%d]" % (epoch, args.n_epoches)) print('Training loss: %.4f, accuracy: %.2f%%' % (train_err, train_acc)) print('Validation loss: %.4f, accuracy: %.2f%%' % (val_err, val_acc))
class ConfusionVisdom(object): '''Plot test confusion matrix in a VisdomLogger ''' def __init__(self, num_classes, title='TBD'): self._confusion = VisdomLogger( 'heatmap', opts={ 'title': '{:s} Confusion Matrix'.format(title), 'columnnames': list(range(num_classes)), 'rownames': list(range(num_classes)) }) check_visdom_server(self._confusion.viz) def log(self, confusion, train=None): assert train is not None,\ 'train should be True or False, not {}'.format(train) if train: pass else: try: self._confusion.log(confusion) except BaseException as e: check_visdom_server(self._confusion.viz) print(e) print("***Retry ConfusionVisdom") self.log(confusion, train)
def log_image(im_logger: VisdomLogger, img: torch.Tensor, batch_size: int = None): if batch_size is None: batch_size = img.shape[0] grid_img = make_grid(img.detach().cpu(), nrow=int(batch_size**0.5), normalize=True, range=(0, 1)).numpy() im_logger.log(grid_img) return grid_img
def test_only(model,train_dataloader, val_dataloader, optimizer, loss_fn, metrics, params, model_dir,logger,restore_file=None): # reload weights from restore_file if specified if restore_file is not None: logging.info("Restoring parameters from {}".format(restore_file)) checkpoint = utils.load_checkpoint(restore_file, model, optimizer) best_val_acc = checkpoint['best_val_acc'] params.current_epoch = checkpoint['epoch'] print('best_val_acc=',best_val_acc) print(optimizer.state_dict()['param_groups'][0]['lr'], checkpoint['epoch']) train_confusion_logger = VisdomLogger('heatmap', port=port, opts={'title': params.experiment_path + 'train_Confusion matrix', 'columnnames': columnnames, 'rownames': rownames},env='Test') test_confusion_logger = VisdomLogger('heatmap', port=port, opts={'title': params.experiment_path + 'test_Confusion matrix', 'columnnames': columnnames, 'rownames': rownames},env='Test') diff_confusion_logger = VisdomLogger('heatmap', port=port, opts={'title': params.experiment_path + 'diff_Confusion matrix', 'columnnames': columnnames, 'rownames': rownames},env='Test') # Evaluate for one epoch on validation set # model.train() model.eval() train_metrics, train_confusion_meter = evaluate(model, loss_fn, train_dataloader, metrics, params, logger) train_confusion_logger.log(train_confusion_meter.value()) model.eval() val_metrics,test_confusion_meter = evaluate(model, loss_fn, val_dataloader, metrics, params, logger) test_confusion_logger.log(test_confusion_meter.value()) diff_confusion_meter = train_confusion_meter.value()-test_confusion_meter.value() diff_confusion_logger.log(diff_confusion_meter) pass
def main(checkpoint_path, batch_size, normalized, visdom_port): checkpoint_path = Path(checkpoint_path) snapshot_path = checkpoint_path.parent.parent.parent / 'snapshot.json' with snapshot_path.open('r') as f: snapshot_dict = json.load(f) mat_id_to_label = snapshot_dict['mat_id_to_label'] label_to_mat_id = {int(v): int(k) for k, v in mat_id_to_label.items()} num_classes = len(label_to_mat_id) + 1 print(f'Loading model checkpoint from {checkpoint_path!r}') checkpoint = torch.load(checkpoint_path) model = RendNet3(num_classes=num_classes, num_roughness_classes=20, num_substances=len(SUBSTANCES), base_model=resnet.resnet18(pretrained=False)) model.load_state_dict(checkpoint['state_dict']) model.train(False) model = model.cuda() validation_dataset = rendering_dataset.MaterialRendDataset( snapshot_dict, snapshot_dict['examples']['validation'], shape=(384, 384), image_transform=transforms.inference_image_transform(INPUT_SIZE), mask_transform=transforms.inference_mask_transform(INPUT_SIZE)) validation_loader = DataLoader( validation_dataset, batch_size=batch_size, num_workers=8, shuffle=False, pin_memory=True, collate_fn=rendering_dataset.collate_fn) pred_counts = collections.defaultdict(collections.Counter) # switch to evaluate mode model.eval() confusion_meter = tnt.meter.ConfusionMeter( k=num_classes, normalized=normalized) pbar = tqdm(validation_loader) for batch_idx, batch_dict in enumerate(pbar): input_tensor = batch_dict['image'].cuda() labels = batch_dict['material_label'].cuda() # compute output output = model.forward(input_tensor) pbar.set_description(f"{output['material'].size()}") # _, pred = output['material'].topk(k=1, dim=1, largest=True, sorted=True) confusion_meter.add(output['material'].cpu(), labels.cpu()) with session_scope() as sess: materials = sess.query(models.Material).filter_by(enabled=True).all() material_id_to_name = {m.id: m.name for m in materials} mat_by_id = {m.id: m for m in materials} class_names = ['background'] class_names.extend([ mat_by_id[label_to_mat_id[i]].name for i in range(1, num_classes) ]) print(len(class_names), ) confusion_matrix = confusion_meter.value() # sorted_confusion_matrix = confusion_matrix[:, inds] # sorted_confusion_matrix = sorted_confusion_matrix[inds, :] # sorted_class_names = [class_names[i] for i in inds] confusion_logger = VisdomLogger( 'heatmap', opts={ 'title': 'Confusion matrix', 'columnnames': class_names, 'rownames': class_names, 'xtickfont': {'size': 8}, 'ytickfont': {'size': 8}, }, env='brdf-classifier-confusion', port=visdom_port) confusion_logger.log(confusion_matrix)
torch.save(model.state_dict(), 'epochs/epoch_%d.pt' % state['epoch']) # Reconstruction visualization. test_sample = next(iter(get_iterator(False))) ground_truth = (test_sample[0].unsqueeze(1).float() / 255.0) <<<<<<< HEAD _, reconstructions = model(Variable(ground_truth)) ======= _, reconstructions = model(Variable(ground_truth).cuda()) >>>>>>> 3da462b9351869b0342b95d99fef37ab3e45a309 reconstruction = reconstructions.cpu().view_as(ground_truth).data ground_truth_logger.log( make_grid(ground_truth, nrow=int(BATCH_SIZE ** 0.5), normalize=True, range=(0, 1)).numpy()) reconstruction_logger.log( make_grid(reconstruction, nrow=int(BATCH_SIZE ** 0.5), normalize=True, range=(0, 1)).numpy()) # def on_start(state): # state['epoch'] = 327 # # engine.hooks['on_start'] = on_start engine.hooks['on_sample'] = on_sample engine.hooks['on_forward'] = on_forward engine.hooks['on_start_epoch'] = on_start_epoch engine.hooks['on_end_epoch'] = on_end_epoch engine.train(processor, get_iterator(True), maxepoch=NUM_EPOCHS, optimizer=optimizer)
loss.backward() optimzier.step() schduler.step() loss_.append(loss.item()) del loss if opt.local_rank == 0: print('[Epoch:{}/{}][batch:{}/{}][loss:{}][learning rate:{}]'.format(epoch,opt.n_epochs,batch,b_r,loss_[-1],optimzier.state_dict()['param_groups'][0]['lr'])) a = np.random.randint(0,31,fout.shape[0]) a_ = np.array(range(fout.shape[0])) output = fout.detach().cpu()[a_,a,:,:]*255 output = output[:,None,:,:] output = torch.cat([output,output,output],1) output_ = make_grid(output,nrow=int(5)).numpy() if opt.local_rank == 0: train_logger.log(output_) Loss_logger.log(epoch,np.mean(np.array(loss_))) psnr_g = [] with torch.no_grad(): for hsi,hsi_g,hsi_resize,msi in Hyper_test: spenet.eval() hsi_g = hsi_g.cuda().float() hsi_resize = hsi_resize.cuda().float() hsi_resize = torch.nn.functional.interpolate(hsi_resize,scale_factor=(8,8),mode='bilinear') # hsi = renet(hsi_resize) hsi = hsi_resize # hsi = [a.cuda().float() for a in hsi] msi = [a.cuda().float() for a in msi] hsi_spe,scale,refined = spenet(hsi,msi[-1]) # hsi_e = get_spe_gt(hsi_g) # hsi_2 = hsi_spe[-1][:,:31,:,:]
loss = focal_loss(classes, focal_label) + margin_loss( classes, margin_label) loss.backward() optimizer.step() # save the metrics meter_loss.add(loss.detach().cpu().item()) meter_accuracy.add(classes.detach().cpu(), target) meter_confusion.add(classes.detach().cpu(), target) if current_step % NUM_STEPS == 0: # print the information about train train_loss_logger.log(current_step // NUM_STEPS, meter_loss.value()[0]) train_accuracy_logger.log(current_step // NUM_STEPS, meter_accuracy.value()[0]) train_confusion_logger.log(meter_confusion.value()) results['train_loss'].append(meter_loss.value()[0]) results['train_accuracy'].append(meter_accuracy.value()[0]) print('[Step %d] Training Loss: %.4f Accuracy: %.2f%%' % (current_step // NUM_STEPS, meter_loss.value()[0], meter_accuracy.value()[0])) reset_meters() # test model periodically model.eval() with torch.no_grad(): for data, target in test_iterator: focal_label, margin_label = target, torch.eye( num_class).index_select(dim=0, index=target) if torch.cuda.is_available(): data, focal_label, margin_label = data.to(
def test_gradient(self): for mesh in self.meshes: object = sl.Object(mesh) self.scene.add_object(object) pose = th.tensor([[0.0596, 0.8315, -0.5523, -0.0651], [0.4715, 0.4642, 0.7498, -0.06036], [0.8798, -0.3051, -0.3644, 0.80551], [0.0000, 0.0000, 0.0000, 1.0000]]) U, S, V = th.svd(pose[:3, :3]) pose[:3, :3] = th.matmul(U, V.t()) object.set_pose(pose) renderer = sl.RenderPass() result = renderer.render(self.scene) gt_mask = (result.instance_index() != 0).float() rgb = result.rgb().detach() gt_rgb = rgb[:, :, :3] gt_rgb_np = gt_rgb.cpu().numpy() gt_pose = 0 if VIS: import visdom from torchnet.logger import VisdomLogger env_name = 'stillleben' vis = visdom.Visdom(port=8097, env=env_name) vis.close(env=env_name) gt_logger = VisdomLogger('image', env=env_name, port=8097, opts=dict(title='gt')) img_logger = VisdomLogger('image', env=env_name, port=8097, opts=dict(title='rgb')) for param in range(6): for obj in self.scene.objects: new_pose = obj.pose().clone() gt_pose = obj.pose().clone() GT_DELTA = th.zeros(6) GT_DELTA[param] = 0.01 new_pose = sl.diff.apply_pose_delta(gt_pose, GT_DELTA) obj.set_pose(new_pose) rendered_result = renderer.render(self.scene) rendered_rgb = rendered_result.rgb() rendered_rgb = rendered_rgb[:, :, :3] rgb_np = rendered_rgb.cpu().numpy() if VIS: img_logger.log(rgb_np.transpose(2, 0, 1)) gt_logger.log(gt_rgb_np.transpose(2, 0, 1)) gt_mask = th.ones_like(gt_mask) grad_wrt_img, l = get_gaussian_pyramid_comparison( gt_rgb.float() / 255.0, rendered_rgb.float() / 255.0, gt_mask.squeeze()) delta = sl.diff.backpropagate_gradient_to_poses( self.scene, rendered_result, grad_wrt_img) print('GT delta', GT_DELTA) print('Delta', delta) self.assertGreater(delta[0][param], 0)
loss_D_B = (loss_D_real + loss_D_fake) * 0.5 loss_D_B.backward() optimizer_D_B.step() loss_meters['loss_G_meter'].add(loss_G.item()) loss_meters['loss_G_identity_meter'].add( (loss_identity_A + loss_identity_B).item()) loss_meters['loss_G_GAN_meter'].add( (loss_GAN_A2B + loss_GAN_B2A).item()) loss_meters['loss_G_cycle_meter'].add( (loss_cycle_ABA + loss_cycle_BAB).item()) loss_meters['loss_D_meter'].add((loss_D_A + loss_D_B).item()) real_A_im_logger.log( make_grid(real_A.detach().cpu(), nrow=int(opt.batchSize**0.5), normalize=True, range=(0, 1)).numpy()) real_B_im_logger.log( make_grid(real_B.detach().cpu(), nrow=int(opt.batchSize**0.5), normalize=True, range=(0, 1)).numpy()) fake_A_im_logger.log( make_grid(fake_A.detach().cpu(), nrow=int(opt.batchSize**0.5), normalize=True, range=(0, 1)).numpy()) fake_B_im_logger.log( make_grid(fake_B.detach().cpu(), nrow=int(opt.batchSize**0.5),
'rownames': list(range(args.num_classes)) }, env=args.env_name) ground_truth_logger = VisdomLogger('image', opts={'title': 'Ground Truth'}, env=args.env_name) reconstruction_logger = VisdomLogger('image', opts={'title': 'Reconstruction'}, env=args.env_name) weight_folder = 'weights/{}'.format(args.env_name.replace(' ', '_')) if not os.path.isdir(weight_folder): os.mkdir(weight_folder) setting_logger.log(str(args)) print("# parameters:", sum(param.numel() for param in model.parameters())) optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 'max', patience=1) train_dataset = datasets.MNIST(root='./data/', train=True, transform=transforms.ToTensor(), download=True) test_dataset = datasets.MNIST(root='./data/', train=False, transform=transforms.ToTensor())
if os.path.isfile('loss.log'): with open("loss.log", "r") as lossfile: loss_list = [] for loss in lossfile: loss_list.append(loss) while len(loss_list) > args.load_loss: loss_list.pop(0) for loss in loss_list: train_loss_logger.log(epoch_offset, float(loss)) epoch_offset += 1 ground_truth_logger_left = VisdomLogger('image', opts={'title': 'Ground Truth, left'}, env=args.env_name) ground_truth_logger_right = VisdomLogger('image', opts={'title': 'Ground Truth, right'}, env=args.env_name) reconstruction_logger_left = VisdomLogger('image', opts={'title': 'Reconstruction, left'}, env=args.env_name) reconstruction_logger_right = VisdomLogger('image', opts={'title': 'Reconstruction, right'}, env=args.env_name) setting_logger.log(str(args)) """ Load training data """ train_dataset = util.MyImageFolder(root='../../data/dumps/', transform=transforms.ToTensor(), target_transform=transforms.ToTensor()) train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=args.batch_size, num_workers=args.num_workers, shuffle=True, drop_last=True) steps = len(train_dataset) // args.batch_size """ Training Loop """ for epoch in range(args.num_epochs):
def test_only(model, train_dataloader, val_dataloader, optimizer, loss_fn, metrics, params, model_dir, logger, restore_file=None): if 'CAD' in params.dataset_name: # modify the model for CAD-60 for fine-tuning out_channel = params.model_args['out_channel'] window_size = params.model_args['window_size'] model.fc7 = nn.Sequential( nn.Linear( (out_channel * 4) * (window_size // 16) * (window_size // 16), 256), # 4*4 for window=64; 8*8 for window=128 nn.ReLU(), nn.Dropout2d(p=0.5)) model.fc8 = nn.Linear(256, 12) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = model.to(device) # reload weights from restore_file if specified if restore_file is not None: logging.info("Restoring parameters from {}".format(restore_file)) checkpoint = utils.load_checkpoint(restore_file, model, optimizer) best_val_acc = checkpoint['best_val_acc'] params.current_epoch = checkpoint['epoch'] print('best_val_acc=', best_val_acc) print(optimizer.state_dict()['param_groups'][0]['lr'], checkpoint['epoch']) train_confusion_logger = VisdomLogger('heatmap', port=port, opts={ 'title': params.experiment_path + 'train_Confusion matrix', 'columnnames': columnnames, 'rownames': rownames }, env='Test') test_confusion_logger = VisdomLogger('heatmap', port=port, opts={ 'title': params.experiment_path + 'test_Confusion matrix', 'columnnames': columnnames, 'rownames': rownames }, env='Test') diff_confusion_logger = VisdomLogger('heatmap', port=port, opts={ 'title': params.experiment_path + 'diff_Confusion matrix', 'columnnames': columnnames, 'rownames': rownames }, env='Test') # Evaluate for one epoch on validation set # model.train() model.eval() train_metrics, train_confusion_meter = evaluate(model, loss_fn, train_dataloader, metrics, params, logger) train_confusion_logger.log(train_confusion_meter.value()) model.eval() val_metrics, test_confusion_meter = evaluate(model, loss_fn, val_dataloader, metrics, params, logger) test_confusion_logger.log(test_confusion_meter.value()) diff_confusion_meter = train_confusion_meter.value( ) - test_confusion_meter.value() diff_confusion_logger.log(diff_confusion_meter) pass
# x1.grad += args.lambda_ * x1_grad # x2.grad += args.lambda_ * x2_grad optimizer.step() meter_accuracy.add(out.data, labels.data) meter_loss.add(loss.data[0]) pbar.set_postfix(loss=meter_loss.value()[0], acc=meter_accuracy.value()[0]) pbar.update() loss = meter_loss.value()[0] acc = meter_accuracy.value()[0] if epoch == 0: setting_logger.log(str(args)) train_loss_logger.log(epoch, loss) train_error_logger.log(epoch, acc) print("\nEpoch{} Train acc:{:4}, loss:{:4}".format( epoch, acc, loss)) scheduler.step(loss) torch.save(model.state_dict(), weight_folder + "/model_{}.pth".format(epoch)) reset_meters() # Test print('Testing...') model.eval() for i, data in enumerate(test_loader):
def train(): end = time.time() batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() env_name = args.experiment_id img_a_logger = VisdomLogger('images', env=env_name, port=8097, opts=dict(title='img_a')) img_b_logger = VisdomLogger('images', env=env_name, port=8097, opts=dict(title='img_b')) rnd_logger = [VisdomLogger('images', env=env_name, port=8097, opts=dict(title='rnd_rgb_{}'.format(_i))) for _i in range(4)] for i in range(start_iter, args.num_iterations + 1): # adjust learning rate and sigma_val (decay after 150k iter) lr = adjust_learning_rate([optimizer], args.learning_rate, i, method=args.lr_type) model.set_sigma(adjust_sigma(args.sigma_val, i)) # load images from multi-view images_a, images_b, viewpoints_a, viewpoints_b = dataset_train.get_random_batch(args.batch_size) images_a = images_a.cuda() images_b = images_b.cuda() viewpoints_a = viewpoints_a.cuda() viewpoints_b = viewpoints_b.cuda() # soft render images render_images, laplacian_loss, flatten_loss = model([images_a, images_b], [viewpoints_a, viewpoints_b], task='train') img_a_logger.log(F.interpolate(images_a.detach().clone(), scale_factor=4).squeeze()[:,:3]) img_b_logger.log(F.interpolate(images_b.detach().clone(), scale_factor=4).squeeze()[:,:3]) for _i in range(4): rnd_logger[_i].log(F.interpolate(render_images[_i].detach().clone(), scale_factor=4).squeeze()[:,:3]) import ipdb; ipdb.set_trace() laplacian_loss = laplacian_loss.mean() flatten_loss = flatten_loss.mean() # compute loss loss = multiview_iou_loss(render_images, images_a, images_b) + \ args.lambda_laplacian * laplacian_loss + \ args.lambda_flatten * flatten_loss losses.update(loss.data.item(), images_a.size(0)) # compute gradient and optimize optimizer.zero_grad() loss.backward() optimizer.step() batch_time.update(time.time() - end) end = time.time() # save checkpoint if i % args.save_freq == 0: model_path = os.path.join(directory_output, 'checkpoint_%07d.pth.tar'%i) torch.save({ 'model': model.state_dict(), 'optimizer': optimizer.state_dict(), }, model_path) # save demo images if i % args.demo_freq == 0: demo_image = images_a[0:1] demo_path = os.path.join(directory_output, 'demo_%07d.obj'%i) demo_v, demo_f = model.reconstruct(demo_image) srf.save_obj(demo_path, demo_v[0], demo_f[0]) imageio.imsave(os.path.join(image_output, '%07d_fake.png' % i), img_cvt(render_images[0][0])) imageio.imsave(os.path.join(image_output, '%07d_input.png' % i), img_cvt(images_a[0])) # print if i % args.print_freq == 0: print('Iter: [{0}/{1}]\t' 'Time {batch_time.val:.3f}\t' 'Loss {loss.val:.3f}\t' 'lr {lr:.6f}\t' 'sv {sv:.6f}\t'.format(i, args.num_iterations, batch_time=batch_time, loss=losses, lr=lr, sv=model.renderer.rasterizer.sigma_val))
train_loss_logger.log(epoch + epoch_offset, loss - loss_recon, name='loss') test_loss_logger.log(epoch + epoch_offset, test_loss, name='loss') """ loss_relation = loss_recon/(loss-loss_recon) if loss_relation > 0.25 and epoch>15: fac = 0.25/loss_relation print("Loss relation = {}. Recon-factor reduced from {} to {}".format(loss_relation, args.recon_factor, args.recon_factor*fac)) args.recon_factor *= fac """ if not args.disable_recon: ground_truth_logger_left.log( make_grid(imgs, nrow=int(args.batch_size**0.5), normalize=True, range=(0, 1)).cpu().numpy()) reconstruction_logger_left.log( make_grid(recon.data, nrow=int(args.batch_size**0.5), normalize=True, range=(0, 1)).cpu().numpy()) train_loss_logger.log(epoch + epoch_offset, loss_recon, name='recon') test_loss_logger.log(epoch + epoch_offset, test_loss_recon, name='recon') with open("loss.log", "a") as myfile:
class statBase(): def __init__(self, args): self.args = args self.lossAvg = tnt.meter.AverageValueMeter() #self.lossSparseMu = tnt.meter.AverageValueMeter() #self.lossSparseVar = tnt.meter.AverageValueMeter() self.train_loss_logger = VisdomPlotLogger('line', opts={'title': 'Train Loss'}, env='PoseCapsules') self.test_loss_logger = VisdomPlotLogger('line', opts={'title': 'Test Loss'}, env='PoseCapsules') self.recon_sum = 0 self.rout_id = 1 if not self.args.disable_recon: self.reconLossAvg = tnt.meter.AverageValueMeter() self.ground_truth_logger_left = VisdomLogger('image', opts={'title': 'Ground Truth, left'}, env='PoseCapsules') self.reconstruction_logger_left = VisdomLogger('image', opts={'title': 'Reconstruction, left'}, env='PoseCapsules') if self.args.regularize: self.regularizeLossAvg = tnt.meter.AverageValueMeter() self.logsigAvg = tnt.meter.AverageValueMeter() self.costmeanAvg = tnt.meter.AverageValueMeter() self.costAvg = tnt.meter.AverageValueMeter() self.aAvg = tnt.meter.AverageValueMeter() def reset(self): self.lossAvg.reset() if not self.args.disable_recon: self.reconLossAvg.reset() if self.args.regularize: self.regularizeLossAvg.reset() self.logsigAvg.reset() self.costmeanAvg.reset() self.costAvg.reset() self.aAvg.reset() def log(self, pbar, output, labels, dict = OrderedDict(), stat=None): if not self.args.disable_loss: dict['loss'] = self.lossAvg.value()[0] if stat is not None: self.logsigAvg.add(stat[-self.rout_id*4 + 0]) self.costmeanAvg.add(stat[-self.rout_id*4 + 1]) self.costAvg.add(stat[-self.rout_id*4 + 2]) self.aAvg.add(stat[-self.rout_id*4 + 3]) stat.clear() dict['logsig'] = self.logsigAvg.value()[0] dict['costmean'] = self.costmeanAvg.value()[0] dict['cost'] = self.costAvg.value()[0] dict['a'] = self.aAvg.value()[0] #dict['mloss'] = self.lossSparseMu.value()[0] #dict['vloss'] = self.lossSparseVar.value()[0] if not self.args.disable_recon: #pbar.set_postfix(loss=self.lossAvg.value()[0], refresh=False) #else: dict['reconloss'] = self.reconLossAvg.value()[0] dict['rsum'] = self.recon_sum #pbar.set_postfix(loss=self.lossAvg.value()[0], rloss=self.reconLossAvg.value()[0], rsum=self.recon_sum, refresh=False) if self.args.regularize: dict['reguloss'] = self.regularizeLossAvg.value()[0] pbar.set_postfix(dict, refresh=False) def endTrainLog(self, epoch, groundtruth_image=None, recon_image=None): #self.train_loss = self.lossAvg.value()[0] if not self.args.disable_loss: self.train_loss_logger.log(epoch, self.lossAvg.value()[0], name='loss') with open("train.log", "a") as myfile: myfile.write(str(self.lossAvg.value()[0]) + '\n') if not self.args.disable_recon: if groundtruth_image is not None: self.ground_truth_logger_left.log(make_grid(groundtruth_image, nrow=int(self.args.batch_size ** 0.5), normalize=True, range=(0, 1)).cpu().numpy()) if recon_image is not None: self.reconstruction_logger_left.log(make_grid(recon_image.data, nrow=int(self.args.batch_size ** 0.5), normalize=True, range=(0, 1)).cpu().numpy()) #self.train_recon_loss = self.reconLossAvg.value()[0] self.train_loss_logger.log(epoch, self.reconLossAvg.value()[0], name='recon') #if self.args.regularize: # self.train_regularize_loss = self.regularizeLossAvg.value()[0] def endTestLog(self, epoch): #loss = self.lossAvg.value()[0] if not self.args.disable_loss: self.test_loss_logger.log(epoch, self.lossAvg.value()[0], name='loss') with open("test.log", "a") as myfile: myfile.write(str(self.lossAvg.value()[0]) + '\n') if not self.args.disable_recon: self.test_loss_logger.log(epoch, self.reconLossAvg.value()[0], name='recon') def load_loss(self, history_count): if os.path.isfile('train.log'): with open("train.log", "r") as lossfile: loss_list = [] for loss in lossfile: loss_list.append(loss) while len(loss_list) > history_count: loss_list.pop(0) epoch = -len(loss_list) for loss in loss_list: self.train_loss_logger.log(epoch, float(loss), name='loss') epoch += 1 if os.path.isfile('test.log'): with open("test.log", "r") as lossfile: loss_list = [] for loss in lossfile: loss_list.append(loss) while len(loss_list) > history_count: loss_list.pop(0) epoch = -len(loss_list) for loss in loss_list: self.test_loss_logger.log(epoch, float(loss), name='loss') epoch += 1
class_criterion_B, train_loader, test_loader, opt) if opt.pretrained: trained_model = load_weight(fullModel, opt.pretrained, verbose=True) # -- Evaluation nTestImages = reid_set.test_inds # [2 ** (n+1) for n in range(5)] cmc, simMat, _, avgSame, avgDiff = compute_cmc(reid_set, nTestImages, trained_model, 128) print(cmc) print(simMat) print(avgSame, avgDiff) sim_logger = VisdomLogger('heatmap', port=8097, opts={ 'title': 'simMat', 'columnnames': list(range(len(simMat[0]))), 'rownames': list(range(len(simMat))) }) cmc_logger = VisdomPlotLogger("line", win="cmc_curve") for i, v in enumerate(cmc): cmc_logger.log(i, v, name="cmc_curve") sim_logger.log(simMat) log.info("Saving results...") with open("cmc.pkl", 'w') as f: pickle.dump(cmc, f) with open("simMat.pkl", 'w') as f: pickle.dump(simMat, f)
def test_only(model, train_dataloader, val_dataloader, optimizer, loss_fn, metrics, params, model_dir, logger, restore_file=None): # reload weights from restore_file if specified if restore_file is not None: logging.info("Restoring parameters from {}".format(restore_file)) checkpoint = utils.load_checkpoint(restore_file, model, optimizer) # checkpoint = utils.load_checkpoint(restore_file, model, optimizer) best_val_acc = checkpoint['best_val_acc'] params.current_epoch = checkpoint['epoch'] print('best_val_acc=', best_val_acc) print(optimizer.state_dict()['param_groups'][0]['lr'], checkpoint['epoch']) train_confusion_logger = VisdomLogger('heatmap', port=port, opts={ 'title': params.experiment_path + 'train_Confusion matrix', 'columnnames': columnnames, 'rownames': rownames }, env='Test') test_confusion_logger = VisdomLogger('heatmap', port=port, opts={ 'title': params.experiment_path + 'test_Confusion matrix', 'columnnames': columnnames, 'rownames': rownames }, env='Test') diff_confusion_logger = VisdomLogger('heatmap', port=port, opts={ 'title': params.experiment_path + 'diff_Confusion matrix', 'columnnames': columnnames, 'rownames': rownames }, env='Test') # Evaluate for one epoch on validation set # model.train() model.eval() train_metrics, train_confusion_meter, train_middle_val, train_middle_label = evaluate( model, loss_fn, train_dataloader, metrics, params, logger) train_confusion_logger.log(train_confusion_meter.value()) model.eval() val_metrics, test_confusion_meter, test_middle_val, test_middle_label = evaluate( model, loss_fn, val_dataloader, metrics, params, logger) test_confusion_logger.log(test_confusion_meter.value()) diff_confusion_meter = train_confusion_meter.value( ) - test_confusion_meter.value() diff_confusion_logger.log(diff_confusion_meter) #---------------------------DWnet Testing------------------------------------------ lr = .001 C = 2**(-30) s = 0.7 N2 = 1 # Number of windows of feature nodes N3 = 7250 # Number of enhancement nodes epochs = 10 # Number of epochs en_list = [] test_acc_list = [] test_time_list = [] #FeaturesPerWindow = [] train_label = train_middle_label[0].reshape([64, 1]) print('train_label:', train_label.shape, train_middle_label[1].shape) for i in range(len(train_middle_label) - 1): train_label = np.vstack( (train_label, np.reshape(train_middle_label[i + 1], [-1, 1]))) print('Shape of train label is:', train_label.shape) np.savetxt('DWnet_result/train_label.txt', train_label) Y = np.zeros([train_label.shape[0], 60]) for i in range(train_label.shape[0]): Y[i, train_label[i]] = 1 np.savetxt('DWnet_result/Y.txt', Y) test_label = np.reshape(test_middle_label[0], [64, 1]) for i in range(len(test_middle_label) - 1): test_label = np.vstack( (test_label, np.reshape(test_middle_label[i + 1], [-1, 1]))) print('Shape of test label is:', test_label.shape) #print(train_middle_val[0]) np.savetxt('DWnet_result/test_label.txt', test_label) Y_TEST = np.zeros([test_label.shape[0], 60]) for i in range(test_label.shape[0]): Y_TEST[i, test_label[i]] = 1 np.savetxt('DWnet_result/Y_TEST.txt', Y_TEST) epoch = 10 for en in range(epoch): print('Enhancement epochs at: ', en, '; The number of enhancement nodes is: ', N3) # N3 = N3 + 50 FeaturesPerWindow = [] train_start_time = time.time() for i in range(N2): print(type(train_middle_val), len(train_middle_val), train_middle_val[0].shape) train_val = train_middle_val[0] for elem in range(len(train_middle_val) - 1): train_val = np.vstack((train_val, train_middle_val[elem + 1])) # print(train_val) print('Train_val shape is:', train_val.shape) FeaturesExtract = train_val print('FeaturesExtract shape is: ', FeaturesExtract.shape) FeaturesPerWindow.append(FeaturesExtract) N1 = FeaturesPerWindow[0].shape[1] FeaturesPerWindow = np.array(FeaturesPerWindow) print('Train features shape is: ', FeaturesPerWindow.shape) FeaturesPerWindow = np.reshape( FeaturesPerWindow, newshape=[-1, FeaturesPerWindow[0].shape[1] * N2]) Features_bias = np.hstack([ FeaturesPerWindow, 0.1 * np.ones(((FeaturesPerWindow.shape[0]), 1)) ]) if N1 * N2 >= N3: # random.seed(67797325) random.seed(int(time.time())) wh = La.orth(2 * rnd.randn(N2 * N1 + 1, N3)) - 1 else: # random.seed(67797325) random.seed(int(time.time())) wh = La.orth((2 * rnd.randn(N2 * N1 + 1, N3).T) - 1).T print(wh) print(wh.shape) EnhancementFeature = np.dot(Features_bias, wh) print(EnhancementFeature.shape) l2 = s / np.max(EnhancementFeature) EnhancementFeature = tansig(EnhancementFeature * l2) FeatureLayerOutput = np.hstack([FeaturesPerWindow, EnhancementFeature]) beta = pinv(FeatureLayerOutput, C) print(FeatureLayerOutput.shape) print(beta.shape) OutputWeights = np.dot(beta, Y) print(OutputWeights.shape) xx = np.dot(FeatureLayerOutput, OutputWeights) TrainingAccuracy = show_accuracy(xx, Y) train_end_time = time.time() train_time = train_end_time - train_start_time print('Training time is' + str(train_time) + 's') print('Training accurate is', TrainingAccuracy * 100, '%') '''Testing Process''' test_start_time = time.time() TestFeatures = [] for j in range(N2): test_val = test_middle_val[0] for elem in range(len(test_middle_val) - 1): test_val = np.vstack((test_val, test_middle_val[elem + 1])) test_pred = test_val TestFeatures.append(test_pred) TestFeatures = np.array(TestFeatures) TestFeatures = np.reshape(TestFeatures, newshape=[-1, TestFeatures[0].shape[1] * N2]) print(TestFeatures.shape) '''Test Enhancement Nodes''' TestFeaturesBias = np.hstack( [TestFeatures, 0.1 * np.ones(((TestFeatures.shape[0]), 1))]) TestEnhancementFeatures = np.dot(TestFeaturesBias, wh) TestEnhancementFeatures = tansig(TestEnhancementFeatures * l2) FinalFeatures = np.hstack([TestFeatures, TestEnhancementFeatures]) TestOutput = np.dot(FinalFeatures, OutputWeights) TestingAccuracy = show_accuracy(TestOutput, Y_TEST) test_end_time = time.time() test_time = test_end_time - test_start_time np.savetxt('./DWnet_result/pred.txt', TestOutput) np.savetxt('./DWnet_result/label.txt', Y_TEST) print('Testing time is' + str(test_time) + 's') print('Testing accurate is', TestingAccuracy * 100, '%') en_list.append(N3) test_time_list.append(test_time) test_acc_list.append(TestingAccuracy) # if (en+1) % 50 == 0: np.savetxt('params/enhance_' + str(en + 1) + '.txt', en_list) np.savetxt('params/test_time_' + str(en + 1) + '.txt', test_time_list) np.savetxt('params/test_acc_' + str(en + 1) + '.txt', test_acc_list) pass
def train_valid_loop(train_loader, dev_loader, test_loader, args, model, fold=None): # -------------------------------------------------------------------------- # TRAIN/VALID LOOP logger.info('-' * 100) stats = { 'timer': utils.Timer(), 'epoch': 0, 'best_valid': 0, 'best_epoch': 0, 'fold': fold } start_epoch = 0 if args.visdom: # add visdom logger code port = args.visdom_port train_loss_logger = VisdomPlotLogger( 'line', port=port, opts={'title': f'{args.model_name} Train Loss'}) train_metric_logger = VisdomPlotLogger( 'line', port=port, opts={'title': f'{args.model_name} Train Class Accuracy'}) idx2label = {i: label for label, i in model.label_dict.items()} label_names = [idx2label[i] for i in range(model.args.label_size)] train_confusion_logger = VisdomLogger( 'heatmap', port=port, opts={ 'title': f'{args.model_name} Train Confusion Matrix', 'columnnames': label_names, 'rownames': label_names }) valid_metric_logger = VisdomPlotLogger( 'line', port=port, opts={'title': f'{args.model_name} Valid Class Accuracy'}) valid_confusion_logger = VisdomLogger( 'heatmap', port=port, opts={ 'title': f'{args.model_name} Valid Confusion Matrix', 'columnnames': label_names, 'rownames': label_names }) train_confusion_meter = tnt.meter.ConfusionMeter(model.args.label_size, normalized=True) valid_confusion_meter = tnt.meter.ConfusionMeter(model.args.label_size, normalized=True) else: train_confusion_meter = None valid_confusion_meter = None try: for epoch in range(start_epoch, args.num_epochs): stats['epoch'] = epoch # Train loss = train(args, train_loader, model, stats) stats['train_loss'] = loss # Validate train train_res, train_cfm = validate( args, train_loader, model, stats, mode='train', confusion_meter=train_confusion_meter) for m in train_res: stats['train_' + m] = train_res[m] # Validate dev val_res, valid_cfm = validate( args, dev_loader, model, stats, mode='dev', confusion_meter=valid_confusion_meter) for m in train_res: stats['dev_' + m] = val_res[m] if args.visdom: train_loss_logger.log(epoch, loss) train_metric_logger.log(epoch, train_res[args.valid_metric]) train_confusion_logger.log(train_cfm) valid_metric_logger.log(epoch, val_res[args.valid_metric]) valid_confusion_logger.log(valid_cfm) train_confusion_meter.reset() valid_confusion_meter.reset() # Save best valid if val_res[args.valid_metric] > stats['best_valid']: logger.info( colored( f'Best valid: {args.valid_metric} = {val_res[args.valid_metric]*100:.2f}% ', 'yellow') + colored( f'(epoch {stats["epoch"]}, {model.updates} updates)', 'yellow')) fold_info = f'.fold_{fold}' if fold is not None else '' model.save(args.model_file + fold_info) stats['best_valid'] = val_res[args.valid_metric] stats['best_epoch'] = epoch logger.info('-' * 100) if args.stats_file: with open(args.stats_file, 'w') as f: out_stats = stats.copy() out_stats['timer'] = out_stats['timer'].time() if fold is None: del out_stats['fold'] f.write(json.dumps(out_stats) + '\n') if epoch - stats['best_epoch'] >= args.early_stopping: logger.info( colored( f'No improvement for {args.early_stopping} epochs, stop training.', 'red')) break except KeyboardInterrupt: logger.info(colored(f'User ended training. stop.', 'red')) logger.info('Load best model...') model = EntityClassifier.load(args.model_file + fold_info, args) # device = torch.device(f"cuda:{args.gpu}" if args.cuda else "cpu") # model.to(device) model.cuda() stats['epoch'] = stats['best_epoch'] if fold is not None: mode = f'fold {fold} test' else: mode = 'test' test_result, _ = validate(args, test_loader, model, stats, mode=mode) return test_result
def test_only(model, train_dataloader, val_dataloader, optimizer, loss_fn, metrics, params, model_dir, logger, restore_file=None): # reload weights from restore_file if specified if restore_file is not None: logging.info("Restoring parameters from {}".format(restore_file)) checkpoint = utils.load_checkpoint(restore_file, model, optimizer) best_val_acc = checkpoint['best_val_acc'] params.current_epoch = checkpoint['epoch'] print('best_val_acc=', best_val_acc) print(optimizer.state_dict()['param_groups'][0]['lr'], checkpoint['epoch']) train_confusion_logger = VisdomLogger('heatmap', port=port, opts={ 'title': params.experiment_path + 'train_Confusion matrix', 'columnnames': columnnames, 'rownames': rownames }, env='Test') test_confusion_logger = VisdomLogger('heatmap', port=port, opts={ 'title': params.experiment_path + 'test_Confusion matrix', 'columnnames': columnnames, 'rownames': rownames }, env='Test') diff_confusion_logger = VisdomLogger('heatmap', port=port, opts={ 'title': params.experiment_path + 'diff_Confusion matrix', 'columnnames': columnnames, 'rownames': rownames }, env='Test') # Evaluate for one epoch on validation set # model.train() model.eval() train_metrics, train_confusion_meter, train_saved_feature = evaluate( model, loss_fn, train_dataloader, metrics, params, logger) train_confusion_logger.log(train_confusion_meter.value()) train_saved_feature_path = os.path.join( feature_path, params.data_type1 + '_' + params.data_type2 + "_train.pkl") with open(train_saved_feature_path, 'wb') as f: pickle.dump(train_saved_feature, f) model.eval() val_metrics, test_confusion_meter, test_saved_feature = evaluate( model, loss_fn, val_dataloader, metrics, params, logger) test_confusion_logger.log(test_confusion_meter.value()) diff_confusion_meter = train_confusion_meter.value( ) - test_confusion_meter.value() diff_confusion_logger.log(diff_confusion_meter) val_saved_feature_path = os.path.join( feature_path, params.data_type1 + '_' + params.data_type2 + "_val.pkl") with open(val_saved_feature_path, 'wb') as f: pickle.dump(test_saved_feature, f) # save the AIF metrics print('save the AIF metrics ing--------------------') AIF1 = model.adat_metrices1_c.data.cpu().numpy() with open('saved_KI_AIF.pkl', 'wb') as f: pickle.dump(AIF1, f) AIF_r1 = model.adat_metrices_r1.data.cpu().numpy() with open('saved_KI_AIF_lay_r1.pkl', 'wb') as f: pickle.dump(AIF_r1, f) AIF_r2 = model.adat_metrices3.data.cpu().numpy() with open('saved_KI_AIF_lay_r2.pkl', 'wb') as f: pickle.dump(AIF_r2, f) print('save the AIF metrics end--------------------') pass