def do_crf_inference(image, annotation, t=5, use_gpu=False): annotation = np.expand_dims(annotation, axis=-1) unary = np.concatenate([annotation, 1 - annotation], axis=-1) num_classes = unary.shape[2] shape = image.shape[0:2] # make input pytorch compatible img_var = torch.tensor( image.transpose(2, 0, 1).reshape([1, 3, shape[0], shape[1]])) unary_var = torch.tensor( unary.transpose(2, 0, 1).reshape([1, num_classes, shape[0], shape[1]])) img_var = img_var.cuda() if use_gpu else img_var unary_var = unary_var.cuda() if use_gpu else unary_var # Create CRF module gausscrf = convcrf.GaussCRF(conf=convcrf.default_conf, shape=shape, nclasses=num_classes, use_gpu=use_gpu) gausscrf = gausscrf.cuda() if use_gpu else gausscrf # Perform CRF inference prediction = gausscrf.forward(unary=unary_var, img=img_var, num_iter=t) prediction = prediction.data.cpu().numpy()[0] return np.argmax(prediction, axis=0)
def do_crf_inference(image, unary, speed_eval, pyinn=False): if pyinn or not hasattr(torch.nn.functional, 'unfold'): # pytorch 0.3 or older requires pyinn. pyinn = True # Cheap and easy trick to make sure that pyinn is loadable. import pyinn # get basic hyperparameters num_classes = unary.shape[2] shape = image.shape[0:2] config = convcrf.default_conf config['filter_size'] = 7 config['pyinn'] = pyinn ## # make input pytorch compatible image = image.transpose(2, 0, 1) # shape: [3, hight, width] # Add batch dimension to image: [1, 3, height, width] image = image.reshape([1, 3, shape[0], shape[1]]) img_var = Variable(torch.Tensor(image)).cuda() unary = unary.transpose(2, 0, 1) # shape: [3, hight, width] # Add batch dimension to unary: [1, 21, height, width] unary = unary.reshape([1, num_classes, shape[0], shape[1]]) unary_var = Variable(torch.Tensor(unary)).cuda() logging.info("Build ConvCRF.") ## # Create CRF module gausscrf = convcrf.GaussCRF(conf=config, shape=shape, nclasses=num_classes) # Cuda computation is required. # A CPU implementation of our message passing is not provided. gausscrf.cuda() logging.info("Start Computation.") # Perform CRF inference prediction = gausscrf.forward(unary=unary_var, img=img_var) if speed_eval: # Evaluate inference speed logging.info("Doing speed evaluation.") start_time = time.time() for i in range(10): # Running ConvCRF 10 times and average total time pred = gausscrf.forward(unary=unary_var, img=img_var) pred.cpu() # wait for all GPU computations to finish duration = (time.time() - start_time) * 1000 / 10 logging.info("Finished running 10 predictions.") logging.info("Avg. Computation time: {} ms".format(duration)) return prediction.data.cpu().numpy()
def do_crf_inference(image, args): if args.pyinn or not hasattr(torch.nn.functional, 'unfold'): # pytorch 0.3 or older requires pyinn. args.pyinn = True # Cheap and easy trick to make sure that pyinn is loadable. import pyinn # get basic hyperparameters #num_classes = unary.shape[2] num_classes = 21 shape = image.shape[0:2] config = convcrf.default_conf config['filter_size'] = 7 config['pyinn'] = args.pyinn if args.normalize: # Warning, applying image normalization affects CRF computation. # The parameter 'col_feats::schan' needs to be adapted. # Normalize image range # This changes the image features and influences CRF output image = image / 255 # mean substraction # CRF is invariant to mean subtraction, output is NOT affected image = image - 0.5 # std normalization # Affect CRF computation image = image / 0.3 # schan = 0.1 is a good starting value for normalized images. # The relation is f_i = image * schan config['col_feats']['schan'] = 0.1 # make input pytorch compatible image = image.transpose(2, 0, 1) # shape: [3, hight, width] # Add batch dimension to image: [1, 3, height, width] image = image.reshape([1, 3, shape[0], shape[1]]) img_var = Variable(torch.Tensor(image)) #.cuda() #unary = unary.transpose(2, 0, 1) # shape: [3, hight, width] # Add batch dimension to unary: [1, 21, height, width] #unary = unary.reshape([1, num_classes, shape[0], shape[1]]) #unary_var = Variable(torch.Tensor(unary))# .cuda() logging.info("Build ConvCRF.") ## # Create CRF module gausscrf = convcrf.GaussCRF(conf=config, shape=shape, nclasses=num_classes) # Cuda computation is required. # A CPU implementation of our message passing is not provided. #gausscrf()#.cuda() logging.info("Start Computation.") # Perform CRF inference #prediction = gausscrf.forward(unary=unary_var, img=img_var) if args.nospeed: # Evaluate inference speed logging.info("Doing speed evaluation.") start_time = time.time() #for i in range(10): ## Running ConvCRF 10 times and average total time #pred = gausscrf.forward(unary=unary_var, img=img_var) #pred.cpu() # wait for all GPU computations to finish duration = (time.time() - start_time) * 1000 / 10 logging.info("Finished running 10 predictions.") logging.info("Avg. Computation time: {} ms".format(duration))
def do_crf_inference(image, unary, args): if args.pyinn or not hasattr(torch.nn.functional, 'unfold'): # pytorch 0.3 or older requires pyinn. args.pyinn = True # Cheap and easy trick to make sure that pyinn is loadable. import pyinn # get basic hyperparameters num_classes = unary.shape[2] shape = image.shape[0:2] config = convcrf.default_conf config['filter_size'] = 7 config['pyinn'] = args.pyinn if args.normalize: # Warning, applying image normalization affects CRF computation. # The parameter 'col_feats::schan' needs to be adapted. # Normalize image range # This changes the image features and influences CRF output image = image / 255 # mean substraction # CRF is invariant to mean subtraction, output is NOT affected image = image - 0.5 # std normalization # Affect CRF computation image = image / 0.3 # schan = 0.1 is a good starting value for normalized images. # The relation is f_i = image / schan config['col_feats']['schan'] = 0.1 # make input pytorch compatible img = image.transpose(2, 0, 1) # shape: [3, hight, width] # Add batch dimension to image: [1, 3, height, width] img = img.reshape([1, 3, shape[0], shape[1]]) img_var = Variable(torch.Tensor(img)).cuda() un = unary.transpose(2, 0, 1) # shape: [3, hight, width] # Add batch dimension to unary: [1, 21, height, width] un = un.reshape([1, num_classes, shape[0], shape[1]]) unary_var = Variable(torch.Tensor(un)).cuda() logging.debug("Build ConvCRF.") ## # Create CRF module gausscrf = convcrf.GaussCRF(conf=config, shape=shape, nclasses=num_classes) # Cuda computation is required. # A CPU implementation of our message passing is not provided. gausscrf.cuda() # Perform ConvCRF inference """ 'Warm up': Our implementation compiles cuda kernels during runtime. The first inference call thus comes with some overhead. """ logging.info("Start Computation.") prediction = gausscrf.forward(unary=unary_var, img=img_var) if args.nospeed: logging.info("Doing speed benchmark with filter size: {}".format( config['filter_size'])) logging.info("Running multiple iteration. This may take a while.") # Our implementation compiles cuda kernels during runtime. # The first inference run is those much slower. # prediction = gausscrf.forward(unary=unary_var, img=img_var) start_time = time.time() for i in range(10): # Running ConvCRF 10 times and report average total time prediction = gausscrf.forward(unary=unary_var, img=img_var) prediction.cpu() # wait for all GPU computations to finish duration = (time.time() - start_time) * 1000 / 10 logging.debug("Finished running 10 predictions.") logging.debug("Avg Computation time: {} ms".format(duration)) # Perform FullCRF inference myfullcrf = fullcrf.FullCRF(config, shape, num_classes) fullprediction = myfullcrf.compute(unary, image, softmax=False) if args.nospeed: start_time = time.time() for i in range(5): # Running FullCRF 5 times and report average total time fullprediction = myfullcrf.compute(unary, image, softmax=False) fullduration = (time.time() - start_time) * 1000 / 5 logging.debug("Finished running 5 predictions.") logging.debug("Avg Computation time: {} ms".format(fullduration)) logging.info("Using FullCRF took {:4.0f} ms ({:2.2f} s)".format( fullduration, fullduration / 1000)) logging.info("Using ConvCRF took {:4.0f} ms ({:2.2f} s)".format( duration, duration / 1000)) logging.info("Congratulation. Using ConvCRF provids a speed-up" " of {:.0f}.".format(fullduration / duration)) logging.info("") return prediction.data.cpu().numpy(), fullprediction
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument('-g', '--gpu', type=int, required=True, help='gpu id') parser.add_argument('--resume', help='checkpoint path') # configurations (same configuration as original work) # https://github.com/shelhamer/fcn.berkeleyvision.org parser.add_argument('--max-iteration', type=int, default=25, help='max iteration') parser.add_argument( '--lr', type=float, default=1.0e-7, help='learning rate', ) parser.add_argument( '--weight-decay', type=float, default=0.0005, help='weight decay', ) parser.add_argument( '--momentum', type=float, default=0.99, help='momentum', ) args = parser.parse_args() args.model = 'FCN8sAtOnce' args.git_hash = git_hash() now = datetime.datetime.now() args.out = osp.join(here, 'logs', now.strftime('%Y%m%d_%H%M%S.%f')) os.makedirs(args.out) with open(osp.join(args.out, 'config.yaml'), 'w') as f: yaml.safe_dump(args.__dict__, f, default_flow_style=False) os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu) cuda = torch.cuda.is_available() torch.manual_seed(1337) if cuda: torch.cuda.manual_seed(1337) # 1. dataset root = osp.expanduser('~/data/datasets') kwargs = {'num_workers': 4, 'pin_memory': True} if cuda else {} train_loader = torch.utils.data.DataLoader(torchfcn.datasets.DAVISClassSeg( root, split='train', transform=True), batch_size=100, shuffle=True, **kwargs) val_loader = torch.utils.data.DataLoader(torchfcn.datasets.DAVISClassSeg( root, split='val', transform=True), batch_size=100, shuffle=False, **kwargs) # 2. model config = convcrf.default_conf config['filter_size'] = 5 config['pyinn'] = False config['trainable'] = True logging.info("Build ConvCRF.") ## # Create CRF module model = convcrf.GaussCRF(conf=config, shape=[224, 224], nclasses=2) start_epoch = 0 start_iteration = 0 if args.resume: checkpoint = torch.load(args.resume) model.load_state_dict(checkpoint['model_state_dict']) start_epoch = checkpoint['epoch'] start_iteration = checkpoint['iteration'] if cuda: model = model.cuda() # for m in model.modules(): # print(dir(m)) # for p in model.parameters(): # print(p.name) optim = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) if args.resume: optim.load_state_dict(checkpoint['optim_state_dict']) trainer = crftrainer.Trainer( cuda=cuda, model=model, optimizer=optim, train_loader=train_loader, val_loader=val_loader, out=args.out, max_iter=args.max_iteration, ) trainer.epoch = start_epoch trainer.iteration = start_iteration trainer.train()
def __init__(self, args): self.args = args # Define Saver self.saver = Saver(args) self.saver.save_experiment_config() # Define Tensorboard Summary self.summary = TensorboardSummary(self.saver.experiment_dir) self.writer = self.summary.create_summary() # Define Dataloader kwargs = {'num_workers': args.workers, 'pin_memory': True} self.train_loader, self.val_loader, self.test_loader, self.nclass = make_data_loader( args, **kwargs) # Define network model = DeepLab(num_classes=self.nclass, backbone=args.backbone, output_stride=args.out_stride, sync_bn=args.sync_bn, freeze_bn=args.freeze_bn, crop_size=args.crop_size, crf_loss=args.crf_loss) train_params = [{ 'params': model.get_1x_lr_params(), 'lr': args.lr }, { 'params': model.get_10x_lr_params(), 'lr': args.lr * 10 }] print(sum(p.numel() for p in model.parameters() if p.requires_grad)) # Define Optimizer optimizer = torch.optim.SGD(train_params, momentum=args.momentum, weight_decay=args.weight_decay, nesterov=args.nesterov) # Define Criterion # whether to use class balanced weights if args.use_balanced_weights: classes_weights_path = os.path.join( Path.db_root_dir(args.dataset), args.dataset + '_classes_weights.npy') if os.path.isfile(classes_weights_path): weight = np.load(classes_weights_path) else: weight = calculate_weigths_labels(args.dataset, self.train_loader, self.nclass) np.save( os.path.join(Path.db_root_dir(args.dataset), args.dataset + '_classes_weights.npy'), weight) weight = torch.from_numpy(weight.astype(np.float32)) else: weight = None self.criterion = SegmentationLosses( weight=weight, cuda=args.cuda, ignore_index=args.ignore_index).build_loss(mode=args.loss_type) self.model, self.optimizer = model, optimizer # Define Evaluator self.evaluator = Evaluator(self.nclass) # Define lr scheduler self.scheduler = LR_Scheduler(args.lr_scheduler, args.lr, args.epochs, len(self.train_loader)) # Using cuda if args.cuda: self.model = torch.nn.DataParallel(self.model, device_ids=self.args.gpu_ids) patch_replication_callback(self.model) self.model = self.model.cuda() if args.crf_loss: _shape = int(args.crop_size / 4) if args.crop_size / 4 != int(args.crop_size / 4): _shape += 1 config = convcrf.default_conf config['filter_size'] = 7 config['col_feats']['schan'] = 0.1 config['trainable'] = True self.convcrf = convcrf.GaussCRF(conf=config, shape=(_shape, _shape), nclasses=self.nclass) self.convcrf.cuda() # Resuming checkpoint self.best_pred = 0.0 if args.resume is not None: if not os.path.isfile(args.resume): raise RuntimeError("=> no checkpoint found at '{}'".format( args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] if args.cuda: self.model.module.load_state_dict(checkpoint['state_dict']) else: self.model.load_state_dict(checkpoint['state_dict']) if not args.ft: self.optimizer.load_state_dict(checkpoint['optimizer']) self.best_pred = checkpoint['best_pred'] print("=> loaded checkpoint '{}' (epoch {})".format( args.resume, checkpoint['epoch'])) # Clear start epoch if fine-tuning if args.ft: args.start_epoch = 0
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument('-g', '--gpu', type=int, required=True, help='gpu id') parser.add_argument('--resume', help='checkpoint path') # configurations (same configuration as original work) # https://github.com/shelhamer/fcn.berkeleyvision.org parser.add_argument( '--max-iteration', type=int, default=3000, help='max iteration' ) parser.add_argument( '--lr', type=float, default=1.0e-4, help='learning rate', ) parser.add_argument( '--weight-decay', type=float, default=0.0005, help='weight decay', ) parser.add_argument( '--momentum', type=float, default=0.99, help='momentum', ) args = parser.parse_args() args.model = 'FCN8sAtOnce' args.git_hash = git_hash() now = datetime.datetime.now() args.out = osp.join(here, 'logs', now.strftime('%Y%m%d_%H%M%S.%f')) os.makedirs(args.out) with open(osp.join(args.out, 'config.yaml'), 'w') as f: yaml.safe_dump(args.__dict__, f, default_flow_style=False) os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu) cuda = torch.cuda.is_available() torch.manual_seed(1337) if cuda: torch.cuda.manual_seed(1337) # 1. dataset root = osp.expanduser('~/data/datasets') kwargs = {'num_workers': 4, 'pin_memory': True} if cuda else {} train_loader = torch.utils.data.DataLoader( torchfcn.datasets.DAVISClassSeg(root, split='train', transform=True), batch_size=1, shuffle=True, **kwargs) val_loader = torch.utils.data.DataLoader( torchfcn.datasets.DAVISClassSeg( root, split='val', transform=True), batch_size=1, shuffle=False, **kwargs) loader = val_loader # 2. model model = torchfcn.models.FCN8sAtOnce(n_class=2) epoch = 0 iteration = 0 checkpoint = torch.load(args.resume) model.load_state_dict(checkpoint['model_state_dict']) if cuda: model = model.cuda() training = False model.eval() n_class = len(loader.dataset.class_names) val_loss = 0 visualizations = [] label_trues, label_preds = [], [] timestamp_start = datetime.datetime.now(pytz.timezone('Asia/Tokyo')) config = convcrf.default_conf config['filter_size'] = 7 config['pyinn'] = False logging.info("Build ConvCRF.") ## # Create CRF module gausscrf = convcrf.GaussCRF(conf=config, shape=[224, 224], nclasses=n_class) # Cuda computation is required. # A CPU implementation of our message passing is not provided. gausscrf.cuda() for batch_idx, (data, target, unary) in tqdm.tqdm( enumerate(loader), total=len(loader), desc='Valid iteration=%d' % iteration, ncols=80, leave=False): data, target, unary = data.cuda(), target.cuda(), unary.cuda() shape = unary.shape[-2:] unary = unary.reshape([loader.batch_size, n_class, shape[0], shape[1]]) data, target, unary_var = Variable(data), Variable(target), Variable(unary) with torch.no_grad(): score = gausscrf.forward(unary=unary_var, img=data) loss = cross_entropy2d(score, target, size_average=False) loss_data = loss.data.item() if np.isnan(loss_data): raise ValueError('loss is nan while validating') val_loss += loss_data / len(data) imgs = data.data.cpu() lbl_pred = score.data.max(1)[1].cpu().numpy()[:, :, :] lbl_true = target.data.cpu() for img, lt, lp in zip(imgs, lbl_true, lbl_pred): img, lt = loader.dataset.untransform(img, lt) label_trues.append(lt) label_preds.append(lp) if len(visualizations) < 9: viz = fcn.utils.visualize_segmentation( lbl_pred=lp, lbl_true=lt, img=img, n_class=n_class) visualizations.append(viz) metrics = torchfcn.utils.label_accuracy_score( label_trues, label_preds, n_class) out = osp.join(args.out, 'visualization_viz') if not osp.exists(out): os.makedirs(out) out_file = osp.join(out, 'iter%012d.jpg' % iteration) scipy.misc.imsave(out_file, fcn.utils.get_tile_image(visualizations)) val_loss /= len(loader) with open(osp.join(args.out, 'log.csv'), 'a') as f: elapsed_time = ( datetime.datetime.now(pytz.timezone('Asia/Tokyo')) - timestamp_start).total_seconds() log = [epoch, iteration] + [''] * 5 + \ [val_loss] + list(metrics) + [elapsed_time] log = map(str, log) f.write(','.join(log) + '\n') mean_iu = metrics[2] print(mean_iu)