def main(args): model = PriSTM( args.embed_dim, args.hidden_size, args.num_layers, args.bidirectional ) trainer = Trainer(gpus=args.gpus, callbacks=[EarlyStopping(monitor="recall")],) batch_size = args.batch_size train_loader = make_loader("train", batch_size) val_loader = make_loader("val", batch_size) test_loader = make_loader("test", batch_size) trainer.fit(model, train_loader, val_loader) trainer.test(model, test_loader)
def main(args): set_seed() loader_config = { "batch_size": args.batch_size, "max_len": args.max_len, "min_len": args.min_len, } train_loader = make_loader("train", **loader_config) tags, val_loader = make_loader("val", return_tags=True, **loader_config) if args.load_title: model = load_model(args.model_name, tags, args.load_title) if model.max_len != args.max_len: warnings.warn("`max_len` of model and data loader do not match") else: device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = BertForPostClassification( args.model_name, tags, args.max_len, args.min_len, freeze_bert=args.freeze_bert, ).to(device) criterion = torch.nn.BCEWithLogitsLoss() optimizer = AdamW( [ {"params": model.bert.parameters(), "lr": 3e-5, "eps": 1e-8}, {"params": model.classifier.parameters()}, ] ) scheduler = get_linear_schedule_with_warmup( optimizer, num_warmup_steps=0, num_training_steps=args.num_epochs * len(train_loader), ) monitor = EarlyStopMonitor(args.patience) logger = Logger(args.num_epochs, args.log_interval) for epoch in range(args.num_epochs): model.train() train_loss = run_epoch(model, train_loader, criterion, optimizer, scheduler) model.eval() with torch.no_grad(): val_loss = run_epoch(model, val_loader, criterion) logger(epoch, train_loss, val_loss) if logger.best == val_loss: save_model(model, f"{args.save_title}", logger) monitor(val_loss) if monitor.stop: break
def create_data_loaders(fold, batch_size, workers): train_ids, val_ids = dataset.get_split(fold) labeled_size = len(train_ids) unlabeled_size = 18000 sampler = dataset.TwoStreamBatchSampler( range(labeled_size), # labeled ids list(range(labeled_size, labeled_size + unlabeled_size)), # unlabeled ids batch_size, # total batch size (labeled + unlabeled) LABELED_BATCH_SIZE # labeled batch size # TODO: was .5 ) train_loader = dataset.DataLoader( dataset=dataset.MeanTeacherTGSDataset( train_ids, transform=dataset.train_transform(), mode='train'), # shuffle=True, num_workers=workers, batch_sampler=sampler, pin_memory=torch.cuda.is_available()) valid_loader = dataset.make_loader( val_ids, transform=dataset.val_transform(), shuffle=False, # batch_size=len(device_ids), batch_size=batch_size, # len(device_ids), workers=workers) return train_loader, valid_loader
def predict(model, test_df, test_images, aug, device, data_dir='input/train'): model.eval() dataloader, ds_size = make_loader(test_df, test_images, conf.batch_size, shuffle=False, test="test", augment=aug) gr_preds = [] cd_preds = [] vd_preds = [] # Iterate over data. t = tqdm(dataloader) for i, samples in enumerate(t): with torch.set_grad_enabled(False): inputs = samples['data'].to(device) px = model(inputs) gr_preds.append(px[0].softmax(dim=1).cpu().data.numpy()) vd_preds.append(px[1].softmax(dim=1).cpu().data.numpy()) cd_preds.append(px[2].softmax(dim=1).cpu().data.numpy()) gr_preds = np.concatenate(gr_preds) vd_preds = np.concatenate(vd_preds) cd_preds = np.concatenate(cd_preds) return np.concatenate([gr_preds, vd_preds, cd_preds], axis=1)
def validate(model, val_df, val_images, aug, device, criterion): model.eval() dataloader, ds_size = make_loader(val_df, val_images, conf.batch_size, shuffle=False, test="valid", augment=aug) all_preds = [] all_trues = [] running_loss = 0.0 # Iterate over data. pbar = tqdm(dataloader) for i, samples in enumerate(pbar): with torch.set_grad_enabled(False): inputs = samples['data'].to(device) outputs = model(inputs) labels = samples['label'].to(device) outputs = model(inputs) loss, _, pred_class = calc_loss(outputs, labels, criterion) all_preds.append(pred_class) all_trues.append(labels.cpu().data.numpy()) loss = loss.mean() running_loss += loss.item() * inputs.size(0) kaggle_score, each_scores = weighted_macro_recall( samples["label"], pred_class) pbar.set_postfix({ "loss": loss.item(), "kaggle": kaggle_score, "graphen": each_scores[0], "vowel": each_scores[1], "consonant": each_scores[2] }) all_preds = np.concatenate(all_preds) all_trues = np.concatenate(all_trues) epoch_loss = running_loss / ds_size epoch_kaggle, epoch_each = weighted_macro_recall(all_trues, all_preds) result = { 'loss': epoch_loss, 'recall': epoch_kaggle, "graphen": epoch_each[0], "vowel": epoch_each[1], "consonant": epoch_each[2] } return all_preds, result
def main(args): set_seed() tags, test_loader = make_loader( "test", args.batch_size, args.max_len, args.min_len, return_tags=True, ) model = load_model(args.model_name, tags, args.save_title) if model.max_len != args.max_len: warnings.warn("`max_len` of model and data loader do not match") accuracy = get_accuracy(model, test_loader) hamming_accuracy = get_hamming_accuracy(model, test_loader) print( f"Accuracy: {accuracy:.4f}, Hamming Accuracy: {hamming_accuracy:.4f}")
def predict(model, ids, transform, kind, batch_size=32): loader = dataset.make_loader( ids, shuffle=False, mode=kind, batch_size=batch_size, transform=transform) preds = np.zeros((len(ids), dataset.NUM_CLASSES), dtype=np.float32) pred_idx = 0 with torch.no_grad(): for batch_num, (inputs, paths) in enumerate(tqdm(loader, desc='predict')): inputs = utils.cuda(inputs) outputs = model(inputs) outputs_npy = torch.sigmoid(outputs).cpu().numpy() for p in outputs_npy: preds[pred_idx] = p pred_idx += 1 return preds
def predict(model, ids, transform, kind, batch_size=32): loader = dataset.make_loader(ids, shuffle=False, mode=kind, batch_size=batch_size, transform=transform) preds = np.zeros((len(ids), 101, 101, 1), dtype=np.float32) pred_idx = 0 with torch.no_grad(): for batch_num, (inputs, paths) in enumerate(tqdm(loader, desc='predict')): inputs = utils.cuda(inputs) outputs = model(inputs) for p in outputs: pred_mask = F.sigmoid(p).data.cpu().numpy() #preds[pred_idx, ..., 0] = pred_mask[0, 13:-14, 13:-14] # remove padding from loader preds[pred_idx, ..., 0] = cv2.resize(pred_mask[0, 27:-27, 27:-27], (101, 101)) # remove padding from loader pred_idx += 1 return preds
def train( model, optimizer, # scheduler, train_df, train_images, aug, device, criterion, epoch=None, undersampling=False): model.train() dataloader, ds_size = make_loader(train_df, train_images, shuffle=True, test="train", mixup=conf.mixup, augment=aug) running_loss = 0.0 all_trues = list() all_preds = list() # Iterate over data. pbar = tqdm(dataloader) optimizer.zero_grad() for i, sample in enumerate(pbar): inputs = sample['data'].to(device) labels = sample['label'].to(device) trues = labels.cpu().data.numpy() all_trues.append(trues) rand = np.random.random() if conf.augmix and rand < conf.augmix_prob: mix1 = sample['mix1'].to(device) mix2 = sample['mix2'].to(device) outputs = model(inputs) mix1 = model(inputs) mix2 = model(inputs) loss, _, pred_class = calc_loss(mix1, labels, criterion) loss = augmix_loss(loss, outputs, mix1, mix2) elif conf.mixup and rand < conf.mixup_prob: inputs, ta, tb, lam = cutmix_data(inputs, labels) outputs = model(inputs) ratio = 1 loss_a, _, pred_class = calc_loss(outputs, ta, criterion, train=True, ohem_ratio=ratio) loss_b, _, _ = calc_loss(outputs, tb, criterion, train=True, ohem_ratio=ratio) loss = lam * loss_a + (1 - lam) * loss_b else: outputs = model(inputs) loss, _, pred_class = calc_loss(outputs, labels, criterion) all_preds.append(pred_class) loss = loss.mean() loss.backward() # statistics running_loss += loss.item() * inputs.size(0) kaggle_score, each_scores = weighted_macro_recall(trues, pred_class) pbar.set_postfix({ "loss": loss.item(), "kaggle": kaggle_score, "graphen": each_scores[0], "vowel": each_scores[1], "consonant": each_scores[2] }) if (i + 1) % conf.accum_time == 0: optimizer.step() optimizer.zero_grad() # scheduler.step() all_preds = np.concatenate(all_preds) all_trues = np.concatenate(all_trues) epoch_loss = running_loss / ds_size epoch_kaggle, epoch_each = weighted_macro_recall(all_trues, all_preds) result = { 'loss': epoch_loss, 'recall': epoch_kaggle, "graphen": epoch_each[0], "vowel": epoch_each[1], "consonant": epoch_each[2] } return all_preds, result
# Load the dataset train_data = PerceptionCarDatasetMerged("PerceptionCarDataset", "PerceptionCarDataset2", mode="train", only_front_camera=ONLY_FRONT_CAMERA) valid_data = PerceptionCarDatasetMerged( "PerceptionCarDataset", "PerceptionCarDataset2", mode="validation", preprocess=PerceptionCarDataset.valid_preprocessing, augment=False, only_front_camera=ONLY_FRONT_CAMERA, split=SPLIT) # Generate the data loaders train_loader = make_loader(train_data, batch_size=BATCH_SIZE, num_workers=4) valid_loader = make_loader(valid_data, batch_size=1, num_workers=4) # Define the model if MODEL_PATH is not None: net = network.NeuralNetworkModel.load(MODEL_PATH, device) net.log("Using {}.".format(MODEL_PATH)) else: arch_class = network.get_model_class(ARCHITECTURE) net = arch_class(only_front_camera=ONLY_FRONT_CAMERA, split=SPLIT) net.to(device=device) # Summary net.log("Train set size: {} samples".format(len(train_data))) net.log("Validation set size: {} samples".format(len(valid_data)))
def main(): args = arguments(opt) model = GoodModel(args) dataloader = make_loader(args)
def main(): parser = argparse.ArgumentParser() arg = parser.add_argument server = False path_default = "./data" if server else "e:/diploma" arg('--jaccard-weight', type=float, default=1) arg('--t', type=float, default=0.07) arg('--pretrain-epochs', type=int, default=100) arg('--train-epochs', type=int, default=100) arg('--train-test-split-file', type=str, default='./data/train_test_id.pickle', help='train test split file path') arg('--pretrain-image-path', type=str, default=f'{path_default}/ham10000_resized/', help='train test split file path') arg('--pretrain-mask-image-path', type=str, default=f'{path_default}/ham_clusters_20/lab/20/', help="images path for pretraining") arg('--image-path', type=str, default=f'{path_default}/task2_h5/', help="h5 images path for training") arg('--batch-size', type=int, default=8, help="n batches") arg('--workers', type=int, default=6, help="n workers") arg('--cuda-driver', type=int, default=1, help="cuda driver") arg('--resume-path', type=str, default=None) arg('--lr', type=float, default=0.001, help="lr") arg('--wandb', type=bool, default=True, help="wandb log") args = parser.parse_args() cudnn.benchmark = True torch.backends.cudnn.enabled = True device = torch.device( f'cuda:{args.cuda_driver}' if torch.cuda.is_available() else 'cpu') num_classes = 5 args.num_classes = 5 model = UNet16(num_classes=num_classes, pretrained="vgg") model = nn.DataParallel(model, device_ids=[args.cuda_driver]) model.to(device) center_layer = model.module.center_Conv2d for p in center_layer.parameters(): p.requires_grad = False pretrain_mask_image_path = args.pretrain_mask_image_path pretrain_image_path = args.pretrain_image_path pretrain_loader = make_pretrain_loader(pretrain_image_path, pretrain_mask_image_path, args, shuffle=True) epoch = 1 optimizer = Adam(model.parameters(), lr=args.lr) scheduler = ReduceLROnPlateau(optimizer, 'min', factor=0.8, patience=5, verbose=True) if args.resume_path is not None: state = torch.load(str(args.resume_path)) epoch = state['epoch'] + 1 model.load_state_dict(state['model']) print('--' * 10) print('Restored previous model, epoch {}'.format(epoch)) print('--' * 10) print(model) print('Start pretraining') criterion = ContrastiveLoss(args.t, device) cuda_available = torch.cuda.is_available() wandb.init(project="pipeline") wandb.run.name = f"pipeline lr = {args.lr}\n pretrain epochs = {args.pretrain_epochs}\ntrain epochs = {args.train_epochs}" wandb.run.save() wandb.watch(model) for epoch in range(epoch, args.pretrain_epochs + 1): model.train() start_time = time.time() losses = [] for ind, (id, image_original, image_transformed, mask_original, mask_transformed) in enumerate(pretrain_loader): start_step = time.time() print() #print(torch.cuda.memory_allocated(device)/ (1024 ** 2)) #print(torch.cuda.memory_reserved(device)/(1024 ** 2)) #print(torch.cuda.max_memory_allocated(device) / (1024 ** 2)) train_image_original = image_original.permute(0, 3, 1, 2) train_image_transformed = image_transformed.permute(0, 3, 1, 2) #print(f"permute time {time.time() - start_step}") #start_image_transfer = time.time() train_image_original = train_image_original.cuda(device, non_blocking=True) train_image_transformed = train_image_transformed.cuda( device, non_blocking=True) #print(f"train transfer : {time.time() - start_image_transfer}") #start_mask_transfer = time.time() mask_original = mask_original.cuda(device, non_blocking=True).type( torch.cuda.ByteTensor if cuda_available else torch.ByteTensor) mask_transformed = mask_transformed.cuda( device, non_blocking=True).type(torch.cuda.ByteTensor if cuda_available else torch.ByteTensor) #print(f"mask_transfer : {time.time() - start_mask_transfer}") #forward_start = time.time() original_result, _ = model(train_image_original) transformed_result, _ = model(train_image_transformed) #print(f"forward start :{time.time() - forward_start}") loss = (criterion(original_result, transformed_result, mask_original, mask_transformed)) losses.append(loss.item()) print(f'epoch={epoch:3d},iter={ind:3d}, loss={loss.item():.4g}') zero_grad_start = time.time() optimizer.zero_grad() #print(f"zero grad time:{time.time() - zero_grad_start}") #start_backward = time.time() loss.backward() #print(f"backward time:{time.time() - start_backward}") #start_optimizer = time.time() optimizer.step() #print(f"oprimizer time:{time.time() - start_optimizer}") print(f"step time:{time.time() - start_step}") avg_loss = np.mean(losses) wandb.log({"pretrain/loss": avg_loss}) epoch_time = time.time() - start_time print(f"epoch time:{epoch_time}") scheduler.step(avg_loss) model_path = f"checkpoint/model_epoch_{epoch}.pt" torch.save( { 'model': model.state_dict(), 'epoch': epoch, 'loss': avg_loss }, str(model_path)) print("Pretraining ended") epoch = 1 ## get train_test_id train_test_id = get_split(args.train_test_split_file) ## train vs. val print('--' * 10) print('num train = {}, num_val = {}'.format( (train_test_id['Split'] == 'train').sum(), (train_test_id['Split'] != 'train').sum())) print('--' * 10) image_path = args.image_path train_loader = make_loader( train_test_id, image_path, args, train=True, shuffle=True, train_test_split_file=args.train_test_split_file) valid_loader = make_loader( train_test_id, image_path, args, train=False, shuffle=True, train_test_split_file=args.train_test_split_file) if True: print('--' * 10) print('check data') train_image, train_mask, train_mask_ind = next(iter(train_loader)) print('train_image.shape', train_image.shape) print('train_mask.shape', train_mask.shape) print('train_mask_ind.shape', train_mask_ind.shape) print('train_image.min', train_image.min().item()) print('train_image.max', train_image.max().item()) print('train_mask.min', train_mask.min().item()) print('train_mask.max', train_mask.max().item()) print('train_mask_ind.min', train_mask_ind.min().item()) print('train_mask_ind.max', train_mask_ind.max().item()) print('--' * 10) valid_fn = validation_binary criterion = LossBinary(jaccard_weight=args.jaccard_weight) meter = AllInOneMeter(device) model.module.projection_head = nn.Conv2d(32, num_classes, 1) center_layer = model.module.center_Conv2d for p in center_layer.parameters(): p.requires_grad = True print(model) optimizer = Adam(model.parameters(), lr=args.lr) scheduler = ReduceLROnPlateau(optimizer, 'min', factor=0.8, patience=5, verbose=True) print("Start fine tuning") for epoch in range(epoch, args.train_epochs + 1): model.train() start_time = time.time() meter.reset() w1 = 1.0 w2 = 0.5 w3 = 0.5 for i, (train_image, train_mask, train_mask_ind) in enumerate(train_loader): train_image = train_image.permute(0, 3, 1, 2) train_mask = train_mask.permute(0, 3, 1, 2) train_image = train_image.to(device) train_mask = train_mask.to( device).type(torch.cuda.FloatTensor if torch.cuda.is_available( ) else torch.FloatTensor) train_mask_ind = train_mask_ind.to( device).type(torch.cuda.FloatTensor if torch.cuda.is_available( ) else torch.FloatTensor) outputs, outputs_mask_ind1 = model(train_image) outputs_mask_ind2 = nn.MaxPool2d( kernel_size=outputs.size()[2:])(outputs) outputs_mask_ind2 = torch.squeeze(outputs_mask_ind2, 2) outputs_mask_ind2 = torch.squeeze(outputs_mask_ind2, 2) train_prob = torch.sigmoid(outputs) train_mask_ind_prob1 = torch.sigmoid(outputs_mask_ind1) train_mask_ind_prob2 = torch.sigmoid(outputs_mask_ind2) loss1 = criterion(outputs, train_mask) loss2 = F.binary_cross_entropy_with_logits(outputs_mask_ind1, train_mask_ind) loss3 = F.binary_cross_entropy_with_logits(outputs_mask_ind2, train_mask_ind) loss = loss1 * w1 + loss2 * w2 + loss3 * w3 print( f'epoch={epoch:3d},iter={i:3d}, loss1={loss1.item():.4g}, loss2={loss2.item():.4g}, loss={loss.item():.4g}' ) optimizer.zero_grad() loss.backward() optimizer.step() meter.add(train_prob, train_mask, train_mask_ind_prob1, train_mask_ind_prob2, train_mask_ind, loss1.item(), loss2.item(), loss3.item(), loss.item()) epoch_time = time.time() - start_time train_metrics = meter.value() train_metrics['epoch_time'] = epoch_time train_metrics['image'] = train_image.data train_metrics['mask'] = train_mask.data train_metrics['prob'] = train_prob.data valid_metrics = valid_fn(model, criterion, valid_loader, device) wandb.log({ "loss/loss": valid_metrics["loss"], "loss/loss1": valid_metrics["loss1"], "loss/loss2": valid_metrics["loss2"], "jaccard_mean/jaccard_mean": valid_metrics["jaccard"], "jaccard_class/jaccard_pigment_network": valid_metrics["jaccard1"], "jaccard_class/jaccard_negative_network": valid_metrics["jaccard2"], "jaccard_class/jaccard_streaks": valid_metrics["jaccard3"], "jaccard_class/jaccard_milia_like_cyst": valid_metrics["jaccard4"], "jaccard_class/jaccard_globules": valid_metrics["jaccard5"] }) scheduler.step(valid_metrics['loss1'])
train_data, val_data, vocab = mydataset.make_dataset(raw_data, opt) opt.max_words = min(opt.max_words, len(vocab) - 4) train_iter, val_iter = mydataset.make_iterator((train_data, val_data), opt) device = T.device(opt.device) start_time = datetime.now().strftime("%Y%m%d-%H%M%S") layout = [('model={:s}', 'cvae'), ('z={:02d}', opt.z_dim), ('time={:s}', start_time), ('data={:s}', opt.dataset)] model_name = '_'.join([t.format(v) for (t, v) in layout]) writer = ut.prepare_writer(model_name, overwrite_existing=False) model = CVAE(opt).to(device) model.embedding.weight.data.copy_(vocab.vectors).to(device) train_gen = mydataset.make_loader(train_iter, opt) val_gen = mydataset.make_loader(val_iter, opt) if opt.mode == 'train': train(model=model, train_loader=train_gen, val_loader=val_gen, tqdm=tqdm.tqdm, device=device, writer=writer, start_time=start_time, dataset_name=opt.dataset, iter_max=opt.max_iter, iter_log=opt.log_iter, iter_save=opt.save_iter, mask_prob=opt.mask_prob) elif opt.mode == 'test':
# torch.set_num_threads(1) # logger logger = Logger(args.logfile) logger.print_args(args) # random seed if args.seed: do_seed(args.seed) logger.log(f'The fixed seed number is {args.seed}') ############### Load Data ############### if 'CS_KD' in args.method: trainloader, testloader = make_loader(args.dataset, batch_size=args.batch_size, aug=args.aug, sampler='CS_KD', num_workers=args.num_threads) logger.log('Dataset for Class-wise Self KD') elif 'DDGSD' in args.method: trainloader, testloader = make_loader(args.dataset, batch_size=args.batch_size, aug=args.aug, sampler='DDGSD', num_workers=args.num_threads) logger.log('Dataset for Data Distortion Guided Self Distillation') else: trainloader, testloader = make_loader(args.dataset, batch_size=args.batch_size, aug=args.aug, num_workers=args.num_threads)
def main(): parser = argparse.ArgumentParser() arg = parser.add_argument arg('--jaccard-weight', type=float, default=1) arg('--root', type=str, default='runs/debug', help='checkpoint root') arg('--image-path', type=str, default='data', help='image path') arg('--batch-size', type=int, default=2) arg('--n-epochs', type=int, default=100) arg('--optimizer', type=str, default='Adam', help='Adam or SGD') arg('--lr', type=float, default=0.001) arg('--workers', type=int, default=10) arg('--model', type=str, default='UNet16', choices=[ 'UNet', 'UNet11', 'UNet16', 'LinkNet34', 'FCDenseNet57', 'FCDenseNet67', 'FCDenseNet103' ]) arg('--model-weight', type=str, default=None) arg('--resume-path', type=str, default=None) arg('--attribute', type=str, default='all', choices=[ 'pigment_network', 'negative_network', 'streaks', 'milia_like_cyst', 'globules', 'all' ]) args = parser.parse_args() ## folder for checkpoint root = Path(args.root) root.mkdir(exist_ok=True, parents=True) image_path = args.image_path #print(args) if args.attribute == 'all': num_classes = 5 else: num_classes = 1 args.num_classes = num_classes ### save initial parameters print('--' * 10) print(args) print('--' * 10) root.joinpath('params.json').write_text( json.dumps(vars(args), indent=True, sort_keys=True)) ## load pretrained model if args.model == 'UNet': model = UNet(num_classes=num_classes) elif args.model == 'UNet11': model = UNet11(num_classes=num_classes, pretrained='vgg') elif args.model == 'UNet16': model = UNet16(num_classes=num_classes, pretrained='vgg') elif args.model == 'LinkNet34': model = LinkNet34(num_classes=num_classes, pretrained=True) elif args.model == 'FCDenseNet103': model = FCDenseNet103(num_classes=num_classes) else: model = UNet(num_classes=num_classes, input_channels=3) ## multiple GPUs device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') if torch.cuda.device_count() > 1: model = nn.DataParallel(model) model.to(device) ## load pretrained model if args.model_weight is not None: state = torch.load(args.model_weight) #epoch = state['epoch'] #step = state['step'] model.load_state_dict(state['model']) print('--' * 10) print('Load pretrained model', args.model_weight) #print('Restored model, epoch {}, step {:,}'.format(epoch, step)) print('--' * 10) ## replace the last layer ## although the model and pre-trained weight have differernt size (the last layer is different) ## pytorch can still load the weight ## I found that the weight for one layer just duplicated for all layers ## therefore, the following code is not necessary # if args.attribute == 'all': # model = list(model.children())[0] # num_filters = 32 # model.final = nn.Conv2d(num_filters, num_classes, kernel_size=1) # print('--' * 10) # print('Load pretrained model and replace the last layer', args.model_weight, num_classes) # print('--' * 10) # if torch.cuda.device_count() > 1: # model = nn.DataParallel(model) # model.to(device) ## model summary print_model_summay(model) ## define loss loss_fn = LossBinary(jaccard_weight=args.jaccard_weight) ## It enables benchmark mode in cudnn. ## benchmark mode is good whenever your input sizes for your network do not vary. This way, cudnn will look for the ## optimal set of algorithms for that particular configuration (which takes some time). This usually leads to faster runtime. ## But if your input sizes changes at each iteration, then cudnn will benchmark every time a new size appears, ## possibly leading to worse runtime performances. cudnn.benchmark = True ## get train_test_id train_test_id = get_split() ## train vs. val print('--' * 10) print('num train = {}, num_val = {}'.format( (train_test_id['Split'] == 'train').sum(), (train_test_id['Split'] != 'train').sum())) print('--' * 10) train_transform = DualCompose( [HorizontalFlip(), VerticalFlip(), ImageOnly(Normalize())]) val_transform = DualCompose([ImageOnly(Normalize())]) ## define data loader train_loader = make_loader(train_test_id, image_path, args, train=True, shuffle=True, transform=train_transform) valid_loader = make_loader(train_test_id, image_path, args, train=False, shuffle=True, transform=val_transform) if True: print('--' * 10) print('check data') train_image, train_mask, train_mask_ind = next(iter(train_loader)) print('train_image.shape', train_image.shape) print('train_mask.shape', train_mask.shape) print('train_mask_ind.shape', train_mask_ind.shape) print('train_image.min', train_image.min().item()) print('train_image.max', train_image.max().item()) print('train_mask.min', train_mask.min().item()) print('train_mask.max', train_mask.max().item()) print('train_mask_ind.min', train_mask_ind.min().item()) print('train_mask_ind.max', train_mask_ind.max().item()) print('--' * 10) valid_fn = validation_binary ########### ## optimizer if args.optimizer == 'Adam': optimizer = Adam(model.parameters(), lr=args.lr) elif args.optimizer == 'SGD': optimizer = SGD(model.parameters(), lr=args.lr, momentum=0.9) ## loss criterion = loss_fn ## change LR scheduler = ReduceLROnPlateau(optimizer, 'min', factor=0.8, patience=5, verbose=True) ########## ## load previous model status previous_valid_loss = 10 model_path = root / 'model.pt' if args.resume_path is not None and model_path.exists(): state = torch.load(str(model_path)) epoch = state['epoch'] step = state['step'] model.load_state_dict(state['model']) epoch = 1 step = 0 try: previous_valid_loss = state['valid_loss'] except: previous_valid_loss = 10 print('--' * 10) print('Restored previous model, epoch {}, step {:,}'.format( epoch, step)) print('--' * 10) else: epoch = 1 step = 0 ######### ## start training log = root.joinpath('train.log').open('at', encoding='utf8') writer = SummaryWriter() meter = AllInOneMeter() #if previous_valid_loss = 10000 print('Start training') print_model_summay(model) previous_valid_jaccard = 0 for epoch in range(epoch, args.n_epochs + 1): model.train() random.seed() #jaccard = [] start_time = time.time() meter.reset() w1 = 1.0 w2 = 0.5 w3 = 0.5 try: train_loss = 0 valid_loss = 0 # if epoch == 1: # freeze_layer_names = get_freeze_layer_names(part='encoder') # set_freeze_layers(model, freeze_layer_names=freeze_layer_names) # #set_train_layers(model, train_layer_names=['module.final.weight','module.final.bias']) # print_model_summay(model) # elif epoch == 5: # w1 = 1.0 # w2 = 0.0 # w3 = 0.5 # freeze_layer_names = get_freeze_layer_names(part='encoder') # set_freeze_layers(model, freeze_layer_names=freeze_layer_names) # # set_train_layers(model, train_layer_names=['module.final.weight','module.final.bias']) # print_model_summay(model) #elif epoch == 3: # set_train_layers(model, train_layer_names=['module.dec5.block.0.conv.weight','module.dec5.block.0.conv.bias', # 'module.dec5.block.1.weight','module.dec5.block.1.bias', # 'module.dec4.block.0.conv.weight','module.dec4.block.0.conv.bias', # 'module.dec4.block.1.weight','module.dec4.block.1.bias', # 'module.dec3.block.0.conv.weight','module.dec3.block.0.conv.bias', # 'module.dec3.block.1.weight','module.dec3.block.1.bias', # 'module.dec2.block.0.conv.weight','module.dec2.block.0.conv.bias', # 'module.dec2.block.1.weight','module.dec2.block.1.bias', # 'module.dec1.conv.weight','module.dec1.conv.bias', # 'module.final.weight','module.final.bias']) # print_model_summa zvgf t5y(model) # elif epoch == 50: # set_freeze_layers(model, freeze_layer_names=None) # print_model_summay(model) for i, (train_image, train_mask, train_mask_ind) in enumerate(train_loader): # inputs, targets = variable(inputs), variable(targets) train_image = train_image.permute(0, 3, 1, 2) train_mask = train_mask.permute(0, 3, 1, 2) train_image = train_image.to(device) train_mask = train_mask.to(device).type(torch.cuda.FloatTensor) train_mask_ind = train_mask_ind.to(device).type( torch.cuda.FloatTensor) # if args.problem_type == 'binary': # train_mask = train_mask.to(device).type(torch.cuda.FloatTensor) # else: # #train_mask = train_mask.to(device).type(torch.cuda.LongTensor) # train_mask = train_mask.to(device).type(torch.cuda.FloatTensor) outputs, outputs_mask_ind1, outputs_mask_ind2 = model( train_image) #print(outputs.size()) #print(outputs_mask_ind1.size()) #print(outputs_mask_ind2.size()) ### note that the last layer in the model is defined differently # if args.problem_type == 'binary': # train_prob = F.sigmoid(outputs) # loss = criterion(outputs, train_mask) # else: # #train_prob = outputs # train_prob = F.sigmoid(outputs) # loss = torch.tensor(0).type(train_mask.type()) # for feat_inx in range(train_mask.shape[1]): # loss += criterion(outputs, train_mask) train_prob = F.sigmoid(outputs) train_mask_ind_prob1 = F.sigmoid(outputs_mask_ind1) train_mask_ind_prob2 = F.sigmoid(outputs_mask_ind2) loss1 = criterion(outputs, train_mask) #loss1 = F.binary_cross_entropy_with_logits(outputs, train_mask) #loss2 = nn.BCEWithLogitsLoss()(outputs_mask_ind1, train_mask_ind) #print(train_mask_ind.size()) #weight = torch.ones_like(train_mask_ind) #weight[:, 0] = weight[:, 0] * 1 #weight[:, 1] = weight[:, 1] * 14 #weight[:, 2] = weight[:, 2] * 14 #weight[:, 3] = weight[:, 3] * 4 #weight[:, 4] = weight[:, 4] * 4 #weight = weight * train_mask_ind + 1 #weight = weight.to(device).type(torch.cuda.FloatTensor) loss2 = F.binary_cross_entropy_with_logits( outputs_mask_ind1, train_mask_ind) loss3 = F.binary_cross_entropy_with_logits( outputs_mask_ind2, train_mask_ind) #loss3 = criterion(outputs_mask_ind2, train_mask_ind) loss = loss1 * w1 + loss2 * w2 + loss3 * w3 #print(loss1.item(), loss2.item(), loss.item()) optimizer.zero_grad() loss.backward() optimizer.step() step += 1 #jaccard += [get_jaccard(train_mask, (train_prob > 0).float()).item()] meter.add(train_prob, train_mask, train_mask_ind_prob1, train_mask_ind_prob2, train_mask_ind, loss1.item(), loss2.item(), loss3.item(), loss.item()) # print(train_mask.data.shape) # print(train_mask.data.sum(dim=-2).shape) # print(train_mask.data.sum(dim=-2).sum(dim=-1).shape) # print(train_mask.data.sum(dim=-2).sum(dim=-1).sum(dim=0).shape) # intersection = train_mask.data.sum(dim=-2).sum(dim=-1) # print(intersection.shape) # print(intersection.dtype) # print(train_mask.data.shape[0]) #torch.zeros([2, 4], dtype=torch.float32) ######################### ## at the end of each epoch, evualte the metrics epoch_time = time.time() - start_time train_metrics = meter.value() train_metrics['epoch_time'] = epoch_time train_metrics['image'] = train_image.data train_metrics['mask'] = train_mask.data train_metrics['prob'] = train_prob.data #train_jaccard = np.mean(jaccard) #train_auc = str(round(mtr1.value()[0],2))+' '+str(round(mtr2.value()[0],2))+' '+str(round(mtr3.value()[0],2))+' '+str(round(mtr4.value()[0],2))+' '+str(round(mtr5.value()[0],2)) valid_metrics = valid_fn(model, criterion, valid_loader, device, num_classes) ############## ## write events write_event(log, step, epoch=epoch, train_metrics=train_metrics, valid_metrics=valid_metrics) #save_weights(model, model_path, epoch + 1, step) ######################### ## tensorboard write_tensorboard(writer, model, epoch, train_metrics=train_metrics, valid_metrics=valid_metrics) ######################### ## save the best model valid_loss = valid_metrics['loss1'] valid_jaccard = valid_metrics['jaccard'] if valid_loss < previous_valid_loss: save_weights(model, model_path, epoch + 1, step, train_metrics, valid_metrics) previous_valid_loss = valid_loss print('Save best model by loss') if valid_jaccard > previous_valid_jaccard: save_weights(model, model_path, epoch + 1, step, train_metrics, valid_metrics) previous_valid_jaccard = valid_jaccard print('Save best model by jaccard') ######################### ## change learning rate scheduler.step(valid_metrics['loss1']) except KeyboardInterrupt: # print('--' * 10) # print('Ctrl+C, saving snapshot') # save_weights(model, model_path, epoch, step) # print('done.') # print('--' * 10) writer.close() #return writer.close()
net = network.NeuralNetworkModel.load(MODEL_PATH, device) net.to(device=device) print("ONLY_FRONT_CAMERA", ONLY_FRONT_CAMERA) input() # Dataset data = PerceptionCarDatasetMerged( "PerceptionCarDataset", "PerceptionCarDataset2", mode=MODE, preprocess=PerceptionCarDataset.valid_preprocessing, only_front_camera=ONLY_FRONT_CAMERA, split=SPLIT, return_image_paths=True) data_loader = make_loader(data, shuffle=False, batch_size=1, num_workers=4) print("Samples: {}".format(len(data))) # Loss function criterion = Customized_Loss() if VISUALIZE: position_errors, orientation_errors = [], [] position_errors_smooth, orientation_errors_smooth = [], [] from visualize import PosePlotter plotter = PosePlotter(update_interval=UPDATE_INTERVAL, trajectory=True) plotter.register("Ground truth", "red") plotter.register("NN prediction", "blue") plotter.register("Smoothed NN prediction", "green") net.eval()
def main(): parser = argparse.ArgumentParser() arg = parser.add_argument arg('--name', type=str) arg('--jaccard-weight', default=0.25, type=float) arg('--device-ids', type=str, default='0', help='For example 0,1 to run on two GPUs') arg('--fold', type=int, help='fold', default=0) arg('--output-dir', default='../data/runs', help='checkpoint root') arg('--batch-size', type=int, default=32) arg('--iter-size', type=int, default=1) arg('--n-epochs', type=int, default=100) arg('--lr', type=float, default=0.0001) arg('--workers', type=int, default=4) arg('--seed', type=int, default=0) arg('--model', type=str, default=models.archs[0], choices=models.archs) arg('--loss', type=str, default='focal', choices=[ 'focal', 'lovasz', 'bjd', 'bce_jaccard', 'bce_dice', 'cos_dice', 'hinge' ]) arg('--focal-gamma', type=float, default=.5) arg('--num-channels', type=int, default=3) arg('--weighted-sampler', action="store_true") arg('--ignore-empty-masks', action='store_true') arg('--remove-suspicious', action='store_true') arg('--resume', action="store_true") args = parser.parse_args() random.seed(args.seed) torch.manual_seed(args.seed) if not args.name: experiment = uuid.uuid4().hex else: experiment = args.name output_dir = Path(args.output_dir) / experiment output_dir.mkdir(exist_ok=True, parents=True) output_dir.joinpath('params.json').write_text( json.dumps(vars(args), indent=True, sort_keys=True)) # in case --resume is provided it will be loaded later model = models.get_model(None, args.model) # model = models.get_model(f"../data/runs/exp81/model_{args.fold}.pth", args.model) if torch.cuda.is_available(): if args.device_ids: device_ids = list(map(int, args.device_ids.split(','))) else: device_ids = None model = nn.DataParallel(model, device_ids=device_ids).cuda() train_ids, val_ids = dataset.get_split(args.fold) cudnn.benchmark = True train_loader = dataset.make_loader( train_ids, num_channels=args.num_channels, transform=dataset.train_transform(), shuffle=True, weighted_sampling=args.weighted_sampler, ignore_empty_masks=args.ignore_empty_masks, remove_suspicious=args.remove_suspicious, batch_size=args.batch_size, workers=args.workers) valid_loader = dataset.make_loader( val_ids, num_channels=args.num_channels, transform=dataset.val_transform(), shuffle=False, #batch_size=len(device_ids), batch_size=args.batch_size, # len(device_ids), workers=args.workers) # optimizer = Adam([p for p in model.parameters() if p.requires_grad], lr=args.lr) optimizer = Adam(model.parameters(), lr=args.lr) # loss = LossBinary(jaccard_weight=args.jaccard_weight) # loss = LossBinaryMixedDiceBCE(dice_weight=0.5, bce_weight=0.5) if args.loss == 'focal': loss = FocalLoss(args.focal_gamma) elif args.loss == 'lovasz': loss = LossLovasz() elif args.loss == 'bjd': loss = BCEDiceJaccardLoss({'bce': 0.25, 'jaccard': None, 'dice': 0.75}) elif args.loss == 'bce_jaccard': loss = LossBinary(args.jaccard_weight) elif args.loss == 'bce_dice': import loss2 bce_weight = 1 dice_weight = 2 loss = loss2.make_loss(bce_weight, dice_weight) elif args.loss == 'cos_dice': import loss2 loss = loss2.make_cos_dice_loss() elif args.loss == 'hinge': loss = LossHinge() else: raise NotImplementedError validation = validation_binary scheduler = ReduceLROnPlateau(optimizer, verbose=True, min_lr=1e-7, factor=0.5) snapshot = utils.fold_snapshot(output_dir, args.fold) if args.resume else None utils.train(experiment=experiment, output_dir=output_dir, optimizer=optimizer, args=args, model=model, criterion=loss, scheduler=scheduler, train_loader=train_loader, valid_loader=valid_loader, validation=validation, fold=args.fold, batch_size=args.batch_size, n_epochs=args.n_epochs, snapshot=snapshot, iter_size=args.iter_size)
def main(): args = build_train_args() random.seed(args.seed) torch.manual_seed(args.seed) experiment = args.name if args.name else uuid.uuid4().hex output_dir = Path(args.output_dir) / experiment output_dir.mkdir(exist_ok=True, parents=True) output_dir.joinpath('params.json').write_text( json.dumps(vars(args), indent=True, sort_keys=True)) # this is different than --resume, which loads optimizer state as well initial_model_path = os.path.join( output_dir, f"model_{args.fold}.pth") if args.load_weights else None model = models.get_model(initial_model_path, args.model) train_ids, val_ids = dataset.get_split(args.fold) cudnn.benchmark = True train_loader = dataset.make_loader(train_ids, num_channels=args.num_channels, transform=dataset.train_transform(), shuffle=True, weighted_sampling=args.weighted_sampler, batch_size=args.batch_size, workers=args.workers) valid_loader = dataset.make_loader( val_ids, num_channels=args.num_channels, transform=dataset.val_transform(), shuffle=False, weighted_sampling=False, batch_size=args.batch_size, # len(device_ids), workers=args.workers) # train last layer only # for m in model.children(): # if m in [model.fc]: # continue # for param in m.parameters(): # param.requires_grad = False # set all layers except fc to lr=0 #fc_params = list(map(id, model.fc.parameters())) #base_params = filter(lambda p: id(p) not in fc_params, model.parameters()) #optimizer = torch.optim.Adam([ # {'params': base_params, 'lr': args.lr * 0.001}, # {'params': model.fc.parameters(), 'lr': args.lr} #], lr=args.lr * 0.001) optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) # optimizer = torch.optim.SGD(model.parameters(), lr=0.03, momentum=0.9, weight_decay=1e-4) # criterion = FocalLoss(gamma=args.focal_gamma) criterion = nn.BCEWithLogitsLoss().cuda() # criterion = losses.f1_loss validation_fn = validation_multi #scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( # optimizer, mode='max', verbose=True, min_lr=1e-7, factor=0.5, patience=5) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1) snapshot = utils.fold_snapshot(output_dir, args.fold) if args.resume else None device_ids = list(map( int, args.device_ids.split(','))) if args.device_ids else None wrapped_model = nn.DataParallel(model, device_ids=device_ids).cuda() # unfreeze # for m in model.children(): # for param in m.parameters(): # param.requires_grad = True # fc1_params = list(map(id, model.fc1.parameters())) # fc2_params = list(map(id, model.fc2.parameters())) # base_params = filter(lambda p: id(p) not in fc1_params and id(p) not in fc2_params, model.parameters()) # # # fc at args.lr and rest with args.lr * 0.1 # optimizer = torch.optim.Adam([ # {'params': base_params}, # {'params': model.fc1.parameters(), 'lr': args.lr}, # {'params': model.fc2.parameters(), 'lr': args.lr}, # ], lr=args.lr * 0.1) # aternatively, add the unfrozen fc2 weight to the current optimizer # optimizer.add_param_group({'params': net.fc2.parameters()}) utils.train(experiment=experiment, output_dir=output_dir, optimizer=optimizer, args=args, model=wrapped_model, criterion=criterion, scheduler=scheduler, train_loader=train_loader, valid_loader=valid_loader, validation_fn=validation_fn, fold=args.fold, batch_size=args.batch_size, n_epochs=args.n_epochs, snapshot=snapshot, iter_size=args.iter_size)