train_loader, val_loader = get_train_val_loaders( csv_path_train=csv_train, csv_path_val=csv_val, batch_size=bs, tg_size=tg_size, label_values=label_values, num_workers=args.num_workers) # grad_acc_steps: if I want to train with a fake_bs=K but the actual bs I want is bs=N, then you use # grad_acc_steps = N/K - 1. # Example: bs=4, fake_bs=4 -> grad_acc_steps = 0 (default) # Example: bs=4, fake_bs=2 -> grad_acc_steps = 1 # Example: bs=4, fake_bs=1 -> grad_acc_steps = 3 print('* Instantiating a {} model'.format(model_name)) model = get_arch(model_name, n_classes=n_classes) model = model.to(device) print("Total params: {0:,}".format( sum(p.numel() for p in model.parameters() if p.requires_grad))) optimizer = torch.optim.Adam(model.parameters(), lr=max_lr) ### TRAINING WITH PSEUDO-LABELS csv_test = args.csv_test path_test_preds = args.path_test_preds checkpoint_folder = args.checkpoint_folder if csv_test is not None: print( 'Training with pseudo-labels, completing training set with predictions on test set' ) from utils.get_loaders import build_pseudo_dataset
load_path_od = args.load_path_od load_path_mac = args.load_path_mac pretrained = args.pretrained bs = args.batch_size csv_test_od = args.csv_test_od csv_test_mac = args.csv_test_mac n_classes = args.n_classes tta = args.tta csv_out = args.csv_out #################################################################################################################### # build results for od-centered with OD model print('* Instantiating model {}, pretrained={}'.format( model_name, pretrained)) model, mean, std = get_arch(model_name, pretrained=pretrained, n_classes=n_classes) model, stats = load_model(model, load_path_od, device='cpu') model = model.to(device) print("Total params: {0:,}".format( sum(p.numel() for p in model.parameters() if p.requires_grad))) print('* Creating Dataloaders, batch size = {:d}'.format(bs)) test_loader = get_test_loader(csv_path_test=csv_test_od, batch_size=bs, mean=mean, std=std) if tta: probs_od, preds_od, labels = test_cls_tta_dihedral(model, test_loader,
load_path_clarity = args.load_path_clarity pretrained = args.pretrained bs = args.batch_size csv_test_q = args.csv_test_q n_classes = args.n_classes tta = args.tta csv_out = args.csv_out #################################################################################################################### # build results for MT model n_classes = 18 print('* Instantiating MT model {}, pretrained={}'.format( model_name_MT, pretrained)) model, mean, std = get_arch(model_name_MT, pretrained=pretrained, n_classes=n_classes) model, stats = load_model(model, load_path_MT, device='cpu') model = model.to(device) print("Total params: {0:,}".format( sum(p.numel() for p in model.parameters() if p.requires_grad))) print('* Creating Dataloaders, batch size = {:d}'.format(bs)) test_loader = get_test_loader(csv_path_test=csv_test_q_MT, batch_size=bs, mean=mean, std=std, qualities=True) probs_tta_q, preds_tta_q, probs_tta_a, preds_tta_a, probs_tta_c, preds_tta_c, probs_tta_f, preds_tta_f \ = test_cls_tta_dihedral_MT(model, test_loader, n=3)
tg_size = (im_size[0], im_size[0]) elif isinstance(im_size, tuple) and len(im_size) == 2: tg_size = (im_size[0], im_size[1]) else: sys.exit('im_size should be a number or a tuple of two numbers') if public: data_path = osp.join('data', dataset) else: data_path = osp.join('private_data', dataset) csv_path = 'test_all.csv' print('* Reading test data from ' + osp.join(data_path, csv_path)) test_dataset = get_test_dataset(data_path, csv_path=csv_path, tg_size=tg_size) print('* Instantiating model = ' + str(model_name)) model = get_arch(model_name, in_c=in_c).to(device) if model_name == 'wnet': model.mode = 'eval' print('* Loading trained weights from ' + experiment_path) try: model, stats = load_model(model, experiment_path, device) except RuntimeError: sys.exit( '---- bad config specification (check layers, n_classes, etc.) ---- ' ) model.eval() save_results_path = osp.join(args.result_path, dataset, experiment_path) print('* Saving predictions to ' + save_results_path) times = [] for i in tqdm(range(len(test_dataset))):
n_classes=1 label_values = [0, 255] print(f"* Creating Dataloaders, batch size = {bs}, workers = {args.num_workers}") train_loader, val_loader = get_train_val_loaders(csv_path_train=csv_train, csv_path_val=csv_val, batch_size=bs, tg_size=tg_size, label_values=label_values, num_workers=args.num_workers) # grad_acc_steps: if I want to train with a fake_bs=K but the actual bs I want is bs=N, then you use # grad_acc_steps = N/K - 1. # Example: bs=4, fake_bs=4 -> grad_acc_steps = 0 (default) # Example: bs=4, fake_bs=2 -> grad_acc_steps = 1 # Example: bs=4, fake_bs=1 -> grad_acc_steps = 3 print('* Instantiating a {} model'.format(model_name)) model = get_arch(model_name, n_classes=n_classes, compose='cat') model = model.to(device) print("Total params: {0:,}".format(sum(p.numel() for p in model.parameters() if p.requires_grad))) optimizer = torch.optim.Adam(model.parameters(), lr=max_lr) ### TRAINING WITH PSEUDO-LABELS csv_test = args.csv_test path_test_preds = args.path_test_preds checkpoint_folder = args.checkpoint_folder if csv_test is not None: print('Training with pseudo-labels, completing training set with predictions on test set') from utils.get_loaders import build_pseudo_dataset tr_im_list, tr_gt_list, tr_mask_list = build_pseudo_dataset(csv_train, csv_test, path_test_preds) train_loader.dataset.im_list = tr_im_list train_loader.dataset.gt_list = tr_gt_list
if isinstance(im_size, tuple) and len(im_size) == 1: tg_size = (im_size[0], im_size[0]) elif isinstance(im_size, tuple) and len(im_size) == 2: tg_size = (im_size[0], im_size[1]) else: sys.exit('im_size should be a number or a tuple of two numbers') data_path = osp.join('data', dataset) csv_path = 'test_all.csv' print('* Reading test data from ' + osp.join(data_path, csv_path)) test_dataset = get_test_dataset(data_path, csv_path=csv_path, tg_size=tg_size) print('* Instantiating model = ' + str(model_name)) model = get_arch(model_name, n_classes=4).to(device) if 'wnet' in model_name: model.mode = 'eval' print('* Loading trained weights from ' + experiment_path) try: model, stats = load_model(model, experiment_path, device) except RuntimeError: sys.exit( '---- bad config specification (check layers, n_classes, etc.) ---- ' ) model.eval() save_results_path = osp.join(args.results_path, dataset, experiment_path) print('* Saving predictions to ' + save_results_path) for i in tqdm(range(len(test_dataset))): im_tens, mask, coords_crop, original_sz, im_name = test_dataset[i]
train_loader, val_loader = get_train_val_loaders( csv_path_train=csv_train, csv_path_val=csv_val, batch_size=bs, tg_size=tg_size, label_values=label_values, num_workers=args.num_workers) # grad_acc_steps: if I want to train with a fake_bs=K but the actual bs I want is bs=N, then you use # grad_acc_steps = N/K - 1. # Example: bs=4, fake_bs=4 -> grad_acc_steps = 0 (default) # Example: bs=4, fake_bs=2 -> grad_acc_steps = 1 # Example: bs=4, fake_bs=1 -> grad_acc_steps = 3 print('* Instantiating a {} model'.format(model_name)) model = get_arch(model_name, in_c=args.in_c, n_classes=n_classes) model = model.to(device) print("Total params: {0:,}".format( sum(p.numel() for p in model.parameters() if p.requires_grad))) optimizer = torch.optim.Adam(model.parameters(), lr=max_lr) ### TRAINING WITH PSEUDO-LABELS csv_test = args.csv_test path_test_preds = args.path_test_preds checkpoint_folder = args.checkpoint_folder if csv_test is not None: print( 'Training with pseudo-labels, completing training set with predictions on test set' ) from utils.get_loaders import build_pseudo_dataset
mask = get_fov(img) print('* FOV mask generated') else: mask = Image.open(mask_path).convert('L') mask = np.array(mask).astype(bool) img, coords_crop = crop_to_fov(img, mask) original_sz = img.size[1], img.size[0] # in numpy convention rsz = p_tr.Resize(tg_size) tnsr = p_tr.ToTensor() tr = p_tr.Compose([rsz, tnsr]) im_tens = tr(img) # only transform image print('* Instantiating model = ' + str(model_name)) model = get_arch(model_name).to(device) if model_name == 'wnet': model.mode = 'eval' print('* Loading trained weights from ' + model_path) model, stats = load_model(model, model_path, device) model.eval() print('* Saving prediction to ' + im_path_out) start_time = time.perf_counter() full_pred, full_pred_bin = create_pred(model, im_tens, mask, coords_crop, original_sz, bin_thresh=bin_thresh, tta=tta)
load_path_f_f2 = 'experiments/best_f_f2' load_path_f_f3 = 'experiments/best_f_f3' load_path_f_f4 = 'experiments/best_f_f4' load_path_c_f1 = 'experiments/best_c_f1' load_path_c_f2 = 'experiments/best_c_f2' load_path_c_f3 = 'experiments/best_c_f3' load_path_c_f4 = 'experiments/best_c_f4' # #################################################################################################################### # # build results for MT model n_classes = 18 # F1 print('* Instantiating MT model {}, pretrained={}, fold 1/MT'.format(get_model_name(load_path_MT_f1), pretrained)) model, mean, std = get_arch(get_model_name(load_path_MT_f1), pretrained=pretrained, n_classes=n_classes) model, stats = load_model(model, load_path_MT_f1, device='cpu') model = model.to(device) print("Total params: {0:,}".format(sum(p.numel() for p in model.parameters() if p.requires_grad))) print('* Creating Dataloaders, batch size = {:d}'.format(bs)) test_loader = get_test_loader(csv_path_test=csv_test_q_MT, batch_size=bs, mean=mean, std=std, qualities=True) probs_qMT_f1, preds_qMT_f1, probs_aMT_f1, preds_aMT_f1, probs_cMT_f1, preds_cMT_f1, probs_fMT_f1, preds_fMT_f1\ = test_cls_tta_dihedral_MT(model, test_loader, n=TTA_N) # F2 print('* Instantiating MT model {}, pretrained={}, fold 2/MT'.format(get_model_name(load_path_MT_f2), pretrained)) model, mean, std = get_arch(get_model_name(load_path_MT_f2), pretrained=pretrained, n_classes=n_classes) model, stats = load_model(model, load_path_MT_f2, device='cpu') model = model.to(device) print("Total params: {0:,}".format(sum(p.numel() for p in model.parameters() if p.requires_grad))) print('* Creating Dataloaders, batch size = {:d}'.format(bs)) test_loader = get_test_loader(csv_path_test=csv_test_q_MT, batch_size=bs, mean=mean, std=std, qualities=True)