def main(): args = get_argparser() # our dataset has two class classes = utils.parse_config(args.config_path) print(len(classes), args.num_classes, classes) assert len(classes) + 1 == args.num_classes, "Number of classes\ in config and argument is not same" # use our dataset and defined transformations dataset = loader.CellDataset(args.root_dir, utils.get_transform(args.model, train=True), args.labels_type, args.model, classes) dataset_test = loader.CellDataset(args.root_dir, utils.get_transform(args.model, train=False), args.labels_type, args.model, classes, mode="Test") indices = torch.arange(len(dataset)).tolist() dataset = torch.utils.data.Subset(dataset, indices[:int(len(indices) * 0.9)]) dataset_test = torch.utils.data.Subset(dataset_test, indices[int(len(indices) * 0.9):]) print("Images in Test set", len(dataset_test), "Images in Train set ", len(dataset)) # define training and validation data loaders data_loader = torch.utils.data.DataLoader(dataset, batch_size=args.batch_size, shuffle=True, num_workers=4, collate_fn=utils.collate_fn) data_loader_test = torch.utils.data.DataLoader(dataset_test, batch_size=1, shuffle=True, num_workers=4, collate_fn=utils.collate_fn) model = models.get_model(args.model, args.weight_path, args.num_classes, args.max_instances, args.maskrcnn_backbone) if args.cuda: device = "cuda:0" model.to(device) else: device = "cpu" # optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, # momentum=0.9, weight_decay=0.0005) print("\n\nStarting Training of ", args.model, "\n\n") optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) model_trainer = trainer.TrainModel(model, optimizer, args.model, device) for epoch in range(args.epochs): model_trainer.train(epoch, data_loader, data_loader_test) print("That's it!")
def main(model_name): model = choose_net(name=model_name, num_classes=num_class, weight_path='github') model.to(device) if torch.cuda.device_count() > 1: net = nn.DataParallel(model) optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate, amsgrad=True) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, Epoches, eta_min=1e-6) # cheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.9, patience=2) kf = KFold(n_splits=5, shuffle=True) for fold, (train_idx, val_idx) in enumerate(kf.split(df)): print(f'fold:{fold+1}...', 'train_size: %d, val_size: %d' % (len(train_idx), len(val_idx))) df_train = df.values[train_idx] df_val = df.values[val_idx] train_dataset = MyData(root=Data_path, df=df_train, phase='train', transform=get_transform(image_size, 'train')) val_dataset = MyData(root=Data_path, df=df_val, phase='test', transform=get_transform(image_size, 'test')) train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=workers, pin_memory=True) val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, drop_last=True) best_acc = 0.0 for epoch in range(Epoches): print('Train {} / {}'.format(epoch + 1, Epoches)) train_loss = train(model, train_loader, optimizer) if isinstance(scheduler, torch.optim.lr_scheduler.ReduceLROnPlateau): scheduler.step(train_loss) else: scheduler.step(epoch) if epoch % 5 == 0: acc = validate(model, val_loader) if acc > best_acc: if torch.cuda.device_count() > 1: torch.save( model.module.state_dict(), Model_path + '/' + f"{model_name}_best_fold{fold + 1}.pth") else: torch.save( model.state_dict(), Model_path + '/' + f"{model_name}_best_fold{fold + 1}.pth")
def augmentBatch3(batch, config): """Shifts the polar radar image by a random amount, does NOT adjust ground truth transform. The keypoints must be unrotated later using the T_aug transform stored in the batch dict. """ rot_max = config['augmentation']['rot_max'] data = batch['data'].numpy() polar = batch['polar'].numpy() azimuths = batch['azimuths'].numpy() mask = batch['mask'].numpy() azimuth_res = 0.9 * np.pi / 180 B, C, H, W = data.shape T_aug = [] for i in range(B): if np.mod(i, config['window_size']) == 0: continue plr = polar[i].squeeze() azm = azimuths[i].squeeze() rot = np.random.uniform(-rot_max, rot_max) rot_azms = int(np.round(rot / azimuth_res)) rot = rot_azms * azimuth_res plr = np.roll(plr, -1 * rot_azms, axis=0) cart = radar_polar_to_cartesian(azm, plr, config['radar_resolution'], config['cart_resolution'], config['cart_pixel_width']) # 1 x H x W data[i] = cart[0] polar_mask = mean_intensity_mask(plr) msk = radar_polar_to_cartesian(azm, polar_mask, config['radar_resolution'], config['cart_resolution'], config['cart_pixel_width']).astype(np.float32) mask[i] = msk[0] T_aug += [torch.from_numpy(get_transform(0, 0, -rot))] polar[i] = plr batch['data'] = torch.from_numpy(data) batch['polar'] = torch.from_numpy(polar) batch['mask'] = torch.from_numpy(mask > 0.5).type(batch['data'].dtype) # make into a binary mask batch['T_aug'] = T_aug return batch
def create_coco_loader(*paths): transform = utils.get_transform(config.image_size, config.central_fraction) datasets = [data.CocoImages(path, transform=transform) for path in paths] dataset = data.Composite(*datasets) data_loader = torch.utils.data.DataLoader( dataset, batch_size=config.preprocess_batch_size, num_workers=config.data_workers, shuffle=False, pin_memory=True, ) return data_loader
def main(): logging.basicConfig( format= '%(asctime)s: %(levelname)s: [%(filename)s:%(lineno)d]: %(message)s', level=logging.INFO) warnings.filterwarnings("ignore") try: ckpt = '../input/efefb2b2/weights/model_b2_386.ckpt' except: print('Set ckpt for evaluation in config.py') return test_dataset = FGVC7Data( root=datasets, phase='test', transform=get_transform([config.image_size[0], config.image_size[1]], 'test')) test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=2, pin_memory=True) num_classes = 5 net = efficientnet(net_name=net_name, num_classes=num_classes, weight_path='kaggle') checkpoint = torch.load(ckpt) state_dict = checkpoint['state_dict'] # Load weights net.load_state_dict(state_dict) print('Network loaded from {}'.format(ckpt)) net = net.to(device) preds = [] image_name = [] with torch.no_grad(): net.eval() pbar = tqdm(total=len(test_loader), unit=' batches') pbar.set_description('Validation') for i, input in enumerate(test_loader): X, _, img_name = input y_pred, y_metric = net(X.to(device)) pred = F.softmax(y_pred, dim=1).cpu().numpy() preds.extend(pred.argmax(1)) image_name.append(img_name[0]) sub = pd.DataFrame({'image_id': image_name, 'label': preds}) print(sub) sub.to_csv("submission.csv", index=False)
def main(file_name, log): set_seed(cfg.SOLVER.SEED) config_file = './configs/' + file_name cfg.merge_from_file(config_file) # os.environ["CUDA_VISIBLE_DEVICES"] = cfg.MODEL.DEVICE_ID USE_CUDA = torch.cuda.is_available() device = torch.device("cuda:0" if USE_CUDA else "cpu") weight_path = cfg.MODEL.MODEL_PATH + cfg.MODEL.NAME + '.pth' model = choose_net(name=cfg.MODEL.NAME, num_classes=cfg.MODEL.CLASSES, weight_path=cfg.MODEL.WEIGHT_FROM) best_acc = 0.0 log.info('Train : {}'.format(cfg.MODEL.NAME)) if os.path.exists(weight_path): checkpoint = torch.load(weight_path) state_dict = checkpoint['state_dict'] best_acc = checkpoint['best_acc'] model.load_state_dict(state_dict) log.info('Network loaded from {}'.format(weight_path)) model.to(device) # model.cuda() if torch.cuda.device_count() > 1: model = nn.DataParallel(model, device_ids=range(torch.cuda.device_count())) optimizer = torch.optim.AdamW(model.parameters(), lr=cfg.SOLVER.BASE_LR, amsgrad=True) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, cfg.SOLVER.MAX_EPOCHS, eta_min=1e-6) train_dataset = FGVC7Data(root=cfg.DATASETS.ROOT_DIR, phase='train', transform=get_transform(cfg.INPUT.SIZE_TRAIN, 'train')) indices = range(len(train_dataset)) split = int(cfg.DATASETS.SPLIT * len(train_dataset)) train_indices = indices[split:] test_indices = indices[:split] train_sampler = SubsetRandomSampler(train_indices) valid_sampler = SubsetRandomSampler(test_indices) train_loader = DataLoader(train_dataset, batch_size=cfg.DATASETS.BATCH_SIZE, sampler=train_sampler, num_workers=cfg.DATASETS.WORKERS, pin_memory=True) val_loader = DataLoader(train_dataset, batch_size=cfg.DATASETS.BATCH_SIZE, sampler=valid_sampler, num_workers=cfg.DATASETS.WORKERS, pin_memory=True) for epoch in range(cfg.SOLVER.MAX_EPOCHS): # pbar = tqdm(total=len(train_loader), unit='batches', ncols=150) # unit 表示迭代速度的单位 # pbar.set_description('Epoch {}/{}'.format(epoch + 1, cfg.SOLVER.MAX_EPOCHS)) train(model, optimizer, epoch, train_loader, log) scheduler.step() if (epoch+1) % 5 == 0: acc = validate(model, val_loader, epoch, log) if acc > best_acc: if torch.cuda.device_count()>1: torch.save({'best_acc':best_acc, 'state_dict':model.module.state_dict()}, weight_path) else: torch.save({'best_acc':best_acc, 'state_dict':model.state_dict()}, weight_path)
def augmentBatch2(batch, config): """Rotates the cartesian radar image by a random amount, does NOT adjust ground truth transform. The keypoints must be unrotated later using the T_aug transform stored in the batch dict. """ rot_max = config['augmentation']['rot_max'] data = batch['data'].numpy() # this seems to return a reference, not a copy mask = batch['mask'].numpy() B, C, H, W = data.shape T_aug = [] for i in range(B): if np.mod(i, config['window_size']) == 0: continue img = data[i].squeeze() mmg = mask[i].squeeze() rot = np.random.uniform(-rot_max, rot_max) M = cv2.getRotationMatrix2D((W / 2, H / 2), rot * 180 / np.pi, 1.0) data[i] = cv2.warpAffine(img, M, (W, H), flags=cv2.INTER_CUBIC).reshape(C, H, W) mask[i] = cv2.warpAffine(mmg, M, (W, H), flags=cv2.INTER_CUBIC).reshape(C, H, W) T_aug += [torch.from_numpy(get_transform(0, 0, -rot))] batch['data'] = torch.from_numpy(data) batch['mask'] = torch.from_numpy(mask > 0.5).type(batch['data'].dtype) # make into a binary mask batch['T_aug'] = T_aug return batch
def get_groundtruth_odometry(self, radar_time, gt_path): """Retrieves the groundtruth 4x4 transform from current time to next Args: radar_time (int): UNIX INT64 time stamp that we want groundtruth for (also the filename for radar) gt_path (AnyStr): path to the ground truth csv file Returns: np.ndarray: 4x4 transformation matrix from current time to next (T_2_1) """ def parse(gps_line): out = [float(x) for x in gps_line.split(',')] out[0] = int(gps_line.split(',')[0]) return out gtfound = False min_delta = 0.1 T_2_1 = np.identity(4, dtype=np.float32) with open(gt_path, 'r') as f: f.readline() lines = f.readlines() for i in range(len(lines) - 1): gt1 = parse(lines[i]) delta = abs(float(gt1[0] - radar_time) / 1.0e9) if delta < min_delta: gt2 = parse(lines[i + 1]) T_enu_r1 = np.matmul(get_transform_boreas(gt1), T_prime) T_enu_r2 = np.matmul(get_transform_boreas(gt2), T_prime) T_r2_r1 = np.matmul(get_inverse_tf(T_enu_r2), T_enu_r1) # 4x4 SE(3) heading, _, _ = rotToYawPitchRoll(T_r2_r1[0:3, 0:3]) T_2_1 = get_transform(T_r2_r1[0, 3], T_r2_r1[1, 3], heading) # 4x4 SE(2) min_delta = delta gtfound = True assert ( gtfound), 'ground truth transform for {} not found in {}'.format( radar_time, gt_path) return T_2_1
def augmentBatch(batch, config): """Rotates the cartesian radar image by a random amount, adjusts the ground truth transform accordingly.""" rot_max = config['augmentation']['rot_max'] batch_size = config['batch_size'] window_size = config['window_size'] data = batch['data'].numpy() mask = batch['mask'].numpy() T_21 = batch['T_21'].numpy() _, C, H, W = data.shape for i in range(batch_size): rot = np.random.uniform(-rot_max, rot_max) T = get_transform(0, 0, rot) for j in range(1, window_size): k = j + i * window_size img = data[k].squeeze() mmg = mask[k].squeeze() M = cv2.getRotationMatrix2D((W / 2, H / 2), rot * 180 * j / np.pi, 1.0) data[k] = cv2.warpAffine(img, M, (W, H), flags=cv2.INTER_CUBIC).reshape(C, H, W) mask[k] = cv2.warpAffine(mmg, M, (W, H), flags=cv2.INTER_CUBIC).reshape(1, H, W) T_21[k - 1] = np.matmul(T, T_21[k - 1]) batch['data'] = torch.from_numpy(data) batch['mask'] = torch.from_numpy(mask > 0.5).type(batch['data'].dtype) # make into a binary mask batch['T_21'] = torch.from_numpy(T_21) return batch
def main(model_name): ################################## # Initialize saving directory ################################## if not os.path.exists(config.save_dir): os.makedirs(config.save_dir) ################################## # Logging setting ################################## logging.basicConfig( stream=sys.stdout, level=logging.INFO, # 设置输出级别 format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') ################################## # Load dataset TODO: 10-fold cross validation ################################## train_dataset = FGVC7Data(root=args.datasets, phase='train', transform=get_transform(config.image_size, 'train')) indices = range(len(train_dataset)) split = int(0.3 * len(train_dataset)) train_indices = indices[split:] test_indices = indices[:split] #train_sampler = SubsetRandomSampler(train_indices) valid_sampler = SubsetRandomSampler(test_indices) train_loader = DataLoader(train_dataset, batch_size=config.batch_size, shuffle=True, num_workers=config.workers, pin_memory=True) validate_loader = DataLoader(train_dataset, batch_size=config.batch_size, sampler=valid_sampler, num_workers=config.workers, pin_memory=True) num_classes = 4 print('Train Size: {}'.format(len(train_indices))) print('Valid Size: {}'.format(len(test_indices))) ################################## # Initialize model ################################## logs = {} # 有 lr, epoch, val_loss start_epoch = 0 num_classes = 5 net = efficientnet(net_name=model_name, num_classes=num_classes, weight_path='github') if config.ckpt: # Load ckpt and get state_dict checkpoint = torch.load(config.ckpt) # Get epoch and some logs logs = checkpoint['logs'] start_epoch = int(logs['epoch']) # Load weights state_dict = checkpoint['state_dict'] net.load_state_dict(state_dict) logging.info('Network loaded from {}'.format(config.ckpt)) #net.re_init() logging.info('Network weights save to {}'.format(config.save_dir)) ################################## # Use cuda ################################## net.to(device) if torch.cuda.device_count() > 1: net = nn.DataParallel(net) ################################## # Optimizer, LR Schedulerextract_features(img) ################################## learning_rate = logs['lr'] if 'lr' in logs else config.learning_rate #optimizer = torch.optim.SGD(net.parameters(), lr=learning_rate, momentum=0.9, weight_decay=1e-5) optimizer = torch.optim.AdamW(net.parameters(), lr=learning_rate, amsgrad=True) #scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.9, patience=2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, config.epochs, eta_min=1e-6) ################################## # ModelCheckpoint ################################## callback_monitor = 'val_{}'.format(raw_metric.name) # topk_accuracy callback = ModelCheckpoint(savepath=os.path.join(config.save_dir, config.model_name), monitor=callback_monitor, mode='max') if callback_monitor in logs: callback.set_best_score(logs[callback_monitor]) else: callback.reset() ################################## # TRAINING ################################## logging.info( 'Start training: Total epochs: {}, Batch size: {}, Training size: {}, Validation size: {}' .format(config.epochs, config.batch_size, len(train_indices), len(test_indices))) logging.info('') for epoch in range(start_epoch, config.epochs): callback.on_epoch_begin() logs['epoch'] = epoch + 1 logs['lr'] = optimizer.param_groups[0]['lr'] logging.info('Epoch {:03d}, LR {:g}'.format( epoch + 1, optimizer.param_groups[0]['lr'])) # 每一个epoch都显示一个进度条 pbar = tqdm(total=len(train_loader), unit='batches') # unit 表示迭代速度的单位 pbar.set_description('Epoch {}/{}'.format(epoch + 1, config.epochs)) train(logs=logs, data_loader=train_loader, net=net, optimizer=optimizer, pbar=pbar) validate(logs=logs, data_loader=validate_loader, net=net, pbar=pbar) if isinstance(scheduler, torch.optim.lr_scheduler.ReduceLROnPlateau): scheduler.step(logs['val_loss']) else: scheduler.step(epoch) callback.on_epoch_end(logs, net) pbar.close()
self.transform = transform def __getitem__(self, item): id = self.ids[item] img_name = self.images_name[item] image = cv2.imread(id) # (C, H, W) image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) label = self.labels[item] if self.phase == 'train' else 0 if self.transform != None: image, _ = self.transform(image, label) return image, label, img_name if __name__ == '__main__': import sys sys.path.append(os.path.abspath('./')) from utils.utils import get_transform data_path = r'F:\competion\6\cassava-leaf-disease-classification' # data_path = r'/home/gongke/data/cassava-leaf-disease-classification' # data_path = r'../data/cassava-leaf-disease-classification' dataset = FGVC7Data(data_path, transform=get_transform((448, 448), 'train'), phase='test') from torch.utils.data import DataLoader loader = DataLoader(dataset, batch_size=1) d = [] for i, input in enumerate(loader): x, _, y = input d.append(y[0]) sub = pd.DataFrame({'image_id': d}) print(sub)