def main(): model = MODEL_DISPATCHER[BASE_MODEL](pretrained=True) model.to(DEVICE) train_dataset = TrainDataset( folds = TRAINING_FOLDS, img_height = IMAGE_HEIGHT, img_width = IMAGE_WIDTH, mean = MODEL_MEAN, std = MODEL_STD ) train_loader = torch.utils.data.DataLoader( dataset = train_dataset, batch_size = TRAIN_BATCH_SIZE, shuffle = True, num_workers = 4 ) valid_dataset = TrainDataset( folds = VALIDATION_FOLDS, img_height = IMAGE_HEIGHT, img_width = IMAGE_WIDTH, mean = MODEL_MEAN, std = MODEL_STD ) valid_loader = torch.utils.data.DataLoader( dataset = valid_dataset, batch_size = TEST_BATCH_SIZE, shuffle = False, num_workers = 4 ) optimizer = torch.optim.Adam(model.parameters(), lr = 1e-4) #optimizer = torch.optim.AdamW(model.parameters()) #optimizer =Over9000(model.parameters(), lr=2e-3, weight_decay=1e-3) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode="max", patience = 0,factor=0.3, verbose=True) early_stopping = EarlyStopping(patience=4, verbose=True) #base_dir = "Project/EducationProject/Bengali_Ai" model_name = "../save_model/{}_folds{}.bin".format(BASE_MODEL, VALIDATION_FOLDS) if torch.cuda.device_count()>1: model = nn.DataParallel(model) for epoch in range(EPOCHS): train_loss, train_score = train(train_dataset, train_loader, model, optimizer) val_loss, val_score = evaluate(valid_dataset, valid_loader, model,optimizer) scheduler.step(val_score) early_stopping(val_score, model, model_name) if early_stopping.early_stop: print("Early stopping") break
def __init__(self, model, cfg): if cfg.scale > 0: self.refiner = model(scale=cfg.scale,group=cfg.group) else: self.refiner = model(multi_scale=True, group=cfg.group) if cfg.loss_fn in ["MSE"]: self.loss_fn = nn.MSELoss() elif cfg.loss_fn in ["L1"]: self.loss_fn = nn.L1Loss() elif cfg.loss_fn in ["SmoothL1"]: self.loss_fn = nn.SmoothL1Loss() self.optim = optim.Adam( filter(lambda p: p.requires_grad, self.refiner.parameters()), cfg.lr) self.train_data = TrainDataset(cfg.train_data_path, scale=cfg.scale, size=cfg.patch_size) self.train_loader = DataLoader(self.train_data, batch_size=cfg.batch_size, num_workers=1, shuffle=True, drop_last=True) self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self.refiner = self.refiner.to(self.device) self.loss_fn = self.loss_fn self.cfg = cfg self.step = 0 self.writer = SummaryWriter(log_dir=os.path.join("runs", cfg.ckpt_name)) if cfg.verbose: num_params = 0 for param in self.refiner.parameters(): num_params += param.nelement() print("# of params:", num_params) os.makedirs(cfg.ckpt_dir, exist_ok=True)
def test(ckpt, gpu): """Run model testing""" model_info = torch.load(ckpt, map_location='cpu') model = Model(ckpt=ckpt, gpu=gpu).eval() if gpu: model = model.cuda() # dataset dataset = TrainDataset(phase='test', shape=model_info['input_size']) dataloader = DataLoader(dataset, batch_size=1) total = 0 correct = 0 for X, y in dataloader: total += 1 if gpu: X = X.cuda() with torch.no_grad(): logit, _, _ = model(X) logit = logit.squeeze() logit = torch.softmax(logit, dim=0) logit = logit.cpu().data.numpy() pred = np.argmax(logit) if pred == y[0].item(): correct += 1 print(correct, total, correct / total)
def main(opt): # モデル定義 model = resnet.resnet152(pretrained=True) if not os.path.exists(opt.output): os.makedirs(opt.output) if torch.cuda.is_available(): # GPUが利用可能か確認 device = 'cuda' else: device = 'cpu' print('device is {0}'.format(device)) model.to(device) model.fc = nn.Linear(2048, opt.ftclass) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9) transform = transform_images() training_data = TrainDataset(opt.input, transform) train_loader = torch.utils.data.DataLoader(training_data, batch_size=32, shuffle=True, num_workers=4, pin_memory=True) for epoch in range(50): train(epoch + 1, model, criterion, optimizer, train_loader, device) if (epoch + 1) % 2 == 0: savefile = os.path.join(opt.output, 'save_{:0>3}.pth'.format(epoch + 1)) torch.save(model.state_dict(), savefile)
def make_loader(df: pd.DataFrame, image_transform,regression=args.regression,shuffle=False,balanced=True) -> DataLoader: return DataLoader( TrainDataset(train_root, df, image_transform, debug=args.debug,regression=regression,balanced=balanced), shuffle=shuffle, batch_size=args.batch_size, num_workers=args.workers, )
def make_loader(df: pd.DataFrame, root, image_transform, name='train') -> DataLoader: if name == 'train': return DataLoader(TrainDatasetTriplet(root, df, debug=args.debug, name=name, imgsize=args.imgsize, class_num=N_CLASSES), shuffle=True, batch_size=args.batch_size, num_workers=args.workers, collate_fn=collate_TrainDatasetTriplet) else: return DataLoader( TrainDataset(root, df, debug=args.debug, name=name, imgsize=args.imgsize, class_num=N_CLASSES), shuffle=True, batch_size=args.batch_size, num_workers=args.workers, )
def make_loader(df: pd.DataFrame, image_transform) -> DataLoader: return DataLoader( TrainDataset(train_root, df, image_transform, debug=args.debug), shuffle=True, batch_size=args.batch_size, num_workers=args.workers, )
def create_multi_source_train_data_loader(args): training_records = broden_dataset.record_list['train'] # 0: object, part, scene # 1: material multi_source_iters = [] for idx_source in range(len(training_records)): dataset = TrainDataset(training_records[idx_source], idx_source, args, batch_per_gpu=args.batch_size_per_gpu) loader_object_part_scene = torchdata.DataLoader( dataset, batch_size=args.num_gpus, # we have modified data_parallel shuffle=False, # we do not use this param collate_fn=user_scattered_collate, num_workers=int(args.workers), drop_last=True, pin_memory=True) multi_source_iters.append(iter(loader_object_part_scene)) # sample from multi source nr_record = [len(records) for records in training_records] sample_prob = np.asarray(nr_record) / np.sum(nr_record) while True: # TODO(LYC):: set random seed. source_idx = np.random.choice(len(training_records), 1, p=sample_prob)[0] yield next(multi_source_iters[source_idx]), source_idx
def __init__(self, model, cfg): self.cfg = cfg self.scales = cfg.scales self.data_path = cfg.data_path self.data_names = cfg.data_names self.refiner = model().cuda() self.loss_fn = nn.L1Loss().cuda() init_param = list(self.refiner.entry.parameters()) + \ list(self.refiner.progression[0].parameters()) + \ list(self.refiner.to_rgb[0].parameters()) self.optim = optim.Adam(init_param, cfg.lr) self.train_data = TrainDataset(self.data_path, self.data_names, self.scales, size=cfg.patch_size) self.writer = SummaryWriter() num_params = 0 for param in self.refiner.parameters(): num_params += param.nelement() print("# of params:", num_params) self.step = 1 self.stage = 0 self.max_stage = len(cfg.scales) - 1
def make_loader(df: pd.DataFrame, root, image_transform, name='train') -> DataLoader: return DataLoader( TrainDataset(root, df, debug=args.debug, name=name, imgsize = args.imgsize), shuffle=True, batch_size=args.batch_size, num_workers=args.workers, )
def Train(rank, world_size, config, resume): setup(rank, world_size) torch.cuda.set_device(rank) # dist.init_process_group(backend='nccl', init_method='tcp://localhost:10006', rank=rank, world_size=world_size) batch_size = int(config['data_loader']["batch_size"] / world_size) train_dataset = TrainDataset(mode="train", path=config["path"], **config["data_set"]) train_sampler = DistributedSampler(train_dataset) train_loader = DataLoader(train_dataset, batch_size=batch_size, sampler=train_sampler, num_workers=8, pin_memory=True) val_dataset = TrainDataset(mode="val", path=config["path"], **config["data_set"]) val_sampler = DistributedSampler(val_dataset) val_loader = DataLoader(val_dataset, batch_size=batch_size, sampler=val_sampler, num_workers=8, pin_memory=True) test_dataset = TestDataset(path=config["path"], name=config["data_set"]["name"]) test_sampler = DistributedSampler(test_dataset) test_loader = DataLoader(test_dataset, batch_size=batch_size, sampler=test_sampler, num_workers=8, pin_memory=True, ) # MODEL # model = EfficientNet.from_name('efficientnet-b0', n_classes=1, pretrained=False).cuda() model = get_instance(models, 'model', config).to(rank) model = DDP(model, device_ids=[rank]) if rank == 0: wandb.init(project="retina-segmentation", config=config, sync_tensorboard=True, name=" UNet_oetd celoss3") logger.info('The patch number of train is %d' % len(train_dataset)) logger.info(f'\n{model}\n') # LOSS loss = get_instance(losses, 'loss', config) # TRAINING trainer = Trainer( model=model, rank=rank, mode="train", loss=loss, config=config, resume=resume, train_loader=train_loader, val_loader=val_loader, test_loader=test_loader ) trainer.train() cleanup()
def get_loader(df, cfg): dataset = TrainDataset(df, cfg.AUDIO_DURATION, cfg.LIKELIHOOD, cfg.NOISE) kwargs = { 'num_workers': cfg.NUM_WORKERS, 'pin_memory': True } if cfg.USE_CUDA else {} loader = DataLoader(dataset, cfg.BATCH_SIZE, **kwargs, drop_last=True) return loader
def get_loader(train_set, train_U2I, n_items, batch_size, cores=6): train_ds = TrainDataset(train_set, train_U2I, n_items) train_dl = DataLoader(train_ds, batch_size=batch_size, shuffle=True, num_workers=cores) return train_dl
def get_loader(source, target, state, opt, stride=None): num_blocks = len(source) rep_factor = 40 # Need to redo this so that only unseen blocks are in the testing train_length = opt.train_len * rep_factor infer_length = opt.infer_len * rep_factor if state == 'train': dataset = TrainDataset(source[0:opt.train_len], target[0:opt.train_len], opt.cube, train_length) sampler = SubsetRandomSampler(range(0, train_length)) return DataLoader(dataset, opt.trainBatchSize, sampler=sampler, num_workers=opt.threads) elif state == 'infer': dataset = TrainDataset(source[opt.infer_len:], target[opt.infer_len:], opt.cube, train_length) sampler = SequentialSampler(range(0, infer_length)) return DataLoader(dataset, opt.inferBatchSize, sampler=sampler, num_workers=opt.threads)
def get_ds(self, phase): assert phase in {"train", "valid"} transform = get_transform(conf_augmentation=self.cfg.Augmentation[phase]) return TrainDataset( conf_dataset=self.cfg.Data.dataset, phase=phase, out_ch=self.cfg.Model.out_channel, transform=transform, )
def __init__(self, model, cfg): if cfg.scale > 0: #there is only a scale, 201904081901 self.refiner = model(scale=cfg.scale, group=cfg.group) else: #there is mutile scales,201904081901 self.refiner = model(multi_scale=True, group=cfg.group) if cfg.loss_fn in ["MSE"]: self.loss_fn = nn.MSELoss() elif cfg.loss_fn in ["L1"]: self.loss_fn = nn.L1Loss() elif cfg.loss_fn in ["SmoothL1"]: self.loss_fn = nn.SmoothL1Loss() self.optim = optim.Adam( filter(lambda p: p.requires_grad, self.refiner.parameters()), cfg.lr) self.train_data = TrainDataset(cfg.train_data_path, scale=cfg.scale, size=cfg.patch_size) self.train_loader = DataLoader(self.train_data, batch_size=cfg.batch_size, num_workers=0, shuffle=True, drop_last=True) #the ways of chosen GPU #the first way os.environ['CUDA_VISIBLE_DEVICES'] = '0,1' self.device = torch.device( "cuda:0,1" if torch.cuda.is_available() else "cpu" ) #tcw201904100941, cuda:1 denotes the GPU of number 1. cuda:0 denotes the GPU of number 0. #automically choose the GPU, if torch.device("cuda" if torch.cuda.is_available() else "cpu") #"The second way is as follows--------------------------" #self.device = torch.device('cuda',1) #the commod is added by tcw 201904100942 #If torch.device('cuda',1), which chooses the GPU of number 1. If torch.device('cuda',0), which chooses the GPU of number 0. self.refiner = self.refiner.to( self.device) #load the model into the self.device self.loss_fn = self.loss_fn self.cfg = cfg self.step = 0 self.writer = SummaryWriter(log_dir=os.path.join( "runs", cfg.ckpt_name)) #log if cfg.verbose: num_params = 0 for param in self.refiner.parameters( ): #model.parameters keep the parameters from all the layers. num_params += param.nelement( ) #.nelement() can count the number of all the parameters. print("# of params:", num_params) if not os.path.exists(cfg.ckpt_dir): #201904072208 tcw #os.makedirs(cfg.ckpt_dir, exist_ok=True) #201904072211tcw, it is given at first, but it is wrong. So, I mark it. os.makedirs(cfg.ckpt_dir, mode=0o777) #2019072211tcw
def fetch_teacher_outputs(args): teacher_outputs = [] # Network Builders builder = ModelBuilder() net_encoder = builder.build_encoder(arch=args.arch_encoder, fc_dim=args.fc_dim, weights=args.weights_encoder) net_decoder = builder.build_decoder(arch=args.arch_decoder, fc_dim=args.fc_dim, num_class=args.num_class, weights=args.weights_decoder) crit = nn.NLLLoss(ignore_index=-1) segmentation_module = SegmentationModule(net_encoder, net_decoder, crit) # dataset and loader, use train dataset_train = TrainDataset(args.list_train, args, batch_per_gpu=args.batch_size_per_gpu) loader_train = torchdata.DataLoader( dataset_train, batch_size=args.num_gpus, # we have modified data_parallel shuffle=False, # we do not use this param collate_fn=user_scattered_collate, num_workers=int(args.workers), drop_last=True, pin_memory=True) segmentation_module.cuda() # here may be some problems for i, batch_data in enumerate(loader_train): batch_data = batch_data[0] # get data list seg_label = as_numpy(batch_data['seg_label'][0]) img_resized_list = batch_data['img_data'] with torch.no_grad(): segSize = (seg_label.shape[0], seg_label.shape[1]) # 预测输出的形状 pred = torch.zeros(1, args.num_class, segSize[0], segSize[1]) for img in img_resized_list: feed_dict = batch_data.copy() feed_dict['img_data'] = img del feed_dict['img_ori'] del feed_dict['info'] feed_dict = async_copy_to(feed_dict, args.gpu_id) # forward pass pred_tmp = segmentation_module(feed_dict, segSize=segSize) pred = pred + pred_tmp.cpu() / len(args.imgSize) _, preds = torch.max(pred, dim=1) preds.as_numpy(preds.squeeze(0)) teacher_outputs.append(preds) return teacher_outputs
def incremental_training(): device = torch.device( 'cuda') if torch.cuda.is_available() else torch.device('cpu') pretrained_model_file = "trained_models/res15_fine_grained.pt" fine_tuned_model_file = "trained_models/res15_fine_grained_fine_tuned.pt" num_utter = 5 num_word = 5 n_batches = 200 epoches = 3 model = Res15(n_labels=26) model.load(pretrained_model_file) model = model.to(device) # print(model) for index, (name, para) in enumerate(model.named_parameters()): if index in [14, 15]: para.requires_grad = True else: para.requires_grad = False print("{} Layer name: {}, Shape: {}, Requires_grad: {}".format( index, name, para.shape, para.requires_grad)) # optimizer step = 0 optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=0.001) criterion = GE2ELoss(device) # dataset train_dataset = TrainDataset("fine_tune_append.txt", "features/train", num_utter, num_word, n_batches) train_loader = DataLoader(dataset=train_dataset, batch_size=1, shuffle=True, num_workers=2) for epoch in range(epoches): model.train() for data in train_loader: step += 1 optimizer.zero_grad() data = data.squeeze( 0 ) # [1, num_utter * num_word, 101, 40] -> [num_utter * num_word, 101, 40] data = data.to(device) _, out = model(data) out = out.reshape(num_word, num_utter, -1) # [10, 10, 12] loss = criterion(out) loss.backward() optimizer.step() print("step #{} loss: {}".format(step, loss)) model.save(fine_tuned_model_file) print("Training Finished. Fine tuned model file are save at {}".format( fine_tuned_model_file))
def __init__(self, split='trainaug', dtype=None, val=True): torch.manual_seed(66) torch.cuda.manual_seed_all(66) self.log_dir = osp.join(settings.LOG_DIR, settings.EXP_NAME) self.model_dir = osp.join(settings.MODEL_DIR, settings.EXP_NAME) ensure_dir(self.log_dir) ensure_dir(self.model_dir) logger.info('set log dir as %s' % self.log_dir) logger.info('set model dir as %s' % self.model_dir) self.step = 1 self.best_mIoU = 0 self.writer = SummaryWriter(self.log_dir) self.split = split train_set = TrainDataset(split=split) self.train_loader = DataLoader(train_set, batch_size=settings.TRAIN_BATCH_SIZE, pin_memory=True, num_workers=settings.NUM_WORKERS, shuffle=True, drop_last=True) val_set = ValDataset(split='val') self.val_loader = DataLoader(val_set, batch_size=1, shuffle=False, num_workers=settings.NUM_WORKERS, drop_last=False) self.net = HamNet(settings.N_CLASSES, settings.N_LAYERS).cuda() params_count(self.net) self.opt = SGD(params=[{ 'params': get_params(self.net, key='1x'), 'lr': 1 * settings.LR, 'weight_decay': settings.WEIGHT_DECAY, }, { 'params': get_params(self.net, key='1y'), 'lr': 1 * settings.LR, 'weight_decay': 0, }, { 'params': get_params(self.net, key='2x'), 'lr': 2 * settings.LR, 'weight_decay': 0.0, }], momentum=settings.LR_MOM) self.net = DataParallel(self.net) patch_replication_callback(self.net)
def main(): print(outf) print("loading dataset ...") trainDataset = TrainDataset(name='/data0/niejiangtao/ICIP2019Deraining/train/train.h5') batchSize = opt.batchSize_per_gpu * len(device_ids) trainLoader = udata.DataLoader(trainDataset, batch_size=batchSize, shuffle=True, num_workers=0) testDataset = TestDataset2(name='/data0/niejiangtao/ICIP2019Deraining/test_a/test.h5') print('testDataset len : {}'.format(len(testDataset))) l1_criterion = nn.L1Loss().cuda() # mask_criterion = nn.MSELoss().cuda() ssim_criterion = SSIM().cuda() model = UNet_v2(n_channels=3, n_classes=3) # model = RESCAN() if len(device_ids) > 1: model = nn.DataParallel(model, device_ids=device_ids) model.cuda() beta1 = 0.9 beta2 = 0.999 optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=opt.lr, weight_decay=weight_decay, betas=(beta1, beta2)) writer = SummaryWriter(outf) for epoch in range(opt.epochs): start = time.time() current_lr = opt.lr / 2**int(epoch / interval) for param_group in optimizer.param_groups: param_group["lr"] = current_lr print("epoch {} learning rate {}".format(epoch, current_lr)) # test(model, testDataset, None, epoch, writer=writer) train_epoch(model, optimizer, trainLoader, l1_criterion, None, ssim_criterion, epoch, writer=writer, radio=radio) if (epoch+1) % 5 == 0: test(model, testDataset, None, epoch, writer=writer) if (epoch+1) % 20 == 0: """ torch.save({ 'epoch': epoch, 'model_state_dict': model.module.state_dict(), 'optimizer_state_dict': optimizer.state_dict() }, os.path.join(outf, 'checkpoint_{}.pth'.format(epoch))) """ torch.save(model.state_dict(), os.path.join(outf, 'model_{}.pth'.format(epoch))) end = time.time() print('epoch {} cost {} hour '.format( epoch, str((end - start) / (60 * 60)))) torch.save(model.state_dict(), os.path.join(outf, 'model.pth')) generate_result(model, outf, testDataset, mat=False, ouput_img=True)
def __init__(self, model, cfg): if cfg.scale > 0: self.refiner = model(scale=cfg.scale, group=cfg.group) #state_dict = torch.load("./checkpoint/carn_10000.pth") #new_state_dict = OrderedDict() #for k, v in state_dict.items(): # name = k # name = k[7:] # remove "module." # new_state_dict[name] = v #self.refiner.load_state_dict(new_state_dict) else: self.refiner = model(multi_scale=True, group=cfg.group) if cfg.loss_fn in ["MSE"]: self.loss_fn = nn.MSELoss() elif cfg.loss_fn in ["L1"]: self.loss_fn = nn.L1Loss() elif cfg.loss_fn in ["SmoothL1"]: self.loss_fn = nn.SmoothL1Loss() elif cfg.loss_fn in ["CrossEntropyLoss"]: self.loss_fn = nn.CrossEntropyLoss(ignore_index=19) self.optim = optim.Adam( filter(lambda p: p.requires_grad, self.refiner.parameters()), cfg.lr) self.train_data = TrainDataset(cfg.train_data_path, scale=cfg.scale, size=cfg.patch_size) self.train_loader = DataLoader(self.train_data, batch_size=cfg.batch_size, num_workers=1, shuffle=True, drop_last=True) self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") self.refiner = self.refiner.to(self.device) self.loss_fn = self.loss_fn self.cfg = cfg self.step = 0 self.writer = SummaryWriter( log_dir=os.path.join("runs", cfg.ckpt_name, str(cfg.scale))) if cfg.verbose: num_params = 0 for param in self.refiner.parameters(): num_params += param.nelement() print("# of params:", num_params) os.makedirs(cfg.ckpt_dir, exist_ok=True)
def __init__(self, src_domain, tgt_domain): self.num_epoch = 10 self.gamma = 1.0 print('construct dataset and dataloader...') train_dataset = TrainDataset(src_domain, tgt_domain) self.NEG_NUM = train_dataset.NEG_NUM self.input_dim = train_dataset.sample_dim self.train_loader = DataLoader(train_dataset, batch_size=32) print('Done!') self.feature_extractor = FeatureExtractor(self.input_dim) self.optimizer = optim.SGD(self.feature_extractor.parameters(), lr=0.1, momentum=0.9)
def get_dataloader(self, dir, name): if name == "train": dataset = TrainDataset(dir, self.image_size, aug_data=args.aug_data) a = DataLoader(dataset, batch_size=self.batch_size, shuffle=True, num_workers=self.num_workers, drop_last=True) self.total_step = len(a) return a elif name == "val": dataset = TestDataset(dir, self.image_size) return DataLoader(dataset, batch_size=1, shuffle=False, num_workers=self.num_workers, drop_last=True) else: print("Incorrect Name for Dataloader!!!") return 0
def load_dataset(args): img_transform = transforms.Compose([ transforms.Resize(256), transforms.RandomCrop(224), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) train_caption, validation_caption, test_caption, train_id, validation_id, val_img_id, test_img_id, img_idx, vocab_table = read_text_data( args) train_dataset = TrainDataset(args.img_dir, vocab_table, img_idx, train_caption, train_id, img_transform) val_img_dataset = ImgDataset(args.img_dir, val_img_id, is_val=True, transform=img_transform, img_idx=img_idx) test_img_dataset = ImgDataset(args.img_dir, test_img_id, transform=img_transform) val_caption_dataset = CaptionDataset(vocab_table, validation_caption, is_val=True, label=validation_id) test_caption_dataset = CaptionDataset(vocab_table, test_caption) train_loader = DataLoader(train_dataset, batch_size=args.batch_size, num_workers=4, shuffle=True, collate_fn=train_collate_fn) val_img_loader = DataLoader(val_img_dataset, batch_size=args.batch_size, num_workers=4) test_img_loader = DataLoader(test_img_dataset, batch_size=args.batch_size, num_workers=4) val_caption_loader = DataLoader(val_caption_dataset, batch_size=args.batch_size, num_workers=4, collate_fn=test_collate_fn) test_caption_loader = DataLoader(test_caption_dataset, batch_size=args.batch_size, num_workers=4, collate_fn=test_collate_fn) return train_loader, val_img_loader, test_img_loader, val_caption_loader, test_caption_loader, vocab_table
def train(self, train_df, val_df, seed, fold): if self.debug: train_df = train_df[:self.get("batch_size")+1] val_df = val_df[:self.get("batch_size")+1] self.params["epochs"] = 1 if self.raw_dirname == "cassava-leaf-disease-classification-merged": val_df = val_df[val_df["source"] == 2021] train_dataset = TrainDataset(train_df, self.data_path, get_transforms('train', self.get("tr_transform_params"), self.get("tr_transforms"))) val_dataset = TrainDataset(val_df, self.data_path, get_transforms('valid', self.get("val_transform_params"))) trainloader, validloader = get_dataloader(train_dataset, val_dataset, self.get("batch_size"), self.get("num_workers")) self.params["seed"] = seed self.params["fold"] = fold self.params["pretrained"] = True model = CassavaClassifierModel(self.params) if self.get("do_retrain"): model.read_weight() model.fit(trainloader, validloader) # valid predict val_preds = model.val_preds val_preds = pd.DataFrame(val_preds, columns=[f"pred_{n}" for n in range(self.params["output_size"])]) val_preds.to_csv(osp.join(self.val_preds_path, f"preds_{seed}_{fold}.csv"), index=False)
def init_train_data(self): batch_size = TRAINING_BATCH_SIZE if self.args.network_type == 'discriminator': batch_size = 1 train_folder = self.args.train_input train_dataset = TrainDataset(train_folder) self.train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) valid_folder = self.args.valid_input valid_dataset = ValidDataset(valid_folder) self.valid_dataloader = DataLoader(valid_dataset, batch_size=batch_size)
def train(model, train_data, train_labels, test_data, test_labels): train_dataset = TrainDataset(train_data, train_labels) train_dataloader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True, collate_fn=train_dataset.collate_fn, drop_last=True) optimizer = optim.Adam(params=model.parameters(), lr=LEARNING_RATE) for i in range(N_EPOCHS): loss = train_epoch(model, train_dataloader, optimizer) print('iteration {0}: average loss: {1}', i, loss) if i != 0 and i % 5 == 0: acc, predictions = evaluate_epoch(model, train_data, train_labels, test_data, test_labels) print('iteration {0}: accuracy: {1}', i, acc)
def check_training_dataset(): args = get_args() training_records = broden_dataset.record_list['train'] for idx_source in [1, 0]: dataset_train = TrainDataset(training_records[idx_source], args, batch_per_gpu=args.batch_size_per_gpu) for idx_record in range(50): print() print("**** source idx: {}, record idx: {} **** ".format( idx_source, idx_record)) print("record: {}".format( training_records[idx_source][idx_record])) batch_dict = dataset_train[idx_record] show_batches(batch_dict, use_imshow=False)
def get_dataloader(data_path, resolution, batch_size): dataset = TrainDataset( file_path=os.path.join(data_path, '*.png'), transform=transforms.Compose([ transforms.Resize(resolution), transforms.ToTensor(), TranformDynamicRange([0, 255], [-1, 1]) ]), ) dataloader = torch.utils.data.DataLoader( dataset=dataset, batch_size=batch_size, shuffle=True, ) return dataloader
def get_loaders(opt): utes, mask, label = load_train_data(opt) samps = opt.samples train_length = int(label.shape[0]) * samps train_dataset = TrainDataset(utes, mask, label, train_length, opt) train_sampler = SubsetRandomSampler(range(0, train_length)) train_loader = DataLoader(train_dataset, opt.trainBatchSize, sampler=train_sampler, num_workers=opt.threads) utes, mask, label = load_infer_data(opt) z_dims = utes.shape[-1] * utes.shape[0] infer_dataset = EvalDataset(utes, mask, label, z_dims) infer_sampler = SequentialSampler(range(0, z_dims - 2)) infer_loader = DataLoader(infer_dataset, opt.inferBatchSize, sampler=infer_sampler, num_workers=opt.threads) return train_loader, infer_loader