'''model''' model = se_resnet101_xuelang(num_classes=160) # model =resnet50(pretrained=False) # model.avgpool = torch.nn.AdaptiveAvgPool2d(output_size=1) # model.fc = torch.nn.Linear(model.fc.in_features,2) # model = inceptionv4(num_classes=160) model = torch.nn.DataParallel(model) base_lr =0.001 resume = None if resume: logging.info('resuming finetune from %s'%resume) model.load_state_dict(torch.load(resume)) model = model.cuda() optimizer = optim.SGD(model.parameters(), lr=base_lr, momentum=0.9, weight_decay=1e-5) criterion = CrossEntropyLoss() exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=6, gamma=0.1) train(model, epoch_num=50, start_epoch=0, optimizer=optimizer, criterion=criterion, exp_lr_scheduler=exp_lr_scheduler, data_set=data_set, data_loader=dataloader, save_dir=save_dir, print_inter=50, val_inter=400)
model.load_state_dict(torch.load(opt.resume)) model = torch.nn.DataParallel(model) model = model.cuda() if opt.optim == "sgd": optimizer = optim.SGD(model.parameters(), lr=opt.learning_rate, momentum=0.9, weight_decay=1e-5) elif opt.optim == "adam": optimizer = optim.Adam(model.parameters(), lr=opt.learning_rate, weight_decay=1e-5) if opt.loss == "CrossEntropyLoss": criterion = CrossEntropyLoss() else: criterion = FocalLoss(class_num=61) exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=6, gamma=0.1) train(model, epoch_num=opt.epochs, start_epoch=opt.start_epoch, optimizer=optimizer, criterion=criterion, exp_lr_scheduler=exp_lr_scheduler, data_set=data_set, data_loader=dataloader, save_dir=save_dir, print_inter=opt.print_interval, val_inter=opt.save_checkpoint_val_interval)
def main(args): # -------------------------------------------------------------------------- # DATA logger.info('-' * 100) logger.info('Load data files') train_exs = util.load_data(args, args.train_file) logger.info('Num train examples = %d' % len(train_exs)) dev_exs = util.load_data(args, args.dev_file) logger.info('Num dev examples = %d' % len(dev_exs)) #test_exs = util.load_data(args, args.test_file) #logger.info('Num dev examples = %d' % len(test_exs)) # -------------------------------------------------------------------------- # MODEL logger.info('-' * 100) start_epoch = 0 if args.checkpoint and os.path.isfile(args.model_file + '.checkpoint'): # Just resume training, no modifications. logger.info('Found a checkpoint...') checkpoint_file = args.model_file + '.checkpoint' model, start_epoch = DocReader.load_checkpoint(checkpoint_file, args) else: # Training starts fresh. But the model state is either pretrained or # newly (randomly) initialized. if args.pretrained: logger.info('Using pretrained model...') model = DocReader.load(args.pretrained, args) if args.expand_dictionary: logger.info('Expanding dictionary for new data...') # Add words in training + dev examples words = util.load_words(args, train_exs + dev_exs) added_words = model.expand_dictionary(words) # Load pretrained embeddings for added words if args.words_embedding_file: model.load_embeddings(added_words, args.words_embedding_file) logger.info('Expanding char dictionary for new data...') # Add words in training + dev examples else: logger.info('Training model from scratch...') model = util.init_from_scratch(args, train_exs, dev_exs) # Set up partial tuning of embeddings if args.tune_partial > 0: logger.info('-' * 100) logger.info('Counting %d most frequent question words' % args.tune_partial) top_words = util.top_words(args, train_exs, model.word_dict) for word in top_words[:5]: logger.info(word) logger.info('...') for word in top_words[-6:-1]: logger.info(word) model.tune_embeddings([w[0] for w in top_words]) # Set up optimizer model.init_optimizer() # Use the GPU? if args.cuda: model.cuda() # Use multiple GPUs? if args.parallel: model.parallelize() # -------------------------------------------------------------------------- # DATA ITERATORS # Three datasets: train and dev. If we sort by length it's faster. logger.info('-' * 100) logger.info('Make data loaders') train_dataset = data.ReaderDataset(train_exs, model) if args.sort_by_len: train_sampler = data.SortedBatchSampler(train_dataset.lengths(), args.batch_size, shuffle=True) else: train_sampler = torch.utils.data.sampler.RandomSampler(train_dataset) train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=args.batch_size, sampler=train_sampler, num_workers=args.data_workers, collate_fn=vector.batchify, # pin_memory= args.cuda, ) dev_dataset = data.ReaderDataset(dev_exs, model) if args.sort_by_len: dev_sampler = data.SortedBatchSampler(dev_dataset.lengths(), args.dev_batch_size, shuffle=False) else: dev_sampler = torch.utils.data.sampler.SequentialSampler(dev_dataset) dev_loader = torch.utils.data.DataLoader( dev_dataset, batch_size=args.dev_batch_size, sampler=dev_sampler, num_workers=args.data_workers, collate_fn=vector.batchify, # pin_memory=args.cuda, ) # ------------------------------------------------------------------------- # PRINT CONFIG logger.info('-' * 100) logger.info('CONFIG:\n%s' % json.dumps(vars(args), indent=4, sort_keys=True)) # -------------------------------------------------------------------------- # -------------------------------------------------------------------------- # TRAIN/VALID LOOP logger.info('-' * 100) logger.info('Starting training...') stats = { 'timer': tool.Timer(), 'epoch': 0, 'best_f1_score': 0, 'best_em_score': 0 } train_saver = tool.DataSaver(args.model_name, "train") dev_saver = tool.DataSaver(args.model_name, "dev") for epoch in range(start_epoch, args.num_epochs): stats['epoch'] = epoch # Train train(args, train_loader, model, stats, train_saver) # Validate unofficial (train) validate_official(args, train_loader, model, stats, train_saver) # Validate unofficial (dev) result = validate_official(args, dev_loader, model, stats, dev_saver) # Save best valid if args.valid_metric is None or args.valid_metric == 'no': model.save(args.model_file) # {'exact_match': exact_match.avg,"f1_score":f1_score_avg.avg} if result['exact_match'] > stats['best_em_score']: stats['best_em_score'] = result['exact_match'] if result['f1_score'] > stats['best_f1_score']: stats['best_f1_score'] = result['f1_score'] logger.info( 'Best f1_score = %.2f Best em_score: = %.2f (epoch %d, %d updates)' % (stats['best_f1_score'], stats['best_em_score'], stats['epoch'], model.updates)) model.save(args.model_file) # save trained data train_saver.save(args.model_dir) dev_saver.save(args.model_dir) return model
lr=base_lr, weight_decay=weight_decay, momentum=0.9) criterion = CrossEntropyLoss() exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=15, gamma=0.33) #exp_lr_scheduler = lr_scheduler.ReduceLROnPlateau(optimizer,mode='min',factor=0.5,patience=4) #exp_lr_scheduler = lr_scheduler.MultiStepLR(optimizer, milestones = [10,20,30,40], gamma=0.1) #exp_lr_scheduler = lr_scheduler.CosineAnnealingLR(optimizer,T_max=5,eta_min=4e-08) iter_per_epoch = len(data_set['train']) // opt.batchsize #print (iter_per_epoch) train(model, model_name=opt.model + '_' + str(opt.width) + '_' + str(opt.start) + '_' + str(opt.cn), epoch_num=100, start_epoch=start_epoch, optimizer=optimizer, criterion=criterion, exp_lr_scheduler=exp_lr_scheduler, data_set=data_set, data_loader=dataloader, save_dir=opt.save_dir, cls_number=cls_number, print_inter=iter_per_epoch // 4, val_inter=iter_per_epoch, mixup=opt.mixup, label_smoothing=opt.label_smoothing, focal_loss=opt.focal_loss) torch.cuda.empty_cache()
def train_main(x_train,x_test,y_train,y_test,model_times=0): if opt.process: #x_train = x_train[:1000] #x_test = x_test[:100] print np.mean(x_train),np.mean(x_test),np.min(x_train),np.min(x_test),np.max(x_train),np.max(x_test) x_train,x_test = process_data(x_train,x_test) print np.mean(x_train),np.mean(x_test),np.min(x_train),np.min(x_test),np.max(x_train),np.max(x_test) if opt.cn == 3: data_transforms = { 'train' : transforms.Compose([ transforms.ToPILImage(), transforms.RandomRotation(degrees=45,resample=Image.BICUBIC), #transforms.RandomRotation(degrees=30,resample=Image.BICUBIC), transforms.RandomHorizontalFlip(), transforms.RandomVerticalFlip(), #transforms.ColorJitter(brightness=0.2,contrast=0.2,saturation=0.2, hue=0.2), transforms.RandomResizedCrop(target_size,scale=(0.64,1.0)), #transforms.RandomResizedCrop(target_size,scale=(0.36,1.0)), transforms.ToTensor(), #transforms.Normalize(mean = (0.485, 0.456, 0.406), std = (0.229, 0.224, 0.225)) ]), 'val': transforms.Compose([ transforms.ToPILImage(), #transforms.Resize(org_size), #transforms.CenterCrop(target_size), transforms.ToTensor(), #transforms.Normalize(mean = (0.485, 0.456, 0.406), std = (0.229, 0.224, 0.225)) ]) } else: data_transforms = { 'train' : Compose([ #RandomRotate((0,45),bound=True), RandomRotate((0,45)), RandomHVShift(), RandomHflip(), RandomVflip(), RandomErasing(), #Resize((target_size,target_size)), RandomResizedCrop((target_size,target_size)), #Normalize() ]), 'val': Compose([ Resize((target_size,target_size)), #Normalize() #CenterCrop((target_size,target_size)), ]) } #traindir = r'/media/disk1/fordata/web_server/multiGPU/cccccc/cloud/train/' ##train_dir #train_dataset = datasets.ImageFolder(traindir,data_transforms['train']) #test_dataset = datasets.ImageFolder(traindir,data_transforms['val']) train_x = torch.stack([torch.Tensor(i) for i in x_train]) train_y = torch.Tensor(y_train) #train_y = torch.stack([torch.Tensor(i) for i in y_train]) val_x = torch.stack([torch.Tensor(i) for i in x_test]) val_y = torch.Tensor(y_test) #val_y = torch.stack([torch.Tensor(i) for i in y_test]) #train_dataset = torch.utils.data.TensorDataset(train_x,train_y) #valid_dataset = torch.utils.data.TensorDataset(val_x,val_y) train_dataset = myTensorDataset(train_x,train_y,data_transforms['train']) valid_dataset = myTensorDataset(val_x,val_y,data_transforms['val']) data_set = {} data_set['train'] = train_dataset data_set['val'] = valid_dataset dataloader = {} dataloader['train'] = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=16) dataloader['val'] = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, num_workers=16) ####print the mean and std of dataset #print (get_mean_and_std(train_dataset,cn=opt.cn)) #print (get_mean_and_std(valid_dataset,cn=opt.cn)) model,start_epoch = load_model(model_name=opt.model,resume=opt.resume,start_epoch=opt.start_epoch,cn=opt.cn, \ save_dir=opt.save_dir,width=opt.width,start=opt.start,cls_number=cls_number,avg_number=opt.avg_number, \ gpus=opt.gpus,model_times=model_times,kfold=opt.kfold) base_lr = opt.baselr weight_decay = opt.wd load_model_flag = False if load_model_flag: conv1_params = list(map(id, model.conv1.parameters())) fc_params = list(map(id, model.fc.parameters())) base_params = filter(lambda p: id(p) not in conv1_params + fc_params,model.parameters()) optimizer = optim.Adam([{'params': base_params},{'params': model.conv1.parameters(), 'lr': base_lr * 10}, \ {'params': model.fc.parameters(), 'lr': base_lr * 10} ], lr=base_lr, weight_decay=weight_decay, amsgrad=True) else: #optimizer = optim.Adam(model.parameters(), lr=base_lr, weight_decay=weight_decay, amsgrad=True) if opt.optimizer == 'Adam': optimizer = optim.Adam(model.parameters(), lr=base_lr, weight_decay=weight_decay, amsgrad=True) elif opt.optimizer == 'SGD': optimizer = optim.SGD(model.parameters(), lr=base_lr, weight_decay=weight_decay,momentum=0.9) criterion = CrossEntropyLoss() exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=15, gamma=0.33) #exp_lr_scheduler = lr_scheduler.ReduceLROnPlateau(optimizer,mode='min',factor=0.5,patience=4) #exp_lr_scheduler = lr_scheduler.MultiStepLR(optimizer, milestones = [10,20,30,40], gamma=0.1) #exp_lr_scheduler = lr_scheduler.CosineAnnealingLR(optimizer,T_max=5,eta_min=4e-08) iter_per_epoch = len(data_set['train']) // opt.batchsize #print (iter_per_epoch) model_name = opt.model + '_' + str(opt.width) + '_' + str(opt.start) + '_' + str(opt.cn) if opt.kfold > 1: model_name = str(model_times) + '_' + model_name train(model, model_name=model_name, end_epoch=opt.end_epoch, start_epoch=start_epoch, optimizer=optimizer, criterion=criterion, exp_lr_scheduler=exp_lr_scheduler, data_set=data_set, data_loader=dataloader, save_dir=opt.save_dir, cls_number=cls_number, print_inter=iter_per_epoch // 4, val_inter=iter_per_epoch, mixup=opt.mixup, label_smoothing=opt.label_smoothing, focal_loss=opt.focal_loss ) torch.cuda.empty_cache()
'lr': base_lr * 10 }, { 'params': model.module.classifier_swap.parameters(), 'lr': base_lr * 10 }, { 'params': model.module.Convmask.parameters(), 'lr': base_lr * 10 }, ], lr=base_lr, momentum=cfg['momentum']) criterion = CrossEntropyLoss() exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=cfg['step_size'], gamma=cfg['gamma']) train( cfg, model, epoch_num=cfg['epoch'], start_epoch=0, optimizer=optimizer, criterion=criterion, exp_lr_scheduler=exp_lr_scheduler, data_set=data_set, data_loader=dataloader, save_dir=save_dir, val_inter=int(cfg['numimage'] / cfg['batch_size']), )
# model.fc = torch.nn.Linear(model.fc.in_features,100) model = multiscale_resnet(num_class=100) base_lr = 0.001 resume = None if resume: logging.info('resuming finetune from %s' % resume) model.load_state_dict(torch.load(resume)) model = model.cuda() optimizer = optim.SGD(model.parameters(), lr=base_lr, momentum=0.9, weight_decay=1e-5) #optimizer = optim.SGD(model.parameters(), lr=base_lr, momentum=0.9, weight_decay=0) #optimizer = optim.Adam(model.parameters(), lr=base_lr, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=False) criterion = CrossEntropyLoss() exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=15, gamma=0.1) best_acc, best_model_wts = train( model, epoch_num=150, start_epoch=0, optimizer=optimizer, criterion=criterion, exp_lr_scheduler=exp_lr_scheduler, data_set=data_set, data_loader=dataloader, save_dir=save_dir, print_inter=50, val_inter=200, )