def load_unlabeled_data(tar, data_dir='dataset/'): from torchvision import transforms folder_tar = data_dir + tar + '/images' transform = { 'weak': transforms.Compose([ transforms.Resize([224, 224]), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]), 'strong': transforms.Compose([ transforms.Resize([256, 256]), transforms.RandomCrop(224), transforms.RandomHorizontalFlip(), RandAugment( ), # Paper: RandAugment: Practical data augmentation with no separate search transforms.ToTensor(), # Cutout(size=16), # (https://github.com/uoguelph-mlrg/Cutout/blob/master/util/cutout.py) transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) } unlabeled_weak_data = datasets.ImageFolder(root=folder_tar, transform=transform['weak']) unlabeled_strong_data = datasets.ImageFolder(root=folder_tar, transform=transform['strong']) return unlabeled_weak_data, unlabeled_strong_data
def __init__(self, data, targets=None, num_classes=None, transform=None, use_strong_transform=False, strong_transform=None, *args, **kwargs): """ Args data: x_data targets: y_data (if not exist, None) num_classes: number of label classes transform: basic transformation of data use_strong_transform: If True, this dataset returns both weakly and strongly augmented images. strong_transform: list of transformation functions for strong augmentation """ super(BasicDataset, self).__init__() self.data = data self.targets = targets self.num_classes = num_classes self.use_strong_transform = use_strong_transform self.transform = transform if use_strong_transform: if strong_transform is None: self.strong_transform = copy.deepcopy(transform) self.strong_transform.transforms.insert(0, RandAugment(3, 5)) else: self.strong_transform = strong_transform
def __init__(self, mean, std): self.weak = transforms.Compose([ transforms.RandomHorizontalFlip(), transforms.RandomCrop(size=32, padding=int(32*0.125), padding_mode='reflect')]) self.strong = transforms.Compose([ transforms.RandomHorizontalFlip(), transforms.RandomCrop(size=32, padding=int(32*0.125), padding_mode='reflect'), RandAugment(n=2, m=10)]) self.normalize = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean=mean, std=std)])
def main(): args = parser.parse_args() args.do_bottleneck_head = False # Setup model print('creating model...') model = create_model(args).cuda() if args.model_path: # make sure to load pretrained ImageNet model state = torch.load(args.model_path, map_location='cpu') filtered_dict = {k: v for k, v in state['model'].items() if (k in model.state_dict() and 'head.fc' not in k)} model.load_state_dict(filtered_dict, strict=False) print('done\n') # COCO Data loading instances_path_val = os.path.join(args.data, 'annotations/instances_val2014.json') instances_path_train = os.path.join(args.data, 'annotations/instances_train2014.json') # data_path_val = args.data # data_path_train = args.data data_path_val = f'{args.data}/val2014' # args.data data_path_train = f'{args.data}/train2014' # args.data val_dataset = CocoDetection(data_path_val, instances_path_val, transforms.Compose([ transforms.Resize((args.image_size, args.image_size)), transforms.ToTensor(), # normalize, # no need, toTensor does normalization ])) train_dataset = CocoDetection(data_path_train, instances_path_train, transforms.Compose([ transforms.Resize((args.image_size, args.image_size)), CutoutPIL(cutout_factor=0.5), RandAugment(), transforms.ToTensor(), # normalize, ])) print("len(val_dataset)): ", len(val_dataset)) print("len(train_dataset)): ", len(train_dataset)) # Pytorch Data loader train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True) val_loader = torch.utils.data.DataLoader( val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=False) # Actuall Training train_multi_label_coco(model, train_loader, val_loader, args.lr)
def create_data_loaders(args): data_path_train = os.path.join(args.data_path, 'imagenet21k_train') train_transform = transforms.Compose([ transforms.Resize((args.image_size, args.image_size)), CutoutPIL(cutout_factor=0.5), RandAugment(), transforms.ToTensor(), ]) data_path_val = os.path.join(args.data_path, 'imagenet21k_val') val_transform = transforms.Compose([ transforms.Resize((args.image_size, args.image_size)), transforms.ToTensor(), ]) train_dataset = ImageFolder(data_path_train, transform=train_transform) val_dataset = ImageFolder(data_path_val, transform=val_transform) print_at_master("length train dataset: {}".format(len(train_dataset))) print_at_master("length val dataset: {}".format(len(val_dataset))) sampler_train = None sampler_val = None if num_distrib() > 1: sampler_train = torch.utils.data.distributed.DistributedSampler( train_dataset) sampler_val = OrderedDistributedSampler(val_dataset) # Pytorch Data loader train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=sampler_train is None, num_workers=args.num_workers, pin_memory=True, sampler=sampler_train) val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers, pin_memory=False, sampler=sampler_val) train_loader = PrefetchLoader(train_loader) val_loader = PrefetchLoader(val_loader) return train_loader, val_loader
def get_transform(cfg): height = cfg.DATASET.HEIGHT width = cfg.DATASET.WIDTH normalize = T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) if cfg.DATASET.TYPE == 'multi_label': valid_transform = T.Compose([ T.Resize([height, width]), T.ToTensor(), # normalize, ]) if cfg.TRAIN.DATAAUG.TYPE == 'randaug': # train_transform = T.Compose([ # T.RandomApply([AutoAugment()], p=cfg.TRAIN.DATAAUG.AUTOAUG_PROB), # T.Resize((height, width), interpolation=3), # CutoutPIL(cutout_factor=0.5), # T.RandomHorizontalFlip(), # T.ToTensor(), # ]) train_transform = T.Compose([ T.Resize((height, width), interpolation=3), RandAugment(), CutoutPIL(cutout_factor=0.5), T.ToTensor(), ]) elif cfg.TRAIN.DATAAUG.TYPE == 'normal': train_transform = T.Compose([ T.Resize((height, width)), T.RandomHorizontalFlip(), T.ToTensor(), # T.RandomErasing(scale=(0.1, 0.5)) # normalize, ]) else: assert False, f'current {cfg.TRAIN.DATAAUG.TYPE} augmentation has not been achieved' else: assert False, 'xxxxxxxx' return train_transform, valid_transform
def augmentation(self, img, randaugment=False): if randaugment: img = Image.fromarray(img) return transforms.Compose([ RandAugment(), transforms.ToTensor(), transforms.RandomCrop(32), transforms.RandomHorizontalFlip(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ])(img) else: return transforms.Compose([ transforms.ToTensor(), transforms.RandomCrop(32), transforms.RandomHorizontalFlip(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ])(img)
from torch.utils.tensorboard import SummaryWriter from randaugment import RandAugment, Cutout # default `log_dir` is "runs" - we'll be more specific here writer = SummaryWriter('runs/effnetlite0_new') # In[3]: data_transforms = { 'train': transforms.Compose([ # transforms.RandomErasing(p=0.5, scale=(0.02, 0.33), ratio=(0.3, 3.3), value=0, inplace=False), # transforms.RandomCrop(size=32, # padding=int(32*0.125), # padding_mode='reflect'), RandAugment(), Cutout(16), transforms.RandomPerspective(distortion_scale=0.5, p=0.5, interpolation=3), transforms.ColorJitter(brightness=0.2, contrast=0, saturation=0, hue=0.), transforms.RandomAffine(degrees=0, translate=(0.2, 0.2)), transforms.ToTensor(), ]), 'valid': transforms.Compose([transforms.ToTensor()]), }
class SVHN_Model: def train_base_model(model_type='ResNext'): transform = transforms.Compose([#transforms.ToPILImage()]) transforms.ToTensor(), transforms.Normalize([.5,.5,.5],[.5,.5,.5]) ]) train_data = datasets.SVHN('Data/',split='train',download=True,transform=transform) test_data = datasets.SVHN('Data/',split='test',download=True,transform=transform) train_dataloader = torch.utils.data.DataLoader( train_data, batch_size=1024, ) test_dataloader = torch.utils.data.DataLoader( test_data, batch_size=1024, shuffle = False ) #Upload petrained model if model_type == 'ResNext': model = models.resnext50_32x4d(pretrained=True) else: model = EfficientNet.from_pretrained('efficientnet-b0') #Add final layer for transefer learning num_ftrs = model.fc.in_features model.fc = nn.Sequential(nn.Linear(num_ftrs, 10), nn.Softmax()) model = model.to(device) #Define training criteria criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=.05,momentum=.9) scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5) #obtained by trial and error model.to(device) train_index = (np.random.random(len(train_data))>=.75) # Training time print('Epoch | Train Loss | Train Accuracy | Val Loss | Val Accuracy | LR ') print('_________________________________________________________________________________') q=0 accuracy = [] for epoch in range(20): tmp = train_index running_loss = 0.0 running_corrects = 0 model.train() for i, data in enumerate(train_dataloader, 0): # get the inputs; data is a list of [inputs, labels] indexes = tmp[:data[0].shape[0]] tmp = tmp[data[0].shape[0]:] inputs = data[0][indexes] labels = data[1][indexes] #inputs, labels = data if torch.cuda.is_available(): inputs = inputs.to(device) labels = labels.to(device) # zero the parameter gradients: Clean the gradient caclulated in the previous iteration optimizer.zero_grad() # forward + backward + optimize outputs = model(inputs) #print(outputs.shape) #print(labels.shape) loss = criterion(outputs, labels.long()) # Calculate gradient of matrix with requires_grad = True loss.backward() # Apply the gradient calculate from last step to the matrix optimizer.step() # Add 1 more iteration count to learning rate scheduler # print statistics running_loss += loss.item() _, preds = torch.max(outputs, 1) running_corrects += torch.sum(preds == labels.data) for el in optimizer.param_groups: lr = el['lr'] scheduler.step() val_corrects = 0 val_loss = 0 model.eval() with torch.no_grad(): for data in test_dataloader: images, labels = data if torch.cuda.is_available(): images = images.to(device) labels = labels.to(device) outputs = model(images) _, preds = torch.max(outputs, 1) val_loss += criterion(outputs, labels.long()) num = torch.sum(preds == labels.data) val_corrects += num #print('Epoch | Train Loss | Train Accuracy | Val Loss | Val Accuracy | LR ') print(" %2d | %.4f | %.4f | %.4f | %.4f | %.5f " %(q,running_loss/sum(train_index),int(running_corrects)/sum(train_index),val_loss/len(test_data),int(val_corrects)/len(test_data),lr)) accuracy.append(int(val_corrects)/len(test_data)) q+=1 print('Finished Training') torch.save(model, 'Data/'+model_type+'_model.pt') return(accuracy,train_index) def train_base_model_total_data(model_type='ResNext'): transform = transforms.Compose([#transforms.ToPILImage()]) transforms.ToTensor(), transforms.Normalize([.5,.5,.5],[.5,.5,.5]) ]) train_data = datasets.SVHN('Data/',split='train',download=True,transform=transform) test_data = datasets.SVHN('Data/',split='test',download=True,transform=transform) train_dataloader = torch.utils.data.DataLoader( train_data, batch_size=1024 ) test_dataloader = torch.utils.data.DataLoader( test_data, batch_size=1024 ) #Upload petrained model if model_type == 'ResNext': model = models.resnext50_32x4d(pretrained=True) else: model = EfficientNet.from_pretrained('efficientnet-b0') #Add final layer for transefer learning num_ftrs = model.fc.in_features model.fc = nn.Sequential(nn.Linear(num_ftrs, 10), nn.Softmax()) model = model.to(device) #Define training criteria criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=0.05,momentum=.9) scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5) #obtained by trial and error model.to(device) # Training time print('Epoch | Train Loss | Train Accuracy | Val Loss | Val Accuracy | LR ') print('_________________________________________________________________________________') q=0 accuracy = [] for epoch in range(25): running_loss = 0.0 running_corrects = 0 model.train() for i, data in enumerate(train_dataloader, 0): # get the inputs; data is a list of [inputs, labels] inputs, labels = data if torch.cuda.is_available(): inputs = inputs.to(device) labels = labels.to(device) # zero the parameter gradients: Clean the gradient caclulated in the previous iteration optimizer.zero_grad() # forward + backward + optimize outputs = model(inputs) #print(outputs.shape) #print(labels.shape) loss = criterion(outputs, labels.long()) # Calculate gradient of matrix with requires_grad = True loss.backward() # Apply the gradient calculate from last step to the matrix optimizer.step() # Add 1 more iteration count to learning rate scheduler # print statistics running_loss += loss.item() _, preds = torch.max(outputs, 1) running_corrects += torch.sum(preds == labels.data) for el in optimizer.param_groups: lr = el['lr'] scheduler.step() val_corrects = 0 val_loss = 0 model.eval() with torch.no_grad(): for data in test_dataloader: images, labels = data if torch.cuda.is_available(): images = images.to(device) labels = labels.to(device) outputs = model(images) _, preds = torch.max(outputs, 1) val_loss += criterion(outputs, labels.long()) num = torch.sum(preds == labels.data) val_corrects += num #print('Epoch | Train Loss | Train Accuracy | Val Loss | Val Accuracy | LR ') print(" %2d | %.4f | %.4f | %.4f | %.4f | %.5f " %(q,running_loss/len(train_data),int(running_corrects)/len(train_data),val_loss/len(test_data),int(val_corrects)/len(test_data),lr)) accuracy.append(int(val_corrects)/len(test_data)) q+=1 print('Finished Training') torch.save(model, 'Data/'+model_type+'_model.pt') return(accuracy) """# Stage 2: Predictions on unlabeled data def student_training(trained,augment = 'Rand', percentile = 0, model_type = 'ResNext' ,save = False, cycle = False, epochs=25): extra = trained==False if cycle: model = torch.load('Data/'+model_type+'_model_cycled.pt') else: model = torch.load('Data/'+model_type+'_model.pt') transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize([.5,.5,.5],[.5,.5,.5]) ]) extra_data = datasets.SVHN('Data/',split='train',download=True,transform=transform) extra_loader = torch.utils.data.DataLoader( extra_data, batch_size=1024,shuffle=False) # Switch some layers (e.g., batch norm, dropout) to evaluation mode model.eval() # Turn off the autograd to save memory usage and speed up prediction_list =[] label_list = [] tmp = extra with torch.no_grad(): for data in extra_loader: images = data[0] labels = data[1] if torch.cuda.is_available(): images = images.to(device) labels = labels.to(device) outputs = model(images) predicted = outputs.data prediction_list.append(predicted.cpu()) label_list.append(labels.cpu()) l_list = torch.cat(label_list).numpy() p_list = torch.cat(prediction_list) p_list = p_list.numpy() p_list[trained] = np.array([0]*10) """### Select threshold""" model = models.resnext50_32x4d(pretrained=True) num_ftrs = model.fc.in_features model.fc = nn.Sequential(nn.Linear(num_ftrs, 10), nn.Softmax()) threshold = np.percentile(p_list,percentile,axis=0) print(threshold) labels = (p_list>=threshold).astype(int)*(np.array([1]*10)) ratio = np.min(sum(labels))/sum(labels) print(np.min(sum(labels))) print(sum(labels)) print(ratio) rands = np.random.random(p_list.shape)<=ratio print(rands) labels = labels*rands print(sum(labels)) indexes = np.array(list(range(np.max(p_list.shape))))[list((np.sum(labels,axis=1)>.1))] labels = np.argmax(labels[indexes],axis=1) train_data = datasets.SVHN('Data/',split='train',download=True,transform=transform) bs = int(sum(trained)/(sum(trained)+sum(extra))*1024) train_dataloader = torch.utils.data.DataLoader( train_data, batch_size=bs, ) """## Stage 2.1: Reload Unlabeled (Extra) Data and apply *RandAugment*""" if augment == 'Rand': rand_transform = transforms.Compose([RandAugment(), transforms.ToTensor(), transforms.Normalize([.5,.5,.5],[.5,.5,.5]) ]) elif augment == 'Traditional': rand_transform = transforms.Compose([transforms.RandomAffine(30), transforms.ToTensor(), transforms.Normalize([.5,.5,.5],[.5,.5,.5]) ]) else: rand_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize([.5,.5,.5],[.5,.5,.5]) ]) aug_data = datasets.SVHN('Data/',split='train',download=True,transform=rand_transform) aug_dataloader = torch.utils.data.DataLoader( extra_data, batch_size=1024-bs,shuffle=False) indices = np.array([False]*aug_data.data.shape[0]) indices[indexes] = True #only need to run this cell when step 1 was skipped test_data = datasets.SVHN('Data/',split='test',download=True,transform=transform) test_dataloader = torch.utils.data.DataLoader( test_data, batch_size=1024 ) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=0.05,momentum=.9) scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=6, gamma=0.5) model.to(device) # Commented out IPython magic to ensure Python compatibility. print('Epoch | Train Loss | Train Accuracy | Val Loss | Val Accuracy | LR ') print('_________________________________________________________________________________') q=0 run_len = train_data.data.shape[0]+labels.shape[0] test_len = test_data.data.shape[0] prev = 0 accuracy = [] while q < epochs: running_loss = 0.0 running_corrects = 0 model.train() inds = indices labs = torch.Tensor(labels) for i in enumerate(zip(train_dataloader,aug_dataloader)): # get the inputs; data is a list of [inputs, labels] b_s = min((1024-bs),i[1][1][0].shape[0]) aug = i[1][1][0][inds[:b_s]] inputs = torch.cat([i[1][0][0],aug]) l = torch.cat([i[1][0][1].float(),labs[:aug.shape[0]].float()]) # randomize the batches ran = torch.randperm(l.shape[0]) inputs = inputs[ran] l = l[ran] if torch.cuda.is_available(): inputs = inputs.to(device) l = l.to(device) # zero the parameter gradients: Clean the gradient caclulated in the previous iteration optimizer.zero_grad() # forward + backward + optimize outputs = model(inputs) #print(outputs.shape) #print(labels.shape) loss = criterion(outputs, l.long()) # Calculate gradient of matrix with requires_grad = True loss.backward() # Apply the gradient calculate from last step to the matrix optimizer.step() # Add 1 more iteration count to learning rate scheduler # print statistics running_loss += loss.item() _, preds = torch.max(outputs, 1) running_corrects += torch.sum(preds == l.data) for el in optimizer.param_groups: lr = el['lr'] scheduler.step() val_corrects = 0 val_loss = 0 model.eval() inds = inds[b_s:] with torch.no_grad(): for data in test_dataloader: images, l = data if torch.cuda.is_available(): images = images.to(device) l = l.to(device) outputs = model(images) _, preds = torch.max(outputs, 1) val_loss += criterion(outputs, l.long()) num = torch.sum(preds == l.data) val_corrects += num print(" %2d | %.4f | %.4f | %.4f | %.4f | %.5f " %(q,running_loss/run_len,int(running_corrects)/run_len,val_loss/len(test_data),int(val_corrects)/len(test_data),lr)) accuracy.append(int(val_corrects)/len(test_data)) #early stopping q+=1 torch.save(model, 'Data/'+model_type+'_model_cycled.pt') print('Finished Training') return(accuracy)
def main(args): ''' Main function for Args: - from_id: start index to file list Returns: - Write file missing data - Write file imputed values ''' # Parameters batch_size = args.batch_size epochs = args.epochs flag_model = args.flag_model # Define your transforms for the training and testing sets data_transforms = { 'train': transforms.Compose([ transforms.RandomRotation(30), transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), RandAugment(), ImageNetPolicy(), Cutout(size=16), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]), 'test': transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) } # Load the datasets with ImageFolder image_datasets = { x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x]) for x in ['train', 'test'] } data_loader = { x: torch.utils.data.DataLoader(image_datasets[x], batch_size=batch_size, shuffe=True, num_workers=4, pin_memory=True) for x in ['train', 'test'] } dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'test']} class_names = image_datasets['train'].classes # we get the class_to_index in the data_Set but what we really need is the cat_to_names so we will create _ = image_datasets['train'].class_to_idx cat_to_name = {_[i]: i for i in list(_.keys())} # Run this to test the data loader images, labels = next(iter(data_loader['test'])) # Loaded pre_trained model model = create_model(flag_model=flag_model, flag_pretrained=True) # Create classifiers for param in model.parameters(): param.requires_grad = True # Configs some last layers for my tasks classification covid fc = nn.Sequential( OrderedDict([('fc1', nn.Linear(2048, 1000, bias=True)), ('BN1', nn.BatchNorm2d(1000, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)), ('dropout1', nn.Dropout(0.7)), ('fc2', nn.Linear(1000, 512)), ('BN2', nn.BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)), ('swish1', Swish()), ('dropout2', nn.Dropout(0.5)), ('fc3', nn.Linear(512, 128)), ('BN3', nn.BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)), ('swish2', Swish()), ('fc4', nn.Linear(128, 3)), ('output', nn.Softmax(dim=1))])) # Connect pretrained model with modified classifer layer model.fc = fc criterion = nn.CrossEntropyLoss() # Optimizer optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9, nesterov=True, weight_decay=0.0001) scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.1) # Send to GPU model.to(device) #Set checkpoint try: checkpoint = torch.load(CHECK_POINT_PATH) print("checkpoint loaded") except: checkpoint = None print("checkpoint not found") if checkpoint == None: CHECK_POINT_PATH = CHECK_POINT_PATH model, best_val_loss, best_val_acc = train_model( model, criterion, optimizer, scheduler, num_epochs=epochs, checkpoint=torch.load(CHECK_POINT_PATH)) torch.save( { 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'best_val_loss': best_val_loss, 'best_val_accuracy': best_val_acc, 'scheduler_state_dict': scheduler.state_dict(), }, CHECK_POINT_PATH) # Load checkpoint path try: checkpoint = torch.load(CHECK_POINT_PATH) print("checkpoint loaded") except: checkpoint = None print("checkpoint not found") load_model(model=model, checkpoint=checkpoint, path=CHECK_POINT_PATH) # Testing since = time.time() model.eval() y_test = [] y_pred = [] for images, labels in data_loader['test']: images = Variable(images.cuda()) labels = Variable(labels.cuda()) outputs = model(images) _, predictions = outputs.max(1) y_test.append(labels.data.cpu().numpy()) y_pred.append(predictions.data.cpu().numpy()) y_test = np.concatenate(y_test) y_pred = np.concatenate(y_pred) pd.DataFrame({ 'true_label': y_test, 'predicted_label': y_pred }).to_csv('Modified_EfficienNet_B0_Covid-19_Test.csv', index=False) time_elapsed = time.time() - since print('Training complete in {:.0f}m {:.0f}s'.format( time_elapsed // 60, time_elapsed % 60)) sns.heatmap(confusion_matrix(y_test, y_pred)) accuracy_score(y_test, y_pred) report = classification_report(y_test, y_pred) print(report)
def main(): args = config.base_parser() # Save file name tr_names = "" for trans in args.transforms: tr_names += "_" + trans save_path = f"{args.dataset}/{args.mode}_{args.mem_manage}_{args.stream_env}_msz{args.memory_size}_rnd{args.rnd_seed}{tr_names}" logging.config.fileConfig("./configuration/logging.conf") logger = logging.getLogger() os.makedirs(f"logs/{args.dataset}", exist_ok=True) fileHandler = logging.FileHandler("logs/{}.log".format(save_path), mode="w") formatter = logging.Formatter( "[%(levelname)s] %(filename)s:%(lineno)d > %(message)s") fileHandler.setFormatter(formatter) logger.addHandler(fileHandler) writer = SummaryWriter("tensorboard") if torch.cuda.is_available(): device = torch.device("cuda") else: device = torch.device("cpu") logger.info(f"Set the device ({device})") # Fix the random seeds # https://hoya012.github.io/blog/reproducible_pytorch/ torch.manual_seed(args.rnd_seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False np.random.seed(args.rnd_seed) random.seed(args.rnd_seed) # Transform Definition mean, std, n_classes, inp_size, _ = get_statistics(dataset=args.dataset) train_transform = [] if "cutout" in args.transforms: train_transform.append(Cutout(size=16)) if "randaug" in args.transforms: train_transform.append(RandAugment()) if "autoaug" in args.transforms: train_transform.append(select_autoaugment(args.dataset)) train_transform = transforms.Compose([ transforms.Resize((inp_size, inp_size)), transforms.RandomCrop(inp_size, padding=4), transforms.RandomHorizontalFlip(), *train_transform, transforms.ToTensor(), transforms.Normalize(mean, std), ]) logger.info(f"Using train-transforms {train_transform}") test_transform = transforms.Compose([ transforms.Resize((inp_size, inp_size)), transforms.ToTensor(), transforms.Normalize(mean, std), ]) logger.info(f"[1] Select a CIL method ({args.mode})") criterion = nn.CrossEntropyLoss(reduction="mean") method = select_method(args, criterion, device, train_transform, test_transform, n_classes) logger.info(f"[2] Incrementally training {args.n_tasks} tasks") task_records = defaultdict(list) for cur_iter in range(args.n_tasks): if args.mode == "joint" and cur_iter > 0: return print("\n" + "#" * 50) print(f"# Task {cur_iter} iteration") print("#" * 50 + "\n") logger.info("[2-1] Prepare a datalist for the current task") task_acc = 0.0 eval_dict = dict() # get datalist cur_train_datalist = get_train_datalist(args, cur_iter) cur_test_datalist = get_test_datalist(args, args.exp_name, cur_iter) # Reduce datalist in Debug mode if args.debug: random.shuffle(cur_train_datalist) random.shuffle(cur_test_datalist) cur_train_datalist = cur_train_datalist[:2560] cur_test_datalist = cur_test_datalist[:2560] logger.info("[2-2] Set environment for the current task") method.set_current_dataset(cur_train_datalist, cur_test_datalist) # Increment known class for current task iteration. method.before_task(cur_train_datalist, cur_iter, args.init_model, args.init_opt) # The way to handle streamed samles logger.info(f"[2-3] Start to train under {args.stream_env}") if args.stream_env == "offline" or args.mode == "joint" or args.mode == "gdumb": # Offline Train task_acc, eval_dict = method.train( cur_iter=cur_iter, n_epoch=args.n_epoch, batch_size=args.batchsize, n_worker=args.n_worker, ) if args.mode == "joint": logger.info(f"joint accuracy: {task_acc}") elif args.stream_env == "online": # Online Train logger.info("Train over streamed data once") method.train( cur_iter=cur_iter, n_epoch=1, batch_size=args.batchsize, n_worker=args.n_worker, ) method.update_memory(cur_iter) # No stremed training data, train with only memory_list method.set_current_dataset([], cur_test_datalist) logger.info("Train over memory") task_acc, eval_dict = method.train( cur_iter=cur_iter, n_epoch=args.n_epoch, batch_size=args.batchsize, n_worker=args.n_worker, ) method.after_task(cur_iter) logger.info("[2-4] Update the information for the current task") method.after_task(cur_iter) task_records["task_acc"].append(task_acc) # task_records['cls_acc'][k][j] = break down j-class accuracy from 'task_acc' task_records["cls_acc"].append(eval_dict["cls_acc"]) # Notify to NSML logger.info("[2-5] Report task result") writer.add_scalar("Metrics/TaskAcc", task_acc, cur_iter) np.save(f"results/{save_path}.npy", task_records["task_acc"]) # Accuracy (A) A_avg = np.mean(task_records["task_acc"]) A_last = task_records["task_acc"][args.n_tasks - 1] # Forgetting (F) acc_arr = np.array(task_records["cls_acc"]) # cls_acc = (k, j), acc for j at k cls_acc = acc_arr.reshape(-1, args.n_cls_a_task).mean(1).reshape( args.n_tasks, -1) for k in range(args.n_tasks): forget_k = [] for j in range(args.n_tasks): if j < k: forget_k.append(cls_acc[:k, j].max() - cls_acc[k, j]) else: forget_k.append(None) task_records["forget"].append(forget_k) F_last = np.mean(task_records["forget"][-1][:-1]) # Intrasigence (I) I_last = args.joint_acc - A_last logger.info(f"======== Summary =======") logger.info( f"A_last {A_last} | A_avg {A_avg} | F_last {F_last} | I_last {I_last}")
def main(): args = parser.parse_args() args.batch_size = args.batch_size # setup model print('creating model...') #state = torch.load(args.model_path, map_location='cpu') #args.num_classes = state['num_classes'] args.do_bottleneck_head = True model = create_model(args).cuda() ema = EMA(model, 0.999) ema.register() #model.load_state_dict(state['model'], strict=True) #model.train() classes_list = np.array(list(idx_to_class.values())) print('done\n') # Data loading code normalize = transforms.Normalize(mean=[0, 0, 0], std=[1, 1, 1]) instances_path_val = os.path.join(args.data, 'annotations/instances_val2017.json') #instances_path_train = os.path.join(args.data, 'annotations/instances_val2017.json')#temprarily use val as train instances_path_train = os.path.join(args.data, 'annotations/instances_train2017.json') data_path_val = os.path.join(args.data, 'val2017') #data_path_train = os.path.join(args.data, 'val2017')#temporarily use val as train data_path_train = os.path.join(args.data, 'train2017') val_dataset = CocoDetection(data_path_val, instances_path_val, transforms.Compose([ transforms.Resize((args.image_size, args.image_size)), RandAugment(), transforms.ToTensor(), normalize, Cutout(n_holes = 1, length = 16) ])) train_dataset = CocoDetection(data_path_train, instances_path_train, transforms.Compose([ transforms.Resize((args.image_size, args.image_size)), RandAugment(), transforms.ToTensor(), normalize, ])) print("len(val_dataset)): ", len(val_dataset)) print("len(train_dataset)): ", len(train_dataset)) val_loader = torch.utils.data.DataLoader( val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=False) train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=False) lr = 0.0002 Epoch = 10 criterion = AsymmetricLoss() params = model.parameters() optimizer = torch.optim.Adam(params, lr=lr)#尝试新的optimizer total_step = len(train_loader) scheduler = lr_scheduler.OneCycleLR(optimizer, max_lr = lr, total_steps = total_step, epochs = Epoch) #total_step = len(train_loader) highest_mAP = 0 trainInfoList = [] Sig = torch.nn.Sigmoid() #f=open('info_train.txt', 'a') for epoch in range(Epoch): for i, (inputData, target) in enumerate(train_loader): f=open('info_train.txt', 'a') #model.train() inputData = inputData.cuda() target = target.cuda() target = target.max(dim=1)[0] #Sig = torch.nn.Sigmoid() output = Sig(model(inputData)) #output[output<args.thre] = 0 #output[output>=args.thre]=1 #print(output.shape) #(batchsize, channel, imhsize, imgsize) #print(inputData.shape) #(batchsize, numclasses) #print(output[0]) #print(target[0]) loss = criterion(output, target) model.zero_grad() loss.backward() optimizer.step() ema.update() #store information if i % 10 == 0: trainInfoList.append([epoch, i, loss.item()]) print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}' .format(epoch, Epoch, i, total_step, loss.item())) f.write('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}\n' .format(epoch, Epoch, i, total_step, loss.item())) if (i+1) % 100 == 0: #储存相应迭代模型 torch.save(model.state_dict(), os.path.join( 'models/', 'model-{}-{}.ckpt'.format(epoch+1, i+1))) #modelName = 'models/' + 'decoder-{}-{}.ckpt'.format(epoch+1, i+1) mAP_score = validate_multi(val_loader, model, args, ema) #model.train() if mAP_score > highest_mAP: highest_mAP = mAP_score print('current highest_mAP = ', highest_mAP) f.write('current highest_mAP = {}\n'.format(highest_mAP)) torch.save(model.state_dict(), os.path.join( 'models/', 'model-highest.ckpt')) f.close() scheduler.step()#修改学习率
def student_training_extra(augment='Rand', percentile=0, model_type='ResNext', save=False, cycle=False, epochs=5): print(percentile) if cycle: model = torch.load('Data/' + model_type + '_model_cycled.pt') else: model = torch.load('Data/' + model_type + '_model.pt') transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize([.5, .5, .5], [.5, .5, .5]) ]) extra_data = datasets.SVHN('Data/', split='extra', download=True, transform=transform) extra_loader = torch.utils.data.DataLoader(extra_data, batch_size=1024, shuffle=False) # Switch some layers (e.g., batch norm, dropout) to evaluation mode model.eval() # Turn off the autograd to save memory usage and speed up prediction_list = [] label_list = [] with torch.no_grad(): for data in extra_loader: images, labels = data if torch.cuda.is_available(): images = images.to(device) labels = labels.to(device) outputs = model(images) predicted = outputs.data prediction_list.append(predicted.cpu()) label_list.append(labels.cpu()) p_list = torch.cat(prediction_list) p_list = p_list.numpy() """### Select threshold""" model = models.resnext50_32x4d(pretrained=True) num_ftrs = model.fc.in_features model.fc = nn.Sequential(nn.Linear(num_ftrs, 10), nn.Softmax()) threshold = np.percentile(np.max(p_list, axis=1), percentile) print(threshold) if threshold < .5: indexes = list(range(p_list.shape[0])) labels = np.argmax(p_list, axis=1) else: print('here') labels = (p_list >= threshold).astype(int) * np.array( [10, 1, 2, 3, 4, 5, 6, 7, 8, 9]) indexes = np.array(list(range(np.max(p_list.shape))))[list( (np.sum(labels, axis=1) > .1))] labels = np.argmax(labels[indexes], axis=1) train_data = datasets.SVHN( root='Data/', split='train', download=True, transform=transform, ) bs = int(73257 / (73257 + labels.shape[0]) * 1024) train_dataloader = torch.utils.data.DataLoader( train_data, batch_size=bs, ) """## Stage 2.1: Reload Unlabeled (Extra) Data and apply *RandAugment*""" if augment == 'Rand': rand_transform = transforms.Compose([ RandAugment(), transforms.ToTensor(), transforms.Normalize([.5, .5, .5], [.5, .5, .5]) ]) elif augment == 'Traditional': rand_transform = transforms.Compose([ transforms.RandomAffine(30), transforms.ToTensor(), transforms.Normalize([.5, .5, .5], [.5, .5, .5]) ]) else: rand_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize([.5, .5, .5], [.5, .5, .5]) ]) aug_data = datasets.SVHN( root='Data/', split='extra', download=True, transform=rand_transform, ) aug_dataloader = torch.utils.data.DataLoader( aug_data, batch_size=(1024 - bs), ) indices = np.array([False] * aug_data.data.shape[0]) indices[indexes] = True #only need to run this cell when step 1 was skipped test_data = datasets.SVHN('Data/', split='test', download=True, transform=transform) test_dataloader = torch.utils.data.DataLoader(test_data, batch_size=1024) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=0.05, momentum=.9) scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5) model.to(device) # Commented out IPython magic to ensure Python compatibility. print( 'Epoch | Train Loss | Train Accuracy | Val Loss | Val Accuracy | LR ' ) print( '_________________________________________________________________________________' ) q = 0 run_len = train_data.data.shape[0] + labels.shape[0] test_len = test_data.data.shape[0] prev = 0 accuracy = [] while q < epochs: running_loss = 0.0 running_corrects = 0 model.train() inds = indices labs = torch.Tensor(labels) for i in enumerate(zip(train_dataloader, aug_dataloader)): # get the inputs; data is a list of [inputs, labels] b_s = min((1024 - bs), i[1][1][0].shape[0]) aug = i[1][1][0][inds[:b_s]] inputs = torch.cat([i[1][0][0], aug]) l = torch.cat( [i[1][0][1].float(), labs[:aug.shape[0]].float()]) # randomize the batches ran = torch.randperm(l.shape[0]) inputs = inputs[ran] l = l[ran] if torch.cuda.is_available(): inputs = inputs.to(device) l = l.to(device) # zero the parameter gradients: Clean the gradient caclulated in the previous iteration optimizer.zero_grad() # forward + backward + optimize outputs = model(inputs) #print(outputs.shape) #print(labels.shape) loss = criterion(outputs, l.long()) # Calculate gradient of matrix with requires_grad = True loss.backward() # Apply the gradient calculate from last step to the matrix optimizer.step() # Add 1 more iteration count to learning rate scheduler # print statistics running_loss += loss.item() _, preds = torch.max(outputs, 1) running_corrects += torch.sum(preds == l.data) for el in optimizer.param_groups: lr = el['lr'] scheduler.step() val_corrects = 0 val_loss = 0 model.eval() inds = inds[b_s:] with torch.no_grad(): for data in test_dataloader: images, l = data if torch.cuda.is_available(): images = images.to(device) l = l.to(device) outputs = model(images) _, preds = torch.max(outputs, 1) val_loss += criterion(outputs, l.long()) num = torch.sum(preds == l.data) val_corrects += num print( " %2d | %.4f | %.4f | %.4f | %.4f | %.5f " % (q, running_loss / run_len, int(running_corrects) / run_len, val_loss / len(test_data), int(val_corrects) / len(test_data), lr)) accuracy.append(int(val_corrects) / len(test_data)) #early stopping if accuracy[-1] > prev: torch.save(model, 'Data/' + model_type + '_model_cycled.pt') prev = accuracy[-1] else: q = epochs q += 1 print('Finished Training') return (accuracy)
if args.seed != 0: torch.manual_seed(args.seed) # Data print('==> Preparing data..') transform_train = transforms.Compose( [transforms.ToTensor(), transforms.Normalize( (0.5, ), (0.5, ), )]) if args.rand_augment: transform_train.transforms.insert(0, RandAugment(args.N, args.M)) if args.augment: transform_train.transforms.insert(0, transforms.RandomCrop(28, padding=4)) transform_train.transforms.insert(1, transforms.RandomHorizontalFlip()) transform_test = transforms.Compose( [transforms.ToTensor(), transforms.Normalize( (0.5, ), (0.5, ), )]) trainset = datasets.FashionMNIST(root='~/data', train=True, download=True,
def main(): args = parser.parse_args() args.do_bottleneck_head = False # setup dist training if 'WORLD_SIZE' in os.environ: args.distributed = int(os.environ['WORLD_SIZE']) > 1 args.device = 'cuda:0' args.world_size = 1 args.rank = 0 # global rank if args.distributed: args.device = 'cuda:%d' % args.local_rank torch.cuda.set_device(args.local_rank) torch.distributed.init_process_group(backend='nccl', init_method='env://') args.world_size = torch.distributed.get_world_size() args.rank = torch.distributed.get_rank() print('Training in distributed mode with multiple processes, 1 GPU per process. Process %d, total %d.' % (args.rank, args.world_size)) else: print('Training with a single process on 1 GPUs.') assert args.rank >= 0 # setup logger logger = setup_logger(output=args.output, distributed_rank=dist.get_rank(), color=False, name="Coco") logger.info("Command: "+' '.join(sys.argv)) if dist.get_rank() == 0: path = os.path.join(args.output, "config.json") with open(path, 'w') as f: json.dump(vars(args), f, indent=2) logger.info("Full config saved to {}".format(path)) os.makedirs(osp.join(args.output, 'tmpdata'), exist_ok=True) # Setup model logger.info('creating model...') model = create_model(args).cuda() if args.model_path: # make sure to load pretrained ImageNet model state = torch.load(args.model_path, map_location='cpu') filtered_dict = {k: v for k, v in state['model'].items() if (k in model.state_dict() and 'head.fc' not in k)} model.load_state_dict(filtered_dict, strict=False) logger.info('done\n') ema = ModelEma(model, 0.9997) # 0.9997^641=0.82 if args.distributed: model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], broadcast_buffers=False) # COCO Data loading instances_path_val = os.path.join(args.data, 'annotations/instances_val2014.json') instances_path_train = os.path.join(args.data, 'annotations/instances_train2014.json') data_path_val = args.data data_path_train = args.data val_dataset = CocoDetection(data_path_val, instances_path_val, transforms.Compose([ transforms.Resize((args.image_size, args.image_size)), transforms.ToTensor(), # normalize, # no need, toTensor does normalization ])) train_dataset = CocoDetection(data_path_train, instances_path_train, transforms.Compose([ transforms.Resize((args.image_size, args.image_size)), CutoutPIL(cutout_factor=0.5), RandAugment(), transforms.ToTensor(), # normalize, ])) logger.info("len(val_dataset)): {}".format(len(val_dataset))) logger.info("len(train_dataset)): {}".format(len(train_dataset))) train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) val_sampler = torch.utils.data.distributed.DistributedSampler(val_dataset, shuffle=False) assert args.batch_size // dist.get_world_size() == args.batch_size / dist.get_world_size(), 'Batch size is not divisible by num of gpus.' # Pytorch Data loader train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=args.batch_size // dist.get_world_size(), shuffle=not args.distributed, num_workers=args.workers, pin_memory=True, sampler=train_sampler, drop_last=True) val_loader = torch.utils.data.DataLoader( val_dataset, batch_size=args.batch_size // dist.get_world_size(), shuffle=False, num_workers=args.workers, pin_memory=False, sampler=val_sampler) # Actuall Training train_multi_label_coco(model, ema, train_loader, val_loader, args.lr, args, logger)
) bs = int(73257/(73257+labels.shape[0])*1024) train_dataloader = torch.utils.data.DataLoader( train_data, batch_size=bs, ) """## Stage 2.1: Reload Unlabeled (Extra) Data and apply *RandAugment*""" if augment == 'Rand': rand_transform = transforms.Compose([RandAugment(), transforms.ToTensor(), transforms.Normalize([.5,.5,.5],[.5,.5,.5]) ]) elif augment == 'Traditional': rand_transform = transforms.Compose([transforms.RandomAffine(30), transforms.ToTensor(), transforms.Normalize([.5,.5,.5],[.5,.5,.5]) ]) else: rand_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize([.5,.5,.5],[.5,.5,.5]) ]) aug_data = datasets.SVHN( root='Data/', split='extra',
def main(): args = parser.parse_args() args.batch_size = args.batch_size args.do_bottleneck_head = False # setup model print('creating model...') model = create_model(args).cuda() if args.model_path: state = torch.load(args.model_path, map_location='cpu') filtered_dict = { k: v for k, v in state['model'].items() if (k in model.state_dict() and 'head.fc' not in k) } model.load_state_dict(filtered_dict, strict=False) # model.load_state_dict(state['model'], strict=False) print('done\n') # Data loading code normalize = transforms.Normalize(mean=[0, 0, 0], std=[1, 1, 1]) instances_path_val = os.path.join(args.data, 'annotations/instances_val2014.json') instances_path_train = os.path.join( args.data, 'annotations/instances_train2014.json') # data_path_val = os.path.join(args.data, 'val2014') # data_path_train = os.path.join(args.data, 'train2014') data_path_val = args.data data_path_train = args.data val_dataset = CocoDetection( data_path_val, instances_path_val, transforms.Compose([ transforms.Resize((args.image_size, args.image_size)), transforms.ToTensor(), # normalize, # no need, toTensor does normalization ])) train_dataset = CocoDetection( data_path_train, instances_path_train, transforms.Compose([ transforms.Resize((args.image_size, args.image_size)), CutoutPIL(cutout_factor=0.5), RandAugment(), transforms.ToTensor(), # normalize, ])) print("len(val_dataset)): ", len(val_dataset)) print("len(train_dataset)): ", len(train_dataset)) val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=False) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=False) train_multi_label_coco(model, train_loader, val_loader, args.lr, gamma_neg=4, gamma_pos=0, clip=0.05)