def load_unlabeled_data(tar, data_dir='dataset/'):
from torchvision import transforms
folder_tar = data_dir + tar + '/images'
transform = {
'weak':
transforms.Compose([
transforms.Resize([224, 224]),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
]),
'strong':
transforms.Compose([
transforms.Resize([256, 256]),
transforms.RandomCrop(224),
transforms.RandomHorizontalFlip(),
RandAugment(
), # Paper: RandAugment: Practical data augmentation with no separate search
transforms.ToTensor(),
# Cutout(size=16), # (https://github.com/uoguelph-mlrg/Cutout/blob/master/util/cutout.py)
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
}
unlabeled_weak_data = datasets.ImageFolder(root=folder_tar,
transform=transform['weak'])
unlabeled_strong_data = datasets.ImageFolder(root=folder_tar,
transform=transform['strong'])
return unlabeled_weak_data, unlabeled_strong_data
def __init__(self,
data,
targets=None,
num_classes=None,
transform=None,
use_strong_transform=False,
strong_transform=None,
*args,
**kwargs):
"""
Args
data: x_data
targets: y_data (if not exist, None)
num_classes: number of label classes
transform: basic transformation of data
use_strong_transform: If True, this dataset returns both weakly and strongly augmented images.
strong_transform: list of transformation functions for strong augmentation
"""
super(BasicDataset, self).__init__()
self.data = data
self.targets = targets
self.num_classes = num_classes
self.use_strong_transform = use_strong_transform
self.transform = transform
if use_strong_transform:
if strong_transform is None:
self.strong_transform = copy.deepcopy(transform)
self.strong_transform.transforms.insert(0, RandAugment(3, 5))
else:
self.strong_transform = strong_transform
def __init__(self, mean, std):
self.weak = transforms.Compose([
transforms.RandomHorizontalFlip(),
transforms.RandomCrop(size=32, padding=int(32*0.125), padding_mode='reflect')])
self.strong = transforms.Compose([
transforms.RandomHorizontalFlip(),
transforms.RandomCrop(size=32, padding=int(32*0.125), padding_mode='reflect'),
RandAugment(n=2, m=10)])
self.normalize = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean=mean, std=std)])
def main():
args = parser.parse_args()
args.do_bottleneck_head = False
# Setup model
print('creating model...')
model = create_model(args).cuda()
if args.model_path: # make sure to load pretrained ImageNet model
state = torch.load(args.model_path, map_location='cpu')
filtered_dict = {k: v for k, v in state['model'].items() if
(k in model.state_dict() and 'head.fc' not in k)}
model.load_state_dict(filtered_dict, strict=False)
print('done\n')
# COCO Data loading
instances_path_val = os.path.join(args.data, 'annotations/instances_val2014.json')
instances_path_train = os.path.join(args.data, 'annotations/instances_train2014.json')
# data_path_val = args.data
# data_path_train = args.data
data_path_val = f'{args.data}/val2014' # args.data
data_path_train = f'{args.data}/train2014' # args.data
val_dataset = CocoDetection(data_path_val,
instances_path_val,
transforms.Compose([
transforms.Resize((args.image_size, args.image_size)),
transforms.ToTensor(),
# normalize, # no need, toTensor does normalization
]))
train_dataset = CocoDetection(data_path_train,
instances_path_train,
transforms.Compose([
transforms.Resize((args.image_size, args.image_size)),
CutoutPIL(cutout_factor=0.5),
RandAugment(),
transforms.ToTensor(),
# normalize,
]))
print("len(val_dataset)): ", len(val_dataset))
print("len(train_dataset)): ", len(train_dataset))
# Pytorch Data loader
train_loader = torch.utils.data.DataLoader(
train_dataset, batch_size=args.batch_size, shuffle=True,
num_workers=args.workers, pin_memory=True)
val_loader = torch.utils.data.DataLoader(
val_dataset, batch_size=args.batch_size, shuffle=False,
num_workers=args.workers, pin_memory=False)
# Actuall Training
train_multi_label_coco(model, train_loader, val_loader, args.lr)
def create_data_loaders(args):
data_path_train = os.path.join(args.data_path, 'imagenet21k_train')
train_transform = transforms.Compose([
transforms.Resize((args.image_size, args.image_size)),
CutoutPIL(cutout_factor=0.5),
RandAugment(),
transforms.ToTensor(),
])
data_path_val = os.path.join(args.data_path, 'imagenet21k_val')
val_transform = transforms.Compose([
transforms.Resize((args.image_size, args.image_size)),
transforms.ToTensor(),
])
train_dataset = ImageFolder(data_path_train, transform=train_transform)
val_dataset = ImageFolder(data_path_val, transform=val_transform)
print_at_master("length train dataset: {}".format(len(train_dataset)))
print_at_master("length val dataset: {}".format(len(val_dataset)))
sampler_train = None
sampler_val = None
if num_distrib() > 1:
sampler_train = torch.utils.data.distributed.DistributedSampler(
train_dataset)
sampler_val = OrderedDistributedSampler(val_dataset)
# Pytorch Data loader
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=sampler_train is None,
num_workers=args.num_workers,
pin_memory=True,
sampler=sampler_train)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
pin_memory=False,
sampler=sampler_val)
train_loader = PrefetchLoader(train_loader)
val_loader = PrefetchLoader(val_loader)
return train_loader, val_loader
def get_transform(cfg):
height = cfg.DATASET.HEIGHT
width = cfg.DATASET.WIDTH
normalize = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
if cfg.DATASET.TYPE == 'multi_label':
valid_transform = T.Compose([
T.Resize([height, width]),
T.ToTensor(),
# normalize,
])
if cfg.TRAIN.DATAAUG.TYPE == 'randaug':
# train_transform = T.Compose([
# T.RandomApply([AutoAugment()], p=cfg.TRAIN.DATAAUG.AUTOAUG_PROB),
# T.Resize((height, width), interpolation=3),
# CutoutPIL(cutout_factor=0.5),
# T.RandomHorizontalFlip(),
# T.ToTensor(),
# ])
train_transform = T.Compose([
T.Resize((height, width), interpolation=3),
RandAugment(),
CutoutPIL(cutout_factor=0.5),
T.ToTensor(),
])
elif cfg.TRAIN.DATAAUG.TYPE == 'normal':
train_transform = T.Compose([
T.Resize((height, width)),
T.RandomHorizontalFlip(),
T.ToTensor(),
# T.RandomErasing(scale=(0.1, 0.5))
# normalize,
])
else:
assert False, f'current {cfg.TRAIN.DATAAUG.TYPE} augmentation has not been achieved'
else:
assert False, 'xxxxxxxx'
return train_transform, valid_transform
def augmentation(self, img, randaugment=False):
if randaugment:
img = Image.fromarray(img)
return transforms.Compose([
RandAugment(),
transforms.ToTensor(),
transforms.RandomCrop(32),
transforms.RandomHorizontalFlip(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])(img)
else:
return transforms.Compose([
transforms.ToTensor(),
transforms.RandomCrop(32),
transforms.RandomHorizontalFlip(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])(img)
from torch.utils.tensorboard import SummaryWriter
from randaugment import RandAugment, Cutout
# default `log_dir` is "runs" - we'll be more specific here
writer = SummaryWriter('runs/effnetlite0_new')
# In[3]:
data_transforms = {
'train':
transforms.Compose([
# transforms.RandomErasing(p=0.5, scale=(0.02, 0.33), ratio=(0.3, 3.3), value=0, inplace=False),
# transforms.RandomCrop(size=32,
# padding=int(32*0.125),
# padding_mode='reflect'),
RandAugment(),
Cutout(16),
transforms.RandomPerspective(distortion_scale=0.5,
p=0.5,
interpolation=3),
transforms.ColorJitter(brightness=0.2,
contrast=0,
saturation=0,
hue=0.),
transforms.RandomAffine(degrees=0, translate=(0.2, 0.2)),
transforms.ToTensor(),
]),
'valid':
transforms.Compose([transforms.ToTensor()]),
}
class SVHN_Model:
def train_base_model(model_type='ResNext'):
transform = transforms.Compose([#transforms.ToPILImage()])
transforms.ToTensor(),
transforms.Normalize([.5,.5,.5],[.5,.5,.5])
])
train_data = datasets.SVHN('Data/',split='train',download=True,transform=transform)
test_data = datasets.SVHN('Data/',split='test',download=True,transform=transform)
train_dataloader = torch.utils.data.DataLoader(
train_data, batch_size=1024,
)
test_dataloader = torch.utils.data.DataLoader(
test_data, batch_size=1024, shuffle = False
)
#Upload petrained model
if model_type == 'ResNext':
model = models.resnext50_32x4d(pretrained=True)
else:
model = EfficientNet.from_pretrained('efficientnet-b0')
#Add final layer for transefer learning
num_ftrs = model.fc.in_features
model.fc = nn.Sequential(nn.Linear(num_ftrs, 10),
nn.Softmax())
model = model.to(device)
#Define training criteria
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=.05,momentum=.9)
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5) #obtained by trial and error
model.to(device)
train_index = (np.random.random(len(train_data))>=.75)
# Training time
print('Epoch | Train Loss | Train Accuracy | Val Loss | Val Accuracy | LR ')
print('_________________________________________________________________________________')
q=0
accuracy = []
for epoch in range(20):
tmp = train_index
running_loss = 0.0
running_corrects = 0
model.train()
for i, data in enumerate(train_dataloader, 0):
# get the inputs; data is a list of [inputs, labels]
indexes = tmp[:data[0].shape[0]]
tmp = tmp[data[0].shape[0]:]
inputs = data[0][indexes]
labels = data[1][indexes]
#inputs, labels = data
if torch.cuda.is_available():
inputs = inputs.to(device)
labels = labels.to(device)
# zero the parameter gradients: Clean the gradient caclulated in the previous iteration
optimizer.zero_grad()
# forward + backward + optimize
outputs = model(inputs)
#print(outputs.shape)
#print(labels.shape)
loss = criterion(outputs, labels.long())
# Calculate gradient of matrix with requires_grad = True
loss.backward()
# Apply the gradient calculate from last step to the matrix
optimizer.step()
# Add 1 more iteration count to learning rate scheduler
# print statistics
running_loss += loss.item()
_, preds = torch.max(outputs, 1)
running_corrects += torch.sum(preds == labels.data)
for el in optimizer.param_groups:
lr = el['lr']
scheduler.step()
val_corrects = 0
val_loss = 0
model.eval()
with torch.no_grad():
for data in test_dataloader:
images, labels = data
if torch.cuda.is_available():
images = images.to(device)
labels = labels.to(device)
outputs = model(images)
_, preds = torch.max(outputs, 1)
val_loss += criterion(outputs, labels.long())
num = torch.sum(preds == labels.data)
val_corrects += num
#print('Epoch | Train Loss | Train Accuracy | Val Loss | Val Accuracy | LR ')
print(" %2d | %.4f | %.4f | %.4f | %.4f | %.5f "
%(q,running_loss/sum(train_index),int(running_corrects)/sum(train_index),val_loss/len(test_data),int(val_corrects)/len(test_data),lr))
accuracy.append(int(val_corrects)/len(test_data))
q+=1
print('Finished Training')
torch.save(model, 'Data/'+model_type+'_model.pt')
return(accuracy,train_index)
def train_base_model_total_data(model_type='ResNext'):
transform = transforms.Compose([#transforms.ToPILImage()])
transforms.ToTensor(),
transforms.Normalize([.5,.5,.5],[.5,.5,.5])
])
train_data = datasets.SVHN('Data/',split='train',download=True,transform=transform)
test_data = datasets.SVHN('Data/',split='test',download=True,transform=transform)
train_dataloader = torch.utils.data.DataLoader(
train_data, batch_size=1024
)
test_dataloader = torch.utils.data.DataLoader(
test_data, batch_size=1024
)
#Upload petrained model
if model_type == 'ResNext':
model = models.resnext50_32x4d(pretrained=True)
else:
model = EfficientNet.from_pretrained('efficientnet-b0')
#Add final layer for transefer learning
num_ftrs = model.fc.in_features
model.fc = nn.Sequential(nn.Linear(num_ftrs, 10),
nn.Softmax())
model = model.to(device)
#Define training criteria
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=0.05,momentum=.9)
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5) #obtained by trial and error
model.to(device)
# Training time
print('Epoch | Train Loss | Train Accuracy | Val Loss | Val Accuracy | LR ')
print('_________________________________________________________________________________')
q=0
accuracy = []
for epoch in range(25):
running_loss = 0.0
running_corrects = 0
model.train()
for i, data in enumerate(train_dataloader, 0):
# get the inputs; data is a list of [inputs, labels]
inputs, labels = data
if torch.cuda.is_available():
inputs = inputs.to(device)
labels = labels.to(device)
# zero the parameter gradients: Clean the gradient caclulated in the previous iteration
optimizer.zero_grad()
# forward + backward + optimize
outputs = model(inputs)
#print(outputs.shape)
#print(labels.shape)
loss = criterion(outputs, labels.long())
# Calculate gradient of matrix with requires_grad = True
loss.backward()
# Apply the gradient calculate from last step to the matrix
optimizer.step()
# Add 1 more iteration count to learning rate scheduler
# print statistics
running_loss += loss.item()
_, preds = torch.max(outputs, 1)
running_corrects += torch.sum(preds == labels.data)
for el in optimizer.param_groups:
lr = el['lr']
scheduler.step()
val_corrects = 0
val_loss = 0
model.eval()
with torch.no_grad():
for data in test_dataloader:
images, labels = data
if torch.cuda.is_available():
images = images.to(device)
labels = labels.to(device)
outputs = model(images)
_, preds = torch.max(outputs, 1)
val_loss += criterion(outputs, labels.long())
num = torch.sum(preds == labels.data)
val_corrects += num
#print('Epoch | Train Loss | Train Accuracy | Val Loss | Val Accuracy | LR ')
print(" %2d | %.4f | %.4f | %.4f | %.4f | %.5f "
%(q,running_loss/len(train_data),int(running_corrects)/len(train_data),val_loss/len(test_data),int(val_corrects)/len(test_data),lr))
accuracy.append(int(val_corrects)/len(test_data))
q+=1
print('Finished Training')
torch.save(model, 'Data/'+model_type+'_model.pt')
return(accuracy)
"""# Stage 2: Predictions on unlabeled data
def student_training(trained,augment = 'Rand', percentile = 0, model_type = 'ResNext' ,save = False, cycle = False, epochs=25):
extra = trained==False
if cycle:
model = torch.load('Data/'+model_type+'_model_cycled.pt')
else:
model = torch.load('Data/'+model_type+'_model.pt')
transform = transforms.Compose([transforms.ToTensor(),
transforms.Normalize([.5,.5,.5],[.5,.5,.5])
])
extra_data = datasets.SVHN('Data/',split='train',download=True,transform=transform)
extra_loader = torch.utils.data.DataLoader(
extra_data, batch_size=1024,shuffle=False)
# Switch some layers (e.g., batch norm, dropout) to evaluation mode
model.eval()
# Turn off the autograd to save memory usage and speed up
prediction_list =[]
label_list = []
tmp = extra
with torch.no_grad():
for data in extra_loader:
images = data[0]
labels = data[1]
if torch.cuda.is_available():
images = images.to(device)
labels = labels.to(device)
outputs = model(images)
predicted = outputs.data
prediction_list.append(predicted.cpu())
label_list.append(labels.cpu())
l_list = torch.cat(label_list).numpy()
p_list = torch.cat(prediction_list)
p_list = p_list.numpy()
p_list[trained] = np.array([0]*10)
"""### Select threshold"""
model = models.resnext50_32x4d(pretrained=True)
num_ftrs = model.fc.in_features
model.fc = nn.Sequential(nn.Linear(num_ftrs, 10),
nn.Softmax())
threshold = np.percentile(p_list,percentile,axis=0)
print(threshold)
labels = (p_list>=threshold).astype(int)*(np.array([1]*10))
ratio = np.min(sum(labels))/sum(labels)
print(np.min(sum(labels)))
print(sum(labels))
print(ratio)
rands = np.random.random(p_list.shape)<=ratio
print(rands)
labels = labels*rands
print(sum(labels))
indexes = np.array(list(range(np.max(p_list.shape))))[list((np.sum(labels,axis=1)>.1))]
labels = np.argmax(labels[indexes],axis=1)
train_data = datasets.SVHN('Data/',split='train',download=True,transform=transform)
bs = int(sum(trained)/(sum(trained)+sum(extra))*1024)
train_dataloader = torch.utils.data.DataLoader(
train_data, batch_size=bs,
)
"""## Stage 2.1: Reload Unlabeled (Extra) Data and apply *RandAugment*"""
if augment == 'Rand':
rand_transform = transforms.Compose([RandAugment(),
transforms.ToTensor(),
transforms.Normalize([.5,.5,.5],[.5,.5,.5])
])
elif augment == 'Traditional':
rand_transform = transforms.Compose([transforms.RandomAffine(30),
transforms.ToTensor(),
transforms.Normalize([.5,.5,.5],[.5,.5,.5])
])
else:
rand_transform = transforms.Compose([transforms.ToTensor(),
transforms.Normalize([.5,.5,.5],[.5,.5,.5])
])
aug_data = datasets.SVHN('Data/',split='train',download=True,transform=rand_transform)
aug_dataloader = torch.utils.data.DataLoader(
extra_data, batch_size=1024-bs,shuffle=False)
indices = np.array([False]*aug_data.data.shape[0])
indices[indexes] = True
#only need to run this cell when step 1 was skipped
test_data = datasets.SVHN('Data/',split='test',download=True,transform=transform)
test_dataloader = torch.utils.data.DataLoader(
test_data, batch_size=1024
)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=0.05,momentum=.9)
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=6, gamma=0.5)
model.to(device)
# Commented out IPython magic to ensure Python compatibility.
print('Epoch | Train Loss | Train Accuracy | Val Loss | Val Accuracy | LR ')
print('_________________________________________________________________________________')
q=0
run_len = train_data.data.shape[0]+labels.shape[0]
test_len = test_data.data.shape[0]
prev = 0
accuracy = []
while q < epochs:
running_loss = 0.0
running_corrects = 0
model.train()
inds = indices
labs = torch.Tensor(labels)
for i in enumerate(zip(train_dataloader,aug_dataloader)):
# get the inputs; data is a list of [inputs, labels]
b_s = min((1024-bs),i[1][1][0].shape[0])
aug = i[1][1][0][inds[:b_s]]
inputs = torch.cat([i[1][0][0],aug])
l = torch.cat([i[1][0][1].float(),labs[:aug.shape[0]].float()])
# randomize the batches
ran = torch.randperm(l.shape[0])
inputs = inputs[ran]
l = l[ran]
if torch.cuda.is_available():
inputs = inputs.to(device)
l = l.to(device)
# zero the parameter gradients: Clean the gradient caclulated in the previous iteration
optimizer.zero_grad()
# forward + backward + optimize
outputs = model(inputs)
#print(outputs.shape)
#print(labels.shape)
loss = criterion(outputs, l.long())
# Calculate gradient of matrix with requires_grad = True
loss.backward()
# Apply the gradient calculate from last step to the matrix
optimizer.step()
# Add 1 more iteration count to learning rate scheduler
# print statistics
running_loss += loss.item()
_, preds = torch.max(outputs, 1)
running_corrects += torch.sum(preds == l.data)
for el in optimizer.param_groups:
lr = el['lr']
scheduler.step()
val_corrects = 0
val_loss = 0
model.eval()
inds = inds[b_s:]
with torch.no_grad():
for data in test_dataloader:
images, l = data
if torch.cuda.is_available():
images = images.to(device)
l = l.to(device)
outputs = model(images)
_, preds = torch.max(outputs, 1)
val_loss += criterion(outputs, l.long())
num = torch.sum(preds == l.data)
val_corrects += num
print(" %2d | %.4f | %.4f | %.4f | %.4f | %.5f "
%(q,running_loss/run_len,int(running_corrects)/run_len,val_loss/len(test_data),int(val_corrects)/len(test_data),lr))
accuracy.append(int(val_corrects)/len(test_data))
#early stopping
q+=1
torch.save(model, 'Data/'+model_type+'_model_cycled.pt')
print('Finished Training')
return(accuracy)
def main(args):
''' Main function for
Args:
- from_id: start index to file list
Returns:
- Write file missing data
- Write file imputed values
'''
# Parameters
batch_size = args.batch_size
epochs = args.epochs
flag_model = args.flag_model
# Define your transforms for the training and testing sets
data_transforms = {
'train':
transforms.Compose([
transforms.RandomRotation(30),
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
RandAugment(),
ImageNetPolicy(),
Cutout(size=16),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]),
'test':
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
}
# Load the datasets with ImageFolder
image_datasets = {
x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x])
for x in ['train', 'test']
}
data_loader = {
x: torch.utils.data.DataLoader(image_datasets[x],
batch_size=batch_size,
shuffe=True,
num_workers=4,
pin_memory=True)
for x in ['train', 'test']
}
dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'test']}
class_names = image_datasets['train'].classes
# we get the class_to_index in the data_Set but what we really need is the cat_to_names so we will create
_ = image_datasets['train'].class_to_idx
cat_to_name = {_[i]: i for i in list(_.keys())}
# Run this to test the data loader
images, labels = next(iter(data_loader['test']))
# Loaded pre_trained model
model = create_model(flag_model=flag_model, flag_pretrained=True)
# Create classifiers
for param in model.parameters():
param.requires_grad = True
# Configs some last layers for my tasks classification covid
fc = nn.Sequential(
OrderedDict([('fc1', nn.Linear(2048, 1000, bias=True)),
('BN1',
nn.BatchNorm2d(1000,
eps=1e-05,
momentum=0.1,
affine=True,
track_running_stats=True)),
('dropout1', nn.Dropout(0.7)),
('fc2', nn.Linear(1000, 512)),
('BN2',
nn.BatchNorm2d(512,
eps=1e-05,
momentum=0.1,
affine=True,
track_running_stats=True)),
('swish1', Swish()), ('dropout2', nn.Dropout(0.5)),
('fc3', nn.Linear(512, 128)),
('BN3',
nn.BatchNorm2d(128,
eps=1e-05,
momentum=0.1,
affine=True,
track_running_stats=True)),
('swish2', Swish()), ('fc4', nn.Linear(128, 3)),
('output', nn.Softmax(dim=1))]))
# Connect pretrained model with modified classifer layer
model.fc = fc
criterion = nn.CrossEntropyLoss()
# Optimizer
optimizer = optim.SGD(model.parameters(),
lr=0.01,
momentum=0.9,
nesterov=True,
weight_decay=0.0001)
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.1)
# Send to GPU
model.to(device)
#Set checkpoint
try:
checkpoint = torch.load(CHECK_POINT_PATH)
print("checkpoint loaded")
except:
checkpoint = None
print("checkpoint not found")
if checkpoint == None:
CHECK_POINT_PATH = CHECK_POINT_PATH
model, best_val_loss, best_val_acc = train_model(
model,
criterion,
optimizer,
scheduler,
num_epochs=epochs,
checkpoint=torch.load(CHECK_POINT_PATH))
torch.save(
{
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'best_val_loss': best_val_loss,
'best_val_accuracy': best_val_acc,
'scheduler_state_dict': scheduler.state_dict(),
}, CHECK_POINT_PATH)
# Load checkpoint path
try:
checkpoint = torch.load(CHECK_POINT_PATH)
print("checkpoint loaded")
except:
checkpoint = None
print("checkpoint not found")
load_model(model=model, checkpoint=checkpoint, path=CHECK_POINT_PATH)
# Testing
since = time.time()
model.eval()
y_test = []
y_pred = []
for images, labels in data_loader['test']:
images = Variable(images.cuda())
labels = Variable(labels.cuda())
outputs = model(images)
_, predictions = outputs.max(1)
y_test.append(labels.data.cpu().numpy())
y_pred.append(predictions.data.cpu().numpy())
y_test = np.concatenate(y_test)
y_pred = np.concatenate(y_pred)
pd.DataFrame({
'true_label': y_test,
'predicted_label': y_pred
}).to_csv('Modified_EfficienNet_B0_Covid-19_Test.csv', index=False)
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
sns.heatmap(confusion_matrix(y_test, y_pred))
accuracy_score(y_test, y_pred)
report = classification_report(y_test, y_pred)
print(report)
def main():
args = config.base_parser()
# Save file name
tr_names = ""
for trans in args.transforms:
tr_names += "_" + trans
save_path = f"{args.dataset}/{args.mode}_{args.mem_manage}_{args.stream_env}_msz{args.memory_size}_rnd{args.rnd_seed}{tr_names}"
logging.config.fileConfig("./configuration/logging.conf")
logger = logging.getLogger()
os.makedirs(f"logs/{args.dataset}", exist_ok=True)
fileHandler = logging.FileHandler("logs/{}.log".format(save_path),
mode="w")
formatter = logging.Formatter(
"[%(levelname)s] %(filename)s:%(lineno)d > %(message)s")
fileHandler.setFormatter(formatter)
logger.addHandler(fileHandler)
writer = SummaryWriter("tensorboard")
if torch.cuda.is_available():
device = torch.device("cuda")
else:
device = torch.device("cpu")
logger.info(f"Set the device ({device})")
# Fix the random seeds
# https://hoya012.github.io/blog/reproducible_pytorch/
torch.manual_seed(args.rnd_seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(args.rnd_seed)
random.seed(args.rnd_seed)
# Transform Definition
mean, std, n_classes, inp_size, _ = get_statistics(dataset=args.dataset)
train_transform = []
if "cutout" in args.transforms:
train_transform.append(Cutout(size=16))
if "randaug" in args.transforms:
train_transform.append(RandAugment())
if "autoaug" in args.transforms:
train_transform.append(select_autoaugment(args.dataset))
train_transform = transforms.Compose([
transforms.Resize((inp_size, inp_size)),
transforms.RandomCrop(inp_size, padding=4),
transforms.RandomHorizontalFlip(),
*train_transform,
transforms.ToTensor(),
transforms.Normalize(mean, std),
])
logger.info(f"Using train-transforms {train_transform}")
test_transform = transforms.Compose([
transforms.Resize((inp_size, inp_size)),
transforms.ToTensor(),
transforms.Normalize(mean, std),
])
logger.info(f"[1] Select a CIL method ({args.mode})")
criterion = nn.CrossEntropyLoss(reduction="mean")
method = select_method(args, criterion, device, train_transform,
test_transform, n_classes)
logger.info(f"[2] Incrementally training {args.n_tasks} tasks")
task_records = defaultdict(list)
for cur_iter in range(args.n_tasks):
if args.mode == "joint" and cur_iter > 0:
return
print("\n" + "#" * 50)
print(f"# Task {cur_iter} iteration")
print("#" * 50 + "\n")
logger.info("[2-1] Prepare a datalist for the current task")
task_acc = 0.0
eval_dict = dict()
# get datalist
cur_train_datalist = get_train_datalist(args, cur_iter)
cur_test_datalist = get_test_datalist(args, args.exp_name, cur_iter)
# Reduce datalist in Debug mode
if args.debug:
random.shuffle(cur_train_datalist)
random.shuffle(cur_test_datalist)
cur_train_datalist = cur_train_datalist[:2560]
cur_test_datalist = cur_test_datalist[:2560]
logger.info("[2-2] Set environment for the current task")
method.set_current_dataset(cur_train_datalist, cur_test_datalist)
# Increment known class for current task iteration.
method.before_task(cur_train_datalist, cur_iter, args.init_model,
args.init_opt)
# The way to handle streamed samles
logger.info(f"[2-3] Start to train under {args.stream_env}")
if args.stream_env == "offline" or args.mode == "joint" or args.mode == "gdumb":
# Offline Train
task_acc, eval_dict = method.train(
cur_iter=cur_iter,
n_epoch=args.n_epoch,
batch_size=args.batchsize,
n_worker=args.n_worker,
)
if args.mode == "joint":
logger.info(f"joint accuracy: {task_acc}")
elif args.stream_env == "online":
# Online Train
logger.info("Train over streamed data once")
method.train(
cur_iter=cur_iter,
n_epoch=1,
batch_size=args.batchsize,
n_worker=args.n_worker,
)
method.update_memory(cur_iter)
# No stremed training data, train with only memory_list
method.set_current_dataset([], cur_test_datalist)
logger.info("Train over memory")
task_acc, eval_dict = method.train(
cur_iter=cur_iter,
n_epoch=args.n_epoch,
batch_size=args.batchsize,
n_worker=args.n_worker,
)
method.after_task(cur_iter)
logger.info("[2-4] Update the information for the current task")
method.after_task(cur_iter)
task_records["task_acc"].append(task_acc)
# task_records['cls_acc'][k][j] = break down j-class accuracy from 'task_acc'
task_records["cls_acc"].append(eval_dict["cls_acc"])
# Notify to NSML
logger.info("[2-5] Report task result")
writer.add_scalar("Metrics/TaskAcc", task_acc, cur_iter)
np.save(f"results/{save_path}.npy", task_records["task_acc"])
# Accuracy (A)
A_avg = np.mean(task_records["task_acc"])
A_last = task_records["task_acc"][args.n_tasks - 1]
# Forgetting (F)
acc_arr = np.array(task_records["cls_acc"])
# cls_acc = (k, j), acc for j at k
cls_acc = acc_arr.reshape(-1, args.n_cls_a_task).mean(1).reshape(
args.n_tasks, -1)
for k in range(args.n_tasks):
forget_k = []
for j in range(args.n_tasks):
if j < k:
forget_k.append(cls_acc[:k, j].max() - cls_acc[k, j])
else:
forget_k.append(None)
task_records["forget"].append(forget_k)
F_last = np.mean(task_records["forget"][-1][:-1])
# Intrasigence (I)
I_last = args.joint_acc - A_last
logger.info(f"======== Summary =======")
logger.info(
f"A_last {A_last} | A_avg {A_avg} | F_last {F_last} | I_last {I_last}")
def main():
args = parser.parse_args()
args.batch_size = args.batch_size
# setup model
print('creating model...')
#state = torch.load(args.model_path, map_location='cpu')
#args.num_classes = state['num_classes']
args.do_bottleneck_head = True
model = create_model(args).cuda()
ema = EMA(model, 0.999)
ema.register()
#model.load_state_dict(state['model'], strict=True)
#model.train()
classes_list = np.array(list(idx_to_class.values()))
print('done\n')
# Data loading code
normalize = transforms.Normalize(mean=[0, 0, 0],
std=[1, 1, 1])
instances_path_val = os.path.join(args.data, 'annotations/instances_val2017.json')
#instances_path_train = os.path.join(args.data, 'annotations/instances_val2017.json')#temprarily use val as train
instances_path_train = os.path.join(args.data, 'annotations/instances_train2017.json')
data_path_val = os.path.join(args.data, 'val2017')
#data_path_train = os.path.join(args.data, 'val2017')#temporarily use val as train
data_path_train = os.path.join(args.data, 'train2017')
val_dataset = CocoDetection(data_path_val,
instances_path_val,
transforms.Compose([
transforms.Resize((args.image_size, args.image_size)),
RandAugment(),
transforms.ToTensor(),
normalize,
Cutout(n_holes = 1, length = 16)
]))
train_dataset = CocoDetection(data_path_train,
instances_path_train,
transforms.Compose([
transforms.Resize((args.image_size, args.image_size)),
RandAugment(),
transforms.ToTensor(),
normalize,
]))
print("len(val_dataset)): ", len(val_dataset))
print("len(train_dataset)): ", len(train_dataset))
val_loader = torch.utils.data.DataLoader(
val_dataset, batch_size=args.batch_size, shuffle=False,
num_workers=args.workers, pin_memory=False)
train_loader = torch.utils.data.DataLoader(
train_dataset, batch_size=args.batch_size, shuffle=True,
num_workers=args.workers, pin_memory=False)
lr = 0.0002
Epoch = 10
criterion = AsymmetricLoss()
params = model.parameters()
optimizer = torch.optim.Adam(params, lr=lr)#尝试新的optimizer
total_step = len(train_loader)
scheduler = lr_scheduler.OneCycleLR(optimizer, max_lr = lr, total_steps = total_step, epochs = Epoch)
#total_step = len(train_loader)
highest_mAP = 0
trainInfoList = []
Sig = torch.nn.Sigmoid()
#f=open('info_train.txt', 'a')
for epoch in range(Epoch):
for i, (inputData, target) in enumerate(train_loader):
f=open('info_train.txt', 'a')
#model.train()
inputData = inputData.cuda()
target = target.cuda()
target = target.max(dim=1)[0]
#Sig = torch.nn.Sigmoid()
output = Sig(model(inputData))
#output[output<args.thre] = 0
#output[output>=args.thre]=1
#print(output.shape) #(batchsize, channel, imhsize, imgsize)
#print(inputData.shape) #(batchsize, numclasses)
#print(output[0])
#print(target[0])
loss = criterion(output, target)
model.zero_grad()
loss.backward()
optimizer.step()
ema.update()
#store information
if i % 10 == 0:
trainInfoList.append([epoch, i, loss.item()])
print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'
.format(epoch, Epoch, i, total_step, loss.item()))
f.write('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}\n'
.format(epoch, Epoch, i, total_step, loss.item()))
if (i+1) % 100 == 0:
#储存相应迭代模型
torch.save(model.state_dict(), os.path.join(
'models/', 'model-{}-{}.ckpt'.format(epoch+1, i+1)))
#modelName = 'models/' + 'decoder-{}-{}.ckpt'.format(epoch+1, i+1)
mAP_score = validate_multi(val_loader, model, args, ema)
#model.train()
if mAP_score > highest_mAP:
highest_mAP = mAP_score
print('current highest_mAP = ', highest_mAP)
f.write('current highest_mAP = {}\n'.format(highest_mAP))
torch.save(model.state_dict(), os.path.join(
'models/', 'model-highest.ckpt'))
f.close()
scheduler.step()#修改学习率
def student_training_extra(augment='Rand',
percentile=0,
model_type='ResNext',
save=False,
cycle=False,
epochs=5):
print(percentile)
if cycle:
model = torch.load('Data/' + model_type + '_model_cycled.pt')
else:
model = torch.load('Data/' + model_type + '_model.pt')
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize([.5, .5, .5], [.5, .5, .5])
])
extra_data = datasets.SVHN('Data/',
split='extra',
download=True,
transform=transform)
extra_loader = torch.utils.data.DataLoader(extra_data,
batch_size=1024,
shuffle=False)
# Switch some layers (e.g., batch norm, dropout) to evaluation mode
model.eval()
# Turn off the autograd to save memory usage and speed up
prediction_list = []
label_list = []
with torch.no_grad():
for data in extra_loader:
images, labels = data
if torch.cuda.is_available():
images = images.to(device)
labels = labels.to(device)
outputs = model(images)
predicted = outputs.data
prediction_list.append(predicted.cpu())
label_list.append(labels.cpu())
p_list = torch.cat(prediction_list)
p_list = p_list.numpy()
"""### Select threshold"""
model = models.resnext50_32x4d(pretrained=True)
num_ftrs = model.fc.in_features
model.fc = nn.Sequential(nn.Linear(num_ftrs, 10), nn.Softmax())
threshold = np.percentile(np.max(p_list, axis=1), percentile)
print(threshold)
if threshold < .5:
indexes = list(range(p_list.shape[0]))
labels = np.argmax(p_list, axis=1)
else:
print('here')
labels = (p_list >= threshold).astype(int) * np.array(
[10, 1, 2, 3, 4, 5, 6, 7, 8, 9])
indexes = np.array(list(range(np.max(p_list.shape))))[list(
(np.sum(labels, axis=1) > .1))]
labels = np.argmax(labels[indexes], axis=1)
train_data = datasets.SVHN(
root='Data/',
split='train',
download=True,
transform=transform,
)
bs = int(73257 / (73257 + labels.shape[0]) * 1024)
train_dataloader = torch.utils.data.DataLoader(
train_data,
batch_size=bs,
)
"""## Stage 2.1: Reload Unlabeled (Extra) Data and apply *RandAugment*"""
if augment == 'Rand':
rand_transform = transforms.Compose([
RandAugment(),
transforms.ToTensor(),
transforms.Normalize([.5, .5, .5], [.5, .5, .5])
])
elif augment == 'Traditional':
rand_transform = transforms.Compose([
transforms.RandomAffine(30),
transforms.ToTensor(),
transforms.Normalize([.5, .5, .5], [.5, .5, .5])
])
else:
rand_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize([.5, .5, .5], [.5, .5, .5])
])
aug_data = datasets.SVHN(
root='Data/',
split='extra',
download=True,
transform=rand_transform,
)
aug_dataloader = torch.utils.data.DataLoader(
aug_data,
batch_size=(1024 - bs),
)
indices = np.array([False] * aug_data.data.shape[0])
indices[indexes] = True
#only need to run this cell when step 1 was skipped
test_data = datasets.SVHN('Data/',
split='test',
download=True,
transform=transform)
test_dataloader = torch.utils.data.DataLoader(test_data,
batch_size=1024)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=0.05, momentum=.9)
scheduler = optim.lr_scheduler.StepLR(optimizer,
step_size=5,
gamma=0.5)
model.to(device)
# Commented out IPython magic to ensure Python compatibility.
print(
'Epoch | Train Loss | Train Accuracy | Val Loss | Val Accuracy | LR '
)
print(
'_________________________________________________________________________________'
)
q = 0
run_len = train_data.data.shape[0] + labels.shape[0]
test_len = test_data.data.shape[0]
prev = 0
accuracy = []
while q < epochs:
running_loss = 0.0
running_corrects = 0
model.train()
inds = indices
labs = torch.Tensor(labels)
for i in enumerate(zip(train_dataloader, aug_dataloader)):
# get the inputs; data is a list of [inputs, labels]
b_s = min((1024 - bs), i[1][1][0].shape[0])
aug = i[1][1][0][inds[:b_s]]
inputs = torch.cat([i[1][0][0], aug])
l = torch.cat(
[i[1][0][1].float(), labs[:aug.shape[0]].float()])
# randomize the batches
ran = torch.randperm(l.shape[0])
inputs = inputs[ran]
l = l[ran]
if torch.cuda.is_available():
inputs = inputs.to(device)
l = l.to(device)
# zero the parameter gradients: Clean the gradient caclulated in the previous iteration
optimizer.zero_grad()
# forward + backward + optimize
outputs = model(inputs)
#print(outputs.shape)
#print(labels.shape)
loss = criterion(outputs, l.long())
# Calculate gradient of matrix with requires_grad = True
loss.backward()
# Apply the gradient calculate from last step to the matrix
optimizer.step()
# Add 1 more iteration count to learning rate scheduler
# print statistics
running_loss += loss.item()
_, preds = torch.max(outputs, 1)
running_corrects += torch.sum(preds == l.data)
for el in optimizer.param_groups:
lr = el['lr']
scheduler.step()
val_corrects = 0
val_loss = 0
model.eval()
inds = inds[b_s:]
with torch.no_grad():
for data in test_dataloader:
images, l = data
if torch.cuda.is_available():
images = images.to(device)
l = l.to(device)
outputs = model(images)
_, preds = torch.max(outputs, 1)
val_loss += criterion(outputs, l.long())
num = torch.sum(preds == l.data)
val_corrects += num
print(
" %2d | %.4f | %.4f | %.4f | %.4f | %.5f "
% (q, running_loss / run_len,
int(running_corrects) / run_len, val_loss / len(test_data),
int(val_corrects) / len(test_data), lr))
accuracy.append(int(val_corrects) / len(test_data))
#early stopping
if accuracy[-1] > prev:
torch.save(model, 'Data/' + model_type + '_model_cycled.pt')
prev = accuracy[-1]
else:
q = epochs
q += 1
print('Finished Training')
return (accuracy)
if args.seed != 0:
torch.manual_seed(args.seed)
# Data
print('==> Preparing data..')
transform_train = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(
(0.5, ),
(0.5, ),
)])
if args.rand_augment:
transform_train.transforms.insert(0, RandAugment(args.N, args.M))
if args.augment:
transform_train.transforms.insert(0, transforms.RandomCrop(28, padding=4))
transform_train.transforms.insert(1, transforms.RandomHorizontalFlip())
transform_test = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(
(0.5, ),
(0.5, ),
)])
trainset = datasets.FashionMNIST(root='~/data',
train=True,
download=True,
def main():
args = parser.parse_args()
args.do_bottleneck_head = False
# setup dist training
if 'WORLD_SIZE' in os.environ:
args.distributed = int(os.environ['WORLD_SIZE']) > 1
args.device = 'cuda:0'
args.world_size = 1
args.rank = 0 # global rank
if args.distributed:
args.device = 'cuda:%d' % args.local_rank
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend='nccl', init_method='env://')
args.world_size = torch.distributed.get_world_size()
args.rank = torch.distributed.get_rank()
print('Training in distributed mode with multiple processes, 1 GPU per process. Process %d, total %d.' % (args.rank, args.world_size))
else:
print('Training with a single process on 1 GPUs.')
assert args.rank >= 0
# setup logger
logger = setup_logger(output=args.output, distributed_rank=dist.get_rank(), color=False, name="Coco")
logger.info("Command: "+' '.join(sys.argv))
if dist.get_rank() == 0:
path = os.path.join(args.output, "config.json")
with open(path, 'w') as f:
json.dump(vars(args), f, indent=2)
logger.info("Full config saved to {}".format(path))
os.makedirs(osp.join(args.output, 'tmpdata'), exist_ok=True)
# Setup model
logger.info('creating model...')
model = create_model(args).cuda()
if args.model_path: # make sure to load pretrained ImageNet model
state = torch.load(args.model_path, map_location='cpu')
filtered_dict = {k: v for k, v in state['model'].items() if
(k in model.state_dict() and 'head.fc' not in k)}
model.load_state_dict(filtered_dict, strict=False)
logger.info('done\n')
ema = ModelEma(model, 0.9997) # 0.9997^641=0.82
if args.distributed:
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], broadcast_buffers=False)
# COCO Data loading
instances_path_val = os.path.join(args.data, 'annotations/instances_val2014.json')
instances_path_train = os.path.join(args.data, 'annotations/instances_train2014.json')
data_path_val = args.data
data_path_train = args.data
val_dataset = CocoDetection(data_path_val,
instances_path_val,
transforms.Compose([
transforms.Resize((args.image_size, args.image_size)),
transforms.ToTensor(),
# normalize, # no need, toTensor does normalization
]))
train_dataset = CocoDetection(data_path_train,
instances_path_train,
transforms.Compose([
transforms.Resize((args.image_size, args.image_size)),
CutoutPIL(cutout_factor=0.5),
RandAugment(),
transforms.ToTensor(),
# normalize,
]))
logger.info("len(val_dataset)): {}".format(len(val_dataset)))
logger.info("len(train_dataset)): {}".format(len(train_dataset)))
train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset)
val_sampler = torch.utils.data.distributed.DistributedSampler(val_dataset, shuffle=False)
assert args.batch_size // dist.get_world_size() == args.batch_size / dist.get_world_size(), 'Batch size is not divisible by num of gpus.'
# Pytorch Data loader
train_loader = torch.utils.data.DataLoader(
train_dataset, batch_size=args.batch_size // dist.get_world_size(),
shuffle=not args.distributed,
num_workers=args.workers, pin_memory=True, sampler=train_sampler, drop_last=True)
val_loader = torch.utils.data.DataLoader(
val_dataset, batch_size=args.batch_size // dist.get_world_size(),
shuffle=False,
num_workers=args.workers, pin_memory=False, sampler=val_sampler)
# Actuall Training
train_multi_label_coco(model, ema, train_loader, val_loader, args.lr, args, logger)
)
bs = int(73257/(73257+labels.shape[0])*1024)
train_dataloader = torch.utils.data.DataLoader(
train_data, batch_size=bs,
)
"""## Stage 2.1: Reload Unlabeled (Extra) Data and apply *RandAugment*"""
if augment == 'Rand':
rand_transform = transforms.Compose([RandAugment(),
transforms.ToTensor(),
transforms.Normalize([.5,.5,.5],[.5,.5,.5])
])
elif augment == 'Traditional':
rand_transform = transforms.Compose([transforms.RandomAffine(30),
transforms.ToTensor(),
transforms.Normalize([.5,.5,.5],[.5,.5,.5])
])
else:
rand_transform = transforms.Compose([transforms.ToTensor(),
transforms.Normalize([.5,.5,.5],[.5,.5,.5])
])
aug_data = datasets.SVHN(
root='Data/', split='extra',
def main():
args = parser.parse_args()
args.batch_size = args.batch_size
args.do_bottleneck_head = False
# setup model
print('creating model...')
model = create_model(args).cuda()
if args.model_path:
state = torch.load(args.model_path, map_location='cpu')
filtered_dict = {
k: v
for k, v in state['model'].items()
if (k in model.state_dict() and 'head.fc' not in k)
}
model.load_state_dict(filtered_dict, strict=False)
# model.load_state_dict(state['model'], strict=False)
print('done\n')
# Data loading code
normalize = transforms.Normalize(mean=[0, 0, 0], std=[1, 1, 1])
instances_path_val = os.path.join(args.data,
'annotations/instances_val2014.json')
instances_path_train = os.path.join(
args.data, 'annotations/instances_train2014.json')
# data_path_val = os.path.join(args.data, 'val2014')
# data_path_train = os.path.join(args.data, 'train2014')
data_path_val = args.data
data_path_train = args.data
val_dataset = CocoDetection(
data_path_val,
instances_path_val,
transforms.Compose([
transforms.Resize((args.image_size, args.image_size)),
transforms.ToTensor(),
# normalize, # no need, toTensor does normalization
]))
train_dataset = CocoDetection(
data_path_train,
instances_path_train,
transforms.Compose([
transforms.Resize((args.image_size, args.image_size)),
CutoutPIL(cutout_factor=0.5),
RandAugment(),
transforms.ToTensor(),
# normalize,
]))
print("len(val_dataset)): ", len(val_dataset))
print("len(train_dataset)): ", len(train_dataset))
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=False)
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=False)
train_multi_label_coco(model,
train_loader,
val_loader,
args.lr,
gamma_neg=4,
gamma_pos=0,
clip=0.05)
