def load_ckp(checkpoint_path, optimizer=None):
"""Loads model parameters (state_dict) from file_path. If optimizer is provided, loads state_dict of
optimizer assuming it is present in checkpoint.
Args:
checkpoint: (string) filename which needs to be loaded
model: (torch.nn.Module) model for which the parameters are loaded
optimizer: (torch.optim) optional: resume optimizer from checkpoint
"""
if torch.cuda.is_available():
map_location = lambda storage, loc: storage.cuda()
else:
map_location = 'cpu'
if not os.path.exists(checkpoint_path):
print("{} File doesn't exist ".format(checkpoint_path))
exit()
# LOAD CHECKPPOINT
checkpoint = torch.load(checkpoint_path, map_location=map_location)
# BUILD MODEL WHICH IS USED TO LOAD MODEL'S WEIGHTS
model = initialize_model(checkpoint['model_name'],
num_classes=config.NO_OF_CLASSES,
feature_extract=True,
use_pretrained=True)
# LOAD A MODEL'S WEIGHTS
model.load_state_dict(checkpoint['state_dict'])
# LOAD MODEL ATTRIBUTES
model.class_to_idx = checkpoint['class_to_idx']
model.best_score = checkpoint['valid_score']
model.model_name = checkpoint['model_name']
if optimizer:
optimizer.load_state_dict(checkpoint['optimizer'])
return model
from utils import *
from visualization.core import *
from visualization.core.utils import image_net_postprocessing
from visualization.core.utils import image_net_preprocessing
from torchvision.transforms import ToTensor, Resize, Compose, ToPILImage
# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
parser = argparse.ArgumentParser(description='Data Visualise')
parser.add_argument('--tag',
type=str,
default='food',
help='model to visualise (default: food)')
args = parser.parse_args()
model, _ = model.initialize_model("resnet", True, 2, use_pretrained=True)
model = model.to(device)
def tensor_to_PIL(tensor):
image = tensor.cpu().clone()
image = image.squeeze(0)
image = ToPILImage()(image)
return image
def test_model(SAVE_MODEL_PATH, TEST_PATH='./Data/', SAVE_PATH='./OUT/'):
# init
if device == torch.device('cpu'):
model.load_state_dict(torch.load(SAVE_MODEL_PATH, map_location='cpu'))
else:
if opt.samples:
faces = face_array(df_inital)
samples(faces, df_inital, emotion_type_dict)
if opt.view_data_counts:
compare(df_inital)
if opt.train:
# resnet,vgg,densenet,inception
model_name = opt.model
num_classes = 7
feature_extract = False
# Initialize the model for this run
model, input_size = initialize_model(model_name,
num_classes,
feature_extract,
use_pretrained=True)
# norm_mean, norm_std = compute_img_mean_std(faces)
norm_mean, norm_std = [0.5073955162068291], [0.2551289894150225]
train_transform = transforms.Compose([
transforms.RandomCrop(44),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(norm_mean, norm_std),
])
# define the transformation of the val images.
val_transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(norm_mean, norm_std)])
from selective_search import find_region_proposals
import cv2
import numpy as np
from model import initialize_model, load_model
from dataset import prepare_image
import torch
import matplotlib.pyplot as plt
import torchvision
from tqdm import tqdm
NUM_CLASSES = 21
image_size = (224, 224)
MODEL_PATH = './checkpoints/regression 12-03-2021 22-38-46 epoch-2.pth'
model = initialize_model(NUM_CLASSES)
model, _, _ = load_model(model, MODEL_PATH)
DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
model = model.to(DEVICE)
print("Model Loaded...")
for param in model.parameters():
param.requires_grad = False
torch.no_grad()
class_list = [
'background', 'person', 'bird', 'cat', 'cow', 'dog', 'horse', 'sheep',
'aeroplane', 'bicycle', 'boat', 'bus', 'car', 'motorbike', 'train',
'bottle', 'chair', 'diningtable', 'pottedplant', 'sofa', 'tvmonitor'
]
def draw_boxes(image, boxes):
new_image = image.copy()
for box in boxes:
def main(ckp_path=None):
"""ckp_path (str): checkpoint_path
Train the model from scratch if ckp_path is None else
Re-Train the model from previous checkpoint
"""
cli_args = get_train_args(__author__, __version__)
# Variables
data_dir = cli_args.data_dir
save_dir = cli_args.save_dir
file_name = cli_args.file_name
use_gpu = cli_args.use_gpu
# LOAD DATA
data_loaders = load_data(data_dir, config.IMG_SIZE, config.BATCH_SIZE)
# BUILD MODEL
if ckp_path == None:
model = initialize_model(model_name=config.MODEL_NAME,
num_classes=config.NO_OF_CLASSES,
feature_extract=True,
use_pretrained=True)
else:
model = load_ckp(ckp_path)
# Device is available or not
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# If the user wants the gpu mode, check if cuda is available
if (use_gpu == True) and (torch.cuda.is_available() == False):
print("GPU mode is not available, using CPU...")
use_gpu = False
# MOVE MODEL TO AVAILBALE DEVICE
model.to(device)
# DEFINE OPTIMIZER
optimizer = optimizer_fn(model_name=config.MODEL_NAME,
model=model,
lr_rate=config.LR_RATE)
# DEFINE SCHEDULER
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode="min",
patience=5,
factor=0.3,
verbose=True)
# DEFINE LOSS FUNCTION
criterion = loss_fn()
# LOAD BEST MODEL'S WEIGHTS
best_model_wts = copy.deepcopy(model.state_dict())
# BEST VALIDATION SCORE
if ckp_path == None:
best_score = -1 # IF MODEL IS TRAIN FROM SCRATCH
else:
best_score = model.best_score # IF MODEL IS RE-TRAIN
# NO OF ITERATION
no_epochs = config.EPOCHS
# KEEP TRACK OF LOSS AND ACCURACY IN EACH EPOCH
stats = {
'train_losses': [],
'valid_losses': [],
'train_accuracies': [],
'valid_accuracies': []
}
print("Models's Training Start......")
for epoch in range(1, no_epochs + 1):
train_loss, train_score = train_fn(data_loaders,
model,
optimizer,
criterion,
device,
phase='train')
val_loss, val_score = eval_fn(data_loaders,
model,
criterion,
device=config.DEVICE,
phase='valid')
scheduler.step(val_loss)
# SAVE MODEL'S WEIGHTS IF MODEL' VALIDATION ACCURACY IS INCREASED
if val_score > best_score:
print(
'Validation score increased ({:.6f} --> {:.6f}). Saving model ...'
.format(best_score, val_score))
best_score = val_score
best_model_wts = copy.deepcopy(
model.state_dict()) #Saving the best model' weights
# MAKE A RECORD OF AVERAGE LOSSES AND ACCURACY IN EACH EPOCH FOR PLOTING
stats['train_losses'].append(train_loss)
stats['valid_losses'].append(val_loss)
stats['train_accuracies'].append(train_score)
stats['valid_accuracies'].append(val_score)
# PRINT TRAINING AND VALIDATION LOOS/ACCURACIES AFTER EACH EPOCH
epoch_len = len(str(no_epochs))
print_msg = (f'[{epoch:>{epoch_len}}/{no_epochs:>{epoch_len}}] ' +
'\t' + f'train_loss: {train_loss:.5f} ' + '\t' +
f'train_score: {train_score:.5f} ' + '\t' +
f'valid_loss: {val_loss:.5f} ' + '\t' +
f'valid_score: {val_score:.5f}')
print(print_msg)
# load best model weights
model.load_state_dict(best_model_wts)
# create checkpoint variable and add important data
model.class_to_idx = data_loaders['train'].dataset.class_to_idx
model.best_score = best_score
model.model_name = config.MODEL_NAME
checkpoint = {
'epoch': no_epochs,
'lr_rate': config.LR_RATE,
'model_name': config.MODEL_NAME,
'batch_size': config.BATCH_SIZE,
'valid_score': best_score,
'optimizer': optimizer.state_dict(),
'state_dict': model.state_dict(),
'class_to_idx': model.class_to_idx
}
# SAVE CHECKPOINT
save_ckp(checkpoint, save_dir, file_name)
print("Models's Training is Successfull......")
return model
model_types = [
'mobile', 'website', 'form', 'chart', 'grid', 'list_', 'dashboard',
'profile', 'checkout', 'landing', 'weather', 'sport', 'game', 'finance',
'travel', 'food', 'ecommerce', 'music', 'pink', 'black', 'white', 'green',
'blue', 'red', 'yellow'
]
model_types = ['sport']
# data loader
df = preprocess()
w2v = loadGloveModel()
test_loader = image_generator()
model_image, _ = model.initialize_model("resnet",
feature_extract=True,
use_pretrained=True)
model_tag = model.CNN()
model = model.MyEnsemble(model_image, model_tag)
model = model.to(device)
def test_model(tag):
if device == torch.device('cpu'):
model.load_state_dict(
torch.load('backup/' + tag + '/checkpoint.pt', map_location='cpu'))
else:
model.load_state_dict(torch.load('backup/' + tag + '/checkpoint.pt'))
model.eval()
id_ratio = {}
def train(model_name, freeze_layers, model_number, optimizer, learning_rate, num_classes = 2000, batch_size = 256, num_epochs = 20, input_size = 224, data_dir = "/home/kylecshan/data/images224/train_ms2000_v5/", save_path = "/home/kylecshan/saved/06_01_2019/triplet/delf_pca768/"):
data_transforms = {
'train': transforms.Compose([
transforms.Resize(input_size),
transforms.CenterCrop(input_size),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]),
'val': transforms.Compose([
transforms.Resize(input_size),
transforms.CenterCrop(input_size),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]),
}
image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x])
for x in ['train', 'val']}
for x in ['train', 'val']:
image_datasets[x] = TripletDataset(image_datasets[x])
# Create training and validation dataloaders
dataloaders_dict = {x: torch.utils.data.DataLoader(image_datasets[x],
batch_size=batch_size,
shuffle=True,
num_workers=4)
for x in ['train', 'val']}
# Detect if we have a GPU available
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# Initialize the model for this run
model_ft = initialize_model(model_name, num_classes, freeze_layers, use_pretrained=True)
# Send the model to GPU
if torch.cuda.device_count() > 0:
print("Let's use", torch.cuda.device_count(), "GPUs!")
model_ft = nn.DataParallel(model_ft)
model_ft = model_ft.to(device)
print("Params to learn:")
params_to_update = []
for name,param in model_ft.named_parameters():
if param.requires_grad == True:
params_to_update.append(param)
print("\t",name)
# Observe that all parameters are being optimized
if optimizer == "Adam":
optimizer_ft = optim.Adam(params_to_update, lr=learning_rate)
elif optimizer == "SGD":
optimizer_ft = optim.SGD(params_to_update, lr=learning_rate, momentum=0.9)
# Setup the loss fxn
criterion = triplet_loss(margin=0.1, norm=True)
# Train and evaluate
save_every = 1
for i in range(0, num_epochs, save_every):
model_ft, val_hist, train_hist = train_model_TL(model_ft, dataloaders_dict, criterion,
optimizer_ft, device, num_epochs=save_every,
is_inception=(model_name=="inception"))
# Save the model parameters as a .pth file
save_dir = save_path + "model" + str(model_number) +"_epoch"+ str(i+save_every-1) + ".pth"
torch.save(model_ft.state_dict(), save_dir)
#Save Validation History (val_hist) to csv file
save_val_csv = save_path + "val" + str(model_number) + ".csv"
with open(save_val_csv, 'ab') as f_val:
np.savetxt(f_val, val_hist)
#Save Training History (train_hist) to csv file
save_train_csv = save_path + "train" + str(model_number) + ".csv"
with open(save_train_csv, 'ab') as f_train:
np.savetxt(f_train, train_hist)
nb_class = 24
data_transforms = transforms.Compose([
transforms.Resize(224),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
test_dataset = ImageFolder('gray_testing/', data_transforms)
test_dataloader = DataLoader(test_dataset,
batch_size=16,
shuffle=True,
num_workers=8)
print(test_dataset.class_to_idx)
confusion_matrix = torch.zeros(nb_class, nb_class)
testing_model, input_size = initialize_model('resnet18', nb_class, False)
testing_model.eval()
testing_model = testing_model.to(device)
testing_model.load_state_dict(
torch.load('model/resnet18_24/model-29.ckpt'))
with torch.no_grad():
for i, (inputs, classes) in enumerate(test_dataloader):
inputs = inputs.to(device)
classes = classes.to(device)
outputs = testing_model(inputs)
_, pred = torch.max(outputs, 1)
for t, p in zip(classes.view(-1), pred.view(-1)):
confusion_matrix[t.long(), p.long()] += 1
print("confusion_matrix", confusion_matrix)
print("per class accuracy:",
confusion_matrix.diag() / confusion_matrix.sum(1))
# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# image loader
image = Image.open(args.i)
image = image.convert('RGB')
input = Compose([Resize((224, 224)),
ToTensor(), image_net_preprocessing])(image).unsqueeze(0)
input = input.to(device)
# model
model_name = "resnet"
feature_extract = True
num_classes = 2
model, _ = model.initialize_model(model_name,
feature_extract,
num_classes,
use_pretrained=True)
model = model.to(device)
def test():
MODEL_PATH = 'backup/' + args.tag + '/checkpoint.pt'
if device == torch.device('cpu'):
model.load_state_dict(torch.load(MODEL_PATH, map_location='cpu'))
else:
model.load_state_dict(torch.load(MODEL_PATH))
model.eval()
model_traced = module2traced(model, input)
first_layer = model_traced[0]
vis = Weights(model, device)
np.savetxt(f, neighbors, delimiter=' ', fmt="%s")
# if i >= 10:
# break
save_index = save_path + 'test.index'
faiss.write_index(testIndex, save_index)
if __name__ == '__main__':
# model_path = '/home/kylecshan/saved/06_01_2019/triplet/delf_pca768/model1_epoch0.pth'
save_path = '/home/kylecshan/saved/06_04_2019/densenet_base/densenet_base'
model_name = "densenet_class"
output_dim = 1664
batch_size = 256
dataloaders_dict = load_data(batch_size=batch_size)
# model = load_model(model_name, model_path)
model = initialize_model(model_name, use_pretrained=True)
save_index(model,
output_dim,
dataloaders_dict,
save_path,
batch_size=batch_size)
retrieve(model,
output_dim,
dataloaders_dict,
save_path,
batch_size=batch_size)
def main():
global args, best_prec1
global cur_itrs
args = parser.parse_args()
print(args.mode)
# STEP1: model
if args.mode=='baseline_train':
model = initialize_model(use_resnet=True, pretrained=False, nclasses=200)
elif args.mode=='pretrain':
model = deeplab_network.deeplabv3_resnet50(num_classes=args.num_classes, output_stride=args.output_stride, pretrained_backbone=False)
set_bn_momentum(model.backbone, momentum=0.01)
elif args.mode=='finetune':
model = initialize_model(use_resnet=True, pretrained=False, nclasses=3)
# load the pretrained model
if args.pretrained_model:
if os.path.isfile(args.pretrained_model):
print("=> loading pretrained model '{}'".format(args.pretrained_model))
checkpoint = torch.load(args.pretrained_model)
args.start_epoch = checkpoint['epoch']
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded pretrained model '{}' (epoch {})".format(args.resume, checkpoint['epoch']))
if torch.cuda.is_available:
model = model.cuda()
# STEP2: criterion and optimizer
if args.mode in ['baseline_train', 'finetune']:
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay)
# train_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=args.step_size, gamma=args.gamma)
elif args.mode=='pretrain':
criterion = nn.MSELoss()
optimizer = torch.optim.SGD(params=[
{'params': model.backbone.parameters(), 'lr': 0.1*args.lr},
{'params': model.classifier.parameters(), 'lr': args.lr},
], lr=args.lr, momentum=0.9, weight_decay=args.weight_decay)
scheduler = PolyLR(optimizer, args.total_itrs, power=0.9)
# STEP3: loss/prec record
if args.mode in ['baseline_train', 'finetune']:
train_losses = []
train_top1s = []
train_top5s = []
test_losses = []
test_top1s = []
test_top5s = []
elif args.mode == 'pretrain':
train_losses = []
test_losses = []
# STEP4: optionlly resume from a checkpoint
if args.resume:
print('resume')
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
if args.mode in ['baseline_train', 'finetune']:
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
datafile = args.resume.split('.pth')[0] + '.npz'
load_data = np.load(datafile)
train_losses = list(load_data['train_losses'])
train_top1s = list(load_data['train_top1s'])
train_top5s = list(load_data['train_top5s'])
test_losses = list(load_data['test_losses'])
test_top1s = list(load_data['test_top1s'])
test_top5s = list(load_data['test_top5s'])
elif args.mode=='pretrain':
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
scheduler.load_state_dict(checkpoint['scheduler'])
cur_itrs = checkpoint['cur_itrs']
datafile = args.resume.split('.pth')[0] + '.npz'
load_data = np.load(datafile)
train_losses = list(load_data['train_losses'])
# test_losses = list(load_data['test_losses'])
print("=> loaded checkpoint '{}' (epoch {})".format(args.resume, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(args.resume))
# STEP5: train!
if args.mode in ['baseline_train', 'finetune']:
# data
from utils import TinyImageNet_data_loader
print('color_distortion:', color_distortion)
train_loader, val_loader = TinyImageNet_data_loader(args.dataset, args.batch_size,color_distortion=args.color_distortion)
# if evaluate the model
if args.evaluate:
print('evaluate this model on validation dataset')
validate(val_loader, model, criterion, args.print_freq)
return
for epoch in range(args.start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch, args.lr)
time1 = time.time() #timekeeping
# train for one epoch
model.train()
loss, top1, top5 = train(train_loader, model, criterion, optimizer, epoch, args.print_freq)
train_losses.append(loss)
train_top1s.append(top1)
train_top5s.append(top5)
# evaluate on validation set
model.eval()
loss, prec1, prec5 = validate(val_loader, model, criterion, args.print_freq)
test_losses.append(loss)
test_top1s.append(prec1)
test_top5s.append(prec5)
# remember the best prec@1 and save checkpoint
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
save_checkpoint({
'epoch': epoch + 1,
'mode': args.mode,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict()
}, is_best, args.mode + '_' + args.dataset +'.pth')
np.savez(args.mode + '_' + args.dataset +'.npz', train_losses=train_losses,train_top1s=train_top1s,train_top5s=train_top5s, test_losses=test_losses,test_top1s=test_top1s, test_top5s=test_top5s)
# np.savez(args.mode + '_' + args.dataset +'.npz', train_losses=train_losses)
time2 = time.time() #timekeeping
print('Elapsed time for epoch:',time2 - time1,'s')
print('ETA of completion:',(time2 - time1)*(args.epochs - epoch - 1)/60,'minutes')
print()
elif args.mode=='pretrain':
#data
from utils import TinyImageNet_data_loader
# args.dataset = 'tiny-imagenet-200'
args.batch_size = 16
train_loader, val_loader = TinyImageNet_data_loader(args.dataset, args.batch_size, col=True)
# if evaluate the model, show some results
if args.evaluate:
print('evaluate this model on validation dataset')
visulization(val_loader, model, args.start_epoch)
return
# for epoch in range(args.start_epoch, args.epochs):
epoch = 0
while True:
if cur_itrs >= args.total_itrs:
return
# adjust_learning_rate(optimizer, epoch, args.lr)
time1 = time.time() #timekeeping
model.train()
# train for one epoch
# loss, _, _ = train(train_loader, model, criterion, optimizer, epoch, args.print_freq, colorization=True,scheduler=scheduler)
# train_losses.append(loss)
# model.eval()
# # evaluate on validation set
# loss, _, _ = validate(val_loader, model, criterion, args.print_freq, colorization=True)
# test_losses.append(loss)
save_checkpoint({
'epoch': epoch + 1,
'mode': args.mode,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
'scheduler':scheduler.state_dict(),
"cur_itrs": cur_itrs
}, True, args.mode + '_' + args.dataset +'.pth')
np.savez(args.mode + '_' + args.dataset +'.npz', train_losses=train_losses)
# scheduler.step()
time2 = time.time() #timekeeping
print('Elapsed time for epoch:',time2 - time1,'s')
print('ETA of completion:',(time2 - time1)*(args.total_itrs - cur_itrs - 1)/60,'minutes')
print()
epoch += 1
feature_extract=False use_pretrained=True cnn_encoding_length=2048 num_classes=2 num_epochs=100 learning_rate=1e-4 log_interval=10 # The interval at which the model will be saved root_dir='../Data/' #--------------------------------------------------------------- # Dataset loader train_loader=Dataset_CRNN(root_dir=root_dir,) # Define the cnn model cnnEnc=m.initialize_model(model_name,cnn_encoding_length,feature_extract,use_pretrained) # To use pretrained model # cnnEnc=MyModel() # To use your own model # Define RNN decoder rnnDec=m.DecoderRNN(CNN_embed_dim=cnn_encoding_length,h_RNN_layers=3, h_RNN=256, h_FC_dim=128, drop_p=0.3, num_classes=num_classes) # Params to update crnn_params=list(cnnEnc.parameters()) + list(rnnDec.parameters()) # Specify the loss to use loss_criterion=F.BCELoss() # Define the optimizer optimizer = torch.optim.Adam(crnn_params, lr=learning_rate) # Specify the device
torch.manual_seed(args.seed)
# data loader
img_tags = load_image_tags(args.tag)
w2v = loadGloveModel()
train_loader, valid_loader, test_loader = image_generator(args.tag)
# Hyperparameters
epochs = args.epochs
learning_rate = args.lr
# Models to choose from [resnet, alexnet, vgg, squeezenet, densenet, inception]
# Flag for feature extracting. When False, we finetune the whole model,
# when True we only update the reshaped layer params
model_name = args.model
feature_extract = args.mode
model_image, _ = model.initialize_model(model_name, feature_extract, use_pretrained=args.pretrain)
model_tag = model.CNN()
model = model.MyEnsemble(model_image, model_tag)
model = model.to(device)
# Gather the parameters to be optimized/updated in this run. If we are
# finetuning we will be updating all parameters. However, if we are
# doing feature extract method, we will only update the parameters
# that we have just initialized, i.e. the parameters with requires_grad
# is True.
params_to_update = model.parameters()
print("-"*10)
print("Params to learn:")
if feature_extract:
params_to_update = []
for name,param in model.named_parameters():
