random.seed(opt.seed)
torch.manual_seed(opt.seed)
torch.cuda.manual_seed(opt.seed)
torch.cuda.manual_seed_all(opt.seed)
"""==================================================================================================="""
##################### NETWORK SETUP ##################
#NOTE: Networks that can be used: 'bninception, resnet50, resnet101, alexnet...'
#>>>> see import pretrainedmodels; pretrainedmodels.model_names
opt.device = torch.device('cuda')
model = netlib.networkselect(opt)
# if opt.wandb_log: wandb.watch(model)
print(
'{} Setup for {} with {} sampling on {} complete with #weights: {}'.format(
opt.loss.upper(), opt.arch.upper(), opt.sampling.upper(),
opt.dataset.upper(), aux.gimme_params(model)))
_ = model.to(opt.device)
to_optim = [{
'params': model.parameters(),
'lr': opt.lr,
'weight_decay': opt.decay
}]
"""============================================================================"""
#################### DATALOADER SETUPS ##################
dataloaders = data.give_dataloaders(opt.dataset, opt)
opt.num_classes = len(dataloaders['training'].dataset.avail_classes)
print("num of classes", opt.num_classes)
print("num of training samples", len(dataloaders['training']))
"""============================================================================"""
json.dump(option_dict, f)
pkl.dump(opt, open(opt.save_path + "/hypa.pkl", "wb"))
#################### CREATE LOGGING FILES ###############
InfoPlotter = aux.InfoPlotter(opt.save_path + '/InfoPlot.svg')
CSV_log_train = aux.CSV_Writer(opt.save_path + '/log_epoch_train.csv',
['Epoch', 'Loss', 'Time'])
CSV_log_val = aux.CSV_Writer(opt.save_path + '/log_epoch_val.csv',
['Epoch', 'Loss', 'Time'])
Progress_Saver = {'Train Loss': [], 'Val Loss': []}
#################### SETUP TRIPLENET ###################
opt.device = torch.device('cuda')
model = netlib.FC_AlexNet(opt.num_classes)
print('TripletNet Setup complete with #weights: {}'.format(
aux.gimme_params(model)))
_ = model.to(opt.device)
#################### TRAINING SETUP ####################
triplet_loss = netlib.TripletLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=opt.tau,
gamma=opt.gamma)
global best_val_loss
best_val_loss = np.inf
#################### TRAINER & EVALUATION FUNCTIONS ############################
def train_one_epoch(dataloader, model, optimizer, triplet_loss, opt,
progress_saver, epoch):
def main():
################### INPUT ARGUMENTS ###################
parser = argparse.ArgumentParser()
parser.add_argument('--num_cluster',
default=200,
type=int,
help='number of kmeans cluster')
parser.add_argument('--lr', default=0.05, type=float, help='Learning Rate')
parser.add_argument('--arch',
default='alexnet',
type=str,
help='Learning Rate')
parser.add_argument('--l2',
default=0.00001,
type=float,
help='L2-Weight-regularization')
parser.add_argument('--perc_data',
default=1,
type=float,
help='L2-Weight-regularization')
parser.add_argument(
'--cluster_intv',
default=1,
type=int,
help='Number of epchs before recomputing supervised cluster labels')
parser.add_argument('--kernels',
default=8,
type=int,
help='Number of cores to use.')
parser.add_argument('--bs',
default=2,
type=int,
help='Mini-Batchsize to use.')
parser.add_argument('--seed',
default=1,
type=int,
help='Random seed for reproducibility.')
parser.add_argument('--n_epochs',
default=100,
type=int,
help='Number of training epochs.')
parser.add_argument('--gpu', default=0, type=int, help='GPU to use.')
parser.add_argument('--iter_update',
default=200,
type=int,
help='Number of iterations before each log entry.')
# parser.add_argument('--save_path', default='/export/home/kroth/Project_Manifold/SAVEDATA', type=str, help='Path to save training information')
# parser.add_argument('--data_path', default='/export/home/kroth/Project_Manifold/LOADDATA', type=str, help='Path to load training information')
parser.add_argument(
'--save_path',
default=
'/home/karsten_dl/Dropbox/Data_Dump/DeepClustering/SAVEDATA/DeepClusterNetwork_Training',
type=str,
help='Path to save training information')
parser.add_argument(
'--data_path',
default='/home/karsten_dl/Dropbox/Data_Dump/DeepClustering/LOADDATA',
type=str,
help='Path to save training information')
parser.add_argument('--savename',
default='',
type=str,
help='Save Folder Name')
parser.add_argument('--pca_dim',
default=128,
type=int,
help='Use sobel filter as initialization.')
parser.add_argument('--no_sobel',
action='store_true',
help='Use sobel filter as initialization.')
parser.add_argument('--no_dim_reduction',
action='store_true',
help='Use sobel filter as initialization.')
parser.add_argument('--make_umap_plots',
action='store_true',
help='Use sobel filter as initialization.')
parser.add_argument('--dont_preload_images',
action='store_true',
help='Use sobel filter as initialization.')
opt = parser.parse_args(['--dont_preload_images', '--perc_data', '0.01'])
opt.use_sobel = not opt.no_sobel
opt.use_dim_reduction = not opt.no_dim_reduction
opt.preload_images = not opt.dont_preload_images
################ FIX SEEDS ##################
torch.manual_seed(opt.seed)
torch.cuda.manual_seed(opt.seed)
np.random.seed(opt.seed)
random.seed(opt.seed)
rng = np.random.RandomState(opt.seed)
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = str(opt.gpu)
opt.device = torch.device('cuda')
################ SET DATALOADER ###########
print('\033[92m' + 'Setting up DataLoader... ' + '\033[0m')
image_dataset = dataset.CelebA_Dataset(opt.data_path,
perc_data=opt.perc_data,
preload_images=opt.preload_images)
image_dataloader = DataLoader(image_dataset,
batch_size=opt.bs,
num_workers=opt.kernels,
pin_memory=True)
input_size = image_dataset[0]['Input Image'].numpy().shape[1:]
print('Done.\n')
################ LOAD MODEL ################
print('\033[92m' + 'Setting up network [{}]... '.format(opt.arch.upper()) +
'\033[0m')
imp.reload(netlib)
model = netlib.AlexNet(opt.num_cluster,
input_size,
use_sobel=opt.use_sobel,
use_batchnorm=True)
_ = model.to(device)
################ SET OPTIMIZER ############
optimizer = torch.optim.SGD(filter(lambda x: x.requires_grad,
model.parameters()),
lr=opt.lr,
momentum=0.9,
weight_decay=opt.l2)
loss_function = nn.CrossEntropyLoss().cuda()
print('Done with [#weights] {}.\n'.format(aux.gimme_params(model)))
############### SET SAVEFOLDER ############
savename = model.name
if opt.savename == '':
savename += '___' + opt.savename
date = datetime.datetime.now()
savename += '___' + '-'.join(
str(x) for x in
[date.year, date.month, date.day, date.hour, date.minute, date.second])
opt.savename = savename
opt.savefolder = opt.save_path + '/' + opt.savename
checkfolder, counter = opt.savefolder, 1
while os.path.exists(checkfolder):
checkfolder = opt.savefolder + '_' + str(counter)
counter += 1
opt.savefolder = checkfolder
os.makedirs(opt.savefolder)
################ SET LOGGING ##############
plot_generator = aux.InfoPlotter(opt.savefolder + '/training_progress.png')
logs = aux.LogSet([
'cluster time', 'train time per iter', 'train time per epoch',
'loss per iter', 'loss per epoch'
])
CSV_Log_per_batch = aux.CSV_Writer(
opt.savefolder + '/training_log_iter-' + str(opt.iter_update) + '.csv',
columns=['Iteration', 'Loss', 'Elapsed Time'])
CSV_Log_per_epoch = aux.CSV_Writer(
opt.savefolder + '/training_log_epoch-' + str(opt.n_epochs) + '.csv',
columns=['Epoch', 'Loss', 'Elapsed Time'])
################ Training Function ###############
print('\033[93m' + 'Starting Training...\n' + '\033[0m')
epoch_iterator = trange(opt.n_epochs, position=0)
epoch_iterator.set_description('Running Training...')
for epoch in epoch_iterator:
opt.epoch = epoch
# opt.make_umap_plots = True if epoch%3==0 else False
image_dataloader.dataset.labels = None
pseudolabels = compute_clusters_and_set_dataloader_labels(
image_dataloader, model, opt)
clustering.adjust_model_compute_clusters_and_set_dataloader_labels(
image_dataloader, model, opt, logs)
aux.train_network_one_epoch(image_dataloader, model, loss_function,
optimizer, opt, epoch, logs,
CSV_Log_per_batch, CSV_Log_per_epoch)
epoch_iterator.set_description(
'Network Training | Curr Loss: {} | Best Loss: {}'.format(
logs.logs['loss per epoch'].log[-1],
np.min(logs.logs['loss per epoch'].log)))
if epoch >= 2:
plot_generator.make_plot(range(epoch + 1),
logs.logs['loss per epoch'].log)
"""============================================================================"""
#################### SEEDS FOR REPROD. #####################
torch.backends.cudnn.deterministic=True
np.random.seed(opt.seed); random.seed(opt.seed)
torch.manual_seed(opt.seed); torch.cuda.manual_seed(opt.seed); torch.cuda.manual_seed_all(opt.seed)
"""============================================================================"""
##################### NETWORK SETUP ##################
opt.device = torch.device('cuda')
#Depending on the choice opt.arch, networkselect() returns the respective network model
model = netlib.networkselect(opt)
print('{} Setup for {} with {} sampling on {} complete with #weights: {}'.format(opt.loss.upper(), opt.arch.upper(), opt.sampling.upper(), opt.dataset.upper(), aux.gimme_params(model)))
#Push to Device
_ = model.to(opt.device)
#Place trainable parameter in list of parameters to train:
if 'fc_lr_mul' in vars(opt).keys() and opt.fc_lr_mul!=0:
all_but_fc_params = list(filter(lambda x: 'last_linear' not in x[0],model.named_parameters()))
fc_params = model.model.last_linear.parameters()
to_optim = [{'params':all_but_fc_params,'lr':opt.lr,'weight_decay':opt.decay},
{'params':fc_params,'lr':opt.lr*opt.fc_lr_mul,'weight_decay':opt.decay}]
else:
to_optim = [{'params':model.parameters(),'lr':opt.lr,'weight_decay':opt.decay}]
def main():
############## SEEDS #########################################
torch.manual_seed(opt.seed)
torch.cuda.manual_seed(opt.seed)
np.random.seed(opt.seed)
random.seed(opt.seed)
torch.backends.cudnn.deterministic = True
bcolor = aux.bcolors
############## GPU SETTINGS ##################################
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = str(opt.gpu)
############## SET DATASETS AND -LOADER ######################
print(bcolor.HEADER + 'Setting DataLoader... ' + bcolor.ENDC)
if opt.dataset == 'cars196':
train_dataset, val_dataset = aux.give_CARS196_datasets(opt)
if opt.dataset == 'cub200':
train_dataset, val_dataset = aux.give_CUB200_datasets(opt)
if opt.dataset == 'online_products':
train_dataset, val_dataset = aux.give_OnlineProducts_datasets(opt)
train_dataloader = torch.utils.data.DataLoader(train_dataset,
batch_size=opt.bs,
num_workers=opt.kernels,
shuffle=True,
pin_memory=True)
val_dataloader = torch.utils.data.DataLoader(val_dataset,
batch_size=opt.bs,
num_workers=opt.kernels,
pin_memory=True)
opt.num_classes = len(train_dataset.avail_classes)
print('Done.')
############## INITIALIZE NETWORK ######################
print(bcolor.HEADER + 'Setting up Network & Log-Files... ' + bcolor.ENDC,
end='')
#################### CREATE SAVING FOLDER ###############
date = datetime.datetime.now()
time_string = '{}-{}-{}-{}-{}-{}'.format(date.year, date.month, date.day,
date.hour, date.minute,
date.second)
checkfolder = opt.save_path + '/{}_JigsawNetwork_'.format(
opt.dataset) + time_string
counter = 1
while os.path.exists(checkfolder):
checkfolder = opt.save_path + '_' + str(counter)
counter += 1
os.makedirs(checkfolder)
opt.save_path = checkfolder
#################### SAVE OPTIONS TO TXT ################
with open(opt.save_path + '/Parameter_Info.txt', 'w') as f:
f.write(aux.gimme_save_string(opt))
pkl.dump(opt, open(opt.save_path + "/hypa.pkl", "wb"))
#################### CREATE LOGGING FILES ###############
InfoPlotter = aux.InfoPlotter(opt.save_path + '/InfoPlot.svg')
full_log = aux.CSV_Writer(opt.save_path + '/log_epoch.csv',
['Epoch', 'Train Loss', 'Val Loss', 'Val Acc'])
Progress_Saver = {'Train Loss': [], 'Val NMI': [], 'Val Recall Sum': []}
#################### SETUP JIGSAW NETWORK ###################
opt.device = torch.device('cuda')
model = netlib.NetworkSelect(opt)
print('JIGSAW Setup for [{}] complete with #weights: {}'.format(
opt.arch, aux.gimme_params(model)))
_ = model.to(opt.device)
global best_val_acc
best_val_acc = 0
################### SET OPTIMIZATION SETUP ##################
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(),
lr=opt.lr,
weight_decay=1e-4)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
opt.tau,
gamma=opt.gamma)
print('Done.')
############## START TRAINING ###########################
print(bcolor.BOLD + bcolor.WARNING +
'Starting Jigsaw Network Training!\n' + bcolor.ENDC + bcolor.ENDC)
for epoch in range(opt.n_epochs):
scheduler.step()
### Training ###
train_one_epoch(opt, epoch, net, optimizer, criterion,
train_dataloader, Metrics)
### Validating ###
evaluate(opt, epoch, net, criterion, val_dataloader, Metrics)
###### Logging Epoch Data ######
full_log.log([
len(Metrics['Train Loss']), Metrics["Train Loss"][-1],
Metrics["Train Acc"][-1], Metrics["Val Acc"][-1]
])
###### Generating Summary Plots #######
InfoPlotter.make_plot(range(epoch + 1), Progress_Saver['Train Loss'],
Progress_Saver['Val Acc'],
['Train Loss', 'Val Acc'])