model_frozen.eval()
model.freeze_encoder()
loss_function = return_loss_function(model_frozen)
mean_epoch_loss, validation_loss = \
decoder_step(model, loss_function, optimizer, data_loader_train, data_loader_val, device)
print(' autoencoder loss: {0:2.5f}, BCE val: {1:2.5f}'.format(
mean_epoch_loss, validation_loss))
if __name__ == "__main__":
device = conf['train']['device']
model = AutoEncoder(in_channels=1,
dec_channels=1,
latent_size=conf['model']['latent_size'])
model = model.to(device)
model.load_state_dict(torch.load(load_path))
dataset_path = os.path.join(conf['data']['dataset_path'],
conf['data']['dataset_file'])
dspites_dataset = Dspites(dataset_path)
train_val = train_val_split(dspites_dataset)
val_test = train_val_split(train_val['val'], val_split=0.2)
data_loader_train = DataLoader(train_val['train'],
batch_size=conf['train']['batch_size'],
shuffle=True,
num_workers=2)
data_loader_val = DataLoader(val_test['val'],
batch_size=200,
shuffle=False,
config = Config()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
autoencoder = AutoEncoder(config)
siamese_network = SiameseNetwork(config)
autoencoder_file = '/autoencoder_epoch175_loss1.1991.pth'
siamese_file = '/siamese_network_epoch175_loss1.1991.pth'
if config.load_model:
autoencoder.load_state_dict(torch.load(config.saved_models_folder + autoencoder_file))
siamese_network.load_state_dict(torch.load(config.saved_models_folder + siamese_file))
autoencoder.to(device)
autoencoder.train()
siamese_network.to(device)
siamese_network.train()
params = list(autoencoder.parameters()) + list(siamese_network.parameters())
optimizer = torch.optim.Adam(params, lr=config.lr, betas=(0.9, 0.999))
transform = transforms.Compose([
transforms.Grayscale(num_output_channels=1),
# transforms.RandomCrop(size=128),
# transforms.RandomRotation(degrees=10),
transforms.ToTensor(),
])
def main():
with open("config.json") as json_file:
conf = json.load(json_file)
dataset_path = os.path.join(conf['data']['dataset_path'],
conf['data']['dataset_file'])
device = conf['train']['device']
model = AutoEncoder(in_channels=1,
dec_channels=1,
latent_size=conf['model']['latent_size'])
model = model.to(device)
model.load_state_dict(torch.load(load_path))
dspites_dataset = Dspites(dataset_path)
train_val = train_val_split(dspites_dataset)
val_test = train_val_split(train_val['val'], val_split=0.2)
data_loader_train = DataLoader(train_val['train'],
batch_size=conf['train']['batch_size'],
shuffle=True,
num_workers=2)
data_loader_val = DataLoader(val_test['val'],
batch_size=200,
shuffle=False,
num_workers=1)
data_loader_test = DataLoader(val_test['train'],
batch_size=200,
shuffle=False,
num_workers=1)
print('autoencoder training')
print('frozen encoder: ', freeze_encoder)
print('train dataset length: ', len(train_val['train']))
print('val dataset length: ', len(val_test['val']))
print('test dataset length: ', len(val_test['train']))
print('latent space size:', conf['model']['latent_size'])
print('batch size:', conf['train']['batch_size'])
loss_function = nn.BCELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
model.train()
if freeze_encoder:
model.freeze_encoder()
for epoch in range(25):
if epoch > 15:
for param in optimizer.param_groups:
param['lr'] = max(0.00001,
param['lr'] / conf['train']['lr_decay'])
print('lr: ', param['lr'])
loss_list = []
model.train()
for batch_i, batch in enumerate(data_loader_train):
augment_transform = np.random.choice(augment_transform_list1)
batch1 = image_batch_transformation(batch, augment_transform)
loss = autoencoder_step(model, batch, device, loss_function)
loss_list.append(loss.item())
optimizer.zero_grad()
loss.backward()
optimizer.step()
mean_epoch_loss = sum(loss_list) / len(loss_list)
model.eval()
validation_loss = autoencoder_validation(data_loader_val, model,
device, loss_function)
if epoch == 0:
min_validation_loss = validation_loss
else:
min_validation_loss = min(min_validation_loss, validation_loss)
print('epoch {0}, loss: {1:2.5f}, validation: {2:2.5f}'.format(
epoch, mean_epoch_loss, validation_loss))
if min_validation_loss == validation_loss:
#pass
torch.save(model.state_dict(), save_path)
model.load_state_dict(torch.load(save_path))
test_results = autoencoder_validation(data_loader_test, model, device,
loss_function)
print('test result: ', test_results)
def main():
save = "./experiments"
log_dir = "./experiments"
input_path = "./data//mnist"
batch_size = 16
lr = 1e-3
latent_size = 12
n_iter = 10
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
checkpoint_dir = f"{save}/checkpoints/autoencoder"
os.makedirs(checkpoint_dir, exist_ok=True)
if device.type == 'cuda':
log_dir = f"{log_dir}/logs/autoencoder/cuda"
else:
log_dir = f"{log_dir}/logs/autoencoder/cpu"
os.makedirs(log_dir, exist_ok=True)
checkpoint_path = f'{checkpoint_dir}/checkpoint_' + datetime.now(
).strftime('%d_%m_%Y_%H:%M:%S')
writer = SummaryWriter(log_dir)
writer = SummaryWriter(log_dir)
data = MNIST(transform=True,
test_size=0.1,
train_batch_size=batch_size,
input_path=input_path)
traindata, valdata, testdata = data.data()
train, val, test = data.loader()
n = 300
x, labels = testdata[np.random.randint(0, len(testdata), n)]
images, labels = torch.from_numpy(x.reshape(
n, 1, 28, 28)), torch.from_numpy(labels).to(device)
img_grid = torchvision.utils.make_grid(images)
# matplotlib_imshow(img_grid, one_channel=True)
writer.add_image(f'{n}_mnist_images', img_grid)
images, labels = images.to(device), labels.to(device)
model = AutoEncoder(28 * 28, latent_size)
optimizer = torch.optim.Adam(params=model.parameters(), lr=lr)
criterion = torch.nn.MSELoss()
model = model.to(device)
writer.add_graph(model, images.view(len(images), 28 * 28))
losses = train_autoencoder(train,
test,
model,
criterion,
optimizer,
device,
checkpoint_path,
writer,
n_iter=n_iter)
with torch.no_grad():
projection = model.encodeur(images.view(len(images), 28 * 28))
writer.add_embedding(projection, metadata=labels, label_img=images)
def run(batch_size, epochs, val_split, num_workers, print_every,
trainval_csv_path, test_csv_path, model_type, tasks, lr, weight_decay,
momentum, dataset_dir):
train_dataset = CustomDatasetFromImages(trainval_csv_path,
data_dir=dataset_dir)
# test_dataset = CustomDatasetFromImages(test_csv_path, data_dir = dataset_dir)
dset_len = len(train_dataset)
val_size = int(val_split * dset_len)
test_size = int(0.15 * dset_len)
train_size = dset_len - val_size - test_size
train_data, val_dataset, test_dataset = torch.utils.data.random_split(
train_dataset, [train_size, val_size, test_size])
train_loader_small = torch.utils.data.DataLoader(dataset=train_data,
batch_size=batch_size,
pin_memory=False,
drop_last=True,
shuffle=True,
num_workers=num_workers)
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=2 * batch_size,
pin_memory=False,
drop_last=True,
shuffle=True,
num_workers=num_workers)
val_loader = torch.utils.data.DataLoader(dataset=val_dataset,
batch_size=batch_size,
pin_memory=False,
drop_last=True,
shuffle=True,
num_workers=num_workers)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=batch_size,
pin_memory=False,
drop_last=True,
shuffle=True,
num_workers=num_workers)
if model_type == 'densenet121':
model = models.densenet121(pretrained=False)
elif model_type == 'resnet101':
model = models.resnet101(pretrained=False)
elif model_type == 'resnet50':
model = models.resnet50(pretrained=False)
elif model_type == 'resnet34':
model = models.resnet34(pretrained=False)
elif model_type == 'vgg19':
model = models.vgg19(pretrained=False)
model = AutoEncoder(model_type, model=model)
model = nn.DataParallel(model)
print(model)
model = model.to('cuda')
criterion = nn.MSELoss(reduction='sum')
# =============================== PRE-TRAIN MODEL ========================
optimizer = torch.optim.SGD(model.parameters(),
weight_decay=weight_decay,
momentum=momentum,
lr=lr,
nesterov=True)
scheduler = ReduceLROnPlateau(optimizer,
factor=0.5,
patience=3,
min_lr=1e-7,
verbose=True)
trainset_percent = (1 - val_split - 0.15)
trainer = AutoTrainer(model,
optimizer,
scheduler,
criterion,
epochs,
print_every=print_every,
trainset_split=trainset_percent)
trainer.train(train_loader, val_loader)
val_loss = trainer.validate(test_loader)
with open(trainer.output_log, 'a+') as out:
print('Test Loss', val_loss, file=out)
shuffle=False,
num_workers=opt.workers,
pin_memory=True,
drop_last=True)
if opt.model == 'ae':
net = AutoEncoder(3, n_classes=1, filters=opt.filters)
elif opt.model == 'unet':
net = UNet(3, n_classes=1, filters=opt.filters)
elif opt.model == 'unet3plus':
net = UNet3Plus(3, n_classes=1, filters=opt.filters)
if device == torch.device('cuda'):
net = nn.DataParallel(net, device_ids=[0, 1, 2, 3])
logger.info(f'use gpu: {net.device_ids}')
net.to(device=device)
# optimizer = optim.RMSprop(net.parameters(), lr=opt.lr, weight_decay=1e-8, momentum=0.9)
optimizer = optim.SGD(net.parameters(), lr=opt.lr, momentum=0.9)
criterion = nn.BCEWithLogitsLoss()
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,
'max',
patience=opt.patience)
train_net(net=net,
device=device,
train_loader=train_loader,
val_loader=val_loader,
epochs=opt.epochs,
optimizer=optimizer,
criterion=criterion,
def main():
loss_function = nn.BCELoss()
with open("config.json") as json_file:
conf = json.load(json_file)
device = conf['train']['device']
dataset_path = os.path.join(conf['data']['dataset_path'],
conf['data']['dataset_file'])
dspites_dataset = Dspites(dataset_path)
train_val = train_val_split(dspites_dataset)
val_test = train_val_split(train_val['val'], val_split=0.2)
data_loader_train = DataLoader(train_val['train'],
batch_size=conf['train']['batch_size'],
shuffle=True,
num_workers=2)
data_loader_val = DataLoader(val_test['val'],
batch_size=200,
shuffle=False,
num_workers=1)
data_loader_test = DataLoader(val_test['train'],
batch_size=200,
shuffle=False,
num_workers=1)
print('metric learning')
print('train dataset length: ', len(train_val['train']))
print('val dataset length: ', len(val_test['val']))
print('test dataset length: ', len(val_test['train']))
print('latent space size:', conf['model']['latent_size'])
print('batch size:', conf['train']['batch_size'])
print('margin:', conf['train']['margin'])
loss_list = []
model = AutoEncoder(in_channels=1,
dec_channels=1,
latent_size=conf['model']['latent_size'])
model = model.to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=conf['train']['lr'])
model.train()
if load_path:
model.load_state_dict(torch.load(load_path))
for epoch in range(10):
for param in optimizer.param_groups:
param['lr'] = max(0.00001, param['lr'] / conf['train']['lr_decay'])
print('lr: ', param['lr'])
loss_list = []
for batch_i, batch in enumerate(data_loader_train):
# if batch_i == 1000:
# break
batch = batch['image']
batch = batch.type(torch.FloatTensor)
batch = batch.to(device)
loss = triplet_step(model, batch, transform1, transform2)
loss_list.append(loss.item())
optimizer.zero_grad()
loss.backward()
optimizer.step()
recall, recall10 = recall_validation(model, data_loader_val,
transform1, transform2, device)
if epoch == 0:
min_validation_recall = recall
else:
min_validation_recall = min(min_validation_recall, recall)
if min_validation_recall == recall and save_path:
torch.save(model.state_dict(), save_path)
print('epoch {0}, loss {1:2.4f}'.format(
epoch,
sum(loss_list) / len(loss_list)))
print('recall@3: {0:2.4f}, recall 10%: {1:2.4f}'.format(
recall, recall10))
model.load_state_dict(torch.load(save_path))
recall, recall10 = recall_validation(model, data_loader_test, transform1,
transform2)
print('test recall@3: {0:2.4f}, recall@3 10%: {1:2.4f}'.format(
recall, recall10))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--filename', required=True, help='Name/path of file')
parser.add_argument('--save_dir', default='./outputs', help='Path to dictionary where will be save results.')
parser.add_argument('--pretrain_epochs', type=int, default=100, help="Number of epochs to pretrain model AE")
parser.add_argument('--epochs', type=int, default=100, help="Number of epochs to train AE and classifier")
parser.add_argument('--dims_layers_ae', type=int, nargs='+', default=[500, 100, 10],
help="Dimensional of layers in AE")
parser.add_argument('--dims_layers_classifier', type=int, nargs='+', default=[10, 5],
help="Dimensional of layers in classifier")
parser.add_argument('--batch_size', type=int, default=50)
parser.add_argument('--lr', type=float, default=0.001, help="Learning rate")
parser.add_argument('--use_dropout', action='store_true', help="Use dropout")
parser.add_argument('--no-cuda', action='store_true', help='disables CUDA training')
parser.add_argument('--seed', type=int, default=1234, help='random seed (default: 1)')
parser.add_argument('--procedure', nargs='+', choices=['pre-training_ae', 'training_classifier', 'training_all'],
help='Procedure which you can use. Choice from: pre-training_ae, training_all, '
'training_classifier')
parser.add_argument('--criterion_classifier', default='BCELoss', choices=['BCELoss', 'HingeLoss'],
help='Kind of loss function')
parser.add_argument('--scale_loss', type=float, default=1., help='Weight for loss of classifier')
parser.add_argument('--earlyStopping', type=int, default=None, help='Number of epochs to early stopping')
parser.add_argument('--use_scheduler', action='store_true')
args = parser.parse_args()
print(args)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
use_cuda = not args.no_cuda and torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")
loaded = np.load(args.filename)
x_train = loaded[f'data_train']
x_test = loaded[f'data_test']
y_train = loaded[f'lab_train']
y_test = loaded[f'lab_test']
del loaded
name_target = PurePosixPath(args.filename).parent.stem
n_split = PurePosixPath(args.filename).stem
save_dir = f'{args.save_dir}/tensorboard/{name_target}_{n_split}'
Path(save_dir).mkdir(parents=True, exist_ok=True)
if args.dims_layers_classifier[0] == -1:
args.dims_layers_classifier[0] = x_test.shape[1]
model_classifier = Classifier(args.dims_layers_classifier, args.use_dropout).to(device)
if args.criterion_classifier == 'HingeLoss':
criterion_classifier = nn.HingeEmbeddingLoss()
print('Use "Hinge" loss.')
else:
criterion_classifier = nn.BCEWithLogitsLoss()
model_ae = None
criterion_ae = None
if 'training_classifier' != args.procedure[0]:
args.dims_layers_ae = [x_train.shape[1]] + args.dims_layers_ae
assert args.dims_layers_ae[-1] == args.dims_layers_classifier[0], \
'Dimension of latent space must be equal with dimension of input classifier!'
model_ae = AutoEncoder(args.dims_layers_ae, args.use_dropout).to(device)
criterion_ae = nn.MSELoss()
optimizer = torch.optim.Adam(list(model_ae.parameters()) + list(model_classifier.parameters()), lr=args.lr)
else:
optimizer = torch.optim.Adam(model_classifier.parameters(), lr=args.lr)
scheduler = None
if args.use_scheduler:
scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda ep: 0.95)
writer = SummaryWriter(save_dir)
total_scores = {'roc_auc': 0, 'acc': 0, 'mcc': 0, 'bal': 0, 'recall': 0,
'max_roc_auc': 0, 'max_acc': 0, 'max_mcc': 0, 'max_bal': 0, 'max_recall': 0,
'pre-fit_time': 0, 'pre-score_time': 0, 'fit_time': 0, 'score_time': 0
}
dir_model_ae = f'{args.save_dir}/models_AE'
Path(dir_model_ae).mkdir(parents=True, exist_ok=True)
# dir_model_classifier = f'{args.save_dir}/models_classifier'
# Path(dir_model_classifier).mkdir(parents=True, exist_ok=True)
path_ae = f'{dir_model_ae}/{name_target}_{n_split}.pth'
# path_classifier = f'{dir_model_classifier}/{name_target}_{n_split}.pth'
if 'pre-training_ae' in args.procedure:
min_val_loss = np.Inf
epochs_no_improve = 0
epoch_tqdm = tqdm(range(args.pretrain_epochs), desc="Epoch pre-train loss")
for epoch in epoch_tqdm:
loss_train, time_trn = train_step(model_ae, None, criterion_ae, None, optimizer, scheduler, x_train,
y_train, device, writer, epoch, args.batch_size, 'pre-training_ae')
loss_test, _ = test_step(model_ae, None, criterion_ae, None, x_test, y_test, device, writer, epoch,
args.batch_size, 'pre-training_ae')
if not np.isfinite(loss_train):
break
total_scores['pre-fit_time'] += time_trn
if loss_test < min_val_loss:
torch.save(model_ae.state_dict(), path_ae)
epochs_no_improve = 0
min_val_loss = loss_test
else:
epochs_no_improve += 1
epoch_tqdm.set_description(
f"Epoch pre-train loss: {loss_train:.5f}, test loss: {loss_test:.5f} (minimal val-loss: {min_val_loss:.5f}, stop: {epochs_no_improve}|{args.earlyStopping})")
if args.earlyStopping is not None and epoch >= args.earlyStopping and epochs_no_improve == args.earlyStopping:
print('\033[1;31mEarly stopping in pre-training model\033[0m')
break
print(f"\033[1;5;33mLoad model AE form '{path_ae}'\033[0m")
if device.type == "cpu":
model_ae.load_state_dict(torch.load(path_ae, map_location=lambda storage, loc: storage))
else:
model_ae.load_state_dict(torch.load(path_ae))
model_ae = model_ae.to(device)
model_ae.eval()
min_val_loss = np.Inf
epochs_no_improve = 0
epoch = None
stage = 'training_classifier' if 'training_classifier' in args.procedure else 'training_all'
epoch_tqdm = tqdm(range(args.epochs), desc="Epoch train loss")
for epoch in epoch_tqdm:
loss_train, time_trn = train_step(model_ae, model_classifier, criterion_ae, criterion_classifier, optimizer,
scheduler, x_train, y_train, device, writer, epoch, args.batch_size,
stage, args.scale_loss)
loss_test, scores_val, time_tst = test_step(model_ae, model_classifier, criterion_ae, criterion_classifier,
x_test, y_test, device, writer, epoch, args.batch_size, stage,
args.scale_loss)
if not np.isfinite(loss_train):
break
total_scores['fit_time'] += time_trn
total_scores['score_time'] += time_tst
if total_scores['max_roc_auc'] < scores_val['roc_auc']:
for key, val in scores_val.items():
total_scores[f'max_{key}'] = val
if loss_test < min_val_loss:
# torch.save(model_ae.state_dict(), path_ae)
# torch.save(model_classifier.state_dict(), path_classifier)
epochs_no_improve = 0
min_val_loss = loss_test
for key, val in scores_val.items():
total_scores[key] = val
else:
epochs_no_improve += 1
epoch_tqdm.set_description(
f"Epoch train loss: {loss_train:.5f}, test loss: {loss_test:.5f} (minimal val-loss: {min_val_loss:.5f}, stop: {epochs_no_improve}|{args.earlyStopping})")
if args.earlyStopping is not None and epoch >= args.earlyStopping and epochs_no_improve == args.earlyStopping:
print('\033[1;31mEarly stopping!\033[0m')
break
total_scores['score_time'] /= epoch + 1
writer.close()
save_file = f'{args.save_dir}/{name_target}.txt'
head = 'idx;params'
temp = f'{n_split};pretrain_epochs:{args.pretrain_epochs},dims_layers_ae:{args.dims_layers_ae},' \
f'dims_layers_classifier:{args.dims_layers_classifier},batch_size:{args.batch_size},lr:{args.lr}' \
f'use_dropout:{args.use_dropout},procedure:{args.procedure},scale_loss:{args.scale_loss},' \
f'earlyStopping:{args.earlyStopping}'
for key, val in total_scores.items():
head = head + f';{key}'
temp = temp + f';{val}'
not_exists = not Path(save_file).exists()
with open(save_file, 'a') as f:
if not_exists:
f.write(f'{head}\n')
f.write(f'{temp}\n')
