def __init__(self,
model,
epsilon,
alpha,
min_val,
max_val,
max_iters,
_type='linf',
loss_type='sim',
regularize='original'):
# Model
self.model = model
#self.projector = projector
self.regularize = regularize
# Maximum perturbation
self.epsilon = epsilon
# Movement multiplier per iteration
self.alpha = alpha
# Minimum value of the pixels
self.min_val = min_val
# Maximum value of the pixels
self.max_val = max_val
# Maximum numbers of iteration to generated adversaries
self.max_iters = max_iters
# The perturbation of epsilon
self._type = _type
# loss type
#self.loss_type = loss_type
self.loss_n = TransformedGridLoss(use_cuda=True,
geometric_model=args.geometric_model)
for name, param in model.FeatureExtraction.state_dict().items():
model.FeatureExtraction.state_dict()[name].copy_(
checkpoint['state_dict']['FeatureExtraction.' + name])
for name, param in model.FeatureRegression.state_dict().items():
model.FeatureRegression.state_dict()[name].copy_(
checkpoint['state_dict']['FeatureRegression.' + name])
for name, param in model.GridDeformation.state_dict().items():
model.GridDeformation.state_dict()[name].copy_(
checkpoint['state_dict']['GridDeformation.' + name])
if args.use_mse_loss:
print('Using MSE loss...')
loss = nn.MSELoss()
else:
print('Using grid loss...')
loss = TransformedGridLoss(use_cuda=use_cuda,
geometric_model=args.geometric_model)
# Dataset and dataloader
dataset = SynthDataset(geometric_model=args.geometric_model,
transform=NormalizeImageDict(['image']),
dataset_csv_file='train.csv',
**arg_groups['dataset'])
dataloader = DataLoader(
dataset, batch_size=args.batch_size, shuffle=True,
num_workers=4) # don't change num_workers, as they copy the rnd seed
dataset_test = SynthDataset(geometric_model=args.geometric_model,
transform=NormalizeImageDict(['image']),
dataset_csv_file='test.csv',
**arg_groups['dataset'])
def main():
args, arg_groups = ArgumentParser(mode='train').parse()
print(args)
use_cuda = torch.cuda.is_available()
device = torch.device('cuda') if use_cuda else torch.device('cpu')
# Seed
torch.manual_seed(args.seed)
if use_cuda:
torch.cuda.manual_seed(args.seed)
# Download dataset if needed and set paths
if args.training_dataset == 'pascal':
if args.dataset_image_path == '' and not os.path.exists(
'datasets/pascal-voc11/TrainVal'):
download_pascal('datasets/pascal-voc11/')
if args.dataset_image_path == '':
args.dataset_image_path = 'datasets/pascal-voc11/'
args.dataset_csv_path = 'training_data/pascal-random'
# CNN model and loss
print('Creating CNN model...')
if args.geometric_model == 'affine':
cnn_output_dim = 6
elif args.geometric_model == 'hom' and args.four_point_hom:
cnn_output_dim = 8
elif args.geometric_model == 'hom' and not args.four_point_hom:
cnn_output_dim = 9
elif args.geometric_model == 'tps':
cnn_output_dim = 18
model = CNNGeometric(use_cuda=use_cuda,
output_dim=cnn_output_dim,
**arg_groups['model'])
if args.geometric_model == 'hom' and not args.four_point_hom:
init_theta = torch.tensor([1, 0, 0, 0, 1, 0, 0, 0, 1], device=device)
model.FeatureRegression.linear.bias.data += init_theta
if args.geometric_model == 'hom' and args.four_point_hom:
init_theta = torch.tensor([-1, -1, 1, 1, -1, 1, -1, 1], device=device)
model.FeatureRegression.linear.bias.data += init_theta
if args.use_mse_loss:
print('Using MSE loss...')
loss = nn.MSELoss()
else:
print('Using grid loss...')
loss = TransformedGridLoss(use_cuda=use_cuda,
geometric_model=args.geometric_model)
# Initialize Dataset objects
dataset = SynthDataset(geometric_model=args.geometric_model,
dataset_csv_path=args.dataset_csv_path,
dataset_csv_file='train.csv',
dataset_image_path=args.dataset_image_path,
transform=NormalizeImageDict(['image']),
random_sample=args.random_sample)
dataset_val = SynthDataset(geometric_model=args.geometric_model,
dataset_csv_path=args.dataset_csv_path,
dataset_csv_file='val.csv',
dataset_image_path=args.dataset_image_path,
transform=NormalizeImageDict(['image']),
random_sample=args.random_sample)
# Set Tnf pair generation func
pair_generation_tnf = SynthPairTnf(geometric_model=args.geometric_model,
use_cuda=use_cuda)
# Initialize DataLoaders
dataloader = DataLoader(dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=4)
dataloader_val = DataLoader(dataset_val,
batch_size=args.batch_size,
shuffle=True,
num_workers=4)
# Optimizer and eventual scheduler
optimizer = optim.Adam(model.FeatureRegression.parameters(), lr=args.lr)
if args.lr_scheduler:
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, T_max=args.lr_max_iter, eta_min=1e-6)
# scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')
else:
scheduler = False
# Train
# Set up names for checkpoints
if args.use_mse_loss:
ckpt = args.trained_model_fn + '_' + args.geometric_model + '_mse_loss' + args.feature_extraction_cnn
checkpoint_path = os.path.join(args.trained_model_dir,
args.trained_model_fn,
ckpt + '.pth.tar')
else:
ckpt = args.trained_model_fn + '_' + args.geometric_model + '_grid_loss' + args.feature_extraction_cnn
checkpoint_path = os.path.join(args.trained_model_dir,
args.trained_model_fn,
ckpt + '.pth.tar')
if not os.path.exists(args.trained_model_dir):
os.mkdir(args.trained_model_dir)
# Set up TensorBoard writer
if not args.log_dir:
tb_dir = os.path.join(args.trained_model_dir,
args.trained_model_fn + '_tb_logs')
else:
tb_dir = os.path.join(args.log_dir, args.trained_model_fn + '_tb_logs')
logs_writer = SummaryWriter(tb_dir)
# add graph, to do so we have to generate a dummy input to pass along with the graph
dummy_input = {
'source_image': torch.rand([args.batch_size, 3, 240, 240],
device=device),
'target_image': torch.rand([args.batch_size, 3, 240, 240],
device=device),
'theta_GT': torch.rand([16, 2, 3], device=device)
}
logs_writer.add_graph(model, dummy_input)
# Start of training
print('Starting training...')
best_val_loss = float("inf")
for epoch in range(1, args.num_epochs + 1):
# we don't need the average epoch loss so we assign it to _
_ = train(epoch,
model,
loss,
optimizer,
dataloader,
pair_generation_tnf,
log_interval=args.log_interval,
scheduler=scheduler,
tb_writer=logs_writer)
val_loss = validate_model(model, loss, dataloader_val,
pair_generation_tnf, epoch, logs_writer)
# remember best loss
is_best = val_loss < best_val_loss
best_val_loss = min(val_loss, best_val_loss)
save_checkpoint(
{
'epoch': epoch + 1,
'args': args,
'state_dict': model.state_dict(),
'best_val_loss': best_val_loss,
'optimizer': optimizer.state_dict(),
}, is_best, checkpoint_path)
logs_writer.close()
print('Done!')
args.training_tnf_csv = 'training_data/pascal-synth-aff'
elif args.training_tnf_csv == '' and args.geometric_model == 'tps':
args.training_tnf_csv = 'training_data/pascal-synth-tps'
# CNN model and loss
print('Creating CNN model...')
model = CNNGeometric(feature_extraction_cnn=args.feature_extraction_cnn,
training=False)
if args.use_mse_loss:
print('Using MSE loss...')
loss = tf.losses
else:
print('Using grid loss...')
loss = TransformedGridLoss(geometric_model=args.geometric_model)
# Dataset and dataloader
dataset = SynthDataset(geometric_model=args.geometric_model,
csv_file=os.path.join(args.training_tnf_csv,
'train.csv'),
training_image_path=args.training_image_path,
transform=NormalizeImageDict(['image']),
random_sample=args.random_sample)
dataset_test = SynthDataset(geometric_model=args.geometric_model,
csv_file=os.path.join(args.training_tnf_csv,
'test.csv'),
training_image_path=args.training_image_path,
transform=NormalizeImageDict(['image']),
random_sample=args.random_sample)
def main():
args = parse_flags()
use_cuda = torch.cuda.is_available()
# Seed
torch.manual_seed(args.seed)
if use_cuda:
torch.cuda.manual_seed(args.seed)
# Download dataset if needed and set paths
if args.training_dataset == 'pascal':
if args.training_image_path == '':
download_pascal('datasets/pascal-voc11/')
args.training_image_path = 'datasets/pascal-voc11/'
if args.training_tnf_csv == '' and args.geometric_model == 'affine':
args.training_tnf_csv = 'training_data/pascal-synth-aff'
elif args.training_tnf_csv == '' and args.geometric_model == 'tps':
args.training_tnf_csv = 'training_data/pascal-synth-tps'
# CNN model and loss
if not args.pretrained:
if args.light_model:
print('Creating light CNN model...')
model = LightCNN(use_cuda=use_cuda,
geometric_model=args.geometric_model)
else:
print('Creating CNN model...')
model = CNNGeometric(
use_cuda=use_cuda,
geometric_model=args.geometric_model,
feature_extraction_cnn=args.feature_extraction_cnn)
else:
model = load_torch_model(args, use_cuda)
if args.loss == 'mse':
print('Using MSE loss...')
loss = MSELoss()
elif args.loss == 'sum':
print('Using the sum of MSE and grid loss...')
loss = GridLossWithMSE(use_cuda=use_cuda,
geometric_model=args.geometric_model)
else:
print('Using grid loss...')
loss = TransformedGridLoss(use_cuda=use_cuda,
geometric_model=args.geometric_model)
# Initialize csv paths
train_csv_path_list = glob(
os.path.join(args.training_tnf_csv, '*train.csv'))
if len(train_csv_path_list) > 1:
print(
"!!!!WARNING!!!! multiple train csv files found, using first in glob order"
)
elif not len(train_csv_path_list):
raise FileNotFoundError(
"No training csv where found in the specified path!!!")
train_csv_path = train_csv_path_list[0]
val_csv_path_list = glob(os.path.join(args.training_tnf_csv, '*val.csv'))
if len(val_csv_path_list) > 1:
print(
"!!!!WARNING!!!! multiple train csv files found, using first in glob order"
)
elif not len(val_csv_path_list):
raise FileNotFoundError(
"No training csv where found in the specified path!!!")
val_csv_path = val_csv_path_list[0]
# Initialize Dataset objects
if args.coupled_dataset:
# Dataset for train and val if dataset is already coupled
dataset = CoupledDataset(geometric_model=args.geometric_model,
csv_file=train_csv_path,
training_image_path=args.training_image_path,
transform=NormalizeImageDict(
['image_a', 'image_b']))
dataset_val = CoupledDataset(
geometric_model=args.geometric_model,
csv_file=val_csv_path,
training_image_path=args.training_image_path,
transform=NormalizeImageDict(['image_a', 'image_b']))
# Set Tnf pair generation func
pair_generation_tnf = CoupledPairTnf(use_cuda=use_cuda)
else:
# Standard Dataset for train and val
dataset = SynthDataset(geometric_model=args.geometric_model,
csv_file=train_csv_path,
training_image_path=args.training_image_path,
transform=NormalizeImageDict(['image']),
random_sample=args.random_sample)
dataset_val = SynthDataset(
geometric_model=args.geometric_model,
csv_file=val_csv_path,
training_image_path=args.training_image_path,
transform=NormalizeImageDict(['image']),
random_sample=args.random_sample)
# Set Tnf pair generation func
pair_generation_tnf = SynthPairTnf(
geometric_model=args.geometric_model, use_cuda=use_cuda)
# Initialize DataLoaders
dataloader = DataLoader(dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=4)
dataloader_val = DataLoader(dataset_val,
batch_size=args.batch_size,
shuffle=True,
num_workers=4)
# Optimizer and eventual scheduler
optimizer = Adam(model.FeatureRegression.parameters(), lr=args.lr)
if args.lr_scheduler:
if args.scheduler_type == 'cosine':
print('Using cosine learning rate scheduler')
scheduler = CosineAnnealingLR(optimizer,
T_max=args.lr_max_iter,
eta_min=args.lr_min)
elif args.scheduler_type == 'decay':
print('Using decay learning rate scheduler')
scheduler = ReduceLROnPlateau(optimizer, 'min')
else:
print(
'Using truncated cosine with decay learning rate scheduler...')
scheduler = TruncateCosineScheduler(optimizer, len(dataloader),
args.num_epochs - 1)
else:
scheduler = False
# Train
# Set up names for checkpoints
if args.loss == 'mse':
ckpt = args.trained_models_fn + '_' + args.geometric_model + '_mse_loss' + args.feature_extraction_cnn
checkpoint_path = os.path.join(args.trained_models_dir,
args.trained_models_fn,
ckpt + '.pth.tar')
elif args.loss == 'sum':
ckpt = args.trained_models_fn + '_' + args.geometric_model + '_sum_loss' + args.feature_extraction_cnn
checkpoint_path = os.path.join(args.trained_models_dir,
args.trained_models_fn,
ckpt + '.pth.tar')
else:
ckpt = args.trained_models_fn + '_' + args.geometric_model + '_grid_loss' + args.feature_extraction_cnn
checkpoint_path = os.path.join(args.trained_models_dir,
args.trained_models_fn,
ckpt + '.pth.tar')
if not os.path.exists(args.trained_models_dir):
os.mkdir(args.trained_models_dir)
# Set up TensorBoard writer
if not args.log_dir:
tb_dir = os.path.join(args.trained_models_dir,
args.trained_models_fn + '_tb_logs')
else:
tb_dir = os.path.join(args.log_dir,
args.trained_models_fn + '_tb_logs')
logs_writer = SummaryWriter(tb_dir)
# add graph, to do so we have to generate a dummy input to pass along with the graph
dummy_input = {
'source_image': torch.rand([args.batch_size, 3, 240, 240]),
'target_image': torch.rand([args.batch_size, 3, 240, 240]),
'theta_GT': torch.rand([16, 2, 3])
}
logs_writer.add_graph(model, dummy_input)
# START OF TRAINING #
print('Starting training...')
best_val_loss = float("inf")
for epoch in range(1, args.num_epochs + 1):
# we don't need the average epoch loss so we assign it to _
_ = train(epoch,
model,
loss,
optimizer,
dataloader,
pair_generation_tnf,
log_interval=args.log_interval,
scheduler=scheduler,
tb_writer=logs_writer)
val_loss = validate_model(model,
loss,
dataloader_val,
pair_generation_tnf,
epoch,
logs_writer,
coupled=args.coupled_dataset)
# remember best loss
is_best = val_loss < best_val_loss
best_val_loss = min(val_loss, best_val_loss)
save_checkpoint(
{
'epoch': epoch + 1,
'args': args,
'state_dict': model.state_dict(),
'best_val_loss': best_val_loss,
'optimizer': optimizer.state_dict(),
}, is_best, checkpoint_path)
logs_writer.close()
print('Done!')