def __init__(self, opt):
self.condition = opt.C_condition
self.n_C = opt.n_C
self.n_critics = opt.n_critics
self.progression = opt.progression
self.USE_CUDA = True if opt.gpu_ids != -1 else False
if opt.CT:
self.CT = True
self.CT_lambda = opt.CT_lambda
self.CT_factor = opt.CT_factor
else:
self.CT = False
if opt.FM:
self.FM = True
self.FM_criterion = self.get_criterion(opt.FM_criterion)
self.FM_lambda = opt.FM_lambda
else:
self.FM = False
if opt.GP_mode == 'Banach':
self.drift_lambda = opt.drift_lambda
self.drift_loss = opt.drift_loss
self.exponent = opt.exponent
self.dual_exponent = 1 / (1 - 1 / self.exponent) if self.exponent != 1 else np.inf
self.sobolev_s = opt.sobolev_s
self.sobolev_c = opt.sobolev_c
elif opt.GP_mode == 'div':
self.k = opt.k
self.p = opt.p
elif opt.GP_mode == 'GP':
self.GP_lambda = opt.GP_lambda
else:
raise NotImplementedError
if opt.VGG:
from models import VGG19
self.VGG = True
self.VGGNet = VGG19().cuda(0) if opt.gpu_ids != -1 else VGG19()
self.VGG_lambda = opt.VGG_lambda
self.VGG_weights = [1.0 / 32, 1.0 / 16, 1.0 / 8, 1.0 / 4, 1.0]
else:
self.VGG = False
def train(**kwargs):
opt.parse(kwargs)
# configure model
model = VGG19()
if opt.cuda:
model.cuda()
# load data
train_data = ImageProcessing(data_root=opt.data_root)
data_loader = DataLoader(train_data,
batch_size=opt.batch_size,
shuffle=False,
num_workers=opt.num_workers)
print(train_data.categories)
for category in train_data.categories:
features, img_pth = [], []
train_data.load_images_path(category)
for data, pth in data_loader:
if opt.cuda:
data = data.cuda()
img_pth.extend(pth)
features.append(model.forward(data))
features = torch.cat(features)
ddt = DDT(features)
ddt.pca()
ddt.project_to_indictor_matrices()
ddt.visualize(category, img_pth)
def __init__(self, opt):
self.condition = opt.C_condition
self.device = torch.device('cuda' if opt.USE_CUDA else 'cpu', 0)
self.n_C = opt.n_C
self.n_critics = opt.n_critics
self.progression = opt.progression
self.USE_CUDA = opt.USE_CUDA
if opt.CT:
self.CT = True
self.CT_lambda = opt.CT_lambda
self.CT_factor = opt.CT_factor
else:
self.CT = False
if opt.FM:
self.FM = True
self.FM_criterion = self.get_criterion(opt.FM_criterion)
self.FM_lambda = opt.FM_lambda
else:
self.FM = False
if opt.GAN_type == 'BWGAN':
self.drift_lambda = opt.drift_lambda
self.drift_loss = opt.drift_loss
self.exponent = opt.exponent
self.dual_exponent = 1 / (
1 - 1 / self.exponent) if self.exponent != 1 else np.inf
self.sobolev_c = opt.sobolev_c
self.sobolev_s = opt.sobolev_s
elif opt.GAN_type == 'WGANDiv':
self.k = opt.k
self.p = opt.p
elif opt.GAN_type == 'WGANGP':
self.GP_lambda = opt.GP_lambda
else:
raise NotImplementedError
if opt.VGG:
from models import VGG19
self.VGG = True
self.VGGNet = VGG19().to(self.device)
self.VGG_lambda = opt.VGG_lambda
self.VGG_weights = [1.0 / 32, 1.0 / 16, 1.0 / 8, 1.0 / 4, 1.0]
else:
self.VGG = False
def get_backbone(args, pretrained=True):
from models import VGG16, VGG19, ResNet50, SEResNet50
if args.backbone == 'resnet':
output_shape = 2048
backbone = ResNet50(pretrained=pretrained, kp=args.kp)
elif args.backbone == 'vgg16':
output_shape = 4096
backbone = VGG16(pretrained=pretrained)
elif args.backbone == 'vgg19':
output_shape = 4096
backbone = VGG19(pretrained=pretrained)
elif args.backbone == 'seresnet':
output_shape = 2048
backbone = SEResNet50(pretrained=pretrained)
return output_shape, backbone
def get_models_genome(model_path):
'''
Load pre-trained model
:return: pretrained image and caption model
'''
image_model = VGG19(pretrained=True)
caption_model = LSTMBranch()
if not os.path.exists(model_path):
print("Not using trained models")
return image_model, caption_model
checkpoint = torch.load(model_path, map_location='cpu')
image_model.load_state_dict(checkpoint['image_model'])
caption_model.load_state_dict(checkpoint['caption_model'])
print('Loaded pretrained models')
return image_model, caption_model
logging_info("Save interval: {}".format(arguments.save_interval
or "At the end"))
logging_info("Plot interval: {}".format(arguments.plot_interval))
# register exit handler
atexit.register(exit_handler, run)
should_continue = query_yes_no("Continue?", default="yes")
if not should_continue:
exit()
# determine compute device
device = torch.device(arguments.device)
# initialize model
model = VGG19(input_channels=3)
vgg_19_path = get_full_path("models", "vgg_19_imagenet",
"vgg_19.minimum.pkl")
if not os.path.isfile(vgg_19_path):
logging_error("Please download the weights and biases of the VGG 19 network from " \
"http://download.tensorflow.org/models/vgg_19_2016_08_28.tar.gz, extract the archive and run the Python script "\
"`extract_vgg_19_weights.py`.")
model.initialize(vgg_19_path)
model.to(device)
# setup extraction layer lists
extract_layers = arguments.style_layers
extract_layers.extend(arguments.content_layers)
# load input data
input_style = image.load(arguments.input_style_file, device=device)
parser.add_argument('--batch-size',
type=int,
default=32,
help='batch size for training (default: 32')
parser.add_argument('--epochs',
type=int,
default=40,
help='number of epochs to train (default: 40)')
parser.add_argument(
'--patience',
type=int,
default=2,
help='number of epochs to wait before reducing lr (default: 2)')
args = parser.parse_args()
MODEL = VGG19(num_classes=17).cuda()
BASE_OPTIMIZER = optim.Adam
DIFF_LR_FACTORS = [9, 3, 1]
# training loop
def train(model, epochs, train_dl, val_dl, fold):
best_score = 0.0
lr0 = args.init_lr_0
iterations = epochs * len(train_dl)
idx = 0
# create optimizer with differential learning rates
optimizer = create_optimizer(MODEL, BASE_OPTIMIZER, args.init_lr_0,
DIFF_LR_FACTORS)
# set up lr schedule based on val loss
lr_scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,
def train():
data_transform = transforms.Compose([
transforms.Resize((224, 224)),
transforms.RandomHorizontalFlip(),
transforms.RandomVerticalFlip(),
transforms.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.3),
transforms.RandomRotation(30),
transforms.RandomAffine(degrees=0, translate=(0.2, 0.2)),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
train_set = IMAGE_Dataset(Path(DATASET_ROOT_train), data_transform)
data_loader = DataLoader(dataset=train_set,
batch_size=32,
shuffle=True,
num_workers=1)
model = VGG19(num_classes=train_set.num_classes)
model = model.cuda(CUDA_DEVICES)
model.train()
best_model_params = copy.deepcopy(model.state_dict())
best_acc = 0.0
num_epochs = 400
criterion = nn.CrossEntropyLoss()
stepsize = 20
base_lr = 0.01
max_lr = 0.1
base_mm = 0.8
max_mm = 0.99
for epoch in range(num_epochs):
newlr = get_triangular_lr(epoch, stepsize, base_lr, max_lr)
mm = get_dynamic_momentum(epoch, stepsize, base_mm, max_mm)
optimizer = torch.optim.SGD(params=model.parameters(),
lr=newlr,
momentum=mm)
print(f'Epoch: {epoch + 1}/{num_epochs}')
print('-' * len(f'Epoch: {epoch + 1}/{num_epochs}'))
training_loss = 0.0
training_corrects = 0
for i, (inputs, labels) in enumerate(data_loader):
inputs = Variable(inputs.cuda(CUDA_DEVICES))
labels = Variable(labels.cuda(CUDA_DEVICES))
optimizer.zero_grad()
outputs = model(inputs)
_, preds = torch.max(outputs.data, 1)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
training_loss += loss.item() * inputs.size(0)
#revise loss.data[0]-->loss.item()
training_corrects += torch.sum(preds == labels.data)
#print(f'training_corrects: {training_corrects}')
training_loss = training_loss / len(train_set)
training_acc = training_corrects.double() / len(train_set)
print(
f'Training loss: {training_loss:.4f}\taccuracy: {training_acc:.4f}\n'
)
test_acc = test(model)
if test_acc > best_acc:
best_acc = test_acc
best_model_params = copy.deepcopy(model.state_dict())
model.load_state_dict(best_model_params)
torch.save(model, f'model-{best_acc:.02f}-best_test_acc.pth')
checkpoint_path = 'checkpoints_vgg/'
IMAGE_SIZE = 32
OUTPUT_FILE_NAME = 'train_output_vgg.txt'
decay_steps = int(len(target_train_data) / batch_size)
learning_rate_decay_factor = 0.95
if not os.path.isdir(filewriter_path): os.mkdir(filewriter_path)
if not os.path.isdir(checkpoint_path): os.mkdir(checkpoint_path)
x = tf.placeholder(tf.float32, [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3])
y = tf.placeholder(tf.float32, [None, num_classes])
keep_prob = tf.placeholder(tf.float32)
model = VGG19(x, keep_prob, num_classes)
score = model.fc8
var_list = [v for v in tf.trainable_variables()]
print(var_list)
initial_x_batch = tf.placeholder(tf.float32,
[batch_size, IMAGE_SIZE, IMAGE_SIZE, 3])
dist_x_batch = distorted_batch(initial_x_batch, IMAGE_SIZE)
with tf.name_scope("cross_ent"):
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=score, labels=y))
with tf.name_scope('train'):
gradients = tf.gradients(loss, var_list)
def main(args):
# Parsing command line arguments
print("========================================================")
print("Process %s, running on %s: starting (%s)" % (
os.getpid(), os.name, time.asctime()))
print("========================================================")
if args.cnn_model == 'vgg':
image_model = VGG19(pretrained=True)
else:
image_model = ResNet50(pretrained = True)
caption_model = LSTMBranch()
if torch.cuda.is_available() and args.use_gpu == True:
image_model = image_model.cuda()
caption_model = caption_model.cuda()
# Get the learnable parameters
image_trainables = [p for p in image_model.parameters() if p.requires_grad]
caption_trainables = [p for p in caption_model.parameters() if p.requires_grad]
params = image_trainables + caption_trainables
transform = transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406),
(0.229, 0.224, 0.225))])
# Obtain the data loader (from file). Note that it runs much faster than before!
print("Dataset being used: ", args.dataset)
if args.dataset == 'flickr':
data_loader_train = get_loader_flickr(transform=transform,
mode='train',
batch_size=args.batch_size,
parse_mode=args.parse_mode)
data_loader_val = get_loader_flickr(transform=transform,
mode='val',
batch_size=args.batch_size,
parse_mode=args.parse_mode)
elif args.dataset == 'coco':
data_loader_train = get_loader_coco(transform=transform,
mode='train',
batch_size=args.batch_size)
data_loader_val = get_loader_coco(transform=transform,
mode='val',
batch_size=args.batch_size)
else:
data_loader_train = get_loader_genome(transform=transform,
mode='train',
batch_size=args.batch_size)
data_loader_val = get_loader_genome(transform=transform,
mode='train',
batch_size=args.batch_size)
# Load saved model
start_epoch, best_loss = load_checkpoint(image_model, caption_model, args.resume)
# optimizer = torch.optim.Adam(params=params, lr=0.01)
optimizer = torch.optim.SGD(params=params, lr=args.lr, momentum=0.9)
total_train_step = math.ceil(
len(data_loader_train.dataset.caption_lengths) / data_loader_train.batch_sampler.batch_size)
# print("Total number of training steps are :", total_train_step)
print("========================================================")
print("Total number of epochs to train: ", args.n_epochs)
print("Loss Type: ", args.loss_type)
if args.loss_type == 'triplet':
print("Sampling strategy: ", args.sampler)
print("Margin for triplet loss: ", args.margin)
print("Learning Rate: ", args.lr)
print("Score Type for similarity: ", args.score_type)
print("========================================================")
epoch = start_epoch
best_epoch = start_epoch
# while (epoch - best_epoch) < args.no_gain_stop and (epoch <= args.n_epochs):
while epoch <= args.n_epochs:
adjust_learning_rate(args.lr, args.lr_decay, optimizer, epoch)
print("========================================================")
print("Epoch: %d Training starting" % epoch)
print("Learning rate : ", get_lr(optimizer))
train_loss = train(data_loader_train, data_loader_val, image_model,
caption_model, args.loss_type, optimizer, epoch,
args.score_type, args.sampler, args.margin,
total_train_step, args.batch_size, args.use_gpu)
print('---------------------------------------------------------')
print("Epoch: %d Validation starting" % epoch)
val_loss = validate(caption_model, image_model, data_loader_val,
epoch, args.loss_type, args.score_type, args.sampler,
args.margin, args.use_gpu)
print("Epoch: ", epoch)
print("Training Loss: ", float(train_loss.data))
print("Validation Loss: ", float(val_loss.data))
print("========================================================")
save_checkpoint({
'epoch': epoch,
'best_loss': min(best_loss, val_loss),
'image_model': image_model.state_dict(),
'caption_model': caption_model.state_dict()
}, val_loss < best_loss)
if (val_loss) < best_loss:
best_epoch = epoch
best_loss = val_loss
epoch += 1
print("Back to main")
resume_filename = 'runs/%s/' % (args.name) + 'model_best.pth.tar'
if os.path.exists(resume_filename):
epoch, best_loss1 = load_checkpoint(image_model, caption_model, args.resume)
val_loss1 = validate(caption_model, image_model, data_loader_val,
epoch, args.loss_type, args.score_type, args.sampler,
args.margin, args.use_gpu)
print("========================================================")
print("========================================================")
print("Final Loss : ", float(val_loss1.data))
print("========================================================")
print("========================================================")
else:
resume_filename = 'runs/%s/' % (args.name) + 'checkpoint.pth.tar'
print("Using last run epoch.")
epoch, best_loss1 = load_checkpoint(image_model, caption_model, args.resume)
val_loss1 = validate(caption_model, image_model, data_loader_val,
epoch, args.loss_type, args.score_type, args.sampler,
args.margin, args.use_gpu)
print("========================================================")
print("========================================================")
print("Final Loss : ", float(val_loss1.data))
print("========================================================")
print("========================================================")
Y = tf.keras.utils.to_categorical(Y,3)
print ('X shape: ', X.shape)
print ('Y shape: ', Y.shape)
n_split=3
losses = []
accuracies = []
for train_index,test_index in KFold(n_split).split(X):
x_train,x_test=X[train_index],X[test_index]
y_train,y_test=Y[train_index],Y[test_index]
model = VGG19.build(width=IMAGE_DIMS[1], height=IMAGE_DIMS[0],
depth=IMAGE_DIMS[2], classes=3,finalAct='softmax')
model.fit(x_train, y_train,epochs=10)
res = model.evaluate(x_test,y_test)
loss = res[0]
acc = res[1]
print('Model evaluation ',res)
losses.append(loss)
accuracies.append(acc)
print('Mean accuracy: ' , np.mean(accuracies))
print('Mean loss: ' , np.mean(loss))
parser.add_argument('--save-ext', type=str, default='jpg', help='The extension name of the output image')
parser.add_argument('--content-size', type=int, default=768, help='New (minimum) size for the content image')
parser.add_argument('--style-size', type=int, default=768, help='New (minimum) size for the style image')
parser.add_argument('--gpu', type=int, default=0, help='ID of the GPU to use; for CPU mode set --gpu = -1')
args = parser.parse_args()
assert args.content_path is not None or args.content_dir is not None, \
'Either --content-path or --content-dir should be given.'
assert args.style_path is not None or args.style_dir is not None, \
'Either --style-path or --style-dir should be given.'
device = torch.device('cuda:%s' % args.gpu if torch.cuda.is_available() and args.gpu != -1 else 'cpu')
encoder = VGG19().to(device)
decoder = Decoder(pretrained_path=args.decoder_path).to(device)
def style_transfer(content, style):
c_features = encoder(content)
s_features = encoder(style)
z_cs = style_decorator(c_features.relu4_1, s_features.relu4_1, args.kernel_size, args.stride, args.alpha)
output = decoder(z_cs, s_features)
return output
if not os.path.exists(args.save_dir):