def set_config(self, config):
"""
Set object config from config dictionary
:param config: config dictionary.
:return: None
"""
self.db = 'db' in config and config['db'] or 'cifar10'
self.net_type = 'net_type' in config and config['net_type'] or 'vgg19'
self.batch_size = 'batch_size' in config and int(
config['batch_size']) or 128
self.epochs = 'epochs' in config and config['epochs'] or 100
self.checkpoint_dir = 'checkpoint_dir' in config and config[
'checkpoint_dir'] or 'results'
self.loss = 'loss' in config and config['loss'] or 'crossentropy'
self.activation = 'activation' in config and config[
'activation'] or 'relu'
self.final_activation = 'final_activation' in config and config[
'final_activation'] or 'softmax'
self.f_a_params = config[
'f_a_params'] if 'f_a_params' in config else {}
self.use_tau = config[
'use_tau'] if 'use_tau' in config and config['use_tau'] else False
self.prob_layer = 'prob_layer' in config and config[
'prob_layer'] or None
self.spp_alpha = 'spp_alpha' in config and config['spp_alpha'] or 0
self.lr = 'lr' in config and config['lr'] or 0.1
self.momentum = 'momentum' in config and config['momentum'] or 0
self.dropout = 'dropout' in config and config['dropout'] or 0
self.task = 'task' in config and config['task'] or 'both'
self.workers = 'workers' in config and config['workers'] or 4
self.queue_size = 'queue_size' in config and config[
'queue_size'] or 1024
self.val_metrics = 'val_metrics' in config and config[
'val_metrics'] or ['acc', 'loss']
self.rescale_factor = 'rescale_factor' in config and config[
'rescale_factor'] or 0
self.augmentation = 'augmentation' in config and config[
'augmentation'] or {}
self._val_type = 'val_type' in config and config[
'val_type'] or 'holdout'
self._holdout = 'holdout' in config and float(config['holdout']) or 0.2
self._n_folds = 'n_folds' in config and int(config['n_folds']) or 5
if 'name' in config:
self.name = config['name']
else:
self.set_auto_name()
# Load dataset
self._ds = Dataset(self._db)
self._setup_validation()
# CUDA for PyTorch
use_cuda = torch.cuda.is_available()
print(use_cuda)
device = torch.device("cuda:0" if use_cuda else "cpu")
print(device)
# Parameters
max_epochs = 80
# Datasets
train_dir = 'data/train/'# IDs
train_label = 'data/train_kaggle.csv'
test_dir = 'data/test/'# Labels
test_label = 'data/sample_solution.csv'
# Generators
train_set = Dataset(train_dir, train_label,'train')
val_set = Dataset(train_dir, train_label,'eval')
# Creating data indices for training and validation splits:
dataset_size = len(train_set)
#print(dataset_size)
indices = list(range(dataset_size))
split = int(np.floor(0.2 * dataset_size))
np.random.shuffle(indices)
train_indices, val_indices = indices[split:], indices[:split]
train_sampler = SubsetRandomSampler(train_indices)
val_sampler = SubsetRandomSampler(val_indices)
train_params = {'batch_size': args.batch_size,
'num_workers': 2,
def main():
# Dataset path
# Visible Images and Color Images
trial = 1
train_color_list = '../idx/train_color_{}'.format(trial) + '.txt'
train_thermal_list = '../idx/train_thermal_{}'.format(trial) + '.txt'
checkpoint_path = 'save_model/'
# Learning params
learning_rate = 0.001
training_iters = 5000
batch_size = 64
display_step = 5
test_step = 500 # 0.5 epoch
margin = 0.5
lambda_intra = 0.1
feature_dim = 1024
test_batch_size = batch_size
# Network params
n_classes = 206
keep_rate = 0.5
suffix = 'BDTR'
# model params
suffix = suffix + '_drop_{}'.format(keep_rate)
suffix = suffix + '_lr_{:1.1e}'.format(learning_rate)
suffix = suffix + '_margin_{}'.format(margin)
suffix = suffix + '_batch_{}'.format(batch_size)
suffix = suffix + '_w_intra_{}'.format(lambda_intra)
suffix = suffix + '_trial_{}_'.format(trial)
# Graph input
x1 = tf.placeholder(tf.float32, [None, 227, 227, 3])
x2 = tf.placeholder(tf.float32, [None, 227, 227, 3])
y0 = tf.placeholder(tf.float32, [None, n_classes])
# y = tf.placeholder(tf.float32, [batch_size, 1])
y1 = tf.placeholder(tf.float32, [None, 1])
y2 = tf.placeholder(tf.float32, [None, 1])
configProt = tf.ConfigProto()
sess = tf.Session(config=configProt)
keep_var = tf.placeholder(tf.float32)
# Construct Model
feat1 = Model().alexnet_visible(x1, keep_var)
feat2 = Model().alexnet_thermal(x2, keep_var)
feat, pred0 = Model().modal_embedding(feat1, feat2, keep_var, feature_dim)
# norm_feat = tf.nn.l2_normalize(feat, 1, epsilon=1e-12)
feature1, feature2 = tf.split(0, 2, feat)
# pdb.set_trace()
rank_loss, prec = compute_birank_loss(feature1, feature2, batch_size,
margin, lambda_intra)
# Loss and optimizer
identity_loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(pred0, y0))
# #################################################################################
# Total Loss:
# Ranking loss performs much better on small dataset
# Identity loss is better for large scale dataset with abundant training samples
# We could adjust the combining weights to achieve different performance
# #################################################################################
total_loss = identity_loss + rank_loss
optimizer = tf.train.MomentumOptimizer(learning_rate,
momentum=0.9).minimize(total_loss)
# Evaluation
correct_pred0 = tf.equal(tf.argmax(pred0, 1), tf.argmax(y0, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred0, tf.float32))
# Init
init = tf.initialize_all_variables()
saver = tf.train.Saver()
# Load dataset
dataset = Dataset(train_color_list, train_thermal_list)
# load testing data (A short test for fast visualization)
test_thermal_list = '../idx/test_thermal_1.txt'
test_color_list = '../idx/test_color_1.txt'
test_color_imgs, test_color_labels = get_test_data(test_color_list)
test_thermal_imgs, test_thermal_labels = get_test_data(test_thermal_list)
if not os.path.isdir(checkpoint_path):
os.makedirs(checkpoint_path)
# Launch the graph
with tf.Session() as sess:
print 'Init variable'
sess.run(init)
print 'Start training'
step = 1
while step < training_iters:
batch_x1, batch_x2, bacth_y0, batch_y1, batch_y2 = dataset.next_batch(
batch_size, 'train')
sess.run(optimizer,
feed_dict={
x1: batch_x1,
x2: batch_x2,
y0: bacth_y0,
y1: batch_y1,
y2: batch_y2,
keep_var: keep_rate
})
# Display training status
if step % display_step == 0:
acc = sess.run(prec,
feed_dict={
x1: batch_x1,
x2: batch_x2,
y0: bacth_y0,
y1: batch_y1,
y2: batch_y2,
keep_var: 1.0
})
batch_loss = sess.run(total_loss,
feed_dict={
x1: batch_x1,
x2: batch_x2,
y0: bacth_y0,
y1: batch_y1,
y2: batch_y2,
keep_var: 1.0
})
print >> sys.stderr, "{} Iter {}: Training Loss = {:.4f}, Top-1 Accuracy = {:.4f}".format(
datetime.now(), step, batch_loss, acc)
# print >> sys.stderr, "diff: {} ".format(diff)
if step % test_step == 0:
# for iround in range
print('Test Step: {}'.format(step))
# get test color feature
num_color_sample = test_color_imgs.shape[0]
ptr1 = 0
test_color_feat = np.zeros((num_color_sample, feature_dim))
# logits = np.zeros((num_color_sample,num_category))
print('Extracting test color features...')
for num in range(int(num_color_sample / test_batch_size) + 1):
real_size = min(test_batch_size, num_color_sample - ptr1)
batch_test_color_imgs = test_color_imgs[ptr1:ptr1 +
real_size, :, :, :]
feature_color = sess.run(feat1,
feed_dict={
x1: batch_test_color_imgs,
keep_var: 1.
})
feature = sess.run(feat,
feed_dict={
feat1: feature_color,
feat2: feature_color,
keep_var: 1.
})
batch_color_feat, _ = tf.split(0, 2, feature)
test_color_feat[ptr1:ptr1 +
real_size, :] = batch_color_feat.eval()
ptr1 += real_size
# get test thermal feature
num_thermal_sample = test_thermal_imgs.shape[0]
ptr2 = 0
test_thermal_feat = np.zeros((num_thermal_sample, feature_dim))
# logits = np.zeros((num_color_sample,num_category))
print('Extracting test thermal features...')
for num in range(
int(num_thermal_sample / test_batch_size) + 1):
real_size = min(test_batch_size, num_thermal_sample - ptr2)
batch_test_thermal_imgs = test_thermal_imgs[
ptr2:ptr2 + real_size, :, :, :]
feature_thermal = sess.run(feat2,
feed_dict={
x2: batch_test_thermal_imgs,
keep_var: 1.
})
feature = sess.run(feat,
feed_dict={
feat1: feature_thermal,
feat2: feature_thermal,
keep_var: 1.
})
_, batch_thermal_feat = tf.split(0, 2, feature)
test_thermal_feat[
ptr2:ptr2 + real_size, :] = batch_thermal_feat.eval()
ptr2 += real_size
query_t_norm = tf.nn.l2_normalize(test_color_feat, dim=1)
test_t_norm = tf.nn.l2_normalize(test_thermal_feat, dim=1)
distmat = tf.matmul(query_t_norm,
test_t_norm,
transpose_a=False,
transpose_b=True)
cmc, mAP = compute_accuracy(-distmat,
test_color_labels,
test_thermal_labels,
topk=20)
print(
'top-1: {:.2%} | top-5: {:.2%} | top-10: {:.2%}| top-20: {:.2%}'
.format(cmc[0], cmc[4], cmc[9], cmc[19]))
print('mAP: {:.2%}'.format(mAP))
# Save the model checkpoint periodically.
if step % 2500 == 0 or step == training_iters:
checkpoint_name = os.path.join(checkpoint_path,
suffix + str(step) + '.ckpt')
save_path = saver.save(sess,
checkpoint_name,
write_meta_graph=False)
step += 1
print "Finish!"
import keras
import matplotlib.pyplot as plt
import numpy as np
import random
from dataset2 import Dataset
ds = Dataset('cinic10')
gen = ds.generate_train(128, {}).__iter__()
fig=plt.figure(figsize=(8, 8))
columns = 8
rows = 3
plt.axis('off')
for i in range(1, columns*rows +1):
x_train, y_train = next(gen)
img = x_train[random.randint(0, x_train.shape[0])]
ax = fig.add_subplot(rows, columns, i)
ax.axis('off')
plt.imshow(img, cmap='brg')
plt.show()
writer = SummaryWriter(log_dir=args.log_dir)
size = (args.image_size, args.image_size)
img_tf = transforms.Compose([
transforms.Resize(size=size),
transforms.ToTensor(),
transforms.Normalize(mean=opt.MEAN, std=opt.STD)
])
mask_tf = transforms.Compose(
[transforms.Resize(size=size),
transforms.ToTensor()])
#dataset_train = Places2(args.root, args.mask_root, img_tf, mask_tf, 'train')
#dataset_val = Places2(args.root, args.mask_root, img_tf, mask_tf, 'val')
dataset_train = Dataset(args.root, args.mask_root, img_tf, mask_tf, True)
dataset_val = Dataset(args.root_val, args.mask_root_val, img_tf, mask_tf, True)
iterator_train = iter(
data.DataLoader(dataset_train,
batch_size=args.batch_size,
sampler=InfiniteSampler(len(dataset_train)),
num_workers=args.n_threads))
print(len(dataset_train))
model = PConvUNet().to(device)
if args.finetune:
lr = args.lr_finetune
model.freeze_enc_bn = True
else:
lr = args.lr
emb_dropout = 0.3
)
def count_parameters(model):
return sum(p.numel() for p in model.parameters() if p.requires_grad)
print(count_parameters(v))
criterion = nn.CrossEntropyLoss()
bce = nn.BCELoss()
sigmoid = nn.Sigmoid()
opt = torch.optim.Adam(v.parameters(), lr=3e-4)
#opt = torch.optim.SGD(v.parameters(), lr=3e-4)
v.cuda()
criterion.cuda()
bce.cuda()
dataloader = torch.utils.data.DataLoader(
Dataset(root_path), batch_size=batch_size, shuffle=False, num_workers=8
)
test_dataloader = torch.utils.data.DataLoader(
TestDataset(test_path), batch_size=batch_size, shuffle=False, num_workers=8
)
def plot_confusion_matrix(cm,classes, epoch, normalize=False,title='Confusion matrix',cmap=plt.cm.Blues):
plt.clf()
"""
This function prints and plots the confusion matrix very prettily.
Normalization can be applied by setting `normalize=True`.
"""
if normalize:
cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
print("Normalized confusion matrix")
else:
class Experiment:
"""
Class that represents a single experiment that can be run and evaluated.
"""
def __init__(self,
name='unnamed',
db='cifar10',
net_type='vgg19',
batch_size=128,
epochs=100,
checkpoint_dir='checkpoint',
loss='categorical_crossentropy',
activation='relu',
final_activation='softmax',
f_a_params={},
use_tau=True,
prob_layer=None,
spp_alpha=1.0,
lr=0.1,
momentum=0.9,
dropout=0,
task='both',
workers=4,
queue_size=1024,
val_metrics=['loss', 'acc'],
rescale_factor=0,
augmentation={},
val_type='holdout',
holdout=0.2,
n_folds=5):
self._name = name
self._db = db
self._net_type = net_type
self._batch_size = batch_size
self._epochs = epochs
self._checkpoint_dir = checkpoint_dir
self._loss = loss
self._activation = activation
self._use_tau = use_tau
self._final_activation = final_activation
self._f_a_params = f_a_params
self._prob_layer = prob_layer
self._spp_alpha = spp_alpha
self._lr = lr
self._momentum = momentum
self._dropout = dropout
self._task = task
self._finished = False
self._workers = workers
self._queue_size = queue_size
self._val_metrics = val_metrics
self._rescale_factor = rescale_factor
self._augmentation = augmentation
self._val_type = val_type
self._holdout = holdout
self._n_folds = n_folds
self._current_fold = 0
self._best_metric = None
self._ds = None
# Model and results file names
self.model_file = 'model.h5'
self.best_model_file = 'best_model.h5'
self.model_file_extra = 'model.txt'
self.csv_file = 'results.csv'
self.evaluation_file = 'evaluation.pickle'
def set_auto_name(self):
"""
Set experiment name based on experiment parameters.
:return: None
"""
self.name = self.get_auto_name()
def get_auto_name(self):
"""
Get experiment auto-generated name based on experiment parameters.
:return: experiment auto-generated name.
"""
return "{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}".format(
self.db, self.net_type, self.batch_size, self.activation,
self.loss, self.final_activation,
self.prob_layer and self.prob_layer or '', self.spp_alpha, self.lr,
self.momentum, self.dropout)
# PROPERTIES
@property
def name(self):
return self._name
@name.setter
def name(self, name):
self._name = name
@name.deleter
def name(self):
del self._name
@property
def db(self):
return self._db
@db.setter
def db(self, db):
self._db = db
@db.deleter
def db(self):
del self._db
@property
def net_type(self):
return self._net_type
@net_type.setter
def net_type(self, net_type):
self._net_type = net_type
@net_type.deleter
def net_type(self):
del self._net_type
@property
def batch_size(self):
return self._batch_size
@batch_size.setter
def batch_size(self, batch_size):
self._batch_size = batch_size
@batch_size.deleter
def batch_size(self):
del self._batch_size
@property
def epochs(self):
return self._epochs
@epochs.setter
def epochs(self, epochs):
self._epochs = epochs
@epochs.deleter
def epochs(self):
del self._epochs
@property
def checkpoint_dir(self):
return self._checkpoint_dir
@checkpoint_dir.setter
def checkpoint_dir(self, checkpoint_dir):
self._checkpoint_dir = checkpoint_dir
@checkpoint_dir.deleter
def checkpoint_dir(self):
del self._checkpoint_dir
@property
def loss(self):
return self._loss
@loss.setter
def loss(self, loss):
self._loss = loss
@loss.deleter
def loss(self):
del self._loss
@property
def activation(self):
return self._activation
@activation.setter
def activation(self, activation):
self._activation = activation
@activation.deleter
def activation(self):
del self._activation
@property
def final_activation(self):
return self._final_activation
@final_activation.setter
def final_activation(self, final_activation):
self._final_activation = final_activation
@final_activation.deleter
def final_activation(self):
del self._final_activation
@property
def f_a_params(self):
return self._f_a_params
@f_a_params.setter
def f_a_params(self, f_a_params):
self._f_a_params = f_a_params
@f_a_params.deleter
def f_a_params(self):
del self._f_a_params
@property
def use_tau(self):
return self._use_tau
@use_tau.setter
def use_tau(self, use_tau):
self._use_tau = use_tau
@use_tau.deleter
def use_tau(self):
del self._use_tau
@property
def prob_layer(self):
return self._prob_layer
@prob_layer.setter
def prob_layer(self, prob_layer):
self._prob_layer = prob_layer
@prob_layer.deleter
def prob_layer(self):
del self._prob_layer
@property
def spp_alpha(self):
return self._spp_alpha
@spp_alpha.setter
def spp_alpha(self, spp_alpha):
self._spp_alpha = spp_alpha
@spp_alpha.deleter
def spp_alpha(self):
del self._spp_alpha
@property
def lr(self):
return self._lr
@lr.setter
def lr(self, lr):
self._lr = lr
@lr.deleter
def lr(self):
del self._lr
@property
def momentum(self):
return self._momentum
@momentum.setter
def momentum(self, momentum):
self._momentum = momentum
@momentum.deleter
def momentum(self):
del self._momentum
@property
def dropout(self):
return self._dropout
@dropout.setter
def dropout(self, dropout):
self._dropout = dropout
@dropout.deleter
def dropout(self):
del self._dropout
@property
def task(self):
return self._task
@task.setter
def task(self, task):
self._task = task
@task.deleter
def task(self):
del self._task
@property
def finished(self):
return self._finished
@finished.setter
def finished(self, finished):
self._finished = finished
@finished.deleter
def finished(self):
del self._finished
@property
def workers(self):
return self._workers
@workers.setter
def workers(self, workers):
self._workers = workers
@workers.deleter
def workers(self):
del self._workers
@property
def queue_size(self):
return self._workers
@queue_size.setter
def queue_size(self, queue_size):
self._queue_size = queue_size
@queue_size.deleter
def queue_size(self):
del self._queue_size
@property
def val_metrics(self):
return self._val_metrics
@val_metrics.setter
def val_metrics(self, val_metrics):
self._val_metrics = val_metrics
@val_metrics.deleter
def val_metrics(self):
del self._val_metrics
@property
def rescale_factor(self):
return self._rescale_factor
@rescale_factor.setter
def rescale_factor(self, rescale_factor):
self._rescale_factor = rescale_factor
@rescale_factor.deleter
def rescale_factor(self):
del self._rescale_factor
@property
def augmentation(self):
return self._augmentation
@augmentation.setter
def augmentation(self, augmentation):
self._augmentation = augmentation
@augmentation.deleter
def augmentation(self):
del self._augmentation
@property
def best_metric(self):
return self._best_metric
@property
def current_fold(self):
return self._current_fold
@current_fold.setter
def current_fold(self, current_fold):
self._current_fold = current_fold
def new_metric(self, metric, maximize=False):
"""
Updates best metric if metric provided is better than the best metric stored.
:param metric: new metric.
:param maximize: maximize metric instead of minimizing.
:return: True if new metric is better than best metric or False otherwise.
"""
if self._best_metric is None or (
maximize and metric > self._best_metric
or not maximize and metric <= self._best_metric):
self._best_metric = metric
return True
return False
# # # # # # #
def run(self):
"""
Run training process.
:return: None
"""
print('=== RUNNING {} ==='.format(self.name))
# Initial epoch. 0 by default
start_epoch = 0
# Load training status
if os.path.isfile(
os.path.join(self.checkpoint_dir, self.model_file_extra)):
# Continue from the epoch where we were and load the best metric
with open(os.path.join(self.checkpoint_dir, self.model_file_extra),
'r') as f:
start_epoch = int(f.readline())
self.new_metric(float(f.readline()))
if start_epoch >= self.epochs:
print("Training already finished. Skipping...")
return
# Get class weights based on frequency
class_weight = self._ds.get_class_weights()
# Learning rate scheduler callback
def lr_exp_scheduler(epoch):
lr = self.lr * np.exp(-0.025 * epoch)
return lr
lr_drop = 20
def lr_scheduler(epoch):
return self.lr * (0.5**(epoch // lr_drop))
# Save epoch callback for training process
def save_epoch(epoch, logs):
# Check whether new metric is better than best metric
if (self.new_metric(logs['val_loss'])):
model.save(
os.path.join(self.checkpoint_dir, self.best_model_file))
print("Best model saved.")
with open(os.path.join(self.checkpoint_dir, self.model_file_extra),
'w') as f:
f.write(str(epoch + 1))
f.write('\n' + str(self.best_metric))
save_epoch_callback = keras.callbacks.LambdaCallback(
on_epoch_end=save_epoch)
# NNet object
net_object = Net(self._ds.img_size, self.activation,
self.final_activation, self.f_a_params, self.use_tau,
self.prob_layer, self._ds.num_channels,
self._ds.num_classes, self.spp_alpha, self.dropout)
# model = self.get_model(net_object, self.net_type)
model = net_object.build(self.net_type)
# Create checkpoint dir if not exists
if not os.path.isdir(self.checkpoint_dir):
os.makedirs(self.checkpoint_dir)
# Check whether a saved model exists
if os.path.isfile(os.path.join(self.checkpoint_dir, self.model_file)):
print("===== RESTORING SAVED MODEL =====")
model.load_weights(
os.path.join(self.checkpoint_dir, self.model_file))
elif os.path.isfile(
os.path.join(self.checkpoint_dir, self.best_model_file)):
print("===== RESTORING SAVED BEST MODEL =====")
model.load_weights(
os.path.join(self.checkpoint_dir, self.best_model_file))
# Cross-entropy loss by default
loss = 'categorical_crossentropy'
metrics = ['accuracy']
lr_decay = 1e-6
# Compile the keras model
model.compile(optimizer=keras.optimizers.SGD(lr=self.lr,
decay=lr_decay,
momentum=0.9,
nesterov=True),
loss=loss,
metrics=metrics)
# Print model summary
model.summary()
print(
F'Training on {self._ds.size_train()} samples, validating on {self._ds.size_val()} samples.'
)
# Run training
model.fit_generator(
self._ds.generate_train(self.batch_size, self.augmentation),
epochs=self.epochs,
initial_epoch=start_epoch,
steps_per_epoch=self._ds.num_batches_train(self.batch_size),
callbacks=[
keras.callbacks.LearningRateScheduler(lr_scheduler),
save_epoch_callback,
keras.callbacks.CSVLogger(os.path.join(self.checkpoint_dir,
self.csv_file),
append=True),
keras.callbacks.EarlyStopping(min_delta=0.0005,
patience=40,
verbose=1)
],
workers=self.workers,
use_multiprocessing=False,
max_queue_size=self.queue_size,
class_weight=class_weight,
validation_data=self._ds.generate_val(self.batch_size),
validation_steps=self._ds.num_batches_val(self.batch_size),
verbose=2)
self.finished = True
# Mark the training as finished in the checkpoint file
with open(os.path.join(self.checkpoint_dir, self.model_file_extra),
'w') as f:
f.write(str(self.epochs))
f.write('\n' + str(self.best_metric))
# Delete model file
if os.path.isfile(os.path.join(self.checkpoint_dir, self.model_file)):
os.remove(os.path.join(self.checkpoint_dir, self.model_file))
def evaluate(self):
"""
Run evaluation on test data.
:return: None
"""
print('=== EVALUATING {} ==='.format(self.name))
# Check if best model file exists
if not os.path.isfile(
os.path.join(self.checkpoint_dir, self.best_model_file)):
print('Best model file not found')
return
# Check if model was already evaluated
if os.path.isfile(
os.path.join(self.checkpoint_dir, self.evaluation_file)):
print('Model already evaluated')
return
all_metrics = {}
# Get the generators for train, validation and test
generators = [
self._ds.generate_train(self.batch_size, {}),
self._ds.generate_val(self.batch_size),
self._ds.generate_test(self.batch_size)
]
steps = [
self._ds.num_batches_train(self.batch_size),
self._ds.num_batches_val(self.batch_size),
self._ds.num_batches_test(self.batch_size)
]
for generator, step, set in zip(generators, steps,
['Train', 'Validation', 'Test']):
print('\n=== {} dataset ===\n'.format(set))
# NNet object
net_object = Net(self._ds.img_size, self.activation,
self.final_activation, self.f_a_params,
self.use_tau, self.prob_layer,
self._ds.num_channels, self._ds.num_classes,
self.spp_alpha, self.dropout)
# model = self.get_model(net_object, self.net_type)
model = net_object.build(self.net_type)
# Load weights
model.load_weights(
os.path.join(self.checkpoint_dir, self.best_model_file))
# Get predictions
predictions = model.predict_generator(generator,
steps=step,
verbose=1)
y_set = None
for x, y in generator:
y_set = np.array(y) if y_set is None else np.vstack((y_set, y))
metrics = self.compute_metrics(y_set, predictions,
self._ds.num_classes)
self.print_metrics(metrics)
all_metrics[set] = metrics
with open(os.path.join(self.checkpoint_dir, self.evaluation_file),
'wb') as f:
pickle.dump({
'config': self.get_config(),
'metrics': all_metrics
}, f)
def compute_metrics(self, y_true, y_pred, num_classes):
# Calculate metric
sess = keras.backend.get_session()
qwk = np_quadratic_weighted_kappa(np.argmax(y_true, axis=1),
np.argmax(y_pred, axis=1), 0,
num_classes - 1)
ms = minimum_sensitivity(y_true, y_pred)
mae = sess.run(K.mean(keras.losses.mean_absolute_error(y_true,
y_pred)))
omae = sess.run(
K.mean(
keras.losses.mean_absolute_error(K.argmax(y_true),
K.argmax(y_pred))))
mse = sess.run(K.mean(keras.losses.mean_squared_error(y_true, y_pred)))
acc = sess.run(
K.mean(keras.metrics.categorical_accuracy(y_true, y_pred)))
top2 = sess.run(top_2_accuracy(y_true, y_pred))
top3 = sess.run(top_3_accuracy(y_true, y_pred))
off1 = accuracy_off1(y_true, y_pred)
conf_mat = confusion_matrix(np.argmax(y_true, axis=1),
np.argmax(y_pred, axis=1))
metrics = {
'QWK': qwk,
'MS': ms,
'MAE': mae,
'OMAE': omae,
'MSE': mse,
'CCR': acc,
'Top-2': top2,
'Top-3': top3,
'1-off': off1,
'Confusion matrix': conf_mat
}
return metrics
def print_metrics(self, metrics):
print('Confusion matrix :\n{}'.format(metrics['Confusion matrix']))
print('QWK: {:.4f}'.format(metrics['QWK']))
print('CCR: {:.4f}'.format(metrics['CCR']))
print('Top-2: {:.4f}'.format(metrics['Top-2']))
print('Top-3: {:.4f}'.format(metrics['Top-3']))
print('1-off: {:.4f}'.format(metrics['1-off']))
print('MAE: {:.4f}'.format(metrics['MAE']))
print('OMAE: {:.4f}'.format(metrics['OMAE']))
print('MSE: {:.4f}'.format(metrics['MSE']))
print('MS: {:.4f}'.format(metrics['MS']))
def get_config(self):
"""
Get config dictionary from object config.
:return: config dictionary.
"""
return {
'name': self.name,
'db': self.db,
'net_type': self.net_type,
'batch_size': self.batch_size,
'epochs': self.epochs,
'checkpoint_dir': self.checkpoint_dir,
'prob_layer': self.prob_layer,
'loss': self.loss,
'activation': self.activation,
'use_tau': self.use_tau,
'final_activation': self.final_activation,
'f_a_params': self.f_a_params,
'spp_alpha': self.spp_alpha,
'lr': self.lr,
'momentum': self.momentum,
'dropout': self.dropout,
'task': self.task,
'workers': self.workers,
'queue_size': self.queue_size,
'val_metrics': self.val_metrics,
'rescale_factor': self.rescale_factor,
'augmentation': self.augmentation,
'val_type': self._val_type,
'holdout': self._holdout,
'n_folds': self._n_folds
}
def set_config(self, config):
"""
Set object config from config dictionary
:param config: config dictionary.
:return: None
"""
self.db = 'db' in config and config['db'] or 'cifar10'
self.net_type = 'net_type' in config and config['net_type'] or 'vgg19'
self.batch_size = 'batch_size' in config and int(
config['batch_size']) or 128
self.epochs = 'epochs' in config and config['epochs'] or 100
self.checkpoint_dir = 'checkpoint_dir' in config and config[
'checkpoint_dir'] or 'results'
self.loss = 'loss' in config and config['loss'] or 'crossentropy'
self.activation = 'activation' in config and config[
'activation'] or 'relu'
self.final_activation = 'final_activation' in config and config[
'final_activation'] or 'softmax'
self.f_a_params = config[
'f_a_params'] if 'f_a_params' in config else {}
self.use_tau = config[
'use_tau'] if 'use_tau' in config and config['use_tau'] else False
self.prob_layer = 'prob_layer' in config and config[
'prob_layer'] or None
self.spp_alpha = 'spp_alpha' in config and config['spp_alpha'] or 0
self.lr = 'lr' in config and config['lr'] or 0.1
self.momentum = 'momentum' in config and config['momentum'] or 0
self.dropout = 'dropout' in config and config['dropout'] or 0
self.task = 'task' in config and config['task'] or 'both'
self.workers = 'workers' in config and config['workers'] or 4
self.queue_size = 'queue_size' in config and config[
'queue_size'] or 1024
self.val_metrics = 'val_metrics' in config and config[
'val_metrics'] or ['acc', 'loss']
self.rescale_factor = 'rescale_factor' in config and config[
'rescale_factor'] or 0
self.augmentation = 'augmentation' in config and config[
'augmentation'] or {}
self._val_type = 'val_type' in config and config[
'val_type'] or 'holdout'
self._holdout = 'holdout' in config and float(config['holdout']) or 0.2
self._n_folds = 'n_folds' in config and int(config['n_folds']) or 5
if 'name' in config:
self.name = config['name']
else:
self.set_auto_name()
# Load dataset
self._ds = Dataset(self._db)
self._setup_validation()
def _setup_validation(self):
if self._ds is None:
raise Exception(
'Cannot setup validation because dataset is not loaded')
# Validation config
if self._val_type == 'kfold':
self._ds.n_folds = self._n_folds
self._ds.set_fold(self._current_fold)
elif self._val_type == 'holdout':
self._ds.n_folds = 1 # 1 fold means holdout
self._ds.holdout = self._holdout
else:
raise Exception('{} is not a valid validation type.'.format(
self._val_type))
def save_to_file(self, path):
"""
Save experiment to pickle file.
:param path: path where pickle file will be saved.
:return: None
"""
pickle.dump(self.get_config(), path)
def load_from_file(self, path):
"""
Load experiment from pickle file.
:param path: path where pickle file is located.
:return: None
"""
if os.path.isfile(path):
self.set_config(pickle.load(path))
def main():
# Dataset path
# Visible Images and Color Images
train_color_list = '../idx/train_color_2tream_1.txt'
train_thermal_list = '../idx/train_thermal_2tream_1.txt'
checkpoint_path = 'log/'
# Learning params
learning_rate = 0.001
training_iters = 5000
batch_size = 64
display_step = 5
test_step = 300 # 0.5 epoch
# Network params
n_classes = 206
keep_rate = 0.5
# Graph input
x1 = tf.placeholder(tf.float32, [batch_size, 227, 227, 3])
x2 = tf.placeholder(tf.float32, [batch_size, 227, 227, 3])
y0 = tf.placeholder(tf.float32, [batch_size*2, n_classes])
# y = tf.placeholder(tf.float32, [batch_size, 1])
y1 = tf.placeholder(tf.float32, [batch_size, 1])
y2 = tf.placeholder(tf.float32, [batch_size, 1])
configProt = tf.ConfigProto()
sess = tf.Session(config = configProt)
keep_var = tf.placeholder(tf.float32)
# Model
feat1 = Model().alexnet_visible(x1, keep_var)
feat2 = Model().alexnet_thermal(x2, keep_var)
feat, pred0 = Model().modal_embedding(feat1, feat2, keep_var)
# norm_feat = tf.nn.l2_normalize(feat, 1, epsilon=1e-12)
feature1, feature2 = tf.split(0, 2, feat)
rank_loss, prec = compute_birank_loss(feature1, feature2, batch_size)
# Loss and optimizer
identity_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred0, y0))
# #################################################################################
# Total Loss:
# Ranking loss performs much better on small dataset
# Identity loss is better for large scale dataset with abundant training samples
# We could adjust the combining weights to achieve different performance
# #################################################################################
total_loss = identity_loss + rank_loss
optimizer = tf.train.MomentumOptimizer(learning_rate, momentum=0.9).minimize(total_loss)
# Evaluation
correct_pred0 = tf.equal(tf.argmax(pred0, 1), tf.argmax(y0, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred0, tf.float32))
# Init
init = tf.initialize_all_variables()
saver = tf.train.Saver()
# Load dataset
dataset = Dataset(train_color_list, train_thermal_list)
# load testing data (A short test for fast visualization)
test_thermal_list = '../idx/test_thermal_1.txt'
test_color_list = '../idx/test_color_1.txt'
test_color_imgs, test_color_labels = get_test_data(test_color_list)
test_thermal_imgs, test_thermal_labels = get_test_data(test_thermal_list)
# Launch the graph
with tf.Session() as sess:
print 'Init variable'
sess.run(init)
print 'Start training'
step = 0
while step < training_iters:
batch_x1, batch_x2, bacth_y0, batch_y1, batch_y2 = dataset.next_batch(batch_size, 'train')
sess.run(optimizer, feed_dict={x1: batch_x1, x2: batch_x2, y0:bacth_y0, y1: batch_y1, y2: batch_y2, keep_var: keep_rate})
# Display training status
if step%display_step == 0:
acc = sess.run(prec, feed_dict={x1: batch_x1, x2: batch_x2, y0:bacth_y0, y1: batch_y1, y2: batch_y2, keep_var: 1.0})
batch_loss = sess.run(total_loss, feed_dict={x1: batch_x1, x2: batch_x2, y0:bacth_y0, y1: batch_y1, y2: batch_y2, keep_var: 1.0})
print >> sys.stderr, "{} Iter {}: Training Loss = {:.4f}, Top-1 Accuracy = {:.4f}".format(datetime.now(), step, batch_loss, acc)
# print >> sys.stderr, "diff: {} ".format(diff)
if step % 100 == 0:
# for iround in range
print 'A short Test'
feature1 = sess.run(feat1, feed_dict={x1:test_color_imgs[:batch_size], keep_var: 1.})
feature2 = sess.run(feat2, feed_dict={x2:test_thermal_imgs[:batch_size], keep_var: 1.})
feature = sess.run(feat, feed_dict={feat1:feature1, feat2:feature2, keep_var: 1.})
test_color_feature, test_thermal_feature = tf.split(0, 2, feature)
# test_feat = sess.run(feat, feed_dict={feat1: test_color_feature, feat2:test_thermal_feature keep_var: 1.})
query_t_norm = tf.nn.l2_normalize(test_color_feature, dim=1)
test_t_norm = tf.nn.l2_normalize(test_thermal_feature, dim=1)
distmat = tf.matmul(query_t_norm, test_t_norm, transpose_a=False, transpose_b=True)
cmc, mAP = compute_accuracy(-distmat, test_color_labels[:batch_size], test_thermal_labels[:batch_size],topk=20)
print('top-1: {:.2%} | top-5: {:.2%} | top-10: {:.2%}| top-20: {:.2%}'.format(cmc[0], cmc[4], cmc[9], cmc[19]))
print('mAP: {:.2%}'.format(mAP))
# Save the model checkpoint periodically.
if step % 2500 == 0 or step == training_iters:
checkpoint_name = os.path.join(checkpoint_path, 'cam_bdtr_trial_1_'+ str(step) +'.ckpt')
save_path = saver.save(sess, checkpoint_name)
step += 1
print "Finish!"