def set_config(self, config): """ Set object config from config dictionary :param config: config dictionary. :return: None """ self.db = 'db' in config and config['db'] or 'cifar10' self.net_type = 'net_type' in config and config['net_type'] or 'vgg19' self.batch_size = 'batch_size' in config and int( config['batch_size']) or 128 self.epochs = 'epochs' in config and config['epochs'] or 100 self.checkpoint_dir = 'checkpoint_dir' in config and config[ 'checkpoint_dir'] or 'results' self.loss = 'loss' in config and config['loss'] or 'crossentropy' self.activation = 'activation' in config and config[ 'activation'] or 'relu' self.final_activation = 'final_activation' in config and config[ 'final_activation'] or 'softmax' self.f_a_params = config[ 'f_a_params'] if 'f_a_params' in config else {} self.use_tau = config[ 'use_tau'] if 'use_tau' in config and config['use_tau'] else False self.prob_layer = 'prob_layer' in config and config[ 'prob_layer'] or None self.spp_alpha = 'spp_alpha' in config and config['spp_alpha'] or 0 self.lr = 'lr' in config and config['lr'] or 0.1 self.momentum = 'momentum' in config and config['momentum'] or 0 self.dropout = 'dropout' in config and config['dropout'] or 0 self.task = 'task' in config and config['task'] or 'both' self.workers = 'workers' in config and config['workers'] or 4 self.queue_size = 'queue_size' in config and config[ 'queue_size'] or 1024 self.val_metrics = 'val_metrics' in config and config[ 'val_metrics'] or ['acc', 'loss'] self.rescale_factor = 'rescale_factor' in config and config[ 'rescale_factor'] or 0 self.augmentation = 'augmentation' in config and config[ 'augmentation'] or {} self._val_type = 'val_type' in config and config[ 'val_type'] or 'holdout' self._holdout = 'holdout' in config and float(config['holdout']) or 0.2 self._n_folds = 'n_folds' in config and int(config['n_folds']) or 5 if 'name' in config: self.name = config['name'] else: self.set_auto_name() # Load dataset self._ds = Dataset(self._db) self._setup_validation()
# CUDA for PyTorch use_cuda = torch.cuda.is_available() print(use_cuda) device = torch.device("cuda:0" if use_cuda else "cpu") print(device) # Parameters max_epochs = 80 # Datasets train_dir = 'data/train/'# IDs train_label = 'data/train_kaggle.csv' test_dir = 'data/test/'# Labels test_label = 'data/sample_solution.csv' # Generators train_set = Dataset(train_dir, train_label,'train') val_set = Dataset(train_dir, train_label,'eval') # Creating data indices for training and validation splits: dataset_size = len(train_set) #print(dataset_size) indices = list(range(dataset_size)) split = int(np.floor(0.2 * dataset_size)) np.random.shuffle(indices) train_indices, val_indices = indices[split:], indices[:split] train_sampler = SubsetRandomSampler(train_indices) val_sampler = SubsetRandomSampler(val_indices) train_params = {'batch_size': args.batch_size, 'num_workers': 2,
def main(): # Dataset path # Visible Images and Color Images trial = 1 train_color_list = '../idx/train_color_{}'.format(trial) + '.txt' train_thermal_list = '../idx/train_thermal_{}'.format(trial) + '.txt' checkpoint_path = 'save_model/' # Learning params learning_rate = 0.001 training_iters = 5000 batch_size = 64 display_step = 5 test_step = 500 # 0.5 epoch margin = 0.5 lambda_intra = 0.1 feature_dim = 1024 test_batch_size = batch_size # Network params n_classes = 206 keep_rate = 0.5 suffix = 'BDTR' # model params suffix = suffix + '_drop_{}'.format(keep_rate) suffix = suffix + '_lr_{:1.1e}'.format(learning_rate) suffix = suffix + '_margin_{}'.format(margin) suffix = suffix + '_batch_{}'.format(batch_size) suffix = suffix + '_w_intra_{}'.format(lambda_intra) suffix = suffix + '_trial_{}_'.format(trial) # Graph input x1 = tf.placeholder(tf.float32, [None, 227, 227, 3]) x2 = tf.placeholder(tf.float32, [None, 227, 227, 3]) y0 = tf.placeholder(tf.float32, [None, n_classes]) # y = tf.placeholder(tf.float32, [batch_size, 1]) y1 = tf.placeholder(tf.float32, [None, 1]) y2 = tf.placeholder(tf.float32, [None, 1]) configProt = tf.ConfigProto() sess = tf.Session(config=configProt) keep_var = tf.placeholder(tf.float32) # Construct Model feat1 = Model().alexnet_visible(x1, keep_var) feat2 = Model().alexnet_thermal(x2, keep_var) feat, pred0 = Model().modal_embedding(feat1, feat2, keep_var, feature_dim) # norm_feat = tf.nn.l2_normalize(feat, 1, epsilon=1e-12) feature1, feature2 = tf.split(0, 2, feat) # pdb.set_trace() rank_loss, prec = compute_birank_loss(feature1, feature2, batch_size, margin, lambda_intra) # Loss and optimizer identity_loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(pred0, y0)) # ################################################################################# # Total Loss: # Ranking loss performs much better on small dataset # Identity loss is better for large scale dataset with abundant training samples # We could adjust the combining weights to achieve different performance # ################################################################################# total_loss = identity_loss + rank_loss optimizer = tf.train.MomentumOptimizer(learning_rate, momentum=0.9).minimize(total_loss) # Evaluation correct_pred0 = tf.equal(tf.argmax(pred0, 1), tf.argmax(y0, 1)) accuracy = tf.reduce_mean(tf.cast(correct_pred0, tf.float32)) # Init init = tf.initialize_all_variables() saver = tf.train.Saver() # Load dataset dataset = Dataset(train_color_list, train_thermal_list) # load testing data (A short test for fast visualization) test_thermal_list = '../idx/test_thermal_1.txt' test_color_list = '../idx/test_color_1.txt' test_color_imgs, test_color_labels = get_test_data(test_color_list) test_thermal_imgs, test_thermal_labels = get_test_data(test_thermal_list) if not os.path.isdir(checkpoint_path): os.makedirs(checkpoint_path) # Launch the graph with tf.Session() as sess: print 'Init variable' sess.run(init) print 'Start training' step = 1 while step < training_iters: batch_x1, batch_x2, bacth_y0, batch_y1, batch_y2 = dataset.next_batch( batch_size, 'train') sess.run(optimizer, feed_dict={ x1: batch_x1, x2: batch_x2, y0: bacth_y0, y1: batch_y1, y2: batch_y2, keep_var: keep_rate }) # Display training status if step % display_step == 0: acc = sess.run(prec, feed_dict={ x1: batch_x1, x2: batch_x2, y0: bacth_y0, y1: batch_y1, y2: batch_y2, keep_var: 1.0 }) batch_loss = sess.run(total_loss, feed_dict={ x1: batch_x1, x2: batch_x2, y0: bacth_y0, y1: batch_y1, y2: batch_y2, keep_var: 1.0 }) print >> sys.stderr, "{} Iter {}: Training Loss = {:.4f}, Top-1 Accuracy = {:.4f}".format( datetime.now(), step, batch_loss, acc) # print >> sys.stderr, "diff: {} ".format(diff) if step % test_step == 0: # for iround in range print('Test Step: {}'.format(step)) # get test color feature num_color_sample = test_color_imgs.shape[0] ptr1 = 0 test_color_feat = np.zeros((num_color_sample, feature_dim)) # logits = np.zeros((num_color_sample,num_category)) print('Extracting test color features...') for num in range(int(num_color_sample / test_batch_size) + 1): real_size = min(test_batch_size, num_color_sample - ptr1) batch_test_color_imgs = test_color_imgs[ptr1:ptr1 + real_size, :, :, :] feature_color = sess.run(feat1, feed_dict={ x1: batch_test_color_imgs, keep_var: 1. }) feature = sess.run(feat, feed_dict={ feat1: feature_color, feat2: feature_color, keep_var: 1. }) batch_color_feat, _ = tf.split(0, 2, feature) test_color_feat[ptr1:ptr1 + real_size, :] = batch_color_feat.eval() ptr1 += real_size # get test thermal feature num_thermal_sample = test_thermal_imgs.shape[0] ptr2 = 0 test_thermal_feat = np.zeros((num_thermal_sample, feature_dim)) # logits = np.zeros((num_color_sample,num_category)) print('Extracting test thermal features...') for num in range( int(num_thermal_sample / test_batch_size) + 1): real_size = min(test_batch_size, num_thermal_sample - ptr2) batch_test_thermal_imgs = test_thermal_imgs[ ptr2:ptr2 + real_size, :, :, :] feature_thermal = sess.run(feat2, feed_dict={ x2: batch_test_thermal_imgs, keep_var: 1. }) feature = sess.run(feat, feed_dict={ feat1: feature_thermal, feat2: feature_thermal, keep_var: 1. }) _, batch_thermal_feat = tf.split(0, 2, feature) test_thermal_feat[ ptr2:ptr2 + real_size, :] = batch_thermal_feat.eval() ptr2 += real_size query_t_norm = tf.nn.l2_normalize(test_color_feat, dim=1) test_t_norm = tf.nn.l2_normalize(test_thermal_feat, dim=1) distmat = tf.matmul(query_t_norm, test_t_norm, transpose_a=False, transpose_b=True) cmc, mAP = compute_accuracy(-distmat, test_color_labels, test_thermal_labels, topk=20) print( 'top-1: {:.2%} | top-5: {:.2%} | top-10: {:.2%}| top-20: {:.2%}' .format(cmc[0], cmc[4], cmc[9], cmc[19])) print('mAP: {:.2%}'.format(mAP)) # Save the model checkpoint periodically. if step % 2500 == 0 or step == training_iters: checkpoint_name = os.path.join(checkpoint_path, suffix + str(step) + '.ckpt') save_path = saver.save(sess, checkpoint_name, write_meta_graph=False) step += 1 print "Finish!"
import keras import matplotlib.pyplot as plt import numpy as np import random from dataset2 import Dataset ds = Dataset('cinic10') gen = ds.generate_train(128, {}).__iter__() fig=plt.figure(figsize=(8, 8)) columns = 8 rows = 3 plt.axis('off') for i in range(1, columns*rows +1): x_train, y_train = next(gen) img = x_train[random.randint(0, x_train.shape[0])] ax = fig.add_subplot(rows, columns, i) ax.axis('off') plt.imshow(img, cmap='brg') plt.show()
writer = SummaryWriter(log_dir=args.log_dir) size = (args.image_size, args.image_size) img_tf = transforms.Compose([ transforms.Resize(size=size), transforms.ToTensor(), transforms.Normalize(mean=opt.MEAN, std=opt.STD) ]) mask_tf = transforms.Compose( [transforms.Resize(size=size), transforms.ToTensor()]) #dataset_train = Places2(args.root, args.mask_root, img_tf, mask_tf, 'train') #dataset_val = Places2(args.root, args.mask_root, img_tf, mask_tf, 'val') dataset_train = Dataset(args.root, args.mask_root, img_tf, mask_tf, True) dataset_val = Dataset(args.root_val, args.mask_root_val, img_tf, mask_tf, True) iterator_train = iter( data.DataLoader(dataset_train, batch_size=args.batch_size, sampler=InfiniteSampler(len(dataset_train)), num_workers=args.n_threads)) print(len(dataset_train)) model = PConvUNet().to(device) if args.finetune: lr = args.lr_finetune model.freeze_enc_bn = True else: lr = args.lr
emb_dropout = 0.3 ) def count_parameters(model): return sum(p.numel() for p in model.parameters() if p.requires_grad) print(count_parameters(v)) criterion = nn.CrossEntropyLoss() bce = nn.BCELoss() sigmoid = nn.Sigmoid() opt = torch.optim.Adam(v.parameters(), lr=3e-4) #opt = torch.optim.SGD(v.parameters(), lr=3e-4) v.cuda() criterion.cuda() bce.cuda() dataloader = torch.utils.data.DataLoader( Dataset(root_path), batch_size=batch_size, shuffle=False, num_workers=8 ) test_dataloader = torch.utils.data.DataLoader( TestDataset(test_path), batch_size=batch_size, shuffle=False, num_workers=8 ) def plot_confusion_matrix(cm,classes, epoch, normalize=False,title='Confusion matrix',cmap=plt.cm.Blues): plt.clf() """ This function prints and plots the confusion matrix very prettily. Normalization can be applied by setting `normalize=True`. """ if normalize: cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] print("Normalized confusion matrix") else:
class Experiment: """ Class that represents a single experiment that can be run and evaluated. """ def __init__(self, name='unnamed', db='cifar10', net_type='vgg19', batch_size=128, epochs=100, checkpoint_dir='checkpoint', loss='categorical_crossentropy', activation='relu', final_activation='softmax', f_a_params={}, use_tau=True, prob_layer=None, spp_alpha=1.0, lr=0.1, momentum=0.9, dropout=0, task='both', workers=4, queue_size=1024, val_metrics=['loss', 'acc'], rescale_factor=0, augmentation={}, val_type='holdout', holdout=0.2, n_folds=5): self._name = name self._db = db self._net_type = net_type self._batch_size = batch_size self._epochs = epochs self._checkpoint_dir = checkpoint_dir self._loss = loss self._activation = activation self._use_tau = use_tau self._final_activation = final_activation self._f_a_params = f_a_params self._prob_layer = prob_layer self._spp_alpha = spp_alpha self._lr = lr self._momentum = momentum self._dropout = dropout self._task = task self._finished = False self._workers = workers self._queue_size = queue_size self._val_metrics = val_metrics self._rescale_factor = rescale_factor self._augmentation = augmentation self._val_type = val_type self._holdout = holdout self._n_folds = n_folds self._current_fold = 0 self._best_metric = None self._ds = None # Model and results file names self.model_file = 'model.h5' self.best_model_file = 'best_model.h5' self.model_file_extra = 'model.txt' self.csv_file = 'results.csv' self.evaluation_file = 'evaluation.pickle' def set_auto_name(self): """ Set experiment name based on experiment parameters. :return: None """ self.name = self.get_auto_name() def get_auto_name(self): """ Get experiment auto-generated name based on experiment parameters. :return: experiment auto-generated name. """ return "{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}".format( self.db, self.net_type, self.batch_size, self.activation, self.loss, self.final_activation, self.prob_layer and self.prob_layer or '', self.spp_alpha, self.lr, self.momentum, self.dropout) # PROPERTIES @property def name(self): return self._name @name.setter def name(self, name): self._name = name @name.deleter def name(self): del self._name @property def db(self): return self._db @db.setter def db(self, db): self._db = db @db.deleter def db(self): del self._db @property def net_type(self): return self._net_type @net_type.setter def net_type(self, net_type): self._net_type = net_type @net_type.deleter def net_type(self): del self._net_type @property def batch_size(self): return self._batch_size @batch_size.setter def batch_size(self, batch_size): self._batch_size = batch_size @batch_size.deleter def batch_size(self): del self._batch_size @property def epochs(self): return self._epochs @epochs.setter def epochs(self, epochs): self._epochs = epochs @epochs.deleter def epochs(self): del self._epochs @property def checkpoint_dir(self): return self._checkpoint_dir @checkpoint_dir.setter def checkpoint_dir(self, checkpoint_dir): self._checkpoint_dir = checkpoint_dir @checkpoint_dir.deleter def checkpoint_dir(self): del self._checkpoint_dir @property def loss(self): return self._loss @loss.setter def loss(self, loss): self._loss = loss @loss.deleter def loss(self): del self._loss @property def activation(self): return self._activation @activation.setter def activation(self, activation): self._activation = activation @activation.deleter def activation(self): del self._activation @property def final_activation(self): return self._final_activation @final_activation.setter def final_activation(self, final_activation): self._final_activation = final_activation @final_activation.deleter def final_activation(self): del self._final_activation @property def f_a_params(self): return self._f_a_params @f_a_params.setter def f_a_params(self, f_a_params): self._f_a_params = f_a_params @f_a_params.deleter def f_a_params(self): del self._f_a_params @property def use_tau(self): return self._use_tau @use_tau.setter def use_tau(self, use_tau): self._use_tau = use_tau @use_tau.deleter def use_tau(self): del self._use_tau @property def prob_layer(self): return self._prob_layer @prob_layer.setter def prob_layer(self, prob_layer): self._prob_layer = prob_layer @prob_layer.deleter def prob_layer(self): del self._prob_layer @property def spp_alpha(self): return self._spp_alpha @spp_alpha.setter def spp_alpha(self, spp_alpha): self._spp_alpha = spp_alpha @spp_alpha.deleter def spp_alpha(self): del self._spp_alpha @property def lr(self): return self._lr @lr.setter def lr(self, lr): self._lr = lr @lr.deleter def lr(self): del self._lr @property def momentum(self): return self._momentum @momentum.setter def momentum(self, momentum): self._momentum = momentum @momentum.deleter def momentum(self): del self._momentum @property def dropout(self): return self._dropout @dropout.setter def dropout(self, dropout): self._dropout = dropout @dropout.deleter def dropout(self): del self._dropout @property def task(self): return self._task @task.setter def task(self, task): self._task = task @task.deleter def task(self): del self._task @property def finished(self): return self._finished @finished.setter def finished(self, finished): self._finished = finished @finished.deleter def finished(self): del self._finished @property def workers(self): return self._workers @workers.setter def workers(self, workers): self._workers = workers @workers.deleter def workers(self): del self._workers @property def queue_size(self): return self._workers @queue_size.setter def queue_size(self, queue_size): self._queue_size = queue_size @queue_size.deleter def queue_size(self): del self._queue_size @property def val_metrics(self): return self._val_metrics @val_metrics.setter def val_metrics(self, val_metrics): self._val_metrics = val_metrics @val_metrics.deleter def val_metrics(self): del self._val_metrics @property def rescale_factor(self): return self._rescale_factor @rescale_factor.setter def rescale_factor(self, rescale_factor): self._rescale_factor = rescale_factor @rescale_factor.deleter def rescale_factor(self): del self._rescale_factor @property def augmentation(self): return self._augmentation @augmentation.setter def augmentation(self, augmentation): self._augmentation = augmentation @augmentation.deleter def augmentation(self): del self._augmentation @property def best_metric(self): return self._best_metric @property def current_fold(self): return self._current_fold @current_fold.setter def current_fold(self, current_fold): self._current_fold = current_fold def new_metric(self, metric, maximize=False): """ Updates best metric if metric provided is better than the best metric stored. :param metric: new metric. :param maximize: maximize metric instead of minimizing. :return: True if new metric is better than best metric or False otherwise. """ if self._best_metric is None or ( maximize and metric > self._best_metric or not maximize and metric <= self._best_metric): self._best_metric = metric return True return False # # # # # # # def run(self): """ Run training process. :return: None """ print('=== RUNNING {} ==='.format(self.name)) # Initial epoch. 0 by default start_epoch = 0 # Load training status if os.path.isfile( os.path.join(self.checkpoint_dir, self.model_file_extra)): # Continue from the epoch where we were and load the best metric with open(os.path.join(self.checkpoint_dir, self.model_file_extra), 'r') as f: start_epoch = int(f.readline()) self.new_metric(float(f.readline())) if start_epoch >= self.epochs: print("Training already finished. Skipping...") return # Get class weights based on frequency class_weight = self._ds.get_class_weights() # Learning rate scheduler callback def lr_exp_scheduler(epoch): lr = self.lr * np.exp(-0.025 * epoch) return lr lr_drop = 20 def lr_scheduler(epoch): return self.lr * (0.5**(epoch // lr_drop)) # Save epoch callback for training process def save_epoch(epoch, logs): # Check whether new metric is better than best metric if (self.new_metric(logs['val_loss'])): model.save( os.path.join(self.checkpoint_dir, self.best_model_file)) print("Best model saved.") with open(os.path.join(self.checkpoint_dir, self.model_file_extra), 'w') as f: f.write(str(epoch + 1)) f.write('\n' + str(self.best_metric)) save_epoch_callback = keras.callbacks.LambdaCallback( on_epoch_end=save_epoch) # NNet object net_object = Net(self._ds.img_size, self.activation, self.final_activation, self.f_a_params, self.use_tau, self.prob_layer, self._ds.num_channels, self._ds.num_classes, self.spp_alpha, self.dropout) # model = self.get_model(net_object, self.net_type) model = net_object.build(self.net_type) # Create checkpoint dir if not exists if not os.path.isdir(self.checkpoint_dir): os.makedirs(self.checkpoint_dir) # Check whether a saved model exists if os.path.isfile(os.path.join(self.checkpoint_dir, self.model_file)): print("===== RESTORING SAVED MODEL =====") model.load_weights( os.path.join(self.checkpoint_dir, self.model_file)) elif os.path.isfile( os.path.join(self.checkpoint_dir, self.best_model_file)): print("===== RESTORING SAVED BEST MODEL =====") model.load_weights( os.path.join(self.checkpoint_dir, self.best_model_file)) # Cross-entropy loss by default loss = 'categorical_crossentropy' metrics = ['accuracy'] lr_decay = 1e-6 # Compile the keras model model.compile(optimizer=keras.optimizers.SGD(lr=self.lr, decay=lr_decay, momentum=0.9, nesterov=True), loss=loss, metrics=metrics) # Print model summary model.summary() print( F'Training on {self._ds.size_train()} samples, validating on {self._ds.size_val()} samples.' ) # Run training model.fit_generator( self._ds.generate_train(self.batch_size, self.augmentation), epochs=self.epochs, initial_epoch=start_epoch, steps_per_epoch=self._ds.num_batches_train(self.batch_size), callbacks=[ keras.callbacks.LearningRateScheduler(lr_scheduler), save_epoch_callback, keras.callbacks.CSVLogger(os.path.join(self.checkpoint_dir, self.csv_file), append=True), keras.callbacks.EarlyStopping(min_delta=0.0005, patience=40, verbose=1) ], workers=self.workers, use_multiprocessing=False, max_queue_size=self.queue_size, class_weight=class_weight, validation_data=self._ds.generate_val(self.batch_size), validation_steps=self._ds.num_batches_val(self.batch_size), verbose=2) self.finished = True # Mark the training as finished in the checkpoint file with open(os.path.join(self.checkpoint_dir, self.model_file_extra), 'w') as f: f.write(str(self.epochs)) f.write('\n' + str(self.best_metric)) # Delete model file if os.path.isfile(os.path.join(self.checkpoint_dir, self.model_file)): os.remove(os.path.join(self.checkpoint_dir, self.model_file)) def evaluate(self): """ Run evaluation on test data. :return: None """ print('=== EVALUATING {} ==='.format(self.name)) # Check if best model file exists if not os.path.isfile( os.path.join(self.checkpoint_dir, self.best_model_file)): print('Best model file not found') return # Check if model was already evaluated if os.path.isfile( os.path.join(self.checkpoint_dir, self.evaluation_file)): print('Model already evaluated') return all_metrics = {} # Get the generators for train, validation and test generators = [ self._ds.generate_train(self.batch_size, {}), self._ds.generate_val(self.batch_size), self._ds.generate_test(self.batch_size) ] steps = [ self._ds.num_batches_train(self.batch_size), self._ds.num_batches_val(self.batch_size), self._ds.num_batches_test(self.batch_size) ] for generator, step, set in zip(generators, steps, ['Train', 'Validation', 'Test']): print('\n=== {} dataset ===\n'.format(set)) # NNet object net_object = Net(self._ds.img_size, self.activation, self.final_activation, self.f_a_params, self.use_tau, self.prob_layer, self._ds.num_channels, self._ds.num_classes, self.spp_alpha, self.dropout) # model = self.get_model(net_object, self.net_type) model = net_object.build(self.net_type) # Load weights model.load_weights( os.path.join(self.checkpoint_dir, self.best_model_file)) # Get predictions predictions = model.predict_generator(generator, steps=step, verbose=1) y_set = None for x, y in generator: y_set = np.array(y) if y_set is None else np.vstack((y_set, y)) metrics = self.compute_metrics(y_set, predictions, self._ds.num_classes) self.print_metrics(metrics) all_metrics[set] = metrics with open(os.path.join(self.checkpoint_dir, self.evaluation_file), 'wb') as f: pickle.dump({ 'config': self.get_config(), 'metrics': all_metrics }, f) def compute_metrics(self, y_true, y_pred, num_classes): # Calculate metric sess = keras.backend.get_session() qwk = np_quadratic_weighted_kappa(np.argmax(y_true, axis=1), np.argmax(y_pred, axis=1), 0, num_classes - 1) ms = minimum_sensitivity(y_true, y_pred) mae = sess.run(K.mean(keras.losses.mean_absolute_error(y_true, y_pred))) omae = sess.run( K.mean( keras.losses.mean_absolute_error(K.argmax(y_true), K.argmax(y_pred)))) mse = sess.run(K.mean(keras.losses.mean_squared_error(y_true, y_pred))) acc = sess.run( K.mean(keras.metrics.categorical_accuracy(y_true, y_pred))) top2 = sess.run(top_2_accuracy(y_true, y_pred)) top3 = sess.run(top_3_accuracy(y_true, y_pred)) off1 = accuracy_off1(y_true, y_pred) conf_mat = confusion_matrix(np.argmax(y_true, axis=1), np.argmax(y_pred, axis=1)) metrics = { 'QWK': qwk, 'MS': ms, 'MAE': mae, 'OMAE': omae, 'MSE': mse, 'CCR': acc, 'Top-2': top2, 'Top-3': top3, '1-off': off1, 'Confusion matrix': conf_mat } return metrics def print_metrics(self, metrics): print('Confusion matrix :\n{}'.format(metrics['Confusion matrix'])) print('QWK: {:.4f}'.format(metrics['QWK'])) print('CCR: {:.4f}'.format(metrics['CCR'])) print('Top-2: {:.4f}'.format(metrics['Top-2'])) print('Top-3: {:.4f}'.format(metrics['Top-3'])) print('1-off: {:.4f}'.format(metrics['1-off'])) print('MAE: {:.4f}'.format(metrics['MAE'])) print('OMAE: {:.4f}'.format(metrics['OMAE'])) print('MSE: {:.4f}'.format(metrics['MSE'])) print('MS: {:.4f}'.format(metrics['MS'])) def get_config(self): """ Get config dictionary from object config. :return: config dictionary. """ return { 'name': self.name, 'db': self.db, 'net_type': self.net_type, 'batch_size': self.batch_size, 'epochs': self.epochs, 'checkpoint_dir': self.checkpoint_dir, 'prob_layer': self.prob_layer, 'loss': self.loss, 'activation': self.activation, 'use_tau': self.use_tau, 'final_activation': self.final_activation, 'f_a_params': self.f_a_params, 'spp_alpha': self.spp_alpha, 'lr': self.lr, 'momentum': self.momentum, 'dropout': self.dropout, 'task': self.task, 'workers': self.workers, 'queue_size': self.queue_size, 'val_metrics': self.val_metrics, 'rescale_factor': self.rescale_factor, 'augmentation': self.augmentation, 'val_type': self._val_type, 'holdout': self._holdout, 'n_folds': self._n_folds } def set_config(self, config): """ Set object config from config dictionary :param config: config dictionary. :return: None """ self.db = 'db' in config and config['db'] or 'cifar10' self.net_type = 'net_type' in config and config['net_type'] or 'vgg19' self.batch_size = 'batch_size' in config and int( config['batch_size']) or 128 self.epochs = 'epochs' in config and config['epochs'] or 100 self.checkpoint_dir = 'checkpoint_dir' in config and config[ 'checkpoint_dir'] or 'results' self.loss = 'loss' in config and config['loss'] or 'crossentropy' self.activation = 'activation' in config and config[ 'activation'] or 'relu' self.final_activation = 'final_activation' in config and config[ 'final_activation'] or 'softmax' self.f_a_params = config[ 'f_a_params'] if 'f_a_params' in config else {} self.use_tau = config[ 'use_tau'] if 'use_tau' in config and config['use_tau'] else False self.prob_layer = 'prob_layer' in config and config[ 'prob_layer'] or None self.spp_alpha = 'spp_alpha' in config and config['spp_alpha'] or 0 self.lr = 'lr' in config and config['lr'] or 0.1 self.momentum = 'momentum' in config and config['momentum'] or 0 self.dropout = 'dropout' in config and config['dropout'] or 0 self.task = 'task' in config and config['task'] or 'both' self.workers = 'workers' in config and config['workers'] or 4 self.queue_size = 'queue_size' in config and config[ 'queue_size'] or 1024 self.val_metrics = 'val_metrics' in config and config[ 'val_metrics'] or ['acc', 'loss'] self.rescale_factor = 'rescale_factor' in config and config[ 'rescale_factor'] or 0 self.augmentation = 'augmentation' in config and config[ 'augmentation'] or {} self._val_type = 'val_type' in config and config[ 'val_type'] or 'holdout' self._holdout = 'holdout' in config and float(config['holdout']) or 0.2 self._n_folds = 'n_folds' in config and int(config['n_folds']) or 5 if 'name' in config: self.name = config['name'] else: self.set_auto_name() # Load dataset self._ds = Dataset(self._db) self._setup_validation() def _setup_validation(self): if self._ds is None: raise Exception( 'Cannot setup validation because dataset is not loaded') # Validation config if self._val_type == 'kfold': self._ds.n_folds = self._n_folds self._ds.set_fold(self._current_fold) elif self._val_type == 'holdout': self._ds.n_folds = 1 # 1 fold means holdout self._ds.holdout = self._holdout else: raise Exception('{} is not a valid validation type.'.format( self._val_type)) def save_to_file(self, path): """ Save experiment to pickle file. :param path: path where pickle file will be saved. :return: None """ pickle.dump(self.get_config(), path) def load_from_file(self, path): """ Load experiment from pickle file. :param path: path where pickle file is located. :return: None """ if os.path.isfile(path): self.set_config(pickle.load(path))
def main(): # Dataset path # Visible Images and Color Images train_color_list = '../idx/train_color_2tream_1.txt' train_thermal_list = '../idx/train_thermal_2tream_1.txt' checkpoint_path = 'log/' # Learning params learning_rate = 0.001 training_iters = 5000 batch_size = 64 display_step = 5 test_step = 300 # 0.5 epoch # Network params n_classes = 206 keep_rate = 0.5 # Graph input x1 = tf.placeholder(tf.float32, [batch_size, 227, 227, 3]) x2 = tf.placeholder(tf.float32, [batch_size, 227, 227, 3]) y0 = tf.placeholder(tf.float32, [batch_size*2, n_classes]) # y = tf.placeholder(tf.float32, [batch_size, 1]) y1 = tf.placeholder(tf.float32, [batch_size, 1]) y2 = tf.placeholder(tf.float32, [batch_size, 1]) configProt = tf.ConfigProto() sess = tf.Session(config = configProt) keep_var = tf.placeholder(tf.float32) # Model feat1 = Model().alexnet_visible(x1, keep_var) feat2 = Model().alexnet_thermal(x2, keep_var) feat, pred0 = Model().modal_embedding(feat1, feat2, keep_var) # norm_feat = tf.nn.l2_normalize(feat, 1, epsilon=1e-12) feature1, feature2 = tf.split(0, 2, feat) rank_loss, prec = compute_birank_loss(feature1, feature2, batch_size) # Loss and optimizer identity_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred0, y0)) # ################################################################################# # Total Loss: # Ranking loss performs much better on small dataset # Identity loss is better for large scale dataset with abundant training samples # We could adjust the combining weights to achieve different performance # ################################################################################# total_loss = identity_loss + rank_loss optimizer = tf.train.MomentumOptimizer(learning_rate, momentum=0.9).minimize(total_loss) # Evaluation correct_pred0 = tf.equal(tf.argmax(pred0, 1), tf.argmax(y0, 1)) accuracy = tf.reduce_mean(tf.cast(correct_pred0, tf.float32)) # Init init = tf.initialize_all_variables() saver = tf.train.Saver() # Load dataset dataset = Dataset(train_color_list, train_thermal_list) # load testing data (A short test for fast visualization) test_thermal_list = '../idx/test_thermal_1.txt' test_color_list = '../idx/test_color_1.txt' test_color_imgs, test_color_labels = get_test_data(test_color_list) test_thermal_imgs, test_thermal_labels = get_test_data(test_thermal_list) # Launch the graph with tf.Session() as sess: print 'Init variable' sess.run(init) print 'Start training' step = 0 while step < training_iters: batch_x1, batch_x2, bacth_y0, batch_y1, batch_y2 = dataset.next_batch(batch_size, 'train') sess.run(optimizer, feed_dict={x1: batch_x1, x2: batch_x2, y0:bacth_y0, y1: batch_y1, y2: batch_y2, keep_var: keep_rate}) # Display training status if step%display_step == 0: acc = sess.run(prec, feed_dict={x1: batch_x1, x2: batch_x2, y0:bacth_y0, y1: batch_y1, y2: batch_y2, keep_var: 1.0}) batch_loss = sess.run(total_loss, feed_dict={x1: batch_x1, x2: batch_x2, y0:bacth_y0, y1: batch_y1, y2: batch_y2, keep_var: 1.0}) print >> sys.stderr, "{} Iter {}: Training Loss = {:.4f}, Top-1 Accuracy = {:.4f}".format(datetime.now(), step, batch_loss, acc) # print >> sys.stderr, "diff: {} ".format(diff) if step % 100 == 0: # for iround in range print 'A short Test' feature1 = sess.run(feat1, feed_dict={x1:test_color_imgs[:batch_size], keep_var: 1.}) feature2 = sess.run(feat2, feed_dict={x2:test_thermal_imgs[:batch_size], keep_var: 1.}) feature = sess.run(feat, feed_dict={feat1:feature1, feat2:feature2, keep_var: 1.}) test_color_feature, test_thermal_feature = tf.split(0, 2, feature) # test_feat = sess.run(feat, feed_dict={feat1: test_color_feature, feat2:test_thermal_feature keep_var: 1.}) query_t_norm = tf.nn.l2_normalize(test_color_feature, dim=1) test_t_norm = tf.nn.l2_normalize(test_thermal_feature, dim=1) distmat = tf.matmul(query_t_norm, test_t_norm, transpose_a=False, transpose_b=True) cmc, mAP = compute_accuracy(-distmat, test_color_labels[:batch_size], test_thermal_labels[:batch_size],topk=20) print('top-1: {:.2%} | top-5: {:.2%} | top-10: {:.2%}| top-20: {:.2%}'.format(cmc[0], cmc[4], cmc[9], cmc[19])) print('mAP: {:.2%}'.format(mAP)) # Save the model checkpoint periodically. if step % 2500 == 0 or step == training_iters: checkpoint_name = os.path.join(checkpoint_path, 'cam_bdtr_trial_1_'+ str(step) +'.ckpt') save_path = saver.save(sess, checkpoint_name) step += 1 print "Finish!"