def _get_restore_vars_dict(self):
"""Load pre-trained variables."""
utils.thick_line()
print('Loading pre-trained variables from:\n',
self.restore_checkpoint_path)
utils.thin_line()
tf.reset_default_graph()
loaded_graph = tf.Graph()
with tf.Session(graph=loaded_graph) as sess:
ckp_path = tf.train.latest_checkpoint(self.restore_checkpoint_path)
loader = tf.train.import_meta_graph(ckp_path + '.meta')
loader.restore(sess, ckp_path)
restore_vars_dict = dict()
restore_vars_dict['w_conv_0'] = sess.run(
loaded_graph.get_tensor_by_name('classifier/conv_0/weights:0'))
restore_vars_dict['b_conv_0'] = sess.run(
loaded_graph.get_tensor_by_name('classifier/conv_0/biases:0'))
restore_vars_dict['w_caps_0'] = sess.run(
loaded_graph.get_tensor_by_name('classifier/caps_0/weights:0'))
restore_vars_dict['b_caps_0'] = sess.run(
loaded_graph.get_tensor_by_name('classifier/caps_0/biases:0'))
restore_vars_dict['w_caps_1'] = sess.run(
loaded_graph.get_tensor_by_name('classifier/caps_1/weights:0'))
# restore_vars_dict['b_caps_1'] = sess.run(
# loaded_graph.get_tensor_by_name('classifier/caps_1/biases:0'))
return restore_vars_dict
def _save_model(self, sess, saver, step, silent=False):
"""Save models."""
save_path = join(self.checkpoint_path, 'models.ckpt')
if not silent:
utils.thin_line()
print('Saving models to {}...'.format(save_path))
saver.save(sess, save_path, global_step=step)
def _get_preds_int(self, preds_vec):
"""Get integer predictions."""
utils.thin_line()
print('Converting prediction vectors to ints...')
preds = np.argmax(np.array(preds_vec), axis=1)
# Save preds
if self.cfg.SAVE_TEST_PRED:
if self.during_training and (self.epoch_train != 'end'):
utils.save_test_pred_is_training(self.test_log_path,
self.epoch_train,
self.step_train,
self.y_test,
preds,
preds_vec,
save_num=20,
pred_is_int=True)
else:
utils.save_test_pred(self.test_log_path,
self.y_test,
preds,
preds_vec,
pred_is_int=True)
return preds
def _get_tensors(self, loaded_graph):
"""Get inputs, labels, loss, and accuracy tensor from <loaded_graph>."""
with loaded_graph.as_default():
utils.thin_line()
print('Loading graph and tensors...')
inputs_ = loaded_graph.get_tensor_by_name('inputs:0')
labels_ = loaded_graph.get_tensor_by_name('labels:0')
input_imgs_ = loaded_graph.get_tensor_by_name('input_imgs:0')
is_training = loaded_graph.get_tensor_by_name('is_training:0')
if self.multi_gpu:
clf_preds_ = loaded_graph.get_tensor_by_name(
'total_clf_preds:0')
if self.cfg.TEST_WITH_REC:
rec_imgs_ = loaded_graph.get_tensor_by_name(
'total_rec_imgs:0')
return inputs_, labels_, input_imgs_, \
is_training, clf_preds_, rec_imgs_
else:
return inputs_, labels_, input_imgs_, is_training, clf_preds_
else:
clf_preds_ = loaded_graph.get_tensor_by_name('clf_preds:0')
if self.cfg.TEST_WITH_REC:
rec_imgs_ = loaded_graph.get_tensor_by_name('rec_imgs:0')
return inputs_, labels_, input_imgs_, \
is_training, clf_preds_, rec_imgs_
else:
return inputs_, labels_, input_imgs_, is_training, clf_preds_
def _scaling(self):
"""
Scaling input images to (0, 1).
"""
utils.thin_line()
print('Scaling features...')
self.x = np.divide(self.x, 255.)
self.x_test = np.divide(self.x_test, 255.)
def _shuffle(self):
"""
Shuffle data sets.
"""
utils.thin_line()
print('Shuffling images and labels...')
self.x, self.y = shuffle(
self.x, self.y, random_state=0)
self.x_test, self.y_test = shuffle(
self.x_test, self.y_test, random_state=0)
def download_data(data_base_name):
"""
Download database.
"""
utils.thick_line()
print('Downloading {} data set...'.format(data_base_name))
utils.thin_line()
if data_base_name == 'mnist':
SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/'
TRAIN_IMAGES = 'train-images-idx3-ubyte.gz'
TRAIN_LABELS = 'train-labels-idx1-ubyte.gz'
TEST_IMAGES = 't10k-images-idx3-ubyte.gz'
TEST_LABELS = 't10k-labels-idx1-ubyte.gz'
source_data_path_ = join(cfg.SOURCE_DATA_PATH, 'mnist')
utils.check_dir([source_data_path_])
utils.download_and_extract_mnist(
url=SOURCE_URL + TRAIN_IMAGES,
save_path=join(source_data_path_, TRAIN_IMAGES),
extract_path=join(source_data_path_, 'train_images'),
data_type='images')
utils.download_and_extract_mnist(
url=SOURCE_URL + TRAIN_LABELS,
save_path=join(source_data_path_, TRAIN_LABELS),
extract_path=join(source_data_path_, 'train_labels'),
data_type='labels')
utils.download_and_extract_mnist(
url=SOURCE_URL + TEST_IMAGES,
save_path=join(source_data_path_, TEST_IMAGES),
extract_path=join(source_data_path_, 'test_images'),
data_type='images')
utils.download_and_extract_mnist(
url=SOURCE_URL + TEST_LABELS,
save_path=join(source_data_path_, TEST_LABELS),
extract_path=join(source_data_path_, 'test_labels'),
data_type='labels')
elif data_base_name == 'cifar10':
SOURCE_URL = 'https://www.cs.toronto.edu/~kriz/'
FILE_NAME = 'cifar-10-python.tar.gz'
utils.check_dir([cfg.SOURCE_DATA_PATH])
utils.download_and_extract_cifar10(url=SOURCE_URL + FILE_NAME,
save_path=cfg.SOURCE_DATA_PATH,
file_name=FILE_NAME,
extract_path=cfg.SOURCE_DATA_PATH)
else:
raise ValueError('Wrong database name!')
utils.thick_line()
def _one_hot_encoding(self):
"""
Scaling images to (0, 1).
"""
utils.thin_line()
print('One-hot-encoding labels...')
encoder = LabelBinarizer()
encoder.fit(self.y)
self.y = encoder.transform(self.y)
self.y_test = encoder.transform(self.y_test)
def _load_data(self):
"""
Load data set from files.
"""
utils.thin_line()
print('Loading {} data set...'.format(self.data_base_name))
self.x = utils.load_data_from_pkl(
join(self.source_data_path, 'train_images.p'))
self.y = utils.load_data_from_pkl(
join(self.source_data_path, 'train_labels.p'))
self.x_test = utils.load_data_from_pkl(
join(self.source_data_path, 'test_images.p'))
self.y_test = utils.load_data_from_pkl(
join(self.source_data_path, 'test_labels.p'))
def _eval_on_full_set(self, sess, epoch_i, step, silent=False):
"""
Evaluate on the full data set and print information.
"""
eval_start_time = time.time()
if not silent:
utils.thick_line()
print('Calculating losses using full data set...')
# Calculate losses and accuracies of full train set
if self.cfg.EVAL_WITH_FULL_TRAIN_SET:
loss_train, clf_loss_train, rec_loss_train, acc_train = \
self._eval_on_batches('train', sess, self.x_train, self.y_train,
self.n_batch_train, silent=silent)
else:
loss_train, clf_loss_train, rec_loss_train, acc_train = \
None, None, None, None
# Calculate losses and accuracies of full valid set
loss_valid, clf_loss_valid, rec_loss_valid, acc_valid = \
self._eval_on_batches('valid', sess, self.x_valid, self.y_valid,
self.n_batch_valid, silent=silent)
if not silent:
utils.print_full_set_eval(epoch_i, self.cfg.EPOCHS, step,
self.start_time, loss_train,
clf_loss_train, rec_loss_train,
acc_train, loss_valid, clf_loss_valid,
rec_loss_valid, acc_valid,
self.cfg.EVAL_WITH_FULL_TRAIN_SET,
self.cfg.WITH_RECONSTRUCTION)
file_path = join(self.train_log_path, 'full_set_eval_log.csv')
if not silent:
utils.thin_line()
print('Saving {}...'.format(file_path))
utils.save_log(file_path, epoch_i + 1, step,
time.time() - self.start_time, loss_train,
clf_loss_train, rec_loss_train, acc_train, loss_valid,
clf_loss_valid, rec_loss_valid, acc_valid,
self.cfg.WITH_RECONSTRUCTION)
if not silent:
utils.thin_line()
print(
'Evaluation done! Using time: {:.2f}'.format(time.time() -
eval_start_time))
def _get_preds_binary(self, preds_vec):
"""Get binary predictions.
-> [0, 0, 1, ..., 0, 1, 0] as labels
"""
utils.thin_line()
print('Converting prediction vectors to binaries...')
preds = np.array(preds_vec)
if self.cfg.MOD_PRED_MODE == 'top_n':
for pred_i in preds:
pos_idx = np.argsort(pred_i)[-self.cfg.MOD_PRED_MAX_NUM:]
neg_idx = np.argsort(pred_i)[:-self.cfg.MOD_PRED_MAX_NUM]
pred_i[pos_idx] = 1
pred_i[neg_idx] = 0
elif self.cfg.MOD_PRED_MODE == 'length_rate':
for pred_i in preds:
pred_i_copy = pred_i.copy()
max_ = pred_i.max()
pred_i[pred_i < (max_ * self.cfg.MOD_PRED_THRESHOLD)] = 0
pred_i[pred_i >= (max_ * self.cfg.MOD_PRED_THRESHOLD)] = 1
if np.sum(pred_i) > self.cfg.MOD_PRED_MAX_NUM:
pos_idx = np.argsort(
pred_i_copy)[-self.cfg.MOD_PRED_MAX_NUM:]
neg_idx = np.argsort(
pred_i_copy)[:-self.cfg.MOD_PRED_MAX_NUM]
pred_i[pos_idx] = 1
pred_i[neg_idx] = 0
else:
raise ValueError('Wrong Mode Name! Find {}!'.format(
self.cfg.MOD_PRED_MODE))
if self.cfg.SAVE_TEST_PRED:
if self.during_training and (self.epoch_train != 'end'):
utils.save_test_pred_is_training(self.test_log_path,
self.epoch_train,
self.step_train,
self.y_test,
preds,
preds_vec,
save_num=20)
else:
utils.save_test_pred(self.test_log_path, self.y_test, preds,
preds_vec)
return np.array(preds, dtype=int)
def __init__(self, cfg):
# Config
self.cfg = cfg
# Get checkpoint path
self.checkpoint_path = join(
cfg.CHECKPOINT_PATH,
'{}/models.ckpt-{}'.format(self.cfg.TEST_VERSION,
self.cfg.TEST_CKP_IDX))
# Get log path, append information if the directory exist.
test_log_path_ = join(
self.cfg.TEST_LOG_PATH, '{}-{}'.format(self.cfg.TEST_VERSION,
self.cfg.TEST_CKP_IDX))
self.test_log_path = test_log_path_
i_append_info = 0
while isdir(self.test_log_path):
i_append_info += 1
self.test_log_path = test_log_path_ + '({})'.format(i_append_info)
# Path for saving images
self.test_image_path = join(self.test_log_path, 'images')
# Check directory of paths
utils.check_dir([self.test_log_path])
if self.cfg.TEST_WITH_RECONSTRUCTION:
if self.cfg.TEST_SAVE_IMAGE_STEP is not None:
utils.check_dir([self.test_image_path])
# Save config
utils.save_config_log(self.test_log_path, self.cfg)
# Load data
utils.thick_line()
print('Loading data...')
utils.thin_line()
preprocessed_path_ = join(cfg.DPP_DATA_PATH, cfg.DATABASE_NAME)
self.x_test = utils.load_data_from_pkl(
join(preprocessed_path_, 'x_test.p'))
self.y_test = utils.load_data_from_pkl(
join(preprocessed_path_, 'y_test.p'))
# Calculate number of batches
self.n_batch_test = len(self.y_test) // self.cfg.TEST_BATCH_SIZE
def pipeline(self, data_base_name):
"""
Pipeline of preprocessing data.
Arg:
data_base_name: name of data base
"""
utils.thick_line()
print('Start Preprocessing...')
start_time = time.time()
self.data_base_name = data_base_name
self.preprocessed_path = join(self.cfg.DPP_DATA_PATH, data_base_name)
self.source_data_path = join(self.cfg.SOURCE_DATA_PATH, data_base_name)
# Load data
self._load_data()
# Augment data
self._augment_data()
# Shuffle data set
# self._shuffle()
# Scaling images to (0, 1)
self._scaling()
# One-hot-encoding labels
self._one_hot_encoding()
# Split data set into train/valid/test
self._split_data()
# Check data format.
self._check_data()
# Save data to pickles
self._save_data()
utils.thin_line()
print('Done! Using {:.3}s'.format(time.time() - start_time))
utils.thick_line()
def _save_data(self):
"""
Save data set to pickle files.
"""
utils.thin_line()
print('Saving pickle files...')
utils.check_dir([self.preprocessed_path])
utils.save_data_to_pkl(
self.x_train, join(self.preprocessed_path, 'x_train.p'))
utils.save_data_to_pkl(
self.y_train, join(self.preprocessed_path, 'y_train.p'))
utils.save_data_to_pkl(
self.x_valid, join(self.preprocessed_path, 'x_valid.p'))
utils.save_data_to_pkl(
self.y_valid, join(self.preprocessed_path, 'y_valid.p'))
utils.save_data_to_pkl(
self.x_test, join(self.preprocessed_path, 'x_test.p'))
utils.save_data_to_pkl(
self.y_test, join(self.preprocessed_path, 'y_test.p'))
def _load_data(self):
utils.thick_line()
print('Loading data...')
utils.thin_line()
if self.cfg.DATABASE_MODE is not None:
preprocessed_path_ = join(
'../data/{}'.format(self.cfg.DATABASE_MODE),
self.cfg.DATABASE_NAME)
else:
preprocessed_path_ = join(self.cfg.DPP_DATA_PATH,
self.cfg.DATABASE_NAME)
x = utils.load_pkls(preprocessed_path_,
'x_test' + self.append_info,
tl=self.tl_encode,
add_n_batch=1)
y = utils.load_pkls(preprocessed_path_, 'y_test' + self.append_info)
imgs = utils.load_pkls(preprocessed_path_,
'imgs_test' + self.append_info)
utils.thin_line()
print('Data info:')
utils.thin_line()
print('x_test: {}\ny_test: {}\nimgs_test: {}'.format(
x.shape, y.shape, imgs.shape))
return x, y, imgs
def _get_preds_vector(self, sess, inputs, preds, is_training):
"""Get prediction vectors of full train set."""
utils.thin_line()
print('Getting prediction vectors...')
pred_all = []
_batch_generator = utils.get_batches(
self.x_test, batch_size=self.cfg.TEST_BATCH_SIZE, keep_last=True)
if len(self.x_test) % self.cfg.TEST_BATCH_SIZE == 0:
n_batch = (len(self.x_test) // self.cfg.TEST_BATCH_SIZE)
else:
n_batch = (len(self.x_test) // self.cfg.TEST_BATCH_SIZE) + 1
for _ in tqdm(range(n_batch), total=n_batch, ncols=100, unit=' batch'):
x_batch = next(_batch_generator)
# The last batch which has less examples
len_batch = len(x_batch)
if len_batch != self.cfg.TEST_BATCH_SIZE:
for i in range(self.cfg.TEST_BATCH_SIZE - len_batch):
x_batch = np.append(x_batch,
np.expand_dims(np.zeros_like(
x_batch[0]),
axis=0),
axis=0)
assert len(x_batch) == self.cfg.TEST_BATCH_SIZE
pred_i = sess.run(preds,
feed_dict={
inputs: x_batch,
is_training: False
})
if len_batch != self.cfg.TEST_BATCH_SIZE:
pred_i = pred_i[:len_batch]
pred_all.extend(list(pred_i))
assert len(pred_all) == len(self.x_test), (len(pred_all),
len(self.x_test))
return np.array(pred_all)
def _get_tensors(self, loaded_graph):
"""
Get inputs, labels, loss, and accuracy tensor from <loaded_graph>
"""
with loaded_graph.as_default():
utils.thin_line()
print('Loading graph and tensors...')
inputs_ = loaded_graph.get_tensor_by_name("inputs:0")
labels_ = loaded_graph.get_tensor_by_name("labels:0")
loss_ = loaded_graph.get_tensor_by_name("loss:0")
accuracy_ = loaded_graph.get_tensor_by_name("accuracy:0")
if self.cfg.TEST_WITH_RECONSTRUCTION:
clf_loss_ = loaded_graph.get_tensor_by_name(
"classifier_loss:0")
rec_loss_ = loaded_graph.get_tensor_by_name("rec_loss:0")
rec_images_ = loaded_graph.get_tensor_by_name("rec_images:0")
return inputs_, labels_, loss_, accuracy_, \
clf_loss_, rec_loss_, rec_images_
else:
return inputs_, labels_, loss_, accuracy_
def _load_data(self):
"""Load preprocessed data."""
utils.thick_line()
print('Loading data...')
x_train = utils.load_pkls(self.preprocessed_path,
'x_train',
tl=self.tl_encode)
x_valid = utils.load_pkls(self.preprocessed_path,
'x_valid',
tl=self.tl_encode,
add_n_batch=1)
imgs_train = utils.load_pkls(self.preprocessed_path, 'imgs_train')
imgs_valid = utils.load_pkls(self.preprocessed_path, 'imgs_valid')
if imgs_train.shape == x_train.shape:
print('[W] imgs_train.shape == x_train.shape')
del imgs_train
del imgs_valid
gc.collect()
imgs_train = x_train
imgs_valid = x_valid
y_train = utils.load_pkls(self.preprocessed_path, 'y_train')
y_valid = utils.load_pkls(self.preprocessed_path, 'y_valid')
utils.thin_line()
print('Data info:')
utils.thin_line()
print('x_train: {}\ny_train: {}\nx_valid: {}\ny_valid: {}'.format(
x_train.shape, y_train.shape, x_valid.shape, y_valid.shape))
print('imgs_train: {}\nimgs_valid: {}'.format(imgs_train.shape,
imgs_valid.shape))
return x_train, y_train, imgs_train, x_valid, y_valid, imgs_valid
def tester(self,
sess,
inputs,
labels,
input_imgs,
is_training,
clf_preds,
rec_imgs,
start_time,
loss=None,
acc=None,
clf_loss=None,
rec_loss=None):
utils.thin_line()
print('Calculating loss and accuracy of test set...')
# Get losses and accuracies
clf_preds_vec_test = self._get_preds_vector(sess, inputs, clf_preds,
is_training)
# Get binary predictions
clf_preds_binary = self._get_preds_binary(preds_vec=clf_preds_vec_test)
# Get evaluation scores for multi-objects detection.
self._get_multi_obj_scores(clf_preds_binary, clf_preds_vec_test)
# Save reconstruction images of multi-objects detection
if self.cfg.TEST_WITH_REC:
self._save_images_mo(sess, rec_imgs, inputs, labels, is_training,
clf_preds_binary, clf_preds_vec_test)
utils.thin_line()
print('Testing finished! Using time: {:.2f}'.format(time.time() -
start_time))
utils.thick_line()
def _split_data(self):
"""
Split data set for training, validation and testing.
"""
utils.thin_line()
print('Splitting train/valid/test set...')
if self.data_base_name == 'mnist':
train_stop = 55000
elif self.data_base_name == 'cifar10':
train_stop = 45000
else:
raise ValueError('Wrong database name!')
if self.cfg.DPP_TEST_AS_VALID:
self.x_train = self.x
self.y_train = self.y
self.x_valid = self.x_test
self.y_valid = self.y_test
else:
self.x_train = self.x[:train_stop]
self.y_train = self.y[:train_stop]
self.x_valid = self.x[train_stop:]
self.y_valid = self.y[train_stop:]
def test(self):
"""
Test models
"""
start_time = time.time()
tf.reset_default_graph()
loaded_graph = tf.Graph()
with tf.Session(graph=loaded_graph) as sess:
# Load saved models
loader = tf.train.import_meta_graph(self.checkpoint_path + '.meta')
loader.restore(sess, self.checkpoint_path)
# Get Tensors from loaded models
if self.cfg.TEST_WITH_RECONSTRUCTION:
inputs, labels, loss, accuracy, \
clf_loss, rec_loss, rec_images = \
self._get_tensors(loaded_graph)
else:
inputs, labels, loss, accuracy = self._get_tensors(
loaded_graph)
clf_loss, rec_loss, rec_images = None, None, None
utils.thick_line()
print('Testing on test set...')
utils.thin_line()
print('Calculating loss and accuracy of test set...')
loss_test, clf_loss_test, rec_loss_test, acc_test = \
self._eval_on_batches(
sess, inputs, labels, loss, accuracy,
clf_loss, rec_loss, rec_images,
self.x_test, self.y_test, self.n_batch_test)
# Print losses and accuracy
utils.thin_line()
print('Test_Loss: {:.4f}'.format(loss_test))
if self.cfg.TEST_WITH_RECONSTRUCTION:
print('Test_Classifier_Loss: {:.4f}\n'.format(clf_loss_test),
'Test_Reconstruction_Loss: {:.4f}'.format(rec_loss_test))
print('Test_Accuracy: {:.2f}%'.format(acc_test * 100))
# Save test log
utils.save_test_log(self.test_log_path, loss_test, acc_test,
clf_loss_test, rec_loss_test,
self.cfg.TEST_WITH_RECONSTRUCTION)
utils.thin_line()
print('Testing finished! Using time: {:.2f}'.format(time.time() -
start_time))
utils.thick_line()
def _load_bottleneck_features(self):
"""Load preprocessed bottleneck features."""
utils.thick_line()
print('Loading data...')
utils.thin_line()
x_train = utils.load_pkls(self.preprocessed_path, 'x_train')
x_valid = utils.load_pkls(self.preprocessed_path,
'x_valid',
add_n_batch=1)
y_train = utils.load_pkls(self.preprocessed_path, 'y_train')
y_valid = utils.load_pkls(self.preprocessed_path, 'y_valid')
utils.thin_line()
print('Data info:')
utils.thin_line()
print('x_train: {}\ny_train: {}\nx_valid: {}\ny_valid: {}'.format(
x_train.shape, y_train.shape, x_valid.shape, y_valid.shape))
return x_train, y_train, x_valid, y_valid
def _trainer(self, sess):
utils.thick_line()
print('Training...')
# Merge all the summaries and create writers
train_summary_path = join(self.summary_path, 'train')
valid_summary_path = join(self.summary_path, 'valid')
utils.check_dir([train_summary_path, valid_summary_path])
train_writer = tf.summary.FileWriter(train_summary_path, sess.graph)
valid_writer = tf.summary.FileWriter(valid_summary_path)
sess.run(tf.global_variables_initializer())
step = 0
for epoch_i in range(self.cfg.EPOCHS):
epoch_start_time = time.time()
utils.thick_line()
print('Training on epoch: {}/{}'.format(epoch_i + 1,
self.cfg.EPOCHS))
if self.cfg.DISPLAY_STEP is not None:
for x_batch, y_batch in utils.get_batches(
self.x_train, self.y_train, self.cfg.BATCH_SIZE):
step += 1
# Training optimizer
sess.run(self.optimizer,
feed_dict={
self.inputs: x_batch,
self.labels: y_batch,
self.step: step - 1,
self.is_training: True
})
# Display training information
if step % self.cfg.DISPLAY_STEP == 0:
self._display_status(sess, x_batch, y_batch, epoch_i,
step)
# Save training logs
if self.cfg.SAVE_LOG_STEP is not None:
if step % self.cfg.SAVE_LOG_STEP == 0:
self._save_logs(sess, train_writer, valid_writer,
x_batch, y_batch, epoch_i, step)
# Save reconstruction images
if self.cfg.SAVE_IMAGE_STEP is not None:
if self.cfg.WITH_RECONSTRUCTION:
if step % self.cfg.SAVE_IMAGE_STEP == 0:
self._save_images(sess,
self.train_image_path,
x_batch,
y_batch,
step,
epoch_i=epoch_i)
# Save models
if self.cfg.SAVE_MODEL_MODE == 'per_batch':
if step % self.cfg.SAVE_MODEL_STEP == 0:
self._save_model(sess, self.saver, step)
# Evaluate on full set
if self.cfg.FULL_SET_EVAL_MODE == 'per_batch':
if step % self.cfg.FULL_SET_EVAL_STEP == 0:
self._eval_on_full_set(sess, epoch_i, step)
utils.thick_line()
else:
utils.thin_line()
train_batch_generator = utils.get_batches(
self.x_train, self.y_train, self.cfg.BATCH_SIZE)
for _ in tqdm(range(self.n_batch_train),
total=self.n_batch_train,
ncols=100,
unit=' batches'):
step += 1
x_batch, y_batch = next(train_batch_generator)
# Training optimizer
sess.run(self.optimizer,
feed_dict={
self.inputs: x_batch,
self.labels: y_batch,
self.step: step - 1,
self.is_training: True
})
# Save training logs
if self.cfg.SAVE_LOG_STEP is not None:
if step % self.cfg.SAVE_LOG_STEP == 0:
self._save_logs(sess, train_writer, valid_writer,
x_batch, y_batch, epoch_i, step)
# Save reconstruction images
if self.cfg.SAVE_IMAGE_STEP is not None:
if self.cfg.WITH_RECONSTRUCTION:
if step % self.cfg.SAVE_IMAGE_STEP == 0:
self._save_images(sess,
self.train_image_path,
x_batch,
y_batch,
step,
silent=True,
epoch_i=epoch_i)
# Save models
if self.cfg.SAVE_MODEL_MODE == 'per_batch':
if step % self.cfg.SAVE_MODEL_STEP == 0:
self._save_model(sess,
self.saver,
step,
silent=True)
# Evaluate on full set
if self.cfg.FULL_SET_EVAL_MODE == 'per_batch':
if step % self.cfg.FULL_SET_EVAL_STEP == 0:
self._eval_on_full_set(sess,
epoch_i,
step,
silent=True)
if self.cfg.SAVE_MODEL_MODE == 'per_epoch':
if (epoch_i + 1) % self.cfg.SAVE_MODEL_STEP == 0:
self._save_model(sess, self.saver, epoch_i)
if self.cfg.FULL_SET_EVAL_MODE == 'per_epoch':
if (epoch_i + 1) % self.cfg.FULL_SET_EVAL_STEP == 0:
self._eval_on_full_set(sess, epoch_i, step)
utils.thin_line()
print('Epoch done! Using time: {:.2f}'.format(time.time() -
epoch_start_time))
utils.thick_line()
print('Training finished! Using time: {:.2f}'.format(time.time() -
self.start_time))
utils.thick_line()
# Evaluate on test set after training
if self.cfg.TEST_AFTER_TRAINING:
self._test_after_training(sess)
utils.thick_line()
print('All task finished! Total time: {:.2f}'.format(time.time() -
self.start_time))
utils.thick_line()
def _test_after_training(self, sess):
"""
Evaluate on the test set after training.
"""
test_start_time = time.time()
utils.thick_line()
print('Testing...')
# Check directory of paths
utils.check_dir([self.test_log_path])
if self.cfg.WITH_RECONSTRUCTION:
if self.cfg.TEST_SAVE_IMAGE_STEP is not None:
utils.check_dir([self.test_image_path])
# Load data
utils.thin_line()
print('Loading test set...')
utils.thin_line()
x_test = utils.load_data_from_pkl(
join(self.preprocessed_path, 'x_test.p'))
y_test = utils.load_data_from_pkl(
join(self.preprocessed_path, 'y_test.p'))
n_batch_test = len(y_test) // self.cfg.BATCH_SIZE
utils.thin_line()
print('Calculating loss and accuracy on test set...')
loss_test_all = []
acc_test_all = []
clf_loss_test_all = []
rec_loss_test_all = []
step = 0
_test_batch_generator = utils.get_batches(x_test, y_test,
self.cfg.BATCH_SIZE)
if self.cfg.WITH_RECONSTRUCTION:
for _ in tqdm(range(n_batch_test),
total=n_batch_test,
ncols=100,
unit=' batches'):
step += 1
test_batch_x, test_batch_y = next(_test_batch_generator)
loss_test_i, clf_loss_i, rec_loss_i, acc_test_i = sess.run(
[self.loss, self.clf_loss, self.rec_loss, self.accuracy],
feed_dict={
self.inputs: test_batch_x,
self.labels: test_batch_y,
self.is_training: False
})
loss_test_all.append(loss_test_i)
acc_test_all.append(acc_test_i)
clf_loss_test_all.append(clf_loss_i)
rec_loss_test_all.append(rec_loss_i)
# Save reconstruct images
if self.cfg.TEST_SAVE_IMAGE_STEP is not None:
if step % self.cfg.TEST_SAVE_IMAGE_STEP == 0:
self._save_images(sess,
self.test_image_path,
test_batch_x,
test_batch_y,
step,
silent=False)
clf_loss_test = sum(clf_loss_test_all) / len(clf_loss_test_all)
rec_loss_test = sum(rec_loss_test_all) / len(rec_loss_test_all)
else:
for _ in tqdm(range(n_batch_test),
total=n_batch_test,
ncols=100,
unit=' batches'):
test_batch_x, test_batch_y = next(_test_batch_generator)
loss_test_i, acc_test_i = sess.run(
[self.loss, self.accuracy],
feed_dict={
self.inputs: test_batch_x,
self.labels: test_batch_y,
self.is_training: False
})
loss_test_all.append(loss_test_i)
acc_test_all.append(acc_test_i)
clf_loss_test, rec_loss_test = None, None
loss_test = sum(loss_test_all) / len(loss_test_all)
acc_test = sum(acc_test_all) / len(acc_test_all)
# Print losses and accuracy
utils.thin_line()
print('Test_Loss: {:.4f}\n'.format(loss_test),
'Test_Accuracy: {:.2f}%'.format(acc_test * 100))
if self.cfg.WITH_RECONSTRUCTION:
utils.thin_line()
print('Test_Train_Loss: {:.4f}\n'.format(clf_loss_test),
'Test_Reconstruction_Loss: {:.4f}'.format(rec_loss_test))
# Save test log
utils.save_test_log(self.test_log_path, loss_test, acc_test,
clf_loss_test, rec_loss_test,
self.cfg.WITH_RECONSTRUCTION)
utils.thin_line()
print('Testing finished! Using time: {:.2f}'.format(time.time() -
test_start_time))
def _save_images(self,
sess,
img_path,
x_batch,
y_batch,
step,
silent=False,
epoch_i=None):
"""
Save reconstructed images.
"""
rec_images_ = sess.run(self.rec_images,
feed_dict={
self.inputs: x_batch,
self.labels: y_batch,
self.is_training: False
})
# Image shape
img_shape = x_batch.shape[1:]
# Get maximum size for square grid of images
save_col_size = math.floor(np.sqrt(rec_images_.shape[0] * 2))
if save_col_size > self.cfg.MAX_IMAGE_IN_COL:
save_col_size = self.cfg.MAX_IMAGE_IN_COL
save_row_size = save_col_size // 2
# Scale to 0-255
rec_images_ = np.array([
np.divide(((img_ - img_.min()) * 255), (img_.max() - img_.min()))
for img_ in rec_images_
])
real_images_ = np.array([
np.divide(((img_ - img_.min()) * 255), (img_.max() - img_.min()))
for img_ in x_batch
])
# Put images in a square arrangement
rec_images_in_square = np.reshape(
rec_images_[:save_row_size * save_col_size],
(save_row_size, save_col_size, *img_shape)).astype(np.uint8)
real_images_in_square = np.reshape(
real_images_[:save_row_size * save_col_size],
(save_row_size, save_col_size, *img_shape)).astype(np.uint8)
if self.cfg.DATABASE_NAME == 'mnist':
mode = 'L'
rec_images_in_square = np.squeeze(rec_images_in_square, 4)
real_images_in_square = np.squeeze(real_images_in_square, 4)
else:
mode = 'RGB'
# Combine images to grid image
thin_gap = 1
thick_gap = 3
avg_gap = (thin_gap + thick_gap) / 2
new_im = Image.new(
mode,
(int((img_shape[1] + thin_gap) * save_col_size - thin_gap +
thick_gap * 2),
int((img_shape[0] + avg_gap) * save_row_size * 2 + thick_gap)),
'white')
for row_i in range(save_row_size * 2):
for col_i in range(save_col_size):
if (row_i + 1) % 2 == 0: # Odd
if mode == 'L':
image = rec_images_in_square[(row_i + 1) // 2 - 1,
col_i, :, :]
else:
image = rec_images_in_square[(row_i + 1) // 2 - 1,
col_i, :, :, :]
im = Image.fromarray(image, mode)
new_im.paste(
im,
(int(col_i * (img_shape[1] + thin_gap) + thick_gap),
int(row_i * img_shape[0] + (row_i + 1) * avg_gap)))
else: # Even
if mode == 'L':
image = real_images_in_square[int((row_i + 1) // 2),
col_i, :, :]
else:
image = real_images_in_square[int((row_i + 1) // 2),
col_i, :, :, :]
im = Image.fromarray(image, mode)
new_im.paste(
im,
(int(col_i * (img_shape[1] + thin_gap) + thick_gap),
int(row_i * (img_shape[0] + avg_gap) + thick_gap)))
if epoch_i is None:
save_image_path = join(img_path, 'batch_{}.jpg'.format(step))
else:
save_image_path = join(
img_path, 'epoch_{}_batch_{}.jpg'.format(epoch_i, step))
if not silent:
utils.thin_line()
print('Saving image to {}...'.format(save_image_path))
new_im.save(save_image_path)
def __init__(self, model, cfg):
"""
Load data and initialize models.
Args:
model: the models which will be trained
"""
# Global start time
self.start_time = time.time()
# Config
self.cfg = cfg
# Get paths from configuration
train_log_path_ = join(cfg.TRAIN_LOG_PATH, cfg.VERSION)
test_log_path_ = join(cfg.TEST_LOG_PATH, cfg.VERSION)
summary_path_ = join(cfg.SUMMARY_PATH, cfg.VERSION)
checkpoint_path_ = join(cfg.CHECKPOINT_PATH, cfg.VERSION)
self.preprocessed_path = join(cfg.DPP_DATA_PATH, cfg.DATABASE_NAME)
# Get log paths, append information if the directory exist.
self.train_log_path = train_log_path_
i_append_info = 0
while isdir(self.train_log_path):
i_append_info += 1
self.train_log_path = train_log_path_ + '({})'.format(
i_append_info)
if i_append_info > 0:
self.summary_path = summary_path_ + '({})'.format(i_append_info)
self.checkpoint_path = checkpoint_path_ + '({})'.format(
i_append_info)
self.test_log_path = test_log_path_ + '({})'.format(i_append_info)
else:
self.summary_path = summary_path_
self.checkpoint_path = checkpoint_path_
self.test_log_path = test_log_path_
# Images saving path
self.train_image_path = join(self.train_log_path, 'images')
self.test_image_path = join(self.test_log_path, 'images')
# Check directory of paths
utils.check_dir([self.train_log_path, self.checkpoint_path])
if cfg.WITH_RECONSTRUCTION:
if cfg.SAVE_IMAGE_STEP is not None:
utils.check_dir([self.train_image_path])
# Load data
utils.thick_line()
print('Loading data...')
utils.thin_line()
self.x_train = utils.load_data_from_pkl(
join(self.preprocessed_path, 'x_train.p'))
self.y_train = utils.load_data_from_pkl(
join(self.preprocessed_path, 'y_train.p'))
self.x_valid = utils.load_data_from_pkl(
join(self.preprocessed_path, 'x_valid.p'))
self.y_valid = utils.load_data_from_pkl(
join(self.preprocessed_path, 'y_valid.p'))
# Calculate number of batches
self.n_batch_train = len(self.y_train) // cfg.BATCH_SIZE
self.n_batch_valid = len(self.y_valid) // cfg.BATCH_SIZE
# Build graph
utils.thick_line()
print('Building graph...')
tf.reset_default_graph()
self.step, self.train_graph, self.inputs, self.labels, self.is_training, \
self.optimizer, self.saver, self.summary, self.loss, self.accuracy,\
self.clf_loss, self.rec_loss, self.rec_images = model.build_graph(
image_size=self.x_train.shape[1:],
num_class=self.y_train.shape[1])
# Save config
utils.save_config_log(self.train_log_path, cfg, model.clf_arch_info,
model.rec_arch_info)
def _eval_on_batches(self, mode, sess, x, y, n_batch, silent=False):
"""
Calculate losses and accuracies of full train set.
"""
loss_all = []
acc_all = []
clf_loss_all = []
rec_loss_all = []
if not silent:
utils.thin_line()
print(
'Calculating loss and accuracy of full {} set...'.format(mode))
_batch_generator = utils.get_batches(x, y, self.cfg.BATCH_SIZE)
if self.cfg.WITH_RECONSTRUCTION:
for _ in tqdm(range(n_batch),
total=n_batch,
ncols=100,
unit=' batches'):
x_batch, y_batch = next(_batch_generator)
loss_i, clf_loss_i, rec_loss_i, acc_i = sess.run(
[
self.loss, self.clf_loss, self.rec_loss,
self.accuracy
],
feed_dict={
self.inputs: x_batch,
self.labels: y_batch,
self.is_training: False
})
loss_all.append(loss_i)
clf_loss_all.append(clf_loss_i)
rec_loss_all.append(rec_loss_i)
acc_all.append(acc_i)
clf_loss = sum(clf_loss_all) / len(clf_loss_all)
rec_loss = sum(rec_loss_all) / len(rec_loss_all)
else:
for _ in tqdm(range(n_batch),
total=n_batch,
ncols=100,
unit=' batches'):
x_batch, y_batch = next(_batch_generator)
loss_i, acc_i = sess.run(
[self.loss, self.accuracy],
feed_dict={
self.inputs: x_batch,
self.labels: y_batch,
self.is_training: False
})
loss_all.append(loss_i)
acc_all.append(acc_i)
clf_loss, rec_loss = None, None
else:
if self.cfg.WITH_RECONSTRUCTION:
for x_batch, y_batch in utils.get_batches(
x, y, self.cfg.BATCH_SIZE):
loss_i, clf_loss_i, rec_loss_i, acc_i = sess.run(
[
self.loss, self.clf_loss, self.rec_loss,
self.accuracy
],
feed_dict={
self.inputs: x_batch,
self.labels: y_batch,
self.is_training: False
})
loss_all.append(loss_i)
clf_loss_all.append(clf_loss_i)
rec_loss_all.append(rec_loss_i)
acc_all.append(acc_i)
clf_loss = sum(clf_loss_all) / len(clf_loss_all)
rec_loss = sum(rec_loss_all) / len(rec_loss_all)
else:
for x_batch, y_batch in utils.get_batches(
x, y, self.cfg.BATCH_SIZE):
loss_i, acc_i = sess.run(
[self.loss, self.accuracy],
feed_dict={
self.inputs: x_batch,
self.labels: y_batch,
self.is_training: False
})
loss_all.append(loss_i)
acc_all.append(acc_i)
clf_loss, rec_loss = None, None
loss = sum(loss_all) / len(loss_all)
accuracy = sum(acc_all) / len(acc_all)
return loss, clf_loss, rec_loss, accuracy
self._trainer(sess)
else:
with tf.Session(graph=self.train_graph,
config=session_cfg) as sess:
self._trainer(sess)
if __name__ == '__main__':
opts, args = getopt.getopt(sys.argv[1:], "g", ['gpu-id'])
for op, value in opts:
if op == "-g":
print('Using /gpu: %d' % int(value))
environ["CUDA_VISIBLE_DEVICES"] = str(int(value))
utils.thick_line()
print('Input [ 1 ] to run normal version.')
print('Input [ 2 ] to run multi-gpu version.')
utils.thin_line()
input_ = input('Input: ')
if input_ == '1':
CapsNet_ = CapsNet(config)
elif input_ == '2':
CapsNet_ = CapsNetDistribute(config)
else:
raise ValueError('Wrong input! Found: ', input_)
Main_ = Main(CapsNet_, config)
Main_.train()
def _save_images_mo(self, sess, rec_imgs, inputs, labels, is_training,
preds_binary, preds_vector):
"""Save reconstructed images."""
utils.thin_line()
print('Getting reconstruction images...')
if len(self.y_test) > self.cfg.MAX_IMAGE_IN_COL**2:
n_test_img = self.cfg.MAX_IMAGE_IN_COL**2
test_img_idx = np.random.choice(len(self.y_test), n_test_img)
else:
test_img_idx = list(range(len(self.y_test)))
rec_imgs_ = []
preds_vec_ = []
if self.cfg.LABEL_FOR_TEST == 'pred':
label_for_img = preds_binary
elif self.cfg.LABEL_FOR_TEST == 'real':
label_for_img = self.y_test
else:
raise ValueError('Wrong LABEL_FOR_TEST Name!')
for x, y_hat, pred_ in tqdm(zip(self.x_test[test_img_idx],
label_for_img[test_img_idx],
preds_vector[test_img_idx]),
total=len(test_img_idx),
ncols=100,
unit=' image'):
# Get new x and y_hat list in which each y contain single object
# [0, 1, 0, 1, 0] -> [[0, 1, 0, 0, 0],
# [0, 0, 0, 1, 0]]
x_new = []
y_hat_new = []
preds_vec_new = []
for i, y_i in enumerate(y_hat):
if y_i == 1:
y_hat_new_i = np.zeros_like(y_hat)
y_hat_new_i[i] = 1
assert y_hat_new_i[i] == y_hat[i]
x_new.append(x)
y_hat_new.append(y_hat_new_i)
preds_vec_new.append(pred_[i])
preds_vec_.append(preds_vec_new)
# Filling x and y tensor to batch size for testing
# [[0, 1, 0, 0, 0],
# [0, 0, 0, 1, 0]] -> [[0, 1, 0, 0, 0],
# [0, 0, 0, 1, 0],
# ...
# [0, 0, 0, 0, 0]]
n_y = len(y_hat_new)
assert n_y == int(np.sum(y_hat))
if n_y > self.cfg.TEST_BATCH_SIZE:
raise ValueError(
'TEST_BATCH_SIZE Must Not Less Than {}!'.format(n_y))
if n_y < self.cfg.TEST_BATCH_SIZE:
for i in range(self.cfg.TEST_BATCH_SIZE - n_y):
x_new = np.append(x_new,
np.expand_dims(np.zeros_like(x), axis=0),
axis=0)
y_hat_new.append(np.zeros_like(y_hat))
assert len(x_new) == self.cfg.TEST_BATCH_SIZE
assert len(y_hat_new) == self.cfg.TEST_BATCH_SIZE
# Get remake images which contain different objects
# y_rec_imgs_ shape: [128, 28, 28, 1] for mnist
y_rec_imgs_ = sess.run(rec_imgs,
feed_dict={
inputs: x_new,
labels: y_hat_new,
is_training: False
})
rec_imgs_.append(y_rec_imgs_[:n_y])
# Get colorful overlapped images
real_imgs_ = utils.img_black_to_color(self.imgs_test[test_img_idx],
same=True)
rec_imgs_overlap = []
rec_imgs_no_overlap = []
for idx, imgs in enumerate(rec_imgs_):
imgs_colored = utils.img_black_to_color(imgs)
imgs_overlap = utils.img_add_overlap(
imgs=imgs_colored,
merge=True,
vec=preds_vec_[idx],
# vec=None,
gamma=0)
imgs_no_overlap = utils.img_add_no_overlap(
imgs=imgs_colored,
num_mul_obj=self.cfg.NUM_MULTI_OBJECT,
vec=preds_vec_[idx],
img_mode='RGB',
resize_filter=Image.ANTIALIAS)
rec_imgs_overlap.append(imgs_overlap)
rec_imgs_no_overlap.append(imgs_no_overlap)
rec_imgs_overlap = np.array(rec_imgs_overlap)
rec_imgs_no_overlap = np.array(rec_imgs_no_overlap)
# Save images
utils.save_imgs(real_imgs=real_imgs_,
rec_imgs=rec_imgs_overlap,
img_path=self.test_image_path,
database_name=self.cfg.DATABASE_NAME,
max_img_in_col=self.cfg.MAX_IMAGE_IN_COL,
silent=False,
test_flag=True,
colorful=True,
append_info='_overlap')
utils.save_imgs(real_imgs=real_imgs_,
rec_imgs=rec_imgs_no_overlap,
img_path=self.test_image_path,
database_name=self.cfg.DATABASE_NAME,
max_img_in_col=self.cfg.MAX_IMAGE_IN_COL,
silent=False,
test_flag=True,
colorful=True,
append_info='_no_overlap')
def _get_multi_obj_scores(self, preds, preds_vec):
"""Get evaluation scores for multi-objects detection."""
utils.thin_line()
print('Calculating evaluation scores for {} detection...'.format(
self.info[1]))
def _f_beta_score(p, r, beta):
if p + r == 0:
return 0.
else:
return ((1 + (beta**2)) * p * r) / ((beta**2) * p + r)
# Calculate scores manually
precision = []
recall = []
accuracy = []
f1score = []
f05score = []
f2score = []
for pred_vec, y_true in zip(preds, self.y_test):
# true positive
tp = np.sum(np.multiply(y_true, pred_vec))
# false positive
fp = np.sum(
np.logical_and(np.equal(y_true, 0), np.equal(pred_vec, 1)))
# false negative
fn = np.sum(
np.logical_and(np.equal(y_true, 1), np.equal(pred_vec, 0)))
# true negative
tn = np.sum(
np.logical_and(np.equal(y_true, 0), np.equal(pred_vec, 0)))
precision_ = tp / (tp + fp)
accuracy_ = (tp + tn) / (tp + fp + tn + fn)
recall_ = tp / (tp + fn)
precision.append(precision_)
accuracy.append(accuracy_)
recall.append(recall_)
f1score.append(_f_beta_score(precision_, recall_, 1.))
f05score.append(_f_beta_score(precision_, recall_, 0.5))
f2score.append(_f_beta_score(precision_, recall_, 2.))
precision = np.mean(precision)
recall = np.mean(recall)
accuracy = np.mean(accuracy)
f1score = np.mean(f1score)
f05score = np.mean(f05score)
f2score = np.mean(f2score)
# true positive
tp = np.sum(np.multiply(preds, self.y_test))
print('TRUE POSITIVE: ', tp)
# false positive
fp = np.sum(
np.logical_and(np.equal(self.y_test, 0), np.equal(preds, 1)))
print('FALSE POSITIVE: ', fp)
# false negative
fn = np.sum(
np.logical_and(np.equal(self.y_test, 1), np.equal(preds, 0)))
print('TRUE NEGATIVE: ', fn)
# true negative
tn = np.sum(
np.logical_and(np.equal(self.y_test, 0), np.equal(preds, 0)))
print('FALSE NEGATIVE: ', tn)
# Calculate scores by using scikit-learn tools
# precision = precision_score(self.y_test, preds, average='samples')
# recall = recall_score(self.y_test, preds, average='samples')
# accuracy = accuracy_score(self.y_test, preds)
# f1score = f1_score(self.y_test, preds, average='samples')
# Top_N
precision_top_n_list = []
if self.cfg.TOP_N_LIST is not None:
for top_n in self.cfg.TOP_N_LIST:
precision_top_n = []
for pred_vec, y_true in zip(preds_vec, self.y_test):
y_pred_idx_top_n = np.argsort(pred_vec)[-top_n:]
y_true_idx = []
for i_, y_i in enumerate(y_true):
if y_i == 1:
y_true_idx.append(i_)
tp_top_n = 0
fp_top_n = 0
for y_idx in y_true_idx:
if y_idx in y_pred_idx_top_n:
tp_top_n += 1
else:
fp_top_n += 1
assert tp_top_n + fp_top_n == len(y_true_idx)
precision_top_n_ = tp_top_n / (tp_top_n + fp_top_n)
precision_top_n.append(precision_top_n_)
precision_top_n = np.mean(precision_top_n)
precision_top_n_list.append(precision_top_n)
assert len(precision_top_n_list) == len(self.cfg.TOP_N_LIST)
# Print evaluation information
utils.print_multi_obj_eval(precision, recall, accuracy, f1score,
f05score, f2score, self.cfg.TOP_N_LIST,
precision_top_n_list)
# Save evaluation scores of multi-objects detection.
if self.during_training and (self.epoch_train != 'end'):
utils.save_multi_obj_scores_is_training(
self.test_log_path, self.epoch_train, self.step_train,
precision, recall, accuracy, f1score, f05score, f2score,
self.cfg.TOP_N_LIST, precision_top_n_list)
else:
utils.save_multi_obj_scores(self.test_log_path, precision, recall,
accuracy, f1score, f05score, f2score,
self.cfg.TOP_N_LIST,
precision_top_n_list)
