def __init__(self, input_shape, num_classes, is_training=True, use_test_queue=False):
self.input_shape = input_shape
self.name = "capsnet"
self.graph = tf.Graph()
with self.graph.as_default():
if is_training:
self.X, self.labels = get_train_batch(cfg.dataset, cfg.batch_size, cfg.num_threads, samples_per_epoch=cfg.samples_per_epoch)
self.Y = tf.one_hot(self.labels, depth=num_classes, axis=1, dtype=tf.float32)
self.inference(num_classes)
self._loss()
self._summary()
self.global_step = tf.Variable(0, name='global_step', trainable=False)
self.optimizer = tf.train.AdamOptimizer()
self.train_op = self.optimizer.minimize(self.total_loss, global_step=self.global_step)
else:
if use_test_queue:
self.X, self.labels = get_test_batch(cfg.dataset, cfg.test_batch_size, cfg.num_threads)
self.Y = tf.one_hot(self.labels, depth=num_classes, axis=1, dtype=tf.float32)
self.inference(num_classes, is_training=False)
self._loss()
self._summary()
else:
self.X = tf.placeholder(tf.float32, shape=self.input_shape)
self.labels = tf.placeholder(tf.int32, shape=(self.input_shape[0],))
self.Y = tf.one_hot(self.labels, depth=num_classes, axis=1, dtype=tf.float32)
self.inference(num_classes, is_training=False)
self._loss()
errors = tf.not_equal(tf.to_int32(self.labels), self.predictions)
self.error_rate = tf.reduce_mean(tf.cast(errors, tf.float32))
def __init__(self,
input_shape,
num_classes,
is_training=True,
use_test_queue=False):
self.input_shape = input_shape
self.name = "resnet"
self.graph = tf.Graph()
with self.graph.as_default():
if is_training:
self.X, self.labels = get_train_batch(
cfg.dataset,
cfg.batch_size,
cfg.num_threads,
samples_per_epoch=cfg.samples_per_epoch)
self.inference(self.X, num_classes)
self.loss()
self._summary()
self.global_step = tf.Variable(0,
name='global_step',
trainable=False)
self.optimizer = tf.train.AdamOptimizer(epsilon=0.1)
self.train_op = self.optimizer.minimize(
self.total_loss, global_step=self.global_step)
else:
if use_test_queue:
self.X, self.labels = get_test_batch(
cfg.dataset, cfg.test_batch_size, cfg.num_threads)
else:
self.X = tf.placeholder(tf.float32, shape=self.input_shape)
self.labels = tf.placeholder(tf.int32,
shape=(self.input_shape[0], ))
self.inference(self.X, num_classes, keep_prob=1.0)
self.loss()
self._summary()
def test(self):
"""
Runs one test step on the generator network.
"""
batch = get_test_batch(c.BATCH_SIZE, num_rec_out=self.num_test_rec)
self.g_model.test_batch(
batch, self.global_step, num_rec_out=self.num_test_rec)
def test(self):
"""
Runs one test step on the generator network.
"""
for ind in range(2):
batch = get_test_batch(c.BATCH_SIZE,
num_rec_out=self.num_test_rec,
index=ind + 1)
self.g_model.test_batch(batch,
self.global_step,
num_rec_out=self.num_test_rec)
shutil.move("../Save/Images/Default/Tests/Step_0",
"../Save/Images/Default/Tests/Step_" + str(ind))
def main(_):
if not os.path.exists(args.ckpt_dir):
os.makedirs(args.ckpt_dir)
if not os.path.exists(args.train_dir):
os.makedirs(args.train_dir)
if not os.path.exists(args.test_dir):
os.makedirs(args.test_dir)
if not os.path.exists(args.feature_dir):
os.makedirs(args.feature_dir)
IMG_W = 32
IMG_H = 32
CAPACITY = 256
if args.phase == 'train':
image_list, label_list = get_train_files(args.train_dir)
train_batch, label_batch = get_train_batch(image_list, label_list,
IMG_W, IMG_H,
args.batch_size, CAPACITY)
run_train(image_batch=train_batch,
label_batch=label_batch,
n_class=args.n_class,
batch_size=args.batch_size,
checkpoint_dir=args.ckpt_dir,
lr=args.lr,
MAX_STEP=args.iters)
elif args.phase == 'test':
test_list = get_test_files(args.test_dir)
test_batch = get_test_batch(test_list, IMG_W, IMG_H, args.batch_size,
CAPACITY)
print(test_batch)
run_test(test_batch, args.n_class, args.ckpt_dir, args.batch_size)
elif args.phase == 'feature_extraction':
image_list, label_list = get_train_files(args.train_dir)
train_batch, label_batch = get_train_batch(image_list, label_list,
IMG_W, IMG_H,
args.batch_size, CAPACITY)
feature_extraction(train_batch, args.n_class, args.ckpt_dir,
args.batch_size)
else:
print('{:*^50}'.format('【Unknown phase!】'))
def test(self):
"""
Runs one test step on the generator network.
"""
all_dirs = sorted(np.array(glob(os.path.join(c.TEST_DIR, '*'))))
print("Total test batch: " + len(all_dirs) + " batch_size = " +
str(c.BATCH_SIZE))
for i in range(c.BATCH_SIZE, len(all_dirs), c.BATCH_SIZE):
eps = all_dirs[i - c.BATCH_SIZE:i]
print("eps: " + eps)
batch = get_test_batch(eps,
c.BATCH_SIZE,
num_rec_out=self.num_test_rec)
self.g_model.test_batch(batch,
eps,
self.global_step,
num_rec_out=self.num_test_rec)
def main(args):
log_dir = os.path.join(os.path.expanduser(args.logs_base_dir),
args.model_name)
if not os.path.isdir(
log_dir): # Create the log directory if it doesn't exist
os.makedirs(log_dir)
model_dir = os.path.join(os.path.expanduser(args.models_base_dir),
args.model_name)
if not os.path.isdir(
model_dir): # Create the model directory if it doesn't exist
os.makedirs(model_dir)
model_path = model_dir + '/' + args.model_name + '.ckpt'
print('pid:' + str(os.getpid()))
os.environ['CUDA_VISIBLE_DEVICES'] = args.CUDA_VISIBLE_DEVICES
# tf.reset_default_graph()
network = importlib.import_module(args.model_def)
max_checkpoints = 3
lstm_model_setting = {}
lstm_model_setting['num_units'] = 128
lstm_model_setting['dimension_projection'] = args.embedding_size
lstm_model_setting['attn_length'] = 10
lstm_model_setting['num_layers'] = 3
test_size = 10
if args.pretrained_model and not args.finetuning:
try:
with open(
os.path.join(os.path.dirname(args.pretrained_model),
'test_speaker.txt'), 'r') as fid:
test_speaker = fid.read().split('\n')
except:
test_speaker = random.sample(os.listdir(args.data_set), test_size)
else:
test_speaker = random.sample(os.listdir(args.data_set), test_size)
with open(os.path.join(model_dir, 'test_speaker.txt'), 'w') as fid:
fid.write('\n'.join(test_speaker))
train_file = [
os.path.join(args.data_set, file_name)
for file_name in os.listdir(args.data_set)
if file_name not in test_speaker
]
test_file = [
os.path.join(args.data_set, file_name) for file_name in test_speaker
]
n_class = len(train_file)
x = tf.placeholder(tf.float32, [None, 199, args.n_fbank], name='inputs')
y = tf.placeholder(tf.int64, [None])
keep_prob = tf.placeholder(tf.float32, name='keep_prob')
lr_placeholder = tf.placeholder(tf.float32, name='learning_rate')
ratio_s_placeholder = tf.placeholder(tf.float32, name='softmax_rate')
ratio_t_placeholder = tf.placeholder(tf.float32, name='triplet_rate')
with tf.device('/cpu:0'):
q = tf.FIFOQueue(args.batch_size * 3, [tf.float32, tf.int64],
shapes=[[199, args.n_fbank], []])
enqueue_op = q.enqueue_many([x, y])
x_b, y_b = q.dequeue_many(args.batch_size)
# with tf.device('/gpu:0'):
logits, embeddings = network.inference(x_b, lstm_model_setting, keep_prob,
n_class)
with tf.name_scope('loss'):
with tf.name_scope('triplet_loss'):
triplet_loss = tf.contrib.losses.metric_learning.triplet_semihard_loss(
y_b, embeddings, margin=0.2)
tf.summary.scalar('train', triplet_loss)
with tf.name_scope('softmax_loss'):
softmax_loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_b,
logits=logits))
tf.summary.scalar('train', softmax_loss)
with tf.name_scope('total_loss'):
total_loss = ratio_s_placeholder * softmax_loss + ratio_t_placeholder * triplet_loss
tf.summary.scalar('train', total_loss)
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(
tf.cast(tf.equal(tf.argmax(logits, 1), y_b), tf.float32))
tf.summary.scalar('train', accuracy)
# opt = tf.train.AdamOptimizer(learning_rate=lr_placeholder)
opt = tf.train.MomentumOptimizer(learning_rate=lr_placeholder,
momentum=0.9,
use_nesterov=True)
global_step = tf.Variable(0, trainable=False)
gradients = opt.compute_gradients(total_loss)
capped_gradients = [(tf.clip_by_value(grad, -5., 5.), var)
for grad, var in gradients if grad is not None]
trainer = opt.apply_gradients(capped_gradients, global_step=global_step)
# trainer = opt.minimize(total_loss, global_step=global_step)
# merge ummaries to write them to file
merged = tf.summary.merge_all()
# checkpoint saver and restorer
if args.only_weight:
all_vars = tf.trainable_variables()
excl_vars = tf.get_collection(tf.GraphKeys.EXCL_RESTORE_VARS)
to_restore = [
item for item in all_vars
if tflearn.utils.check_restore_tensor(item, excl_vars)
]
elif args.finetuning:
all_vars = tf.global_variables()
excl_vars = tf.get_collection(tf.GraphKeys.EXCL_RESTORE_VARS)
to_restore = [
item for item in all_vars
if tflearn.utils.check_restore_tensor(item, excl_vars)
]
else:
to_restore = None
restorer = tf.train.Saver(var_list=to_restore, max_to_keep=max_checkpoints)
saver = tf.train.Saver(max_to_keep=max_checkpoints)
coord = tf.train.Coordinator()
config = tf.ConfigProto()
config.allow_soft_placement = True
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
if args.pretrained_model:
restorer.restore(sess, args.pretrained_model)
# enqueuing batches procedure
def enqueue_batches():
while not coord.should_stop():
batch_feats, batch_labels = utils.get_train_batch(
train_file, args.batch_size)
sess.run(enqueue_op,
feed_dict={
x: batch_feats,
y: batch_labels
})
# creating and starting parallel threads to fill the queue
num_threads = 3
for i in range(num_threads):
t = threading.Thread(target=enqueue_batches)
t.setDaemon(True)
t.start()
train_writer = tf.summary.FileWriter(log_dir, sess.graph)
start_time = time.time()
step = sess.run(global_step)
while step <= args.max_step:
if args.config_file:
ratio_s, ratio_t, lr, display_step, evaluate_step = get_config(
args.config_file)
else:
ratio_s = 1
ratio_t = 0.5
lr = 0.001
display_step = 100
evaluate_step = 1000
_, step = sess.run(
[trainer, global_step],
feed_dict={
keep_prob: args.keep_prob,
lr_placeholder: lr,
ratio_s_placeholder: ratio_s,
ratio_t_placeholder: ratio_t
})
if step % display_step == 0:
train_tl, train_sl, train_l, train_acc, result = sess.run(
[triplet_loss, softmax_loss, total_loss, accuracy, merged],
feed_dict={
keep_prob: 1,
lr_placeholder: lr,
ratio_s_placeholder: ratio_s,
ratio_t_placeholder: ratio_t
})
int_time = time.time()
print(
'Step: {:09d} --- Loss: {:.7f} Cross Entropy: {:.07f} Triplet Loss: {:.07f} Training accuracy: {:.4f} Learning Rate: {} PID: {} Elapsed time: {}'
.format(step, train_l, train_sl, train_tl, train_acc, lr,
os.getpid(),
utils.format_time(int_time - start_time)))
train_writer.add_summary(result, step)
if step % evaluate_step == 0:
enroll_feats, enroll_labels, test_feats, test_labels = utils.get_test_batch(
test_file, args.batch_size)
embs = sess.run(embeddings,
feed_dict={
x_b:
np.vstack((enroll_feats,
test_feats[:args.batch_size -
len(enroll_feats)])),
keep_prob:
1
})
enroll_list = utils.enroll(embs[:len(enroll_feats)],
enroll_labels)
embsp = sess.run(embeddings,
feed_dict={
x_b:
test_feats[args.batch_size -
len(enroll_feats):],
keep_prob:
1
})
predict_labels = utils.speaker_identification(
np.vstack((embs[len(enroll_feats):], embsp)), enroll_list)
test_acc = accuracy_score(test_labels, predict_labels)
print('===================')
int_time = time.time()
print('Elapsed time: {}'.format(
utils.format_time(int_time - start_time)))
print('Validation accuracy: {:.04f}'.format(test_acc))
# save weights to file
save_path = saver.save(sess, model_path)
print('Variables saved in file: %s' % save_path)
print('Logs saved in dir: %s' % log_dir)
summary = tf.Summary()
summary.value.add(tag='accuracy/val', simple_value=test_acc)
train_writer.add_summary(summary, step)
print('===================')
end_time = time.time()
print('Elapsed time: {}'.format(
utils.format_time(end_time - start_time)))
save_path = saver.save(sess, model_path)
print('Variables saved in file: %s' % save_path)
print('Logs saved in dir: %s' % log_dir)
coord.request_stop()
coord.join()
def infer(modelpath='result/TRAINED_ADVERSARIAL.model', no_of_preds=7):
"""
Visualizes the predictions of the saved model
:param modelpath: path of the saved Generative model to test on
:param no_of_preds: No of recursive predcitions to make
:return:
"""
# Initiate the generator model
model = MultiScaleGenerator(c.SCALE_FMS_G, c.SCALE_KERNEL_SIZES_G)
chainer.serializers.load_npz(modelpath, model)
# Get input fot the model as well as ground truth future frames
# Here future frames is already a list, NOT an array
input_frames, ground_truth = get_test_batch(no_of_preds=no_of_preds)
# Create a split version for visualization
inputs = np.split(input_frames, [3, 6, 9], 1)
# Plot the input frames
fig = plt.figure(figsize=(10, 15))
for i in range(4):
f, ax1 = plt.subplots(1, 1)
# Denormalize
ax1.imshow(np.transpose(denormalize_frames(inputs[i][0]), (1, 2, 0)))
ax1.set_xticks([])
ax1.set_yticks([])
ax1.set_xlabel('INPUTS')
f.savefig("inference/INPUT" + str(i) + '.png', tight=True)
plt.close()
# Get prediction
predictions = model.predict(input_frames, no_of_predictions=no_of_preds)
# TODO: Visualize the multi scale outputs
# Create a list of the predicted frames for viz
for i, frame in enumerate(predictions):
predictions[i] = np.transpose(denormalize_frames(frame)[0], (1, 2, 0))
# Create a list of grount truth frames for viz
for i, frame in enumerate(ground_truth):
ground_truth[i] = np.transpose(denormalize_frames(frame), (1, 2, 0))
# Plot Outputs
fig = plt.figure(figsize=(10, 15))
for i in range(len(ground_truth)):
f, ax1 = plt.subplots(1, 1)
ax1.imshow(predictions[i])
ax1.set_xticks([])
ax1.set_yticks([])
ax1.set_xlabel('OUTPUTS', color='r')
f.savefig("inference/OUTPUTS" + str(i) + '.png', tight=True)
plt.close()
# Plot comparison
fig = plt.figure(figsize=(10, 15))
for i in range(len(predictions)):
f, (ax1, ax2) = plt.subplots(1, 2)
ax1.imshow(ground_truth[i])
ax1.set_xticks([])
ax1.set_yticks([])
ax1.set_xlabel('GROUND TRUTH')
ax2.imshow(predictions[i])
ax2.set_xticks([])
ax2.set_yticks([])
ax2.set_xlabel('ADVERSARIAL PREDICTIONS', color='r')
f.savefig('inference/comparison' + str(i) + '.png', tight=True)
def test(self, test_no=1):
batch = get_test_batch(c.BATCH_SIZE_TEST)
self.GAN_model.generator_test_batch(batch, test_no)
