def test():
def imshow(ax, normalize_image, title):
img = ((normalize_image + 1) * 127.5).astype(np.uint8)[0]
ax.imshow(img)
ax.axes.get_xaxis().set_visible(False)
ax.axes.get_yaxis().set_visible(False)
ax.set_title(title)
epochs = [50, 100, 150, 200]
models = ["facade_%d" % epoch for epoch in epochs]
gans = []
for model in models:
p2p = Pix2Pix()
p2p.load_model(model)
gans.append(p2p)
for real, cond in load_batch("facade", "test", 256, 1):
ax = plt.subplot(231)
imshow(ax, cond, "conditional")
ax = plt.subplot(232)
imshow(ax, real, "real")
for gi, (epoch, gan) in enumerate(zip(epochs, gans)):
fake = gan.generator.predict(cond)
ax = plt.subplot(2, 3, gi + 3)
imshow(ax, fake, "epoch %d" % epoch)
plt.show()
def train(self, dataset, batch_size=1, epochs=100, save=True):
"""
Train the gan.
:param dataset: str, name of dataset, such as "facade"
:param batch_size: int, batch size
:param epochs: int, training epochs
:param save: bool, whether save model after training
"""
for ei in range(epochs):
if ei % 10 == 0 and ei > 0:
h5_file = "%s/%s_%d.h5" % (MODEL_DIR, dataset, ei)
self.gan.save_weights(h5_file)
is_valid = np.ones((batch_size, 32, 32, 1))
is_invalid = np.zeros((batch_size, 32, 32, 1))
for bi, (real, cond) in enumerate(load_batch(dataset, "train", 256, batch_size)):
fake = self.generator.predict(cond)
# train discriminator
d_bc_real, d_acc_real = self.discriminator.train_on_batch([real, cond], is_valid)
d_bc_fake, d_acc_fake = self.discriminator.train_on_batch([fake, cond], is_invalid)
d_bc = 0.5 * (d_bc_real + d_bc_fake)
d_acc = 0.5 * (d_acc_real + d_acc_fake)
# train generator
g_metrics = self.gan.train_on_batch([real, cond], [is_valid, real])
g_loss, g_bc, g_mae = g_metrics
print("epoch-%d batch-%d | d_bc:%.4f d_acc:%.4f | g_loss: %.4f g_bc:%.4f g_mae:%.4f" %
(ei + 1, bi + 1, d_bc, d_acc, g_loss, g_bc, g_mae))
h5_file = "%s/%s_%d.h5" % (MODEL_DIR, dataset, epochs)
self.gan.save_weights(h5_file)
if save:
h5_file = "%s/%s.h5" % (MODEL_DIR, dataset)
self.gan.save_weights(h5_file)
def get_cost(f_cost, samples, batch, mat, costs, pgb, f_update=None, lrate=0.):
'''
Compute the cost of the given batches and update the parameters if it is necessary
'''
x, mask_x, y, mask_y = load_batch(samples, batch, mat)
costs.extend(f_cost(x, mask_x, y, mask_y))
if f_update:
f_update(lrate)
if numpy.isnan(costs[-1]) or numpy.isinf(costs[-1]):
pgb.pause()
raise FloatingPointError, 'Bad cost detected!'
pgb.disp(costs, 'COST')
def get_cost(f_cost, samples, batch, mat, costs, pgb, f_update=None, lrate=0.):
'''
Compute the cost of the given batches and update the parameters if it is necessary
'''
x, mask_x, y, mask_y = load_batch(samples, batch, mat)
costs.extend(f_cost(x, mask_x, y, mask_y))
if f_update:
f_update(lrate)
if numpy.isnan(costs[-1]) or numpy.isinf(costs[-1]):
pgb.pause()
raise FloatingPointError, 'Bad cost detected!'
pgb.disp(costs, 'COST')
def predict(f_enc, f_dec, samples, batches, mat, max_len, header):
'''
Sample words and compute the prediction error
'''
preds = []
errs = []
progress = ProgressBar(numpy.sum([len(batch) for batch in batches]), 20,
header)
for batch in batches:
x, mask_x, y, mask_y = load_batch(samples, batch, mat)
[prev_h] = f_enc(x, mask_x)
n_steps = mask_x.sum(0)
n_samples = x.shape[1]
sents = numpy.zeros((n_samples, max_len), 'int32')
# First step - No embedded word is fed into the decoder
sents[:, 0], prev_h = f_dec(numpy.asarray([-1] * n_samples, 'int32'),
prev_h)
n_ends = n_steps - (sents[:, 0] == 0)
for i in range(1, max_len - 1):
prev_words = sents[:, i - 1]
if not n_ends.any():
break
next_words, prev_h = f_dec(prev_words, prev_h)
sents[:, i] = next_words * (n_ends > 0)
n_ends -= (next_words == 0) * (n_ends > 0)
for i in range(n_samples):
idx = 0
while idx < max_len and n_steps[i] > 0:
if sents[i, idx] == 0:
n_steps[i] -= 1
idx += 1
preds.append(sents[i, :idx].tolist())
y = numpy.concatenate(
[y, numpy.zeros((max_len - len(y), n_samples), 'int32')]).T
mask_y = numpy.concatenate(
[mask_y,
numpy.zeros((max_len - len(mask_y), n_samples), 'int32')]).T
errs.extend(((sents != y) * mask_y * 1.).sum(1) / mask_y.sum(1))
progress.disp(errs, ' ERR')
return preds, numpy.mean(errs)
def predict(f_enc, f_dec, samples, batches, mat, max_len, header):
'''
Sample words and compute the prediction error
'''
preds = []
errs = []
progress = ProgressBar(numpy.sum([len(batch) for batch in batches]), 20, header)
for batch in batches:
x, mask_x, y, mask_y = load_batch(samples, batch, mat)
[prev_h] = f_enc(x, mask_x)
n_steps = mask_x.sum(0)
n_samples = x.shape[1]
sents = numpy.zeros((n_samples, max_len), 'int32')
# First step - No embedded word is fed into the decoder
sents[:, 0], prev_h = f_dec(numpy.asarray([-1] * n_samples, 'int32'), prev_h)
n_ends = n_steps - (sents[:, 0] == 0)
for i in range(1, max_len - 1):
prev_words = sents[:, i - 1]
if not n_ends.any():
break
next_words, prev_h = f_dec(prev_words, prev_h)
sents[:, i] = next_words * (n_ends > 0)
n_ends -= (next_words == 0) * (n_ends > 0)
for i in range(n_samples):
idx = 0
while idx < max_len and n_steps[i] > 0:
if sents[i, idx] == 0:
n_steps[i] -= 1
idx += 1
preds.append(sents[i, : idx].tolist())
y = numpy.concatenate([y, numpy.zeros((max_len - len(y), n_samples), 'int32')]).T
mask_y = numpy.concatenate([mask_y, numpy.zeros((max_len - len(mask_y), n_samples), 'int32')]).T
errs.extend(((sents != y) * mask_y * 1.).sum(1) / mask_y.sum(1))
progress.disp(errs, ' ERR')
return preds, numpy.mean(errs)
def get_input(inFile,
inType,
modelType,
offset=0.,
scale=1.,
winLen=None,
winShift=10):
if winLen is None:
winLen = model.input_shape[3]
typeInfo = modelType.split('_')
timeDistributed = (len(typeInfo) > 1) and any(
[x in typeInfo[1] for x in ['dist', 'stateful']])
if inType == 'tinyfeats':
feaStream = audioproc.load_bin(inFile)
if not timeDistributed:
feaStream, starts = fbank_stream(feaStream, winLen)
elif inType == 'audio':
if os.path.isfile(inFile):
feaStream = audioproc.wav2fbank(inFile)
nFiles = 1
if not timeDistributed:
feaStream, starts = fbank_stream(feaStream, winLen)
elif os.path.isdir(inFile): # assumes each clip is winLen
feaStream, files = audioproc.wav2fbank_batch(inFile)
nFiles = len(files)
if timeDistributed:
feaStream = np.reshape(
feaStream, (nFiles, feaStream.shape[0], feaStream.shape[1]))
elif inType == 'serverfeats':
feaStream = audioproc.load_serverfeats(inFile)
elif inType == 'features':
feaStream = data.load_batch(inFile, var='features')
feaStream = data.apply_norm(feaStream, offset, scale)
feaStream = data.reshape_for_model(feaStream, modelType)
return feaStream
def predict(f_enc, f_dec, samples, batches, mat, beam_size, max_len, header):
'''
Sample words and compute the prediction error
'''
preds = []
errs = []
progress = ProgressBar(numpy.sum([len(batch) for batch in batches]), 20,
header)
for batch in batches:
x, mask_x, y, mask_y = load_batch(samples, batch, mat)
[init_h] = f_enc(x, mask_x)
n_steps = mask_x.sum(0)
n_samples = x.shape[1]
prev_sents = numpy.zeros((beam_size, n_samples, max_len), 'int32')
# First step - No embedded word is fed into the decoder
prev_words = numpy.asarray([-1] * n_samples, 'int32')
prev_sents[:, :, 0], prev_log_prob, prev_h = f_dec(prev_words, init_h)
prev_h = numpy.tile(prev_h, (beam_size, 1, 1))
prev_n_ends = n_steps - (prev_sents[:, :, 0] == 0)
for i in range(1, max_len - 1):
hypo_sents = [[]] * n_samples
hypo_log_prob = [[]] * n_samples
hypo_h = [[]] * n_samples
hypo_n_ends = [[]] * n_samples
has_hypos = numpy.asarray([False] * n_samples)
for j in range(beam_size):
if not prev_n_ends[j].any():
continue
next_words, next_log_prob, next_h = f_dec(
prev_sents[j, :, i - 1], prev_h[j])
for k in range(n_samples):
if prev_n_ends[j, k] > 0:
has_hypos[k] = True
next_sents = numpy.tile(prev_sents[j, k],
(beam_size, 1))
next_sents[:, i] = next_words[:, k]
hypo_sents[k].extend(next_sents)
hypo_log_prob[k].extend(next_log_prob[:, k] +
prev_log_prob[j, k])
hypo_h[k].extend([next_h[k]] * beam_size)
hypo_n_ends[k].extend(prev_n_ends[j, k] -
(next_words[:, k] == 0))
else:
hypo_sents[k].append(prev_sents[j, k].copy())
hypo_log_prob[k].append(prev_log_prob[j, k])
hypo_h[k].append(prev_h[j, k].copy())
hypo_n_ends[k].append(0)
if not has_hypos.any():
break
for j in range(n_samples):
if not has_hypos[j]:
continue
indices = numpy.argsort(hypo_log_prob[j])[:-beam_size - 1:-1]
for k in range(beam_size):
prev_sents[k, j] = hypo_sents[j][indices[k]]
prev_log_prob[k, j] = hypo_log_prob[j][indices[k]]
prev_h[k, j] = hypo_h[j][indices[k]]
prev_n_ends[k, j] = hypo_n_ends[j][indices[k]]
sents = prev_sents[prev_log_prob.argmax(0), numpy.arange(n_samples)]
for i in range(n_samples):
idx = 0
while idx < max_len and n_steps[i] > 0:
if sents[i, idx] == 0:
n_steps[i] -= 1
idx += 1
preds.append(sents[i, :idx].tolist())
y = numpy.concatenate(
[y, numpy.zeros((max_len - len(y), n_samples), 'int32')]).T
mask_y = numpy.concatenate(
[mask_y,
numpy.zeros((max_len - len(mask_y), n_samples), 'int32')]).T
errs.extend(((sents != y) * mask_y * 1.).sum(1) / mask_y.sum(1))
progress.disp(errs, ' ERR')
return preds, numpy.mean(errs)
return np.sum(np.square(u - v)) / u.shape[0]
if __name__ == '__main__':
# config
config = Config()
# model with config
model = Model(config)
error = []
# to keep it simple, we define the number of beatches we wanna do
for batch_indx in range(config.batch_number):
# load random snippets from one file
x, t = load_batch(config) #
# for each of them generate an initial state
initial_state = np.zeros((config.batch_number, 6))
with tf.GradientTape() as tape:
output = model(x)
train_loss = config.loss_function(t, output)
train_error = mse(t, output)
gradients = tape.gradient(train_loss, model.trainable_variables)
if batch_indx % PRINT_STEP == 0:
print("* Batch: [{:4d}/{}] === Loss: {:04.5f} === Error: {:04.5f}".
format(batch_indx, config.batch_number, train_loss,
train_error))
config.optimizer.apply_gradients(
zip(gradients, model.trainable_variables))
def train():
tf.global_variables_initializer().run()
could_load, checkpoint_counter = load()
if could_load:
start_epoch = (int)(checkpoint_counter / num_batches)
start_batch_id = checkpoint_counter - start_epoch * num_batches
counter = checkpoint_counter
print("Checkpoint Load Successed")
else:
start_epoch = 0
start_batch_id = 0
counter = 1
print("train from scratch...")
train_iter = []
train_loss = []
IOU = 0.5
F1 = 0.7
# utils.count_params()
print("Total train image:{}".format(len(train_img)))
print("Total validate image:{}".format(len(valid_img)))
print("Total epoch:{}".format(args.num_epochs))
print("Batch size:{}".format(args.batch_size))
print("Learning rate:{}".format(args.learning_rate))
print("Checkpoint step:{}".format(args.checkpoint_step))
print("Data Argument:")
print("h_flip: {}".format(args.h_flip))
print("v_flip: {}".format(args.v_flip))
print("rotate: {}".format(args.rotation))
print("clip size: {}".format(args.clip_size))
loss_tmp = []
for i in range(start_epoch, args.num_epochs):
epoch_time = time.time()
id_list = np.random.permutation(len(train_img))
# if (i > args.start_valid):
# if (i - args.start_valid) % args.valid_step == 0:
# val_iou = validation()
# print("last iou valu:{}".format(IOU))
# print("new_iou value:{}".format(val_iou))
# if val_iou > IOU:
# print("Save the checkpoint...")
# saver.save(sess, './mapmodfy/checkpoint/model.ckpt', global_step=counter, write_meta_graph=True)
# IOU = val_iou
for j in range(start_batch_id, num_batches):
img_d = []
lab_d = []
for ind in range(args.batch_size):
id = id_list[j * args.batch_size + ind]
img_d.append(train_img[id])
lab_d.append(train_label[id])
x_batch, y_batch = load_batch(img_d, lab_d)
# print(x_batch)
feed_dict = {img: x_batch, label: y_batch, is_training: True}
_, loss, pred1 = sess.run(
[train_step, sigmoid_cross_entropy_loss, pred],
feed_dict=feed_dict)
loss_tmp.append(loss)
# print(loss)
if (counter % 200 == 0):
tmp = np.median(loss_tmp)
train_iter.append(counter)
train_loss.append(tmp)
print('Epoch', i, '|Iter', counter, '|Loss', tmp)
loss_tmp.clear()
counter += 1
start_batch_id = 0
print('Time:', time.time() - epoch_time)
# if(i > args.start_valid and (i+1)%5==0):
# saver.save(sess, './checkpoint/model.ckpt', global_step=counter)
if (i > args.start_valid):
if (i - args.start_valid) % args.valid_step == 0:
val_iou = validation()
print("last iou valu:{}".format(IOU))
print("new_iou value:{}".format(val_iou))
if val_iou > IOU:
print("Save the checkpoint...")
saver.save(sess,
'./checkpoint/model.ckpt',
global_step=counter,
write_meta_graph=True)
IOU = val_iou
saver.save(sess, './checkpoint/model.ckpt', global_step=counter)
def fit(self,
train,
val,
max_epochs=None,
batch_size=512,
class_weights=None,
early_stopping=0,
save_model=False):
outpath = os.path.join(self.outpath, 'cnn_model/')
if os.path.isdir(outpath):
shutil.rmtree(outpath)
os.makedirs(outpath, exist_ok=True)
os.mkdir(os.path.join(outpath, 'model_weights/'))
if max_epochs is None:
max_epochs = 300
self.epochs = 0
train_n = len(train)
val_n = len(val)
decreasing = 0
tloss = []
tacc = []
vloss = []
vacc = []
stop = False
# iterate through every epoch
while not stop:
sys.stderr.flush()
print('Epoch: %i' % self.epochs)
print('Training:')
shuffle(train)
for i in tqdm(range(0, train_n, batch_size)):
batch_paths = train[i:i + batch_size]
if i + batch_size >= train_n:
batch_paths = train[i:]
x, y = data.load_batch(batch_paths, len(batch_paths))
metrics = self.train_on_batch(x,
y,
class_weight=class_weights,
return_dict=True)
tloss.append(metrics['loss'])
tacc.append(metrics['acc'])
del x, y
# print metrics
avg_loss = sum(tloss) / len(tloss)
avg_acc = sum(tacc) / len(tacc)
self.train_loss.append(avg_loss)
self.train_acc.append(avg_acc)
print('\nTrain Loss: %f Train Accuracy: %f' %
(avg_loss, avg_acc))
print('Validation:')
# test on val for each epoch
for i in tqdm(range(0, val_n, batch_size)):
batch_paths = val[i:i + batch_size]
if i + batch_size >= val_n:
batch_paths = val[i:]
x, y = data.load_batch(batch_paths, len(batch_paths))
metrics = self.model.test_on_batch(x, y, return_dict=True)
vloss.append(metrics['loss'])
vacc.append(metrics['acc'])
del x, y
# print metrics
avg_loss = sum(vloss) / len(vloss)
avg_acc = sum(vacc) / len(vacc)
self.val_loss.append(avg_loss)
self.val_acc.append(avg_acc)
print('Validation Loss: %f Validation Accuracy: %f' %
(avg_loss, avg_acc))
# check stopping criteria
if self.epochs >= max_epochs - 1:
stop = True
if early_stopping > 0:
if len(self.val_loss) >= early_stopping:
if self.val_loss[-1] > self.val_loss[-2]:
if decreasing >= early_stopping:
stop = True
else:
decreasing += 1
else:
decreasing = 0
# save if model improves
if self.epochs >= 1 and self.val_loss[-1] < self.val_loss[-2]:
self.model.save_weights(
os.path.join(outpath, 'model_weights/',
'weights_' + str(self.epochs)))
if save_model:
self.model.save(os.path.join(outpath, 'model'))
self.epochs += 1
def train():
tf.global_variables_initializer().run()
could_load, checkpoint_counter = load()
if could_load:
start_epoch = (int)(checkpoint_counter / num_batches)
start_batch_id = checkpoint_counter - start_epoch * num_batches
counter = checkpoint_counter
print("Checkpoint Load Successed")
else:
start_epoch = 0
start_batch_id = 0
counter = 1
print("train from scratch...")
train_iter=[]
train_loss=[]
utils.count_params()
print("Total image:{}".format(len(train_img)))
print("Total epoch:{}".format(args.num_epochs))
print("Batch size:{}".format(args.batch_size))
print("Learning rate:{}".format(args.learning_rate))
print("Checkpoint step:{}".format(args.checkpoint_step))
print("Data Argument:")
print("h_flip: {}".format(args.h_flip))
print("v_flip: {}".format(args.v_flip))
print("rotate: {}".format(args.rotation))
print("clip size: {}".format(args.clip_size))
for i in range(start_epoch,args.num_epochs):
id_list = np.random.permutation(len(train_img))
epoch_time=time.time()
for j in range(start_batch_id,num_batches):
img_d=[]
lab_d=[]
for ind in range(args.batch_size):
id = id_list[j * args.batch_size + ind]
img_d.append(train_img[id])
lab_d.append(train_label[id])
x_batch, y_batch = load_batch(img_d,lab_d,args)
feed_dict = {img: x_batch,
label: y_batch
}
loss_tmp = []
_, loss, pred1 = sess.run([train_step, sigmoid_cross_entropy_loss, pred], feed_dict=feed_dict)
loss_tmp.append(loss)
if (counter % 100 == 0):
tmp = np.mean(loss_tmp)
train_iter.append(counter)
train_loss.append(tmp)
print('Epoch', i, '|Iter', counter, '|Loss', tmp)
counter += 1
start_batch_id=0
print('Time:', time.time() - epoch_time)
#if((i+1)%10==0 ):#lr dst from 10 every 10 epoches by 0.1
#learning_rate = 0.1 * learning_rate
#last_checkpoint_name = "checkpoint/latest_model_epoch_" + "_pspet.ckpt"
# print("Saving latest checkpoint")
# saver.save(sess, last_checkpoint_name)
if((i+1)%args.checkpoint_step==0):#save 20,30,40,50 checkpoint
args.learning_rate=0.1*args.learning_rate
print(args.learning_rate)
saver.save(sess,'./checkpoint/model.ckpt',global_step=counter,write_meta_graph=True)
"""
host = host_subplot(111)
plt.subplots_adjust(right=0.8)
p1, = host.plot(train_iter, train_loss, label="training loss")
host.legend(loc=5)
host.axis["left"].label.set_color(p1.get_color())
host.set_xlim([0, counter])
plt.draw()
plt.show()
"""
fig1, ax1 = plt.subplots(figsize=(11, 8))
ax1.plot(train_iter, train_loss)
ax1.set_title("Training loss vs Iter")
ax1.set_xlabel("Iter")
ax1.set_ylabel("Training loss")
plt.savefig('Training loss_vs_Iter.png')
plt.clf()
remain_time=(args.num_epochs - 1 - i) * (time.time() - epoch_time)
m, s = divmod(remain_time, 60)
h, m = divmod(m, 60)
print("Remaining training time = %d hours %d minutes %d seconds\n" % (h, m, s))
def train(verbose):
X_train, Y_train, X_dev, Y_dev = data.load_batch('mfcc') #load training and validation batches
net = network.RNN_network()
total_loss = net.loss()
training_losses = []
validation_losses = []
#adaptive learning rate
global_step = tf.Variable(0, trainable=False)
# adaptive_learning_rate = tf.train.exponential_decay(learning_rate_init, global_step, 1400, learning_decay, staircase=True)
optimizer_a = tf.train.AdamOptimizer(learning_rate)
# optimizer = optimizer_a.minimize(total_loss, global_step=global_step)
#add gradient clipping
gradients, variables = zip(*optimizer_a.compute_gradients(total_loss))
gradients, _ = tf.clip_by_global_norm(gradients, clip_norm=1.0)
optimizer = optimizer_a.apply_gradients(zip(gradients, variables), global_step=global_step)
run_options = tf.RunOptions(report_tensor_allocations_upon_oom = True)
with tf.Session() as sess:
t0 = time.time()
sess.run(tf.global_variables_initializer(), options=run_options)
net.load_state(sess, CKPT_PATH)
n_train_batch = len(X_train)
print("Number of training batch:", n_train_batch)
idx_train = list(range(n_train_batch))
n_dev_batch = len(X_dev)
print("Number of validation batch:", n_dev_batch)
idx_dev = list(range(n_dev_batch))
for epoch_idx in range(num_epochs):
np.random.shuffle(idx_train)
loss_epoch = 0
#training mode
for i in range(n_train_batch):
_total_loss, _train_step, net_output = sess.run(
[total_loss, optimizer, net()],
feed_dict={
net.batchX_placeholder:X_train[idx_train[i]],
net.batchY_placeholder:Y_train[idx_train[i]]
})
loss_epoch += _total_loss
#check for NaNs in network output
if (np.any(np.isnan(net_output))):
print("\nepoch: " + repr(epoch_idx) + " Nan output!")
if verbose == 1:
print("batch_loss:", _total_loss)
t1 = time.time()
print("\nepoch: " + repr(epoch_idx) + " || loss_epoch: " + repr(loss_epoch) + " || ", end=' ')
timer(t0, t1)
training_losses.append(loss_epoch)
if epoch_idx % 10 == 0:
tf.train.Saver().save(sess, CKPT_PATH, global_step=epoch_idx)
if epoch_idx % 1 == 0: #validation mode
dev_loss_epoch = 0
for j in range(n_dev_batch):
_total_dev_loss = sess.run(
[total_loss],
feed_dict={
net.batchX_placeholder: X_dev[idx_dev[j]],
net.batchY_placeholder: Y_dev[idx_dev[j]]
})
dev_loss_epoch += _total_dev_loss[0] #dev loss across all validation batches
print('********epoch: '+ repr(epoch_idx) + " || validation loss: " + repr(dev_loss_epoch) + " || ", end=' ')
validation_losses.append(dev_loss_epoch)
tf.train.Saver().save(sess, SAVE_PATH + "/" + repr(time.time()) + "/" + "save.ckpt")
print("finished.")
training_losses = np.array(training_losses)
np.save(SAVE_PATH + "training_losses.npy", training_losses)
validation__losses = np.array(validation_losses)
np.save(SAVE_PATH + "validation_losses.npy", validation__losses)
def main(_):
#Create the log directory here. Must be done here otherwise import will activate this unneededly.
if not os.path.exists(FLAGS.log_dir):
os.mkdir(FLAGS.log_dir)
labels_dict = load_labels_into_dict(FLAGS.labels_file)
#======================= TRAINING PROCESS =========================
#Now we start to construct the graph and build our model
with tf.Graph().as_default() as graph:
tf.logging.set_verbosity(tf.logging.INFO) #Set the verbosity to INFO level
#####################
# Data Loading #
dataset = get_split('train', FLAGS.dataset_dir, FLAGS.num_classes, FLAGS.labels_file, file_pattern=FLAGS.file_pattern, file_pattern_for_counting=FLAGS.file_pattern_for_counting)
images, _, labels = load_batch(dataset, FLAGS.preprocessing, FLAGS.batch_size, FLAGS.image_size)
num_batches_per_epoch = int(dataset.num_samples / FLAGS.batch_size)
num_steps_per_epoch = num_batches_per_epoch
decay_steps = int(FLAGS.epochs_before_decay * num_steps_per_epoch)
######################
# Select the network #
######################
network_fn = nets_factory.get_network_fn(FLAGS.model_name, num_classes=(dataset.num_classes), weight_decay=FLAGS.weight_decay, is_training=True)
logits, end_points = network_fn(images)
final_tensor = tf.identity(end_points['Predictions'], name="final_result")
tf.summary.histogram('activations', final_tensor)
#Perform one-hot-encoding of the labels (Try one-hot-encoding within the load_batch function!) ?
one_hot_labels = slim.one_hot_encoding(labels, dataset.num_classes)
with tf.name_scope('cross_entropy_loss'):
#Performs the equivalent to tf.nn.sparse__entropy_with_logits but enhanced with checks
loss = tf.losses.softmax_cross_entropy(onehot_labels=one_hot_labels, logits = logits)
#Optionally calculate a weighted loss
if FLAGS.class_weight:
loss = tf.losses.compute_weighted_loss(loss, weights=FLAGS.class_weight)
total_loss = tf.losses.get_total_loss() #obtain the regularization losses as well
#Create the global step for monitoring the learning_rate and training.
global_step = tf.train.get_or_create_global_step()
#Define decaying learning rate
lr = tf.train.exponential_decay(learning_rate = FLAGS.learning_rate,
global_step = global_step,
decay_steps = decay_steps,
decay_rate = FLAGS.learning_rate_decay,
staircase = True)
#TODO: add options to decide optimizer
optimizer = tf.train.AdamOptimizer(learning_rate = lr)
variables_to_train = _get_variables_to_train()
train_op = slim.learning.create_train_op(total_loss, optimizer, variables_to_train=variables_to_train)
accuracy, prediction = _add_evaluation_step(final_tensor, one_hot_labels)
tf.summary.scalar('losses/Total_Loss', total_loss)
tf.summary.scalar('learning_rate', lr)
my_summary_op = tf.summary.merge_all()
sv = tf.train.Supervisor(logdir = FLAGS.log_dir, summary_op = None, init_fn=_get_init_fn())
#Run the managed session
with sv.managed_session() as sess:
for step in range(num_steps_per_epoch * FLAGS.num_epochs):
if step % num_batches_per_epoch == 0:
logging.info('Epoch %s/%s', step/num_batches_per_epoch + 1, FLAGS.num_epochs)
learning_rate_value, accuracy_value = sess.run([lr, accuracy])
logging.info('Current Learning Rate: %s', learning_rate_value)
logging.info('Current Batch Accuracy: %s', accuracy_value)
if FLAGS.save_every_epoch:
sv.saver.save(sess, sv.save_path, global_step = sv.global_step)
if step % FLAGS.log_every_n_steps == 0:
loss, _ = _train_step(sess, train_op, sv.global_step, log=True)
summaries = sess.run(my_summary_op)
sv.summary_computed(sess, summaries)
else:
loss, _ = _train_step(sess, train_op, sv.global_step)
#Log the final training loss and train accuracy
logging.info('Final Loss: %s', loss)
logging.info('Final Train Accuracy: %s', sess.run(accuracy))
logging.info('Finished training! Saving model to disk now {}'.format(sv.save_path))
sv.saver.save(sess, sv.save_path, global_step = sv.global_step)
img_name_vector, cap_vector, test_size=0.2, random_state=0)
# feel free to change these parameters according to your system's configuration
BATCH_SIZE = 64
BUFFER_SIZE = 1000
embedding_dim = 512
units = 512
vocab_size = len(tokenizer.word_index) + 1
num_steps = len(img_name_train) // BATCH_SIZE
# shape of the vector extracted from InceptionV3 is (64, 2048)
# these two variables represent that
features_shape = 2048
attention_features_shape = 65
batch = load_batch(img_name_train, cap_train)
encoder = CNN_Encoder_Adaptive(embedding_dim)
decoder = RNN_Decoder_Adaptive(embedding_dim, units, vocab_size)
checkpoint_path = "./checkpoints/train_adaptive"
ckpt = tf.train.Checkpoint(encoder=encoder, decoder=decoder)
status = ckpt.restore(tf.train.latest_checkpoint(checkpoint_path))
# captions on the validation set
for i in range(len(val_img_name_vector)):
rid = np.random.randint(0, len(val_img_name_vector))
image = val_img_name_vector[rid]
real_caption = ' '.join(
[tokenizer.index_word[i] for i in val_cap_vector[rid] if i not in [0]])
result, attention_plot = evaluate(
cap_vector, tokenizer, max_length = preprocess_caption(train_captions, 5000)
# Create training and validation sets using 80-20 split
img_name_train, img_name_val, cap_train, cap_val = train_test_split(
img_name_vector, cap_vector, test_size=0.1, random_state=0)
vocab_size = len(tokenizer.word_index) + 1
print('vocab_size:' + str(vocab_size))
num_steps = len(img_name_train) // BATCH_SIZE
val_num_steps = len(img_name_val) // BATCH_SIZE
# shape of the vector extracted from InceptionV3 is (64, 2048)
# these two variables represent that
features_shape = 2048
attention_features_shape = 64
dataset = load_batch(img_name_train, cap_train, BATCH_SIZE, BUFFER_SIZE)
val_dataset = load_batch(img_name_val, cap_val, BATCH_SIZE, BUFFER_SIZE)
encoder = CNN_Encoder(embedding_dim)
decoder = RNN_Decoder(embedding_dim, units, vocab_size)
optimizer = tf.train.AdamOptimizer()
checkpoint_path = "./checkpoints/train"
ckpt = tf.train.Checkpoint(encoder=encoder, decoder=decoder)
ckpt_manager = tf.train.CheckpointManager(ckpt,
checkpoint_path,
max_to_keep=50)
start_epoch = 0
if ckpt_manager.latest_checkpoint:
def predict(f_enc, f_dec, samples, batches, mat, beam_size, max_len, header):
'''
Sample words and compute the prediction error
'''
preds = []
errs = []
progress = ProgressBar(numpy.sum([len(batch) for batch in batches]), 20, header)
for batch in batches:
x, mask_x, y, mask_y = load_batch(samples, batch, mat)
[init_h] = f_enc(x, mask_x)
n_steps = mask_x.sum(0)
n_samples = x.shape[1]
prev_sents = numpy.zeros((beam_size, n_samples, max_len), 'int32')
# First step - No embedded word is fed into the decoder
prev_words = numpy.asarray([-1] * n_samples, 'int32')
prev_sents[:, :, 0], prev_log_prob, prev_h = f_dec(prev_words, init_h)
prev_h = numpy.tile(prev_h, (beam_size, 1, 1))
prev_n_ends = n_steps - (prev_sents[:, :, 0] == 0)
for i in range(1, max_len - 1):
hypo_sents = [[]] * n_samples
hypo_log_prob = [[]] * n_samples
hypo_h = [[]] * n_samples
hypo_n_ends = [[]] * n_samples
has_hypos = numpy.asarray([False] * n_samples)
for j in range(beam_size):
if not prev_n_ends[j].any():
continue
next_words, next_log_prob, next_h = f_dec(prev_sents[j, :, i - 1], prev_h[j])
for k in range(n_samples):
if prev_n_ends[j, k] > 0:
has_hypos[k] = True
next_sents = numpy.tile(prev_sents[j, k], (beam_size, 1))
next_sents[:, i] = next_words[:, k]
hypo_sents[k].extend(next_sents)
hypo_log_prob[k].extend(next_log_prob[:, k] + prev_log_prob[j, k])
hypo_h[k].extend([next_h[k]] * beam_size)
hypo_n_ends[k].extend(prev_n_ends[j, k] - (next_words[:, k] == 0))
else:
hypo_sents[k].append(prev_sents[j, k].copy())
hypo_log_prob[k].append(prev_log_prob[j, k])
hypo_h[k].append(prev_h[j, k].copy())
hypo_n_ends[k].append(0)
if not has_hypos.any():
break
for j in range(n_samples):
if not has_hypos[j]:
continue
indices = numpy.argsort(hypo_log_prob[j])[: -beam_size - 1: -1]
for k in range(beam_size):
prev_sents[k, j] = hypo_sents[j][indices[k]]
prev_log_prob[k, j] = hypo_log_prob[j][indices[k]]
prev_h[k, j] = hypo_h[j][indices[k]]
prev_n_ends[k, j] = hypo_n_ends[j][indices[k]]
sents = prev_sents[prev_log_prob.argmax(0), numpy.arange(n_samples)]
for i in range(n_samples):
idx = 0
while idx < max_len and n_steps[i] > 0:
if sents[i, idx] == 0:
n_steps[i] -= 1
idx += 1
preds.append(sents[i, : idx].tolist())
y = numpy.concatenate([y, numpy.zeros((max_len - len(y), n_samples), 'int32')]).T
mask_y = numpy.concatenate([mask_y, numpy.zeros((max_len - len(mask_y), n_samples), 'int32')]).T
errs.extend(((sents != y) * mask_y * 1.).sum(1) / mask_y.sum(1))
progress.disp(errs, ' ERR')
return preds, numpy.mean(errs)