def main():
## Sets
#train_set_path='../Sets/demoset.h5'
#validation_set_path='../Sets/demoset.h5'
#test_set_path='../Sets/demoset.h5'
train_set_path = '../Sets/trainset.h5'
validation_set_path = '../Sets/validationset1.h5'
test_set_path = '../Sets/testset.h5'
## Database location
DataBasePath = '../Database/lines'
transferFLAG = False
testFLAG = False
batch_size = 16
num_epochs = 250
learning_rate = 0.0003
num_epochs_before_validation = 10
restore_checkpoint_at_epoch = 0
import datetime
now = now = datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S")
## Log file
if testFLAG:
indicator = sys.argv[0].split('/')[-1].split('-')[0]
files_path = './train-{}/eval-{}/'.format(indicator, now)
log_path = './train-{}/eval-{}/log/'.format(indicator, now)
log_file_path = './train-{}/eval-{}/log/log.txt'.format(indicator, now)
models_path = './train-{}/models/'.format(indicator)
TensorBoard_dir = './train-{}/eval-{}/TensorBoard_files/'.format(
indicator, now)
tf.gfile.MakeDirs(files_path)
tf.gfile.MakeDirs(log_path)
tf.gfile.MakeDirs(TensorBoard_dir)
copyfile(sys.argv[0], files_path + '{}-'.format(now) + sys.argv[0])
log_file_indicator = initialize_log(log_file_path, mode='w')
elif restore_checkpoint_at_epoch == 0 or transferFLAG:
files_path = './train-{}/'.format(now)
log_path = './train-{}/log'.format(now)
log_file_path = './train-{}/log/log.txt'.format(now)
models_path = './train-{}/models/'.format(now)
TensorBoard_dir = './train-{}/TensorBoard_files/'.format(now)
tf.gfile.MakeDirs(files_path)
tf.gfile.MakeDirs(log_path)
tf.gfile.MakeDirs(models_path)
tf.gfile.MakeDirs(TensorBoard_dir)
copyfile(sys.argv[0], files_path + '{}-'.format(now) + sys.argv[0])
copyfile(sys.argv[0], './{}-'.format(now) + sys.argv[0])
log_file_indicator = initialize_log(log_file_path, mode='w')
else:
indicator = sys.argv[0].split('/')[-1].split('-')[0]
log_path = './train-{}/log'.format(indicator)
log_file_path = './train-{}/log/log.txt'.format(indicator)
models_path = './train-{}/models/'.format(indicator)
TensorBoard_dir = './train-{}/TensorBoard_files/'.format(indicator)
log_file_indicator = initialize_log(log_file_path, mode='a')
log_file_indicator.write(
('#' * 100 + '\n') * 5 +
'\n\nRecovering after break or pause in epoch ' +
str(restore_checkpoint_at_epoch) + '\n\n' + ('#' * 100 + '\n') * 5)
if transferFLAG:
model_for_transfer_path = './model_for_transfer'
num_steps = ceil(num_epochs / num_epochs_before_validation)
validSet, valid_imageHeight, valid_imageWidth, valid_labels = load_dataset(
validation_set_path, DataBasePath, log_file_indicator)
if not testFLAG:
trainSet, train_imageHeight, train_imageWidth, train_labels = load_dataset(
train_set_path, DataBasePath, log_file_indicator)
imageHeight, labels = check_valid_and_test_sets(
train_imageHeight, valid_imageHeight, train_imageHeight,
train_labels, valid_labels, train_labels, log_file_indicator)
train_writer = tf.summary.FileWriter(TensorBoard_dir + 'train_task')
valid_vs_writer = tf.summary.FileWriter(TensorBoard_dir +
'valid_task_validset')
valid_ts_writer = tf.summary.FileWriter(TensorBoard_dir +
'valid_task_trainset')
else:
testSet, test_imageHeight, test_imageWidth, test_labels = load_dataset(
test_set_path, DataBasePath, log_file_indicator)
imageHeight, labels = check_valid_and_test_sets(
test_imageHeight, valid_imageHeight, test_imageHeight, test_labels,
valid_labels, test_labels, log_file_indicator)
test_writer = tf.summary.FileWriter(TensorBoard_dir + 'test_validset')
valid_writer = tf.summary.FileWriter(TensorBoard_dir + 'valid_testset')
log_file_indicator.flush()
# The number of classes is the amount of labels plus 1 for blanck
num_classes = len(labels) + 1
train_start = time.time()
network_train = Network()
if transferFLAG:
epoch = restore_checkpoint_at_epoch
transfer(epoch, network_train, imageHeight, train_imageWidth,
num_classes, log_file_indicator, model_for_transfer_path,
models_path, train_writer)
if not testFLAG:
for step in range(
ceil(restore_checkpoint_at_epoch /
num_epochs_before_validation), num_steps):
train(step, network_train, num_epochs_before_validation,
batch_size, learning_rate, trainSet, imageHeight,
train_imageWidth, num_classes, log_file_indicator,
models_path, train_writer, transferFLAG)
epoch = (step + 1) * num_epochs_before_validation - 1
validation(epoch, network_train, batch_size, 'validation',
validSet, imageHeight, valid_imageWidth, labels,
num_classes, log_file_indicator, models_path,
valid_vs_writer)
validation(epoch, network_train, batch_size, 'train', trainSet,
imageHeight, train_imageWidth, labels, num_classes,
log_file_indicator, models_path, valid_ts_writer)
train_end = time.time()
train_duration = train_end - train_start
print('Training completed in: ' +
seconds_to_days_hours_min_sec(train_duration))
log_file_indicator.write(
'\nTraining completed in: ' +
seconds_to_days_hours_min_sec(train_duration, day_flag=True) +
'\n')
else:
epoch = restore_checkpoint_at_epoch
text = '\nEvaluating model at epoch {}...\n'.format(epoch)
print(text)
log_file_indicator.write(text)
validation(epoch, network_train, batch_size, 'validation', validSet,
imageHeight, valid_imageWidth, labels, num_classes,
log_file_indicator, models_path, valid_writer)
validation(epoch, network_train, batch_size, 'test', testSet,
imageHeight, test_imageWidth, labels, num_classes,
log_file_indicator, models_path, test_writer)
test_end = time.time()
test_duration = test_end - train_start
print('Evaluation completed in: ' +
seconds_to_days_hours_min_sec(test_duration))
log_file_indicator.write(
'\nEvaluation completed in: ' +
seconds_to_days_hours_min_sec(test_duration, day_flag=True) + '\n')
total_parameters = 0
for variable in tf.trainable_variables():
# shape is an array of tf.Dimension
shape = variable.get_shape()
print(shape)
print(len(shape))
variable_parameters = 1
for dim in shape:
print(dim)
variable_parameters *= dim.value
print(variable_parameters)
total_parameters += variable_parameters
print(total_parameters)
if not testFLAG:
train_writer.close()
valid_ts_writer.close()
valid_vs_writer.close()
else:
test_writer.close()
valid_writer.close()
log_file_indicator.flush()
log_file_indicator.close()
def validation(epoch,
network,
batchSize,
set_name,
Set,
imageHeight,
imageWidth,
labels,
num_classes,
log_indicator,
models_path,
valid_writer,
AACHEN_init=False,
AACHEN_h5_file=[],
dataAugmentation=False):
nameList, inputs, targetList, seqLengths, heights, transcriptionList, transcriptionsLenList = Set
SetSize = len(nameList)
n_batches = ceil(SetSize / batchSize)
nameList_copy, inputs_copy, targetList_copy, seqLengths_copy, heights_copy, transcriptionList_copy, transcriptionsLenList_copy = list(
nameList), list(inputs), list(targetList), list(seqLengths), list(
heights), list(transcriptionList), list(transcriptionsLenList)
if dataAugmentation:
inputs_copy = pack_images(inputs_copy, imageHeight, imageWidth)
setTotalChars = np.sum(transcriptionsLenList)
EDnorm = 0
EDabs = 0
totalCost = 0
graph, saver, inputs_mask_ph, seq_len_ph, targets_ph, targets_len_ph, learning_rate_ph, n_batches_ph, setTotalChars_ph, previousEDabs_ph, previousEDnorm_ph, previousCost_ph, optimizer, batch_cost, cost, errors, ED, predictions, merged = network.create(
imageHeight, imageWidth, num_classes, True)
if type(inputs_mask_ph) == list:
mask_ph = inputs_mask_ph[1]
inputs_ph = inputs_mask_ph[0]
else:
inputs_ph = inputs_mask_ph
if type(saver) == list:
saver = saver[0]
with tf.Session(graph=graph) as sess:
if AACHEN_init:
text = ('\nInitializing weights from AACHEN framework\n')
print(text)
log_indicator.write(text)
init = tf.global_variables_initializer()
feed_dict = initialize_from_AACHEN(graph, AACHEN_h5_file,
log_indicator)
sess.run(init, feed_dict=feed_dict)
else:
saver.restore(sess=sess,
save_path=tf.train.latest_checkpoint(models_path))
valid_start = time.time()
prev_percent = -1
text = '\n' * 4 + "Muestras epoch " + str(
epoch) + " in " + set_name + " set.\n"
print(text)
log_indicator.write(text)
word_errors = 0
num_words = 0
for batch in range(n_batches):
BatchNameList, BatchInputs, BatchTargetSparse, BatchSeqLengths, BatchHeights, BatchTranscriptions, BatchTransLen = get_batch(
batchSize, nameList_copy, inputs_copy, targetList_copy,
seqLengths_copy, heights_copy, transcriptionList_copy,
transcriptionsLenList_copy)
feed = {
inputs_ph: BatchInputs,
targets_ph: BatchTargetSparse,
targets_len_ph: BatchTransLen,
seq_len_ph: BatchSeqLengths,
n_batches_ph: n_batches,
setTotalChars_ph: setTotalChars,
previousEDabs_ph: EDabs,
previousEDnorm_ph: EDnorm,
previousCost_ph: totalCost
}
if type(inputs_mask_ph) == list:
mask = np.zeros(
[len(BatchNameList), imageHeight, imageWidth, 1])
for img in range(len(BatchNameList)):
mask[img, :BatchHeights[img], :BatchSeqLengths[img],
0] = np.ones(
[BatchHeights[img], BatchSeqLengths[img]])
feed[mask_ph] = mask
summary, batchCost, totalCost, [
EDnorm, EDabs
], BatchOutpusSparse, errors_output = sess.run(
[merged, batch_cost, cost, ED, predictions[0], errors], feed)
BatchOutput = sess.run(
tf.sparse_tensor_to_dense(tf.SparseTensor(
BatchOutpusSparse.indices, BatchOutpusSparse.values,
BatchOutpusSparse.dense_shape),
default_value=num_classes))
labels[num_classes] = ' '
for ind in range(len(BatchNameList)):
obtained_transcription = ' '.join(
list(map(labels.get, list(BatchOutput[ind])))).strip()
text = str('| Name:').ljust(10) + str(
BatchNameList[ind]
).rjust(15) + ' | ' + str("Target:").ljust(10) + ''.join(
BatchTranscriptions[ind]).rjust(100) + " |\n" + str(
'| Errors: ').ljust(10) + str(
errors_output[ind]).rjust(15) + ' | ' + str(
"Output:").ljust(10) + str(
obtained_transcription).rjust(
100) + ' |\n' + '-' * 88 + '\n'
print(text)
log_indicator.write(text)
word_errors += levenshtein(
''.join(BatchTranscriptions[ind].split()).split('|'),
''.join(obtained_transcription.split()).split('|'))
num_words += len(''.join(
BatchTranscriptions[ind].split()).split('|'))
batch_end = time.time()
time_elapsed = floor(1000 * (batch_end - valid_start)) / 1000
prev_percent = floor(10000 * (batch + 1) / n_batches) / 100
remaining_time = floor(
1000 * (100 * (time_elapsed + eps) /
(prev_percent + eps) - time_elapsed)) / 1000
print('Epoch ' + str(epoch) + '. Evaluated ' +
str(len(BatchNameList)) + ' sequences in batch ' +
str(batch + 1) + '/' + str(n_batches) + '. Cost Function: ' +
str(batchCost) + '.\nTime elapsed: ' +
seconds_to_days_hours_min_sec(time_elapsed) +
'. Remaining time: ' +
seconds_to_days_hours_min_sec(remaining_time) + '\n')
print('[' + int(prev_percent) * '|' +
(100 - int(prev_percent)) * ' ' + '] ' + str(prev_percent) +
'%\n')
WER = word_errors / num_words
valid_writer.add_summary(summary, epoch)
print_valid_results(epoch, set_name, SetSize, totalCost,
[EDnorm, EDabs], WER, log_indicator)
def main():
## Sets
train_set_path = '../Sets/cv1/demo.h5'
validation_set_path = '../Sets/cv1/demo.h5'
test_set_path = '../Sets/cv1/demo.h5'
# train_set_path='../Sets/cv1/train.h5'
# validation_set_path='../Sets/cv1/valid.h5'
# test_set_path='../Sets/cv1/test.h5'
## Database location
DataBasePath = '../Database/washingtondb-v1.0/data/line_images_normalized'
batch_size = 15
num_epochs = 250
learning_rate = 0.0003
num_epochs_before_validation = 10
restore_checkpoint_at_epoch = 0
num_steps = ceil(num_epochs / num_epochs_before_validation)
import datetime
from shutil import copyfile
now = now = datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S")
## Log file
files_path = './train-{}/'.format(now)
log_path = './train-{}/log/'.format(now)
log_file_path = './train-{}/log/log.txt'.format(now)
models_path = './train-{}/models/'.format(now)
TensorBoard_dir = './train-{}/TensorBoard_files/'.format(now)
if restore_checkpoint_at_epoch == 0:
tf.gfile.MakeDirs(files_path)
copyfile(sys.argv[0], files_path + '{}-'.format(now) + sys.argv[0])
tf.gfile.MakeDirs(log_path)
log_file_indicator = initialize_log(log_file_path, mode='w')
if tf.gfile.Exists(TensorBoard_dir):
tf.gfile.DeleteRecursively(TensorBoard_dir)
tf.gfile.MakeDirs(TensorBoard_dir)
if tf.gfile.Exists(models_path):
tf.gfile.DeleteRecursively(models_path)
tf.gfile.MakeDirs(models_path)
else:
log_file_indicator = initialize_log(log_file_path, mode='a')
log_file_indicator.write(
('#' * 100 + '\n') * 5 +
'\n\nRecovering after break or pause in epoch ' +
str(restore_checkpoint_at_epoch) + '\n\n' + ('#' * 100 + '\n') * 5)
trainSet, train_imageHeight, train_imageWidth, train_labels = load_dataset(
train_set_path,
DataBasePath,
log_file_indicator,
database='Washington')
validSet, valid_imageHeight, valid_imageWidth, valid_labels = load_dataset(
validation_set_path,
DataBasePath,
log_file_indicator,
database='Washington')
testSet, test_imageHeight, test_imageWidth, test_labels = load_dataset(
test_set_path, DataBasePath, log_file_indicator, database='Washington')
log_file_indicator.flush()
imageHeight, labels = check_valid_and_test_sets(
train_imageHeight, valid_imageHeight, test_imageHeight, train_labels,
valid_labels, test_labels, log_file_indicator)
# The number of classes is the amount of labels plus 1 for blanck
num_classes = len(labels) + 1
train_start = time.time()
network_train = Network()
train_writer = tf.summary.FileWriter(TensorBoard_dir + 'train_task')
valid_vs_writer = tf.summary.FileWriter(TensorBoard_dir +
'valid_task_validset')
valid_ts_writer = tf.summary.FileWriter(TensorBoard_dir +
'valid_task_trainset')
for step in range(
ceil(restore_checkpoint_at_epoch / num_epochs_before_validation),
num_steps):
train(step, network_train, num_epochs_before_validation, batch_size,
learning_rate, trainSet, imageHeight, train_imageWidth,
num_classes, log_file_indicator, models_path, train_writer)
epoch = (step + 1) * num_epochs_before_validation - 1
validation(epoch, network_train, batch_size, 'validation', validSet,
imageHeight, valid_imageWidth, labels, num_classes,
log_file_indicator, models_path, valid_vs_writer)
validation(epoch, network_train, batch_size, 'train', trainSet,
imageHeight, train_imageWidth, labels, num_classes,
log_file_indicator, models_path, valid_ts_writer)
train_end = time.time()
train_duration = train_end - train_start
print('Training completed in: ' +
seconds_to_days_hours_min_sec(train_duration))
train_writer.close()
valid_ts_writer.close()
valid_vs_writer.close()
log_file_indicator.write(
'\nTraining completed in: ' +
seconds_to_days_hours_min_sec(train_duration, day_flag=True) + '\n')
log_file_indicator.flush()
log_file_indicator.close()
def train(step,
network,
num_epochs,
batchSize,
learning_rate,
trainSet,
imageHeight,
imageWidth,
num_classes,
log_indicator,
models_path,
train_writer,
model_for_transfer_path,
transferFLAG=False,
dataAugmentation=False):
train_nameList, train_inputs, train_targetList, train_seqLengths, train_heights, train_transcriptionList, train_transcriptionsLenList = trainSet
trainSetSize = len(train_nameList)
n_train_batches = ceil(trainSetSize / batchSize)
graph, saver, inputs_mask_ph, seq_len_ph, targets_ph, targets_len_ph, learning_rate_ph, n_batches_ph, setTotalChars_ph, previousEDabs_ph, previousEDnorm_ph, previousCost_ph, optimizer, batch_cost, cost, errors, ED, predictions, merged = network.create(
imageHeight, imageWidth, num_classes, False)
if type(inputs_mask_ph) == list:
mask_ph = inputs_mask_ph[1]
inputs_ph = inputs_mask_ph[0]
else:
inputs_ph = inputs_mask_ph
if type(saver) == list:
saver = saver[0]
with tf.Session(graph=graph) as sess:
if step == 0 and not transferFLAG:
train_writer.add_graph(sess.graph)
text = ('\nTransfering weights from {}\n'.format(
model_for_transfer_path))
print(text)
log_indicator.write(text)
tf.global_variables_initializer().run()
else:
saver.restore(
sess=sess,
save_path=tf.train.latest_checkpoint(model_for_transfer_path))
for epoch in range(num_epochs):
totalCost = 0
epoch_start = time.time()
prev_percent = -1
if dataAugmentation:
train_inputs_copy = distort(train_inputs)
train_inputs_copy = pack_images(train_inputs_copy, imageHeight,
imageWidth)
else:
train_inputs_copy = list(train_inputs)
for batch in range(n_train_batches):
train_nameList_shuffled, train_inputs_shuffled, train_targetList_shuffled, train_seqLengths_shuffled, train_heihts_shuffled, train_transcriptionList_shuffled, train_transcriptionsLenList_shuffled = shuffle_set(
train_nameList, train_inputs_copy, train_targetList,
train_seqLengths, train_heights, train_transcriptionList,
train_transcriptionsLenList)
trainBatchNameList, trainBatchInputs, trainBatchTargetSparse, trainBatchSeqLengths, trainHeights, trainBatchTranscriptions, trainBatchTransLen = get_batch(
batchSize, train_nameList_shuffled, train_inputs_shuffled,
train_targetList_shuffled, train_seqLengths_shuffled,
train_heihts_shuffled, train_transcriptionList_shuffled,
train_transcriptionsLenList_shuffled)
feed = {
inputs_ph: trainBatchInputs,
targets_ph: trainBatchTargetSparse,
targets_len_ph: trainBatchTransLen,
seq_len_ph: trainBatchSeqLengths,
learning_rate_ph: learning_rate,
n_batches_ph: n_train_batches,
previousCost_ph: totalCost
}
if type(inputs_mask_ph) == list:
mask = np.zeros(
[len(trainBatchNameList), imageHeight, imageWidth, 1])
for img in range(len(trainBatchNameList)):
mask[img, :trainHeights[img], :
trainBatchSeqLengths[img], 0] = np.ones([
trainHeights[img], trainBatchSeqLengths[img]
])
feed[mask_ph] = mask
summary, _, batchCost, totalCost = sess.run(
[merged, optimizer, batch_cost, cost], feed)
batch_end = time.time()
time_elapsed = floor(1000 * (batch_end - epoch_start)) / 1000
prev_percent = floor(10000 *
(batch + 1) / n_train_batches) / 100
remaining_time = max([
0,
floor(1000 * (100 * (time_elapsed + eps) /
(prev_percent + eps) - time_elapsed)) / 1000
])
print('Epoch ' + str(epoch + step * num_epochs) +
'. Computed ' + str(len(trainBatchNameList)) +
' sequences in batch ' + str(batch + 1) + '/' +
str(n_train_batches) + '. Cost Function: ' +
str(batchCost) + '.\nTime elapsed: ' +
seconds_to_days_hours_min_sec(time_elapsed) +
'. Remaining time: ' +
seconds_to_days_hours_min_sec(remaining_time) + '\n')
print('[' + int(prev_percent) * '|' +
(100 - int(prev_percent)) * ' ' + '] ' +
str(prev_percent) + '%\n')
epoch_end = time.time()
epoch_duration = epoch_end - epoch_start
train_writer.add_summary(summary, epoch + step * num_epochs)
print(
'Epoch ' + str(epoch + step * num_epochs) + ' completed in: ' +
seconds_to_days_hours_min_sec(epoch_duration, day_flag=False) +
'\n' * 2)
log_indicator.write(
'\nEpoch ' + str(epoch + step * num_epochs) +
' completed in: ' +
seconds_to_days_hours_min_sec(epoch_duration, day_flag=False) +
'. Cost: ' + str(totalCost) + '\n')
log_indicator.flush()
saver.save(sess,
models_path + 'model',
global_step=step * num_epochs + epoch)
