def compute_accuracy(convert, ground_truth_labels, prediction_labels):
ground_truth_str = convert.sparse_tensor_to_str(ground_truth_labels)
predictions_str = convert.sparse_tensor_to_str(prediction_labels)
accuracy = evaluation_tools.compute_accuracy(ground_truth_str,
predictions_str,
mode='full_sequence')
return accuracy
def train_shadownet(dataset_dir, weights_path, char_dict_path, ord_map_dict_path, need_decode=False):
"""
:param dataset_dir:
:param weights_path:
:param char_dict_path:
:param ord_map_dict_path:
:param need_decode:
:return:
"""
# prepare dataset
train_dataset = shadownet_data_feed_pipline.CrnnDataFeeder(
dataset_dir=dataset_dir,
char_dict_path=char_dict_path,
ord_map_dict_path=ord_map_dict_path,
flags='train'
)
val_dataset = shadownet_data_feed_pipline.CrnnDataFeeder(
dataset_dir=dataset_dir,
char_dict_path=char_dict_path,
ord_map_dict_path=ord_map_dict_path,
flags='val'
)
train_images, train_labels, train_images_paths = train_dataset.inputs(
batch_size=CFG.TRAIN.BATCH_SIZE
)
val_images, val_labels, val_images_paths = val_dataset.inputs(
batch_size=CFG.TRAIN.BATCH_SIZE
)
# declare crnn net
shadownet = crnn_net.ShadowNet(
phase='train',
hidden_nums=CFG.ARCH.HIDDEN_UNITS,
layers_nums=CFG.ARCH.HIDDEN_LAYERS,
num_classes=CFG.ARCH.NUM_CLASSES
)
shadownet_val = crnn_net.ShadowNet(
phase='test',
hidden_nums=CFG.ARCH.HIDDEN_UNITS,
layers_nums=CFG.ARCH.HIDDEN_LAYERS,
num_classes=CFG.ARCH.NUM_CLASSES
)
# set up decoder
decoder = tf_io_pipline_fast_tools.CrnnFeatureReader(
char_dict_path=char_dict_path,
ord_map_dict_path=ord_map_dict_path
)
# set up training graph
with tf.device('/gpu:1'):
# compute loss and seq distance
train_inference_ret, train_ctc_loss = shadownet.compute_loss(
inputdata=train_images,
labels=train_labels,
name='shadow_net',
reuse=False
)
val_inference_ret, val_ctc_loss = shadownet_val.compute_loss(
inputdata=val_images,
labels=val_labels,
name='shadow_net',
reuse=True
)
train_decoded, train_log_prob = tf.nn.ctc_beam_search_decoder(
train_inference_ret,
CFG.ARCH.SEQ_LENGTH * np.ones(CFG.TRAIN.BATCH_SIZE),
merge_repeated=False
)
val_decoded, val_log_prob = tf.nn.ctc_beam_search_decoder(
val_inference_ret,
CFG.ARCH.SEQ_LENGTH * np.ones(CFG.TRAIN.BATCH_SIZE),
merge_repeated=False
)
train_sequence_dist = tf.reduce_mean(
tf.edit_distance(tf.cast(train_decoded[0], tf.int32), train_labels),
name='train_edit_distance'
)
val_sequence_dist = tf.reduce_mean(
tf.edit_distance(tf.cast(val_decoded[0], tf.int32), val_labels),
name='val_edit_distance'
)
# set learning rate
global_step = tf.Variable(0, name='global_step', trainable=False)
learning_rate = tf.train.exponential_decay(
learning_rate=CFG.TRAIN.LEARNING_RATE,
global_step=global_step,
decay_steps=CFG.TRAIN.LR_DECAY_STEPS,
decay_rate=CFG.TRAIN.LR_DECAY_RATE,
staircase=CFG.TRAIN.LR_STAIRCASE)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
optimizer = tf.train.MomentumOptimizer(
learning_rate=learning_rate, momentum=0.9).minimize(
loss=train_ctc_loss, global_step=global_step)
# Set tf summary
tboard_save_dir = 'tboard/crnn_syn90k'
os.makedirs(tboard_save_dir, exist_ok=True)
tf.summary.scalar(name='train_ctc_loss', tensor=train_ctc_loss)
tf.summary.scalar(name='val_ctc_loss', tensor=val_ctc_loss)
tf.summary.scalar(name='learning_rate', tensor=learning_rate)
if need_decode:
tf.summary.scalar(name='train_seq_distance', tensor=train_sequence_dist)
tf.summary.scalar(name='val_seq_distance', tensor=val_sequence_dist)
merge_summary_op = tf.summary.merge_all()
# Set saver configuration
saver = tf.train.Saver()
model_save_dir = 'model/crnn_syn90k'
os.makedirs(model_save_dir, exist_ok=True)
#train_start_time = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime(time.time()))
#model_name = 'shadownet_{:s}.ckpt'.format(str(train_start_time))
model_name = 'shadownet.ckpt'
model_save_path = ops.join(model_save_dir, model_name)
# Set sess configuration
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TRAIN.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess = tf.Session(config=sess_config)
summary_writer = tf.summary.FileWriter(tboard_save_dir)
summary_writer.add_graph(sess.graph)
# Set the training parameters
train_epochs = CFG.TRAIN.EPOCHS
with sess.as_default():
epoch = 0
if weights_path is None:
logger.info('Training from scratch')
init = tf.global_variables_initializer()
sess.run(init)
else:
logger.info('Restore model from {:s}'.format(weights_path))
saver.restore(sess=sess, save_path=weights_path)
epoch = sess.run(tf.train.get_global_step())
patience_counter = 1
cost_history = [np.inf]
while epoch < train_epochs:
epoch += 1
# setup early stopping
if epoch > 1 and CFG.TRAIN.EARLY_STOPPING:
# We always compare to the first point where cost didn't improve
if cost_history[-1 - patience_counter] - cost_history[-1] > CFG.TRAIN.PATIENCE_DELTA:
patience_counter = 1
else:
patience_counter += 1
if patience_counter > CFG.TRAIN.PATIENCE_EPOCHS:
logger.info("Cost didn't improve beyond {:f} for {:d} epochs, stopping early.".
format(CFG.TRAIN.PATIENCE_DELTA, patience_counter))
break
if need_decode and epoch % 500 == 0:
# train part
_, train_ctc_loss_value, train_seq_dist_value, \
train_predictions, train_labels_sparse, merge_summary_value = sess.run(
[optimizer, train_ctc_loss, train_sequence_dist,
train_decoded, train_labels, merge_summary_op])
train_labels_str = decoder.sparse_tensor_to_str(train_labels_sparse)
train_predictions = decoder.sparse_tensor_to_str(train_predictions[0])
avg_train_accuracy = evaluation_tools.compute_accuracy(train_labels_str, train_predictions)
if epoch % CFG.TRAIN.DISPLAY_STEP == 0:
logger.info('Epoch_Train: {:d} cost= {:9f} seq distance= {:9f} train accuracy= {:9f}'.format(
epoch + 1, train_ctc_loss_value, train_seq_dist_value, avg_train_accuracy))
# validation part
val_ctc_loss_value, val_seq_dist_value, \
val_predictions, val_labels_sparse = sess.run(
[val_ctc_loss, val_sequence_dist, val_decoded, val_labels])
val_labels_str = decoder.sparse_tensor_to_str(val_labels_sparse)
val_predictions = decoder.sparse_tensor_to_str(val_predictions[0])
avg_val_accuracy = evaluation_tools.compute_accuracy(val_labels_str, val_predictions)
if epoch % CFG.TRAIN.DISPLAY_STEP == 0:
logger.info('Epoch_Val: {:d} cost= {:9f} seq distance= {:9f} train accuracy= {:9f}'.format(
epoch + 1, val_ctc_loss_value, val_seq_dist_value, avg_val_accuracy))
else:
_, train_ctc_loss_value, merge_summary_value = sess.run(
[optimizer, train_ctc_loss, merge_summary_op])
if epoch % CFG.TRAIN.DISPLAY_STEP == 0:
logger.info('Epoch_Train: {:d} cost= {:9f}'.format(epoch + 1, train_ctc_loss_value))
# record history train ctc loss
cost_history.append(train_ctc_loss_value)
# add training sumary
summary_writer.add_summary(summary=merge_summary_value, global_step=epoch)
if epoch % 2000 == 0:
saver.save(sess=sess, save_path=model_save_path, global_step=epoch)
return np.array(cost_history[1:]) # Don't return the first np.inf
def evaluate_shadownet(dataset_dir,
weights_path,
char_dict_path,
ord_map_dict_path,
is_visualize=False,
is_process_all_data=False):
"""
:param dataset_dir:
:param weights_path:
:param char_dict_path:
:param ord_map_dict_path:
:param is_visualize:
:param is_process_all_data:
:return:
"""
# prepare dataset
test_dataset = shadownet_data_feed_pipline.CrnnDataFeeder(
dataset_dir=dataset_dir,
char_dict_path=char_dict_path,
ord_map_dict_path=ord_map_dict_path,
flags='test')
test_images, test_labels, test_images_paths = test_dataset.inputs(
batch_size=CFG.TEST.BATCH_SIZE)
# set up test sample count
if is_process_all_data:
log.info('Start computing test dataset sample counts')
t_start = time.time()
test_sample_count = test_dataset.sample_counts()
log.info(
'Computing test dataset sample counts finished, cost time: {:.5f}'.
format(time.time() - t_start))
num_iterations = int(math.ceil(test_sample_count /
CFG.TEST.BATCH_SIZE))
else:
num_iterations = 1
# declare crnn net
shadownet = crnn_net.ShadowNet(phase='test',
hidden_nums=CFG.ARCH.HIDDEN_UNITS,
layers_nums=CFG.ARCH.HIDDEN_LAYERS,
num_classes=CFG.ARCH.NUM_CLASSES)
# set up decoder
decoder = tf_io_pipline_fast_tools.CrnnFeatureReader(
char_dict_path=char_dict_path, ord_map_dict_path=ord_map_dict_path)
# compute inference result
test_inference_ret = shadownet.inference(inputdata=test_images,
name='shadow_net',
reuse=False)
test_decoded, test_log_prob = tf.nn.ctc_beam_search_decoder(
test_inference_ret,
CFG.ARCH.SEQ_LENGTH * np.ones(CFG.TEST.BATCH_SIZE),
beam_width=1,
merge_repeated=False)
# recover image from [-1.0, 1.0] ---> [0.0, 255.0]
test_images = tf.multiply(tf.add(test_images, 1.0),
127.5,
name='recoverd_test_images')
# Set saver configuration
saver = tf.train.Saver()
# Set sess configuration
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TRAIN.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess = tf.Session(config=sess_config)
with sess.as_default():
saver.restore(sess=sess, save_path=weights_path)
log.info('Start predicting...')
per_char_accuracy = 0.0
full_sequence_accuracy = 0.0
total_labels_char_list = []
total_predictions_char_list = []
while True:
try:
for epoch in range(num_iterations):
test_predictions_value, test_images_value, test_labels_value, \
test_images_paths_value = sess.run(
[test_decoded, test_images, test_labels, test_images_paths])
test_images_paths_value = np.reshape(
test_images_paths_value,
newshape=test_images_paths_value.shape[0])
test_images_paths_value = [
tmp.decode('utf-8') for tmp in test_images_paths_value
]
test_images_names_value = [
ops.split(tmp)[1] for tmp in test_images_paths_value
]
test_labels_value = decoder.sparse_tensor_to_str(
test_labels_value)
test_predictions_value = decoder.sparse_tensor_to_str(
test_predictions_value[0])
per_char_accuracy += evaluation_tools.compute_accuracy(
test_labels_value,
test_predictions_value,
display=False,
mode='per_char')
full_sequence_accuracy += evaluation_tools.compute_accuracy(
test_labels_value,
test_predictions_value,
display=False,
mode='full_sequence')
for index, test_image in enumerate(test_images_value):
log.info(
'Predict {:s} image with gt label: {:s} **** predicted label: {:s}'
.format(test_images_names_value[index],
test_labels_value[index],
test_predictions_value[index]))
if is_visualize:
plt.imshow(
np.array(test_image, np.uint8)[:, :,
(2, 1, 0)])
plt.show()
test_labels_char_list_value = [
s for s in test_labels_value[index]
]
test_predictions_char_list_value = [
s for s in test_predictions_value[index]
]
if not test_labels_char_list_value or not test_predictions_char_list_value:
continue
if len(test_labels_char_list_value) != len(
test_predictions_char_list_value):
min_length = min(
len(test_labels_char_list_value),
len(test_predictions_char_list_value))
test_labels_char_list_value = test_labels_char_list_value[:
min_length
-
1]
test_predictions_char_list_value = test_predictions_char_list_value[:
min_length
-
1]
assert len(test_labels_char_list_value) == len(test_predictions_char_list_value), \
log.error('{}, {}'.format(test_labels_char_list_value, test_predictions_char_list_value))
total_labels_char_list.extend(
test_labels_char_list_value)
total_predictions_char_list.extend(
test_predictions_char_list_value)
if is_visualize:
plt.imshow(
np.array(test_image, np.uint8)[:, :,
(2, 1, 0)])
except tf.errors.OutOfRangeError:
log.error('End of tfrecords sequence')
break
except Exception as err:
log.error(err)
break
avg_per_char_accuracy = per_char_accuracy / num_iterations
avg_full_sequence_accuracy = full_sequence_accuracy / num_iterations
log.info('Mean test per char accuracy is {:5f}'.format(
avg_per_char_accuracy))
log.info('Mean test full sequence accuracy is {:5f}'.format(
avg_full_sequence_accuracy))
# compute confusion matrix
cnf_matrix = confusion_matrix(total_labels_char_list,
total_predictions_char_list)
np.set_printoptions(precision=2)
evaluation_tools.plot_confusion_matrix(cm=cnf_matrix, normalize=True)
plt.show()
def evaluate_shadownet(dataset_dir,
weights_path,
char_dict_path,
ord_map_dict_path,
is_visualize=False,
is_process_all_data=False):
"""
:param dataset_dir:
:param weights_path:
:param char_dict_path:
:param ord_map_dict_path:
:param is_visualize:
:param is_process_all_data:
:return:
"""
# prepare dataset
test_dataset = shadownet_data_feed_pipline.CrnnDataFeeder(
dataset_dir=dataset_dir,
char_dict_path=char_dict_path,
ord_map_dict_path=ord_map_dict_path,
flags='test')
test_images, test_labels, test_images_paths = test_dataset.inputs(
batch_size=CFG.TEST.BATCH_SIZE, num_epochs=1)
# set up test sample count
if is_process_all_data:
log.info('Start computing test dataset sample counts')
t_start = time.time()
test_sample_count = test_dataset.sample_counts()
log.info(
'Computing test dataset sample counts finished, cost time: {:.5f}'.
format(time.time() - t_start))
num_iterations = int(math.ceil(test_sample_count /
CFG.TEST.BATCH_SIZE))
else:
num_iterations = 1
# declare crnn net
shadownet = crnn_model.ShadowNet(phase='test',
hidden_nums=CFG.ARCH.HIDDEN_UNITS,
layers_nums=CFG.ARCH.HIDDEN_LAYERS,
num_classes=CFG.ARCH.NUM_CLASSES)
# set up decoder
decoder = tf_io_pipline_tools.TextFeatureIO(
char_dict_path=char_dict_path,
ord_map_dict_path=ord_map_dict_path).reader
# compute inference result
test_inference_ret = shadownet.inference(inputdata=test_images,
name='shadow_net',
reuse=False)
test_decoded, test_log_prob = tf.nn.ctc_beam_search_decoder(
test_inference_ret,
CFG.ARCH.SEQ_LENGTH * np.ones(CFG.TEST.BATCH_SIZE),
beam_width=1,
merge_repeated=False)
# recover image from [-1.0, 1.0] ---> [0.0, 255.0]
test_images = tf.multiply(tf.add(test_images, 1.0),
127.5,
name='recoverd_test_images')
# Set saver configuration
saver = tf.train.Saver()
# Set sess configuration
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TRAIN.GPU_MEMORY_FRACTION
sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
sess = tf.Session(config=sess_config)
with sess.as_default():
saver.restore(sess=sess, save_path=weights_path)
log.info('Start predicting...')
accuracy = 0
while True:
try:
for epoch in range(num_iterations):
test_predictions_value, test_images_value, test_labels_value, \
test_images_paths_value = sess.run(
[test_decoded, test_images, test_labels, test_images_paths]
)
test_images_paths_value = np.reshape(
test_images_paths_value,
newshape=test_images_paths_value.shape[0])
test_images_paths_value = [
tmp.decode('utf-8') for tmp in test_images_paths_value
]
test_images_names_value = [
ops.split(tmp)[1] for tmp in test_images_paths_value
]
test_labels_value = decoder.sparse_tensor_to_str(
test_labels_value)
test_predictions_value = decoder.sparse_tensor_to_str(
test_predictions_value[0])
accuracy += evaluation_tools.compute_accuracy(
test_labels_value,
test_predictions_value,
display=False)
for index, test_image in enumerate(test_images_value):
print(
'Predict {:s} image with gt label: {:s} **** predicted label: {:s}'
.format(test_images_names_value[index],
test_labels_value[index],
test_predictions_value[index]))
# avoid accidentally displaying for the whole dataset
if is_visualize:
plt.imshow(
np.array(test_image, np.uint8)[:, :,
(2, 1, 0)])
plt.show()
except tf.errors.OutOfRangeError:
print('End of tfrecords sequence')
break
except Exception as err:
print(err)
break
# we compute a mean of means, so we need the sample sizes to be constant
# (BATCH_SIZE) for this to equal the actual mean
accuracy /= num_iterations
log.info('Mean test accuracy is {:5f}'.format(accuracy))