def check_loading(self, num_gpu, is_sorted):
print('----- num_gpu: ' + str(num_gpu) + ', is_sorted: ' +
str(is_sorted) + ' -----')
batch_size = 64
dataset = Dataset(data_type='train',
label_type_main='character',
label_type_sub='phone61',
batch_size=batch_size,
num_stack=3,
num_skip=3,
is_sorted=is_sorted,
is_progressbar=True,
num_gpu=num_gpu)
tf.reset_default_graph()
with tf.Session().as_default() as sess:
print('=> Loading mini-batch...')
map_file_path_char = '../metrics/mapping_files/ctc/char2num.txt'
map_file_path_phone = '../metrics/mapping_files/ctc/phone2num_61.txt'
mini_batch = dataset.next_batch(session=sess)
iter_per_epoch = int(dataset.data_num / (batch_size * num_gpu)) + 1
for i in range(iter_per_epoch + 1):
return_tuple = mini_batch.__next__()
inputs = return_tuple[0]
labels_char_st = return_tuple[1]
labels_phone_st = return_tuple[2]
if num_gpu > 1:
for inputs_gpu in inputs:
print(inputs_gpu.shape)
labels_char_st = labels_char_st[0]
labels_phone_st = labels_phone_st[0]
labels_char = sparsetensor2list(labels_char_st,
batch_size=len(inputs))
labels_phone = sparsetensor2list(labels_phone_st,
batch_size=len(inputs))
if num_gpu == 1:
for inputs_i, labels_i in zip(inputs, labels_char):
if len(inputs_i) < len(labels_i):
print(len(inputs_i))
print(len(labels_i))
raise ValueError
for inputs_i, labels_i in zip(inputs, labels_phone):
if len(inputs_i) < len(labels_i):
print(len(inputs_i))
print(len(labels_i))
raise ValueError
str_true_char = num2char(labels_char[0], map_file_path_char)
str_true_char = re.sub(r'_', ' ', str_true_char)
str_true_phone = num2phone(labels_phone[0],
map_file_path_phone)
def decode_test(session, decode_op, network, dataset, label_type,
save_path=None):
"""Visualize label outputs of CTC model.
Args:
session: session of training model
decode_op: operation for decoding
network: network to evaluate
dataset: An instance of a `Dataset` class
label_type: string, phone39 or phone48 or phone61 or character
save_path: path to save decoding results
"""
# Batch size is expected to be 1
iteration = dataset.data_num
# Make data generator
mini_batch = dataset.next_batch(batch_size=1)
if label_type == 'character':
map_file_path = '../metrics/mapping_files/ctc/char2num.txt'
else:
map_file_path = '../metrics/mapping_files/ctc/phone2num_' + \
label_type[5:7] + '.txt'
if save_path is not None:
sys.stdout = open(join(network.model_dir, 'decode.txt'), 'w')
for step in range(iteration):
# Create feed dictionary for next mini batch
inputs, labels_true_st, inputs_seq_len, input_names = mini_batch.__next__()
feed_dict = {
network.inputs: inputs,
network.inputs_seq_len: inputs_seq_len,
network.keep_prob_input: 1.0,
network.keep_prob_hidden: 1.0
}
# Visualize
labels_pred_st = session.run(decode_op, feed_dict=feed_dict)
labels_true = sparsetensor2list(labels_true_st, batch_size=1)
labels_pred = sparsetensor2list(labels_pred_st, batch_size=1)
if label_type == 'character':
print('----- wav: %s -----' % input_names[0])
print('True: %s' % num2char(
labels_true[0], map_file_path))
print('Pred: %s' % num2char(
labels_pred[0], map_file_path))
else:
print('----- wav: %s -----' % input_names[0])
print('True: %s' % num2phone(
labels_true[0], map_file_path))
print('Pred: %s' % num2phone(
labels_pred[0], map_file_path))
def check_training(self):
print('----- multitask -----')
tf.reset_default_graph()
with tf.Graph().as_default():
# Load batch data
batch_size = 4
inputs, labels_true_char_st, labels_true_phone_st, inputs_seq_len = generate_data(
label_type='multitask', model='ctc', batch_size=batch_size)
# Define placeholders
inputs_pl = tf.placeholder(tf.float32,
shape=[None, None, inputs.shape[-1]],
name='input')
indices_pl = tf.placeholder(tf.int64, name='indices')
values_pl = tf.placeholder(tf.int32, name='values')
shape_pl = tf.placeholder(tf.int64, name='shape')
labels_pl = tf.SparseTensor(indices_pl, values_pl, shape_pl)
indices_second_pl = tf.placeholder(tf.int64, name='indices_second')
values_second_pl = tf.placeholder(tf.int32, name='values_second')
shape_second_pl = tf.placeholder(tf.int64, name='shape_second')
labels_second_pl = tf.SparseTensor(indices_second_pl,
values_second_pl,
shape_second_pl)
inputs_seq_len_pl = tf.placeholder(tf.int64,
shape=[None],
name='inputs_seq_len')
keep_prob_input_pl = tf.placeholder(tf.float32,
name='keep_prob_input')
keep_prob_hidden_pl = tf.placeholder(tf.float32,
name='keep_prob_hidden')
# Define model graph
output_size_main = 26
output_size_second = 61
network = Multitask_BLSTM_CTC(
batch_size=batch_size,
input_size=inputs[0].shape[1],
num_unit=256,
num_layer_main=2,
num_layer_second=1,
output_size_main=output_size_main,
output_size_second=output_size_second,
main_task_weight=0.8,
parameter_init=0.1,
clip_grad=5.0,
clip_activation=50,
dropout_ratio_input=1.0,
dropout_ratio_hidden=1.0,
num_proj=None,
weight_decay=1e-6)
# Add to the graph each operation
loss_op, logits_main, logits_second = network.compute_loss(
inputs_pl, labels_pl, labels_second_pl, inputs_seq_len_pl,
keep_prob_input_pl, keep_prob_hidden_pl)
learning_rate = 1e-3
train_op = network.train(loss_op,
optimizer='rmsprop',
learning_rate_init=learning_rate,
is_scheduled=False)
decode_op_main, decode_op_second = network.decoder(
logits_main,
logits_second,
inputs_seq_len_pl,
decode_type='beam_search',
beam_width=20)
ler_op_main, ler_op_second = network.compute_ler(
decode_op_main, decode_op_second, labels_pl, labels_second_pl)
# Add the variable initializer operation
init_op = tf.global_variables_initializer()
# Count total parameters
parameters_dict, total_parameters = count_total_parameters(
tf.trainable_variables())
for parameter_name in sorted(parameters_dict.keys()):
print("%s %d" %
(parameter_name, parameters_dict[parameter_name]))
print("Total %d variables, %s M parameters" %
(len(parameters_dict.keys()), "{:,}".format(
total_parameters / 1000000)))
# Make feed dict
feed_dict = {
inputs_pl: inputs,
labels_pl: labels_true_char_st,
labels_second_pl: labels_true_phone_st,
inputs_seq_len_pl: inputs_seq_len,
keep_prob_input_pl: network.dropout_ratio_input,
keep_prob_hidden_pl: network.dropout_ratio_hidden,
network.lr: learning_rate
}
with tf.Session() as sess:
# Initialize parameters
sess.run(init_op)
# Wrapper for tfdbg
# sess = tf_debug.LocalCLIDebugWrapperSession(sess)
# Train model
max_steps = 400
start_time_global = time.time()
start_time_step = time.time()
ler_train_char_pre = 1
not_improved_count = 0
for step in range(max_steps):
# Compute loss
_, loss_train = sess.run([train_op, loss_op],
feed_dict=feed_dict)
# Gradient check
# grads = sess.run(network.clipped_grads, feed_dict=feed_dict)
# for grad in grads:
# print(np.max(grad))
if (step + 1) % 10 == 0:
# Change to evaluation mode
feed_dict[keep_prob_input_pl] = 1.0
feed_dict[keep_prob_hidden_pl] = 1.0
# Compute accuracy
ler_train_char, ler_train_phone = sess.run(
[ler_op_main, ler_op_second], feed_dict=feed_dict)
duration_step = time.time() - start_time_step
print(
'Step %d: loss = %.3f / cer = %.4f / per = %.4f (%.3f sec)\n'
% (step + 1, loss_train, ler_train_char,
ler_train_phone, duration_step))
start_time_step = time.time()
# Visualize
labels_pred_char_st, labels_pred_phone_st = sess.run(
[decode_op_main, decode_op_second],
feed_dict=feed_dict)
labels_true_char = sparsetensor2list(
labels_true_char_st, batch_size=batch_size)
labels_true_phone = sparsetensor2list(
labels_true_phone_st, batch_size=batch_size)
labels_pred_char = sparsetensor2list(
labels_pred_char_st, batch_size=batch_size)
labels_pred_phone = sparsetensor2list(
labels_pred_phone_st, batch_size=batch_size)
# character
print('Character')
print(' True: %s' % num2alpha(labels_true_char[0]))
print(' Pred: %s' % num2alpha(labels_pred_char[0]))
print('Phone')
print(' True: %s' % num2phone(labels_true_phone[0]))
print(' Pred: %s' % num2phone(labels_pred_phone[0]))
print('----------------------------------------')
if ler_train_char >= ler_train_char_pre:
not_improved_count += 1
else:
not_improved_count = 0
if not_improved_count >= 5:
print('Modle is Converged.')
break
ler_train_char_pre = ler_train_char
# Change to training mode
network.is_training = True
duration_global = time.time() - start_time_global
print('Total time: %.3f sec' % (duration_global))
def do_eval_per(session,
decode_op,
per_op,
network,
dataset,
label_type,
eval_batch_size=None,
is_progressbar=False,
is_multitask=False):
"""Evaluate trained model by Phone Error Rate.
Args:
session: session of training model
decode_op: operation for decoding
per_op: operation for computing phone error rate
network: network to evaluate
dataset: An instance of a `Dataset' class
label_type: string, phone39 or phone48 or phone61
eval_batch_size: int, the batch size when evaluating the model
is_progressbar: if True, visualize the progressbar
is_multitask: if True, evaluate the multitask model
Returns:
per_mean: An average of PER
"""
if eval_batch_size is not None:
batch_size = eval_batch_size
else:
batch_size = dataset.batch_size
train_label_type = label_type
if is_multitask:
eval_label_type = dataset.label_type_sub
else:
eval_label_type = dataset.label_type
num_examples = dataset.data_num
iteration = int(num_examples / batch_size)
if (num_examples / batch_size) != int(num_examples / batch_size):
iteration += 1
per_mean = 0
# Make data generator
mini_batch = dataset.next_batch(batch_size=batch_size)
train_phone2num_map_file_path = '../metrics/mapping_files/ctc/' + \
train_label_type + '_to_num.txt'
eval_phone2num_map_file_path = '../metrics/mapping_files/ctc/' + \
eval_label_type + '_to_num.txt'
phone2num_39_map_file_path = '../metrics/mapping_files/ctc/phone39_to_num.txt'
phone2phone_map_file_path = '../metrics/mapping_files/phone2phone.txt'
for step in wrap_iterator(range(iteration), is_progressbar):
# Create feed dictionary for next mini batch
if not is_multitask:
inputs, labels_true, inputs_seq_len, _ = mini_batch.__next__()
else:
inputs, _, labels_true, inputs_seq_len, _ = mini_batch.__next__()
feed_dict = {
network.inputs: inputs,
network.inputs_seq_len: inputs_seq_len,
network.keep_prob_input: 1.0,
network.keep_prob_hidden: 1.0
}
batch_size_each = len(inputs_seq_len)
# Evaluate by 39 phones
labels_pred_st = session.run(decode_op, feed_dict=feed_dict)
labels_pred = sparsetensor2list(labels_pred_st, batch_size_each)
labels_pred_mapped, labels_true_mapped = [], []
for i_batch in range(batch_size_each):
###############
# Hypothesis
###############
# Convert from num to phone (-> list of phone strings)
phone_pred_list = num2phone(
labels_pred[i_batch], train_phone2num_map_file_path).split(' ')
# Mapping to 39 phones (-> list of phone strings)
phone_pred_list = map_to_39phone(phone_pred_list, train_label_type,
phone2phone_map_file_path)
# Convert from phone to num (-> list of phone indices)
phone_pred_list = phone2num(phone_pred_list,
phone2num_39_map_file_path)
labels_pred_mapped.append(phone_pred_list)
###############
# Reference
###############
# Convert from num to phone (-> list of phone strings)
phone_true_list = num2phone(
labels_true[i_batch], eval_phone2num_map_file_path).split(' ')
# Mapping to 39 phones (-> list of phone strings)
phone_true_list = map_to_39phone(phone_true_list, eval_label_type,
phone2phone_map_file_path)
# Convert from phone to num (-> list of phone indices)
phone_true_list = phone2num(phone_true_list,
phone2num_39_map_file_path)
labels_true_mapped.append(phone_true_list)
# Compute edit distance
labels_true_st = list2sparsetensor(labels_true_mapped, padded_value=-1)
labels_pred_st = list2sparsetensor(labels_pred_mapped, padded_value=-1)
per_each = compute_edit_distance(session, labels_true_st,
labels_pred_st)
per_mean += per_each * batch_size_each
per_mean /= dataset.data_num
return per_mean
def do_eval_cer(session,
decode_op,
network,
dataset,
eval_batch_size=None,
is_progressbar=False,
is_multitask=False):
"""Evaluate trained model by Character Error Rate.
Args:
session: session of training model
decode_op: operation for decoding
network: network to evaluate
dataset: An instance of a `Dataset` class
eval_batch_size: int, the batch size when evaluating the model
is_progressbar: if True, visualize the progressbar
is_multitask: if True, evaluate the multitask model
Return:
cer_mean: An average of CER
"""
if eval_batch_size is not None:
batch_size = eval_batch_size
else:
batch_size = dataset.batch_size
num_examples = dataset.data_num
iteration = int(num_examples / batch_size)
if (num_examples / batch_size) != int(num_examples / batch_size):
iteration += 1
cer_sum = 0
# Make data generator
mini_batch = dataset.next_batch(batch_size=batch_size)
map_file_path = '../metrics/mapping_files/ctc/character_to_num.txt'
for step in wrap_iterator(range(iteration), is_progressbar):
# Create feed dictionary for next mini batch
if not is_multitask:
inputs, labels_true, inputs_seq_len, _ = mini_batch.__next__()
else:
inputs, labels_true, _, inputs_seq_len, _ = mini_batch.__next__()
feed_dict = {
network.inputs: inputs,
network.inputs_seq_len: inputs_seq_len,
network.keep_prob_input: 1.0,
network.keep_prob_hidden: 1.0
}
batch_size_each = len(inputs_seq_len)
labels_pred_st = session.run(decode_op, feed_dict=feed_dict)
labels_pred = sparsetensor2list(labels_pred_st, batch_size_each)
for i_batch in range(batch_size_each):
# Convert from list to string
str_true = num2char(labels_true[i_batch], map_file_path)
str_pred = num2char(labels_pred[i_batch], map_file_path)
# Remove silence(_) labels
str_true = re.sub(r'[_,.\'-?!]+', "", str_true)
str_pred = re.sub(r'[_,.\'-?!]+', "", str_pred)
# Compute edit distance
cer_each = Levenshtein.distance(str_pred, str_true) / len(
list(str_true))
cer_sum += cer_each
cer_mean = cer_sum / dataset.data_num
return cer_mean
def do_eval_cer(session,
decode_op,
network,
dataset,
label_type,
is_test=None,
eval_batch_size=None,
is_progressbar=False,
is_multitask=False,
is_main=False):
"""Evaluate trained model by Character Error Rate.
Args:
session: session of training model
decode_op: operation for decoding
network: network to evaluate
dataset: An instance of `Dataset` class
label_type: string, kanji or kana or phone
is_test: bool, set to True when evaluating by the test set
eval_batch_size: int, the batch size when evaluating the model
is_progressbar: if True, visualize progressbar
is_multitask: if True, evaluate the multitask model
is_main: if True, evaluate the main task
Return:
cer_mean: An average of CER
"""
if eval_batch_size is None:
batch_size = dataset.batch_size
else:
batch_size = eval_batch_size
num_examples = dataset.data_num
iteration = int(num_examples / batch_size)
if (num_examples / batch_size) != int(num_examples / batch_size):
iteration += 1
cer_sum = 0
# Make data generator
mini_batch = dataset.next_batch(batch_size=batch_size)
if label_type == 'kanji':
map_file_path = '../metrics/mapping_files/ctc/kanji2num.txt'
elif label_type == 'kana':
map_file_path = '../metrics/mapping_files/ctc/kana2num.txt'
elif label_type == 'phone':
map_file_path == '../metrics/mapping_files/ctc/phone2num.txt'
for step in wrap_iterator(range(iteration), is_progressbar):
# Create feed dictionary for next mini batch
if not is_multitask:
inputs, labels_true, inputs_seq_len, _ = mini_batch.__next__()
else:
if is_main:
inputs, labels_true, _, inputs_seq_len, _ = mini_batch.__next__(
)
else:
inputs, _, labels_true, inputs_seq_len, _ = mini_batch.__next__(
)
feed_dict = {
network.inputs: inputs,
network.inputs_seq_len: inputs_seq_len,
network.keep_prob_input: 1.0,
network.keep_prob_hidden: 1.0
}
batch_size_each = len(inputs_seq_len)
labels_pred_st = session.run(decode_op, feed_dict=feed_dict)
labels_pred = sparsetensor2list(labels_pred_st, batch_size_each)
for i_batch in range(batch_size_each):
# Convert from list to string
if label_type != 'phone' and is_test:
str_true = ''.join(labels_true[i_batch])
# NOTE: 漢字とかなの場合はテストデータのラベルはそのまま保存してある
else:
str_true = num2char(labels_true[i_batch], map_file_path)
str_pred = num2char(labels_pred[i_batch], map_file_path)
# Remove silence(_) & noise(NZ) labels
str_true = re.sub(r'[_NZー]+', "", str_true)
str_pred = re.sub(r'[_NZー]+', "", str_pred)
# Compute edit distance
cer_each = Levenshtein.distance(str_pred, str_true) / len(
list(str_true))
cer_sum += cer_each
cer_mean = cer_sum / dataset.data_num
return cer_mean
def check_loading(self, label_type, num_gpu, is_sorted):
print('----- label_type: ' + label_type + ', num_gpu: ' +
str(num_gpu) + ', is_sorted: ' + str(is_sorted) + ' -----')
batch_size = 64
eos_index = 2 if label_type == 'character' else 1
dataset = Dataset(data_type='train',
label_type=label_type,
batch_size=batch_size,
eos_index=eos_index,
is_sorted=is_sorted,
is_progressbar=True,
num_gpu=num_gpu)
tf.reset_default_graph()
with tf.Session().as_default() as sess:
print('=> Loading mini-batch...')
if label_type == 'character':
att_map_file_path = '../metrics/mapping_files/attention/char2num.txt'
ctc_map_file_path = '../metrics/mapping_files/ctc/char2num.txt'
map_fn = num2char
else:
att_map_file_path = '../metrics/mapping_files/attention/phone2num_' + \
label_type[5:7] + '.txt'
ctc_map_file_path = '../metrics/mapping_files/ctc/phone2num_' + \
label_type[5:7] + '.txt'
map_fn = num2phone
mini_batch = dataset.next_batch(session=sess)
iter_per_epoch = int(dataset.data_num / (batch_size * num_gpu)) + 1
for i in range(iter_per_epoch + 1):
return_tuple = mini_batch.__next__()
inputs = return_tuple[0]
att_labels = return_tuple[1]
ctc_labels_st = return_tuple[2]
att_labels_seq_len = return_tuple[4]
if num_gpu > 1:
for inputs_gpu in inputs:
print(inputs_gpu.shape)
inputs = inputs[0]
att_labels = att_labels[0]
ctc_labels_st = ctc_labels_st[0]
att_labels_seq_len = att_labels_seq_len[0]
ctc_labels = sparsetensor2list(ctc_labels_st,
batch_size=len(inputs))
if num_gpu == 1:
for inputs_i, labels_i in zip(inputs, ctc_labels):
if len(inputs_i) < len(labels_i):
print(len(inputs_i))
print(len(labels_i))
raise ValueError
att_str_true = map_fn(att_labels[0][0:att_labels_seq_len[0]],
att_map_file_path)
ctc_str_true = map_fn(ctc_labels[0], ctc_map_file_path)
att_str_true = re.sub(r'_', ' ', att_str_true)
ctc_str_true = re.sub(r'_', ' ', ctc_str_true)