def main(_):
initialize_globals()
if not FLAGS.test_files:
log_error('You need to specify what files to use for evaluation via '
'the --test_files flag.')
exit(1)
global alphabet
alphabet = Alphabet(FLAGS.alphabet_config_path)
# sort examples by length, improves packing of batches and timesteps
test_data = preprocess(
FLAGS.test_files.split(','),
FLAGS.test_batch_size,
alphabet=alphabet,
numcep=Config.n_input,
numcontext=Config.n_context,
hdf5_cache_path=FLAGS.hdf5_test_set).sort_values(
by="features_len",
ascending=False)
from DeepSpeech import create_inference_graph
graph = create_inference_graph(batch_size=FLAGS.test_batch_size, n_steps=-1)
samples = evaluate(test_data, graph, alphabet)
if FLAGS.test_output_file:
# Save decoded tuples as JSON, converting NumPy floats to Python floats
json.dump(samples, open(FLAGS.test_output_file, 'w'), default=lambda x: float(x))
def main(_):
initialize_globals()
if not FLAGS.test_files:
log_error('You need to specify what files to use for evaluation via '
'the --test_files flag.')
exit(1)
#if FLAGS.embeddings_output_dir:
# prefix = FLAGS.embeddings_output_dir
# print('Prefix :', prefix)
# #print('LAYER4 :', LAYER4)
# EMBEDDINGS = prefix + 'embeddings/'
# LAYER4 = EMBEDDINGS + 'layer4/'
# LAYER5 = EMBEDDINGS + 'layer5/'
# LAYER6 = EMBEDDINGS + 'layer6/'
# c.TEXT = EMBEDDINGS + 'text/'
# print('LAYER4 :', LAYER4)
# sort examples by length, improves packing of batches and timesteps
test_data = preprocess(FLAGS.test_files.split(','),
FLAGS.test_batch_size,
alphabet=Config.alphabet,
numcep=Config.n_input,
numcontext=Config.n_context,
hdf5_cache_path=FLAGS.hdf5_test_set).sort_values(
by="features_len", ascending=False)
#print('test_data', test_data)
#print(test_data.fname[1])
#return 1
#print(test_data[0].fname)
print('Batch Size: ', FLAGS.test_batch_size)
from DeepSpeech import create_inference_graph
graph = create_inference_graph(batch_size=FLAGS.test_batch_size,
n_steps=-1)
samples = evaluate(test_data, graph)
if FLAGS.test_output_file:
# Save decoded tuples as JSON, converting NumPy floats to Python floats
json.dump(samples,
open(FLAGS.test_output_file, 'w'),
default=lambda x: float(x))
def main(_):
initialize_globals()
if not FLAGS.test_files:
log_error('You need to specify what files to use for evaluation via '
'the --test_files flag.')
exit(1)
global alphabet
alphabet = Alphabet(FLAGS.alphabet_config_path)
scorer = Scorer(FLAGS.lm_weight, FLAGS.valid_word_count_weight,
FLAGS.lm_binary_path, FLAGS.lm_trie_path,
alphabet)
# sort examples by length, improves packing of batches and timesteps
test_data = preprocess(
FLAGS.test_files.split(','),
FLAGS.test_batch_size,
alphabet=alphabet,
numcep=N_FEATURES,
numcontext=N_CONTEXT,
hdf5_cache_path=FLAGS.hdf5_test_set).sort_values(
by="features_len",
ascending=False)
def create_windows(features):
num_strides = len(features) - (N_CONTEXT * 2)
# Create a view into the array with overlapping strides of size
# numcontext (past) + 1 (present) + numcontext (future)
window_size = 2*N_CONTEXT+1
features = np.lib.stride_tricks.as_strided(
features,
(num_strides, window_size, N_FEATURES),
(features.strides[0], features.strides[0], features.strides[1]),
writeable=False)
return features
# Create overlapping windows over the features
test_data['features'] = test_data['features'].apply(create_windows)
with tf.Session() as session:
inputs, outputs, layers = create_inference_graph(batch_size=FLAGS.test_batch_size, n_steps=-1)
# Transpose to batch major for decoder
transposed = tf.transpose(outputs['outputs'], [1, 0, 2])
labels_ph = tf.placeholder(tf.int32, [FLAGS.test_batch_size, None], name="labels")
label_lengths_ph = tf.placeholder(tf.int32, [FLAGS.test_batch_size], name="label_lengths")
sparse_labels = tf.cast(ctc_label_dense_to_sparse(labels_ph, label_lengths_ph, FLAGS.test_batch_size), tf.int32)
loss = tf.nn.ctc_loss(labels=sparse_labels,
inputs=layers['raw_logits'],
sequence_length=inputs['input_lengths'])
# Create a saver using variables from the above newly created graph
mapping = {v.op.name: v for v in tf.global_variables() if not v.op.name.startswith('previous_state_')}
saver = tf.train.Saver(mapping)
# Restore variables from training checkpoint
checkpoint = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir)
if not checkpoint:
log_error('Checkpoint directory ({}) does not contain a valid checkpoint state.'.format(FLAGS.checkpoint_dir))
exit(1)
checkpoint_path = checkpoint.model_checkpoint_path
saver.restore(session, checkpoint_path)
logitses = []
losses = []
print('Computing acoustic model predictions...')
batch_count = len(test_data) // FLAGS.test_batch_size
bar = progressbar.ProgressBar(max_value=batch_count,
widget=progressbar.AdaptiveETA)
# First pass, compute losses and transposed logits for decoding
for batch in bar(split_data(test_data, FLAGS.test_batch_size)):
session.run(outputs['initialize_state'])
features = pad_to_dense(batch['features'].values)
features_len = batch['features_len'].values
labels = pad_to_dense(batch['transcript'].values)
label_lengths = batch['transcript_len'].values
logits, loss = session.run([transposed, loss], feed_dict={
inputs['input']: features,
inputs['input_lengths']: features_len,
labels_ph: labels,
label_lengths_ph: label_lengths
})
logitses.append(logits)
losses.extend(loss)
ground_truths = []
predictions = []
distances = []
print('Decoding predictions...')
bar = progressbar.ProgressBar(max_value=batch_count,
widget=progressbar.AdaptiveETA)
# Get number of accessible CPU cores for this process
num_processes = len(os.sched_getaffinity(0))
# Second pass, decode logits and compute WER and edit distance metrics
for logits, batch in bar(zip(logitses, split_data(test_data, FLAGS.test_batch_size))):
seq_lengths = batch['features_len'].values.astype(np.int32)
decoded = ctc_beam_search_decoder_batch(logits, seq_lengths, alphabet, FLAGS.beam_width,
num_processes=num_processes, scorer=scorer)
ground_truths.extend(alphabet.decode(l) for l in batch['transcript'])
predictions.extend(d[0][1] for d in decoded)
distances.extend(levenshtein(a, b) for a, b in zip(labels, predictions))
wer, samples = calculate_report(ground_truths, predictions, distances, losses)
mean_edit_distance = np.mean(distances)
mean_loss = np.mean(losses)
# Take only the first report_count items
report_samples = itertools.islice(samples, FLAGS.report_count)
print('Test - WER: %f, loss: %f, mean edit distance: %f' %
(wer, mean_loss, mean_edit_distance))
print('-' * 80)
for sample in report_samples:
print('WER: %f, loss: %f, edit distance: %f' %
(sample.wer, sample.loss, sample.distance))
print(' - src: "%s"' % sample.src)
print(' - res: "%s"' % sample.res)
print('-' * 80)
if FLAGS.test_output_file:
json.dump(samples, open(FLAGS.test_output_file, 'w'), default=lambda x: float(x))
def activations_common_voice_pertubed_sets(input_dir,
output_dir,
test_only=False,
prune_percentage=0,
scores_file=None,
random=False,
verbose=True,
randomly_initialized=False):
'''Obtains activations for wavs in input_dir and saves them to output_dir'''
inputs, outputs, layers = create_inference_graph(batch_size=1, n_steps=-1)
intermediate_layer_names = [
'layer_1', 'layer_2', 'layer_3', 'rnn_output', 'layer_4', 'layer_5'
]
intermediate_layers = [
l for n, l in layers.items() if n in intermediate_layer_names
]
pertubed_sets = json.load(open('data/pertubed_input_sets_balanced.json'))
skip_sets = []
if test_only: skip_sets = json.load(open('./results/set_ids_used.json'))
if not prune_percentage: base_path = '{}/activations'.format(output_dir)
else:
base_path = '{}/activations/pruned-{}'.format(output_dir,
prune_percentage * 100)
if random: base_path += '-random'
with tfv1.Session(config=Config.session_config) as session:
# Create a saver using variables from the above newly created graph
if not randomly_initialized:
saver = tfv1.train.Saver()
# Restore variables from training checkpoint
loaded = False
if not loaded and FLAGS.load in ['auto', 'last']:
loaded = try_loading(session,
saver,
'checkpoint',
'most recent',
load_step=False)
if not loaded and FLAGS.load in ['auto', 'best']:
loaded = try_loading(session,
saver,
'best_dev_checkpoint',
'best validation',
load_step=False)
if not loaded:
print('Could not load checkpoint from {}'.format(
FLAGS.checkpoint_dir))
sys.exit(1)
else:
initializer = tfv1.global_variables_initializer()
session.run(initializer)
###### PRUNING PART ######
if verbose:
if not prune_percentage: print('No pruning done.')
else:
if verbose: print('-' * 80)
if verbose: print('pruning with {}%...'.format(prune_percentage))
scores_per_layer = np.load(scores_file)
layer_masks = prune_matrices(scores_per_layer,
prune_percentage=prune_percentage,
random=random,
verbose=verbose,
skip_lstm=False)
n_layers_to_prune = len(layer_masks)
i = 0
for index, v in enumerate(tf.trainable_variables()):
lstm_layer_name = 'cudnn_lstm/rnn/multi_rnn_cell/cell_0/cudnn_compatible_lstm_cell/kernel:0'
if 'weights' not in v.name and v.name != lstm_layer_name:
continue
if (i >= n_layers_to_prune):
break # if i < total_ops, it is not yet the last layer
# make mask into the shape of the weights
if v.name == lstm_layer_name:
if skip_lstm: continue
# Shape of LSTM weights: [(2*neurons), (4*neurons)]
cell_template = np.ones((2, 4))
mask = np.repeat(layer_masks[i], v.shape[0] // 2, axis=0)
mask = mask.reshape(
[layer_masks[i].shape[0], v.shape[0] // 2])
mask = np.swapaxes(mask, 0, 1)
mask = np.kron(mask, cell_template)
else:
idx = layer_masks[i] == 1
mask = np.repeat(layer_masks[i], v.shape[0], axis=0)
mask = mask.reshape([layer_masks[i].shape[0], v.shape[0]])
mask = np.swapaxes(mask, 0, 1)
# apply mask to weights
session.run(v.assign(tf.multiply(v, mask)))
i += 1
###### END PRUNING PART ######
# Default states for LSTM cell
previous_state_c = np.zeros([1, Config.n_cell_dim])
previous_state_h = np.zeros([1, Config.n_cell_dim])
sets_to_process = [
set for set in pertubed_sets if str(set['set_id']) not in skip_sets
]
print('{} sets found'.format(len(sets_to_process)))
for set in sets_to_process:
print('Processing set {}, {} items...'.format(
set['set_id'], set['set_length']))
# Only process files that are not yet available in results directory
create_dir_if_not_exists('{}/{}'.format(
base_path, set['set_id'])) # Check if directory exists
files_done = [
f[:-4]
for f in os.listdir('{}/{}'.format(base_path, set['set_id']))
if f.endswith('.npy')
]
for item in set['set_items']:
file_name = item['path'][:-4]
print(file_name)
if file_name in files_done:
print('Skipped.')
continue
print('current file: {}'.format(file_name))
input_file_path = '{}/{}.wav'.format(input_dir, file_name)
# Prepare features
features, features_len = audiofile_to_features(input_file_path)
features = tf.expand_dims(features, 0)
features_len = tf.expand_dims(features_len, 0)
features = create_overlapping_windows(features).eval(
session=session)
features_len = features_len.eval(session=session)
feed_dict = {
inputs['input']: features,
inputs['input_lengths']: features_len,
inputs['previous_state_c']: previous_state_c,
inputs['previous_state_h']: previous_state_h,
}
intermediate_activations = session.run(intermediate_layers,
feed_dict=feed_dict)
# Save activations of actual input
save_to_path_activations = '{}/{}/{}.npy'.format(
base_path, set['set_id'], file_name)
write_numpy_to_file(save_to_path_activations,
np.array(intermediate_activations))
print('Activations for {} are saved to: {}'.format(
file_name, save_to_path_activations))
return True
def main(_):
initialize_globals()
if not FLAGS.test_files:
log_error('You need to specify what files to use for evaluation via '
'the --test_files flag.')
exit(1)
global alphabet
alphabet = Alphabet(os.path.abspath(FLAGS.alphabet_config_path))
# sort examples by length, improves packing of batches and timesteps
test_data = preprocess(FLAGS.test_files.split(','),
FLAGS.test_batch_size,
alphabet=alphabet,
numcep=N_FEATURES,
numcontext=N_CONTEXT,
hdf5_cache_path=FLAGS.hdf5_test_set).sort_values(
by="features_len", ascending=False)
def create_windows(features):
num_strides = len(features) - (N_CONTEXT * 2)
# Create a view into the array with overlapping strides of size
# numcontext (past) + 1 (present) + numcontext (future)
window_size = 2 * N_CONTEXT + 1
features = np.lib.stride_tricks.as_strided(
features, (num_strides, window_size, N_FEATURES),
(features.strides[0], features.strides[0], features.strides[1]),
writeable=False)
return features
test_data['features'] = test_data['features'].apply(create_windows)
with tf.Session() as session:
inputs, outputs = create_inference_graph(
batch_size=FLAGS.test_batch_size, n_steps=N_STEPS)
seq_lengths_ph = tf.placeholder(tf.int32, [FLAGS.test_batch_size])
decode_logits_ph = tf.placeholder(
tf.float32, [None, FLAGS.test_batch_size,
alphabet.size() + 1])
labels_ph = tf.placeholder(tf.int32, [FLAGS.test_batch_size, None])
label_lengths_ph = tf.placeholder(tf.int32, [FLAGS.test_batch_size])
decoded, _ = decode_with_lm(decode_logits_ph,
seq_lengths_ph,
merge_repeated=False,
beam_width=FLAGS.beam_width)
sparse_labels = tf.cast(
ctc_label_dense_to_sparse(labels_ph, label_lengths_ph,
FLAGS.test_batch_size), tf.int32)
loss = tf.nn.ctc_loss(labels=sparse_labels,
inputs=decode_logits_ph,
sequence_length=seq_lengths_ph)
distance = tf.edit_distance(tf.cast(decoded[0], tf.int32),
sparse_labels)
# Create a saver using variables from the above newly created graph
mapping = {
v.op.name: v
for v in tf.global_variables()
if not v.op.name.startswith('previous_state_')
}
saver = tf.train.Saver(mapping)
# Restore variables from training checkpoint
checkpoint = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir)
if not checkpoint:
log_error(
'Checkpoint directory ({}) does not contain a valid checkpoint state.'
.format(FLAGS.checkpoint_dir))
exit(1)
checkpoint_path = checkpoint.model_checkpoint_path
saver.restore(session, checkpoint_path)
logitses = []
batch_count = len(test_data) // FLAGS.test_batch_size
bar = progressbar.ProgressBar(max_value=batch_count - 1,
widget=progressbar.AdaptiveETA)
for batch in bar(split_data(test_data, FLAGS.test_batch_size)):
session.run(outputs['initialize_state'])
batch_features = pad_to_dense(batch['features'].values)
batch_features_len = batch['features_len'].values
full_step_len = np.full_like(batch_features_len, N_STEPS)
logits = np.empty([0, FLAGS.test_batch_size, alphabet.size() + 1])
for i in range(0, batch_features.shape[1], N_STEPS):
chunk_features = batch_features[:, i:i + N_STEPS, :, :]
chunk_features_len = np.minimum(batch_features_len,
full_step_len)
# pad with zeros if the chunk does not have enough steps
steps_in_chunk = chunk_features.shape[1]
if steps_in_chunk < FLAGS.n_steps:
chunk_features = np.pad(
chunk_features,
((0, 0), (0, FLAGS.n_steps - steps_in_chunk), (0, 0),
(0, 0)),
mode='constant',
constant_values=0)
output = session.run(outputs['outputs'],
feed_dict={
inputs['input']:
chunk_features,
inputs['input_lengths']:
chunk_features_len,
})
logits = np.concatenate((logits, output))
# we have processed N_STEPS so subtract from remaining steps
batch_features_len -= N_STEPS
# clip to zero
batch_features_len = np.maximum(
batch_features_len, np.zeros_like(batch_features_len))
logitses.append(logits)
ground_truths = []
predictions = []
distances = []
losses = []
bar = progressbar.ProgressBar(max_value=batch_count - 1,
widget=progressbar.AdaptiveETA)
for logits, batch in bar(
zip(logitses, split_data(test_data, FLAGS.test_batch_size))):
seq_lengths = batch['features_len'].values
labels = pad_to_dense(batch['transcript'].values)
label_lengths = batch['transcript_len'].values
decoded_, loss_, distance_, sparse_labels_ = session.run(
[decoded, loss, distance, sparse_labels],
feed_dict={
decode_logits_ph: logits,
seq_lengths_ph: seq_lengths,
labels_ph: labels,
label_lengths_ph: label_lengths
})
ground_truths.extend(
sparse_tensor_value_to_texts(sparse_labels_, alphabet))
predictions.extend(
sparse_tensor_value_to_texts(decoded_[0], alphabet))
distances.extend(distance_)
losses.extend(loss_)
wer, samples = calculate_report(ground_truths, predictions, distances,
losses)
mean_edit_distance = np.mean(distances)
mean_loss = np.mean(losses)
# Filter out all items with WER=0 and take only the first report_count items
report_samples = itertools.islice((s for s in samples if s.wer > 0),
FLAGS.report_count)
print('Test - WER: %f, loss: %f, mean edit distance: %f' %
(wer, mean_loss, mean_edit_distance))
print('-' * 80)
for sample in report_samples:
print('WER: %f, loss: %f, mean edit distance: %f' %
(sample.wer, sample.loss, sample.distance))
print(' - src: "%s"' % sample.src)
print(' - res: "%s"' % sample.res)
print('-' * 80)
if FLAGS.test_output_file:
json.dump(samples,
open(FLAGS.test_output_file, 'w'),
default=lambda x: float(x))