def create_tf_example(
file_id, start, end, sub_segment, sub_segment_id, sub_segment_len, label
):
# Drop 1/4th of speech signals to balance data
if label == 1:
drop = np.random.randint(0, 4)
if drop > 0:
return None
# MFCC feature extraction
signal_to_process = np.copy(sub_segment)
signal_to_process = np.float32(signal_to_process)
features = extract_features(
signal_to_process, freq=16000, n_mfcc=5, size=512, step=16
)
features = np.reshape(features, -1)
feature_dict = {
"signal/id": bytes_feature(file_id.encode()),
"segment/start": int64_feature(int(start)),
"segment/end": int64_feature(int(end)),
"subsegment/id": int64_feature(sub_segment_id),
"subsegment/length": int64_feature(sub_segment_len),
"subsegment/signal": float_list_feature(sub_segment.tolist()),
"subsegment/features": float_list_feature(features.tolist()),
"subsegment/label": int64_feature(label),
}
example = tf.train.Example(features=tf.train.Features(feature=feature_dict))
return example
def create_tf_example(
file_id,
start,
end,
sub_segment,
sub_segment_id,
sub_segment_len,
label,
):
"""Build a TF training example from raw data.
Args:
file_id (int): file ID
start (int): global audio segment start frame
end (int): global audio segment end frame
sub_segment (np.ndarray): sub audio segment signal
sub_segment_id (int): sub audio segment ID
sub_segment_len (int): sub audio segment length
label (int): sub audio segment ID (0 or 1)
Returns:
example (tf.core.example.example_pb2.Example): TF training example
"""
# Drop 1/4th of speech signals to balance data
if label == 1:
drop = np.random.randint(0, 4)
if drop > 0:
return None
# MFCC feature extraction
signal_to_process = np.copy(sub_segment)
signal_to_process = np.float32(signal_to_process)
features = extract_features(
signal_to_process, freq=16000, n_mfcc=5, size=512, step=16
)
features = np.reshape(features, -1)
feature_dict = {
"signal/id": bytes_feature(file_id.encode()),
"segment/start": int64_feature(int(start)),
"segment/end": int64_feature(int(end)),
"subsegment/id": int64_feature(sub_segment_id),
"subsegment/length": int64_feature(sub_segment_len),
"subsegment/signal": float_list_feature(sub_segment.tolist()),
"subsegment/features": float_list_feature(features.tolist()),
"subsegment/label": int64_feature(label),
}
example = tf.train.Example(features=tf.train.Features(feature=feature_dict))
return example
def main(_):
np.random.seed(0)
# Directories
data_dir = os.path.join(FLAGS.data_dir, 'test-clean/')
label_dir = os.path.join(FLAGS.data_dir, 'labels/')
_, _, test = split_data(label_dir, split='0.7/0.15', random_seed=0)
file_it = file_iter(data_dir, label_dir, files=test)
# TensorFlow inputs
features_input_ph = tf.placeholder(shape=FEAT_SIZE, dtype=tf.float32)
features_input_op = tf.transpose(features_input_ph, perm=[1, 0])
features_input_op = tf.expand_dims(features_input_op, axis=0)
# TensorFlow exported model
speech_predictor = tf.contrib.predictor.from_saved_model(export_dir=FLAGS.exported_model)
init = tf.initializers.global_variables()
classes = ['Noise', 'Speech']
# Iterate though test data
with tf.Session() as sess:
for signal, labels, fn in file_it:
sess.run(init)
print('\nPrediction on file {} ...'.format(fn))
signal_input = deque(signal[:FLAGS.seq_len].tolist(), maxlen=FLAGS.seq_len)
preds, pred_time = [], []
pointer = FLAGS.seq_len
while pointer < len(signal):
start = time()
# Preprocess signal & extract features
signal_to_process = np.copy(signal_input)
signal_to_process = np.float32(signal_to_process)
features = extract_features(signal_to_process, freq=16000, n_mfcc=5, size=512, step=16)
# Prediction
features_input = sess.run(features_input_op, feed_dict={features_input_ph: features})
speech_prob = speech_predictor({'features_input': features_input})['speech'][0]
speech_pred = classes[int(np.round(speech_prob))]
# Time prediction & processing
end = time()
dt = end - start
pred_time.append(dt)
print('Prediction = {} | proba = {:.2f} | time = {:.2f} s'.format(speech_pred, speech_prob[0], dt))
# For visualization
preds.append([pointer - FLAGS.seq_len, pointer, np.round(speech_prob)])
# Update signal segment
signal_input.extend(signal[pointer + FLAGS.stride:pointer + FLAGS.stride + FLAGS.seq_len])
pointer += FLAGS.seq_len + FLAGS.stride
print('Average prediction time = {:.2f} ms'.format(np.mean(pred_time) * 1e3))
# Smoothing & hangover
if FLAGS.smoothing:
preds = smooth_predictions(preds)
# Visualization
visualize_predictions(signal, fn, preds)
def main(_):
np.random.seed(0)
file_it = file_iter(FLAGS.data_dir)
if not tf.gfile.IsDirectory(FLAGS.out_dir):
tf.gfile.MakeDirs(FLAGS.out_dir)
# TensorFlow inputs
features_input_ph = tf.placeholder(shape=(16, 65), dtype=tf.float32)
features_input_op = tf.transpose(features_input_ph, perm=[1, 0])
features_input_op = tf.expand_dims(features_input_op, axis=0)
# TensorFlow exported model
speech_predictor = tf.contrib.predictor.from_saved_model(
export_dir=FLAGS.exported_model)
init = tf.initializers.global_variables()
classes = ["Noise", "Speech"]
# Iterate though test data
with tf.Session() as sess:
for signal, fn in file_it:
sess.run(init)
print("\nPrediction on file {} ...".format(fn))
signal_input = deque(signal[:1024].tolist(), maxlen=1024)
labels = {"speech_segments": []}
preds, pred_time = [], []
pointer = 1024
while pointer < len(signal):
start = time.time()
# Preprocess signal & extract features
signal_to_process = np.copy(signal_input)
signal_to_process = np.float32(signal_to_process)
signal_to_process = np.add(signal_to_process, 1.0)
signal_to_process = np.divide(signal_to_process, 2.0)
features = extract_features(signal_to_process,
freq=16000,
n_mfcc=5,
size=512,
step=16)
# Prediction
features_input = sess.run(
features_input_op, feed_dict={features_input_ph: features})
speech_prob = speech_predictor(
{"features_input": features_input})["speech"][0]
speech_pred = classes[int(np.round(speech_prob))]
# Time prediction & processing
end = time.time()
dt = end - start
pred_time.append(dt)
if FLAGS.viz:
print("Prediction = {} | proba = {:.2f} | time = {:.2f} s".
format(speech_pred, speech_prob[0], dt))
# For visualization
preds.append([pointer - 1024, pointer, np.round(speech_prob)])
# For label recording
if np.round(speech_prob) > 0:
labels["speech_segments"].append({
"start_time": pointer - 1024,
"end_time": pointer
})
# Update signal segment
signal_input.extend(signal[pointer + 1:pointer + 1 + 1024])
pointer += 1024 + 1
print("Average prediction time = {:.2f} ms".format(
np.mean(pred_time) * 1e3))
# Visualization
if FLAGS.viz:
visualize_predictions(signal, fn, preds)
# Record labels to .json
if not FLAGS.viz:
out_fn = "{}.json".format(fn.split(".")[0])
out_fp = os.path.join(FLAGS.out_dir, out_fn)
with open(out_fp, "w") as f:
json.dump(labels, f)
print("{} predictions recorded to {}".format(
len(labels["speech_segments"]), FLAGS.out_dir))
def automatic_labeling(data_dir, exported_model, visualize=False):
"""Run automatic labeling over a given dataset of raw audio signals, given a pre-trained VAD model.
Args:
data_dir (str): path to raw dataset directory
exported_model (str): path to exported pre-trained TF model directory
visualize (bool, optional): option to visualize automatic labeling. Defaults to False.
"""
np.random.seed(0)
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
tf.logging.set_verbosity(tf.logging.INFO)
test_data_dir = os.path.join(data_dir, "test-clean/")
labels_dir = os.path.join(data_dir, "labels/")
file_it = file_iter(test_data_dir)
if not tf.gfile.IsDirectory(labels_dir):
tf.gfile.MakeDirs(labels_dir)
# TensorFlow inputs
features_input_ph = tf.placeholder(shape=(16, 65), dtype=tf.float32)
features_input_op = tf.transpose(features_input_ph, perm=[1, 0])
features_input_op = tf.expand_dims(features_input_op, axis=0)
# TensorFlow exported model
speech_predictor = tf.contrib.predictor.from_saved_model(
export_dir=exported_model)
init = tf.initializers.global_variables()
classes = ["Noise", "Speech"]
# Iterate though test data
with tf.Session() as sess:
for signal, fn in file_it:
sess.run(init)
logger.info(f"Prediction on file {fn} ...")
signal_input = deque(signal[:1024].tolist(), maxlen=1024)
labels = {"speech_segments": []}
preds, pred_time = [], []
pointer = 1024
while pointer < len(signal):
start = time.time()
# Preprocess signal & extract features
signal_to_process = np.copy(signal_input)
signal_to_process = np.float32(signal_to_process)
signal_to_process = np.add(signal_to_process, 1.0)
signal_to_process = np.divide(signal_to_process, 2.0)
features = extract_features(signal_to_process,
freq=16000,
n_mfcc=5,
size=512,
step=16)
# Prediction
features_input = sess.run(
features_input_op, feed_dict={features_input_ph: features})
speech_prob = speech_predictor(
{"features_input": features_input})["speech"][0]
speech_pred = classes[int(np.round(speech_prob))]
# Time prediction & processing
end = time.time()
dt = end - start
pred_time.append(dt)
if visualize:
logger.info(
f"Prediction = {speech_pred} | proba = {speech_prob[0]:.2f} | time = {dt:.2f} s"
)
# For visualization
preds.append([pointer - 1024, pointer, np.round(speech_prob)])
# For label recording
if np.round(speech_prob) > 0:
labels["speech_segments"].append({
"start_time": pointer - 1024,
"end_time": pointer
})
# Update signal segment
signal_input.extend(signal[pointer + 1:pointer + 1 + 1024])
pointer += 1024 + 1
logger.info(
f"Average prediction time = {np.mean(pred_time) * 1e3:.2f} ms")
# Visualization
if visualize:
visualize_predictions(signal, fn, preds)
# Record labels to .json
if not visualize:
base_name = fn.split(".")[0]
out_fn = f"{base_name}.json"
out_fp = os.path.join(labels_dir, out_fn)
with open(out_fp, "w") as f:
json.dump(labels, f)
nb_preds = len(labels["speech_segments"])
logger.info(f"{nb_preds} predictions recorded to {labels_dir}")
def run_inference(params, data_dir, exported_model):
"""Run Voice Activity Detection CNN inference over raw audio signals.
Args:
params (dict): dictionary of inference parameters
data_dir (str): path to raw dataset directory
exported_model (str): path to exported pre-trained TF model directory
"""
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
tf.logging.set_verbosity(tf.logging.INFO)
np.random.seed(0)
input_size = params["input_size"]
stride = params["stride"]
smoothing = params["smoothing"]
# Directories
test_data_dir = os.path.join(data_dir, "test-clean/")
label_dir = os.path.join(data_dir, "labels/")
_, _, test = split_data(label_dir, split="0.7/0.15", random_seed=0)
file_it = file_iter(test_data_dir, label_dir, files=test)
# TensorFlow inputs
features_input_ph = tf.placeholder(shape=FEAT_SIZE, dtype=tf.float32)
features_input_op = tf.transpose(features_input_ph, perm=[1, 0])
features_input_op = tf.expand_dims(features_input_op, axis=0)
# TensorFlow exported model
speech_predictor = tf.contrib.predictor.from_saved_model(export_dir=exported_model)
init = tf.initializers.global_variables()
classes = ["Noise", "Speech"]
# Iterate though test data
with tf.Session() as sess:
for signal, labels, fn in file_it:
sess.run(init)
logger.info(f"Prediction on file {fn} ...")
signal_input = deque(signal[:input_size].tolist(), maxlen=input_size)
preds, pred_time = [], []
pointer = input_size
while pointer < len(signal):
start = time.time()
# Preprocess signal & extract features
signal_to_process = np.copy(signal_input)
signal_to_process = np.float32(signal_to_process)
features = extract_features(
signal_to_process, freq=16000, n_mfcc=5, size=512, step=16
)
# Prediction
features_input = sess.run(
features_input_op, feed_dict={features_input_ph: features}
)
speech_prob = speech_predictor({"features_input": features_input})[
"speech"
][0]
speech_pred = classes[int(np.round(speech_prob))]
# Time prediction & processing
end = time.time()
dt = end - start
pred_time.append(dt)
logger.info(
f"Prediction = {speech_pred} | proba = {speech_prob[0]:.2f} | time = {dt:.2f} s"
)
# For visualization
preds.append([pointer - input_size, pointer, np.round(speech_prob)])
# Update signal segment
signal_input.extend(
signal[pointer + stride : pointer + stride + input_size]
)
pointer += input_size + stride
logger.info(f"Average prediction time = {np.mean(pred_time) * 1e3:.2f} ms")
# Smoothing & hangover
if smoothing:
preds = smooth_predictions(preds)
# Visualization
visualize_predictions(signal, fn, preds)