def compute_mfcc(self):
"""
Compute the MFCCs of the two waves,
and store them internally.
"""
if (
(self.real_wave_path is not None) and
(os.path.isfile(self.real_wave_path))
):
self._log("Computing MFCCs for real wave...")
wave = AudioFile(self.real_wave_path, logger=self.logger)
wave.extract_mfcc(self.frame_rate)
self.real_wave_full_mfcc = wave.audio_mfcc
self.real_wave_length = wave.audio_length
self._log("Computing MFCCs for real wave... done")
else:
self._log(["Input file '%s' cannot be read", self.real_wave_path], Logger.CRITICAL)
raise OSError("Input file cannot be read")
if (
(self.synt_wave_path is not None) and
(os.path.isfile(self.synt_wave_path))
):
self._log("Computing MFCCs for synt wave...")
wave = AudioFile(self.synt_wave_path, logger=self.logger)
wave.extract_mfcc(self.frame_rate)
self.synt_wave_full_mfcc = wave.audio_mfcc
self.synt_wave_length = wave.audio_length
self._log("Computing MFCCs for synt wave... done")
else:
self._log(["Input file '%s' cannot be read", self.synt_wave_path], Logger.CRITICAL)
raise OSError("Input file cannot be read")
def _extract_mfcc(self, audio_file_path):
"""
Extract the MFCCs of the real full wave.
"""
self._log("Extracting MFCCs from real full wave")
try:
audio_file = AudioFile(audio_file_path, logger=self.logger)
audio_file.extract_mfcc()
self._log("Extracting MFCCs from real full wave: succeeded")
return (True, audio_file.audio_mfcc, audio_file.audio_length)
except Exception as e:
self._log("Extracting MFCCs from real full wave: failed")
self._log(["Message: %s", str(e)])
return (False, None, None)
def _extract_mfcc(self, audio_file_path):
"""
Extract the MFCCs of the real full wave.
"""
self._log("Extracting MFCCs from real full wave")
try:
audio_file = AudioFile(audio_file_path, logger=self.logger)
audio_file.extract_mfcc()
self._log("Extracting MFCCs from real full wave: succeeded")
return (True, audio_file.audio_mfcc, audio_file.audio_length)
except Exception as e:
self._log("Extracting MFCCs from real full wave: failed")
self._log(["Message: %s", str(e)])
return (False, None, None)
def main():
""" Entry point """
if len(sys.argv) < 3:
usage()
return
file_path = sys.argv[1]
save_path = sys.argv[2]
if not gf.can_run_c_extension():
print "[WARN] Unable to load Python C Extensions"
print "[WARN] Running the slower pure Python code"
print "[WARN] See the README file for directions to compile the Python C Extensions"
audiofile = AudioFile(file_path)
audiofile.load_data()
audiofile.extract_mfcc()
audiofile.clear_data()
numpy.savetxt(save_path, audiofile.audio_mfcc)
print "[INFO] MFCCs saved to %s" % (save_path)
def compute_mfcc(self):
"""
Compute the MFCCs of the wave,
and store them internally.
"""
if (self.wave_path is not None) and (os.path.isfile(self.wave_path)):
self._log("Computing MFCCs for wave...")
try:
wave = AudioFile(self.wave_path, logger=self.logger)
wave.extract_mfcc(self.frame_rate)
self.wave_mfcc = wave.audio_mfcc
self.wave_len = wave.audio_length
except IOError as e:
self._log("IOError", Logger.CRITICAL)
self._log(["Message: %s", e])
raise e
self._log("Computing MFCCs for wave... done")
else:
self._log(["Input file '%s' cannot be read", self.wave_path], Logger.CRITICAL)
raise OSError("Input file cannot be read")
def compute_mfcc(self):
"""
Compute the MFCCs of the wave,
and store them internally.
"""
if (self.wave_path is not None) and (os.path.isfile(self.wave_path)):
self._log("Computing MFCCs for wave...")
try:
wave = AudioFile(self.wave_path, logger=self.logger)
wave.extract_mfcc(self.frame_rate)
self.wave_mfcc = wave.audio_mfcc
self.wave_len = wave.audio_length
except IOError as e:
self._log("IOError", Logger.CRITICAL)
self._log(["Message: %s", e])
raise e
self._log("Computing MFCCs for wave... done")
else:
self._log(["Input file '%s' cannot be read", self.wave_path],
Logger.CRITICAL)
raise OSError("Input file cannot be read")
def _detect_start(self,
min_start_length,
max_start_length,
metric,
backwards=False):
""" Detect start """
self._log(["Min start length: %.3f", min_start_length])
self._log(["Max start length: %.3f", max_start_length])
self._log(["Metric: %s", metric])
self._log(["Backwards: %s", str(backwards)])
audio_rate = self.text_file.characters / self.audio_file.audio_length
self._log(["Audio rate: %.3f", audio_rate])
self._log("Synthesizing query...")
tmp_handler, tmp_file_path = tempfile.mkstemp(suffix=".wav",
dir=gf.custom_tmp_dir())
synt = Synthesizer(logger=self.logger)
synt_duration = max_start_length * self.QUERY_FACTOR
self._log(["Synthesizing %.3f seconds", synt_duration])
result = synt.synthesize(self.text_file,
tmp_file_path,
quit_after=synt_duration,
backwards=backwards)
self._log("Synthesizing query... done")
query_file = AudioFile(tmp_file_path)
if backwards:
self._log("Reversing query")
query_file.reverse()
self._log("Extracting MFCCs for query...")
query_file.extract_mfcc(frame_rate=self.frame_rate)
query_file.clear_data()
self._log("Extracting MFCCs for query... done")
self._log("Cleaning up...")
self._cleanup(tmp_handler, tmp_file_path)
self._log("Cleaning up... done")
query_characters = result[2]
query_len = query_file.audio_length
query_mfcc = query_file.audio_mfcc
query_rate = query_characters / query_len
stretch_factor = max(1, query_rate / audio_rate)
self._log(["Audio rate: %.3f", audio_rate])
self._log(["Query rate: %.3f", query_rate])
self._log(["Stretch factor: %.3f", stretch_factor])
audio_mfcc = self.audio_file.audio_mfcc
self._log(["Actual audio has %d frames", audio_mfcc.shape[1]])
audio_mfcc_end_index = int(max_start_length * self.AUDIO_FACTOR *
self.frame_rate)
self._log(["Limiting audio to first %d frames", audio_mfcc_end_index])
audio_mfcc_end_index = min(audio_mfcc_end_index, audio_mfcc.shape[1])
audio_mfcc = audio_mfcc[:, 0:audio_mfcc_end_index]
self._log(["Limited audio has %d frames", audio_mfcc.shape[1]])
l, o = audio_mfcc.shape
l, n = query_mfcc.shape
# minimum length of a matched interval in the real audio
stretched_match_minimum_length = int(n * stretch_factor)
self._log(["Audio has %d frames == %.3f seconds", o, self._i2t(o)])
self._log(["Query has %d frames == %.3f seconds", n, self._i2t(n)])
self._log(["Stretch factor: %.3f", stretch_factor])
self._log(
["Required minimum length: %.3f", stretched_match_minimum_length])
self._log("Speech intervals:")
for interval in self.audio_speech:
self._log([
" %d %d == %.3f %.3f",
self._t2i(interval[0]),
self._t2i(interval[1]), interval[0], interval[1]
])
admissible_intervals = [
x for x in self.audio_speech
if ((x[0] >= min_start_length) and (x[0] <= max_start_length))
]
self._log("AdmissibleSpeech intervals:")
for interval in admissible_intervals:
self._log([
" %d %d == %.3f %.3f",
self._t2i(interval[0]),
self._t2i(interval[1]), interval[0], interval[1]
])
candidates = []
runs_with_min_length = 0
runs_no_improvement = 0
runs_min_distortion = numpy.inf
runs_min_value = numpy.inf
for interval in admissible_intervals:
if runs_no_improvement >= self.MAX_RUNS_NO_IMPROVEMENT:
self._log(" Breaking: too many runs without improvement")
break
if runs_with_min_length >= self.MAX_RUNS_WITH_MIN_LENGTH:
self._log(
" Breaking: too many runs with minimum required length")
break
start_time = interval[0]
start_index = self._t2i(start_time)
self._log([
"Evaluating interval starting at %d == %.3f ", start_index,
start_time
])
if start_index > o:
self._log(" Breaking: start index outside audio window")
break
req_end_index = start_index + stretched_match_minimum_length
req_end_time = self._i2t(req_end_index)
if req_end_index > o:
self._log(
" Breaking: not enough audio left in shifted window")
break
end_index = min(start_index + 2 * n, o)
end_time = self._i2t(end_index)
self._log([" Start %d == %.3f", start_index, start_time])
self._log([" Req end %d == %.3f", req_end_index, req_end_time])
self._log([" Eff end %d == %.3f", end_index, end_time])
audio_mfcc_sub = audio_mfcc[:, start_index:end_index]
l, m = audio_mfcc_sub.shape
self._log("Computing DTW...")
aligner = DTWAligner(None,
None,
frame_rate=self.frame_rate,
logger=self.logger)
aligner.real_wave_full_mfcc = audio_mfcc_sub
aligner.synt_wave_full_mfcc = query_mfcc
aligner.real_wave_length = self._i2t(m)
aligner.synt_wave_length = self._i2t(n)
acm = aligner.compute_accumulated_cost_matrix()
# transpose, so we have an n x m accumulated cost matrix
acm = acm.transpose()
last_row = acm[-1, :]
self._log("Computing DTW... done")
# find the minimum, but its index must be >= stretched_match_minimum_length
candidate_argmin_index = numpy.argmin(
last_row[stretched_match_minimum_length:])
candidate_length_index = stretched_match_minimum_length + candidate_argmin_index
candidate_length_time = self._i2t(candidate_length_index)
candidate_value = last_row[candidate_length_index]
candidate_end_index = start_index + candidate_length_index
candidate_end_time = self._i2t(candidate_end_index)
candidate_distortion = candidate_value / candidate_length_index
# check if the candidate has minimum length
if candidate_length_index == stretched_match_minimum_length:
runs_with_min_length += 1
else:
runs_with_min_length = 0
# check if the candidate improved the global minimum value
if metric == SDMetric.VALUE:
if candidate_value < runs_min_value:
runs_min_value = candidate_value
runs_no_improvement = 0
else:
runs_no_improvement += 1
if metric == SDMetric.DISTORTION:
if candidate_distortion < runs_min_distortion:
runs_min_distortion = candidate_distortion
runs_no_improvement = 0
else:
runs_no_improvement += 1
# append to the list of candidates
self._log([
" Interval start: %d == %.6f", start_index, start_time
])
self._log(
[" Interval end: %d == %.6f", end_index, end_time])
self._log([
" Candidate start: %d == %.6f", start_index, start_time
])
self._log([
" Candidate end: %d == %.6f", candidate_end_index,
candidate_end_time
])
self._log([
" Candidate length: %d == %.6f", candidate_length_index,
candidate_length_time
])
self._log([" Candidate value: %.6f", candidate_value])
self._log([" Candidate distortion: %.6f", candidate_distortion])
candidates.append({
"start_index": start_index,
"length": candidate_length_index,
"value": candidate_value,
"distortion": candidate_distortion
})
# select best candidate and return its start time
# if we have no best candidate, return 0.0
best_candidate = self._select_best_candidate(candidates, metric)
if best_candidate is None:
return 0.0
sd_time = self._i2t(max(best_candidate["start_index"], 0))
self._log(["Returning time %.3f", sd_time])
return sd_time
def _detect_start(self, min_start_length, max_start_length, metric, backwards=False):
""" Detect start """
self._log(["Min start length: %.3f", min_start_length])
self._log(["Max start length: %.3f", max_start_length])
self._log(["Metric: %s", metric])
self._log(["Backwards: %s", str(backwards)])
audio_rate = self.text_file.characters / self.audio_file.audio_length
self._log(["Audio rate: %.3f", audio_rate])
self._log("Synthesizing query...")
tmp_handler, tmp_file_path = tempfile.mkstemp(
suffix=".wav",
dir=gf.custom_tmp_dir()
)
synt = Synthesizer(logger=self.logger)
synt_duration = max_start_length * self.QUERY_FACTOR
self._log(["Synthesizing %.3f seconds", synt_duration])
result = synt.synthesize(
self.text_file,
tmp_file_path,
quit_after=synt_duration,
backwards=backwards
)
self._log("Synthesizing query... done")
query_file = AudioFile(tmp_file_path)
if backwards:
self._log("Reversing query")
query_file.reverse()
self._log("Extracting MFCCs for query...")
query_file.extract_mfcc(frame_rate=self.frame_rate)
query_file.clear_data()
self._log("Extracting MFCCs for query... done")
self._log("Cleaning up...")
self._cleanup(tmp_handler, tmp_file_path)
self._log("Cleaning up... done")
query_characters = result[2]
query_len = query_file.audio_length
query_mfcc = query_file.audio_mfcc
query_rate = query_characters / query_len
stretch_factor = max(1, query_rate / audio_rate)
self._log(["Audio rate: %.3f", audio_rate])
self._log(["Query rate: %.3f", query_rate])
self._log(["Stretch factor: %.3f", stretch_factor])
audio_mfcc = self.audio_file.audio_mfcc
self._log(["Actual audio has %d frames", audio_mfcc.shape[1]])
audio_mfcc_end_index = int(max_start_length * self.AUDIO_FACTOR * self.frame_rate)
self._log(["Limiting audio to first %d frames", audio_mfcc_end_index])
audio_mfcc_end_index = min(audio_mfcc_end_index, audio_mfcc.shape[1])
audio_mfcc = audio_mfcc[:, 0:audio_mfcc_end_index]
self._log(["Limited audio has %d frames", audio_mfcc.shape[1]])
l, o = audio_mfcc.shape
l, n = query_mfcc.shape
# minimum length of a matched interval in the real audio
stretched_match_minimum_length = int(n * stretch_factor)
self._log(["Audio has %d frames == %.3f seconds", o, self._i2t(o)])
self._log(["Query has %d frames == %.3f seconds", n, self._i2t(n)])
self._log(["Stretch factor: %.3f", stretch_factor])
self._log(["Required minimum length: %.3f", stretched_match_minimum_length])
self._log("Speech intervals:")
for interval in self.audio_speech:
self._log([" %d %d == %.3f %.3f", self._t2i(interval[0]), self._t2i(interval[1]), interval[0], interval[1]])
admissible_intervals = [x for x in self.audio_speech if ((x[0] >= min_start_length) and (x[0] <= max_start_length))]
self._log("AdmissibleSpeech intervals:")
for interval in admissible_intervals:
self._log([" %d %d == %.3f %.3f", self._t2i(interval[0]), self._t2i(interval[1]), interval[0], interval[1]])
candidates = []
runs_with_min_length = 0
runs_no_improvement = 0
runs_min_distortion = numpy.inf
runs_min_value = numpy.inf
for interval in admissible_intervals:
if runs_no_improvement >= self.MAX_RUNS_NO_IMPROVEMENT:
self._log(" Breaking: too many runs without improvement")
break
if runs_with_min_length >= self.MAX_RUNS_WITH_MIN_LENGTH:
self._log(" Breaking: too many runs with minimum required length")
break
start_time = interval[0]
start_index = self._t2i(start_time)
self._log(["Evaluating interval starting at %d == %.3f ", start_index, start_time])
if start_index > o:
self._log(" Breaking: start index outside audio window")
break
req_end_index = start_index + stretched_match_minimum_length
req_end_time = self._i2t(req_end_index)
if req_end_index > o:
self._log(" Breaking: not enough audio left in shifted window")
break
end_index = min(start_index + 2 * n, o)
end_time = self._i2t(end_index)
self._log([" Start %d == %.3f", start_index, start_time])
self._log([" Req end %d == %.3f", req_end_index, req_end_time])
self._log([" Eff end %d == %.3f", end_index, end_time])
audio_mfcc_sub = audio_mfcc[:, start_index:end_index]
l, m = audio_mfcc_sub.shape
self._log("Computing DTW...")
aligner = DTWAligner(None, None, frame_rate=self.frame_rate, logger=self.logger)
aligner.real_wave_full_mfcc = audio_mfcc_sub
aligner.synt_wave_full_mfcc = query_mfcc
aligner.real_wave_length = self._i2t(m)
aligner.synt_wave_length = self._i2t(n)
acm = aligner.compute_accumulated_cost_matrix()
# transpose, so we have an n x m accumulated cost matrix
acm = acm.transpose()
last_row = acm[-1, :]
self._log("Computing DTW... done")
# find the minimum, but its index must be >= stretched_match_minimum_length
candidate_argmin_index = numpy.argmin(last_row[stretched_match_minimum_length:])
candidate_length_index = stretched_match_minimum_length + candidate_argmin_index
candidate_length_time = self._i2t(candidate_length_index)
candidate_value = last_row[candidate_length_index]
candidate_end_index = start_index + candidate_length_index
candidate_end_time = self._i2t(candidate_end_index)
candidate_distortion = candidate_value / candidate_length_index
# check if the candidate has minimum length
if candidate_length_index == stretched_match_minimum_length:
runs_with_min_length += 1
else:
runs_with_min_length = 0
# check if the candidate improved the global minimum value
if metric == SDMetric.VALUE:
if candidate_value < runs_min_value:
runs_min_value = candidate_value
runs_no_improvement = 0
else:
runs_no_improvement += 1
if metric == SDMetric.DISTORTION:
if candidate_distortion < runs_min_distortion:
runs_min_distortion = candidate_distortion
runs_no_improvement = 0
else:
runs_no_improvement += 1
# append to the list of candidates
self._log([" Interval start: %d == %.6f", start_index, start_time])
self._log([" Interval end: %d == %.6f", end_index, end_time])
self._log([" Candidate start: %d == %.6f", start_index, start_time])
self._log([" Candidate end: %d == %.6f", candidate_end_index, candidate_end_time])
self._log([" Candidate length: %d == %.6f", candidate_length_index, candidate_length_time])
self._log([" Candidate value: %.6f", candidate_value])
self._log([" Candidate distortion: %.6f", candidate_distortion])
candidates.append({
"start_index": start_index,
"length": candidate_length_index,
"value": candidate_value,
"distortion": candidate_distortion
})
# select best candidate and return its start time
# if we have no best candidate, return 0.0
best_candidate = self._select_best_candidate(candidates, metric)
if best_candidate is None:
return 0.0
sd_time = self._i2t(max(best_candidate["start_index"], 0))
self._log(["Returning time %.3f", sd_time])
return sd_time
