def test_mod(self):
tv1 = TimeValue("1.100")
tv2 = TimeValue("2.200")
tv3 = TimeValue("0.000")
self.check(tv2 % tv1, tv3)
d = Decimal("1.100")
self.check(tv2 % d, tv3)
def test_truediv(self):
tv1 = TimeValue("1")
tv2 = TimeValue("2")
tv3 = TimeValue("0.5")
self.check(tv1 / tv2, tv3)
d = Decimal("2")
self.check(tv1 / d, tv3)
def test_floordiv(self):
tv1 = TimeValue("1.100")
tv2 = TimeValue("2.200")
tv3 = TimeValue("0.000")
self.check(tv1 // tv2, tv3)
d = Decimal("2.200")
self.check(tv1 // d, tv3)
def test_mul(self):
tv1 = TimeValue("1.100")
tv2 = TimeValue("2.200")
tv3 = TimeValue("2.420")
self.check(tv1 * tv2, tv3)
self.check(tv2 * tv1, tv3)
d = Decimal("2.200")
self.check(tv1 * d, tv3)
def test_fragment_rate_valid(self):
text = TextFragment(lines=[u"Hello", u"World"])
frag = SyncMapFragment(text_fragment=text,
begin=TimeValue("1.234"),
end=TimeValue("6.234"))
self.assertEqual(frag.audio_duration, 5)
self.assertEqual(frag.chars, 10)
self.assertEqual(frag.rate, 2.000)
self.assertEqual(frag.rate, Decimal("2.000"))
def test_sub(self):
tv1 = TimeValue("1.100")
tv2 = TimeValue("2.200")
tv3 = TimeValue("-1.100")
tv4 = TimeValue("1.100")
self.check(tv1 - tv2, tv3)
self.check(tv2 - tv1, tv4)
d = Decimal("2.200")
self.check(tv1 - d, tv3)
def test_add(self):
tv1 = TimeValue("1.100")
tv2 = TimeValue("2.200")
tv3 = TimeValue("3.300")
self.check(tv1 + tv2, tv3)
self.check(tv2 + tv1, tv3)
d = Decimal("2.200")
self.check(tv1 + d, tv3)
self.check(d + tv1, tv3)
def test_numpy_decimal(self):
tv1 = TimeValue("1.000")
n1 = numpy.array(
[Decimal("0.000"),
Decimal("1.000"),
Decimal("2.000")],
dtype=Decimal)
n2 = numpy.array(
[TimeValue("1.000"),
TimeValue("2.000"),
TimeValue("3.000")],
dtype=TimeValue)
n3 = numpy.array(
[TimeValue("0.000"),
TimeValue("1.000"),
TimeValue("2.000")],
dtype=TimeValue)
self.check_numpy(n1 + tv1, n2)
self.check_numpy(n1 * tv1, n3)
class SD(Loggable):
"""
The SD ("start detector").
Given an audio file and a text, detects the audio head and/or tail,
using a voice activity detector (via :class:`~aeneas.vad.VAD`) and
performing an alignment with a partial portion of the text
(via :class:`~aeneas.dtw.DTWAligner`).
This implementation relies on the following heuristic:
1. synthesize text until
``max_head_length`` times :data:`aeneas.sd.SD.QUERY_FACTOR`
seconds are reached;
2. consider only the first
``max_head_length`` times :data:`aeneas.sd.SD.AUDIO_FACTOR`
seconds of the audio file;
3. compute the best partial alignment of 1. with 2., and return
the corresponding time value.
(Similarly for the audio tail.)
:param real_wave_mfcc: the audio file
:type real_wave_mfcc: :class:`~aeneas.audiofile.AudioFileMFCC`
:param text_file: the text file
:type text_file: :class:`~aeneas.textfile.TextFile`
:param rconf: a runtime configuration
:type rconf: :class:`~aeneas.runtimeconfiguration.RuntimeConfiguration`
:param logger: the logger object
:type logger: :class:`~aeneas.logger.Logger`
"""
QUERY_FACTOR = Decimal("1.0")
"""
Multiply the max head/tail length by this factor
to get the minimum query length to be synthesized.
Default: ``1.0``.
.. versionadded:: 1.5.0
"""
AUDIO_FACTOR = Decimal("2.5")
"""
Multiply the max head/tail length by this factor
to get the minimum length in the audio that will be searched
for.
Set it to be at least ``1.0 + QUERY_FACTOR * 1.5``.
Default: ``2.5``.
.. versionadded:: 1.5.0
"""
MAX_LENGTH = TimeValue("10.000")
"""
Try detecting audio head or tail up to this many seconds.
Default: ``10.000``.
.. versionadded:: 1.2.0
"""
MIN_LENGTH = TimeValue("0.000")
"""
Try detecting audio head or tail of at least this many seconds.
Default: ``0.000``.
.. versionadded:: 1.2.0
"""
TAG = u"SD"
def __init__(self, real_wave_mfcc, text_file, rconf=None, logger=None):
super(SD, self).__init__(rconf=rconf, logger=logger)
self.real_wave_mfcc = real_wave_mfcc
self.text_file = text_file
def detect_interval(
self,
min_head_length=None,
max_head_length=None,
min_tail_length=None,
max_tail_length=None
):
"""
Detect the interval of the audio file
containing the fragments in the text file.
Return the audio interval as a tuple of two
:class:`~aeneas.timevalue.TimeValue` objects,
representing the begin and end time, in seconds,
with respect to the full wave duration.
If one of the parameters is ``None``, the default value
(``0.0`` for min, ``10.0`` for max) will be used.
:param min_head_length: estimated minimum head length
:type min_head_length: :class:`~aeneas.timevalue.TimeValue`
:param max_head_length: estimated maximum head length
:type max_head_length: :class:`~aeneas.timevalue.TimeValue`
:param min_tail_length: estimated minimum tail length
:type min_tail_length: :class:`~aeneas.timevalue.TimeValue`
:param max_tail_length: estimated maximum tail length
:type max_tail_length: :class:`~aeneas.timevalue.TimeValue`
:rtype: (:class:`~aeneas.timevalue.TimeValue`, :class:`~aeneas.timevalue.TimeValue`)
:raises: TypeError: if one of the parameters is not ``None`` or a number
:raises: ValueError: if one of the parameters is negative
"""
head = self.detect_head(min_head_length, max_head_length)
tail = self.detect_tail(min_tail_length, max_tail_length)
begin = head
end = self.real_wave_mfcc.audio_length - tail
self.log([u"Audio length: %.3f", self.real_wave_mfcc.audio_length])
self.log([u"Head length: %.3f", head])
self.log([u"Tail length: %.3f", tail])
self.log([u"Begin: %.3f", begin])
self.log([u"End: %.3f", end])
if (begin >= TimeValue("0.000")) and (end > begin):
self.log([u"Returning %.3f %.3f", begin, end])
return (begin, end)
self.log(u"Returning (0.000, 0.000)")
return (TimeValue("0.000"), TimeValue("0.000"))
def detect_head(self, min_head_length=None, max_head_length=None):
"""
Detect the audio head, returning its duration, in seconds.
:param min_head_length: estimated minimum head length
:type min_head_length: :class:`~aeneas.timevalue.TimeValue`
:param max_head_length: estimated maximum head length
:type max_head_length: :class:`~aeneas.timevalue.TimeValue`
:rtype: :class:`~aeneas.timevalue.TimeValue`
:raises: TypeError: if one of the parameters is not ``None`` or a number
:raises: ValueError: if one of the parameters is negative
"""
return self._detect(min_head_length, max_head_length, tail=False)
def detect_tail(self, min_tail_length=None, max_tail_length=None):
"""
Detect the audio tail, returning its duration, in seconds.
:param min_tail_length: estimated minimum tail length
:type min_tail_length: :class:`~aeneas.timevalue.TimeValue`
:param max_tail_length: estimated maximum tail length
:type max_tail_length: :class:`~aeneas.timevalue.TimeValue`
:rtype: :class:`~aeneas.timevalue.TimeValue`
:raises: TypeError: if one of the parameters is not ``None`` or a number
:raises: ValueError: if one of the parameters is negative
"""
return self._detect(min_tail_length, max_tail_length, tail=True)
def _detect(self, min_length, max_length, tail=False):
"""
Detect the head or tail within ``min_length`` and ``max_length`` duration.
If detecting the tail, the real wave MFCC and the query are reversed
so that the tail detection problem reduces to a head detection problem.
Return the duration of the head or tail, in seconds.
:param min_length: estimated minimum length
:type min_length: :class:`~aeneas.timevalue.TimeValue`
:param max_length: estimated maximum length
:type max_length: :class:`~aeneas.timevalue.TimeValue`
:rtype: :class:`~aeneas.timevalue.TimeValue`
:raises: TypeError: if one of the parameters is not ``None`` or a number
:raises: ValueError: if one of the parameters is negative
"""
def _sanitize(value, default, name):
if value is None:
value = default
try:
value = TimeValue(value)
except (TypeError, ValueError, InvalidOperation) as exc:
self.log_exc(u"The value of %s is not a number" % (name), exc, True, TypeError)
if value < 0:
self.log_exc(u"The value of %s is negative" % (name), None, True, ValueError)
return value
min_length = _sanitize(min_length, self.MIN_LENGTH, "min_length")
max_length = _sanitize(max_length, self.MAX_LENGTH, "max_length")
mws = self.rconf.mws
min_length_frames = int(min_length / mws)
max_length_frames = int(max_length / mws)
self.log([u"MFCC window shift s: %.3f", mws])
self.log([u"Min start length s: %.3f", min_length])
self.log([u"Min start length frames: %d", min_length_frames])
self.log([u"Max start length s: %.3f", max_length])
self.log([u"Max start length frames: %d", max_length_frames])
self.log([u"Tail?: %s", str(tail)])
self.log(u"Synthesizing query...")
synt_duration = max_length * self.QUERY_FACTOR
self.log([u"Synthesizing at least %.3f seconds", synt_duration])
tmp_handler, tmp_file_path = gf.tmp_file(suffix=u".wav", root=self.rconf[RuntimeConfiguration.TMP_PATH])
synt = Synthesizer(rconf=self.rconf, logger=self.logger)
anchors, total_time, synthesized_chars = synt.synthesize(
self.text_file,
tmp_file_path,
quit_after=synt_duration,
backwards=tail
)
self.log(u"Synthesizing query... done")
self.log(u"Extracting MFCCs for query...")
query_mfcc = AudioFileMFCC(tmp_file_path, rconf=self.rconf, logger=self.logger)
self.log(u"Extracting MFCCs for query... done")
self.log(u"Cleaning up...")
gf.delete_file(tmp_handler, tmp_file_path)
self.log(u"Cleaning up... done")
search_window = max_length * self.AUDIO_FACTOR
search_window_end = min(int(search_window / mws), self.real_wave_mfcc.all_length)
self.log([u"Query MFCC length (frames): %d", query_mfcc.all_length])
self.log([u"Real MFCC length (frames): %d", self.real_wave_mfcc.all_length])
self.log([u"Search window end (s): %.3f", search_window])
self.log([u"Search window end (frames): %d", search_window_end])
if tail:
self.log(u"Tail => reversing real_wave_mfcc and query_mfcc")
self.real_wave_mfcc.reverse()
query_mfcc.reverse()
# NOTE: VAD will be run here, if not done before
speech_intervals = self.real_wave_mfcc.intervals(speech=True, time=False)
if len(speech_intervals) < 1:
self.log(u"No speech intervals, hence no start found")
if tail:
self.real_wave_mfcc.reverse()
return TimeValue("0.000")
# generate a list of begin indices
search_end = None
candidates_begin = []
for interval in speech_intervals:
if (interval[0] >= min_length_frames) and (interval[0] <= max_length_frames):
candidates_begin.append(interval[0])
search_end = interval[1]
if search_end >= search_window_end:
break
# for each begin index, compute the acm cost
# to match the query
# note that we take the min over the last column of the acm
# meaning that we allow to match the entire query wave
# against a portion of the real wave
candidates = []
for candidate_begin in candidates_begin:
self.log([u"Candidate interval starting at %d == %.3f", candidate_begin, candidate_begin * mws])
try:
rwm = AudioFileMFCC(
mfcc_matrix=self.real_wave_mfcc.all_mfcc[:, candidate_begin:search_end],
rconf=self.rconf,
logger=self.logger
)
dtw = DTWAligner(
real_wave_mfcc=rwm,
synt_wave_mfcc=query_mfcc,
rconf=self.rconf,
logger=self.logger
)
acm = dtw.compute_accumulated_cost_matrix()
last_column = acm[:, -1]
min_value = numpy.min(last_column)
min_index = numpy.argmin(last_column)
self.log([u"Candidate interval: %d %d == %.3f %.3f", candidate_begin, search_end, candidate_begin * mws, search_end * mws])
self.log([u" Min value: %.6f", min_value])
self.log([u" Min index: %d == %.3f", min_index, min_index * mws])
candidates.append((min_value, candidate_begin, min_index))
except Exception as exc:
self.log_exc(u"An unexpected error occurred while running _detect", exc, False, None)
# reverse again the real wave
if tail:
self.log(u"Tail => reversing real_wave_mfcc again")
self.real_wave_mfcc.reverse()
# return
if len(candidates) < 1:
self.log(u"No candidates found")
return TimeValue("0.000")
self.log(u"Candidates:")
for candidate in candidates:
self.log([u" Value: %.6f Begin Time: %.3f Min Index: %d", candidate[0], candidate[1] * mws, candidate[2]])
best = sorted(candidates)[0][1]
self.log([u"Best candidate: %d == %.3f", best, best * mws])
return best * mws
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 1:
return self.print_help()
if self.has_option([u"-e", u"--examples"]):
return self.print_examples(False)
if self.has_option(u"--examples-all"):
return self.print_examples(True)
if self.has_option([u"--list-parameters"]):
return self.print_parameters()
parameter = self.has_option_with_value(u"--list-values")
if parameter is not None:
return self.print_values(parameter)
elif self.has_option(u"--list-values"):
return self.print_values(u"?")
# NOTE list() is needed for Python3, where keys() is not a list!
demo = self.has_option(list(self.DEMOS.keys()))
demo_parameters = u""
download_from_youtube = self.has_option([u"-y", u"--youtube"])
largest_audio = self.has_option(u"--largest-audio")
keep_audio = self.has_option(u"--keep-audio")
output_html = self.has_option(u"--output-html")
validate = not self.has_option(u"--skip-validator")
print_faster_rate = self.has_option(u"--faster-rate")
print_rates = self.has_option(u"--rates")
print_zero = self.has_option(u"--zero")
if demo:
validate = False
for key in self.DEMOS:
if self.has_option(key):
demo_parameters = self.DEMOS[key]
audio_file_path = demo_parameters[u"audio"]
text_file_path = demo_parameters[u"text"]
config_string = demo_parameters[u"config"]
sync_map_file_path = demo_parameters[u"syncmap"]
# TODO allow injecting rconf options directly from DEMOS options field
if key == u"--example-cewsubprocess":
self.rconf[
RuntimeConfiguration.CEW_SUBPROCESS_ENABLED] = True
elif key == u"--example-ctw-espeak":
self.rconf[RuntimeConfiguration.TTS] = "custom"
self.rconf[
RuntimeConfiguration.TTS_PATH] = self.CTW_ESPEAK
elif key == u"--example-ctw-speect":
self.rconf[RuntimeConfiguration.TTS] = "custom"
self.rconf[
RuntimeConfiguration.TTS_PATH] = self.CTW_SPEECT
elif key == u"--example-festival":
self.rconf[RuntimeConfiguration.TTS] = "festival"
elif key == u"--example-mws":
self.rconf[
RuntimeConfiguration.MFCC_WINDOW_LENGTH] = "1.500"
self.rconf[
RuntimeConfiguration.MFCC_WINDOW_SHIFT] = "0.500"
elif key == u"--example-multilevel-tts":
self.rconf[RuntimeConfiguration.TTS_L1] = "festival"
self.rconf[RuntimeConfiguration.TTS_L2] = "festival"
self.rconf[RuntimeConfiguration.TTS_L3] = "espeak"
elif key == u"--example-words-festival-cache":
self.rconf[RuntimeConfiguration.TTS] = "festival"
self.rconf[RuntimeConfiguration.TTS_CACHE] = True
elif key == u"--example-faster-rate":
print_faster_rate = True
elif key == u"--example-no-zero":
print_zero = True
elif key == u"--example-py":
self.rconf[RuntimeConfiguration.C_EXTENSIONS] = False
elif key == u"--example-rates":
print_rates = True
elif key == u"--example-youtube":
download_from_youtube = True
break
else:
if len(self.actual_arguments) < 4:
return self.print_help()
audio_file_path = self.actual_arguments[0]
text_file_path = self.actual_arguments[1]
config_string = self.actual_arguments[2]
sync_map_file_path = self.actual_arguments[3]
html_file_path = None
if output_html:
keep_audio = True
html_file_path = sync_map_file_path + u".html"
if download_from_youtube:
youtube_url = audio_file_path
if (not download_from_youtube) and (
not self.check_input_file(audio_file_path)):
return self.ERROR_EXIT_CODE
if not self.check_input_file(text_file_path):
return self.ERROR_EXIT_CODE
if not self.check_output_file(sync_map_file_path):
return self.ERROR_EXIT_CODE
if (html_file_path
is not None) and (not self.check_output_file(html_file_path)):
return self.ERROR_EXIT_CODE
self.check_c_extensions()
if demo:
msg = []
msg.append(u"Running example task with arguments:")
if download_from_youtube:
msg.append(u" YouTube URL: %s" % youtube_url)
else:
msg.append(u" Audio file: %s" % audio_file_path)
msg.append(u" Text file: %s" % text_file_path)
msg.append(u" Config string: %s" % config_string)
msg.append(u" Sync map file: %s" % sync_map_file_path)
if len(demo_parameters[u"options"]) > 0:
msg.append(u" Options: %s" %
demo_parameters[u"options"])
self.print_info(u"\n".join(msg))
if validate:
self.print_info(
u"Validating config string (specify --skip-validator to bypass)..."
)
validator = Validator(logger=self.logger)
result = validator.check_configuration_string(config_string,
is_job=False,
external_name=True)
if not result.passed:
self.print_error(u"The given config string is not valid:")
self.print_generic(result.pretty_print())
return self.ERROR_EXIT_CODE
self.print_info(u"Validating config string... done")
if download_from_youtube:
try:
self.print_info(u"Downloading audio from '%s' ..." %
youtube_url)
downloader = Downloader(logger=self.logger)
audio_file_path = downloader.audio_from_youtube(
youtube_url,
download=True,
output_file_path=None,
largest_audio=largest_audio)
self.print_info(u"Downloading audio from '%s' ... done" %
youtube_url)
except ImportError:
self.print_no_pafy_error()
return self.ERROR_EXIT_CODE
except Exception as exc:
self.print_error(
u"An unexpected error occurred while downloading audio from YouTube:"
)
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
else:
audio_extension = gf.file_extension(audio_file_path)
if audio_extension.lower() not in AudioFile.FILE_EXTENSIONS:
self.print_warning(
u"Your audio file path has extension '%s', which is uncommon for an audio file."
% audio_extension)
self.print_warning(
u"Attempting at executing your Task anyway.")
self.print_warning(
u"If it fails, you might have swapped the first two arguments."
)
self.print_warning(
u"The audio file path should be the first argument, the text file path the second."
)
try:
self.print_info(u"Creating task...")
task = Task(config_string, logger=self.logger)
task.audio_file_path_absolute = audio_file_path
task.text_file_path_absolute = text_file_path
task.sync_map_file_path_absolute = sync_map_file_path
self.print_info(u"Creating task... done")
except Exception as exc:
self.print_error(
u"An unexpected error occurred while creating the task:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Executing task...")
executor = ExecuteTask(task=task,
rconf=self.rconf,
logger=self.logger)
executor.execute()
self.print_info(u"Executing task... done")
except Exception as exc:
self.print_error(
u"An unexpected error occurred while executing the task:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Creating output sync map file...")
path = task.output_sync_map_file()
self.print_info(u"Creating output sync map file... done")
self.print_success(u"Created file '%s'" % path)
except Exception as exc:
self.print_error(
u"An unexpected error occurred while writing the sync map file:"
)
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
if output_html:
try:
parameters = {}
parameters[gc.PPN_TASK_OS_FILE_FORMAT] = task.configuration[
"o_format"]
parameters[
gc.PPN_TASK_OS_FILE_EAF_AUDIO_REF] = task.configuration[
"o_eaf_audio_ref"]
parameters[
gc.PPN_TASK_OS_FILE_SMIL_AUDIO_REF] = task.configuration[
"o_smil_audio_ref"]
parameters[
gc.PPN_TASK_OS_FILE_SMIL_PAGE_REF] = task.configuration[
"o_smil_page_ref"]
self.print_info(u"Creating output HTML file...")
task.sync_map.output_html_for_tuning(audio_file_path,
html_file_path,
parameters)
self.print_info(u"Creating output HTML file... done")
self.print_success(u"Created file '%s'" % html_file_path)
except Exception as exc:
self.print_error(
u"An unexpected error occurred while writing the HTML file:"
)
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
if download_from_youtube:
if keep_audio:
self.print_info(
u"Option --keep-audio set: keeping downloaded file '%s'" %
audio_file_path)
else:
gf.delete_file(None, audio_file_path)
if print_zero:
zero_duration = [
l for l in task.sync_map.fragments_tree.vleaves_not_empty
if l.begin == l.end
]
if len(zero_duration) > 0:
self.print_warning(u"Fragments with zero duration:")
for fragment in zero_duration:
self.print_generic(u" %s" % fragment)
if print_rates:
self.print_info(u"Fragments with rates:")
for fragment in task.sync_map.fragments_tree.vleaves_not_empty:
self.print_generic(u" %s (rate: %.3f chars/s)" %
(fragment, fragment.rate))
if print_faster_rate:
max_rate = task.configuration["aba_rate_value"]
if max_rate is not None:
faster = [
l for l in task.sync_map.fragments_tree.vleaves_not_empty
if l.rate >= max_rate + Decimal("0.001")
]
if len(faster) > 0:
self.print_warning(
u"Fragments with rate greater than %.3f:" % max_rate)
for fragment in faster:
self.print_generic(u" %s (rate: %.3f chars/s)" %
(fragment, fragment.rate))
return self.NO_ERROR_EXIT_CODE
def new_time(begin, end, current):
""" Compute new time """
# NOTE self.parameters[0] is an int
percent = max(min(Decimal(self.parameters[0]) / 100, 100), 0)
return (begin + (end + 1 - begin) * percent) * self.rconf.mws