def test_relative_path(self):
tests = [
("res", "aeneas/tools/somefile.py", "aeneas/tools/res"),
("res/foo", "aeneas/tools/somefile.py", "aeneas/tools/res/foo"),
("res/bar.baz", "aeneas/tools/somefile.py", "aeneas/tools/res/bar.baz"),
("res/bar/baz/foo", "aeneas/tools/somefile.py", "aeneas/tools/res/bar/baz/foo"),
("res/bar/baz/foo.bar", "aeneas/tools/somefile.py", "aeneas/tools/res/bar/baz/foo.bar"),
]
for test in tests:
self.assertEqual(gf.relative_path(test[0], test[1]), test[2])
def test_relative_path(self):
tests = [("res", "aeneas/tools/somefile.py", "aeneas/tools/res"),
("res/foo", "aeneas/tools/somefile.py",
"aeneas/tools/res/foo"),
("res/bar.baz", "aeneas/tools/somefile.py",
"aeneas/tools/res/bar.baz"),
("res/bar/baz/foo", "aeneas/tools/somefile.py",
"aeneas/tools/res/bar/baz/foo"),
("res/bar/baz/foo.bar", "aeneas/tools/somefile.py",
"aeneas/tools/res/bar/baz/foo.bar")]
for test in tests:
self.assertEqual(gf.relative_path(test[0], test[1]), test[2])
class ReadAudioCLI(AbstractCLIProgram):
"""
Read audio file properties.
"""
AUDIO_FILE = gf.relative_path("res/audio.mp3", __file__)
NAME = gf.file_name_without_extension(__file__)
HELP = {
"description":
u"Read audio file properties.",
"synopsis": [(u"AUDIO_FILE", True)],
"options":
[u"-f, --full : load samples from file, possibly converting to WAVE"],
"parameters": [],
"examples": [u"%s" % (AUDIO_FILE)]
}
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 1:
return self.print_help()
audio_file_path = self.actual_arguments[0]
try:
audiofile = AudioFile(audio_file_path,
rconf=self.rconf,
logger=self.logger)
audiofile.read_properties()
if self.has_option([u"-f", u"--full"]):
audiofile.read_samples_from_file()
self.print_generic(audiofile.__unicode__())
return self.NO_ERROR_EXIT_CODE
except OSError:
self.print_error(u"Cannot read file '%s'" % (audio_file_path))
self.print_error(
u"Make sure the input file path is written/escaped correctly")
except AudioFileProbeError:
self.print_error(u"Unable to call the ffprobe executable '%s'" %
(self.rconf[RuntimeConfiguration.FFPROBE_PATH]))
self.print_error(u"Make sure the path to ffprobe is correct")
except AudioFileUnsupportedFormatError:
self.print_error(u"Cannot read properties of file '%s'" %
(audio_file_path))
self.print_error(
u"Make sure the input file has a format supported by ffprobe")
return self.ERROR_EXIT_CODE
class FFMPEGWrapperCLI(AbstractCLIProgram):
"""
Convert audio files to mono WAV using the ``ffmpeg`` wrapper.
"""
INPUT_FILE = gf.relative_path("res/audio.mp3", __file__)
OUTPUT_FILE = "output/audio.wav"
NAME = gf.file_name_without_extension(__file__)
HELP = {
"description":
u"Convert audio files to mono WAV using the ffmpeg wrapper.",
"synopsis": [(u"INPUT_FILE OUTPUT_FILE", True)],
"examples": [u"%s %s" % (INPUT_FILE, OUTPUT_FILE)]
}
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 2:
return self.print_help()
input_file_path = self.actual_arguments[0]
output_file_path = self.actual_arguments[1]
if not self.check_input_file(input_file_path):
return self.ERROR_EXIT_CODE
if not self.check_output_file(output_file_path):
return self.ERROR_EXIT_CODE
try:
converter = FFMPEGWrapper(rconf=self.rconf, logger=self.logger)
converter.convert(input_file_path, output_file_path)
self.print_success(u"Converted '%s' into '%s'" %
(input_file_path, output_file_path))
return self.NO_ERROR_EXIT_CODE
except FFMPEGPathError:
self.print_error(u"Unable to call the ffmpeg executable '%s'" %
(self.rconf[RuntimeConfiguration.FFMPEG_PATH]))
self.print_error(u"Make sure the path to ffmpeg is correct")
except OSError:
self.print_error(u"Cannot convert file '%s' into '%s'" %
(input_file_path, output_file_path))
self.print_error(
u"Make sure the input file has a format supported by ffmpeg")
return self.ERROR_EXIT_CODE
class FFPROBEWrapperCLI(AbstractCLIProgram):
"""
Read audio file properties using the ``ffprobe`` wrapper.
"""
AUDIO_FILE = gf.relative_path("res/audio.mp3", __file__)
NAME = gf.file_name_without_extension(__file__)
HELP = {
"description":
u"Read audio file properties using the ffprobe wrapper.",
"synopsis": [(u"AUDIO_FILE", True)],
"examples": [u"%s" % (AUDIO_FILE)]
}
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 1:
return self.print_help()
audio_file_path = self.actual_arguments[0]
if not self.check_input_file(audio_file_path):
return self.ERROR_EXIT_CODE
try:
prober = FFPROBEWrapper(rconf=self.rconf, logger=self.logger)
dictionary = prober.read_properties(audio_file_path)
for key in sorted(dictionary.keys()):
self.print_generic(u"%s %s" % (key, dictionary[key]))
return self.NO_ERROR_EXIT_CODE
except FFPROBEPathError:
self.print_error(u"Unable to call the ffprobe executable '%s'" %
(self.rconf[RuntimeConfiguration.FFPROBE_PATH]))
self.print_error(u"Make sure the path to ffprobe is correct")
except (FFPROBEUnsupportedFormatError, FFPROBEParsingError):
self.print_error(u"Cannot read properties of file '%s'" %
(audio_file_path))
self.print_error(
u"Make sure the input file has a format supported by ffprobe")
return self.ERROR_EXIT_CODE
class PlotWaveformCLI(AbstractCLIProgram):
"""
Plot a waveform and labels to file.
"""
AUDIO_FILE = gf.relative_path("res/audio.mp3", __file__)
VAD_FILE = gf.relative_path("res/sonnet.vad", __file__)
OUTPUT_FILE = "output/sonnet.png"
NAME = gf.file_name_without_extension(__file__)
HELP = {
"description":
u"Plot a waveform and labels to file.",
"synopsis":
[(u"AUDIO_FILE OUTPUT_FILE [-i LABEL_FILE [-i LABEL_FILE [...]]]",
True)],
"examples": [u"%s %s -i %s" % (AUDIO_FILE, OUTPUT_FILE, VAD_FILE)],
"options": [
u"--fast : enable fast waveform rendering (default: False)",
u"--hzoom=ZOOM : horizontal zoom (int, default: 5)",
u"--label=LABEL : label for the plot (str)",
u"--text : show fragment text instead of identifier",
u"--time-step=STEP : print time ticks every STEP seconds (int)",
u"--vzoom=ZOOM : vertical zoom (int, default: 30)"
]
}
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 2:
return self.print_help()
input_file_path = self.actual_arguments[0]
output_file_path = self.actual_arguments[1]
if not self.check_input_file(input_file_path):
return self.ERROR_EXIT_CODE
if not self.check_output_file(output_file_path):
return self.ERROR_EXIT_CODE
fast = self.has_option("--fast")
fragment_text = self.has_option("--text")
h_zoom = gf.safe_int(self.has_option_with_value("--hzoom"), 5)
label = self.has_option_with_value("--label")
time_step = gf.safe_int(self.has_option_with_value("--time-step"), 0)
v_zoom = gf.safe_int(self.has_option_with_value("--vzoom"), 30)
labels = not self.has_option("--no-labels")
begin_times = not self.has_option("--no-begin-times")
end_times = not self.has_option("--no-end-times")
begin_guides = not self.has_option("--no-begin-guides")
end_guides = not self.has_option("--no-end-guides")
try:
# import or ImportError
from aeneas.plotter import PlotLabelset
from aeneas.plotter import PlotTimeScale
from aeneas.plotter import PlotWaveform
from aeneas.plotter import Plotter
# create plotter object
self.print_info(u"Plotting to file...")
plotter = Plotter(rconf=self.rconf, logger=self.logger)
# add waveform
afm = AudioFile(input_file_path,
rconf=self.rconf,
logger=self.logger)
afm.read_samples_from_file()
plotter.add_waveform(
PlotWaveform(afm,
label=label,
fast=fast,
rconf=self.rconf,
logger=self.logger))
# add time scale, if requested
if time_step > 0:
plotter.add_timescale(
PlotTimeScale(afm.audio_length,
time_step=time_step,
rconf=self.rconf,
logger=self.logger))
# add labelsets, if any
for i in range(len(self.actual_arguments)):
if (self.actual_arguments[i]
== "-i") and (i + 1 < len(self.actual_arguments)):
label_file_path = self.actual_arguments[i + 1]
extension = gf.file_extension(label_file_path)
if extension == "vad":
labelset = self._read_syncmap_file(
label_file_path, SyncMapFormat.TSV, False)
ls = PlotLabelset(labelset,
parameters=None,
rconf=self.rconf,
logger=self.logger)
ls.parameters["labels"] = False
ls.parameters["begin_time"] = begin_times
ls.parameters["end_time"] = end_times
ls.parameters["begin_guide"] = begin_guides
ls.parameters["end_guide"] = end_guides
plotter.add_labelset(ls)
if extension in SyncMapFormat.ALLOWED_VALUES:
labelset = self._read_syncmap_file(
label_file_path, extension, fragment_text)
ls = PlotLabelset(labelset,
parameters=None,
rconf=self.rconf,
logger=self.logger)
ls.parameters["labels"] = labels
ls.parameters["begin_time"] = begin_times
ls.parameters["end_time"] = end_times
ls.parameters["begin_guide"] = begin_guides
ls.parameters["end_guide"] = end_guides
plotter.add_labelset(ls)
# output to file
plotter.draw_png(output_file_path, h_zoom=h_zoom, v_zoom=v_zoom)
self.print_info(u"Plotting to file... done")
self.print_success(u"Created file '%s'" % output_file_path)
return self.NO_ERROR_EXIT_CODE
except ImportError:
self.print_error(
u"You need to install Python module Pillow to output image to file. Run:"
)
self.print_error(u"$ pip install Pillow")
self.print_error(u"or, to install for all users:")
self.print_error(u"$ sudo pip install Pillow")
except Exception as exc:
self.print_error(
u"An unexpected error occurred while generating the image file:"
)
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
def _read_syncmap_file(self, path, extension, text=False):
""" Read labels from a SyncMap file """
syncmap = SyncMap(logger=self.logger)
syncmap.read(extension, path, parameters=None)
if text:
return [(f.begin, f.end, u" ".join(f.text_fragment.lines))
for f in syncmap.fragments]
return [(f.begin, f.end, f.text_fragment.identifier)
for f in syncmap.fragments]
class ValidateCLI(AbstractCLIProgram):
"""
Perform validation in one of the following modes:
1. a container
2. a job configuration string
3. a task configuration string
4. a container + configuration string from wizard
5. a job TXT configuration file
6. a job XML configuration file
"""
CONFIG_FILE_TXT = gf.relative_path("res/config.txt", __file__)
CONFIG_FILE_XML = gf.relative_path("res/config.xml", __file__)
CONTAINER_FILE = gf.relative_path("res/job.zip", __file__)
JOB_CONFIG_STRING = u"job_language=ita|os_job_file_name=output.zip|os_job_file_container=zip|is_hierarchy_type=flat"
TASK_CONFIG_STRING = u"task_language=ita|is_text_type=plain|os_task_file_name=output.txt|os_task_file_format=txt"
WRONG_CONFIG_STRING = u"job_language=ita|invalid=string"
GOOD_CONFIG_STRING = u"is_hierarchy_type=flat|is_hierarchy_prefix=assets/|is_text_file_relative_path=.|is_text_file_name_regex=.*\.xhtml|is_text_type=unparsed|is_audio_file_relative_path=.|is_audio_file_name_regex=.*\.mp3|is_text_unparsed_id_regex=f[0-9]+|is_text_unparsed_id_sort=numeric|os_job_file_name=demo_sync_job_output|os_job_file_container=zip|os_job_file_hierarchy_type=flat|os_job_file_hierarchy_prefix=assets/|os_task_file_name=\\$PREFIX.xhtml.smil|os_task_file_format=smil|os_task_file_smil_page_ref=\\$PREFIX.xhtml|os_task_file_smil_audio_ref=../Audio/\\$PREFIX.mp3|job_language=eng|job_description=Demo Sync Job"
NAME = gf.file_name_without_extension(__file__)
HELP = {
"description":
u"Perform validation of a config string or a container",
"synopsis":
[(u"config CONFIG.TXT", True), (u"config CONFIG.XML", True),
(u"container CONTAINER", True), (u"job CONFIG_STRING", True),
(u"task CONFIG_STRING", True),
(u"wizard CONFIG_STRING CONTAINER", True)],
"examples": [
u"config %s" % (CONFIG_FILE_TXT),
u"config %s" % (CONFIG_FILE_XML),
u"container %s" % (CONTAINER_FILE),
u"job \"%s\"" % (JOB_CONFIG_STRING),
u"task \"%s\"" % (TASK_CONFIG_STRING),
u"wizard \"%s\" %s" % (GOOD_CONFIG_STRING, CONTAINER_FILE)
]
}
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 2:
return self.print_help()
mode = self.actual_arguments[0]
validator = Validator(rconf=self.rconf, logger=self.logger)
if mode == u"config":
config_file_path = self.actual_arguments[1]
config_txt = None
if config_file_path.lower().endswith(u".txt"):
config_txt = True
elif config_file_path.lower().endswith(u".xml"):
config_txt = False
else:
return self.print_help()
if not self.check_input_file(config_file_path):
return self.ERROR_EXIT_CODE
contents = gf.read_file_bytes(config_file_path)
if contents is None:
return self.ERROR_EXIT_CODE
if config_txt:
result = validator.check_config_txt(contents)
msg = u"TXT configuration"
else:
result = validator.check_config_xml(contents)
msg = "XML configuration"
elif mode == u"container":
container_path = self.actual_arguments[1]
result = validator.check_container(container_path)
msg = "container"
elif mode == u"job":
config_string = self.actual_arguments[1]
result = validator.check_configuration_string(config_string,
is_job=True)
msg = u"job configuration string"
elif mode == u"task":
config_string = self.actual_arguments[1]
result = validator.check_configuration_string(config_string,
is_job=False,
external_name=True)
msg = u"task configuration string"
elif mode == u"wizard":
if (len(self.actual_arguments) <
3) or (self.actual_arguments[2].startswith(u"-")):
return self.print_help()
config_string = self.actual_arguments[1]
container_path = self.actual_arguments[2]
if not self.check_input_file(container_path):
return self.ERROR_EXIT_CODE
result = validator.check_container(container_path,
config_string=config_string)
msg = "container with configuration string from wizard"
else:
return self.print_help()
if result.passed:
self.print_success(u"Valid %s" % msg)
for warning in result.warnings:
self.print_warning(u"%s" % warning)
return self.NO_ERROR_EXIT_CODE
else:
self.print_error(u"Invalid %s" % msg)
for error in result.errors:
self.print_error(u"%s" % error)
return self.ERROR_EXIT_CODE
class SynthesizeTextCLI(AbstractCLIProgram):
"""
Synthesize several text fragments,
producing a WAV audio file.
"""
OUTPUT_FILE = "output/synthesized.wav"
TEXT_FILE_MPLAIN = gf.relative_path("res/mplain.txt", __file__)
TEXT_FILE_MUNPARSED = gf.relative_path("res/munparsed2.xhtml", __file__)
TEXT_FILE_PARSED = gf.relative_path("res/parsed.txt", __file__)
TEXT_FILE_PLAIN = gf.relative_path("res/plain.txt", __file__)
TEXT_FILE_SUBTITLES = gf.relative_path("res/subtitles.txt", __file__)
TEXT_FILE_UNPARSED = gf.relative_path("res/unparsed.xhtml", __file__)
NAME = gf.file_name_without_extension(__file__)
HELP = {
"description":
u"Synthesize several text fragments.",
"synopsis":
[(u"list 'fragment 1|fragment 2|...|fragment N' LANGUAGE OUTPUT_FILE",
True),
(u"[mplain|munparsed|parsed|plain|subtitles|unparsed] TEXT_FILE LANGUAGE OUTPUT_FILE",
True)],
"examples": [
u"list 'From|fairest|creatures|we|desire|increase' eng %s" %
(OUTPUT_FILE),
u"mplain %s eng %s" % (TEXT_FILE_MPLAIN, OUTPUT_FILE),
u"munparsed %s eng %s --l1-id-regex=p[0-9]+ --l2-id-regex=s[0-9]+ --l3-id-regex=w[0-9]+"
% (TEXT_FILE_MUNPARSED, OUTPUT_FILE),
u"parsed %s eng %s" % (TEXT_FILE_PARSED, OUTPUT_FILE),
u"plain %s eng %s" % (TEXT_FILE_PLAIN, OUTPUT_FILE),
u"subtitles %s eng %s" % (TEXT_FILE_SUBTITLES, OUTPUT_FILE),
u"unparsed %s eng %s --id-regex=f[0-9]*" %
(TEXT_FILE_UNPARSED, OUTPUT_FILE),
u"unparsed %s eng %s --class-regex=ra" %
(TEXT_FILE_UNPARSED, OUTPUT_FILE),
u"unparsed %s eng %s --id-regex=f[0-9]* --sort=numeric" %
(TEXT_FILE_UNPARSED, OUTPUT_FILE),
u"plain %s eng %s --start=5" % (TEXT_FILE_PLAIN, OUTPUT_FILE),
u"plain %s eng %s --end=10" % (TEXT_FILE_PLAIN, OUTPUT_FILE),
u"plain %s eng %s --start=5 --end=10" %
(TEXT_FILE_PLAIN, OUTPUT_FILE),
u"plain %s eng %s --backwards" % (TEXT_FILE_PLAIN, OUTPUT_FILE),
u"plain %s eng %s --quit-after=10.0" %
(TEXT_FILE_PLAIN, OUTPUT_FILE),
],
"options": [
u"--class-regex=REGEX : extract text from elements with class attribute matching REGEX (unparsed)",
u"--end=INDEX : slice the text file until fragment INDEX",
u"--id-format=FMT : use FMT for generating text id attributes (subtitles, plain)",
u"--id-regex=REGEX : extract text from elements with id attribute matching REGEX (unparsed)",
u"--l1-id-regex=REGEX : extract text from level 1 elements with id attribute matching REGEX (munparsed)",
u"--l2-id-regex=REGEX : extract text from level 2 elements with id attribute matching REGEX (munparsed)",
u"--l3-id-regex=REGEX : extract text from level 3 elements with id attribute matching REGEX (munparsed)",
u"--quit-after=DUR : synthesize fragments until DUR seconds or the end of text is reached",
u"--sort=ALGORITHM : sort the matched element id attributes using ALGORITHM (lexicographic, numeric, unsorted)",
u"--start=INDEX : slice the text file from fragment INDEX",
u"-b, --backwards : synthesize from the last fragment to the first one",
]
}
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 4:
return self.print_help()
text_format = gf.safe_unicode(self.actual_arguments[0])
if text_format == u"list":
text = gf.safe_unicode(self.actual_arguments[1])
elif text_format in TextFileFormat.ALLOWED_VALUES:
text = self.actual_arguments[1]
if not self.check_input_file(text):
return self.ERROR_EXIT_CODE
else:
return self.print_help()
l1_id_regex = self.has_option_with_value(u"--l1-id-regex")
l2_id_regex = self.has_option_with_value(u"--l2-id-regex")
l3_id_regex = self.has_option_with_value(u"--l3-id-regex")
id_regex = self.has_option_with_value(u"--id-regex")
class_regex = self.has_option_with_value(u"--class-regex")
sort = self.has_option_with_value(u"--sort")
backwards = self.has_option([u"-b", u"--backwards"])
quit_after = gf.safe_float(self.has_option_with_value(u"--quit-after"),
None)
start_fragment = gf.safe_int(self.has_option_with_value(u"--start"),
None)
end_fragment = gf.safe_int(self.has_option_with_value(u"--end"), None)
parameters = {
gc.PPN_TASK_IS_TEXT_MUNPARSED_L1_ID_REGEX: l1_id_regex,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L2_ID_REGEX: l2_id_regex,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L3_ID_REGEX: l3_id_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_CLASS_REGEX: class_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_ID_REGEX: id_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_ID_SORT: sort,
}
if (text_format == TextFileFormat.MUNPARSED) and (
(l1_id_regex is None) or (l2_id_regex is None) or
(l3_id_regex is None)):
self.print_error(
u"You must specify --l1-id-regex and --l2-id-regex and --l3-id-regex for munparsed format"
)
return self.ERROR_EXIT_CODE
if (text_format == TextFileFormat.UNPARSED) and (
id_regex is None) and (class_regex is None):
self.print_error(
u"You must specify --id-regex and/or --class-regex for unparsed format"
)
return self.ERROR_EXIT_CODE
language = gf.safe_unicode(self.actual_arguments[2])
output_file_path = self.actual_arguments[3]
if not self.check_output_file(output_file_path):
return self.ERROR_EXIT_CODE
text_file = self.get_text_file(text_format, text, parameters)
if text_file is None:
self.print_error(
u"Unable to build a TextFile from the given parameters")
return self.ERROR_EXIT_CODE
elif len(text_file) == 0:
self.print_error(u"No text fragments found")
return self.ERROR_EXIT_CODE
text_file.set_language(language)
self.print_info(u"Read input text with %d fragments" %
(len(text_file)))
if start_fragment is not None:
self.print_info(u"Slicing from index %d" % (start_fragment))
if end_fragment is not None:
self.print_info(u"Slicing to index %d" % (end_fragment))
text_slice = text_file.get_slice(start_fragment, end_fragment)
self.print_info(u"Synthesizing %d fragments" % (len(text_slice)))
if quit_after is not None:
self.print_info(u"Stop synthesizing upon reaching %.3f seconds" %
(quit_after))
try:
synt = Synthesizer(rconf=self.rconf, logger=self.logger)
synt.synthesize(text_slice,
output_file_path,
quit_after=quit_after,
backwards=backwards)
self.print_success(u"Created file '%s'" % output_file_path)
synt.clear_cache()
return self.NO_ERROR_EXIT_CODE
except ImportError as exc:
tts = self.rconf[RuntimeConfiguration.TTS]
if tts == Synthesizer.AWS:
self.print_error(
u"You need to install Python module boto3 to use the AWS Polly TTS API wrapper. Run:"
)
self.print_error(u"$ pip install boto3")
self.print_error(u"or, to install for all users:")
self.print_error(u"$ sudo pip install boto3")
elif tts == Synthesizer.NUANCE:
self.print_error(
u"You need to install Python module requests to use the Nuance TTS API wrapper. Run:"
)
self.print_error(u"$ pip install requests")
self.print_error(u"or, to install for all users:")
self.print_error(u"$ sudo pip install requests")
else:
self.print_error(
u"An unexpected error occurred while synthesizing text:")
self.print_error(u"%s" % exc)
except Exception as exc:
self.print_error(
u"An unexpected error occurred while synthesizing text:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
class ExtractMFCCCLI(AbstractCLIProgram):
"""
Extract MFCCs from a given audio file.
"""
INPUT_FILE = gf.relative_path("res/audio.wav", __file__)
OUTPUT_FILE = "output/audio.wav.mfcc.txt"
NAME = gf.file_name_without_extension(__file__)
HELP = {
"description":
u"Extract MFCCs from a given audio file as a fat matrix.",
"synopsis": [(u"AUDIO_FILE OUTPUT_FILE", True)],
"examples": [u"%s %s" % (INPUT_FILE, OUTPUT_FILE)],
"options": [
u"-b, --binary : output MFCCs as a float64 binary file",
u"-d, --delete-first : do not output the 0th MFCC coefficient",
u"-n, --npy : output MFCCs as a NumPy .npy binary file",
u"-t, --transpose : transpose the MFCCs matrix, returning a tall matrix",
u"-z, --npz : output MFCCs as a NumPy compressed .npz binary file",
u"--format=FMT : output to text file using format FMT (default: '%.18e')"
]
}
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 2:
return self.print_help()
input_file_path = self.actual_arguments[0]
output_file_path = self.actual_arguments[1]
output_text_format = self.has_option_with_value(u"--format")
if output_text_format is None:
output_text_format = u"%.18e"
output_binary = self.has_option([u"-b", u"--binary"])
output_npz = self.has_option([u"-z", u"--npz"])
output_npy = self.has_option([u"-n", u"--npy"])
delete_first = self.has_option([u"-d", u"--delete-first"])
transpose = self.has_option([u"-t", u"--transpose"])
self.check_c_extensions("cmfcc")
if not self.check_input_file(input_file_path):
return self.ERROR_EXIT_CODE
if not self.check_output_file(output_file_path):
return self.ERROR_EXIT_CODE
try:
mfccs = AudioFileMFCC(input_file_path,
rconf=self.rconf,
logger=self.logger).all_mfcc
if delete_first:
mfccs = mfccs[1:, :]
if transpose:
mfccs = mfccs.transpose()
if output_binary:
# save as a raw C float64 binary file
mapped = numpy.memmap(output_file_path,
dtype="float64",
mode="w+",
shape=mfccs.shape)
mapped[:] = mfccs[:]
mapped.flush()
del mapped
elif output_npz:
# save as a .npz compressed binary file
with io.open(output_file_path, "wb") as output_file:
numpy.savez(output_file, mfccs)
elif output_npy:
# save as a .npy binary file
with io.open(output_file_path, "wb") as output_file:
numpy.save(output_file, mfccs)
else:
# save as a text file
# NOTE: in Python 2, passing the fmt value a Unicode string crashes NumPy
# hence, converting back to bytes, which works in Python 3 too
numpy.savetxt(output_file_path,
mfccs,
fmt=gf.safe_bytes(output_text_format))
self.print_info(u"MFCCs shape: %d %d" % (mfccs.shape))
self.print_success(u"MFCCs saved to '%s'" % (output_file_path))
return self.NO_ERROR_EXIT_CODE
except AudioFileConverterError:
self.print_error(u"Unable to call the ffmpeg executable '%s'" %
(self.rconf[RuntimeConfiguration.FFMPEG_PATH]))
self.print_error(u"Make sure the path to ffmpeg is correct")
except (AudioFileUnsupportedFormatError, AudioFileNotInitializedError):
self.print_error(u"Cannot read file '%s'" % (input_file_path))
self.print_error(u"Check that its format is supported by ffmpeg")
except OSError:
self.print_error(u"Cannot write file '%s'" % (output_file_path))
return self.ERROR_EXIT_CODE
class ConvertSyncMapCLI(AbstractCLIProgram):
"""
Convert a sync map from a format to another.
"""
AUDIO = gf.relative_path("res/audio.mp3", __file__)
SMIL_PARAMETERS = "--audio-ref=audio/sonnet001.mp3 --page-ref=text/sonnet001.xhtml"
SYNC_MAP_CSV = gf.relative_path("res/sonnet.csv", __file__)
SYNC_MAP_JSON = gf.relative_path("res/sonnet.json", __file__)
SYNC_MAP_ZZZ = gf.relative_path("res/sonnet.zzz", __file__)
OUTPUT_HTML = "output/sonnet.html"
OUTPUT_MAP_DAT = "output/syncmap.dat"
OUTPUT_MAP_JSON = "output/syncmap.json"
OUTPUT_MAP_SMIL = "output/syncmap.smil"
OUTPUT_MAP_SRT = "output/syncmap.srt"
OUTPUT_MAP_TXT = "output/syncmap.txt"
NAME = gf.file_name_without_extension(__file__)
HELP = {
"description":
u"Convert a sync map from a format to another.",
"synopsis": [
(u"INPUT_SYNCMAP OUTPUT_SYNCMAP", True),
(u"INPUT_SYNCMAP OUTPUT_HTML AUDIO_FILE --output-html", True),
],
"examples": [
u"%s %s" % (SYNC_MAP_JSON, OUTPUT_MAP_SRT),
u"%s %s --output-format=txt" % (SYNC_MAP_JSON, OUTPUT_MAP_DAT),
u"%s %s --input-format=csv" % (SYNC_MAP_ZZZ, OUTPUT_MAP_TXT),
u"%s %s --language=en" % (SYNC_MAP_CSV, OUTPUT_MAP_JSON),
u"%s %s %s" % (SYNC_MAP_JSON, OUTPUT_MAP_SMIL, SMIL_PARAMETERS),
u"%s %s %s --output-html" % (SYNC_MAP_JSON, OUTPUT_HTML, AUDIO)
],
"options": [
u"--audio-ref=REF : use REF for the audio ref attribute (smil, smilh, smilm)",
u"--input-format=FMT : input sync map file has format FMT",
u"--language=CODE : set language to CODE",
u"--output-format=FMT : output sync map file has format FMT",
u"--output-html : output HTML file for fine tuning",
u"--page-ref=REF : use REF for the text ref attribute (smil, smilh, smilm)"
]
}
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 2:
return self.print_help()
input_file_path = self.actual_arguments[0]
output_file_path = self.actual_arguments[1]
output_html = self.has_option(u"--output-html")
if not self.check_input_file(input_file_path):
return self.ERROR_EXIT_CODE
input_sm_format = self.has_option_with_value(u"--input-format")
if input_sm_format is None:
input_sm_format = gf.file_extension(input_file_path)
if not self.check_format(input_sm_format):
return self.ERROR_EXIT_CODE
if not self.check_output_file(output_file_path):
return self.ERROR_EXIT_CODE
if output_html:
if len(self.actual_arguments) < 3:
return self.print_help()
audio_file_path = self.actual_arguments[2]
if not self.check_input_file(audio_file_path):
return self.ERROR_EXIT_CODE
else:
output_sm_format = self.has_option_with_value(u"--output-format")
if output_sm_format is None:
output_sm_format = gf.file_extension(output_file_path)
if not self.check_format(output_sm_format):
return self.ERROR_EXIT_CODE
# TODO add a way to specify a text file for input formats like SMIL
# that do not carry the source text
language = self.has_option_with_value(u"--language")
audio_ref = self.has_option_with_value(u"--audio-ref")
page_ref = self.has_option_with_value(u"--page-ref")
parameters = {
gc.PPN_SYNCMAP_LANGUAGE: language,
gc.PPN_TASK_OS_FILE_SMIL_AUDIO_REF: audio_ref,
gc.PPN_TASK_OS_FILE_SMIL_PAGE_REF: page_ref
}
try:
self.print_info(u"Reading sync map in '%s' format from file '%s'" %
(input_sm_format, input_file_path))
self.print_info(u"Reading sync map...")
syncmap = SyncMap(logger=self.logger)
syncmap.read(input_sm_format, input_file_path, parameters)
self.print_info(u"Reading sync map... done")
self.print_info(u"Read %d sync map fragments" % (len(syncmap)))
except Exception as exc:
self.print_error(
u"An unexpected error occurred while reading the input sync map:"
)
self.print_error(u"%s" % (exc))
return self.ERROR_EXIT_CODE
if output_html:
try:
self.print_info(u"Writing HTML file...")
syncmap.output_html_for_tuning(audio_file_path,
output_file_path, parameters)
self.print_info(u"Writing HTML file... done")
self.print_success(u"Created HTML file '%s'" %
(output_file_path))
return self.NO_ERROR_EXIT_CODE
except Exception as exc:
self.print_error(
u"An unexpected error occurred while writing the output HTML file:"
)
self.print_error(u"%s" % (exc))
else:
try:
self.print_info(u"Writing sync map...")
syncmap.write(output_sm_format, output_file_path, parameters)
self.print_info(u"Writing sync map... done")
self.print_success(u"Created '%s' sync map file '%s'" %
(output_sm_format, output_file_path))
return self.NO_ERROR_EXIT_CODE
except Exception as exc:
self.print_error(
u"An unexpected error occurred while writing the output sync map:"
)
self.print_error(u"%s" % (exc))
return self.ERROR_EXIT_CODE
def check_format(self, sm_format):
"""
Return ``True`` if the given sync map format is allowed,
and ``False`` otherwise.
:param sm_format: the sync map format to be checked
:type sm_format: Unicode string
:rtype: bool
"""
if sm_format not in SyncMapFormat.ALLOWED_VALUES:
self.print_error(u"Sync map format '%s' is not allowed" %
(sm_format))
self.print_info(u"Allowed formats:")
self.print_generic(u" ".join(SyncMapFormat.ALLOWED_VALUES))
return False
return True
class ExecuteJobCLI(AbstractCLIProgram):
"""
Execute a Job, passed as a container or
as a container and a configuration string
(i.e., from a wizard).
"""
CONTAINER_FILE = gf.relative_path("res/job.zip", __file__)
CONTAINER_FILE_NO_CONFIG = gf.relative_path("res/job_no_config.zip",
__file__)
OUTPUT_DIRECTORY = "output/"
CONFIG_STRING = u"is_hierarchy_type=flat|is_hierarchy_prefix=assets/|is_text_file_relative_path=.|is_text_file_name_regex=.*\.xhtml|is_text_type=unparsed|is_audio_file_relative_path=.|is_audio_file_name_regex=.*\.mp3|is_text_unparsed_id_regex=f[0-9]+|is_text_unparsed_id_sort=numeric|os_job_file_name=demo_sync_job_output|os_job_file_container=zip|os_job_file_hierarchy_type=flat|os_job_file_hierarchy_prefix=assets/|os_task_file_name=\\$PREFIX.xhtml.smil|os_task_file_format=smil|os_task_file_smil_page_ref=\\$PREFIX.xhtml|os_task_file_smil_audio_ref=../Audio/\\$PREFIX.mp3|job_language=eng|job_description=Demo Sync Job"
NAME = gf.file_name_without_extension(__file__)
HELP = {
"description":
u"Execute a Job, passed as a container.",
"synopsis": [(u"CONTAINER OUTPUT_DIR [CONFIG_STRING]", True)],
"examples": [
u"%s %s" % (CONTAINER_FILE, OUTPUT_DIRECTORY),
u"%s %s --cewsubprocess" % (CONTAINER_FILE, OUTPUT_DIRECTORY),
u"%s %s \"%s\"" %
(CONTAINER_FILE_NO_CONFIG, OUTPUT_DIRECTORY, CONFIG_STRING)
],
"options": [
u"--cewsubprocess : run cew in separate process (see docs)",
u"--skip-validator : do not validate the given container and/or config string"
]
}
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 2:
return self.print_help()
container_path = self.actual_arguments[0]
output_directory_path = self.actual_arguments[1]
config_string = None
if (len(self.actual_arguments)) > 2 and (
not self.actual_arguments[2].startswith(u"-")):
config_string = self.actual_arguments[2]
validate = not self.has_option(u"--skip-validator")
if self.has_option(u"--cewsubprocess"):
self.rconf[RuntimeConfiguration.CEW_SUBPROCESS_ENABLED] = True
if not self.check_input_file_or_directory(container_path):
return self.ERROR_EXIT_CODE
if not self.check_output_directory(output_directory_path):
return self.ERROR_EXIT_CODE
if validate:
try:
self.print_info(
u"Validating the container (specify --skip-validator to bypass)..."
)
validator = Validator(rconf=self.rconf, logger=self.logger)
result = validator.check_container(container_path,
config_string=config_string)
if not result.passed:
self.print_error(u"The given container is not valid:")
self.print_error(result.pretty_print())
return self.ERROR_EXIT_CODE
self.print_info(u"Validating the container... done")
except Exception as exc:
self.print_error(
u"An unexpected error occurred while validating the container:"
)
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Loading job from container...")
executor = ExecuteJob(rconf=self.rconf, logger=self.logger)
executor.load_job_from_container(container_path, config_string)
self.print_info(u"Loading job from container... done")
except Exception as exc:
self.print_error(
u"An unexpected error occurred while loading the job:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Executing...")
executor.execute()
self.print_info(u"Executing... done")
except Exception as exc:
self.print_error(
u"An unexpected error occurred while executing the job:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Creating output container...")
path = executor.write_output_container(output_directory_path)
self.print_info(u"Creating output container... done")
self.print_success(u"Created output file '%s'" % path)
executor.clean(True)
return self.NO_ERROR_EXIT_CODE
except Exception as exc:
self.print_error(
u"An unexpected error occurred while writing the output container:"
)
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
class ExecuteTaskCLI(AbstractCLIProgram):
"""
Execute a Task, that is, a pair of audio/text files
and a configuration string.
"""
AUDIO_FILE = gf.relative_path("res/audio.mp3", __file__)
CTW_ESPEAK = gf.relative_path("../extra/ctw_espeak.py", __file__)
CTW_SPEECT = gf.relative_path("../extra/ctw_speect/ctw_speect.py",
__file__)
DEMOS = {
u"--example-aftercurrent": {
u"description":
u"input: plain text (plain), output: AUD, aba beforenext 0.200",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/plain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=plain|os_task_file_format=aud|task_adjust_boundary_algorithm=aftercurrent|task_adjust_boundary_aftercurrent_value=0.200",
u"syncmap": "output/sonnet.aftercurrent0200.aud",
u"options": u"",
u"show": False
},
u"--example-beforenext": {
u"description":
u"input: plain text (plain), output: AUD, aba beforenext 0.200",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/plain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=plain|os_task_file_format=aud|task_adjust_boundary_algorithm=beforenext|task_adjust_boundary_beforenext_value=0.200",
u"syncmap": "output/sonnet.beforenext0200.aud",
u"options": u"",
u"show": False
},
u"--example-cewsubprocess": {
u"description":
u"input: plain text, output: TSV, run via cewsubprocess",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/plain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=plain|os_task_file_format=tsv",
u"syncmap": "output/sonnet.cewsubprocess.tsv",
u"options": u"-r=\"cew_subprocess_enabled=True\"",
u"show": False
},
u"--example-ctw-espeak": {
u"description":
u"input: plain text, output: TSV, tts engine: ctw espeak",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/plain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=plain|os_task_file_format=tsv",
u"syncmap": "output/sonnet.ctw_espeak.tsv",
u"options": u"-r=\"tts=custom|tts_path=%s\"" % CTW_ESPEAK,
u"show": False
},
u"--example-ctw-speect": {
u"description":
u"input: plain text, output: TSV, tts engine: ctw speect",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/plain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=plain|os_task_file_format=tsv",
u"syncmap": "output/sonnet.ctw_speect.tsv",
u"options": u"-r=\"tts=custom|tts_path=%s\"" % CTW_SPEECT,
u"show": False
},
u"--example-eaf": {
u"description": u"input: plain text, output: EAF",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/plain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=plain|os_task_file_format=eaf",
u"syncmap": "output/sonnet.eaf",
u"options": u"",
u"show": True
},
u"--example-faster-rate": {
u"description":
u"input: plain text (plain), output: SRT, print faster than 14.0 chars/s",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/plain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=plain|os_task_file_format=srt|task_adjust_boundary_algorithm=rate|task_adjust_boundary_rate_value=14.000",
u"syncmap": "output/sonnet.faster.srt",
u"options": u"--faster-rate",
u"show": False
},
u"--example-festival": {
u"description":
u"input: plain text, output: TSV, tts engine: Festival",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/plain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=plain|os_task_file_format=tsv",
u"syncmap": "output/sonnet.festival.tsv",
u"options": u"-r=\"tts=festival\"",
u"show": False
},
u"--example-flatten-12": {
u"description":
u"input: mplain text (multilevel), output: JSON, levels to output: 1 and 2",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/mplain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=mplain|os_task_file_format=json|os_task_file_levels=12",
u"syncmap": "output/sonnet.flatten12.json",
u"options": u"",
u"show": False
},
u"--example-flatten-2": {
u"description":
u"input: mplain text (multilevel), output: JSON, levels to output: 2",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/mplain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=mplain|os_task_file_format=json|os_task_file_levels=2",
u"syncmap": "output/sonnet.flatten2.json",
u"options": u"",
u"show": False
},
u"--example-flatten-3": {
u"description":
u"input: mplain text (multilevel), output: JSON, levels to output: 3",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/mplain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=mplain|os_task_file_format=json|os_task_file_levels=3",
u"syncmap": "output/sonnet.flatten3.json",
u"options": u"",
u"show": False
},
u"--example-head-tail": {
u"description":
u"input: plain text, output: TSV, explicit head and tail",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/plain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=plain|os_task_file_format=tsv|is_audio_file_head_length=0.400|is_audio_file_tail_length=0.500|os_task_file_head_tail_format=stretch",
u"syncmap": "output/sonnet.headtail.tsv",
u"options": u"",
u"show": False
},
u"--example-json": {
u"description": u"input: plain text, output: JSON",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/plain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=plain|os_task_file_format=json",
u"syncmap": "output/sonnet.json",
u"options": u"",
u"show": True
},
u"--example-mplain-json": {
u"description":
u"input: multilevel plain text (mplain), output: JSON",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/mplain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=mplain|os_task_file_format=json",
u"syncmap": "output/sonnet.mplain.json",
u"options": u"",
u"show": False
},
u"--example-mplain-smil": {
u"description":
u"input: multilevel plain text (mplain), output: SMIL",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/mplain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=mplain|os_task_file_format=smil|os_task_file_smil_audio_ref=p001.mp3|os_task_file_smil_page_ref=p001.xhtml",
u"syncmap": "output/sonnet.mplain.smil",
u"options": u"",
u"show": True
},
u"--example-multilevel-tts": {
u"description":
u"input: multilevel plain text (mplain), different TTS engines, output: JSON",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/mplain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=mplain|os_task_file_format=json",
u"syncmap": "output/sonnet.mplain.json",
u"options":
u"-r=\"tts_l1=festival|tts_l2=festival|tts_l3=espeak\"",
u"show": False
},
u"--example-munparsed-json": {
u"description":
u"input: multilevel unparsed text (munparsed), output: JSON",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/munparsed2.xhtml", __file__),
u"config":
u"task_language=eng|is_text_type=munparsed|is_text_munparsed_l1_id_regex=p[0-9]+|is_text_munparsed_l2_id_regex=s[0-9]+|is_text_munparsed_l3_id_regex=w[0-9]+|os_task_file_format=json",
u"syncmap": "output/sonnet.munparsed.json",
u"options": u"",
u"show": False
},
u"--example-munparsed-smil": {
u"description":
u"input: multilevel unparsed text (munparsed), output: SMIL",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/munparsed.xhtml", __file__),
u"config":
u"task_language=eng|is_text_type=munparsed|is_text_munparsed_l1_id_regex=p[0-9]+|is_text_munparsed_l2_id_regex=p[0-9]+s[0-9]+|is_text_munparsed_l3_id_regex=p[0-9]+s[0-9]+w[0-9]+|os_task_file_format=smil|os_task_file_smil_audio_ref=p001.mp3|os_task_file_smil_page_ref=p001.xhtml",
u"syncmap": "output/sonnet.munparsed.smil",
u"options": u"",
u"show": True
},
u"--example-mws": {
u"description":
u"input: plain text, output: JSON, resolution: 0.500s",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/plain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=plain|os_task_file_format=json",
u"syncmap": "output/sonnet.mws.json",
u"options":
u"-r=\"mfcc_window_length=1.500|mfcc_window_shift=0.500\"",
u"show": False
},
u"--example-no-zero": {
u"description":
u"input: multilevel plain text (mplain), output: JSON, no zero duration",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/mplain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=mplain|os_task_file_format=json|os_task_file_no_zero=True",
u"syncmap": "output/sonnet.nozero.json",
u"options": u"--zero",
u"show": False
},
u"--example-offset": {
u"description":
u"input: plain text (plain), output: AUD, aba offset 0.200",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/plain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=plain|os_task_file_format=aud|task_adjust_boundary_algorithm=offset|task_adjust_boundary_offset_value=0.200",
u"syncmap": "output/sonnet.offset0200.aud",
u"options": u"",
u"show": False
},
u"--example-percent": {
u"description":
u"input: plain text (plain), output: AUD, aba percent 50",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/plain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=plain|os_task_file_format=aud|task_adjust_boundary_algorithm=percent|task_adjust_boundary_percent_value=50",
u"syncmap": "output/sonnet.percent50.aud",
u"options": u"",
u"show": False
},
u"--example-py": {
u"description": u"input: plain text, output: JSON, pure python",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/plain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=plain|os_task_file_format=json",
u"syncmap": "output/sonnet.cext.json",
u"options": u"-r=\"c_extensions=False\"",
u"show": False
},
u"--example-rates": {
u"description":
u"input: plain text (plain), output: SRT, max rate 14.0 chars/s, print rates",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/plain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=plain|os_task_file_format=srt|task_adjust_boundary_algorithm=rate|task_adjust_boundary_rate_value=14.000",
u"syncmap": "output/sonnet.rates.srt",
u"options": u"--rates",
u"show": False
},
u"--example-sd": {
u"description":
u"input: plain text, output: TSV, head/tail detection",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/plain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=plain|os_task_file_format=tsv|is_audio_file_detect_head_max=10.000|is_audio_file_detect_tail_max=10.000",
u"syncmap": "output/sonnet.sd.tsv",
u"options": u"",
u"show": False
},
u"--example-srt": {
u"description": u"input: subtitles text, output: SRT",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/subtitles.txt", __file__),
u"config":
u"task_language=eng|is_text_type=subtitles|os_task_file_format=srt",
u"syncmap": "output/sonnet.srt",
u"options": u"",
u"show": True
},
u"--example-smil": {
u"description": u"input: unparsed text, output: SMIL",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/page.xhtml", __file__),
u"config":
u"task_language=eng|is_text_type=unparsed|is_text_unparsed_id_regex=f[0-9]+|is_text_unparsed_id_sort=numeric|os_task_file_format=smil|os_task_file_smil_audio_ref=p001.mp3|os_task_file_smil_page_ref=p001.xhtml",
u"syncmap": "output/sonnet.smil",
u"options": u"",
u"show": True
},
u"--example-tsv": {
u"description": u"input: parsed text, output: TSV",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/parsed.txt", __file__),
u"config":
u"task_language=eng|is_text_type=parsed|os_task_file_format=tsv",
u"syncmap": "output/sonnet.tsv",
u"options": u"",
u"show": True
},
u"--example-words": {
u"description":
u"input: single word granularity plain text, output: AUD",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/words.txt", __file__),
u"config":
u"task_language=eng|is_text_type=plain|os_task_file_format=aud",
u"syncmap": "output/sonnet.words.aud",
u"options": u"",
u"show": True
},
u"--example-words-festival-cache": {
u"description":
u"input: single word granularity plain text, output: AUD, tts engine: Festival, TTS cache on",
u"audio": AUDIO_FILE,
u"text": gf.relative_path("res/words.txt", __file__),
u"config":
u"task_language=eng|is_text_type=plain|os_task_file_format=aud",
u"syncmap": "output/sonnet.words.aud",
u"options": u"-r=\"tts=festival|tts_cache=True\"",
u"show": False
},
u"--example-youtube": {
u"description": u"input: audio from YouTube, output: TXT",
u"audio": "https://www.youtube.com/watch?v=rU4a7AA8wM0",
u"text": gf.relative_path("res/plain.txt", __file__),
u"config":
u"task_language=eng|is_text_type=plain|os_task_file_format=txt",
u"syncmap": "output/sonnet.txt",
u"options": u"-y",
u"show": True
}
}
PARAMETERS = [
u" task_language : language (*)",
u"",
u" is_audio_file_detect_head_max : detect audio head, at most this many seconds",
u" is_audio_file_detect_head_min : detect audio head, at least this many seconds",
u" is_audio_file_detect_tail_max : detect audio tail, at most this many seconds",
u" is_audio_file_detect_tail_min : detect audio tail, at least this many seconds",
u" is_audio_file_head_length : ignore this many seconds from the begin of the audio file",
u" is_audio_file_process_length : process this many seconds of the audio file",
u" is_audio_file_tail_length : ignore this many seconds from the end of the audio file",
u"",
u" is_text_type : input text format (*)",
u" is_text_mplain_word_separator : word separator (mplain)",
u" is_text_munparsed_l1_id_regex : regex matching level 1 id attributes (munparsed)",
u" is_text_munparsed_l2_id_regex : regex matching level 2 id attributes (munparsed)",
u" is_text_munparsed_l3_id_regex : regex matching level 3 id attributes (munparsed)",
u" is_text_unparsed_class_regex : regex matching class attributes (unparsed)",
u" is_text_unparsed_id_regex : regex matching id attributes (unparsed)",
u" is_text_unparsed_id_sort : sort matched elements by id (unparsed) (*)",
u" is_text_file_ignore_regex : ignore text matched by regex for audio alignment purposes",
u" is_text_file_transliterate_map : apply the given transliteration map for audio alignment purposes",
u"",
u" os_task_file_format : output sync map format (*)",
u" os_task_file_id_regex : id regex for the output sync map (subtitles, plain)",
u" os_task_file_head_tail_format : format audio head/tail (*)",
u" os_task_file_levels : output the specified levels (mplain)",
u" os_task_file_no_zero : if True, do not allow zero-length fragments",
u" os_task_file_smil_audio_ref : value for the audio ref (smil, smilh, smilm)",
u" os_task_file_smil_page_ref : value for the text ref (smil, smilh, smilm)",
u"",
u" task_adjust_boundary_algorithm : adjust sync map fragments using algorithm (*)",
u" task_adjust_boundary_aftercurrent_value : offset value, in seconds (aftercurrent)",
u" task_adjust_boundary_beforenext_value : offset value, in seconds (beforenext)",
u" task_adjust_boundary_offset_value : offset value, in seconds (offset)",
u" task_adjust_boundary_percent_value : percent value, in [0..100], (percent)",
u" task_adjust_boundary_rate_value : max rate, in characters/s (rate, rateaggressive)",
]
VALUES = {
"espeak": sorted(ESPEAKTTSWrapper.LANGUAGE_TO_VOICE_CODE.keys()),
"espeak-ng": sorted(ESPEAKNGTTSWrapper.LANGUAGE_TO_VOICE_CODE.keys()),
"festival": sorted(FESTIVALTTSWrapper.LANGUAGE_TO_VOICE_CODE.keys()),
"nuance": sorted(NuanceTTSWrapper.LANGUAGE_TO_VOICE_CODE.keys()),
"task_language": Language.ALLOWED_VALUES,
"is_text_type": TextFileFormat.ALLOWED_VALUES,
"is_text_unparsed_id_sort": IDSortingAlgorithm.ALLOWED_VALUES,
"os_task_file_format": SyncMapFormat.ALLOWED_VALUES,
"os_task_file_head_tail_format": SyncMapHeadTailFormat.ALLOWED_VALUES,
"task_adjust_boundary_algorithm":
AdjustBoundaryAlgorithm.ALLOWED_VALUES,
}
NAME = gf.file_name_without_extension(__file__)
HELP = {
"description":
u"Execute a Task.",
"synopsis": [
(u"--list-parameters", False),
(u"--list-values[=PARAM]", False),
(u"AUDIO_FILE TEXT_FILE CONFIG_STRING OUTPUT_FILE", True),
(u"YOUTUBE_URL TEXT_FILE CONFIG_STRING OUTPUT_FILE -y", True),
],
"examples": [u"--examples", u"--examples-all"],
"options": [
u"--faster-rate : print fragments with rate > task_adjust_boundary_rate_value",
u"--keep-audio : do not delete the audio file downloaded from YouTube (-y only)",
u"--largest-audio : download largest audio stream (-y only)",
u"--list-parameters : list all parameters",
u"--list-values : list all parameters for which values can be listed",
u"--list-values=PARAM : list all allowed values for parameter PARAM",
u"--output-html : output HTML file for fine tuning",
u"--rates : print rate of each fragment",
u"--skip-validator : do not validate the given config string",
u"--zero : print fragments with zero duration",
u"-y, --youtube : download audio from YouTube video",
]
}
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 1:
return self.print_help()
if self.has_option([u"-e", u"--examples"]):
return self.print_examples(False)
if self.has_option(u"--examples-all"):
return self.print_examples(True)
if self.has_option([u"--list-parameters"]):
return self.print_parameters()
parameter = self.has_option_with_value(u"--list-values")
if parameter is not None:
return self.print_values(parameter)
elif self.has_option(u"--list-values"):
return self.print_values(u"?")
# NOTE list() is needed for Python3, where keys() is not a list!
demo = self.has_option(list(self.DEMOS.keys()))
demo_parameters = u""
download_from_youtube = self.has_option([u"-y", u"--youtube"])
largest_audio = self.has_option(u"--largest-audio")
keep_audio = self.has_option(u"--keep-audio")
output_html = self.has_option(u"--output-html")
validate = not self.has_option(u"--skip-validator")
print_faster_rate = self.has_option(u"--faster-rate")
print_rates = self.has_option(u"--rates")
print_zero = self.has_option(u"--zero")
if demo:
validate = False
for key in self.DEMOS:
if self.has_option(key):
demo_parameters = self.DEMOS[key]
audio_file_path = demo_parameters[u"audio"]
text_file_path = demo_parameters[u"text"]
config_string = demo_parameters[u"config"]
sync_map_file_path = demo_parameters[u"syncmap"]
# TODO allow injecting rconf options directly from DEMOS options field
if key == u"--example-cewsubprocess":
self.rconf[
RuntimeConfiguration.CEW_SUBPROCESS_ENABLED] = True
elif key == u"--example-ctw-espeak":
self.rconf[RuntimeConfiguration.TTS] = "custom"
self.rconf[
RuntimeConfiguration.TTS_PATH] = self.CTW_ESPEAK
elif key == u"--example-ctw-speect":
self.rconf[RuntimeConfiguration.TTS] = "custom"
self.rconf[
RuntimeConfiguration.TTS_PATH] = self.CTW_SPEECT
elif key == u"--example-festival":
self.rconf[RuntimeConfiguration.TTS] = "festival"
elif key == u"--example-mws":
self.rconf[
RuntimeConfiguration.MFCC_WINDOW_LENGTH] = "1.500"
self.rconf[
RuntimeConfiguration.MFCC_WINDOW_SHIFT] = "0.500"
elif key == u"--example-multilevel-tts":
self.rconf[RuntimeConfiguration.TTS_L1] = "festival"
self.rconf[RuntimeConfiguration.TTS_L2] = "festival"
self.rconf[RuntimeConfiguration.TTS_L3] = "espeak"
elif key == u"--example-words-festival-cache":
self.rconf[RuntimeConfiguration.TTS] = "festival"
self.rconf[RuntimeConfiguration.TTS_CACHE] = True
elif key == u"--example-faster-rate":
print_faster_rate = True
elif key == u"--example-no-zero":
print_zero = True
elif key == u"--example-py":
self.rconf[RuntimeConfiguration.C_EXTENSIONS] = False
elif key == u"--example-rates":
print_rates = True
elif key == u"--example-youtube":
download_from_youtube = True
break
else:
if len(self.actual_arguments) < 4:
return self.print_help()
audio_file_path = self.actual_arguments[0]
text_file_path = self.actual_arguments[1]
config_string = self.actual_arguments[2]
sync_map_file_path = self.actual_arguments[3]
html_file_path = None
if output_html:
keep_audio = True
html_file_path = sync_map_file_path + u".html"
if download_from_youtube:
youtube_url = audio_file_path
if (not download_from_youtube) and (
not self.check_input_file(audio_file_path)):
return self.ERROR_EXIT_CODE
if not self.check_input_file(text_file_path):
return self.ERROR_EXIT_CODE
if not self.check_output_file(sync_map_file_path):
return self.ERROR_EXIT_CODE
if (html_file_path
is not None) and (not self.check_output_file(html_file_path)):
return self.ERROR_EXIT_CODE
self.check_c_extensions()
if demo:
msg = []
msg.append(u"Running example task with arguments:")
if download_from_youtube:
msg.append(u" YouTube URL: %s" % youtube_url)
else:
msg.append(u" Audio file: %s" % audio_file_path)
msg.append(u" Text file: %s" % text_file_path)
msg.append(u" Config string: %s" % config_string)
msg.append(u" Sync map file: %s" % sync_map_file_path)
if len(demo_parameters[u"options"]) > 0:
msg.append(u" Options: %s" %
demo_parameters[u"options"])
self.print_info(u"\n".join(msg))
if validate:
self.print_info(
u"Validating config string (specify --skip-validator to bypass)..."
)
validator = Validator(logger=self.logger)
result = validator.check_configuration_string(config_string,
is_job=False,
external_name=True)
if not result.passed:
self.print_error(u"The given config string is not valid:")
self.print_generic(result.pretty_print())
return self.ERROR_EXIT_CODE
self.print_info(u"Validating config string... done")
if download_from_youtube:
try:
self.print_info(u"Downloading audio from '%s' ..." %
youtube_url)
downloader = Downloader(logger=self.logger)
audio_file_path = downloader.audio_from_youtube(
youtube_url,
download=True,
output_file_path=None,
largest_audio=largest_audio)
self.print_info(u"Downloading audio from '%s' ... done" %
youtube_url)
except ImportError:
self.print_no_pafy_error()
return self.ERROR_EXIT_CODE
except Exception as exc:
self.print_error(
u"An unexpected error occurred while downloading audio from YouTube:"
)
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
else:
audio_extension = gf.file_extension(audio_file_path)
if audio_extension.lower() not in AudioFile.FILE_EXTENSIONS:
self.print_warning(
u"Your audio file path has extension '%s', which is uncommon for an audio file."
% audio_extension)
self.print_warning(
u"Attempting at executing your Task anyway.")
self.print_warning(
u"If it fails, you might have swapped the first two arguments."
)
self.print_warning(
u"The audio file path should be the first argument, the text file path the second."
)
try:
self.print_info(u"Creating task...")
task = Task(config_string, logger=self.logger)
task.audio_file_path_absolute = audio_file_path
task.text_file_path_absolute = text_file_path
task.sync_map_file_path_absolute = sync_map_file_path
self.print_info(u"Creating task... done")
except Exception as exc:
self.print_error(
u"An unexpected error occurred while creating the task:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Executing task...")
executor = ExecuteTask(task=task,
rconf=self.rconf,
logger=self.logger)
executor.execute()
self.print_info(u"Executing task... done")
except Exception as exc:
self.print_error(
u"An unexpected error occurred while executing the task:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Creating output sync map file...")
path = task.output_sync_map_file()
self.print_info(u"Creating output sync map file... done")
self.print_success(u"Created file '%s'" % path)
except Exception as exc:
self.print_error(
u"An unexpected error occurred while writing the sync map file:"
)
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
if output_html:
try:
parameters = {}
parameters[gc.PPN_TASK_OS_FILE_FORMAT] = task.configuration[
"o_format"]
parameters[
gc.PPN_TASK_OS_FILE_EAF_AUDIO_REF] = task.configuration[
"o_eaf_audio_ref"]
parameters[
gc.PPN_TASK_OS_FILE_SMIL_AUDIO_REF] = task.configuration[
"o_smil_audio_ref"]
parameters[
gc.PPN_TASK_OS_FILE_SMIL_PAGE_REF] = task.configuration[
"o_smil_page_ref"]
self.print_info(u"Creating output HTML file...")
task.sync_map.output_html_for_tuning(audio_file_path,
html_file_path,
parameters)
self.print_info(u"Creating output HTML file... done")
self.print_success(u"Created file '%s'" % html_file_path)
except Exception as exc:
self.print_error(
u"An unexpected error occurred while writing the HTML file:"
)
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
if download_from_youtube:
if keep_audio:
self.print_info(
u"Option --keep-audio set: keeping downloaded file '%s'" %
audio_file_path)
else:
gf.delete_file(None, audio_file_path)
if print_zero:
zero_duration = [
l for l in task.sync_map.fragments_tree.vleaves_not_empty
if l.begin == l.end
]
if len(zero_duration) > 0:
self.print_warning(u"Fragments with zero duration:")
for fragment in zero_duration:
self.print_generic(u" %s" % fragment)
if print_rates:
self.print_info(u"Fragments with rates:")
for fragment in task.sync_map.fragments_tree.vleaves_not_empty:
self.print_generic(u" %s (rate: %.3f chars/s)" %
(fragment, fragment.rate))
if print_faster_rate:
max_rate = task.configuration["aba_rate_value"]
if max_rate is not None:
faster = [
l for l in task.sync_map.fragments_tree.vleaves_not_empty
if l.rate >= max_rate + Decimal("0.001")
]
if len(faster) > 0:
self.print_warning(
u"Fragments with rate greater than %.3f:" % max_rate)
for fragment in faster:
self.print_generic(u" %s (rate: %.3f chars/s)" %
(fragment, fragment.rate))
return self.NO_ERROR_EXIT_CODE
def print_examples(self, full=False):
"""
Print the examples and exit.
:param bool full: if ``True``, print all examples; otherwise,
print only selected ones
"""
msg = []
i = 1
for key in sorted(self.DEMOS.keys()):
example = self.DEMOS[key]
if full or example["show"]:
msg.append(u"Example %d (%s)" % (i, example[u"description"]))
# NOTE too verbose now that we have dozens of examples
# COMMENTED msg.append(u" $ CONFIG_STRING=\"%s\"" % (example[u"config"]))
# COMMENTED msg.append(u" $ %s %s %s \"$CONFIG_STRING\" %s %s" % (
# COMMENTED self.invoke,
# COMMENTED example[u"audio"],
# COMMENTED example[u"text"],
# COMMENTED example[u"syncmap"],
# COMMENTED example[u"options"]
# COMMENTED ))
# COMMENTED msg.append(u" or")
msg.append(u" $ %s %s" % (self.invoke, key))
msg.append(u"")
i += 1
self.print_generic(u"\n" + u"\n".join(msg) + u"\n")
return self.HELP_EXIT_CODE
def print_parameters(self):
"""
Print the list of parameters and exit.
"""
self.print_info(
u"You can use --list-values=PARAM on parameters marked by (*)")
self.print_info(u"Available parameters:")
self.print_generic(u"\n" + u"\n".join(self.PARAMETERS) + u"\n")
return self.HELP_EXIT_CODE
def print_values(self, parameter):
"""
Print the list of values for the given parameter and exit.
If ``parameter`` is invalid, print the list of
parameter names that have allowed values.
:param parameter: the parameter name
:type parameter: Unicode string
"""
if parameter in self.VALUES:
self.print_info(u"Available values for parameter '%s':" %
parameter)
self.print_generic(u", ".join(self.VALUES[parameter]))
return self.HELP_EXIT_CODE
if parameter not in [u"?", u""]:
self.print_error(u"Invalid parameter name '%s'" % parameter)
self.print_info(u"Parameters for which values can be listed:")
self.print_generic(u", ".join(sorted(self.VALUES.keys())))
return self.HELP_EXIT_CODE
class ReadTextCLI(AbstractCLIProgram):
"""
Read text fragments from file.
"""
TEXT_FILE_MPLAIN = gf.relative_path("res/mplain.txt", __file__)
TEXT_FILE_MUNPARSED = gf.relative_path("res/munparsed2.xhtml", __file__)
TEXT_FILE_PARSED = gf.relative_path("res/parsed.txt", __file__)
TEXT_FILE_PLAIN = gf.relative_path("res/plain.txt", __file__)
TEXT_FILE_SUBTITLES = gf.relative_path("res/subtitles.txt", __file__)
TEXT_FILE_UNPARSED = gf.relative_path("res/unparsed.xhtml", __file__)
NAME = gf.file_name_without_extension(__file__)
HELP = {
"description":
u"Read text fragments from file.",
"synopsis":
[(u"list 'fragment 1|fragment 2|...|fragment N'", True),
(u"[mplain|munparsed|parsed|plain|subtitles|unparsed] TEXT_FILE",
True)],
"options": [
u"--class-regex=REGEX : extract text from elements with class attribute matching REGEX (unparsed)",
u"--id-regex=REGEX : extract text from elements with id attribute matching REGEX (unparsed)",
u"--id-format=FMT : use FMT for generating text id attributes (subtitles, plain)",
u"--l1-id-regex=REGEX : extract text from level 1 elements with id attribute matching REGEX (munparsed)",
u"--l2-id-regex=REGEX : extract text from level 2 elements with id attribute matching REGEX (munparsed)",
u"--l3-id-regex=REGEX : extract text from level 3 elements with id attribute matching REGEX (munparsed)",
u"--sort=ALGORITHM : sort the matched element id attributes using ALGORITHM (lexicographic, numeric, unsorted)",
],
"examples": [
u"list 'From|fairest|creatures|we|desire|increase'",
u"mplain %s" % (TEXT_FILE_MPLAIN),
u"munparsed %s --l1-id-regex=p[0-9]+ --l2-id-regex=s[0-9]+ --l3-id-regex=w[0-9]+"
% (TEXT_FILE_MUNPARSED),
u"parsed %s" % (TEXT_FILE_PARSED),
u"plain %s" % (TEXT_FILE_PLAIN),
u"plain %s --id-format=Word%%06d" % (TEXT_FILE_PLAIN),
u"subtitles %s" % (TEXT_FILE_SUBTITLES),
u"subtitles %s --id-format=Sub%%03d" % (TEXT_FILE_SUBTITLES),
u"unparsed %s --id-regex=f[0-9]*" % (TEXT_FILE_UNPARSED),
u"unparsed %s --class-regex=ra --sort=unsorted" %
(TEXT_FILE_UNPARSED),
u"unparsed %s --id-regex=f[0-9]* --sort=numeric" %
(TEXT_FILE_UNPARSED),
u"unparsed %s --id-regex=f[0-9]* --sort=lexicographic" %
(TEXT_FILE_UNPARSED)
]
}
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 2:
return self.print_help()
text_format = gf.safe_unicode(self.actual_arguments[0])
if text_format == u"list":
text = gf.safe_unicode(self.actual_arguments[1])
elif text_format in TextFileFormat.ALLOWED_VALUES:
text = self.actual_arguments[1]
if not self.check_input_file(text):
return self.ERROR_EXIT_CODE
else:
return self.print_help()
l1_id_regex = self.has_option_with_value(u"--l1-id-regex")
l2_id_regex = self.has_option_with_value(u"--l2-id-regex")
l3_id_regex = self.has_option_with_value(u"--l3-id-regex")
id_regex = self.has_option_with_value(u"--id-regex")
id_format = self.has_option_with_value(u"--id-format")
class_regex = self.has_option_with_value(u"--class-regex")
sort = self.has_option_with_value(u"--sort")
parameters = {
gc.PPN_TASK_IS_TEXT_MUNPARSED_L1_ID_REGEX: l1_id_regex,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L2_ID_REGEX: l2_id_regex,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L3_ID_REGEX: l3_id_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_ID_REGEX: id_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_CLASS_REGEX: class_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_ID_SORT: sort,
gc.PPN_TASK_OS_FILE_ID_REGEX: id_format
}
if (text_format == TextFileFormat.MUNPARSED) and (
(l1_id_regex is None) or (l2_id_regex is None) or
(l3_id_regex is None)):
self.print_error(
u"You must specify --l1-id-regex and --l2-id-regex and --l3-id-regex for munparsed format"
)
return self.ERROR_EXIT_CODE
if (text_format == TextFileFormat.UNPARSED) and (
id_regex is None) and (class_regex is None):
self.print_error(
u"You must specify --id-regex and/or --class-regex for unparsed format"
)
return self.ERROR_EXIT_CODE
if (text_format in [TextFileFormat.PLAIN, TextFileFormat.SUBTITLES
]) and (id_format is not None):
try:
identifier = id_format % 1
except (TypeError, ValueError):
self.print_error(
u"The given string '%s' is not a valid id format" %
id_format)
return self.ERROR_EXIT_CODE
text_file = self.get_text_file(text_format, text, parameters)
if text_file is None:
self.print_error(
u"Unable to build a TextFile from the given parameters")
elif len(text_file) == 0:
self.print_error(u"No text fragments found")
else:
self.print_generic(text_file.__unicode__())
return self.NO_ERROR_EXIT_CODE
return self.ERROR_EXIT_CODE
class RunVADCLI(AbstractCLIProgram):
"""
Extract a list of speech intervals from the given audio file,
using the MFCC energy-based VAD algorithm.
"""
INPUT_FILE = gf.relative_path("res/audio.mp3", __file__)
OUTPUT_BOTH = "output/both.txt"
OUTPUT_NONSPEECH = "output/nonspeech.txt"
OUTPUT_SPEECH = "output/speech.txt"
MODES = [u"both", u"nonspeech", u"speech"]
NAME = gf.file_name_without_extension(__file__)
HELP = {
"description":
u"Extract a list of speech intervals using the MFCC energy-based VAD.",
"synopsis":
[(u"AUDIO_FILE [%s] [OUTPUT_FILE]" % (u"|".join(MODES)), True)],
"examples": [
u"%s both %s" % (INPUT_FILE, OUTPUT_BOTH),
u"%s nonspeech %s" % (INPUT_FILE, OUTPUT_NONSPEECH),
u"%s speech %s" % (INPUT_FILE, OUTPUT_SPEECH)
],
"options": [
u"-i, --index : output intervals as indices instead of seconds",
]
}
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 2:
return self.print_help()
audio_file_path = self.actual_arguments[0]
mode = self.actual_arguments[1]
if mode not in [u"speech", u"nonspeech", u"both"]:
return self.print_help()
output_file_path = None
if len(self.actual_arguments) >= 3:
output_file_path = self.actual_arguments[2]
output_time = not self.has_option([u"-i", u"--index"])
self.check_c_extensions("cmfcc")
if not self.check_input_file(audio_file_path):
return self.ERROR_EXIT_CODE
if (output_file_path is not None) and (
not self.check_output_file(output_file_path)):
return self.ERROR_EXIT_CODE
self.print_info(u"Reading audio...")
try:
audio_file_mfcc = AudioFileMFCC(audio_file_path,
rconf=self.rconf,
logger=self.logger)
except AudioFileConverterError:
self.print_error(u"Unable to call the ffmpeg executable '%s'" %
(self.rconf[RuntimeConfiguration.FFMPEG_PATH]))
self.print_error(u"Make sure the path to ffmpeg is correct")
return self.ERROR_EXIT_CODE
except (AudioFileUnsupportedFormatError, AudioFileNotInitializedError):
self.print_error(u"Cannot read file '%s'" % (audio_file_path))
self.print_error(u"Check that its format is supported by ffmpeg")
return self.ERROR_EXIT_CODE
except Exception as exc:
self.print_error(
u"An unexpected error occurred while reading the audio file:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
self.print_info(u"Reading audio... done")
self.print_info(u"Executing VAD...")
audio_file_mfcc.run_vad()
self.print_info(u"Executing VAD... done")
speech = audio_file_mfcc.intervals(speech=True, time=output_time)
nonspeech = audio_file_mfcc.intervals(speech=False, time=output_time)
if mode == u"speech":
intervals = speech
elif mode == u"nonspeech":
intervals = nonspeech
elif mode == u"both":
speech = [[x[0], x[1], u"speech"] for x in speech]
nonspeech = [[x[0], x[1], u"nonspeech"] for x in nonspeech]
intervals = sorted(speech + nonspeech)
intervals = [tuple(interval) for interval in intervals]
self.write_to_file(output_file_path, intervals, output_time)
return self.NO_ERROR_EXIT_CODE
def write_to_file(self, output_file_path, intervals, time):
"""
Write intervals to file.
:param output_file_path: path of the output file to be written;
if ``None``, print to stdout
:type output_file_path: string (path)
:param intervals: a list of tuples, each representing an interval
:type intervals: list of tuples
"""
msg = []
if len(intervals) > 0:
if len(intervals[0]) == 2:
template = u"%.3f\t%.3f" if time else u"%d\t%d"
else:
template = u"%.3f\t%.3f\t%s" if time else u"%d\t%d\t%s"
msg = [template % (interval) for interval in intervals]
if output_file_path is None:
self.print_info(u"Intervals detected:")
for line in msg:
self.print_generic(line)
else:
with io.open(output_file_path, "w",
encoding="utf-8") as output_file:
output_file.write(u"\n".join(msg))
self.print_success(u"Created file '%s'" % output_file_path)
class RunSDCLI(AbstractCLIProgram):
"""
Detect the audio head and/or tail of the given audio file.
"""
AUDIO_FILE = gf.relative_path("res/audio.mp3", __file__)
PARAMETERS_HEAD = "--min-head=0.0 --max-head=5.0"
PARAMETERS_TAIL = "--min-tail=1.0 --max-tail=5.0"
TEXT_FILE = gf.relative_path("res/parsed.txt", __file__)
NAME = gf.file_name_without_extension(__file__)
HELP = {
"description":
u"Detect the audio head and/or tail of the given audio file.",
"synopsis":
[(u"list 'fragment 1|fragment 2|...|fragment N' LANGUAGE AUDIO_FILE",
True),
(u"[mplain|munparsed|parsed|plain|subtitles|unparsed] TEXT_FILE LANGUAGE AUDIO_FILE",
True)],
"examples": [
u"parsed %s eng %s" % (TEXT_FILE, AUDIO_FILE),
u"parsed %s eng %s %s" % (TEXT_FILE, AUDIO_FILE, PARAMETERS_HEAD),
u"parsed %s eng %s %s" % (TEXT_FILE, AUDIO_FILE, PARAMETERS_TAIL),
u"parsed %s eng %s %s %s" %
(TEXT_FILE, AUDIO_FILE, PARAMETERS_HEAD, PARAMETERS_TAIL),
],
"options": [
u"--class-regex=REGEX : extract text from elements with class attribute matching REGEX (unparsed)",
u"--id-regex=REGEX : extract text from elements with id attribute matching REGEX (unparsed)",
u"--l1-id-regex=REGEX : extract text from level 1 elements with id attribute matching REGEX (munparsed)",
u"--l2-id-regex=REGEX : extract text from level 2 elements with id attribute matching REGEX (munparsed)",
u"--l3-id-regex=REGEX : extract text from level 3 elements with id attribute matching REGEX (munparsed)",
u"--max-head=DUR : audio head has at most DUR seconds",
u"--max-tail=DUR : audio tail has at most DUR seconds",
u"--min-head=DUR : audio head has at least DUR seconds",
u"--min-tail=DUR : audio tail has at least DUR seconds",
u"--sort=ALGORITHM : sort the matched element id attributes using ALGORITHM (lexicographic, numeric, unsorted)"
]
}
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 4:
return self.print_help()
text_format = gf.safe_unicode(self.actual_arguments[0])
if text_format == u"list":
text = gf.safe_unicode(self.actual_arguments[1])
elif text_format in TextFileFormat.ALLOWED_VALUES:
text = self.actual_arguments[1]
if not self.check_input_file(text):
return self.ERROR_EXIT_CODE
else:
return self.print_help()
l1_id_regex = self.has_option_with_value(u"--l1-id-regex")
l2_id_regex = self.has_option_with_value(u"--l2-id-regex")
l3_id_regex = self.has_option_with_value(u"--l3-id-regex")
id_regex = self.has_option_with_value(u"--id-regex")
class_regex = self.has_option_with_value(u"--class-regex")
sort = self.has_option_with_value(u"--sort")
parameters = {
gc.PPN_TASK_IS_TEXT_MUNPARSED_L1_ID_REGEX: l1_id_regex,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L2_ID_REGEX: l2_id_regex,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L3_ID_REGEX: l3_id_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_CLASS_REGEX: class_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_ID_REGEX: id_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_ID_SORT: sort,
}
if (text_format == TextFileFormat.MUNPARSED) and (
(l1_id_regex is None) or (l2_id_regex is None) or
(l3_id_regex is None)):
self.print_error(
u"You must specify --l1-id-regex and --l2-id-regex and --l3-id-regex for munparsed format"
)
return self.ERROR_EXIT_CODE
if (text_format == TextFileFormat.UNPARSED) and (
id_regex is None) and (class_regex is None):
self.print_error(
u"You must specify --id-regex and/or --class-regex for unparsed format"
)
return self.ERROR_EXIT_CODE
language = gf.safe_unicode(self.actual_arguments[2])
audio_file_path = self.actual_arguments[3]
if not self.check_input_file(audio_file_path):
return self.ERROR_EXIT_CODE
text_file = self.get_text_file(text_format, text, parameters)
if text_file is None:
self.print_error(
u"Unable to build a TextFile from the given parameters")
return self.ERROR_EXIT_CODE
elif len(text_file) == 0:
self.print_error(u"No text fragments found")
return self.ERROR_EXIT_CODE
text_file.set_language(language)
self.print_info(u"Read input text with %d fragments" %
(len(text_file)))
self.print_info(u"Reading audio...")
try:
audio_file_mfcc = AudioFileMFCC(audio_file_path,
rconf=self.rconf,
logger=self.logger)
except AudioFileConverterError:
self.print_error(u"Unable to call the ffmpeg executable '%s'" %
(self.rconf[RuntimeConfiguration.FFMPEG_PATH]))
self.print_error(u"Make sure the path to ffmpeg is correct")
return self.ERROR_EXIT_CODE
except (AudioFileUnsupportedFormatError, AudioFileNotInitializedError):
self.print_error(u"Cannot read file '%s'" % (audio_file_path))
self.print_error(u"Check that its format is supported by ffmpeg")
return self.ERROR_EXIT_CODE
except Exception as exc:
self.print_error(
u"An unexpected error occurred while reading the audio file:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
self.print_info(u"Reading audio... done")
self.print_info(u"Running VAD...")
audio_file_mfcc.run_vad()
self.print_info(u"Running VAD... done")
min_head = gf.safe_float(self.has_option_with_value(u"--min-head"),
None)
max_head = gf.safe_float(self.has_option_with_value(u"--max-head"),
None)
min_tail = gf.safe_float(self.has_option_with_value(u"--min-tail"),
None)
max_tail = gf.safe_float(self.has_option_with_value(u"--max-tail"),
None)
self.print_info(u"Detecting audio interval...")
start_detector = SD(audio_file_mfcc,
text_file,
rconf=self.rconf,
logger=self.logger)
start, end = start_detector.detect_interval(min_head, max_head,
min_tail, max_tail)
self.print_info(u"Detecting audio interval... done")
self.print_result(audio_file_mfcc.audio_length, start, end)
return self.NO_ERROR_EXIT_CODE
def print_result(self, audio_len, start, end):
"""
Print result of SD.
:param audio_len: the length of the entire audio file, in seconds
:type audio_len: float
:param start: the start position of the spoken text
:type start: float
:param end: the end position of the spoken text
:type end: float
"""
msg = []
zero = 0
head_len = start
text_len = end - start
tail_len = audio_len - end
msg.append(u"")
msg.append(u"Head: %.3f %.3f (%.3f)" % (zero, start, head_len))
msg.append(u"Text: %.3f %.3f (%.3f)" % (start, end, text_len))
msg.append(u"Tail: %.3f %.3f (%.3f)" % (end, audio_len, tail_len))
msg.append(u"")
zero_h = gf.time_to_hhmmssmmm(0)
start_h = gf.time_to_hhmmssmmm(start)
end_h = gf.time_to_hhmmssmmm(end)
audio_len_h = gf.time_to_hhmmssmmm(audio_len)
head_len_h = gf.time_to_hhmmssmmm(head_len)
text_len_h = gf.time_to_hhmmssmmm(text_len)
tail_len_h = gf.time_to_hhmmssmmm(tail_len)
msg.append("Head: %s %s (%s)" % (zero_h, start_h, head_len_h))
msg.append("Text: %s %s (%s)" % (start_h, end_h, text_len_h))
msg.append("Tail: %s %s (%s)" % (end_h, audio_len_h, tail_len_h))
msg.append(u"")
self.print_info(u"\n".join(msg))
