def parse(self, input_text, syncmap):
from lxml import etree
# namespaces
xsi = "http://www.w3.org/2001/XMLSchema-instance"
ns_map = {"xsi": xsi}
# get root
root = etree.fromstring(gf.safe_bytes(input_text))
# get time slots
time_slots = dict()
for ts in root.iter("TIME_SLOT"):
time_slots[ts.get("TIME_SLOT_ID")] = gf.time_from_ssmmm(ts.get("TIME_VALUE")) / 1000
# parse annotations
for alignable in root.iter("ALIGNABLE_ANNOTATION"):
identifier = gf.safe_unicode(alignable.get("ANNOTATION_ID"))
begin = time_slots[alignable.get("TIME_SLOT_REF1")]
end = time_slots[alignable.get("TIME_SLOT_REF2")]
lines = []
for value in alignable.iter("ANNOTATION_VALUE"):
lines.append(gf.safe_unicode(value.text))
self._add_fragment(
syncmap=syncmap,
identifier=identifier,
lines=lines,
begin=begin,
end=end
)
def parse(self, input_text, syncmap):
from lxml import etree
# namespaces
xsi = "http://www.w3.org/2001/XMLSchema-instance"
ns_map = {"xsi": xsi}
# get root
root = etree.fromstring(gf.safe_bytes(input_text))
# get time slots
time_slots = dict()
for ts in root.iter("TIME_SLOT"):
time_slots[ts.get("TIME_SLOT_ID")] = gf.time_from_ssmmm(
ts.get("TIME_VALUE")) / 1000
# parse annotations
for alignable in root.iter("ALIGNABLE_ANNOTATION"):
identifier = gf.safe_unicode(alignable.get("ANNOTATION_ID"))
begin = time_slots[alignable.get("TIME_SLOT_REF1")]
end = time_slots[alignable.get("TIME_SLOT_REF2")]
lines = []
for value in alignable.iter("ANNOTATION_VALUE"):
lines.append(gf.safe_unicode(value.text))
self._add_fragment(syncmap=syncmap,
identifier=identifier,
lines=lines,
begin=begin,
end=end)
def _read_unparsed(self, lines):
"""
Read text fragments from an unparsed format text file.
:param list lines: the lines of the unparsed text file
"""
def filter_attributes():
""" Return a dict with the bs4 filter parameters """
attributes = {}
for attribute_name, filter_name in [
("class", gc.PPN_JOB_IS_TEXT_UNPARSED_CLASS_REGEX),
("id", gc.PPN_JOB_IS_TEXT_UNPARSED_ID_REGEX)
]:
if filter_name in self.parameters:
regex_string = self.parameters[filter_name]
if regex_string is not None:
self.log([u"Regex for %s: '%s'", attribute_name, regex_string])
regex = re.compile(r".*\b" + regex_string + r"\b.*")
attributes[attribute_name] = regex
return attributes
#
# TODO better and/or parametric parsing,
# for example, removing tags but keeping text, etc.
#
self.log(u"Parsing fragments from unparsed text format")
# transform text in a soup object
self.log(u"Creating soup")
soup = BeautifulSoup("\n".join(lines), "lxml")
# extract according to class_regex and id_regex
text_from_id = {}
ids = []
filter_attributes = filter_attributes()
self.log([u"Finding elements matching attributes '%s'", filter_attributes])
nodes = soup.findAll(attrs=filter_attributes)
for node in nodes:
try:
f_id = gf.safe_unicode(node["id"])
f_text = gf.safe_unicode(node.text)
text_from_id[f_id] = f_text
ids.append(f_id)
except KeyError:
self.log_warn(u"KeyError while parsing a node")
# sort by ID as requested
id_sort = gf.safe_get(
dictionary=self.parameters,
key=gc.PPN_JOB_IS_TEXT_UNPARSED_ID_SORT,
default_value=IDSortingAlgorithm.UNSORTED,
can_return_none=False
)
self.log([u"Sorting text fragments using '%s'", id_sort])
sorted_ids = IDSortingAlgorithm(id_sort).sort(ids)
# append to fragments
self.log(u"Appending fragments")
self._create_text_fragments([(key, [text_from_id[key]]) for key in sorted_ids])
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 2:
return self.print_help()
text_format = gf.safe_unicode(self.actual_arguments[0])
if text_format == u"list":
text = gf.safe_unicode(self.actual_arguments[1])
elif text_format in TextFileFormat.ALLOWED_VALUES:
text = self.actual_arguments[1]
if not self.check_input_file(text):
return self.ERROR_EXIT_CODE
else:
return self.print_help()
l1_id_regex = self.has_option_with_value(u"--l1-id-regex")
l2_id_regex = self.has_option_with_value(u"--l2-id-regex")
l3_id_regex = self.has_option_with_value(u"--l3-id-regex")
id_regex = self.has_option_with_value(u"--id-regex")
id_format = self.has_option_with_value(u"--id-format")
class_regex = self.has_option_with_value(u"--class-regex")
sort = self.has_option_with_value(u"--sort")
parameters = {
gc.PPN_TASK_IS_TEXT_MUNPARSED_L1_ID_REGEX : l1_id_regex,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L2_ID_REGEX : l2_id_regex,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L3_ID_REGEX : l3_id_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_ID_REGEX : id_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_CLASS_REGEX : class_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_ID_SORT : sort,
gc.PPN_TASK_OS_FILE_ID_REGEX : id_format
}
if (text_format == TextFileFormat.MUNPARSED) and ((l1_id_regex is None) or (l2_id_regex is None) or (l3_id_regex is None)):
self.print_error(u"You must specify --l1-id-regex and --l2-id-regex and --l3-id-regex for munparsed format")
return self.ERROR_EXIT_CODE
if (text_format == TextFileFormat.UNPARSED) and (id_regex is None) and (class_regex is None):
self.print_error(u"You must specify --id-regex and/or --class-regex for unparsed format")
return self.ERROR_EXIT_CODE
if (text_format in [TextFileFormat.PLAIN, TextFileFormat.SUBTITLES]) and (id_format is not None):
try:
identifier = id_format % 1
except (TypeError, ValueError):
self.print_error(u"The given string '%s' is not a valid id format" % id_format)
return self.ERROR_EXIT_CODE
text_file = self.get_text_file(text_format, text, parameters)
if text_file is None:
self.print_error(u"Unable to build a TextFile from the given parameters")
elif len(text_file) == 0:
self.print_error(u"No text fragments found")
else:
self.print_generic(text_file.__unicode__())
return self.NO_ERROR_EXIT_CODE
return self.ERROR_EXIT_CODE
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 2:
return self.print_help()
text_format = gf.safe_unicode(self.actual_arguments[0])
if text_format == u"list":
text = gf.safe_unicode(self.actual_arguments[1])
elif text_format in TextFileFormat.ALLOWED_VALUES:
text = self.actual_arguments[1]
if not self.check_input_file(text):
return self.ERROR_EXIT_CODE
else:
return self.print_help()
l1_id_regex = self.has_option_with_value(u"--l1-id-regex")
l2_id_regex = self.has_option_with_value(u"--l2-id-regex")
l3_id_regex = self.has_option_with_value(u"--l3-id-regex")
id_regex = self.has_option_with_value(u"--id-regex")
id_format = self.has_option_with_value(u"--id-format")
class_regex = self.has_option_with_value(u"--class-regex")
sort = self.has_option_with_value(u"--sort")
parameters = {
gc.PPN_TASK_IS_TEXT_MUNPARSED_L1_ID_REGEX: l1_id_regex,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L2_ID_REGEX: l2_id_regex,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L3_ID_REGEX: l3_id_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_ID_REGEX: id_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_CLASS_REGEX: class_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_ID_SORT: sort,
gc.PPN_TASK_OS_FILE_ID_REGEX: id_format
}
if (text_format == TextFileFormat.MUNPARSED) and ((l1_id_regex is None) or (l2_id_regex is None) or (l3_id_regex is None)):
self.print_error(u"You must specify --l1-id-regex and --l2-id-regex and --l3-id-regex for munparsed format")
return self.ERROR_EXIT_CODE
if (text_format == TextFileFormat.UNPARSED) and (id_regex is None) and (class_regex is None):
self.print_error(u"You must specify --id-regex and/or --class-regex for unparsed format")
return self.ERROR_EXIT_CODE
if (text_format in [TextFileFormat.PLAIN, TextFileFormat.SUBTITLES]) and (id_format is not None):
try:
identifier = id_format % 1
except (TypeError, ValueError):
self.print_error(u"The given string '%s' is not a valid id format" % id_format)
return self.ERROR_EXIT_CODE
text_file = self.get_text_file(text_format, text, parameters)
if text_file is None:
self.print_error(u"Unable to build a TextFile from the given parameters")
elif len(text_file) == 0:
self.print_error(u"No text fragments found")
else:
self.print_generic(text_file.__unicode__())
return self.NO_ERROR_EXIT_CODE
return self.ERROR_EXIT_CODE
def main():
"""
Run ``aeneas.cew``, reading input text from file and writing audio and interval data to file.
"""
# make sure we have enough parameters
if len(sys.argv) < 6:
print("You must pass five arguments: QUIT_AFTER BACKWARDS TEXT_FILE_PATH AUDIO_FILE_PATH DATA_FILE_PATH")
return 1
# read parameters
c_quit_after = float(sys.argv[1]) # NOTE: cew needs float, not TimeValue
c_backwards = int(sys.argv[2])
text_file_path = sys.argv[3]
audio_file_path = sys.argv[4]
data_file_path = sys.argv[5]
# read (voice_code, text) from file
s_text = []
with io.open(text_file_path, "r", encoding="utf-8") as text:
for line in text.readlines():
# NOTE: not using strip() to avoid removing trailing blank characters
line = line.replace(u"\n", u"").replace(u"\r", u"")
idx = line.find(" ")
if idx > 0:
f_voice_code = line[:idx]
f_text = line[idx+1:]
#print("%s => '%s' and '%s'" % (line, f_voice_code, f_text))
s_text.append((f_voice_code, f_text))
# convert to bytes/unicode as required by subprocess
c_text = []
if gf.PY2:
for f_voice_code, f_text in s_text:
c_text.append((gf.safe_bytes(f_voice_code), gf.safe_bytes(f_text)))
else:
for f_voice_code, f_text in s_text:
c_text.append((gf.safe_unicode(f_voice_code), gf.safe_unicode(f_text)))
try:
import aeneas.cew.cew
sr, sf, intervals = aeneas.cew.cew.synthesize_multiple(
audio_file_path,
c_quit_after,
c_backwards,
c_text
)
with io.open(data_file_path, "w", encoding="utf-8") as data:
data.write(u"%d\n" % (sr))
data.write(u"%d\n" % (sf))
data.write(u"\n".join([u"%.3f %.3f" % (i[0], i[1]) for i in intervals]))
except Exception as exc:
print(u"Unexpected error: %s" % str(exc))
def main():
"""
Run ``aeneas.cew``, reading input text from file and writing audio and interval data to file.
"""
# make sure we have enough parameters
if len(sys.argv) < 6:
print(
"You must pass five arguments: QUIT_AFTER BACKWARDS TEXT_FILE_PATH AUDIO_FILE_PATH DATA_FILE_PATH"
)
return 1
# read parameters
c_quit_after = float(sys.argv[1]) # NOTE: cew needs float, not TimeValue
c_backwards = int(sys.argv[2])
text_file_path = sys.argv[3]
audio_file_path = sys.argv[4]
data_file_path = sys.argv[5]
# read (voice_code, text) from file
s_text = []
with io.open(text_file_path, "r", encoding="utf-8") as text:
for line in text.readlines():
# NOTE: not using strip() to avoid removing trailing blank characters
line = line.replace(u"\n", u"").replace(u"\r", u"")
idx = line.find(" ")
if idx > 0:
f_voice_code = line[:idx]
f_text = line[idx + 1:]
#print("%s => '%s' and '%s'" % (line, f_voice_code, f_text))
s_text.append((f_voice_code, f_text))
# convert to bytes/unicode as required by subprocess
c_text = []
if gf.PY2:
for f_voice_code, f_text in s_text:
c_text.append((gf.safe_bytes(f_voice_code), gf.safe_bytes(f_text)))
else:
for f_voice_code, f_text in s_text:
c_text.append(
(gf.safe_unicode(f_voice_code), gf.safe_unicode(f_text)))
try:
import aeneas.cew.cew
sr, sf, intervals = aeneas.cew.cew.synthesize_multiple(
audio_file_path, c_quit_after, c_backwards, c_text)
with io.open(data_file_path, "w", encoding="utf-8") as data:
data.write(u"%d\n" % (sr))
data.write(u"%d\n" % (sf))
data.write(u"\n".join(
[u"%.3f %.3f" % (i[0], i[1]) for i in intervals]))
except Exception as exc:
print(u"Unexpected error: %s" % str(exc))
def test_safe_unicode(self):
tests = [
("", u""),
("foo", u"foo"),
("foà", u"foà"),
(u"", u""),
(u"foo", u"foo"),
(u"foà", u"foà"),
]
self.assertIsNone(gf.safe_unicode(None))
for test in tests:
self.assertEqual(gf.safe_unicode(test[0]), test[1])
def test_safe_unicode(self):
tests = [
("", u""),
("foo", u"foo"),
("foà", u"foà"),
(u"", u""),
(u"foo", u"foo"),
(u"foà", u"foà"),
]
self.assertIsNone(gf.safe_unicode(None))
for test in tests:
self.assertEqual(gf.safe_unicode(test[0]), test[1])
def _get_lines_from_node_text(cls, node):
"""
Given an ``lxml`` node, get lines from ``node.text``,
where the line separator is ``<br xmlns=... />``.
"""
# TODO more robust parsing
from lxml import etree
parts = ([node.text] + list(chain(*([etree.tostring(c, with_tail=False), c.tail] for c in node.getchildren()))) + [node.tail])
parts = [gf.safe_unicode(p) for p in parts]
parts = [p.strip() for p in parts if not p.startswith(u"<br ")]
parts = [p for p in parts if len(p) > 0]
uparts = []
for part in parts:
uparts.append(gf.safe_unicode(part))
return uparts
def parse(self, input_text, syncmap):
"""
Read from SMIL file.
Limitations:
1. parses only ``<par>`` elements, in order
2. timings must have ``hh:mm:ss.mmm`` or ``ss.mmm`` format (autodetected)
3. both ``clipBegin`` and ``clipEnd`` attributes of ``<audio>`` must be populated
"""
from lxml import etree
smil_ns = "{http://www.w3.org/ns/SMIL}"
root = etree.fromstring(gf.safe_bytes(input_text))
for par in root.iter(smil_ns + "par"):
for child in par:
if child.tag == (smil_ns + "text"):
identifier = gf.safe_unicode(gf.split_url(child.get("src"))[1])
elif child.tag == (smil_ns + "audio"):
begin_text = child.get("clipBegin")
if ":" in begin_text:
begin = gf.time_from_hhmmssmmm(begin_text)
else:
begin = gf.time_from_ssmmm(begin_text)
end_text = child.get("clipEnd")
if ":" in end_text:
end = gf.time_from_hhmmssmmm(end_text)
else:
end = gf.time_from_ssmmm(end_text)
# TODO read text from additional text_file?
self._add_fragment(
syncmap=syncmap,
identifier=identifier,
lines=[u""],
begin=begin,
end=end
)
def check_raw_string(self, string, is_bstring=True):
"""
Check whether the given string
is properly UTF-8 encoded (if ``is_bytes`` is ``True``),
it is not empty, and
it does not contain reserved characters.
:param string string: the byte string or Unicode string to be checked
:param bool is_bstring: if True, string is a byte string
:rtype: :class:`~aeneas.validator.ValidatorResult`
"""
self.log(u"Checking the given byte string")
self.result = ValidatorResult()
if self._are_safety_checks_disabled(u"check_raw_string"):
return self.result
if is_bstring:
self._check_utf8_encoding(string)
if not self.result.passed:
return self.result
string = gf.safe_unicode(string)
self._check_not_empty(string)
if not self.result.passed:
return self.result
self._check_reserved_characters(string)
return self.result
def json_string(self):
"""
Return a JSON representation of the sync map.
:rtype: string
.. versionadded:: 1.3.1
"""
def visit_children(node):
""" Recursively visit the fragments_tree """
output_fragments = []
for child in node.children_not_empty:
fragment = child.value
text = fragment.text_fragment
output_fragments.append({
"id": text.identifier,
"language": text.language,
"lines": text.lines,
"begin": gf.time_to_ssmmm(fragment.begin),
"end": gf.time_to_ssmmm(fragment.end),
"children": visit_children(child)
})
return output_fragments
output_fragments = visit_children(self.fragments_tree)
return gf.safe_unicode(
json.dumps({"fragments": output_fragments}, indent=1, sort_keys=True)
)
def check_config_txt(self, contents, is_config_string=False):
"""
Check whether the given TXT config file contents
(if ``is_config_string`` is ``False``) or
TXT config string (if ``is_config_string`` is ``True``)
is well-formed and it has all the required parameters.
:param string contents: the TXT config file contents or TXT config string
:param bool is_config_string: if ``True``, contents is a config string
:rtype: :class:`~aeneas.validator.ValidatorResult`
"""
self.log(u"Checking contents TXT config file")
self.result = ValidatorResult()
if self._are_safety_checks_disabled(u"check_config_txt"):
return self.result
is_bstring = gf.is_bytes(contents)
if is_bstring:
self.log(u"Checking that contents is well formed")
self.check_raw_string(contents, is_bstring=True)
if not self.result.passed:
return self.result
contents = gf.safe_unicode(contents)
if not is_config_string:
self.log(u"Converting file contents to config string")
contents = gf.config_txt_to_string(contents)
self.log(u"Checking required parameters")
required_parameters = self.TXT_REQUIRED_PARAMETERS
parameters = gf.config_string_to_dict(contents, self.result)
self._check_required_parameters(required_parameters, parameters)
self.log([u"Checking contents: returning %s", self.result.passed])
return self.result
def json_string(self):
"""
Return a JSON representation of the sync map.
:rtype: string
.. versionadded:: 1.3.1
"""
def visit_children(node):
""" Recursively visit the fragments_tree """
output_fragments = []
for child in node.children_not_empty:
fragment = child.value
text = fragment.text_fragment
output_fragments.append({
"id":
text.identifier,
"language":
text.language,
"lines":
text.lines,
"begin":
gf.time_to_ssmmm(fragment.begin),
"end":
gf.time_to_ssmmm(fragment.end),
"children":
visit_children(child)
})
return output_fragments
output_fragments = visit_children(self.fragments_tree)
return gf.safe_unicode(
json.dumps({"fragments": output_fragments},
indent=1,
sort_keys=True))
def _get_lines_from_node_text(cls, node):
"""
Given an ``lxml`` node, get lines from ``node.text``,
where the line separator is ``<br xmlns=... />``.
"""
# TODO more robust parsing
from lxml import etree
parts = ([node.text] + list(
chain(*([etree.tostring(c, with_tail=False), c.tail]
for c in node.getchildren()))) + [node.tail])
parts = [gf.safe_unicode(p) for p in parts]
parts = [p.strip() for p in parts if not p.startswith(u"<br ")]
parts = [p for p in parts if len(p) > 0]
uparts = []
for part in parts:
uparts.append(gf.safe_unicode(part))
return uparts
def parse(self, input_text, syncmap):
from lxml import etree
root = etree.fromstring(gf.safe_bytes(input_text))
for frag in root:
identifier = gf.safe_unicode(frag.get("id"))
begin = gf.time_from_ssmmm(frag.get("begin"))
end = gf.time_from_ssmmm(frag.get("end"))
lines = []
for child in frag:
if child.tag == "line":
lines.append(gf.safe_unicode(child.text))
self._add_fragment(
syncmap=syncmap,
identifier=identifier,
lines=lines,
begin=begin,
end=end
)
def _synthesize_single_c_extension(self, text, voice_code, output_file_path):
"""
Synthesize a single text fragment, using the cew extension.
Return the duration of the synthesized text, in seconds.
:rtype: (bool, (:class:`~aeneas.timevalue.TimeValue`, ))
"""
self.log(u"Synthesizing using C extension...")
end = None
if self.rconf[RuntimeConfiguration.CEW_SUBPROCESS_ENABLED]:
self.log(u"Using cewsubprocess to call aeneas.cew")
try:
self.log(u"Importing aeneas.cewsubprocess...")
from aeneas.cewsubprocess import CEWSubprocess
self.log(u"Importing aeneas.cewsubprocess... done")
self.log(u"Calling aeneas.cewsubprocess...")
cewsub = CEWSubprocess(rconf=self.rconf, logger=self.logger)
end = cewsub.synthesize_single(output_file_path, voice_code, text)
self.log(u"Calling aeneas.cewsubprocess... done")
except Exception as exc:
self.log_exc(u"An unexpected error occurred while running cewsubprocess", exc, False, None)
# NOTE not critical, try calling aeneas.cew directly
#return (False, None)
if end is None:
self.log(u"Preparing c_text...")
if gf.PY2:
# Python 2 => pass byte strings
c_text = gf.safe_bytes(text)
else:
# Python 3 => pass Unicode strings
c_text = gf.safe_unicode(text)
self.log(u"Preparing c_text... done")
self.log(u"Calling aeneas.cew directly")
try:
self.log(u"Importing aeneas.cew...")
import aeneas.cew.cew
self.log(u"Importing aeneas.cew... done")
self.log(u"Calling aeneas.cew...")
sr, begin, end = aeneas.cew.cew.synthesize_single(
output_file_path,
voice_code,
c_text
)
end = TimeValue(end)
self.log(u"Calling aeneas.cew... done")
except Exception as exc:
self.log_exc(u"An unexpected error occurred while running cew", exc, False, None)
return (False, None)
self.log(u"Synthesizing using C extension... done")
return (True, (end, ))
def _tree_to_string(cls, root_element, xml_declaration=True, pretty_print=True):
"""
Return an ``lxml`` tree as a Unicode string.
"""
from lxml import etree
return gf.safe_unicode(etree.tostring(
root_element,
encoding="UTF-8",
method="xml",
xml_declaration=xml_declaration,
pretty_print=pretty_print
))
def _tree_to_string(cls,
root_element,
xml_declaration=True,
pretty_print=True):
"""
Return an ``lxml`` tree as a Unicode string.
"""
from lxml import etree
return gf.safe_unicode(
etree.tostring(root_element,
encoding="UTF-8",
method="xml",
xml_declaration=xml_declaration,
pretty_print=pretty_print))
def parse(self, input_text, syncmap):
from lxml import etree
root = etree.fromstring(gf.safe_bytes(input_text))
for frag in root:
for child in frag:
if child.tag == "identifier":
identifier = gf.safe_unicode(child.text)
elif child.tag == "start":
begin = gf.time_from_ssmmm(child.text)
elif child.tag == "end":
end = gf.time_from_ssmmm(child.text)
# TODO read text from additional text_file?
self._add_fragment(syncmap=syncmap,
identifier=identifier,
lines=[u""],
begin=begin,
end=end)
def parse(self, input_text, syncmap):
from lxml import etree
ttml_ns = "{http://www.w3.org/ns/ttml}"
xml_ns = "{http://www.w3.org/XML/1998/namespace}"
root = etree.fromstring(gf.safe_bytes(input_text))
language = root.get(xml_ns + "lang")
for elem in root.iter(ttml_ns + "p"):
identifier = gf.safe_unicode(elem.get(xml_ns + "id"))
begin = gf.time_from_ttml(elem.get("begin"))
end = gf.time_from_ttml(elem.get("end"))
fragment_lines = self._get_lines_from_node_text(elem)
self._add_fragment(syncmap=syncmap,
identifier=identifier,
language=language,
lines=fragment_lines,
begin=begin,
end=end)
def format(self, syncmap):
smil_data = []
smil_ids = []
for fragment in syncmap.fragments:
text = fragment.text_fragment
smil_data.append({
"id": text.identifier,
"begin": gf.time_to_ssmmm(fragment.begin),
"end": gf.time_to_ssmmm(fragment.end)
})
smil_ids.append(text.identifier)
return gf.safe_unicode(
json.dumps(obj={
"smil_ids": smil_ids,
"smil_data": smil_data
},
indent=1,
sort_keys=True))
def parse(self, input_text, syncmap):
from lxml import etree
root = etree.fromstring(gf.safe_bytes(input_text))
for frag in root:
for child in frag:
if child.tag == "identifier":
identifier = gf.safe_unicode(child.text)
elif child.tag == "start":
begin = gf.time_from_ssmmm(child.text)
elif child.tag == "end":
end = gf.time_from_ssmmm(child.text)
# TODO read text from additional text_file?
self._add_fragment(
syncmap=syncmap,
identifier=identifier,
lines=[u""],
begin=begin,
end=end
)
def check_configuration_string(
self,
config_string,
is_job=True,
external_name=False
):
"""
Check whether the given job or task configuration string
is well-formed (if ``is_bstring`` is ``True``)
and it has all the required parameters.
:param string config_string: the byte string or Unicode string to be checked
:param bool is_job: if ``True``, ``config_string`` is a job config string
:param bool external_name: if ``True``, the task name is provided externally,
and it is not required to appear
in the config string
:rtype: :class:`~aeneas.validator.ValidatorResult`
"""
if is_job:
self.log(u"Checking job configuration string")
else:
self.log(u"Checking task configuration string")
self.result = ValidatorResult()
if self._are_safety_checks_disabled(u"check_configuration_string"):
return self.result
if is_job:
required_parameters = self.JOB_REQUIRED_PARAMETERS
elif external_name:
required_parameters = self.TASK_REQUIRED_PARAMETERS_EXTERNAL_NAME
else:
required_parameters = self.TASK_REQUIRED_PARAMETERS
is_bstring = gf.is_bytes(config_string)
if is_bstring:
self.log(u"Checking that config_string is well formed")
self.check_raw_string(config_string, is_bstring=True)
if not self.result.passed:
return self.result
config_string = gf.safe_unicode(config_string)
self.log(u"Checking required parameters")
parameters = gf.config_string_to_dict(config_string, self.result)
self._check_required_parameters(required_parameters, parameters)
self.log([u"Checking config_string: returning %s", self.result.passed])
return self.result
def parse(self, input_text, syncmap):
from lxml import etree
ttml_ns = "{http://www.w3.org/ns/ttml}"
xml_ns = "{http://www.w3.org/XML/1998/namespace}"
root = etree.fromstring(gf.safe_bytes(input_text))
language = root.get(xml_ns + "lang")
for elem in root.iter(ttml_ns + "p"):
identifier = gf.safe_unicode(elem.get(xml_ns + "id"))
begin = gf.time_from_ttml(elem.get("begin"))
end = gf.time_from_ttml(elem.get("end"))
fragment_lines = self._get_lines_from_node_text(elem)
self._add_fragment(
syncmap=syncmap,
identifier=identifier,
language=language,
lines=fragment_lines,
begin=begin,
end=end
)
def format(self, syncmap):
smil_data = []
smil_ids = []
for fragment in syncmap.fragments:
text = fragment.text_fragment
smil_data.append({
"id": text.identifier,
"begin": gf.time_to_ssmmm(fragment.begin),
"end": gf.time_to_ssmmm(fragment.end)
})
smil_ids.append(text.identifier)
return gf.safe_unicode(
json.dumps(
obj={
"smil_ids": smil_ids,
"smil_data": smil_data
},
indent=1,
sort_keys=True
)
)
def format(self, syncmap):
try:
import tgt
except ImportError as exc:
self.log_exc(u"Python module tgt is not installed", exc, True, ImportError)
# from https://github.com/hbuschme/TextGridTools/blob/master/tgt/io.py
textgrid = tgt.TextGrid()
tier = tgt.IntervalTier(name="Token")
for fragment in syncmap.fragments:
begin = float(fragment.begin)
end = float(fragment.end)
text = fragment.text_fragment.text
if text == u"":
text = u"SIL"
interval = tgt.Interval(begin, end, text=text)
tier.add_interval(interval)
textgrid.add_tier(tier)
if self.variant == self.DEFAULT:
msg = tgt.io.export_to_long_textgrid(textgrid)
else:
msg = tgt.io.export_to_short_textgrid(textgrid)
return gf.safe_unicode(msg)
def _read_munparsed(self, lines):
"""
Read text fragments from an munparsed format text file.
:param list lines: the lines of the unparsed text file
"""
def nodes_at_level(root, level):
""" Return a dict with the bs4 filter parameters """
LEVEL_TO_REGEX_MAP = [
None,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L1_ID_REGEX,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L2_ID_REGEX,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L3_ID_REGEX,
]
attribute_name = "id"
regex_string = self.parameters[LEVEL_TO_REGEX_MAP[level]]
indent = u" " * 2 * (level - 1)
self.log([u"%sRegex for %s: '%s'", indent, attribute_name, regex_string])
regex = re.compile(r".*\b" + regex_string + r"\b.*")
return root.findAll(attrs={ attribute_name: regex })
#
# TODO better and/or parametric parsing,
# for example, removing tags but keeping text, etc.
#
self.log(u"Parsing fragments from munparsed text format")
# transform text in a soup object
self.log(u"Creating soup")
soup = BeautifulSoup("\n".join(lines), "lxml")
# extract according to class_regex and id_regex
text_from_id = {}
ids = []
self.log(u"Finding l1 elements")
tree = Tree()
for l1_node in nodes_at_level(soup, 1):
has_word = False
try:
l1_id = gf.safe_unicode(l1_node["id"])
self.log([u"Found l1 node with id: '%s'", l1_id])
l1_text = []
paragraph_node = Tree()
paragraph_text = []
for l2_node in nodes_at_level(l1_node, 2):
l2_id = gf.safe_unicode(l2_node["id"])
self.log([u" Found l2 node with id: '%s'", l2_id])
l2_text = []
sentence_node = Tree()
paragraph_node.add_child(sentence_node)
sentence_text = []
for l3_node in nodes_at_level(l2_node, 3):
l3_id = gf.safe_unicode(l3_node["id"])
l3_text = gf.safe_unicode(l3_node.text)
self.log([u" Found l3 node with id: '%s'", l3_id])
self.log([u" Found l3 node with text: '%s'", l3_text])
word_fragment = TextFragment(
identifier=l3_id,
lines=[l3_text],
filtered_lines=[l3_text]
)
word_node = Tree(value=word_fragment)
sentence_node.add_child(word_node)
sentence_text.append(l3_text)
has_word = True
sentence_text = u" ".join(sentence_text)
paragraph_text.append(sentence_text)
sentence_node.value = TextFragment(
identifier=l2_id,
lines=[sentence_text],
filtered_lines=[sentence_text]
)
self.log([u" Found l2 node with text: '%s'" % sentence_text])
if has_word:
paragraph_text = u" ".join(paragraph_text)
paragraph_node.value = TextFragment(
identifier=l1_id,
lines=[paragraph_text],
filtered_lines=[paragraph_text]
)
tree.add_child(paragraph_node)
self.log([u"Found l1 node with text: '%s'" % paragraph_text])
else:
self.log(u"Found l1 node but it has no words, skipping")
except KeyError:
self.log_warn(u"KeyError while parsing a l1 node")
# append to fragments
self.log(u"Storing tree")
self.fragments_tree = tree
def _read_munparsed(self, lines):
"""
Read text fragments from an munparsed format text file.
:param list lines: the lines of the unparsed text file
"""
from bs4 import BeautifulSoup
def nodes_at_level(root, level):
""" Return a dict with the bs4 filter parameters """
LEVEL_TO_REGEX_MAP = [
None,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L1_ID_REGEX,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L2_ID_REGEX,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L3_ID_REGEX,
]
attribute_name = "id"
regex_string = self.parameters[LEVEL_TO_REGEX_MAP[level]]
indent = u" " * 2 * (level - 1)
self.log([
u"%sRegex for %s: '%s'", indent, attribute_name, regex_string
])
regex = re.compile(r".*\b" + regex_string + r"\b.*")
return root.findAll(attrs={attribute_name: regex})
#
# TODO better and/or parametric parsing,
# for example, removing tags but keeping text, etc.
#
self.log(u"Parsing fragments from munparsed text format")
# transform text in a soup object
self.log(u"Creating soup")
soup = BeautifulSoup("\n".join(lines), "lxml")
# extract according to class_regex and id_regex
text_from_id = {}
ids = []
self.log(u"Finding l1 elements")
tree = Tree()
for l1_node in nodes_at_level(soup, 1):
has_word = False
try:
l1_id = gf.safe_unicode(l1_node["id"])
self.log([u"Found l1 node with id: '%s'", l1_id])
l1_text = []
paragraph_node = Tree()
paragraph_text = []
for l2_node in nodes_at_level(l1_node, 2):
l2_id = gf.safe_unicode(l2_node["id"])
self.log([u" Found l2 node with id: '%s'", l2_id])
l2_text = []
sentence_node = Tree()
paragraph_node.add_child(sentence_node)
sentence_text = []
for l3_node in nodes_at_level(l2_node, 3):
l3_id = gf.safe_unicode(l3_node["id"])
l3_text = gf.safe_unicode(l3_node.text)
self.log([u" Found l3 node with id: '%s'", l3_id])
self.log(
[u" Found l3 node with text: '%s'", l3_text])
word_fragment = TextFragment(identifier=l3_id,
lines=[l3_text],
filtered_lines=[l3_text])
word_node = Tree(value=word_fragment)
sentence_node.add_child(word_node)
sentence_text.append(l3_text)
has_word = True
sentence_text = u" ".join(sentence_text)
paragraph_text.append(sentence_text)
sentence_node.value = TextFragment(
identifier=l2_id,
lines=[sentence_text],
filtered_lines=[sentence_text])
self.log(
[u" Found l2 node with text: '%s'" % sentence_text])
if has_word:
paragraph_text = u" ".join(paragraph_text)
paragraph_node.value = TextFragment(
identifier=l1_id,
lines=[paragraph_text],
filtered_lines=[paragraph_text])
tree.add_child(paragraph_node)
self.log(
[u"Found l1 node with text: '%s'" % paragraph_text])
else:
self.log(u"Found l1 node but it has no words, skipping")
except KeyError:
self.log_warn(u"KeyError while parsing a l1 node")
# append to fragments
self.log(u"Storing tree")
self.fragments_tree = tree
def read_properties(self, audio_file_path):
"""
Read the properties of an audio file
and return them as a dictionary.
Example: ::
d["index"]=0
d["codec_name"]=mp3
d["codec_long_name"]=MP3 (MPEG audio layer 3)
d["profile"]=unknown
d["codec_type"]=audio
d["codec_time_base"]=1/44100
d["codec_tag_string"]=[0][0][0][0]
d["codec_tag"]=0x0000
d["sample_fmt"]=s16p
d["sample_rate"]=44100
d["channels"]=1
d["channel_layout"]=mono
d["bits_per_sample"]=0
d["id"]=N/A
d["r_frame_rate"]=0/0
d["avg_frame_rate"]=0/0
d["time_base"]=1/14112000
d["start_pts"]=0
d["start_time"]=0.000000
d["duration_ts"]=1545083190
d["duration"]=109.487188
d["bit_rate"]=128000
d["max_bit_rate"]=N/A
d["bits_per_raw_sample"]=N/A
d["nb_frames"]=N/A
d["nb_read_frames"]=N/A
d["nb_read_packets"]=N/A
d["DISPOSITION:default"]=0
d["DISPOSITION:dub"]=0
d["DISPOSITION:original"]=0
d["DISPOSITION:comment"]=0
d["DISPOSITION:lyrics"]=0
d["DISPOSITION:karaoke"]=0
d["DISPOSITION:forced"]=0
d["DISPOSITION:hearing_impaired"]=0
d["DISPOSITION:visual_impaired"]=0
d["DISPOSITION:clean_effects"]=0
d["DISPOSITION:attached_pic"]=0
:param string audio_file_path: the path of the audio file to analyze
:rtype: dict
:raises: TypeError: if ``audio_file_path`` is None
:raises: OSError: if the file at ``audio_file_path`` cannot be read
:raises: FFPROBEParsingError: if the call to ``ffprobe`` does not produce any output
:raises: FFPROBEPathError: if the path to the ``ffprobe`` executable cannot be called
:raises: FFPROBEUnsupportedFormatError: if the file has a format not supported by ``ffprobe``
"""
# test if we can read the file at audio_file_path
if audio_file_path is None:
self.log_exc(u"The audio file path is None", None, True, TypeError)
if not gf.file_can_be_read(audio_file_path):
self.log_exc(u"Input file '%s' cannot be read" % (audio_file_path), None, True, OSError)
# call ffprobe
arguments = [self.rconf[RuntimeConfiguration.FFPROBE_PATH]]
arguments.extend(self.FFPROBE_PARAMETERS)
arguments.append(audio_file_path)
self.log([u"Calling with arguments '%s'", arguments])
try:
proc = subprocess.Popen(
arguments,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE
)
(stdoutdata, stderrdata) = proc.communicate()
proc.stdout.close()
proc.stdin.close()
proc.stderr.close()
except OSError as exc:
self.log_exc(u"Unable to call the '%s' ffprobe executable" % (self.rconf[RuntimeConfiguration.FFPROBE_PATH]), exc, True, FFPROBEPathError)
self.log(u"Call completed")
# check there is some output
if (stdoutdata is None) or (len(stderrdata) == 0):
self.log_exc(u"ffprobe produced no output", None, True, FFPROBEParsingError)
# decode stdoutdata and stderrdata to Unicode string
try:
stdoutdata = gf.safe_unicode(stdoutdata)
stderrdata = gf.safe_unicode(stderrdata)
except UnicodeDecodeError as exc:
self.log_exc(u"Unable to decode ffprobe out/err", exc, True, FFPROBEParsingError)
# dictionary for the results
results = {
self.STDOUT_CHANNELS : None,
self.STDOUT_CODEC_NAME : None,
self.STDOUT_DURATION : None,
self.STDOUT_SAMPLE_RATE : None
}
# scan the first audio stream the ffprobe stdout output
# TODO more robust parsing
# TODO deal with multiple audio streams
for line in stdoutdata.splitlines():
if line == self.STDOUT_END_STREAM:
self.log(u"Reached end of the stream")
break
elif len(line.split("=")) == 2:
key, value = line.split("=")
results[key] = value
self.log([u"Found property '%s'='%s'", key, value])
try:
self.log([u"Duration found in stdout: '%s'", results[self.STDOUT_DURATION]])
results[self.STDOUT_DURATION] = TimeValue(results[self.STDOUT_DURATION])
self.log(u"Valid duration")
except:
self.log_warn(u"Invalid duration")
results[self.STDOUT_DURATION] = None
# try scanning ffprobe stderr output
for line in stderrdata.splitlines():
match = self.STDERR_DURATION_REGEX.search(line)
if match is not None:
self.log([u"Found matching line '%s'", line])
results[self.STDOUT_DURATION] = gf.time_from_hhmmssmmm(line)
self.log([u"Extracted duration '%.3f'", results[self.STDOUT_DURATION]])
break
if results[self.STDOUT_DURATION] is None:
self.log_exc(u"No duration found in stdout or stderr. Unsupported audio file format?", None, True, FFPROBEUnsupportedFormatError)
# return dictionary
self.log(u"Returning dict")
return results
def _analyze_txt_config(self, config_string=None):
"""
Analyze the given container and return the corresponding job.
If ``config_string`` is ``None``,
try reading it from the TXT config file inside the container.
:param string config_string: the configuration string
:rtype: :class:`~aeneas.job.Job`
"""
self.log(u"Analyzing container with TXT config string")
if config_string is None:
self.log(u"Analyzing container with TXT config file")
config_entry = self.container.entry_config_txt
self.log([u"Found TXT config entry '%s'", config_entry])
config_dir = os.path.dirname(config_entry)
self.log([u"Directory of TXT config entry: '%s'", config_dir])
self.log([u"Reading TXT config entry: '%s'", config_entry])
config_contents = self.container.read_entry(config_entry)
self.log(u"Converting config contents to config string")
config_contents = gf.safe_unicode(config_contents)
config_string = gf.config_txt_to_string(config_contents)
else:
self.log([
u"Analyzing container with TXT config string '%s'",
config_string
])
config_dir = ""
self.log(u"Creating the Job object")
job = Job(config_string)
self.log(u"Getting entries")
entries = self.container.entries
self.log(u"Converting config string into config dict")
parameters = gf.config_string_to_dict(config_string)
self.log(u"Calculating the path of the tasks root directory")
tasks_root_directory = gf.norm_join(
config_dir, parameters[gc.PPN_JOB_IS_HIERARCHY_PREFIX])
self.log(
[u"Path of the tasks root directory: '%s'", tasks_root_directory])
self.log(u"Calculating the path of the sync map root directory")
sync_map_root_directory = gf.norm_join(
config_dir, parameters[gc.PPN_JOB_OS_HIERARCHY_PREFIX])
job_os_hierarchy_type = parameters[gc.PPN_JOB_OS_HIERARCHY_TYPE]
self.log([
u"Path of the sync map root directory: '%s'",
sync_map_root_directory
])
text_file_relative_path = parameters[
gc.PPN_JOB_IS_TEXT_FILE_RELATIVE_PATH]
self.log(
[u"Relative path for text file: '%s'", text_file_relative_path])
text_file_name_regex = re.compile(
r"" + parameters[gc.PPN_JOB_IS_TEXT_FILE_NAME_REGEX])
self.log([
u"Regex for text file: '%s'",
parameters[gc.PPN_JOB_IS_TEXT_FILE_NAME_REGEX]
])
audio_file_relative_path = parameters[
gc.PPN_JOB_IS_AUDIO_FILE_RELATIVE_PATH]
self.log(
[u"Relative path for audio file: '%s'", audio_file_relative_path])
audio_file_name_regex = re.compile(
r"" + parameters[gc.PPN_JOB_IS_AUDIO_FILE_NAME_REGEX])
self.log([
u"Regex for audio file: '%s'",
parameters[gc.PPN_JOB_IS_AUDIO_FILE_NAME_REGEX]
])
if parameters[gc.PPN_JOB_IS_HIERARCHY_TYPE] == HierarchyType.FLAT:
self.log(u"Looking for text/audio pairs in flat hierarchy")
text_files = self._find_files(entries, tasks_root_directory,
text_file_relative_path,
text_file_name_regex)
self.log([u"Found text files: '%s'", text_files])
audio_files = self._find_files(entries, tasks_root_directory,
audio_file_relative_path,
audio_file_name_regex)
self.log([u"Found audio files: '%s'", audio_files])
self.log(u"Matching files in flat hierarchy...")
matched_tasks = self._match_files_flat_hierarchy(
text_files, audio_files)
self.log(u"Matching files in flat hierarchy... done")
for task_info in matched_tasks:
self.log([u"Creating task: '%s'", str(task_info)])
task = self._create_task(task_info, config_string,
sync_map_root_directory,
job_os_hierarchy_type)
job.add_task(task)
if parameters[gc.PPN_JOB_IS_HIERARCHY_TYPE] == HierarchyType.PAGED:
self.log(u"Looking for text/audio pairs in paged hierarchy")
# find all subdirectories of tasks_root_directory
# that match gc.PPN_JOB_IS_TASK_DIRECTORY_NAME_REGEX
matched_directories = self._match_directories(
entries, tasks_root_directory,
parameters[gc.PPN_JOB_IS_TASK_DIRECTORY_NAME_REGEX])
for matched_directory in matched_directories:
# rebuild the full path
matched_directory_full_path = gf.norm_join(
tasks_root_directory, matched_directory)
self.log([
u"Looking for text/audio pairs in directory '%s'",
matched_directory_full_path
])
# look for text and audio files there
text_files = self._find_files(entries,
matched_directory_full_path,
text_file_relative_path,
text_file_name_regex)
self.log([u"Found text files: '%s'", text_files])
audio_files = self._find_files(entries,
matched_directory_full_path,
audio_file_relative_path,
audio_file_name_regex)
self.log([u"Found audio files: '%s'", audio_files])
# if we have found exactly one text and one audio file,
# create a Task
if (len(text_files) == 1) and (len(audio_files) == 1):
self.log([
u"Exactly one text file and one audio file in '%s'",
matched_directory
])
task_info = [
matched_directory, text_files[0], audio_files[0]
]
self.log([u"Creating task: '%s'", str(task_info)])
task = self._create_task(task_info, config_string,
sync_map_root_directory,
job_os_hierarchy_type)
job.add_task(task)
elif len(text_files) > 1:
self.log([
u"More than one text file in '%s'", matched_directory
])
elif len(audio_files) > 1:
self.log([
u"More than one audio file in '%s'", matched_directory
])
else:
self.log(
[u"No text nor audio file in '%s'", matched_directory])
return job
def _analyze_txt_config(self, config_string=None):
"""
Analyze the given container and return the corresponding job.
If ``config_string`` is ``None``,
try reading it from the TXT config file inside the container.
:param string config_string: the configuration string
:rtype: :class:`~aeneas.job.Job`
"""
self.log(u"Analyzing container with TXT config string")
if config_string is None:
self.log(u"Analyzing container with TXT config file")
config_entry = self.container.entry_config_txt
self.log([u"Found TXT config entry '%s'", config_entry])
config_dir = os.path.dirname(config_entry)
self.log([u"Directory of TXT config entry: '%s'", config_dir])
self.log([u"Reading TXT config entry: '%s'", config_entry])
config_contents = self.container.read_entry(config_entry)
self.log(u"Converting config contents to config string")
config_contents = gf.safe_unicode(config_contents)
config_string = gf.config_txt_to_string(config_contents)
else:
self.log([u"Analyzing container with TXT config string '%s'", config_string])
config_dir = ""
self.log(u"Creating the Job object")
job = Job(config_string)
self.log(u"Getting entries")
entries = self.container.entries
self.log(u"Converting config string into config dict")
parameters = gf.config_string_to_dict(config_string)
self.log(u"Calculating the path of the tasks root directory")
tasks_root_directory = gf.norm_join(
config_dir,
parameters[gc.PPN_JOB_IS_HIERARCHY_PREFIX]
)
self.log([u"Path of the tasks root directory: '%s'", tasks_root_directory])
self.log(u"Calculating the path of the sync map root directory")
sync_map_root_directory = gf.norm_join(
config_dir,
parameters[gc.PPN_JOB_OS_HIERARCHY_PREFIX]
)
job_os_hierarchy_type = parameters[gc.PPN_JOB_OS_HIERARCHY_TYPE]
self.log([u"Path of the sync map root directory: '%s'", sync_map_root_directory])
text_file_relative_path = parameters[gc.PPN_JOB_IS_TEXT_FILE_RELATIVE_PATH]
self.log([u"Relative path for text file: '%s'", text_file_relative_path])
text_file_name_regex = re.compile(r"" + parameters[gc.PPN_JOB_IS_TEXT_FILE_NAME_REGEX])
self.log([u"Regex for text file: '%s'", parameters[gc.PPN_JOB_IS_TEXT_FILE_NAME_REGEX]])
audio_file_relative_path = parameters[gc.PPN_JOB_IS_AUDIO_FILE_RELATIVE_PATH]
self.log([u"Relative path for audio file: '%s'", audio_file_relative_path])
audio_file_name_regex = re.compile(r"" + parameters[gc.PPN_JOB_IS_AUDIO_FILE_NAME_REGEX])
self.log([u"Regex for audio file: '%s'", parameters[gc.PPN_JOB_IS_AUDIO_FILE_NAME_REGEX]])
if parameters[gc.PPN_JOB_IS_HIERARCHY_TYPE] == HierarchyType.FLAT:
self.log(u"Looking for text/audio pairs in flat hierarchy")
text_files = self._find_files(
entries,
tasks_root_directory,
text_file_relative_path,
text_file_name_regex
)
self.log([u"Found text files: '%s'", text_files])
audio_files = self._find_files(
entries,
tasks_root_directory,
audio_file_relative_path,
audio_file_name_regex
)
self.log([u"Found audio files: '%s'", audio_files])
self.log(u"Matching files in flat hierarchy...")
matched_tasks = self._match_files_flat_hierarchy(
text_files,
audio_files
)
self.log(u"Matching files in flat hierarchy... done")
for task_info in matched_tasks:
self.log([u"Creating task: '%s'", str(task_info)])
task = self._create_task(
task_info,
config_string,
sync_map_root_directory,
job_os_hierarchy_type
)
job.add_task(task)
if parameters[gc.PPN_JOB_IS_HIERARCHY_TYPE] == HierarchyType.PAGED:
self.log(u"Looking for text/audio pairs in paged hierarchy")
# find all subdirectories of tasks_root_directory
# that match gc.PPN_JOB_IS_TASK_DIRECTORY_NAME_REGEX
matched_directories = self._match_directories(
entries,
tasks_root_directory,
parameters[gc.PPN_JOB_IS_TASK_DIRECTORY_NAME_REGEX]
)
for matched_directory in matched_directories:
# rebuild the full path
matched_directory_full_path = gf.norm_join(
tasks_root_directory,
matched_directory
)
self.log([u"Looking for text/audio pairs in directory '%s'", matched_directory_full_path])
# look for text and audio files there
text_files = self._find_files(
entries,
matched_directory_full_path,
text_file_relative_path,
text_file_name_regex
)
self.log([u"Found text files: '%s'", text_files])
audio_files = self._find_files(
entries,
matched_directory_full_path,
audio_file_relative_path,
audio_file_name_regex
)
self.log([u"Found audio files: '%s'", audio_files])
# if we have found exactly one text and one audio file,
# create a Task
if (len(text_files) == 1) and (len(audio_files) == 1):
self.log([u"Exactly one text file and one audio file in '%s'", matched_directory])
task_info = [
matched_directory,
text_files[0],
audio_files[0]
]
self.log([u"Creating task: '%s'", str(task_info)])
task = self._create_task(
task_info,
config_string,
sync_map_root_directory,
job_os_hierarchy_type
)
job.add_task(task)
elif len(text_files) > 1:
self.log([u"More than one text file in '%s'", matched_directory])
elif len(audio_files) > 1:
self.log([u"More than one audio file in '%s'", matched_directory])
else:
self.log([u"No text nor audio file in '%s'", matched_directory])
return job
def run(self, arguments, show_help=True):
"""
Program entry point.
Please note that the first item in ``arguments`` is discarded,
as it is assumed to be the script/invocation name;
pass a "dumb" placeholder if you call this method with
an argument different that ``sys.argv``.
:param arguments: the list of arguments
:type arguments: list
:param show_help: if ``False``, do not show help on ``-h`` and ``--help``
:type show_help: bool
:rtype: int
"""
# convert arguments into Unicode strings
if self.use_sys:
# check that sys.stdin.encoding and sys.stdout.encoding are set to utf-8
if not gf.FROZEN:
if sys.stdin.encoding not in ["UTF-8", "UTF8"]:
self.print_warning(u"The default input encoding is not UTF-8.")
self.print_warning(u"You might want to set 'PYTHONIOENCODING=UTF-8' in your shell.")
if sys.stdout.encoding not in ["UTF-8", "UTF8"]:
self.print_warning(u"The default output encoding is not UTF-8.")
self.print_warning(u"You might want to set 'PYTHONIOENCODING=UTF-8' in your shell.")
# decode using sys.stdin.encoding
args = [gf.safe_unicode_stdin(arg) for arg in arguments]
else:
# decode using utf-8 (but you should pass Unicode strings as parameters anyway)
args = [gf.safe_unicode(arg) for arg in arguments]
if show_help:
if u"-h" in args:
return self.print_help(short=True)
if u"--help" in args:
return self.print_help(short=False)
if u"--version" in args:
return self.print_name_version()
# store formal arguments
self.formal_arguments_raw = arguments
self.formal_arguments = args
# to obtain the actual arguments,
# remove the first one and "special" switches
args = args[1:]
set_args = set(args)
# set verbosity, if requested
for flag in set([u"-v", u"--verbose"]) & set_args:
self.verbose = True
args.remove(flag)
for flag in set([u"-vv", u"--very-verbose"]) & set_args:
self.verbose = True
self.very_verbose = True
args.remove(flag)
# set RuntimeConfiguration string, if specified
for flag in [u"-r", u"--runtime-configuration"]:
rconf_string = self.has_option_with_value(flag, actual_arguments=False)
if rconf_string is not None:
self.rconf = RuntimeConfiguration(rconf_string)
args.remove("%s=%s" % (flag, rconf_string))
# set log file path, if requested
log_path = None
for flag in [u"-l", u"--log"]:
log_path = self.has_option_with_value(flag, actual_arguments=False)
if log_path is not None:
args.remove("%s=%s" % (flag, log_path))
elif flag in set_args:
handler, log_path = gf.tmp_file(suffix=u".log", root=self.rconf[RuntimeConfiguration.TMP_PATH])
args.remove(flag)
if log_path is not None:
self.log_file_path = log_path
# if no actual arguments left, print help
if (len(args) < 1) and (show_help):
return self.print_help(short=True)
# store actual arguments
self.actual_arguments = args
# create logger
self.logger = Logger(tee=self.verbose, tee_show_datetime=self.very_verbose)
self.log([u"Formal arguments: %s", self.formal_arguments])
self.log([u"Actual arguments: %s", self.actual_arguments])
self.log([u"Runtime configuration: '%s'", self.rconf.config_string()])
# perform command
exit_code = self.perform_command()
self.log([u"Execution completed with code %d", exit_code])
# output log if requested
if self.log_file_path is not None:
self.log([u"User requested saving log to file '%s'", self.log_file_path])
self.logger.write(self.log_file_path)
if self.use_sys:
self.print_info(u"Log written to file '%s'" % self.log_file_path)
return self.exit(exit_code)
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 4:
return self.print_help()
text_format = gf.safe_unicode(self.actual_arguments[0])
if text_format == u"list":
text = gf.safe_unicode(self.actual_arguments[1])
elif text_format in TextFileFormat.ALLOWED_VALUES:
text = self.actual_arguments[1]
if not self.check_input_file(text):
return self.ERROR_EXIT_CODE
else:
return self.print_help()
l1_id_regex = self.has_option_with_value(u"--l1-id-regex")
l2_id_regex = self.has_option_with_value(u"--l2-id-regex")
l3_id_regex = self.has_option_with_value(u"--l3-id-regex")
id_regex = self.has_option_with_value(u"--id-regex")
class_regex = self.has_option_with_value(u"--class-regex")
sort = self.has_option_with_value(u"--sort")
backwards = self.has_option([u"-b", u"--backwards"])
quit_after = gf.safe_float(self.has_option_with_value(u"--quit-after"), None)
start_fragment = gf.safe_int(self.has_option_with_value(u"--start"), None)
end_fragment = gf.safe_int(self.has_option_with_value(u"--end"), None)
parameters = {
gc.PPN_TASK_IS_TEXT_MUNPARSED_L1_ID_REGEX: l1_id_regex,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L2_ID_REGEX: l2_id_regex,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L3_ID_REGEX: l3_id_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_CLASS_REGEX: class_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_ID_REGEX: id_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_ID_SORT: sort,
}
if (text_format == TextFileFormat.MUNPARSED) and ((l1_id_regex is None) or (l2_id_regex is None) or (l3_id_regex is None)):
self.print_error(u"You must specify --l1-id-regex and --l2-id-regex and --l3-id-regex for munparsed format")
return self.ERROR_EXIT_CODE
if (text_format == TextFileFormat.UNPARSED) and (id_regex is None) and (class_regex is None):
self.print_error(u"You must specify --id-regex and/or --class-regex for unparsed format")
return self.ERROR_EXIT_CODE
language = gf.safe_unicode(self.actual_arguments[2])
output_file_path = self.actual_arguments[3]
if not self.check_output_file(output_file_path):
return self.ERROR_EXIT_CODE
text_file = self.get_text_file(text_format, text, parameters)
if text_file is None:
self.print_error(u"Unable to build a TextFile from the given parameters")
return self.ERROR_EXIT_CODE
elif len(text_file) == 0:
self.print_error(u"No text fragments found")
return self.ERROR_EXIT_CODE
text_file.set_language(language)
self.print_info(u"Read input text with %d fragments" % (len(text_file)))
if start_fragment is not None:
self.print_info(u"Slicing from index %d" % (start_fragment))
if end_fragment is not None:
self.print_info(u"Slicing to index %d" % (end_fragment))
text_slice = text_file.get_slice(start_fragment, end_fragment)
self.print_info(u"Synthesizing %d fragments" % (len(text_slice)))
if quit_after is not None:
self.print_info(u"Stop synthesizing upon reaching %.3f seconds" % (quit_after))
try:
synt = Synthesizer(rconf=self.rconf, logger=self.logger)
synt.synthesize(
text_slice,
output_file_path,
quit_after=quit_after,
backwards=backwards
)
self.print_success(u"Created file '%s'" % output_file_path)
synt.clear_cache()
return self.NO_ERROR_EXIT_CODE
except ImportError as exc:
tts = self.rconf[RuntimeConfiguration.TTS]
if tts == Synthesizer.AWS:
self.print_error(u"You need to install Python module boto3 to use the AWS Polly TTS API wrapper. Run:")
self.print_error(u"$ pip install boto3")
self.print_error(u"or, to install for all users:")
self.print_error(u"$ sudo pip install boto3")
elif tts == Synthesizer.NUANCE:
self.print_error(u"You need to install Python module requests to use the Nuance TTS API wrapper. Run:")
self.print_error(u"$ pip install requests")
self.print_error(u"or, to install for all users:")
self.print_error(u"$ sudo pip install requests")
else:
self.print_error(u"An unexpected error occurred while synthesizing text:")
self.print_error(u"%s" % exc)
except Exception as exc:
self.print_error(u"An unexpected error occurred while synthesizing text:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 1:
return self.print_help()
if self.has_option([u"-e", u"--examples"]):
return self.print_examples(False)
if self.has_option(u"--examples-all"):
return self.print_examples(True)
if self.has_option([u"--list-parameters"]):
return self.print_parameters()
parameter = self.has_option_with_value(u"--list-values")
if parameter is not None:
return self.print_values(parameter)
elif self.has_option(u"--list-values"):
return self.print_values(u"?")
# NOTE list() is needed for Python3, where keys() is not a list!
demo = self.has_option(list(self.DEMOS.keys()))
demo_parameters = u""
download_from_youtube = self.has_option([u"-y", u"--youtube"])
largest_audio = self.has_option(u"--largest-audio")
keep_audio = self.has_option(u"--keep-audio")
output_html = self.has_option(u"--output-html")
validate = not self.has_option(u"--skip-validator")
print_faster_rate = self.has_option(u"--faster-rate")
print_rates = self.has_option(u"--rate")
print_zero = self.has_option(u"--zero")
presets_word = self.has_option(u"--presets-word")
if demo:
validate = False
for key in self.DEMOS:
if self.has_option(key):
demo_parameters = self.DEMOS[key]
audio_file_path = demo_parameters[u"audio"]
text_file_path = demo_parameters[u"text"]
config_string = demo_parameters[u"config"]
sync_map_file_path = demo_parameters[u"syncmap"]
# TODO allow injecting rconf options directly from DEMOS options field
if key == u"--example-cewsubprocess":
self.rconf[RuntimeConfiguration.CEW_SUBPROCESS_ENABLED] = True
elif key == u"--example-ctw-espeak":
self.rconf[RuntimeConfiguration.TTS] = "custom"
self.rconf[RuntimeConfiguration.TTS_PATH] = self.CTW_ESPEAK
elif key == u"--example-ctw-speect":
self.rconf[RuntimeConfiguration.TTS] = "custom"
self.rconf[RuntimeConfiguration.TTS_PATH] = self.CTW_SPEECT
elif key == u"--example-festival":
self.rconf[RuntimeConfiguration.TTS] = "festival"
elif key == u"--example-mws":
self.rconf[RuntimeConfiguration.MFCC_WINDOW_LENGTH] = "1.500"
self.rconf[RuntimeConfiguration.MFCC_WINDOW_SHIFT] = "0.500"
elif key == u"--example-multilevel-tts":
self.rconf[RuntimeConfiguration.TTS_L1] = "festival"
self.rconf[RuntimeConfiguration.TTS_L2] = "festival"
self.rconf[RuntimeConfiguration.TTS_L3] = "espeak"
elif key == u"--example-words-festival-cache":
self.rconf[RuntimeConfiguration.TTS] = "festival"
self.rconf[RuntimeConfiguration.TTS_CACHE] = True
elif key == u"--example-faster-rate":
print_faster_rate = True
elif key == u"--example-no-zero":
print_zero = True
elif key == u"--example-py":
self.rconf[RuntimeConfiguration.C_EXTENSIONS] = False
elif key == u"--example-rate":
print_rates = True
elif key == u"--example-remove-nonspeech-rateaggressive":
print_rates = True
elif key == u"--example-youtube":
download_from_youtube = True
break
else:
if len(self.actual_arguments) < 4:
return self.print_help()
audio_file_path = self.actual_arguments[0]
text_file_path = self.actual_arguments[1]
config_string = self.actual_arguments[2]
sync_map_file_path = self.actual_arguments[3]
if presets_word:
self.print_info(u"Preset for word-level alignment")
self.rconf[RuntimeConfiguration.MFCC_MASK_NONSPEECH] = True
self.rconf[RuntimeConfiguration.MFCC_MASK_NONSPEECH_L3] = True
html_file_path = None
if output_html:
keep_audio = True
html_file_path = sync_map_file_path + u".html"
if download_from_youtube:
youtube_url = gf.safe_unicode(audio_file_path)
if (not download_from_youtube) and (not self.check_input_file(audio_file_path)):
return self.ERROR_EXIT_CODE
if not self.check_input_file(text_file_path):
return self.ERROR_EXIT_CODE
if not self.check_output_file(sync_map_file_path):
return self.ERROR_EXIT_CODE
if (html_file_path is not None) and (not self.check_output_file(html_file_path)):
return self.ERROR_EXIT_CODE
self.check_c_extensions()
if demo:
msg = []
msg.append(u"Running example task with arguments:")
if download_from_youtube:
msg.append(u" YouTube URL: %s" % youtube_url)
else:
msg.append(u" Audio file: %s" % audio_file_path)
msg.append(u" Text file: %s" % text_file_path)
msg.append(u" Config string: %s" % config_string)
msg.append(u" Sync map file: %s" % sync_map_file_path)
if len(demo_parameters[u"options"]) > 0:
msg.append(u" Options: %s" % demo_parameters[u"options"])
self.print_info(u"\n".join(msg))
if validate:
self.print_info(u"Validating config string (specify --skip-validator to bypass)...")
validator = Validator(logger=self.logger)
result = validator.check_configuration_string(config_string, is_job=False, external_name=True)
if not result.passed:
self.print_error(u"The given config string is not valid:")
self.print_generic(result.pretty_print())
return self.ERROR_EXIT_CODE
self.print_info(u"Validating config string... done")
if download_from_youtube:
try:
self.print_info(u"Downloading audio from '%s' ..." % youtube_url)
downloader = Downloader(logger=self.logger)
audio_file_path = downloader.audio_from_youtube(
youtube_url,
download=True,
output_file_path=None,
largest_audio=largest_audio
)
self.print_info(u"Downloading audio from '%s' ... done" % youtube_url)
except ImportError:
self.print_no_dependency_error()
return self.ERROR_EXIT_CODE
except Exception as exc:
self.print_error(u"An unexpected error occurred while downloading audio from YouTube:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
else:
audio_extension = gf.file_extension(audio_file_path)
if audio_extension.lower() not in AudioFile.FILE_EXTENSIONS:
self.print_warning(u"Your audio file path has extension '%s', which is uncommon for an audio file." % audio_extension)
self.print_warning(u"Attempting at executing your Task anyway.")
self.print_warning(u"If it fails, you might have swapped the first two arguments.")
self.print_warning(u"The audio file path should be the first argument, the text file path the second.")
try:
self.print_info(u"Creating task...")
task = Task(config_string, logger=self.logger)
task.audio_file_path_absolute = audio_file_path
task.text_file_path_absolute = text_file_path
task.sync_map_file_path_absolute = sync_map_file_path
self.print_info(u"Creating task... done")
except Exception as exc:
self.print_error(u"An unexpected error occurred while creating the task:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Executing task...")
executor = ExecuteTask(task=task, rconf=self.rconf, logger=self.logger)
executor.execute()
self.print_info(u"Executing task... done")
except Exception as exc:
self.print_error(u"An unexpected error occurred while executing the task:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Creating output sync map file...")
path = task.output_sync_map_file()
self.print_info(u"Creating output sync map file... done")
self.print_success(u"Created file '%s'" % path)
except Exception as exc:
self.print_error(u"An unexpected error occurred while writing the sync map file:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
if output_html:
try:
parameters = {}
parameters[gc.PPN_TASK_OS_FILE_FORMAT] = task.configuration["o_format"]
parameters[gc.PPN_TASK_OS_FILE_EAF_AUDIO_REF] = task.configuration["o_eaf_audio_ref"]
parameters[gc.PPN_TASK_OS_FILE_SMIL_AUDIO_REF] = task.configuration["o_smil_audio_ref"]
parameters[gc.PPN_TASK_OS_FILE_SMIL_PAGE_REF] = task.configuration["o_smil_page_ref"]
self.print_info(u"Creating output HTML file...")
task.sync_map.output_html_for_tuning(audio_file_path, html_file_path, parameters)
self.print_info(u"Creating output HTML file... done")
self.print_success(u"Created file '%s'" % html_file_path)
except Exception as exc:
self.print_error(u"An unexpected error occurred while writing the HTML file:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
if download_from_youtube:
if keep_audio:
self.print_info(u"Option --keep-audio set: keeping downloaded file '%s'" % audio_file_path)
else:
gf.delete_file(None, audio_file_path)
if print_zero:
zero_duration = [l for l in task.sync_map_leaves(SyncMapFragment.REGULAR) if l.begin == l.end]
if len(zero_duration) > 0:
self.print_warning(u"Fragments with zero duration:")
for fragment in zero_duration:
self.print_generic(u" %s" % (fragment.pretty_print))
if print_rates:
self.print_info(u"Fragments with rates:")
for fragment in task.sync_map_leaves(SyncMapFragment.REGULAR):
self.print_generic(u" %s\t%.3f" % (fragment.pretty_print, fragment.rate or 0.0))
if print_faster_rate:
max_rate = task.configuration["aba_rate_value"]
if max_rate is not None:
faster = [l for l in task.sync_map_leaves(SyncMapFragment.REGULAR) if l.rate >= max_rate + Decimal("0.001")]
if len(faster) > 0:
self.print_warning(u"Fragments with rate greater than %.3f:" % max_rate)
for fragment in faster:
self.print_generic(u" %s\t%.3f" % (fragment.pretty_print, fragment.rate or 0.0))
return self.NO_ERROR_EXIT_CODE
def _read_unparsed(self, lines):
"""
Read text fragments from an unparsed format text file.
:param list lines: the lines of the unparsed text file
"""
from bs4 import BeautifulSoup
def filter_attributes():
""" Return a dict with the bs4 filter parameters """
attributes = {}
for attribute_name, filter_name in [
("class", gc.PPN_TASK_IS_TEXT_UNPARSED_CLASS_REGEX),
("id", gc.PPN_TASK_IS_TEXT_UNPARSED_ID_REGEX)
]:
if filter_name in self.parameters:
regex_string = self.parameters[filter_name]
if regex_string is not None:
self.log([
u"Regex for %s: '%s'", attribute_name, regex_string
])
regex = re.compile(r".*\b" + regex_string + r"\b.*")
attributes[attribute_name] = regex
return attributes
#
# TODO better and/or parametric parsing,
# for example, removing tags but keeping text, etc.
#
self.log(u"Parsing fragments from unparsed text format")
# transform text in a soup object
self.log(u"Creating soup")
soup = BeautifulSoup("\n".join(lines), "lxml")
# extract according to class_regex and id_regex
text_from_id = {}
ids = []
filter_attributes = filter_attributes()
self.log(
[u"Finding elements matching attributes '%s'", filter_attributes])
nodes = soup.findAll(attrs=filter_attributes)
for node in nodes:
try:
f_id = gf.safe_unicode(node["id"])
f_text = gf.safe_unicode(node.text)
text_from_id[f_id] = f_text
ids.append(f_id)
except KeyError:
self.log_warn(u"KeyError while parsing a node")
# sort by ID as requested
id_sort = gf.safe_get(dictionary=self.parameters,
key=gc.PPN_TASK_IS_TEXT_UNPARSED_ID_SORT,
default_value=IDSortingAlgorithm.UNSORTED,
can_return_none=False)
self.log([u"Sorting text fragments using '%s'", id_sort])
sorted_ids = IDSortingAlgorithm(id_sort).sort(ids)
# append to fragments
self.log(u"Appending fragments")
self._create_text_fragments([(key, [text_from_id[key]])
for key in sorted_ids])
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 1:
return self.print_help()
if self.has_option([u"-e", u"--examples"]):
return self.print_examples(False)
if self.has_option(u"--examples-all"):
return self.print_examples(True)
if self.has_option([u"--list-parameters"]):
return self.print_parameters()
parameter = self.has_option_with_value(u"--list-values")
if parameter is not None:
return self.print_values(parameter)
elif self.has_option(u"--list-values"):
return self.print_values(u"?")
# NOTE list() is needed for Python3, where keys() is not a list!
demo = self.has_option(list(self.DEMOS.keys()))
demo_parameters = u""
download_from_youtube = self.has_option([u"-y", u"--youtube"])
largest_audio = self.has_option(u"--largest-audio")
keep_audio = self.has_option(u"--keep-audio")
output_html = self.has_option(u"--output-html")
validate = not self.has_option(u"--skip-validator")
print_faster_rate = self.has_option(u"--faster-rate")
print_rates = self.has_option(u"--rate")
print_zero = self.has_option(u"--zero")
presets_word = self.has_option(u"--presets-word")
if demo:
validate = False
for key in self.DEMOS:
if self.has_option(key):
demo_parameters = self.DEMOS[key]
audio_file_path = demo_parameters[u"audio"]
text_file_path = demo_parameters[u"text"]
config_string = demo_parameters[u"config"]
sync_map_file_path = demo_parameters[u"syncmap"]
# TODO allow injecting rconf options directly from DEMOS options field
if key == u"--example-cewsubprocess":
self.rconf[
RuntimeConfiguration.CEW_SUBPROCESS_ENABLED] = True
elif key == u"--example-ctw-espeak":
self.rconf[RuntimeConfiguration.TTS] = "custom"
self.rconf[
RuntimeConfiguration.TTS_PATH] = self.CTW_ESPEAK
elif key == u"--example-ctw-speect":
self.rconf[RuntimeConfiguration.TTS] = "custom"
self.rconf[
RuntimeConfiguration.TTS_PATH] = self.CTW_SPEECT
elif key == u"--example-festival":
self.rconf[RuntimeConfiguration.TTS] = "festival"
elif key == u"--example-mws":
self.rconf[
RuntimeConfiguration.MFCC_WINDOW_LENGTH] = "1.500"
self.rconf[
RuntimeConfiguration.MFCC_WINDOW_SHIFT] = "0.500"
elif key == u"--example-multilevel-tts":
self.rconf[RuntimeConfiguration.TTS_L1] = "festival"
self.rconf[RuntimeConfiguration.TTS_L2] = "festival"
self.rconf[RuntimeConfiguration.TTS_L3] = "espeak"
elif key == u"--example-words-festival-cache":
self.rconf[RuntimeConfiguration.TTS] = "festival"
self.rconf[RuntimeConfiguration.TTS_CACHE] = True
elif key == u"--example-faster-rate":
print_faster_rate = True
elif key == u"--example-no-zero":
print_zero = True
elif key == u"--example-py":
self.rconf[RuntimeConfiguration.C_EXTENSIONS] = False
elif key == u"--example-rate":
print_rates = True
elif key == u"--example-remove-nonspeech-rateaggressive":
print_rates = True
elif key == u"--example-youtube":
download_from_youtube = True
break
else:
if len(self.actual_arguments) < 4:
return self.print_help()
audio_file_path = self.actual_arguments[0]
text_file_path = self.actual_arguments[1]
config_string = self.actual_arguments[2]
sync_map_file_path = self.actual_arguments[3]
if presets_word:
self.print_info(u"Preset for word-level alignment")
self.rconf[RuntimeConfiguration.MFCC_MASK_NONSPEECH] = True
self.rconf[RuntimeConfiguration.MFCC_MASK_NONSPEECH_L3] = True
html_file_path = None
if output_html:
keep_audio = True
html_file_path = sync_map_file_path + u".html"
if download_from_youtube:
youtube_url = gf.safe_unicode(audio_file_path)
if (not download_from_youtube) and (
not self.check_input_file(audio_file_path)):
return self.ERROR_EXIT_CODE
if not self.check_input_file(text_file_path):
return self.ERROR_EXIT_CODE
if not self.check_output_file(sync_map_file_path):
return self.ERROR_EXIT_CODE
if (html_file_path
is not None) and (not self.check_output_file(html_file_path)):
return self.ERROR_EXIT_CODE
self.check_c_extensions()
if demo:
msg = []
msg.append(u"Running example task with arguments:")
if download_from_youtube:
msg.append(u" YouTube URL: %s" % youtube_url)
else:
msg.append(u" Audio file: %s" % audio_file_path)
msg.append(u" Text file: %s" % text_file_path)
msg.append(u" Config string: %s" % config_string)
msg.append(u" Sync map file: %s" % sync_map_file_path)
if len(demo_parameters[u"options"]) > 0:
msg.append(u" Options: %s" %
demo_parameters[u"options"])
self.print_info(u"\n".join(msg))
if validate:
self.print_info(
u"Validating config string (specify --skip-validator to bypass)..."
)
validator = Validator(logger=self.logger)
result = validator.check_configuration_string(config_string,
is_job=False,
external_name=True)
if not result.passed:
self.print_error(u"The given config string is not valid:")
self.print_generic(result.pretty_print())
return self.ERROR_EXIT_CODE
self.print_info(u"Validating config string... done")
if download_from_youtube:
try:
self.print_info(u"Downloading audio from '%s' ..." %
youtube_url)
downloader = Downloader(logger=self.logger)
audio_file_path = downloader.audio_from_youtube(
youtube_url,
download=True,
output_file_path=None,
largest_audio=largest_audio)
self.print_info(u"Downloading audio from '%s' ... done" %
youtube_url)
except ImportError:
self.print_no_dependency_error()
return self.ERROR_EXIT_CODE
except Exception as exc:
self.print_error(
u"An unexpected error occurred while downloading audio from YouTube:"
)
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
else:
audio_extension = gf.file_extension(audio_file_path)
if audio_extension.lower() not in AudioFile.FILE_EXTENSIONS:
self.print_warning(
u"Your audio file path has extension '%s', which is uncommon for an audio file."
% audio_extension)
self.print_warning(
u"Attempting at executing your Task anyway.")
self.print_warning(
u"If it fails, you might have swapped the first two arguments."
)
self.print_warning(
u"The audio file path should be the first argument, the text file path the second."
)
try:
self.print_info(u"Creating task...")
task = Task(config_string, logger=self.logger)
task.audio_file_path_absolute = audio_file_path
task.text_file_path_absolute = text_file_path
task.sync_map_file_path_absolute = sync_map_file_path
self.print_info(u"Creating task... done")
except Exception as exc:
self.print_error(
u"An unexpected error occurred while creating the task:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Executing task...")
executor = ExecuteTask(task=task,
rconf=self.rconf,
logger=self.logger)
executor.execute()
self.print_info(u"Executing task... done")
except Exception as exc:
self.print_error(
u"An unexpected error occurred while executing the task:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Creating output sync map file...")
path = task.output_sync_map_file()
self.print_info(u"Creating output sync map file... done")
self.print_success(u"Created file '%s'" % path)
except Exception as exc:
self.print_error(
u"An unexpected error occurred while writing the sync map file:"
)
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
if output_html:
try:
parameters = {}
parameters[gc.PPN_TASK_OS_FILE_FORMAT] = task.configuration[
"o_format"]
parameters[
gc.PPN_TASK_OS_FILE_EAF_AUDIO_REF] = task.configuration[
"o_eaf_audio_ref"]
parameters[
gc.PPN_TASK_OS_FILE_SMIL_AUDIO_REF] = task.configuration[
"o_smil_audio_ref"]
parameters[
gc.PPN_TASK_OS_FILE_SMIL_PAGE_REF] = task.configuration[
"o_smil_page_ref"]
self.print_info(u"Creating output HTML file...")
task.sync_map.output_html_for_tuning(audio_file_path,
html_file_path,
parameters)
self.print_info(u"Creating output HTML file... done")
self.print_success(u"Created file '%s'" % html_file_path)
except Exception as exc:
self.print_error(
u"An unexpected error occurred while writing the HTML file:"
)
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
if download_from_youtube:
if keep_audio:
self.print_info(
u"Option --keep-audio set: keeping downloaded file '%s'" %
audio_file_path)
else:
gf.delete_file(None, audio_file_path)
if print_zero:
zero_duration = [
l for l in task.sync_map_leaves(SyncMapFragment.REGULAR)
if l.begin == l.end
]
if len(zero_duration) > 0:
self.print_warning(u"Fragments with zero duration:")
for fragment in zero_duration:
self.print_generic(u" %s" % (fragment.pretty_print))
if print_rates:
self.print_info(u"Fragments with rates:")
for fragment in task.sync_map_leaves(SyncMapFragment.REGULAR):
self.print_generic(
u" %s\t%.3f" %
(fragment.pretty_print, fragment.rate or 0.0))
if print_faster_rate:
max_rate = task.configuration["aba_rate_value"]
if max_rate is not None:
faster = [
l for l in task.sync_map_leaves(SyncMapFragment.REGULAR)
if l.rate >= max_rate + Decimal("0.001")
]
if len(faster) > 0:
self.print_warning(
u"Fragments with rate greater than %.3f:" % max_rate)
for fragment in faster:
self.print_generic(
u" %s\t%.3f" %
(fragment.pretty_print, fragment.rate or 0.0))
return self.NO_ERROR_EXIT_CODE
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 4:
return self.print_help()
text_format = gf.safe_unicode(self.actual_arguments[0])
if text_format == u"list":
text = gf.safe_unicode(self.actual_arguments[1])
elif text_format in TextFileFormat.ALLOWED_VALUES:
text = self.actual_arguments[1]
if not self.check_input_file(text):
return self.ERROR_EXIT_CODE
else:
return self.print_help()
l1_id_regex = self.has_option_with_value(u"--l1-id-regex")
l2_id_regex = self.has_option_with_value(u"--l2-id-regex")
l3_id_regex = self.has_option_with_value(u"--l3-id-regex")
id_regex = self.has_option_with_value(u"--id-regex")
class_regex = self.has_option_with_value(u"--class-regex")
sort = self.has_option_with_value(u"--sort")
backwards = self.has_option([u"-b", u"--backwards"])
quit_after = gf.safe_float(self.has_option_with_value(u"--quit-after"),
None)
start_fragment = gf.safe_int(self.has_option_with_value(u"--start"),
None)
end_fragment = gf.safe_int(self.has_option_with_value(u"--end"), None)
parameters = {
gc.PPN_TASK_IS_TEXT_MUNPARSED_L1_ID_REGEX: l1_id_regex,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L2_ID_REGEX: l2_id_regex,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L3_ID_REGEX: l3_id_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_CLASS_REGEX: class_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_ID_REGEX: id_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_ID_SORT: sort,
}
if (text_format == TextFileFormat.MUNPARSED) and (
(l1_id_regex is None) or (l2_id_regex is None) or
(l3_id_regex is None)):
self.print_error(
u"You must specify --l1-id-regex and --l2-id-regex and --l3-id-regex for munparsed format"
)
return self.ERROR_EXIT_CODE
if (text_format == TextFileFormat.UNPARSED) and (
id_regex is None) and (class_regex is None):
self.print_error(
u"You must specify --id-regex and/or --class-regex for unparsed format"
)
return self.ERROR_EXIT_CODE
language = gf.safe_unicode(self.actual_arguments[2])
output_file_path = self.actual_arguments[3]
if not self.check_output_file(output_file_path):
return self.ERROR_EXIT_CODE
text_file = self.get_text_file(text_format, text, parameters)
if text_file is None:
self.print_error(
u"Unable to build a TextFile from the given parameters")
return self.ERROR_EXIT_CODE
elif len(text_file) == 0:
self.print_error(u"No text fragments found")
return self.ERROR_EXIT_CODE
text_file.set_language(language)
self.print_info(u"Read input text with %d fragments" %
(len(text_file)))
if start_fragment is not None:
self.print_info(u"Slicing from index %d" % (start_fragment))
if end_fragment is not None:
self.print_info(u"Slicing to index %d" % (end_fragment))
text_slice = text_file.get_slice(start_fragment, end_fragment)
self.print_info(u"Synthesizing %d fragments" % (len(text_slice)))
if quit_after is not None:
self.print_info(u"Stop synthesizing upon reaching %.3f seconds" %
(quit_after))
try:
synt = Synthesizer(rconf=self.rconf, logger=self.logger)
synt.synthesize(text_slice,
output_file_path,
quit_after=quit_after,
backwards=backwards)
self.print_success(u"Created file '%s'" % output_file_path)
synt.clear_cache()
return self.NO_ERROR_EXIT_CODE
except ImportError as exc:
tts = self.rconf[RuntimeConfiguration.TTS]
if tts == Synthesizer.AWS:
self.print_error(
u"You need to install Python module boto3 to use the AWS Polly TTS API wrapper. Run:"
)
self.print_error(u"$ pip install boto3")
self.print_error(u"or, to install for all users:")
self.print_error(u"$ sudo pip install boto3")
elif tts == Synthesizer.NUANCE:
self.print_error(
u"You need to install Python module requests to use the Nuance TTS API wrapper. Run:"
)
self.print_error(u"$ pip install requests")
self.print_error(u"or, to install for all users:")
self.print_error(u"$ sudo pip install requests")
else:
self.print_error(
u"An unexpected error occurred while synthesizing text:")
self.print_error(u"%s" % exc)
except Exception as exc:
self.print_error(
u"An unexpected error occurred while synthesizing text:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
def _synthesize_multiple_c_extension(self, text_file, output_file_path, quit_after=None, backwards=False):
"""
Synthesize multiple text fragments, using the cfw extension.
Return a tuple (anchors, total_time, num_chars).
:rtype: (bool, (list, :class:`~aeneas.exacttiming.TimeValue`, int))
"""
self.log(u"Synthesizing using C extension...")
# convert parameters from Python values to C values
try:
c_quit_after = float(quit_after)
except TypeError:
c_quit_after = 0.0
c_backwards = 0
if backwards:
c_backwards = 1
self.log([u"output_file_path: %s", output_file_path])
self.log([u"c_quit_after: %.3f", c_quit_after])
self.log([u"c_backwards: %d", c_backwards])
self.log(u"Preparing u_text...")
u_text = []
fragments = text_file.fragments
for fragment in fragments:
f_lang = fragment.language
f_text = fragment.filtered_text
if f_lang is None:
f_lang = self.DEFAULT_LANGUAGE
f_voice_code = self.VOICE_CODE_TO_SUBPROCESS[self._language_to_voice_code(f_lang)]
if f_text is None:
f_text = u""
u_text.append((f_voice_code, f_text))
self.log(u"Preparing u_text... done")
# call C extension
sr = None
sf = None
intervals = None
self.log(u"Preparing c_text...")
if gf.PY2:
# Python 2 => pass byte strings
c_text = [(gf.safe_bytes(t[0]), gf.safe_bytes(t[1])) for t in u_text]
else:
# Python 3 => pass Unicode strings
c_text = [(gf.safe_unicode(t[0]), gf.safe_unicode(t[1])) for t in u_text]
self.log(u"Preparing c_text... done")
self.log(u"Calling aeneas.cfw directly")
try:
self.log(u"Importing aeneas.cfw...")
import aeneas.cfw.cfw
self.log(u"Importing aeneas.cfw... done")
self.log(u"Calling aeneas.cfw...")
sr, sf, intervals = aeneas.cfw.cfw.synthesize_multiple(
output_file_path,
c_quit_after,
c_backwards,
c_text
)
self.log(u"Calling aeneas.cfw... done")
except Exception as exc:
self.log_exc(u"An unexpected error occurred while running cfw", exc, False, None)
return (False, None)
self.log([u"sr: %d", sr])
self.log([u"sf: %d", sf])
# create output
anchors = []
current_time = TimeValue("0.000")
num_chars = 0
if backwards:
fragments = fragments[::-1]
for i in range(sf):
# get the correct fragment
fragment = fragments[i]
# store for later output
anchors.append([
TimeValue(intervals[i][0]),
fragment.identifier,
fragment.filtered_text
])
# increase the character counter
num_chars += fragment.characters
# update current_time
current_time = TimeValue(intervals[i][1])
# return output
# NOTE anchors do not make sense if backwards == True
self.log([u"Returning %d time anchors", len(anchors)])
self.log([u"Current time %.3f", current_time])
self.log([u"Synthesized %d characters", num_chars])
self.log(u"Synthesizing using C extension... done")
return (True, (anchors, current_time, num_chars))
def read_properties(self, audio_file_path):
"""
Read the properties of an audio file
and return them as a dictionary.
Example: ::
d["index"]=0
d["codec_name"]=mp3
d["codec_long_name"]=MP3 (MPEG audio layer 3)
d["profile"]=unknown
d["codec_type"]=audio
d["codec_time_base"]=1/44100
d["codec_tag_string"]=[0][0][0][0]
d["codec_tag"]=0x0000
d["sample_fmt"]=s16p
d["sample_rate"]=44100
d["channels"]=1
d["channel_layout"]=mono
d["bits_per_sample"]=0
d["id"]=N/A
d["r_frame_rate"]=0/0
d["avg_frame_rate"]=0/0
d["time_base"]=1/14112000
d["start_pts"]=0
d["start_time"]=0.000000
d["duration_ts"]=1545083190
d["duration"]=109.487188
d["bit_rate"]=128000
d["max_bit_rate"]=N/A
d["bits_per_raw_sample"]=N/A
d["nb_frames"]=N/A
d["nb_read_frames"]=N/A
d["nb_read_packets"]=N/A
d["DISPOSITION:default"]=0
d["DISPOSITION:dub"]=0
d["DISPOSITION:original"]=0
d["DISPOSITION:comment"]=0
d["DISPOSITION:lyrics"]=0
d["DISPOSITION:karaoke"]=0
d["DISPOSITION:forced"]=0
d["DISPOSITION:hearing_impaired"]=0
d["DISPOSITION:visual_impaired"]=0
d["DISPOSITION:clean_effects"]=0
d["DISPOSITION:attached_pic"]=0
:param string audio_file_path: the path of the audio file to analyze
:rtype: dict
:raises: TypeError: if ``audio_file_path`` is None
:raises: OSError: if the file at ``audio_file_path`` cannot be read
:raises: FFPROBEParsingError: if the call to ``ffprobe`` does not produce any output
:raises: FFPROBEPathError: if the path to the ``ffprobe`` executable cannot be called
:raises: FFPROBEUnsupportedFormatError: if the file has a format not supported by ``ffprobe``
"""
# test if we can read the file at audio_file_path
if audio_file_path is None:
self.log_exc(u"The audio file path is None", None, True, TypeError)
if not gf.file_can_be_read(audio_file_path):
self.log_exc(u"Input file '%s' cannot be read" % (audio_file_path),
None, True, OSError)
# call ffprobe
arguments = [self.rconf[RuntimeConfiguration.FFPROBE_PATH]]
arguments.extend(self.FFPROBE_PARAMETERS)
arguments.append(audio_file_path)
self.log([u"Calling with arguments '%s'", arguments])
try:
proc = subprocess.Popen(arguments,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdoutdata, stderrdata) = proc.communicate()
proc.stdout.close()
proc.stdin.close()
proc.stderr.close()
except OSError as exc:
self.log_exc(
u"Unable to call the '%s' ffprobe executable" %
(self.rconf[RuntimeConfiguration.FFPROBE_PATH]), exc, True,
FFPROBEPathError)
self.log(u"Call completed")
# check there is some output
if (stdoutdata is None) or (len(stderrdata) == 0):
self.log_exc(u"ffprobe produced no output", None, True,
FFPROBEParsingError)
# decode stdoutdata and stderrdata to Unicode string
try:
stdoutdata = gf.safe_unicode(stdoutdata)
stderrdata = gf.safe_unicode(stderrdata)
except UnicodeDecodeError as exc:
self.log_exc(u"Unable to decode ffprobe out/err", exc, True,
FFPROBEParsingError)
# dictionary for the results
results = {
self.STDOUT_CHANNELS: None,
self.STDOUT_CODEC_NAME: None,
self.STDOUT_DURATION: None,
self.STDOUT_SAMPLE_RATE: None
}
# scan the first audio stream the ffprobe stdout output
# TODO more robust parsing
# TODO deal with multiple audio streams
for line in stdoutdata.splitlines():
if line == self.STDOUT_END_STREAM:
self.log(u"Reached end of the stream")
break
elif len(line.split("=")) == 2:
key, value = line.split("=")
results[key] = value
self.log([u"Found property '%s'='%s'", key, value])
try:
self.log([
u"Duration found in stdout: '%s'",
results[self.STDOUT_DURATION]
])
results[self.STDOUT_DURATION] = TimeValue(
results[self.STDOUT_DURATION])
self.log(u"Valid duration")
except:
self.log_warn(u"Invalid duration")
results[self.STDOUT_DURATION] = None
# try scanning ffprobe stderr output
for line in stderrdata.splitlines():
match = self.STDERR_DURATION_REGEX.search(line)
if match is not None:
self.log([u"Found matching line '%s'", line])
results[self.STDOUT_DURATION] = gf.time_from_hhmmssmmm(
line)
self.log([
u"Extracted duration '%.3f'",
results[self.STDOUT_DURATION]
])
break
if results[self.STDOUT_DURATION] is None:
self.log_exc(
u"No duration found in stdout or stderr. Unsupported audio file format?",
None, True, FFPROBEUnsupportedFormatError)
# return dictionary
self.log(u"Returning dict")
return results
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 4:
return self.print_help()
text_format = gf.safe_unicode(self.actual_arguments[0])
if text_format == u"list":
text = gf.safe_unicode(self.actual_arguments[1])
elif text_format in TextFileFormat.ALLOWED_VALUES:
text = self.actual_arguments[1]
if not self.check_input_file(text):
return self.ERROR_EXIT_CODE
else:
return self.print_help()
l1_id_regex = self.has_option_with_value(u"--l1-id-regex")
l2_id_regex = self.has_option_with_value(u"--l2-id-regex")
l3_id_regex = self.has_option_with_value(u"--l3-id-regex")
id_regex = self.has_option_with_value(u"--id-regex")
class_regex = self.has_option_with_value(u"--class-regex")
sort = self.has_option_with_value(u"--sort")
parameters = {
gc.PPN_TASK_IS_TEXT_MUNPARSED_L1_ID_REGEX : l1_id_regex,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L2_ID_REGEX : l2_id_regex,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L3_ID_REGEX : l3_id_regex,
gc.PPN_JOB_IS_TEXT_UNPARSED_ID_REGEX : id_regex,
gc.PPN_JOB_IS_TEXT_UNPARSED_CLASS_REGEX : class_regex,
gc.PPN_JOB_IS_TEXT_UNPARSED_ID_SORT : sort
}
if (text_format == TextFileFormat.MUNPARSED) and ((l1_id_regex is None) or (l2_id_regex is None) or (l3_id_regex is None)):
self.print_error(u"You must specify --l1-id-regex and --l2-id-regex and --l3-id-regex for munparsed format")
return self.ERROR_EXIT_CODE
if (text_format == TextFileFormat.UNPARSED) and (id_regex is None) and (class_regex is None):
self.print_error(u"You must specify --id-regex and/or --class-regex for unparsed format")
return self.ERROR_EXIT_CODE
language = gf.safe_unicode(self.actual_arguments[2])
audio_file_path = self.actual_arguments[3]
if not self.check_input_file(audio_file_path):
return self.ERROR_EXIT_CODE
text_file = self.get_text_file(text_format, text, parameters)
if text_file is None:
self.print_error(u"Unable to build a TextFile from the given parameters")
return self.ERROR_EXIT_CODE
elif len(text_file) == 0:
self.print_error(u"No text fragments found")
return self.ERROR_EXIT_CODE
text_file.set_language(language)
self.print_info(u"Read input text with %d fragments" % (len(text_file)))
self.print_info(u"Reading audio...")
try:
audio_file_mfcc = AudioFileMFCC(audio_file_path, rconf=self.rconf, logger=self.logger)
except AudioFileConverterError:
self.print_error(u"Unable to call the ffmpeg executable '%s'" % (self.rconf[RuntimeConfiguration.FFMPEG_PATH]))
self.print_error(u"Make sure the path to ffmpeg is correct")
return self.ERROR_EXIT_CODE
except (AudioFileUnsupportedFormatError, AudioFileNotInitializedError):
self.print_error(u"Cannot read file '%s'" % (audio_file_path))
self.print_error(u"Check that its format is supported by ffmpeg")
return self.ERROR_EXIT_CODE
except Exception as exc:
self.print_error(u"An unexpected error occurred while reading the audio file:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
self.print_info(u"Reading audio... done")
self.print_info(u"Running VAD...")
audio_file_mfcc.run_vad()
self.print_info(u"Running VAD... done")
min_head = gf.safe_float(self.has_option_with_value(u"--min-head"), None)
max_head = gf.safe_float(self.has_option_with_value(u"--max-head"), None)
min_tail = gf.safe_float(self.has_option_with_value(u"--min-tail"), None)
max_tail = gf.safe_float(self.has_option_with_value(u"--max-tail"), None)
self.print_info(u"Detecting audio interval...")
start_detector = SD(audio_file_mfcc, text_file, rconf=self.rconf, logger=self.logger)
start, end = start_detector.detect_interval(min_head, max_head, min_tail, max_tail)
self.print_info(u"Detecting audio interval... done")
self.print_result(audio_file_mfcc.audio_length, start, end)
return self.NO_ERROR_EXIT_CODE
def run(self, arguments, show_help=True):
"""
Program entry point.
Please note that the first item in ``arguments`` is discarded,
as it is assumed to be the script/invocation name;
pass a "dumb" placeholder if you call this method with
an argument different that ``sys.argv``.
:param arguments: the list of arguments
:type arguments: list
:param show_help: if ``False``, do not show help on ``-h`` and ``--help``
:type show_help: bool
:rtype: int
"""
# convert arguments into Unicode strings
if self.use_sys:
# check that sys.stdin.encoding and sys.stdout.encoding are set to utf-8
if not gf.FROZEN:
if sys.stdin.encoding not in ["UTF-8", "UTF8"]:
self.print_warning(
u"The default input encoding is not UTF-8.")
self.print_warning(
u"You might want to set 'PYTHONIOENCODING=UTF-8' in your shell."
)
if sys.stdout.encoding not in ["UTF-8", "UTF8"]:
self.print_warning(
u"The default output encoding is not UTF-8.")
self.print_warning(
u"You might want to set 'PYTHONIOENCODING=UTF-8' in your shell."
)
# decode using sys.stdin.encoding
args = [gf.safe_unicode_stdin(arg) for arg in arguments]
else:
# decode using utf-8 (but you should pass Unicode strings as parameters anyway)
args = [gf.safe_unicode(arg) for arg in arguments]
if show_help:
if u"-h" in args:
return self.print_help(short=True)
if u"--help" in args:
return self.print_help(short=False)
if u"--version" in args:
return self.print_name_version()
# store formal arguments
self.formal_arguments_raw = arguments
self.formal_arguments = args
# to obtain the actual arguments,
# remove the first one and "special" switches
args = args[1:]
set_args = set(args)
# set verbosity, if requested
for flag in set([u"-v", u"--verbose"]) & set_args:
self.verbose = True
args.remove(flag)
for flag in set([u"-vv", u"--very-verbose"]) & set_args:
self.verbose = True
self.very_verbose = True
args.remove(flag)
# set RuntimeConfiguration string, if specified
for flag in [u"-r", u"--runtime-configuration"]:
rconf_string = self.has_option_with_value(flag,
actual_arguments=False)
if rconf_string is not None:
self.rconf = RuntimeConfiguration(rconf_string)
args.remove("%s=%s" % (flag, rconf_string))
# set log file path, if requested
log_path = None
for flag in [u"-l", u"--log"]:
log_path = self.has_option_with_value(flag, actual_arguments=False)
if log_path is not None:
args.remove("%s=%s" % (flag, log_path))
elif flag in set_args:
handler, log_path = gf.tmp_file(
suffix=u".log",
root=self.rconf[RuntimeConfiguration.TMP_PATH])
args.remove(flag)
if log_path is not None:
self.log_file_path = log_path
# if no actual arguments left, print help
if (len(args) < 1) and (show_help):
return self.print_help(short=True)
# store actual arguments
self.actual_arguments = args
# create logger
self.logger = Logger(tee=self.verbose,
tee_show_datetime=self.very_verbose)
self.log([u"Formal arguments: %s", self.formal_arguments])
self.log([u"Actual arguments: %s", self.actual_arguments])
self.log([u"Runtime configuration: '%s'", self.rconf.config_string()])
# perform command
exit_code = self.perform_command()
self.log([u"Execution completed with code %d", exit_code])
# output log if requested
if self.log_file_path is not None:
self.log([
u"User requested saving log to file '%s'", self.log_file_path
])
self.logger.write(self.log_file_path)
if self.use_sys:
self.print_info(u"Log written to file '%s'" %
self.log_file_path)
return self.exit(exit_code)
def _synthesize_multiple_c_extension(self, text_file, output_file_path, quit_after=None, backwards=False):
"""
Synthesize multiple text fragments, using the cew extension.
Return a tuple (anchors, total_time, num_chars).
:rtype: (bool, (list, :class:`~aeneas.timevalue.TimeValue`, int))
"""
self.log(u"Synthesizing using C extension...")
# convert parameters from Python values to C values
try:
c_quit_after = float(quit_after)
except TypeError:
c_quit_after = 0.0
c_backwards = 0
if backwards:
c_backwards = 1
self.log([u"output_file_path: %s", output_file_path])
self.log([u"c_quit_after: %.3f", c_quit_after])
self.log([u"c_backwards: %d", c_backwards])
self.log(u"Preparing u_text...")
u_text = []
fragments = text_file.fragments
for fragment in fragments:
f_lang = fragment.language
f_text = fragment.filtered_text
if f_lang is None:
f_lang = self.DEFAULT_LANGUAGE
f_voice_code = self._language_to_voice_code(f_lang)
if f_text is None:
f_text = u""
u_text.append((f_voice_code, f_text))
self.log(u"Preparing u_text... done")
# call C extension
sr = None
sf = None
intervals = None
if self.rconf[RuntimeConfiguration.CEW_SUBPROCESS_ENABLED]:
self.log(u"Using cewsubprocess to call aeneas.cew")
try:
self.log(u"Importing aeneas.cewsubprocess...")
from aeneas.cewsubprocess import CEWSubprocess
self.log(u"Importing aeneas.cewsubprocess... done")
self.log(u"Calling aeneas.cewsubprocess...")
cewsub = CEWSubprocess(rconf=self.rconf, logger=self.logger)
sr, sf, intervals = cewsub.synthesize_multiple(output_file_path, c_quit_after, c_backwards, u_text)
self.log(u"Calling aeneas.cewsubprocess... done")
except Exception as exc:
self.log_exc(u"An unexpected error occurred while running cewsubprocess", exc, False, None)
# NOTE not critical, try calling aeneas.cew directly
#return (False, None)
if sr is None:
self.log(u"Preparing c_text...")
if gf.PY2:
# Python 2 => pass byte strings
c_text = [(gf.safe_bytes(t[0]), gf.safe_bytes(t[1])) for t in u_text]
else:
# Python 3 => pass Unicode strings
c_text = [(gf.safe_unicode(t[0]), gf.safe_unicode(t[1])) for t in u_text]
self.log(u"Preparing c_text... done")
self.log(u"Calling aeneas.cew directly")
try:
self.log(u"Importing aeneas.cew...")
import aeneas.cew.cew
self.log(u"Importing aeneas.cew... done")
self.log(u"Calling aeneas.cew...")
sr, sf, intervals = aeneas.cew.cew.synthesize_multiple(
output_file_path,
c_quit_after,
c_backwards,
c_text
)
self.log(u"Calling aeneas.cew... done")
except Exception as exc:
self.log_exc(u"An unexpected error occurred while running cew", exc, False, None)
return (False, None)
self.log([u"sr: %d", sr])
self.log([u"sf: %d", sf])
# create output
anchors = []
current_time = TimeValue("0.000")
num_chars = 0
if backwards:
fragments = fragments[::-1]
for i in range(sf):
# get the correct fragment
fragment = fragments[i]
# store for later output
anchors.append([
TimeValue(intervals[i][0]),
fragment.identifier,
fragment.filtered_text
])
# increase the character counter
num_chars += fragment.characters
# update current_time
current_time = TimeValue(intervals[i][1])
# return output
# NOTE anchors do not make sense if backwards == True
self.log([u"Returning %d time anchors", len(anchors)])
self.log([u"Current time %.3f", current_time])
self.log([u"Synthesized %d characters", num_chars])
self.log(u"Synthesizing using C extension... done")
return (True, (anchors, current_time, num_chars))
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 4:
return self.print_help()
text_format = gf.safe_unicode(self.actual_arguments[0])
if text_format == u"list":
text = gf.safe_unicode(self.actual_arguments[1])
elif text_format in TextFileFormat.ALLOWED_VALUES:
text = self.actual_arguments[1]
if not self.check_input_file(text):
return self.ERROR_EXIT_CODE
else:
return self.print_help()
l1_id_regex = self.has_option_with_value(u"--l1-id-regex")
l2_id_regex = self.has_option_with_value(u"--l2-id-regex")
l3_id_regex = self.has_option_with_value(u"--l3-id-regex")
id_regex = self.has_option_with_value(u"--id-regex")
class_regex = self.has_option_with_value(u"--class-regex")
sort = self.has_option_with_value(u"--sort")
parameters = {
gc.PPN_TASK_IS_TEXT_MUNPARSED_L1_ID_REGEX: l1_id_regex,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L2_ID_REGEX: l2_id_regex,
gc.PPN_TASK_IS_TEXT_MUNPARSED_L3_ID_REGEX: l3_id_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_CLASS_REGEX: class_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_ID_REGEX: id_regex,
gc.PPN_TASK_IS_TEXT_UNPARSED_ID_SORT: sort,
}
if (text_format == TextFileFormat.MUNPARSED) and (
(l1_id_regex is None) or (l2_id_regex is None) or
(l3_id_regex is None)):
self.print_error(
u"You must specify --l1-id-regex and --l2-id-regex and --l3-id-regex for munparsed format"
)
return self.ERROR_EXIT_CODE
if (text_format == TextFileFormat.UNPARSED) and (
id_regex is None) and (class_regex is None):
self.print_error(
u"You must specify --id-regex and/or --class-regex for unparsed format"
)
return self.ERROR_EXIT_CODE
language = gf.safe_unicode(self.actual_arguments[2])
audio_file_path = self.actual_arguments[3]
if not self.check_input_file(audio_file_path):
return self.ERROR_EXIT_CODE
text_file = self.get_text_file(text_format, text, parameters)
if text_file is None:
self.print_error(
u"Unable to build a TextFile from the given parameters")
return self.ERROR_EXIT_CODE
elif len(text_file) == 0:
self.print_error(u"No text fragments found")
return self.ERROR_EXIT_CODE
text_file.set_language(language)
self.print_info(u"Read input text with %d fragments" %
(len(text_file)))
self.print_info(u"Reading audio...")
try:
audio_file_mfcc = AudioFileMFCC(audio_file_path,
rconf=self.rconf,
logger=self.logger)
except AudioFileConverterError:
self.print_error(u"Unable to call the ffmpeg executable '%s'" %
(self.rconf[RuntimeConfiguration.FFMPEG_PATH]))
self.print_error(u"Make sure the path to ffmpeg is correct")
return self.ERROR_EXIT_CODE
except (AudioFileUnsupportedFormatError, AudioFileNotInitializedError):
self.print_error(u"Cannot read file '%s'" % (audio_file_path))
self.print_error(u"Check that its format is supported by ffmpeg")
return self.ERROR_EXIT_CODE
except Exception as exc:
self.print_error(
u"An unexpected error occurred while reading the audio file:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
self.print_info(u"Reading audio... done")
self.print_info(u"Running VAD...")
audio_file_mfcc.run_vad()
self.print_info(u"Running VAD... done")
min_head = gf.safe_float(self.has_option_with_value(u"--min-head"),
None)
max_head = gf.safe_float(self.has_option_with_value(u"--max-head"),
None)
min_tail = gf.safe_float(self.has_option_with_value(u"--min-tail"),
None)
max_tail = gf.safe_float(self.has_option_with_value(u"--max-tail"),
None)
self.print_info(u"Detecting audio interval...")
start_detector = SD(audio_file_mfcc,
text_file,
rconf=self.rconf,
logger=self.logger)
start, end = start_detector.detect_interval(min_head, max_head,
min_tail, max_tail)
self.print_info(u"Detecting audio interval... done")
self.print_result(audio_file_mfcc.audio_length, start, end)
return self.NO_ERROR_EXIT_CODE