def test_check_job_configuration_05(self):
logger = Logger()
validator = Validator(logger=logger)
string = "malformed="
result = validator.check_job_configuration(string)
self.assertFalse(result.passed)
self.assertGreater(len(result.errors), 0)
def test_check_job_configuration_01(self):
logger = Logger()
validator = Validator(logger=logger)
string = u"dummy config string with bad encoding é".encode("latin-1")
result = validator.check_job_configuration(string)
self.assertFalse(result.passed)
self.assertGreater(len(result.errors), 0)
def test_check_container_xml_05(self):
logger = Logger()
validator = Validator(logger=logger)
container_path = get_abs_path("res/validator/job_xml_config_bad_4")
result = validator.check_container(container_path)
self.assertFalse(result.passed)
self.assertGreater(len(result.errors), 0)
def test_check_container_xml_07(self):
logger = Logger()
validator = Validator(logger=logger)
container_path = get_abs_path("res/validator/job_xml_config_not_root_nested")
result = validator.check_container(container_path)
self.assertTrue(result.passed)
self.assertEqual(len(result.errors), 0)
def test_check_task_configuration_23(self):
logger = Logger()
validator = Validator(logger=logger)
string = "task_language=it|is_text_type=plain|os_task_file_name=output.txt|os_task_file_format=smil"
result = validator.check_task_configuration(string)
self.assertFalse(result.passed)
self.assertGreater(len(result.errors), 0)
def test_check_job_configuration_08(self):
logger = Logger()
validator = Validator(logger=logger)
string = "job_language=it|os_job_file_name=output.zip"
result = validator.check_job_configuration(string)
self.assertFalse(result.passed)
self.assertGreater(len(result.errors), 0)
def test_check_task_configuration_19(self):
logger = Logger()
validator = Validator(logger=logger)
string = "task_language=it|is_text_type=unparsed|is_text_unparsed_class_regex=ra|is_text_unparsed_id_regex=f[0-9]*|os_task_file_name=output.txt|os_task_file_format=txt"
result = validator.check_task_configuration(string)
self.assertFalse(result.passed)
self.assertGreater(len(result.errors), 0)
def test_check_task_configuration_20(self):
logger = Logger()
validator = Validator(logger=logger)
string = "task_language=it|is_text_type=plain|os_task_file_name=output.txt|os_task_file_format=smil|os_task_file_smil_page_ref=page.xhtml|os_task_file_smil_audio_ref=../Audio/audio.mp3"
result = validator.check_task_configuration(string)
self.assertTrue(result.passed)
self.assertEqual(len(result.errors), 0)
def test_check_task_configuration_08(self):
logger = Logger()
validator = Validator(logger=logger)
string = "task_language=it|is_text_type=plain|missing=other"
result = validator.check_task_configuration(string)
self.assertFalse(result.passed)
self.assertGreater(len(result.errors), 0)
def test_check_task_configuration_16(self):
logger = Logger()
validator = Validator(logger=logger)
string = "task_language=it|is_text_type=unparsed|is_text_unparsed_id_regex=f[0-9]*|is_text_unparsed_id_sort=numeric|os_task_file_name=output.txt|os_task_file_format=txt"
result = validator.check_task_configuration(string)
self.assertTrue(result.passed)
self.assertEqual(len(result.errors), 0)
def test_check_task_configuration_06(self):
logger = Logger()
validator = Validator(logger=logger)
string = "not=relevant|config=string"
result = validator.check_task_configuration(string)
self.assertFalse(result.passed)
self.assertGreater(len(result.errors), 0)
def test_check_job_configuration_17(self):
logger = Logger()
validator = Validator(logger=logger)
string = "job_language=it|os_job_file_name=output.zip|os_job_file_container=zip|os_job_file_hierarchy_type=flat"
result = validator.check_job_configuration(string)
self.assertTrue(result.passed)
self.assertEqual(len(result.errors), 0)
def test_check_job_configuration_14(self):
logger = Logger()
validator = Validator(logger=logger)
string = "job_language=it|os_job_file_name=output.zip|os_job_file_container=zip|is_hierarchy_type=paged"
result = validator.check_job_configuration(string)
self.assertFalse(result.passed)
self.assertGreater(len(result.errors), 0)
def test_check_job_configuration_02(self):
logger = Logger()
validator = Validator(logger=logger)
string = "dummy config string with ~ reserved characters"
result = validator.check_job_configuration(string)
self.assertFalse(result.passed)
self.assertGreater(len(result.errors), 0)
def test_check_job_configuration_07(self):
logger = Logger()
validator = Validator(logger=logger)
string = "job_language=it|missing=other"
result = validator.check_job_configuration(string)
self.assertFalse(result.passed)
self.assertGreater(len(result.errors), 0)
def container(self, path, expected):
validator = Validator()
result = validator.check_container(gf.absolute_path(path, __file__))
self.assertEqual(result.passed, expected)
if expected:
self.assertEqual(len(result.errors), 0)
else:
self.assertGreater(len(result.errors), 0)
def tc(self, string, expected):
validator = Validator()
result = validator.check_task_configuration(string)
self.assertEqual(result.passed, expected)
if expected:
self.assertEqual(len(result.errors), 0)
else:
self.assertGreater(len(result.errors), 0)
def container(self, path, expected):
validator = Validator()
result = validator.check_container(gf.absolute_path(path, __file__))
self.assertEqual(result.passed, expected)
if expected:
self.assertEqual(len(result.errors), 0)
else:
self.assertGreater(len(result.errors), 0)
def tc(self, string, expected):
validator = Validator()
result = validator.check_configuration_string(string,
is_job=False,
external_name=False)
self.assertEqual(result.passed, expected)
if expected:
self.assertEqual(len(result.errors), 0)
else:
self.assertGreater(len(result.errors), 0)
def tc(self, string, expected):
validator = Validator()
result = validator.check_configuration_string(
string,
is_job=False,
external_name=False
)
self.assertEqual(result.passed, expected)
if expected:
self.assertEqual(len(result.errors), 0)
else:
self.assertGreater(len(result.errors), 0)
def main():
""" Entry point """
if len(sys.argv) < 3:
usage()
return
validator = Validator()
mode = sys.argv[1]
result = None
msg = ""
if mode == "config":
if sys.argv[2].endswith(".txt"):
try:
config_file = open(sys.argv[2], "r")
config_contents = config_file.read()
config_file.close()
result = validator.check_contents_txt_config_file(config_contents, True)
msg = "TXT configuration"
except:
print "[ERRO] Unable to read file %s" % sys.argv[2]
elif sys.argv[2].endswith(".xml"):
try:
config_file = open(sys.argv[2], "r")
config_contents = config_file.read()
config_file.close()
result = validator.check_contents_xml_config_file(config_contents)
msg = "XML configuration"
except:
print "[ERRO] Unable to read file %s" % sys.argv[2]
else:
usage()
return
elif mode == "container":
result = validator.check_container(sys.argv[2])
msg = "container"
elif mode == "job":
result = validator.check_job_configuration(sys.argv[2])
msg = "job configuration string"
elif mode == "task":
result = validator.check_task_configuration(sys.argv[2])
msg = "task configuration string"
elif mode == "wizard":
if len(sys.argv) < 4:
usage()
return
result = validator.check_container_from_wizard(sys.argv[2], sys.argv[3])
msg = "container + configuration string from wizard"
else:
usage()
return
if result.passed:
print "[INFO] Valid %s" % msg
if len(result.warnings) > 0:
for warning in result.warnings:
print "[WARN] " + warning
else:
print "[INFO] Invalid %s" % msg
for error in result.errors:
print "[ERRO] " + error
def load_job_from_container(self, container_path, config_string=None):
"""
Validate the given container, and, if it is well formed,
load the job from it.
If ``config_string`` is ``None``,
the container must contain a configuration file;
otherwise use the provided config string
(i.e., the wizard case).
Return ``True`` if the job has been loaded successfully,
``False`` otherwise.
:param container_path: the path to the input container
:type container_path: string (path)
:param config_string: the configuration string (from wizard)
:type config_string: string
:rtype: bool
"""
self._log("Loading job from container...")
# validate container
self._log("Validating container...")
validator = Validator(logger=self.logger)
if config_string is None:
validator_result = validator.check_container(container_path)
else:
validator_result = validator.check_container_from_wizard(
container_path, config_string)
if not validator_result.passed:
self._log("Validating container: failed")
self._log("Loading job from container: failed")
return False
self._log("Validating container: succeeded")
try:
# create working directory where the input container
# will be decompressed
self.working_directory = tempfile.mkdtemp(dir=gf.custom_tmp_dir())
self._log(
["Created working directory '%s'", self.working_directory])
# decompress
self._log("Decompressing input container...")
input_container = Container(container_path, logger=self.logger)
input_container.decompress(self.working_directory)
self._log("Decompressing input container... done")
# create job from the working directory
self._log("Creating job from working directory...")
working_container = Container(self.working_directory,
logger=self.logger)
analyzer = AnalyzeContainer(working_container, logger=self.logger)
if config_string is None:
self.job = analyzer.analyze()
else:
self.job = analyzer.analyze_from_wizard(config_string)
self._log("Creating job from working directory... done")
# set absolute path for text file and audio file
# for each task in the job
self._log("Setting absolute paths for tasks...")
for task in self.job.tasks:
task.text_file_path_absolute = gf.norm_join(
self.working_directory, task.text_file_path)
task.audio_file_path_absolute = gf.norm_join(
self.working_directory, task.audio_file_path)
self._log("Setting absolute paths for tasks... done")
# return
self._log("Loading job from container: succeeded")
return True
except:
# failure: clean and return
self.clean()
self._log("Loading job from container: failed")
return False
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 1:
return self.print_help()
if self.has_option([u"-e", u"--examples"]):
return self.print_examples(False)
if self.has_option(u"--examples-all"):
return self.print_examples(True)
if self.has_option([u"--list-parameters"]):
return self.print_parameters()
parameter = self.has_option_with_value(u"--list-values")
if parameter is not None:
return self.print_values(parameter)
elif self.has_option(u"--list-values"):
return self.print_values(u"?")
# NOTE list() is needed for Python3, where keys() is not a list!
demo = self.has_option(list(self.DEMOS.keys()))
demo_parameters = u""
download_from_youtube = self.has_option([u"-y", u"--youtube"])
largest_audio = self.has_option(u"--largest-audio")
keep_audio = self.has_option(u"--keep-audio")
output_html = self.has_option(u"--output-html")
validate = not self.has_option(u"--skip-validator")
print_faster_rate = self.has_option(u"--faster-rate")
print_rates = self.has_option(u"--rates")
print_zero = self.has_option(u"--zero")
if demo:
validate = False
for key in self.DEMOS:
if self.has_option(key):
demo_parameters = self.DEMOS[key]
audio_file_path = demo_parameters[u"audio"]
text_file_path = demo_parameters[u"text"]
config_string = demo_parameters[u"config"]
sync_map_file_path = demo_parameters[u"syncmap"]
# TODO allow injecting rconf options directly from DEMOS options field
if key == u"--example-cewsubprocess":
self.rconf[RuntimeConfiguration.CEW_SUBPROCESS_ENABLED] = True
elif key == u"--example-ctw-espeak":
self.rconf[RuntimeConfiguration.TTS] = "custom"
self.rconf[RuntimeConfiguration.TTS_PATH] = self.CTW_ESPEAK
elif key == u"--example-ctw-speect":
self.rconf[RuntimeConfiguration.TTS] = "custom"
self.rconf[RuntimeConfiguration.TTS_PATH] = self.CTW_SPEECT
elif key == u"--example-festival":
self.rconf[RuntimeConfiguration.TTS] = "festival"
self.rconf[RuntimeConfiguration.TTS_PATH] = "text2wave"
elif key == u"--example-mws":
self.rconf[RuntimeConfiguration.MFCC_WINDOW_LENGTH] = "1.500"
self.rconf[RuntimeConfiguration.MFCC_WINDOW_SHIFT] = "0.500"
elif key == u"--example-faster-rate":
print_faster_rate = True
elif key == u"--example-no-zero":
print_zero = True
elif key == u"--example-py":
self.rconf[RuntimeConfiguration.C_EXTENSIONS] = False
elif key == u"--example-rates":
print_rates = True
elif key == u"--example-youtube":
download_from_youtube = True
break
else:
if len(self.actual_arguments) < 4:
return self.print_help()
audio_file_path = self.actual_arguments[0]
text_file_path = self.actual_arguments[1]
config_string = self.actual_arguments[2]
sync_map_file_path = self.actual_arguments[3]
html_file_path = None
if output_html:
keep_audio = True
html_file_path = sync_map_file_path + u".html"
if download_from_youtube:
youtube_url = audio_file_path
if (not download_from_youtube) and (not self.check_input_file(audio_file_path)):
return self.ERROR_EXIT_CODE
if not self.check_input_file(text_file_path):
return self.ERROR_EXIT_CODE
if not self.check_output_file(sync_map_file_path):
return self.ERROR_EXIT_CODE
if (html_file_path is not None) and (not self.check_output_file(html_file_path)):
return self.ERROR_EXIT_CODE
self.check_c_extensions()
if demo:
msg = []
msg.append(u"Running example task with arguments:")
if download_from_youtube:
msg.append(u" YouTube URL: %s" % youtube_url)
else:
msg.append(u" Audio file: %s" % audio_file_path)
msg.append(u" Text file: %s" % text_file_path)
msg.append(u" Config string: %s" % config_string)
msg.append(u" Sync map file: %s" % sync_map_file_path)
if len(demo_parameters[u"options"]) > 0:
msg.append(u" Options: %s" % demo_parameters[u"options"])
self.print_info(u"\n".join(msg))
if validate:
self.print_info(u"Validating config string (specify --skip-validator to bypass)...")
validator = Validator(logger=self.logger)
result = validator.check_configuration_string(config_string, is_job=False, external_name=True)
if not result.passed:
self.print_error(u"The given config string is not valid:")
self.print_generic(result.pretty_print())
return self.ERROR_EXIT_CODE
self.print_info(u"Validating config string... done")
if download_from_youtube:
try:
self.print_info(u"Downloading audio from '%s' ..." % youtube_url)
downloader = Downloader(logger=self.logger)
audio_file_path = downloader.audio_from_youtube(
youtube_url,
download=True,
output_file_path=None,
largest_audio=largest_audio
)
self.print_info(u"Downloading audio from '%s' ... done" % youtube_url)
except ImportError:
self.print_no_pafy_error()
return self.ERROR_EXIT_CODE
except Exception as exc:
self.print_error(u"An unexpected error occurred while downloading audio from YouTube:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Creating task...")
task = Task(config_string, logger=self.logger)
task.audio_file_path_absolute = audio_file_path
task.text_file_path_absolute = text_file_path
task.sync_map_file_path_absolute = sync_map_file_path
self.print_info(u"Creating task... done")
except Exception as exc:
self.print_error(u"An unexpected error occurred while creating the task:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Executing task...")
executor = ExecuteTask(task=task, rconf=self.rconf, logger=self.logger)
executor.execute()
self.print_info(u"Executing task... done")
except Exception as exc:
self.print_error(u"An unexpected error occurred while executing the task:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Creating output sync map file...")
path = task.output_sync_map_file()
self.print_info(u"Creating output sync map file... done")
self.print_success(u"Created file '%s'" % path)
except Exception as exc:
self.print_error(u"An unexpected error occurred while writing the sync map file:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
if output_html:
try:
parameters = {}
parameters[gc.PPN_TASK_OS_FILE_FORMAT] = task.configuration["o_format"]
parameters[gc.PPN_TASK_OS_FILE_SMIL_AUDIO_REF] = task.configuration["o_smil_audio_ref"]
parameters[gc.PPN_TASK_OS_FILE_SMIL_PAGE_REF] = task.configuration["o_smil_page_ref"]
self.print_info(u"Creating output HTML file...")
task.sync_map.output_html_for_tuning(audio_file_path, html_file_path, parameters)
self.print_info(u"Creating output HTML file... done")
self.print_success(u"Created file '%s'" % html_file_path)
except Exception as exc:
self.print_error(u"An unexpected error occurred while writing the HTML file:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
if download_from_youtube:
if keep_audio:
self.print_info(u"Option --keep-audio set: keeping downloaded file '%s'" % audio_file_path)
else:
gf.delete_file(None, audio_file_path)
if print_zero:
zero_duration = [l for l in task.sync_map.fragments_tree.vleaves_not_empty if l.begin == l.end]
if len(zero_duration) > 0:
self.print_warning(u"Fragments with zero duration:")
for fragment in zero_duration:
self.print_generic(u" %s" % fragment)
if print_rates:
self.print_info(u"Fragments with rates:")
for fragment in task.sync_map.fragments_tree.vleaves_not_empty:
self.print_generic(u" %s (rate: %.3f chars/s)" % (fragment, fragment.rate))
if print_faster_rate:
max_rate = task.configuration["aba_rate_value"]
if max_rate is not None:
faster = [l for l in task.sync_map.fragments_tree.vleaves_not_empty if l.rate >= max_rate + Decimal("0.001")]
if len(faster) > 0:
self.print_warning(u"Fragments with rate greater than %.3f:" % max_rate)
for fragment in faster:
self.print_generic(u" %s (rate: %.3f chars/s)" % (fragment, fragment.rate))
return self.NO_ERROR_EXIT_CODE
def file_encoding(self, path, expected):
validator = Validator()
result = validator.check_file_encoding(get_abs_path(path))
self.assertEqual(result.passed, expected)
def string_well_formed(self, bstring, expected):
validator = Validator()
validator.check_raw_string(bstring, is_bstring=True)
self.assertEqual(validator.result.passed, expected)
def main():
""" Entry point """
if len(sys.argv) < 3:
usage()
return
validator = Validator()
mode = sys.argv[1]
result = None
msg = ""
if mode == "config":
if sys.argv[2].endswith(".txt"):
try:
config_file = open(sys.argv[2], "r")
config_contents = config_file.read()
config_file.close()
result = validator.check_contents_txt_config_file(
config_contents, True)
msg = "TXT configuration"
except:
print "[ERRO] Unable to read file %s" % sys.argv[2]
elif sys.argv[2].endswith(".xml"):
try:
config_file = open(sys.argv[2], "r")
config_contents = config_file.read()
config_file.close()
result = validator.check_contents_xml_config_file(
config_contents)
msg = "XML configuration"
except:
print "[ERRO] Unable to read file %s" % sys.argv[2]
else:
usage()
return
elif mode == "container":
result = validator.check_container(sys.argv[2])
msg = "container"
elif mode == "job":
result = validator.check_job_configuration(sys.argv[2])
msg = "job configuration string"
elif mode == "task":
result = validator.check_task_configuration(sys.argv[2])
msg = "task configuration string"
elif mode == "wizard":
if len(sys.argv) < 4:
usage()
return
result = validator.check_container_from_wizard(sys.argv[2],
sys.argv[3])
msg = "container + configuration string from wizard"
else:
usage()
return
if result.passed:
print "[INFO] Valid %s" % msg
if len(result.warnings) > 0:
for warning in result.warnings:
print "[WARN] " + warning
else:
print "[INFO] Invalid %s" % msg
for error in result.errors:
print "[ERRO] " + error
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 1:
return self.print_help()
if self.has_option([u"-e", u"--examples"]):
return self.print_examples(False)
if self.has_option(u"--examples-all"):
return self.print_examples(True)
if self.has_option([u"--list-parameters"]):
return self.print_parameters()
parameter = self.has_option_with_value(u"--list-values")
if parameter is not None:
return self.print_values(parameter)
elif self.has_option(u"--list-values"):
return self.print_values(u"?")
# NOTE list() is needed for Python3, where keys() is not a list!
demo = self.has_option(list(self.DEMOS.keys()))
demo_parameters = u""
download_from_youtube = self.has_option([u"-y", u"--youtube"])
largest_audio = self.has_option(u"--largest-audio")
keep_audio = self.has_option(u"--keep-audio")
output_html = self.has_option(u"--output-html")
validate = not self.has_option(u"--skip-validator")
print_faster_rate = self.has_option(u"--faster-rate")
print_rates = self.has_option(u"--rates")
print_zero = self.has_option(u"--zero")
if demo:
validate = False
for key in self.DEMOS:
if self.has_option(key):
demo_parameters = self.DEMOS[key]
audio_file_path = demo_parameters[u"audio"]
text_file_path = demo_parameters[u"text"]
config_string = demo_parameters[u"config"]
sync_map_file_path = demo_parameters[u"syncmap"]
# TODO allow injecting rconf options directly from DEMOS options field
if key == u"--example-cewsubprocess":
self.rconf[
RuntimeConfiguration.CEW_SUBPROCESS_ENABLED] = True
elif key == u"--example-ctw-espeak":
self.rconf[RuntimeConfiguration.TTS] = "custom"
self.rconf[
RuntimeConfiguration.TTS_PATH] = self.CTW_ESPEAK
elif key == u"--example-ctw-speect":
self.rconf[RuntimeConfiguration.TTS] = "custom"
self.rconf[
RuntimeConfiguration.TTS_PATH] = self.CTW_SPEECT
elif key == u"--example-festival":
self.rconf[RuntimeConfiguration.TTS] = "festival"
elif key == u"--example-mws":
self.rconf[
RuntimeConfiguration.MFCC_WINDOW_LENGTH] = "1.500"
self.rconf[
RuntimeConfiguration.MFCC_WINDOW_SHIFT] = "0.500"
elif key == u"--example-multilevel-tts":
self.rconf[RuntimeConfiguration.TTS_L1] = "festival"
self.rconf[RuntimeConfiguration.TTS_L2] = "festival"
self.rconf[RuntimeConfiguration.TTS_L3] = "espeak"
elif key == u"--example-words-festival-cache":
self.rconf[RuntimeConfiguration.TTS] = "festival"
self.rconf[RuntimeConfiguration.TTS_CACHE] = True
elif key == u"--example-faster-rate":
print_faster_rate = True
elif key == u"--example-no-zero":
print_zero = True
elif key == u"--example-py":
self.rconf[RuntimeConfiguration.C_EXTENSIONS] = False
elif key == u"--example-rates":
print_rates = True
elif key == u"--example-youtube":
download_from_youtube = True
break
else:
if len(self.actual_arguments) < 4:
return self.print_help()
audio_file_path = self.actual_arguments[0]
text_file_path = self.actual_arguments[1]
config_string = self.actual_arguments[2]
sync_map_file_path = self.actual_arguments[3]
html_file_path = None
if output_html:
keep_audio = True
html_file_path = sync_map_file_path + u".html"
if download_from_youtube:
youtube_url = audio_file_path
if (not download_from_youtube) and (
not self.check_input_file(audio_file_path)):
return self.ERROR_EXIT_CODE
if not self.check_input_file(text_file_path):
return self.ERROR_EXIT_CODE
if not self.check_output_file(sync_map_file_path):
return self.ERROR_EXIT_CODE
if (html_file_path
is not None) and (not self.check_output_file(html_file_path)):
return self.ERROR_EXIT_CODE
self.check_c_extensions()
if demo:
msg = []
msg.append(u"Running example task with arguments:")
if download_from_youtube:
msg.append(u" YouTube URL: %s" % youtube_url)
else:
msg.append(u" Audio file: %s" % audio_file_path)
msg.append(u" Text file: %s" % text_file_path)
msg.append(u" Config string: %s" % config_string)
msg.append(u" Sync map file: %s" % sync_map_file_path)
if len(demo_parameters[u"options"]) > 0:
msg.append(u" Options: %s" %
demo_parameters[u"options"])
self.print_info(u"\n".join(msg))
if validate:
self.print_info(
u"Validating config string (specify --skip-validator to bypass)..."
)
validator = Validator(logger=self.logger)
result = validator.check_configuration_string(config_string,
is_job=False,
external_name=True)
if not result.passed:
self.print_error(u"The given config string is not valid:")
self.print_generic(result.pretty_print())
return self.ERROR_EXIT_CODE
self.print_info(u"Validating config string... done")
if download_from_youtube:
try:
self.print_info(u"Downloading audio from '%s' ..." %
youtube_url)
downloader = Downloader(logger=self.logger)
audio_file_path = downloader.audio_from_youtube(
youtube_url,
download=True,
output_file_path=None,
largest_audio=largest_audio)
self.print_info(u"Downloading audio from '%s' ... done" %
youtube_url)
except ImportError:
self.print_no_pafy_error()
return self.ERROR_EXIT_CODE
except Exception as exc:
self.print_error(
u"An unexpected error occurred while downloading audio from YouTube:"
)
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
else:
audio_extension = gf.file_extension(audio_file_path)
if audio_extension.lower() not in AudioFile.FILE_EXTENSIONS:
self.print_warning(
u"Your audio file path has extension '%s', which is uncommon for an audio file."
% audio_extension)
self.print_warning(
u"Attempting at executing your Task anyway.")
self.print_warning(
u"If it fails, you might have swapped the first two arguments."
)
self.print_warning(
u"The audio file path should be the first argument, the text file path the second."
)
try:
self.print_info(u"Creating task...")
task = Task(config_string, logger=self.logger)
task.audio_file_path_absolute = audio_file_path
task.text_file_path_absolute = text_file_path
task.sync_map_file_path_absolute = sync_map_file_path
self.print_info(u"Creating task... done")
except Exception as exc:
self.print_error(
u"An unexpected error occurred while creating the task:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Executing task...")
executor = ExecuteTask(task=task,
rconf=self.rconf,
logger=self.logger)
executor.execute()
self.print_info(u"Executing task... done")
except Exception as exc:
self.print_error(
u"An unexpected error occurred while executing the task:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Creating output sync map file...")
path = task.output_sync_map_file()
self.print_info(u"Creating output sync map file... done")
self.print_success(u"Created file '%s'" % path)
except Exception as exc:
self.print_error(
u"An unexpected error occurred while writing the sync map file:"
)
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
if output_html:
try:
parameters = {}
parameters[gc.PPN_TASK_OS_FILE_FORMAT] = task.configuration[
"o_format"]
parameters[
gc.PPN_TASK_OS_FILE_EAF_AUDIO_REF] = task.configuration[
"o_eaf_audio_ref"]
parameters[
gc.PPN_TASK_OS_FILE_SMIL_AUDIO_REF] = task.configuration[
"o_smil_audio_ref"]
parameters[
gc.PPN_TASK_OS_FILE_SMIL_PAGE_REF] = task.configuration[
"o_smil_page_ref"]
self.print_info(u"Creating output HTML file...")
task.sync_map.output_html_for_tuning(audio_file_path,
html_file_path,
parameters)
self.print_info(u"Creating output HTML file... done")
self.print_success(u"Created file '%s'" % html_file_path)
except Exception as exc:
self.print_error(
u"An unexpected error occurred while writing the HTML file:"
)
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
if download_from_youtube:
if keep_audio:
self.print_info(
u"Option --keep-audio set: keeping downloaded file '%s'" %
audio_file_path)
else:
gf.delete_file(None, audio_file_path)
if print_zero:
zero_duration = [
l for l in task.sync_map.fragments_tree.vleaves_not_empty
if l.begin == l.end
]
if len(zero_duration) > 0:
self.print_warning(u"Fragments with zero duration:")
for fragment in zero_duration:
self.print_generic(u" %s" % fragment)
if print_rates:
self.print_info(u"Fragments with rates:")
for fragment in task.sync_map.fragments_tree.vleaves_not_empty:
self.print_generic(u" %s (rate: %.3f chars/s)" %
(fragment, fragment.rate))
if print_faster_rate:
max_rate = task.configuration["aba_rate_value"]
if max_rate is not None:
faster = [
l for l in task.sync_map.fragments_tree.vleaves_not_empty
if l.rate >= max_rate + Decimal("0.001")
]
if len(faster) > 0:
self.print_warning(
u"Fragments with rate greater than %.3f:" % max_rate)
for fragment in faster:
self.print_generic(u" %s (rate: %.3f chars/s)" %
(fragment, fragment.rate))
return self.NO_ERROR_EXIT_CODE
def test_check_reserved_characters(self):
validator = Validator()
self.assertFalse(validator._check_reserved_characters("string with ~ reserved char"))
self.assertTrue(validator._check_reserved_characters("string without reserved char"))
def test_check_file_encoding_false_01(self):
logger = Logger()
validator = Validator(logger=logger)
input_file_path = get_abs_path("res/validator/encoding_iso8859.txt")
result = validator.check_file_encoding(input_file_path)
self.assertFalse(result.passed)
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 2:
return self.print_help()
mode = self.actual_arguments[0]
validator = Validator(rconf=self.rconf, logger=self.logger)
if mode == u"config":
config_file_path = self.actual_arguments[1]
config_txt = None
if config_file_path.lower().endswith(u".txt"):
config_txt = True
elif config_file_path.lower().endswith(u".xml"):
config_txt = False
else:
return self.print_help()
if not self.check_input_file(config_file_path):
return self.ERROR_EXIT_CODE
contents = gf.read_file_bytes(config_file_path)
if contents is None:
return self.ERROR_EXIT_CODE
if config_txt:
result = validator.check_config_txt(contents)
msg = u"TXT configuration"
else:
result = validator.check_config_xml(contents)
msg = "XML configuration"
elif mode == u"container":
container_path = self.actual_arguments[1]
result = validator.check_container(container_path)
msg = "container"
elif mode == u"job":
config_string = self.actual_arguments[1]
result = validator.check_configuration_string(config_string, is_job=True)
msg = u"job configuration string"
elif mode == u"task":
config_string = self.actual_arguments[1]
result = validator.check_configuration_string(config_string, is_job=False, external_name=True)
msg = u"task configuration string"
elif mode == u"wizard":
if (len(self.actual_arguments) < 3) or (self.actual_arguments[2].startswith(u"-")):
return self.print_help()
config_string = self.actual_arguments[1]
container_path = self.actual_arguments[2]
if not self.check_input_file(container_path):
return self.ERROR_EXIT_CODE
result = validator.check_container(container_path, config_string=config_string)
msg = "container with configuration string from wizard"
else:
return self.print_help()
if result.passed:
self.print_success(u"Valid %s" % msg)
for warning in result.warnings:
self.print_warning(u"%s" % warning)
return self.NO_ERROR_EXIT_CODE
else:
self.print_error(u"Invalid %s" % msg)
for error in result.errors:
self.print_error(u"%s" % error)
return self.ERROR_EXIT_CODE
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 2:
return self.print_help()
container_path = self.actual_arguments[0]
output_directory_path = self.actual_arguments[1]
config_string = None
if (len(self.actual_arguments)) > 2 and (not self.actual_arguments[2].startswith(u"-")):
config_string = self.actual_arguments[2]
validate = not self.has_option(u"--skip-validator")
if self.has_option(u"--cewsubprocess"):
self.rconf[RuntimeConfiguration.CEW_SUBPROCESS_ENABLED] = True
if not self.check_input_file_or_directory(container_path):
return self.ERROR_EXIT_CODE
if not self.check_output_directory(output_directory_path):
return self.ERROR_EXIT_CODE
if validate:
try:
self.print_info(u"Validating the container (specify --skip-validator to bypass)...")
validator = Validator(rconf=self.rconf, logger=self.logger)
result = validator.check_container(container_path, config_string=config_string)
if not result.passed:
self.print_error(u"The given container is not valid:")
self.print_error(result.pretty_print())
return self.ERROR_EXIT_CODE
self.print_info(u"Validating the container... done")
except Exception as exc:
self.print_error(u"An unexpected error occurred while validating the container:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Loading job from container...")
executor = ExecuteJob(rconf=self.rconf, logger=self.logger)
executor.load_job_from_container(container_path, config_string)
self.print_info(u"Loading job from container... done")
except Exception as exc:
self.print_error(u"An unexpected error occurred while loading the job:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Executing...")
executor.execute()
self.print_info(u"Executing... done")
except Exception as exc:
self.print_error(u"An unexpected error occurred while executing the job:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Creating output container...")
path = executor.write_output_container(output_directory_path)
self.print_info(u"Creating output container... done")
self.print_success(u"Created output file '%s'" % path)
executor.clean(True)
return self.NO_ERROR_EXIT_CODE
except Exception as exc:
self.print_error(u"An unexpected error occurred while writing the output container:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
def load_job_from_container(self, container_path, config_string=None):
"""
Validate the given container, and, if it is well formed,
load the job from it.
If ``config_string`` is ``None``,
the container must contain a configuration file;
otherwise use the provided config string
(i.e., the wizard case).
Return ``True`` if the job has been loaded successfully,
``False`` otherwise.
:param container_path: the path to the input container
:type container_path: string (path)
:param config_string: the configuration string (from wizard)
:type config_string: string
:rtype: bool
"""
self._log("Loading job from container...")
# validate container
self._log("Validating container...")
validator = Validator(logger=self.logger)
if config_string == None:
validator_result = validator.check_container(container_path)
else:
validator_result = validator.check_container_from_wizard(
container_path,
config_string
)
if not validator_result.passed:
self._log("Validating container: failed")
self._log("Loading job from container: failed")
return False
self._log("Validating container: succeeded")
try:
# create working directory where the input container
# will be decompressed
self.working_directory = tempfile.mkdtemp(dir=gf.custom_tmp_dir())
self._log("Created working directory '%s'" % self.working_directory)
# decompress
self._log("Decompressing input container...")
input_container = Container(container_path, logger=self.logger)
input_container.decompress(self.working_directory)
self._log("Decompressing input container... done")
# create job from the working directory
self._log("Creating job from working directory...")
working_container = Container(
self.working_directory,
logger=self.logger
)
analyzer = AnalyzeContainer(working_container, logger=self.logger)
if config_string == None:
self.job = analyzer.analyze()
else:
self.job = analyzer.analyze_from_wizard(config_string)
self._log("Creating job from working directory... done")
# set absolute path for text file and audio file
# for each task in the job
self._log("Setting absolute paths for tasks...")
for task in self.job.tasks:
task.text_file_path_absolute = gf.norm_join(
self.working_directory,
task.text_file_path
)
task.audio_file_path_absolute = gf.norm_join(
self.working_directory,
task.audio_file_path
)
self._log("Setting absolute paths for tasks... done")
# return
self._log("Loading job from container: succeeded")
return True
except:
# failure: clean and return
self.clean()
self._log("Loading job from container: failed")
return False
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 2:
return self.print_help()
container_path = self.actual_arguments[0]
output_directory_path = self.actual_arguments[1]
config_string = None
if (len(self.actual_arguments)) > 2 and (
not self.actual_arguments[2].startswith(u"-")):
config_string = self.actual_arguments[2]
validate = not self.has_option(u"--skip-validator")
if self.has_option(u"--cewsubprocess"):
self.rconf[RuntimeConfiguration.CEW_SUBPROCESS_ENABLED] = True
if not self.check_input_file_or_directory(container_path):
return self.ERROR_EXIT_CODE
if not self.check_output_directory(output_directory_path):
return self.ERROR_EXIT_CODE
if validate:
try:
self.print_info(
u"Validating the container (specify --skip-validator to bypass)..."
)
validator = Validator(rconf=self.rconf, logger=self.logger)
result = validator.check_container(container_path,
config_string=config_string)
if not result.passed:
self.print_error(u"The given container is not valid:")
self.print_error(result.pretty_print())
return self.ERROR_EXIT_CODE
self.print_info(u"Validating the container... done")
except Exception as exc:
self.print_error(
u"An unexpected error occurred while validating the container:"
)
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Loading job from container...")
executor = ExecuteJob(rconf=self.rconf, logger=self.logger)
executor.load_job_from_container(container_path, config_string)
self.print_info(u"Loading job from container... done")
except Exception as exc:
self.print_error(
u"An unexpected error occurred while loading the job:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Executing...")
executor.execute()
self.print_info(u"Executing... done")
except Exception as exc:
self.print_error(
u"An unexpected error occurred while executing the job:")
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
try:
self.print_info(u"Creating output container...")
path = executor.write_output_container(output_directory_path)
self.print_info(u"Creating output container... done")
self.print_success(u"Created output file '%s'" % path)
executor.clean(True)
return self.NO_ERROR_EXIT_CODE
except Exception as exc:
self.print_error(
u"An unexpected error occurred while writing the output container:"
)
self.print_error(u"%s" % exc)
return self.ERROR_EXIT_CODE
def file_encoding(self, path, expected):
validator = Validator()
result = validator.check_file_encoding(gf.absolute_path(path, __file__))
self.assertEqual(result.passed, expected)
def string_well_formed(self, bstring, expected):
validator = Validator()
validator.check_raw_string(bstring, is_bstring=True)
self.assertEqual(validator.result.passed, expected)
def file_encoding(self, path, expected):
validator = Validator()
result = validator.check_file_encoding(gf.absolute_path(
path, __file__))
self.assertEqual(result.passed, expected)
def test_check_file_encoding_true_02(self):
logger = Logger()
validator = Validator(logger=logger)
input_file_path = get_abs_path("res/validator/encoding_utf8_bom.xhtml")
result = validator.check_file_encoding(input_file_path)
self.assertTrue(result.passed)
def perform_command(self):
"""
Perform command and return the appropriate exit code.
:rtype: int
"""
if len(self.actual_arguments) < 2:
return self.print_help()
mode = self.actual_arguments[0]
validator = Validator(rconf=self.rconf, logger=self.logger)
if mode == u"config":
config_file_path = self.actual_arguments[1]
config_txt = None
if config_file_path.lower().endswith(u".txt"):
config_txt = True
elif config_file_path.lower().endswith(u".xml"):
config_txt = False
else:
return self.print_help()
if not self.check_input_file(config_file_path):
return self.ERROR_EXIT_CODE
contents = gf.read_file_bytes(config_file_path)
if contents is None:
return self.ERROR_EXIT_CODE
if config_txt:
result = validator.check_config_txt(contents)
msg = u"TXT configuration"
else:
result = validator.check_config_xml(contents)
msg = "XML configuration"
elif mode == u"container":
container_path = self.actual_arguments[1]
result = validator.check_container(container_path)
msg = "container"
elif mode == u"job":
config_string = self.actual_arguments[1]
result = validator.check_configuration_string(config_string,
is_job=True)
msg = u"job configuration string"
elif mode == u"task":
config_string = self.actual_arguments[1]
result = validator.check_configuration_string(config_string,
is_job=False,
external_name=True)
msg = u"task configuration string"
elif mode == u"wizard":
if (len(self.actual_arguments) <
3) or (self.actual_arguments[2].startswith(u"-")):
return self.print_help()
config_string = self.actual_arguments[1]
container_path = self.actual_arguments[2]
if not self.check_input_file(container_path):
return self.ERROR_EXIT_CODE
result = validator.check_container(container_path,
config_string=config_string)
msg = "container with configuration string from wizard"
else:
return self.print_help()
if result.passed:
self.print_success(u"Valid %s" % msg)
for warning in result.warnings:
self.print_warning(u"%s" % warning)
return self.NO_ERROR_EXIT_CODE
else:
self.print_error(u"Invalid %s" % msg)
for error in result.errors:
self.print_error(u"%s" % error)
return self.ERROR_EXIT_CODE
def test_check_string_encoding(self):
validator = Validator()
self.assertFalse(validator._check_string_encoding(u"abcdé".encode("latin-1")))
self.assertTrue(validator._check_string_encoding(u"abcdé".encode("utf-8")))
