def test_check_job_configuration_05(self): logger = Logger() validator = Validator(logger=logger) string = "malformed=" result = validator.check_job_configuration(string) self.assertFalse(result.passed) self.assertGreater(len(result.errors), 0)
def test_check_job_configuration_01(self): logger = Logger() validator = Validator(logger=logger) string = u"dummy config string with bad encoding é".encode("latin-1") result = validator.check_job_configuration(string) self.assertFalse(result.passed) self.assertGreater(len(result.errors), 0)
def test_check_container_xml_05(self): logger = Logger() validator = Validator(logger=logger) container_path = get_abs_path("res/validator/job_xml_config_bad_4") result = validator.check_container(container_path) self.assertFalse(result.passed) self.assertGreater(len(result.errors), 0)
def test_check_container_xml_07(self): logger = Logger() validator = Validator(logger=logger) container_path = get_abs_path("res/validator/job_xml_config_not_root_nested") result = validator.check_container(container_path) self.assertTrue(result.passed) self.assertEqual(len(result.errors), 0)
def test_check_task_configuration_23(self): logger = Logger() validator = Validator(logger=logger) string = "task_language=it|is_text_type=plain|os_task_file_name=output.txt|os_task_file_format=smil" result = validator.check_task_configuration(string) self.assertFalse(result.passed) self.assertGreater(len(result.errors), 0)
def test_check_job_configuration_08(self): logger = Logger() validator = Validator(logger=logger) string = "job_language=it|os_job_file_name=output.zip" result = validator.check_job_configuration(string) self.assertFalse(result.passed) self.assertGreater(len(result.errors), 0)
def test_check_task_configuration_19(self): logger = Logger() validator = Validator(logger=logger) string = "task_language=it|is_text_type=unparsed|is_text_unparsed_class_regex=ra|is_text_unparsed_id_regex=f[0-9]*|os_task_file_name=output.txt|os_task_file_format=txt" result = validator.check_task_configuration(string) self.assertFalse(result.passed) self.assertGreater(len(result.errors), 0)
def test_check_task_configuration_20(self): logger = Logger() validator = Validator(logger=logger) string = "task_language=it|is_text_type=plain|os_task_file_name=output.txt|os_task_file_format=smil|os_task_file_smil_page_ref=page.xhtml|os_task_file_smil_audio_ref=../Audio/audio.mp3" result = validator.check_task_configuration(string) self.assertTrue(result.passed) self.assertEqual(len(result.errors), 0)
def test_check_task_configuration_08(self): logger = Logger() validator = Validator(logger=logger) string = "task_language=it|is_text_type=plain|missing=other" result = validator.check_task_configuration(string) self.assertFalse(result.passed) self.assertGreater(len(result.errors), 0)
def test_check_task_configuration_16(self): logger = Logger() validator = Validator(logger=logger) string = "task_language=it|is_text_type=unparsed|is_text_unparsed_id_regex=f[0-9]*|is_text_unparsed_id_sort=numeric|os_task_file_name=output.txt|os_task_file_format=txt" result = validator.check_task_configuration(string) self.assertTrue(result.passed) self.assertEqual(len(result.errors), 0)
def test_check_task_configuration_06(self): logger = Logger() validator = Validator(logger=logger) string = "not=relevant|config=string" result = validator.check_task_configuration(string) self.assertFalse(result.passed) self.assertGreater(len(result.errors), 0)
def test_check_job_configuration_17(self): logger = Logger() validator = Validator(logger=logger) string = "job_language=it|os_job_file_name=output.zip|os_job_file_container=zip|os_job_file_hierarchy_type=flat" result = validator.check_job_configuration(string) self.assertTrue(result.passed) self.assertEqual(len(result.errors), 0)
def test_check_job_configuration_14(self): logger = Logger() validator = Validator(logger=logger) string = "job_language=it|os_job_file_name=output.zip|os_job_file_container=zip|is_hierarchy_type=paged" result = validator.check_job_configuration(string) self.assertFalse(result.passed) self.assertGreater(len(result.errors), 0)
def test_check_job_configuration_02(self): logger = Logger() validator = Validator(logger=logger) string = "dummy config string with ~ reserved characters" result = validator.check_job_configuration(string) self.assertFalse(result.passed) self.assertGreater(len(result.errors), 0)
def test_check_job_configuration_07(self): logger = Logger() validator = Validator(logger=logger) string = "job_language=it|missing=other" result = validator.check_job_configuration(string) self.assertFalse(result.passed) self.assertGreater(len(result.errors), 0)
def container(self, path, expected): validator = Validator() result = validator.check_container(gf.absolute_path(path, __file__)) self.assertEqual(result.passed, expected) if expected: self.assertEqual(len(result.errors), 0) else: self.assertGreater(len(result.errors), 0)
def tc(self, string, expected): validator = Validator() result = validator.check_task_configuration(string) self.assertEqual(result.passed, expected) if expected: self.assertEqual(len(result.errors), 0) else: self.assertGreater(len(result.errors), 0)
def tc(self, string, expected): validator = Validator() result = validator.check_configuration_string(string, is_job=False, external_name=False) self.assertEqual(result.passed, expected) if expected: self.assertEqual(len(result.errors), 0) else: self.assertGreater(len(result.errors), 0)
def tc(self, string, expected): validator = Validator() result = validator.check_configuration_string( string, is_job=False, external_name=False ) self.assertEqual(result.passed, expected) if expected: self.assertEqual(len(result.errors), 0) else: self.assertGreater(len(result.errors), 0)
def main(): """ Entry point """ if len(sys.argv) < 3: usage() return validator = Validator() mode = sys.argv[1] result = None msg = "" if mode == "config": if sys.argv[2].endswith(".txt"): try: config_file = open(sys.argv[2], "r") config_contents = config_file.read() config_file.close() result = validator.check_contents_txt_config_file(config_contents, True) msg = "TXT configuration" except: print "[ERRO] Unable to read file %s" % sys.argv[2] elif sys.argv[2].endswith(".xml"): try: config_file = open(sys.argv[2], "r") config_contents = config_file.read() config_file.close() result = validator.check_contents_xml_config_file(config_contents) msg = "XML configuration" except: print "[ERRO] Unable to read file %s" % sys.argv[2] else: usage() return elif mode == "container": result = validator.check_container(sys.argv[2]) msg = "container" elif mode == "job": result = validator.check_job_configuration(sys.argv[2]) msg = "job configuration string" elif mode == "task": result = validator.check_task_configuration(sys.argv[2]) msg = "task configuration string" elif mode == "wizard": if len(sys.argv) < 4: usage() return result = validator.check_container_from_wizard(sys.argv[2], sys.argv[3]) msg = "container + configuration string from wizard" else: usage() return if result.passed: print "[INFO] Valid %s" % msg if len(result.warnings) > 0: for warning in result.warnings: print "[WARN] " + warning else: print "[INFO] Invalid %s" % msg for error in result.errors: print "[ERRO] " + error
def load_job_from_container(self, container_path, config_string=None): """ Validate the given container, and, if it is well formed, load the job from it. If ``config_string`` is ``None``, the container must contain a configuration file; otherwise use the provided config string (i.e., the wizard case). Return ``True`` if the job has been loaded successfully, ``False`` otherwise. :param container_path: the path to the input container :type container_path: string (path) :param config_string: the configuration string (from wizard) :type config_string: string :rtype: bool """ self._log("Loading job from container...") # validate container self._log("Validating container...") validator = Validator(logger=self.logger) if config_string is None: validator_result = validator.check_container(container_path) else: validator_result = validator.check_container_from_wizard( container_path, config_string) if not validator_result.passed: self._log("Validating container: failed") self._log("Loading job from container: failed") return False self._log("Validating container: succeeded") try: # create working directory where the input container # will be decompressed self.working_directory = tempfile.mkdtemp(dir=gf.custom_tmp_dir()) self._log( ["Created working directory '%s'", self.working_directory]) # decompress self._log("Decompressing input container...") input_container = Container(container_path, logger=self.logger) input_container.decompress(self.working_directory) self._log("Decompressing input container... done") # create job from the working directory self._log("Creating job from working directory...") working_container = Container(self.working_directory, logger=self.logger) analyzer = AnalyzeContainer(working_container, logger=self.logger) if config_string is None: self.job = analyzer.analyze() else: self.job = analyzer.analyze_from_wizard(config_string) self._log("Creating job from working directory... done") # set absolute path for text file and audio file # for each task in the job self._log("Setting absolute paths for tasks...") for task in self.job.tasks: task.text_file_path_absolute = gf.norm_join( self.working_directory, task.text_file_path) task.audio_file_path_absolute = gf.norm_join( self.working_directory, task.audio_file_path) self._log("Setting absolute paths for tasks... done") # return self._log("Loading job from container: succeeded") return True except: # failure: clean and return self.clean() self._log("Loading job from container: failed") return False
def perform_command(self): """ Perform command and return the appropriate exit code. :rtype: int """ if len(self.actual_arguments) < 1: return self.print_help() if self.has_option([u"-e", u"--examples"]): return self.print_examples(False) if self.has_option(u"--examples-all"): return self.print_examples(True) if self.has_option([u"--list-parameters"]): return self.print_parameters() parameter = self.has_option_with_value(u"--list-values") if parameter is not None: return self.print_values(parameter) elif self.has_option(u"--list-values"): return self.print_values(u"?") # NOTE list() is needed for Python3, where keys() is not a list! demo = self.has_option(list(self.DEMOS.keys())) demo_parameters = u"" download_from_youtube = self.has_option([u"-y", u"--youtube"]) largest_audio = self.has_option(u"--largest-audio") keep_audio = self.has_option(u"--keep-audio") output_html = self.has_option(u"--output-html") validate = not self.has_option(u"--skip-validator") print_faster_rate = self.has_option(u"--faster-rate") print_rates = self.has_option(u"--rates") print_zero = self.has_option(u"--zero") if demo: validate = False for key in self.DEMOS: if self.has_option(key): demo_parameters = self.DEMOS[key] audio_file_path = demo_parameters[u"audio"] text_file_path = demo_parameters[u"text"] config_string = demo_parameters[u"config"] sync_map_file_path = demo_parameters[u"syncmap"] # TODO allow injecting rconf options directly from DEMOS options field if key == u"--example-cewsubprocess": self.rconf[RuntimeConfiguration.CEW_SUBPROCESS_ENABLED] = True elif key == u"--example-ctw-espeak": self.rconf[RuntimeConfiguration.TTS] = "custom" self.rconf[RuntimeConfiguration.TTS_PATH] = self.CTW_ESPEAK elif key == u"--example-ctw-speect": self.rconf[RuntimeConfiguration.TTS] = "custom" self.rconf[RuntimeConfiguration.TTS_PATH] = self.CTW_SPEECT elif key == u"--example-festival": self.rconf[RuntimeConfiguration.TTS] = "festival" self.rconf[RuntimeConfiguration.TTS_PATH] = "text2wave" elif key == u"--example-mws": self.rconf[RuntimeConfiguration.MFCC_WINDOW_LENGTH] = "1.500" self.rconf[RuntimeConfiguration.MFCC_WINDOW_SHIFT] = "0.500" elif key == u"--example-faster-rate": print_faster_rate = True elif key == u"--example-no-zero": print_zero = True elif key == u"--example-py": self.rconf[RuntimeConfiguration.C_EXTENSIONS] = False elif key == u"--example-rates": print_rates = True elif key == u"--example-youtube": download_from_youtube = True break else: if len(self.actual_arguments) < 4: return self.print_help() audio_file_path = self.actual_arguments[0] text_file_path = self.actual_arguments[1] config_string = self.actual_arguments[2] sync_map_file_path = self.actual_arguments[3] html_file_path = None if output_html: keep_audio = True html_file_path = sync_map_file_path + u".html" if download_from_youtube: youtube_url = audio_file_path if (not download_from_youtube) and (not self.check_input_file(audio_file_path)): return self.ERROR_EXIT_CODE if not self.check_input_file(text_file_path): return self.ERROR_EXIT_CODE if not self.check_output_file(sync_map_file_path): return self.ERROR_EXIT_CODE if (html_file_path is not None) and (not self.check_output_file(html_file_path)): return self.ERROR_EXIT_CODE self.check_c_extensions() if demo: msg = [] msg.append(u"Running example task with arguments:") if download_from_youtube: msg.append(u" YouTube URL: %s" % youtube_url) else: msg.append(u" Audio file: %s" % audio_file_path) msg.append(u" Text file: %s" % text_file_path) msg.append(u" Config string: %s" % config_string) msg.append(u" Sync map file: %s" % sync_map_file_path) if len(demo_parameters[u"options"]) > 0: msg.append(u" Options: %s" % demo_parameters[u"options"]) self.print_info(u"\n".join(msg)) if validate: self.print_info(u"Validating config string (specify --skip-validator to bypass)...") validator = Validator(logger=self.logger) result = validator.check_configuration_string(config_string, is_job=False, external_name=True) if not result.passed: self.print_error(u"The given config string is not valid:") self.print_generic(result.pretty_print()) return self.ERROR_EXIT_CODE self.print_info(u"Validating config string... done") if download_from_youtube: try: self.print_info(u"Downloading audio from '%s' ..." % youtube_url) downloader = Downloader(logger=self.logger) audio_file_path = downloader.audio_from_youtube( youtube_url, download=True, output_file_path=None, largest_audio=largest_audio ) self.print_info(u"Downloading audio from '%s' ... done" % youtube_url) except ImportError: self.print_no_pafy_error() return self.ERROR_EXIT_CODE except Exception as exc: self.print_error(u"An unexpected error occurred while downloading audio from YouTube:") self.print_error(u"%s" % exc) return self.ERROR_EXIT_CODE try: self.print_info(u"Creating task...") task = Task(config_string, logger=self.logger) task.audio_file_path_absolute = audio_file_path task.text_file_path_absolute = text_file_path task.sync_map_file_path_absolute = sync_map_file_path self.print_info(u"Creating task... done") except Exception as exc: self.print_error(u"An unexpected error occurred while creating the task:") self.print_error(u"%s" % exc) return self.ERROR_EXIT_CODE try: self.print_info(u"Executing task...") executor = ExecuteTask(task=task, rconf=self.rconf, logger=self.logger) executor.execute() self.print_info(u"Executing task... done") except Exception as exc: self.print_error(u"An unexpected error occurred while executing the task:") self.print_error(u"%s" % exc) return self.ERROR_EXIT_CODE try: self.print_info(u"Creating output sync map file...") path = task.output_sync_map_file() self.print_info(u"Creating output sync map file... done") self.print_success(u"Created file '%s'" % path) except Exception as exc: self.print_error(u"An unexpected error occurred while writing the sync map file:") self.print_error(u"%s" % exc) return self.ERROR_EXIT_CODE if output_html: try: parameters = {} parameters[gc.PPN_TASK_OS_FILE_FORMAT] = task.configuration["o_format"] parameters[gc.PPN_TASK_OS_FILE_SMIL_AUDIO_REF] = task.configuration["o_smil_audio_ref"] parameters[gc.PPN_TASK_OS_FILE_SMIL_PAGE_REF] = task.configuration["o_smil_page_ref"] self.print_info(u"Creating output HTML file...") task.sync_map.output_html_for_tuning(audio_file_path, html_file_path, parameters) self.print_info(u"Creating output HTML file... done") self.print_success(u"Created file '%s'" % html_file_path) except Exception as exc: self.print_error(u"An unexpected error occurred while writing the HTML file:") self.print_error(u"%s" % exc) return self.ERROR_EXIT_CODE if download_from_youtube: if keep_audio: self.print_info(u"Option --keep-audio set: keeping downloaded file '%s'" % audio_file_path) else: gf.delete_file(None, audio_file_path) if print_zero: zero_duration = [l for l in task.sync_map.fragments_tree.vleaves_not_empty if l.begin == l.end] if len(zero_duration) > 0: self.print_warning(u"Fragments with zero duration:") for fragment in zero_duration: self.print_generic(u" %s" % fragment) if print_rates: self.print_info(u"Fragments with rates:") for fragment in task.sync_map.fragments_tree.vleaves_not_empty: self.print_generic(u" %s (rate: %.3f chars/s)" % (fragment, fragment.rate)) if print_faster_rate: max_rate = task.configuration["aba_rate_value"] if max_rate is not None: faster = [l for l in task.sync_map.fragments_tree.vleaves_not_empty if l.rate >= max_rate + Decimal("0.001")] if len(faster) > 0: self.print_warning(u"Fragments with rate greater than %.3f:" % max_rate) for fragment in faster: self.print_generic(u" %s (rate: %.3f chars/s)" % (fragment, fragment.rate)) return self.NO_ERROR_EXIT_CODE
def file_encoding(self, path, expected): validator = Validator() result = validator.check_file_encoding(get_abs_path(path)) self.assertEqual(result.passed, expected)
def string_well_formed(self, bstring, expected): validator = Validator() validator.check_raw_string(bstring, is_bstring=True) self.assertEqual(validator.result.passed, expected)
def main(): """ Entry point """ if len(sys.argv) < 3: usage() return validator = Validator() mode = sys.argv[1] result = None msg = "" if mode == "config": if sys.argv[2].endswith(".txt"): try: config_file = open(sys.argv[2], "r") config_contents = config_file.read() config_file.close() result = validator.check_contents_txt_config_file( config_contents, True) msg = "TXT configuration" except: print "[ERRO] Unable to read file %s" % sys.argv[2] elif sys.argv[2].endswith(".xml"): try: config_file = open(sys.argv[2], "r") config_contents = config_file.read() config_file.close() result = validator.check_contents_xml_config_file( config_contents) msg = "XML configuration" except: print "[ERRO] Unable to read file %s" % sys.argv[2] else: usage() return elif mode == "container": result = validator.check_container(sys.argv[2]) msg = "container" elif mode == "job": result = validator.check_job_configuration(sys.argv[2]) msg = "job configuration string" elif mode == "task": result = validator.check_task_configuration(sys.argv[2]) msg = "task configuration string" elif mode == "wizard": if len(sys.argv) < 4: usage() return result = validator.check_container_from_wizard(sys.argv[2], sys.argv[3]) msg = "container + configuration string from wizard" else: usage() return if result.passed: print "[INFO] Valid %s" % msg if len(result.warnings) > 0: for warning in result.warnings: print "[WARN] " + warning else: print "[INFO] Invalid %s" % msg for error in result.errors: print "[ERRO] " + error
def perform_command(self): """ Perform command and return the appropriate exit code. :rtype: int """ if len(self.actual_arguments) < 1: return self.print_help() if self.has_option([u"-e", u"--examples"]): return self.print_examples(False) if self.has_option(u"--examples-all"): return self.print_examples(True) if self.has_option([u"--list-parameters"]): return self.print_parameters() parameter = self.has_option_with_value(u"--list-values") if parameter is not None: return self.print_values(parameter) elif self.has_option(u"--list-values"): return self.print_values(u"?") # NOTE list() is needed for Python3, where keys() is not a list! demo = self.has_option(list(self.DEMOS.keys())) demo_parameters = u"" download_from_youtube = self.has_option([u"-y", u"--youtube"]) largest_audio = self.has_option(u"--largest-audio") keep_audio = self.has_option(u"--keep-audio") output_html = self.has_option(u"--output-html") validate = not self.has_option(u"--skip-validator") print_faster_rate = self.has_option(u"--faster-rate") print_rates = self.has_option(u"--rates") print_zero = self.has_option(u"--zero") if demo: validate = False for key in self.DEMOS: if self.has_option(key): demo_parameters = self.DEMOS[key] audio_file_path = demo_parameters[u"audio"] text_file_path = demo_parameters[u"text"] config_string = demo_parameters[u"config"] sync_map_file_path = demo_parameters[u"syncmap"] # TODO allow injecting rconf options directly from DEMOS options field if key == u"--example-cewsubprocess": self.rconf[ RuntimeConfiguration.CEW_SUBPROCESS_ENABLED] = True elif key == u"--example-ctw-espeak": self.rconf[RuntimeConfiguration.TTS] = "custom" self.rconf[ RuntimeConfiguration.TTS_PATH] = self.CTW_ESPEAK elif key == u"--example-ctw-speect": self.rconf[RuntimeConfiguration.TTS] = "custom" self.rconf[ RuntimeConfiguration.TTS_PATH] = self.CTW_SPEECT elif key == u"--example-festival": self.rconf[RuntimeConfiguration.TTS] = "festival" elif key == u"--example-mws": self.rconf[ RuntimeConfiguration.MFCC_WINDOW_LENGTH] = "1.500" self.rconf[ RuntimeConfiguration.MFCC_WINDOW_SHIFT] = "0.500" elif key == u"--example-multilevel-tts": self.rconf[RuntimeConfiguration.TTS_L1] = "festival" self.rconf[RuntimeConfiguration.TTS_L2] = "festival" self.rconf[RuntimeConfiguration.TTS_L3] = "espeak" elif key == u"--example-words-festival-cache": self.rconf[RuntimeConfiguration.TTS] = "festival" self.rconf[RuntimeConfiguration.TTS_CACHE] = True elif key == u"--example-faster-rate": print_faster_rate = True elif key == u"--example-no-zero": print_zero = True elif key == u"--example-py": self.rconf[RuntimeConfiguration.C_EXTENSIONS] = False elif key == u"--example-rates": print_rates = True elif key == u"--example-youtube": download_from_youtube = True break else: if len(self.actual_arguments) < 4: return self.print_help() audio_file_path = self.actual_arguments[0] text_file_path = self.actual_arguments[1] config_string = self.actual_arguments[2] sync_map_file_path = self.actual_arguments[3] html_file_path = None if output_html: keep_audio = True html_file_path = sync_map_file_path + u".html" if download_from_youtube: youtube_url = audio_file_path if (not download_from_youtube) and ( not self.check_input_file(audio_file_path)): return self.ERROR_EXIT_CODE if not self.check_input_file(text_file_path): return self.ERROR_EXIT_CODE if not self.check_output_file(sync_map_file_path): return self.ERROR_EXIT_CODE if (html_file_path is not None) and (not self.check_output_file(html_file_path)): return self.ERROR_EXIT_CODE self.check_c_extensions() if demo: msg = [] msg.append(u"Running example task with arguments:") if download_from_youtube: msg.append(u" YouTube URL: %s" % youtube_url) else: msg.append(u" Audio file: %s" % audio_file_path) msg.append(u" Text file: %s" % text_file_path) msg.append(u" Config string: %s" % config_string) msg.append(u" Sync map file: %s" % sync_map_file_path) if len(demo_parameters[u"options"]) > 0: msg.append(u" Options: %s" % demo_parameters[u"options"]) self.print_info(u"\n".join(msg)) if validate: self.print_info( u"Validating config string (specify --skip-validator to bypass)..." ) validator = Validator(logger=self.logger) result = validator.check_configuration_string(config_string, is_job=False, external_name=True) if not result.passed: self.print_error(u"The given config string is not valid:") self.print_generic(result.pretty_print()) return self.ERROR_EXIT_CODE self.print_info(u"Validating config string... done") if download_from_youtube: try: self.print_info(u"Downloading audio from '%s' ..." % youtube_url) downloader = Downloader(logger=self.logger) audio_file_path = downloader.audio_from_youtube( youtube_url, download=True, output_file_path=None, largest_audio=largest_audio) self.print_info(u"Downloading audio from '%s' ... done" % youtube_url) except ImportError: self.print_no_pafy_error() return self.ERROR_EXIT_CODE except Exception as exc: self.print_error( u"An unexpected error occurred while downloading audio from YouTube:" ) self.print_error(u"%s" % exc) return self.ERROR_EXIT_CODE else: audio_extension = gf.file_extension(audio_file_path) if audio_extension.lower() not in AudioFile.FILE_EXTENSIONS: self.print_warning( u"Your audio file path has extension '%s', which is uncommon for an audio file." % audio_extension) self.print_warning( u"Attempting at executing your Task anyway.") self.print_warning( u"If it fails, you might have swapped the first two arguments." ) self.print_warning( u"The audio file path should be the first argument, the text file path the second." ) try: self.print_info(u"Creating task...") task = Task(config_string, logger=self.logger) task.audio_file_path_absolute = audio_file_path task.text_file_path_absolute = text_file_path task.sync_map_file_path_absolute = sync_map_file_path self.print_info(u"Creating task... done") except Exception as exc: self.print_error( u"An unexpected error occurred while creating the task:") self.print_error(u"%s" % exc) return self.ERROR_EXIT_CODE try: self.print_info(u"Executing task...") executor = ExecuteTask(task=task, rconf=self.rconf, logger=self.logger) executor.execute() self.print_info(u"Executing task... done") except Exception as exc: self.print_error( u"An unexpected error occurred while executing the task:") self.print_error(u"%s" % exc) return self.ERROR_EXIT_CODE try: self.print_info(u"Creating output sync map file...") path = task.output_sync_map_file() self.print_info(u"Creating output sync map file... done") self.print_success(u"Created file '%s'" % path) except Exception as exc: self.print_error( u"An unexpected error occurred while writing the sync map file:" ) self.print_error(u"%s" % exc) return self.ERROR_EXIT_CODE if output_html: try: parameters = {} parameters[gc.PPN_TASK_OS_FILE_FORMAT] = task.configuration[ "o_format"] parameters[ gc.PPN_TASK_OS_FILE_EAF_AUDIO_REF] = task.configuration[ "o_eaf_audio_ref"] parameters[ gc.PPN_TASK_OS_FILE_SMIL_AUDIO_REF] = task.configuration[ "o_smil_audio_ref"] parameters[ gc.PPN_TASK_OS_FILE_SMIL_PAGE_REF] = task.configuration[ "o_smil_page_ref"] self.print_info(u"Creating output HTML file...") task.sync_map.output_html_for_tuning(audio_file_path, html_file_path, parameters) self.print_info(u"Creating output HTML file... done") self.print_success(u"Created file '%s'" % html_file_path) except Exception as exc: self.print_error( u"An unexpected error occurred while writing the HTML file:" ) self.print_error(u"%s" % exc) return self.ERROR_EXIT_CODE if download_from_youtube: if keep_audio: self.print_info( u"Option --keep-audio set: keeping downloaded file '%s'" % audio_file_path) else: gf.delete_file(None, audio_file_path) if print_zero: zero_duration = [ l for l in task.sync_map.fragments_tree.vleaves_not_empty if l.begin == l.end ] if len(zero_duration) > 0: self.print_warning(u"Fragments with zero duration:") for fragment in zero_duration: self.print_generic(u" %s" % fragment) if print_rates: self.print_info(u"Fragments with rates:") for fragment in task.sync_map.fragments_tree.vleaves_not_empty: self.print_generic(u" %s (rate: %.3f chars/s)" % (fragment, fragment.rate)) if print_faster_rate: max_rate = task.configuration["aba_rate_value"] if max_rate is not None: faster = [ l for l in task.sync_map.fragments_tree.vleaves_not_empty if l.rate >= max_rate + Decimal("0.001") ] if len(faster) > 0: self.print_warning( u"Fragments with rate greater than %.3f:" % max_rate) for fragment in faster: self.print_generic(u" %s (rate: %.3f chars/s)" % (fragment, fragment.rate)) return self.NO_ERROR_EXIT_CODE
def test_check_reserved_characters(self): validator = Validator() self.assertFalse(validator._check_reserved_characters("string with ~ reserved char")) self.assertTrue(validator._check_reserved_characters("string without reserved char"))
def test_check_file_encoding_false_01(self): logger = Logger() validator = Validator(logger=logger) input_file_path = get_abs_path("res/validator/encoding_iso8859.txt") result = validator.check_file_encoding(input_file_path) self.assertFalse(result.passed)
def perform_command(self): """ Perform command and return the appropriate exit code. :rtype: int """ if len(self.actual_arguments) < 2: return self.print_help() mode = self.actual_arguments[0] validator = Validator(rconf=self.rconf, logger=self.logger) if mode == u"config": config_file_path = self.actual_arguments[1] config_txt = None if config_file_path.lower().endswith(u".txt"): config_txt = True elif config_file_path.lower().endswith(u".xml"): config_txt = False else: return self.print_help() if not self.check_input_file(config_file_path): return self.ERROR_EXIT_CODE contents = gf.read_file_bytes(config_file_path) if contents is None: return self.ERROR_EXIT_CODE if config_txt: result = validator.check_config_txt(contents) msg = u"TXT configuration" else: result = validator.check_config_xml(contents) msg = "XML configuration" elif mode == u"container": container_path = self.actual_arguments[1] result = validator.check_container(container_path) msg = "container" elif mode == u"job": config_string = self.actual_arguments[1] result = validator.check_configuration_string(config_string, is_job=True) msg = u"job configuration string" elif mode == u"task": config_string = self.actual_arguments[1] result = validator.check_configuration_string(config_string, is_job=False, external_name=True) msg = u"task configuration string" elif mode == u"wizard": if (len(self.actual_arguments) < 3) or (self.actual_arguments[2].startswith(u"-")): return self.print_help() config_string = self.actual_arguments[1] container_path = self.actual_arguments[2] if not self.check_input_file(container_path): return self.ERROR_EXIT_CODE result = validator.check_container(container_path, config_string=config_string) msg = "container with configuration string from wizard" else: return self.print_help() if result.passed: self.print_success(u"Valid %s" % msg) for warning in result.warnings: self.print_warning(u"%s" % warning) return self.NO_ERROR_EXIT_CODE else: self.print_error(u"Invalid %s" % msg) for error in result.errors: self.print_error(u"%s" % error) return self.ERROR_EXIT_CODE
def perform_command(self): """ Perform command and return the appropriate exit code. :rtype: int """ if len(self.actual_arguments) < 2: return self.print_help() container_path = self.actual_arguments[0] output_directory_path = self.actual_arguments[1] config_string = None if (len(self.actual_arguments)) > 2 and (not self.actual_arguments[2].startswith(u"-")): config_string = self.actual_arguments[2] validate = not self.has_option(u"--skip-validator") if self.has_option(u"--cewsubprocess"): self.rconf[RuntimeConfiguration.CEW_SUBPROCESS_ENABLED] = True if not self.check_input_file_or_directory(container_path): return self.ERROR_EXIT_CODE if not self.check_output_directory(output_directory_path): return self.ERROR_EXIT_CODE if validate: try: self.print_info(u"Validating the container (specify --skip-validator to bypass)...") validator = Validator(rconf=self.rconf, logger=self.logger) result = validator.check_container(container_path, config_string=config_string) if not result.passed: self.print_error(u"The given container is not valid:") self.print_error(result.pretty_print()) return self.ERROR_EXIT_CODE self.print_info(u"Validating the container... done") except Exception as exc: self.print_error(u"An unexpected error occurred while validating the container:") self.print_error(u"%s" % exc) return self.ERROR_EXIT_CODE try: self.print_info(u"Loading job from container...") executor = ExecuteJob(rconf=self.rconf, logger=self.logger) executor.load_job_from_container(container_path, config_string) self.print_info(u"Loading job from container... done") except Exception as exc: self.print_error(u"An unexpected error occurred while loading the job:") self.print_error(u"%s" % exc) return self.ERROR_EXIT_CODE try: self.print_info(u"Executing...") executor.execute() self.print_info(u"Executing... done") except Exception as exc: self.print_error(u"An unexpected error occurred while executing the job:") self.print_error(u"%s" % exc) return self.ERROR_EXIT_CODE try: self.print_info(u"Creating output container...") path = executor.write_output_container(output_directory_path) self.print_info(u"Creating output container... done") self.print_success(u"Created output file '%s'" % path) executor.clean(True) return self.NO_ERROR_EXIT_CODE except Exception as exc: self.print_error(u"An unexpected error occurred while writing the output container:") self.print_error(u"%s" % exc) return self.ERROR_EXIT_CODE
def load_job_from_container(self, container_path, config_string=None): """ Validate the given container, and, if it is well formed, load the job from it. If ``config_string`` is ``None``, the container must contain a configuration file; otherwise use the provided config string (i.e., the wizard case). Return ``True`` if the job has been loaded successfully, ``False`` otherwise. :param container_path: the path to the input container :type container_path: string (path) :param config_string: the configuration string (from wizard) :type config_string: string :rtype: bool """ self._log("Loading job from container...") # validate container self._log("Validating container...") validator = Validator(logger=self.logger) if config_string == None: validator_result = validator.check_container(container_path) else: validator_result = validator.check_container_from_wizard( container_path, config_string ) if not validator_result.passed: self._log("Validating container: failed") self._log("Loading job from container: failed") return False self._log("Validating container: succeeded") try: # create working directory where the input container # will be decompressed self.working_directory = tempfile.mkdtemp(dir=gf.custom_tmp_dir()) self._log("Created working directory '%s'" % self.working_directory) # decompress self._log("Decompressing input container...") input_container = Container(container_path, logger=self.logger) input_container.decompress(self.working_directory) self._log("Decompressing input container... done") # create job from the working directory self._log("Creating job from working directory...") working_container = Container( self.working_directory, logger=self.logger ) analyzer = AnalyzeContainer(working_container, logger=self.logger) if config_string == None: self.job = analyzer.analyze() else: self.job = analyzer.analyze_from_wizard(config_string) self._log("Creating job from working directory... done") # set absolute path for text file and audio file # for each task in the job self._log("Setting absolute paths for tasks...") for task in self.job.tasks: task.text_file_path_absolute = gf.norm_join( self.working_directory, task.text_file_path ) task.audio_file_path_absolute = gf.norm_join( self.working_directory, task.audio_file_path ) self._log("Setting absolute paths for tasks... done") # return self._log("Loading job from container: succeeded") return True except: # failure: clean and return self.clean() self._log("Loading job from container: failed") return False
def perform_command(self): """ Perform command and return the appropriate exit code. :rtype: int """ if len(self.actual_arguments) < 2: return self.print_help() container_path = self.actual_arguments[0] output_directory_path = self.actual_arguments[1] config_string = None if (len(self.actual_arguments)) > 2 and ( not self.actual_arguments[2].startswith(u"-")): config_string = self.actual_arguments[2] validate = not self.has_option(u"--skip-validator") if self.has_option(u"--cewsubprocess"): self.rconf[RuntimeConfiguration.CEW_SUBPROCESS_ENABLED] = True if not self.check_input_file_or_directory(container_path): return self.ERROR_EXIT_CODE if not self.check_output_directory(output_directory_path): return self.ERROR_EXIT_CODE if validate: try: self.print_info( u"Validating the container (specify --skip-validator to bypass)..." ) validator = Validator(rconf=self.rconf, logger=self.logger) result = validator.check_container(container_path, config_string=config_string) if not result.passed: self.print_error(u"The given container is not valid:") self.print_error(result.pretty_print()) return self.ERROR_EXIT_CODE self.print_info(u"Validating the container... done") except Exception as exc: self.print_error( u"An unexpected error occurred while validating the container:" ) self.print_error(u"%s" % exc) return self.ERROR_EXIT_CODE try: self.print_info(u"Loading job from container...") executor = ExecuteJob(rconf=self.rconf, logger=self.logger) executor.load_job_from_container(container_path, config_string) self.print_info(u"Loading job from container... done") except Exception as exc: self.print_error( u"An unexpected error occurred while loading the job:") self.print_error(u"%s" % exc) return self.ERROR_EXIT_CODE try: self.print_info(u"Executing...") executor.execute() self.print_info(u"Executing... done") except Exception as exc: self.print_error( u"An unexpected error occurred while executing the job:") self.print_error(u"%s" % exc) return self.ERROR_EXIT_CODE try: self.print_info(u"Creating output container...") path = executor.write_output_container(output_directory_path) self.print_info(u"Creating output container... done") self.print_success(u"Created output file '%s'" % path) executor.clean(True) return self.NO_ERROR_EXIT_CODE except Exception as exc: self.print_error( u"An unexpected error occurred while writing the output container:" ) self.print_error(u"%s" % exc) return self.ERROR_EXIT_CODE
def file_encoding(self, path, expected): validator = Validator() result = validator.check_file_encoding(gf.absolute_path(path, __file__)) self.assertEqual(result.passed, expected)
def file_encoding(self, path, expected): validator = Validator() result = validator.check_file_encoding(gf.absolute_path( path, __file__)) self.assertEqual(result.passed, expected)
def test_check_file_encoding_true_02(self): logger = Logger() validator = Validator(logger=logger) input_file_path = get_abs_path("res/validator/encoding_utf8_bom.xhtml") result = validator.check_file_encoding(input_file_path) self.assertTrue(result.passed)
def test_check_string_encoding(self): validator = Validator() self.assertFalse(validator._check_string_encoding(u"abcdé".encode("latin-1"))) self.assertTrue(validator._check_string_encoding(u"abcdé".encode("utf-8")))