def prepare(**kwargs): """Prepare XMLFILE for 'readalongs align' from PLAINTEXTFILE. PLAINTEXTFILE must be plain text encoded in utf-8, with one sentence per line, paragraph breaks marked by a blank line, and page breaks marked by two blank lines. PLAINTEXTFILE: Path to the plain text input file, or - for stdin XMLFILE: Path to the XML output file, or - for stdout [default: PLAINTEXTFILE.xml] """ if kwargs["debug"]: LOGGER.setLevel("DEBUG") LOGGER.info( "Running readalongs prepare(lang={}, force-overwrite={}, plaintextfile={}, xmlfile={})." .format( kwargs["language"], kwargs["force_overwrite"], kwargs["plaintextfile"], kwargs["xmlfile"], )) input_file = kwargs["plaintextfile"] out_file = kwargs["xmlfile"] if not out_file: try: out_file = input_file.name except Exception: # For unit testing: simulated stdin stream has no .name attrib out_file = "<stdin>" if out_file == "<stdin>": # actual intput_file.name when cli input is "-" out_file = "-" else: if out_file.endswith(".txt"): out_file = out_file[:-4] out_file += ".xml" if out_file == "-": filehandle, filename = create_input_tei( input_file_handle=input_file, text_language=kwargs["language"], ) with io.open(filename) as f: sys.stdout.write(f.read()) else: if not out_file.endswith(".xml"): out_file += ".xml" if os.path.exists(out_file) and not kwargs["force_overwrite"]: raise click.BadParameter( "Output file %s exists already, use -f to overwrite." % out_file) filehandle, filename = create_input_tei( input_file_handle=input_file, text_language=kwargs["language"], output_file=out_file, ) LOGGER.info("Wrote {}".format(out_file))
def test_create_input_tei_errors(self): """create_input_tei should raise a AssertionError when parameters are missing.""" # These used to be RuntimeError, but that was not right: *programmer* # errors can and should dump stack traces, unlike *user* errors, which # warrant nice friendly messages. with self.assertRaises(AssertionError): # missing input_file_name or input_file_handle _, _ = create_input_tei() with self.assertRaises(AssertionError): # missing text_languages _, _ = create_input_tei( input_file_name=os.path.join(self.data_dir, "fra.txt"))
def testAlignText(self): txt_path = os.path.join(self.data_dir, "ej-fra.txt") wav_path = os.path.join(self.data_dir, "ej-fra.m4a") # tempfh, temp_fn = create_input_xml(txt_path, text_language='git', save_temps="unit") tempfh, temp_fn = create_input_tei(input_file_name=txt_path, text_language="fra", save_temps=None) results = align_audio(temp_fn, wav_path, unit="w", save_temps=None) # Verify that the same IDs are in the output converted_path = os.path.join(self.data_dir, "ej-fra-converted.xml") xml = etree.parse(converted_path).getroot() words = results["words"] xml_words = xml.xpath(".//w") self.assertEqual(len(words), len(xml_words)) for w, xw in zip(words, xml_words): self.assertEqual(xw.attrib["id"], w["id"])
def test_align_text(self): """Basic alignment test case with plain text input""" txt_path = os.path.join(self.data_dir, "ej-fra.txt") wav_path = os.path.join(self.data_dir, "ej-fra.m4a") _, temp_fn = create_input_tei(input_file_name=txt_path, text_languages=("fra", ), save_temps=None) results = align_audio(temp_fn, wav_path, unit="w", save_temps=None) # Verify that the same IDs are in the output converted_path = os.path.join(self.data_dir, "ej-fra-converted.xml") xml = etree.parse(converted_path).getroot() words = results["words"] xml_words = xml.xpath(".//w") self.assertEqual(len(words), len(xml_words)) for w, xw in zip(words, xml_words): self.assertEqual(xw.attrib["id"], w["id"])
def align(**kwargs): """Align TEXTFILE and AUDIOFILE and create output files as OUTPUT_BASE.* in directory OUTPUT_BASE/. TEXTFILE: Input text file path (in XML, or plain text with -i) AUDIOFILE: Input audio file path, in any format supported by ffmpeg OUTPUT_BASE: Base name for output files """ config = kwargs.get("config", None) if config: if config.endswith("json"): try: with open(config) as f: config = json.load(f) except json.decoder.JSONDecodeError: LOGGER.error(f"Config file at {config} is not valid json.") else: raise click.BadParameter(f"Config file '{config}' must be in JSON format") output_dir = kwargs["output_base"] if os.path.exists(output_dir): if not os.path.isdir(output_dir): raise click.UsageError( f"Output folder '{output_dir}' already exists but is a not a directory." ) if not kwargs["force_overwrite"]: raise click.UsageError( f"Output folder '{output_dir}' already exists, use -f to overwrite." ) else: os.mkdir(output_dir) # Make sure we can write to the output directory, for early error checking and user # friendly error messages. try: with TemporaryFile(dir=output_dir): pass except Exception: raise click.UsageError( f"Cannot write into output folder '{output_dir}'. Please verify permissions." ) output_basename = os.path.basename(output_dir) output_base = os.path.join(output_dir, output_basename) temp_base = None if kwargs["save_temps"]: temp_dir = os.path.join(output_dir, "tempfiles") if not os.path.isdir(temp_dir): if os.path.exists(temp_dir) and kwargs["force_overwrite"]: os.unlink(temp_dir) os.mkdir(temp_dir) temp_base = os.path.join(temp_dir, output_basename) if kwargs["debug"]: LOGGER.setLevel("DEBUG") if kwargs["text_input"]: if not kwargs["language"]: LOGGER.warn("No input language provided, using undetermined mapping") tempfile, kwargs["textfile"] = create_input_tei( input_file_name=kwargs["textfile"], text_language=kwargs["language"], save_temps=temp_base, ) if kwargs["output_xhtml"]: tokenized_xml_path = "%s.xhtml" % output_base else: _, input_ext = os.path.splitext(kwargs["textfile"]) tokenized_xml_path = "%s%s" % (output_base, input_ext) if os.path.exists(tokenized_xml_path) and not kwargs["force_overwrite"]: raise click.BadParameter( "Output file %s exists already, use -f to overwrite." % tokenized_xml_path ) smil_path = output_base + ".smil" if os.path.exists(smil_path) and not kwargs["force_overwrite"]: raise click.BadParameter( "Output file %s exists already, use -f to overwrite." % smil_path ) _, audio_ext = os.path.splitext(kwargs["audiofile"]) audio_path = output_base + audio_ext if os.path.exists(audio_path) and not kwargs["force_overwrite"]: raise click.BadParameter( "Output file %s exists already, use -f to overwrite." % audio_path ) unit = kwargs.get("unit", "w") bare = kwargs.get("bare", False) if ( not unit ): # .get() above should handle this but apparently the way kwargs is implemented unit = "w" # unit could still be None here. try: results = align_audio( kwargs["textfile"], kwargs["audiofile"], unit=unit, bare=bare, config=config, save_temps=temp_base, ) except RuntimeError as e: LOGGER.error(e) exit(1) if kwargs["text_grid"]: audio = read_audio_from_file(kwargs["audiofile"]) duration = audio.frame_count() / audio.frame_rate words, sentences = return_words_and_sentences(results) textgrid = write_to_text_grid(words, sentences, duration) textgrid.to_file(output_base + ".TextGrid") textgrid.to_eaf().to_file(output_base + ".eaf") if kwargs["closed_captioning"]: words, sentences = return_words_and_sentences(results) webvtt_sentences = write_to_subtitles(sentences) webvtt_sentences.save(output_base + "_sentences.vtt") webvtt_sentences.save_as_srt(output_base + "_sentences.srt") webvtt_words = write_to_subtitles(words) webvtt_words.save(output_base + "_words.vtt") webvtt_words.save_as_srt(output_base + "_words.srt") if kwargs["output_xhtml"]: convert_to_xhtml(results["tokenized"]) save_minimal_index_html( os.path.join(output_dir, "index.html"), os.path.basename(tokenized_xml_path), os.path.basename(smil_path), os.path.basename(audio_path), ) save_xml(tokenized_xml_path, results["tokenized"]) smil = make_smil( os.path.basename(tokenized_xml_path), os.path.basename(audio_path), results ) shutil.copy(kwargs["audiofile"], audio_path) save_txt(smil_path, smil)
def test_create_input_tei_no_input(self): with self.assertRaises(RuntimeError): (fh, fname) = create_input_tei()
def prepare(**kwargs): """Prepare XMLFILE for 'readalongs align' from PLAINTEXTFILE. PLAINTEXTFILE must be plain text encoded in UTF-8, with one sentence per line, paragraph breaks marked by a blank line, and page breaks marked by two blank lines. PLAINTEXTFILE: Path to the plain text input file, or - for stdin XMLFILE: Path to the XML output file, or - for stdout [default: PLAINTEXTFILE.xml] """ if kwargs["debug"]: LOGGER.setLevel("DEBUG") LOGGER.info( "Running readalongs prepare(lang={}, force-overwrite={}, plaintextfile={}, xmlfile={})." .format( kwargs["language"], kwargs["force_overwrite"], kwargs["plaintextfile"], kwargs["xmlfile"], )) input_file = kwargs["plaintextfile"] out_file = kwargs["xmlfile"] if not out_file: out_file = get_click_file_name(input_file) if out_file != "-": if out_file.endswith(".txt"): out_file = out_file[:-4] out_file += ".xml" languages = kwargs["language"] if not kwargs["lang_no_append_und"] and "und" not in languages: languages.append("und") try: if out_file == "-": _, filename = create_input_tei( input_file_handle=input_file, text_languages=languages, ) with io.open(filename, encoding="utf8") as f: sys.stdout.write(f.read()) else: if not out_file.endswith(".xml"): out_file += ".xml" if os.path.exists(out_file) and not kwargs["force_overwrite"]: raise click.BadParameter( "Output file %s exists already, use -f to overwrite." % out_file) _, filename = create_input_tei( input_file_handle=input_file, text_languages=languages, output_file=out_file, ) except RuntimeError as e: raise click.UsageError(e) from e LOGGER.info("Wrote {}".format(out_file))
def align(**kwargs): """Align TEXTFILE and AUDIOFILE and create output files as OUTPUT_BASE.* in directory OUTPUT_BASE/. TEXTFILE: Input text file path (in XML or plain text) \b If TEXTFILE has a .xml extension or starts with an XML declaration line, it is parsed as XML and can be in one of three formats: - the output of 'readalongs prepare', - the output of 'readalongs tokenize', or - the output of 'readalongs g2p'. \b If TEXTFILE has a .txt extension or does not start with an XML declaration line, is it read as plain text with the following conventions: - The text should be plain UTF-8 text without any markup. - Paragraph breaks are indicated by inserting one blank line. - Page breaks are indicated by inserting two blank lines. One can add the known ARPABET phonetics in the XML for words (<w> elements) that are not correctly handled by g2p in the output of 'readalongs tokenize' or 'readalongs g2p', via the ARPABET attribute. One can add anchor elements in the XML, e.g., '<anchor time="2.345s"/>', to mark known anchor points between the audio and text stream. AUDIOFILE: Input audio file path, in any format supported by ffmpeg OUTPUT_BASE: Output files will be saved as OUTPUT_BASE/OUTPUT_BASE.* """ config_file = kwargs.get("config", None) config = None if config_file: if config_file.endswith("json"): try: with open(config_file, encoding="utf8") as f: config = json.load(f) except json.decoder.JSONDecodeError as e: raise click.BadParameter( f"Config file at {config_file} is not in valid JSON format." ) from e else: raise click.BadParameter( f"Config file '{config_file}' must be in JSON format") output_dir = kwargs["output_base"] if os.path.exists(output_dir): if not os.path.isdir(output_dir): raise click.UsageError( f"Output folder '{output_dir}' already exists but is a not a directory." ) if not kwargs["force_overwrite"]: raise click.UsageError( f"Output folder '{output_dir}' already exists, use -f to overwrite." ) else: os.mkdir(output_dir) # Make sure we can write to the output directory, for early error checking and user # friendly error messages. try: with TemporaryFile(dir=output_dir): pass except Exception as e: raise click.UsageError( f"Cannot write into output folder '{output_dir}'. Please verify permissions." ) from e if kwargs["g2p_fallback"] is not None: raise click.BadParameter( "The --g2p-fallback option is obsolete.\n" "Specify multiple languages with the -l/--language option instead,\n" "or by adding the 'fallback-langs' attribute where relevant in your XML input." ) output_basename = os.path.basename(output_dir) temp_base = None if kwargs["save_temps"]: temp_dir = os.path.join(output_dir, "tempfiles") if not os.path.isdir(temp_dir): if os.path.exists(temp_dir) and kwargs["force_overwrite"]: os.unlink(temp_dir) os.mkdir(temp_dir) temp_base = os.path.join(temp_dir, output_basename) if kwargs["debug"]: LOGGER.setLevel("DEBUG") if kwargs["text_input"] is not None: raise click.BadParameter( "The -i option is obsolete. .txt files are now read as plain text, " ".xml as XML, and other files based on whether they start with <?xml or not." ) # Determine if the file is plain text or XML textfile_name = kwargs["textfile"] if textfile_name.endswith(".xml"): textfile_is_plaintext = False # .xml is XML elif textfile_name.endswith(".txt"): textfile_is_plaintext = True # .txt is plain text else: # Files other than .xml or .txt are parsed using etree. If the parse is # successful or the first syntax error is past the first line, the file # is assumed to be XML. Plain text files will yield an error in the # first few characters of line 1, typically complaining about not # finding "<" at the start. # There are many valid "magic numbers" for XML files, depending on # their encoding (utf8, utf16, endianness, etc). If we looked for # "<?xml " at the beginning, that would only catch some of the valid # XML encodings that etree can parse. # We could also use python-magic or filetype, but why introduce another # dependency when we can ask the library we're already using!? try: _ = etree.parse(textfile_name) textfile_is_plaintext = False except etree.XMLSyntaxError as e: textfile_is_plaintext = e.position <= (1, 10) if textfile_is_plaintext: if not kwargs["language"]: raise click.BadParameter( "No input language specified for plain text input. " "Please provide the -l/--language switch.") languages = kwargs["language"] if not kwargs["lang_no_append_und"] and "und" not in languages: languages.append("und") plain_textfile = kwargs["textfile"] try: _, xml_textfile = create_input_tei( input_file_name=plain_textfile, text_languages=languages, save_temps=temp_base, ) except RuntimeError as e: raise click.UsageError(e) from e else: xml_textfile = kwargs["textfile"] bare = kwargs.get("bare", False) try: results = align_audio( xml_textfile, kwargs["audiofile"], bare=bare, config=config, save_temps=temp_base, verbose_g2p_warnings=kwargs["g2p_verbose"], ) except RuntimeError as e: raise click.UsageError(e) from e # LOGGER.error(e) # sys.exit(1) output_formats = kwargs["output_formats"] save_readalong( align_results=results, output_dir=output_dir, output_basename=output_basename, config=config, audiofile=kwargs["audiofile"], audiosegment=results["audio"], output_formats=output_formats, )