def run(self, input_filename, output_filename=None): """ Perform the TGA estimation process. :param input_filename: (str) Name of the input file with the aligned syllables :param output_filename: (str) Name of the resulting file with TGA """ self.print_filename(input_filename) self.print_options() self.print_diagnosis(input_filename) # Get the tier to syllabify parser = sppasRW(input_filename) trs_input = parser.read() tier_input = sppasFindTier.aligned_syllables(trs_input) # Create the transcription result trs_output = sppasTranscription("Time Group Analyzer") trs_output.set_meta('tga_result_of', input_filename) # Estimate TGA on the tier trs_output = self.convert(tier_input) # Save in a file if output_filename is not None: if len(trs_output) > 0: parser = sppasRW(output_filename) parser.write(trs_output) self.print_filename(output_filename, status=0) else: raise EmptyOutputError return trs_output
def run(self, input_filename, output_filename=None): """ Run the INTSINT annotation process on an input file. :param input_filename: (str) the input file name with momel :param output_filename: (str) the output file name of the INTSINT tier :returns: (sppasTranscription) """ self.print_filename(input_filename) # Get the tier to be annotated. parser = sppasRW(input_filename) trs_input = parser.read() tier_input = sppasFindTier.pitch_anchors(trs_input) # Annotate the tier targets = sppasIntsint.tier_to_anchors(tier_input) tones = self.intsint.annotate(targets) tier_intsint = sppasIntsint.tones_to_tier(tones, tier_input) # Create the transcription result trs_output = sppasTranscription(self.name) trs_output.append(tier_intsint) trs_output.set_meta('intsint_result_of', input_filename) # Save in a file if output_filename is not None: parser = sppasRW(output_filename) parser.write(trs_output) self.print_filename(output_filename, status=0) return trs_output
def run(self, input_filename, output_filename=None): """ Perform the Syllabification process. :param input_filename: (str) Name of the input file with the aligned phonemes :param output_filename: (str) Name of the resulting file with syllabification """ self.print_filename(input_filename) self.print_options() self.print_diagnosis(input_filename) # Get the tier to syllabify parser = sppasRW(input_filename) trs_input = parser.read() tier_input = sppasFindTier.aligned_phones(trs_input) # Create the transcription result trs_output = sppasTranscription("Syllabification") trs_output.set_meta('syllabification_result_of', input_filename) # Syllabify the tier if self._options['usesphons'] is True: tier_syll = self.convert(tier_input) trs_output.append(tier_syll) if self._options['createclasses']: trs_output.append(self.make_classes(tier_syll)) # Extra tier: syllabify between given intervals if self._options['usesintervals'] is True: intervals = trs_input.find(self._options['tiername']) if intervals is None: self.print_message(MSG_NO_TIER.format(tiername=self._options['tiername']), indent=2, status=WARNING_ID) else: tier_syll_int = self.convert(tier_input, intervals) tier_syll_int.set_name("SyllAlign-Intervals") tier_syll_int.set_meta('syllabification_used_intervals', intervals.get_name()) trs_output.append(tier_syll_int) if self._options['createclasses']: t = self.make_classes(tier_syll_int) t.set_name("SyllClassAlign-Intervals") trs_output.append(t) # Save in a file if output_filename is not None: if len(trs_output) > 0: parser = sppasRW(output_filename) parser.write(trs_output) self.print_filename(output_filename, status=0) else: raise EmptyOutputError return trs_output
def compare_folders(self, samples_folder, expected_result_dir, sa): # Apply Alignment on each sample for filename in os.listdir(os.path.join(paths.samples, samples_folder)): if filename.endswith(".wav") is False: continue # Get the expected result expected_result_filename = os.path.join( expected_result_dir, filename[:-4] + "-palign.xra") if os.path.exists(expected_result_filename) is False: print("no existing alignment result {:s}".format(expected_result_filename)) continue parser = sppasRW(expected_result_filename) expected_result = parser.read() expected_tier_phones = expected_result.find('PhonAlign') if expected_tier_phones is None: print("malformed alignment result for:", filename) continue # Estimate a result and check if it's like expected. audio_file = os.path.join(paths.samples, samples_folder, filename) phn_file = os.path.join(expected_result_dir, filename.replace('.wav', '-phon.xra')) tok_file = os.path.join(expected_result_dir, filename.replace('.wav', '-token.xra')) result_file = os.path.join(paths.samples, samples_folder, filename.replace('.wav', '-palign.xra')) expected_result = sa.run([audio_file, phn_file], [tok_file], result_file) print('Evaluate:', audio_file) self.compare_tiers(expected_tier_phones, expected_result.find('PhonAlign'))
def get_tier(filename, tier_name, verbose=True): """ Get a tier from a file. :param filename: (str) Name of the annotated file. :param tier_name: (str) Name of the tier :param verbose: (bool) Print message :returns: (Tier) """ # Read an annotated file. if verbose: print("Read file: {:s}".format(filename)) parser = sppasRW(filename) trs = parser.read() if verbose: print(" ... [ OK ] ") # Get the expected tier if verbose: print("Get tier {:s}".format(tier_name)) tier = trs.find(tier_name, case_sensitive=False) if tier is None: print("Tier not found.") sys.exit(1) if verbose: print(" ... [ OK ] ") return tier
def _merge(self): """Merge all annotated files.""" self._progress.set_header("Merge all annotations in a file") self._progress.update(0, "") # Get the list of files with the ".wav" extension filelist = self.get_annot_files( pattern="", extensions=sppas.src.audiodata.aio.extensions) total = len(filelist) output_format = self._parameters.get_output_format() for i, f in enumerate(filelist): nbfiles = 0 # Change f, to allow "replace" to work properly basef = os.path.splitext(f)[0] self._logfile.print_message("File: " + f, indent=0) self._progress.set_text( os.path.basename(f) + " (" + str(i + 1) + "/" + str(total) + ")") # Add all files content in the same order than to annotate trs = sppasTranscription() nbfiles += self.__add_trs(trs, basef + output_format) for s in range(self._parameters.get_step_numbers()): ann_key = self._parameters.get_step_key(s) a = self._get_instance(ann_key) pattern = a.get_pattern() if len(pattern) > 0: nbfiles += self.__add_trs(trs, basef + pattern + output_format) if nbfiles > 1: try: info_tier = sppasMetaInfoTier(trs) tier = info_tier.create_time_tier( trs.get_min_loc().get_midpoint(), trs.get_max_loc().get_midpoint()) trs.append(tier) parser = sppasRW(basef + "-merge.xra") parser.write(trs) self._logfile.print_message(basef + "-merge.xra", indent=1, status=0) except Exception as e: self._logfile.print_message(str(e), indent=1, status=-1) self._progress.set_fraction(float((i + 1)) / float(total)) self._logfile.print_newline() self._progress.update(1, "Completed.") self._progress.set_header("")
def run(self, input_filename, output_filename=None): """ Run the Phonetization process on an input file. :param input_filename (str) Name of the file including a tokenization :param output_filename (str) Name of the resulting file with phonetization :returns: (sppasTranscription) """ self.print_filename(input_filename) self.print_options() self.print_diagnosis(input_filename) # Get the tier to be phonetized. pattern = "" if self._options['usestdtokens'] is True: pattern = "std" parser = sppasRW(input_filename) trs_input = parser.read() tier_input = sppasFindTier.tokenization(trs_input, pattern) # Phonetize the tier tier_phon = self.convert(tier_input) # Create the transcription result trs_output = sppasTranscription("Phonetization") if tier_phon is not None: trs_output.append(tier_phon) trs_output.set_meta('text_phonetization_result_of', input_filename) trs_output.set_meta('text_phonetization_dict', self.phonetizer.get_dict_filename()) # Save in a file if output_filename is not None: if len(trs_output) > 0: parser = sppasRW(output_filename) parser.write(trs_output) self.print_filename(output_filename, status=0) else: raise EmptyOutputError return trs_output
def test_TGA_on_sample(self): # This is one of the samples proposed in-line by Dafydd path = os.path.join(DATA, "tga.TextGrid") parser = sppasRW(path) trs = parser.read() tier = trs.find('Syllables') t = sppasTGA() timegroups = t.syllables_to_timegroups(tier) tg_dur = t.timegroups_to_durations(tier, timegroups) tga = TimeGroupAnalysis(tg_dur) occurrences = tga.len() self.assertEqual(34, len(occurrences)) total = tga.total() mean = tga.mean() median = tga.median() stdev = tga.stdev() npvi = tga.nPVI() reglin = tga.intercept_slope_original() self.assertEqual(3, occurrences['tg_1']) self.assertEqual(0.57, round(total['tg_1'], 2)) self.assertEqual(0.19, round(mean['tg_1'], 2)) self.assertEqual(0.14, round(median['tg_1'], 2)) self.assertEqual(0.13928, round(stdev['tg_1'], 5)) self.assertEqual(94, round(npvi['tg_1'], 0)) i, s = reglin['tg_1'] self.assertEqual(0.025, round(i, 3)) self.assertEqual(0.165, round(s, 3)) self.assertEqual(4, occurrences['tg_33']) self.assertEqual(0.78, round(total['tg_33'], 2)) self.assertEqual(0.195, round(mean['tg_33'], 3)) self.assertEqual(0.06062, round(stdev['tg_33'], 5)) self.assertEqual(53, round(npvi['tg_33'], 0)) i, s = reglin['tg_33'] self.assertEqual(0.156, round(i, 3)) self.assertEqual(0.026, round(s, 3)) # do the job from a tier trs1 = t.convert(tier) t.set_intercept_slope_annotationpro(True) t.set_intercept_slope_original(False) self.assertEqual(10, len(trs1)) # do the same from the file trs2 = t.run(path) self.assertEqual(10, len(trs2))
def test_samples(self): """ ... Compare the current result is the same as the existing one. """ for samples_folder in os.listdir(SAMPLES_PATH): if samples_folder.startswith("samples-") is False: continue expected_result_dir = os.path.join(SAMPLES_PATH, "annotation-results", samples_folder) # Create a TextNormalizer for the given set of samples lang = samples_folder[-3:] vocab = os.path.join(RESOURCES_PATH, "vocab", lang + ".vocab") tn = sppasTextNorm(vocab, lang) tn.set_faked(True) tn.set_std(True) tn.set_custom(True) # Apply TextNormalization on each sample for filename in os.listdir( os.path.join(SAMPLES_PATH, samples_folder)): if filename.endswith(".TextGrid") is False: continue # Get the expected result expected_result_filename = os.path.join( expected_result_dir, filename[:-9] + "-token.xra") if os.path.exists(expected_result_filename) is False: continue parser = sppasRW(expected_result_filename) expected_result = parser.read() # Estimate the result and check if it's like expected. result = tn.run( os.path.join(SAMPLES_PATH, samples_folder, filename)) expected_tier_tokens = expected_result.find('Tokens') if expected_tier_tokens is not None: self.compare_tiers(expected_tier_tokens, result.find('Tokens')) expected_tier_tokens = expected_result.find('TokensStd') if expected_tier_tokens is not None: self.compare_tiers(expected_tier_tokens, result.find('TokensStd')) expected_tier_tokens = expected_result.find('TokensCustom') if expected_tier_tokens is not None: self.compare_tiers(expected_tier_tokens, result.find('TokensCustom'))
def convert(self, input_audio_filename, input_filename): """Return a tier with transcription aligned to the audio. :param input_audio_filename: (str) Input audio file :param input_filename: (str) Input transcription file """ # Get audio and the channel we'll work on audio_speech = sppas.src.audiodata.aio.open(input_audio_filename) n = audio_speech.get_nchannels() if n != 1: raise AudioChannelError(n) idx = audio_speech.extract_channel() channel = audio_speech.get_channel(idx) # Get the units we'll work on parser = sppasRW(input_filename) trs = parser.read() if len(trs) > 1: pass if len(trs[0]) == 0: pass units = list() for a in trs[0]: units.append(a.serialize_labels()) ipus = [u for u in units if u != SIL_ORTHO] # Create the instance to fill in IPUs filler = FillIPUs(channel, units) filler.set_min_ipu(self._options['min_ipu']) filler.set_min_sil(self._options['min_sil']) n = filler.fix_threshold_durations() if n != len(ipus): return # Process the data. tracks = filler.get_tracks(time_domain=True) tier = sppasSearchIPUs.tracks_to_tier(tracks, channel.get_duration(), filler.get_vagueness()) tier.set_name('Transcription') self._set_meta(filler, tier) i = 0 for a in tier: if a.get_best_tag().is_silence() is False: a.set_labels([sppasLabel(sppasTag(ipus[i]))]) i += 1 return tier
def __add_trs(self, trs, trs_inputfile): try: parser = sppasRW(trs_inputfile) trs_input = parser.read(trs_inputfile) except IOError: return 0 for tier in trs_input: already_in = False if trs.is_empty() is False: tier_name = tier.get_name() for t in trs: if t.get_name() == tier_name: already_in = True if already_in is False: trs.append(tier) return 1
def test_samples(self): """... Compare the current result is the same as the existing one.""" # the place where are the samples to be tested. samples_path = os.path.join(paths.samples, "annotation-results") # each samples folder is tested for samples_folder in os.listdir(samples_path): if samples_folder.startswith("samples-") is False: continue # Create a Syllabifier for the given set of samples of the given language lang = samples_folder[-3:] rules_file = os.path.join(paths.resources, "syll", "syllConfig-"+lang+".txt") if os.path.exists(rules_file) is False: continue tn = sppasSyll() tn.load_resources(rules_file) # Apply Syllabification on each sample for filename in os.listdir(os.path.join(samples_path, samples_folder)): if filename.endswith("-palign.xra") is False: continue # Get the expected result expected_result_filename = os.path.join(samples_path, samples_folder, filename[:-11] + "-salign.xra") if os.path.exists(expected_result_filename) is False: print("no match palign/salign for: {:s}".format(filename)) continue try: parser = sppasRW(expected_result_filename) expected_result = parser.read() except AioEncodingError: continue # Estimate the result and check if it's like expected. input_file = os.path.join(samples_path, samples_folder, filename) result = tn.run([input_file]) expected_tier_syll = expected_result.find('SyllAlign') if expected_tier_syll is not None: self.compare_tiers(expected_tier_syll, result.find('SyllAlign'))
def test_samples(self): """... Compare the current result is the same as the existing one.""" # Test the automatic annotation with its default parameters only. # the place where are the samples to be tested. samples_path = os.path.join(paths.samples, "annotation-results") # each samples folder is tested for samples_folder in os.listdir(samples_path): if samples_folder.startswith("samples-") is False: continue # Create a Phonetizer for the given set of samples of the given language lang = samples_folder[-3:] pron_dict = os.path.join(paths.resources, "dict", lang + ".dict") tn = sppasPhon() tn.load_resources(dict_filename=pron_dict) # Apply Phonetization on each sample for filename in os.listdir( os.path.join(samples_path, samples_folder)): if filename.endswith("-token.xra") is False: continue # Get the expected result expected_result_filename = os.path.join( samples_path, samples_folder, filename[:-10] + "-phon.xra") if os.path.exists(expected_result_filename) is False: print("no match token/phon for:", filename) continue parser = sppasRW(expected_result_filename) expected_result = parser.read() # Estimate the result and check if it's like expected. input_file = os.path.join(samples_path, samples_folder, filename) result = tn.run([input_file]) expected_tier_phones = expected_result.find('Phones') if expected_tier_phones is not None: self.compare_tiers(expected_tier_phones, result.find('Phones'))
def run(self, input_file, opt_input_file=None, output_file=None): """Run the automatic annotation process on an input. input_filename is a tuple (audio, raw transcription) :param input_file: (list of str) (audio, ortho) :param opt_input_file: (list of str) ignored :param output_file: (str) the output file name :returns: (sppasTranscription) """ input_audio_filename = input_file[0] input_trans_filename = input_file[1] tier = self.convert(input_audio_filename, input_trans_filename) if tier is None: self.logfile.print_message(_info(1296), indent=2, status=-1) return None # Create the transcription to put the result trs_output = sppasTranscription(self.name) trs_output.set_meta('fill_ipus_result_of', input_audio_filename) trs_output.set_meta('fill_ipus_result_of_trs', input_trans_filename) trs_output.append(tier) extm = os.path.splitext(input_audio_filename)[1].lower()[1:] media = sppasMedia(os.path.abspath(input_audio_filename), mime_type="audio/" + extm) tier.set_media(media) # Save in a file if output_file is not None: parser = sppasRW(output_file) parser.write(trs_output) return trs_output
def count(self, *datafiles): """Count ngrams of order n from data files. :param datafiles: (*args) is a set of file names, with UTF-8 encoding. If the file contains more than one tier, only the first one is used. """ for filename in datafiles: parser = sppasRW(filename) trs = parser.read() if len(trs) == 0: continue tier = trs[0] for ann in tier: labels = ann.get_labels() for label in labels: for tag, score in label: if tag.is_empty() is False and\ tag.is_silence() is False: self.append_sentence(tag.get_content()) if self._n == 1: self._datacounts[((self._ss),)] = 0
along with this script. If not, see <http://www.gnu.org/licenses/>. """ import sys import os.path SPPAS_IS_HERE = os.path.join("..", "..") sys.path.append(SPPAS_IS_HERE) from sppas.src.anndata import sppasRW # ---------------------------------------------------------------------------- # Variables # ---------------------------------------------------------------------------- input_filename = 'F_F_B003-P9-merge.TextGrid' output_filename = input_filename.replace('.TextGrid', '.csv') # ---------------------------------------------------------------------------- # Main # ---------------------------------------------------------------------------- if __name__ == '__main__': # Create a parser object then parse the input file. parser = sppasRW(input_filename) trs = parser.read() # Save the Transcription object into a file. parser.set_filename(output_filename) parser.write(trs)