def run(self, input_filename, output_filename=None):
""" Perform the TGA estimation process.
:param input_filename: (str) Name of the input file with the aligned syllables
:param output_filename: (str) Name of the resulting file with TGA
"""
self.print_filename(input_filename)
self.print_options()
self.print_diagnosis(input_filename)
# Get the tier to syllabify
parser = sppasRW(input_filename)
trs_input = parser.read()
tier_input = sppasFindTier.aligned_syllables(trs_input)
# Create the transcription result
trs_output = sppasTranscription("Time Group Analyzer")
trs_output.set_meta('tga_result_of', input_filename)
# Estimate TGA on the tier
trs_output = self.convert(tier_input)
# Save in a file
if output_filename is not None:
if len(trs_output) > 0:
parser = sppasRW(output_filename)
parser.write(trs_output)
self.print_filename(output_filename, status=0)
else:
raise EmptyOutputError
return trs_output
def run(self, input_filename, output_filename=None):
""" Run the INTSINT annotation process on an input file.
:param input_filename: (str) the input file name with momel
:param output_filename: (str) the output file name of the INTSINT tier
:returns: (sppasTranscription)
"""
self.print_filename(input_filename)
# Get the tier to be annotated.
parser = sppasRW(input_filename)
trs_input = parser.read()
tier_input = sppasFindTier.pitch_anchors(trs_input)
# Annotate the tier
targets = sppasIntsint.tier_to_anchors(tier_input)
tones = self.intsint.annotate(targets)
tier_intsint = sppasIntsint.tones_to_tier(tones, tier_input)
# Create the transcription result
trs_output = sppasTranscription(self.name)
trs_output.append(tier_intsint)
trs_output.set_meta('intsint_result_of', input_filename)
# Save in a file
if output_filename is not None:
parser = sppasRW(output_filename)
parser.write(trs_output)
self.print_filename(output_filename, status=0)
return trs_output
def run(self, input_filename, output_filename=None):
""" Perform the Syllabification process.
:param input_filename: (str) Name of the input file with the aligned phonemes
:param output_filename: (str) Name of the resulting file with syllabification
"""
self.print_filename(input_filename)
self.print_options()
self.print_diagnosis(input_filename)
# Get the tier to syllabify
parser = sppasRW(input_filename)
trs_input = parser.read()
tier_input = sppasFindTier.aligned_phones(trs_input)
# Create the transcription result
trs_output = sppasTranscription("Syllabification")
trs_output.set_meta('syllabification_result_of', input_filename)
# Syllabify the tier
if self._options['usesphons'] is True:
tier_syll = self.convert(tier_input)
trs_output.append(tier_syll)
if self._options['createclasses']:
trs_output.append(self.make_classes(tier_syll))
# Extra tier: syllabify between given intervals
if self._options['usesintervals'] is True:
intervals = trs_input.find(self._options['tiername'])
if intervals is None:
self.print_message(MSG_NO_TIER.format(tiername=self._options['tiername']),
indent=2, status=WARNING_ID)
else:
tier_syll_int = self.convert(tier_input, intervals)
tier_syll_int.set_name("SyllAlign-Intervals")
tier_syll_int.set_meta('syllabification_used_intervals', intervals.get_name())
trs_output.append(tier_syll_int)
if self._options['createclasses']:
t = self.make_classes(tier_syll_int)
t.set_name("SyllClassAlign-Intervals")
trs_output.append(t)
# Save in a file
if output_filename is not None:
if len(trs_output) > 0:
parser = sppasRW(output_filename)
parser.write(trs_output)
self.print_filename(output_filename, status=0)
else:
raise EmptyOutputError
return trs_output
def compare_folders(self, samples_folder, expected_result_dir, sa):
# Apply Alignment on each sample
for filename in os.listdir(os.path.join(paths.samples, samples_folder)):
if filename.endswith(".wav") is False:
continue
# Get the expected result
expected_result_filename = os.path.join(
expected_result_dir,
filename[:-4] + "-palign.xra")
if os.path.exists(expected_result_filename) is False:
print("no existing alignment result {:s}".format(expected_result_filename))
continue
parser = sppasRW(expected_result_filename)
expected_result = parser.read()
expected_tier_phones = expected_result.find('PhonAlign')
if expected_tier_phones is None:
print("malformed alignment result for:", filename)
continue
# Estimate a result and check if it's like expected.
audio_file = os.path.join(paths.samples, samples_folder, filename)
phn_file = os.path.join(expected_result_dir, filename.replace('.wav', '-phon.xra'))
tok_file = os.path.join(expected_result_dir, filename.replace('.wav', '-token.xra'))
result_file = os.path.join(paths.samples, samples_folder, filename.replace('.wav', '-palign.xra'))
expected_result = sa.run([audio_file, phn_file], [tok_file], result_file)
print('Evaluate:', audio_file)
self.compare_tiers(expected_tier_phones,
expected_result.find('PhonAlign'))
def get_tier(filename, tier_name, verbose=True):
""" Get a tier from a file.
:param filename: (str) Name of the annotated file.
:param tier_name: (str) Name of the tier
:param verbose: (bool) Print message
:returns: (Tier)
"""
# Read an annotated file.
if verbose:
print("Read file: {:s}".format(filename))
parser = sppasRW(filename)
trs = parser.read()
if verbose:
print(" ... [ OK ] ")
# Get the expected tier
if verbose:
print("Get tier {:s}".format(tier_name))
tier = trs.find(tier_name, case_sensitive=False)
if tier is None:
print("Tier not found.")
sys.exit(1)
if verbose:
print(" ... [ OK ] ")
return tier
def _merge(self):
"""Merge all annotated files."""
self._progress.set_header("Merge all annotations in a file")
self._progress.update(0, "")
# Get the list of files with the ".wav" extension
filelist = self.get_annot_files(
pattern="", extensions=sppas.src.audiodata.aio.extensions)
total = len(filelist)
output_format = self._parameters.get_output_format()
for i, f in enumerate(filelist):
nbfiles = 0
# Change f, to allow "replace" to work properly
basef = os.path.splitext(f)[0]
self._logfile.print_message("File: " + f, indent=0)
self._progress.set_text(
os.path.basename(f) + " (" + str(i + 1) + "/" + str(total) +
")")
# Add all files content in the same order than to annotate
trs = sppasTranscription()
nbfiles += self.__add_trs(trs, basef + output_format)
for s in range(self._parameters.get_step_numbers()):
ann_key = self._parameters.get_step_key(s)
a = self._get_instance(ann_key)
pattern = a.get_pattern()
if len(pattern) > 0:
nbfiles += self.__add_trs(trs,
basef + pattern + output_format)
if nbfiles > 1:
try:
info_tier = sppasMetaInfoTier(trs)
tier = info_tier.create_time_tier(
trs.get_min_loc().get_midpoint(),
trs.get_max_loc().get_midpoint())
trs.append(tier)
parser = sppasRW(basef + "-merge.xra")
parser.write(trs)
self._logfile.print_message(basef + "-merge.xra",
indent=1,
status=0)
except Exception as e:
self._logfile.print_message(str(e), indent=1, status=-1)
self._progress.set_fraction(float((i + 1)) / float(total))
self._logfile.print_newline()
self._progress.update(1, "Completed.")
self._progress.set_header("")
def run(self, input_filename, output_filename=None):
""" Run the Phonetization process on an input file.
:param input_filename (str) Name of the file including a tokenization
:param output_filename (str) Name of the resulting file with phonetization
:returns: (sppasTranscription)
"""
self.print_filename(input_filename)
self.print_options()
self.print_diagnosis(input_filename)
# Get the tier to be phonetized.
pattern = ""
if self._options['usestdtokens'] is True:
pattern = "std"
parser = sppasRW(input_filename)
trs_input = parser.read()
tier_input = sppasFindTier.tokenization(trs_input, pattern)
# Phonetize the tier
tier_phon = self.convert(tier_input)
# Create the transcription result
trs_output = sppasTranscription("Phonetization")
if tier_phon is not None:
trs_output.append(tier_phon)
trs_output.set_meta('text_phonetization_result_of', input_filename)
trs_output.set_meta('text_phonetization_dict',
self.phonetizer.get_dict_filename())
# Save in a file
if output_filename is not None:
if len(trs_output) > 0:
parser = sppasRW(output_filename)
parser.write(trs_output)
self.print_filename(output_filename, status=0)
else:
raise EmptyOutputError
return trs_output
def test_TGA_on_sample(self):
# This is one of the samples proposed in-line by Dafydd
path = os.path.join(DATA, "tga.TextGrid")
parser = sppasRW(path)
trs = parser.read()
tier = trs.find('Syllables')
t = sppasTGA()
timegroups = t.syllables_to_timegroups(tier)
tg_dur = t.timegroups_to_durations(tier, timegroups)
tga = TimeGroupAnalysis(tg_dur)
occurrences = tga.len()
self.assertEqual(34, len(occurrences))
total = tga.total()
mean = tga.mean()
median = tga.median()
stdev = tga.stdev()
npvi = tga.nPVI()
reglin = tga.intercept_slope_original()
self.assertEqual(3, occurrences['tg_1'])
self.assertEqual(0.57, round(total['tg_1'], 2))
self.assertEqual(0.19, round(mean['tg_1'], 2))
self.assertEqual(0.14, round(median['tg_1'], 2))
self.assertEqual(0.13928, round(stdev['tg_1'], 5))
self.assertEqual(94, round(npvi['tg_1'], 0))
i, s = reglin['tg_1']
self.assertEqual(0.025, round(i, 3))
self.assertEqual(0.165, round(s, 3))
self.assertEqual(4, occurrences['tg_33'])
self.assertEqual(0.78, round(total['tg_33'], 2))
self.assertEqual(0.195, round(mean['tg_33'], 3))
self.assertEqual(0.06062, round(stdev['tg_33'], 5))
self.assertEqual(53, round(npvi['tg_33'], 0))
i, s = reglin['tg_33']
self.assertEqual(0.156, round(i, 3))
self.assertEqual(0.026, round(s, 3))
# do the job from a tier
trs1 = t.convert(tier)
t.set_intercept_slope_annotationpro(True)
t.set_intercept_slope_original(False)
self.assertEqual(10, len(trs1))
# do the same from the file
trs2 = t.run(path)
self.assertEqual(10, len(trs2))
def test_samples(self):
""" ... Compare the current result is the same as the existing one. """
for samples_folder in os.listdir(SAMPLES_PATH):
if samples_folder.startswith("samples-") is False:
continue
expected_result_dir = os.path.join(SAMPLES_PATH,
"annotation-results",
samples_folder)
# Create a TextNormalizer for the given set of samples
lang = samples_folder[-3:]
vocab = os.path.join(RESOURCES_PATH, "vocab", lang + ".vocab")
tn = sppasTextNorm(vocab, lang)
tn.set_faked(True)
tn.set_std(True)
tn.set_custom(True)
# Apply TextNormalization on each sample
for filename in os.listdir(
os.path.join(SAMPLES_PATH, samples_folder)):
if filename.endswith(".TextGrid") is False:
continue
# Get the expected result
expected_result_filename = os.path.join(
expected_result_dir, filename[:-9] + "-token.xra")
if os.path.exists(expected_result_filename) is False:
continue
parser = sppasRW(expected_result_filename)
expected_result = parser.read()
# Estimate the result and check if it's like expected.
result = tn.run(
os.path.join(SAMPLES_PATH, samples_folder, filename))
expected_tier_tokens = expected_result.find('Tokens')
if expected_tier_tokens is not None:
self.compare_tiers(expected_tier_tokens,
result.find('Tokens'))
expected_tier_tokens = expected_result.find('TokensStd')
if expected_tier_tokens is not None:
self.compare_tiers(expected_tier_tokens,
result.find('TokensStd'))
expected_tier_tokens = expected_result.find('TokensCustom')
if expected_tier_tokens is not None:
self.compare_tiers(expected_tier_tokens,
result.find('TokensCustom'))
def convert(self, input_audio_filename, input_filename):
"""Return a tier with transcription aligned to the audio.
:param input_audio_filename: (str) Input audio file
:param input_filename: (str) Input transcription file
"""
# Get audio and the channel we'll work on
audio_speech = sppas.src.audiodata.aio.open(input_audio_filename)
n = audio_speech.get_nchannels()
if n != 1:
raise AudioChannelError(n)
idx = audio_speech.extract_channel()
channel = audio_speech.get_channel(idx)
# Get the units we'll work on
parser = sppasRW(input_filename)
trs = parser.read()
if len(trs) > 1:
pass
if len(trs[0]) == 0:
pass
units = list()
for a in trs[0]:
units.append(a.serialize_labels())
ipus = [u for u in units if u != SIL_ORTHO]
# Create the instance to fill in IPUs
filler = FillIPUs(channel, units)
filler.set_min_ipu(self._options['min_ipu'])
filler.set_min_sil(self._options['min_sil'])
n = filler.fix_threshold_durations()
if n != len(ipus):
return
# Process the data.
tracks = filler.get_tracks(time_domain=True)
tier = sppasSearchIPUs.tracks_to_tier(tracks, channel.get_duration(),
filler.get_vagueness())
tier.set_name('Transcription')
self._set_meta(filler, tier)
i = 0
for a in tier:
if a.get_best_tag().is_silence() is False:
a.set_labels([sppasLabel(sppasTag(ipus[i]))])
i += 1
return tier
def __add_trs(self, trs, trs_inputfile):
try:
parser = sppasRW(trs_inputfile)
trs_input = parser.read(trs_inputfile)
except IOError:
return 0
for tier in trs_input:
already_in = False
if trs.is_empty() is False:
tier_name = tier.get_name()
for t in trs:
if t.get_name() == tier_name:
already_in = True
if already_in is False:
trs.append(tier)
return 1
def test_samples(self):
"""... Compare the current result is the same as the existing one."""
# the place where are the samples to be tested.
samples_path = os.path.join(paths.samples, "annotation-results")
# each samples folder is tested
for samples_folder in os.listdir(samples_path):
if samples_folder.startswith("samples-") is False:
continue
# Create a Syllabifier for the given set of samples of the given language
lang = samples_folder[-3:]
rules_file = os.path.join(paths.resources, "syll", "syllConfig-"+lang+".txt")
if os.path.exists(rules_file) is False:
continue
tn = sppasSyll()
tn.load_resources(rules_file)
# Apply Syllabification on each sample
for filename in os.listdir(os.path.join(samples_path, samples_folder)):
if filename.endswith("-palign.xra") is False:
continue
# Get the expected result
expected_result_filename = os.path.join(samples_path, samples_folder,
filename[:-11] + "-salign.xra")
if os.path.exists(expected_result_filename) is False:
print("no match palign/salign for: {:s}".format(filename))
continue
try:
parser = sppasRW(expected_result_filename)
expected_result = parser.read()
except AioEncodingError:
continue
# Estimate the result and check if it's like expected.
input_file = os.path.join(samples_path, samples_folder, filename)
result = tn.run([input_file])
expected_tier_syll = expected_result.find('SyllAlign')
if expected_tier_syll is not None:
self.compare_tiers(expected_tier_syll, result.find('SyllAlign'))
def test_samples(self):
"""... Compare the current result is the same as the existing one."""
# Test the automatic annotation with its default parameters only.
# the place where are the samples to be tested.
samples_path = os.path.join(paths.samples, "annotation-results")
# each samples folder is tested
for samples_folder in os.listdir(samples_path):
if samples_folder.startswith("samples-") is False:
continue
# Create a Phonetizer for the given set of samples of the given language
lang = samples_folder[-3:]
pron_dict = os.path.join(paths.resources, "dict", lang + ".dict")
tn = sppasPhon()
tn.load_resources(dict_filename=pron_dict)
# Apply Phonetization on each sample
for filename in os.listdir(
os.path.join(samples_path, samples_folder)):
if filename.endswith("-token.xra") is False:
continue
# Get the expected result
expected_result_filename = os.path.join(
samples_path, samples_folder, filename[:-10] + "-phon.xra")
if os.path.exists(expected_result_filename) is False:
print("no match token/phon for:", filename)
continue
parser = sppasRW(expected_result_filename)
expected_result = parser.read()
# Estimate the result and check if it's like expected.
input_file = os.path.join(samples_path, samples_folder,
filename)
result = tn.run([input_file])
expected_tier_phones = expected_result.find('Phones')
if expected_tier_phones is not None:
self.compare_tiers(expected_tier_phones,
result.find('Phones'))
def run(self, input_file, opt_input_file=None, output_file=None):
"""Run the automatic annotation process on an input.
input_filename is a tuple (audio, raw transcription)
:param input_file: (list of str) (audio, ortho)
:param opt_input_file: (list of str) ignored
:param output_file: (str) the output file name
:returns: (sppasTranscription)
"""
input_audio_filename = input_file[0]
input_trans_filename = input_file[1]
tier = self.convert(input_audio_filename, input_trans_filename)
if tier is None:
self.logfile.print_message(_info(1296), indent=2, status=-1)
return None
# Create the transcription to put the result
trs_output = sppasTranscription(self.name)
trs_output.set_meta('fill_ipus_result_of', input_audio_filename)
trs_output.set_meta('fill_ipus_result_of_trs', input_trans_filename)
trs_output.append(tier)
extm = os.path.splitext(input_audio_filename)[1].lower()[1:]
media = sppasMedia(os.path.abspath(input_audio_filename),
mime_type="audio/" + extm)
tier.set_media(media)
# Save in a file
if output_file is not None:
parser = sppasRW(output_file)
parser.write(trs_output)
return trs_output
def count(self, *datafiles):
"""Count ngrams of order n from data files.
:param datafiles: (*args) is a set of file names, with UTF-8 encoding.
If the file contains more than one tier, only the first one is used.
"""
for filename in datafiles:
parser = sppasRW(filename)
trs = parser.read()
if len(trs) == 0:
continue
tier = trs[0]
for ann in tier:
labels = ann.get_labels()
for label in labels:
for tag, score in label:
if tag.is_empty() is False and\
tag.is_silence() is False:
self.append_sentence(tag.get_content())
if self._n == 1:
self._datacounts[((self._ss),)] = 0
along with this script. If not, see <http://www.gnu.org/licenses/>.
"""
import sys
import os.path
SPPAS_IS_HERE = os.path.join("..", "..")
sys.path.append(SPPAS_IS_HERE)
from sppas.src.anndata import sppasRW
# ----------------------------------------------------------------------------
# Variables
# ----------------------------------------------------------------------------
input_filename = 'F_F_B003-P9-merge.TextGrid'
output_filename = input_filename.replace('.TextGrid', '.csv')
# ----------------------------------------------------------------------------
# Main
# ----------------------------------------------------------------------------
if __name__ == '__main__':
# Create a parser object then parse the input file.
parser = sppasRW(input_filename)
trs = parser.read()
# Save the Transcription object into a file.
parser.set_filename(output_filename)
parser.write(trs)
