def remove_dloc(cls, morph_tier):
"""Remove dloc|dloc=DISLOC words on the morphology tier.
dloc|dloc=DISLOC stands for `„` on the utterance.
"""
morph_tier = morph_tier.replace('dloc|dloc=DISLOC', '')
return CHATUtteranceCleaner.remove_redundant_whitespaces(morph_tier)
def remove_terminator(cls, utterance):
"""Remove utterance terminator.
Also removes the colon and the dash.
"""
utterance = CHATUtteranceCleaner.remove_terminator(utterance)
return utterance.rstrip('-').rstrip(':')
def test_remove_omissions_multiple_omissions_no_space_before_terminator(
self):
"""Test with 3 omissions and no space before terminator."""
actual_output = CHATUtteranceCleaner.remove_omissions(
'0but where 0is my 0truck?')
desired_output = 'where my?'
self.assertEqual(actual_output, desired_output)
def remove_non_words(cls, morph_tier):
"""Remove all non-words from the morphology tier.
Non-words have the POS tag 'tag'.
"""
non_words_regex = re.compile(r'tag\|\S+')
morph_tier = non_words_regex.sub('', morph_tier)
return CHATUtteranceCleaner.remove_redundant_whitespaces(morph_tier)
def add_repetitions(cls, raw_utt, morph_tier):
"""Add repetitions to morphology tier."""
# check if there are any repetitions
if '[x ' in raw_utt:
# execute same cleaning steps except those for scoped symbols
for cleaning_method in [
CHATUtteranceCleaner.remove_terminator,
CHATUtteranceCleaner.unify_untranscribed,
CHATUtteranceCleaner.remove_events,
CHATUtteranceCleaner.remove_omissions,
CHATUtteranceCleaner.remove_linkers,
CHATUtteranceCleaner.remove_separators,
CHATUtteranceCleaner.remove_ca,
CHATUtteranceCleaner.remove_pauses_between_words,
CHATUtteranceCleaner.remove_commas,
CHATUtteranceCleaner.null_event_utterances
]:
raw_utt = cleaning_method(raw_utt)
# remove scoped symbols except for repetitions
scope_regex = re.compile(r'\[[^x].*?\]')
raw_utt = scope_regex.sub('', raw_utt)
raw_utt = CHATUtteranceCleaner.remove_redundant_whitespaces(
raw_utt)
# get words from utterance and morphology tier
utt_words = re.split(r'(?<!\[x) (?!\[x)', raw_utt)
morph_words = morph_tier.split(' ')
# check for misalignments
if len(utt_words) == len(morph_words):
morph_new = []
group = []
for uw, mw in zip(utt_words, morph_words):
morph_new.append(mw)
match = re.search(r'\[x (\d+)', uw)
if uw.startswith('<'):
group = [mw]
elif match:
reps = int(match.group(1))
if group:
group.append(mw)
morph_new += (reps - 1) * group
group = []
else:
morph_new += (reps - 1) * [mw]
elif group:
group.append(mw)
return ' '.join(morph_new)
return morph_tier
def remove_non_words(cls, morph_tier):
"""Remove all non-words from the morphology tier.
Non-words include:
end|end („)
cm|cm (,)
bq|bq (“)
eq|eq (”)
"""
non_words_regex = re.compile(r'end\|end'
r'|cm\|cm'
r'|bq\|bq'
r'|eq\|eq')
morph_tier = non_words_regex.sub('', morph_tier)
return CHATUtteranceCleaner.remove_redundant_whitespaces(morph_tier)
def test_remove_terminator_exclamation_point(self):
"""Test remove_terminator with exclamation_point."""
actual_output = CHATUtteranceCleaner.remove_terminator('sit down !')
desired_output = 'sit down'
self.assertEqual(actual_output, desired_output)
def test_remove_terminator_question_mark(self):
"""Test remove_terminator with question mark."""
actual_output = CHATUtteranceCleaner.remove_terminator(
'is that a carrot ?')
desired_output = 'is that a carrot'
self.assertEqual(actual_output, desired_output)
def test_remove_redundant_whitespace_leading_trailing_whitespace(self):
"""Test remove_redundant_whitespace with lead/end spaces."""
actual_output = CHATUtteranceCleaner.remove_redundant_whitespaces(
' h ')
desired_output = 'h'
self.assertEqual(actual_output, desired_output)
def test_remove_pauses_between_words_multiple_pauses(self):
"""Test remove_pauses with 3 pauses of different length."""
actual_output = CHATUtteranceCleaner.remove_pauses_between_words(
"I (.) don't (..) know (...) this.")
desired_output = "I don't know this."
self.assertEqual(actual_output, desired_output)
def test_remove_terminator_period(self):
"""Test remove_terminator with period."""
actual_output = CHATUtteranceCleaner.remove_terminator('I got cold .')
desired_output = 'I got cold'
self.assertEqual(actual_output, desired_output)
def remove_double_hashes(cls, morph_tier):
"""Remove ## from the morphology tier."""
morph_tier = re.sub(r'(^| )##( |$)', r'\1\2', morph_tier)
return CHATUtteranceCleaner.remove_redundant_whitespaces(morph_tier)
def clean_morph_tier(cls, morph_tier):
return CHATUtteranceCleaner.remove_terminator(morph_tier)
def test_remove_scoped_symbols_empty_string(self):
"""Test remove_scoped_symbols with an empty string."""
actual_output = CHATUtteranceCleaner.remove_scoped_symbols('')
desired_output = ''
self.assertEqual(actual_output, desired_output)
def test_remove_ca_marked_question(self):
"""Test remove_ca with marked question („)."""
actual_output = CHATUtteranceCleaner.remove_ca('Hey there„ what up no')
desired_output = 'Hey there what up no'
self.assertEqual(actual_output, desired_output)
def test_remove_redundant_whitespace_leading_trailing_creturns(self):
"""remove_redundant_whitespace with lead/end carriage return."""
actual_output = CHATUtteranceCleaner.remove_redundant_whitespaces(
'\rh\r')
desired_output = 'h'
self.assertEqual(actual_output, desired_output)
def test_remove_scoped_symbols_two_levels_nested(self):
"""Test remove_scoped_symbols with 2 levels of nestedness."""
utterance = "<that's mine <she said [=! cries]>> [=! slaps leg]"
actual_output = CHATUtteranceCleaner.remove_scoped_symbols(utterance)
desired_output = "that's mine she said"
self.assertEqual(actual_output, desired_output)
def test_remove_scoped_symbols_one_level_nested(self):
"""Test remove_scoped_symbols with 1 level of nestedness."""
utterance = "<that's mine [=! cries]>"
actual_output = CHATUtteranceCleaner.remove_scoped_symbols(utterance)
desired_output = "that's mine"
self.assertEqual(actual_output, desired_output)
def test_remove_scoped_symbols_not_nested(self):
"""Test remove_scoped_symbols with 2 not nested symbol pairs."""
utterance = "<that's mine> [=! cries]"
actual_output = CHATUtteranceCleaner.remove_scoped_symbols(utterance)
desired_output = "that's mine"
self.assertEqual(actual_output, desired_output)
def test_remove_pauses_between_words_empty_string(self):
"""Test remove_pauses with an empty string."""
actual_output = CHATUtteranceCleaner.remove_pauses_between_words('')
desired_output = ''
self.assertEqual(actual_output, desired_output)
def test_remove_terminator_trailing_off(self):
"""Test remove_terminator with trailing off."""
actual_output = CHATUtteranceCleaner.remove_terminator(
'*SAR: smells good enough for +...')
desired_output = '*SAR: smells good enough for'
self.assertEqual(actual_output, desired_output)
def test_remove_ca_falling_rising_mark(self):
"""Test remove_ca with 3 rising (↑) and 1 falling (↓) mark."""
actual_output = CHATUtteranceCleaner.remove_ca(
'Hey↑ there↓ what up↑ no↑')
desired_output = 'Hey there what up no'
self.assertEqual(actual_output, desired_output)
def test_remove_ca_quotations(self):
"""Test remove_ca with opening and closing quotations."""
actual_output = CHATUtteranceCleaner.remove_ca(
'“Hey there what up no”')
desired_output = 'Hey there what up no'
self.assertEqual(actual_output, desired_output)
def test_remove_redundant_whitespace_combinations(self):
"""Test remove_redundant_whitespace with mixed space chars."""
actual_output = CHATUtteranceCleaner.remove_redundant_whitespaces(
'\n\t \r\r h \nh \t\t\n\r')
desired_output = 'h h'
self.assertEqual(actual_output, desired_output)
def clean_morpheme_word(cls, morpheme_word):
return CHATUtteranceCleaner.remove_terminator(morpheme_word)
def test_remove_redundant_whitespace_empty_string(self):
"""Test remove_redundant_whitespace with empty string."""
actual_output = CHATUtteranceCleaner.remove_redundant_whitespaces('')
desired_output = ''
self.assertEqual(actual_output, desired_output)
def test_remove_scoped_symbols_no_withespace(self):
"""Test remove_scoped_symbols with missing whitespace."""
utterance = '0[=! just testing something]'
actual_output = CHATUtteranceCleaner.remove_scoped_symbols(utterance)
desired_output = '0'
self.assertEqual(actual_output, desired_output)
def test_remove_ca_satellite_marker(self):
"""Test remove_ca with satellite marker (‡)."""
actual_output = CHATUtteranceCleaner.remove_ca('no ‡ Mommy no go')
desired_output = 'no Mommy no go'
self.assertEqual(actual_output, desired_output)
def clean_morph_tier(cls, morph_tier):
morph_tier = CHATUtteranceCleaner.remove_terminator(morph_tier)
return cls.remove_non_words(morph_tier)
def test_remove_redundant_whitespace_multiple_creturns(self):
"""remove_redundant_whitespace with multiple carriage return."""
actual_output = CHATUtteranceCleaner.remove_redundant_whitespaces(
'\r\rh\r\rh\r\r')
desired_output = 'h h'
self.assertEqual(actual_output, desired_output)