def test_add_unknown_word_to_utterances_with_zero_max_unknownword(self):
# Given
utterances = [text_to_utterance("yo")]
replacement_string = "yo"
unknown_word_prob = 1
max_unknown_words = 0
random_state = np.random.RandomState
# When / Then
with self.fail_if_exception("Failed to augment utterances with "
"unknown_word_prob=0"):
add_unknown_word_to_utterances(utterances, replacement_string,
unknown_word_prob,
max_unknown_words, random_state)
def test_add_unknown_words_to_utterances(self):
# Given
base_utterances = {
"data": [
{
"text": "hello "
},
{
"text": " you ",
"entity": "you"
},
{
"text": " how are you "
},
{
"text": "cat",
"entity": "cat"
}
]
}
utterances = []
for _ in range(6):
utterances.append(deepcopy(base_utterances))
rand_it = cycle([0, 1])
def mocked_rand():
return next(rand_it)
max_unknown_words = 3
rg_it = cycle([i for i in range(1, max_unknown_words + 1)])
def mocked_randint(a, b): # pylint: disable=unused-argument
return next(rg_it)
unknownword_prob = .5
random_state = MagicMock()
random_state_rand = MagicMock()
random_state_rand.side_effect = mocked_rand
random_state_choice = MagicMock()
random_state_choice.side_effect = mocked_randint
random_state.rand = random_state_rand
random_state.randint = random_state_choice
# When
replacement_string = "unknownword"
noisy_utterances = add_unknown_word_to_utterances(
utterances, unknown_word_prob=unknownword_prob,
replacement_string=replacement_string,
max_unknown_words=max_unknown_words,
random_state=random_state
)
# Then
expected_utterances = [
{
"data": [
{
"text": "hello "
},
{
"text": " you ",
"entity": "you"
},
{
"text": " how are you "
},
{
"text": "cat",
"entity": "cat"
},
{
"text": " unknownword"
}
]
},
{
"data": [
{
"text": "hello "
},
{
"text": " you ",
"entity": "you"
},
{
"text": " how are you "
},
{
"text": "cat",
"entity": "cat"
},
]
},
{
"data": [
{
"text": "hello "
},
{
"text": " you ",
"entity": "you"
},
{
"text": " how are you "
},
{
"text": "cat",
"entity": "cat"
},
{
"text": " unknownword unknownword"
}
]
},
{
"data": [
{
"text": "hello "
},
{
"text": " you ",
"entity": "you"
},
{
"text": " how are you "
},
{
"text": "cat",
"entity": "cat"
},
]
},
{
"data": [
{
"text": "hello "
},
{
"text": " you ",
"entity": "you"
},
{
"text": " how are you "
},
{
"text": "cat",
"entity": "cat"
},
{
"text": " unknownword unknownword unknownword"
}
]
},
{
"data": [
{
"text": "hello "
},
{
"text": " you ",
"entity": "you"
},
{
"text": " how are you "
},
{
"text": "cat",
"entity": "cat"
},
]
}
]
self.assertEqual(expected_utterances, noisy_utterances)
def test_add_unknown_words_to_utterances(self):
# Given
utterances = [{
"data": [{
"text": "hello "
}, {
"text": " you ",
"entity": "you"
}, {
"text": " how are you "
}, {
"text": "dude",
"entity": "you"
}]
}, {
"data": [{
"text": "hello "
}, {
"text": "dude",
"entity": "you"
}, {
"text": " how are you "
}, {
"text": " you ",
"entity": "you"
}]
}]
unknownword_prob = .5
random_state = np.random.RandomState(1)
# When
replacement_string = "unknownword"
noisy_utterances = add_unknown_word_to_utterances(
utterances,
unknown_word_prob=unknownword_prob,
replacement_string=replacement_string,
random_state=random_state)
# Then
expected_utterances = [{
"data": [{
"text": "hello "
}, {
"text": " unknownword ",
"entity": "you"
}, {
"text": " how are you "
}, {
"text": "dude",
"entity": "you"
}]
}, {
"data": [{
"text": "hello "
}, {
"text": "unknownword",
"entity": "you"
}, {
"text": " how are you "
}, {
"text": " unknownword ",
"entity": "you"
}]
}]
self.assertEqual(expected_utterances, noisy_utterances)
def test_add_unknown_words_to_utterances(self):
# Given
utterances = [
{
"data": [
{
"text": "hello "
},
{
"text": " you ",
"entity": "you"
},
{
"text": " how are you "
},
{
"text": "dude",
"entity": "you"
}
]
},
{
"data": [
{
"text": "hello "
},
{
"text": "dude",
"entity": "you"
},
{
"text": " how are you "
},
{
"text": " you ",
"entity": "you"
}
]
}
]
unknownword_prob = .5
random_state = np.random.RandomState(1)
# When
replacement_string = "unknownword"
noisy_utterances = add_unknown_word_to_utterances(
utterances, unknown_word_prob=unknownword_prob,
replacement_string=replacement_string, random_state=random_state
)
# Then
expected_utterances = [
{
"data": [
{
"text": "hello "
},
{
"text": " unknownword ",
"entity": "you"
},
{
"text": " how are you "
},
{
"text": "dude",
"entity": "you"
}
]
},
{
"data": [
{
"text": "hello "
},
{
"text": "unknownword",
"entity": "you"
},
{
"text": " how are you "
},
{
"text": " unknownword ",
"entity": "you"
}
]
}
]
self.assertEqual(expected_utterances, noisy_utterances)