def test_is_valid_token_sequence__with_non_string_token_sequence__returns_null(
self):
self.assertIsNone(is_valid_token_sequence([], 1))
self.assertIsNone(is_valid_token_sequence([], True))
self.assertIsNone(is_valid_token_sequence([], -1.0))
self.assertIsNone(is_valid_token_sequence([], complex(-1.0)))
self.assertIsNone(is_valid_token_sequence([], [1]))
self.assertIsNone(is_valid_token_sequence([], (1, )))
self.assertIsNone(is_valid_token_sequence([], {1}))
self.assertIsNone(is_valid_token_sequence([], {'a': 1}))
self.assertIsNone(is_valid_token_sequence([], range(1)))
self.assertIsNone(is_valid_token_sequence([], int))
def test_is_valid_token_sequence__with_valid_token_sequence__returns_true(
self, token_alphabet, token_length, num_tokens, token_seq_sep,
terminate_token_seq_with_sep):
tokens = [
''.join(
np.random.choice(list(token_alphabet),
size=token_length).tolist())
for i in range(num_tokens)
]
token_seq_len = np.random.choice(range(1, num_tokens + 1))
token_seq = '{}'.format(token_seq_sep).join(
np.random.choice(tokens, size=token_seq_len, replace=False))
if terminate_token_seq_with_sep:
token_seq += token_seq_sep
self.assertTrue(
is_valid_token_sequence(tokens, token_seq, token_seq_sep))
def test_is_valid_token_sequence__with_invalid_token_sequence_containing_duplicate_token__returns_false(
self, token_alphabet, token_length, num_tokens, token_seq_sep,
terminate_token_seq_with_sep):
tokens = [
''.join(
np.random.choice(list(token_alphabet),
size=token_length).tolist())
for i in range(num_tokens)
]
token_seq_len = np.random.choice(range(1, num_tokens + 1))
token_seq = '{}'.format(token_seq_sep).join(
np.random.choice(tokens, size=token_seq_len, replace=False))
token_seq += token_seq_sep + [t for t in tokens if t in token_seq][0]
if terminate_token_seq_with_sep:
token_seq += token_seq_sep
self.assertFalse(
is_valid_token_sequence(tokens, token_seq, token_seq_sep))
def test_is_valid_token_sequence__with_some_non_string_or_empty_string_fixed_tokens__returns_null(
self):
self.assertIsNone(is_valid_token_sequence(['token', ''], 'seq'))
self.assertIsNone(is_valid_token_sequence(['token', 1], 'seq'))
self.assertIsNone(is_valid_token_sequence(['token', True], 'seq'))
self.assertIsNone(is_valid_token_sequence(['token', -1.0], 'seq'))
self.assertIsNone(
is_valid_token_sequence(['token', complex(-1.0)], 'seq'))
self.assertIsNone(is_valid_token_sequence(['token', [1]], 'seq'))
self.assertIsNone(is_valid_token_sequence(['token', (1, )], 'seq'))
self.assertIsNone(is_valid_token_sequence(['token', {1}], 'seq'))
self.assertIsNone(is_valid_token_sequence(['token', {'a': 1}], 'seq'))
self.assertIsNone(is_valid_token_sequence(['token', range(1)], 'seq'))
self.assertIsNone(is_valid_token_sequence(['token', int], 'seq'))