def test_multi_input(self):
op = Tokenize(inputs='x', outputs='x')
data = op.forward(data=self.multi_input, state={})
self.assertTrue(is_equal(data, self.multi_output))
def test_lower_case(self):
op = Tokenize(inputs='x', outputs='x', to_lower_case=True)
data = op.forward(data=self.lower_case_input, state={})
self.assertTrue(
is_equal(data, [['to', 'test', 'lowercase', 'parameter']]))
def test_single_input_tokenize_function(self):
op = Tokenize(inputs='x', outputs='x', tokenize_fn=self.tokenize_fn)
data = op.forward(data=self.single_input, state={})
self.assertTrue(is_equal(data, self.tokenize_fn_output))
