def test_multi_input(self): op = Tokenize(inputs='x', outputs='x') data = op.forward(data=self.multi_input, state={}) self.assertTrue(is_equal(data, self.multi_output))
def test_lower_case(self): op = Tokenize(inputs='x', outputs='x', to_lower_case=True) data = op.forward(data=self.lower_case_input, state={}) self.assertTrue( is_equal(data, [['to', 'test', 'lowercase', 'parameter']]))
def test_single_input_tokenize_function(self): op = Tokenize(inputs='x', outputs='x', tokenize_fn=self.tokenize_fn) data = op.forward(data=self.single_input, state={}) self.assertTrue(is_equal(data, self.tokenize_fn_output))