def test_python_tokenize_returns_expected(self, types_to_skip, source, expected): tokenizer = python_tokenizer.PythonTokenizer() if types_to_skip is not None: tokenizer.update_types_to_skip(types_to_skip) actual = tokenizer.tokenize(source) self.assertListEqual(expected, actual)
def setUp(self): super().setUp() self.tokenizer = python_tokenizer.PythonTokenizer() subword_vocabulary_path = self.create_tempfile( content=self._VOCABULARY_CONTENT).full_path self.subword_text_encoder = text_encoder.SubwordTextEncoder( subword_vocabulary_path)
def test_python_tokenize_abstract_returns_positioning( self, source, expected_positions_and_kinds): tokenizer = python_tokenizer.PythonTokenizer() agnostic = tokenizer.tokenize_and_abstract(source) actual_positions_and_kinds = tuple( (m.metadata.start.line, m.metadata.start.column, m.metadata.end.line, m.metadata.end.column, m.kind) for m in agnostic) self.assertSequenceEqual(expected_positions_and_kinds, actual_positions_and_kinds)
def test_python_tokenize_raises_on_unknown_python_token_kind( self, unused_function_mock): tokenizer = python_tokenizer.PythonTokenizer() with self.assertRaisesRegex(ValueError, 'While trying to turn'): tokenizer.tokenize('source')
def test_python_untokenize_raises_as_expected(self, tokens): tokenizer = python_tokenizer.PythonTokenizer() with self.assertRaises(ValueError): tokenizer.untokenize(tokens)
def test_python_with_comment_roundtrip(self, source, expected): tokenizer = python_tokenizer.PythonTokenizer() tokenizer.update_types_to_skip([]) self.assertEqual(expected, tokenizer.untokenize(tokenizer.tokenize(source)))
def test_python_roundtrip(self, source, expected): tokenizer = python_tokenizer.PythonTokenizer() self.assertEqual(expected, tokenizer.untokenize(tokenizer.tokenize(source)))
def test_python_tokenize_respects_mappings(self, mappings, source, expected): tokenizer = python_tokenizer.PythonTokenizer() tokenizer.update_mappings(mappings) actual = tokenizer.tokenize(source) self.assertListEqual(expected, actual)
def test_python_tokenize_respects_reserved(self, reserved, source, expected): tokenizer = python_tokenizer.PythonTokenizer() tokenizer.replace_reserved_keywords(reserved) actual = tokenizer.tokenize(source) self.assertListEqual(expected, actual)
def test_python_tokenize_handles_tokenization_errors(self, bad_code): tokenizer = python_tokenizer.PythonTokenizer() actual = tokenizer.tokenize(bad_code) self.assertListEqual([_ERROR_NAME, _EOS_NAME], actual)
def test_python_tokenize_handles_tokenization_errors(self, bad_code): tokenizer = python_tokenizer.PythonTokenizer() tokenizer.replace_reserved_keywords(('___ERROR___', )) actual = tokenizer.tokenize(bad_code) self.assertListEqual([_ERROR_NAME, _EOS_NAME], actual)