def test_python_tokenize_returns_expected(self, types_to_skip, source,
expected):
tokenizer = python_tokenizer.PythonTokenizer()
if types_to_skip is not None:
tokenizer.update_types_to_skip(types_to_skip)
actual = tokenizer.tokenize(source)
self.assertListEqual(expected, actual)
def setUp(self):
super().setUp()
self.tokenizer = python_tokenizer.PythonTokenizer()
subword_vocabulary_path = self.create_tempfile(
content=self._VOCABULARY_CONTENT).full_path
self.subword_text_encoder = text_encoder.SubwordTextEncoder(
subword_vocabulary_path)
def test_python_tokenize_abstract_returns_positioning(
self, source, expected_positions_and_kinds):
tokenizer = python_tokenizer.PythonTokenizer()
agnostic = tokenizer.tokenize_and_abstract(source)
actual_positions_and_kinds = tuple(
(m.metadata.start.line, m.metadata.start.column,
m.metadata.end.line, m.metadata.end.column, m.kind)
for m in agnostic)
self.assertSequenceEqual(expected_positions_and_kinds,
actual_positions_and_kinds)
def test_python_tokenize_raises_on_unknown_python_token_kind(
self, unused_function_mock):
tokenizer = python_tokenizer.PythonTokenizer()
with self.assertRaisesRegex(ValueError, 'While trying to turn'):
tokenizer.tokenize('source')
def test_python_untokenize_raises_as_expected(self, tokens):
tokenizer = python_tokenizer.PythonTokenizer()
with self.assertRaises(ValueError):
tokenizer.untokenize(tokens)
def test_python_with_comment_roundtrip(self, source, expected):
tokenizer = python_tokenizer.PythonTokenizer()
tokenizer.update_types_to_skip([])
self.assertEqual(expected,
tokenizer.untokenize(tokenizer.tokenize(source)))
def test_python_roundtrip(self, source, expected):
tokenizer = python_tokenizer.PythonTokenizer()
self.assertEqual(expected,
tokenizer.untokenize(tokenizer.tokenize(source)))
def test_python_tokenize_respects_mappings(self, mappings, source,
expected):
tokenizer = python_tokenizer.PythonTokenizer()
tokenizer.update_mappings(mappings)
actual = tokenizer.tokenize(source)
self.assertListEqual(expected, actual)
def test_python_tokenize_respects_reserved(self, reserved, source,
expected):
tokenizer = python_tokenizer.PythonTokenizer()
tokenizer.replace_reserved_keywords(reserved)
actual = tokenizer.tokenize(source)
self.assertListEqual(expected, actual)
def test_python_tokenize_handles_tokenization_errors(self, bad_code):
tokenizer = python_tokenizer.PythonTokenizer()
actual = tokenizer.tokenize(bad_code)
self.assertListEqual([_ERROR_NAME, _EOS_NAME], actual)
def test_python_tokenize_handles_tokenization_errors(self, bad_code):
tokenizer = python_tokenizer.PythonTokenizer()
tokenizer.replace_reserved_keywords(('___ERROR___', ))
actual = tokenizer.tokenize(bad_code)
self.assertListEqual([_ERROR_NAME, _EOS_NAME], actual)
