def generate_tokens(self): """Tokenize the file and yield the tokens. :raises flake8.exceptions.InvalidSyntax: If a :class:`tokenize.TokenError` is raised while generating tokens. """ try: for token in tokenize.generate_tokens(self.next_line): if token[2][0] > self.total_lines: break self.tokens.append(token) yield token except (tokenize.TokenError, SyntaxError) as exc: raise exceptions.InvalidSyntax(exception=exc)
def file_tokens(self): """Return the complete set of tokens for a file. Accessing this attribute *may* raise an InvalidSyntax exception. :raises: flake8.exceptions.InvalidSyntax """ if self._file_tokens is None: line_iter = iter(self.lines) try: self._file_tokens = list( tokenize.generate_tokens(lambda: next(line_iter))) except tokenize.TokenError as exc: raise exceptions.InvalidSyntax(exc.message, exception=exc) return self._file_tokens
def generate_tokens(self): """Tokenize the file and yield the tokens. :raises flake8.exceptions.InvalidSyntax: If a :class:`tokenize.TokenError` is raised while generating tokens. """ try: for token in tokenize.generate_tokens(self.next_line): if token[2][0] > self.total_lines: break self.tokens.append(token) yield token # NOTE(sigmavirus24): pycodestyle was catching both a SyntaxError # and a tokenize.TokenError. In looking a the source on Python 2 and # Python 3, the SyntaxError should never arise from generate_tokens. # If we were using tokenize.tokenize, we would have to catch that. Of # course, I'm going to be unsurprised to be proven wrong at a later # date. except tokenize.TokenError as exc: raise exceptions.InvalidSyntax(exc.message, exception=exc)
"""Tests for the flake8.exceptions module.""" import pickle import pytest from flake8 import exceptions @pytest.mark.parametrize( 'err', (exceptions.FailedToLoadPlugin( plugin_name='plugin_name', exception=ValueError('boom!'), ), exceptions.InvalidSyntax(exception=ValueError('Unexpected token: $')), exceptions.PluginRequestedUnknownParameters( plugin={'plugin_name': 'plugin_name'}, exception=ValueError('boom!'), ), exceptions.PluginExecutionFailed( plugin={'plugin_name': 'plugin_name'}, exception=ValueError('boom!'), )), ) def test_pickleable(err): """Ensure that our exceptions can cross pickle boundaries.""" for proto in range(pickle.HIGHEST_PROTOCOL + 1): new_err = pickle.loads(pickle.dumps(err, protocol=proto)) assert str(err) == str(new_err) orig_e = err.original_exception new_e = new_err.original_exception assert (type(orig_e), orig_e.args) == (type(new_e), new_e.args)
class TestInvalidSyntax(_ExceptionTest): """Tests for the InvalidSyntax exception.""" err = exceptions.InvalidSyntax(exception=ValueError('Unexpected token: $'))