def generate_tokens(self):
"""Tokenize the file and yield the tokens.
:raises flake8.exceptions.InvalidSyntax:
If a :class:`tokenize.TokenError` is raised while generating
tokens.
"""
try:
for token in tokenize.generate_tokens(self.next_line):
if token[2][0] > self.total_lines:
break
self.tokens.append(token)
yield token
except (tokenize.TokenError, SyntaxError) as exc:
raise exceptions.InvalidSyntax(exception=exc)
def file_tokens(self):
"""Return the complete set of tokens for a file.
Accessing this attribute *may* raise an InvalidSyntax exception.
:raises: flake8.exceptions.InvalidSyntax
"""
if self._file_tokens is None:
line_iter = iter(self.lines)
try:
self._file_tokens = list(
tokenize.generate_tokens(lambda: next(line_iter)))
except tokenize.TokenError as exc:
raise exceptions.InvalidSyntax(exc.message, exception=exc)
return self._file_tokens
def generate_tokens(self):
"""Tokenize the file and yield the tokens.
:raises flake8.exceptions.InvalidSyntax:
If a :class:`tokenize.TokenError` is raised while generating
tokens.
"""
try:
for token in tokenize.generate_tokens(self.next_line):
if token[2][0] > self.total_lines:
break
self.tokens.append(token)
yield token
# NOTE(sigmavirus24): pycodestyle was catching both a SyntaxError
# and a tokenize.TokenError. In looking a the source on Python 2 and
# Python 3, the SyntaxError should never arise from generate_tokens.
# If we were using tokenize.tokenize, we would have to catch that. Of
# course, I'm going to be unsurprised to be proven wrong at a later
# date.
except tokenize.TokenError as exc:
raise exceptions.InvalidSyntax(exc.message, exception=exc)
"""Tests for the flake8.exceptions module."""
import pickle
import pytest
from flake8 import exceptions
@pytest.mark.parametrize(
'err',
(exceptions.FailedToLoadPlugin(
plugin_name='plugin_name',
exception=ValueError('boom!'),
), exceptions.InvalidSyntax(exception=ValueError('Unexpected token: $')),
exceptions.PluginRequestedUnknownParameters(
plugin={'plugin_name': 'plugin_name'},
exception=ValueError('boom!'),
),
exceptions.PluginExecutionFailed(
plugin={'plugin_name': 'plugin_name'},
exception=ValueError('boom!'),
)),
)
def test_pickleable(err):
"""Ensure that our exceptions can cross pickle boundaries."""
for proto in range(pickle.HIGHEST_PROTOCOL + 1):
new_err = pickle.loads(pickle.dumps(err, protocol=proto))
assert str(err) == str(new_err)
orig_e = err.original_exception
new_e = new_err.original_exception
assert (type(orig_e), orig_e.args) == (type(new_e), new_e.args)
class TestInvalidSyntax(_ExceptionTest):
"""Tests for the InvalidSyntax exception."""
err = exceptions.InvalidSyntax(exception=ValueError('Unexpected token: $'))
