def test__parser__lexer_fail_via_parse():
"""Test the how the parser fails and reports errors while lexing."""
lexer = Lexer(config=FluffConfig())
_, vs = lexer.lex("Select \u0394")
assert vs
assert len(vs) == 1
err = vs[0]
assert isinstance(err, SQLLexError)
assert err.line_pos == 8
def test__dialect__ansi__file_lex(raw, res, caplog):
"""Test we don't drop bits on simple examples."""
config = FluffConfig(overrides=dict(dialect="ansi"))
lexer = Lexer(config=config)
with caplog.at_level(logging.DEBUG):
tokens, _ = lexer.lex(raw)
# From just the initial parse, check we're all there
raw_list = [token.raw for token in tokens]
assert "".join(token.raw for token in tokens) == raw
assert raw_list == res
def test__parser__lexer_fail():
"""Test the how the lexer fails and reports errors."""
lex = Lexer(config=FluffConfig())
_, vs = lex.lex("Select \u0394")
assert len(vs) == 1
err = vs[0]
assert isinstance(err, SQLLexError)
assert err.pos_marker().char_pos == 7
def test__parser__lexer_fail():
"""Test the how the lexer fails and reports errors."""
lex = Lexer(config=FluffConfig(overrides={"dialect": "ansi"}))
_, vs = lex.lex("Select \u0394")
assert len(vs) == 1
err = vs[0]
assert isinstance(err, SQLLexError)
assert err.line_pos == 8
def lex(raw, config):
"""Basic parsing for the tests below."""
# Set up the lexer
lex = Lexer(config=config)
# Lex the string for matching. For a good test, this would
# arguably happen as a fixture, but it's easier to pass strings
# as parameters than pre-lexed segment strings.
seg_list, vs = lex.lex(raw)
assert not vs
print(seg_list)
return seg_list
def test__parser__lexer_obj(raw, res, caplog):
"""Test the lexer splits as expected in a selection of cases."""
lex = Lexer(config=FluffConfig())
with caplog.at_level(logging.DEBUG):
lexing_segments, _ = lex.lex(raw)
assert [seg.raw for seg in lexing_segments] == res
def _lex_templated_file(
templated_file: TemplatedFile, config: FluffConfig
) -> Tuple[Optional[Sequence[BaseSegment]], List[SQLLexError],
FluffConfig]:
"""Lex a templated file.
NOTE: This potentially mutates the config, so make sure to
use the returned one.
"""
violations = []
linter_logger.info("LEXING RAW (%s)", templated_file.fname)
# Get the lexer
lexer = Lexer(config=config)
# Lex the file and log any problems
try:
tokens, lex_vs = lexer.lex(templated_file)
# We might just get the violations as a list
violations += lex_vs
linter_logger.info("Lexed tokens: %s",
[seg.raw for seg in tokens] if tokens else None)
except SQLLexError as err:
linter_logger.info("LEXING FAILED! (%s): %s", templated_file.fname,
err)
violations.append(err)
return None, violations, config
if not tokens: # pragma: no cover TODO?
return None, violations, config
# Check that we've got sensible indentation from the lexer.
# We might need to suppress if it's a complicated file.
templating_blocks_indent = config.get("template_blocks_indent",
"indentation")
if isinstance(templating_blocks_indent, str):
force_block_indent = templating_blocks_indent.lower().strip(
) == "force"
else:
force_block_indent = False
templating_blocks_indent = bool(templating_blocks_indent)
# If we're forcing it through we don't check.
if templating_blocks_indent and not force_block_indent:
indent_balance = sum(
getattr(elem, "indent_val", 0)
for elem in cast(Tuple[BaseSegment, ...], tokens))
if indent_balance != 0:
linter_logger.debug(
"Indent balance test failed for %r. Template indents will not be "
"linted for this file.",
templated_file.fname,
)
# Don't enable the templating blocks.
templating_blocks_indent = False
# The file will have been lexed without config, so check all indents
# are enabled.
new_tokens = []
for token in cast(Tuple[BaseSegment, ...], tokens):
if token.is_meta:
token = cast(MetaSegment, token)
if token.indent_val != 0:
# Don't allow it if we're not linting templating block indents.
if not templating_blocks_indent:
continue
new_tokens.append(token)
# Return new buffer
return new_tokens, violations, config