def test__parser__lexer_fail_via_parse(): """Test the how the parser fails and reports errors while lexing.""" lexer = Lexer(config=FluffConfig()) _, vs = lexer.lex("Select \u0394") assert vs assert len(vs) == 1 err = vs[0] assert isinstance(err, SQLLexError) assert err.line_pos == 8
def test__dialect__ansi__file_lex(raw, res, caplog): """Test we don't drop bits on simple examples.""" config = FluffConfig(overrides=dict(dialect="ansi")) lexer = Lexer(config=config) with caplog.at_level(logging.DEBUG): tokens, _ = lexer.lex(raw) # From just the initial parse, check we're all there raw_list = [token.raw for token in tokens] assert "".join(token.raw for token in tokens) == raw assert raw_list == res
def test__parser__lexer_fail(): """Test the how the lexer fails and reports errors.""" lex = Lexer(config=FluffConfig()) _, vs = lex.lex("Select \u0394") assert len(vs) == 1 err = vs[0] assert isinstance(err, SQLLexError) assert err.pos_marker().char_pos == 7
def test__parser__lexer_fail(): """Test the how the lexer fails and reports errors.""" lex = Lexer(config=FluffConfig(overrides={"dialect": "ansi"})) _, vs = lex.lex("Select \u0394") assert len(vs) == 1 err = vs[0] assert isinstance(err, SQLLexError) assert err.line_pos == 8
def lex(raw, config): """Basic parsing for the tests below.""" # Set up the lexer lex = Lexer(config=config) # Lex the string for matching. For a good test, this would # arguably happen as a fixture, but it's easier to pass strings # as parameters than pre-lexed segment strings. seg_list, vs = lex.lex(raw) assert not vs print(seg_list) return seg_list
def test__parser__lexer_obj(raw, res, caplog): """Test the lexer splits as expected in a selection of cases.""" lex = Lexer(config=FluffConfig()) with caplog.at_level(logging.DEBUG): lexing_segments, _ = lex.lex(raw) assert [seg.raw for seg in lexing_segments] == res
def _lex_templated_file( templated_file: TemplatedFile, config: FluffConfig ) -> Tuple[Optional[Sequence[BaseSegment]], List[SQLLexError], FluffConfig]: """Lex a templated file. NOTE: This potentially mutates the config, so make sure to use the returned one. """ violations = [] linter_logger.info("LEXING RAW (%s)", templated_file.fname) # Get the lexer lexer = Lexer(config=config) # Lex the file and log any problems try: tokens, lex_vs = lexer.lex(templated_file) # We might just get the violations as a list violations += lex_vs linter_logger.info("Lexed tokens: %s", [seg.raw for seg in tokens] if tokens else None) except SQLLexError as err: linter_logger.info("LEXING FAILED! (%s): %s", templated_file.fname, err) violations.append(err) return None, violations, config if not tokens: # pragma: no cover TODO? return None, violations, config # Check that we've got sensible indentation from the lexer. # We might need to suppress if it's a complicated file. templating_blocks_indent = config.get("template_blocks_indent", "indentation") if isinstance(templating_blocks_indent, str): force_block_indent = templating_blocks_indent.lower().strip( ) == "force" else: force_block_indent = False templating_blocks_indent = bool(templating_blocks_indent) # If we're forcing it through we don't check. if templating_blocks_indent and not force_block_indent: indent_balance = sum( getattr(elem, "indent_val", 0) for elem in cast(Tuple[BaseSegment, ...], tokens)) if indent_balance != 0: linter_logger.debug( "Indent balance test failed for %r. Template indents will not be " "linted for this file.", templated_file.fname, ) # Don't enable the templating blocks. templating_blocks_indent = False # The file will have been lexed without config, so check all indents # are enabled. new_tokens = [] for token in cast(Tuple[BaseSegment, ...], tokens): if token.is_meta: token = cast(MetaSegment, token) if token.indent_val != 0: # Don't allow it if we're not linting templating block indents. if not templating_blocks_indent: continue new_tokens.append(token) # Return new buffer return new_tokens, violations, config