def test__linter__path_from_paths__exts():
"""Test configuration of file discovery."""
lntr = Linter(config=FluffConfig(overrides={"sql_file_exts": ".txt"}))
paths = normalise_paths(lntr.paths_from_path("test/fixtures/linter"))
assert "test.fixtures.linter.passing.sql" not in paths
assert "test.fixtures.linter.passing_cap_extension.SQL" not in paths
assert "test.fixtures.linter.discovery_file.txt" in paths
def test_linter_noqa_with_templating():
"""Similar to test_linter_noqa, but uses templating (Jinja)."""
lntr = Linter(
config=FluffConfig(
overrides={
"dialect": "bigquery", # Use bigquery to allow hash comments.
"templater": "jinja",
"rules": "L016",
}
)
)
sql = "\n"
'"{%- set a_var = ["1", "2"] -%}\n'
"SELECT\n"
" this_is_just_a_very_long_line_for_demonstration_purposes_of_a_bug_involving_"
"templated_sql_files, --noqa: L016\n"
" this_is_not_so_big a, --Inline comment --noqa: L012\n"
" this_is_not_so_big b, /* Block comment */ --noqa: L012\n"
" this_is_not_so_big c, # hash comment --noqa: L012\n"
" this_is_just_a_very_long_line_for_demonstration_purposes_of_a_bug_involving_"
"templated_sql_files, --noqa: L01*\n"
"FROM\n"
" a_table\n"
" "
result = lntr.lint_string(sql)
assert not result.get_violations()
def test__linter__path_from_paths__default():
"""Test .sql files are found by default."""
lntr = Linter()
paths = normalise_paths(lntr.paths_from_path("test/fixtures/linter"))
assert "test.fixtures.linter.passing.sql" in paths
assert "test.fixtures.linter.passing_cap_extension.SQL" in paths
assert "test.fixtures.linter.discovery_file.txt" not in paths
def assert_rule_pass_in_sql(code, sql, configs=None, msg=None):
"""Assert that a given rule doesn't fail on the given sql."""
# Configs allows overrides if we want to use them.
if configs is None:
configs = {}
core = configs.setdefault("core", {})
core["rules"] = code
cfg = FluffConfig(configs=configs)
linter = Linter(config=cfg)
# This section is mainly for aid in debugging.
rendered = linter.render_string(sql,
fname="<STR>",
config=cfg,
encoding="utf-8")
parsed = linter.parse_rendered(rendered, recurse=True)
if parsed.violations:
if msg:
print(msg) # pragma: no cover
pytest.fail(parsed.violations[0].desc() + "\n" +
parsed.tree.stringify())
print(f"Parsed:\n {parsed.tree.stringify()}")
# Note that lint_string() runs the templater and parser again, in order to
# test the whole linting pipeline in the same way that users do. In other
# words, the "rendered" and "parsed" variables above are irrelevant to this
# line of code.
lint_result = linter.lint_string(sql, config=cfg, fname="<STR>")
lerrs = lint_result.violations
print(f"Errors Found: {lerrs}")
if any(v.rule.code == code for v in lerrs):
if msg:
print(msg) # pragma: no cover
pytest.fail(f"Found {code} failures in query which should pass.",
pytrace=False)
def test__attempt_to_change_templater_warning(caplog):
"""Test warning when changing templater in .sqlfluff file in subdirectory."""
initial_config = FluffConfig(
configs={"core": {
"templater": "jinja",
"dialect": "ansi"
}})
lntr = Linter(config=initial_config)
updated_config = FluffConfig(
configs={"core": {
"templater": "python",
"dialect": "ansi"
}})
logger = logging.getLogger("sqlfluff")
original_propagate_value = logger.propagate
try:
logger.propagate = True
with caplog.at_level(logging.WARNING, logger="sqlfluff.linter"):
lntr.render_string(
in_str="select * from table",
fname="test.sql",
config=updated_config,
encoding="utf-8",
)
assert "Attempt to set templater to " in caplog.text
finally:
logger.propagate = original_propagate_value
def get_linter_and_formatter(
cfg: FluffConfig,
silent: bool = False) -> Tuple[Linter, CallbackFormatter]:
"""Get a linter object given a config."""
try:
# We're just making sure it exists at this stage.
# It will be fetched properly in the linter.
dialect_selector(cfg.get("dialect"))
except KeyError: # pragma: no cover
click.echo(f"Error: Unknown dialect '{cfg.get('dialect')}'")
sys.exit(66)
if not silent:
# Instantiate the linter and return it (with an output function)
formatter = CallbackFormatter(
callback=_callback_handler(cfg=cfg),
verbosity=cfg.get("verbose"),
output_line_length=cfg.get("output_line_length"),
)
return Linter(config=cfg, formatter=formatter), formatter
else:
# Instantiate the linter and return. NB: No formatter
# in the Linter and a black formatter otherwise.
formatter = CallbackFormatter(callback=lambda m: None, verbosity=0)
return Linter(config=cfg), formatter
def test__dialect__ansi_specific_segment_not_parse(raw, err_locations, caplog):
"""Test queries do not parse, with parsing errors raised properly."""
lnt = Linter()
parsed = lnt.parse_string(raw)
assert len(parsed.violations) > 0
locs = [(v.line_no(), v.line_pos()) for v in parsed.violations]
assert locs == err_locations
def test__linter__raises_malformed_noqa():
"""A badly formatted noqa gets raised as a parsing error."""
lntr = Linter()
result = lntr.lint_string_wrapped("select 1 --noqa missing semicolon")
with pytest.raises(SQLParseError):
result.check_tuples()
def test_non_selects_unparseable(raw: str) -> None:
"""Test that non-SELECT commands are not parseable."""
cfg = FluffConfig(configs={"core": {"dialect": "soql"}})
lnt = Linter(config=cfg)
result = lnt.lint_string(raw)
assert len(result.violations) == 1
assert isinstance(result.violations[0], SQLParseError)
def test__linter__lint_string_vs_file(path):
"""Test the linter finds the same things on strings and files."""
with open(path) as f:
sql_str = f.read()
lntr = Linter()
assert (lntr.lint_string(sql_str).check_tuples() == lntr.lint_path(
path).check_tuples())
def test__linter__path_from_paths__file():
"""Test extracting paths from a file path."""
lntr = Linter()
paths = lntr.paths_from_path("test/fixtures/linter/indentation_errors.sql")
assert normalise_paths(paths) == {
"test.fixtures.linter.indentation_errors.sql"
}
def test_linter_noqa_prs():
"""Test "noqa" feature to ignore PRS at the higher "Linter" level."""
lntr = Linter(dialect="ansi")
sql = "SELEC * FROM foo -- noqa: PRS\n"
result = lntr.lint_string(sql)
violations = result.get_violations()
assert not violations
def parse(
sql: str,
dialect: str = "ansi",
config_path: Optional[str] = None,
) -> Dict[str, Any]:
"""Parse a SQL string.
Args:
sql (:obj:`str`): The SQL to be parsed.
dialect (:obj:`str`, optional): A reference to the dialect of the SQL
to be parsed. Defaults to `ansi`.
config_path (:obj:`Optional[str]`, optional): A path to a .sqlfluff config.
Defaults to None.
Returns:
:obj:`Dict[str, Any]` JSON containing the parsed structure.
"""
cfg = get_simple_config(
dialect=dialect,
config_path=config_path,
)
linter = Linter(config=cfg)
parsed = linter.parse_string(sql)
# If we encounter any parsing errors, raise them in a combined issue.
if parsed.violations:
raise APIParsingError(parsed.violations)
# Return a JSON representation of the parse tree.
if parsed.tree is None: # pragma: no cover
return {}
return parsed.tree.as_record(show_raw=True)
def lint(
sql: str,
dialect: str = "ansi",
rules: Optional[List[str]] = None,
exclude_rules: Optional[List[str]] = None,
config_path: Optional[str] = None,
) -> List[Dict[str, Any]]:
"""Lint a SQL string.
Args:
sql (:obj:`str`): The SQL to be linted.
dialect (:obj:`str`, optional): A reference to the dialect of the SQL
to be linted. Defaults to `ansi`.
rules (:obj:`Optional[List[str]`, optional): A list of rule
references to lint for. Defaults to None.
exclude_rules (:obj:`Optional[List[str]`, optional): A list of rule
references to avoid linting for. Defaults to None.
config_path (:obj:`Optional[str]`, optional): A path to a .sqlfluff config.
Defaults to None.
Returns:
:obj:`List[Dict[str, Any]]` for each violation found.
"""
cfg = get_simple_config(
dialect=dialect,
rules=rules,
exclude_rules=exclude_rules,
config_path=config_path,
)
linter = Linter(config=cfg)
result = linter.lint_string_wrapped(sql)
result_records = result.as_records()
# Return just the violations for this file
return [] if not result_records else result_records[0]["violations"]
def assert_rule_fail_in_sql(code, sql, configs=None, line_numbers=None):
"""Assert that a given rule does fail on the given sql."""
# Set up the config to only use the rule we are testing.
cfg = FluffConfig(configs=configs, overrides={"rules": code})
# Lint it using the current config (while in fix mode)
linted = Linter(config=cfg).lint_string(sql, fix=True)
lerrs = linted.get_violations()
print(f"Errors Found: {lerrs}")
for e in lerrs:
if e.desc().startswith("Unexpected exception"):
pytest.fail(f"Linter failed with {e.desc()}")
parse_errors = list(filter(lambda v: type(v) == SQLParseError, lerrs))
if parse_errors:
pytest.fail(f"Found the following parse errors in test case: {parse_errors}")
if not any(v.rule.code == code for v in lerrs):
pytest.fail(
f"No {code} failures found in query which should fail.",
pytrace=False,
)
if line_numbers:
actual_line_numbers = [e.line_no for e in lerrs]
if line_numbers != actual_line_numbers:
pytest.fail(
"Expected errors on lines {}, but got errors on lines {}".format(
line_numbers, actual_line_numbers
)
)
# The query should already have been fixed if possible so just return the raw.
return linted.tree.raw
def test_linter_noqa():
"""Test "noqa" feature at the higher "Linter" level."""
lntr = Linter(config=FluffConfig(overrides={
"rules": "L012",
}))
sql = """
SELECT
col_a a,
col_b b, --noqa: disable=L012
col_c c,
col_d d, --noqa: enable=L012
col_e e,
col_f f,
col_g g, --noqa
col_h h,
col_i i, --noqa:L012
col_j j,
col_k k, --noqa:L013
col_l l,
col_m m,
col_n n, --noqa: disable=all
col_o o,
col_p p --noqa: enable=all
FROM foo
"""
result = lntr.lint_string(sql)
violations = result.get_violations()
assert {3, 6, 7, 8, 10, 12, 13, 14, 15,
18} == {v.line_no
for v in violations}
def test__linter__path_from_paths__ignore(path):
"""Test extracting paths from a dot."""
lntr = Linter()
paths = lntr.paths_from_path(path)
# We should only get query_b, because of the sqlfluffignore files.
assert normalise_paths(paths) == {
"test.fixtures.linter.sqlfluffignore.path_b.query_b.sql"
}
def test__templated_sections_do_not_raise_lint_error(in_dbt_project_dir): # noqa
"""Test that the dbt test has only a new line lint error."""
lntr = Linter(config=FluffConfig(configs=DBT_FLUFF_CONFIG))
lnt = lntr.lint_string(fname="tests/test.sql")
print(lnt.violations)
assert len(lnt.violations) == 1
# Newlines are removed by dbt templater
assert lnt.violations[0].rule.code == "L009"
def test__templated_sections_do_not_raise_lint_error(in_dbt_project_dir,
fname): # noqa
"""Test that the dbt test has only a new line lint error."""
lntr = Linter(config=FluffConfig(configs=DBT_FLUFF_CONFIG))
lnt = lntr.lint_path(path="models/my_new_project/" + fname)
violations = lnt.check_tuples()
print(violations)
assert len(violations) == 0
def test__linter__skip_dbt_model_disabled(in_dbt_project_dir): # noqa
"""Test that the linter skips disabled dbt models."""
conf = FluffConfig(configs={"core": {"templater": "dbt"}})
lntr = Linter(config=conf)
linted_path = lntr.lint_path(path="models/my_new_project/disabled_model.sql")
linted_file = linted_path.files[0]
assert linted_file.path == "models/my_new_project/disabled_model.sql"
assert not linted_file.templated_file
def test__linter__lint_ephemeral_3_level(project_dir): # noqa
"""Test linter can lint a project with 3-level ephemeral dependencies."""
# This was previously crashing inside dbt, in a function named
# inject_ctes_into_sql(). (issue 2671).
conf = FluffConfig(configs=DBT_FLUFF_CONFIG)
lntr = Linter(config=conf)
model_file_path = os.path.join(project_dir, "models/ephemeral_3_level")
lntr.lint_path(path=model_file_path)
def assert_rule_raises_violations_in_file(rule, fpath, violations, fluff_config):
"""Assert that a given rule raises given errors in specific positions of a file."""
lntr = Linter(config=fluff_config)
lnt = lntr.lint_path(fpath)
# Reformat the test data to match the format we're expecting. We use
# sets because we really don't care about order and if one is missing,
# we don't care about the orders of the correct ones.
assert set(lnt.check_tuples()) == {(rule, v[0], v[1]) for v in violations}
def test__rules__std_file(rule, path, violations):
"""Test the linter finds the given errors in (and only in) the right places."""
# Use config to look for only the rule we care about.
lntr = Linter(config=FluffConfig(overrides=dict(rules=rule)))
lnt = lntr.lint_path(path)
# Reformat the test data to match the format we're expecting. We use
# sets because we really don't care about order and if one is missing,
# we don't care about the orders of the correct ones.
assert set(lnt.check_tuples()) == {(rule, v[0], v[1]) for v in violations}
def test__linter__linting_result_get_violations():
"""Test that we can get violations from a LintingResult."""
lntr = Linter()
result = lntr.lint_paths([
"test/fixtures/linter/comma_errors.sql",
"test/fixtures/linter/whitespace_errors.sql",
])
all([type(v) == SQLLintError for v in result.get_violations()])
def test__linter__linting_result_check_tuples_by_path(by_path, result_type):
"""Test that a LintingResult can partition violations by the source files."""
lntr = Linter()
result = lntr.lint_paths([
"test/fixtures/linter/comma_errors.sql",
"test/fixtures/linter/whitespace_errors.sql",
])
check_tuples = result.check_tuples(by_path=by_path)
isinstance(check_tuples, result_type)
def test__linter__path_from_paths__dir():
"""Test extracting paths from directories."""
lntr = Linter()
paths = lntr.paths_from_path("test/fixtures/lexer")
assert normalise_paths(paths) == {
"test.fixtures.lexer.block_comment.sql",
"test.fixtures.lexer.inline_comment.sql",
"test.fixtures.lexer.basic.sql",
}
def test__linter__encoding(fname, config_encoding, lexerror):
"""Test linter deals with files with different encoding."""
lntr = Linter(config=FluffConfig(overrides={
"rules": "L001",
"encoding": config_encoding,
}))
result = lntr.lint_paths([fname])
assert lexerror == (SQLLexError
in [type(v) for v in result.get_violations()])
def test__dbt_templated_models_do_not_raise_lint_error(
project_dir,
fname # noqa: F811
):
"""Test that templated dbt models do not raise a linting error."""
lntr = Linter(config=FluffConfig(configs=DBT_FLUFF_CONFIG))
lnt = lntr.lint_path(
path=os.path.join(project_dir, "models/my_new_project/", fname))
violations = lnt.check_tuples()
assert len(violations) == 0
def test__linter__path_from_paths__dot():
"""Test extracting paths from a dot."""
lntr = Linter()
paths = lntr.paths_from_path(".")
# Use set theory to check that we get AT LEAST these files
assert normalise_paths(paths) >= {
"test.fixtures.lexer.block_comment.sql",
"test.fixtures.lexer.inline_comment.sql",
"test.fixtures.lexer.basic.sql",
}
def test__linter__path_from_paths__explicit_ignore():
"""Test ignoring files that were passed explicitly."""
lntr = Linter()
paths = lntr.paths_from_path(
"test/fixtures/linter/sqlfluffignore/path_a/query_a.sql",
ignore_non_existent_files=True,
ignore_files=True,
working_path="test/fixtures/linter/sqlfluffignore/",
)
assert len(paths) == 0