def test__linter__lint_ephemeral_3_level(project_dir): # noqa
"""Test linter can lint a project with 3-level ephemeral dependencies."""
# This was previously crashing inside dbt, in a function named
# inject_ctes_into_sql(). (issue 2671).
conf = FluffConfig(configs=DBT_FLUFF_CONFIG)
lntr = Linter(config=conf)
model_file_path = os.path.join(project_dir, "models/ephemeral_3_level")
lntr.lint_path(path=model_file_path)
def test__linter__lint_string_vs_file(path):
"""Test the linter finds the same things on strings and files."""
with open(path) as f:
sql_str = f.read()
lntr = Linter()
assert (lntr.lint_string(sql_str).check_tuples() == lntr.lint_path(
path).check_tuples())
def test__templated_sections_do_not_raise_lint_error(in_dbt_project_dir,
fname): # noqa
"""Test that the dbt test has only a new line lint error."""
lntr = Linter(config=FluffConfig(configs=DBT_FLUFF_CONFIG))
lnt = lntr.lint_path(path="models/my_new_project/" + fname)
violations = lnt.check_tuples()
print(violations)
assert len(violations) == 0
def test__linter__skip_dbt_model_disabled(in_dbt_project_dir): # noqa
"""Test that the linter skips disabled dbt models."""
conf = FluffConfig(configs={"core": {"templater": "dbt"}})
lntr = Linter(config=conf)
linted_path = lntr.lint_path(path="models/my_new_project/disabled_model.sql")
linted_file = linted_path.files[0]
assert linted_file.path == "models/my_new_project/disabled_model.sql"
assert not linted_file.templated_file
def assert_rule_raises_violations_in_file(rule, fpath, violations, fluff_config):
"""Assert that a given rule raises given errors in specific positions of a file."""
lntr = Linter(config=fluff_config)
lnt = lntr.lint_path(fpath)
# Reformat the test data to match the format we're expecting. We use
# sets because we really don't care about order and if one is missing,
# we don't care about the orders of the correct ones.
assert set(lnt.check_tuples()) == {(rule, v[0], v[1]) for v in violations}
def test__rules__std_file(rule, path, violations):
"""Test the linter finds the given errors in (and only in) the right places."""
# Use config to look for only the rule we care about.
lntr = Linter(config=FluffConfig(overrides=dict(rules=rule)))
lnt = lntr.lint_path(path)
# Reformat the test data to match the format we're expecting. We use
# sets because we really don't care about order and if one is missing,
# we don't care about the orders of the correct ones.
assert set(lnt.check_tuples()) == {(rule, v[0], v[1]) for v in violations}
def test__dbt_templated_models_do_not_raise_lint_error(
project_dir,
fname # noqa: F811
):
"""Test that templated dbt models do not raise a linting error."""
lntr = Linter(config=FluffConfig(configs=DBT_FLUFF_CONFIG))
lnt = lntr.lint_path(
path=os.path.join(project_dir, "models/my_new_project/", fname))
violations = lnt.check_tuples()
assert len(violations) == 0
def test__config__rules_group_with_exclude():
"""Test linting when a rules group is selected and rules are excluded."""
lntr = Linter(config=FluffConfig.from_path(
"test/fixtures/config/rules_group_with_exclude"))
lnt = lntr.lint_path(
"test/fixtures/config/rules_group_with_exclude/test.sql")
violations = lnt.check_tuples(by_path=True)
for k in violations:
assert ("L010", 15, 1) in violations[k]
assert "L019" not in [c[0] for c in violations[k]]
def test__linter__mask_templated_violations(ignore_templated_areas,
check_tuples):
"""Test linter masks files properly around templated content."""
lntr = Linter(config=FluffConfig(
overrides={
"rules": "L006",
"ignore_templated_areas": ignore_templated_areas,
}))
linted = lntr.lint_path(
path="test/fixtures/templater/jinja_h_macros/jinja.sql")
assert linted.check_tuples() == check_tuples
def test__linter__skip_dbt_model_disabled(project_dir): # noqa
"""Test that the linter skips disabled dbt models."""
conf = FluffConfig(configs=DBT_FLUFF_CONFIG)
lntr = Linter(config=conf)
model_file_path = os.path.join(project_dir,
"models/my_new_project/disabled_model.sql")
linted_path = lntr.lint_path(path=model_file_path)
# Check that the file is still there
assert len(linted_path.files) == 1
linted_file = linted_path.files[0]
assert linted_file.path == model_file_path
assert not linted_file.templated_file
assert not linted_file.tree
def test__config__rules_set_to_none():
"""Test linting when rules are set to 'None'.
Ensure that all rules are still run.
"""
lntr = Linter(
config=FluffConfig.from_path("test/fixtures/config/rules_set_to_none"))
lnt = lntr.lint_path("test/fixtures/config/rules_set_to_none/test.sql")
violations = lnt.check_tuples(by_path=True)
for k in violations:
assert ("L050", 1, 1) in violations[k]
assert ("L044", 12, 1) in violations[k]
assert ("L010", 12, 10) in violations[k]
def test__linter__skip_file(path, project_dir): # noqa
"""Test that the linter skips disabled dbt models and macros."""
conf = FluffConfig(configs=DBT_FLUFF_CONFIG)
lntr = Linter(config=conf)
model_file_path = os.path.join(project_dir, path)
linted_path = lntr.lint_path(path=model_file_path)
# Check that the file is still there
assert len(linted_path.files) == 1
linted_file = linted_path.files[0]
# Normalise paths to control for OS variance
assert os.path.normpath(
linted_file.path) == os.path.normpath(model_file_path)
assert not linted_file.templated_file
assert not linted_file.tree
def test__config__glob_include_config_tests():
"""Test linting with a glob pattern in rules.
This looks like a linter test but it's actually a config
test.
"""
lntr = Linter(
config=FluffConfig.from_path("test/fixtures/config/glob_include"))
lnt = lntr.lint_path("test/fixtures/config/glob_include/test.sql")
violations = lnt.check_tuples(by_path=True)
for k in violations:
assert ("L050", 1, 1) in violations[k]
assert ("L051", 12, 1) in violations[k]
assert ("L052", 12, 9) in violations[k]
assert ("L027", 10, 8) in violations[k]
assert "L044" not in [c[0] for c in violations[k]]
def test__config__nested_config_tests():
"""Test linting with overriden config in nested paths.
This looks like a linter test but it's actually a config
test.
"""
lntr = Linter(config=FluffConfig(overrides=dict(exclude_rules="L002")))
lnt = lntr.lint_path("test/fixtures/config/inheritance_b")
violations = lnt.check_tuples(by_path=True)
for k in violations:
if k.endswith("nested\\example.sql"):
assert ("L003", 1, 4) in violations[k]
assert ("L009", 1, 12) in violations[k]
assert "L002" not in [c[0] for c in violations[k]]
elif k.endswith("inheritance_b\\example.sql"):
assert ("L003", 1, 4) in violations[k]
assert "L002" not in [c[0] for c in violations[k]]
assert "L009" not in [c[0] for c in violations[k]]
def violations(src_path: str) -> List[Violation]:
"""Return list of violations.
Given the path to a .sql file, analyze it and return a list of
violations (i.e. formatting or style issues).
"""
linter = Linter(config=FluffConfig.from_root())
linted_path = linter.lint_path(src_path,
ignore_non_existent_files=True)
result = []
for violation in linted_path.get_violations():
try:
# Normal SQLFluff warnings
message = f"{violation.rule_code()}: {violation.description}"
except AttributeError:
# Parse errors
message = str(violation)
result.append(Violation(violation.line_no, message))
return result
def test__dbt_templated_models_fix_does_not_corrupt_file(
project_dir,
path,
caplog # noqa: F811
):
"""Test issues where previously "sqlfluff fix" corrupted the file."""
test_glob = os.path.join(project_dir, os.path.dirname(path), "*FIXED.sql")
_clean_path(test_glob)
lntr = Linter(config=FluffConfig(configs=DBT_FLUFF_CONFIG))
with caplog.at_level(logging.INFO, logger="sqlfluff.linter"):
lnt = lntr.lint_path(os.path.join(project_dir, path), fix=True)
try:
lnt.persist_changes(fixed_file_suffix="FIXED")
with open(os.path.join(project_dir, path + ".after")) as f:
comp_buff = f.read()
with open(os.path.join(project_dir, path.replace(".sql",
"FIXED.sql"))) as f:
fixed_buff = f.read()
assert fixed_buff == comp_buff
finally:
_clean_path(test_glob)
def test__dbt_templated_models_fix_does_not_corrupt_file(
project_dir,
path # noqa: F811
):
"""Test fix for issue 1608. Previously "sqlfluff fix" corrupted the file."""
for fsp in glob.glob(os.path.join(project_dir, "snapshots", "*FIXED.sql")):
os.remove(fsp)
lntr = Linter(config=FluffConfig(configs=DBT_FLUFF_CONFIG))
lnt = lntr.lint_path(os.path.join(project_dir, path), fix=True)
try:
lnt.persist_changes(fixed_file_suffix="FIXED")
with open(os.path.join(project_dir, path + ".after")) as f:
comp_buff = f.read()
with open(os.path.join(project_dir, path.replace(".sql",
"FIXED.sql"))) as f:
fixed_buff = f.read()
assert fixed_buff == comp_buff
finally:
for fsp in glob.glob(
os.path.join(project_dir, "snapshots", "*FIXED.sql")):
os.remove(fsp)
def auto_fix_test(dialect, folder, caplog):
"""A test for roundtrip testing, take a file buffer, lint, fix and lint.
This is explicitly different from the linter version of this, in that
it uses the command line rather than the direct api.
"""
# Log just the rules logger for this test.
# NOTE: In debugging it may be instructive to enable some of
# the other loggers listed here to debug particular issues.
# Enabling all of them results in very long logs so use
# wisely.
# caplog.set_level(logging.DEBUG, logger="sqlfluff.templater")
# caplog.set_level(logging.DEBUG, logger="sqlfluff.lexer")
caplog.set_level(logging.DEBUG, logger="sqlfluff.linter")
caplog.set_level(logging.DEBUG, logger="sqlfluff.rules")
filename = "testing.sql"
# Lets get the path of a file to use
tempdir_path = tempfile.mkdtemp()
filepath = os.path.join(tempdir_path, filename)
cfgpath = os.path.join(tempdir_path, ".sqlfluff")
src_filepath = os.path.join(*base_auto_fix_path, dialect, folder,
"before.sql")
cmp_filepath = os.path.join(*base_auto_fix_path, dialect, folder,
"after.sql")
vio_filepath = os.path.join(*base_auto_fix_path, dialect, folder,
"violations.json")
cfg_filepath = os.path.join(*base_auto_fix_path, dialect, folder,
".sqlfluff")
test_conf_filepath = os.path.join(*base_auto_fix_path, dialect, folder,
"test-config.yml")
# Load the config file for the test:
with open(test_conf_filepath) as cfg_file:
cfg = yaml.safe_load(cfg_file)
print("## Config: ", cfg)
rules = ",".join(cfg["test-config"]["rules"])
# Open the example file and write the content to it
print_buff = ""
with open(filepath, mode="w") as dest_file:
with open(src_filepath, mode="r") as source_file:
for line in source_file:
dest_file.write(line)
print_buff += line
# Copy the config file too
try:
with open(cfgpath, mode="w") as dest_file:
with open(cfg_filepath, mode="r") as source_file:
print("## Config File Found.")
for line in source_file:
dest_file.write(line)
except FileNotFoundError:
# No config file? No biggie
print("## No Config File Found.")
pass
print("## Input file:\n{0}".format(print_buff))
# Do we need to do a violations check?
try:
with open(vio_filepath, mode="r") as vio_file:
violations = json.load(vio_file)
except FileNotFoundError:
# No violations file. Let's not worry
violations = None
# Run the fix command
cfg = FluffConfig.from_root(overrides=dict(rules=rules, dialect=dialect))
lnt = Linter(config=cfg)
res = lnt.lint_path(filepath, fix=True)
print("## Templated file:\n{0}".format(res.tree.raw))
# We call the check_tuples here, even to makes sure any non-linting
# violations are raised, and the test fails.
vs = set(res.check_tuples())
# If we have a violations structure, let's enforce it.
if violations:
# Format the violations file
expected_vs = set()
for rule_key in violations["violations"]["linting"]:
for elem in violations["violations"]["linting"][rule_key]:
expected_vs.add((rule_key, *elem))
assert expected_vs == vs
# Actually do the fixes
res = res.persist_changes()
# Read the fixed file
with open(filepath, mode="r") as fixed_file:
fixed_buff = fixed_file.read()
# Clearup once read
shutil.rmtree(tempdir_path)
# Read the comparison file
with open(cmp_filepath, mode="r") as comp_file:
comp_buff = comp_file.read()
# Make sure we were successful
assert res
# Assert that we fixed as expected
assert fixed_buff == comp_buff
def auto_fix_test(rules, dialect, folder):
"""A test for roundtrip testing, take a file buffer, lint, fix and lint.
This is explicitly different from the linter version of this, in that
it uses the command line rather than the direct api.
"""
filename = "testing.sql"
# Lets get the path of a file to use
tempdir_path = tempfile.mkdtemp()
filepath = os.path.join(tempdir_path, filename)
cfgpath = os.path.join(tempdir_path, ".sqlfluff")
src_filepath = os.path.join(*base_auto_fix_path, dialect, folder,
"before.sql")
cmp_filepath = os.path.join(*base_auto_fix_path, dialect, folder,
"after.sql")
vio_filepath = os.path.join(*base_auto_fix_path, dialect, folder,
"violations.json")
cfg_filepath = os.path.join(*base_auto_fix_path, dialect, folder,
".sqlfluff")
# Open the example file and write the content to it
print_buff = ""
with open(filepath, mode="w") as dest_file:
with open(src_filepath, mode="r") as source_file:
for line in source_file:
dest_file.write(line)
print_buff += line
# Copy the config file too
try:
with open(cfgpath, mode="w") as dest_file:
with open(cfg_filepath, mode="r") as source_file:
for line in source_file:
dest_file.write(line)
except FileNotFoundError:
# No config file? No biggie
pass
print("## Input file:\n{0}".format(print_buff))
# Do we need to do a violations check?
try:
with open(vio_filepath, mode="r") as vio_file:
violations = json.load(vio_file)
except FileNotFoundError:
# No violations file. Let's not worry
violations = None
# Run the fix command
cfg = FluffConfig.from_root(overrides=dict(rules=rules, dialect=dialect))
lnt = Linter(config=cfg)
res = lnt.lint_path(filepath, fix=True)
# If we have a violations structure, let's enforce it.
if violations:
vs = set(res.check_tuples())
# Format the violations file
expected_vs = set()
for rule_key in violations["violations"]["linting"]:
for elem in violations["violations"]["linting"][rule_key]:
expected_vs.add((rule_key, *elem))
assert expected_vs == vs
# Actually do the fixes
res = do_fixes(lnt, res)
# Read the fixed file
with open(filepath, mode="r") as fixed_file:
fixed_buff = fixed_file.read()
# Clearup once read
shutil.rmtree(tempdir_path)
# Read the comparison file
with open(cmp_filepath, mode="r") as comp_file:
comp_buff = comp_file.read()
# Make sure we were successful
assert res
# Assert that we fixed as expected
assert fixed_buff == comp_buff
