def render_string(self, in_str: str, fname: str, config: FluffConfig,
encoding: str) -> RenderedFile:
"""Template the file."""
linter_logger.info("TEMPLATING RAW [%s] (%s)", self.templater.name,
fname)
# Start the templating timer
t0 = time.monotonic()
# Newlines are normalised to unix-style line endings (\n).
# The motivation is that Jinja normalises newlines during templating and
# we want consistent mapping between the raw and templated slices.
in_str = self._normalise_newlines(in_str)
# Since Linter.__init__() does not require a dialect to be specified,
# check for one now. (We're processing a string, not a file, so we're
# not going to pick up a .sqlfluff or other config file to provide a
# missing dialect at this point.)
config.verify_dialect_specified()
if not config.get("templater_obj") == self.templater:
linter_logger.warning((
f"Attempt to set templater to {config.get('templater_obj').name} "
f"failed. Using {self.templater.name} templater. Templater cannot "
"be set in a .sqlfluff file in a subdirectory of the current "
"working directory. It can be set in a .sqlfluff in the current "
"working directory. See Nesting section of the docs for more "
"details."))
try:
templated_file, templater_violations = self.templater.process(
in_str=in_str,
fname=fname,
config=config,
formatter=self.formatter)
except SQLFluffSkipFile as s: # pragma: no cover
linter_logger.warning(str(s))
templated_file = None
templater_violations = []
if not templated_file:
linter_logger.info("TEMPLATING FAILED: %s", templater_violations)
# Record time
time_dict = {"templating": time.monotonic() - t0}
return RenderedFile(
templated_file,
templater_violations,
config,
time_dict,
fname,
encoding,
in_str,
)
def assert_rule_pass_in_sql(code, sql, configs=None):
"""Assert that a given rule doesn't fail on the given sql."""
# Configs allows overrides if we want to use them.
cfg = FluffConfig(configs=configs)
r = get_rule_from_set(code, config=cfg)
parsed = Linter(config=cfg).parse_string(sql)
if parsed.violations:
pytest.fail(parsed.violations[0].desc() + "\n" + parsed.tree.stringify())
print(f"Parsed:\n {parsed.tree.stringify()}")
lerrs, _, _, _ = r.crawl(parsed.tree, dialect=cfg.get("dialect_obj"))
print(f"Errors Found: {lerrs}")
if any(v.rule.code == code for v in lerrs):
pytest.fail(f"Found {code} failures in query which should pass.", pytrace=False)
def _load_raw_file_and_config(
fname: str,
root_config: FluffConfig) -> Tuple[str, FluffConfig, str]:
"""Load a raw file and the associated config."""
file_config = root_config.make_child_from_path(fname)
encoding = get_encoding(fname=fname, config=file_config)
# Check file size before loading.
limit = file_config.get("large_file_skip_byte_limit")
if limit:
# Get the file size
file_size = os.path.getsize(fname)
if file_size > limit:
raise SQLFluffSkipFile(
f"Length of file {fname!r} is {file_size} bytes which is over "
f"the limit of {limit} bytes. Skipping to avoid parser lock. "
"Users can increase this limit in their config by setting the "
"'large_file_skip_byte_limit' value, or disable by setting it "
"to zero.")
with open(fname, encoding=encoding,
errors="backslashreplace") as target_file:
raw_file = target_file.read()
# Scan the raw file for config commands.
file_config.process_raw_file_for_config(raw_file)
# Return the raw file and config
return raw_file, file_config, encoding
def assert_rule_pass_in_sql(code, sql, configs=None, msg=None):
"""Assert that a given rule doesn't fail on the given sql."""
# Configs allows overrides if we want to use them.
if configs is None:
configs = {}
core = configs.setdefault("core", {})
core["rules"] = code
cfg = FluffConfig(configs=configs)
linter = Linter(config=cfg)
# This section is mainly for aid in debugging.
rendered = linter.render_string(sql,
fname="<STR>",
config=cfg,
encoding="utf-8")
parsed = linter.parse_rendered(rendered, recurse=True)
if parsed.violations:
if msg:
print(msg) # pragma: no cover
pytest.fail(parsed.violations[0].desc() + "\n" +
parsed.tree.stringify())
print(f"Parsed:\n {parsed.tree.stringify()}")
# Note that lint_string() runs the templater and parser again, in order to
# test the whole linting pipeline in the same way that users do. In other
# words, the "rendered" and "parsed" variables above are irrelevant to this
# line of code.
lint_result = linter.lint_string(sql, config=cfg, fname="<STR>")
lerrs = lint_result.violations
print(f"Errors Found: {lerrs}")
if any(v.rule.code == code for v in lerrs):
if msg:
print(msg) # pragma: no cover
pytest.fail(f"Found {code} failures in query which should pass.",
pytrace=False)
def render_string(self, in_str: str, fname: str,
config: FluffConfig) -> RenderedFile:
"""Template the file."""
linter_logger.info("TEMPLATING RAW [%s] (%s)", self.templater.name,
fname)
# Start the templating timer
t0 = time.monotonic()
if not config.get("templater_obj") == self.templater:
linter_logger.warning((
f"Attempt to set templater to {config.get('templater_obj').name} failed. Using {self.templater.name} "
"templater. Templater cannot be set in a .sqlfluff file in a subdirectory of the current working "
"directory. It can be set in a .sqlfluff in the current working directory. See Nesting section of the "
"docs for more details."))
try:
templated_file, templater_violations = self.templater.process(
in_str=in_str,
fname=fname,
config=config,
formatter=self.formatter)
except SQLTemplaterSkipFile as s:
linter_logger.warning(str(s))
templated_file = None
templater_violations = []
if not templated_file:
linter_logger.info("TEMPLATING FAILED: %s", templater_violations)
# Record time
time_dict = {"templating": time.monotonic() - t0}
return RenderedFile(templated_file, templater_violations, config,
time_dict, fname)
def assert_rule_fail_in_sql(code, sql, configs=None, line_numbers=None):
"""Assert that a given rule does fail on the given sql."""
# Set up the config to only use the rule we are testing.
cfg = FluffConfig(configs=configs, overrides={"rules": code})
# Lint it using the current config (while in fix mode)
linted = Linter(config=cfg).lint_string(sql, fix=True)
lerrs = linted.get_violations()
print(f"Errors Found: {lerrs}")
for e in lerrs:
if e.desc().startswith("Unexpected exception"):
pytest.fail(f"Linter failed with {e.desc()}")
parse_errors = list(filter(lambda v: type(v) == SQLParseError, lerrs))
if parse_errors:
pytest.fail(f"Found the following parse errors in test case: {parse_errors}")
if not any(v.rule.code == code for v in lerrs):
pytest.fail(
f"No {code} failures found in query which should fail.",
pytrace=False,
)
if line_numbers:
actual_line_numbers = [e.line_no for e in lerrs]
if line_numbers != actual_line_numbers:
pytest.fail(
"Expected errors on lines {}, but got errors on lines {}".format(
line_numbers, actual_line_numbers
)
)
# The query should already have been fixed if possible so just return the raw.
return linted.tree.raw
def test_rule_set_return_informative_error_when_rule_not_registered():
"""Assert that a rule that throws an exception returns it as a validation."""
cfg = FluffConfig(overrides={"dialect": "ansi"})
with pytest.raises(ValueError) as e:
get_rule_from_set("L000", config=cfg)
e.match("'L000' not in")
def __init__(
self,
config: Optional[FluffConfig] = None,
formatter: Any = None,
dialect: Optional[str] = None,
rules: Optional[List[str]] = None,
user_rules: Optional[List[BaseRule]] = None,
exclude_rules: Optional[List[str]] = None,
) -> None:
# Store the config object
self.config = FluffConfig.from_kwargs(
config=config,
dialect=dialect,
rules=rules,
exclude_rules=exclude_rules,
# Don't require a dialect to be provided yet. Defer this until we
# are actually linting something, since the directory we are linting
# from may provide additional configuration, including a dialect.
require_dialect=False,
)
# Get the dialect and templater
self.dialect = self.config.get("dialect_obj")
self.templater = self.config.get("templater_obj")
# Store the formatter for output
self.formatter = formatter
# Store references to user rule classes
self.user_rules = user_rules or []
def test__rules__std_file(rule, path, violations):
"""Test the linter finds the given errors in (and only in) the right places."""
assert_rule_raises_violations_in_file(
rule=rule,
fpath=path,
violations=violations,
fluff_config=FluffConfig(overrides=dict(rules=rule)),
)
def test__rules__std_file_dbt(rule, path, violations, in_dbt_project_dir): # noqa
"""Test the linter finds the given errors in (and only in) the right places (DBT)."""
assert_rule_raises_violations_in_file(
rule=rule,
fpath=path,
violations=violations,
fluff_config=FluffConfig(configs=DBT_FLUFF_CONFIG, overrides=dict(rules=rule)),
)
def test__rule_test_case(test_case):
"""Run the tests."""
res = rules__test_helper(test_case)
if res is not None and res != test_case.fail_str:
cfg = FluffConfig(configs=test_case.configs)
rule = get_rule_from_set(test_case.rule, config=cfg)
assert is_fix_compatible(
rule
), f'Rule {test_case.rule} returned fixes but does not specify "@document_fix_compatible".'
def test__rule_test_case(test_case, caplog):
"""Run the tests."""
with caplog.at_level(logging.DEBUG, logger="sqlfluff.rules"):
res = rules__test_helper(test_case)
if res is not None and res != test_case.fail_str:
cfg = FluffConfig(configs=test_case.configs)
rule = get_rule_from_set(test_case.rule, config=cfg)
assert is_fix_compatible(
rule
), f'Rule {test_case.rule} returned fixes but does not specify "@document_fix_compatible".'
def get_encoding(fname: str, config: FluffConfig) -> str:
"""Get the encoding of the file (autodetect)."""
encoding_config = config.get("encoding", default="autodetect")
if encoding_config == "autodetect":
with open(fname, "rb") as f:
data = f.read()
return chardet.detect(data)["encoding"]
return encoding_config
def _get_macros_path(self, config: FluffConfig) -> Optional[List[str]]:
if config:
macros_path = config.get_section(
(self.templater_selector, self.name, "load_macros_from_path"))
if macros_path:
result = [
s.strip() for s in macros_path.split(",") if s.strip()
]
if result:
return result
return None
def test_rule_exception_is_caught_to_validation():
"""Assert that a rule that throws an exception on _eval returns it as a validation."""
@std_rule_set.register
class Rule_LXXX(BaseCrawler):
"""Rule that throws an exception."""
def _eval(self, segment, parent_stack, **kwargs):
raise Exception(
"Catch me or I'll deny any linting results from you")
linter = Linter(config=FluffConfig(overrides=dict(rules="LXXX")))
assert linter.lint_string("select 1").check_tuples() == [("LXXX", 1, 1)]
def _load_raw_file_and_config(
fname: str,
root_config: FluffConfig) -> Tuple[str, FluffConfig, str]:
"""Load a raw file and the associated config."""
file_config = root_config.make_child_from_path(fname)
encoding = get_encoding(fname=fname, config=file_config)
with open(fname, encoding=encoding,
errors="backslashreplace") as target_file:
raw_file = target_file.read()
# Scan the raw file for config commands.
file_config.process_raw_file_for_config(raw_file)
# Return the raw file and config
return raw_file, file_config, encoding
def assert_rule_pass_in_sql(code, sql, configs=None):
"""Assert that a given rule doesn't fail on the given sql."""
# Configs allows overrides if we want to use them.
cfg = FluffConfig(configs=configs)
r = get_rule_from_set(code, config=cfg)
linter = Linter(config=cfg)
rendered = linter.render_string(sql,
fname="<STR>",
config=cfg,
encoding="utf-8")
parsed = linter.parse_rendered(rendered, recurse=True)
if parsed.violations:
pytest.fail(parsed.violations[0].desc() + "\n" +
parsed.tree.stringify())
print(f"Parsed:\n {parsed.tree.stringify()}")
lerrs, _, _, _ = r.crawl(parsed.tree, [],
dialect=cfg.get("dialect_obj"),
templated_file=rendered[0])
print(f"Errors Found: {lerrs}")
if any(v.rule.code == code for v in lerrs):
pytest.fail(f"Found {code} failures in query which should pass.",
pytrace=False)
def __init__(
self,
config: Optional[FluffConfig] = None,
last_resort_lexer: Optional[SingletonMatcher] = None,
dialect: Optional[str] = None,
):
# Allow optional config and dialect
self.config = FluffConfig.from_kwargs(config=config, dialect=dialect)
lexer_struct = self.config.get("dialect_obj").get_lexer_struct()
self.matcher = RepeatedMultiMatcher.from_struct(lexer_struct)
self.last_resort_lexer = last_resort_lexer or RegexMatcher.from_shorthand(
"<unlexable>", r"[^\t\n\,\.\ \-\+\*\\\/\'\"\;\:\[\]\(\)\|]*", is_code=True
)
def test__rules__std_L003_process_raw_stack(generate_test_segments):
"""Test the _process_raw_stack function.
Note: This test probably needs expanding. It doesn't
really check enough of the full functionality.
"""
cfg = FluffConfig()
r = get_rule_from_set("L003", config=cfg)
test_stack = generate_test_segments(["bar", "\n", " ", "foo", "baar", " \t "])
res = r._process_raw_stack(test_stack)
print(res)
assert sorted(res.keys()) == [1, 2]
assert res[2]["indent_size"] == 5
def test__rules__runaway_fail_catch():
"""Test that we catch runaway rules."""
runaway_limit = 5
my_query = "SELECT * FROM foo"
# Set up the config to only use the rule we are testing.
cfg = FluffConfig(overrides={"rules": "T001", "runaway_limit": runaway_limit})
# Lint it using the current config (while in fix mode)
linter = Linter(config=cfg, user_rules=[Rule_T001])
# In theory this step should result in an infinite
# loop, but the loop limit should catch it.
linted = linter.lint_string(my_query, fix=True)
# We should have a lot of newlines in there.
# The number should equal the runaway limit
assert linted.tree.raw.count("\n") == runaway_limit
def test__rules__runaway_fail_catch():
"""Test that we catch runaway rules."""
runaway_limit = 5
my_query = "SELECT * FROM foo"
# Set up the config to only use the rule we are testing.
cfg = FluffConfig(overrides={
"rules": "T001",
"runaway_limit": runaway_limit
})
# Lint it using the current config (while in fix mode)
linter = Linter(config=cfg, user_rules=[Rule_T001])
# In theory this step should result in an infinite
# loop, but the loop limit should catch it.
linted = linter.lint_string(my_query, fix=True)
# When the linter hits the runaway limit, it returns the original SQL tree.
assert linted.tree.raw == my_query
def __init__(
self,
config: Optional[FluffConfig] = None,
last_resort_lexer: Optional[StringLexer] = None,
dialect: Optional[str] = None,
):
# Allow optional config and dialect
self.config = FluffConfig.from_kwargs(config=config, dialect=dialect)
# Store the matchers
self.lexer_matchers = self.config.get(
"dialect_obj").get_lexer_matchers()
self.last_resort_lexer = last_resort_lexer or RegexLexer(
"<unlexable>",
r"[^\t\n\,\.\ \-\+\*\\\/\'\"\;\:\[\]\(\)\|]*",
UnlexableSegment,
)
def assert_rule_fail_in_sql(code, sql, configs=None):
"""Assert that a given rule does fail on the given sql."""
# Set up the config to only use the rule we are testing.
cfg = FluffConfig(configs=configs, overrides={"rules": code})
# Lint it using the current config (while in fix mode)
linted = Linter(config=cfg).lint_string(sql, fix=True)
lerrs = linted.get_violations()
print("Errors Found: {0}".format(lerrs))
parse_errors = list(filter(lambda v: type(v) == SQLParseError, lerrs))
if parse_errors:
pytest.fail(f"Found the following parse errors in test case: {parse_errors}")
if not any(v.rule.code == code for v in lerrs):
pytest.fail(
"No {0} failures found in query which should fail.".format(code),
pytrace=False,
)
# The query should already have been fixed if possible so just return the raw.
return linted.tree.raw
def __init__(
self,
config: Optional[FluffConfig] = None,
formatter: Any = None,
dialect: Optional[str] = None,
rules: Optional[Union[str, List[str]]] = None,
user_rules: Optional[Union[str, List[str]]] = None,
) -> None:
# Store the config object
self.config = FluffConfig.from_kwargs(config=config,
dialect=dialect,
rules=rules)
# Get the dialect and templater
self.dialect = self.config.get("dialect_obj")
self.templater = self.config.get("templater_obj")
# Store the formatter for output
self.formatter = formatter
# Store references to user rule classes
self.user_rules = user_rules or []
def test_rule_exception_is_caught_to_validation():
"""Assert that a rule that throws an exception returns it as a validation."""
std_rule_set = get_ruleset()
@std_rule_set.register
class Rule_T000(BaseRule):
"""Rule that throws an exception."""
groups = ("all", )
crawl_behaviour = RootOnlyCrawler()
def _eval(self, segment, parent_stack, **kwargs):
raise Exception(
"Catch me or I'll deny any linting results from you")
linter = Linter(
config=FluffConfig(overrides=dict(rules="T000", dialect="ansi")),
user_rules=[Rule_T000],
)
assert linter.lint_string("select 1").check_tuples() == [("T000", 1, 1)]
def test__rules__std_L007_before():
"""Verify that L007 returns the correct error message when before is used."""
sql = """
SELECT
a,
b
FROM foo
WHERE
a = 1
AND b = 2
"""
config = FluffConfig(
configs={"rules": {"L007": {"operator_new_lines": "before"}}},
overrides={"dialect": "ansi"},
)
# The sqlfluff.lint API doesn't allow us to pass config so need to do what it does
linter = Linter(config=config)
result_records = linter.lint_string_wrapped(sql).as_records()
result = result_records[0]["violations"]
assert "L007" in [r["code"] for r in result]
assert before_description in [r["description"] for r in result]
def _wrapped(self,
*,
in_str: str,
fname: str,
config: FluffConfig = None,
**kwargs):
if config:
limit = config.get("large_file_skip_char_limit")
if limit:
templater_logger.warning(
"The config value large_file_skip_char_limit was found set. "
"This feature will be removed in a future release, please "
"use the more efficient 'large_file_skip_byte_limit' instead."
)
if limit and len(in_str) > limit:
raise SQLFluffSkipFile(
f"Length of file {fname!r} is over {limit} characters. "
"Skipping to avoid parser lock. Users can increase this limit "
"in their config by setting the 'large_file_skip_char_limit' "
"value, or disable by setting it to zero.")
return func(self, in_str=in_str, fname=fname, config=config, **kwargs)
def test__rules__std_L003_process_raw_stack(generate_test_segments, test_elems,
result):
"""Test the _process_raw_stack function.
Note: This test probably needs expanding. It doesn't
really check enough of the full functionality.
"""
cfg = FluffConfig()
r = get_rule_from_set("L003", config=cfg)
test_stack = generate_test_segments(test_elems)
res = r._process_raw_stack(test_stack, {})
print(res)
# Verify structure
assert isinstance(res, dict)
assert all(isinstance(k, int) for k in res.keys())
assert all(isinstance(v, dict) for v in res.values())
# Check keys are all present
assert all(
v.keys() == {
"line_no",
"templated_line",
"line_buffer",
"indent_buffer",
"indent_size",
"indent_balance",
"hanging_indent",
"clean_indent",
} for v in res.values())
for k in res:
# For testing purposes, we won't be checking the buffer fields. They're
# just too hard to create in the test cases and aren't critical in
# determining what course of action to take. Most of the logic uses the
# values which we *are* still testing.
del res[k]["line_buffer"]
del res[k]["indent_buffer"]
# We also don't check the "templated_file" flag. These tests don't
# exercise that code.
del res[k]["templated_line"]
assert res == result
def test__rules__std_L007_after():
"""Verify orrect error message when after is explicitly used."""
sql = """
SELECT
a,
b
FROM foo
WHERE
a = 1 AND
b = 2
"""
config = FluffConfig(
configs={"rules": {
"L007": {
"operator_new_lines": "after"
}
}})
# The sqlfluff.lint API doesn't allow us to pass config so need to do what it does
linter = Linter(config=config)
result_records = linter.lint_string_wrapped(sql).as_records()
result = result_records[0]["violations"]
assert "L007" in [r["code"] for r in result]
assert after_description in [r["description"] for r in result]
def assert_rule_fail_in_sql(code, sql, configs=None, line_numbers=None):
"""Assert that a given rule does fail on the given sql."""
print("# Asserting Rule Fail in SQL")
# Set up the config to only use the rule we are testing.
overrides = {"rules": code}
if configs is None or "core" not in configs or "dialect" not in configs[
"core"]:
overrides["dialect"] = "ansi"
cfg = FluffConfig(configs=configs, overrides=overrides)
# Lint it using the current config (while in fix mode)
linted = Linter(config=cfg).lint_string(sql, fix=True)
lerrs = linted.get_violations()
print("Errors Found:")
for e in lerrs:
print(" " + repr(e))
if e.desc().startswith("Unexpected exception"):
pytest.fail(f"Linter failed with {e.desc()}") # pragma: no cover
parse_errors = list(
filter(lambda v: isinstance(v, (SQLParseError, SQLTemplaterError)),
lerrs))
if parse_errors:
pytest.fail(
f"Found the following parse errors in test case: {parse_errors}")
if not any(v.rule.code == code for v in lerrs):
print(f"Parsed File:\n{linted.tree.stringify()}")
pytest.fail(
f"No {code} failures found in query which should fail.",
pytrace=False,
)
if line_numbers:
actual_line_numbers = [e.line_no for e in lerrs]
if line_numbers != actual_line_numbers: # pragma: no cover
pytest.fail(
"Expected errors on lines {}, but got errors on lines {}".
format(line_numbers, actual_line_numbers))
fixed, _ = linted.fix_string()
return fixed, linted.violations
