def render_string(self, in_str: str, fname: str, config: FluffConfig, encoding: str) -> RenderedFile: """Template the file.""" linter_logger.info("TEMPLATING RAW [%s] (%s)", self.templater.name, fname) # Start the templating timer t0 = time.monotonic() # Newlines are normalised to unix-style line endings (\n). # The motivation is that Jinja normalises newlines during templating and # we want consistent mapping between the raw and templated slices. in_str = self._normalise_newlines(in_str) # Since Linter.__init__() does not require a dialect to be specified, # check for one now. (We're processing a string, not a file, so we're # not going to pick up a .sqlfluff or other config file to provide a # missing dialect at this point.) config.verify_dialect_specified() if not config.get("templater_obj") == self.templater: linter_logger.warning(( f"Attempt to set templater to {config.get('templater_obj').name} " f"failed. Using {self.templater.name} templater. Templater cannot " "be set in a .sqlfluff file in a subdirectory of the current " "working directory. It can be set in a .sqlfluff in the current " "working directory. See Nesting section of the docs for more " "details.")) try: templated_file, templater_violations = self.templater.process( in_str=in_str, fname=fname, config=config, formatter=self.formatter) except SQLFluffSkipFile as s: # pragma: no cover linter_logger.warning(str(s)) templated_file = None templater_violations = [] if not templated_file: linter_logger.info("TEMPLATING FAILED: %s", templater_violations) # Record time time_dict = {"templating": time.monotonic() - t0} return RenderedFile( templated_file, templater_violations, config, time_dict, fname, encoding, in_str, )
def assert_rule_pass_in_sql(code, sql, configs=None): """Assert that a given rule doesn't fail on the given sql.""" # Configs allows overrides if we want to use them. cfg = FluffConfig(configs=configs) r = get_rule_from_set(code, config=cfg) parsed = Linter(config=cfg).parse_string(sql) if parsed.violations: pytest.fail(parsed.violations[0].desc() + "\n" + parsed.tree.stringify()) print(f"Parsed:\n {parsed.tree.stringify()}") lerrs, _, _, _ = r.crawl(parsed.tree, dialect=cfg.get("dialect_obj")) print(f"Errors Found: {lerrs}") if any(v.rule.code == code for v in lerrs): pytest.fail(f"Found {code} failures in query which should pass.", pytrace=False)
def _load_raw_file_and_config( fname: str, root_config: FluffConfig) -> Tuple[str, FluffConfig, str]: """Load a raw file and the associated config.""" file_config = root_config.make_child_from_path(fname) encoding = get_encoding(fname=fname, config=file_config) # Check file size before loading. limit = file_config.get("large_file_skip_byte_limit") if limit: # Get the file size file_size = os.path.getsize(fname) if file_size > limit: raise SQLFluffSkipFile( f"Length of file {fname!r} is {file_size} bytes which is over " f"the limit of {limit} bytes. Skipping to avoid parser lock. " "Users can increase this limit in their config by setting the " "'large_file_skip_byte_limit' value, or disable by setting it " "to zero.") with open(fname, encoding=encoding, errors="backslashreplace") as target_file: raw_file = target_file.read() # Scan the raw file for config commands. file_config.process_raw_file_for_config(raw_file) # Return the raw file and config return raw_file, file_config, encoding
def assert_rule_pass_in_sql(code, sql, configs=None, msg=None): """Assert that a given rule doesn't fail on the given sql.""" # Configs allows overrides if we want to use them. if configs is None: configs = {} core = configs.setdefault("core", {}) core["rules"] = code cfg = FluffConfig(configs=configs) linter = Linter(config=cfg) # This section is mainly for aid in debugging. rendered = linter.render_string(sql, fname="<STR>", config=cfg, encoding="utf-8") parsed = linter.parse_rendered(rendered, recurse=True) if parsed.violations: if msg: print(msg) # pragma: no cover pytest.fail(parsed.violations[0].desc() + "\n" + parsed.tree.stringify()) print(f"Parsed:\n {parsed.tree.stringify()}") # Note that lint_string() runs the templater and parser again, in order to # test the whole linting pipeline in the same way that users do. In other # words, the "rendered" and "parsed" variables above are irrelevant to this # line of code. lint_result = linter.lint_string(sql, config=cfg, fname="<STR>") lerrs = lint_result.violations print(f"Errors Found: {lerrs}") if any(v.rule.code == code for v in lerrs): if msg: print(msg) # pragma: no cover pytest.fail(f"Found {code} failures in query which should pass.", pytrace=False)
def render_string(self, in_str: str, fname: str, config: FluffConfig) -> RenderedFile: """Template the file.""" linter_logger.info("TEMPLATING RAW [%s] (%s)", self.templater.name, fname) # Start the templating timer t0 = time.monotonic() if not config.get("templater_obj") == self.templater: linter_logger.warning(( f"Attempt to set templater to {config.get('templater_obj').name} failed. Using {self.templater.name} " "templater. Templater cannot be set in a .sqlfluff file in a subdirectory of the current working " "directory. It can be set in a .sqlfluff in the current working directory. See Nesting section of the " "docs for more details.")) try: templated_file, templater_violations = self.templater.process( in_str=in_str, fname=fname, config=config, formatter=self.formatter) except SQLTemplaterSkipFile as s: linter_logger.warning(str(s)) templated_file = None templater_violations = [] if not templated_file: linter_logger.info("TEMPLATING FAILED: %s", templater_violations) # Record time time_dict = {"templating": time.monotonic() - t0} return RenderedFile(templated_file, templater_violations, config, time_dict, fname)
def assert_rule_fail_in_sql(code, sql, configs=None, line_numbers=None): """Assert that a given rule does fail on the given sql.""" # Set up the config to only use the rule we are testing. cfg = FluffConfig(configs=configs, overrides={"rules": code}) # Lint it using the current config (while in fix mode) linted = Linter(config=cfg).lint_string(sql, fix=True) lerrs = linted.get_violations() print(f"Errors Found: {lerrs}") for e in lerrs: if e.desc().startswith("Unexpected exception"): pytest.fail(f"Linter failed with {e.desc()}") parse_errors = list(filter(lambda v: type(v) == SQLParseError, lerrs)) if parse_errors: pytest.fail(f"Found the following parse errors in test case: {parse_errors}") if not any(v.rule.code == code for v in lerrs): pytest.fail( f"No {code} failures found in query which should fail.", pytrace=False, ) if line_numbers: actual_line_numbers = [e.line_no for e in lerrs] if line_numbers != actual_line_numbers: pytest.fail( "Expected errors on lines {}, but got errors on lines {}".format( line_numbers, actual_line_numbers ) ) # The query should already have been fixed if possible so just return the raw. return linted.tree.raw
def test_rule_set_return_informative_error_when_rule_not_registered(): """Assert that a rule that throws an exception returns it as a validation.""" cfg = FluffConfig(overrides={"dialect": "ansi"}) with pytest.raises(ValueError) as e: get_rule_from_set("L000", config=cfg) e.match("'L000' not in")
def __init__( self, config: Optional[FluffConfig] = None, formatter: Any = None, dialect: Optional[str] = None, rules: Optional[List[str]] = None, user_rules: Optional[List[BaseRule]] = None, exclude_rules: Optional[List[str]] = None, ) -> None: # Store the config object self.config = FluffConfig.from_kwargs( config=config, dialect=dialect, rules=rules, exclude_rules=exclude_rules, # Don't require a dialect to be provided yet. Defer this until we # are actually linting something, since the directory we are linting # from may provide additional configuration, including a dialect. require_dialect=False, ) # Get the dialect and templater self.dialect = self.config.get("dialect_obj") self.templater = self.config.get("templater_obj") # Store the formatter for output self.formatter = formatter # Store references to user rule classes self.user_rules = user_rules or []
def test__rules__std_file(rule, path, violations): """Test the linter finds the given errors in (and only in) the right places.""" assert_rule_raises_violations_in_file( rule=rule, fpath=path, violations=violations, fluff_config=FluffConfig(overrides=dict(rules=rule)), )
def test__rules__std_file_dbt(rule, path, violations, in_dbt_project_dir): # noqa """Test the linter finds the given errors in (and only in) the right places (DBT).""" assert_rule_raises_violations_in_file( rule=rule, fpath=path, violations=violations, fluff_config=FluffConfig(configs=DBT_FLUFF_CONFIG, overrides=dict(rules=rule)), )
def test__rule_test_case(test_case): """Run the tests.""" res = rules__test_helper(test_case) if res is not None and res != test_case.fail_str: cfg = FluffConfig(configs=test_case.configs) rule = get_rule_from_set(test_case.rule, config=cfg) assert is_fix_compatible( rule ), f'Rule {test_case.rule} returned fixes but does not specify "@document_fix_compatible".'
def test__rule_test_case(test_case, caplog): """Run the tests.""" with caplog.at_level(logging.DEBUG, logger="sqlfluff.rules"): res = rules__test_helper(test_case) if res is not None and res != test_case.fail_str: cfg = FluffConfig(configs=test_case.configs) rule = get_rule_from_set(test_case.rule, config=cfg) assert is_fix_compatible( rule ), f'Rule {test_case.rule} returned fixes but does not specify "@document_fix_compatible".'
def get_encoding(fname: str, config: FluffConfig) -> str: """Get the encoding of the file (autodetect).""" encoding_config = config.get("encoding", default="autodetect") if encoding_config == "autodetect": with open(fname, "rb") as f: data = f.read() return chardet.detect(data)["encoding"] return encoding_config
def _get_macros_path(self, config: FluffConfig) -> Optional[List[str]]: if config: macros_path = config.get_section( (self.templater_selector, self.name, "load_macros_from_path")) if macros_path: result = [ s.strip() for s in macros_path.split(",") if s.strip() ] if result: return result return None
def test_rule_exception_is_caught_to_validation(): """Assert that a rule that throws an exception on _eval returns it as a validation.""" @std_rule_set.register class Rule_LXXX(BaseCrawler): """Rule that throws an exception.""" def _eval(self, segment, parent_stack, **kwargs): raise Exception( "Catch me or I'll deny any linting results from you") linter = Linter(config=FluffConfig(overrides=dict(rules="LXXX"))) assert linter.lint_string("select 1").check_tuples() == [("LXXX", 1, 1)]
def _load_raw_file_and_config( fname: str, root_config: FluffConfig) -> Tuple[str, FluffConfig, str]: """Load a raw file and the associated config.""" file_config = root_config.make_child_from_path(fname) encoding = get_encoding(fname=fname, config=file_config) with open(fname, encoding=encoding, errors="backslashreplace") as target_file: raw_file = target_file.read() # Scan the raw file for config commands. file_config.process_raw_file_for_config(raw_file) # Return the raw file and config return raw_file, file_config, encoding
def assert_rule_pass_in_sql(code, sql, configs=None): """Assert that a given rule doesn't fail on the given sql.""" # Configs allows overrides if we want to use them. cfg = FluffConfig(configs=configs) r = get_rule_from_set(code, config=cfg) linter = Linter(config=cfg) rendered = linter.render_string(sql, fname="<STR>", config=cfg, encoding="utf-8") parsed = linter.parse_rendered(rendered, recurse=True) if parsed.violations: pytest.fail(parsed.violations[0].desc() + "\n" + parsed.tree.stringify()) print(f"Parsed:\n {parsed.tree.stringify()}") lerrs, _, _, _ = r.crawl(parsed.tree, [], dialect=cfg.get("dialect_obj"), templated_file=rendered[0]) print(f"Errors Found: {lerrs}") if any(v.rule.code == code for v in lerrs): pytest.fail(f"Found {code} failures in query which should pass.", pytrace=False)
def __init__( self, config: Optional[FluffConfig] = None, last_resort_lexer: Optional[SingletonMatcher] = None, dialect: Optional[str] = None, ): # Allow optional config and dialect self.config = FluffConfig.from_kwargs(config=config, dialect=dialect) lexer_struct = self.config.get("dialect_obj").get_lexer_struct() self.matcher = RepeatedMultiMatcher.from_struct(lexer_struct) self.last_resort_lexer = last_resort_lexer or RegexMatcher.from_shorthand( "<unlexable>", r"[^\t\n\,\.\ \-\+\*\\\/\'\"\;\:\[\]\(\)\|]*", is_code=True )
def test__rules__std_L003_process_raw_stack(generate_test_segments): """Test the _process_raw_stack function. Note: This test probably needs expanding. It doesn't really check enough of the full functionality. """ cfg = FluffConfig() r = get_rule_from_set("L003", config=cfg) test_stack = generate_test_segments(["bar", "\n", " ", "foo", "baar", " \t "]) res = r._process_raw_stack(test_stack) print(res) assert sorted(res.keys()) == [1, 2] assert res[2]["indent_size"] == 5
def test__rules__runaway_fail_catch(): """Test that we catch runaway rules.""" runaway_limit = 5 my_query = "SELECT * FROM foo" # Set up the config to only use the rule we are testing. cfg = FluffConfig(overrides={"rules": "T001", "runaway_limit": runaway_limit}) # Lint it using the current config (while in fix mode) linter = Linter(config=cfg, user_rules=[Rule_T001]) # In theory this step should result in an infinite # loop, but the loop limit should catch it. linted = linter.lint_string(my_query, fix=True) # We should have a lot of newlines in there. # The number should equal the runaway limit assert linted.tree.raw.count("\n") == runaway_limit
def test__rules__runaway_fail_catch(): """Test that we catch runaway rules.""" runaway_limit = 5 my_query = "SELECT * FROM foo" # Set up the config to only use the rule we are testing. cfg = FluffConfig(overrides={ "rules": "T001", "runaway_limit": runaway_limit }) # Lint it using the current config (while in fix mode) linter = Linter(config=cfg, user_rules=[Rule_T001]) # In theory this step should result in an infinite # loop, but the loop limit should catch it. linted = linter.lint_string(my_query, fix=True) # When the linter hits the runaway limit, it returns the original SQL tree. assert linted.tree.raw == my_query
def __init__( self, config: Optional[FluffConfig] = None, last_resort_lexer: Optional[StringLexer] = None, dialect: Optional[str] = None, ): # Allow optional config and dialect self.config = FluffConfig.from_kwargs(config=config, dialect=dialect) # Store the matchers self.lexer_matchers = self.config.get( "dialect_obj").get_lexer_matchers() self.last_resort_lexer = last_resort_lexer or RegexLexer( "<unlexable>", r"[^\t\n\,\.\ \-\+\*\\\/\'\"\;\:\[\]\(\)\|]*", UnlexableSegment, )
def assert_rule_fail_in_sql(code, sql, configs=None): """Assert that a given rule does fail on the given sql.""" # Set up the config to only use the rule we are testing. cfg = FluffConfig(configs=configs, overrides={"rules": code}) # Lint it using the current config (while in fix mode) linted = Linter(config=cfg).lint_string(sql, fix=True) lerrs = linted.get_violations() print("Errors Found: {0}".format(lerrs)) parse_errors = list(filter(lambda v: type(v) == SQLParseError, lerrs)) if parse_errors: pytest.fail(f"Found the following parse errors in test case: {parse_errors}") if not any(v.rule.code == code for v in lerrs): pytest.fail( "No {0} failures found in query which should fail.".format(code), pytrace=False, ) # The query should already have been fixed if possible so just return the raw. return linted.tree.raw
def __init__( self, config: Optional[FluffConfig] = None, formatter: Any = None, dialect: Optional[str] = None, rules: Optional[Union[str, List[str]]] = None, user_rules: Optional[Union[str, List[str]]] = None, ) -> None: # Store the config object self.config = FluffConfig.from_kwargs(config=config, dialect=dialect, rules=rules) # Get the dialect and templater self.dialect = self.config.get("dialect_obj") self.templater = self.config.get("templater_obj") # Store the formatter for output self.formatter = formatter # Store references to user rule classes self.user_rules = user_rules or []
def test_rule_exception_is_caught_to_validation(): """Assert that a rule that throws an exception returns it as a validation.""" std_rule_set = get_ruleset() @std_rule_set.register class Rule_T000(BaseRule): """Rule that throws an exception.""" groups = ("all", ) crawl_behaviour = RootOnlyCrawler() def _eval(self, segment, parent_stack, **kwargs): raise Exception( "Catch me or I'll deny any linting results from you") linter = Linter( config=FluffConfig(overrides=dict(rules="T000", dialect="ansi")), user_rules=[Rule_T000], ) assert linter.lint_string("select 1").check_tuples() == [("T000", 1, 1)]
def test__rules__std_L007_before(): """Verify that L007 returns the correct error message when before is used.""" sql = """ SELECT a, b FROM foo WHERE a = 1 AND b = 2 """ config = FluffConfig( configs={"rules": {"L007": {"operator_new_lines": "before"}}}, overrides={"dialect": "ansi"}, ) # The sqlfluff.lint API doesn't allow us to pass config so need to do what it does linter = Linter(config=config) result_records = linter.lint_string_wrapped(sql).as_records() result = result_records[0]["violations"] assert "L007" in [r["code"] for r in result] assert before_description in [r["description"] for r in result]
def _wrapped(self, *, in_str: str, fname: str, config: FluffConfig = None, **kwargs): if config: limit = config.get("large_file_skip_char_limit") if limit: templater_logger.warning( "The config value large_file_skip_char_limit was found set. " "This feature will be removed in a future release, please " "use the more efficient 'large_file_skip_byte_limit' instead." ) if limit and len(in_str) > limit: raise SQLFluffSkipFile( f"Length of file {fname!r} is over {limit} characters. " "Skipping to avoid parser lock. Users can increase this limit " "in their config by setting the 'large_file_skip_char_limit' " "value, or disable by setting it to zero.") return func(self, in_str=in_str, fname=fname, config=config, **kwargs)
def test__rules__std_L003_process_raw_stack(generate_test_segments, test_elems, result): """Test the _process_raw_stack function. Note: This test probably needs expanding. It doesn't really check enough of the full functionality. """ cfg = FluffConfig() r = get_rule_from_set("L003", config=cfg) test_stack = generate_test_segments(test_elems) res = r._process_raw_stack(test_stack, {}) print(res) # Verify structure assert isinstance(res, dict) assert all(isinstance(k, int) for k in res.keys()) assert all(isinstance(v, dict) for v in res.values()) # Check keys are all present assert all( v.keys() == { "line_no", "templated_line", "line_buffer", "indent_buffer", "indent_size", "indent_balance", "hanging_indent", "clean_indent", } for v in res.values()) for k in res: # For testing purposes, we won't be checking the buffer fields. They're # just too hard to create in the test cases and aren't critical in # determining what course of action to take. Most of the logic uses the # values which we *are* still testing. del res[k]["line_buffer"] del res[k]["indent_buffer"] # We also don't check the "templated_file" flag. These tests don't # exercise that code. del res[k]["templated_line"] assert res == result
def test__rules__std_L007_after(): """Verify orrect error message when after is explicitly used.""" sql = """ SELECT a, b FROM foo WHERE a = 1 AND b = 2 """ config = FluffConfig( configs={"rules": { "L007": { "operator_new_lines": "after" } }}) # The sqlfluff.lint API doesn't allow us to pass config so need to do what it does linter = Linter(config=config) result_records = linter.lint_string_wrapped(sql).as_records() result = result_records[0]["violations"] assert "L007" in [r["code"] for r in result] assert after_description in [r["description"] for r in result]
def assert_rule_fail_in_sql(code, sql, configs=None, line_numbers=None): """Assert that a given rule does fail on the given sql.""" print("# Asserting Rule Fail in SQL") # Set up the config to only use the rule we are testing. overrides = {"rules": code} if configs is None or "core" not in configs or "dialect" not in configs[ "core"]: overrides["dialect"] = "ansi" cfg = FluffConfig(configs=configs, overrides=overrides) # Lint it using the current config (while in fix mode) linted = Linter(config=cfg).lint_string(sql, fix=True) lerrs = linted.get_violations() print("Errors Found:") for e in lerrs: print(" " + repr(e)) if e.desc().startswith("Unexpected exception"): pytest.fail(f"Linter failed with {e.desc()}") # pragma: no cover parse_errors = list( filter(lambda v: isinstance(v, (SQLParseError, SQLTemplaterError)), lerrs)) if parse_errors: pytest.fail( f"Found the following parse errors in test case: {parse_errors}") if not any(v.rule.code == code for v in lerrs): print(f"Parsed File:\n{linted.tree.stringify()}") pytest.fail( f"No {code} failures found in query which should fail.", pytrace=False, ) if line_numbers: actual_line_numbers = [e.line_no for e in lerrs] if line_numbers != actual_line_numbers: # pragma: no cover pytest.fail( "Expected errors on lines {}, but got errors on lines {}". format(line_numbers, actual_line_numbers)) fixed, _ = linted.fix_string() return fixed, linted.violations