def test__config__glob_include_config_tests(): """Test linting with a glob pattern in rules. This looks like a linter test but it's actually a config test. """ lntr = Linter( config=FluffConfig.from_path("test/fixtures/config/glob_include")) lnt = lntr.lint_path("test/fixtures/config/glob_include/test.sql") violations = lnt.check_tuples(by_path=True) for k in violations: assert ("L050", 1, 1) in violations[k] assert ("L051", 12, 1) in violations[k] assert ("L052", 12, 9) in violations[k] assert ("L027", 10, 8) in violations[k] assert "L044" not in [c[0] for c in violations[k]]
def assert_structure(yaml_loader, path, code_only=True): """Check that a parsed sql file matches the yaml file with the same name.""" lntr = Linter() p = list(lntr.parse_path(path + ".sql")) parsed = p[0][0] if parsed is None: print(p) raise RuntimeError(p[0][1]) # Whitespace is important here to test how that's treated tpl = parsed.to_tuple(code_only=code_only, show_raw=True) # Check nothing unparsable if "unparsable" in parsed.type_set(): print(parsed.stringify()) raise ValueError("Input file is contains unparsable.") expected = yaml_loader(path + ".yml") assert tpl == expected
def test_linter_noqa_with_templating(): """Similar to test_linter_noqa, but uses templating (Jinja).""" lntr = Linter(config=FluffConfig(overrides={ "templater": "jinja", "rules": "L016", })) sql = """ {%- set a_var = ["1", "2"] -%} SELECT this_is_just_a_very_long_line_for_demonstration_purposes_of_a_bug_involving_templated_sql_files, --noqa: L016 this_is_not_so_big FROM a_table """ result = lntr.lint_string(sql) assert not result.get_violations()
def assert_rule_pass_in_sql(code, sql, configs=None): """Assert that a given rule doesn't fail on the given sql.""" # Configs allows overrides if we want to use them. cfg = FluffConfig(configs=configs) r = get_rule_from_set(code, config=cfg) parsed = Linter(config=cfg).parse_string(sql) if parsed.violations: pytest.fail(parsed.violations[0].desc() + "\n" + parsed.tree.stringify()) print("Parsed:\n {0}".format(parsed.tree.stringify())) lerrs, _, _, _ = r.crawl(parsed.tree, dialect=cfg.get("dialect_obj")) print("Errors Found: {0}".format(lerrs)) if any(v.rule.code == code for v in lerrs): pytest.fail( "Found {0} failures in query which should pass.".format(code), pytrace=False)
def test__rules__runaway_fail_catch(): """Test that we catch runaway rules.""" runaway_limit = 5 my_query = "SELECT * FROM foo" # Set up the config to only use the rule we are testing. cfg = FluffConfig(overrides={ "rules": "T001", "runaway_limit": runaway_limit }) # Lint it using the current config (while in fix mode) linter = Linter(config=cfg, user_rules=[Rule_T001]) # In theory this step should result in an infinite # loop, but the loop limit should catch it. linted = linter.lint_string(my_query, fix=True) # We should have a lot of newlines in there. # The number should equal the runaway limit assert linted.tree.raw.count("\n") == runaway_limit
def test_rule_exception_is_caught_to_validation(): """Assert that a rule that throws an exception on _eval returns it as a validation.""" std_rule_set = get_ruleset() @std_rule_set.register class Rule_T000(BaseCrawler): """Rule that throws an exception.""" def _eval(self, segment, parent_stack, **kwargs): raise Exception("Catch me or I'll deny any linting results from you") linter = Linter( config=FluffConfig(overrides=dict(rules="T000")), user_rules=[Rule_T000], ) assert linter.lint_string("select 1").check_tuples() == [("T000", 1, 1)]
def test__rules__runaway_fail_catch(): """Test that we catch runaway rules.""" runaway_limit = 5 my_query = "SELECT * FROM foo" # Set up the config to only use the rule we are testing. cfg = FluffConfig(overrides={ "rules": "T001", "runaway_limit": runaway_limit, "dialect": "ansi" }) # Lint it using the current config (while in fix mode) linter = Linter(config=cfg, user_rules=[Rule_T001]) # In theory this step should result in an infinite # loop, but the loop limit should catch it. linted = linter.lint_string(my_query, fix=True) # When the linter hits the runaway limit, it returns the original SQL tree. assert linted.tree.raw == my_query
def fix( sql: str, dialect: str = "ansi", rules: Optional[List[str]] = None, exclude_rules: Optional[List[str]] = None, config_path: Optional[str] = None, fix_even_unparsable: Optional[bool] = None, ) -> str: """Fix a SQL string. Args: sql (:obj:`str`): The SQL to be fixed. dialect (:obj:`str`, optional): A reference to the dialect of the SQL to be fixed. Defaults to `ansi`. rules (:obj:`Optional[List[str]`, optional): A subset of rule references to fix for. Defaults to None. exclude_rules (:obj:`Optional[List[str]`, optional): A subset of rule references to avoid fixing for. Defaults to None. config_path (:obj:`Optional[str]`, optional): A path to a .sqlfluff config. Defaults to None. Returns: :obj:`str` for the fixed SQL if possible. """ cfg = get_simple_config( dialect=dialect, rules=rules, exclude_rules=exclude_rules, config_path=config_path, ) linter = Linter(config=cfg) result = linter.lint_string_wrapped(sql, fix=True) if fix_even_unparsable is None: fix_even_unparsable = cfg.get("fix_even_unparsable") should_fix = True if not fix_even_unparsable: # If fix_even_unparsable wasn't set, check for templating or parse # errors and suppress fixing if there were any. _, num_filtered_errors = result.count_tmp_prs_errors() if num_filtered_errors > 0: should_fix = False if should_fix: sql = result.paths[0].files[0].fix_string()[0] return sql
def assert_rule_fail_in_sql(code, sql, configs=None): """Assert that a given rule does fail on the given sql.""" # Set up the config to only use the rule we are testing. cfg = FluffConfig(configs=configs, overrides={"rules": code}) # Lint it using the current config (while in fix mode) linted = Linter(config=cfg).lint_string(sql, fix=True) lerrs = linted.get_violations() print("Errors Found: {0}".format(lerrs)) parse_errors = list(filter(lambda v: type(v) == SQLParseError, lerrs)) if parse_errors: pytest.fail(f"Found the following parse errors in test case: {parse_errors}") if not any(v.rule.code == code for v in lerrs): pytest.fail( "No {0} failures found in query which should fail.".format(code), pytrace=False, ) # The query should already have been fixed if possible so just return the raw. return linted.tree.raw
def test__config__nested_config_tests(): """Test linting with overriden config in nested paths. This looks like a linter test but it's actually a config test. """ lntr = Linter(config=FluffConfig(overrides=dict(exclude_rules="L002"))) lnt = lntr.lint_path("test/fixtures/config/inheritance_b") violations = lnt.check_tuples(by_path=True) for k in violations: if k.endswith("nested\\example.sql"): assert ("L003", 1, 4) in violations[k] assert ("L009", 1, 12) in violations[k] assert "L002" not in [c[0] for c in violations[k]] elif k.endswith("inheritance_b\\example.sql"): assert ("L003", 1, 4) in violations[k] assert "L002" not in [c[0] for c in violations[k]] assert "L009" not in [c[0] for c in violations[k]]
def test__dialect__ansi_parse_indented_joins(sql_string, indented_joins, meta_loc): """Test parsing of meta segments using Conditional works with indented_joins.""" lnt = Linter(config=FluffConfig( configs={"indentation": { "indented_joins": indented_joins }}, overrides={"dialect": "ansi"}, )) parsed = lnt.parse_string(sql_string) # Check that there's nothing unparsable assert "unparsable" not in parsed.tree.type_set() # Check all the segments that *should* be metas, ARE. # NOTE: This includes the end of file marker. res_meta_locs = tuple( idx for idx, raw_seg in enumerate(parsed.tree.get_raw_segments()) if raw_seg.is_meta) assert res_meta_locs == meta_loc
def assert_rule_fail_in_sql(code, sql, configs=None, runaway_limit=20): """Assert that a given rule does fail on the given sql.""" # Configs allows overrides if we want to use them. cfg = FluffConfig(configs=configs) r = get_rule_from_set(code, config=cfg) parsed, _, _ = Linter(config=cfg).parse_string(sql) print("Parsed:\n {0}".format(parsed.stringify())) lerrs, _, _, _ = r.crawl(parsed, dialect=cfg.get("dialect_obj"), fix=True) print("Errors Found: {0}".format(lerrs)) if not any(v.rule.code == code for v in lerrs): pytest.fail( "No {0} failures found in query which should fail.".format(code), pytrace=False, ) fixed = parsed # use this as our buffer (yes it's a bit of misnomer right here) loop_idx = 0 while loop_idx < runaway_limit: # We get the errors again, but this time skip the assertion # because we're in the loop. If we asserted on every loop then # we're stuffed. lerrs, _, _, _ = r.crawl(fixed, dialect=cfg.get("dialect_obj"), fix=True) print("Errors Found: {0}".format(lerrs)) fixes = [] for e in lerrs: fixes += e.fixes if not fixes: print("Done") break print("Fixes to apply: {0}".format(fixes)) l_fixes = fixes # Save the fixes to compare to later fixed, fixes = fixed.apply_fixes(fixes) # iterate until all fixes applied if fixes: if fixes == l_fixes: raise RuntimeError( "Fixes aren't being applied: {0!r}".format(fixes)) loop_idx += 1 else: raise ValueError( "Runaway loop limit reached for rule! This example never stabilises." ) return fixed.raw
def test__attempt_to_change_templater_warning(caplog): """Test warning if user tries to change templater in .sqlfluff file in subdirectory.""" initial_config = FluffConfig(configs={"core": {"templater": "jinja"}}) lntr = Linter(config=initial_config) updated_config = FluffConfig(configs={"core": {"templater": "dbt"}}) logger = logging.getLogger("sqlfluff") original_propagate_value = logger.propagate try: logger.propagate = True with caplog.at_level(logging.WARNING, logger="sqlfluff.linter"): lntr.render_string( in_str="select * from table", fname="test.sql", config=updated_config, encoding="utf-8", ) assert "Attempt to set templater to " in caplog.text finally: logger.propagate = original_propagate_value
def parse(sql, dialect="ansi"): """Parse a sql string or file. Args: sql (:obj:`str` or file-like object): The sql to be linted either as a string or a subclass of :obj:`TextIOBase`. dialect (:obj:`str`, optional): A reference to the dialect of the sql to be linted. Defaults to `ansi`. Returns: :obj:`ParsedString` containing the parsed structure. """ sql = _unify_str_or_file(sql) linter = Linter(dialect=dialect) parsed = linter.parse_string(sql) # If we encounter any parsing errors, raise them in a combined issue. if parsed.violations: raise APIParsingError(parsed.violations) return parsed
def violations(src_path: str) -> List[Violation]: """Return list of violations. Given the path to a .sql file, analyze it and return a list of violations (i.e. formatting or style issues). """ linter = Linter(config=FluffConfig.from_root()) linted_path = linter.lint_path(src_path, ignore_non_existent_files=True) result = [] for violation in linted_path.get_violations(): try: # Normal SQLFluff warnings message = f"{violation.rule_code()}: {violation.description}" except AttributeError: # Parse errors message = str(violation) result.append(Violation(violation.line_no, message)) return result
def assert_rule_pass_in_sql(code, sql, configs=None, msg=None): """Assert that a given rule doesn't fail on the given sql.""" # Configs allows overrides if we want to use them. print("# Asserting Rule Pass in SQL") if configs is None: configs = {} core = configs.setdefault("core", {}) core["rules"] = code overrides = {} if "dialect" not in configs["core"]: overrides["dialect"] = "ansi" cfg = FluffConfig(configs=configs, overrides=overrides) linter = Linter(config=cfg) # This section is mainly for aid in debugging. rendered = linter.render_string(sql, fname="<STR>", config=cfg, encoding="utf-8") parsed = linter.parse_rendered(rendered, recurse=True) if parsed.violations: if msg: print(msg) # pragma: no cover pytest.fail(parsed.violations[0].desc() + "\n" + parsed.tree.stringify()) print(f"Parsed:\n {parsed.tree.stringify()}") # Note that lint_string() runs the templater and parser again, in order to # test the whole linting pipeline in the same way that users do. In other # words, the "rendered" and "parsed" variables above are irrelevant to this # line of code. lint_result = linter.lint_string(sql, config=cfg, fname="<STR>") lerrs = lint_result.violations if any(v.rule.code == code for v in lerrs): print("Errors Found:") for e in lerrs: print(" " + repr(e)) if msg: print(msg) # pragma: no cover pytest.fail(f"Found {code} failures in query which should pass.", pytrace=False)
def test_linter_noqa(): """Test "noqa" feature at the higher "Linter" level.""" lntr = Linter( config=FluffConfig( overrides={ "dialect": "bigquery", # Use bigquery to allow hash comments. "rules": "L012, L019", } ) ) sql = """ SELECT col_a a, col_b b, --noqa: disable=L012 col_c c, col_d d, --noqa: enable=L012 col_e e, col_f f, col_g g, --noqa col_h h, col_i i, --noqa:L012 col_j j, col_k k, --noqa:L013 col_l l, col_m m, col_n n, --noqa: disable=all col_o o, col_p p, --noqa: enable=all col_q q, --Inline comment --noqa: L012 col_r r, /* Block comment */ --noqa: L012 col_s s # hash comment --noqa: L012 -- We trigger both L012 (implicit aliasing) -- and L019 (leading commas) here to -- test glob ignoring of multiple rules. , col_t t --noqa: L01* , col_u u -- Some comment --noqa: L01* , col_v v -- We can ignore both L012 and L019 -- noqa: L01[29] FROM foo """ result = lntr.lint_string(sql) violations = result.get_violations() assert {3, 6, 7, 8, 10, 12, 13, 14, 15, 18} == {v.line_no for v in violations}
def test__linter__linting_parallel_thread(force_error, monkeypatch): """Run linter in parallel mode using threads. Similar to test__linter__linting_result_get_violations but uses a thread pool of 1 worker to test parallel mode without subprocesses. This lets the tests capture code coverage information for the backend parts of parallel execution without having to jump through hoops. """ if not force_error: monkeypatch.setattr(Linter, "allow_process_parallelism", False) else: def _create_pool(*args, **kwargs): class ErrorPool: def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): pass def imap_unordered(self, *args, **kwargs): yield runner.DelayedException(ValueError()) return ErrorPool() monkeypatch.setattr(runner.MultiProcessRunner, "_create_pool", _create_pool) config = FluffConfig(overrides={"dialect": "ansi"}) output_stream = make_output_stream(config, None, os.devnull) lntr = Linter( formatter=OutputStreamFormatter(output_stream, False, verbosity=0), dialect="ansi", ) result = lntr.lint_paths( ("test/fixtures/linter/comma_errors.sql", ), processes=2, ) all([type(v) == SQLLintError for v in result.get_violations()])
def get_linter_and_formatter( cfg: FluffConfig, output_stream: Optional[OutputStream] = None ) -> Tuple[Linter, OutputStreamFormatter]: """Get a linter object given a config.""" try: # We're just making sure it exists at this stage. # It will be fetched properly in the linter. dialect = cfg.get("dialect") if dialect: dialect_selector(dialect) except KeyError: # pragma: no cover click.echo(f"Error: Unknown dialect '{cfg.get('dialect')}'") sys.exit(EXIT_ERROR) formatter = OutputStreamFormatter( output_stream=output_stream or make_output_stream(cfg), nocolor=cfg.get("nocolor"), verbosity=cfg.get("verbose"), output_line_length=cfg.get("output_line_length"), ) return Linter(config=cfg, formatter=formatter), formatter
def test__rules__std_L009_and_L052_interaction() -> None: """Test interaction between L009 and L052 doesn't stop L052 from being applied.""" # Test sql with no final newline and no final semicolon. sql = "SELECT foo FROM bar" # Ensure final semicolon requirement is active. cfg = FluffConfig() cfg.set_value(config_path=["rules", "L052", "require_final_semicolon"], val=True) linter = Linter(config=cfg) # Return linted/fixed file. linted_file = linter.lint_string(sql, fix=True) # Check expected lint errors are raised. assert set([v.rule.code for v in linted_file.violations]) == {"L009", "L052"} # Check file is fixed. assert linted_file.fix_string()[0] == "SELECT foo FROM bar;\n"
def fix(sql, dialect="ansi", rules=None): """Fix a sql string or file. Args: sql (:obj:`str` or file-like object): The sql to be linted either as a string or a subclass of :obj:`TextIOBase`. dialect (:obj:`str`, optional): A reference to the dialect of the sql to be linted. Defaults to `ansi`. rules (:obj:`str` or iterable of :obj:`str`, optional): A subset of rule reference to lint for. Returns: :obj:`str` for the fixed sql if possible. """ sql = _unify_str_or_file(sql) linter = Linter(dialect=dialect, rules=rules) result = linter.lint_string_wrapped(sql, fix=True) fixed_string = result.paths[0].files[0].fix_string()[0] return fixed_string
def test__dbt_templated_models_fix_does_not_corrupt_file( project_dir, path, caplog # noqa: F811 ): """Test issues where previously "sqlfluff fix" corrupted the file.""" test_glob = os.path.join(project_dir, os.path.dirname(path), "*FIXED.sql") _clean_path(test_glob) lntr = Linter(config=FluffConfig(configs=DBT_FLUFF_CONFIG)) with caplog.at_level(logging.INFO, logger="sqlfluff.linter"): lnt = lntr.lint_path(os.path.join(project_dir, path), fix=True) try: lnt.persist_changes(fixed_file_suffix="FIXED") with open(os.path.join(project_dir, path + ".after")) as f: comp_buff = f.read() with open(os.path.join(project_dir, path.replace(".sql", "FIXED.sql"))) as f: fixed_buff = f.read() assert fixed_buff == comp_buff finally: _clean_path(test_glob)
def test__dbt_templated_models_fix_does_not_corrupt_file( project_dir, path # noqa: F811 ): """Test fix for issue 1608. Previously "sqlfluff fix" corrupted the file.""" for fsp in glob.glob(os.path.join(project_dir, "snapshots", "*FIXED.sql")): os.remove(fsp) lntr = Linter(config=FluffConfig(configs=DBT_FLUFF_CONFIG)) lnt = lntr.lint_path(os.path.join(project_dir, path), fix=True) try: lnt.persist_changes(fixed_file_suffix="FIXED") with open(os.path.join(project_dir, path + ".after")) as f: comp_buff = f.read() with open(os.path.join(project_dir, path.replace(".sql", "FIXED.sql"))) as f: fixed_buff = f.read() assert fixed_buff == comp_buff finally: for fsp in glob.glob( os.path.join(project_dir, "snapshots", "*FIXED.sql")): os.remove(fsp)
def lint(sql, dialect="ansi", rules=None): """Lint a sql string or file. Args: sql (:obj:`str` or file-like object): The sql to be linted either as a string or a subclass of :obj:`TextIOBase`. dialect (:obj:`str`, optional): A reference to the dialect of the sql to be linted. Defaults to `ansi`. rules (:obj:`str` or iterable of :obj:`str`, optional): A subset of rule reference to lint for. Returns: :obj:`list` of :obj:`dict` for each violation found. """ sql = _unify_str_or_file(sql) linter = Linter(dialect=dialect, rules=rules) result = linter.lint_string_wrapped(sql) result_records = result.as_records() # Return just the violations for this file return [] if not result_records else result_records[0]["violations"]
def test__rules__std_L007_before(): """Verify that L007 returns the correct error message when before is used.""" sql = """ SELECT a, b FROM foo WHERE a = 1 AND b = 2 """ config = FluffConfig( configs={"rules": {"L007": {"operator_new_lines": "before"}}}, overrides={"dialect": "ansi"}, ) # The sqlfluff.lint API doesn't allow us to pass config so need to do what it does linter = Linter(config=config) result_records = linter.lint_string_wrapped(sql).as_records() result = result_records[0]["violations"] assert "L007" in [r["code"] for r in result] assert before_description in [r["description"] for r in result]
def test__linter__linting_parallel_thread(force_error, monkeypatch): """Run linter in parallel mode using threads. Similar to test__linter__linting_result_get_violations but uses a thread pool of 1 worker to test parallel mode without subprocesses. This lets the tests capture code coverage information for the backend parts of parallel execution without having to jump through hoops. """ monkeypatch.setattr(Linter, "MIN_THRESHOLD_PARALLEL", 1) if not force_error: monkeypatch.setattr(Linter, "PARALLEL_CLS", runner.MultiThreadRunner) else: def _create_pool(*args, **kwargs): class ErrorPool: def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): pass def imap_unordered(self, *args, **kwargs): yield runner.DelayedException(ValueError()) return ErrorPool() monkeypatch.setattr(runner.MultiProcessRunner, "_create_pool", _create_pool) lntr = Linter( formatter=CallbackFormatter(callback=lambda m: None, verbosity=0)) result = lntr.lint_paths( ("test/fixtures/linter/comma_errors.sql", ), parallel=1, ) all([type(v) == SQLLintError for v in result.get_violations()])
def test_linter_noqa_prs(): """Test "noqa" feature to ignore PRS at the higher "Linter" level.""" lntr = Linter( config=FluffConfig( overrides={ "dialect": "bigquery", "exclude_rules": "L050", } ) ) sql = """ CREATE TABLE IF NOT EXISTS Test.events (userID STRING, eventName STRING, eventID INTEGER, device STRUCT < mobileBrandName STRING, -- noqa: PRS mobileModelName STRING>); Insert into Test.events VALUES ("1","abc",123,STRUCT("htc","10")); """ result = lntr.lint_string(sql) violations = result.get_violations() assert not violations
def assert_rule_pass_in_sql(code, sql, configs=None): """Assert that a given rule doesn't fail on the given sql.""" # Configs allows overrides if we want to use them. cfg = FluffConfig(configs=configs) r = get_rule_from_set(code, config=cfg) linter = Linter(config=cfg) rendered = linter.render_string(sql, fname="<STR>", config=cfg, encoding="utf-8") parsed = linter.parse_rendered(rendered, recurse=True) if parsed.violations: pytest.fail(parsed.violations[0].desc() + "\n" + parsed.tree.stringify()) print(f"Parsed:\n {parsed.tree.stringify()}") lerrs, _, _, _ = r.crawl(parsed.tree, [], dialect=cfg.get("dialect_obj"), templated_file=rendered[0]) print(f"Errors Found: {lerrs}") if any(v.rule.code == code for v in lerrs): pytest.fail(f"Found {code} failures in query which should pass.", pytrace=False)
def test__rules__std_L007_after(): """Verify orrect error message when after is explicitly used.""" sql = """ SELECT a, b FROM foo WHERE a = 1 AND b = 2 """ config = FluffConfig( configs={"rules": { "L007": { "operator_new_lines": "after" } }}) # The sqlfluff.lint API doesn't allow us to pass config so need to do what it does linter = Linter(config=config) result_records = linter.lint_string_wrapped(sql).as_records() result = result_records[0]["violations"] assert "L007" in [r["code"] for r in result] assert after_description in [r["description"] for r in result]
def _validate_dialect_specific_statements(dialect, segment_cls, raw, stmt_count): """This validates one or multiple statements against specified segment class. It even validates the number of parsed statements with the number of expected statements. """ lnt = Linter(dialect=dialect) parsed = lnt.parse_string(raw) assert len(parsed.violations) == 0 # Find any unparsable statements typs = parsed.tree.type_set() assert "unparsable" not in typs # Find the expected type in the parsed segment child_segments = [ seg for seg in parsed.tree.recursive_crawl(segment_cls.type) ] assert len(child_segments) == stmt_count # Check if all child segments are the correct type for c in child_segments: assert isinstance(c, segment_cls)