def _format_config(self, linter): """Format the config of a `Linter`.""" text_buffer = StringIO() # Only show version information if verbosity is high enough if self._verbosity > 0: text_buffer.write("==== sqlfluff ====\n") config_content = [ ("sqlfluff", get_package_version()), ("python", get_python_version()), ("dialect", linter.dialect.name), ("verbosity", self._verbosity), ] text_buffer.write(cli_table(config_content, col_width=25)) text_buffer.write("\n") if linter.config.get("rule_whitelist"): text_buffer.write( cli_table( [("rules", ", ".join( linter.config.get("rule_whitelist")))], col_width=41, )) if self._verbosity > 1: text_buffer.write("== Raw Config:\n") text_buffer.write(format_config_vals( linter.config.iter_vals())) return text_buffer.getvalue()
def format_linting_stats(result, verbose=0): """Format a set of stats given a `LintingResult`.""" text_buffer = StringIO() all_stats = result.stats() text_buffer.write("==== summary ====\n") if verbose >= 2: output_fields = [ "files", "violations", "clean files", "unclean files", "avg per file", "unclean rate", "status", ] special_formats = {"unclean rate": "{0:.0%}"} else: output_fields = ["violations", "status"] special_formats = {} # Generate content tuples, applying special formats for some fields summary_content = [( key, special_formats[key].format(all_stats[key]) if key in special_formats else all_stats[key], ) for key in output_fields] # Render it all as a table text_buffer.write(cli_table(summary_content, max_label_width=14)) return text_buffer.getvalue()
def _print_out_violations_and_timing( bench: bool, code_only: bool, total_time: float, verbose: int, parsed_strings: List[ParsedString], ) -> int: """Used by human formatting during the parse.""" violations_count = 0 timing = TimingSummary() for parsed_string in parsed_strings: timing.add(parsed_string.time_dict) if parsed_string.tree: click.echo(parsed_string.tree.stringify(code_only=code_only)) else: # TODO: Make this prettier click.echo("...Failed to Parse...") # pragma: no cover violations_count += len(parsed_string.violations) if parsed_string.violations: click.echo("==== parsing violations ====") # pragma: no cover for v in parsed_string.violations: click.echo(format_violation(v)) # pragma: no cover if parsed_string.violations and parsed_string.config.get( "dialect") == "ansi": click.echo(format_dialect_warning()) # pragma: no cover if verbose >= 2: click.echo("==== timings ====") click.echo(cli_table(parsed_string.time_dict.items())) if verbose >= 2 or bench: click.echo("==== overall timings ====") click.echo(cli_table([("Clock time", total_time)])) timing_summary = timing.summary() for step in timing_summary: click.echo(f"=== {step} ===") click.echo(cli_table(timing_summary[step].items())) return violations_count
def format_dialects(dialect_readout, verbose=0): """Format the dialects yielded by `dialect_readout`.""" text_buffer = StringIO() text_buffer.write("==== sqlfluff - dialects ====\n") readouts = [ (d["label"], "{name} dialect [inherits from '{inherits_from}']".format(**d)) for d in dialect_readout() ] text_buffer.write( cli_table(readouts, col_width=60, cols=1, label_color="blue", val_align="right") ) return text_buffer.getvalue()
def format_rules(linter, verbose=0): """Format the a set of rules given a `Linter`.""" text_buffer = StringIO() text_buffer.write("==== sqlfluff - rules ====\n") text_buffer.write( cli_table( linter.rule_tuples(), col_width=80, cols=1, label_color="blue", val_align="left", )) return text_buffer.getvalue()
def format_dialects(dialect_readout, verbose=0): """Format the dialects yielded by `dialect_readout`.""" text_buffer = StringIO() text_buffer.write("==== sqlfluff - dialects ====\n") readouts = [( dialect.label, f"{dialect.name} dialect [inherits from '{dialect.inherits_from}']", ) for dialect in dialect_readout()] text_buffer.write( cli_table( readouts, col_width=60, cols=1, label_color=Color.blue, val_align="right", )) return text_buffer.getvalue()
def parse(path, code_only, format, profiler, bench, nofail, logger=None, **kwargs): """Parse SQL files and just spit out the result. PATH is the path to a sql file or directory to lint. This can be either a file ('path/to/file.sql'), a path ('directory/of/sql/files'), a single ('-') character to indicate reading from *stdin* or a dot/blank ('.'/' ') which will be interpreted like passing the current working directory as a path argument. """ # Initialise the benchmarker bencher = BenchIt() # starts the timer c = get_config(**kwargs) # We don't want anything else to be logged if we want json or yaml output non_human_output = format in ("json", "yaml") lnt, formatter = get_linter_and_formatter(c, silent=non_human_output) verbose = c.get("verbose") recurse = c.get("recurse") formatter.dispatch_config(lnt) # Set up logging. set_logging_level(verbosity=verbose, logger=logger, stderr_output=non_human_output) # TODO: do this better nv = 0 if profiler: # Set up the profiler if required try: import cProfile except ImportError: click.echo("The cProfiler is not available on your platform.") sys.exit(1) pr = cProfile.Profile() pr.enable() bencher("Parse setup") try: # handle stdin if specified via lone '-' if "-" == path: # put the parser result in a list to iterate later result = [ lnt.parse_string(sys.stdin.read(), "stdin", recurse=recurse, config=lnt.config), ] else: # A single path must be specified for this command # TODO: Remove verbose result = lnt.parse_path(path, recurse=recurse) # iterative print for human readout if format == "human": for parsed_string in result: if parsed_string.tree: click.echo( parsed_string.tree.stringify(code_only=code_only)) else: # TODO: Make this prettier click.echo("...Failed to Parse...") nv += len(parsed_string.violations) if parsed_string.violations: click.echo("==== parsing violations ====") for v in parsed_string.violations: click.echo(format_violation(v)) if (parsed_string.violations and parsed_string.config.get("dialect") == "ansi"): click.echo(format_dialect_warning()) if verbose >= 2: click.echo("==== timings ====") click.echo(cli_table(parsed_string.time_dict.items())) bencher("Output details for file") else: # collect result and print as single payload # will need to zip in the file paths filepaths = ["stdin"] if "-" == path else lnt.paths_from_path(path) result = [ dict( filepath=filepath, segments=parsed.as_record(code_only=code_only, show_raw=True) if parsed else None, ) for filepath, (parsed, _, _, _, _) in zip(filepaths, result) ] if format == "yaml": # For yaml dumping always dump double quoted strings if they contain tabs or newlines. yaml.add_representer(str, quoted_presenter) click.echo(yaml.dump(result)) elif format == "json": click.echo(json.dumps(result)) except IOError: click.echo( colorize( "The path {0!r} could not be accessed. Check it exists.". format(path), "red", )) sys.exit(1) if profiler: pr.disable() profiler_buffer = StringIO() ps = pstats.Stats(pr, stream=profiler_buffer).sort_stats("cumulative") ps.print_stats() click.echo("==== profiler stats ====") # Only print the first 50 lines of it click.echo("\n".join(profiler_buffer.getvalue().split("\n")[:50])) if bench: click.echo("\n\n==== bencher stats ====") bencher.display() if nv > 0 and not nofail: sys.exit(66) else: sys.exit(0)
def test__cli__helpers__cli_table(): """Test making tables.""" vals = [("a", 3), ("b", "c"), ("d", 4.7654), ("e", 9)] txt = cli_table(vals, col_width=7, divider_char="|", label_color=None) # NB: No trailing newline assert txt == "a: 3|b: c\nd: 4.77|e: 9"
def parse( path, code_only, include_meta, format, profiler, bench, nofail, logger=None, **kwargs, ): """Parse SQL files and just spit out the result. PATH is the path to a sql file or directory to lint. This can be either a file ('path/to/file.sql'), a path ('directory/of/sql/files'), a single ('-') character to indicate reading from *stdin* or a dot/blank ('.'/' ') which will be interpreted like passing the current working directory as a path argument. """ c = get_config(**kwargs) # We don't want anything else to be logged if we want json or yaml output non_human_output = format in ("json", "yaml") lnt, formatter = get_linter_and_formatter(c, silent=non_human_output) verbose = c.get("verbose") recurse = c.get("recurse") formatter.dispatch_config(lnt) # Set up logging. set_logging_level(verbosity=verbose, logger=logger, stderr_output=non_human_output) # TODO: do this better nv = 0 if profiler: # Set up the profiler if required try: import cProfile except ImportError: # pragma: no cover click.echo("The cProfiler is not available on your platform.") sys.exit(1) pr = cProfile.Profile() pr.enable() try: t0 = time.monotonic() # handle stdin if specified via lone '-' if "-" == path: # put the parser result in a list to iterate later result = [ lnt.parse_string( sys.stdin.read(), "stdin", recurse=recurse, config=lnt.config ), ] else: # A single path must be specified for this command result = lnt.parse_path(path, recurse=recurse) total_time = time.monotonic() - t0 # iterative print for human readout if format == "human": timing = TimingSummary() for parsed_string in result: timing.add(parsed_string.time_dict) if parsed_string.tree: click.echo(parsed_string.tree.stringify(code_only=code_only)) else: # TODO: Make this prettier click.echo("...Failed to Parse...") # pragma: no cover nv += len(parsed_string.violations) if parsed_string.violations: click.echo("==== parsing violations ====") # pragma: no cover for v in parsed_string.violations: click.echo(format_violation(v)) # pragma: no cover if ( parsed_string.violations and parsed_string.config.get("dialect") == "ansi" ): click.echo(format_dialect_warning()) # pragma: no cover if verbose >= 2: click.echo("==== timings ====") click.echo(cli_table(parsed_string.time_dict.items())) if verbose >= 2 or bench: click.echo("==== overall timings ====") click.echo(cli_table([("Clock time", total_time)])) timing_summary = timing.summary() for step in timing_summary: click.echo(f"=== {step} ===") click.echo(cli_table(timing_summary[step].items())) else: result = [ dict( filepath=linted_result.fname, segments=linted_result.tree.as_record( code_only=code_only, show_raw=True, include_meta=include_meta ) if linted_result.tree else None, ) for linted_result in result ] if format == "yaml": # For yaml dumping always dump double quoted strings if they contain tabs or newlines. yaml.add_representer(str, quoted_presenter) click.echo(yaml.dump(result)) elif format == "json": click.echo(json.dumps(result)) except OSError: # pragma: no cover click.echo( colorize( f"The path {path!r} could not be accessed. Check it exists.", "red", ), err=True, ) sys.exit(1) if profiler: pr.disable() profiler_buffer = StringIO() ps = pstats.Stats(pr, stream=profiler_buffer).sort_stats("cumulative") ps.print_stats() click.echo("==== profiler stats ====") # Only print the first 50 lines of it click.echo("\n".join(profiler_buffer.getvalue().split("\n")[:50])) if nv > 0 and not nofail: sys.exit(66) # pragma: no cover else: sys.exit(0)
def fix(force, paths, processes, bench=False, fixed_suffix="", logger=None, **kwargs): """Fix SQL files. PATH is the path to a sql file or directory to lint. This can be either a file ('path/to/file.sql'), a path ('directory/of/sql/files'), a single ('-') character to indicate reading from *stdin* or a dot/blank ('.'/' ') which will be interpreted like passing the current working directory as a path argument. """ # some quick checks fixing_stdin = ("-",) == paths config = get_config(**kwargs) lnt, formatter = get_linter_and_formatter(config, silent=fixing_stdin) verbose = config.get("verbose") exit_code = 0 formatter.dispatch_config(lnt) # Set up logging. set_logging_level(verbosity=verbose, logger=logger, stderr_output=fixing_stdin) # handle stdin case. should output formatted sql to stdout and nothing else. if fixing_stdin: stdin = sys.stdin.read() result = lnt.lint_string_wrapped(stdin, fname="stdin", fix=True) templater_error = result.num_violations(types=SQLTemplaterError) > 0 unfixable_error = result.num_violations(types=SQLLintError, fixable=False) > 0 if result.num_violations(types=SQLLintError, fixable=True) > 0: stdout = result.paths[0].files[0].fix_string()[0] else: stdout = stdin if templater_error: click.echo( colorize("Fix aborted due to unparseable template variables.", "red"), err=True, ) click.echo( colorize("Use '--ignore templating' to attempt to fix anyway.", "red"), err=True, ) if unfixable_error: click.echo(colorize("Unfixable violations detected.", "red"), err=True) click.echo(stdout, nl=False) sys.exit(1 if templater_error or unfixable_error else 0) # Lint the paths (not with the fix argument at this stage), outputting as we go. click.echo("==== finding fixable violations ====") try: result = lnt.lint_paths( paths, fix=True, ignore_non_existent_files=False, processes=processes ) except OSError: click.echo( colorize( "The path(s) {!r} could not be accessed. Check it/they exist(s).".format( paths ), "red", ), err=True, ) sys.exit(1) # NB: We filter to linting violations here, because they're # the only ones which can be potentially fixed. if result.num_violations(types=SQLLintError, fixable=True) > 0: click.echo("==== fixing violations ====") click.echo( "{} fixable linting violations found".format( result.num_violations(types=SQLLintError, fixable=True) ) ) if force: click.echo(colorize("FORCE MODE", "red") + ": Attempting fixes...") success = do_fixes( lnt, result, formatter, types=SQLLintError, fixed_file_suffix=fixed_suffix, ) if not success: sys.exit(1) # pragma: no cover else: click.echo( "Are you sure you wish to attempt to fix these? [Y/n] ", nl=False ) c = click.getchar().lower() click.echo("...") if c in ("y", "\r", "\n"): click.echo("Attempting fixes...") success = do_fixes( lnt, result, formatter, types=SQLLintError, fixed_file_suffix=fixed_suffix, ) if not success: sys.exit(1) # pragma: no cover else: _completion_message(config) elif c == "n": click.echo("Aborting...") exit_code = 1 else: # pragma: no cover click.echo("Invalid input, please enter 'Y' or 'N'") click.echo("Aborting...") exit_code = 1 else: click.echo("==== no fixable linting violations found ====") _completion_message(config) if result.num_violations(types=SQLLintError, fixable=False) > 0: click.echo( " [{} unfixable linting violations found]".format( result.num_violations(types=SQLLintError, fixable=False) ) ) exit_code = 1 if result.num_violations(types=SQLTemplaterError) > 0: click.echo( " [{} templating errors found]".format( result.num_violations(types=SQLTemplaterError) ) ) exit_code = 1 if bench: click.echo("==== overall timings ====") click.echo(cli_table([("Clock time", result.total_time)])) timing_summary = result.timing_summary() for step in timing_summary: click.echo(f"=== {step} ===") click.echo(cli_table(timing_summary[step].items())) sys.exit(exit_code)
def lint( paths, processes, format, annotation_level, nofail, disregard_sqlfluffignores, logger=None, bench=False, **kwargs, ): """Lint SQL files via passing a list of files or using stdin. PATH is the path to a sql file or directory to lint. This can be either a file ('path/to/file.sql'), a path ('directory/of/sql/files'), a single ('-') character to indicate reading from *stdin* or a dot/blank ('.'/' ') which will be interpreted like passing the current working directory as a path argument. Linting SQL files: sqlfluff lint path/to/file.sql sqlfluff lint directory/of/sql/files Linting a file via stdin (note the lone '-' character): cat path/to/file.sql | sqlfluff lint - echo 'select col from tbl' | sqlfluff lint - """ config = get_config(**kwargs) non_human_output = format != "human" lnt, formatter = get_linter_and_formatter(config, silent=non_human_output) verbose = config.get("verbose") formatter.dispatch_config(lnt) # Set up logging. set_logging_level(verbosity=verbose, logger=logger, stderr_output=non_human_output) # add stdin if specified via lone '-' if ("-",) == paths: result = lnt.lint_string_wrapped(sys.stdin.read(), fname="stdin") else: # Output the results as we go if verbose >= 1: click.echo(format_linting_result_header()) try: result = lnt.lint_paths( paths, ignore_non_existent_files=False, ignore_files=not disregard_sqlfluffignores, processes=processes, ) except OSError: click.echo( colorize( "The path(s) {!r} could not be accessed. Check it/they exist(s).".format( paths ), "red", ) ) sys.exit(1) # Output the final stats if verbose >= 1: click.echo(format_linting_stats(result, verbose=verbose)) if format == "json": click.echo(json.dumps(result.as_records())) elif format == "yaml": click.echo(yaml.dump(result.as_records())) elif format == "github-annotation": github_result = [] for record in result.as_records(): filepath = record["filepath"] for violation in record["violations"]: # NOTE: The output format is designed for this GitHub action: # https://github.com/yuzutech/annotations-action # It is similar, but not identical, to the native GitHub format: # https://docs.github.com/en/rest/reference/checks#annotations-items github_result.append( { "file": filepath, "line": violation["line_no"], "start_column": violation["line_pos"], "end_column": violation["line_pos"], "title": "SQLFluff", "message": f"{violation['code']}: {violation['description']}", "annotation_level": annotation_level, } ) click.echo(json.dumps(github_result)) if bench: click.echo("==== overall timings ====") click.echo(cli_table([("Clock time", result.total_time)])) timing_summary = result.timing_summary() for step in timing_summary: click.echo(f"=== {step} ===") click.echo(cli_table(timing_summary[step].items())) if not nofail: if not non_human_output: _completion_message(config) sys.exit(result.stats()["exit code"]) else: sys.exit(0)
def lint( paths, processes, format, nofail, disregard_sqlfluffignores, logger=None, bench=False, **kwargs, ): """Lint SQL files via passing a list of files or using stdin. PATH is the path to a sql file or directory to lint. This can be either a file ('path/to/file.sql'), a path ('directory/of/sql/files'), a single ('-') character to indicate reading from *stdin* or a dot/blank ('.'/' ') which will be interpreted like passing the current working directory as a path argument. Linting SQL files: sqlfluff lint path/to/file.sql sqlfluff lint directory/of/sql/files Linting a file via stdin (note the lone '-' character): cat path/to/file.sql | sqlfluff lint - echo 'select col from tbl' | sqlfluff lint - """ c = get_config(**kwargs) non_human_output = format in ("json", "yaml") lnt, formatter = get_linter_and_formatter(c, silent=non_human_output) verbose = c.get("verbose") formatter.dispatch_config(lnt) # Set up logging. set_logging_level(verbosity=verbose, logger=logger, stderr_output=non_human_output) # add stdin if specified via lone '-' if ("-", ) == paths: result = lnt.lint_string_wrapped(sys.stdin.read(), fname="stdin") else: # Output the results as we go if verbose >= 1: click.echo(format_linting_result_header()) try: result = lnt.lint_paths( paths, ignore_non_existent_files=False, ignore_files=not disregard_sqlfluffignores, processes=processes, ) except OSError: click.echo( colorize( "The path(s) {!r} could not be accessed. Check it/they exist(s)." .format(paths), "red", )) sys.exit(1) # Output the final stats if verbose >= 1: click.echo(format_linting_stats(result, verbose=verbose)) if format == "json": click.echo(json.dumps(result.as_records())) elif format == "yaml": click.echo(yaml.dump(result.as_records())) if bench: click.echo("==== overall timings ====") click.echo(cli_table([("Clock time", result.total_time)])) timing_summary = result.timing_summary() for step in timing_summary: click.echo(f"=== {step} ===") click.echo(cli_table(timing_summary[step].items())) if not nofail: if not non_human_output: click.echo("All Finished 📜 🎉!") sys.exit(result.stats()["exit code"]) else: sys.exit(0)
def fix( force: bool, paths: Tuple[str], processes: int, bench: bool = False, fixed_suffix: str = "", logger: Optional[logging.Logger] = None, disable_progress_bar: Optional[bool] = False, extra_config_path: Optional[str] = None, ignore_local_config: bool = False, **kwargs, ) -> NoReturn: """Fix SQL files. PATH is the path to a sql file or directory to lint. This can be either a file ('path/to/file.sql'), a path ('directory/of/sql/files'), a single ('-') character to indicate reading from *stdin* or a dot/blank ('.'/' ') which will be interpreted like passing the current working directory as a path argument. """ # some quick checks fixing_stdin = ("-", ) == paths config = get_config(extra_config_path, ignore_local_config, **kwargs) fix_even_unparsable = config.get("fix_even_unparsable") lnt, formatter = get_linter_and_formatter(config, silent=fixing_stdin) verbose = config.get("verbose") progress_bar_configuration.disable_progress_bar = disable_progress_bar exit_code = 0 formatter.dispatch_config(lnt) # Set up logging. set_logging_level(verbosity=verbose, logger=logger, stderr_output=fixing_stdin) # handle stdin case. should output formatted sql to stdout and nothing else. if fixing_stdin: stdin = sys.stdin.read() result = lnt.lint_string_wrapped(stdin, fname="stdin", fix=True) templater_error = result.num_violations(types=SQLTemplaterError) > 0 unfixable_error = result.num_violations(types=SQLLintError, fixable=False) > 0 if not fix_even_unparsable: exit_code = _handle_files_with_tmp_or_prs_errors(result) if result.num_violations(types=SQLLintError, fixable=True) > 0: stdout = result.paths[0].files[0].fix_string()[0] else: stdout = stdin if templater_error: click.echo( colorize( "Fix aborted due to unparseable template variables.", Color.red, ), err=True, ) click.echo( colorize( "Use --fix-even-unparsable' to attempt to fix the SQL anyway.", Color.red, ), err=True, ) if unfixable_error: click.echo(colorize("Unfixable violations detected.", Color.red), err=True) click.echo(stdout, nl=False) sys.exit(1 if templater_error or unfixable_error else exit_code) # Lint the paths (not with the fix argument at this stage), outputting as we go. click.echo("==== finding fixable violations ====") try: result = lnt.lint_paths( paths, fix=True, ignore_non_existent_files=False, processes=processes, ) except OSError: click.echo( colorize( f"The path(s) '{paths}' could not be accessed. Check it/they exist(s).", Color.red, ), err=True, ) sys.exit(1) if not fix_even_unparsable: exit_code = _handle_files_with_tmp_or_prs_errors(result) # NB: We filter to linting violations here, because they're # the only ones which can be potentially fixed. if result.num_violations(types=SQLLintError, fixable=True) > 0: click.echo("==== fixing violations ====") click.echo( f"{result.num_violations(types=SQLLintError, fixable=True)} fixable " "linting violations found") if force: click.echo( f"{colorize('FORCE MODE', Color.red)}: Attempting fixes...") success = do_fixes( lnt, result, formatter, types=SQLLintError, fixed_file_suffix=fixed_suffix, ) if not success: sys.exit(1) # pragma: no cover else: click.echo("Are you sure you wish to attempt to fix these? [Y/n] ", nl=False) c = click.getchar().lower() click.echo("...") if c in ("y", "\r", "\n"): click.echo("Attempting fixes...") success = do_fixes( lnt, result, formatter, types=SQLLintError, fixed_file_suffix=fixed_suffix, ) if not success: sys.exit(1) # pragma: no cover else: _completion_message(config) elif c == "n": click.echo("Aborting...") exit_code = 1 else: # pragma: no cover click.echo("Invalid input, please enter 'Y' or 'N'") click.echo("Aborting...") exit_code = 1 else: click.echo("==== no fixable linting violations found ====") _completion_message(config) error_types = [ ( dict(types=SQLLintError, fixable=False), " [{} unfixable linting violations found]", 1, ), ] for num_violations_kwargs, message_format, error_level in error_types: num_violations = result.num_violations(**num_violations_kwargs) if num_violations > 0: click.echo(message_format.format(num_violations)) exit_code = max(exit_code, error_level) if bench: click.echo("==== overall timings ====") click.echo(cli_table([("Clock time", result.total_time)])) timing_summary = result.timing_summary() for step in timing_summary: click.echo(f"=== {step} ===") click.echo(cli_table(timing_summary[step].items())) sys.exit(exit_code)
def test__cli__helpers__cli_table(): vals = [('a', 3), ('b', 'c'), ('d', 4.7654), ('e', 9)] txt = cli_table(vals, col_width=7, divider_char='|', label_color=None) # NB: No trailing newline assert txt == 'a: 3|b: c\nd: 4.77|e: 9'