def demo_benchmark():
b = BenchIt() # starts the timer
# do stuff
sleep(1); b()
sleep(.2)
sleep(.3)
sleep(.4)
b("More stuff Done")
for i in range(1, 5):
sleep(.1); b()
b.display()
def parse(path,
code_only,
format,
profiler,
bench,
nofail,
logger=None,
**kwargs):
"""Parse SQL files and just spit out the result.
PATH is the path to a sql file or directory to lint. This can be either a
file ('path/to/file.sql'), a path ('directory/of/sql/files'), a single ('-')
character to indicate reading from *stdin* or a dot/blank ('.'/' ') which will
be interpreted like passing the current working directory as a path argument.
"""
# Initialise the benchmarker
bencher = BenchIt() # starts the timer
c = get_config(**kwargs)
# We don't want anything else to be logged if we want a yaml output
lnt, formatter = get_linter_and_formatter(c,
silent=format
in ("json", "yaml"))
verbose = c.get("verbose")
recurse = c.get("recurse")
formatter.dispatch_config(lnt)
# Set up logging.
set_logging_level(verbosity=verbose, logger=logger)
# TODO: do this better
nv = 0
if profiler:
# Set up the profiler if required
try:
import cProfile
except ImportError:
click.echo("The cProfiler is not available on your platform.")
sys.exit(1)
pr = cProfile.Profile()
pr.enable()
bencher("Parse setup")
try:
# handle stdin if specified via lone '-'
if "-" == path:
# put the parser result in a list to iterate later
config = lnt.config.make_child_from_path("stdin")
result = [(
# TODO: Remove verbose
*lnt.parse_string(
sys.stdin.read(), "stdin", recurse=recurse, config=config),
config,
)]
else:
# A single path must be specified for this command
# TODO: Remove verbose
result = lnt.parse_path(path, recurse=recurse)
# iterative print for human readout
if format == "human":
for parsed, violations, time_dict, f_cfg in result:
if parsed:
click.echo(parsed.stringify(code_only=code_only))
else:
# TODO: Make this prettier
click.echo("...Failed to Parse...")
nv += len(violations)
if violations:
click.echo("==== parsing violations ====")
for v in violations:
click.echo(format_violation(v))
if violations and f_cfg.get("dialect") == "ansi":
click.echo(format_dialect_warning())
if verbose >= 2:
click.echo("==== timings ====")
click.echo(cli_table(time_dict.items()))
bencher("Output details for file")
else:
# collect result and print as single payload
# will need to zip in the file paths
filepaths = ["stdin"] if "-" == path else lnt.paths_from_path(path)
result = [
dict(
filepath=filepath,
segments=parsed.as_record(code_only=code_only,
show_raw=True),
) for filepath, (parsed, _, _, _) in zip(filepaths, result)
]
if format == "yaml":
# For yaml dumping always dump double quoted strings if they contain tabs or newlines.
def quoted_presenter(dumper, data):
"""Representer which always double quotes string values needing escapes."""
if "\n" in data or "\t" in data or "'" in data:
return dumper.represent_scalar("tag:yaml.org,2002:str",
data,
style='"')
else:
return dumper.represent_scalar("tag:yaml.org,2002:str",
data,
style="")
yaml.add_representer(str, quoted_presenter)
click.echo(yaml.dump(result))
elif format == "json":
click.echo(json.dumps(result))
except IOError:
click.echo(
colorize(
"The path {0!r} could not be accessed. Check it exists.".
format(path),
"red",
))
sys.exit(1)
if profiler:
pr.disable()
profiler_buffer = StringIO()
ps = pstats.Stats(pr, stream=profiler_buffer).sort_stats("cumulative")
ps.print_stats()
click.echo("==== profiler stats ====")
# Only print the first 50 lines of it
click.echo("\n".join(profiler_buffer.getvalue().split("\n")[:50]))
if bench:
click.echo("\n\n==== bencher stats ====")
bencher.display()
if nv > 0 and not nofail:
sys.exit(66)
else:
sys.exit(0)
def fix(force,
paths,
bench=False,
fixed_suffix="",
no_safety=False,
logger=None,
**kwargs):
"""Fix SQL files.
PATH is the path to a sql file or directory to lint. This can be either a
file ('path/to/file.sql'), a path ('directory/of/sql/files'), a single ('-')
character to indicate reading from *stdin* or a dot/blank ('.'/' ') which will
be interpreted like passing the current working directory as a path argument.
"""
c = get_config(**kwargs)
lnt, formatter = get_linter_and_formatter(c, silent=("-", ) == paths)
verbose = c.get("verbose")
bencher = BenchIt()
formatter.dispatch_config(lnt)
# Set up logging.
set_logging_level(verbosity=verbose, logger=logger)
# Check that if fix is specified, that we have picked only a subset of rules
if no_safety:
click.echo(
colorize("NO SAFETY", "red") +
": Attempting fixes for all enabled rules.")
elif lnt.config.get("rule_whitelist") is None:
click.echo(("The fix option is only available in combination"
" with --rules. This is for your own safety! To"
" disable this safety feature use --no-safety or --s."))
sys.exit(1)
# handle stdin case. should output formatted sql to stdout and nothing else.
if ("-", ) == paths:
stdin = sys.stdin.read()
# TODO: Remove verbose
result = lnt.lint_string_wrapped(stdin, fname="stdin", fix=True)
stdout = result.paths[0].files[0].fix_string()[0]
click.echo(stdout, nl=False)
sys.exit()
# Lint the paths (not with the fix argument at this stage), outputting as we go.
click.echo("==== finding fixable violations ====")
try:
result = lnt.lint_paths(paths,
fix=True,
ignore_non_existent_files=False)
except IOError:
click.echo(
colorize(
"The path(s) {0!r} could not be accessed. Check it/they exist(s)."
.format(paths),
"red",
))
sys.exit(1)
# NB: We filter to linting violations here, because they're
# the only ones which can be potentially fixed.
if result.num_violations(types=SQLLintError, fixable=True) > 0:
click.echo("==== fixing violations ====")
click.echo("{0} fixable linting violations found".format(
result.num_violations(types=SQLLintError, fixable=True)))
if force:
click.echo(colorize("FORCE MODE", "red") + ": Attempting fixes...")
# TODO: Remove verbose
success = do_fixes(
lnt,
result,
formatter,
types=SQLLintError,
fixed_file_suffix=fixed_suffix,
)
if not success:
sys.exit(1)
else:
click.echo("Are you sure you wish to attempt to fix these? [Y/n] ",
nl=False)
c = click.getchar().lower()
click.echo("...")
if c == "y":
click.echo("Attempting fixes...")
# TODO: Remove verbose
success = do_fixes(
lnt,
result,
formatter,
types=SQLLintError,
fixed_file_suffix=fixed_suffix,
)
if not success:
sys.exit(1)
elif c == "n":
click.echo("Aborting...")
else:
click.echo("Invalid input :(")
click.echo("Aborting...")
else:
click.echo("==== no fixable linting violations found ====")
if result.num_violations(types=SQLLintError, fixable=False) > 0:
click.echo(" [{0} unfixable linting violations found]".format(
result.num_violations(types=SQLLintError, fixable=False)))
if bench:
click.echo("\n\n==== bencher stats ====")
bencher.display()
sys.exit(0)
def fix(force,
paths,
parallel,
bench=False,
fixed_suffix="",
logger=None,
**kwargs):
"""Fix SQL files.
PATH is the path to a sql file or directory to lint. This can be either a
file ('path/to/file.sql'), a path ('directory/of/sql/files'), a single ('-')
character to indicate reading from *stdin* or a dot/blank ('.'/' ') which will
be interpreted like passing the current working directory as a path argument.
"""
# some quick checks
fixing_stdin = ("-", ) == paths
c = get_config(**kwargs)
lnt, formatter = get_linter_and_formatter(c, silent=fixing_stdin)
verbose = c.get("verbose")
bencher = BenchIt()
formatter.dispatch_config(lnt)
# Set up logging.
set_logging_level(verbosity=verbose,
logger=logger,
stderr_output=fixing_stdin)
# handle stdin case. should output formatted sql to stdout and nothing else.
if fixing_stdin:
stdin = sys.stdin.read()
# TODO: Remove verbose
result = lnt.lint_string_wrapped(stdin, fname="stdin", fix=True)
stdout = result.paths[0].files[0].fix_string()[0]
click.echo(stdout, nl=False)
sys.exit()
# Lint the paths (not with the fix argument at this stage), outputting as we go.
click.echo("==== finding fixable violations ====")
try:
result = lnt.lint_paths(paths,
fix=True,
ignore_non_existent_files=False,
parallel=parallel)
except IOError:
click.echo(
colorize(
"The path(s) {0!r} could not be accessed. Check it/they exist(s)."
.format(paths),
"red",
))
sys.exit(1)
# NB: We filter to linting violations here, because they're
# the only ones which can be potentially fixed.
if result.num_violations(types=SQLLintError, fixable=True) > 0:
click.echo("==== fixing violations ====")
click.echo("{0} fixable linting violations found".format(
result.num_violations(types=SQLLintError, fixable=True)))
if force:
click.echo(colorize("FORCE MODE", "red") + ": Attempting fixes...")
# TODO: Remove verbose
success = do_fixes(
lnt,
result,
formatter,
types=SQLLintError,
fixed_file_suffix=fixed_suffix,
)
if not success:
sys.exit(1)
else:
click.echo("Are you sure you wish to attempt to fix these? [Y/n] ",
nl=False)
c = click.getchar().lower()
click.echo("...")
if c in ("y", "\r", "\n"):
click.echo("Attempting fixes...")
# TODO: Remove verbose
success = do_fixes(
lnt,
result,
formatter,
types=SQLLintError,
fixed_file_suffix=fixed_suffix,
)
if not success:
sys.exit(1)
else:
click.echo("All Finished 📜 🎉!")
elif c == "n":
click.echo("Aborting...")
else:
click.echo("Invalid input, please enter 'Y' or 'N'")
click.echo("Aborting...")
else:
click.echo("==== no fixable linting violations found ====")
if result.num_violations(types=SQLLintError, fixable=False) > 0:
click.echo(" [{0} unfixable linting violations found]".format(
result.num_violations(types=SQLLintError, fixable=False)))
click.echo("All Finished 📜 🎉!")
if bench:
click.echo("\n\n==== bencher stats ====")
bencher.display()
sys.exit(0)
def parse(path, code_only, format, profiler, bench, **kwargs):
"""Parse SQL files and just spit out the result.
PATH is the path to a sql file or directory to lint. This can be either a
file ('path/to/file.sql'), a path ('directory/of/sql/files'), a single ('-')
character to indicate reading from *stdin* or a dot/blank ('.'/' ') which will
be interpreted like passing the current working directory as a path argument.
"""
# Initialise the benchmarker
bencher = BenchIt() # starts the timer
c = get_config(**kwargs)
# We don't want anything else to be logged if we want a yaml output
lnt = get_linter(c, silent=format in ('json', 'yaml'))
verbose = c.get('verbose')
recurse = c.get('recurse')
config_string = format_config(lnt, verbose=verbose)
if len(config_string) > 0:
lnt.log(config_string)
# TODO: do this better
nv = 0
if profiler:
# Set up the profiler if required
try:
import cProfile
except ImportError:
lnt.log('The cProfiler is not available on your platform.')
sys.exit(1)
pr = cProfile.Profile()
pr.enable()
bencher("Parse setup")
try:
# handle stdin if specified via lone '-'
if '-' == path:
# put the parser result in a list to iterate later
config = lnt.config.make_child_from_path('stdin')
result = [lnt.parse_string(
sys.stdin.read(),
'stdin',
verbosity=verbose,
recurse=recurse,
config=config
)]
else:
# A single path must be specified for this command
result = lnt.parse_path(path, verbosity=verbose, recurse=recurse)
# iterative print for human readout
if format == 'human':
for parsed, violations, time_dict in result:
if parsed:
lnt.log(parsed.stringify(code_only=code_only))
else:
# TODO: Make this prettier
lnt.log('...Failed to Parse...')
nv += len(violations)
for v in violations:
lnt.log(format_violation(v, verbose=verbose))
if verbose >= 2:
lnt.log("==== timings ====")
lnt.log(cli_table(time_dict.items()))
bencher("Output details for file")
else:
# collect result and print as single payload
# will need to zip in the file paths
filepaths = ['stdin'] if '-' == path else lnt.paths_from_path(path)
result = [
dict(
filepath=filepath,
segments=parsed.as_record(code_only=code_only, show_raw=True)
)
for filepath, (parsed, _, _) in zip(filepaths, result)
]
if format == 'yaml':
# For yaml dumping always dump double quoted strings if they contain tabs or newlines.
def quoted_presenter(dumper, data):
"""Representer which always double quotes string values needing escapes."""
if '\n' in data or '\t' in data:
return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='"')
else:
return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='')
yaml.add_representer(str, quoted_presenter)
click.echo(yaml.dump(result))
elif format == 'json':
click.echo(json.dumps(result))
except IOError:
click.echo(colorize('The path {0!r} could not be accessed. Check it exists.'.format(path), 'red'))
sys.exit(1)
if profiler:
pr.disable()
profiler_buffer = StringIO()
ps = pstats.Stats(
pr, stream=profiler_buffer
).sort_stats('cumulative')
ps.print_stats()
lnt.log("==== profiler stats ====")
# Only print the first 50 lines of it
lnt.log('\n'.join(profiler_buffer.getvalue().split('\n')[:50]))
if bench:
lnt.log("\n\n==== bencher stats ====")
bencher.display()
if nv > 0:
sys.exit(66)
else:
sys.exit(0)
