def raw_seg_list():
"""A generic list of raw segments to test against."""
return [
RawSegment("bar", FilePositionMarker()),
RawSegment("foo", FilePositionMarker().advance_by("bar")),
RawSegment("bar", FilePositionMarker().advance_by("barfoo")),
]
def test__parser__base_segments_base_compare():
"""Test comparison of base segments."""
rs1 = RawSegment("foobar", FilePositionMarker())
rs2 = RawSegment("foobar", FilePositionMarker())
ds1 = DummySegment([rs1])
ds2 = DummySegment([rs2])
dsa2 = DummyAuxSegment([rs2])
# Check for equality
assert ds1 == ds2
# Check a different match on the same details are not the same
assert ds1 != dsa2
def test__parser__lexer_multimatcher(caplog):
"""Test the RepeatedMultiMatcher."""
matcher = RepeatedMultiMatcher(
SingletonMatcher("dot", ".",
RawSegment.make(".", name="dot", is_code=True)),
RegexMatcher("test", r"#[^#]*#", RawSegment.make("test", name="test")),
)
start_pos = FilePositionMarker()
with caplog.at_level(logging.DEBUG):
res = matcher.match("..#..#..#", start_pos)
assert res.new_string == "#" # Should match right up to the final element
assert res.new_pos == start_pos.advance_by("..#..#..")
assert len(res.segments) == 5
assert res.segments[2].raw == "#..#"
def _crawl_tree(cls, tree, variable_names, raw):
"""Crawl the tree looking for occurrences of the undeclared values."""
# First iterate through children
for elem in tree.iter_child_nodes():
yield from cls._crawl_tree(elem, variable_names, raw)
# Then assess self
if isinstance(tree, jinja2.nodes.Name) and tree.name in variable_names:
line_no = tree.lineno
line = raw.split("\n")[line_no - 1]
pos = line.index(tree.name) + 1
# Generate the charpos. +1 is for the newline characters themselves
charpos = (sum(
len(raw_line) + 1
for raw_line in raw.split("\n")[:line_no - 1]) + pos)
# NB: The positions returned here will be *inconsistent* with those
# from the linter at the moment, because these are references to the
# structure of the file *before* templating.
yield SQLTemplaterError(
"Undefined jinja template variable: {0!r}".format(tree.name),
pos=FilePositionMarker(None, line_no, pos, charpos),
)
def assert_matches(instring, matcher, matchstring):
"""Assert that a matcher does or doesn't work on a string.
The optional `matchstring` argument, which can optionally
be None, allows to either test positive matching of a
particular string or negative matching (that it explicitly)
doesn't match.
"""
start_pos = FilePositionMarker()
res = matcher.match(instring, start_pos)
# Check we've got the right type
assert isinstance(res, LexMatch)
if matchstring is None:
assert res.new_string == instring
assert res.new_pos == start_pos
assert res.segments == () # tuple
else:
new_pos = start_pos.advance_by(matchstring)
assert res.new_string == instring[len(matchstring):]
assert res.new_pos == new_pos
assert len(res.segments) == 1
assert res.segments[0].raw == matchstring
def test__parser__base_segments_raw_compare():
"""Test comparison of raw segments."""
rs1 = RawSegment("foobar", FilePositionMarker())
rs2 = RawSegment("foobar", FilePositionMarker())
assert rs1 == rs2
def test__parser__base_segments_raw_init():
"""Test initialisation. Other tests just use the fixture."""
RawSegment("foobar", FilePositionMarker())
def raw_seg():
"""Construct a raw segment as a fixture."""
fp = FilePositionMarker().advance_by("abc")
return RawSegment("foobar", fp)
result = Linter.parse_noqa(input, 0)
if not isinstance(expected, type):
assert result == expected
else:
# With exceptions, just check the type, not the contents.
assert isinstance(result, expected)
@pytest.mark.parametrize(
"noqa,violations,expected",
[
[
[],
[
DummyLintError(
FilePositionMarker(statement_index=None, line_no=1))
],
[
0,
],
],
[
[dict(comment="noqa: L001", line_no=1)],
[
DummyLintError(
FilePositionMarker(statement_index=None, line_no=1))
],
[],
],
[
[dict(comment="noqa: L001", line_no=2)],
def process(
self, *, in_str: str, fname: Optional[str] = None, config=None
) -> Tuple[Optional[TemplatedFile], list]:
"""Process a string and return the new string.
Note that the arguments are enforced as keywords
because Templaters can have differences in their
`process` method signature.
A Templater that only supports reading from a file
would need the following signature:
process(*, fname, in_str=None, config=None)
(arguments are swapped)
Args:
in_str (:obj:`str`): The input string.
fname (:obj:`str`, optional): The filename of this string. This is
mostly for loading config files at runtime.
config (:obj:`FluffConfig`): A specific config to use for this
templating operation. Only necessary for some templaters.
"""
if not config:
raise ValueError(
"For the jinja templater, the `process()` method requires a config object."
)
# Load the context
live_context = self.get_context(fname=fname, config=config)
# Apply dbt builtin functions if we're allowed.
apply_dbt_builtins = config.get_section(
(self.templater_selector, self.name, "apply_dbt_builtins")
)
if apply_dbt_builtins:
# This feels a bit wrong defining these here, they should probably
# be configurable somewhere sensible. But for now they're not.
# TODO: Come up with a better solution.
dbt_builtins = self._generate_dbt_builtins()
for name in dbt_builtins:
# Only apply if it hasn't already been set at this stage.
if name not in live_context:
live_context[name] = dbt_builtins[name]
env = self._get_jinja_env()
# Load macros from path (if applicable)
macros_path = config.get_section(
(self.templater_selector, self.name, "load_macros_from_path")
)
if macros_path:
live_context.update(
self._extract_macros_from_path(macros_path, env=env, ctx=live_context)
)
# Load config macros, these will take precedence over macros from the path
live_context.update(
self._extract_macros_from_config(config=config, env=env, ctx=live_context)
)
live_context.update(self._extract_libraries_from_config(config=config))
# Load the template, passing the global context.
try:
template = env.from_string(in_str, globals=live_context)
except TemplateSyntaxError as err:
# Something in the template didn't parse, return the original
# and a violation around what happened.
(len(line) for line in in_str.split("\n")[: err.lineno])
return (
TemplatedFile(source_str=in_str, fname=fname),
[
SQLTemplaterError(
"Failure to parse jinja template: {0}.".format(err),
pos=FilePositionMarker(
None,
err.lineno,
None,
# Calculate the charpos for sorting.
sum(
len(line)
for line in in_str.split("\n")[: err.lineno - 1]
),
),
)
],
)
violations = []
# Attempt to identify any undeclared variables. The majority
# will be found during the _crawl_tree step rather than this
# first Exception which serves only to catch catastrophic errors.
try:
syntax_tree = env.parse(in_str)
undefined_variables = meta.find_undeclared_variables(syntax_tree)
except Exception as err:
# TODO: Add a url here so people can get more help.
raise SQLTemplaterError(
"Failure in identifying Jinja variables: {0}.".format(err)
)
# Get rid of any that *are* actually defined.
for val in live_context:
if val in undefined_variables:
undefined_variables.remove(val)
if undefined_variables:
# Lets go through and find out where they are:
for val in self._crawl_tree(syntax_tree, undefined_variables, in_str):
violations.append(val)
try:
# NB: Passing no context. Everything is loaded when the template is loaded.
out_str = template.render()
# Slice the file once rendered.
raw_sliced, sliced_file, out_str = self.slice_file(
in_str, out_str, config=config
)
return (
TemplatedFile(
source_str=in_str,
templated_str=out_str,
fname=fname,
sliced_file=sliced_file,
raw_sliced=raw_sliced,
),
violations,
)
except (TemplateError, TypeError) as err:
templater_logger.info("Unrecoverable Jinja Error: %s", err)
violations.append(
SQLTemplaterError(
(
"Unrecoverable failure in Jinja templating: {0}. Have you configured "
"your variables? https://docs.sqlfluff.com/en/latest/configuration.html"
).format(err)
)
)
return None, violations