def generate_test_segments_func(elems):
"""Roughly generate test segments.
This function isn't totally robust, but good enough
for testing. Use with caution.
"""
buff = []
raw_file = "".join(elems)
templated_file = TemplatedFile.from_string(raw_file)
idx = 0
for elem in elems:
if elem == "<indent>":
buff.append(
Indent(pos_marker=PositionMarker.from_point(
idx, idx, templated_file)))
continue
elif elem == "<dedent>":
buff.append(
Dedent(pos_marker=PositionMarker.from_point(
idx, idx, templated_file)))
continue
seg_kwargs = {}
if set(elem) <= {" ", "\t"}:
SegClass = WhitespaceSegment
elif set(elem) <= {"\n"}:
SegClass = NewlineSegment
elif elem == "(":
SegClass = SymbolSegment
seg_kwargs = {"name": "bracket_open"}
elif elem == ")":
SegClass = SymbolSegment
seg_kwargs = {"name": "bracket_close"}
elif elem.startswith("--"):
SegClass = CommentSegment
seg_kwargs = {"name": "inline_comment"}
elif elem.startswith('"'):
SegClass = CodeSegment
seg_kwargs = {"name": "double_quote"}
elif elem.startswith("'"):
SegClass = CodeSegment
seg_kwargs = {"name": "single_quote"}
else:
SegClass = CodeSegment
# Set a none position marker which we'll realign at the end.
buff.append(
SegClass(raw=elem,
pos_marker=PositionMarker(
slice(idx, idx + len(elem)),
slice(idx, idx + len(elem)),
templated_file,
),
**seg_kwargs))
idx += len(elem)
return tuple(buff)
def test__parser__base_segments_stubs():
"""Test stub methods that have no implementation in base class."""
template = TemplatedFile.from_string("foobar")
rs1 = RawSegment("foobar", PositionMarker(slice(0, 6), slice(0, 6), template))
base_segment = BaseSegment(segments=[rs1])
with pytest.raises(NotImplementedError):
base_segment.edit("foo")
def test__parser__base_segments_raw_compare():
"""Test comparison of raw segments."""
template = TemplatedFile.from_string("foobar")
rs1 = RawSegment("foobar",
PositionMarker(slice(0, 6), slice(0, 6), template))
rs2 = RawSegment("foobar",
PositionMarker(slice(0, 6), slice(0, 6), template))
assert rs1 == rs2
def test_markers__setting_position_raw():
"""Test that we can correctly infer positions from strings & locations."""
templ = TemplatedFile.from_string("foobar")
# Check inference in the template
assert templ.get_line_pos_of_char_pos(2, source=True) == (1, 3)
assert templ.get_line_pos_of_char_pos(2, source=False) == (1, 3)
# Now check it passes through
pos = PositionMarker(slice(2, 5), slice(2, 5), templ)
# Can we infer positions correctly?
assert pos.working_loc == (1, 3)
def test__parser__base_segments_base_compare():
"""Test comparison of base segments."""
template = TemplatedFile.from_string("foobar")
rs1 = RawSegment("foobar", PositionMarker(slice(0, 6), slice(0, 6), template))
rs2 = RawSegment("foobar", PositionMarker(slice(0, 6), slice(0, 6), template))
ds1 = DummySegment([rs1])
ds2 = DummySegment([rs2])
dsa2 = DummyAuxSegment([rs2])
# Check for equality
assert ds1 == ds2
# Check a different match on the same details are not the same
assert ds1 != dsa2
def lex(
self, raw: Union[str, TemplatedFile]
) -> Tuple[Tuple[BaseSegment, ...], List[SQLLexError]]:
"""Take a string or TemplatedFile and return segments.
If we fail to match the *whole* string, then we must have
found something that we cannot lex. If that happens we should
package it up as unlexable and keep track of the exceptions.
"""
# Make sure we've got a string buffer and a template
# regardless of what was passed in.
if isinstance(raw, str):
template = TemplatedFile.from_string(raw)
str_buff = raw
else:
template = raw
str_buff = str(template)
# Lex the string to get a tuple of LexedElement
element_buffer: List[LexedElement] = []
while True:
res = self.lex_match(str_buff, self.lexer_matchers)
element_buffer += res.elements
if res.forward_string:
resort_res = self.last_resort_lexer.match(res.forward_string)
if not resort_res:
# If we STILL can't match, then just panic out.
raise SQLLexError(
f"Fatal. Unable to lex characters: {0!r}".format(
res.forward_string[:10] +
"..." if len(res.forward_string) > 9 else res.
forward_string))
str_buff = resort_res.forward_string
element_buffer += resort_res.elements
else:
break
# Map tuple LexedElement to list of TemplateElement.
# This adds the template_slice to the object.
templated_buffer = self.map_template_slices(element_buffer, template)
# Turn lexed elements into segments.
segments: Tuple[RawSegment, ...] = self.elements_to_segments(
templated_buffer, template)
# Generate any violations
violations: List[SQLLexError] = self.violations_from_segments(segments)
return segments, violations
def test_linted_file_ignore_masked_violations(
noqa: dict, violations: List[SQLBaseError], expected
):
"""Test that _ignore_masked_violations() correctly filters violations."""
ignore_mask = [Linter.parse_noqa(**c) for c in noqa]
lf = linter.LintedFile(
path="",
violations=violations,
time_dict={},
tree=None,
ignore_mask=ignore_mask,
templated_file=TemplatedFile.from_string(""),
)
result = lf._ignore_masked_violations(violations)
expected_violations = [v for i, v in enumerate(violations) if i in expected]
assert expected_violations == result
def test_markers__setting_position_working():
"""Test that we can correctly set positions manually."""
templ = TemplatedFile.from_string("foobar")
pos = PositionMarker(slice(2, 5), slice(2, 5), templ, 4, 4)
# Can we NOT infer when we're told.
assert pos.working_loc == (4, 4)
def generate_test_segments_func(elems):
"""Roughly generate test segments.
This function isn't totally robust, but good enough
for testing. Use with caution.
"""
buff = []
raw_file = "".join(elems)
templated_file = TemplatedFile.from_string(raw_file)
idx = 0
for elem in elems:
if elem == "<indent>":
buff.append(
Indent(pos_marker=PositionMarker.from_point(
idx, idx, templated_file)))
continue
elif elem == "<dedent>":
buff.append(
Dedent(pos_marker=PositionMarker.from_point(
idx, idx, templated_file)))
continue
if set(elem) <= {" ", "\t"}:
cls = RawSegment.make(" ",
name="whitespace",
type="whitespace",
_is_code=False)
elif set(elem) <= {"\n"}:
cls = RawSegment.make("\n",
name="newline",
type="newline",
_is_code=False)
elif elem == "(":
cls = RawSegment.make("(", name="bracket_open")
elif elem == ")":
cls = RawSegment.make(")", name="bracket_close")
elif elem.startswith("--"):
cls = RawSegment.make("--",
name="inline_comment",
_is_code=False)
elif elem.startswith('"'):
cls = RawSegment.make('"', name="double_quote")
elif elem.startswith("'"):
cls = RawSegment.make("'", name="single_quote")
else:
cls = RawSegment.make("")
# Set a none position marker which we'll realign at the end.
buff.append(
cls(
elem,
pos_marker=PositionMarker(
slice(idx, idx + len(elem)),
slice(idx, idx + len(elem)),
templated_file,
),
))
idx += len(elem)
return tuple(buff)
def test__linted_file__slice_source_file_using_patches(source_patches,
source_only_slices,
raw_source_string,
expected_result,
caplog):
"""Test _slice_source_file_using_patches.
This is part of fix_string().
"""
with caplog.at_level(logging.DEBUG, logger="sqlfluff.linter"):
result = LintedFile._slice_source_file_using_patches(
source_patches, source_only_slices, raw_source_string)
assert result == expected_result
templated_file_1 = TemplatedFile.from_string("abc")
templated_file_2 = TemplatedFile(
"{# blah #}{{ foo }}bc",
"<testing>",
"abc",
[
TemplatedFileSlice("comment", slice(0, 10), slice(0, 0)),
TemplatedFileSlice("templated", slice(10, 19), slice(0, 1)),
TemplatedFileSlice("literal", slice(19, 21), slice(1, 3)),
],
[
RawFileSlice("{# blah #}", "comment", 0),
RawFileSlice("{{ foo }}", "templated", 10),
RawFileSlice("bc", "literal", 19),
],
)