def _eval_multiple_select_target_elements(self, select_targets_info,
segment):
"""Multiple select targets. Ensure each is on a separate line."""
# Insert newline before every select target.
fixes = []
for i, select_target in enumerate(select_targets_info.select_targets):
base_segment = (segment if not i else
select_targets_info.select_targets[i - 1])
if (base_segment.pos_marker.working_line_no ==
select_target.pos_marker.working_line_no):
# Find and delete any whitespace before the select target.
start_seg = select_targets_info.select_idx
# If any select modifier (e.g. distinct ) is present, start
# there rather than at the beginning.
modifier = segment.get_child("select_clause_modifier")
if modifier:
start_seg = segment.segments.index(modifier)
ws_to_delete = segment.select_children(
start_seg=segment.segments[start_seg]
if not i else select_targets_info.select_targets[i - 1],
select_if=lambda s: s.is_type("whitespace"),
loop_while=lambda s: s.is_type("whitespace", "comma") or s.
is_meta,
)
fixes += [LintFix.delete(ws) for ws in ws_to_delete]
fixes.append(
LintFix.create_before(select_target, [NewlineSegment()]))
# If we are at the last select target check if the FROM clause
# is on the same line, and if so move it to its own line.
if select_targets_info.from_segment:
if (i + 1 == len(select_targets_info.select_targets)) and (
select_target.pos_marker.working_line_no
== select_targets_info.from_segment.pos_marker.
working_line_no):
fixes.extend([
LintFix.delete(ws)
for ws in select_targets_info.pre_from_whitespace
])
fixes.append(
LintFix.create_before(
select_targets_info.from_segment,
[NewlineSegment()],
))
if fixes:
return LintResult(anchor=segment, fixes=fixes)
def _eval(self, context: RuleContext) -> Optional[LintResult]:
"""Fully qualify JOINs."""
# Config type hints
self.fully_qualify_join_types: str
# We are only interested in JOIN clauses.
if not context.segment.is_type("join_clause"):
return None
join_clause_keywords = [
segment for segment in context.segment.segments if segment.type == "keyword"
]
# Identify LEFT/RIGHT/OUTER JOIN and if the next keyword is JOIN.
if (
self.fully_qualify_join_types in ["outer", "both"]
and join_clause_keywords[0].raw_upper in ["RIGHT", "LEFT", "FULL"]
and join_clause_keywords[1].raw_upper == "JOIN"
):
# Define basic-level OUTER capitalization based on JOIN
outer_kw = ("outer", "OUTER")[join_clause_keywords[1].raw == "JOIN"]
# Insert OUTER after LEFT/RIGHT/FULL
return LintResult(
context.segment.segments[0],
fixes=[
LintFix.create_after(
context.segment.segments[0],
[WhitespaceSegment(), KeywordSegment(outer_kw)],
)
],
)
# Identify lone JOIN by looking at first child segment.
if (
self.fully_qualify_join_types in ["inner", "both"]
and join_clause_keywords[0].raw_upper == "JOIN"
):
# Define basic-level INNER capitalization based on JOIN
inner_kw = ("inner", "INNER")[join_clause_keywords[0].raw == "JOIN"]
# Replace lone JOIN with INNER JOIN.
return LintResult(
context.segment.segments[0],
fixes=[
LintFix.create_before(
context.segment.segments[0],
[KeywordSegment(inner_kw), WhitespaceSegment()],
)
],
)
return None
def _eval(self, context: RuleContext) -> Optional[List[LintResult]]:
"""Single whitespace expected in mother middle segment."""
error_buffer: List[LintResult] = []
if context.segment.is_type(self.expected_mother_segment_type):
last_code = None
mid_segs: List[BaseSegment] = []
for seg in context.segment.iter_segments(
expanding=self.expand_children):
if seg.is_code:
if (last_code and self.matches_target_tuples(
last_code, [self.pre_segment_identifier])
and self.matches_target_tuples(
seg, [self.post_segment_identifier])):
# Do we actually have the right amount of whitespace?
raw_inner = "".join(s.raw for s in mid_segs)
if raw_inner != " " and not (self.allow_newline and
any(s.name == "newline"
for s in mid_segs)):
if not raw_inner.strip():
# There's some whitespace and/or newlines, or nothing
fixes = []
if raw_inner:
# There's whitespace and/or newlines. Drop those.
fixes += [
LintFix.delete(mid_seg)
for mid_seg in mid_segs
]
# Enforce a single space
fixes += [
LintFix.create_before(
seg,
[WhitespaceSegment()],
)
]
else:
# Don't otherwise suggest a fix for now.
# Only whitespace & newlines are covered.
# At least a comment section between `AS` and `(` can
# result in an unfixable error.
# TODO: Enable more complex fixing here.
fixes = None # pragma: no cover
error_buffer.append(
LintResult(anchor=last_code, fixes=fixes))
mid_segs = []
if not seg.is_meta:
last_code = seg
else:
mid_segs.append(seg)
return error_buffer or None
def _eval(self, context: RuleContext) -> Optional[LintResult]:
"""Looking for DISTINCT before a bracket.
Look for DISTINCT keyword immediately followed by open parenthesis.
"""
# We trigger on `select_clause` and look for `select_clause_modifier`
if context.segment.is_type("select_clause"):
children = context.functional.segment.children()
modifier = children.select(sp.is_type("select_clause_modifier"))
first_element = children.select(
sp.is_type("select_clause_element")).first()
if not modifier or not first_element:
return None
# is the first element only an expression with only brackets?
expression = (first_element.children(
sp.is_type("expression")).first() or first_element)
bracketed = expression.children(sp.is_type("bracketed")).first()
if bracketed:
fixes = []
# If there's nothing else in the expression, remove the brackets.
if len(expression[0].segments) == 1:
# Remove the brackets and strip any meta segments.
fixes.append(
LintFix.replace(
bracketed[0],
self.filter_meta(bracketed[0].segments)[1:-1]), )
# If no whitespace between DISTINCT and expression, add it.
if not children.select(sp.is_whitespace(),
start_seg=modifier[0],
stop_seg=first_element[0]):
fixes.append(
LintFix.create_before(
first_element[0],
[WhitespaceSegment()],
))
# If no fixes, no problem.
if fixes:
return LintResult(anchor=modifier[0], fixes=fixes)
return None
def _coerce_indent_to(
self,
desired_indent: str,
current_indent_buffer: Tuple[RawSegment, ...],
current_anchor: BaseSegment,
) -> List[LintFix]:
"""Generate fixes to make an indent a certain size."""
# If there shouldn't be an indent at all, just delete.
if len(desired_indent) == 0:
fixes = [LintFix.delete(elem) for elem in current_indent_buffer]
# If we don't have any indent and we should, then add a single
elif len("".join(elem.raw for elem in current_indent_buffer)) == 0:
fixes = [
LintFix.create_before(
current_anchor,
[
WhitespaceSegment(
raw=desired_indent,
),
],
),
]
# Otherwise edit the first element to be the right size
else:
# Edit the first element of this line's indent and remove any other
# indents.
fixes = [
LintFix.replace(
current_indent_buffer[0],
[
WhitespaceSegment(
raw=desired_indent,
),
],
),
] + [LintFix.delete(elem) for elem in current_indent_buffer[1:]]
return fixes
def _eval(self, context: RuleContext) -> LintResult:
"""WITH clause closing bracket should be aligned with WITH keyword.
Look for a with clause and evaluate the position of closing brackets.
"""
# We only trigger on start_bracket (open parenthesis)
if context.segment.is_type("with_compound_statement"):
raw_stack_buff = list(context.raw_stack)
# Look for the with keyword
for seg in context.segment.segments:
if seg.name.lower() == "with":
seg_line_no = seg.pos_marker.line_no
break
else: # pragma: no cover
# This *could* happen if the with statement is unparsable,
# in which case then the user will have to fix that first.
if any(
s.is_type("unparsable")
for s in context.segment.segments):
return LintResult()
# If it's parsable but we still didn't find a with, then
# we should raise that.
raise RuntimeError("Didn't find WITH keyword!")
def indent_size_up_to(segs):
seg_buff = []
# Get any segments running up to the WITH
for elem in reversed(segs):
if elem.is_type("newline"):
break
elif elem.is_meta:
continue
else:
seg_buff.append(elem)
# reverse the indent if we have one
if seg_buff:
seg_buff = list(reversed(seg_buff))
indent_str = "".join(seg.raw for seg in seg_buff).replace(
"\t", " " * self.tab_space_size)
indent_size = len(indent_str)
return indent_size, indent_str
balance = 0
with_indent, with_indent_str = indent_size_up_to(raw_stack_buff)
for seg in context.segment.iter_segments(
expanding=["common_table_expression", "bracketed"],
pass_through=True):
if seg.name == "start_bracket":
balance += 1
elif seg.name == "end_bracket":
balance -= 1
if balance == 0:
closing_bracket_indent, _ = indent_size_up_to(
raw_stack_buff)
indent_diff = closing_bracket_indent - with_indent
# Is indent of closing bracket not the same as
# indent of WITH keyword.
if seg.pos_marker.line_no == seg_line_no:
# Skip if it's the one-line version. That's ok
pass
elif indent_diff < 0:
return LintResult(
anchor=seg,
fixes=[
LintFix.create_before(
seg,
[
WhitespaceSegment(" " *
(-indent_diff))
],
)
],
)
elif indent_diff > 0:
# Is it all whitespace before the bracket on this line?
assert seg.pos_marker
prev_segs_on_line = [
elem for elem in context.segment.raw_segments
if cast(PositionMarker, elem.pos_marker
).line_no == seg.pos_marker.line_no
and cast(PositionMarker, elem.pos_marker
).line_pos < seg.pos_marker.line_pos
]
if all(
elem.is_type("whitespace")
for elem in prev_segs_on_line):
# We can move it back, it's all whitespace
fixes = [
LintFix.create_before(
seg,
[WhitespaceSegment(with_indent_str)],
)
] + [
LintFix.delete(elem)
for elem in prev_segs_on_line
]
else:
# We have to move it to a newline
fixes = [
LintFix.create_before(
seg,
[
NewlineSegment(),
WhitespaceSegment(with_indent_str),
],
)
]
return LintResult(anchor=seg, fixes=fixes)
else:
raw_stack_buff.append(seg)
return LintResult()
def _eval(self, context: RuleContext) -> EvalResultType:
"""Operators should be surrounded by a single whitespace.
Rewritten to assess direct children of a segment to make
whitespace insertion more sensible.
We only need to handle *missing* whitespace because excess
whitespace is handled by L039.
NOTE: We also allow bracket characters either side.
"""
# Iterate through children of this segment looking for any of the
# target types. We also check for whether any of the children start
# or end with the targets.
# We ignore any targets which start or finish this segment. They'll
# be dealt with by the parent segment. That also means that we need
# to have at least three children.
if self._require_three_children and len(context.segment.segments) <= 2:
return LintResult()
violations = []
for idx, sub_seg in enumerate(context.segment.segments):
check_before = False
check_after = False
before_anchor = sub_seg
after_anchor = sub_seg
# Skip anything which is whitespace
if sub_seg.is_whitespace:
continue
# Skip any non-code elements
if not sub_seg.is_code:
continue
# Is it a target in itself?
if self.matches_target_tuples(sub_seg, self._target_elems):
self.logger.debug(
"Found Target [main] @%s: %r", sub_seg.pos_marker, sub_seg.raw
)
check_before = True
check_after = True
# Is it a compound segment ending or starting with the target?
elif sub_seg.segments:
# Get first and last raw segments.
raw_list = list(sub_seg.get_raw_segments())
if len(raw_list) > 1:
leading = raw_list[0]
trailing = raw_list[-1]
if self.matches_target_tuples(leading, self._target_elems):
before_anchor = leading
self.logger.debug(
"Found Target [leading] @%s: %r",
before_anchor.pos_marker,
before_anchor.raw,
)
check_before = True
if self.matches_target_tuples(trailing, self._target_elems):
after_anchor = trailing
self.logger.debug(
"Found Target [trailing] @%s: %r",
after_anchor.pos_marker,
after_anchor.raw,
)
check_after = True
if check_before:
prev_seg = self._find_segment(
idx, context.segment.segments, before=True
)
if self._missing_whitespace(prev_seg, before=True):
self.logger.debug(
"Missing Whitespace Before %r. Found %r instead.",
before_anchor.raw,
prev_seg.raw,
)
violations.append(
LintResult(
anchor=before_anchor,
description="Missing whitespace before {}".format(
before_anchor.raw
),
fixes=[
LintFix.create_before(
# NB the anchor here is always in the parent and not
# anchor
anchor_segment=sub_seg,
edit_segments=[WhitespaceSegment(raw=" ")],
)
],
)
)
if check_after:
next_seg = self._find_segment(
idx, context.segment.segments, before=False
)
if self._missing_whitespace(next_seg, before=False):
self.logger.debug(
"Missing Whitespace After %r. Found %r instead.",
after_anchor.raw,
next_seg.raw,
)
violations.append(
LintResult(
anchor=after_anchor,
description="Missing whitespace after {}".format(
after_anchor.raw
),
fixes=[
LintFix.create_before(
# NB the anchor here is always in the parent and not
# anchor
anchor_segment=next_seg,
edit_segments=[WhitespaceSegment(raw=" ")],
)
],
)
)
if violations:
return violations
return LintResult()
def _eval(self, context: RuleContext) -> Optional[LintResult]:
"""Implicit aliasing of table/column not allowed. Use explicit `AS` clause.
We look for the alias segment, and then evaluate its parent and whether
it contains an AS keyword. This is the _eval function for both L011 and L012.
The use of `raw_stack` is just for working out how much whitespace to add.
"""
# Config type hints
self.aliasing: str
fixes = []
if context.segment.is_type("alias_expression"):
if context.parent_stack[-1].is_type(*self._target_elems):
if any(e.name.lower() == "as"
for e in context.segment.segments):
if self.aliasing == "implicit":
if context.segment.segments[0].name.lower() == "as":
# Remove the AS as we're using implict aliasing
fixes.append(
LintFix.delete(context.segment.segments[0]))
anchor = context.raw_stack[-1]
# Remove whitespace before (if exists) or after (if not)
if (len(context.raw_stack) > 0
and context.raw_stack[-1].type
== "whitespace"):
fixes.append(
LintFix.delete(context.raw_stack[-1]))
elif (len(context.segment.segments) > 0
and context.segment.segments[1].type
== "whitespace"):
fixes.append(
LintFix.delete(
context.segment.segments[1]))
return LintResult(anchor=anchor, fixes=fixes)
elif self.aliasing != "implicit":
insert_buff: List[Union[WhitespaceSegment,
KeywordSegment]] = []
# Add initial whitespace if we need to...
if context.raw_stack[-1].name not in [
"whitespace", "newline"
]:
insert_buff.append(WhitespaceSegment())
# Add an AS (Uppercase for now, but could be corrected later)
insert_buff.append(KeywordSegment("AS"))
# Add a trailing whitespace if we need to
if context.segment.segments[0].name not in [
"whitespace",
"newline",
]:
insert_buff.append(WhitespaceSegment())
return LintResult(
anchor=context.segment,
fixes=[
LintFix.create_before(
context.segment.segments[0],
insert_buff,
)
],
)
return None
def _eval_single_select_target_element(self, select_targets_info,
context: RuleContext):
select_clause = context.functional.segment
parent_stack = context.parent_stack
wildcards = select_clause.children(
sp.is_type("select_clause_element")).children(
sp.is_type("wildcard_expression"))
is_wildcard = bool(wildcards)
if is_wildcard:
wildcard_select_clause_element = wildcards[0]
if (select_targets_info.select_idx <
select_targets_info.first_new_line_idx <
select_targets_info.first_select_target_idx) and (
not is_wildcard):
# Do we have a modifier?
select_children = select_clause.children()
modifier: Optional[Segments]
modifier = select_children.first(
sp.is_type("select_clause_modifier"))
# Prepare the select clause which will be inserted
insert_buff = [
WhitespaceSegment(),
select_children[select_targets_info.first_select_target_idx],
]
# Check if the modifier is one we care about
if modifier:
# If it's already on the first line, ignore it.
if (select_children.index(modifier.get()) <
select_targets_info.first_new_line_idx):
modifier = None
fixes = [
# Delete the first select target from its original location.
# We'll add it to the right section at the end, once we know
# what to add.
LintFix.delete(
select_children[
select_targets_info.first_select_target_idx], ),
]
# If we have a modifier to move:
if modifier:
# Add it to the insert
insert_buff = [WhitespaceSegment(), modifier[0]] + insert_buff
modifier_idx = select_children.index(modifier.get())
# Delete the whitespace after it (which is two after, thanks to indent)
if (len(select_children) > modifier_idx + 1
and select_children[modifier_idx + 2].is_whitespace):
fixes += [
LintFix.delete(select_children[modifier_idx + 2], ),
]
# Delete the modifier itself
fixes += [
LintFix.delete(modifier[0], ),
]
# Set the position marker for removing the preceding
# whitespace and newline, which we'll use below.
start_idx = modifier_idx
else:
# Set the position marker for removing the preceding
# whitespace and newline, which we'll use below.
start_idx = select_targets_info.first_select_target_idx
if parent_stack and parent_stack[-1].is_type("select_statement"):
select_stmt = parent_stack[-1]
select_clause_idx = select_stmt.segments.index(
select_clause.get())
after_select_clause_idx = select_clause_idx + 1
if len(select_stmt.segments) > after_select_clause_idx:
def _fixes_for_move_after_select_clause(
stop_seg: BaseSegment,
add_newline: bool = True,
) -> List[LintFix]:
"""Cleans up by moving leftover select_clause segments.
Context: Some of the other fixes we make in
_eval_single_select_target_element() leave the
select_clause in an illegal state -- a select_clause's
*rightmost children cannot be whitespace or comments*.
This function addresses that by moving these segments
up the parse tree to an ancestor segment chosen by
_choose_anchor_segment(). After these fixes are applied,
these segments may, for example, be *siblings* of
select_clause.
"""
start_seg = (
modifier[0] if modifier else select_children[
select_targets_info.first_new_line_idx])
move_after_select_clause = select_children.select(
start_seg=start_seg,
stop_seg=stop_seg,
)
fixes = [
LintFix.delete(seg)
for seg in move_after_select_clause
]
fixes.append(
LintFix.create_after(
self._choose_anchor_segment(
context,
"create_after",
select_clause[0],
filter_meta=True,
),
([NewlineSegment()] if add_newline else []) +
list(move_after_select_clause),
))
return fixes
if select_stmt.segments[after_select_clause_idx].is_type(
"newline"):
# Since we're deleting the newline, we should also delete all
# whitespace before it or it will add random whitespace to
# following statements. So walk back through the segment
# deleting whitespace until you get the previous newline, or
# something else.
to_delete = select_children.reversed().select(
loop_while=sp.is_type("whitespace"),
start_seg=select_children[start_idx],
)
fixes += [LintFix.delete(seg) for seg in to_delete]
# The select_clause is immediately followed by a
# newline. Delete the newline in order to avoid leaving
# behind an empty line after fix, *unless* we stopped
# due to something other than a newline.
delete_last_newline = select_children[
start_idx - len(to_delete) - 1].is_type("newline")
# Delete the newline if we decided to.
if delete_last_newline:
fixes.append(
LintFix.delete(
select_stmt.
segments[after_select_clause_idx], ))
fixes += _fixes_for_move_after_select_clause(
to_delete[-1], )
elif select_stmt.segments[after_select_clause_idx].is_type(
"whitespace"):
# The select_clause has stuff after (most likely a comment)
# Delete the whitespace immediately after the select clause
# so the other stuff aligns nicely based on where the select
# clause started
fixes += [
LintFix.delete(
select_stmt.segments[after_select_clause_idx],
),
]
fixes += _fixes_for_move_after_select_clause(
select_children[
select_targets_info.first_select_target_idx], )
elif select_stmt.segments[after_select_clause_idx].is_type(
"dedent"):
# Again let's strip back the whitespace, but simpler
# as don't need to worry about new line so just break
# if see non-whitespace
to_delete = select_children.reversed().select(
loop_while=sp.is_type("whitespace"),
start_seg=select_children[select_clause_idx - 1],
)
fixes += [LintFix.delete(seg) for seg in to_delete]
if to_delete:
fixes += _fixes_for_move_after_select_clause(
to_delete[-1],
# If we deleted a newline, create a newline.
any(seg for seg in to_delete
if seg.is_type("newline")),
)
else:
fixes += _fixes_for_move_after_select_clause(
select_children[
select_targets_info.first_select_target_idx], )
fixes += [
# Insert the select_clause in place of the first newline in the
# Select statement
LintFix.replace(
select_children[select_targets_info.first_new_line_idx],
insert_buff,
),
]
return LintResult(
anchor=select_clause.get(),
fixes=fixes,
)
# If we have a wildcard on the same line as the FROM keyword, but not the same
# line as the SELECT keyword, we need to move the FROM keyword to its own line.
# i.e.
# SELECT
# * FROM foo
if select_targets_info.from_segment:
if (is_wildcard and
(select_clause[0].pos_marker.working_line_no !=
select_targets_info.from_segment.pos_marker.working_line_no)
and
(wildcard_select_clause_element.pos_marker.working_line_no ==
select_targets_info.from_segment.pos_marker.working_line_no)):
fixes = [
LintFix.delete(ws)
for ws in select_targets_info.pre_from_whitespace
]
fixes.append(
LintFix.create_before(
select_targets_info.from_segment,
[NewlineSegment()],
))
return LintResult(anchor=select_clause.get(), fixes=fixes)
return None
def _eval(self, context: RuleContext) -> Optional[LintResult]:
"""Look for USING in a join clause."""
segment = context.functional.segment
parent_stack = context.functional.parent_stack
# We are not concerned with non join clauses
if not segment.all(sp.is_type("join_clause")):
return None
using_anchor = segment.children(sp.is_keyword("using")).first()
# If there is no evidence of a USING then we exit
if len(using_anchor) == 0:
return None
anchor = using_anchor.get()
description = "Found USING statement. Expected only ON statements."
# All returns from here out will be some form of linting error.
# we prepare the variable here
unfixable_result = LintResult(
anchor=anchor,
description=description,
)
tables_in_join = parent_stack.last().children(
sp.is_type("join_clause", "from_expression_element"))
# If we have more than 2 tables we won't try to fix join.
# TODO: if this is table 2 of 3 it is still fixable
if len(tables_in_join) > 2:
return unfixable_result
parent_select = parent_stack.last(sp.is_type("select_statement")).get()
if not parent_select: # pragma: no cover
return unfixable_result
select_info = get_select_statement_info(parent_select, context.dialect)
if not select_info: # pragma: no cover
return unfixable_result
to_delete, insert_after_anchor = _extract_deletion_sequence_and_anchor(
tables_in_join.last())
table_a, table_b = select_info.table_aliases
edit_segments = [
KeywordSegment(raw="ON"),
WhitespaceSegment(raw=" "),
] + _generate_join_conditions(
table_a.ref_str,
table_b.ref_str,
select_info.using_cols,
)
fixes = [
LintFix.create_before(
anchor_segment=insert_after_anchor,
edit_segments=edit_segments,
),
*[LintFix.delete(seg) for seg in to_delete],
]
return LintResult(
anchor=anchor,
description=description,
fixes=fixes,
)
def _process_current_line(
self, res: dict, memory: dict, context: RuleContext
) -> LintResult:
"""Checks indentation of one line of code, returning a LintResult.
The _eval() function calls it for the current line of code:
- When passed a newline segment (thus ending a line)
- When passed the *final* segment in the entire parse tree (which may
not be a newline)
"""
this_line_no = max(res.keys())
this_line = res.pop(this_line_no)
self.logger.debug(
"Evaluating line #%s. %s",
this_line_no,
# Don't log the line or indent buffer, it's too noisy.
self._strip_buffers(this_line),
)
trigger_segment = memory["trigger"]
# Is this line just comments? (Disregard trailing newline if present.)
check_comment_line = this_line["line_buffer"]
if check_comment_line and all(
seg.is_type(
"whitespace", "comment", "indent" # dedent is a subtype of indent
)
for seg in check_comment_line
):
# Comment line, deal with it later.
memory["comment_lines"].append(this_line_no)
self.logger.debug(" Comment Line. #%s", this_line_no)
return LintResult(memory=memory)
# Is it a hanging indent?
# Find last meaningful line indent.
last_code_line = None
for k in sorted(res.keys(), reverse=True):
if any(seg.is_code for seg in res[k]["line_buffer"]):
last_code_line = k
break
if len(res) > 0 and last_code_line:
last_line_hanger_indent = res[last_code_line]["hanging_indent"]
# Let's just deal with hanging indents here.
if (
# NB: Hangers are only allowed if there was content after the last
# indent on the previous line. Otherwise it's just an indent.
this_line["indent_size"] == last_line_hanger_indent
# Or they're if the indent balance is the same and the indent is the
# same AND the previous line was a hanger
or (
this_line["indent_size"] == res[last_code_line]["indent_size"]
and this_line["indent_balance"]
== res[last_code_line]["indent_balance"]
and last_code_line in memory["hanging_lines"]
)
) and (
# There MUST also be a non-zero indent. Otherwise we're just on the
# baseline.
this_line["indent_size"]
> 0
):
# This is a HANGER
memory["hanging_lines"].append(this_line_no)
self.logger.debug(" Hanger Line. #%s", this_line_no)
self.logger.debug(
" Last Line: %s", self._strip_buffers(res[last_code_line])
)
return LintResult(memory=memory)
# Is this an indented first line?
elif len(res) == 0:
if this_line["indent_size"] > 0:
self.logger.debug(" Indented First Line. #%s", this_line_no)
return LintResult(
anchor=trigger_segment,
memory=memory,
description="First line has unexpected indent",
fixes=[LintFix.delete(elem) for elem in this_line["indent_buffer"]],
)
# Special handling for template end blocks on a line by themselves.
if self._is_template_block_end_line(
this_line["line_buffer"], context.templated_file
):
block_lines = {
k: (
"end"
if self._is_template_block_end_line(
res[k]["line_buffer"], context.templated_file
)
else "start",
res[k]["indent_balance"],
"".join(
seg.raw or getattr(seg, "source_str", "")
for seg in res[k]["line_buffer"]
),
)
for k in res
if self._is_template_block_end_line(
res[k]["line_buffer"], context.templated_file
)
or self._is_template_block_start_line(
res[k]["line_buffer"], context.templated_file
)
}
# For a template block end on a line by itself, search for a
# matching block start on a line by itself. If there is one, match
# its indentation. Question: Could we avoid treating this as a
# special case? It has some similarities to the non-templated test
# case test/fixtures/linter/indentation_error_contained.sql, in tha
# both have lines where indent_balance drops 2 levels from one line
# to the next, making it a bit unclear how to indent that line.
template_block_level = -1
for k in sorted(block_lines.keys(), reverse=True):
if block_lines[k][0] == "end":
template_block_level -= 1
else:
template_block_level += 1
if template_block_level != 0:
continue
# Found prior template block line with the same indent balance.
# Is this a problem line?
if k in memory["problem_lines"] + memory["hanging_lines"]:
# Skip it if it is
return LintResult(memory=memory)
self.logger.debug(" [template block end] Comparing to #%s", k)
if this_line["indent_size"] == res[k]["indent_size"]:
# All good.
return LintResult(memory=memory)
# Indents don't match even though balance is the same...
memory["problem_lines"].append(this_line_no)
# The previous indent.
desired_indent = "".join(elem.raw for elem in res[k]["indent_buffer"])
# Make fixes
fixes = self._coerce_indent_to(
desired_indent=desired_indent,
current_indent_buffer=this_line["indent_buffer"],
current_anchor=this_line["line_buffer"][0],
)
self.logger.debug(
" !! Indentation does not match #%s. Fixes: %s", k, fixes
)
return LintResult(
anchor=trigger_segment,
memory=memory,
description="Indentation not consistent with line #{}".format(k),
# See above for logic
fixes=fixes,
)
# Assuming it's not a hanger, let's compare it to the other previous
# lines. We do it in reverse so that closer lines are more relevant.
for k in sorted(res.keys(), reverse=True):
# Is this a problem line?
if k in memory["problem_lines"] + memory["hanging_lines"]:
# Skip it if it is
continue
# Is this an empty line?
if not any(
elem.is_code or elem.is_type("placeholder")
for elem in res[k]["line_buffer"]
):
# Skip if it is
continue
# Work out the difference in indent
indent_diff = this_line["indent_balance"] - res[k]["indent_balance"]
# If we're comparing to a previous, more deeply indented line, then skip and
# keep looking.
if indent_diff < 0:
continue
# Is the indent balance the same?
if indent_diff == 0:
self.logger.debug(" [same indent balance] Comparing to #%s", k)
if this_line["indent_size"] != res[k]["indent_size"]:
# Indents don't match even though balance is the same...
memory["problem_lines"].append(this_line_no)
# Work out desired indent
if res[k]["indent_size"] == 0:
desired_indent = ""
elif this_line["indent_size"] == 0:
desired_indent = self._make_indent(
indent_unit=self.indent_unit,
tab_space_size=self.tab_space_size,
)
else:
# The previous indent.
desired_indent = "".join(
elem.raw for elem in res[k]["indent_buffer"]
)
# Make fixes
fixes = self._coerce_indent_to(
desired_indent=desired_indent,
current_indent_buffer=this_line["indent_buffer"],
current_anchor=trigger_segment,
)
self.logger.debug(
" !! Indentation does not match #%s. Fixes: %s", k, fixes
)
return LintResult(
anchor=trigger_segment,
memory=memory,
description="Indentation not consistent with line #{}".format(
k
),
# See above for logic
fixes=fixes,
)
# Are we at a deeper indent?
elif indent_diff > 0:
self.logger.debug(" [deeper indent balance] Comparing to #%s", k)
# NB: We shouldn't need to deal with correct hanging indents
# here, they should already have been dealt with before. We
# may still need to deal with *creating* hanging indents if
# appropriate.
self.logger.debug(
" Comparison Line: %s", self._strip_buffers(res[k])
)
# Check to see if we've got a whole number of multiples. If
# we do then record the number for later, otherwise raise
# an error. We do the comparison here so we have a reference
# point to do the repairs. We need a sensible previous line
# to base the repairs off. If there's no indent at all, then
# we should also take this route because there SHOULD be one.
if this_line["indent_size"] % self.tab_space_size != 0:
memory["problem_lines"].append(this_line_no)
# The default indent is the one just reconstructs it from
# the indent size.
default_indent = "".join(
elem.raw for elem in res[k]["indent_buffer"]
) + self._make_indent(
indent_unit=self.indent_unit,
tab_space_size=self.tab_space_size,
num=indent_diff,
)
# If we have a clean indent, we can just add steps in line
# with the difference in the indent buffers. simples.
if this_line["clean_indent"]:
self.logger.debug(" Use clean indent.")
desired_indent = default_indent
# If we have the option of a hanging indent then use it.
elif res[k]["hanging_indent"]:
self.logger.debug(" Use hanging indent.")
desired_indent = " " * res[k]["hanging_indent"]
else: # pragma: no cover
self.logger.debug(" Use default indent.")
desired_indent = default_indent
# Make fixes
fixes = self._coerce_indent_to(
desired_indent=desired_indent,
current_indent_buffer=this_line["indent_buffer"],
current_anchor=trigger_segment,
)
return LintResult(
anchor=trigger_segment,
memory=memory,
description=(
"Indentation not hanging or a multiple of {} spaces"
).format(self.tab_space_size),
fixes=fixes,
)
else:
# We'll need this value later.
this_indent_num = this_line["indent_size"] // self.tab_space_size
# We know that the indent balance is higher, what actually is
# the difference in indent counts? It should be a whole number
# if we're still here.
comp_indent_num = res[k]["indent_size"] // self.tab_space_size
# The indent number should be at least 1, and can be UP TO
# and including the difference in the indent balance.
if comp_indent_num == this_indent_num:
# We have two lines indented the same, but with a different starting
# indent balance. This is either a problem OR a sign that one of the
# opening indents wasn't used. We account for the latter and then
# have a violation if that wasn't the case.
# Does the comparison line have enough unused indent to get us back
# to where we need to be? NB: This should only be applied if this is
# a CLOSING bracket.
# First work out if we have some closing brackets, and if so, how
# many.
b_idx = 0
b_num = 0
while True:
if len(this_line["line_buffer"][b_idx:]) == 0:
break
elem = this_line["line_buffer"][b_idx]
if not elem.is_code:
b_idx += 1
continue
else:
if elem.is_type("end_bracket", "end_square_bracket"):
b_idx += 1
b_num += 1
continue
break # pragma: no cover
if b_num >= indent_diff:
# It does. This line is fine.
pass
else:
# It doesn't. That means we *should* have an indent when
# compared to this line and we DON'T.
memory["problem_lines"].append(this_line_no)
return LintResult(
anchor=trigger_segment,
memory=memory,
description="Indent expected and not found compared to line"
" #{}".format(k),
# Add in an extra bit of whitespace for the indent
fixes=[
LintFix.create_before(
trigger_segment,
[
WhitespaceSegment(
raw=self._make_indent(
indent_unit=self.indent_unit,
tab_space_size=self.tab_space_size,
),
),
],
),
],
)
elif this_indent_num < comp_indent_num:
memory["problem_lines"].append(this_line_no)
return LintResult(
anchor=trigger_segment,
memory=memory,
description="Line under-indented compared to line #{}".format(
k
),
fixes=[
LintFix.create_before(
trigger_segment,
[
WhitespaceSegment(
# Make the minimum indent for it to be ok.
raw=self._make_indent(
num=comp_indent_num - this_indent_num,
indent_unit=self.indent_unit,
tab_space_size=self.tab_space_size,
),
),
],
),
],
)
elif this_indent_num > comp_indent_num + indent_diff:
# Calculate the lowest ok indent:
desired_indent = self._make_indent(
num=comp_indent_num - this_indent_num,
indent_unit=self.indent_unit,
tab_space_size=self.tab_space_size,
)
# Make fixes
fixes = self._coerce_indent_to(
desired_indent=desired_indent,
current_indent_buffer=this_line["indent_buffer"],
current_anchor=trigger_segment,
)
memory["problem_lines"].append(this_line_no)
return LintResult(
anchor=trigger_segment,
memory=memory,
description="Line over-indented compared to line #{}".format(k),
fixes=fixes,
)
# This was a valid comparison, so if it doesn't flag then
# we can assume that we're ok.
self.logger.debug(" Indent deemed ok comparing to #%s", k)
# Given that this line is ok, consider if the preceding lines are
# comments. If they are, lint the indentation of the comment(s).
fixes = []
for n in range(this_line_no - 1, -1, -1):
if n in memory["comment_lines"]:
# The previous line WAS a comment.
prev_line = res[n]
if this_line["indent_size"] != prev_line["indent_size"]:
# It's not aligned.
# Find the anchor first.
anchor: BaseSegment = None # type: ignore
for seg in prev_line["line_buffer"]:
if seg.is_type("comment"):
anchor = seg
break
# Make fixes.
fixes += self._coerce_indent_to(
desired_indent="".join(
elem.raw for elem in this_line["indent_buffer"]
),
current_indent_buffer=prev_line["indent_buffer"],
current_anchor=anchor,
)
memory["problem_lines"].append(n)
else:
break
if fixes:
return LintResult(
anchor=anchor,
memory=memory,
description="Comment not aligned with following line.",
fixes=fixes,
)
# Otherwise all good.
return LintResult(memory=memory)
# NB: At shallower indents, we don't check, we just check the
# previous lines with the same balance. Deeper indents can check
# themselves.
# If we get to here, then we're all good for now.
return LintResult(memory=memory)
def _validate_one_reference(
single_table_references: str,
allow_select_alias: bool,
ref: BaseSegment,
this_ref_type: str,
standalone_aliases: List[str],
table_ref_str: str,
col_alias_names: List[str],
seen_ref_types: Set[str],
) -> Optional[LintResult]:
# We skip any unqualified wildcard references (i.e. *). They shouldn't
# count.
if not ref.is_qualified() and ref.is_type(
"wildcard_identifier"): # type: ignore
return None
# Oddball case: Column aliases provided via function calls in by
# FROM or JOIN. References to these don't need to be qualified.
# Note there could be a table with a column by the same name as
# this alias, so avoid bogus warnings by just skipping them
# entirely rather than trying to enforce anything.
if ref.raw in standalone_aliases:
return None
# Certain dialects allow use of SELECT alias in WHERE clauses
if allow_select_alias and ref.raw in col_alias_names:
return None
if single_table_references == "consistent":
if seen_ref_types and this_ref_type not in seen_ref_types:
return LintResult(
anchor=ref,
description=f"{this_ref_type.capitalize()} reference "
f"{ref.raw!r} found in single table select which is "
"inconsistent with previous references.",
)
return None
if single_table_references != this_ref_type:
if single_table_references == "unqualified":
# If this is qualified we must have a "table", "."" at least
fixes = [LintFix.delete(el) for el in ref.segments[:2]]
return LintResult(
anchor=ref,
fixes=fixes,
description="{} reference {!r} found in single table "
"select.".format(this_ref_type.capitalize(), ref.raw),
)
fixes = [
LintFix.create_before(
ref.segments[0] if len(ref.segments) else ref,
edit_segments=[
CodeSegment(
raw=table_ref_str,
name="naked_identifier",
type="identifier",
),
SymbolSegment(raw=".", type="symbol", name="dot"),
],
)
]
return LintResult(
anchor=ref,
fixes=fixes,
description="{} reference {!r} found in single table "
"select.".format(this_ref_type.capitalize(), ref.raw),
)
return None
def generate_fixes_to_coerce(
self,
segments: List[RawSegment],
indent_section: "Section",
crawler: Rule_L016,
indent: int,
) -> List[LintFix]:
"""Generate a list of fixes to create a break at this point.
The `segments` argument is necessary to extract anchors
from the existing segments.
"""
fixes = []
# Generate some sample indents:
unit_indent = crawler._make_indent(
indent_unit=crawler.indent_unit,
tab_space_size=crawler.tab_space_size,
)
indent_p1 = indent_section.raw + unit_indent
if unit_indent in indent_section.raw:
indent_m1 = indent_section.raw.replace(unit_indent, "", 1)
else:
indent_m1 = indent_section.raw
if indent > 0:
new_indent = indent_p1
elif indent < 0:
new_indent = indent_m1
else:
new_indent = indent_section.raw
create_anchor = self.find_segment_at(
segments, self.segments[-1].get_end_loc())
if self.role == "pausepoint":
# Assume that this means there isn't a breakpoint
# and that we'll break with the same indent as the
# existing line.
# NOTE: Deal with commas and binary operators differently here.
# Maybe only deal with commas to start with?
if any(
seg.is_type("binary_operator")
for seg in self.segments): # pragma: no cover
raise NotImplementedError(
"Don't know how to deal with binary operators here yet!!"
)
# Remove any existing whitespace
for elem in self.segments:
if not elem.is_meta and elem.is_type("whitespace"):
fixes.append(LintFix.delete(elem))
# Create a newline and a similar indent
fixes.append(
LintFix.create_before(
create_anchor,
[
NewlineSegment(),
WhitespaceSegment(new_indent),
],
))
return fixes
if self.role == "breakpoint":
# Can we determine the required indent just from
# the info in this segment only?
# Remove anything which is already here
for elem in self.segments:
if not elem.is_meta:
fixes.append(LintFix.delete(elem))
# Create a newline, create an indent of the relevant size
fixes.append(
LintFix.create_before(
create_anchor,
[
NewlineSegment(),
WhitespaceSegment(new_indent),
],
))
return fixes
raise ValueError(f"Unexpected break generated at {self}"
) # pragma: no cover
def _eval(self, context: RuleContext) -> Optional[LintResult]:
"""Line is too long.
This only triggers on newline segments, evaluating the whole line.
The detection is simple, the fixing is much trickier.
"""
# Config type hints
self.max_line_length: int
self.ignore_comment_lines: bool
self.ignore_comment_clauses: bool
if not context.memory:
memory: dict = {"comment_clauses": set()}
else:
memory = context.memory
if context.segment.name == "newline":
# iterate to buffer the whole line up to this point
this_line = self._gen_line_so_far(context.raw_stack)
else:
if self.ignore_comment_clauses and context.segment.is_type(
"comment_clause", "comment_equals_clause"):
comment_segment = context.functional.segment.children().first(
sp.is_name("quoted_literal"))
if comment_segment:
memory["comment_clauses"].add(comment_segment.get())
# Otherwise we're all good
return LintResult(memory=memory)
# Now we can work out the line length and deal with the content
line_len = self._compute_source_length(this_line, memory)
if line_len > self.max_line_length:
# Problem, we'll be reporting a violation. The
# question is, can we fix it?
# We'll need the indent, so let's get it for fixing.
line_indent = []
for s in this_line:
if s.name == "whitespace":
line_indent.append(s)
else:
break
# Don't even attempt to handle template placeholders as gets
# complicated if logic changes (e.g. moving for loops). Most of
# these long lines will likely be single line Jinja comments.
# They will remain as unfixable.
if this_line[-1].type == "placeholder":
self.logger.info("Unfixable template segment: %s",
this_line[-1])
return LintResult(anchor=context.segment, memory=memory)
# Does the line end in an inline comment that we can move back?
if this_line[-1].name == "inline_comment":
# Is this line JUST COMMENT (with optional preceding whitespace) if
# so, user will have to fix themselves.
if len(this_line) == 1 or all(
elem.name == "whitespace" or elem.is_meta
for elem in this_line[:-1]):
self.logger.info(
"Unfixable inline comment, alone on line: %s",
this_line[-1])
if self.ignore_comment_lines:
return LintResult(memory=memory)
else:
return LintResult(anchor=context.segment,
memory=memory)
self.logger.info(
"Attempting move of inline comment at end of line: %s",
this_line[-1],
)
# Set up to delete the original comment and the preceding whitespace
delete_buffer = [LintFix.delete(this_line[-1])]
idx = -2
while True:
if (len(this_line) >= abs(idx)
and this_line[idx].name == "whitespace"):
delete_buffer.append(LintFix.delete(this_line[idx]))
idx -= 1
else:
break # pragma: no cover
create_elements = line_indent + [
this_line[-1],
cast(RawSegment, context.segment),
]
if (self._compute_source_length(create_elements, memory) >
self.max_line_length):
# The inline comment is NOT on a line by itself, but even if
# we move it onto a line by itself, it's still too long. In
# this case, the rule should do nothing, otherwise it
# triggers an endless cycle of "fixes" that simply keeps
# adding blank lines.
self.logger.info(
"Unfixable inline comment, too long even on a line by itself: "
"%s",
this_line[-1],
)
if self.ignore_comment_lines:
return LintResult(memory=memory)
else:
return LintResult(anchor=context.segment,
memory=memory)
# Create a newline before this one with the existing comment, an
# identical indent AND a terminating newline, copied from the current
# target segment.
create_buffer = [
LintFix.create_before(this_line[0], create_elements)
]
return LintResult(
anchor=context.segment,
fixes=delete_buffer + create_buffer,
memory=memory,
)
fixes = self._eval_line_for_breaks(this_line)
if fixes:
return LintResult(anchor=context.segment,
fixes=fixes,
memory=memory)
return LintResult(anchor=context.segment, memory=memory)
return LintResult(memory=memory)
def _eval(self, context: RuleContext) -> Optional[LintResult]:
"""Enforce comma placement.
For leading commas we're looking for trailing commas, so
we look for newline segments. For trailing commas we're
looking for leading commas, so we look for the comma itself.
We also want to handle proper whitespace removal/addition. We remove
any trailing whitespace after the leading comma, when converting a
leading comma to a trailing comma. We add whitespace after the leading
comma when converting a trailing comma to a leading comma.
"""
# Config type hints
self.comma_style: str
if not context.memory:
memory: Dict[str, Any] = {
# Trailing comma keys
#
# Do we have a fix in place for removing a leading
# comma violation, and inserting a new trailing comma?
"insert_trailing_comma": False,
# A list of whitespace segments that come after a
# leading comma violation, to be removed during fixing.
"whitespace_deletions": None,
# The leading comma violation segment to be removed during fixing
"last_leading_comma_seg": None,
# The newline segment where we're going to insert our new trailing
# comma during fixing
"anchor_for_new_trailing_comma_seg": None,
#
# Leading comma keys
#
# Do we have a fix in place for removing a trailing
# comma violation, and inserting a new leading comma?
"insert_leading_comma": False,
# The trailing comma violation segment to be removed during fixing
"last_trailing_comma_segment": None,
}
else:
memory = context.memory
if self.comma_style == "trailing":
# A comma preceded by a new line == a leading comma
if context.segment.is_type("comma"):
last_seg = self._last_code_seg(context.raw_stack)
if (last_seg and last_seg.is_type("newline")
and not last_seg.is_templated):
# Recorded where the fix should be applied
memory["last_leading_comma_seg"] = context.segment
last_comment_seg = self._last_comment_seg(
context.raw_stack)
inline_comment = (last_comment_seg.pos_marker.line_no
== last_seg.pos_marker.line_no
if last_comment_seg else False)
# If we have a comment right before the newline, then anchor
# the fix at the comment instead
memory["anchor_for_new_trailing_comma_seg"] = (
last_seg if not inline_comment else last_comment_seg)
# Trigger fix routine
memory["insert_trailing_comma"] = True
memory["whitespace_deletions"] = []
return LintResult(memory=memory)
# Have we found a leading comma violation?
if memory["insert_trailing_comma"]:
# Search for trailing whitespace to delete after the leading
# comma violation
if context.segment.is_type("whitespace"):
memory["whitespace_deletions"] += [context.segment]
return LintResult(memory=memory)
else:
# We've run out of whitespace to delete, time to fix
last_leading_comma_seg = memory["last_leading_comma_seg"]
# Scan backwards to find the last code segment, skipping
# over lines that are either entirely blank or just a
# comment. We want to place the comma immediately after it.
last_code_seg = None
while last_code_seg is None or last_code_seg.is_type(
"newline"):
last_code_seg = self._last_code_seg(
context.raw_stack[:context.raw_stack.index(
last_code_seg if last_code_seg else
memory["last_leading_comma_seg"])])
return LintResult(
anchor=last_leading_comma_seg,
description=
"Found leading comma. Expected only trailing.",
fixes=[
LintFix.delete(last_leading_comma_seg),
*[
LintFix.delete(d)
for d in memory["whitespace_deletions"]
],
LintFix.create_before(
anchor_segment=self._get_following_seg(
context.raw_stack, last_code_seg),
edit_segments=[last_leading_comma_seg],
),
],
)
elif self.comma_style == "leading":
# A new line preceded by a comma == a trailing comma
if context.segment.is_type("newline"):
last_seg = self._last_code_seg(context.raw_stack)
# no code precedes the current position: no issue
if last_seg is None:
return None
if last_seg.is_type(
"comma") and not context.segment.is_templated:
# Trigger fix routine
memory["insert_leading_comma"] = True
# Record where the fix should be applied
memory["last_trailing_comma_segment"] = last_seg
return LintResult(memory=memory)
# Have we found a trailing comma violation?
if memory["insert_leading_comma"]:
# Only insert the comma here if this isn't a comment/whitespace segment
if context.segment.is_code:
last_comma_seg = memory["last_trailing_comma_segment"]
# Create whitespace to insert after the new leading comma
new_whitespace_seg = WhitespaceSegment()
return LintResult(
anchor=last_comma_seg,
description=
"Found trailing comma. Expected only leading.",
fixes=[
LintFix.delete(last_comma_seg),
LintFix.create_before(
anchor_segment=context.segment,
edit_segments=[
last_comma_seg, new_whitespace_seg
],
),
],
)
# Otherwise, no issue
return None
