def _eval_multiple_select_target_elements(self, select_targets_info, segment):
"""Multiple select targets. Ensure each is on a separate line."""
# Insert newline before every select target.
fixes = []
for i, select_target in enumerate(select_targets_info.select_targets):
base_segment = (
segment if not i else select_targets_info.select_targets[i - 1]
)
if (
base_segment.pos_marker.working_line_no
== select_target.pos_marker.working_line_no
):
# Find and delete any whitespace before the select target.
start_seg = select_targets_info.select_idx
# If any select modifier (e.g. distinct ) is present, start
# there rather than at the beginning.
modifier = segment.get_child("select_clause_modifier")
if modifier:
start_seg = segment.segments.index(modifier)
ws_to_delete = segment.select_children(
start_seg=segment.segments[start_seg]
if not i
else select_targets_info.select_targets[i - 1],
select_if=lambda s: s.is_type("whitespace"),
loop_while=lambda s: s.is_type("whitespace", "comma") or s.is_meta,
)
fixes += [LintFix.delete(ws) for ws in ws_to_delete]
fixes.append(LintFix.create_before(select_target, [NewlineSegment()]))
# If we are at the last select target check if the FROM clause
# is on the same line, and if so move it to its own line.
if select_targets_info.from_segment:
if (i + 1 == len(select_targets_info.select_targets)) and (
select_target.pos_marker.working_line_no
== select_targets_info.from_segment.pos_marker.working_line_no
):
fixes.extend(
[
LintFix.delete(ws)
for ws in select_targets_info.pre_from_whitespace
]
)
fixes.append(
LintFix.create_before(
select_targets_info.from_segment,
[NewlineSegment()],
)
)
if fixes:
return LintResult(anchor=segment, fixes=fixes)
def _eval(self, segment, raw_stack, **kwargs):
"""Stars make newlines."""
if segment.is_type("star"):
return LintResult(
anchor=segment,
fixes=[LintFix("create", segment, NewlineSegment())],
)
def _eval_multiple_select_target_elements(self, select_targets_info,
segment):
"""Multiple select targets. Ensure each is on a separate line."""
# Insert newline before every select target.
fixes = []
for i, select_target in enumerate(select_targets_info.select_targets):
base_segment = (segment if not i else
select_targets_info.select_targets[i - 1])
if (base_segment.pos_marker.working_line_no ==
select_target.pos_marker.working_line_no):
# Find and delete any whitespace before the select target.
start_seg = select_targets_info.select_idx
# If any select modifier (e.g. distinct ) is present, start
# there rather than at the beginning.
modifier = segment.get_child("select_clause_modifier")
if modifier:
start_seg = segment.segments.index(modifier)
ws_to_delete = segment.select_children(
start_seg=segment.segments[start_seg]
if not i else select_targets_info.select_targets[i - 1],
select_if=lambda s: s.is_type("whitespace"),
loop_while=lambda s: s.is_type("whitespace", "comma") or s.
is_meta,
)
fixes += [LintFix("delete", ws) for ws in ws_to_delete]
fixes.append(LintFix("create", select_target,
NewlineSegment()))
if fixes:
return LintResult(anchor=segment, fixes=fixes)
def _generate_fixes(
whitespace_segment: Optional[BaseSegment],
) -> Optional[List[LintFix]]:
if whitespace_segment:
return [
LintFix.replace(
anchor_segment=whitespace_segment,
# NB: Currently we are just inserting a Newline here. This alone will
# produce not properly indented SQL. We rely on L003 to deal with
# indentation later.
# As a future improvement we could maybe add WhitespaceSegment( ... )
# here directly.
edit_segments=[NewlineSegment()],
)
]
else:
# We should rarely reach here as set operators are always surrounded by either
# WhitespaceSegment or NewlineSegment.
# However, in exceptional cases the WhitespaceSegment might be enclosed in the
# surrounding segment hierachy and not accessible by the rule logic.
# At the time of writing this is true for `tsql` as covered in the test
# `test_fail_autofix_in_tsql_disabled`. If we encounter such case, we skip
# fixing.
return []
def _eval(self, context: RuleContext) -> LintResult:
"""Nested CASE statement in ELSE clause could be flattened."""
segment = FunctionalContext(context).segment
assert segment.select(sp.is_type("case_expression"))
case1_children = segment.children()
case1_last_when = case1_children.last(sp.is_type("when_clause")).get()
case1_else_clause = case1_children.select(sp.is_type("else_clause"))
case1_else_expressions = case1_else_clause.children(
sp.is_type("expression"))
expression_children = case1_else_expressions.children()
case2 = expression_children.select(sp.is_type("case_expression"))
# The len() checks below are for safety, to ensure the CASE inside
# the ELSE is not part of a larger expression. In that case, it's
# not safe to simplify in this way -- we'd be deleting other code.
if (not case1_last_when or len(case1_else_expressions) > 1
or len(expression_children) > 1 or not case2):
return LintResult()
# We can assert that this exists because of the previous check.
assert case1_last_when
# We can also assert that we'll also have an else clause because
# otherwise the case2 check above would fail.
case1_else_clause_seg = case1_else_clause.get()
assert case1_else_clause_seg
# Delete stuff between the last "WHEN" clause and the "ELSE" clause.
case1_to_delete = case1_children.select(start_seg=case1_last_when,
stop_seg=case1_else_clause_seg)
# Delete the nested "CASE" expression.
fixes = case1_to_delete.apply(lambda seg: LintFix.delete(seg))
# Determine the indentation to use when we move the nested "WHEN"
# and "ELSE" clauses, based on the indentation of case1_last_when.
# If no whitespace segments found, use default indent.
indent = (case1_children.select(
stop_seg=case1_last_when).reversed().select(
sp.is_type("whitespace")))
indent_str = "".join(seg.raw
for seg in indent) if indent else self.indent
# Move the nested "when" and "else" clauses after the last outer
# "when".
nested_clauses = case2.children(
sp.is_type("when_clause", "else_clause"))
create_after_last_when = nested_clauses.apply(
lambda seg: [NewlineSegment(),
WhitespaceSegment(indent_str), seg])
segments = [
item for sublist in create_after_last_when for item in sublist
]
fixes.append(
LintFix.create_after(case1_last_when, segments, source=segments))
# Delete the outer "else" clause.
fixes.append(LintFix.delete(case1_else_clause_seg))
return LintResult(case2[0], fixes=fixes)
def _eval(self, context: RuleContext) -> Optional[LintResult]:
"""Files must end with a single trailing newline.
We only care about the segment and the siblings which come after it
for this rule, we discard the others into the kwargs argument.
"""
# We only care about the final segment of the parse tree.
if not self.is_final_segment(context):
return None
# Include current segment for complete stack and reverse.
parent_stack: Segments = context.functional.parent_stack
complete_stack: Segments = (context.functional.raw_stack +
context.functional.segment)
reversed_complete_stack = complete_stack.reversed()
# Find the trailing newline segments.
trailing_newlines = reversed_complete_stack.select(
select_if=sp.is_type("newline"),
loop_while=sp.or_(sp.is_whitespace(), sp.is_type("dedent")),
)
trailing_literal_newlines = trailing_newlines.select(
loop_while=lambda seg: sp.templated_slices(
seg, context.templated_file).all(tsp.is_slice_type("literal")))
if not trailing_literal_newlines:
# We make an edit to create this segment after the child of the FileSegment.
if len(parent_stack) == 1:
fix_anchor_segment = context.segment
else:
fix_anchor_segment = parent_stack[1]
return LintResult(
anchor=context.segment,
fixes=[
LintFix.create_after(
fix_anchor_segment,
[NewlineSegment()],
)
],
)
elif len(trailing_literal_newlines) > 1:
# Delete extra newlines.
return LintResult(
anchor=context.segment,
fixes=[
LintFix.delete(d) for d in trailing_literal_newlines[1:]
],
)
else:
# Single newline, no need for fix.
return None
def _eval(self, context: RuleContext) -> Optional[LintResult]:
"""Files must end with a single trailing newline.
We only care about the segment and the siblings which come after it
for this rule, we discard the others into the kwargs argument.
"""
# We only care about the final segment of the parse tree.
parent_stack, segment = get_last_segment(FunctionalContext(context).segment)
self.logger.debug("Found last segment as: %s", segment)
trailing_newlines = Segments(*get_trailing_newlines(context.segment))
trailing_literal_newlines = trailing_newlines
self.logger.debug(
"Untemplated trailing newlines: %s", trailing_literal_newlines
)
if context.templated_file:
trailing_literal_newlines = trailing_newlines.select(
loop_while=lambda seg: sp.templated_slices(
seg, context.templated_file
).all(tsp.is_slice_type("literal"))
)
self.logger.debug("Templated trailing newlines: %s", trailing_literal_newlines)
if not trailing_literal_newlines:
# We make an edit to create this segment after the child of the FileSegment.
if len(parent_stack) == 1:
fix_anchor_segment = segment[0]
else:
fix_anchor_segment = parent_stack[1]
self.logger.debug("Anchor on: %s", fix_anchor_segment)
return LintResult(
anchor=segment[0],
fixes=[
LintFix.create_after(
fix_anchor_segment,
[NewlineSegment()],
)
],
)
elif len(trailing_literal_newlines) > 1:
# Delete extra newlines.
return LintResult(
anchor=segment[0],
fixes=[LintFix.delete(d) for d in trailing_literal_newlines[1:]],
)
else:
# Single newline, no need for fix.
return None
def _eval(self, segment, **kwargs):
"""Select clause modifiers must appear on same line as SELECT."""
if segment.is_type("select_clause"):
# Does the select clause have modifiers?
select_modifier = segment.get_child("select_clause_modifier")
if not select_modifier:
return None # No. We're done.
select_modifier_idx = segment.segments.index(select_modifier)
# Does the select clause contain a newline?
newline = segment.get_child("newline")
if not newline:
return None # No. We're done.
newline_idx = segment.segments.index(newline)
# Is there a newline before the select modifier?
if newline_idx > select_modifier_idx:
return None # No, we're done.
# Yes to all the above. We found an issue.
# E.g.: " DISTINCT\n"
replace_newline_with = [
WhitespaceSegment(),
select_modifier,
NewlineSegment(),
]
fixes = [
# E.g. "\n" -> " DISTINCT\n.
LintFix("edit", newline, replace_newline_with),
# E.g. "DISTINCT" -> X
LintFix("delete", select_modifier),
]
# E.g. " " after "DISTINCT"
ws_to_delete = segment.select_children(
start_seg=select_modifier,
select_if=lambda s: s.is_type("whitespace"),
loop_while=lambda s: s.is_type("whitespace") or s.is_meta,
)
# E.g. " " -> X
fixes += [LintFix("delete", ws) for ws in ws_to_delete]
return LintResult(
anchor=segment,
fixes=fixes,
)
def _fixes_for_move_after_select_clause(
stop_seg: BaseSegment,
delete_segments: Optional[Segments] = None,
add_newline: bool = True,
) -> List[LintFix]:
"""Cleans up by moving leftover select_clause segments.
Context: Some of the other fixes we make in
_eval_single_select_target_element() leave leftover
child segments that need to be moved to become
*siblings* of the select_clause.
"""
start_seg = (
modifier[0]
if modifier
else select_children[select_targets_info.first_new_line_idx]
)
move_after_select_clause = select_children.select(
start_seg=start_seg,
stop_seg=stop_seg,
)
# :TRICKY: Below, we have a couple places where we
# filter to guard against deleting the same segment
# multiple times -- this is illegal.
# :TRICKY: Use IdentitySet rather than set() since
# different segments may compare as equal.
all_deletes = IdentitySet(
fix.anchor for fix in fixes if fix.edit_type == "delete"
)
fixes_ = []
for seg in delete_segments or []:
if seg not in all_deletes:
fixes.append(LintFix.delete(seg))
all_deletes.add(seg)
fixes_ += [
LintFix.delete(seg)
for seg in move_after_select_clause
if seg not in all_deletes
]
fixes_.append(
LintFix.create_after(
select_clause[0],
([NewlineSegment()] if add_newline else [])
+ list(move_after_select_clause),
)
)
return fixes_
def _fixes_for_move_after_select_clause(
stop_seg: BaseSegment,
add_newline: bool = True,
) -> List[LintFix]:
"""Cleans up by moving leftover select_clause segments.
Context: Some of the other fixes we make in
_eval_single_select_target_element() leave the
select_clause in an illegal state -- a select_clause's
*rightmost children cannot be whitespace or comments*.
This function addresses that by moving these segments
up the parse tree to an ancestor segment chosen by
_choose_anchor_segment(). After these fixes are applied,
these segments may, for example, be *siblings* of
select_clause.
"""
start_seg = (
modifier[0] if modifier else select_children[
select_targets_info.first_new_line_idx])
move_after_select_clause = select_children.select(
start_seg=start_seg,
stop_seg=stop_seg,
)
fixes = [
LintFix.delete(seg)
for seg in move_after_select_clause
]
fixes.append(
LintFix.create_after(
self._choose_anchor_segment(
context,
"create_after",
select_clause[0],
filter_meta=True,
),
([NewlineSegment()] if add_newline else []) +
list(move_after_select_clause),
))
return fixes
def _eval(self, segment, siblings_post, parent_stack, **kwargs):
"""Files must end with a trailing newline.
We only care about the segment and the siblings which come after it
for this rule, we discard the others into the kwargs argument.
"""
if len(self.filter_meta(siblings_post)) > 0:
# This can only fail on the last segment
return None
elif len(segment.segments) > 0:
# This can only fail on the last base segment
return None
elif segment.name == "newline":
# If this is the last segment, and it's a newline then we're good
return None
elif segment.is_meta:
# We can't fail on a meta segment
return None
else:
# So this looks like the end of the file, but we
# need to check that each parent segment is also the last.
# We do this with reference to the templated file, because it's
# the best we can do given the information available.
file_len = len(segment.pos_marker.templated_file.templated_str)
pos = segment.pos_marker.templated_slice.stop
# Does the length of the file equal the end of the templated position?
if file_len != pos:
return None
# We're going to make an edit because we're appending to the end and there's
# no segment after it to match on. Otherwise we would never get a match!
return LintResult(
anchor=segment,
fixes=[LintFix("edit", segment,
[segment, NewlineSegment()])],
)
def _eval_single_select_target_element(self, select_targets_info,
select_clause, parent_stack):
is_wildcard = False
for segment in select_clause.segments:
if segment.is_type("select_clause_element"):
for sub_segment in segment.segments:
if sub_segment.is_type("wildcard_expression"):
is_wildcard = True
if is_wildcard:
return None
elif (select_targets_info.select_idx <
select_targets_info.first_new_line_idx <
select_targets_info.first_select_target_idx):
# Do we have a modifier?
modifier = select_clause.get_child("select_clause_modifier")
# Prepare the select clause which will be inserted
# In most (but not all) case we'll want to replace the newline with
# the statement and a newline, but in some cases however (see #1424)
# we don't need the final newline.
copy_with_newline = True
insert_buff = [
WhitespaceSegment(),
select_clause.segments[
select_targets_info.first_select_target_idx],
]
# Check if the modifier is one we care about
if modifier:
# If it's already on the first line, ignore it.
if (select_clause.segments.index(modifier) <
select_targets_info.first_new_line_idx):
modifier = None
fixes = [
# Delete the first select target from its original location.
# We'll add it to the right section at the end, once we know
# what to add.
LintFix(
"delete",
select_clause.segments[
select_targets_info.first_select_target_idx],
),
]
start_idx = 0
# If we have a modifier to move:
if modifier:
# Add it to the insert
insert_buff = [WhitespaceSegment(), modifier] + insert_buff
modifier_idx = select_clause.segments.index(modifier)
# Delete the whitespace after it (which is two after, thanks to indent)
if (len(select_clause.segments) > modifier_idx + 1
and select_clause.segments[modifier_idx +
2].is_whitespace):
fixes += [
LintFix(
"delete",
select_clause.segments[modifier_idx + 2],
),
]
# Delete the modifier itself
fixes += [
LintFix(
"delete",
modifier,
),
]
# Set the position marker for removing the preceding
# whitespace and newline, which we'll use below.
start_idx = modifier_idx
else:
# Set the position marker for removing the preceding
# whitespace and newline, which we'll use below.
start_idx = select_targets_info.first_select_target_idx
if parent_stack and parent_stack[-1].is_type("select_statement"):
select_stmt = parent_stack[-1]
select_clause_idx = select_stmt.segments.index(select_clause)
after_select_clause_idx = select_clause_idx + 1
if len(select_stmt.segments) > after_select_clause_idx:
if select_stmt.segments[after_select_clause_idx].is_type(
"newline"):
# The select_clause is immediately followed by a
# newline. Delete the newline in order to avoid leaving
# behind an empty line after fix.
delete_last_newline = True
# Since, we're deleting the newline, we should also delete all
# whitespace before it or it will add random whitespace to
# following statements. So walk back through the segment
# deleting whitespace until you get the previous newline, or
# something else.
idx = 1
while start_idx - idx < len(select_clause.segments):
# Delete any whitespace
if select_clause.segments[start_idx - idx].is_type(
"whitespace"):
fixes += [
LintFix(
"delete",
select_clause.segments[start_idx -
idx],
),
]
# Once we see a newline, then we're done
if select_clause.segments[start_idx - idx].is_type(
"newline", ):
break
# If we see anything other than whitespace,
# then we're done, but in this case we want to
# keep the final newline.
if not select_clause.segments[start_idx -
idx].is_type(
"whitespace",
"newline"):
delete_last_newline = False
break
idx += 1
# Finally delete the newline, unless we've decided not to
if delete_last_newline:
fixes.append(
LintFix(
"delete",
select_stmt.
segments[after_select_clause_idx],
))
elif select_stmt.segments[after_select_clause_idx].is_type(
"whitespace"):
# The select_clause has stuff after (most likely a comment)
# Delete the whitespace immeadiately after the select clause
# so the other stuff aligns nicely based on where the select
# clause started
fixes += [
LintFix(
"delete",
select_stmt.segments[after_select_clause_idx],
),
]
elif select_stmt.segments[after_select_clause_idx].is_type(
"dedent"):
# The end of the select statement, so this is the one
# case we don't want the newline added to end of
# select_clause (see #1424)
copy_with_newline = False
# Again let's strip back the whitespace, bnut simpler
# as don't need to worry about new line so just break
# if see non-whitespace
idx = 1
start_idx = select_clause_idx - 1
while start_idx - idx < len(select_clause.segments):
# Delete any whitespace
if select_clause.segments[start_idx - idx].is_type(
"whitespace"):
fixes += [
LintFix(
"delete",
select_clause.segments[start_idx -
idx],
),
]
# Once we see a newline, then we're done
if select_clause.segments[start_idx -
idx].is_type("newline"):
break
# If we see anything other than whitespace,
# then we're done, but in this case we want to
# keep the final newline.
if not select_clause.segments[start_idx -
idx].is_type(
"whitespace",
"newline"):
copy_with_newline = True
break
idx += 1
if copy_with_newline:
insert_buff = insert_buff + [NewlineSegment()]
fixes += [
# Insert the select_clause in place of the first newlin in the
# Select statement
LintFix(
"edit",
select_clause.segments[
select_targets_info.first_new_line_idx],
insert_buff,
),
]
return LintResult(
anchor=select_clause,
fixes=fixes,
)
else:
return None
def _eval(self, segment, raw_stack, **kwargs):
"""WITH clause closing bracket should be aligned with WITH keyword.
Look for a with clause and evaluate the position of closing brackets.
"""
# We only trigger on start_bracket (open parenthesis)
if segment.is_type("with_compound_statement"):
raw_stack_buff = list(raw_stack)
# Look for the with keyword
for seg in segment.segments:
if seg.name.lower() == "with":
seg_line_no = seg.pos_marker.line_no
break
else: # pragma: no cover
# This *could* happen if the with statement is unparsable,
# in which case then the user will have to fix that first.
if any(s.is_type("unparsable") for s in segment.segments):
return LintResult()
# If it's parsable but we still didn't find a with, then
# we should raise that.
raise RuntimeError("Didn't find WITH keyword!")
def indent_size_up_to(segs):
seg_buff = []
# Get any segments running up to the WITH
for elem in reversed(segs):
if elem.is_type("newline"):
break
elif elem.is_meta:
continue
else:
seg_buff.append(elem)
# reverse the indent if we have one
if seg_buff:
seg_buff = list(reversed(seg_buff))
indent_str = "".join(seg.raw for seg in seg_buff).replace(
"\t", " " * self.tab_space_size)
indent_size = len(indent_str)
return indent_size, indent_str
balance = 0
with_indent, with_indent_str = indent_size_up_to(raw_stack_buff)
for seg in segment.iter_segments(
expanding=["common_table_expression", "bracketed"],
pass_through=True):
if seg.name == "start_bracket":
balance += 1
elif seg.name == "end_bracket":
balance -= 1
if balance == 0:
closing_bracket_indent, _ = indent_size_up_to(
raw_stack_buff)
indent_diff = closing_bracket_indent - with_indent
# Is indent of closing bracket not the same as
# indent of WITH keyword.
if seg.pos_marker.line_no == seg_line_no:
# Skip if it's the one-line version. That's ok
pass
elif indent_diff < 0:
return LintResult(
anchor=seg,
fixes=[
LintFix(
"create",
seg,
WhitespaceSegment(" " *
(-indent_diff)),
)
],
)
elif indent_diff > 0:
# Is it all whitespace before the bracket on this line?
prev_segs_on_line = [
elem for elem in segment.iter_segments(
expanding=[
"common_table_expression", "bracketed"
],
pass_through=True,
) if elem.pos_marker.line_no == seg.pos_marker.
line_no and elem.pos_marker.line_pos <
seg.pos_marker.line_pos
]
if all(
elem.is_type("whitespace")
for elem in prev_segs_on_line):
# We can move it back, it's all whitespace
fixes = [
LintFix(
"create",
seg,
[WhitespaceSegment(with_indent_str)],
)
] + [
LintFix("delete", elem)
for elem in prev_segs_on_line
]
else:
# We have to move it to a newline
fixes = [
LintFix(
"create",
seg,
[
NewlineSegment(),
WhitespaceSegment(with_indent_str),
],
)
]
return LintResult(anchor=seg, fixes=fixes)
else:
raw_stack_buff.append(seg)
return LintResult()
def _eval_single_select_target_element(self, select_targets_info,
context: RuleContext):
select_clause = context.functional.segment
parent_stack = context.parent_stack
wildcards = select_clause.children(
sp.is_type("select_clause_element")).children(
sp.is_type("wildcard_expression"))
is_wildcard = bool(wildcards)
if is_wildcard:
wildcard_select_clause_element = wildcards[0]
if (select_targets_info.select_idx <
select_targets_info.first_new_line_idx <
select_targets_info.first_select_target_idx) and (
not is_wildcard):
# Do we have a modifier?
select_children = select_clause.children()
modifier: Optional[Segments]
modifier = select_children.first(
sp.is_type("select_clause_modifier"))
# Prepare the select clause which will be inserted
insert_buff = [
WhitespaceSegment(),
select_children[select_targets_info.first_select_target_idx],
]
# Check if the modifier is one we care about
if modifier:
# If it's already on the first line, ignore it.
if (select_children.index(modifier.get()) <
select_targets_info.first_new_line_idx):
modifier = None
fixes = [
# Delete the first select target from its original location.
# We'll add it to the right section at the end, once we know
# what to add.
LintFix.delete(
select_children[
select_targets_info.first_select_target_idx], ),
]
# If we have a modifier to move:
if modifier:
# Add it to the insert
insert_buff = [WhitespaceSegment(), modifier[0]] + insert_buff
modifier_idx = select_children.index(modifier.get())
# Delete the whitespace after it (which is two after, thanks to indent)
if (len(select_children) > modifier_idx + 1
and select_children[modifier_idx + 2].is_whitespace):
fixes += [
LintFix.delete(select_children[modifier_idx + 2], ),
]
# Delete the modifier itself
fixes += [
LintFix.delete(modifier[0], ),
]
# Set the position marker for removing the preceding
# whitespace and newline, which we'll use below.
start_idx = modifier_idx
else:
# Set the position marker for removing the preceding
# whitespace and newline, which we'll use below.
start_idx = select_targets_info.first_select_target_idx
if parent_stack and parent_stack[-1].is_type("select_statement"):
select_stmt = parent_stack[-1]
select_clause_idx = select_stmt.segments.index(
select_clause.get())
after_select_clause_idx = select_clause_idx + 1
if len(select_stmt.segments) > after_select_clause_idx:
def _fixes_for_move_after_select_clause(
stop_seg: BaseSegment,
add_newline: bool = True,
) -> List[LintFix]:
"""Cleans up by moving leftover select_clause segments.
Context: Some of the other fixes we make in
_eval_single_select_target_element() leave the
select_clause in an illegal state -- a select_clause's
*rightmost children cannot be whitespace or comments*.
This function addresses that by moving these segments
up the parse tree to an ancestor segment chosen by
_choose_anchor_segment(). After these fixes are applied,
these segments may, for example, be *siblings* of
select_clause.
"""
start_seg = (
modifier[0] if modifier else select_children[
select_targets_info.first_new_line_idx])
move_after_select_clause = select_children.select(
start_seg=start_seg,
stop_seg=stop_seg,
)
fixes = [
LintFix.delete(seg)
for seg in move_after_select_clause
]
fixes.append(
LintFix.create_after(
self._choose_anchor_segment(
context,
"create_after",
select_clause[0],
filter_meta=True,
),
([NewlineSegment()] if add_newline else []) +
list(move_after_select_clause),
))
return fixes
if select_stmt.segments[after_select_clause_idx].is_type(
"newline"):
# Since we're deleting the newline, we should also delete all
# whitespace before it or it will add random whitespace to
# following statements. So walk back through the segment
# deleting whitespace until you get the previous newline, or
# something else.
to_delete = select_children.reversed().select(
loop_while=sp.is_type("whitespace"),
start_seg=select_children[start_idx],
)
fixes += [LintFix.delete(seg) for seg in to_delete]
# The select_clause is immediately followed by a
# newline. Delete the newline in order to avoid leaving
# behind an empty line after fix, *unless* we stopped
# due to something other than a newline.
delete_last_newline = select_children[
start_idx - len(to_delete) - 1].is_type("newline")
# Delete the newline if we decided to.
if delete_last_newline:
fixes.append(
LintFix.delete(
select_stmt.
segments[after_select_clause_idx], ))
fixes += _fixes_for_move_after_select_clause(
to_delete[-1], )
elif select_stmt.segments[after_select_clause_idx].is_type(
"whitespace"):
# The select_clause has stuff after (most likely a comment)
# Delete the whitespace immediately after the select clause
# so the other stuff aligns nicely based on where the select
# clause started
fixes += [
LintFix.delete(
select_stmt.segments[after_select_clause_idx],
),
]
fixes += _fixes_for_move_after_select_clause(
select_children[
select_targets_info.first_select_target_idx], )
elif select_stmt.segments[after_select_clause_idx].is_type(
"dedent"):
# Again let's strip back the whitespace, but simpler
# as don't need to worry about new line so just break
# if see non-whitespace
to_delete = select_children.reversed().select(
loop_while=sp.is_type("whitespace"),
start_seg=select_children[select_clause_idx - 1],
)
fixes += [LintFix.delete(seg) for seg in to_delete]
if to_delete:
fixes += _fixes_for_move_after_select_clause(
to_delete[-1],
# If we deleted a newline, create a newline.
any(seg for seg in to_delete
if seg.is_type("newline")),
)
else:
fixes += _fixes_for_move_after_select_clause(
select_children[
select_targets_info.first_select_target_idx], )
fixes += [
# Insert the select_clause in place of the first newline in the
# Select statement
LintFix.replace(
select_children[select_targets_info.first_new_line_idx],
insert_buff,
),
]
return LintResult(
anchor=select_clause.get(),
fixes=fixes,
)
# If we have a wildcard on the same line as the FROM keyword, but not the same
# line as the SELECT keyword, we need to move the FROM keyword to its own line.
# i.e.
# SELECT
# * FROM foo
if select_targets_info.from_segment:
if (is_wildcard and
(select_clause[0].pos_marker.working_line_no !=
select_targets_info.from_segment.pos_marker.working_line_no)
and
(wildcard_select_clause_element.pos_marker.working_line_no ==
select_targets_info.from_segment.pos_marker.working_line_no)):
fixes = [
LintFix.delete(ws)
for ws in select_targets_info.pre_from_whitespace
]
fixes.append(
LintFix.create_before(
select_targets_info.from_segment,
[NewlineSegment()],
))
return LintResult(anchor=select_clause.get(), fixes=fixes)
return None
def make_newline(cls, raw=None):
"""Make a newline segment."""
# Default the newline to \n
raw = raw or "\n"
return NewlineSegment(raw=raw, pos_marker=None)
def generate_fixes_to_coerce(
self, segments, indent_section, crawler, indent
):
"""Generate a list of fixes to create a break at this point.
The `segments` argument is necessary to extract anchors
from the existing segments.
"""
fixes = []
# Generate some sample indents:
unit_indent = crawler._make_indent(
indent_unit=crawler.indent_unit,
tab_space_size=crawler.tab_space_size,
)
indent_p1 = indent_section.raw + unit_indent
if unit_indent in indent_section.raw:
indent_m1 = indent_section.raw.replace(unit_indent, "", 1)
else:
indent_m1 = indent_section.raw
if indent > 0:
new_indent = indent_p1
elif indent < 0:
new_indent = indent_m1
else:
new_indent = indent_section.raw
create_anchor = self.find_segment_at(
segments, self.segments[-1].get_end_loc()
)
if self.role == "pausepoint":
# Assume that this means there isn't a breakpoint
# and that we'll break with the same indent as the
# existing line.
# NOTE: Deal with commas and binary operators differently here.
# Maybe only deal with commas to start with?
if any(
seg.is_type("binary_operator") for seg in self.segments
): # pragma: no cover
raise NotImplementedError(
"Don't know how to deal with binary operators here yet!!"
)
# Remove any existing whitespace
for elem in self.segments:
if not elem.is_meta and elem.is_type("whitespace"):
fixes.append(LintFix("delete", elem))
# Create a newline and a similar indent
fixes.append(
LintFix(
"create",
create_anchor,
[
NewlineSegment(),
WhitespaceSegment(new_indent),
],
)
)
return fixes
if self.role == "breakpoint":
# Can we determine the required indent just from
# the info in this segment only?
# Remove anything which is already here
for elem in self.segments:
if not elem.is_meta:
fixes.append(LintFix("delete", elem))
# Create a newline, create an indent of the relevant size
fixes.append(
LintFix(
"create",
create_anchor,
[
NewlineSegment(),
WhitespaceSegment(new_indent),
],
)
)
return fixes
raise ValueError(
f"Unexpected break generated at {self}"
) # pragma: no cover
def _eval(self, context: RuleContext):
"""WITH clause closing bracket should be aligned with WITH keyword.
Look for a with clause and evaluate the position of closing brackets.
"""
# We only trigger on start_bracket (open parenthesis)
assert context.segment.is_type("with_compound_statement")
raw_stack_buff = list(context.raw_stack)
# Look for the with keyword
for seg in context.segment.segments:
if seg.name.lower() == "with":
seg_line_no = seg.pos_marker.line_no
break
else: # pragma: no cover
# This *could* happen if the with statement is unparsable,
# in which case then the user will have to fix that first.
if any(s.is_type("unparsable") for s in context.segment.segments):
return LintResult()
# If it's parsable but we still didn't find a with, then
# we should raise that.
raise RuntimeError("Didn't find WITH keyword!")
# Find the end brackets for the CTE *query* (i.e. ignore optional
# list of CTE columns).
cte_end_brackets = IdentitySet()
for cte in (FunctionalContext(context).segment.children(
sp.is_type("common_table_expression")).iterate_segments()):
cte_end_bracket = (cte.children().last(
sp.is_type("bracketed")).children().last(
sp.is_type("end_bracket")))
if cte_end_bracket:
cte_end_brackets.add(cte_end_bracket[0])
for seg in context.segment.iter_segments(
expanding=["common_table_expression", "bracketed"],
pass_through=True):
if seg not in cte_end_brackets:
if not seg.is_type("start_bracket"):
raw_stack_buff.append(seg)
continue
if seg.pos_marker.line_no == seg_line_no:
# Skip if it's the one-line version. That's ok
continue
# Is it all whitespace before the bracket on this line?
assert seg.pos_marker
contains_non_whitespace = False
for elem in context.segment.raw_segments:
if (cast(PositionMarker,
elem.pos_marker).line_no == seg.pos_marker.line_no
and cast(PositionMarker, elem.pos_marker).line_pos <=
seg.pos_marker.line_pos):
if elem is seg:
break
elif elem.is_type("newline"):
contains_non_whitespace = False
elif not elem.is_type("dedent") and not elem.is_type(
"whitespace"):
contains_non_whitespace = True
if contains_non_whitespace:
# We have to move it to a newline
return LintResult(
anchor=seg,
fixes=[LintFix.create_before(
seg,
[
NewlineSegment(),
],
)],
)
def _eval_single_select_target_element(
self, select_targets_info, select_clause, parent_stack
):
is_wildcard = False
for segment in select_clause.segments:
if segment.is_type("select_clause_element"):
for sub_segment in segment.segments:
if sub_segment.is_type("wildcard_expression"):
is_wildcard = True
if is_wildcard:
return None
elif (
select_targets_info.select_idx
< select_targets_info.first_new_line_idx
< select_targets_info.first_select_target_idx
):
# Do we have a modifier?
modifier = select_clause.get_child("select_clause_modifier")
# there is a newline between select and select target
insert_buff = [
WhitespaceSegment(),
select_clause.segments[select_targets_info.first_select_target_idx],
NewlineSegment(),
]
# Also move any modifiers if present
if modifier:
# If it's already on the first line, ignore it.
if (
select_clause.segments.index(modifier)
< select_targets_info.first_new_line_idx
):
modifier = None
# Otherwise we need to move it too
else:
insert_buff = [WhitespaceSegment(), modifier] + insert_buff
fixes = [
# Replace "newline" with <<select_target>>, "newline".
LintFix(
"edit",
select_clause.segments[select_targets_info.first_new_line_idx],
insert_buff,
),
# Delete the first select target from its original location.
LintFix(
"delete",
select_clause.segments[select_targets_info.first_select_target_idx],
),
]
# Also delete the original modifier if present
if modifier:
fixes += [
LintFix(
"delete",
modifier,
),
]
if (
select_targets_info.first_select_target_idx
- select_targets_info.first_new_line_idx
== 2
and select_clause.segments[
select_targets_info.first_new_line_idx + 1
].is_whitespace
):
# If the select target is preceded by a single whitespace
# segment, delete that as well. This addresses the bug fix
# tested in L036.yml's "test_cte" scenario.
fixes.append(
LintFix(
"delete",
select_clause.segments[
select_targets_info.first_new_line_idx + 1
],
),
)
if parent_stack and parent_stack[-1].is_type("select_statement"):
select_stmt = parent_stack[-1]
select_clause_idx = select_stmt.segments.index(select_clause)
after_select_clause_idx = select_clause_idx + 1
if len(select_stmt.segments) > after_select_clause_idx:
if select_stmt.segments[after_select_clause_idx].is_type("newline"):
# The select_clause is immediately followed by a
# newline. Delete the newline in order to avoid leaving
# behind an empty line after fix.
fixes.append(
LintFix(
"delete", select_stmt.segments[after_select_clause_idx]
)
)
return LintResult(
anchor=select_clause,
fixes=fixes,
)
else:
return None
def _eval(self, context: RuleContext) -> Optional[List[LintResult]]:
"""Blank line expected but not found after CTE definition."""
# Config type hints
self.comma_style: str
error_buffer = []
if context.segment.is_type("with_compound_statement"):
# First we need to find all the commas, the end brackets, the
# things that come after that and the blank lines in between.
# Find all the closing brackets. They are our anchor points.
bracket_indices = []
expanded_segments = list(
context.segment.iter_segments(expanding=["common_table_expression"])
)
for idx, seg in enumerate(expanded_segments):
if seg.is_type("bracketed"):
# Check if the preceding keyword is AS, otherwise it's a column name
# definition in the CTE.
preceding_keyword = next(
(
s
for s in expanded_segments[:idx][::-1]
if s.is_type("keyword")
),
None,
)
if (
preceding_keyword is not None
and preceding_keyword.raw.upper() == "AS"
):
bracket_indices.append(idx)
# Work through each point and deal with it individually
for bracket_idx in bracket_indices:
forward_slice = expanded_segments[bracket_idx:]
seg_idx = 1
line_idx = 0
comma_seg_idx = 0
blank_lines = 0
comma_line_idx = None
line_blank = False
comma_style = None
line_starts = {}
comment_lines = []
self.logger.info(
"## CTE closing bracket found at %s, idx: %s. Forward slice: %.20r",
forward_slice[0].pos_marker,
bracket_idx,
"".join(elem.raw for elem in forward_slice),
)
# Work forward to map out the following segments.
while (
forward_slice[seg_idx].is_type("comma")
or not forward_slice[seg_idx].is_code
):
if forward_slice[seg_idx].is_type("newline"):
if line_blank:
# It's a blank line!
blank_lines += 1
line_blank = True
line_idx += 1
line_starts[line_idx] = seg_idx + 1
elif forward_slice[seg_idx].is_type("comment"):
# Lines with comments aren't blank
line_blank = False
comment_lines.append(line_idx)
elif forward_slice[seg_idx].is_type("comma"):
# Keep track of where the comma is.
# We'll evaluate it later.
comma_line_idx = line_idx
comma_seg_idx = seg_idx
seg_idx += 1
# Infer the comma style (NB this could be different for each case!)
if comma_line_idx is None:
comma_style = "final"
elif line_idx == 0:
comma_style = "oneline"
elif comma_line_idx == 0:
comma_style = "trailing"
elif comma_line_idx == line_idx:
comma_style = "leading"
else:
comma_style = "floating"
# Readout of findings
self.logger.info(
"blank_lines: %s, comma_line_idx: %s. final_line_idx: %s, "
"final_seg_idx: %s",
blank_lines,
comma_line_idx,
line_idx,
seg_idx,
)
self.logger.info(
"comma_style: %r, line_starts: %r, comment_lines: %r",
comma_style,
line_starts,
comment_lines,
)
if blank_lines < 1:
# We've got an issue
self.logger.info("!! Found CTE without enough blank lines.")
# Based on the current location of the comma we insert newlines
# to correct the issue.
fix_type = "create_before" # In most cases we just insert newlines.
if comma_style == "oneline":
# Here we respect the target comma style to insert at the
# relevant point.
if self.comma_style == "trailing":
# Add a blank line after the comma
fix_point = forward_slice[comma_seg_idx + 1]
# Optionally here, if the segment we've landed on is
# whitespace then we REPLACE it rather than inserting.
if forward_slice[comma_seg_idx + 1].is_type("whitespace"):
fix_type = "replace"
elif self.comma_style == "leading":
# Add a blank line before the comma
fix_point = forward_slice[comma_seg_idx]
# In both cases it's a double newline.
num_newlines = 2
else:
# In the following cases we only care which one we're in
# when comments don't get in the way. If they *do*, then
# we just work around them.
if not comment_lines or line_idx - 1 not in comment_lines:
self.logger.info("Comment routines not applicable")
if comma_style in ("trailing", "final", "floating"):
# Detected an existing trailing comma or it's a final
# CTE, OR the comma isn't leading or trailing.
# If the preceding segment is whitespace, replace it
if forward_slice[seg_idx - 1].is_type("whitespace"):
fix_point = forward_slice[seg_idx - 1]
fix_type = "replace"
else:
# Otherwise add a single newline before the end
# content.
fix_point = forward_slice[seg_idx]
elif comma_style == "leading":
# Detected an existing leading comma.
fix_point = forward_slice[comma_seg_idx]
else:
self.logger.info("Handling preceding comments")
offset = 1
while line_idx - offset in comment_lines:
offset += 1
fix_point = forward_slice[
line_starts[line_idx - (offset - 1)]
]
# Note: There is an edge case where this isn't enough, if
# comments are in strange places, but we'll catch them on
# the next iteration.
num_newlines = 1
fixes = [
LintFix(
fix_type,
fix_point,
[NewlineSegment()] * num_newlines,
)
]
# Create a result, anchored on the start of the next content.
error_buffer.append(
LintResult(anchor=forward_slice[seg_idx], fixes=fixes)
)
# Return the buffer if we have one.
return error_buffer or None
def _eval(self, context: RuleContext) -> Optional[LintResult]:
"""Select clause modifiers must appear on same line as SELECT."""
# We only care about select_clause.
if not context.segment.is_type("select_clause"):
return None
# Get children of select_clause and the corresponding select keyword.
child_segments = context.functional.segment.children()
select_keyword = child_segments[0]
# See if we have a select_clause_modifier.
select_clause_modifier_seg = child_segments.first(
sp.is_type("select_clause_modifier"))
# Rule doesn't apply if there's no select clause modifier.
if not select_clause_modifier_seg:
return None
select_clause_modifier = select_clause_modifier_seg[0]
# Are there any newlines between the select keyword
# and the select clause modifier.
leading_newline_segments = child_segments.select(
select_if=sp.is_type("newline"),
loop_while=sp.or_(sp.is_whitespace(), sp.is_meta()),
start_seg=select_keyword,
)
# Rule doesn't apply if select clause modifier
# is already on the same line as the select keyword.
if not leading_newline_segments:
return None
# We should also check if the following select clause element
# is on the same line as the select clause modifier.
trailing_newline_segments = child_segments.select(
select_if=sp.is_type("newline"),
loop_while=sp.or_(sp.is_whitespace(), sp.is_meta()),
start_seg=select_clause_modifier,
)
# We will insert these segments directly after the select keyword.
edit_segments = [
WhitespaceSegment(),
select_clause_modifier,
]
if not trailing_newline_segments:
# if the first select clause element is on the same line
# as the select clause modifier then also insert a newline.
edit_segments.append(NewlineSegment())
fixes = []
# Move select clause modifier after select keyword.
fixes.append(
LintFix.create_after(
anchor_segment=select_keyword,
edit_segments=edit_segments,
))
# Delete original newlines between select keyword and select clause modifier
# and delete the original select clause modifier.
fixes.extend((LintFix.delete(s) for s in leading_newline_segments))
fixes.append(LintFix.delete(select_clause_modifier))
# If there is whitespace (on the same line) after the select clause modifier
# then also delete this.
trailing_whitespace_segments = child_segments.select(
select_if=sp.is_whitespace(),
loop_while=sp.or_(sp.is_type("whitespace"), sp.is_meta()),
start_seg=select_clause_modifier,
)
if trailing_whitespace_segments:
fixes.extend(
(LintFix.delete(s) for s in trailing_whitespace_segments))
return LintResult(
anchor=context.segment,
fixes=fixes,
)
