def RetainHorizontalSpacing(self, first_column, depth):
"""Retains a token's horizontal spacing."""
previous = self.previous_token
if not previous:
return
if previous.is_pseudo:
previous = previous.previous_token
if not previous:
return
cur_lineno = self.lineno
prev_lineno = previous.lineno
if previous.is_multiline_string:
prev_lineno += previous.value.count('\n')
if (cur_lineno != prev_lineno
or (previous.is_pseudo and previous.value != ')'
and cur_lineno != previous.previous_token.lineno)):
self.spaces_required_before = (self.column - first_column +
depth * style.Get('INDENT_WIDTH'))
return
cur_column = self.column
prev_column = previous.node.column
prev_len = len(previous.value)
if previous.is_pseudo and previous.value == ')':
prev_column -= 1
prev_len = 0
if previous.is_multiline_string:
prev_len = len(previous.value.split('\n')[-1])
if '\n' in previous.value:
prev_column = 0 # Last line starts in column 0.
self.spaces_required_before = cur_column - (prev_column + prev_len)
def Visit_dictsetmaker(self, node): # pylint: disable=invalid-name
# dictsetmaker ::= (test ':' test (comp_for |
# (',' test ':' test)* [','])) |
# (test (comp_for | (',' test)* [',']))
for child in node.children:
self.Visit(child)
comp_for = False
dict_maker = False
for child in node.children:
if pytree_utils.NodeName(child) == 'comp_for':
comp_for = True
_AppendFirstLeafTokenSubtype(
child, format_token.Subtype.DICT_SET_GENERATOR)
elif pytree_utils.NodeName(child) == 'COLON':
dict_maker = True
if not comp_for and dict_maker:
last_was_colon = False
for child in node.children:
if dict_maker:
if last_was_colon:
if style.Get('INDENT_DICTIONARY_VALUE'):
_InsertPseudoParentheses(child)
else:
_AppendFirstLeafTokenSubtype(
child, format_token.Subtype.DICTIONARY_VALUE)
elif (child is not None
and (isinstance(child, pytree.Node)
or child.value not in '{:,')):
# Mark the first leaf of a key entry as a DICTIONARY_KEY. We
# normally want to split before them if the dictionary cannot exist
# on a single line.
_AppendFirstLeafTokenSubtype(
child, format_token.Subtype.DICTIONARY_KEY)
last_was_colon = pytree_utils.NodeName(child) == 'COLON'
def Visit_dictsetmaker(self, node): # pylint: disable=invalid-name
# dictsetmaker ::= (test ':' test (comp_for |
# (',' test ':' test)* [','])) |
# (test (comp_for | (',' test)* [',']))
dict_maker = False
if len(node.children) > 1:
index = 0
while index < len(node.children):
if node.children[index].type != token.COMMENT:
break
index += 1
if index < len(node.children):
child = node.children[index + 1]
dict_maker = isinstance(child, pytree.Leaf) and child.value == ':'
last_was_colon = False
for child in node.children:
if pytree_utils.NodeName(child) == 'comp_for':
_AppendFirstLeafTokenSubtype(child,
format_token.Subtype.DICT_SET_GENERATOR)
else:
if dict_maker:
if last_was_colon:
if style.Get('INDENT_DICTIONARY_VALUE'):
_InsertPseudoParentheses(child)
else:
_AppendFirstLeafTokenSubtype(
child, format_token.Subtype.DICTIONARY_VALUE)
elif (child is not None and
(isinstance(child, pytree.Node) or child.value not in '{:,')):
# Mark the first leaf of a key entry as a DICTIONARY_KEY. We
# normally want to split before them if the dictionary cannot exist
# on a single line.
_AppendFirstLeafTokenSubtype(child,
format_token.Subtype.DICTIONARY_KEY)
last_was_colon = isinstance(child, pytree.Leaf) and child.value == ':'
self.Visit(child)
def Visit_dictsetmaker(self, node): # pylint: disable=invalid-name
# dictsetmaker ::= (test ':' test (comp_for |
# (',' test ':' test)* [','])) |
# (test (comp_for | (',' test)* [',']))
dict_maker = False
if len(node.children) > 1:
index = 0
while index < len(node.children):
if node.children[index].type != token.COMMENT:
break
index += 1
if index < len(node.children):
child = node.children[index + 1]
dict_maker = isinstance(child, pytree.Leaf) and child.value == ':'
last_was_comma = False
last_was_colon = False
for child in node.children:
if pytree_utils.NodeName(child) == 'comp_for':
self._AppendFirstLeafTokenSubtype(
child, format_token.Subtype.DICT_SET_GENERATOR)
else:
if dict_maker:
if last_was_comma:
self._AppendFirstLeafTokenSubtype(
child, format_token.Subtype.DICTIONARY_KEY)
elif last_was_colon:
if pytree_utils.NodeName(child) == 'power':
self._AppendSubtypeRec(child, format_token.Subtype.NONE)
else:
self._AppendFirstLeafTokenSubtype(
child, format_token.Subtype.DICTIONARY_VALUE)
if style.Get('INDENT_DICTIONARY_VALUE'):
_InsertPseudoParentheses(child)
last_was_comma = isinstance(child, pytree.Leaf) and child.value == ','
last_was_colon = isinstance(child, pytree.Leaf) and child.value == ':'
self.Visit(child)
def assertCodeEqual(self, expected_code, code):
if code != expected_code:
msg = ['Code format mismatch:', 'Expected:']
linelen = style.Get('COLUMN_LIMIT')
for line in expected_code.splitlines():
if len(line) > linelen:
msg.append('!> %s' % line)
else:
msg.append(' > %s' % line)
msg.append('Actual:')
for line in code.splitlines():
if len(line) > linelen:
msg.append('!> %s' % line)
else:
msg.append(' > %s' % line)
msg.append('Diff:')
msg.extend(
difflib.unified_diff(
code.splitlines(),
expected_code.splitlines(),
fromfile='actual',
tofile='expected',
lineterm=''))
self.fail('\n'.join(msg))
def _GetNewlineColumn(self):
"""Return the new column on the newline."""
current = self.next_token
previous = current.previous_token
top_of_stack = self.stack[-1]
if isinstance(current.spaces_required_before, list):
# Don't set the value here, as we need to look at the lines near
# this one to determine the actual horizontal alignment value.
return 0
elif current.spaces_required_before > 2 or self.line.disable:
return current.spaces_required_before
if current.OpensScope():
return top_of_stack.indent if self.paren_level else self.first_indent
if current.ClosesScope():
if (previous.OpensScope() or
(previous.is_comment and previous.previous_token is not None and
previous.previous_token.OpensScope())):
return max(0,
top_of_stack.indent - style.Get('CONTINUATION_INDENT_WIDTH'))
return top_of_stack.closing_scope_indent
if (previous and previous.is_string and current.is_string and
format_token.Subtype.DICTIONARY_VALUE in current.subtypes):
return previous.column
if style.Get('INDENT_DICTIONARY_VALUE'):
if previous and (previous.value == ':' or previous.is_pseudo_paren):
if format_token.Subtype.DICTIONARY_VALUE in current.subtypes:
return top_of_stack.indent
if (_IsCompoundStatement(self.line.first) and
(not style.Get('DEDENT_CLOSING_BRACKETS') or
style.Get('SPLIT_BEFORE_FIRST_ARGUMENT'))):
token_indent = (
len(self.line.first.whitespace_prefix.split('\n')[-1]) +
style.Get('INDENT_WIDTH'))
if token_indent == top_of_stack.indent:
return top_of_stack.indent + style.Get('CONTINUATION_INDENT_WIDTH')
return top_of_stack.indent
def _GetNewlineColumn(self):
"""Return the new column on the newline."""
current = self.next_token
previous = current.previous_token
top_of_stack = self.stack[-1]
if current.spaces_required_before > 2 or self.line.disable:
return current.spaces_required_before
if current.OpensScope():
return top_of_stack.indent if self.paren_level else self.first_indent
if current.ClosesScope():
if (previous.OpensScope() or
(previous.is_comment and previous.previous_token is not None
and previous.previous_token.OpensScope())):
return max(
0, top_of_stack.indent -
style.Get('CONTINUATION_INDENT_WIDTH'))
return top_of_stack.closing_scope_indent
if (previous and previous.is_string and current.is_string
and format_token.Subtype.DICTIONARY_VALUE in current.subtypes):
return previous.column
if style.Get('INDENT_DICTIONARY_VALUE'):
if previous and (previous.value == ':'
or previous.is_pseudo_paren):
if format_token.Subtype.DICTIONARY_VALUE in current.subtypes:
return top_of_stack.indent
if (self.line.first.value in _COMPOUND_STMTS
and (not style.Get('DEDENT_CLOSING_BRACKETS')
or style.Get('SPLIT_BEFORE_FIRST_ARGUMENT'))):
token_indent = (
len(self.line.first.whitespace_prefix.split('\n')[-1]) +
style.Get('INDENT_WIDTH'))
if token_indent == top_of_stack.indent:
return top_of_stack.indent + style.Get(
'CONTINUATION_INDENT_WIDTH')
return top_of_stack.indent
def main(argv):
"""Main program.
Arguments:
argv: command-line arguments, such as sys.argv (including the program name
in argv[0]).
Returns:
0 if there were no changes, non-zero otherwise.
Raises:
YapfError: if none of the supplied files were Python files.
"""
parser = argparse.ArgumentParser(description='Formatter for Python code.')
parser.add_argument(
'-v',
'--version',
action='store_true',
help='show version number and exit')
diff_inplace_group = parser.add_mutually_exclusive_group()
diff_inplace_group.add_argument(
'-d',
'--diff',
action='store_true',
help='print the diff for the fixed source')
diff_inplace_group.add_argument(
'-i',
'--in-place',
action='store_true',
help='make changes to files in place')
lines_recursive_group = parser.add_mutually_exclusive_group()
lines_recursive_group.add_argument(
'-r',
'--recursive',
action='store_true',
help='run recursively over directories')
lines_recursive_group.add_argument(
'-l',
'--lines',
metavar='START-END',
action='append',
default=None,
help='range of lines to reformat, one-based')
parser.add_argument(
'-e',
'--exclude',
metavar='PATTERN',
action='append',
default=None,
help='patterns for files to exclude from formatting')
parser.add_argument(
'--style',
action='store',
help=('specify formatting style: either a style name (for example "pep8" '
'or "google"), or the name of a file with style settings. The '
'default is pep8 unless a %s or %s file located in one of the '
'parent directories of the source file (or current directory for '
'stdin)' % (style.LOCAL_STYLE, style.SETUP_CONFIG)))
parser.add_argument(
'--style-help',
action='store_true',
help=('show style settings and exit; this output can be '
'saved to .style.yapf to make your settings '
'permanent'))
parser.add_argument(
'--no-local-style',
action='store_true',
help="don't search for local style definition")
parser.add_argument('--verify', action='store_true', help=argparse.SUPPRESS)
parser.add_argument(
'-p',
'--parallel',
action='store_true',
help=('Run yapf in parallel when formatting multiple files. Requires '
'concurrent.futures in Python 2.X'))
parser.add_argument('files', nargs='*')
args = parser.parse_args(argv[1:])
if args.version:
print('yapf {}'.format(__version__))
return 0
if args.style_help:
style.SetGlobalStyle(style.CreateStyleFromConfig(args.style))
print('[style]')
for option, docstring in sorted(style.Help().items()):
for line in docstring.splitlines():
print('#', line and ' ' or '', line, sep='')
print(option.lower(), '=', style.Get(option), sep='')
print()
return 0
if args.lines and len(args.files) > 1:
parser.error('cannot use -l/--lines with more than one file')
lines = _GetLines(args.lines) if args.lines is not None else None
if not args.files:
# No arguments specified. Read code from stdin.
if args.in_place or args.diff:
parser.error('cannot use --in-place or --diff flags when reading '
'from stdin')
original_source = []
while True:
try:
# Use 'raw_input' instead of 'sys.stdin.read', because otherwise the
# user will need to hit 'Ctrl-D' more than once if they're inputting
# the program by hand. 'raw_input' throws an EOFError exception if
# 'Ctrl-D' is pressed, which makes it easy to bail out of this loop.
original_source.append(py3compat.raw_input())
except EOFError:
break
style_config = args.style
if style_config is None and not args.no_local_style:
style_config = file_resources.GetDefaultStyleForDir(os.getcwd())
source = [line.rstrip() for line in original_source]
reformatted_source, _ = yapf_api.FormatCode(
py3compat.unicode('\n'.join(source) + '\n'),
filename='<stdin>',
style_config=style_config,
lines=lines,
verify=args.verify)
file_resources.WriteReformattedCode('<stdout>', reformatted_source)
return 0
files = file_resources.GetCommandLineFiles(args.files, args.recursive,
args.exclude)
if not files:
raise errors.YapfError('Input filenames did not match any python files')
FormatFiles(
files,
lines,
style_config=args.style,
no_local_style=args.no_local_style,
in_place=args.in_place,
print_diff=args.diff,
verify=args.verify,
parallel=args.parallel)
return 0
def _AddTokenOnNewline(self, dry_run, must_split):
"""Adds a line break and necessary indentation.
Appends the next token to the state and updates information necessary for
indentation.
Arguments:
dry_run: (bool) Don't commit whitespace changes to the FormatToken if
True.
must_split: (bool) A newline was required before this token.
Returns:
The split penalty for splitting after the current state.
"""
current = self.next_token
previous = current.previous_token
self.column = self._GetNewlineColumn()
if not dry_run:
current.AddWhitespacePrefix(newlines_before=1, spaces=self.column)
if not current.is_comment:
self.stack[-1].last_space = self.column
self.start_of_line_level = self.paren_level
self.lowest_level_on_line = self.paren_level
if (previous.OpensScope()
or (previous.is_comment and previous.previous_token is not None
and previous.previous_token.OpensScope())):
self.stack[-1].closing_scope_indent = max(
0,
self.stack[-1].indent - style.Get('CONTINUATION_INDENT_WIDTH'))
self.stack[-1].split_before_closing_bracket = True
# Calculate the split penalty.
penalty = current.split_penalty
if must_split:
# Don't penalize for a must split.
return penalty
if previous.is_pseudo_paren and previous.value == '(':
# Small penalty for splitting after a pseudo paren.
penalty += 50
# Add a penalty for each increasing newline we add, but don't penalize for
# splitting before an if-expression or list comprehension.
if current.value not in {'if', 'for'}:
last = self.stack[-1]
last.num_line_splits += 1
penalty += (style.Get('SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT') *
last.num_line_splits)
if current.OpensScope() and previous.OpensScope():
# Prefer to keep opening brackets coalesced (unless it's at the beginning
# of a function call).
pprev = previous.previous_token
if not pprev or not pprev.is_name:
penalty += 10
return penalty + 10
def test_positive_case(self):
self.__setup_import_splitter(True)
self.assertTrue(style.Get('FORCE_LONG_LINES_WRAPPING'))
def MustSplit(self):
"""Returns True if the line must split before the next token."""
current = self.next_token
previous = current.previous_token
if current.is_pseudo_paren:
return False
if current.must_break_before:
return True
if not previous:
return False
if self.stack[
-1].split_before_closing_bracket and current.value in '}]':
# Split before the closing bracket if we can.
return current.node_split_penalty != split_penalty.UNBREAKABLE
###########################################################################
# List Splitting
if (style.Get('DEDENT_CLOSING_BRACKETS')
or style.Get('SPLIT_BEFORE_FIRST_ARGUMENT')):
bracket = current if current.ClosesScope() else previous
if format_token.Subtype.SUBSCRIPT_BRACKET not in bracket.subtypes:
if bracket.OpensScope():
if style.Get('COALESCE_BRACKETS'):
if current.OpensScope():
# Prefer to keep all opening brackets together.
return False
if (not _IsLastScopeInLine(bracket)
or unwrapped_line.IsSurroundedByBrackets(bracket)):
last_token = bracket.matching_bracket
else:
last_token = _LastTokenInLine(bracket.matching_bracket)
if not self._FitsOnLine(bracket, last_token):
# Split before the first element if the whole list can't fit on a
# single line.
self.stack[-1].split_before_closing_bracket = True
return True
elif style.Get(
'DEDENT_CLOSING_BRACKETS') and current.ClosesScope():
# Split before and dedent the closing bracket.
return self.stack[-1].split_before_closing_bracket
if (current.is_name or current.is_string) and previous.value == ',':
# If the list has function calls in it and the full list itself cannot
# fit on the line, then we want to split. Otherwise, we'll get something
# like this:
#
# X = [
# Bar(xxx='some string',
# yyy='another long string',
# zzz='a third long string'), Bar(
# xxx='some string',
# yyy='another long string',
# zzz='a third long string')
# ]
#
# or when a string formatting syntax.
func_call_or_string_format = False
if current.is_name:
tok = current.next_token
while tok and (tok.is_name or tok.value == '.'):
tok = tok.next_token
func_call_or_string_format = tok.value == '('
elif current.is_string:
tok = current.next_token
while tok and tok.is_string:
tok = tok.next_token
func_call_or_string_format = tok.value == '%'
if func_call_or_string_format:
open_bracket = unwrapped_line.IsSurroundedByBrackets(current)
if open_bracket and open_bracket.value in '[{':
if not self._FitsOnLine(open_bracket,
open_bracket.matching_bracket):
return True
###########################################################################
# Dict/Set Splitting
if (style.Get('EACH_DICT_ENTRY_ON_SEPARATE_LINE')
and format_token.Subtype.DICTIONARY_KEY in current.subtypes
and not current.is_comment):
# Place each dictionary entry onto its own line.
if previous.value == '{' and previous.previous_token:
opening = _GetOpeningBracket(previous.previous_token)
if (opening and opening.value == '(' and opening.previous_token
and opening.previous_token.is_name):
# This is a dictionary that's an argument to a function.
if self._FitsOnLine(previous, previous.matching_bracket):
return False
return True
if (style.Get('SPLIT_BEFORE_DICT_SET_GENERATOR') and
format_token.Subtype.DICT_SET_GENERATOR in current.subtypes):
# Split before a dict/set generator.
return True
if (format_token.Subtype.DICTIONARY_VALUE in current.subtypes
or (previous.is_pseudo_paren and previous.value == '('
and not current.is_comment)):
# Split before the dictionary value if we can't fit every dictionary
# entry on its own line.
if not current.OpensScope():
opening = _GetOpeningBracket(current)
if not self._EachDictEntryFitsOnOneLine(opening):
return True
if previous.value == '{':
# Split if the dict/set cannot fit on one line and ends in a comma.
closing = previous.matching_bracket
if (not self._FitsOnLine(previous, closing)
and closing.previous_token.value == ','):
self.stack[-1].split_before_closing_bracket = True
return True
###########################################################################
# Argument List Splitting
if (style.Get('SPLIT_BEFORE_NAMED_ASSIGNS') and not current.is_comment
and format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST
in current.subtypes):
if (previous.value not in {'=', ':', '*', '**'}
and current.value not in ':=,)'
and not _IsFunctionDefinition(previous)):
# If we're going to split the lines because of named arguments, then we
# want to split after the opening bracket as well. But not when this is
# part of a function definition.
if previous.value == '(':
# Make sure we don't split after the opening bracket if the
# continuation indent is greater than the opening bracket:
#
# a(
# b=1,
# c=2)
if (self._FitsOnLine(previous, previous.matching_bracket)
and
unwrapped_line.IsSurroundedByBrackets(previous)):
# An argument to a function is a function call with named
# assigns.
return False
column = self.column - self.stack[-1].last_space
return column > style.Get('CONTINUATION_INDENT_WIDTH')
opening = _GetOpeningBracket(current)
if opening:
arglist_length = (opening.matching_bracket.total_length -
opening.total_length +
self.stack[-1].indent)
return arglist_length > self.column_limit
if style.Get('SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED'):
# Split before arguments in a function call or definition if the
# arguments are terminated by a comma.
opening = _GetOpeningBracket(current)
if opening and opening.previous_token and opening.previous_token.is_name:
if previous.value in '(,':
if opening.matching_bracket.previous_token.value == ',':
return True
if ((current.is_name or current.value in {'*', '**'})
and previous.value == ','):
# If we have a function call within an argument list and it won't fit on
# the remaining line, but it will fit on a line by itself, then go ahead
# and split before the call.
opening = _GetOpeningBracket(current)
if (opening and opening.value == '(' and opening.previous_token
and (opening.previous_token.is_name
or opening.previous_token.value in {'*', '**'})):
is_func_call = False
token = current
while token:
if token.value == '(':
is_func_call = True
break
if (not (token.is_name or token.value in {'*', '**'})
and token.value != '.'):
break
token = token.next_token
if is_func_call:
if not self._FitsOnLine(current, opening.matching_bracket):
return True
pprevious = previous.previous_token
if (current.is_name and pprevious and pprevious.is_name
and previous.value == '('):
if (not self._FitsOnLine(previous, previous.matching_bracket)
and _IsFunctionCallWithArguments(current)):
# There is a function call, with more than 1 argument, where the first
# argument is itself a function call with arguments. In this specific
# case, if we split after the first argument's opening '(', then the
# formatting will look bad for the rest of the arguments. E.g.:
#
# outer_function_call(inner_function_call(
# inner_arg1, inner_arg2),
# outer_arg1, outer_arg2)
#
# Instead, enforce a split before that argument to keep things looking
# good.
return True
if (previous.OpensScope() and not current.OpensScope()
and format_token.Subtype.SUBSCRIPT_BRACKET
not in previous.subtypes):
if not current.is_comment:
if pprevious and not pprevious.is_keyword and not pprevious.is_name:
# We want to split if there's a comment in the container.
token = current
while token != previous.matching_bracket:
if token.is_comment:
return True
token = token.next_token
if previous.value == '(':
pptoken = previous.previous_token
if not pptoken or not pptoken.is_name:
# Split after the opening of a tuple if it doesn't fit on the current
# line and it's not a function call.
if self._FitsOnLine(previous, previous.matching_bracket):
return False
elif not self._FitsOnLine(previous, previous.matching_bracket):
if (self.column_limit - self.column) / float(
self.column_limit) < 0.3:
# Try not to squish all of the arguments off to the right.
return current.next_token != previous.matching_bracket
else:
# Split after the opening of a container if it doesn't fit on the
# current line or if it has a comment.
if not self._FitsOnLine(previous, previous.matching_bracket):
return True
###########################################################################
# List Comprehension Splitting
if (format_token.Subtype.COMP_FOR in current.subtypes
and format_token.Subtype.COMP_FOR not in previous.subtypes):
# Split at the beginning of a list comprehension.
length = _GetLengthOfSubtype(current,
format_token.Subtype.COMP_FOR,
format_token.Subtype.COMP_IF)
if length + self.column > self.column_limit:
return True
if (format_token.Subtype.COMP_IF in current.subtypes
and format_token.Subtype.COMP_IF not in previous.subtypes):
# Split at the beginning of an if expression.
length = _GetLengthOfSubtype(current, format_token.Subtype.COMP_IF)
if length + self.column > self.column_limit:
return True
###########################################################################
# Original Formatting Splitting
# These checks rely upon the original formatting. This is in order to
# attempt to keep hand-written code in the same condition as it was before.
# However, this may cause the formatter to fail to be idempotent.
if (style.Get('SPLIT_BEFORE_BITWISE_OPERATOR')
and current.value in '&|'
and previous.lineno < current.lineno):
# Retain the split before a bitwise operator.
return True
if (current.is_comment and
previous.lineno < current.lineno - current.value.count('\n')):
# If a comment comes in the middle of an unwrapped line (like an if
# conditional with comments interspersed), then we want to split if the
# original comments were on a separate line.
return True
return False
def Visit_comp_if(self, node): # pylint: disable=invalid-name
# comp_if ::= 'if' old_test [comp_iter]
_SetSplitPenalty(node.children[0],
style.Get('SPLIT_PENALTY_BEFORE_IF_EXPR'))
_SetStronglyConnected(*node.children[1:])
self.DefaultNodeVisit(node)
def Reformat(uwlines, verify=False, lines=None):
"""Reformat the unwrapped lines.
Arguments:
uwlines: (list of unwrapped_line.UnwrappedLine) Lines we want to format.
verify: (bool) True if reformatted code should be verified for syntax.
lines: (set of int) The lines which can be modified or None if there is no
line range restriction.
Returns:
A string representing the reformatted code.
"""
final_lines = []
prev_uwline = None # The previous line.
indent_width = style.Get('INDENT_WIDTH')
for uwline in _SingleOrMergedLines(uwlines):
if style.Get('PRESERVE_DICTIONARY_FORMATTING'):
for tok in uwline.tokens:
if tok.value == '{':
uwline.disable = True
first_token = uwline.first
_FormatFirstToken(first_token, uwline.depth, prev_uwline, final_lines)
indent_amt = indent_width * uwline.depth
state = format_decision_state.FormatDecisionState(uwline, indent_amt)
state.MoveStateToNextToken()
if not uwline.disable:
if uwline.first.is_comment:
uwline.first.node.value = uwline.first.node.value.rstrip()
elif uwline.last.is_comment:
uwline.last.node.value = uwline.last.node.value.rstrip()
if prev_uwline and prev_uwline.disable:
# Keep the vertical spacing between a disabled and enabled formatting
# region.
_RetainRequiredVerticalSpacingBetweenTokens(
uwline.first, prev_uwline.last, lines)
if any(tok.is_comment for tok in uwline.tokens):
_RetainVerticalSpacingBeforeComments(uwline)
if (_LineContainsI18n(uwline) or uwline.disable
or _LineHasContinuationMarkers(uwline)):
_RetainHorizontalSpacing(uwline)
_RetainRequiredVerticalSpacing(uwline, prev_uwline, lines)
_EmitLineUnformatted(state)
elif _CanPlaceOnSingleLine(uwline) and not any(
tok.must_split for tok in uwline.tokens):
# The unwrapped line fits on one line.
while state.next_token:
state.AddTokenToState(newline=False, dry_run=False)
else:
if not _AnalyzeSolutionSpace(state):
# Failsafe mode. If there isn't a solution to the line, then just emit
# it as is.
state = format_decision_state.FormatDecisionState(
uwline, indent_amt)
state.MoveStateToNextToken()
_RetainHorizontalSpacing(uwline)
_RetainRequiredVerticalSpacing(uwline, prev_uwline, None)
_EmitLineUnformatted(state)
final_lines.append(uwline)
prev_uwline = uwline
return _FormatFinalLines(final_lines, verify)
def test_disabled(self):
self.__setup(False)
self.assertFalse(style.Get('DISABLE_ALL_WARNINGS'))
def _SpaceRequiredBetween(left, right):
"""Return True if a space is required between the left and right token."""
lval = left.value
rval = right.value
if (left.is_pseudo_paren and _IsIdNumberStringToken(right)
and left.previous_token
and _IsIdNumberStringToken(left.previous_token)):
# Space between keyword... tokens and pseudo parens.
return True
if left.is_pseudo_paren or right.is_pseudo_paren:
# There should be a space after the ':' in a dictionary.
if left.OpensScope():
return True
# The closing pseudo-paren shouldn't affect spacing.
return False
if left.is_continuation or right.is_continuation:
# The continuation node's value has all of the spaces it needs.
return False
if right.name in pytree_utils.NONSEMANTIC_TOKENS:
# No space before a non-semantic token.
return False
if _IsIdNumberStringToken(left) and _IsIdNumberStringToken(right):
# Spaces between keyword, string, number, and identifier tokens.
return True
if lval == ',' and rval == ':':
# We do want a space between a comma and colon.
return True
if rval in ':,':
# Otherwise, we never want a space before a colon or comma.
return False
if lval == ',' and rval in ']})':
# Add a space between ending ',' and closing bracket if requested.
return style.Get('SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET')
if lval == ',':
# We want a space after a comma.
return True
if lval == 'from' and rval == '.':
# Space before the '.' in an import statement.
return True
if lval == '.' and rval == 'import':
# Space after the '.' in an import statement.
return True
if lval == '=' and rval == '.':
# Space between equal and '.' as in "X = ...".
return True
if ((right.is_keyword or right.is_name)
and (left.is_keyword or left.is_name)):
# Don't merge two keywords/identifiers.
return True
if (format_token.Subtype.SUBSCRIPT_COLON in left.subtypes
or format_token.Subtype.SUBSCRIPT_COLON in right.subtypes):
# A subscript shouldn't have spaces separating its colons.
return False
if (format_token.Subtype.TYPED_NAME in left.subtypes
or format_token.Subtype.TYPED_NAME in right.subtypes):
# A typed argument should have a space after the colon.
return True
if left.is_string:
if (rval == '='
and format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST
in right.subtypes):
# If there is a type hint, then we don't want to add a space between the
# equal sign and the hint.
return False
if rval not in '[)]}.':
# A string followed by something other than a subscript, closing bracket,
# or dot should have a space after it.
return True
if left.is_binary_op and lval != '**' and _IsUnaryOperator(right):
# Space between the binary opertor and the unary operator.
return True
if left.is_keyword and _IsUnaryOperator(right):
# Handle things like "not -3 < x".
return True
if _IsUnaryOperator(left) and _IsUnaryOperator(right):
# No space between two unary operators.
return False
if left.is_binary_op or right.is_binary_op:
if lval == '**' or rval == '**':
# Space around the "power" operator.
return style.Get('SPACES_AROUND_POWER_OPERATOR')
# Enforce spaces around binary operators except the blacklisted ones.
blacklist = style.Get('NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS')
return lval not in blacklist and rval not in blacklist
if (_IsUnaryOperator(left) and lval != 'not'
and (right.is_name or right.is_number or rval == '(')):
# The previous token was a unary op. No space is desired between it and
# the current token.
return False
if (format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN in left.subtypes
or format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN in right.subtypes):
# A named argument or default parameter shouldn't have spaces around it.
return style.Get('SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN')
if (format_token.Subtype.VARARGS_LIST in left.subtypes
or format_token.Subtype.VARARGS_LIST in right.subtypes):
return False
if (format_token.Subtype.VARARGS_STAR in left.subtypes
or format_token.Subtype.KWARGS_STAR_STAR in left.subtypes):
# Don't add a space after a vararg's star or a keyword's star-star.
return False
if lval == '@' and format_token.Subtype.DECORATOR in left.subtypes:
# Decorators shouldn't be separated from the 'at' sign.
return False
if lval == '.' or rval == '.':
# Don't place spaces between dots.
return False
if ((lval == '(' and rval == ')') or (lval == '[' and rval == ']')
or (lval == '{' and rval == '}')):
# Empty objects shouldn't be separted by spaces.
return False
if (lval in pytree_utils.OPENING_BRACKETS
and rval in pytree_utils.OPENING_BRACKETS):
# Nested objects' opening brackets shouldn't be separated.
return False
if (lval in pytree_utils.CLOSING_BRACKETS
and rval in pytree_utils.CLOSING_BRACKETS):
# Nested objects' closing brackets shouldn't be separated.
return False
if lval in pytree_utils.CLOSING_BRACKETS and rval in '([':
# A call, set, dictionary, or subscript that has a call or subscript after
# it shouldn't have a space between them.
return False
if lval in pytree_utils.OPENING_BRACKETS and _IsIdNumberStringToken(right):
# Don't separate the opening bracket from the first item.
return False
if left.is_name and rval in '([':
# Don't separate a call or array access from the name.
return False
if rval in pytree_utils.CLOSING_BRACKETS:
# Don't separate the closing bracket from the last item.
# FIXME(morbo): This might be too permissive.
return False
if lval == 'print' and rval == '(':
# Special support for the 'print' function.
return False
if lval in pytree_utils.OPENING_BRACKETS and _IsUnaryOperator(right):
# Don't separate a unary operator from the opening bracket.
return False
if (lval in pytree_utils.OPENING_BRACKETS
and (format_token.Subtype.VARARGS_STAR in right.subtypes
or format_token.Subtype.KWARGS_STAR_STAR in right.subtypes)):
# Don't separate a '*' or '**' from the opening bracket.
return False
if rval == ';':
# Avoid spaces before a semicolon. (Why is there a semicolon?!)
return False
if lval == '(' and rval == 'await':
# Special support for the 'await' keyword. Don't separate the 'await'
# keyword from an opening paren.
return False
return True
def Reformat(llines, verify=False, lines=None):
"""Reformat the logical lines.
Arguments:
llines: (list of logical_line.LogicalLine) Lines we want to format.
verify: (bool) True if reformatted code should be verified for syntax.
lines: (set of int) The lines which can be modified or None if there is no
line range restriction.
Returns:
A string representing the reformatted code.
"""
final_lines = []
prev_line = None # The previous line.
indent_width = style.Get('INDENT_WIDTH')
for lline in _SingleOrMergedLines(llines):
first_token = lline.first
_FormatFirstToken(first_token, lline.depth, prev_line, final_lines)
indent_amt = indent_width * lline.depth
state = format_decision_state.FormatDecisionState(lline, indent_amt)
state.MoveStateToNextToken()
if not lline.disable:
if lline.first.is_comment:
lline.first.node.value = lline.first.node.value.rstrip()
elif lline.last.is_comment:
lline.last.node.value = lline.last.node.value.rstrip()
if prev_line and prev_line.disable:
# Keep the vertical spacing between a disabled and enabled formatting
# region.
_RetainRequiredVerticalSpacingBetweenTokens(
lline.first, prev_line.last, lines)
if any(tok.is_comment for tok in lline.tokens):
_RetainVerticalSpacingBeforeComments(lline)
if lline.disable or _LineHasContinuationMarkers(lline):
_RetainHorizontalSpacing(lline)
_RetainRequiredVerticalSpacing(lline, prev_line, lines)
_EmitLineUnformatted(state)
elif (_LineContainsPylintDisableLineTooLong(lline)
or _LineContainsI18n(lline)):
# Don't modify vertical spacing, but fix any horizontal spacing issues.
_RetainRequiredVerticalSpacing(lline, prev_line, lines)
_EmitLineUnformatted(state)
elif _CanPlaceOnSingleLine(lline) and not any(tok.must_break_before
for tok in lline.tokens):
# The logical line fits on one line.
while state.next_token:
state.AddTokenToState(newline=False, dry_run=False)
elif not _AnalyzeSolutionSpace(state):
# Failsafe mode. If there isn't a solution to the line, then just emit
# it as is.
state = format_decision_state.FormatDecisionState(
lline, indent_amt)
state.MoveStateToNextToken()
_RetainHorizontalSpacing(lline)
_RetainRequiredVerticalSpacing(lline, prev_line, None)
_EmitLineUnformatted(state)
final_lines.append(lline)
prev_line = lline
_AlignTrailingComments(final_lines)
return _FormatFinalLines(final_lines, verify)
def test_enabled(self):
self.__setup(True)
self.assertTrue(style.Get('DISABLE_ALL_WARNINGS'))
def test_negative_case(self):
self.__setup_import_splitter(False)
self.assertFalse(style.Get('WARN_NOT_COMMENTED_GLOBAL_VARS'))
def test_positive_case(self):
self.__setup_import_splitter(True)
self.assertTrue(style.Get('WARN_NOT_COMMENTED_GLOBAL_VARS'))
def test_negative_case(self):
self.__setup_import_splitter(False)
self.assertFalse(style.Get('FORCE_LONG_LINES_WRAPPING'))
def Visit_trailer(self, node): # pylint: disable=invalid-name
# trailer ::= '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
if node.children[0].value == '.':
before = style.Get('SPLIT_BEFORE_DOT')
_SetSplitPenalty(
node.children[0],
VERY_STRONGLY_CONNECTED if before else DOTTED_NAME)
_SetSplitPenalty(
node.children[1],
DOTTED_NAME if before else VERY_STRONGLY_CONNECTED)
elif len(node.children) == 2:
# Don't split an empty argument list if at all possible.
_SetSplitPenalty(node.children[1], VERY_STRONGLY_CONNECTED)
elif len(node.children) == 3:
name = pytree_utils.NodeName(node.children[1])
if name in {'argument', 'comparison'}:
# Don't split an argument list with one element if at all possible.
_SetStronglyConnected(node.children[1])
if (len(node.children[1].children) > 1
and pytree_utils.NodeName(
node.children[1].children[1]) == 'comp_for'):
# Don't penalize splitting before a comp_for expression.
_SetSplitPenalty(
pytree_utils.FirstLeafNode(node.children[1]), 0)
else:
_SetSplitPenalty(
pytree_utils.FirstLeafNode(node.children[1]),
ONE_ELEMENT_ARGUMENT)
elif (node.children[0].type == grammar_token.LSQB
and len(node.children[1].children) > 2
and (name.endswith('_test') or name.endswith('_expr'))):
_SetStronglyConnected(node.children[1].children[0])
_SetStronglyConnected(node.children[1].children[2])
# Still allow splitting around the operator.
split_before = (
(name.endswith('_test')
and style.Get('SPLIT_BEFORE_LOGICAL_OPERATOR'))
or (name.endswith('_expr')
and style.Get('SPLIT_BEFORE_BITWISE_OPERATOR')))
if split_before:
_SetSplitPenalty(
pytree_utils.LastLeafNode(
node.children[1].children[1]), 0)
else:
_SetSplitPenalty(
pytree_utils.FirstLeafNode(
node.children[1].children[2]), 0)
# Don't split the ending bracket of a subscript list.
_RecAnnotate(node.children[-1],
pytree_utils.Annotation.SPLIT_PENALTY,
VERY_STRONGLY_CONNECTED)
elif name not in {
'arglist', 'argument', 'term', 'or_test', 'and_test',
'comparison', 'atom', 'power'
}:
# Don't split an argument list with one element if at all possible.
stypes = pytree_utils.GetNodeAnnotation(
pytree_utils.FirstLeafNode(node),
pytree_utils.Annotation.SUBTYPE)
if stypes and subtypes.SUBSCRIPT_BRACKET in stypes:
_IncreasePenalty(node, SUBSCRIPT)
# Bump up the split penalty for the first part of a subscript. We
# would rather not split there.
_IncreasePenalty(node.children[1], CONNECTED)
else:
_SetStronglyConnected(node.children[1], node.children[2])
if name == 'arglist':
_SetStronglyConnected(node.children[-1])
self.DefaultNodeVisit(node)
def test_enabled(self):
self.__setup(True)
self.assertTrue(style.Get('CHECK_SCRIPT_CODE_ENCAPSULATION'))
def Visit_power(self, node): # pylint: disable=invalid-name,missing-docstring
# power ::= atom trailer* ['**' factor]
self.DefaultNodeVisit(node)
# When atom is followed by a trailer, we can not break between them.
# E.g. arr[idx] - no break allowed between 'arr' and '['.
if (len(node.children) > 1
and pytree_utils.NodeName(node.children[1]) == 'trailer'):
# children[1] itself is a whole trailer: we don't want to
# mark all of it as unbreakable, only its first token: (, [ or .
first = pytree_utils.FirstLeafNode(node.children[1])
if first.value != '.':
_SetUnbreakable(node.children[1].children[0])
# A special case when there are more trailers in the sequence. Given:
# atom tr1 tr2
# The last token of tr1 and the first token of tr2 comprise an unbreakable
# region. For example: foo.bar.baz(1)
# We can't put breaks between either of the '.', '(', or '[' and the names
# *preceding* them.
prev_trailer_idx = 1
while prev_trailer_idx < len(node.children) - 1:
cur_trailer_idx = prev_trailer_idx + 1
cur_trailer = node.children[cur_trailer_idx]
if pytree_utils.NodeName(cur_trailer) != 'trailer':
break
# Now we know we have two trailers one after the other
prev_trailer = node.children[prev_trailer_idx]
if prev_trailer.children[-1].value != ')':
# Set the previous node unbreakable if it's not a function call:
# atom tr1() tr2
# It may be necessary (though undesirable) to split up a previous
# function call's parentheses to the next line.
_SetStronglyConnected(prev_trailer.children[-1])
_SetStronglyConnected(cur_trailer.children[0])
prev_trailer_idx = cur_trailer_idx
# We don't want to split before the last ')' of a function call. This also
# takes care of the special case of:
# atom tr1 tr2 ... trn
# where the 'tr#' are trailers that may end in a ')'.
for trailer in node.children[1:]:
if pytree_utils.NodeName(trailer) != 'trailer':
break
if trailer.children[0].value in '([':
if len(trailer.children) > 2:
stypes = pytree_utils.GetNodeAnnotation(
trailer.children[0], pytree_utils.Annotation.SUBTYPE)
if stypes and subtypes.SUBSCRIPT_BRACKET in stypes:
_SetStronglyConnected(
pytree_utils.FirstLeafNode(trailer.children[1]))
last_child_node = pytree_utils.LastLeafNode(trailer)
if last_child_node.value.strip().startswith('#'):
last_child_node = last_child_node.prev_sibling
if not (style.Get('INDENT_CLOSING_BRACKETS')
or style.Get('DEDENT_CLOSING_BRACKETS')):
last = pytree_utils.LastLeafNode(
last_child_node.prev_sibling)
if last.value != ',':
if last_child_node.value == ']':
_SetUnbreakable(last_child_node)
else:
_SetSplitPenalty(last_child_node,
VERY_STRONGLY_CONNECTED)
else:
# If the trailer's children are '()', then make it a strongly
# connected region. It's sometimes necessary, though undesirable, to
# split the two.
_SetStronglyConnected(trailer.children[-1])
def Reformat(uwlines, verify=True):
"""Reformat the unwrapped lines.
Arguments:
uwlines: (list of unwrapped_line.UnwrappedLine) Lines we want to format.
verify: (bool) True if reformatted code should be verified for syntax.
Returns:
A string representing the reformatted code.
"""
final_lines = []
prev_uwline = None # The previous line.
for uwline in _SingleOrMergedLines(uwlines):
first_token = uwline.first
_FormatFirstToken(first_token, uwline.depth, prev_uwline)
indent_amt = style.Get('INDENT_WIDTH') * uwline.depth
state = format_decision_state.FormatDecisionState(uwline, indent_amt)
if not uwline.disable:
if uwline.first.is_comment:
uwline.first.node.value = uwline.first.node.value.rstrip()
elif uwline.last.is_comment:
uwline.last.node.value = uwline.last.node.value.rstrip()
if prev_uwline and prev_uwline.disable:
# Keep the vertical spacing between a disabled and enabled formatting
# region.
_RetainVerticalSpacingBetweenTokens(uwline.first, prev_uwline.last)
if any(tok.is_comment for tok in uwline.tokens):
_RetainVerticalSpacingBeforeComments(uwline)
if (_LineContainsI18n(uwline) or uwline.disable or
_LineHasContinuationMarkers(uwline)):
_RetainHorizontalSpacing(uwline)
_RetainVerticalSpacing(uwline, prev_uwline)
_EmitLineUnformatted(state)
elif _CanPlaceOnSingleLine(uwline):
# The unwrapped line fits on one line.
while state.next_token:
state.AddTokenToState(newline=False, dry_run=False)
else:
if not _AnalyzeSolutionSpace(state, dry_run=False):
# Failsafe mode. If there isn't a solution to the line, then just emit
# it as is.
state = format_decision_state.FormatDecisionState(uwline, indent_amt)
_RetainHorizontalSpacing(uwline)
_RetainVerticalSpacing(uwline, prev_uwline)
_EmitLineUnformatted(state)
final_lines.append(uwline)
prev_uwline = uwline
formatted_code = []
for line in final_lines:
formatted_line = []
for tok in line.tokens:
if not tok.is_pseudo_paren:
formatted_line.append(tok.whitespace_prefix)
formatted_line.append(tok.value)
else:
if (not tok.next_token.whitespace_prefix.startswith('\n') and
not tok.next_token.whitespace_prefix.startswith(' ')):
if (tok.previous_token.value == ':' or
tok.next_token.value not in ',}])'):
formatted_line.append(' ')
formatted_code.append(''.join(formatted_line))
if verify:
verifier.VerifyCode(formatted_code[-1])
return ''.join(formatted_code) + '\n'
def _SpaceRequiredBetween(left, right):
"""Return True if a space is required between the left and right token."""
if left.is_continuation or right.is_continuation:
# The continuation node's value has all of the spaces it needs.
return False
if right.name in pytree_utils.NONSEMANTIC_TOKENS:
# No space before a non-semantic token.
return False
if _IsIdNumberStringToken(left) and _IsIdNumberStringToken(right):
# Spaces between keyword, string, number, and identifier tokens.
return True
if right.value in ':,':
# We never want a space before a colon or comma.
return False
if left.value == ',' and right.value in ']})':
# Add a space between ending ',' and closing bracket if requested.
return style.Get('SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET')
if left.value == ',':
# We want a space after a comma.
return True
if left.value == 'from' and right.value == '.':
# Space before the '.' in an import statement.
return True
if left.value == '.' and right.value == 'import':
# Space after the '.' in an import statement.
return True
if ((right.is_keyword or right.is_name)
and (left.is_keyword or left.is_name)):
# Don't merge two keywords/identifiers.
return True
if left.is_string and right.value not in '[)]}.':
# A string followed by something other than a subscript, closing bracket,
# or dot should have a space after it.
return True
if _IsBinaryOperator(left) and _IsUnaryOperator(right):
# Space between the binary opertor and the unary operator.
return True
if _IsUnaryOperator(left) and _IsUnaryOperator(right):
# No space between two unary operators.
return False
if _IsBinaryOperator(left) or _IsBinaryOperator(right):
# Enforce spaces around binary operators.
return True
if (_IsUnaryOperator(left) and left.value != 'not'
and (right.is_name or right.is_number or right.value == '(')):
# The previous token was a unary op. No space is desired between it and
# the current token.
return False
if (left.subtype == format_token.Subtype.SUBSCRIPT_COLON
or right.subtype == format_token.Subtype.SUBSCRIPT_COLON):
# A subscript shouldn't have spaces separating its colons.
return False
if (left.subtype == format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN
or right.subtype == format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN):
# A named argument or default parameter shouldn't have spaces around it.
return False
if left.subtype in {
format_token.Subtype.VARARGS_STAR,
format_token.Subtype.KWARGS_STAR_STAR
}:
# Don't add a space after a vararg's star or a keyword's star-star.
return False
if left.value == '@':
# Decorators shouldn't be separated from the 'at' sign.
return False
if left.value == '.' or right.value == '.':
# Don't place spaces between dots.
return False
if ((left.value == '(' and right.value == ')')
or (left.value == '[' and right.value == ']')
or (left.value == '{' and right.value == '}')):
# Empty objects shouldn't be separted by spaces.
return False
if (left.value in pytree_utils.OPENING_BRACKETS
and right.value in pytree_utils.OPENING_BRACKETS):
# Nested objects' opening brackets shouldn't be separated.
return False
if (left.value in pytree_utils.CLOSING_BRACKETS
and right.value in pytree_utils.CLOSING_BRACKETS):
# Nested objects' closing brackets shouldn't be separated.
return False
if left.value in pytree_utils.CLOSING_BRACKETS and right.value in '([':
# A call, set, dictionary, or subscript that has a call or subscript after
# it shouldn't have a space between them.
return False
if (left.value in pytree_utils.OPENING_BRACKETS
and _IsIdNumberStringToken(right)):
# Don't separate the opening bracket from the first item.
return False
if left.is_name and right.value in '([':
# Don't separate a call or array access from the name.
return False
if right.value in pytree_utils.CLOSING_BRACKETS:
# Don't separate the closing bracket from the last item.
# FIXME(morbo): This might be too permissive.
return False
if left.value == 'print' and right.value == '(':
# Special support for the 'print' function.
return False
if left.value in pytree_utils.OPENING_BRACKETS and _IsUnaryOperator(right):
# Don't separate a unary operator from the opening bracket.
return False
if (left.value in pytree_utils.OPENING_BRACKETS and right.subtype in {
format_token.Subtype.VARARGS_STAR,
format_token.Subtype.KWARGS_STAR_STAR
}):
# Don't separate a '*' or '**' from the opening bracket.
return False
if right.value == ';':
# Avoid spaces before a semicolon. (Why is there a semicolon?!)
return False
return True
def _CalculateNumberOfNewlines(first_token, indent_depth, prev_uwline):
"""Calculate the number of newlines we need to add.
Arguments:
first_token: (format_token.FormatToken) The first token in the unwrapped
line.
indent_depth: (int) The line's indentation depth.
prev_uwline: (list of unwrapped_line.UnwrappedLine) The unwrapped line
previous to this line.
Returns:
The number of newlines needed before the first token.
"""
# TODO(morbo): Special handling for imports.
# TODO(morbo): Create a knob that can tune these.
if prev_uwline is None:
# The first line in the file. Don't add blank lines.
# FIXME(morbo): Is this correct?
if first_token.newlines is not None:
pytree_utils.SetNodeAnnotation(first_token.GetPytreeNode(),
pytree_utils.Annotation.NEWLINES, None)
return 0
if first_token.is_docstring:
# The docstring shouldn't have a newline before it.
# TODO(morbo): Add a knob to adjust this.
return NO_BLANK_LINES
prev_last_token = prev_uwline.last
if prev_last_token.is_docstring:
if not indent_depth and first_token.value in {'class', 'def'}:
# Separate a class or function from the module-level docstring with two
# blank lines.
return TWO_BLANK_LINES
if _NoBlankLinesBeforeCurrentToken(prev_last_token.value, first_token,
prev_last_token):
return NO_BLANK_LINES
else:
return ONE_BLANK_LINE
if first_token.value in {'class', 'def', '@'}:
# TODO(morbo): This can go once the blank line calculator is more
# sophisticated.
if not indent_depth:
# This is a top-level class or function.
is_inline_comment = prev_last_token.whitespace_prefix.count('\n') == 0
if prev_last_token.is_comment and not is_inline_comment:
# This token follows a non-inline comment.
if _NoBlankLinesBeforeCurrentToken(prev_last_token.value, first_token,
prev_last_token):
# Assume that the comment is "attached" to the current line.
# Therefore, we want two blank lines before the comment.
prev_last_token.AdjustNewlinesBefore(TWO_BLANK_LINES)
if first_token.newlines is not None:
pytree_utils.SetNodeAnnotation(first_token.GetPytreeNode(),
pytree_utils.Annotation.NEWLINES,
None)
return NO_BLANK_LINES
elif prev_uwline.first.value in {'class', 'def'}:
if not style.Get('BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF'):
pytree_utils.SetNodeAnnotation(first_token.node,
pytree_utils.Annotation.NEWLINES, None)
return NO_BLANK_LINES
# Calculate how many newlines were between the original lines. We want to
# retain that formatting if it doesn't violate one of the style guide rules.
if first_token.is_comment:
first_token_lineno = first_token.lineno - first_token.value.count('\n')
else:
first_token_lineno = first_token.lineno
if first_token_lineno - prev_last_token.lineno > 1:
return ONE_BLANK_LINE
return NO_BLANK_LINES
def _CalculateNumberOfNewlines(first_token, indent_depth, prev_uwline,
final_lines):
"""Calculate the number of newlines we need to add.
Arguments:
first_token: (format_token.FormatToken) The first token in the unwrapped
line.
indent_depth: (int) The line's indentation depth.
prev_uwline: (list of unwrapped_line.UnwrappedLine) The unwrapped line
previous to this line.
final_lines: (list of unwrapped_line.UnwrappedLine) The unwrapped lines
that have already been processed.
Returns:
The number of newlines needed before the first token.
"""
# TODO(morbo): Special handling for imports.
# TODO(morbo): Create a knob that can tune these.
if prev_uwline is None:
# The first line in the file. Don't add blank lines.
# FIXME(morbo): Is this correct?
if first_token.newlines is not None:
pytree_utils.SetNodeAnnotation(first_token.node,
pytree_utils.Annotation.NEWLINES,
None)
return 0
if first_token.is_docstring:
if (prev_uwline.first.value == 'class'
and style.Get('BLANK_LINE_BEFORE_CLASS_DOCSTRING')):
# Enforce a blank line before a class's docstring.
return ONE_BLANK_LINE
elif (prev_uwline.first.value.startswith('#')
and style.Get('BLANK_LINE_BEFORE_MODULE_DOCSTRING')):
# Enforce a blank line before a module's docstring.
return ONE_BLANK_LINE
# The docstring shouldn't have a newline before it.
return NO_BLANK_LINES
prev_last_token = prev_uwline.last
if prev_last_token.is_docstring:
if (not indent_depth
and first_token.value in {'class', 'def', 'async'}):
# Separate a class or function from the module-level docstring with
# appropriate number of blank lines.
return 1 + style.Get('BLANK_LINES_AROUND_TOP_LEVEL_DEFINITION')
if _NoBlankLinesBeforeCurrentToken(prev_last_token.value, first_token,
prev_last_token):
return NO_BLANK_LINES
else:
return ONE_BLANK_LINE
if first_token.value in {'class', 'def', 'async', '@'}:
# TODO(morbo): This can go once the blank line calculator is more
# sophisticated.
if not indent_depth:
# This is a top-level class or function.
is_inline_comment = prev_last_token.whitespace_prefix.count(
'\n') == 0
if (not prev_uwline.disable and prev_last_token.is_comment
and not is_inline_comment):
# This token follows a non-inline comment.
if _NoBlankLinesBeforeCurrentToken(prev_last_token.value,
first_token,
prev_last_token):
# Assume that the comment is "attached" to the current line.
# Therefore, we want two blank lines before the comment.
index = len(final_lines) - 1
while index > 0:
if not final_lines[index - 1].is_comment:
break
index -= 1
if final_lines[index - 1].first.value == '@':
final_lines[index].first.AdjustNewlinesBefore(
NO_BLANK_LINES)
else:
prev_last_token.AdjustNewlinesBefore(1 + style.Get(
'BLANK_LINES_AROUND_TOP_LEVEL_DEFINITION'))
if first_token.newlines is not None:
pytree_utils.SetNodeAnnotation(
first_token.node, pytree_utils.Annotation.NEWLINES,
None)
return NO_BLANK_LINES
elif _IsClassOrDef(prev_uwline):
if not style.Get('BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF'):
pytree_utils.SetNodeAnnotation(
first_token.node, pytree_utils.Annotation.NEWLINES, None)
return NO_BLANK_LINES
# Calculate how many newlines were between the original lines. We want to
# retain that formatting if it doesn't violate one of the style guide rules.
if first_token.is_comment:
first_token_lineno = first_token.lineno - first_token.value.count('\n')
else:
first_token_lineno = first_token.lineno
prev_last_token_lineno = prev_last_token.lineno
if prev_last_token.is_multiline_string:
prev_last_token_lineno += prev_last_token.value.count('\n')
if first_token_lineno - prev_last_token_lineno > 1:
return ONE_BLANK_LINE
return NO_BLANK_LINES
def test_disabled(self):
self.__setup(False)
self.assertFalse(style.Get('CHECK_SCRIPT_CODE_ENCAPSULATION'))
def _SplitPenalty(prev_token, cur_token):
"""Return the penalty for breaking the line before the current token."""
pval = prev_token.value
cval = cur_token.value
if pval == 'not':
return split_penalty.UNBREAKABLE
if cur_token.node_split_penalty > 0:
return cur_token.node_split_penalty
if style.Get('SPLIT_BEFORE_LOGICAL_OPERATOR'):
# Prefer to split before 'and' and 'or'.
if pval in _LOGICAL_OPERATORS:
return style.Get('SPLIT_PENALTY_LOGICAL_OPERATOR')
if cval in _LOGICAL_OPERATORS:
return 0
else:
# Prefer to split after 'and' and 'or'.
if pval in _LOGICAL_OPERATORS:
return 0
if cval in _LOGICAL_OPERATORS:
return style.Get('SPLIT_PENALTY_LOGICAL_OPERATOR')
if style.Get('SPLIT_BEFORE_BITWISE_OPERATOR'):
# Prefer to split before '&', '|', and '^'.
if pval in _BITWISE_OPERATORS:
return style.Get('SPLIT_PENALTY_BITWISE_OPERATOR')
if cval in _BITWISE_OPERATORS:
return 0
else:
# Prefer to split after '&', '|', and '^'.
if pval in _BITWISE_OPERATORS:
return 0
if cval in _BITWISE_OPERATORS:
return style.Get('SPLIT_PENALTY_BITWISE_OPERATOR')
if (format_token.Subtype.COMP_FOR in cur_token.subtypes
or format_token.Subtype.COMP_IF in cur_token.subtypes):
# We don't mind breaking before the 'for' or 'if' of a list comprehension.
return 0
if format_token.Subtype.UNARY_OPERATOR in prev_token.subtypes:
# Try not to break after a unary operator.
return style.Get('SPLIT_PENALTY_AFTER_UNARY_OPERATOR')
if pval == ',':
# Breaking after a comma is fine, if need be.
return 0
if prev_token.is_binary_op:
# We would rather not split after an equality operator.
return 20
if (format_token.Subtype.VARARGS_STAR in prev_token.subtypes
or format_token.Subtype.KWARGS_STAR_STAR in prev_token.subtypes):
# Don't split after a varargs * or kwargs **.
return split_penalty.UNBREAKABLE
if prev_token.OpensScope() and cval != '(':
# Slightly prefer
return style.Get('SPLIT_PENALTY_AFTER_OPENING_BRACKET')
if cval == ':':
# Don't split before a colon.
return split_penalty.UNBREAKABLE
if cval == '=':
# Don't split before an assignment.
return split_penalty.UNBREAKABLE
if (format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN in prev_token.subtypes
or format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN
in cur_token.subtypes):
# Don't break before or after an default or named assignment.
return split_penalty.UNBREAKABLE
if cval == '==':
# We would rather not split before an equality operator.
return split_penalty.STRONGLY_CONNECTED
if cur_token.ClosesScope():
# Give a slight penalty for splitting before the closing scope.
return 100
if pval in _TERM_OPERATORS or cval in _TERM_OPERATORS:
return 50
return 0
def _CalculateComprehensionState(self, newline):
"""Makes required changes to comprehension state.
Args:
newline: Whether the current token is to be added on a newline.
Returns:
The penalty for the token-newline combination given the current
comprehension state.
"""
current = self.next_token
previous = current.previous_token
top_of_stack = self.comp_stack[-1] if self.comp_stack else None
penalty = 0
if top_of_stack is not None:
# Check if the token terminates the current comprehension.
if current == top_of_stack.closing_bracket:
last = self.comp_stack.pop()
# Lightly penalize comprehensions that are split across multiple lines.
if last.has_interior_split:
penalty += style.Get('SPLIT_PENALTY_COMPREHENSION')
return penalty
if newline:
top_of_stack.has_interior_split = True
if (format_token.Subtype.COMP_EXPR in current.subtypes
and format_token.Subtype.COMP_EXPR not in previous.subtypes):
self.comp_stack.append(object_state.ComprehensionState(current))
return penalty
if (current.value == 'for'
and format_token.Subtype.COMP_FOR in current.subtypes):
if top_of_stack.for_token is not None:
# Treat nested comprehensions like normal comp_if expressions.
# Example:
# my_comp = [
# a.qux + b.qux
# for a in foo
# --> for b in bar <--
# if a.zut + b.zut
# ]
if (style.Get('SPLIT_COMPLEX_COMPREHENSION')
and top_of_stack.has_split_at_for != newline
and (top_of_stack.has_split_at_for
or not top_of_stack.HasTrivialExpr())):
penalty += split_penalty.UNBREAKABLE
else:
top_of_stack.for_token = current
top_of_stack.has_split_at_for = newline
# Try to keep trivial expressions on the same line as the comp_for.
if (style.Get('SPLIT_COMPLEX_COMPREHENSION') and newline
and top_of_stack.HasTrivialExpr()):
penalty += split_penalty.CONNECTED
if (format_token.Subtype.COMP_IF in current.subtypes
and format_token.Subtype.COMP_IF not in previous.subtypes):
# Penalize breaking at comp_if when it doesn't match the newline structure
# in the rest of the comprehension.
if (style.Get('SPLIT_COMPLEX_COMPREHENSION')
and top_of_stack.has_split_at_for != newline
and (top_of_stack.has_split_at_for
or not top_of_stack.HasTrivialExpr())):
penalty += split_penalty.UNBREAKABLE
return penalty