def testIsMultilineString(self):
tok = format_token.FormatToken(pytree.Leaf(token.STRING,
'"""hello"""'))
self.assertTrue(tok.is_multiline_string)
tok = format_token.FormatToken(
pytree.Leaf(token.STRING, 'r"""hello"""'))
self.assertTrue(tok.is_multiline_string)
def testSimple(self):
tok = format_token.FormatToken(pytree.Leaf(token.STRING, "'hello world'"))
self.assertEqual("FormatToken(name=STRING, value='hello world', lineno=0)",
str(tok))
self.assertTrue(tok.is_string)
tok = format_token.FormatToken(pytree.Leaf(token.COMMENT, '# A comment'))
self.assertEqual('FormatToken(name=COMMENT, value=# A comment, lineno=0)',
str(tok))
self.assertTrue(tok.is_comment)
def AppendNode(self, node):
"""Convenience method to append a pytree node directly.
Wraps the node with a FormatToken.
Arguments:
node: the node to append
"""
self.AppendToken(format_token.FormatToken(node))
def DefaultLeafVisit(self, leaf):
"""Default visitor for tree leaves.
A tree leaf is always just gets appended to the current logical line.
Arguments:
leaf: the leaf to visit.
"""
if leaf.type in _WHITESPACE_TOKENS:
self._StartNewLine()
elif leaf.type != grammar_token.COMMENT or leaf.value.strip():
# Add non-whitespace tokens and comments that aren't empty.
self._cur_logical_line.AppendToken(format_token.FormatToken(leaf))
def _SingleOrMergedLines(uwlines):
"""Generate the lines we want to format.
Arguments:
uwlines: (list of unwrapped_line.UnwrappedLine) Lines we want to format.
Yields:
Either a single line, if the current line cannot be merged with the
succeeding line, or the next two lines merged into one line.
"""
index = 0
last_was_merged = False
while index < len(uwlines):
if uwlines[index].disable:
uwline = uwlines[index]
index += 1
while index < len(uwlines):
column = uwline.last.column + 2
if uwlines[index].lineno != uwline.lineno:
break
if uwline.last.value != ':':
leaf = pytree.Leaf(type=token.SEMI,
value=';',
context=('', (uwline.lineno, column)))
uwline.AppendToken(format_token.FormatToken(leaf))
for tok in uwlines[index].tokens:
uwline.AppendToken(tok)
index += 1
yield uwline
elif line_joiner.CanMergeMultipleLines(uwlines[index:],
last_was_merged):
# TODO(morbo): This splice is potentially very slow. Come up with a more
# performance-friendly way of determining if two lines can be merged.
next_uwline = uwlines[index + 1]
for tok in next_uwline.tokens:
uwlines[index].AppendToken(tok)
if (len(next_uwline.tokens) == 1
and next_uwline.first.is_multiline_string):
# This may be a multiline shebang. In that case, we want to retain the
# formatting. Otherwise, it could mess up the shell script's syntax.
uwlines[index].disable = True
yield uwlines[index]
index += 2
last_was_merged = True
else:
yield uwlines[index]
index += 1
last_was_merged = False
def _CreateLogicalLines(tokens):
"""Separate tokens into logical lines.
Arguments:
tokens: (list of tokenizer.TokenInfo) Tokens generated by tokenizer.
Returns:
A list of LogicalLines.
"""
logical_lines = []
cur_logical_line = []
prev_tok = None
depth = 0
for tok in tokens:
tok = py3compat.TokenInfo(*tok)
if tok.type == tokenize.NEWLINE:
# End of a logical line.
logical_lines.append(
logical_line.LogicalLine(depth, cur_logical_line))
cur_logical_line = []
prev_tok = None
elif tok.type == tokenize.INDENT:
depth += 1
elif tok.type == tokenize.DEDENT:
depth -= 1
elif tok.type not in {tokenize.NL, tokenize.ENDMARKER}:
if (prev_tok and prev_tok.line.rstrip().endswith('\\')
and prev_tok.start[0] < tok.start[0]):
# Insert a token for a line continuation.
ctok = py3compat.TokenInfo(type=CONTINUATION,
string='\\',
start=(prev_tok.start[0],
prev_tok.start[1] + 1),
end=(prev_tok.end[0],
prev_tok.end[0] + 2),
line=prev_tok.line)
ctok.lineno = ctok.start[0]
ctok.column = ctok.start[1]
ctok.value = '\\'
cur_logical_line.append(
format_token.FormatToken(ctok, 'CONTINUATION'))
tok.lineno = tok.start[0]
tok.column = tok.start[1]
tok.value = tok.string
cur_logical_line.append(
format_token.FormatToken(tok, token.tok_name[tok.type]))
prev_tok = tok
# Link the FormatTokens in each line together to for a doubly linked list.
for line in logical_lines:
previous = line.first
bracket_stack = [previous] if previous.OpensScope() else []
for tok in line.tokens[1:]:
tok.previous_token = previous
previous.next_token = tok
previous = tok
# Set up the "matching_bracket" attribute.
if tok.OpensScope():
bracket_stack.append(tok)
elif tok.ClosesScope():
bracket_stack[-1].matching_bracket = tok
tok.matching_bracket = bracket_stack.pop()
return logical_lines
def _MakeFormatTokenLeaf(token_type, token_value): return format_token.FormatToken(pytree.Leaf(token_type, token_value))
def testIsImportStatement(self):
tok = format_token.FormatToken(pytree.Leaf(token.NAME, 'import'))
self.assertTrue(tok.is_import_keyword)
tok = format_token.FormatToken(pytree.Leaf(token.STRING, "'import'"))
self.assertFalse(tok.is_import_keyword)