def testFirstLast(self):
toks = _MakeFormatTokenList([(token.DOT, '.'), (token.LPAR, '('),
(token.VBAR, '|')])
lline = logical_line.LogicalLine(20, toks)
self.assertEqual(20, lline.depth)
self.assertEqual('DOT', lline.first.name)
self.assertEqual('VBAR', lline.last.name)
def __init__(self):
# A list of all logical lines finished visiting so far.
self._logical_lines = []
# Builds up a "current" logical line while visiting pytree nodes. Some nodes
# will finish a line and start a new one.
self._cur_logical_line = logical_line.LogicalLine(0)
# Current indentation depth.
self._cur_depth = 0
def testSimpleFunctionDefWithNoSplitting(self):
code = textwrap.dedent(r"""
def f(a, b):
pass
""")
llines = yapf_test_helper.ParseAndUnwrap(code)
lline = logical_line.LogicalLine(0, _FilterLine(llines[0]))
lline.CalculateFormattingInformation()
# Add: 'f'
state = format_decision_state.FormatDecisionState(lline, 0)
state.MoveStateToNextToken()
self.assertEqual('f', state.next_token.value)
self.assertFalse(state.CanSplit(False))
# Add: '('
state.AddTokenToState(False, True)
self.assertEqual('(', state.next_token.value)
self.assertFalse(state.CanSplit(False))
self.assertFalse(state.MustSplit())
# Add: 'a'
state.AddTokenToState(False, True)
self.assertEqual('a', state.next_token.value)
self.assertTrue(state.CanSplit(False))
self.assertFalse(state.MustSplit())
# Add: ','
state.AddTokenToState(False, True)
self.assertEqual(',', state.next_token.value)
self.assertFalse(state.CanSplit(False))
self.assertFalse(state.MustSplit())
# Add: 'b'
state.AddTokenToState(False, True)
self.assertEqual('b', state.next_token.value)
self.assertTrue(state.CanSplit(False))
self.assertFalse(state.MustSplit())
# Add: ')'
state.AddTokenToState(False, True)
self.assertEqual(')', state.next_token.value)
self.assertTrue(state.CanSplit(False))
self.assertFalse(state.MustSplit())
# Add: ':'
state.AddTokenToState(False, True)
self.assertEqual(':', state.next_token.value)
self.assertFalse(state.CanSplit(False))
self.assertFalse(state.MustSplit())
clone = state.Clone()
self.assertEqual(repr(state), repr(clone))
def _StartNewLine(self):
"""Finish current line and start a new one.
Place the currently accumulated line into the _logical_lines list and
start a new one.
"""
if self._cur_logical_line.tokens:
self._logical_lines.append(self._cur_logical_line)
_MatchBrackets(self._cur_logical_line)
_IdentifyParameterLists(self._cur_logical_line)
_AdjustSplitPenalty(self._cur_logical_line)
self._cur_logical_line = logical_line.LogicalLine(self._cur_depth)
def _CreateLogicalLines(tokens):
"""Separate tokens into logical lines.
Arguments:
tokens: (list of tokenizer.TokenInfo) Tokens generated by tokenizer.
Returns:
A list of LogicalLines.
"""
logical_lines = []
cur_logical_line = []
prev_tok = None
depth = 0
for tok in tokens:
tok = py3compat.TokenInfo(*tok)
if tok.type == tokenize.NEWLINE:
# End of a logical line.
logical_lines.append(
logical_line.LogicalLine(depth, cur_logical_line))
cur_logical_line = []
prev_tok = None
elif tok.type == tokenize.INDENT:
depth += 1
elif tok.type == tokenize.DEDENT:
depth -= 1
elif tok.type not in {tokenize.NL, tokenize.ENDMARKER}:
if (prev_tok and prev_tok.line.rstrip().endswith('\\')
and prev_tok.start[0] < tok.start[0]):
# Insert a token for a line continuation.
ctok = py3compat.TokenInfo(type=CONTINUATION,
string='\\',
start=(prev_tok.start[0],
prev_tok.start[1] + 1),
end=(prev_tok.end[0],
prev_tok.end[0] + 2),
line=prev_tok.line)
ctok.lineno = ctok.start[0]
ctok.column = ctok.start[1]
ctok.value = '\\'
cur_logical_line.append(
format_token.FormatToken(ctok, 'CONTINUATION'))
tok.lineno = tok.start[0]
tok.column = tok.start[1]
tok.value = tok.string
cur_logical_line.append(
format_token.FormatToken(tok, token.tok_name[tok.type]))
prev_tok = tok
# Link the FormatTokens in each line together to for a doubly linked list.
for line in logical_lines:
previous = line.first
bracket_stack = [previous] if previous.OpensScope() else []
for tok in line.tokens[1:]:
tok.previous_token = previous
previous.next_token = tok
previous = tok
# Set up the "matching_bracket" attribute.
if tok.OpensScope():
bracket_stack.append(tok)
elif tok.ClosesScope():
bracket_stack[-1].matching_bracket = tok
tok.matching_bracket = bracket_stack.pop()
return logical_lines
def testAppendToken(self):
lline = logical_line.LogicalLine(0)
lline.AppendToken(_MakeFormatTokenLeaf(token.LPAR, '('))
lline.AppendToken(_MakeFormatTokenLeaf(token.RPAR, ')'))
self.assertEqual(['LPAR', 'RPAR'], [tok.name for tok in lline.tokens])
def testAsCode(self):
toks = _MakeFormatTokenList([(token.DOT, '.'), (token.LPAR, '('),
(token.VBAR, '|')])
lline = logical_line.LogicalLine(2, toks)
self.assertEqual(' . ( |', lline.AsCode())
def testConstruction(self):
toks = _MakeFormatTokenList([(token.DOT, '.'), (token.VBAR, '|')])
lline = logical_line.LogicalLine(20, toks)
self.assertEqual(20, lline.depth)
self.assertEqual(['DOT', 'VBAR'], [tok.name for tok in lline.tokens])
def testAppendNode(self):
lline = logical_line.LogicalLine(0)
lline.AppendNode(pytree.Leaf(token.LPAR, '('))
lline.AppendNode(pytree.Leaf(token.RPAR, ')'))
self.assertEqual(['LPAR', 'RPAR'], [tok.name for tok in lline.tokens])
