def testFirstLast(self): toks = _MakeFormatTokenList([(token.DOT, '.'), (token.LPAR, '('), (token.VBAR, '|')]) lline = logical_line.LogicalLine(20, toks) self.assertEqual(20, lline.depth) self.assertEqual('DOT', lline.first.name) self.assertEqual('VBAR', lline.last.name)
def __init__(self): # A list of all logical lines finished visiting so far. self._logical_lines = [] # Builds up a "current" logical line while visiting pytree nodes. Some nodes # will finish a line and start a new one. self._cur_logical_line = logical_line.LogicalLine(0) # Current indentation depth. self._cur_depth = 0
def testSimpleFunctionDefWithNoSplitting(self): code = textwrap.dedent(r""" def f(a, b): pass """) llines = yapf_test_helper.ParseAndUnwrap(code) lline = logical_line.LogicalLine(0, _FilterLine(llines[0])) lline.CalculateFormattingInformation() # Add: 'f' state = format_decision_state.FormatDecisionState(lline, 0) state.MoveStateToNextToken() self.assertEqual('f', state.next_token.value) self.assertFalse(state.CanSplit(False)) # Add: '(' state.AddTokenToState(False, True) self.assertEqual('(', state.next_token.value) self.assertFalse(state.CanSplit(False)) self.assertFalse(state.MustSplit()) # Add: 'a' state.AddTokenToState(False, True) self.assertEqual('a', state.next_token.value) self.assertTrue(state.CanSplit(False)) self.assertFalse(state.MustSplit()) # Add: ',' state.AddTokenToState(False, True) self.assertEqual(',', state.next_token.value) self.assertFalse(state.CanSplit(False)) self.assertFalse(state.MustSplit()) # Add: 'b' state.AddTokenToState(False, True) self.assertEqual('b', state.next_token.value) self.assertTrue(state.CanSplit(False)) self.assertFalse(state.MustSplit()) # Add: ')' state.AddTokenToState(False, True) self.assertEqual(')', state.next_token.value) self.assertTrue(state.CanSplit(False)) self.assertFalse(state.MustSplit()) # Add: ':' state.AddTokenToState(False, True) self.assertEqual(':', state.next_token.value) self.assertFalse(state.CanSplit(False)) self.assertFalse(state.MustSplit()) clone = state.Clone() self.assertEqual(repr(state), repr(clone))
def _StartNewLine(self): """Finish current line and start a new one. Place the currently accumulated line into the _logical_lines list and start a new one. """ if self._cur_logical_line.tokens: self._logical_lines.append(self._cur_logical_line) _MatchBrackets(self._cur_logical_line) _IdentifyParameterLists(self._cur_logical_line) _AdjustSplitPenalty(self._cur_logical_line) self._cur_logical_line = logical_line.LogicalLine(self._cur_depth)
def _CreateLogicalLines(tokens): """Separate tokens into logical lines. Arguments: tokens: (list of tokenizer.TokenInfo) Tokens generated by tokenizer. Returns: A list of LogicalLines. """ logical_lines = [] cur_logical_line = [] prev_tok = None depth = 0 for tok in tokens: tok = py3compat.TokenInfo(*tok) if tok.type == tokenize.NEWLINE: # End of a logical line. logical_lines.append( logical_line.LogicalLine(depth, cur_logical_line)) cur_logical_line = [] prev_tok = None elif tok.type == tokenize.INDENT: depth += 1 elif tok.type == tokenize.DEDENT: depth -= 1 elif tok.type not in {tokenize.NL, tokenize.ENDMARKER}: if (prev_tok and prev_tok.line.rstrip().endswith('\\') and prev_tok.start[0] < tok.start[0]): # Insert a token for a line continuation. ctok = py3compat.TokenInfo(type=CONTINUATION, string='\\', start=(prev_tok.start[0], prev_tok.start[1] + 1), end=(prev_tok.end[0], prev_tok.end[0] + 2), line=prev_tok.line) ctok.lineno = ctok.start[0] ctok.column = ctok.start[1] ctok.value = '\\' cur_logical_line.append( format_token.FormatToken(ctok, 'CONTINUATION')) tok.lineno = tok.start[0] tok.column = tok.start[1] tok.value = tok.string cur_logical_line.append( format_token.FormatToken(tok, token.tok_name[tok.type])) prev_tok = tok # Link the FormatTokens in each line together to for a doubly linked list. for line in logical_lines: previous = line.first bracket_stack = [previous] if previous.OpensScope() else [] for tok in line.tokens[1:]: tok.previous_token = previous previous.next_token = tok previous = tok # Set up the "matching_bracket" attribute. if tok.OpensScope(): bracket_stack.append(tok) elif tok.ClosesScope(): bracket_stack[-1].matching_bracket = tok tok.matching_bracket = bracket_stack.pop() return logical_lines
def testAppendToken(self): lline = logical_line.LogicalLine(0) lline.AppendToken(_MakeFormatTokenLeaf(token.LPAR, '(')) lline.AppendToken(_MakeFormatTokenLeaf(token.RPAR, ')')) self.assertEqual(['LPAR', 'RPAR'], [tok.name for tok in lline.tokens])
def testAsCode(self): toks = _MakeFormatTokenList([(token.DOT, '.'), (token.LPAR, '('), (token.VBAR, '|')]) lline = logical_line.LogicalLine(2, toks) self.assertEqual(' . ( |', lline.AsCode())
def testConstruction(self): toks = _MakeFormatTokenList([(token.DOT, '.'), (token.VBAR, '|')]) lline = logical_line.LogicalLine(20, toks) self.assertEqual(20, lline.depth) self.assertEqual(['DOT', 'VBAR'], [tok.name for tok in lline.tokens])
def testAppendNode(self): lline = logical_line.LogicalLine(0) lline.AppendNode(pytree.Leaf(token.LPAR, '(')) lline.AppendNode(pytree.Leaf(token.RPAR, ')')) self.assertEqual(['LPAR', 'RPAR'], [tok.name for tok in lline.tokens])