def p_redirection_heredoc(p): '''redirection : LESS_LESS WORD | NUMBER LESS_LESS WORD | REDIR_WORD LESS_LESS WORD | LESS_LESS_MINUS WORD | NUMBER LESS_LESS_MINUS WORD | REDIR_WORD LESS_LESS_MINUS WORD''' parserobj = p.context assert isinstance(parserobj, _parser) output = bast.node(kind='word', word=p[len(p) - 1], parts=[], pos=p.lexspan(len(p) - 1)) if len(p) == 3: p[0] = bast.node(kind='redirect', input=None, type=p[1], heredoc=None, output=output, pos=(p.lexpos(1), p.endlexpos(2))) else: p[0] = bast.node(kind='redirect', input=p[1], type=p[2], heredoc=None, output=output, pos=(p.lexpos(1), p.endlexpos(3))) if p.slice[len(p) - 2].ttype == tokenizer.tokentype.LESS_LESS: parserobj.redirstack.append((p[0], False)) else: parserobj.redirstack.append((p[0], True))
def p_redirection(p): '''redirection : GREATER WORD | LESS WORD | NUMBER GREATER WORD | NUMBER LESS WORD | REDIR_WORD GREATER WORD | REDIR_WORD LESS WORD | GREATER_GREATER WORD | NUMBER GREATER_GREATER WORD | REDIR_WORD GREATER_GREATER WORD | GREATER_BAR WORD | NUMBER GREATER_BAR WORD | REDIR_WORD GREATER_BAR WORD | LESS_GREATER WORD | NUMBER LESS_GREATER WORD | REDIR_WORD LESS_GREATER WORD | LESS_LESS_LESS WORD | NUMBER LESS_LESS_LESS WORD | REDIR_WORD LESS_LESS_LESS WORD | LESS_AND NUMBER | NUMBER LESS_AND NUMBER | REDIR_WORD LESS_AND NUMBER | GREATER_AND NUMBER | NUMBER GREATER_AND NUMBER | REDIR_WORD GREATER_AND NUMBER | LESS_AND WORD | NUMBER LESS_AND WORD | REDIR_WORD LESS_AND WORD | GREATER_AND WORD | NUMBER GREATER_AND WORD | REDIR_WORD GREATER_AND WORD | GREATER_AND DASH | NUMBER GREATER_AND DASH | REDIR_WORD GREATER_AND DASH | LESS_AND DASH | NUMBER LESS_AND DASH | REDIR_WORD LESS_AND DASH | AND_GREATER WORD | AND_GREATER_GREATER WORD''' parserobj = p.context if len(p) == 3: output = p[2] if p.slice[2].ttype == tokenizer.tokentype.WORD: output = _expandword(parserobj, p.slice[2]) p[0] = bast.node(kind='redirect', input=None, type=p[1], heredoc=None, output=output, pos=(p.lexpos(1), p.endlexpos(2))) else: output = p[3] if p.slice[3].ttype == tokenizer.tokentype.WORD: output = _expandword(parserobj, p.slice[3]) p[0] = bast.node(kind='redirect', input=p[1], type=p[2], heredoc=None, output=output, pos=(p.lexpos(1), p.endlexpos(3)))
def p_shell_command(p): '''shell_command : for_command | case_command | WHILE compound_list DO compound_list DONE | UNTIL compound_list DO compound_list DONE | select_command | if_command | subshell | group_command | arith_command | cond_command | arith_for_command''' if len(p) == 2: p[0] = p[1] else: # while or until assert p[2].kind == 'list' parts = _makeparts(p) kind = parts[0].word assert kind in ('while', 'until') p[0] = bast.node( kind='compound', redirects=[], list=[bast.node(kind=kind, parts=parts, pos=_partsspan(parts))], pos=_partsspan(parts)) assert p[0].kind == 'compound'
def _expandword(parser, tokenword): if parser._expansionlimit == -1: # we enter this branch in the following conditions: # - currently parsing a substitution as a result of an expansion # - the previous expansion had limit == 0 # # this means that this node is a descendant of a substitution in an # unexpanded word and will be filtered in the limit == 0 condition below # # (the reason we even expand when limit == 0 is to get quote removal) node = bast.node(kind='word', word=tokenword, pos=(tokenword.lexpos, tokenword.endlexpos), parts=[]) return node else: quoted = bool(tokenword.flags & flags.word.QUOTED) doublequoted = quoted and tokenword.value[0] == '"' # TODO set qheredocument parts, expandedword = subst._expandwordinternal( parser, tokenword, 0, doublequoted, 0, 0) # limit reached, don't include substitutions (still expanded to get # quote removal though) if parser._expansionlimit == 0: parts = [node for node in parts if 'substitution' not in node.kind] node = bast.node(kind='word', word=expandedword, pos=(tokenword.lexpos, tokenword.endlexpos), parts=parts) return node
def p_group_command(p): '''group_command : LEFT_CURLY compound_list RIGHT_CURLY''' lcurly = bast.node(kind='reservedword', word=p[1], pos=p.lexspan(1)) rcurly = bast.node(kind='reservedword', word=p[3], pos=p.lexspan(3)) parts = [lcurly, p[2], rcurly] p[0] = bast.node(kind='compound', list=parts, redirects=[], pos=_partsspan(parts))
def p_subshell(p): '''subshell : LEFT_PAREN compound_list RIGHT_PAREN''' lparen = bast.node(kind='reservedword', word=p[1], pos=p.lexspan(1)) rparen = bast.node(kind='reservedword', word=p[3], pos=p.lexspan(3)) parts = [lparen, p[2], rparen] p[0] = bast.node(kind='compound', list=parts, redirects=[], pos=_partsspan(parts))
def p_list0(p): '''list0 : list1 NEWLINE newline_list | list1 AMPERSAND newline_list | list1 SEMICOLON newline_list''' parts = p[1] if len(parts) > 1 or p.slice[2].ttype != tokenizer.tokentype.NEWLINE: parts.append(bast.node(kind='operator', op=p[2], pos=p.lexspan(2))) p[0] = bast.node(kind='list', parts=parts, pos=_partsspan(parts)) else: p[0] = parts[0]
def p_if_command(p): '''if_command : IF compound_list THEN compound_list FI | IF compound_list THEN compound_list ELSE compound_list FI | IF compound_list THEN compound_list elif_clause FI''' # we currently don't distinguish the various lists that make up the # command, because it's not needed later on. if there will be a need # we can always add different nodes for elif/else. parts = _makeparts(p) p[0] = bast.node( kind='compound', redirects=[], list=[bast.node(kind='if', parts=parts, pos=_partsspan(parts))], pos=_partsspan(parts))
def _extractcommandsubst(parserobj, string, sindex, sxcommand=False): if string[sindex] == '(': raise NotImplementedError('arithmetic expansion') #return _extractdelimitedstring(parserobj, string, sindex, '$(', '(', '(', sxcommand=True) else: node, si = _parsedolparen(parserobj, string, sindex) si += 1 return bast.node(kind='commandsubstitution', command=node, pos=(sindex - 2, si)), si
def p_pipeline(p): '''pipeline : pipeline BAR newline_list pipeline | pipeline BAR_AND newline_list pipeline | command''' if len(p) == 2: p[0] = [p[1]] else: p[0] = p[1] p[0].append(bast.node(kind='pipe', pipe=p[2], pos=p.lexspan(2))) p[0].extend(p[len(p) - 1])
def p_compound_list(p): '''compound_list : list | newline_list list1''' if len(p) == 2: p[0] = p[1] else: parts = p[2] if len(parts) > 1: p[0] = bast.node(kind='list', parts=parts, pos=_partsspan(parts)) else: p[0] = parts[0]
def p_simple_list(p): '''simple_list : simple_list1 | simple_list1 AMPERSAND | simple_list1 SEMICOLON''' tok = p.lexer heredoc.gatherheredocuments(tok) if len(p) == 3 or len(p[1]) > 1: parts = p[1] if len(p) == 3: parts.append(bast.node(kind='operator', op=p[2], pos=p.lexspan(2))) p[0] = bast.node(kind='list', parts=parts, pos=_partsspan(parts)) else: assert len(p[1]) == 1 p[0] = p[1][0] if (len(p) == 2 and p.lexer._parserstate & flags.parser.CMDSUBST and p.lexer._current_token.nopos() == p.lexer._shell_eof_token): # accept the input p.accept()
def p_elif_clause(p): '''elif_clause : ELIF compound_list THEN compound_list | ELIF compound_list THEN compound_list ELSE compound_list | ELIF compound_list THEN compound_list elif_clause''' parts = [] for i in range(1, len(p)): if isinstance(p[i], bast.node): parts.append(p[i]) else: parts.append( bast.node(kind='reservedword', word=p[i], pos=p.lexspan(i))) p[0] = parts
def p_simple_list1(p): '''simple_list1 : simple_list1 AND_AND newline_list simple_list1 | simple_list1 OR_OR newline_list simple_list1 | simple_list1 AMPERSAND simple_list1 | simple_list1 SEMICOLON simple_list1 | pipeline_command''' if len(p) == 2: p[0] = [p[1]] else: p[0] = p[1] p[0].append(bast.node(kind='operator', op=p[2], pos=p.lexspan(2))) p[0].extend(p[len(p) - 1])
def p_function_def(p): '''function_def : WORD LEFT_PAREN RIGHT_PAREN newline_list function_body | FUNCTION WORD LEFT_PAREN RIGHT_PAREN newline_list function_body | FUNCTION WORD newline_list function_body''' parts = _makeparts(p) body = parts[-1] name = parts[bast.findfirstkind(parts, 'word')] p[0] = bast.node(kind='function', name=name, body=body, parts=parts, pos=_partsspan(parts))
def p_for_command(p): '''for_command : FOR WORD newline_list DO compound_list DONE | FOR WORD newline_list LEFT_CURLY compound_list RIGHT_CURLY | FOR WORD SEMICOLON newline_list DO compound_list DONE | FOR WORD SEMICOLON newline_list LEFT_CURLY compound_list RIGHT_CURLY | FOR WORD newline_list IN word_list list_terminator newline_list DO compound_list DONE | FOR WORD newline_list IN word_list list_terminator newline_list LEFT_CURLY compound_list RIGHT_CURLY | FOR WORD newline_list IN list_terminator newline_list DO compound_list DONE | FOR WORD newline_list IN list_terminator newline_list LEFT_CURLY compound_list RIGHT_CURLY''' parts = _makeparts(p) # find the operatornode that we might have there due to # list_terminator/newline_list and convert it to a reservedword so its # considered as part of the for loop for i, part in enumerate(parts): if part.kind == 'operator' and part.op == ';': parts[i] = bast.node(kind='reservedword', word=';', pos=part.pos) break # there could be only one in there... p[0] = bast.node( kind='compound', redirects=[], list=[bast.node(kind='for', parts=parts, pos=_partsspan(parts))], pos=_partsspan(parts))
def p_list1(p): '''list1 : list1 AND_AND newline_list list1 | list1 OR_OR newline_list list1 | list1 AMPERSAND newline_list list1 | list1 SEMICOLON newline_list list1 | list1 NEWLINE newline_list list1 | pipeline_command''' if len(p) == 2: p[0] = [p[1]] else: p[0] = p[1] # XXX newline p[0].append(bast.node(kind='operator', op=p[2], pos=p.lexspan(2))) p[0].extend(p[len(p) - 1])
def _paramexpand(parserobj, string, sindex): node = None zindex = sindex + 1 c = string[zindex] if zindex < len(string) else None if c and c in '0123456789$#?-!*@': # XXX 7685 node = bast.node(kind='parameter', value=c, pos=(sindex, zindex+1)) elif c == '{': # XXX 7863 # TODO not start enough, doesn't consider escaping zindex = string.find('}', zindex + 1) node = bast.node(kind='parameter', value=string[sindex + 2:zindex], pos=(sindex, zindex+1)) # TODO # return _parameterbraceexpand(string, zindex) elif c == '(': return _extractcommandsubst(parserobj, string, zindex + 1) elif c == '[': raise NotImplementedError('arithmetic substitution') #return _extractarithmeticsubst(string, zindex + 1) else: tindex = zindex for zindex in range(tindex, len(string) + 1): if zindex == len(string): break if not string[zindex].isalnum() and not string[zindex] == '_': break temp1 = string[sindex:zindex] if temp1: return (bast.node(kind='parameter', value=temp1[1:], pos=(sindex, zindex)), zindex) if zindex < len(string): zindex += 1 return node, zindex
def p_command(p): '''command : simple_command | shell_command | shell_command redirection_list | function_def | coproc''' if isinstance(p[1], bast.node): p[0] = p[1] if len(p) == 3: assert p[0].kind == 'compound' p[0].redirects.extend(p[2]) assert p[0].pos[0] < p[0].redirects[-1].pos[1] p[0].pos = (p[0].pos[0], p[0].redirects[-1].pos[1]) else: p[0] = bast.node(kind='command', parts=p[1], pos=_partsspan(p[1]))
def p_pipeline_command(p): '''pipeline_command : pipeline | BANG pipeline_command | timespec pipeline_command | timespec list_terminator | BANG list_terminator''' if len(p) == 2: if len(p[1]) == 1: p[0] = p[1][0] else: p[0] = bast.node(kind='pipeline', parts=p[1], pos=(p[1][0].pos[0], p[1][-1].pos[1])) else: # XXX timespec node = bast.node(kind='reservedword', word='!', pos=p.lexspan(1)) if p[2].kind == 'pipeline': p[0] = p[2] p[0].parts.insert(0, node) p[0].pos = (p[0].parts[0].pos[0], p[0].parts[-1].pos[1]) else: p[0] = bast.node(kind='pipeline', parts=[node, p[2]], pos=(node.pos[0], p[2].pos[1]))
def makeheredoc(tokenizer, redirnode, lineno, killleading): # redirword = string_quote_removal(redirectnode.word) redirword = redirnode.output.word document = [] startpos = tokenizer._shell_input_line_index #fullline = self.tok.readline(bool(redirword.output.flags & flags.word.QUOTED)) fullline = tokenizer.readline(False) while fullline: if killleading: while fullline[0] == '\t': fullline = fullline[1:] if not fullline: continue if fullline[:-1] == redirword and fullline[len(redirword)] == '\n': document.append(fullline[:-1]) # document_done break document.append(fullline) #fullline = self.readline(bool(redirnode.flags & flags.word.QUOTED)) fullline = tokenizer.readline(False) if not fullline: raise errors.ParsingError( "here-document at line %d delimited by end-of-file (wanted %r)" % (lineno, redirword), tokenizer._shell_input_line, tokenizer._shell_input_line_index) document = ''.join(document) endpos = tokenizer._shell_input_line_index - 1 assert hasattr(redirnode, 'heredoc') redirnode.heredoc = bast.node(kind='heredoc', value=document, pos=(startpos, endpos)) # if the heredoc immediately follows this node, fix its end pos if redirnode.pos[1] + 1 == startpos: redirnode.pos = (redirnode.pos[0], endpos) return document
def _makeparts(p): parts = [] for i in range(1, len(p)): if isinstance(p[i], bast.node): parts.append(p[i]) elif isinstance(p[i], list): parts.extend(p[i]) elif isinstance(p.slice[i], tokenizer.token): if p.slice[i].ttype == tokenizer.tokentype.WORD: parserobj = p.context parts.append(_expandword(parserobj, p.slice[i])) else: parts.append( bast.node(kind='reservedword', word=p[i], pos=p.lexspan(i))) else: pass return parts
def normalize_command(node, current=None): bash_grammar = BashGrammar() bash_grammar.name2type = bg.name2type if not node or not node.parts: return input = node.parts num_tokens = len(node.parts) bast_node = input[0] if bast_node.kind == 'assignment': normalize(bast_node, current, 'assignment') elif bast_node.kind == 'redirect': normalize(bast_node, current, 'redirect') elif bast_node.kind == 'commandsubstitution': normalize(bast_node, current, 'commandsubstitution') elif bast_node.kind == 'word' and not bast_node.parts: token = normalize_word(bast_node) head = UtilityNode(token, parent=current, lsb=current.get_right_child()) if current: current.add_child(head) # If utility grammar is not known, parse into a simple two-level tree if not bg.consume(token): raise errors.LintParsingError( "Warning: grammar not found - utility {}".format(token), num_tokens, 0) for bast_node in input[1:]: if bast_node.kind == 'word' and ( not bast_node.parts or (bast_node.parts[0].kind == 'parameter' and bast_node.word.startswith('-'))): token = normalize_word(bast_node) if token.startswith('-'): child = FlagNode(token, parent=head, lsb=head.get_right_child()) else: child = ArgumentNode(token, arg_type='Unknown', parent=head, lsb=head.get_right_child()) head.add_child(child) else: normalize(bast_node, head) return current, i = head, 1 bash_grammar.grammar = { head.value: copy.deepcopy(bg.grammar[head.value]) } bash_grammar.consume(head.value) while i < len(input): bast_node = input[i] # '--': signal the end of options if bast_node.kind == 'word' and bast_node.word == '--': op = OperatorNode('--', parent=current, lsb=current.get_right_child()) current.add_child(op) bash_grammar.push('--', OPERATOR_S) i += 1 continue # examine each possible next states in order matched = False for next_state in bash_grammar.next_states: if next_state.is_compound_flag(): # Next state is a flag if bast_node.kind != 'word' or ( bast_node.parts and not (bast_node.word.startswith('-') and bast_node.parts[0].kind == 'parameter')): continue if is_parenthesis(bast_node, current): flag = FlagNode(bast_node.word, parent=current, lsb=current.get_right_child()) current.add_child(flag) matched = True i += 1 break elif is_unary_logic_op(bast_node, current): flag = UnaryLogicOpNode( bast_node.word, parent=current, lsb=current.get_right_child()) current.add_child(flag) matched = True i += 1 break elif is_binary_logic_op(bast_node, current): flag = BinaryLogicOpNode( bast_node.word, parent=current, lsb=current.get_right_child()) current.add_child(flag) matched = True i += 1 break else: token = normalize_word(bast_node) try: result = bash_grammar.push( token, COMPOUND_FLAG_S) except ValueError as e: raise errors.FlagError(e.args[0], num_tokens, i) if result: for flag_token, flag_arg in result: flag = FlagNode( flag_token, parent=current, lsb=current.get_right_child()) current.add_child(flag) if flag_arg == '__OPEN__': # Incomplete AST, expecting flag argument current = flag elif flag_arg is not None: # Argument is specified with flag argument = ArgumentNode( flag_arg[0], arg_type=flag_arg[1], parent=flag, lsb=flag.get_right_child()) flag.add_child(argument) matched = True i += 1 break elif next_state.is_command(): # Next state is a nested bash command new_command_node = bast.node(kind="command", word="", parts=[], pos=(-1, -1)) if next_state.type == ARG_COMMAND_S: if bast_node.kind == 'word' and not bast_node.parts: token = normalize_word(bast_node) if constants.with_quotation(token): subcommand = token[1:-1] start_pos = bast_node.pos[0] + 1 tree = safe_bashlex_parse( subcommand, start_pos=start_pos, verbose=verbose) if tree is None: raise errors.SubCommandError( 'Error in subcommand string: {}'. format(token), num_tokens, i) normalize(tree[0], current) bash_grammar.push(token, next_state.type) i += 1 else: continue else: normalize(bast_node, current, 'command') i += 1 elif next_state.type == EXEC_COMMAND_S: new_input = [] j = i while j < len(input): if hasattr(input[j], 'word') and \ input[j].word in next_state.stop_tokens: break else: new_input.append(input[j]) j += 1 new_command_node.parts = new_input normalize_command(new_command_node, current) if j < len(input): current.value += ('::' + input[j].word) bash_grammar.push(input[j], EXEC_COMMAND_S) else: if verbose: print( "Warning: -exec missing stop token - ; added" ) current.value += ('::' + ';') bash_grammar.push(';', EXEC_COMMAND_S) i = j + 1 else: # Interpret all of the rest of the tokens as content of the nested command new_command_node.parts = input[i:] normalize_command(new_command_node, current) bash_grammar.push('', next_state.type) i = len(input) current = current.utility matched = True break elif next_state.is_argument(): # Next state is an argument if bast_node.kind == 'word' and not bast_node.parts: token = normalize_word(bast_node) if next_state.is_list and next_state.list_separator != ' ': list_separator = next_state.list_separator argument = ArgumentNode( token, arg_type=next_state.arg_type, parent=current, lsb=current.get_right_child(), list_members=token.split(list_separator), list_separator=list_separator) else: argument = ArgumentNode( token, arg_type=next_state.arg_type, parent=current, lsb=current.get_right_child()) current.add_child(argument) status = bash_grammar.push(token, ARG_S) else: normalize(bast_node, current, next_state.arg_type) status = bash_grammar.push('', ARG_S) if status != '__SAME_PARENT__': current = current.utility i += 1 matched = True break if not matched: if bast_node.kind == 'redirect' or bast_node.kind == 'operator': i += 1 matched = True else: raise errors.LintParsingError('Unmatched token', num_tokens, i) if bash_grammar.allow_eof(): post_process_command(head) return else: raise errors.LintParsingError('Incomplete command', num_tokens, i) else: if bast_node.parts: normalize(bast_node, current) else: raise errors.LintParsingError( 'Utility needs to be a BAST node of "Word" type" {}'. format(bast_node), num_tokens, 0)
def _expandwordinternal(parserobj, wordtoken, qheredocument, qdoublequotes, quoted, isexp): # bash/subst.c L8132 istring = '' parts = [] tindex = [0] sindex = [0] string = wordtoken.value def nextchar(): sindex[0] += 1 if sindex[0] < len(string): return string[sindex[0]] def peekchar(): if sindex[0]+1 < len(string): return string[sindex[0]+1] while True: if sindex[0] == len(string): break # goto finished_with_string c = string[sindex[0]] if c in '<>': if (nextchar() != '(' or qheredocument or qdoublequotes or (wordtoken.flags & set([flags.word.DQUOTE, flags.word.NOPROCSUB]))): sindex[0] -= 1 # goto add_character sindex[0] += 1 istring += c else: tindex = sindex[0] + 1 node, sindex[0] = _extractprocesssubst(parserobj, string, tindex) parts.append(bast.node(kind='processsubstitution', command=node, pos=(tindex - 2, sindex[0]))) istring += string[tindex - 2:sindex[0]] # goto dollar_add_string # TODO # elif c == '=': # pass # elif c == ':': # pass elif c == '~': if (wordtoken.flags & set([flags.word.NOTILDE, flags.word.DQUOTE]) or (sindex[0] > 0 and not (wordtoken.flags & flags.word.NOTILDE)) or qdoublequotes or qheredocument): wordtoken.flags.clear() wordtoken.flags.add(flags.word.ITILDE) sindex[0] += 1 istring += c else: stopatcolon = wordtoken.flags & set([flags.word.ASSIGNRHS, flags.word.ASSIGNMENT, flags.word.TILDEEXP]) expand = True for i in range(sindex[0], len(string)): r = string[i] if r == '/': break if r in "\\'\"": expand = False break if stopatcolon and r == ':': break else: # go one past the end if we didn't exit early i += 1 if i > sindex[0] and expand: node = bast.node(kind='tilde', value=string[sindex[0]:i], pos=(sindex[0], i)) parts.append(node) istring += string[sindex[0]:i] sindex[0] = i elif c == '$' and len(string) > 1: tindex = sindex[0] node, sindex[0] = _paramexpand(parserobj, string, sindex[0]) if node: parts.append(node) istring += string[tindex:sindex[0]] elif c == '`': tindex = sindex[0] # bare instance of `` if nextchar() == '`': sindex[0] += 1 istring += '``' else: x = _stringextract(string, sindex[0], "`") if x == -1: raise errors.ParsingError('bad substitution: no closing "`" ' 'in %s' % string) else: if wordtoken.flags & flags.word.NOCOMSUB: pass else: sindex[0] = x word = string[tindex+1:sindex[0]] command, ttindex = _recursiveparse(parserobj, word, 0) _adjustpositions(command, tindex+1, len(string)) ttindex += 1 # ttindex is on the closing char # assert sindex[0] == ttindex # go one past the closing ` sindex[0] += 1 node = bast.node(kind='commandsubstitution', command=command, pos=(tindex, sindex[0])) parts.append(node) istring += string[tindex:sindex[0]] elif c == '\\': istring += string[sindex[0]+1:sindex[0]+2] sindex[0] += 2 elif c == '"': sindex[0] += 1 continue # 8513 #if qdoublequotes or qheredocument: # sindex[0] += 1 #else: # tindex = sindex[0] + 1 # parts, sindex[0] = _stringextractdoublequoted(string, sindex[0]) # if tindex == 1 and sindex[0] == len(string): # quotedstate = 'wholly' # else: # quotedstate = 'partially' elif c == "'": # entire string surronded by single quotes, no expansion is # going to happen if sindex[0] == 0 and string[-1] == "'": return [], string[1:-1] # check if we're inside double quotes if not qdoublequotes: # look for the closing ', we know we have one or otherwise # this wouldn't tokenize due to unmatched ' tindex = sindex[0] sindex[0] = string.find("'", sindex[0]) + 1 istring += string[tindex+1:sindex[0]-1] else: # this is a single quote inside double quotes, add it istring += c sindex[0] += 1 else: istring += string[sindex[0]:sindex[0]+1] sindex[0] += 1 if parts: class v(bast.nodevisitor): def visitnode(self, node): assert node.pos[1] + wordtoken.lexpos <= wordtoken.endlexpos node.pos = (node.pos[0] + wordtoken.lexpos, node.pos[1] + wordtoken.lexpos) visitor = v() for node in parts: visitor.visit(node) return parts, istring
def p_list_terminator(p): '''list_terminator : NEWLINE | SEMICOLON | EOF''' if p[1] == ';': p[0] = bast.node(kind='operator', op=';', pos=p.lexspan(1))