def process_expressions(tree): node = tree.children[0] while id(node) != id(tree): if len(node.children) > 0: if node.rule == u'ASSIGNMENT': process_expr_tree(node) remove_parentheses(node) aux_node = ParseTree('VARLIST') aux_node.parent = node aux_node2 = ParseTree('EXPRLIST') aux_node2.parent = node aux_node.value = lexer.Token('DEV', u'1') aux_node2.value = lexer.Token('DEV', u'2') node.children[0].parent = aux_node node.children[0].next = None node.children[1].next = None node.children[1].parent = aux_node2 aux_node.children.append(node.children[0]) aux_node2.children.append(node.children[1]) node.children = [aux_node, aux_node2] comma_detection(node) node = node.up_node() elif node.rule == u'EXPR' or node.rule == u'READ': process_expr_tree(node) remove_parentheses(node) comma_detection(node) node = node.up_node() else: node = node.children[0] else: node = node.up_node()
def setUp(self): self.brief_tok = lexer.Token('WORD', 'This is brief.', 0, 0, 0) self.name_tok = lexer.Token('WORD', 'var', 0, 0, 0) self.type_tok = lexer.Token('WORD', 'int', 0, 0, 0) self.tok_see = lexer.Token('WORD', 'See', 0, 0, 0) self.tok_sig = lexer.Token('WORD', 'payload', 0, 0, 0) self.formatter = raw_doc.DoxFormatter()
def setUp(self): self.tok_name = lexer.Token('WORD', 'name', 0, 0, 0) self.tok_text = lexer.Token('WORD', 'text', 0, 0, 0) self.tok_inout = lexer.Token('PARAM_IN_OUT', '[in,out]', 0, 0, 0) self.txt_name = raw_doc.RawText([self.tok_name]) self.txt_text = raw_doc.RawText([self.tok_text]) self.formatter = raw_doc.DoxFormatter()
def processValues(self, entry, node, value_nodes): result = [] enum_name = translate(node.key) #print value_nodes for key, value in value_nodes.children.iteritems(): raw_var = raw_doc.RawVariable() #print value var_name = value.key if '::' in enum_name: var_name = enum_name.split('::')[0] + '::' + var_name t = lexer.Token('WORD', enum_name, 0, 0, 0) raw_var.type = raw_doc.RawText([t]) t = lexer.Token('WORD', enum_name + '#' + var_name, 0, 0, 0) raw_var.name = raw_doc.RawText([t]) processed = set() if value.children.get('summary'): self.processSummary(raw_var, value.children['summary']) processed.add('summary') unhandled = set(value.children.keys()) - processed if unhandled: print 'Missed %s in %s' % (unhandled, child) sys.exit(1) #print raw_var.getFormatted(formatter) result.append(raw_var) return result
def setUp(self): self.brief_tok = lexer.Token('WORD', 'This is brief.', 0, 0, 0) self.name_tok = lexer.Token('WORD', 'Adaption', 0, 0, 0) self.title_tok = lexer.Token('WORD', 'Adaption Title', 0, 0, 0) self.tok_see = lexer.Token('WORD', 'See', 0, 0, 0) self.tok_sig = lexer.Token('WORD', 'payload', 0, 0, 0) self.formatter = raw_doc.DoxFormatter()
def processDescription(self, entry, node): for text in node.texts: if text.startswith('type=text:'): t = lexer.Token('WORD', text[len('type=text:'):], 0, 0, 0) raw_text = raw_doc.RawText(translateTokens([t])) else: t = lexer.Token('WORD', text, 0, 0, 0) raw_text = raw_doc.RawText(translateTokens([t])) entry.addParagraph(raw_doc.RawParagraph(raw_text))
def testConstructionWithTokens(self): tokens = [ lexer.Token('WORD', 'test', 0, 0, 0), lexer.Token('SPACE', ' ', 0, 0, 0), lexer.Token('WORD', 'foo', 0, 0, 0) ] text = raw_doc.RawText(tokens) self.assertEqual(text.tokens, tokens) self.failIf(text.empty)
def processSummary(self, entry, node): for text in node.texts: if text.startswith('type=text:'): t = lexer.Token('WORD', text[len('type=text:'):], 0, 0, 0) raw_text = raw_doc.RawText(translateTokens([t])) else: t = lexer.Token('WORD', text, 0, 0, 0) raw_text = raw_doc.RawText(translateTokens([t])) entry.addBrief(raw_doc.RawBrief(raw_text))
def setUp(self): self.path_t = lexer.Token('WORD', 'apath', 0, 0, 0) self.path = raw_doc.RawText([self.path_t]) self.snippet_t0 = lexer.Token('WORD', 'The', 0, 0, 0) self.snippet_t1 = lexer.Token('SPACE', ' ', 0, 0, 0) self.snippet_t2 = lexer.Token('WORD', 'snippet', 0, 0, 0) self.snippet = raw_doc.RawText( [self.snippet_t0, self.snippet_t1, self.snippet_t2]) self.formatter = raw_doc.DoxFormatter()
def passage(tokenQueue): """ This function takes a token queue as input and returns a tuple containing a token and a token queue. >>> tokenQueue = make_test_queue("passage") >>> _passage, tokenQueue = passage(tokenQueue) >>> tokenQueue [Token('EOF', 'EOF', 'None')] >>> _passage Token('None', 'PASSAGE', 'None') Recall that the first component of a passage is the passage command. >>> _passage.children[0] Token('passage', 'PASSAGECOMMAND', '\passage') Passages can be followed by other passages. >>> tokenQueue = make_test_queue("passage2") >>> _passage, tokenQueue = passage(tokenQueue) Second passage should remain in tokenQueue, looping happens in story function. >>> tokenQueue [Token('passage', 'PASSAGECOMMAND', '\passage'), Token('{', 'LEFTCURLY', '{'), Token('second', 'CHARACTER', 'second'), Token('}', 'RIGHTCURLY', '}')] >>> _passage Token('None', 'PASSAGE', 'None') """ token = next_token(tokenQueue) #print("tokenQueue after getting next token in passage:", tokenQueue) _passage = lexer.Token(token_type="PASSAGE") _passage.children.append(token) _argument, tokenQueue = argument( tokenQueue) #get arguments (right now just psgtitle) #print("tokenQueue after running argument in passage:", tokenQueue) if _argument == False: return (False, None) elif len(_argument) == 1: _passage.children.append(_argument[0]) else: for i in _argument: _passage.children.append(i) token = lexer.Token() while token.token_type != "EOF" and token.token_type != "PASSAGECOMMAND": _text, tokenQueue = text(tokenQueue) #print("tokenQueue in while loop of passage:", tokenQueue) if _text: _passage.children.append(_text) token = next_token(tokenQueue) #print(token) else: print("Parsing error: text in passage not correctly formatted") return (False, None) #print(token) tokenQueue.put(token) return (_passage, tokenQueue)
def processTags(self, entry, node, tag_nodes): result = [] group_name = translate(node.key) for key, value in tag_nodes.children.iteritems(): raw_tag = raw_doc.RawTag() tag_name = value.key t = lexer.Token('WORD', group_name + '#' + tag_name, 0, 0, 0) raw_tag.name = raw_doc.RawText([t]) processed = set() if value.children.get('summary'): self.processSummary(raw_tag, value.children['summary']) processed.add('summary') if value.children.get('remarks'): self.processRemarks(raw_tag, value.children['remarks']) processed.add('remarks') if value.children.get('remark'): self.processRemarks(raw_tag, value.children['remark']) processed.add('remark') if value.children.get('text'): self.processRemarks(raw_tag, value.children['text']) processed.add('text') if value.children.get('type'): ts = [] if value.children['type'].texts: ts.append(lexer.Token('WORD', ' Types: ', 0, 0, 0)) ts.append( lexer.Token('WORD', value.children['type'].texts[0], 0, 0, 0)) for txt in value.children['type'].texts[1:]: ts.append(lexer.Token('WORD', ', ' + txt, 0, 0, 0)) raw_text = raw_doc.RawText(ts) entry.addParagraph(raw_doc.RawParagraph(raw_text)) processed.add('type') processed.add('signature') # Ignore. processed.add('include') # TODO(holtgrew): Required here? processed.add('see') # TODO(holtgrew): Required here? # We do not have double-linking any more. processed.add('function') unhandled = set(value.children.keys()) - processed if unhandled: print 'Missed %s in %s processed: %s' % (unhandled, node, processed) sys.exit(1) #print raw_tag.getFormatted(formatter) result.append(raw_tag) return result
def processRemarks(self, entry, node, is_first=True): if is_first: raw_text = raw_doc.RawText( [lexer.Token('WORD', 'Remarks', 0, 0, 0)]) raw_section = raw_doc.RawSection(raw_text) entry.addParagraph(raw_section) for text in node.texts: if text.startswith('type=text:'): t = lexer.Token('WORD', text[len('type=text:'):], 0, 0, 0) raw_text = raw_doc.RawText(translateTokens([t])) else: t = lexer.Token('WORD', text, 0, 0, 0) raw_text = raw_doc.RawText(translateTokens([t])) entry.addParagraph(raw_doc.RawParagraph(raw_text))
def story(tokenQueue): """ The story function takes a token queue as input and returns a token if parsing is successful or False otherwise. >>> tokenQueue = make_test_queue("storypass") >>> _story = story(tokenQueue) >>> _story Token('None', 'STORY', 'None') """ _story = lexer.Token(token_type="STORY") _preamble, tokenQueue = preamble(tokenQueue) #get the preamble #debug... print(tokenQueue) if _preamble == False: # if the preamble is NOT correctly formatted... #stop right here! print("Parsing Error: Preamble not formatted correctly.") return False else: # if preamble is correctly formatted, _story.children.append( _preamble) # append the preamble to the story tree. passages = [] # used to store all passages while tokenQueue.queue[0].token_type != "EOF": print('passages:', passages) _passage, tokenQueue = passage(tokenQueue) # get passage passages.append(_passage) # add passage to passage list for pas in passages: # for each passage... _story.children.append(pas) # append passage to story tree return _story
def _parse(self, lr_table, token_list, code_parser): code_parser.on_parse_start() cursor = 0 state_stack = [0] token_stack = [SYMBOL_END] for token in token_list: action, value = lr_table.get_action(state_stack[-1], token.get_symbol()) while action == ACTION_R: p = self.grammar.get_context().get_production(value) prod_list = [] for _ in xrange(len(p.get_production())): prod_list.insert(0, token_stack.pop()) state_stack.pop() token_stack.append(lexer.Token(p.get_name())) state_stack.append( lr_table.get_goto(state_stack[-1], token_stack[-1].get_symbol())) action, value = lr_table.get_action(state_stack[-1], token.get_symbol()) code_parser.do_semantics(p, token_stack[-1], prod_list) if action == ACTION_S: token_stack.append(token) state_stack.append(value) cursor += 1 elif action == ACTION_ACC: print "==============================lr parse acc==============================" else: print "error" return
def unary_prefix_evaluator(parser, sym): arg = parser.parse_to(sym.rprio) if arg is None: return CompositeNode(sym.token, [Node(lexer.Token('ERROR', 'MISSING VALUE'))]) else: return CompositeNode(sym.token, [arg])
def processFiles(self, entry, node): for text in node.texts: if text.startswith('../'): text = text[len('../'):] t = lexer.Token('WORD', text, 0, 0, 0) raw_text = raw_doc.RawText(translateTokens([t])) entry.addParagraph(raw_doc.RawInclude(raw_text))
def binary_evaluator(parser, left_arg, sym): right_arg = parser.parse_to(sym.rprio) if right_arg is None: return CompositeNode( sym.token, [left_arg, Node(lexer.Token('ERROR', 'MISSING VALUE'))]) else: return CompositeNode(sym.token, [left_arg, right_arg])
def match(self, nodes): types = [node.type for node in nodes] trail = ['expression', 'colon', 'expression', 'comma' ] * (len(types) // 4 + 1) if all(a in b.split('/') for a, b in zip(trail, types)) and len(types) % 4 in [0, 3]: children = [] for i in range(0, len(types), 4): children.append( ASTNode(lexer.Token('mapping/expression', ''), [nodes[i], nodes[i + 2]])) return (len(nodes), ASTNode(lexer.Token('bracket_expr/list/expression', ''), [ ASTNode( lexer.Token('comma/comma_expr/expression', ','), children) ]))
def needLeft(tok, toks): if (tok.tT.tType not in ['Identifier', 'OperatorOnly' ]) or (tok.tT.text not in op.withLeft): # insert space or adjacency subop return L.Token(tT=L.TokTT(text=(" " if tok.whiteB4 else ""),tType='OperatorOnly'), indent=-1,whiteB4=False,location=tok.location),\ U.prependGen(tok,toks) return tok, toks
def _parse_array_offset(self, expr, env): """ Parses the array offset. """ self._match('[') num = self._parse_logical_expression(env) self._match(']') of_type = expr.get_type().get_of_type() width = ast.Constant(lexer.Num(of_type.get_width()), ty.Type.Int) expr_id = expr offset = ast.Binary(lexer.Token('*'), num, width) if type(expr) is ast.Access: expr_id = expr.get_access_id() offset = ast.Binary(lexer.Token('+'), expr.get_offset(), offset) return ast.Access(lexer.Word('[]', lexer.Tag.INDEX), of_type, expr_id, offset)
def _get_next_token(self): try: self.cur_token = self.lexer.token() if self.cur_token is None: self.cur_token = lexer.Token(None, None, None) except lexer.LexerError as e: self._error('Lexer error at position %d' % e.pos)
def argument(tokenQueue): """ This function takes a token queue as input and returns a tuple containing a list of tokens (which may be length 1) and a token queue. >>> _argument, tokenQueue = argument(make_test_queue("argumentpass")) >>> _argument [Token('None', 'ARGUMENT', 'None')] >>> tokenQueue [Token('EOF', 'EOF', 'None')] The list _argument contains a list of the ARGUMENT type tokens. >>> _argument[0] Token('None', 'ARGUMENT', 'None') Each token in this list should have children, which are the actual arguments contained in curly braces. >>> _argument[0].children[0] Token('argument', 'CHARACTER', 'argument') A preamble macro cannot be an argument. >>> _argument, tokenQueue = argument(make_test_queue("argumentfail")) Parsing Error: argument provided is not a valid argument type. """ _argument = lexer.Token(token_type="ARGUMENT") token = next_token(tokenQueue) arguments = [] if token.token_type == "LEFTCURLY": pass else: return (False, None) while token.token_type == "LEFTCURLY": anothertoken = next_token(tokenQueue) if anothertoken.token_type == "CHARACTER": _argument.children.append(anothertoken) anothertoken = next_token(tokenQueue) if anothertoken.token_type == "RIGHTCURLY": arguments.append(_argument) token = next_token(tokenQueue) else: return (False, None) elif anothertoken.token_type == "MACROCOMMAND": _macro, tokenQueue = macro(token, tokenQueue) #print(_macro.children) _argument.children.append(_macro) if _argument: return (_argument, tokenQueue) else: print( "Parsing Error: argument provided is not a valid argument type." ) return (False, None) tokenQueue.put(token) return (arguments, tokenQueue)
def _processTextNode(self, node): texts = [] for text in node.texts: if text.startswith('type=text:'): t = lexer.Token('WORD', text[len('type=text:'):], 0, 0, 0) raw_text = raw_doc.RawParagraph(raw_doc.RawText([t])) elif text.startswith('type=section:#'): t = lexer.Token('WORD', text[len('type=section:#'):], 0, 0, 0) raw_text = raw_doc.RawSection(raw_doc.RawText([t])) elif text.startswith('type=subsection:#.#'): t = lexer.Token('WORD', text[len('type=subsection:#.#'):], 0, 0, 0) raw_text = raw_doc.RawSection(raw_doc.RawText([t]), 1) else: t = lexer.Token('WORD', text, 0, 0, 0) raw_par = raw_doc.RawParagraph(raw_doc.RawText([t])) pars.append(raw_par) return pars
def jumping(self, true_label, false_label, frame): """ Generate Expression's Jumping Code Arguments: true_label: The true branch. false_label: The false branch. """ const = Constant(lexer.Num(0), ty.Type.Int) self.emitjumps(Rel(lexer.Token('>'), self.reduce(frame), const), true_label, false_label, frame)
def parse_for_value(self, tk): if tk.kind == 'NUMBER' or tk.kind == 'ID': self.values_stack.append(Node(tk)) self.waiting_value = False elif tk.lexem in self.prefix_actions: self.prefix_actions[tk.lexem](self) elif tk.lexem in self.prefix_operators: self.push_operator(self.prefix_operators[tk.lexem]) else: self.values_stack.append(Node(lexer.Token('', 'ERROR'))) self.parse_for_operator(tk)
def processExtends(self, entry, node): for text in node.texts: if '\u0001' in text: continue # do not add inherited if text.startswith('Class.'): t = lexer.Token('WORD', text[len('Class.'):], 0, 0, 0) raw_text = raw_doc.RawText([t]) entry.addExtends(raw_doc.RawExtends(raw_text)) elif text.startswith('Spec.'): t = lexer.Token('WORD', text[len('Spec.'):], 0, 0, 0) raw_text = raw_doc.RawText([t]) entry.addExtends(raw_doc.RawExtends(raw_text)) elif text.startswith('Concept.'): t = lexer.Token('WORD', text[len('Concept.'):], 0, 0, 0) raw_text = raw_doc.RawText([t]) if hasattr(entry, 'addImplements'): entry.addImplements(raw_doc.RawImplements(raw_text)) else: entry.addExtends(raw_doc.RawExtends(raw_text)) else: assert False, str(node)
def processExample(self, entry, node): # Add example header. raw_text = raw_doc.RawText([lexer.Token('WORD', 'Examples', 0, 0, 0)]) raw_section = raw_doc.RawSection(raw_text) entry.addParagraph(raw_section) for text in node.texts: if text.startswith('type=text:'): t = lexer.Token('WORD', text[len('type=text:'):], 0, 0, 0) raw_text = raw_doc.RawText(translateTokens([t])) raw_par = raw_doc.RawParagraph(raw_text) elif text.startswith('type=code:'): t = lexer.Token('WORD', '{.cpp}\n' + text[len('type=code:'):] + '\n', 0, 0, 0) raw_text = raw_doc.RawText(translateTokens([t])) raw_par = raw_doc.RawCode(raw_text) else: t = lexer.Token('WORD', text, 0, 0, 0) raw_text = raw_doc.RawText(translateTokens([t])) raw_par = raw_doc.RawParagraph(raw_text) entry.addParagraph(raw_par)
def _get_next_token(self): """Advances the parser's internal lexer to the next token. This method doesn't return anything; it assigns self.cur_token to the next token in the input stream. """ try: self.cur_token = self.lexer.token() if self.cur_token is None: self.cur_token = lexer.Token(None, None, None) except lexer.LexerError as e: self._error('Lexer error at position {}: {}'.format(e.pos, e))
def needNoLeft(tok, toks, opCtx, noneOK): if (tok.tT.tType in ['Identifier','OperatorOnly']) and ((posSubop(tok.tT,opCtx)) or \ ((tok.tT.text not in op.noLeft) and (tok.tT.text in op.withLeft))): # should have had a left or operand preceding subop if noneOK: # none it is then return None, U.prependGen(tok, toks) # better insert defaultOperand defOperandTok = L.Token(tT=L.TokTT(text='!!defaultOperand', tType='OperatorOnly'), indent=None, whiteB4=False, location=tok.location) return defOperandTok, U.prependGen(tok, toks) # backup a bit return tok, toks
def migratePages(node): #print >>sys.stderr, 'Migrating pages...' pages = [] #print node for name, child in node.children.iteritems(): if name == 'Glossary': continue page = raw_doc.RawPage() page.title.tokens.append(lexer.Token('WORD', name, 0, 0, 0)) s = 'Page' page.name.tokens.append( lexer.Token('WORD', s + page.title.text, 0, 0, 0)) if child.children.get('summary'): for text in child.children['summary'].texts: if text.startswith('type=text:'): t = lexer.Token('WORD', text[len('type=text:'):], 0, 0, 0) else: t = lexer.Token('WORD', text, 0, 0, 0) raw_text = raw_doc.RawText(translateTokens([t])) raw_brief = raw_doc.RawBrief(raw_text) page.briefs.append(raw_brief) if child.children.get('description'): for text in child.children['description'].texts: if text.startswith('type=text:'): t = lexer.Token('WORD', text[len('type=text:'):], 0, 0, 0) raw_par = raw_doc.RawParagraph( raw_doc.RawText(translateTokens([t]))) elif text.startswith('type=section:#'): t = lexer.Token('WORD', text[len('type=section:#'):], 0, 0, 0) raw_par = raw_doc.RawSection( raw_doc.RawText(translateTokens([t]))) elif text.startswith('type=subsection:#.#'): t = lexer.Token('WORD', text[len('type=subsection:#.#'):], 0, 0, 0) raw_par = raw_doc.RawSection( raw_doc.RawText(translateTokens([t])), 1) else: t = lexer.Token('WORD', text, 0, 0, 0) raw_par = raw_doc.RawParagraph( raw_doc.RawText(translateTokens([t]))) page.body.addParagraph(raw_par) pages.append(page) print 'RESULTING PAGES %s' % [p.title.text for p in pages] return pages