def clone_token(old_token, new_type): token = LexToken() token.type = new_type token.value = old_token.value token.lineno = old_token.lineno token.lexpos = old_token.lexpos return token
def new_dedent(amount, token): tok = LexToken() tok.type = "DEDENT" tok.value = amount tok.lineno = token.lineno tok.lexpos = token.lexpos return tok
def emit_autoend(self): tok = LexToken() tok.type = "AUTO_END" tok.value = "" tok.lineno = self.lineno tok.lexpos = self.lexpos return tok
def token(self, value, ty=None): t = LexToken() t.type = ty if ty != None else value t.value = value t.lineno = -1 t.lexpos = -1 return t
def token(self): t = LexToken() c = self.cur if c >= len(self.str): return None c = self.str[c] if c == "\\": t.type = "BACKSLASH" elif c == "/": t.type = "DIVIDE" elif c == "[": t.type = "LSBRACKET" elif c == "]": t.type = "RSBRACKET" elif c == "*": t.type = "STAR" elif c == "\n" or c == "\r": t.type = "LT" elif re.match(r"[a-zA-Z0-9_$]+", c) != None: t.type = "ID_PART" else: t.type = "UCHAR" t.value = c t.lineno = 0 t.lexpos = self.cur self.cur += 1 print(t) return t
def make_tok(type_, value, lineno, lexpos): token = LexToken() token.type = type_ token.value = value token.lineno = lineno token.lexpos = lexpos return token
def _to_yacc(self, token_type, token_data): token = LexToken() token.type = token_type token.value = (token_type, token_data) token.lineno = 0 # TODO: file offset token.lexpos = 0 self.__to_yacc(token)
def str2tok(tok_string): t = LexToken() t.type, t.value, t.lineno, t.lexpos = re.fullmatch( pattern=r'LexToken\(([A-Z_]+),\'([^\']+)\',([0-9]+),([0-9]+)\)', string=tok_string ).groups() return t
def _new_token(type, token): tok = LexToken() tok.type = type tok.value = token.value tok.lineno = token.lineno tok.lexpos = token.lexpos return tok
def _lextoken( self, type_, value ) : tok = LexToken() tok.type = type_ tok.value = value tok.lineno = self.lexer.lineno tok.lexpos = self.lexer.lexpos return tok
def remove_first_and_last_char(original): token = LexToken() token.type = original.type token.value = original.value[1:-1] token.lineno = original.lineno token.lexpos = original.lexpos token.lexer = original.lexer return token
def p_class0(self, p): """ class : CLASS TYPEID '{' feature_list '}' ';' """ tok = LexToken() tok.type = "OBJECTID" tok.value = "object" tok.lineno = -1 #this is just inserted not present tok.lexpos = -1 p[0] = class_(p[2], tok, p[4], self.filename)
def correct_tag_name(original): token = LexToken() token.type = original.type token.value = original.value[:-1] token.lineno = original.lineno token.lexpos = original.lexpos token.lexer = original.lexer return token
def createFunctionDefinition(self, def_token, var_token, params, val_node): lamToken = LexToken() lamToken.value = 'lambda' lamToken.type = 'LAMBDA' return LetNode(def_token, [ VariableNode(var_token), LambdaNode(lamToken, [Node(None, None, nodes(params)), val_node]), ])
def new_tok(lexpos, tok_type, lineno, value): # Create a token for return tok = LexToken() tok.value = value tok.lineno = lineno tok.lexpos = lexpos tok.type = tok_type return tok
def newtok(tok, ttype=None): if tok.type != ttype and (ttype != None or tok.value != ""): if tok.type != None: push(tok) tok = LexToken() tok.type = ttype tok.value = "" return tok
def make_ws(tok): lt = LexToken() lt.type = "CPP_WS" lt.value = "\n" lt.lexer = lexer lt.lineno = tok.lineno lt.lexpos = tok.lexpos return lt
def gen_token(value, type, line, lexpos): t = LexToken() t.value = value t.type = type t.line = line t.lexpos = lexpos t.lexer = self return t
def p_expr4(self, p): """ expr : OBJECTID '(' expr_arg_list ')' """ tok = LexToken() tok.type = "OBJECTID" tok.value = "self" tok.lineno = -1 #this is just inserted not present tok.lexpos = -1 p[0] = dispatch(object_(tok), p[1], p[3]) p[0].lineno = p.lineno(1)
def p_expr5(self, p): """ expr : OBJECTID '(' ')' """ tok = LexToken() tok.type = "OBJECTID" tok.value = "self" tok.lineno = -1 #this is just inserted not present tok.lexpos = -1 p[0] = dispatch(object_(tok), p[1], nil_Expressions()) p.lineno = p.lineno(1)
def _new_token(self, new_type, new_value, lineno: int, lexpos: int): """ Creates a new token with the given data. :return: new token with the given data. """ token = LexToken() token.type = new_type token.value = new_value token.lineno = lineno token.lexpos = lexpos return token
def to_tokens(self, token_list): result = [] for values in token_list: token = LexToken() token.type = values[0] token.value = values[1] token.lineno = values[2] token.lexpos = values[3] token.lexer = self.lexer result.append(token) return result
def _gen_token(self, type, value='', lnum=None, position=0, lexpos=None): """ Generates a LexToken with the paramaters given. """ tok = LexToken() tok.lexer = self.lex tok.type = type tok.value = value tok.line_position = position # I think this will work... tok.lineno = self.lex.lineno if lnum is None else lnum tok.lexpos = self.lex.lexpos if lexpos is None else lexpos return tok
def _new_token(self, type=None, value=None, lexpos=None, lineno=None) -> LexToken: """ Creates a new lexer token with the given properties. :return: a new lexer token with the given properties. """ token = LexToken() token.type = type token.value = value token.lexpos = lexpos token.lineno = lineno
def p_error(self, p): # TODO if p: self._errors.append(p) pass # self._parser.errok() else: # hack handle eof, don't know why ply behaves this way from ply.lex import LexToken tok = LexToken() tok.value = self.lexer.lexdata[self.lexer.lexpos:] tok.lineno = self.lexer.lineno tok.type = 'error' tok.lexpos = self.lexer.lexpos self._parser.errok() return tok
def peek(self): p = self.lexer.token() if p is None: return None t = LexToken() t.type = p.type t.value = p.value t.lexpos = p.lexpos t.lineno = self.lexer.lineno t._comments = self.lexer.comments t._comment = self.lexer.comment t._comment_id = self.lexer.comment_id p.lineno = self.lexer.lineno p.lexer = self self.peeks.append([t, self.lexer.lexpos, self.lineno]) return p
def _replace_escaped_characters(self, original): def replace_escape(matchobj): if matchobj.group("newline"): return '\n' if matchobj.group("blank"): return ' ' if matchobj.group("keep"): return matchobj.group("keep") if matchobj.group("tab"): return '\t' return None token = LexToken() token.type = original.type token.value = self.tag_escape_replacement.sub(replace_escape, original.value) token.lineno = original.lineno token.lexpos = original.lexpos token.lexer = original.lexer return token
def t_ID(t): global last_id #p = t.lexer._lexwithprev.peek_i(0) #if p is not None and p.type == "COLON": # t.type = "ID_COLON" # t.lexer._lexwithprev.next() #else: # t.type = reserved.get(t.value,'ID') # Check for reserved words ld = t.lexer.lexdata li = t.lexpos + len(t.value) while li < len(ld) and ld[li] in [" ", "\n", "\r", "\t"]: li += 1 if li < len(ld) and ld[li] == ":" and t.value not in reserved: t.type = "ID_COLON" else: t.type = reserved.get(t.value, 'ID') # Check for reserved words if class_property_validate(ld, t.lexpos) and last_id != "CLASS_PROP_PRE": t2 = LexToken() t2.type = t.type t2.value = t.value t2.lineno = t.lineno t2.lexer = t.lexer t2.lexpos = t.lexpos t.lexer._lexwithprev.push(t2) t.type = "CLASS_PROP_PRE" t.value = "" last_id = t.type #sys.exit() return t last_id = t.type return t
def _new_token(type, value, pos): o = LexToken() o.type = type o.value = value o.lineno, o.lexpos = pos return o
'MINUS': '-', 'TIMES': '*', 'DIVIDE': '/', 'MODULE': '%', 'SEMICOLON': ';', 'COMMA': ',', 'NULL': 'null', 'ATTRIBUTION': '=', 'IDENT': 'ident', 'FLOAT_CONSTANT': 'float_constant', 'INT_CONSTANT': 'int_constant', 'STRING_CONSTANT': 'string_constant', 'STACK_BOT': '$' } STACK_BOT_TOKEN = LexToken() STACK_BOT_TOKEN.type = 'STACK_BOT' class CC20202Parser: def __init__(self): curr_file_folder = os.path.dirname(__file__) grammar_path = os.path.join(curr_file_folder, '..', '..', 'grammar', 'ConvCC-2020-2.csf') cfg_proc = CfgProcessor() cfg_proc.read(grammar_path) self.cfg = cfg_proc.cfg self.start_symbol = cfg_proc.cfg.start_symbol self.mat = cfg_proc.generate_matrix()