def clone_token(old_token, new_type):
token = LexToken()
token.type = new_type
token.value = old_token.value
token.lineno = old_token.lineno
token.lexpos = old_token.lexpos
return token
def new_dedent(amount, token):
tok = LexToken()
tok.type = "DEDENT"
tok.value = amount
tok.lineno = token.lineno
tok.lexpos = token.lexpos
return tok
def emit_autoend(self):
tok = LexToken()
tok.type = "AUTO_END"
tok.value = ""
tok.lineno = self.lineno
tok.lexpos = self.lexpos
return tok
def token(self, value, ty=None):
t = LexToken()
t.type = ty if ty != None else value
t.value = value
t.lineno = -1
t.lexpos = -1
return t
def token(self):
t = LexToken()
c = self.cur
if c >= len(self.str):
return None
c = self.str[c]
if c == "\\": t.type = "BACKSLASH"
elif c == "/": t.type = "DIVIDE"
elif c == "[": t.type = "LSBRACKET"
elif c == "]": t.type = "RSBRACKET"
elif c == "*": t.type = "STAR"
elif c == "\n" or c == "\r": t.type = "LT"
elif re.match(r"[a-zA-Z0-9_$]+", c) != None:
t.type = "ID_PART"
else:
t.type = "UCHAR"
t.value = c
t.lineno = 0
t.lexpos = self.cur
self.cur += 1
print(t)
return t
def make_tok(type_, value, lineno, lexpos):
token = LexToken()
token.type = type_
token.value = value
token.lineno = lineno
token.lexpos = lexpos
return token
def _to_yacc(self, token_type, token_data):
token = LexToken()
token.type = token_type
token.value = (token_type, token_data)
token.lineno = 0 # TODO: file offset
token.lexpos = 0
self.__to_yacc(token)
def str2tok(tok_string):
t = LexToken()
t.type, t.value, t.lineno, t.lexpos = re.fullmatch(
pattern=r'LexToken\(([A-Z_]+),\'([^\']+)\',([0-9]+),([0-9]+)\)',
string=tok_string
).groups()
return t
def _new_token(type, token):
tok = LexToken()
tok.type = type
tok.value = token.value
tok.lineno = token.lineno
tok.lexpos = token.lexpos
return tok
def _lextoken( self, type_, value ) :
tok = LexToken()
tok.type = type_
tok.value = value
tok.lineno = self.lexer.lineno
tok.lexpos = self.lexer.lexpos
return tok
def remove_first_and_last_char(original):
token = LexToken()
token.type = original.type
token.value = original.value[1:-1]
token.lineno = original.lineno
token.lexpos = original.lexpos
token.lexer = original.lexer
return token
def p_class0(self, p):
""" class : CLASS TYPEID '{' feature_list '}' ';' """
tok = LexToken()
tok.type = "OBJECTID"
tok.value = "object"
tok.lineno = -1 #this is just inserted not present
tok.lexpos = -1
p[0] = class_(p[2], tok, p[4], self.filename)
def correct_tag_name(original):
token = LexToken()
token.type = original.type
token.value = original.value[:-1]
token.lineno = original.lineno
token.lexpos = original.lexpos
token.lexer = original.lexer
return token
def createFunctionDefinition(self, def_token, var_token, params, val_node):
lamToken = LexToken()
lamToken.value = 'lambda'
lamToken.type = 'LAMBDA'
return LetNode(def_token, [
VariableNode(var_token),
LambdaNode(lamToken, [Node(None, None, nodes(params)), val_node]),
])
def new_tok(lexpos, tok_type, lineno, value):
# Create a token for return
tok = LexToken()
tok.value = value
tok.lineno = lineno
tok.lexpos = lexpos
tok.type = tok_type
return tok
def newtok(tok, ttype=None):
if tok.type != ttype and (ttype != None or tok.value != ""):
if tok.type != None:
push(tok)
tok = LexToken()
tok.type = ttype
tok.value = ""
return tok
def make_ws(tok):
lt = LexToken()
lt.type = "CPP_WS"
lt.value = "\n"
lt.lexer = lexer
lt.lineno = tok.lineno
lt.lexpos = tok.lexpos
return lt
def gen_token(value, type, line, lexpos):
t = LexToken()
t.value = value
t.type = type
t.line = line
t.lexpos = lexpos
t.lexer = self
return t
def p_expr4(self, p):
""" expr : OBJECTID '(' expr_arg_list ')' """
tok = LexToken()
tok.type = "OBJECTID"
tok.value = "self"
tok.lineno = -1 #this is just inserted not present
tok.lexpos = -1
p[0] = dispatch(object_(tok), p[1], p[3])
p[0].lineno = p.lineno(1)
def p_expr5(self, p):
""" expr : OBJECTID '(' ')' """
tok = LexToken()
tok.type = "OBJECTID"
tok.value = "self"
tok.lineno = -1 #this is just inserted not present
tok.lexpos = -1
p[0] = dispatch(object_(tok), p[1], nil_Expressions())
p.lineno = p.lineno(1)
def _new_token(self, new_type, new_value, lineno: int, lexpos: int):
"""
Creates a new token with the given data.
:return: new token with the given data.
"""
token = LexToken()
token.type = new_type
token.value = new_value
token.lineno = lineno
token.lexpos = lexpos
return token
def to_tokens(self, token_list):
result = []
for values in token_list:
token = LexToken()
token.type = values[0]
token.value = values[1]
token.lineno = values[2]
token.lexpos = values[3]
token.lexer = self.lexer
result.append(token)
return result
def _gen_token(self, type, value='', lnum=None, position=0, lexpos=None):
"""
Generates a LexToken with the paramaters given.
"""
tok = LexToken()
tok.lexer = self.lex
tok.type = type
tok.value = value
tok.line_position = position
# I think this will work...
tok.lineno = self.lex.lineno if lnum is None else lnum
tok.lexpos = self.lex.lexpos if lexpos is None else lexpos
return tok
def _new_token(self,
type=None,
value=None,
lexpos=None,
lineno=None) -> LexToken:
"""
Creates a new lexer token with the given properties.
:return: a new lexer token with the given properties.
"""
token = LexToken()
token.type = type
token.value = value
token.lexpos = lexpos
token.lineno = lineno
def p_error(self, p):
# TODO
if p:
self._errors.append(p)
pass # self._parser.errok()
else:
# hack handle eof, don't know why ply behaves this way
from ply.lex import LexToken
tok = LexToken()
tok.value = self.lexer.lexdata[self.lexer.lexpos:]
tok.lineno = self.lexer.lineno
tok.type = 'error'
tok.lexpos = self.lexer.lexpos
self._parser.errok()
return tok
def peek(self):
p = self.lexer.token()
if p is None:
return None
t = LexToken()
t.type = p.type
t.value = p.value
t.lexpos = p.lexpos
t.lineno = self.lexer.lineno
t._comments = self.lexer.comments
t._comment = self.lexer.comment
t._comment_id = self.lexer.comment_id
p.lineno = self.lexer.lineno
p.lexer = self
self.peeks.append([t, self.lexer.lexpos, self.lineno])
return p
def _replace_escaped_characters(self, original):
def replace_escape(matchobj):
if matchobj.group("newline"):
return '\n'
if matchobj.group("blank"):
return ' '
if matchobj.group("keep"):
return matchobj.group("keep")
if matchobj.group("tab"):
return '\t'
return None
token = LexToken()
token.type = original.type
token.value = self.tag_escape_replacement.sub(replace_escape,
original.value)
token.lineno = original.lineno
token.lexpos = original.lexpos
token.lexer = original.lexer
return token
def t_ID(t):
global last_id
#p = t.lexer._lexwithprev.peek_i(0)
#if p is not None and p.type == "COLON":
# t.type = "ID_COLON"
# t.lexer._lexwithprev.next()
#else:
# t.type = reserved.get(t.value,'ID') # Check for reserved words
ld = t.lexer.lexdata
li = t.lexpos + len(t.value)
while li < len(ld) and ld[li] in [" ", "\n", "\r", "\t"]:
li += 1
if li < len(ld) and ld[li] == ":" and t.value not in reserved:
t.type = "ID_COLON"
else:
t.type = reserved.get(t.value, 'ID') # Check for reserved words
if class_property_validate(ld, t.lexpos) and last_id != "CLASS_PROP_PRE":
t2 = LexToken()
t2.type = t.type
t2.value = t.value
t2.lineno = t.lineno
t2.lexer = t.lexer
t2.lexpos = t.lexpos
t.lexer._lexwithprev.push(t2)
t.type = "CLASS_PROP_PRE"
t.value = ""
last_id = t.type
#sys.exit()
return t
last_id = t.type
return t
def _new_token(type, value, pos):
o = LexToken()
o.type = type
o.value = value
o.lineno, o.lexpos = pos
return o
'MINUS': '-',
'TIMES': '*',
'DIVIDE': '/',
'MODULE': '%',
'SEMICOLON': ';',
'COMMA': ',',
'NULL': 'null',
'ATTRIBUTION': '=',
'IDENT': 'ident',
'FLOAT_CONSTANT': 'float_constant',
'INT_CONSTANT': 'int_constant',
'STRING_CONSTANT': 'string_constant',
'STACK_BOT': '$'
}
STACK_BOT_TOKEN = LexToken()
STACK_BOT_TOKEN.type = 'STACK_BOT'
class CC20202Parser:
def __init__(self):
curr_file_folder = os.path.dirname(__file__)
grammar_path = os.path.join(curr_file_folder, '..', '..', 'grammar',
'ConvCC-2020-2.csf')
cfg_proc = CfgProcessor()
cfg_proc.read(grammar_path)
self.cfg = cfg_proc.cfg
self.start_symbol = cfg_proc.cfg.start_symbol
self.mat = cfg_proc.generate_matrix()