def make_ws(tok):
lt = LexToken()
lt.type = "CPP_WS"
lt.value = "\n"
lt.lexer = lexer
lt.lineno = tok.lineno
lt.lexpos = tok.lexpos
return lt
def correct_tag_name(original):
token = LexToken()
token.type = original.type
token.value = original.value[:-1]
token.lineno = original.lineno
token.lexpos = original.lexpos
token.lexer = original.lexer
return token
def remove_first_and_last_char(original):
token = LexToken()
token.type = original.type
token.value = original.value[1:-1]
token.lineno = original.lineno
token.lexpos = original.lexpos
token.lexer = original.lexer
return token
def gen_token(value, type, line, lexpos):
t = LexToken()
t.value = value
t.type = type
t.line = line
t.lexpos = lexpos
t.lexer = self
return t
def gen_token(value, type, line, lexpos):
t = LexToken()
t.value = value
t.type = type
t.line = line
t.lexpos = lexpos
t.lexer = self
return t
def to_tokens(self, token_list):
result = []
for values in token_list:
token = LexToken()
token.type = values[0]
token.value = values[1]
token.lineno = values[2]
token.lexpos = values[3]
token.lexer = self.lexer
result.append(token)
return result
def _gen_token(self, type, value='', lnum=None, position=0, lexpos=None):
"""
Generates a LexToken with the paramaters given.
"""
tok = LexToken()
tok.lexer = self.lex
tok.type = type
tok.value = value
tok.line_position = position
# I think this will work...
tok.lineno = self.lex.lineno if lnum is None else lnum
tok.lexpos = self.lex.lexpos if lexpos is None else lexpos
return tok
def _gen_token(self, type, value='', lnum=None, position=0, lexpos=None):
"""
Generates a LexToken with the paramaters given.
"""
tok = LexToken()
tok.lexer = self.lex
tok.type = type
tok.value = value
tok.line_position = position
# I think this will work...
tok.lineno = self.lex.lineno if lnum is None else lnum
tok.lexpos = self.lex.lexpos if lexpos is None else lexpos
return tok
def _replace_escaped_characters(self, original):
def replace_escape(matchobj):
if matchobj.group("newline"):
return '\n'
if matchobj.group("blank"):
return ' '
if matchobj.group("keep"):
return matchobj.group("keep")
if matchobj.group("tab"):
return '\t'
return None
token = LexToken()
token.type = original.type
token.value = self.tag_escape_replacement.sub(replace_escape,
original.value)
token.lineno = original.lineno
token.lexpos = original.lexpos
token.lexer = original.lexer
return token
def t_ID(t):
global last_id
#p = t.lexer._lexwithprev.peek_i(0)
#if p is not None and p.type == "COLON":
# t.type = "ID_COLON"
# t.lexer._lexwithprev.next()
#else:
# t.type = reserved.get(t.value,'ID') # Check for reserved words
ld = t.lexer.lexdata
li = t.lexpos + len(t.value)
while li < len(ld) and ld[li] in [" ", "\n", "\r", "\t"]:
li += 1
if li < len(ld) and ld[li] == ":" and t.value not in reserved:
t.type = "ID_COLON"
else:
t.type = reserved.get(t.value, 'ID') # Check for reserved words
if class_property_validate(ld, t.lexpos) and last_id != "CLASS_PROP_PRE":
t2 = LexToken()
t2.type = t.type
t2.value = t.value
t2.lineno = t.lineno
t2.lexer = t.lexer
t2.lexpos = t.lexpos
t.lexer._lexwithprev.push(t2)
t.type = "CLASS_PROP_PRE"
t.value = ""
last_id = t.type
#sys.exit()
return t
last_id = t.type
return t