def get_location(self, position):
loc = Location(self.source_name, 0, 0, 0)
loc.update(self.data[:position])
return loc
class Lexer(object):
'''Custom, ply-based lexer. Takes the tokens the same way the ply
lexer takes but makes some extra additional comfort features.'''
source_name = None
error_context = None
lexer_object = None
current_location = None
error_tokens = None
warning_tokens = None
error_message = 'Invalid token: @token@'
token_list = None
def __init__(self, source_name, error_context):
self.source_name = source_name
self.error_context = error_context
self.current_location = Location(self.source_name, 0, 0, 0)
self.data = ''
# Create logger for lexer
self.log = logger.getLogger('lexer-%d' % logger.sysid(self))
# Create ply lexer object
log_wrapper = logger.PlyLoggerWrapping(self.log)
self.lexer_object = lex.lex(module=self, debuglog=log_wrapper, errorlog=log_wrapper, debug=1)
self.log.info('Created lexer; class ID=lexer-class-%d' % logger.sysid(self.__class__))
@property
def lineno(self):
if self.lexer_object is None:
return None
return self.lexer_object.lineno
@property
def lexpos(self):
if self.lexer_object is None:
return None
return self.lexer_object.lexpos
def get_location(self, position):
loc = Location(self.source_name, 0, 0, 0)
loc.update(self.data[:position])
return loc
def input(self, data):
self.data = data
self.lexer_object.input(data)
self.log.debug('Lexer got input; length=%d' % len(data))
def token(self):
prev_pos = self.lexer_object.lexpos
token = self.lexer_object.token()
if token is not None:
token.location = self.current_location.clone()
self.log.debug('Got next token; value=%r, type=%s, location=%s' % (token.value, token.type, token.location))
else:
self.log.debug('End of file reached; location=%s' % self.current_location)
data = self.lexer_object.lexdata[prev_pos:self.lexer_object.lexpos]
self.current_location.update(data)
if token is not None:
if token.type in self.error_tokens:
error = replace(self.error_tokens[token.type], type=token.type, value=token.value, location=token.location)
self.error_context.error(error, token.location)
self.log.error(error)
return None
if token.type in self.warning_tokens:
warning = replace(self.warning_tokens[token.type], type=token.type, value=token.value, location=token.location)
self.error_context.warning(warning, token.location)
self.log.warning(warning)
return token
def ignore_token(self, token):
self.log.debug('Ignoring token; value=%r, type=%s, location=%s' % (token.value, token.type, self.current_location))
@property
def tokens(self):
return self.token_list
def t_error(self, t):
token = t.value[0]
self.error_context.error(replace(self.error_message, token=token), self.current_location.clone())
self.lexer_object.skip(len(token))
self.log.error('Error parsing character %r' % token)