def get_tokens_unprocessed(self, text): scanner = Scanner(text, re.DOTALL | re.MULTILINE | re.IGNORECASE) stack = ['initial'] in_function_block = False in_property_block = False was_dot = False next_token_is_function = False next_token_is_property = False collect_labels = False block_labels = set() brace_balance = [0, 0] while not scanner.eos: token = Error if stack[-1] == 'initial': if scanner.scan(r'\s+'): token = Text elif scanner.scan(r'\{.*?\}|\(\*.*?\*\)'): if scanner.match.startswith('$'): token = Comment.Preproc else: token = Comment.Multiline elif scanner.scan(r'//.*?$'): token = Comment.Single elif scanner.scan(r'[-+*\/=<>:;,.@\^]'): token = Operator # stop label highlighting on next ";" if collect_labels and scanner.match == ';': collect_labels = False elif scanner.scan(r'[\(\)\[\]]+'): token = Punctuation # abort function naming ``foo = Function(...)`` next_token_is_function = False # if we are in a function block we count the open # braces because ootherwise it's impossible to # determine the end of the modifier context if in_function_block or in_property_block: if scanner.match == '(': brace_balance[0] += 1 elif scanner.match == ')': brace_balance[0] -= 1 elif scanner.match == '[': brace_balance[1] += 1 elif scanner.match == ']': brace_balance[1] -= 1 elif scanner.scan(r'[A-Za-z_][A-Za-z_0-9]*'): lowercase_name = scanner.match.lower() if lowercase_name == 'result': token = Name.Builtin.Pseudo elif lowercase_name in self.keywords: token = Keyword # if we are in a special block and a # block ending keyword occours (and the parenthesis # is balanced) we end the current block context if (in_function_block or in_property_block) and \ lowercase_name in self.BLOCK_KEYWORDS and \ brace_balance[0] <= 0 and \ brace_balance[1] <= 0: in_function_block = False in_property_block = False brace_balance = [0, 0] block_labels = set() if lowercase_name in ('label', 'goto'): collect_labels = True elif lowercase_name == 'asm': stack.append('asm') elif lowercase_name == 'property': in_property_block = True next_token_is_property = True elif lowercase_name in ('procedure', 'operator', 'function', 'constructor', 'destructor'): in_function_block = True next_token_is_function = True # we are in a function block and the current name # is in the set of registered modifiers. highlight # it as pseudo keyword elif in_function_block and \ lowercase_name in self.FUNCTION_MODIFIERS: token = Keyword.Pseudo # if we are in a property highlight some more # modifiers elif in_property_block and \ lowercase_name in ('read', 'write'): token = Keyword.Pseudo next_token_is_function = True # if the last iteration set next_token_is_function # to true we now want this name highlighted as # function. so do that and reset the state elif next_token_is_function: # Look if the next token is a dot. If yes it's # not a function, but a class name and the # part after the dot a function name if scanner.test(r'\s*\.\s*'): token = Name.Class # it's not a dot, our job is done else: token = Name.Function next_token_is_function = False # same for properties elif next_token_is_property: token = Name.Property next_token_is_property = False # Highlight this token as label and add it # to the list of known labels elif collect_labels: token = Name.Label block_labels.add(scanner.match.lower()) # name is in list of known labels elif lowercase_name in block_labels: token = Name.Label elif lowercase_name in self.BUILTIN_TYPES: token = Keyword.Type elif lowercase_name in self.DIRECTIVES: token = Keyword.Pseudo # builtins are just builtins if the token # before isn't a dot elif not was_dot and lowercase_name in self.builtins: token = Name.Builtin else: token = Name elif scanner.scan(r"'"): token = String stack.append('string') elif scanner.scan(r'\#(\d+|\$[0-9A-Fa-f]+)'): token = String.Char elif scanner.scan(r'\$[0-9A-Fa-f]+'): token = Number.Hex elif scanner.scan(r'\d+(?![eE]|\.[^.])'): token = Number.Integer elif scanner.scan(r'\d+(\.\d+([eE][+-]?\d+)?|[eE][+-]?\d+)'): token = Number.Float else: # if the stack depth is deeper than once, pop if len(stack) > 1: stack.pop() scanner.get_char() elif stack[-1] == 'string': if scanner.scan(r"''"): token = String.Escape elif scanner.scan(r"'"): token = String stack.pop() elif scanner.scan(r"[^']*"): token = String else: scanner.get_char() stack.pop() elif stack[-1] == 'asm': if scanner.scan(r'\s+'): token = Text elif scanner.scan(r'end'): token = Keyword stack.pop() elif scanner.scan(r'\{.*?\}|\(\*.*?\*\)'): if scanner.match.startswith('$'): token = Comment.Preproc else: token = Comment.Multiline elif scanner.scan(r'//.*?$'): token = Comment.Single elif scanner.scan(r"'"): token = String stack.append('string') elif scanner.scan(r'@@[A-Za-z_][A-Za-z_0-9]*'): token = Name.Label elif scanner.scan(r'[A-Za-z_][A-Za-z_0-9]*'): lowercase_name = scanner.match.lower() if lowercase_name in self.ASM_INSTRUCTIONS: token = Keyword elif lowercase_name in self.ASM_REGISTERS: token = Name.Builtin else: token = Name elif scanner.scan(r'[-+*\/=<>:;,.@\^]+'): token = Operator elif scanner.scan(r'[\(\)\[\]]+'): token = Punctuation elif scanner.scan(r'\$[0-9A-Fa-f]+'): token = Number.Hex elif scanner.scan(r'\d+(?![eE]|\.[^.])'): token = Number.Integer elif scanner.scan(r'\d+(\.\d+([eE][+-]?\d+)?|[eE][+-]?\d+)'): token = Number.Float else: scanner.get_char() stack.pop() # save the dot!!!11 if scanner.match.strip(): was_dot = scanner.match == '.' yield scanner.start_pos, token, scanner.match or ''
def get_tokens_unprocessed(self, text): scanner = Scanner(text, re.DOTALL | re.MULTILINE | re.IGNORECASE) stack = ['initial'] in_function_block = False in_property_block = False was_dot = False next_token_is_function = False next_token_is_property = False collect_labels = False block_labels = set() brace_balance = [0, 0] while not scanner.eos: token = Error if stack[-1] == 'initial': if scanner.scan(r'\s+'): token = Text elif scanner.scan(r'\{.*?\}|\(\*.*?\*\)'): if scanner.match.startswith('$'): token = Comment.Preproc else: token = Comment.Multiline elif scanner.scan(r'//.*?$'): token = Comment.Single elif scanner.scan(r'[-+*\/=<>:;,.@\^]'): token = Operator # stop label highlighting on next ";" if collect_labels and scanner.match == ';': collect_labels = False elif scanner.scan(r'[\(\)\[\]]+'): token = Punctuation # abort function naming ``foo = Function(...)`` next_token_is_function = False # if we are in a function block we count the open # braces because ootherwise it's impossible to # determine the end of the modifier context if in_function_block or in_property_block: if scanner.match == '(': brace_balance[0] += 1 elif scanner.match == ')': brace_balance[0] -= 1 elif scanner.match == '[': brace_balance[1] += 1 elif scanner.match == ']': brace_balance[1] -= 1 elif scanner.scan(r'[A-Za-z_][A-Za-z_0-9]*'): lowercase_name = scanner.match.lower() if lowercase_name == 'result': token = Name.Builtin.Pseudo elif lowercase_name in self.keywords: token = Keyword # if we are in a special block and a # block ending keyword occours (and the parenthesis # is balanced) we end the current block context if (in_function_block or in_property_block) and \ lowercase_name in self.BLOCK_KEYWORDS and \ brace_balance[0] <= 0 and \ brace_balance[1] <= 0: in_function_block = False in_property_block = False brace_balance = [0, 0] block_labels = set() if lowercase_name in ('label', 'goto'): collect_labels = True elif lowercase_name == 'asm': stack.append('asm') elif lowercase_name == 'property': in_property_block = True next_token_is_property = True elif lowercase_name in ('procedure', 'operator', 'function', 'constructor', 'destructor'): in_function_block = True next_token_is_function = True # we are in a function block and the current name # is in the set of registered modifiers. highlight # it as pseudo keyword elif in_function_block and \ lowercase_name in self.FUNCTION_MODIFIERS: token = Keyword.Pseudo # if we are in a property highlight some more # modifiers elif in_property_block and \ lowercase_name in ('read', 'write'): token = Keyword.Pseudo next_token_is_function = True # if the last iteration set next_token_is_function # to true we now want this name highlighted as # function. so do that and reset the state elif next_token_is_function: # Look if the next token is a dot. If yes it's # not a function, but a class name and the # part after the dot a function name if scanner.test(r'\s*\.\s*'): token = Name.Class # it's not a dot, our job is done else: token = Name.Function next_token_is_function = False # same for properties elif next_token_is_property: token = Name.Property next_token_is_property = False # Highlight this token as label and add it # to the list of known labels elif collect_labels: token = Name.Label block_labels.add(scanner.match.lower()) # name is in list of known labels elif lowercase_name in block_labels: token = Name.Label elif lowercase_name in self.BUILTIN_TYPES: token = Keyword.Type elif lowercase_name in self.DIRECTIVES: token = Keyword.Pseudo # builtins are just builtins if the token # before isn't a dot elif not was_dot and lowercase_name in self.builtins: token = Name.Builtin else: token = Name elif scanner.scan(r"'"): token = String stack.append('string') elif scanner.scan(r'\#(\d+|\$[0-9A-Fa-f]+)'): token = String.Char elif scanner.scan(r'\$[0-9A-Fa-f]+'): token = Number.Hex elif scanner.scan(r'\d+(?![eE]|\.[^.])'): token = Number.Integer elif scanner.scan(r'\d+(\.\d+([eE][+-]?\d+)?|[eE][+-]?\d+)'): token = Number.Float else: # if the stack depth is deeper than once, pop if len(stack) > 1: stack.pop() scanner.get_char() elif stack[-1] == 'string': if scanner.scan(r"''"): token = String.Escape elif scanner.scan(r"'"): token = String stack.pop() elif scanner.scan(r"[^']*"): token = String else: scanner.get_char() stack.pop() elif stack[-1] == 'asm': if scanner.scan(r'\s+'): token = Text elif scanner.scan(r'end'): token = Keyword stack.pop() elif scanner.scan(r'\{.*?\}|\(\*.*?\*\)'): if scanner.match.startswith('$'): token = Comment.Preproc else: token = Comment.Multiline elif scanner.scan(r'//.*?$'): token = Comment.Single elif scanner.scan(r"'"): token = String stack.append('string') elif scanner.scan(r'@@[A-Za-z_][A-Za-z_0-9]*'): token = Name.Label elif scanner.scan(r'[A-Za-z_][A-Za-z_0-9]*'): lowercase_name = scanner.match.lower() if lowercase_name in self.ASM_INSTRUCTIONS: token = Keyword elif lowercase_name in self.ASM_REGISTERS: token = Name.Builtin else: token = Name elif scanner.scan(r'[-+*\/=<>:;,.@\^]+'): token = Operator elif scanner.scan(r'[\(\)\[\]]+'): token = Punctuation elif scanner.scan(r'\$[0-9A-Fa-f]+'): token = Number.Hex elif scanner.scan(r'\d+(?![eE]|\.[^.])'): token = Number.Integer elif scanner.scan(r'\d+(\.\d+([eE][+-]?\d+)?|[eE][+-]?\d+)'): token = Number.Float else: scanner.get_char() stack.pop() # save the dot!!!11 if scanner.match.strip(): was_dot = scanner.match == '.' yield scanner.start_pos, token, scanner.match or ''