def debug_tokenize_fix(self): lineno = 1 result = [] upto = 0 for match in tag_re.finditer(self.template_string): start, end = match.span() if start > upto: token_string = self.template_string[upto:start] if token_string.strip(): result.append( self.create_token(token_string, (upto, start), lineno, in_tag=False)) lineno += token_string.count('\n') upto = start token_string = self.template_string[start:end] if token_string.strip(): result.append( self.create_token(token_string, (start, end), lineno, in_tag=True)) lineno += token_string.count('\n') upto = end last_bit = self.template_string[upto:] if last_bit.strip(): result.append( self.create_token(last_bit, (upto, upto + len(last_bit)), lineno, in_tag=False)) return result
def tokenize(self): "Return a list of tokens from a given template_string" result, upto = [], 0 for match in tag_re.finditer(self.template_string): start, end = match.span() if start > upto: result.append(self.create_token(self.template_string[upto:start], (upto, start), False)) upto = start result.append(self.create_token(self.template_string[start:end], (start, end), True)) upto = end last_bit = self.template_string[upto:] if last_bit: result.append(self.create_token(last_bit, (upto, upto + len(last_bit)), False)) return result
def tokenize(self): "Return a list of tokens from a given template_string" result, upto = [], 0 for match in tag_re.finditer(self.template_string): start, end = match.span() if start > upto: result.append(self.create_token(self.template_string[upto:start], (upto, start), False)) upto = start result.append(self.create_token(self.template_string[start:end], (start, end), True)) upto = end last_bit = self.template_string[upto:] if last_bit: result.append(self.create_token(last_bit, (upto, upto + len(last_bit)), False)) return result