def get_tokens_unprocessed(self, text): erlexer = ErlangLexer(**self.options) curcode = '' insertions = [] for match in line_re.finditer(text): line = match.group() m = self._prompt_re.match(line) if m is not None: end = m.end() insertions.append((len(curcode), [(0, Generic.Prompt, line[:end])])) curcode += line[end:] else: if curcode: for item in do_insertions(insertions, erlexer.get_tokens_unprocessed(curcode)): yield item curcode = '' insertions = [] if line.startswith('*'): yield match.start(), Generic.Traceback, line else: yield match.start(), Generic.Output, line if curcode: for item in do_insertions(insertions, erlexer.get_tokens_unprocessed(curcode)): yield item
def get_tokens_unprocessed(self, text): mlexer = MatlabLexer(**self.options) curcode = "" insertions = [] for match in line_re.finditer(text): line = match.group() if line.startswith(">>"): insertions.append((len(curcode), [(0, Generic.Prompt, line[:3])])) curcode += line[3:] elif line.startswith("???"): idx = len(curcode) # without is showing error on same line as before...? line = "\n" + line token = (0, Generic.Traceback, line) insertions.append((idx, [token])) else: if curcode: for item in do_insertions(insertions, mlexer.get_tokens_unprocessed(curcode)): yield item curcode = "" insertions = [] yield match.start(), Generic.Output, line if curcode: # or item: for item in do_insertions(insertions, mlexer.get_tokens_unprocessed(curcode)): yield item
def get_tokens_unprocessed(self, text): mlexer = MatlabLexer(**self.options) curcode = '' insertions = [] for match in line_re.finditer(text): line = match.group() if line.startswith('>>'): insertions.append( (len(curcode), [(0, Generic.Prompt, line[:3])])) curcode += line[3:] elif line.startswith('???'): idx = len(curcode) # without is showing error on same line as before...? line = "\n" + line token = (0, Generic.Traceback, line) insertions.append((idx, [ token, ])) else: if curcode: for item in do_insertions( insertions, mlexer.get_tokens_unprocessed(curcode)): yield item curcode = '' insertions = [] yield match.start(), Generic.Output, line if curcode: # or item: for item in do_insertions(insertions, mlexer.get_tokens_unprocessed(curcode)): yield item
def get_tokens_unprocessed(self, text): hslexer = HaskellLexer(**self.options) style = self.options.get('litstyle') if style is None: style = (text.lstrip()[0] in '%\\') and 'latex' or 'bird' code = '' insertions = [] if style == 'bird': # bird-style for match in line_re.finditer(text): line = match.group() m = bird_re.match(line) if m: insertions.append((len(code), [(0, Comment.Special, m.group(1))])) code += m.group(2) else: insertions.append((len(code), [(0, Text, line)])) else: # latex-style from pygments3.lexers.text import TexLexer lxlexer = TexLexer(**self.options) codelines = 0 latex = '' for match in line_re.finditer(text): line = match.group() if codelines: if line.lstrip().startswith('\\end{code}'): codelines = 0 latex += line else: code += line elif line.lstrip().startswith('\\begin{code}'): codelines = 1 latex += line insertions.append((len(code), list(lxlexer.get_tokens_unprocessed(latex)))) latex = '' else: latex += line insertions.append((len(code), list(lxlexer.get_tokens_unprocessed(latex)))) for item in do_insertions(insertions, hslexer.get_tokens_unprocessed(code)): yield item