def get_tokens_unprocessed(self, text):
offset = 0
if re.search(r'^----\s*$', text, re.MULTILINE):
py, _, text = text.partition('----')
lexer = PythonLexer(**self.options)
for i, token, value in lexer.get_tokens_unprocessed(py):
yield i, token, value
offset = i + 1
yield offset, Text, u'----'
offset += 1
lexer = HtmlDjangoLexer(**self.options)
for i, token, value in lexer.get_tokens_unprocessed(text):
yield offset + i, token, value
def get_tokens_unprocessed(self, text):
for index, token, value in \
PythonLexer.get_tokens_unprocessed(self, text):
if token is Name and value in self.EXTRA_KEYWORDS:
yield index, Keyword.Pseudo, value
else:
yield index, token, value
def get_tokens_unprocessed(self, text):
pylexer = PythonLexer(**self.options)
tblexer = PythonTracebackLexer(**self.options)
# print '\nTEXT > \n', text, '\n TEXT'
for line in text.splitlines():
lstrip = line.lstrip()
if lstrip.startswith('Out'):
line = lstrip + '\n'
elif lstrip.startswith('...'):
line = line + '\n'
else:
line = line + '\n'
input_prompt = self.input_prompt.match(line)
output_prompt = self.output_prompt.match(line)
if input_prompt is not None:
yield (0, Generic.Prompt, input_prompt.group())
code = line[input_prompt.end():]
for item in pylexer.get_tokens_unprocessed(code):
yield item
elif output_prompt is not None:
# Use the 'error' token for output. We should probably make
# our own token, but error is typicaly in a bright color like
# red, so it works fine for our output prompts.
yield (0, Generic.Error, output_prompt.group())
index = output_prompt.end()
yield index, Generic.Output, line[index:]
else:
yield 0, Generic.Output, line
def get_tokens_unprocessed(self, text):
offset = 0
if re.search(r'^----\s*$', text, re.MULTILINE):
py, _, text = text.partition('----')
lexer = PythonLexer(**self.options)
for i, token, value in lexer.get_tokens_unprocessed(py):
yield i, token, value
offset = i + 1
yield offset, Text, u'----'
offset += 1
lexer = HtmlDjangoLexer(**self.options)
for i, token, value in lexer.get_tokens_unprocessed(text):
yield offset + i, token, value
def get_tokens_unprocessed(self, text):
pylexer = PythonLexer(**self.options)
tblexer = PythonTracebackLexer(**self.options)
# print '\nTEXT > \n', text, '\n TEXT'
for line in text.splitlines():
lstrip = line.lstrip()
if lstrip.startswith('Out'):
line = lstrip + '\n'
elif lstrip.startswith('...'):
line = line + '\n'
else:
line = line + '\n'
input_prompt = self.input_prompt.match(line)
output_prompt = self.output_prompt.match(line)
if input_prompt is not None:
yield (0, Generic.Prompt, input_prompt.group())
code = line[input_prompt.end():]
for item in pylexer.get_tokens_unprocessed(code):
yield item
elif output_prompt is not None:
# Use the 'error' token for output. We should probably make
# our own token, but error is typicaly in a bright color like
# red, so it works fine for our output prompts.
yield (0, Generic.Error, output_prompt.group())
index = output_prompt.end()
yield index, Generic.Output, line[index:]
else:
yield 0, Generic.Output, line
def get_tokens_unprocessed(self, text):
for index, token, value in PythonLexer.get_tokens_unprocessed(self, text):
if token is Name and value in self.lpy_modules:
# Colourize previously detected modules
yield index, Keyword, value
else:
yield index, token, value
def get_tokens_unprocessed(self, text):
for index, token, value in PythonLexer.get_tokens_unprocessed(
self, text):
if token is Name and value in self._extra_commands:
yield index, Name.Builtin, value
else:
yield index, token, value
def get_tokens_unprocessed(self, text):
for index, token, value in \
PythonLexer.get_tokens_unprocessed(self, text):
if token is Name and value in self.EXTRA_KEYWORDS:
yield index, Keyword.Pseudo, value
else:
yield index, token, value
def get_tokens_unprocessed(self, text):
for index, token, value in PythonLexer.get_tokens_unprocessed(
self, text):
if token is Name and value in self.lpy_modules:
# Colourize previously detected modules
yield index, Keyword, value
else:
yield index, token, value
def get_tokens_unprocessed(self, text):
pylexer = PythonLexer(**self.options)
tblexer = PythonTracebackLexer(**self.options)
curcode = ''
insertions = []
for match in line_re.finditer(text):
line = match.group()
input_prompt = self.input_prompt.match(line)
continue_prompt = self.continue_prompt.match(line.rstrip())
output_prompt = self.output_prompt.match(line)
if line.startswith("#"):
insertions.append((len(curcode), [(0, Comment, line)]))
elif input_prompt is not None:
insertions.append(
(len(curcode), [(0, Other, input_prompt.group())]))
curcode += line[input_prompt.end():]
elif continue_prompt is not None:
insertions.append(
(len(curcode), [(0, Other, continue_prompt.group())]))
curcode += line[continue_prompt.end():]
elif output_prompt is not None:
# Use the 'error' token for output. We should probably make
# our own token, but error is typicaly in a bright color like
# red, so it works fine for our output prompts.
insertions.append((
len(curcode),
[(
0,
Other, #Generic.Error,
output_prompt.group())]))
curcode += line[output_prompt.end():]
else:
if curcode:
for item in do_insertions(
insertions,
pylexer.get_tokens_unprocessed(curcode)):
yield item
curcode = ''
insertions = []
yield match.start(), Generic.Output, line
if curcode:
for item in do_insertions(insertions,
pylexer.get_tokens_unprocessed(curcode)):
yield item
def get_tokens_unprocessed(self, text):
pylexer = PythonLexer(**self.options)
tblexer = PythonTracebackLexer(**self.options)
curcode = ''
insertions = []
for match in line_re.finditer(text):
line = match.group()
input_prompt = self.input_prompt.match(line)
continue_prompt = self.continue_prompt.match(line.rstrip())
output_prompt = self.output_prompt.match(line)
if line.startswith("#"):
insertions.append((len(curcode),
[(0, Comment, line)]))
elif line.startswith("<warning>"):
insertions.append((len(curcode),
[(0, Generic.Error, line[9:])]))
elif input_prompt is not None:
insertions.append((len(curcode),
[(0, Generic.Prompt, input_prompt.group())]))
curcode += line[input_prompt.end():]
elif continue_prompt is not None:
insertions.append((len(curcode),
[(0, Generic.Prompt, continue_prompt.group())]))
curcode += line[continue_prompt.end():]
elif output_prompt is not None:
# Use the 'error' token for output. We should probably make
# our own token, but error is typicaly in a bright color like
# red, so it works fine for our output prompts.
insertions.append((len(curcode),
[(0, Generic.Error, output_prompt.group())]))
curcode += line[output_prompt.end():]
else:
if curcode:
for item in do_insertions(insertions,
pylexer.get_tokens_unprocessed(curcode)):
yield item
curcode = ''
insertions = []
yield match.start(), Generic.Output, line
if curcode:
for item in do_insertions(insertions,
pylexer.get_tokens_unprocessed(curcode)):
yield item
def get_tokens_unprocessed(self, text):
pylexer = PythonLexer(**self.options)
tblexer = PythonTracebackLexer(**self.options)
curcode = ''
insertions = []
for match in line_re.finditer(text):
line = match.group()
input_prompt = self.input_prompt.match(line)
continue_prompt = self.continue_prompt.match(line.rstrip())
output_prompt = self.output_prompt.match(line)
if line.startswith("#"):
insertions.append((len(curcode), [(0, Comment, line)]))
elif input_prompt is not None:
insertions.append((len(curcode), [(0, Generic.Prompt,
input_prompt.group())]))
curcode += line[input_prompt.end():]
elif continue_prompt is not None:
insertions.append((len(curcode), [(0, Generic.Prompt,
continue_prompt.group())]))
curcode += line[continue_prompt.end():]
elif output_prompt is not None:
insertions.append((len(curcode), [(0, Generic.Output,
output_prompt.group())]))
curcode += line[output_prompt.end():]
else:
if curcode:
for item in do_insertions(
insertions,
pylexer.get_tokens_unprocessed(curcode)):
yield item
curcode = ''
insertions = []
yield match.start(), Generic.Output, line
if curcode:
for item in do_insertions(insertions,
pylexer.get_tokens_unprocessed(curcode)):
yield item
def get_tokens_unprocessed(self, text):
pylexer = PythonLexer(**self.options)
tblexer = PythonTracebackLexer(**self.options)
curcode = ''
insertions = []
for match in line_re.finditer(text):
line = match.group()
input_prompt = self.input_prompt.match(line)
continue_prompt = self.continue_prompt.match(line.rstrip())
output_prompt = self.output_prompt.match(line)
if line.startswith("#"):
insertions.append((len(curcode),
[(0, Comment, line)]))
elif input_prompt is not None:
insertions.append((len(curcode),
[(0, Generic.Prompt, input_prompt.group())]))
curcode += line[input_prompt.end():]
elif continue_prompt is not None:
insertions.append((len(curcode),
[(0, Generic.Prompt, continue_prompt.group())]))
curcode += line[continue_prompt.end():]
elif output_prompt is not None:
insertions.append((len(curcode),
[(0, Generic.Output, output_prompt.group())]))
curcode += line[output_prompt.end():]
else:
if curcode:
for item in do_insertions(insertions,
pylexer.get_tokens_unprocessed(curcode)):
yield item
curcode = ''
insertions = []
yield match.start(), Generic.Output, line
if curcode:
for item in do_insertions(insertions,
pylexer.get_tokens_unprocessed(curcode)):
yield item
def get_tokens_unprocessed(self, text):
for index, token, value in PythonLexer.get_tokens_unprocessed(self, text):
if value.startswith("###"):
continue
if token == Token.Error and value == "$":
yield index, Token.Keyword, value
elif token in [ Name, Operator.Word ] and value in KEYWORDS:
yield index, Token.Keyword, value
elif token in Name and value in PROPERTIES:
yield index, Name.Attribute, value
else:
yield index, token, value
def get_tokens_unprocessed(self, text):
for index, token, value in PythonLexer.get_tokens_unprocessed(self, text):
if value.startswith("###"):
continue
if token == Token.Error and value == "$":
yield index, Token.Keyword, value
elif token in [ Name, Operator.Word ] and value in KEYWORDS:
yield index, Token.Keyword, value
elif token in Name and value in PROPERTIES:
yield index, Name.Attribute, value
else:
yield index, token, value
def get_tokens_unprocessed(self, text):
pylexer = PythonLexer(**self.options)
for pos, type_, value in pylexer.get_tokens_unprocessed(text):
if type_ == Token.Error and value == '$':
type_ = Comment.Preproc
yield pos, type_, value
def get_tokens_unprocessed(self, text):
for index, token, value in PythonLexer.get_tokens_unprocessed(self, text):
if token is Name and value in self._extra_commands:
yield index, Name.Builtin, value
else:
yield index, token, value
def get_tokens_unprocessed(self, text):
pylexer = PythonLexer(**self.options)
for pos, type_, value in pylexer.get_tokens_unprocessed(text):
if type_ == Token.Error and value == '$':
type_ = Comment.Preproc
yield pos, type_, value
