def dump_json_output(output, output_path, json_style="pretty"):
"""
Dumps the output to JSON in the output file.
"""
fp = resolve_output_path(output_path)
highlit = json_style == "pretty" and fp == sys.stdout and fp.isatty()
json_encoder = ExtendedJsonEncoder(**JSON_PARAMS[json_style])
if highlit:
ex_json_lexer = ExtendedJsonLexer()
# The LexerContext stores the state of the lexer after each call to get_tokens_unprocessed
lexer_context = LexerContext("", 0)
for chunk in json_encoder.iterencode(output):
lexer_context.text = chunk
lexer_context.pos = 0
lexer_context.end = len(chunk)
token_generator = (
(token_type, value)
for (index, token_type,
value) in ex_json_lexer.get_tokens_unprocessed(
context=lexer_context))
fp.write(pygments.format(token_generator,
get_terminal_formatter()))
else:
for chunk in json_encoder.iterencode(output):
fp.write(chunk)
fp.write("\n")
def intp_string_callback(self, match, ctx):
yield match.start(1), String.Other, match.group(1)
nctx = LexerContext(match.group(3), 0, ['interpolated-string'])
for i, t, v in self.get_tokens_unprocessed(context=nctx):
yield match.start(3) + i, t, v
yield match.start(4), String.Other, match.group(4) # end
ctx.pos = match.end()
def intp_regex_callback(self, match, ctx):
yield match.start(1), String.Regex, match.group(1) # begin
nctx = LexerContext(match.group(3), 0, ['interpolated-regex'])
for i, t, v in self.get_tokens_unprocessed(context=nctx):
yield match.start(3)+i, t, v
yield match.start(4), String.Regex, match.group(4) # end[mixounse]*
ctx.pos = match.end()
def get_tokens_unprocessed(self, text, stack=('root', )):
"""
Split ``text`` into (tokentype, text) pairs.
``stack`` is the inital stack (default: ``['root']``)
"""
tokendefs = self._tokens
self.ctx = ctx = LexerContext(text, 0)
ctx.stack = list(stack)
statetokens = tokendefs[ctx.stack[-1]]
while 1:
for rexmatch, action, new_state in statetokens:
self.m = m = rexmatch(text, ctx.pos, ctx.end)
if m:
if action is not None:
if type(action) is _TokenType:
yield ctx.pos, action, m.group()
ctx.pos = m.end()
else:
if not isinstance(self, ExtendedRegexLexer):
for item in action(self, m):
yield item
ctx.pos = m.end()
else:
for item in action(self, m, ctx):
yield item
if not new_state:
# altered the state stack?
statetokens = tokendefs[ctx.stack[-1]]
if new_state is not None:
# state transition
if isinstance(new_state, tuple):
for state in new_state:
if state == '#pop':
ctx.stack.pop()
elif state == '#push':
ctx.stack.append(ctx.stack[-1])
else:
ctx.stack.append(state)
elif isinstance(new_state, int):
# pop
del ctx.stack[new_state:]
elif new_state == '#push':
ctx.stack.append(ctx.stack[-1])
else:
assert False, 'wrong state def: %r' % new_state
statetokens = tokendefs[ctx.stack[-1]]
break
else:
try:
if ctx.pos >= ctx.end:
break
if text[ctx.pos] == '\n':
# at EOL, reset state to 'root'
ctx.stack = ['root']
statetokens = tokendefs['root']
yield ctx.pos, Text, u'\n'
ctx.pos += 1
continue
yield ctx.pos, Error, text[ctx.pos]
ctx.pos += 1
except IndexError:
break
def get_tokens_unprocessed(self, text=None, context=None):
ctx = LexerContext(text, 0)
ctx.indent = 0
return ExtendedRegexLexer.get_tokens_unprocessed(self,
text,
context=ctx)
def get_tokens_unprocessed(self, text=None, context=None):
ctx = LexerContext(text, 0)
ctx.indent = 0
return ExtendedRegexLexer.get_tokens_unprocessed(self, text, context=ctx)