def main():
# Printing a manually constructed list of (Token, text) tuples.
text = [
(Token.Keyword, "print"),
(Token.Punctuation, "("),
(Token.Literal.String.Double, '"'),
(Token.Literal.String.Double, "hello"),
(Token.Literal.String.Double, '"'),
(Token.Punctuation, ")"),
(Token.Text, "\n"),
]
print_formatted_text(PygmentsTokens(text))
# Printing the output of a pygments lexer.
tokens = list(pygments.lex('print("Hello")', lexer=PythonLexer()))
print_formatted_text(PygmentsTokens(tokens))
# With a custom style.
style = Style.from_dict(
{
"pygments.keyword": "underline",
"pygments.literal.string": "bg:#00ff00 #ffffff",
}
)
print_formatted_text(PygmentsTokens(tokens), style=style)
def _get_toolbar_tokens(is_connected, servers, info):
if is_connected:
hosts = ', '.join(re.sub(r'^https?:\/\/', '', a) for a in servers)
return PygmentsTokens([(Token.Toolbar.Status.Key, 'USER: '******'--'),
(Token.Toolbar.Status, ' | '),
(Token.Toolbar.Status.Key, 'SCHEMA: '),
(Token.Toolbar.Status, info.schema or 'doc'),
(Token.Toolbar.Status, ' | '),
(Token.Toolbar.Status.Key, 'HOSTS: '),
(Token.Toolbar.Status, hosts)])
else:
return PygmentsTokens([(Token.Toolbar.Status, 'not connected')])
def pt_init(self):
def get_prompt_tokens():
return [(Token.Prompt, self.prompt)]
if self._ptcomp is None:
compl = IPCompleter(
shell=self.shell,
namespace={},
global_namespace={},
parent=self.shell,
)
self._ptcomp = IPythonPTCompleter(compl)
options = dict(
message=(lambda: PygmentsTokens(get_prompt_tokens())),
editing_mode=getattr(EditingMode, self.shell.editing_mode.upper()),
key_bindings=create_ipython_shortcuts(self.shell),
history=self.shell.debugger_history,
completer=self._ptcomp,
enable_history_search=True,
mouse_support=self.shell.mouse_support,
complete_style=self.shell.pt_complete_style,
style=self.shell.style,
color_depth=self.shell.color_depth,
)
if not PTK3:
options['inputhook'] = self.shell.inputhook
self.pt_loop = asyncio.new_event_loop()
self.pt_app = PromptSession(**options)
def print_color(self, string, end="\n", **kwargs):
"""Prints a color string using prompt-toolkit color management."""
if isinstance(string, str):
tokens = partial_color_tokenize(string)
else:
# assume this is a list of (Token, str) tuples and just print
tokens = string
tokens = PygmentsTokens(tokens)
env = XSH.env
style_overrides_env = env.get("XONSH_STYLE_OVERRIDES", {})
if HAS_PYGMENTS:
self.styler.style_name = env.get("XONSH_COLOR_STYLE")
self.styler.override(style_overrides_env)
proxy_style = _style_from_pygments_cls(
pyghooks.xonsh_style_proxy(self.styler))
else:
proxy_style = merge_styles([
_style_from_pygments_dict(DEFAULT_STYLE_DICT),
_style_from_pygments_dict(style_overrides_env),
])
ptk_print(
tokens,
style=proxy_style,
end=end,
include_default_pygments_style=False,
**kwargs,
)
def pretty_print(data, arg=None):
"""
:param data:
:param arg:
"""
lexer = None
if arg == 'JSON':
data = json.dumps(json.loads(data), indent=2)
lexer = JsonLexer()
elif arg == 'XML':
lexer = HtmlLexer()
elif arg == 'ASCII':
if not data.startswith(conf.error_marker):
rows = data[1:].split("\n")
rows = [r.replace('"', '').split(";") for r in rows if r]
data = tabulate(rows, **conf.tabulate_opts)
return click.echo_via_pager(data)
else:
return eprint(data)
if lexer:
tokens = list(pygments.lex(data, lexer=lexer))
print_formatted_text(PygmentsTokens(tokens))
else:
print(data)
def pt_init(self, pt_session_options=None):
"""Initialize the prompt session and the prompt loop
and store them in self.pt_app and self.pt_loop.
Additional keyword arguments for the PromptSession class
can be specified in pt_session_options.
"""
if pt_session_options is None:
pt_session_options = {}
def get_prompt_tokens():
return [(Token.Prompt, self.prompt)]
if self._ptcomp is None:
compl = IPCompleter(shell=self.shell,
namespace={},
global_namespace={},
parent=self.shell)
# add a completer for all the do_ methods
methods_names = [m[3:] for m in dir(self) if m.startswith("do_")]
def gen_comp(self, text):
return [m for m in methods_names if m.startswith(text)]
import types
newcomp = types.MethodType(gen_comp, compl)
compl.custom_matchers.insert(0, newcomp)
# end add completer.
self._ptcomp = IPythonPTCompleter(compl)
# setup history only when we start pdb
if self.shell.debugger_history is None:
if self.shell.debugger_history_file is not None:
p = Path(self.shell.debugger_history_file).expanduser()
if not p.exists():
p.touch()
self.debugger_history = FileHistory(os.path.expanduser(str(p)))
else:
self.debugger_history = InMemoryHistory()
options = dict(
message=(lambda: PygmentsTokens(get_prompt_tokens())),
editing_mode=getattr(EditingMode, self.shell.editing_mode.upper()),
key_bindings=create_ipython_shortcuts(self.shell),
history=self.debugger_history,
completer=self._ptcomp,
enable_history_search=True,
mouse_support=self.shell.mouse_support,
complete_style=self.shell.pt_complete_style,
style=getattr(self.shell, "style", None),
color_depth=self.shell.color_depth,
)
if not PTK3:
options['inputhook'] = self.shell.inputhook
options.update(pt_session_options)
self.pt_loop = asyncio.new_event_loop()
self.pt_app = PromptSession(**options)
def _format_exception_output(self, e: BaseException) -> PygmentsTokens:
# Instead of just calling ``traceback.format_exc``, we take the
# traceback and skip the bottom calls of this framework.
t, v, tb = sys.exc_info()
# Required for pdb.post_mortem() to work.
sys.last_type, sys.last_value, sys.last_traceback = t, v, tb
tblist = list(traceback.extract_tb(tb))
for line_nr, tb_tuple in enumerate(tblist):
if tb_tuple[0] == "<stdin>":
tblist = tblist[line_nr:]
break
l = traceback.format_list(tblist)
if l:
l.insert(0, "Traceback (most recent call last):\n")
l.extend(traceback.format_exception_only(t, v))
tb_str = "".join(l)
# Format exception and write to output.
# (We use the default style. Most other styles result
# in unreadable colors for the traceback.)
if self.enable_syntax_highlighting:
tokens = list(_lex_python_traceback(tb_str))
else:
tokens = [(Token, tb_str)]
return PygmentsTokens(tokens)
def _try_jsonify(self, source):
# If it is a string, first check whether it can be decoded as JSON
if isinstance(source, str):
try:
source = json.loads(source)
except JSONDecodeError:
_logger.debug(f'Source string is not JSON: {source!r}')
try:
if not isinstance(source, str):
source = json.dumps(
source,
cls=PeekEncoder,
app=self.app,
indent=2 if self.pretty_print else None)
tokens = []
for t in pygments.lex(source, lexer=self.payload_lexer):
tokens.append(t)
if t[0] is Token.Error:
_logger.debug(f'Source string is not valid payload type: {t!r}')
return source, source
return PygmentsTokens(tokens), source
except Exception as e:
_logger.debug(f'Cannot render object as json: {source!r}, {e}')
return source, source
def do_show(self, fmt=None, fmt2=None, *args):
if fmt is not None:
self.pyre = PyreParser()
fmt_v4 = ['ipt', 'iptables', 'ip4', 'ipt4', 'iptables4']
fmt_v6 = ['ipt6', 'ip6tables', 'ip6', 'iptables6']
ip4, ip6 = self.pyre.parse_lines(self.buffer)
if fmt in fmt_v4 or fmt2 in fmt_v4:
msg('green',
'Current session parsed into IPv4 iptables rules:')
msg('blue', '### Begin IPTables v4 Rules ###')
for l in ip4:
print(l)
msg('blue', '### End IPTables v4 Rules ###')
if fmt in fmt_v6 or fmt2 in fmt_v6:
msg('green',
'Current session parsed into IPv6 iptables rules:')
msg('blue', '### Begin IPTables v6 Rules ###')
for l in ip6:
print(l)
msg('blue', '### End IPTables v6 Rules ###')
return
msg('green', 'Current PyreWall rules executed during this session:')
msg('### Begin Pyre Rules ###')
tokens = list()
for l in self.buffer:
if empty(l.strip()): continue
tokens += list(pygments.lex(l.strip(), lexer=PyreLexer()))
print_formatted_text(PygmentsTokens(tokens), style=self.style, end='')
# print(l.strip())
msg('### End Pyre Rules ###')
def format_output(self, result):
locals: Dict[str, Any] = self.get_locals()
locals["_"] = locals["_%i" % self.current_statement_index] = result
if result is None:
return None
else:
out_prompt = self.get_output_prompt()
try:
result_str = "%r\n" % (result, )
except UnicodeDecodeError:
# In Python 2: `__repr__` should return a bytestring,
# so to put it in a unicode context could raise an
# exception that the 'ascii' codec can't decode certain
# characters. Decode as utf-8 in that case.
result_str = "%s\n" % repr(result).decode( # type: ignore
"utf-8")
# Align every line to the first one.
line_sep = "\n" + " " * fragment_list_width(out_prompt)
result_str = line_sep.join(result_str.splitlines()).strip("")
# Write output tokens.
if self.enable_syntax_highlighting:
formatted_output = FormattedText(
merge_formatted_text([
out_prompt,
PygmentsTokens(list(_lex_python_result(result_str))),
])())
formatted_output.pop()
else:
formatted_output = FormattedText(out_prompt +
[("", result_str)])
return formatted_output
def print_python(path: list, nodes: dict) -> None:
tokens = []
block_code = path_to_python(path, nodes)
tokens.extend(list(pygments.lex(block_code, lexer=PythonLexer())))
print_formatted_text(PygmentsTokens(tokens))
def _invalidate():
new_prompt = self.tokens.process()
formatted_tokens = tokenize_ansi(
PygmentsTokens(partial_color_tokenize(new_prompt))
)
setattr(self.session, self.name, formatted_tokens)
self.session.app.invalidate()
def _extra_prompt_options(self):
"""
Return the current layout option for the current Terminal InteractiveShell
"""
def get_message():
return PygmentsTokens(self.prompts.in_prompt_tokens())
return {
'complete_in_thread': False,
'lexer':IPythonPTLexer(),
'reserve_space_for_menu':self.space_for_menu,
'message': get_message,
'prompt_continuation': (
lambda width, lineno, is_soft_wrap:
PygmentsTokens(self.prompts.continuation_prompt_tokens(width))),
'multiline': True,
'complete_style': self.pt_complete_style,
# Highlight matching brackets, but only when this setting is
# enabled, and only when the DEFAULT_BUFFER has the focus.
'input_processors': [ConditionalProcessor(
processor=HighlightMatchingBracketProcessor(chars='[](){}'),
filter=HasFocus(DEFAULT_BUFFER) & ~IsDone() &
Condition(lambda: self.highlight_matching_brackets))],
}
def display_bibs(labels, bibs, meta=False):
r"""
Display a list of bib entries on screen with flying colors.
Parameters
----------
labels: List of Strings
Header labels to show above each Bib() entry.
bibs: List of Bib() objects
BibTeX entries to display.
meta: Bool
If True, also display the meta-information.
Examples
--------
>>> import bibmanager.bib_manager as bm
>>> e1 = '''@Misc{JonesEtal2001scipy,
author = {Eric Jones and Travis Oliphant and Pearu Peterson},
title = {{SciPy}: Open source scientific tools for {Python}},
year = {2001},
}'''
>>> e2 = '''@Misc{Jones2001,
author = {Eric Jones and Travis Oliphant and Pearu Peterson},
title = {SciPy: Open source scientific tools for Python},
year = {2001},
}'''
>>> bibs = [bm.Bib(e1), bm.Bib(e2)]
>>> bm.display_bibs(["DATABASE:\n", "NEW:\n"], bibs)
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
DATABASE:
@Misc{JonesEtal2001scipy,
author = {Eric Jones and Travis Oliphant and Pearu Peterson},
title = {{SciPy}: Open source scientific tools for {Python}},
year = {2001},
}
NEW:
@Misc{Jones2001,
author = {Eric Jones and Travis Oliphant and Pearu Peterson},
title = {SciPy: Open source scientific tools for Python},
year = {2001},
}
"""
style = prompt_toolkit.styles.style_from_pygments_cls(
pygments.styles.get_style_by_name(cm.get('style')))
if labels is None:
labels = ["" for _ in bibs]
tokens = [(Token.Comment, u.BANNER)]
for label, bib in zip(labels, bibs):
tokens += [(Token.Text, label)]
if meta:
tokens += [(Token.Comment, bib.meta())]
tokens += list(pygments.lex(bib.content, lexer=BibTeXLexer()))
tokens += [(Token.Text, "\n")]
print_formatted_text(PygmentsTokens(tokens),
end="",
style=style,
output=create_output(sys.stdout))
def print_poc(target: Target, path: list, nodes: dict,
receive_data_after_each_request, receive_data_after_fuzz) -> None:
tokens = []
exploit_code = get_exploit_code(target, path, nodes, receive_data_after_each_request, receive_data_after_fuzz)
tokens.extend(list(pygments.lex(exploit_code, lexer=PythonLexer())))
print_formatted_text(PygmentsTokens(tokens))
def dump_summary(retcode, start, end):
tokens = [(Token.NORMAL, "\n" + "=" * 80 + "\n[Return "),
(Token.NORMAL if retcode == 0 else Token.ALERM, "%d" % retcode),
(Token.NORMAL, "] [Start %s] [End %s] [Elapsed %.3f sec]\n" %
(time.strftime("%H:%M:%S", time.localtime(start)) +
".%d" % int(1000 * (start - int(start))),
time.strftime("%H:%M:%S", time.localtime(end)) +
".%d" % int(1000 * (end - int(end))), end - start))]
print_formatted_text(PygmentsTokens(tokens), style=SUMMARY_STYLE)
def _show_pyre_rules(self):
msg('green', 'Current PyreWall rules executed during this session:')
msg('### Begin Pyre Rules ###')
tokens = list()
for l in self.buffer:
if empty(l.strip()): continue
tokens += list(pygments.lex(l.strip(), lexer=PyreLexer()))
print_formatted_text(PygmentsTokens(tokens), style=self.style, end='')
msg('### End Pyre Rules ###')
def show_result(self, result: object) -> None:
"""
Show __repr__ for an `eval` result.
"""
out_prompt = to_formatted_text(self.get_output_prompt())
# If the repr is valid Python code, use the Pygments lexer.
result_repr = repr(result)
try:
compile(result_repr, "", "eval")
except SyntaxError:
formatted_result_repr = to_formatted_text(result_repr)
else:
formatted_result_repr = to_formatted_text(
PygmentsTokens(list(_lex_python_result(result_repr))))
# If __pt_repr__ is present, take this. This can return
# prompt_toolkit formatted text.
if hasattr(result, "__pt_repr__"):
try:
formatted_result_repr = to_formatted_text(
getattr(result, "__pt_repr__")())
if isinstance(formatted_result_repr, list):
formatted_result_repr = FormattedText(
formatted_result_repr)
except:
pass
# Align every line to the prompt.
line_sep = "\n" + " " * fragment_list_width(out_prompt)
indented_repr: StyleAndTextTuples = []
lines = list(split_lines(formatted_result_repr))
for i, fragment in enumerate(lines):
indented_repr.extend(fragment)
# Add indentation separator between lines, not after the last line.
if i != len(lines) - 1:
indented_repr.append(("", line_sep))
# Write output tokens.
if self.enable_syntax_highlighting:
formatted_output = merge_formatted_text(
[out_prompt, indented_repr])
else:
formatted_output = FormattedText(out_prompt + [(
"", fragment_list_to_text(formatted_result_repr))])
print_formatted_text(
formatted_output,
style=self._current_style,
style_transformation=self.style_transformation,
include_default_pygments_style=False,
output=self.app.output,
)
self.app.output.flush()
def render(output):
dump = json.dumps(output, indent=4)
tokens = list(json_lexer.get_tokens(dump))
with pager(options="-FRSX") as less:
if is_flat(output):
table = tabulate(*tabularize(output), tablefmt="psql").encode("utf-8")
less.write(table)
else:
print_formatted_text(PygmentsTokens(tokens), style=style, file=less)
def _render_highlighted_block(self, content, language):
code = indent(content, " " * 2)
lexer = get_lexer(language or "")
if lexer:
formatted_text = PygmentsTokens(
pygments.lex(code=code, lexer=lexer))
else:
formatted_text = to_formatted_text(
code,
style="",
)
return formatted_text
def _eval_and_print_result(self, input: str):
try:
output = pprint.pformat(
self.runeval(input,
self.frame_history.exec_frame.raw_frame.f_globals,
self.frame_history.exec_frame.frame_locals))
tokens = pygments.lex(output, lexer=PythonLexer())
print_formatted_text(PygmentsTokens(tokens))
except Exception as err:
self._print_exception(err)
else:
self.eval_count += 1
def test_pygments_tokens():
text = [
(('A', 'B'), 'hello'), # Token.A.B
(('C', 'D', 'E'), 'hello'), # Token.C.D.E
((), 'world'), # Token
]
assert to_formatted_text(PygmentsTokens(text)) == [
('class:pygments.a.b', 'hello'),
('class:pygments.c.d.e', 'hello'),
('class:pygments', 'world'),
]
