def guess_lexer(self, lang=None):
if lang is None:
lang = self.config["lang"]
try:
return find_lexer_class_by_name(self.config["lexer"])
except pygments.util.ClassNotFound:
try:
return find_lexer_class_by_name(str.lower(lang))
except pygments.util.ClassNotFound:
return None
def colorize_source(source, lexer_name):
try:
lexer = find_lexer_class_by_name(lexer_name)()
except ClassNotFound:
return source
formatter = Terminal256Formatter()
return highlight(source, lexer, formatter)
def pretty_dumps(json_dict):
"""Format a json dictionary to a colorful and indented string."""
if json_dict is None:
return "None"
formatted_json = json.dumps(json_dict, indent=4)
return highlight(formatted_json,
lexers.find_lexer_class_by_name("JSON")(),
formatters.find_formatter_class("terminal")())
def format_code(self, lang: Optional[str], text: str) -> str:
if lang:
langclass = LANG_TAG.format(lang)
else:
langclass = ""
# Check for code hilite extension
if not self.checked_for_codehilite:
for ext in self.md.registeredExtensions:
if isinstance(ext, CodeHiliteExtension):
self.codehilite_conf = ext.config
break
self.checked_for_codehilite = True
# If config is not empty, then the codehighlite extension
# is enabled, so we call it to highlite the code
if self.codehilite_conf:
highliter = CodeHilite(
text,
linenums=self.codehilite_conf["linenums"][0],
guess_lang=self.codehilite_conf["guess_lang"][0],
css_class=self.codehilite_conf["css_class"][0],
style=self.codehilite_conf["pygments_style"][0],
use_pygments=self.codehilite_conf["use_pygments"][0],
lang=(lang or None),
noclasses=self.codehilite_conf["noclasses"][0],
)
code = highliter.hilite().rstrip("\n")
else:
code = CODE_WRAP.format(langclass, self._escape(text))
# To support our "view in playground" feature, the frontend
# needs to know what Pygments language was used for
# highlighting this code block. We record this in a data
# attribute attached to the outer `pre` element.
# Unfortunately, the pygments API doesn't offer a way to add
# this, so we need to do it in a post-processing step.
if lang:
div_tag = lxml.html.fromstring(code)
# For the value of our data element, we get the lexer
# subclass name instead of directly using the language,
# since that canonicalizes aliases (Eg: `js` and
# `javascript` will be mapped to `JavaScript`).
try:
code_language = find_lexer_class_by_name(lang).name
except ClassNotFound:
# If there isn't a Pygments lexer by this name, we
# still tag it with the user's data-code-language
# value, since this allows hooking up a "playground"
# for custom "languages" that aren't known to Pygments.
code_language = lang
div_tag.attrib["data-code-language"] = code_language
code = lxml.html.tostring(div_tag, encoding="unicode")
return code
def formatPygmentsCodeSnippet(data: dict, html, filepath, lineno):
startline = data.get('startline', None)
endline = data.get('endline', None)
name = data.get('name', None)
file = full_filepath(data['file'], filepath)
# Get the GitHub/GitLab links for this file
git_service, git_link = get_git_remote_link(file, startline, endline)
# Select the lines between startline and endline
filecontents, ctrstart = clip_file_contents(file, startline, endline)
# Select the right lexer based on the filename and contents
if 'lexer' in data:
lexer = find_lexer_class_by_name(data['lexer'])()
else:
lex_filename = path.basename(file)
if lex_filename == 'CMakeLists.txt':
lex_filename += '.cmake'
lexer = guess_lexer_for_filename(lex_filename, filecontents)
# Select the right formatter based on the lexer
cssclass = 'pygments{}'.format(lineno)
if lexer.name == "Arduino" and not "style" in data:
formatter = HtmlFormatter(cssclass=cssclass, style='arduino')
else:
style = data.get('style', VSCodeStyle)
formatter = HtmlFormatter(cssclass=cssclass, style=style)
# Extract the CSS from the formatter, and set the line number offset
css = formatter.get_style_defs('.' + cssclass)
css += '\n.pygments{} pre.snippet{} {{ counter-reset: line {}; }}' \
.format(lineno, lineno, ctrstart)
# Syntax highlight the code
htmlc = highlight(filecontents, lexer, formatter)
# Set the right classes
htmlc = htmlc.replace('<pre>',
'<pre class="lineNumbers snippet{}">'.format(lineno))
htmlc = htmlc.replace('\n</pre></div>', '</pre></div>')
# Construct the final HTML code
datastr = ''
if name is not None:
datastr += '<h4 class="snippet-name">' + name + '</h4>\n'
datastr += '<div class="codesnippet"><style>' + css + '</style>\n'
if git_link is not None and git_service == 'github':
datastr += '<a href="' + git_link + '" title="Open on GitHub">'
datastr += '<img class="github-mark" src="/Images/GitHub-Mark.svg"/>'
datastr += '</a>\n'
if git_link is not None and git_service == 'gitlab':
datastr += '<a href="' + git_link + '" title="Open on GitLab">'
datastr += '<img class="gitlab-mark" src="/Images/GitLab-Mark.svg"/>'
datastr += '</a>\n'
datastr += htmlc + '</div>'
return datastr, file
def __fancy(text, language, fmt):
if not configs._on_rtd and __USE_PYGMENTS:
try:
lang_lex = lexers.find_lexer_class_by_name(language)
fmt = formatters.get_formatter_by_name(fmt)
highlighted = pygments.highlight(text, lang_lex(), fmt)
return highlighted
except:
return text
else:
return text
def __init__(self, lexer_name='yaml'):
self.buf = Buffer()
self.buf.text = ''
self.lexer = PygmentsLexer(find_lexer_class_by_name(lexer_name))
self.window = HSplit([
HorizontalLine(),
Window(content=BufferControl(buffer=self.buf, lexer=self.lexer))
],
height=Dimension(min=5, max=20, weight=1))
super(InfoWindow, self).__init__(content=self.window,
filter=has_focus(self))
def __fancy(text, language, fmt):
if not configs._on_rtd and __USE_PYGMENTS:
try:
lang_lex = lexers.find_lexer_class_by_name(language)
fmt = formatters.get_formatter_by_name(fmt)
highlighted = pygments.highlight(text, lang_lex(), fmt)
return highlighted
except:
return text
else:
return text
def _refresh_source(self):
d = self.declaration
code = d.source
Lexer = (lexers.find_lexer_class_by_name(d.language)
if d.language else lexers.guess_lexer(code))
if Lexer is None:
raise ValueError("Could not determine the proper lexer for {}. "
"Please define it explicitly.".format(d.language))
formatter = formatters.get_formatter_by_name(d.output_format)
if not formatter:
raise ValueError("Could not determine the formatter for {}. "
"Please define it explicitly.".format(
d.output_format))
source = pygments.highlight(code, lexer=Lexer(), formatter=formatter)
super(CodeComponent, self).set_source(source)
def view_crawler_app(request, crawler_id):
crawler_item = Crawler.objects.get(id=crawler_id)
crawler_app = crawler_item.crawler_app
try:
code_text = ''
file_object = open(crawler_app, 'r')
for line in file_object:
code_text = code_text + line
finally:
file_object.close()
highted_code = highlight(code_text,
lexers.find_lexer_class_by_name('python')(),
formatters.find_formatter_class('html')())
model = {'code_text': highted_code}
return render(request, 'html/crawler_app_view.html', model)
def show_paste(request, paste_id):
"""View a Paste with links to delete or edit if you're the appropriate
user."""
p = get_object_or_404(Paste, pk=paste_id)
formatter = HtmlFormatter()
try:
body_html = highlight(
p.body,
lexers.find_lexer_class_by_name(p.language.name)(),
formatter,
)
except (ClassNotFound, AttributeError):
body_html = highlight(p.body, TextLexer(), formatter)
return render(
request,
'show_paste.html',
{
'paste': p.view(),
'paste_body_html': body_html,
'can_edit': user_can_edit_paste(request.user, p),
'is_admin': admin_using_powers(request.user, p),
'rendered': False,
},
)
def get_buffer_window(buffer,
height=20,
lexer_name="",
style="fg:black bg:ansiwhite"):
"""Generate a editable text window for pager in terminal
:param buffer: Buffer object storing the text
:type buffer: str
:param lexer_name: If the editable text should be highlighted with
some kind of grammar, examples are ``yaml``, ``python`` ...
:type lexer_name: str
:param height: Max height of the text area
:type height: int
:param style: Color setting of backgorund and foreground
:type style: str
:return: Container with window information
:rtype : prompt_toolkit.layout.containers.Window
"""
import prompt_toolkit.layout.containers
import prompt_toolkit.layout.controls
from prompt_toolkit.layout import Dimension
from prompt_toolkit.lexers import PygmentsLexer
from pygments.lexers import find_lexer_class_by_name
return prompt_toolkit.layout.containers.Window(
height=None if not height else Dimension(min=0, max=height),
width=None,
style=style,
always_hide_cursor=False,
content=prompt_toolkit.layout.controls.BufferControl(
buffer=buffer,
lexer=PygmentsLexer(find_lexer_class_by_name(lexer_name))))
def highlight(self, code):
Lexer = find_lexer_class_by_name(self.syntax)
formatter = HtmlFormatter()
highlighted = highlight(code, Lexer(), formatter)
return highlighted
def init_lexers() -> None:
lexers.find_lexer_class_by_name("nasm")
lexers.find_lexer_class_by_name("awk")
lexers.find_lexer_class_by_name("bash")
lexers.find_lexer_class_by_name("brainfuck")
lexers.find_lexer_class_by_name("c")
lexers.find_lexer_class_by_name("lisp")
lexers.find_lexer_class_by_name("csharp")
lexers.find_lexer_class_by_name("cpp")
lexers.find_lexer_class_by_name("d")
lexers.find_lexer_class_by_name("ruby")
lexers.find_lexer_class_by_name("emacs")
lexers.find_lexer_class_by_name("elixir")
lexers.find_lexer_class_by_name("haskell")
lexers.find_lexer_class_by_name("go")
lexers.find_lexer_class_by_name("java")
lexers.find_lexer_class_by_name("javascript")
lexers.find_lexer_class_by_name("julia")
lexers.find_lexer_class_by_name("kotlin")
lexers.find_lexer_class_by_name("perl")
lexers.find_lexer_class_by_name("php")
lexers.find_lexer_class_by_name("python3")
lexers.find_lexer_class_by_name("python2")
lexers.find_lexer_class_by_name("rust")
lexers.find_lexer_class_by_name("swift")
lexers.find_lexer_class_by_name("zig")
lexers.find_lexer_class_by_name("nim")
lexers.find_lexer_class_by_name("scala")
lexers.find_lexer_class_by_name("typescript")
lexers.find_lexer_class_by_name("lua")
lexers.find_lexer_class_by_name("crystal")
lexers.find_lexer_class_by_name("text")
def test_pygments():
# This function exists after version 2.2.0 only
from pygments.lexers import find_lexer_class_by_name
yaml = find_lexer_class_by_name('yaml')
assert(yaml is not None)
def text_area(title, text, lexer_name="", height=10, full_screen=False):
"""
Small implementation of an editor/pager for small pieces of text.
:param title: Title of the text_area
:type title: str
:param text: Editable text
:type text: str
:param lexer_name: If the editable text should be highlighted with
some kind of grammar, examples are ``yaml``, ``python`` ...
:type lexer_name: str
:param height: Max height of the text area
:type height: int
:param full_screen: Wether or not the text area should be full screen.
:type full_screen: bool
"""
from prompt_toolkit import Application
from prompt_toolkit.enums import EditingMode
from prompt_toolkit.buffer import Buffer
from prompt_toolkit.layout.containers import HSplit, Window, WindowAlign
from prompt_toolkit.layout.controls import (BufferControl,
FormattedTextControl)
from prompt_toolkit.layout.layout import Layout
from prompt_toolkit.layout import Dimension
from prompt_toolkit.key_binding import KeyBindings
from prompt_toolkit.lexers import PygmentsLexer
from pygments.lexers import find_lexer_class_by_name
assert (type(title) == str)
assert (type(text) == str)
assert (type(lexer_name) == str)
assert (type(height) == int)
assert (type(full_screen) == bool)
kb = KeyBindings()
buffer1 = Buffer()
buffer1.text = text
@kb.add('c-q')
def exit_(event):
event.app.exit(0)
@kb.add('c-s')
def save_(event):
event.app.return_text = buffer1.text
class App(Application):
return_text = None
text_height = Dimension(min=0, max=height) if height is not None else None
pygment_lexer = find_lexer_class_by_name(lexer_name)
lexer = PygmentsLexer(pygment_lexer)
text_window = Window(height=text_height,
content=BufferControl(buffer=buffer1, lexer=lexer))
root_container = HSplit([
Window(char='-',
align=WindowAlign.CENTER,
height=1,
content=FormattedTextControl(text=[('fg:ansiblack bg:ansiwhite',
title)]),
always_hide_cursor=True),
text_window,
Window(height=1,
width=None,
align=WindowAlign.CENTER,
char='-',
content=FormattedTextControl(
text=[('fg:ansiblack bg:ansiwhite',
"Quit [Ctrl-q] Save [Ctrl-s]")])),
])
layout = Layout(root_container)
layout.focus(text_window)
app = App(editing_mode=(EditingMode.EMACS if papis.config.get(
'editmode', section='tui') == 'emacs' else EditingMode.VI),
layout=layout,
key_bindings=kb,
full_screen=full_screen)
app.run()
return app.return_text
def init_lexers() -> None:
"""Declaring all lexers of the languages supported by piston API."""
lexers.find_lexer_class_by_name("nasm")
lexers.find_lexer_class_by_name("awk")
lexers.find_lexer_class_by_name("bash")
lexers.find_lexer_class_by_name("brainfuck")
lexers.find_lexer_class_by_name("c")
lexers.find_lexer_class_by_name("lisp")
lexers.find_lexer_class_by_name("csharp")
lexers.find_lexer_class_by_name("cpp")
lexers.find_lexer_class_by_name("d")
lexers.find_lexer_class_by_name("ruby")
lexers.find_lexer_class_by_name("emacs")
lexers.find_lexer_class_by_name("elixir")
lexers.find_lexer_class_by_name("haskell")
lexers.find_lexer_class_by_name("go")
lexers.find_lexer_class_by_name("java")
lexers.find_lexer_class_by_name("javascript")
lexers.find_lexer_class_by_name("julia")
lexers.find_lexer_class_by_name("kotlin")
lexers.find_lexer_class_by_name("perl")
lexers.find_lexer_class_by_name("php")
lexers.find_lexer_class_by_name("python3")
lexers.find_lexer_class_by_name("python2")
lexers.find_lexer_class_by_name("rust")
lexers.find_lexer_class_by_name("swift")
lexers.find_lexer_class_by_name("zig")
lexers.find_lexer_class_by_name("nim")
lexers.find_lexer_class_by_name("scala")
lexers.find_lexer_class_by_name("typescript")
lexers.find_lexer_class_by_name("lua")
lexers.find_lexer_class_by_name("crystal")
lexers.find_lexer_class_by_name("text")
def print_highlighted(txt, lexer=None, formatter="terminal"):
lexer = find_lexer_class_by_name(lexer) if lexer else guess_lexer(txt)
print(pygments.highlight(txt, lexer, get_formatter_by_name(formatter)))
def unified_diff(from_path: Union[Path, str],
to_path: Union[Path, str],
n: int = 3,
lineterm: str = "\n",
encoding: Optional[str] = None,
no_pygments: bool = False) -> str:
r"""
Return the :func:`unified_diff <difflib.unified_diff>` between two files.
Any errors, such as not being able to read a file, will |fail| the application
abruptly.
Parameters
----------
from_path : pathlib.Path or str
The file to diff from (the "original" file).
to_path : pathlib.Path or str
The file to diff to (the "changed" file).
n : int
Number of context lines. Default: ``3``. Pass-through parameter to
:func:`difflib.unified_diff`.
lineterm : str
Default: ``"\n"``. Pass-through parameter to :func:`difflib.unified_diff`.
encoding : str or None
The encoding to open files with. Default: ``None`` implies default.
Pass-through parameter to :func:`python:open`.
no_pygments : bool
Whether or not an attempt to colorize the output using
`Pygments <http://pygments.org/>`_ using the ``console`` formatter. If Pygments
is not installed, no errors will ensue.
Default: ``False``, always try and make pretty output. Set to ``True`` if you
need to enforce that the returned string does not have colors.
Return
------
str
A string ready to be printed to the console.
"""
# Make sure we have paths we can work with.
if isinstance(from_path, str):
from_path = Path(from_path)
if isinstance(to_path, str):
to_path = Path(to_path)
if not from_path.is_file():
fail("unified_diff: from_path '{from_path}' does not exist!".format(
from_path=str(from_path)))
if not to_path.is_file():
fail("unified_diff: to_path '{to_path}' does not exist!".format(
to_path=str(to_path)))
try:
# difflib wants list of strings, read them in
with from_path.open(encoding=encoding) as from_file:
from_lines = from_file.readlines()
with to_path.open(encoding=encoding) as to_file:
to_lines = to_file.readlines()
# Compute the unified diff <3
diff_generator = difflib.unified_diff(from_lines,
to_lines,
fromfile=str(from_path),
tofile=str(to_path),
n=n,
lineterm=lineterm)
diff_text = "".join(diff_generator)
# Pygments will turn empty string (no diff) into \n, quit now.
if diff_text == "":
return diff_text
if not no_pygments:
try:
import pygments
from pygments import lexers, formatters
lex = lexers.find_lexer_class_by_name("diff")
fmt = formatters.get_formatter_by_name("console")
diff_text = pygments.highlight(diff_text, lex(), fmt)
except: # noqa: E722
pass
return diff_text
except Exception as e:
fail(
"unified_diff: unable to diff '{from_path}' with '{to_path}': {e}".
format(from_path=str(from_path), to_path=str(to_path), e=e))
def _default_lexer(self):
d = self.declaration
if d.language:
return lexers.find_lexer_class_by_name(d.language)()
return lexers.guess_lexer(d.source)
def text_area(
title: str,
text: str,
lexer_name: str = "",
height: int = 10,
full_screen: bool = False) -> str:
"""
Small implementation of an editor/pager for small pieces of text.
:param title: Title of the text_area
:type title: str
:param text: Editable text
:type text: str
:param lexer_name: If the editable text should be highlighted with
some kind of grammar, examples are ``yaml``, ``python`` ...
:type lexer_name: str
:param height: Max height of the text area
:type height: int
:param full_screen: Wether or not the text area should be full screen.
:type full_screen: bool
"""
from prompt_toolkit import Application
from prompt_toolkit.enums import EditingMode
from prompt_toolkit.buffer import Buffer
from prompt_toolkit.layout.containers import HSplit, Window, WindowAlign
from prompt_toolkit.layout.controls import (
BufferControl, FormattedTextControl
)
from prompt_toolkit.layout.layout import Layout
from prompt_toolkit.utils import Event
from prompt_toolkit.layout import Dimension
from prompt_toolkit.key_binding import KeyBindings
from prompt_toolkit.lexers import PygmentsLexer
from pygments.lexers import find_lexer_class_by_name
kb = KeyBindings()
buffer1 = Buffer()
buffer1.text = text
@kb.add('c-q') # type: ignore
def exit_(event: Event) -> None:
event.app.exit(0)
@kb.add('c-s') # type: ignore
def save_(event: Event) -> None:
event.app.return_text = buffer1.text
class App(Application): # type: ignore
# TODO: add stubs to be able to remove type ignore above
return_text = "" # type: str
text_height = Dimension(min=0, max=height) if height is not None else None
pygment_lexer = find_lexer_class_by_name(lexer_name)
lexer = PygmentsLexer(pygment_lexer)
text_window = Window(
height=text_height,
style='bg:black fg:ansiwhite',
content=BufferControl(buffer=buffer1, lexer=lexer))
root_container = HSplit([
Window(
align=WindowAlign.LEFT,
style='bg:ansiwhite',
height=1,
content=FormattedTextControl(
text=[('fg:ansiblack bg:ansiwhite', title)]
),
always_hide_cursor=True
),
text_window,
Window(
height=1,
width=None,
align=WindowAlign.LEFT,
style='bg:ansiwhite',
content=FormattedTextControl(
text=[(
'fg:ansiblack bg:ansiwhite',
"Quit [Ctrl-q] Save [Ctrl-s]"
)]
)
),
])
layout = Layout(root_container)
layout.focus(text_window)
app = App(
editing_mode=EditingMode.EMACS,
layout=layout,
key_bindings=kb,
full_screen=full_screen)
app.run()
return app.return_text
def test_unified_diff(capsys):
"""Validate that |unified_diff| diffs / errors as expected."""
# Invalid from_file should exit.
with pytest.raises(SystemExit):
unified_diff("i_am_not_here", "tox.ini")
captured = capsys.readouterr()
assert captured.out == ""
assert "unified_diff: from_path 'i_am_not_here' does not exist!" in captured.err
# Invalid to_file should exit.
with pytest.raises(SystemExit):
unified_diff("tox.ini", "i_am_not_here")
captured = capsys.readouterr()
assert captured.out == ""
assert "unified_diff: to_path 'i_am_not_here' does not exist!" in captured.err
# Diff between a file and itself should result in the empty string.
empty = unified_diff("tox.ini", "tox.ini")
assert empty == ""
# Do some diffing.
expected_diff_template = textwrap.dedent('''\
--- {backup}
+++ {cmake_lists_txt}
@@ -1,6 +1,6 @@
cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
project(super_project)
-export(PACKAGE super_project)
+# export(PACKAGE super_project)
message(STATUS "PLEASE STOP DOING export(PACKAGE)!")
message(STATUS "At the very least, give us an option() to stop it!")
''').replace("@@\n\n cmake", "@@\n \n cmake")
# NOTE: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ this took a while to figure out xD
# looking at diff of diff in tox output was very confusing hehehe.
def filter_diff(no_pygments):
filter_town, cmake_lists_txt = _make_dummy()
backup = filter_file(cmake_lists_txt,
r"^(\s*export\s*\(PACKAGE.*\).*)$",
r"# \1",
line_based=True)
diff = unified_diff(backup, cmake_lists_txt, no_pygments=no_pygments)
expected_diff = expected_diff_template.format(
backup=str(backup), cmake_lists_txt=str(cmake_lists_txt))
return (diff, expected_diff)
for no_pygments in (True, False):
diff, expected_diff = filter_diff(no_pygments)
if no_pygments:
assert diff == expected_diff
else:
import pygments
from pygments import lexers, formatters
lex = lexers.find_lexer_class_by_name("diff")
fmt = formatters.get_formatter_by_name("console")
assert diff == pygments.highlight(expected_diff, lex(), fmt)
# Force in an error just for shiggles (and because we can).
def superfail(*args, **kwargs):
raise ValueError("superfail")
lexers.find_lexer_class_by_name = superfail
# Attempt to call pygments code now that this raises. Result: original text.
diff, expected_diff = filter_diff(False)
assert diff == expected_diff
# Lastly, make sure the catch-all exception prints the expected message.
import difflib
difflib.unified_diff = superfail
with pytest.raises(SystemExit):
unified_diff("tox.ini", "tox.ini")
captured = capsys.readouterr()
assert captured.out == ""
expected = "unified_diff: unable to diff 'tox.ini' with 'tox.ini': superfail"
assert captured.err.strip().endswith(expected)
# Cleanup.
filter_town, cmake_lists_txt = _make_dummy()
rm_rf(filter_town)
