def set_lexer_from_filename(self, filename):
"""
Change the lexer based on the filename (actually only the extension is
needed)
:param filename: Filename or extension
"""
if filename.endswith("~"):
filename = filename[0:len(filename) - 1]
try:
self._lexer = get_lexer_for_filename(filename)
_logger().debug('lexer for filename (%s): %r', filename,
self._lexer)
except ClassNotFound:
_logger().warning('failed to get lexer from filename: %s, using '
'plain text instead...', filename)
self._lexer = TextLexer()
return False
except ImportError:
# import error while loading some pygments plugins, the editor
# should not crash
_logger().warning('failed to get lexer from filename: %s, using '
'plain text instead...', filename)
self._lexer = TextLexer()
return False
else:
return True
def set_lexer_from_filename(self, filename):
"""
Change the lexer based on the filename (actually only the extension is
needed)
:param filename: Filename or extension
"""
if filename.endswith("~"):
filename = filename[0:len(filename) - 1]
try:
self._lexer = get_lexer_for_filename(filename)
_logger().debug('lexer for filename (%s): %r', filename,
self._lexer)
except ClassNotFound:
_logger().warning(
'failed to get lexer from filename: %s, using '
'plain text instead...', filename)
self._lexer = TextLexer()
return False
except ImportError:
# import error while loading some pygments plugins, the editor
# should not crash
_logger().warning(
'failed to get lexer from filename: %s, using '
'plain text instead...', filename)
self._lexer = TextLexer()
return False
else:
return True
def set_lexer_from_filename(self, filename):
"""
Change the lexer based on the filename (actually only the extension is
needed)
:param filename: Filename or extension
"""
self._lexer = None
if filename.endswith("~"):
filename = filename[0:len(filename) - 1]
try:
self._lexer = get_lexer_for_filename(filename)
except (ClassNotFound, ImportError):
print(('class not found for url', filename))
try:
m = mimetypes.guess_type(filename)
print(m)
self._lexer = get_lexer_for_mimetype(m[0])
except (ClassNotFound, IndexError, ImportError):
self._lexer = get_lexer_for_mimetype('text/plain')
if self._lexer is None:
_logger().warning(
'failed to get lexer from filename: %s, using '
'plain text instead...', filename)
self._lexer = TextLexer()
def getOutput(buildRetCode, buildLog, stdInputList, cmdArgsList, exitTypeList, stdoutStrList):
s = '<pre>\n'
if buildRetCode!=0: # build error
s += buildLog
else:
assert(len(stdInputList)==len(cmdArgsList))
for i in range(len(stdInputList)):
stdInput = stdInputList[i]
cmdArgs = cmdArgsList[i]
exitType = exitTypeList[i]
stdoutStr = stdoutStrList[i]
if exitType == 0:
s += '(STD_INPUT: %s)\n'%stdInput
s += '(CMD_ARGS: %s)\n'%cmdArgs
# success, unistr = getUnicodeStr(stdoutStr)
# s += highlight(unistr, TextLexer(), HtmlFormatter())
s += highlight(stdoutStr, TextLexer(), HtmlFormatter())
elif exitType == -1:
s += highlight(stdoutStr, TextLexer(), HtmlFormatter())
elif exitType == 1: # time out
s += '(STD_INPUT: %s)\n'%stdInput
s += '(CMD_ARGS: %s)\n'%cmdArgs
s += 'Timeout'
s += '\n'
return s
def render(file, size, browser=False, color=True, fontSize="10pt", margin="0.5in", relative=True, title=None):
"""Render file with filename as HTML page(s) of specified size."""
code = file.read().decode("utf-8", "ignore")
if code.strip() and color:
try:
lexer = get_lexer_for_filename(file.filename)
except:
try:
assert code.startswith("#!") # else, e.g., a .gitignore file with a dotfile is mistaken by GuessLexer
lexer = guess_lexer(code.splitlines()[0])
except:
lexer = TextLexer()
string = highlight(code, lexer, HtmlFormatter(linenos="inline", nobackground=True))
else:
string = highlight(code, TextLexer(), HtmlFormatter(linenos="inline", nobackground=True))
if not title:
title = file.filename
stylesheets = [
CSS(string="@page {{ border-top: 1px #808080 solid; margin: {}; padding-top: 1em; size: {}; }}".format(margin, size)),
CSS(string="@page {{ @top-right {{ color: #808080; content: '{}'; padding-bottom: 1em; vertical-align: bottom; }} }}".format(
title.replace("'", "\'"))),
CSS(string="* {{ font-family: monospace; font-size: {}; margin: 0; overflow-wrap: break-word; white-space: pre-wrap; }}".format(fontSize)),
CSS(string=HtmlFormatter().get_style_defs('.highlight')),
CSS(string=".highlight { background: initial; }"),
CSS(string=".lineno { color: #808080; }"),
CSS(string=".lineno:after { content: ' '; }")]
HTML(string=string).write_pdf('/tmp/output.pdf', stylesheets=stylesheets)
def get_lexer(self):
# this function gets the lexer depending on the files name
try:
lexer = get_lexer_for_filename(self.short_name)
except pygments.util.ClassNotFound:
lexer = TextLexer()
lexer.add_filter('tokenmerge')
return lexer
def _ensure_lexer(self):
if self.lexer is not None:
return
try:
lexer = pygments_cache.get_lexer_for_filename(self.name)
except pygments.util.ClassNotFound:
lexer = TextLexer()
lexer = Python3Lexer() if isinstance(lexer, PythonLexer) else lexer
lexer.add_filter(NonEmptyFilter())
lexer.add_filter('tokenmerge')
self.lexer = self.line_kwargs['lexer'] = lexer
def __init__(self, name, main_display, tabsize, multiline_window=1500,
number_of_windows=1):
self.name = name
self.file = f = open(name)
try:
lexer = guess_lexer_for_filename(name, f.readline())
except TypeError:
try:
lexer = get_lexer_by_name(os.path.splitext(name)[1][1:])
except pygments.util.ClassNotFound:
lexer = TextLexer()
except pygments.util.ClassNotFound:
lexer = TextLexer()
lexer = Python3Lexer() if isinstance(lexer, PythonLexer) else lexer
lexer.add_filter(NonEmptyFilter())
lexer.add_filter('tokenmerge')
f.seek(0)
self.lines = []
self.focus = 0
self.clipboard = None
self.clipboard_pos = None
self.lexer = lexer
self.w_pos = {}
self.all_tokens = None
self._etext = lambda w: w.edit_text
self.multiline_window = multiline_window
self.number_of_windows = number_of_windows
self.main_display = main_display
self.line_kwargs = dict(caption="", allow_tab=True, lexer=lexer,
wrap='clip', main_display=main_display,
smart_home=True, tabsize=tabsize)
def preview():
_csrf_protect()
# Set lexer
try:
lexer = get_lexer_by_name(request.form['lexer'])
except ClassNotFound:
lexer = TextLexer()
kwargs = {}
# Set option values for pygments
# font_size
try:
font_size = int(request.form['font_size'])
if font_size > 0:
kwargs['font_size'] = font_size
except ValueError:
pass
if not 'font_size' in kwargs.keys():
kwargs['font_size'] = DEFAULT_FONT_SIZE
# line_numbers
if not 'is_display_linenum' in request.form:
kwargs['line_numbers'] = False
binary = _generate(request.form['text'], lexer, kwargs)
return Response(binary.encode("base64"), mimetype='text/plain')
def view_file(filename):
""" Views file with syntax highlighting (if applicable)
Args:
filename (str): Full path to filename to render view response for.
"""
folder = filename.split(_AEON_TOPDIR).pop().strip('/')
filename = os.path.join(_AEON_TOPDIR, filename)
try:
with open(filename, 'r') as f:
data = f.read()
# lexer = guess_lexer_for_filename(filename, data)
formatter = HtmlFormatter(linenos=True)
try:
lexer = get_lexer_for_filename(filename)
code = highlight(data, lexer, formatter)
except ClassNotFound:
lexer = TextLexer()
code = highlight(data, lexer, formatter)
stat = os.stat(filename)
return render_template('view.html',
content=code,
folder=folder,
stat=stat,
filename=filename)
except (OSError, IOError) as e:
code = e[0]
reason = e[1]
flash(
'Error: Could not view file {filename}: {reason} ({code})'.format(
filename=filename, reason=reason, code=code), 'danger')
return render_template('view.html')
def _get_pygments_lexer(language):
if language == 'ipython2':
try:
from IPython.lib.lexers import IPythonLexer
except ImportError:
warn("IPython lexer unavailable, falling back on Python")
language = 'python'
else:
return IPythonLexer()
elif language == 'ipython3':
try:
from IPython.lib.lexers import IPython3Lexer
except ImportError:
warn("IPython3 lexer unavailable, falling back on Python 3")
language = 'python3'
else:
return IPython3Lexer()
try:
return get_lexer_by_name(language, stripall=True)
except ClassNotFound:
warn("No lexer found for language %r. Treating as plain text." % language)
from pygments.lexers.special import TextLexer
return TextLexer()
def languageLexerToPlain(self, event=None):
print("Setting language to: Plain Text")
self.lexer = TextLexer()
self.currentLanguageName = "Plain Text"
self.create_tags()
self.recolorize()
self.updateStatusBar()
def get_text(self):
quote = re.compile('^> (.*?)\n', re.MULTILINE|re.DOTALL)
quote_results = quote.findall(self.text)
#print self.text.encode('ascii', 'ignore')
for quote_m in quote_results:
#print quote_m.encode('ascii', 'ignore')
self.text = self.text.replace("> " + quote_m + "\n", "<blockquote>" + quote_m + "</blockquote>" )
pre = re.compile('```(.*?)```', re.MULTILINE|re.DOTALL)
pre_results = pre.findall(self.text)
for pre_m in pre_results:
from pygments import highlight
from pygments.lexers import guess_lexer
from pygments.lexers.special import TextLexer
from pygments.formatters import HtmlFormatter
from pygments.util import ClassNotFound
try:
lexer = guess_lexer(self.unescape_html(pre_m[:2048]))
except (ClassNotFound, ValueError):
lexer = TextLexer()
formatted = highlight(self.unescape_html(pre_m), lexer, HtmlFormatter())
self.text = self.text.replace("```" + pre_m + "```", formatted )
code = re.compile('`(.*?)`', re.MULTILINE|re.DOTALL)
code_results = code.findall(self.text)
for code_m in code_results:
self.text = self.text.replace("`" + code_m + "`", "<code>" + code_m + "</code>" )
p = re.compile('<(.*?)>')
results = p.findall(self.text)
for m in results:
if m.startswith('@U'):
slack_id = m.replace("@",'').split("|")[0]
try:
user = User.objects.get(slack_id = slack_id)
except:
user = None
if user:
self.text = self.text.replace("<" + m + ">", user.slack_username )
if m.startswith('http'):
link = m.split("|")[0]
try:
title = m.split("|")[1]
except IndexError:
title = ''
if False: #link.endswith(('png', 'jpg', 'jpeg', 'gif',)):
self.text = self.text.replace("<" + m + ">", "<a href='"+link + "' title='"+title+"'>" + "<img width=300px src='"+link+"'>" + "</a>" )
else:
self.text = self.text.replace("<" + m + ">", "<a href='"+link + "' title='"+title+"'>" + link + "</a>" )
return self.text
def format_html(filename, source):
"""Format the source text given as HTML with Pygments."""
try:
lexer = guess_lexer_for_filename(filename, source, stripnl=False)
except ClassNotFound:
lexer = TextLexer(stripnl=False)
formatter = HtmlFormatter(nowrap=True)
return highlight(source, lexer, formatter)
def format_rtf(t):
tokensource = list(TextLexer().get_tokens(t))
fmt = RtfFormatter()
buf = StringIO()
fmt.format(tokensource, buf)
result = buf.getvalue()
buf.close()
return result
def _pygments_highlight(source,
output_formatter,
language="ipython",
metadata=None):
"""
Return a syntax-highlighted version of the input source
Parameters
----------
source : str
source of the cell to highlight
output_formatter : Pygments formatter
language : str
language to highlight the syntax of
metadata : NotebookNode cell metadata
metadata of the cell to highlight
"""
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.util import ClassNotFound
# If the cell uses a magic extension language,
# use the magic language instead.
if language.startswith(
"ipython") and metadata and "magics_language" in metadata:
language = metadata["magics_language"]
lexer = None
if language == "ipython2":
try:
from IPython.lib.lexers import IPythonLexer
except ImportError:
warn("IPython lexer unavailable, falling back on Python")
language = "python"
else:
lexer = IPythonLexer()
elif language == "ipython3":
try:
from IPython.lib.lexers import IPython3Lexer
except ImportError:
warn("IPython3 lexer unavailable, falling back on Python 3")
language = "python3"
else:
lexer = IPython3Lexer()
if lexer is None:
try:
lexer = get_lexer_by_name(language, stripall=True)
except ClassNotFound:
warn("No lexer found for language %r. Treating as plain text." %
language)
from pygments.lexers.special import TextLexer
lexer = TextLexer()
return highlight(source, lexer, output_formatter)
def _get_lexer(filename: Optional[PurePath], text: str) -> Lexer:
if filename:
with suppress(ClassNotFound):
return get_lexer_for_filename(normcase(filename.name))
with suppress(ClassNotFound):
return guess_lexer(text)
return TextLexer()
def detect_from_name():
_csrf_protect()
# Detect lexer
try:
lexer = get_lexer_for_filename(request.form['name'])
except ClassNotFound:
lexer = TextLexer()
return Response(lexer.aliases[0], mimetype='text/plain')
def jinja_pygments_formatter(data):
formatter = HtmlFormatter(cssclass='codehilite')
if isinstance(data, dict) or isinstance(data, list):
data = jsonutils.dumps(data, indent=4, sort_keys=True)
lexer = JsonLexer()
elif six.string_types or six.text_type:
try:
data = jsonutils.dumps(jsonutils.loads(data),
indent=4,
sort_keys=True)
lexer = JsonLexer()
except (ValueError, TypeError):
lexer = TextLexer()
else:
lexer = TextLexer()
lexer.stripall = True
return highlight(Markup(data).unescape(), lexer, formatter)
def guess_lexer_for_filename(filename):
from pygments.lexers import get_lexer_for_filename
from pygments.util import ClassNotFound
try:
lexer = get_lexer_for_filename(filename)
except ClassNotFound:
from pygments.lexers.special import TextLexer
lexer = TextLexer()
return lexer
def _parse_src(cls, src_contents, src_filename):
"""
Return a stream of `(token_type, value)` tuples
parsed from `src_contents` (str)
Uses `src_filename` to guess the type of file
so it can highlight syntax correctly.
"""
# Parse the source into tokens
try:
lexer = guess_lexer_for_filename(src_filename, src_contents)
except ClassNotFound:
lexer = TextLexer()
# Ensure that we don't strip newlines from
# the source file when lexing.
lexer.stripnl = False
return pygments.lex(src_contents, lexer)
def format_data(data):
formatter = HtmlFormatter(cssclass="codehilite")
if isinstance(data, bool) or isinstance(data, int) or isinstance(
data, float):
return highlight(str(data), TextLexer(), formatter)
elif isinstance(data, str):
try:
data = json.dumps(json.loads(data), indent=4, sort_keys=True)
lexer = JsonLexer()
except (ValueError, TypeError):
lexer = TextLexer()
elif isinstance(data, dict) or isinstance(data, list):
data = json.dumps(data, indent=4, sort_keys=True)
lexer = JsonLexer()
else:
lexer = TextLexer()
lexer.stripall = True
return highlight(data, lexer, formatter)
def run(self):
self.assert_has_content()
try:
lexer = get_lexer_by_name(self.arguments[0])
except ValueError:
# no lexer found - use the text one instead of an exception
lexer = TextLexer()
# take an arbitrary option if more than one is given
formatter = self.options and VARIANTS[self.options.keys()[0]] or DEFAULT
parsed = highlight(u'\n'.join(self.content), lexer, formatter)
return [nodes.raw('', parsed, format='html')]
def _parse_src(cls, src_contents, src_filename):
"""
Return a stream of `(token_type, value)` tuples
parsed from `src_contents` (str)
Uses `src_filename` to guess the type of file
so it can highlight syntax correctly.
"""
# Parse the source into tokens
try:
lexer = guess_lexer_for_filename(src_filename, src_contents)
except ClassNotFound:
lexer = TextLexer()
# Ensure that we don't strip newlines from
# the source file when lexing.
lexer.stripnl = False
return pygments.lex(src_contents, lexer)
def getOutput(buildRetCode, buildLog, userInputList, exitTypeList, stdoutStrList):
s = '<pre>\n'
if buildRetCode!=0: # build error
s += buildLog
else:
for i in range(len(userInputList)):
userInput = userInputList[i]
exitType = exitTypeList[i]
stdoutStr = stdoutStrList[i]
if exitType == 0:
s += '(user input: %s)\n'%userInput
# success, unistr = getUnicodeStr(stdoutStr)
# s += highlight(unistr, TextLexer(), HtmlFormatter())
s += highlight(stdoutStr, TextLexer(), HtmlFormatter())
elif exitType == -1:
s += highlight(stdoutStr, TextLexer(), HtmlFormatter())
elif exitType == 1: # time out
s += '(user input: %s)\n'%userInput
s += 'Timeout'
s += '\n'
return s
def set_mime_type(self, mime_type):
"""
Update the highlighter lexer based on a mime type.
:param mime_type: mime type of the new lexer to setup.
"""
try:
self.set_lexer_from_mime_type(mime_type)
except ClassNotFound:
_logger().exception('failed to get lexer from mimetype')
self._lexer = TextLexer()
return False
except ImportError:
# import error while loading some pygments plugins, the editor
# should not crash
_logger().warning('failed to get lexer from mimetype (%s)' %
mime_type)
self._lexer = TextLexer()
return False
else:
return True
def set_mime_type(self, mime_type):
"""
Update the highlighter lexer based on a mime type.
:param mime_type: mime type of the new lexer to setup.
"""
try:
self.set_lexer_from_mime_type(mime_type)
except ClassNotFound:
_logger().exception('failed to get lexer from mimetype')
self._lexer = TextLexer()
return False
except ImportError:
# import error while loading some pygments plugins, the editor
# should not crash
_logger().warning('failed to get lexer from mimetype (%s)' %
mime_type)
self._lexer = TextLexer()
return False
else:
return True
def get_context(self, value, parent_context=None):
context = super().get_context(value, parent_context=parent_context)
try:
lexer = get_lexer_by_name(value['language'], stripall=True)
except ClassNotFound:
lexer = TextLexer(stripall=True)
formatter = HtmlFormatter(linenos=None,
cssclass='codehilite',
noclasses=False)
context['html_code'] = highlight(value['code'], lexer, formatter)
context['html_code'] = mark_safe(context['html_code'])
return context
def __init__(self, name, main_display, tabsize, multiline_window=1500,
number_of_windows=1):
self.name = name
self.file = f = open(name)
try:
lexer = guess_lexer_for_filename(name, f.readline())
except TypeError:
try:
lexer = get_lexer_by_name(os.path.splitext(name)[1][1:])
except pygments.util.ClassNotFound:
lexer = TextLexer()
except pygments.util.ClassNotFound:
lexer = TextLexer()
lexer = Python3Lexer() if isinstance(lexer, PythonLexer) else lexer
lexer.add_filter(NonEmptyFilter())
lexer.add_filter('tokenmerge')
f.seek(0)
self.lines = []
self.focus = 0
self.clipboard = None
self.clipboard_pos = None
self.lexer = lexer
self.w_pos = {}
self.all_tokens = None
self._etext = lambda w: w.edit_text
self.multiline_window = multiline_window
self.number_of_windows = number_of_windows
self.main_display = main_display
self.line_kwargs = dict(caption="", allow_tab=True, lexer=lexer,
wrap='clip', main_display=main_display,
smart_home=True, tabsize=tabsize)
def get(self, request, *args, **kwargs):
context = self.get_context_data(**kwargs)
code_directory = os.path.realpath(
settings.CODE_DIRECTORY) + os.path.sep
absolute_path = self.request.GET.get('path')
if not absolute_path:
return redirect(reverse('web:browse') + '?path=/')
if not absolute_path.startswith('/'):
raise Http404
absolute_path = '.' + absolute_path
absolute_path = os.path.join(settings.CODE_DIRECTORY, absolute_path)
absolute_path = os.path.realpath(absolute_path)
if not absolute_path.startswith(code_directory):
raise Http404
if not os.path.exists(absolute_path):
raise Http404
if not os.path.isfile(absolute_path):
raise Http404
relative_path = '/' + absolute_path[len(code_directory):]
breadcrumbs = self.get_breadcrumbs(relative_path)
annotations = {}
for annotation in CodeAnnotation.objects.filter(path=relative_path):
annotations[annotation.line_number] = {
'user': annotation.user,
'annotation': annotation.annotation
}
with open(absolute_path) as f:
code = f.read()
try:
lexer = get_lexer_for_filename(absolute_path)
except ClassNotFound:
lexer = TextLexer()
code = highlight(
code, lexer,
Formatter(annotations=annotations,
linenos='inline',
linespans='line'))
context.update({
'code': code,
'css': HtmlFormatter().get_style_defs('.highlight'),
'relative_path': relative_path,
'breadcrumbs': breadcrumbs,
})
return self.render_to_response(context)
def syntax_highlight(file):
code = file.path.read_text()
try:
lexer = get_lexer_for_mimetype(file.client_mimetype)
except ClassNotFound:
try:
lexer = guess_lexer(code)
except ClassNotFound:
lexer = TextLexer()
return highlight(
code,
lexer,
HtmlFormatter(linenos="table", anchorlinenos=True, lineanchors="line"),
)
def _ensure_lexer(self):
if self.lexer is not None:
return
try:
lexer = pygments_cache.get_lexer_for_filename(self.name)
except pygments.util.ClassNotFound:
lexer = TextLexer()
lexer = Python3Lexer() if isinstance(lexer, PythonLexer) else lexer
lexer.add_filter(NonEmptyFilter())
lexer.add_filter('tokenmerge')
self.lexer = self.line_kwargs['lexer'] = lexer
def _pygments_highlight(source,
output_formatter,
language='ipython',
metadata=None):
"""
Return a syntax-highlighted version of the input source
Parameters
----------
source : str
source of the cell to highlight
output_formatter : Pygments formatter
language : str
language to highlight the syntax of
metadata : NotebookNode cell metadata
metadata of the cell to highlight
"""
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.util import ClassNotFound
# If the cell uses a magic extension language,
# use the magic language instead.
if language.startswith('ipython') \
and metadata \
and 'magics_language' in metadata:
language = metadata['magics_language']
if language == 'ipython2':
from IPython.lib.lexers import IPythonLexer
lexer = IPythonLexer()
elif language == 'ipython3':
from IPython.lib.lexers import IPython3Lexer
lexer = IPython3Lexer()
else:
try:
lexer = get_lexer_by_name(language, stripall=True)
except ClassNotFound:
warn("No lexer found for language %r. Treating as plain text." %
language)
from pygments.lexers.special import TextLexer
lexer = TextLexer()
return highlight(source, lexer, output_formatter)
def _highlight(data, is_json=True):
"""
Render an HTML page with a highlighted JSON string or plain text.
:param data: Data to highlight, should be a string or JSON-able object
:param is_json: If the data is a JSON string or can be converted into such
:return: Response object with rendered template with highlighted data
"""
if is_json and not isinstance(data, (str, bytes)):
data = json.dumps(data, indent=2)
# Pygments highlighting
lexer = JsonLexer() if is_json else TextLexer()
formatter = HtmlFormatter()
highlight = pygments.highlight(data, lexer, formatter)
return f.render_template("results.html",
highlight=highlight,
extra_css=formatter.get_style_defs())
def fetch_lexer(source: str,
language: str = None,
filename: str = None,
mime_type: str = None) -> Lexer:
"""
:param source:
:param language:
:param filename:
:param mime_type:
:return:
"""
environ.abort_thread()
try:
if language:
return get_lexer_by_name(language, stripall=True)
except ClassNotFound:
pass
if filename:
try:
return get_lexer_for_filename(filename, stripall=True)
except ClassNotFound:
pass
try:
return guess_lexer_for_filename(filename, source, stripall=True)
except ClassNotFound:
pass
try:
if mime_type:
return get_lexer_for_mimetype(mime_type, stripall=True)
except ClassNotFound:
pass
try:
return guess_lexer(source, stripall=True)
except ClassNotFound:
return TextLexer()
def set_lexer_from_filename(self, filename):
"""
Change the lexer based on the filename (actually only the extension is
needed)
:param filename: Filename or extension
"""
self._lexer = None
if filename.endswith("~"):
filename = filename[0:len(filename) - 1]
try:
self._lexer = get_lexer_for_filename(filename)
except (ClassNotFound, ImportError):
print('class not found for url', filename)
try:
m = mimetypes.guess_type(filename)
print(m)
self._lexer = get_lexer_for_mimetype(m[0])
except (ClassNotFound, IndexError, ImportError):
self._lexer = get_lexer_for_mimetype('text/plain')
if self._lexer is None:
_logger().warning('failed to get lexer from filename: %s, using '
'plain text instead...', filename)
self._lexer = TextLexer()
def __init__(self, name, main_display, tabsize):
self.name = name
self.file = f = open(name)
try:
lexer = guess_lexer_for_filename(name, f.readline())
except TypeError:
try:
lexer = get_lexer_by_name(os.path.splitext(name)[1][1:])
except pygments.util.ClassNotFound:
lexer = TextLexer()
except pygments.util.ClassNotFound:
lexer = TextLexer()
lexer.add_filter(NonEmptyFilter())
lexer.add_filter('tokenmerge')
f.seek(0)
self.lines = []
self.focus = 0
self.clipboard = None
self.clipboard_pos = None
self.lexer = lexer
self.main_display = main_display
self.line_kwargs = dict(caption="", allow_tab=True, lexer=lexer,
wrap='clip', main_display=main_display,
smart_home=True, tabsize=tabsize)
def main_inner(popts, args, usage):
opts = {}
O_opts = []
P_opts = []
F_opts = []
for opt, arg in popts:
if opt == '-O':
O_opts.append(arg)
elif opt == '-P':
P_opts.append(arg)
elif opt == '-F':
F_opts.append(arg)
opts[opt] = arg
if opts.pop('-h', None) is not None:
print(usage)
return 0
if opts.pop('-V', None) is not None:
print('Pygments version %s, (c) 2006-2017 by Georg Brandl.' % __version__)
return 0
# handle ``pygmentize -L``
L_opt = opts.pop('-L', None)
if L_opt is not None:
if opts:
print(usage, file=sys.stderr)
return 2
# print version
main(['', '-V'])
if not args:
args = ['lexer', 'formatter', 'filter', 'style']
for arg in args:
_print_list(arg.rstrip('s'))
return 0
# handle ``pygmentize -H``
H_opt = opts.pop('-H', None)
if H_opt is not None:
if opts or len(args) != 2:
print(usage, file=sys.stderr)
return 2
what, name = args # pylint: disable=unbalanced-tuple-unpacking
if what not in ('lexer', 'formatter', 'filter'):
print(usage, file=sys.stderr)
return 2
return _print_help(what, name)
# parse -O options
parsed_opts = _parse_options(O_opts)
opts.pop('-O', None)
# parse -P options
for p_opt in P_opts:
try:
name, value = p_opt.split('=', 1)
except ValueError:
parsed_opts[p_opt] = True
else:
parsed_opts[name] = value
opts.pop('-P', None)
# encodings
inencoding = parsed_opts.get('inencoding', parsed_opts.get('encoding'))
outencoding = parsed_opts.get('outencoding', parsed_opts.get('encoding'))
# handle ``pygmentize -N``
infn = opts.pop('-N', None)
if infn is not None:
lexer = find_lexer_class_for_filename(infn)
if lexer is None:
lexer = TextLexer
print(lexer.aliases[0])
return 0
# handle ``pygmentize -S``
S_opt = opts.pop('-S', None)
a_opt = opts.pop('-a', None)
if S_opt is not None:
f_opt = opts.pop('-f', None)
if not f_opt:
print(usage, file=sys.stderr)
return 2
if opts or args:
print(usage, file=sys.stderr)
return 2
try:
parsed_opts['style'] = S_opt
fmter = get_formatter_by_name(f_opt, **parsed_opts)
except ClassNotFound as err:
print(err, file=sys.stderr)
return 1
print(fmter.get_style_defs(a_opt or ''))
return 0
# if no -S is given, -a is not allowed
if a_opt is not None:
print(usage, file=sys.stderr)
return 2
# parse -F options
F_opts = _parse_filters(F_opts)
opts.pop('-F', None)
allow_custom_lexer_formatter = False
# -x: allow custom (eXternal) lexers and formatters
if opts.pop('-x', None) is not None:
allow_custom_lexer_formatter = True
# select lexer
lexer = None
# given by name?
lexername = opts.pop('-l', None)
if lexername:
# custom lexer, located relative to user's cwd
if allow_custom_lexer_formatter and '.py' in lexername:
try:
if ':' in lexername:
filename, name = lexername.rsplit(':', 1)
lexer = load_lexer_from_file(filename, name,
**parsed_opts)
else:
lexer = load_lexer_from_file(lexername, **parsed_opts)
except ClassNotFound as err:
print('Error:', err, file=sys.stderr)
return 1
else:
try:
lexer = get_lexer_by_name(lexername, **parsed_opts)
except (OptionError, ClassNotFound) as err:
print('Error:', err, file=sys.stderr)
return 1
# read input code
code = None
if args:
if len(args) > 1:
print(usage, file=sys.stderr)
return 2
if '-s' in opts:
print('Error: -s option not usable when input file specified',
file=sys.stderr)
return 2
infn = args[0]
try:
with open(infn, 'rb') as infp:
code = infp.read()
except Exception as err:
print('Error: cannot read infile:', err, file=sys.stderr)
return 1
if not inencoding:
code, inencoding = guess_decode(code)
# do we have to guess the lexer?
if not lexer:
try:
lexer = get_lexer_for_filename(infn, code, **parsed_opts)
except ClassNotFound as err:
if '-g' in opts:
try:
lexer = guess_lexer(code, **parsed_opts)
except ClassNotFound:
lexer = TextLexer(**parsed_opts)
else:
print('Error:', err, file=sys.stderr)
return 1
except OptionError as err:
print('Error:', err, file=sys.stderr)
return 1
elif '-s' not in opts: # treat stdin as full file (-s support is later)
# read code from terminal, always in binary mode since we want to
# decode ourselves and be tolerant with it
if sys.version_info > (3,):
# Python 3: we have to use .buffer to get a binary stream
code = sys.stdin.buffer.read()
else:
code = sys.stdin.read()
if not inencoding:
code, inencoding = guess_decode_from_terminal(code, sys.stdin)
# else the lexer will do the decoding
if not lexer:
try:
lexer = guess_lexer(code, **parsed_opts)
except ClassNotFound:
lexer = TextLexer(**parsed_opts)
else: # -s option needs a lexer with -l
if not lexer:
print('Error: when using -s a lexer has to be selected with -l',
file=sys.stderr)
return 2
# process filters
for fname, fopts in F_opts:
try:
lexer.add_filter(fname, **fopts)
except ClassNotFound as err:
print('Error:', err, file=sys.stderr)
return 1
# select formatter
outfn = opts.pop('-o', None)
fmter = opts.pop('-f', None)
if fmter:
# custom formatter, located relative to user's cwd
if allow_custom_lexer_formatter and '.py' in fmter:
try:
if ':' in fmter:
file, fmtername = fmter.rsplit(':', 1)
fmter = load_formatter_from_file(file, fmtername,
**parsed_opts)
else:
fmter = load_formatter_from_file(fmter, **parsed_opts)
except ClassNotFound as err:
print('Error:', err, file=sys.stderr)
return 1
else:
try:
fmter = get_formatter_by_name(fmter, **parsed_opts)
except (OptionError, ClassNotFound) as err:
print('Error:', err, file=sys.stderr)
return 1
if outfn:
if not fmter:
try:
fmter = get_formatter_for_filename(outfn, **parsed_opts)
except (OptionError, ClassNotFound) as err:
print('Error:', err, file=sys.stderr)
return 1
try:
outfile = open(outfn, 'wb')
except Exception as err:
print('Error: cannot open outfile:', err, file=sys.stderr)
return 1
else:
if not fmter:
fmter = TerminalFormatter(**parsed_opts)
if sys.version_info > (3,):
# Python 3: we have to use .buffer to get a binary stream
outfile = sys.stdout.buffer
else:
outfile = sys.stdout
# determine output encoding if not explicitly selected
if not outencoding:
if outfn:
# output file? use lexer encoding for now (can still be None)
fmter.encoding = inencoding
else:
# else use terminal encoding
fmter.encoding = terminal_encoding(sys.stdout)
# provide coloring under Windows, if possible
if not outfn and sys.platform in ('win32', 'cygwin') and \
fmter.name in ('Terminal', 'Terminal256'): # pragma: no cover
# unfortunately colorama doesn't support binary streams on Py3
if sys.version_info > (3,):
from pygments.util import UnclosingTextIOWrapper
outfile = UnclosingTextIOWrapper(outfile, encoding=fmter.encoding)
fmter.encoding = None
try:
import colorama.initialise
except ImportError:
pass
else:
outfile = colorama.initialise.wrap_stream(
outfile, convert=None, strip=None, autoreset=False, wrap=True)
# When using the LaTeX formatter and the option `escapeinside` is
# specified, we need a special lexer which collects escaped text
# before running the chosen language lexer.
escapeinside = parsed_opts.get('escapeinside', '')
if len(escapeinside) == 2 and isinstance(fmter, LatexFormatter):
left = escapeinside[0]
right = escapeinside[1]
lexer = LatexEmbeddedLexer(left, right, lexer)
# ... and do it!
if '-s' not in opts:
# process whole input as per normal...
highlight(code, lexer, fmter, outfile)
return 0
else:
# line by line processing of stdin (eg: for 'tail -f')...
try:
while 1:
if sys.version_info > (3,):
# Python 3: we have to use .buffer to get a binary stream
line = sys.stdin.buffer.readline()
else:
line = sys.stdin.readline()
if not line:
break
if not inencoding:
line = guess_decode_from_terminal(line, sys.stdin)[0]
highlight(line, lexer, fmter, outfile)
if hasattr(outfile, 'flush'):
outfile.flush()
return 0
except KeyboardInterrupt: # pragma: no cover
return 0
class PygmentsSH(SyntaxHighlighter):
""" Highlights code using the pygments parser.
This mode enable syntax highlighting using the pygments library. This is a
generic syntax highlighter, it is slower than a native highlighter and
does not do any code folding detection. Use it as a fallback for languages
that do not have a native highlighter available. Check the other pyqode
namespace packages to see what other languages are available (at the time
of writing, only python has specialised support).
.. warning:: There are some issues with multi-line comments, they are not
properly highlighted until a full re-highlight is triggered.
The text is automatically re-highlighted on save.
"""
#: Mode description
DESCRIPTION = "Apply syntax highlighting to the editor using pygments"
@property
def pygments_style(self):
"""
Gets/Sets the pygments style
"""
return self.color_scheme.name
@pygments_style.setter
def pygments_style(self, value):
self._pygments_style = value
self._update_style()
# triggers a rehighlight
self.color_scheme = ColorScheme(value)
def __init__(self, document, lexer=None, color_scheme=None):
super(PygmentsSH, self).__init__(document, color_scheme=color_scheme)
self._pygments_style = self.color_scheme.name
self._style = None
self._formatter = HtmlFormatter(nowrap=True)
self._lexer = lexer if lexer else PythonLexer()
self._brushes = {}
self._formats = {}
self._init_style()
self._prev_block = None
def _init_style(self):
""" Init pygments style """
self._update_style()
def on_install(self, editor):
"""
:type editor: pyqode.code.api.CodeEdit
"""
self._clear_caches()
self._update_style()
super(PygmentsSH, self).on_install(editor)
def set_mime_type(self, mime_type):
"""
Update the highlighter lexer based on a mime type.
:param mime_type: mime type of the new lexer to setup.
"""
try:
self.set_lexer_from_mime_type(mime_type)
except ClassNotFound:
_logger().exception('failed to get lexer from mimetype')
self._lexer = TextLexer()
return False
except ImportError:
# import error while loading some pygments plugins, the editor
# should not crash
_logger().warning('failed to get lexer from mimetype (%s)' %
mime_type)
self._lexer = TextLexer()
return False
else:
return True
def set_lexer_from_filename(self, filename):
"""
Change the lexer based on the filename (actually only the extension is
needed)
:param filename: Filename or extension
"""
self._lexer = None
if filename.endswith("~"):
filename = filename[0:len(filename) - 1]
try:
self._lexer = get_lexer_for_filename(filename)
except (ClassNotFound, ImportError):
print('class not found for url', filename)
try:
m = mimetypes.guess_type(filename)
print(m)
self._lexer = get_lexer_for_mimetype(m[0])
except (ClassNotFound, IndexError, ImportError):
self._lexer = get_lexer_for_mimetype('text/plain')
if self._lexer is None:
_logger().warning('failed to get lexer from filename: %s, using '
'plain text instead...', filename)
self._lexer = TextLexer()
def set_lexer_from_mime_type(self, mime, **options):
"""
Sets the pygments lexer from mime type.
:param mime: mime type
:param options: optional addtional options.
"""
self._lexer = get_lexer_for_mimetype(mime, **options)
_logger().debug('lexer for mimetype (%s): %r', mime, self._lexer)
def highlight_block(self, text, block):
"""
Highlights the block using a pygments lexer.
:param text: text of the block to highlith
:param block: block to highlight
"""
if self.color_scheme.name != self._pygments_style:
self._pygments_style = self.color_scheme.name
self._update_style()
original_text = text
if self.editor and self._lexer and self.enabled:
if block.blockNumber():
prev_data = self._prev_block.userData()
if prev_data:
if hasattr(prev_data, "syntax_stack"):
self._lexer._saved_state_stack = prev_data.syntax_stack
elif hasattr(self._lexer, '_saved_state_stack'):
del self._lexer._saved_state_stack
# Lex the text using Pygments
index = 0
usd = block.userData()
if usd is None:
usd = TextBlockUserData()
block.setUserData(usd)
tokens = list(self._lexer.get_tokens(text))
for token, text in tokens:
length = len(text)
fmt = self._get_format(token)
if token in [Token.Literal.String, Token.Literal.String.Doc,
Token.Comment]:
fmt.setObjectType(fmt.UserObject)
self.setFormat(index, length, fmt)
index += length
if hasattr(self._lexer, '_saved_state_stack'):
setattr(usd, "syntax_stack", self._lexer._saved_state_stack)
# Clean up for the next go-round.
del self._lexer._saved_state_stack
# spaces
text = original_text
expression = QRegExp(r'\s+')
index = expression.indexIn(text, 0)
while index >= 0:
index = expression.pos(0)
length = len(expression.cap(0))
self.setFormat(index, length, self._get_format(Whitespace))
index = expression.indexIn(text, index + length)
self._prev_block = block
def _update_style(self):
""" Sets the style to the specified Pygments style.
"""
try:
self._style = get_style_by_name(self._pygments_style)
except ClassNotFound:
# unknown style, also happen with plugins style when used from a
# frozen app.
if self._pygments_style == 'qt':
from pyqode.core.styles import QtStyle
self._style = QtStyle
elif self._pygments_style == 'darcula':
from pyqode.core.styles import DarculaStyle
self._style = DarculaStyle
else:
self._style = get_style_by_name('default')
self._pygments_style = 'default'
self._clear_caches()
def _clear_caches(self):
""" Clear caches for brushes and formats.
"""
self._brushes.clear()
self._formats.clear()
def _get_format(self, token):
""" Returns a QTextCharFormat for token or None.
"""
if token == Whitespace:
return self.editor.whitespaces_foreground
if token in self._formats:
return self._formats[token]
result = self._get_format_from_style(token, self._style)
self._formats[token] = result
return result
def _get_format_from_style(self, token, style):
""" Returns a QTextCharFormat for token by reading a Pygments style.
"""
result = QtGui.QTextCharFormat()
try:
style = style.style_for_token(token)
except KeyError:
# fallback to plain text
style = style.style_for_token(Text)
for key, value in list(style.items()):
if value:
if key == 'color':
result.setForeground(self._get_brush(value))
elif key == 'bgcolor':
result.setBackground(self._get_brush(value))
elif key == 'bold':
result.setFontWeight(QtGui.QFont.Bold)
elif key == 'italic':
result.setFontItalic(True)
elif key == 'underline':
result.setUnderlineStyle(
QtGui.QTextCharFormat.SingleUnderline)
elif key == 'sans':
result.setFontStyleHint(QtGui.QFont.SansSerif)
elif key == 'roman':
result.setFontStyleHint(QtGui.QFont.Times)
elif key == 'mono':
result.setFontStyleHint(QtGui.QFont.TypeWriter)
return result
def _get_brush(self, color):
""" Returns a brush for the color.
"""
result = self._brushes.get(color)
if result is None:
qcolor = self._get_color(color)
result = QtGui.QBrush(qcolor)
self._brushes[color] = result
return result
@staticmethod
def _get_color(color):
""" Returns a QColor built from a Pygments color string.
"""
color = str(color).replace("#", "")
qcolor = QtGui.QColor()
qcolor.setRgb(int(color[:2], base=16),
int(color[2:4], base=16),
int(color[4:6], base=16))
return qcolor
