def highlight(el, lang):
text = textContent(el)
if lang in ["idl", "webidl"]:
widl = parser.Parser(text, IDLUI())
marker = HighlightMarker()
nested = parseHTML(unicode(widl.markup(marker)))
coloredText = collections.deque()
for n in childNodes(flattenHighlighting(nested)):
if isElement(n):
coloredText.append(ColoredText(textContent(n), n.get('class')))
else:
coloredText.append(ColoredText(n, None))
else:
if lang in customLexers:
lexer = customLexers[lang]
else:
try:
lexer = get_lexer_by_name(lang, encoding="utf-8", stripAll=True)
except pyg.util.ClassNotFound:
die("'{0}' isn't a known syntax-highlighting language. See http://pygments.org/docs/lexers/. Seen on:\n{1}", lang, outerHTML(el), el=el)
return
coloredText = parsePygments(pyg.highlight(text, lexer, formatters.RawTokenFormatter()))
# empty span at beginning
# extra linebreak at the end
mergeHighlighting(el, coloredText)
addClass(el, "highlight")
def highlightWithPygments(text, lang, el):
import pygments
from pygments import formatters
lexer = lexerFromLang(lang)
if lexer is None:
die("'{0}' isn't a known syntax-highlighting language. See http://pygments.org/docs/lexers/. Seen on:\n{1}", lang, outerHTML(el), el=el)
return
rawTokens = pygments.highlight(text, lexer, formatters.RawTokenFormatter())
coloredText = coloredTextFromRawTokens(rawTokens)
return coloredText
def scan(self, data, file, options, expire_at):
highlight = pygments.highlight(
data,
self.lexer,
formatters.RawTokenFormatter(),
)
highlight_list = highlight.split(b'\n')
ordered_highlights = []
for hl in highlight_list:
split_highlight = hl.split(b'\t')
if len(split_highlight) == 2:
token = split_highlight[0].decode()
value = split_highlight[1].decode().strip('\'"').strip()
highlight_entry = {'token': token, 'value': value}
if highlight_entry['value']:
ordered_highlights.append(highlight_entry)
self.event.setdefault('tokens', [])
self.event.setdefault('comments', [])
self.event.setdefault('keywords', [])
self.event.setdefault('labels', [])
self.event.setdefault('strings', [])
self.event.setdefault('text', [])
self.event.setdefault('variables', [])
position = 0
while position < len(ordered_highlights):
ohlp = ordered_highlights[position]
if ohlp['token'] not in self.event['tokens']:
self.event['tokens'].append(ohlp['token'])
if ohlp['token'] == 'Token.Comment.Single':
if ohlp['value'] not in self.event['comments']:
self.event['comments'].append(ohlp['value'])
elif ohlp['token'] == 'Token.Keyword':
if ohlp['value'] not in self.event['keywords']:
self.event['keywords'].append(ohlp['value'])
elif ohlp['token'] == 'Token.Name.Label':
if ohlp['value'] not in self.event['labels']:
self.event['labels'].append(ohlp['value'])
elif ohlp['token'] == 'Token.Literal.String.Double':
if ohlp['value'] not in self.event['strings']:
self.event['strings'].append(ohlp['value'])
elif ohlp['token'] == 'Token.Literal.String.Single':
if ohlp['value'] not in self.event['strings']:
self.event['strings'].append(ohlp['value'])
elif ohlp['token'] == 'Token.Text':
if ohlp['value'] not in self.event['text']:
self.event['text'].append(ohlp['value'])
elif ohlp['token'] == 'Token.Name.Variable':
if ohlp['value'] not in self.event['variables']:
self.event['variables'].append(ohlp['value'])
position += 1
def scan(self, file_object, options):
highlight = pygments.highlight(file_object.data,
self.lexer,
formatters.RawTokenFormatter())
highlight_list = highlight.split(b"\n")
ordered_highlights = []
for hl in highlight_list:
split_highlight = hl.split(b"\t")
if len(split_highlight) == 2:
token = split_highlight[0].decode()
value = split_highlight[1].decode().strip("'\"").strip()
highlight_entry = {"token": token, "value": value}
if highlight_entry["value"]:
ordered_highlights.append(highlight_entry)
self.metadata.setdefault("tokens", [])
self.metadata.setdefault("comments", [])
self.metadata.setdefault("keywords", [])
self.metadata.setdefault("labels", [])
self.metadata.setdefault("strings", [])
self.metadata.setdefault("text", [])
self.metadata.setdefault("variables", [])
position = 0
while position < len(ordered_highlights):
ohlp = ordered_highlights[position]
if ohlp["token"] not in self.metadata["tokens"]:
self.metadata["tokens"].append(ohlp["token"])
if ohlp["token"] == "Token.Comment.Single":
if ohlp["value"] not in self.metadata["comments"]:
self.metadata["comments"].append(ohlp["value"])
elif ohlp["token"] == "Token.Keyword":
if ohlp["value"] not in self.metadata["keywords"]:
self.metadata["keywords"].append(ohlp["value"])
elif ohlp["token"] == "Token.Name.Label":
if ohlp["value"] not in self.metadata["labels"]:
self.metadata["labels"].append(ohlp["value"])
elif ohlp["token"] == "Token.Literal.String.Double":
if ohlp["value"] not in self.metadata["strings"]:
self.metadata["strings"].append(ohlp["value"])
elif ohlp["token"] == "Token.Literal.String.Single":
if ohlp["value"] not in self.metadata["strings"]:
self.metadata["strings"].append(ohlp["value"])
elif ohlp["token"] == "Token.Text":
if ohlp["value"] not in self.metadata["text"]:
self.metadata["text"].append(ohlp["value"])
elif ohlp["token"] == "Token.Name.Variable":
if ohlp["value"] not in self.metadata["variables"]:
self.metadata["variables"].append(ohlp["value"])
position += 1
def highlightWithPygments(text, lang):
import pygments
from pygments import formatters
lexer = lexerFromLang(lang)
if lexer is None:
die(
"'{0}' isn't a known syntax-highlighting language. See http://pygments.org/docs/lexers/.",
lang)
return
rawTokens = str(pygments.highlight(text, lexer,
formatters.RawTokenFormatter()),
encoding="utf-8")
coloredText = coloredTextFromRawTokens(rawTokens)
return coloredText
def scan(self, file_object, options):
highlight = pygments.highlight(file_object.data, self.lexer,
formatters.RawTokenFormatter())
highlight_list = highlight.split(b"\n")
ordered_highlights = []
for hl in highlight_list:
split_highlight = hl.split(b"\t")
if len(split_highlight) == 2:
token = split_highlight[0].decode()
value = split_highlight[1].decode().strip("'\"").strip()
highlight_entry = {"token": token, "value": value}
if highlight_entry["value"]:
ordered_highlights.append(highlight_entry)
self.metadata.setdefault("tokens", [])
self.metadata.setdefault("comments", [])
self.metadata.setdefault("functions", [])
self.metadata.setdefault("names", [])
self.metadata.setdefault("operators", [])
self.metadata.setdefault("strings", [])
position = 0
while position < len(ordered_highlights):
ohlp = ordered_highlights[position]
if ohlp["token"] not in self.metadata["tokens"]:
self.metadata["tokens"].append(ohlp["token"])
if ohlp["token"] == "Token.Comment":
if ohlp["value"] not in self.metadata["comments"]:
self.metadata["comments"].append(ohlp["value"])
elif ohlp["token"] == "Token.Name.Function":
if ohlp["value"] not in self.metadata["functions"]:
self.metadata["functions"].append(ohlp["value"])
elif ohlp["token"] == "Token.Name":
if ohlp["value"] not in self.metadata["names"]:
self.metadata["names"].append(ohlp["value"])
elif ohlp["token"] == "Token.Operator":
if ohlp["value"] not in self.metadata["operators"]:
self.metadata["operators"].append(ohlp["value"])
elif ohlp["token"] == "Token.Literal.String":
if ohlp["value"] not in self.metadata["strings"]:
self.metadata["strings"].append(ohlp["value"])
position += 1
