def highlight(el, lang): text = textContent(el) if lang in ["idl", "webidl"]: widl = parser.Parser(text, IDLUI()) marker = HighlightMarker() nested = parseHTML(unicode(widl.markup(marker))) coloredText = collections.deque() for n in childNodes(flattenHighlighting(nested)): if isElement(n): coloredText.append(ColoredText(textContent(n), n.get('class'))) else: coloredText.append(ColoredText(n, None)) else: if lang in customLexers: lexer = customLexers[lang] else: try: lexer = get_lexer_by_name(lang, encoding="utf-8", stripAll=True) except pyg.util.ClassNotFound: die("'{0}' isn't a known syntax-highlighting language. See http://pygments.org/docs/lexers/. Seen on:\n{1}", lang, outerHTML(el), el=el) return coloredText = parsePygments(pyg.highlight(text, lexer, formatters.RawTokenFormatter())) # empty span at beginning # extra linebreak at the end mergeHighlighting(el, coloredText) addClass(el, "highlight")
def highlightWithPygments(text, lang, el): import pygments from pygments import formatters lexer = lexerFromLang(lang) if lexer is None: die("'{0}' isn't a known syntax-highlighting language. See http://pygments.org/docs/lexers/. Seen on:\n{1}", lang, outerHTML(el), el=el) return rawTokens = pygments.highlight(text, lexer, formatters.RawTokenFormatter()) coloredText = coloredTextFromRawTokens(rawTokens) return coloredText
def scan(self, data, file, options, expire_at): highlight = pygments.highlight( data, self.lexer, formatters.RawTokenFormatter(), ) highlight_list = highlight.split(b'\n') ordered_highlights = [] for hl in highlight_list: split_highlight = hl.split(b'\t') if len(split_highlight) == 2: token = split_highlight[0].decode() value = split_highlight[1].decode().strip('\'"').strip() highlight_entry = {'token': token, 'value': value} if highlight_entry['value']: ordered_highlights.append(highlight_entry) self.event.setdefault('tokens', []) self.event.setdefault('comments', []) self.event.setdefault('keywords', []) self.event.setdefault('labels', []) self.event.setdefault('strings', []) self.event.setdefault('text', []) self.event.setdefault('variables', []) position = 0 while position < len(ordered_highlights): ohlp = ordered_highlights[position] if ohlp['token'] not in self.event['tokens']: self.event['tokens'].append(ohlp['token']) if ohlp['token'] == 'Token.Comment.Single': if ohlp['value'] not in self.event['comments']: self.event['comments'].append(ohlp['value']) elif ohlp['token'] == 'Token.Keyword': if ohlp['value'] not in self.event['keywords']: self.event['keywords'].append(ohlp['value']) elif ohlp['token'] == 'Token.Name.Label': if ohlp['value'] not in self.event['labels']: self.event['labels'].append(ohlp['value']) elif ohlp['token'] == 'Token.Literal.String.Double': if ohlp['value'] not in self.event['strings']: self.event['strings'].append(ohlp['value']) elif ohlp['token'] == 'Token.Literal.String.Single': if ohlp['value'] not in self.event['strings']: self.event['strings'].append(ohlp['value']) elif ohlp['token'] == 'Token.Text': if ohlp['value'] not in self.event['text']: self.event['text'].append(ohlp['value']) elif ohlp['token'] == 'Token.Name.Variable': if ohlp['value'] not in self.event['variables']: self.event['variables'].append(ohlp['value']) position += 1
def scan(self, file_object, options): highlight = pygments.highlight(file_object.data, self.lexer, formatters.RawTokenFormatter()) highlight_list = highlight.split(b"\n") ordered_highlights = [] for hl in highlight_list: split_highlight = hl.split(b"\t") if len(split_highlight) == 2: token = split_highlight[0].decode() value = split_highlight[1].decode().strip("'\"").strip() highlight_entry = {"token": token, "value": value} if highlight_entry["value"]: ordered_highlights.append(highlight_entry) self.metadata.setdefault("tokens", []) self.metadata.setdefault("comments", []) self.metadata.setdefault("keywords", []) self.metadata.setdefault("labels", []) self.metadata.setdefault("strings", []) self.metadata.setdefault("text", []) self.metadata.setdefault("variables", []) position = 0 while position < len(ordered_highlights): ohlp = ordered_highlights[position] if ohlp["token"] not in self.metadata["tokens"]: self.metadata["tokens"].append(ohlp["token"]) if ohlp["token"] == "Token.Comment.Single": if ohlp["value"] not in self.metadata["comments"]: self.metadata["comments"].append(ohlp["value"]) elif ohlp["token"] == "Token.Keyword": if ohlp["value"] not in self.metadata["keywords"]: self.metadata["keywords"].append(ohlp["value"]) elif ohlp["token"] == "Token.Name.Label": if ohlp["value"] not in self.metadata["labels"]: self.metadata["labels"].append(ohlp["value"]) elif ohlp["token"] == "Token.Literal.String.Double": if ohlp["value"] not in self.metadata["strings"]: self.metadata["strings"].append(ohlp["value"]) elif ohlp["token"] == "Token.Literal.String.Single": if ohlp["value"] not in self.metadata["strings"]: self.metadata["strings"].append(ohlp["value"]) elif ohlp["token"] == "Token.Text": if ohlp["value"] not in self.metadata["text"]: self.metadata["text"].append(ohlp["value"]) elif ohlp["token"] == "Token.Name.Variable": if ohlp["value"] not in self.metadata["variables"]: self.metadata["variables"].append(ohlp["value"]) position += 1
def highlightWithPygments(text, lang): import pygments from pygments import formatters lexer = lexerFromLang(lang) if lexer is None: die( "'{0}' isn't a known syntax-highlighting language. See http://pygments.org/docs/lexers/.", lang) return rawTokens = str(pygments.highlight(text, lexer, formatters.RawTokenFormatter()), encoding="utf-8") coloredText = coloredTextFromRawTokens(rawTokens) return coloredText
def scan(self, file_object, options): highlight = pygments.highlight(file_object.data, self.lexer, formatters.RawTokenFormatter()) highlight_list = highlight.split(b"\n") ordered_highlights = [] for hl in highlight_list: split_highlight = hl.split(b"\t") if len(split_highlight) == 2: token = split_highlight[0].decode() value = split_highlight[1].decode().strip("'\"").strip() highlight_entry = {"token": token, "value": value} if highlight_entry["value"]: ordered_highlights.append(highlight_entry) self.metadata.setdefault("tokens", []) self.metadata.setdefault("comments", []) self.metadata.setdefault("functions", []) self.metadata.setdefault("names", []) self.metadata.setdefault("operators", []) self.metadata.setdefault("strings", []) position = 0 while position < len(ordered_highlights): ohlp = ordered_highlights[position] if ohlp["token"] not in self.metadata["tokens"]: self.metadata["tokens"].append(ohlp["token"]) if ohlp["token"] == "Token.Comment": if ohlp["value"] not in self.metadata["comments"]: self.metadata["comments"].append(ohlp["value"]) elif ohlp["token"] == "Token.Name.Function": if ohlp["value"] not in self.metadata["functions"]: self.metadata["functions"].append(ohlp["value"]) elif ohlp["token"] == "Token.Name": if ohlp["value"] not in self.metadata["names"]: self.metadata["names"].append(ohlp["value"]) elif ohlp["token"] == "Token.Operator": if ohlp["value"] not in self.metadata["operators"]: self.metadata["operators"].append(ohlp["value"]) elif ohlp["token"] == "Token.Literal.String": if ohlp["value"] not in self.metadata["strings"]: self.metadata["strings"].append(ohlp["value"]) position += 1