def get_tokens_unprocessed(self, text): for index, token, value in RubyLexer.get_tokens_unprocessed( self, text): if token is Name and value in self.EXTRA_KEYWORDS: yield index, Keyword, value else: yield index, token, value
class ProgrammingLexer(object): """lexes a string with multiple programming lexers and returns tokens""" lexers = { 'actionscript': ActionScript3Lexer(), 'c': CLexer(), 'cpp': CppLexer(), 'cs': CSharpLexer(), 'java': JavaLexer(), 'javascript': JavascriptLexer(), 'perl': PerlLexer(), 'php': PhpLexer(startinline=True), 'python': PythonLexer(), 'ruby': RubyLexer(), 'vb': VbNetLexer(), } matched_languages = [] data = None def __init__(self, matched_langs, data_string): self.matched_languages = matched_langs self.data = data_string def lex(self): """ For every possible matched language, we run a lexer to see if we can eliminate it as a possible match. If we detect errors, or have no lexer matches, we remove it from the list. :return: the list of lexer results :rtype: list """ results = {} threads = [] # Looping through each matched language that has a lexer for lexer_id, lexer in \ [[lexid, lxr] for lexid, lxr in self.lexers.items() if lexid in self.matched_languages]: # Creating a thread for each lexer thread = ProgrammingLexerThread(lexer_id, lexer, self.data) thread.start() threads.append(thread) for thr in threads: thr.join() for thr in [th for th in threads if th.result]: results[thr.thread_id] = thr.result return results
def __init__(self, **options): from pygments.lexers.agile import RubyLexer self.ruby_lexer = RubyLexer(**options) Lexer.__init__(self, **options)
class ErbLexer(Lexer): """ Generic `ERB <http://ruby-doc.org/core/classes/ERB.html>`_ (Ruby Templating) lexer. Just highlights ruby code between the preprocessor directives, other data is left untouched by the lexer. All options are also forwarded to the `RubyLexer`. """ name = 'ERB' aliases = ['erb'] mimetypes = ['application/x-ruby-templating'] _block_re = re.compile(r'(<%%|%%>|<%=|<%#|<%-|<%|-%>|%>|^%[^%].*?$)', re.M) def __init__(self, **options): from pygments.lexers.agile import RubyLexer self.ruby_lexer = RubyLexer(**options) Lexer.__init__(self, **options) def get_tokens_unprocessed(self, text): """ Since ERB doesn't allow "<%" and other tags inside of ruby blocks we have to use a split approach here that fails for that too. """ tokens = self._block_re.split(text) tokens.reverse() state = idx = 0 try: while True: # text if state == 0: val = tokens.pop() yield idx, Other, val idx += len(val) state = 1 # block starts elif state == 1: tag = tokens.pop() # literals if tag in ('<%%', '%%>'): yield idx, Other, tag idx += 3 state = 0 # comment elif tag == '<%#': yield idx, Comment.Preproc, tag val = tokens.pop() yield idx + 3, Comment, val idx += 3 + len(val) state = 2 # blocks or output elif tag in ('<%', '<%=', '<%-'): yield idx, Comment.Preproc, tag idx += len(tag) data = tokens.pop() r_idx = 0 for r_idx, r_token, r_value in \ self.ruby_lexer.get_tokens_unprocessed(data): yield r_idx + idx, r_token, r_value idx += len(data) state = 2 elif tag in ('%>', '-%>'): yield idx, Error, tag idx += len(tag) state = 0 # % raw ruby statements else: yield idx, Comment.Preproc, tag[0] r_idx = 0 for r_idx, r_token, r_value in \ self.ruby_lexer.get_tokens_unprocessed(tag[1:]): yield idx + 1 + r_idx, r_token, r_value idx += len(tag) state = 0 # block ends elif state == 2: tag = tokens.pop() if tag not in ('%>', '-%>'): yield idx, Other, tag else: yield idx, Comment.Preproc, tag idx += len(tag) state = 0 except IndexError: return def analyse_text(text): if '<%' in text and '%>' in text: return 0.4
def get_tokens_unprocessed(self, text): for index, token, value in RubyLexer.get_tokens_unprocessed(self, text): if token is Name and value in self.EXTRA_KEYWORDS: yield index, Keyword, value else: yield index, token, value
#show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # -- Settings for symfony doc extension --------------------------------------------------- # enable highlighting for PHP code not between ``<?php ... ?>`` by default lexers['php'] = PhpLexer(startinline=True) lexers['php-annotations'] = PhpLexer(startinline=True) lexers['php-standalone'] = PhpLexer(startinline=True) lexers['php-symfony'] = PhpLexer(startinline=True) lexers['varnish2'] = RubyLexer() lexers['varnish3'] = RubyLexer() lexers['varnish4'] = RubyLexer() config_block = { 'varnish2': 'Varnish 2', 'varnish3': 'Varnish 3', 'varnish4': 'Varnish 4' } # use PHP as the primary domain primary_domain = 'php' # set url for API links api_url = 'http://api.symfony.com/master/%s'