예제 #1
0
파일: asm.py 프로젝트: Cold5nap/sobolIter
class LlvmLexer(RegexLexer):
    """
    For LLVM assembly code.
    """
    name = 'LLVM'
    aliases = ['llvm']
    filenames = ['*.ll']
    mimetypes = ['text/x-llvm']

    #: optional Comment or Whitespace
    string = r'"[^"]*?"'
    identifier = r'([-a-zA-Z$._][\w\-$.]*|' + string + ')'

    tokens = {
        'root': [
            include('whitespace'),

            # Before keywords, because keywords are valid label names :(...
            (identifier + r'\s*:', Name.Label),
            include('keyword'),
            (r'%' + identifier, Name.Variable),
            (r'@' + identifier, Name.Variable.Global),
            (r'%\d+', Name.Variable.Anonymous),
            (r'@\d+', Name.Variable.Global),
            (r'#\d+', Name.Variable.Global),
            (r'!' + identifier, Name.Variable),
            (r'!\d+', Name.Variable.Anonymous),
            (r'c?' + string, String),
            (r'0[xX][a-fA-F0-9]+', Number),
            (r'-?\d+(?:[.]\d+)?(?:[eE][-+]?\d+(?:[.]\d+)?)?', Number),
            (r'[=<>{}\[\]()*.,!]|x\b', Punctuation)
        ],
        'whitespace': [(r'(\n|\s)+', Text), (r';.*?\n', Comment)],
        'keyword': [
            # Regular keywords
            (words((
                'acq_rel', 'acquire', 'add', 'addrspace', 'addrspacecast',
                'afn', 'alias', 'aliasee', 'align', 'alignLog2', 'alignstack',
                'alloca', 'allocsize', 'allOnes', 'alwaysinline', 'amdgpu_cs',
                'amdgpu_es', 'amdgpu_gs', 'amdgpu_hs', 'amdgpu_kernel',
                'amdgpu_ls', 'amdgpu_ps', 'amdgpu_vs', 'and', 'any',
                'anyregcc', 'appending', 'arcp', 'argmemonly', 'args',
                'arm_aapcs_vfpcc', 'arm_aapcscc', 'arm_apcscc', 'ashr', 'asm',
                'atomic', 'atomicrmw', 'attributes', 'available_externally',
                'avr_intrcc', 'avr_signalcc', 'bit', 'bitcast', 'bitMask',
                'blockaddress', 'br', 'branchFunnel', 'builtin', 'byArg',
                'byte', 'byteArray', 'byval', 'c', 'call', 'callee', 'caller',
                'calls', 'catch', 'catchpad', 'catchret', 'catchswitch', 'cc',
                'ccc', 'cleanup', 'cleanuppad', 'cleanupret', 'cmpxchg',
                'cold', 'coldcc', 'comdat', 'common', 'constant', 'contract',
                'convergent', 'critical', 'cxx_fast_tlscc', 'datalayout',
                'declare', 'default', 'define', 'deplibs', 'dereferenceable',
                'dereferenceable_or_null', 'distinct', 'dllexport',
                'dllimport', 'dso_local', 'dso_preemptable', 'dsoLocal', 'eq',
                'exact', 'exactmatch', 'extern_weak', 'external',
                'externally_initialized', 'extractelement', 'extractvalue',
                'fadd', 'false', 'fast', 'fastcc', 'fcmp', 'fdiv', 'fence',
                'filter', 'flags', 'fmul', 'fpext', 'fptosi', 'fptoui',
                'fptrunc', 'freeze', 'frem', 'from', 'fsub', 'funcFlags',
                'function', 'gc', 'getelementptr', 'ghccc', 'global', 'guid',
                'gv', 'hash', 'hhvm_ccc', 'hhvmcc', 'hidden', 'hot', 'hotness',
                'icmp', 'ifunc', 'inaccessiblemem_or_argmemonly',
                'inaccessiblememonly', 'inalloca', 'inbounds', 'indir',
                'indirectbr', 'info', 'initialexec', 'inline', 'inlineBits',
                'inlinehint', 'inrange', 'inreg', 'insertelement',
                'insertvalue', 'insts', 'intel_ocl_bicc', 'inteldialect',
                'internal', 'inttoptr', 'invoke', 'jumptable', 'kind',
                'landingpad', 'largest', 'linkage', 'linkonce', 'linkonce_odr',
                'live', 'load', 'local_unnamed_addr', 'localdynamic',
                'localexec', 'lshr', 'max', 'metadata', 'min', 'minsize',
                'module', 'monotonic', 'msp430_intrcc', 'mul', 'musttail',
                'naked', 'name', 'nand', 'ne', 'nest', 'ninf', 'nnan',
                'noalias', 'nobuiltin', 'nocapture', 'nocf_check',
                'noduplicate', 'noduplicates', 'noimplicitfloat', 'noinline',
                'none', 'nonlazybind', 'nonnull', 'norecurse', 'noRecurse',
                'noredzone', 'noreturn', 'notail', 'notEligibleToImport',
                'nounwind', 'nsw', 'nsz', 'null', 'nuw', 'oeq', 'offset',
                'oge', 'ogt', 'ole', 'olt', 'one', 'opaque', 'optforfuzzing',
                'optnone', 'optsize', 'or', 'ord', 'path', 'personality',
                'phi', 'poison', 'prefix', 'preserve_allcc', 'preserve_mostcc',
                'private', 'prologue', 'protected', 'ptrtoint', 'ptx_device',
                'ptx_kernel', 'readnone', 'readNone', 'readonly', 'readOnly',
                'reassoc', 'refs', 'relbf', 'release', 'resByArg', 'resume',
                'ret', 'returnDoesNotAlias', 'returned', 'returns_twice',
                'safestack', 'samesize', 'sanitize_address',
                'sanitize_hwaddress', 'sanitize_memory', 'sanitize_thread',
                'sdiv', 'section', 'select', 'seq_cst', 'sext', 'sge', 'sgt',
                'shadowcallstack', 'shl', 'shufflevector', 'sideeffect',
                'signext', 'single', 'singleImpl', 'singleImplName', 'sitofp',
                'sizeM1', 'sizeM1BitWidth', 'sle', 'slt', 'source_filename',
                'speculatable', 'spir_func', 'spir_kernel', 'srem', 'sret',
                'ssp', 'sspreq', 'sspstrong', 'store', 'strictfp', 'sub',
                'summaries', 'summary', 'swiftcc', 'swifterror', 'swiftself',
                'switch', 'syncscope', 'tail', 'target', 'thread_local', 'to',
                'token', 'triple', 'true', 'trunc', 'type',
                'typeCheckedLoadConstVCalls', 'typeCheckedLoadVCalls',
                'typeid', 'typeIdInfo', 'typeTestAssumeConstVCalls',
                'typeTestAssumeVCalls', 'typeTestRes', 'typeTests', 'udiv',
                'ueq', 'uge', 'ugt', 'uitofp', 'ule', 'ult', 'umax', 'umin',
                'undef', 'une', 'uniformRetVal', 'uniqueRetVal', 'unknown',
                'unnamed_addr', 'uno', 'unordered', 'unreachable', 'unsat',
                'unwind', 'urem', 'uselistorder', 'uselistorder_bb', 'uwtable',
                'va_arg', 'variable', 'vFuncId', 'virtualConstProp', 'void',
                'volatile', 'weak', 'weak_odr', 'webkit_jscc', 'win64cc',
                'within', 'wpdRes', 'wpdResolutions', 'writeonly',
                'x86_64_sysvcc', 'x86_fastcallcc', 'x86_intrcc', 'x86_mmx',
                'x86_regcallcc', 'x86_stdcallcc', 'x86_thiscallcc',
                'x86_vectorcallcc', 'xchg', 'xor', 'zeroext',
                'zeroinitializer', 'zext', 'immarg', 'willreturn'),
                   suffix=r'\b'), Keyword),

            # Types
            (words(('void', 'half', 'bfloat', 'float', 'double', 'fp128',
                    'x86_fp80', 'ppc_fp128', 'label', 'metadata', 'token')),
             Keyword.Type),

            # Integer types
            (r'i[1-9]\d*', Keyword.Type)
        ]
    }
예제 #2
0
파일: asm.py 프로젝트: Cold5nap/sobolIter
class LlvmMirLexer(RegexLexer):
    """
    Lexer for the overall LLVM MIR document format.

    MIR is a human readable serialization format that's used to represent LLVM's
    machine specific intermediate representation. It allows LLVM's developers to
    see the state of the compilation process at various points, as well as test
    individual pieces of the compiler.

    For more information on LLVM MIR see https://llvm.org/docs/MIRLangRef.html.

    .. versionadded:: 2.6
    """
    name = 'LLVM-MIR'
    aliases = ['llvm-mir']
    filenames = ['*.mir']

    tokens = {
        'root': [
            # Comments are hashes at the YAML level
            (r'#.*', Comment),
            # Documents starting with | are LLVM-IR
            (r'--- \|$', Keyword, 'llvm_ir'),
            # Other documents are MIR
            (r'---', Keyword, 'llvm_mir'),
            # Consume everything else in one token for efficiency
            (r'[^-#]+|.', Text),
        ],
        'llvm_ir': [
            # Documents end with '...' or '---'
            (r'(\.\.\.|(?=---))', Keyword, '#pop'),
            # Delegate to the LlvmLexer
            (r'((?:.|\n)+?)(?=(\.\.\.|---))', bygroups(using(LlvmLexer))),
        ],
        'llvm_mir': [
            # Comments are hashes at the YAML level
            (r'#.*', Comment),
            # Documents end with '...' or '---'
            (r'(\.\.\.|(?=---))', Keyword, '#pop'),
            # Handle the simple attributes
            (r'name:', Keyword, 'name'),
            (words(('alignment', ), suffix=':'), Keyword, 'number'),
            (words(('legalized', 'regBankSelected', 'tracksRegLiveness',
                    'selected', 'exposesReturnsTwice'),
                   suffix=':'), Keyword, 'boolean'),
            # Handle the attributes don't highlight inside
            (words(('registers', 'stack', 'fixedStack', 'liveins', 'frameInfo',
                    'machineFunctionInfo'),
                   suffix=':'), Keyword),
            # Delegate the body block to the LlvmMirBodyLexer
            (r'body: *\|', Keyword, 'llvm_mir_body'),
            # Consume everything else
            (r'.+', Text),
            (r'\n', Text),
        ],
        'name': [
            (r'[^\n]+', Name),
            default('#pop'),
        ],
        'boolean': [
            (r' *(true|false)', Name.Builtin),
            default('#pop'),
        ],
        'number': [
            (r' *[0-9]+', Number),
            default('#pop'),
        ],
        'llvm_mir_body': [
            # Documents end with '...' or '---'.
            # We have to pop llvm_mir_body and llvm_mir
            (r'(\.\.\.|(?=---))', Keyword, '#pop:2'),
            # Delegate the body block to the LlvmMirBodyLexer
            (r'((?:.|\n)+?)(?=\.\.\.|---)', bygroups(using(LlvmMirBodyLexer))),
            # The '...' is optional. If we didn't already find it then it isn't
            # there. There might be a '---' instead though.
            (r'(?!\.\.\.|---)((?:.|\n)+)', bygroups(using(LlvmMirBodyLexer))),
        ],
    }
예제 #3
0
class SystemVerilogLexer(RegexLexer):
    """
    Extends verilog lexer to recognise all SystemVerilog keywords from IEEE
    1800-2009 standard.

    .. versionadded:: 1.5
    """
    name = 'systemverilog'
    aliases = ['systemverilog', 'sv']
    filenames = ['*.sv', '*.svh']
    mimetypes = ['text/x-systemverilog']

    # : optional Comment or Whitespace
    _ws = r'(?:\s|//.*?\n|/[*].*?[*]/)+'

    tokens = {
        'root': [
            (r'^\s*`define', Comment.Preproc, 'macro'),
            (r'^(\s*)(package)(\s+)', bygroups(Text, Keyword.Namespace, Text)),
            (r'^(\s*)(import)(\s+)', bygroups(Text, Keyword.Namespace,
                                              Text), 'import'),
            (r'\n', Text),
            (r'\s+', Text),
            (r'\\\n', Text),  # line continuation
            (r'/(\\\n)?/(\n|(.|\n)*?[^\\]\n)', Comment.Single),
            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
            (r'[{}#@]', Punctuation),
            (r'L?"', String, 'string'),
            (r"L?'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'",
             String.Char),
            (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[lL]?', Number.Float),
            (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
            (r'([0-9]+)|(\'h)[0-9a-fA-F]+', Number.Hex),
            (r'([0-9]+)|(\'b)[01]+', Number.Bin),
            (r'([0-9]+)|(\'d)[0-9]+', Number.Integer),
            (r'([0-9]+)|(\'o)[0-7]+', Number.Oct),
            (r'\'[01xz]', Number),
            (r'\d+[Ll]?', Number.Integer),
            (r'\*/', Error),
            (r'[~!%^&*+=|?:<>/-]', Operator),
            (r'[()\[\],.;\']', Punctuation),
            (r'`[a-zA-Z_]\w*', Name.Constant),
            (words(
                ('accept_on', 'alias', 'always', 'always_comb', 'always_ff',
                 'always_latch', 'and', 'assert', 'assign', 'assume',
                 'automatic', 'before', 'begin', 'bind', 'bins', 'binsof',
                 'bit', 'break', 'buf', 'bufif0', 'bufif1', 'byte', 'case',
                 'casex', 'casez', 'cell', 'chandle', 'checker', 'class',
                 'clocking', 'cmos', 'config', 'const', 'constraint',
                 'context', 'continue', 'cover', 'covergroup', 'coverpoint',
                 'cross', 'deassign', 'default', 'defparam', 'design',
                 'disable', 'dist', 'do', 'edge', 'else', 'end', 'endcase',
                 'endchecker', 'endclass', 'endclocking', 'endconfig',
                 'endfunction', 'endgenerate', 'endgroup', 'endinterface',
                 'endmodule', 'endpackage', 'endprimitive', 'endprogram',
                 'endproperty', 'endsequence', 'endspecify', 'endtable',
                 'endtask', 'enum', 'event', 'eventually', 'expect', 'export',
                 'extends', 'extern', 'final', 'first_match', 'for', 'force',
                 'foreach', 'forever', 'fork', 'forkjoin', 'function',
                 'generate', 'genvar', 'global', 'highz0', 'highz1', 'if',
                 'iff', 'ifnone', 'ignore_bins', 'illegal_bins', 'implies',
                 'import', 'incdir', 'include', 'initial', 'inout', 'input',
                 'inside', 'instance', 'int', 'integer', 'interface',
                 'intersect', 'join', 'join_any', 'join_none', 'large', 'let',
                 'liblist', 'library', 'local', 'localparam', 'logic',
                 'longint', 'macromodule', 'matches', 'medium', 'modport',
                 'module', 'nand', 'negedge', 'new', 'nexttime', 'nmos', 'nor',
                 'noshowcancelled', 'not', 'notif0', 'notif1', 'null', 'or',
                 'output', 'package', 'packed', 'parameter', 'pmos', 'posedge',
                 'primitive', 'priority', 'program', 'property', 'protected',
                 'pull0', 'pull1', 'pulldown', 'pullup', 'pulsestyle_ondetect',
                 'pulsestyle_onevent', 'pure', 'rand', 'randc', 'randcase',
                 'randsequence', 'rcmos', 'real', 'realtime', 'ref', 'reg',
                 'reject_on', 'release', 'repeat', 'restrict', 'return',
                 'rnmos', 'rpmos', 'rtran', 'rtranif0', 'rtranif1', 's_always',
                 's_eventually', 's_nexttime', 's_until', 's_until_with',
                 'scalared', 'sequence', 'shortint', 'shortreal',
                 'showcancelled', 'signed', 'small', 'solve', 'specify',
                 'specparam', 'static', 'string', 'strong', 'strong0',
                 'strong1', 'struct', 'super', 'supply0', 'supply1',
                 'sync_accept_on', 'sync_reject_on', 'table', 'tagged', 'task',
                 'this', 'throughout', 'time', 'timeprecision', 'timeunit',
                 'tran', 'tranif0', 'tranif1', 'tri', 'tri0', 'tri1', 'triand',
                 'trior', 'trireg', 'type', 'typedef', 'union', 'unique',
                 'unique0', 'unsigned', 'until', 'until_with', 'untyped',
                 'use', 'uwire', 'var', 'vectored', 'virtual', 'void', 'wait',
                 'wait_order', 'wand', 'weak', 'weak0', 'weak1', 'while',
                 'wildcard', 'wire', 'with', 'within', 'wor', 'xnor', 'xor'),
                suffix=r'\b'), Keyword),
            (words(('`__FILE__', '`__LINE__', '`begin_keywords', '`celldefine',
                    '`default_nettype', '`define', '`else', '`elsif',
                    '`end_keywords', '`endcelldefine', '`endif', '`ifdef',
                    '`ifndef', '`include', '`line', '`nounconnected_drive',
                    '`pragma', '`resetall', '`timescale', '`unconnected_drive',
                    '`undef', '`undefineall'),
                   suffix=r'\b'), Comment.Preproc),
            (words(
                ('$display', '$displayb', '$displayh', '$displayo', '$dumpall',
                 '$dumpfile', '$dumpflush', '$dumplimit', '$dumpoff',
                 '$dumpon', '$dumpports', '$dumpportsall', '$dumpportsflush',
                 '$dumpportslimit', '$dumpportsoff', '$dumpportson',
                 '$dumpvars', '$fclose', '$fdisplay', '$fdisplayb',
                 '$fdisplayh', '$fdisplayo', '$feof', '$ferror', '$fflush',
                 '$fgetc', '$fgets', '$finish', '$fmonitor', '$fmonitorb',
                 '$fmonitorh', '$fmonitoro', '$fopen', '$fread', '$fscanf',
                 '$fseek', '$fstrobe', '$fstrobeb', '$fstrobeh', '$fstrobeo',
                 '$ftell', '$fwrite', '$fwriteb', '$fwriteh', '$fwriteo',
                 '$monitor', '$monitorb', '$monitorh', '$monitoro',
                 '$monitoroff', '$monitoron', '$plusargs', '$random',
                 '$readmemb', '$readmemh', '$rewind', '$sformat', '$sformatf',
                 '$sscanf', '$strobe', '$strobeb', '$strobeh', '$strobeo',
                 '$swrite', '$swriteb', '$swriteh', '$swriteo', '$test',
                 '$ungetc', '$value$plusargs', '$write', '$writeb', '$writeh',
                 '$writememb', '$writememh', '$writeo'),
                suffix=r'\b'), Name.Builtin),
            (r'(class)(\s+)', bygroups(Keyword, Text), 'classname'),
            (words(('byte', 'shortint', 'int', 'longint', 'integer', 'time',
                    'bit', 'logic', 'reg', 'supply0', 'supply1', 'tri',
                    'triand', 'trior', 'tri0', 'tri1', 'trireg', 'uwire',
                    'wire', 'wand', 'wo'
                    'shortreal', 'real', 'realtime'),
                   suffix=r'\b'), Keyword.Type),
            ('[a-zA-Z_]\w*:(?!:)', Name.Label),
            ('[a-zA-Z_]\w*', Name),
        ],
        'classname': [
            (r'[a-zA-Z_]\w*', Name.Class, '#pop'),
        ],
        'string': [
            (r'"', String, '#pop'),
            (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})',
             String.Escape),
            (r'[^\\"\n]+', String),  # all other characters
            (r'\\\n', String),  # line continuation
            (r'\\', String),  # stray backslash
        ],
        'macro': [
            (r'[^/\n]+', Comment.Preproc),
            (r'/[*](.|\n)*?[*]/', Comment.Multiline),
            (r'//.*?\n', Comment.Single, '#pop'),
            (r'/', Comment.Preproc),
            (r'(?<=\\)\n', Comment.Preproc),
            (r'\n', Comment.Preproc, '#pop'),
        ],
        'import': [(r'[\w:]+\*?', Name.Namespace, '#pop')]
    }

    def get_tokens_unprocessed(self, text):
        for index, token, value in \
                RegexLexer.get_tokens_unprocessed(self, text):
            # Convention: mark all upper case names as constants
            if token is Name:
                if value.isupper():
                    token = Name.Constant
            yield index, token, value
예제 #4
0
class VhdlLexer(RegexLexer):
    """
    For VHDL source code.

    .. versionadded:: 1.5
    """
    name = 'vhdl'
    aliases = ['vhdl']
    filenames = ['*.vhdl', '*.vhd']
    mimetypes = ['text/x-vhdl']
    flags = re.MULTILINE | re.IGNORECASE

    tokens = {
        'root': [
            (r'\n', Text),
            (r'\s+', Text),
            (r'\\\n', Text),  # line continuation
            (r'--.*?$', Comment.Single),
            (r"'(U|X|0|1|Z|W|L|H|-)'", String.Char),
            (r'[~!%^&*+=|?:<>/-]', Operator),
            (r"'[a-z_]\w*", Name.Attribute),
            (r'[()\[\],.;\']', Punctuation),
            (r'"[^\n\\]*"', String),
            (r'(library)(\s+)([a-z_]\w*)',
             bygroups(Keyword, Text, Name.Namespace)),
            (r'(use)(\s+)(entity)', bygroups(Keyword, Text, Keyword)),
            (r'(use)(\s+)([a-z_][\w.]*)',
             bygroups(Keyword, Text, Name.Namespace)),
            (r'(entity|component)(\s+)([a-z_]\w*)',
             bygroups(Keyword, Text, Name.Class)),
            (r'(architecture|configuration)(\s+)([a-z_]\w*)(\s+)'
             r'(of)(\s+)([a-z_]\w*)(\s+)(is)',
             bygroups(Keyword, Text, Name.Class, Text, Keyword, Text,
                      Name.Class, Text, Keyword)),
            (r'(end)(\s+)', bygroups(using(this), Text), 'endblock'),
            include('types'),
            include('keywords'),
            include('numbers'),
            (r'[a-z_]\w*', Name),
        ],
        'endblock': [
            include('keywords'),
            (r'[a-z_]\w*', Name.Class),
            (r'(\s+)', Text),
            (r';', Punctuation, '#pop'),
        ],
        'types': [
            (words(('boolean', 'bit', 'character', 'severity_level', 'integer',
                    'time', 'delay_length', 'natural', 'positive', 'string',
                    'bit_vector', 'file_open_kind', 'file_open_status',
                    'std_ulogic', 'std_ulogic_vector', 'std_logic',
                    'std_logic_vector'),
                   suffix=r'\b'), Keyword.Type),
        ],
        'keywords': [
            (words(
                ('abs', 'access', 'after', 'alias', 'all', 'and',
                 'architecture', 'array', 'assert', 'attribute', 'begin',
                 'block', 'body', 'buffer', 'bus', 'case', 'component',
                 'configuration', 'constant', 'disconnect', 'downto', 'else',
                 'elsif', 'end', 'entity', 'exit', 'file', 'for', 'function',
                 'generate', 'generic', 'group', 'guarded', 'if', 'impure',
                 'in', 'inertial', 'inout', 'is', 'label', 'library',
                 'linkage', 'literal', 'loop', 'map', 'mod', 'nand', 'new',
                 'next', 'nor', 'not', 'null', 'of', 'on', 'open', 'or',
                 'others', 'out', 'package', 'port', 'postponed', 'procedure',
                 'process', 'pure', 'range', 'record', 'register', 'reject',
                 'return', 'rol', 'ror', 'select', 'severity', 'signal',
                 'shared', 'sla', 'sli', 'sra', 'srl', 'subtype', 'then', 'to',
                 'transport', 'type', 'units', 'until', 'use', 'variable',
                 'wait', 'when', 'while', 'with', 'xnor', 'xor'),
                suffix=r'\b'), Keyword),
        ],
        'numbers': [
            (r'\d{1,2}#[0-9a-f_]+#?', Number.Integer),
            (r'\d+', Number.Integer),
            (r'(\d+\.\d*|\.\d+|\d+)E[+-]?\d+', Number.Float),
            (r'X"[0-9a-f_]+"', Number.Hex),
            (r'O"[0-7_]+"', Number.Oct),
            (r'B"[01_]+"', Number.Bin),
        ],
    }
예제 #5
0
파일: ampl.py 프로젝트: Igorshu90/Hal1.1
class AmplLexer(RegexLexer):
    """
    For `AMPL <http://ampl.com/>`_ source code.

    .. versionadded:: 2.2
    """
    name = 'Ampl'
    aliases = ['ampl']
    filenames = ['*.run']

    tokens = {
        'root': [
            (r'\n', Text),
            (r'\s+', Text.Whitespace),
            (r'#.*?\n', Comment.Single),
            (r'/[*](.|\n)*?[*]/', Comment.Multiline),
            (words(
                ('call', 'cd', 'close', 'commands', 'data', 'delete',
                 'display', 'drop', 'end', 'environ', 'exit', 'expand',
                 'include', 'load', 'model', 'objective', 'option', 'problem',
                 'purge', 'quit', 'redeclare', 'reload', 'remove', 'reset',
                 'restore', 'shell', 'show', 'solexpand', 'solution', 'solve',
                 'update', 'unload', 'xref', 'coeff', 'coef', 'cover', 'obj',
                 'interval', 'default', 'from', 'to', 'to_come', 'net_in',
                 'net_out', 'dimen', 'dimension', 'check', 'complements',
                 'write', 'function', 'pipe', 'format', 'if', 'then', 'else',
                 'in', 'while', 'repeat', 'for'),
                suffix=r'\b'), Keyword.Reserved),
            (r'(integer|binary|symbolic|ordered|circular|reversed|INOUT|IN|OUT|LOCAL)',
             Keyword.Type),
            (r'\".*?\"', String.Double),
            (r'\'.*?\'', String.Single),
            (r'[()\[\]{},;:]+', Punctuation),
            (r'\b(\w+)(\.)(astatus|init0|init|lb0|lb1|lb2|lb|lrc|'
             r'lslack|rc|relax|slack|sstatus|status|ub0|ub1|ub2|'
             r'ub|urc|uslack|val)',
             bygroups(Name.Variable, Punctuation, Keyword.Reserved)),
            (r'(set|param|var|arc|minimize|maximize|subject to|s\.t\.|subj to|'
             r'node|table|suffix|read table|write table)(\s+)(\w+)',
             bygroups(Keyword.Declaration, Text, Name.Variable)),
            (r'(param)(\s*)(:)(\s*)(\w+)(\s*)(:)(\s*)((\w|\s)+)',
             bygroups(Keyword.Declaration, Text, Punctuation, Text,
                      Name.Variable, Text, Punctuation, Text, Name.Variable)),
            (r'(let|fix|unfix)(\s*)((?:\{.*\})?)(\s*)(\w+)',
             bygroups(Keyword.Declaration, Text, using(this), Text,
                      Name.Variable)),
            (words(('abs', 'acos', 'acosh', 'alias', 'asin', 'asinh', 'atan',
                    'atan2', 'atanh', 'ceil', 'ctime', 'cos', 'exp', 'floor',
                    'log', 'log10', 'max', 'min', 'precision', 'round', 'sin',
                    'sinh', 'sqrt', 'tan', 'tanh', 'time', 'trunc', 'Beta',
                    'Cauchy', 'Exponential', 'Gamma', 'Irand224', 'Normal',
                    'Normal01', 'Poisson', 'Uniform', 'Uniform01', 'num',
                    'num0', 'ichar', 'char', 'length', 'substr', 'sprintf',
                    'match', 'sub', 'gsub', 'print', 'printf', 'next', 'nextw',
                    'prev', 'prevw', 'first', 'last', 'ord', 'ord0', 'card',
                    'arity', 'indexarity'),
                   prefix=r'\b',
                   suffix=r'\b'), Name.Builtin),
            (r'(\+|\-|\*|/|\*\*|=|<=|>=|==|\||\^|<|>|\!|\.\.|:=|\&|\!=|<<|>>)',
             Operator),
            (words(('or', 'exists', 'forall', 'and', 'in', 'not', 'within',
                    'union', 'diff', 'difference', 'symdiff', 'inter',
                    'intersect', 'intersection', 'cross', 'setof', 'by',
                    'less', 'sum', 'prod', 'product', 'div', 'mod'),
                   suffix=r'\b'), Keyword.Reserved
             ),  # Operator.Name but not enough emphasized with that
            (r'(\d+\.(?!\.)\d*|\.(?!.)\d+)([eE][+-]?\d+)?', Number.Float),
            (r'\d+([eE][+-]?\d+)?', Number.Integer),
            (r'[+-]?Infinity', Number.Integer),
            (r'(\w+|(\.(?!\.)))', Text)
        ]
    }
예제 #6
0
파일: c_cpp.py 프로젝트: zoulianmp/PythonQt
class CFamilyLexer(RegexLexer):
    """
    For C family source code.  This is used as a base class to avoid repetitious
    definitions.
    """

    #: optional Comment or Whitespace
    _ws = r'(?:\s|//.*?\n|/[*].*?[*]/)+'
    #: only one /* */ style comment
    _ws1 = r'\s*(?:/[*].*?[*]/\s*)*'

    tokens = {
        'whitespace': [
            # preprocessor directives: without whitespace
            ('^#if\s+0', Comment.Preproc, 'if0'),
            ('^#', Comment.Preproc, 'macro'),
            # or with whitespace
            ('^(' + _ws1 + r')(#if\s+0)', bygroups(using(this),
                                                   Comment.Preproc), 'if0'),
            ('^(' + _ws1 + ')(#)', bygroups(using(this),
                                            Comment.Preproc), 'macro'),
            (r'\n', Text),
            (r'\s+', Text),
            (r'\\\n', Text),  # line continuation
            (r'//(\n|(.|\n)*?[^\\]\n)', Comment.Single),
            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
        ],
        'statements': [
            (r'L?"', String, 'string'),
            (r"L?'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'",
             String.Char),
            (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*', Number.Float),
            (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
            (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex),
            (r'0[0-7]+[LlUu]*', Number.Oct),
            (r'\d+[LlUu]*', Number.Integer),
            (r'\*/', Error),
            (r'[~!%^&*+=|?:<>/-]', Operator),
            (r'[()\[\],.]', Punctuation),
            (words(
                ('auto', 'break', 'case', 'const', 'continue', 'default', 'do',
                 'else', 'enum', 'extern', 'for', 'goto', 'if', 'register',
                 'restricted', 'return', 'sizeof', 'static', 'struct',
                 'switch', 'typedef', 'union', 'volatile', 'while'),
                suffix=r'\b'), Keyword),
            (r'(bool|int|long|float|short|double|char|unsigned|signed|void|'
             r'[a-z_][a-z0-9_]*_t)\b', Keyword.Type),
            (words(('inline', '_inline', '__inline', 'naked', 'restrict',
                    'thread', 'typename'),
                   suffix=r'\b'), Keyword.Reserved),
            # Vector intrinsics
            (r'(__m(128i|128d|128|64))\b', Keyword.Reserved),
            # Microsoft-isms
            (words(('asm', 'int8', 'based', 'except', 'int16', 'stdcall',
                    'cdecl', 'fastcall', 'int32', 'declspec', 'finally',
                    'int64', 'try', 'leave', 'wchar_t', 'w64', 'unaligned',
                    'raise', 'noop', 'identifier', 'forceinline', 'assume'),
                   prefix=r'__',
                   suffix=r'\b'), Keyword.Reserved),
            (r'(true|false|NULL)\b', Name.Builtin),
            (r'([a-zA-Z_]\w*)(\s*)(:)(?!:)',
             bygroups(Name.Label, Text, Punctuation)),
            ('[a-zA-Z_]\w*', Name),
        ],
        'root': [
            include('whitespace'),
            # functions
            (
                r'((?:[\w*\s])+?(?:\s|[*]))'  # return arguments
                r'([a-zA-Z_]\w*)'  # method name
                r'(\s*\([^;]*?\))'  # signature
                r'(' + _ws + r')?({)',
                bygroups(using(this), Name.Function, using(this), using(this),
                         Punctuation),
                'function'),
            # function declarations
            (
                r'((?:[\w*\s])+?(?:\s|[*]))'  # return arguments
                r'([a-zA-Z_]\w*)'  # method name
                r'(\s*\([^;]*?\))'  # signature
                r'(' + _ws + r')?(;)',
                bygroups(using(this), Name.Function, using(this), using(this),
                         Punctuation)),
            default('statement'),
        ],
        'statement': [
            include('whitespace'),
            include('statements'),
            ('[{}]', Punctuation),
            (';', Punctuation, '#pop'),
        ],
        'function': [
            include('whitespace'),
            include('statements'),
            (';', Punctuation),
            (r'\{', Punctuation, '#push'),
            (r'\}', Punctuation, '#pop'),
        ],
        'string': [
            (r'"', String, '#pop'),
            (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|'
             r'u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|[0-7]{1,3})', String.Escape),
            (r'[^\\"\n]+', String),  # all other characters
            (r'\\\n', String),  # line continuation
            (r'\\', String),  # stray backslash
        ],
        'macro': [
            (r'[^/\n]+', Comment.Preproc),
            (r'/[*](.|\n)*?[*]/', Comment.Multiline),
            (r'//.*?\n', Comment.Single, '#pop'),
            (r'/', Comment.Preproc),
            (r'(?<=\\)\n', Comment.Preproc),
            (r'\n', Comment.Preproc, '#pop'),
        ],
        'if0': [
            (r'^\s*#if.*?(?<!\\)\n', Comment.Preproc, '#push'),
            (r'^\s*#el(?:se|if).*\n', Comment.Preproc, '#pop'),
            (r'^\s*#endif.*?(?<!\\)\n', Comment.Preproc, '#pop'),
            (r'.*?\n', Comment),
        ]
    }

    stdlib_types = [
        'size_t', 'ssize_t', 'off_t', 'wchar_t', 'ptrdiff_t', 'sig_atomic_t',
        'fpos_t', 'clock_t', 'time_t', 'va_list', 'jmp_buf', 'FILE', 'DIR',
        'div_t', 'ldiv_t', 'mbstate_t', 'wctrans_t', 'wint_t', 'wctype_t'
    ]
    c99_types = [
        '_Bool', '_Complex', 'int8_t', 'int16_t', 'int32_t', 'int64_t',
        'uint8_t', 'uint16_t', 'uint32_t', 'uint64_t', 'int_least8_t',
        'int_least16_t', 'int_least32_t', 'int_least64_t', 'uint_least8_t',
        'uint_least16_t', 'uint_least32_t', 'uint_least64_t', 'int_fast8_t',
        'int_fast16_t', 'int_fast32_t', 'int_fast64_t', 'uint_fast8_t',
        'uint_fast16_t', 'uint_fast32_t', 'uint_fast64_t', 'intptr_t',
        'uintptr_t', 'intmax_t', 'uintmax_t'
    ]

    def __init__(self, **options):
        self.stdlibhighlighting = get_bool_opt(options, 'stdlibhighlighting',
                                               True)
        self.c99highlighting = get_bool_opt(options, 'c99highlighting', True)
        RegexLexer.__init__(self, **options)

    def get_tokens_unprocessed(self, text):
        for index, token, value in \
                RegexLexer.get_tokens_unprocessed(self, text):
            if token is Name:
                if self.stdlibhighlighting and value in self.stdlib_types:
                    token = Keyword.Type
                elif self.c99highlighting and value in self.c99_types:
                    token = Keyword.Type
            yield index, token, value
예제 #7
0
파일: shell.py 프로젝트: kylemitra/BME-547
 def _make_begin_state(compound,
                       _core_token=_core_token,
                       _core_token_compound=_core_token_compound,
                       _keyword_terminator=_keyword_terminator,
                       _nl=_nl,
                       _punct=_punct,
                       _string=_string,
                       _space=_space,
                       _start_label=_start_label,
                       _stoken=_stoken,
                       _token_terminator=_token_terminator,
                       _variable=_variable,
                       _ws=_ws):
     rest = '(?:%s|%s|[^"%%%s%s%s])*' % (_string, _variable, _nl, _punct,
                                         ')' if compound else '')
     rest_of_line = r'(?:(?:[^%s^]|\^[%s]?[\w\W])*)' % (_nl, _nl)
     rest_of_line_compound = r'(?:(?:[^%s^)]|\^[%s]?[^)])*)' % (_nl, _nl)
     set_space = r'((?:(?:\^[%s]?)?[^\S\n])*)' % _nl
     suffix = ''
     if compound:
         _keyword_terminator = r'(?:(?=\))|%s)' % _keyword_terminator
         _token_terminator = r'(?:(?=\))|%s)' % _token_terminator
         suffix = '/compound'
     return [
         ((r'\)', Punctuation, '#pop') if compound else
          (r'\)((?=\()|%s)%s' % (_token_terminator, rest_of_line),
           Comment.Single)),
         (r'(?=%s)' % _start_label, Text, 'follow%s' % suffix),
         (_space, using(this, state='text')),
         include('redirect%s' % suffix), (r'[%s]+' % _nl, Text),
         (r'\(', Punctuation, 'root/compound'), (r'@+', Punctuation),
         (r'((?:for|if|rem)(?:(?=(?:\^[%s]?)?/)|(?:(?!\^)|'
          r'(?<=m))(?:(?=\()|%s)))(%s?%s?(?:\^[%s]?)?/(?:\^[%s]?)?\?)' %
          (_nl, _token_terminator, _space,
           _core_token_compound if compound else _core_token, _nl, _nl),
          bygroups(Keyword, using(this,
                                  state='text')), 'follow%s' % suffix),
         (r'(goto%s)(%s(?:\^[%s]?)?/(?:\^[%s]?)?\?%s)' %
          (_keyword_terminator, rest, _nl, _nl, rest),
          bygroups(Keyword, using(this,
                                  state='text')), 'follow%s' % suffix),
         (words(('assoc', 'break', 'cd', 'chdir', 'cls', 'color', 'copy',
                 'date', 'del', 'dir', 'dpath', 'echo', 'endlocal', 'erase',
                 'exit', 'ftype', 'keys', 'md', 'mkdir', 'mklink', 'move',
                 'path', 'pause', 'popd', 'prompt', 'pushd', 'rd', 'ren',
                 'rename', 'rmdir', 'setlocal', 'shift', 'start', 'time',
                 'title', 'type', 'ver', 'verify', 'vol'),
                suffix=_keyword_terminator), Keyword, 'follow%s' % suffix),
         (r'(call)(%s?)(:)' % _space,
          bygroups(Keyword, using(this, state='text'),
                   Punctuation), 'call%s' % suffix),
         (r'call%s' % _keyword_terminator, Keyword),
         (r'(for%s(?!\^))(%s)(/f%s)' %
          (_token_terminator, _space, _token_terminator),
          bygroups(Keyword, using(this, state='text'),
                   Keyword), ('for/f', 'for')),
         (r'(for%s(?!\^))(%s)(/l%s)' %
          (_token_terminator, _space, _token_terminator),
          bygroups(Keyword, using(this, state='text'),
                   Keyword), ('for/l', 'for')),
         (r'for%s(?!\^)' % _token_terminator, Keyword, ('for2', 'for')),
         (r'(goto%s)(%s?)(:?)' % (_keyword_terminator, _space),
          bygroups(Keyword, using(this, state='text'),
                   Punctuation), 'label%s' % suffix),
         (r'(if(?:(?=\()|%s)(?!\^))(%s?)((?:/i%s)?)(%s?)((?:not%s)?)(%s?)' %
          (_token_terminator, _space, _token_terminator, _space,
           _token_terminator, _space),
          bygroups(Keyword, using(this, state='text'), Keyword,
                   using(this, state='text'), Keyword,
                   using(this, state='text')), ('(?', 'if')),
         (r'rem(((?=\()|%s)%s?%s?.*|%s%s)' %
          (_token_terminator, _space, _stoken, _keyword_terminator,
           rest_of_line_compound if compound else rest_of_line),
          Comment.Single, 'follow%s' % suffix),
         (r'(set%s)%s(/a)' % (_keyword_terminator, set_space),
          bygroups(Keyword, using(this, state='text'),
                   Keyword), 'arithmetic%s' % suffix),
         (r'(set%s)%s((?:/p)?)%s((?:(?:(?:\^[%s]?)?[^"%s%s^=%s]|'
          r'\^[%s]?[^"=])+)?)((?:(?:\^[%s]?)?=)?)' %
          (_keyword_terminator, set_space, set_space, _nl, _nl, _punct,
           ')' if compound else '', _nl, _nl),
          bygroups(Keyword, using(this, state='text'), Keyword,
                   using(this, state='text'), using(this, state='variable'),
                   Punctuation), 'follow%s' % suffix),
         default('follow%s' % suffix)
     ]
예제 #8
0
파일: css.py 프로젝트: 0xDEC0DE8/diaphora
 (words((
     'azimuth', 'background-attachment', 'background-color',
     'background-image', 'background-position', 'background-repeat',
     'background', 'border-bottom-color', 'border-bottom-style',
     'border-bottom-width', 'border-left-color', 'border-left-style',
     'border-left-width', 'border-right', 'border-right-color',
     'border-right-style', 'border-right-width', 'border-top-color',
     'border-top-style', 'border-top-width', 'border-bottom',
     'border-collapse', 'border-left', 'border-width', 'border-color',
     'border-spacing', 'border-style', 'border-top', 'border', 'caption-side',
     'clear', 'clip', 'color', 'content', 'counter-increment', 'counter-reset',
     'cue-after', 'cue-before', 'cue', 'cursor', 'direction', 'display',
     'elevation', 'empty-cells', 'float', 'font-family', 'font-size',
     'font-size-adjust', 'font-stretch', 'font-style', 'font-variant',
     'font-weight', 'font', 'height', 'letter-spacing', 'line-height',
     'list-style-type', 'list-style-image', 'list-style-position',
     'list-style', 'margin-bottom', 'margin-left', 'margin-right',
     'margin-top', 'margin', 'marker-offset', 'marks', 'max-height', 'max-width',
     'min-height', 'min-width', 'opacity', 'orphans', 'outline', 'outline-color',
     'outline-style', 'outline-width', 'overflow', 'padding-bottom',
     'padding-left', 'padding-right', 'padding-top', 'padding', 'page',
     'page-break-after', 'page-break-before', 'page-break-inside',
     'pause-after', 'pause-before', 'pause', 'pitch', 'pitch-range',
     'play-during', 'position', 'quotes', 'richness', 'right', 'size',
     'speak-header', 'speak-numeral', 'speak-punctuation', 'speak',
     'speech-rate', 'stress', 'table-layout', 'text-align', 'text-decoration',
     'text-indent', 'text-shadow', 'text-transform', 'top', 'unicode-bidi',
     'vertical-align', 'visibility', 'voice-family', 'volume', 'white-space',
     'widows', 'width', 'word-spacing', 'z-index', 'bottom', 'left',
     'above', 'absolute', 'always', 'armenian', 'aural', 'auto', 'avoid', 'baseline',
     'behind', 'below', 'bidi-override', 'blink', 'block', 'bold', 'bolder', 'both',
     'capitalize', 'center-left', 'center-right', 'center', 'circle',
     'cjk-ideographic', 'close-quote', 'collapse', 'condensed', 'continuous',
     'crop', 'crosshair', 'cross', 'cursive', 'dashed', 'decimal-leading-zero',
     'decimal', 'default', 'digits', 'disc', 'dotted', 'double', 'e-resize', 'embed',
     'extra-condensed', 'extra-expanded', 'expanded', 'fantasy', 'far-left',
     'far-right', 'faster', 'fast', 'fixed', 'georgian', 'groove', 'hebrew', 'help',
     'hidden', 'hide', 'higher', 'high', 'hiragana-iroha', 'hiragana', 'icon',
     'inherit', 'inline-table', 'inline', 'inset', 'inside', 'invert', 'italic',
     'justify', 'katakana-iroha', 'katakana', 'landscape', 'larger', 'large',
     'left-side', 'leftwards', 'level', 'lighter', 'line-through', 'list-item',
     'loud', 'lower-alpha', 'lower-greek', 'lower-roman', 'lowercase', 'ltr',
     'lower', 'low', 'medium', 'message-box', 'middle', 'mix', 'monospace',
     'n-resize', 'narrower', 'ne-resize', 'no-close-quote', 'no-open-quote',
     'no-repeat', 'none', 'normal', 'nowrap', 'nw-resize', 'oblique', 'once',
     'open-quote', 'outset', 'outside', 'overline', 'pointer', 'portrait', 'px',
     'relative', 'repeat-x', 'repeat-y', 'repeat', 'rgb', 'ridge', 'right-side',
     'rightwards', 's-resize', 'sans-serif', 'scroll', 'se-resize',
     'semi-condensed', 'semi-expanded', 'separate', 'serif', 'show', 'silent',
     'slow', 'slower', 'small-caps', 'small-caption', 'smaller', 'soft', 'solid',
     'spell-out', 'square', 'static', 'status-bar', 'super', 'sw-resize',
     'table-caption', 'table-cell', 'table-column', 'table-column-group',
     'table-footer-group', 'table-header-group', 'table-row',
     'table-row-group', 'text', 'text-bottom', 'text-top', 'thick', 'thin',
     'transparent', 'ultra-condensed', 'ultra-expanded', 'underline',
     'upper-alpha', 'upper-latin', 'upper-roman', 'uppercase', 'url',
     'visible', 'w-resize', 'wait', 'wider', 'x-fast', 'x-high', 'x-large', 'x-loud',
     'x-low', 'x-small', 'x-soft', 'xx-large', 'xx-small', 'yes'), suffix=r'\b'),
예제 #9
0
    def gen_rubystrings_rules():
        def intp_regex_callback(self, match, ctx):
            yield match.start(1), String.Regex, match.group(1)  # begin
            nctx = LexerContext(match.group(3), 0, ['interpolated-regex'])
            for i, t, v in self.get_tokens_unprocessed(context=nctx):
                yield match.start(3) + i, t, v
            yield match.start(4), String.Regex, match.group(
                4)  # end[mixounse]*
            ctx.pos = match.end()

        def intp_string_callback(self, match, ctx):
            yield match.start(1), String.Other, match.group(1)
            nctx = LexerContext(match.group(3), 0, ['interpolated-string'])
            for i, t, v in self.get_tokens_unprocessed(context=nctx):
                yield match.start(3) + i, t, v
            yield match.start(4), String.Other, match.group(4)  # end
            ctx.pos = match.end()

        states = {}
        states['strings'] = [
            # easy ones
            (r'\:@{0,2}[a-zA-Z_]\w*[!?]?', String.Symbol),
            (words(RUBY_OPERATORS, prefix=r'\:@{0,2}'), String.Symbol),
            (r":'(\\\\|\\'|[^'])*'", String.Symbol),
            (r"'(\\\\|\\'|[^'])*'", String.Single),
            (r':"', String.Symbol, 'simple-sym'),
            (r'([a-zA-Z_]\w*)(:)(?!:)',
             bygroups(String.Symbol, Punctuation)),  # Since Ruby 1.9
            (r'"', String.Double, 'simple-string'),
            (r'(?<!\.)`', String.Backtick, 'simple-backtick'),
        ]

        # double-quoted string and symbol
        for name, ttype, end in ('string', String.Double, '"'), \
                                ('sym', String.Symbol, '"'), \
                                ('backtick', String.Backtick, '`'):
            states['simple-' + name] = [
                include('string-intp-escaped'),
                (r'[^\\%s#]+' % end, ttype),
                (r'[\\#]', ttype),
                (end, ttype, '#pop'),
            ]

        # braced quoted strings
        for lbrace, rbrace, bracecc, name in \
                ('\\{', '\\}', '{}', 'cb'), \
                ('\\[', '\\]', '\\[\\]', 'sb'), \
                ('\\(', '\\)', '()', 'pa'), \
                ('<', '>', '<>', 'ab'):
            states[name + '-intp-string'] = [
                (r'\\[\\' + bracecc + ']', String.Other),
                (lbrace, String.Other, '#push'),
                (rbrace, String.Other, '#pop'),
                include('string-intp-escaped'),
                (r'[\\#' + bracecc + ']', String.Other),
                (r'[^\\#' + bracecc + ']+', String.Other),
            ]
            states['strings'].append(
                (r'%[QWx]?' + lbrace, String.Other, name + '-intp-string'))
            states[name + '-string'] = [
                (r'\\[\\' + bracecc + ']', String.Other),
                (lbrace, String.Other, '#push'),
                (rbrace, String.Other, '#pop'),
                (r'[\\#' + bracecc + ']', String.Other),
                (r'[^\\#' + bracecc + ']+', String.Other),
            ]
            states['strings'].append(
                (r'%[qsw]' + lbrace, String.Other, name + '-string'))
            states[name + '-regex'] = [
                (r'\\[\\' + bracecc + ']', String.Regex),
                (lbrace, String.Regex, '#push'),
                (rbrace + '[mixounse]*', String.Regex, '#pop'),
                include('string-intp'),
                (r'[\\#' + bracecc + ']', String.Regex),
                (r'[^\\#' + bracecc + ']+', String.Regex),
            ]
            states['strings'].append(
                (r'%r' + lbrace, String.Regex, name + '-regex'))

        # these must come after %<brace>!
        states['strings'] += [
            # %r regex
            (r'(%r([\W_]))((?:\\\2|(?!\2).)*)(\2[mixounse]*)',
             intp_regex_callback),
            # regular fancy strings with qsw
            (r'%[qsw]([\W_])((?:\\\1|(?!\1).)*)\1', String.Other),
            (r'(%[QWx]([\W_]))((?:\\\2|(?!\2).)*)(\2)', intp_string_callback),
            # special forms of fancy strings after operators or
            # in method calls with braces
            (r'(?<=[-+/*%=<>&!^|~,(])(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
             bygroups(Text, String.Other, None)),
            # and because of fixed width lookbehinds the whole thing a
            # second time for line startings...
            (r'^(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
             bygroups(Text, String.Other, None)),
            # all regular fancy strings without qsw
            (r'(%([^a-zA-Z0-9\s]))((?:\\\2|(?!\2).)*)(\2)',
             intp_string_callback),
        ]

        return states
예제 #10
0
class MznDefLexer(MznLexer):
    name = 'MiniZincDef'
    aliases = ['minizincdef']

    tokens = {
        'root': [
            (r'<[0-9a-zA-Z- ]+>', Comment.Special),
            (r'"/\\"', String),
            (r'"""', String),
            (r'"\\\("', String),
            (r'"\\.\\.\\."', Comment.Special),
            (r'\[[^ ]', String.Regex, 'regex'),
            (r'::=', Comment.Special),
            (words(('|',), prefix=r'^\s*'), Comment.Special),
            inherit
        ],
        'main__1': [
            (r'<[0-9a-zA-Z- ]+>', Comment.Special),
            (r'"/\\"', String),
            (r'"""', String),
            (r'"\\\("', String),
            (r'"\\.\\.\\."', String),
            (r'\[[^ ]', String.Regex, 'regex'),
            (r'::=', Comment.Special),
            (words(('|',), prefix=r'^\s*'), Comment.Special),
            inherit  
        ],
        'main__2': [
            (r'<[0-9a-zA-Z- ]+>', Comment.Special),
            (r'"/\\"', String),
            (r'"""', String),
            (r'"\\\("', String),
            (r'"\\.\\.\\."', String),
            (r'\[[^ ]', String.Regex, 'regex'),
            (r'::=', Comment.Special),
            (words(('|',), prefix=r'^\s*'), Comment.Special),
            inherit  
        ],
        'main__3': [
            (r'<[0-9a-zA-Z- ]+>', Comment.Special),
            (r'"/\\"', String),
            (r'"""', String),
            (r'"\\\("', String),
            (r'"\\.\\.\\."', String),
            (r'\[[^ ]', String.Regex, 'regex'),
            (r'::=', Comment.Special),
            (words(('|',), prefix=r'^\s*'), Comment.Special),
            inherit  
        ],
        'main__4': [
            (r'<[0-9a-zA-Z- ]+>', Comment.Special),
            (r'"/\\"', String),
            (r'"""', String),
            (r'"\\\("', String),
            (r'"\\.\\.\\."', String),
            (r'\[[^ ]', String.Regex, 'regex'),
            (r'::=', Comment.Special),
            (words(('|',), prefix=r'^\s*'), Comment.Special),
            inherit  
        ],
        'regex': [
            (r'\]', String.Regex, '#pop'),
            (r'.', String.Regex)
        ]
    }
예제 #11
0
class FancyLexer(RegexLexer):
    """
    Pygments Lexer For `Fancy <http://www.fancy-lang.org/>`_.

    Fancy is a self-hosted, pure object-oriented, dynamic,
    class-based, concurrent general-purpose programming language
    running on Rubinius, the Ruby VM.

    .. versionadded:: 1.5
    """
    name = 'Fancy'
    filenames = ['*.fy', '*.fancypack']
    aliases = ['fancy', 'fy']
    mimetypes = ['text/x-fancysrc']

    tokens = {
        # copied from PerlLexer:
        'balanced-regex': [
            (r'/(\\\\|\\/|[^/])*/[egimosx]*', String.Regex, '#pop'),
            (r'!(\\\\|\\!|[^!])*![egimosx]*', String.Regex, '#pop'),
            (r'\\(\\\\|[^\\])*\\[egimosx]*', String.Regex, '#pop'),
            (r'\{(\\\\|\\\}|[^}])*\}[egimosx]*', String.Regex, '#pop'),
            (r'<(\\\\|\\>|[^>])*>[egimosx]*', String.Regex, '#pop'),
            (r'\[(\\\\|\\\]|[^\]])*\][egimosx]*', String.Regex, '#pop'),
            (r'\((\\\\|\\\)|[^)])*\)[egimosx]*', String.Regex, '#pop'),
            (r'@(\\\\|\\@|[^@])*@[egimosx]*', String.Regex, '#pop'),
            (r'%(\\\\|\\%|[^%])*%[egimosx]*', String.Regex, '#pop'),
            (r'\$(\\\\|\\\$|[^$])*\$[egimosx]*', String.Regex, '#pop'),
        ],
        'root': [
            (r'\s+', Text),

            # balanced delimiters (copied from PerlLexer):
            (r's\{(\\\\|\\\}|[^}])*\}\s*', String.Regex, 'balanced-regex'),
            (r's<(\\\\|\\>|[^>])*>\s*', String.Regex, 'balanced-regex'),
            (r's\[(\\\\|\\\]|[^\]])*\]\s*', String.Regex, 'balanced-regex'),
            (r's\((\\\\|\\\)|[^)])*\)\s*', String.Regex, 'balanced-regex'),
            (r'm?/(\\\\|\\/|[^/\n])*/[gcimosx]*', String.Regex),
            (r'm(?=[/!\\{<\[(@%$])', String.Regex, 'balanced-regex'),

            # Comments
            (r'#(.*?)\n', Comment.Single),
            # Symbols
            (r'\'([^\'\s\[\](){}]+|\[\])', String.Symbol),
            # Multi-line DoubleQuotedString
            (r'"""(\\\\|\\"|[^"])*"""', String),
            # DoubleQuotedString
            (r'"(\\\\|\\"|[^"])*"', String),
            # keywords
            (r'(def|class|try|catch|finally|retry|return|return_local|match|'
             r'case|->|=>)\b', Keyword),
            # constants
            (r'(self|super|nil|false|true)\b', Name.Constant),
            (r'[(){};,/?|:\\]', Punctuation),
            # names
            (words(
                ('Object', 'Array', 'Hash', 'Directory', 'File', 'Class',
                 'String', 'Number', 'Enumerable', 'FancyEnumerable', 'Block',
                 'TrueClass', 'NilClass', 'FalseClass', 'Tuple', 'Symbol',
                 'Stack', 'Set', 'FancySpec', 'Method', 'Package', 'Range'),
                suffix=r'\b'), Name.Builtin),
            # functions
            (r'[a-zA-Z](\w|[-+?!=*/^><%])*:', Name.Function),
            # operators, must be below functions
            (r'[-+*/~,<>=&!?%^\[\].$]+', Operator),
            ('[A-Z]\w*', Name.Constant),
            ('@[a-zA-Z_]\w*', Name.Variable.Instance),
            ('@@[a-zA-Z_]\w*', Name.Variable.Class),
            ('@@?', Operator),
            ('[a-zA-Z_]\w*', Name),
            # numbers - / checks are necessary to avoid mismarking regexes,
            # see comment in RubyLexer
            (r'(0[oO]?[0-7]+(?:_[0-7]+)*)(\s*)([/?])?',
             bygroups(Number.Oct, Text, Operator)),
            (r'(0[xX][0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*)(\s*)([/?])?',
             bygroups(Number.Hex, Text, Operator)),
            (r'(0[bB][01]+(?:_[01]+)*)(\s*)([/?])?',
             bygroups(Number.Bin, Text, Operator)),
            (r'([\d]+(?:_\d+)*)(\s*)([/?])?',
             bygroups(Number.Integer, Text, Operator)),
            (r'\d+([eE][+-]?[0-9]+)|\d+\.\d+([eE][+-]?[0-9]+)?', Number.Float),
            (r'\d+', Number.Integer)
        ]
    }
예제 #12
0
class RubyLexer(ExtendedRegexLexer):
    """
    For `Ruby <http://www.ruby-lang.org>`_ source code.
    """

    name = 'Ruby'
    aliases = ['rb', 'ruby', 'duby']
    filenames = [
        '*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby'
    ]
    mimetypes = ['text/x-ruby', 'application/x-ruby']

    flags = re.DOTALL | re.MULTILINE

    def heredoc_callback(self, match, ctx):
        # okay, this is the hardest part of parsing Ruby...
        # match: 1 = <<-?, 2 = quote? 3 = name 4 = quote? 5 = rest of line

        start = match.start(1)
        yield start, Operator, match.group(1)  # <<-?
        yield match.start(2), String.Heredoc, match.group(2)  # quote ", ', `
        yield match.start(3), Name.Constant, match.group(3)  # heredoc name
        yield match.start(4), String.Heredoc, match.group(4)  # quote again

        heredocstack = ctx.__dict__.setdefault('heredocstack', [])
        outermost = not bool(heredocstack)
        heredocstack.append((match.group(1) == '<<-', match.group(3)))

        ctx.pos = match.start(5)
        ctx.end = match.end(5)
        # this may find other heredocs
        for i, t, v in self.get_tokens_unprocessed(context=ctx):
            yield i, t, v
        ctx.pos = match.end()

        if outermost:
            # this is the outer heredoc again, now we can process them all
            for tolerant, hdname in heredocstack:
                lines = []
                for match in line_re.finditer(ctx.text, ctx.pos):
                    if tolerant:
                        check = match.group().strip()
                    else:
                        check = match.group().rstrip()
                    if check == hdname:
                        for amatch in lines:
                            yield amatch.start(), String.Heredoc, amatch.group(
                            )
                        yield match.start(), Name.Constant, match.group()
                        ctx.pos = match.end()
                        break
                    else:
                        lines.append(match)
                else:
                    # end of heredoc not found -- error!
                    for amatch in lines:
                        yield amatch.start(), Error, amatch.group()
            ctx.end = len(ctx.text)
            del heredocstack[:]

    def gen_rubystrings_rules():
        def intp_regex_callback(self, match, ctx):
            yield match.start(1), String.Regex, match.group(1)  # begin
            nctx = LexerContext(match.group(3), 0, ['interpolated-regex'])
            for i, t, v in self.get_tokens_unprocessed(context=nctx):
                yield match.start(3) + i, t, v
            yield match.start(4), String.Regex, match.group(
                4)  # end[mixounse]*
            ctx.pos = match.end()

        def intp_string_callback(self, match, ctx):
            yield match.start(1), String.Other, match.group(1)
            nctx = LexerContext(match.group(3), 0, ['interpolated-string'])
            for i, t, v in self.get_tokens_unprocessed(context=nctx):
                yield match.start(3) + i, t, v
            yield match.start(4), String.Other, match.group(4)  # end
            ctx.pos = match.end()

        states = {}
        states['strings'] = [
            # easy ones
            (r'\:@{0,2}[a-zA-Z_]\w*[!?]?', String.Symbol),
            (words(RUBY_OPERATORS, prefix=r'\:@{0,2}'), String.Symbol),
            (r":'(\\\\|\\'|[^'])*'", String.Symbol),
            (r"'(\\\\|\\'|[^'])*'", String.Single),
            (r':"', String.Symbol, 'simple-sym'),
            (r'([a-zA-Z_]\w*)(:)(?!:)',
             bygroups(String.Symbol, Punctuation)),  # Since Ruby 1.9
            (r'"', String.Double, 'simple-string'),
            (r'(?<!\.)`', String.Backtick, 'simple-backtick'),
        ]

        # double-quoted string and symbol
        for name, ttype, end in ('string', String.Double, '"'), \
                                ('sym', String.Symbol, '"'), \
                                ('backtick', String.Backtick, '`'):
            states['simple-' + name] = [
                include('string-intp-escaped'),
                (r'[^\\%s#]+' % end, ttype),
                (r'[\\#]', ttype),
                (end, ttype, '#pop'),
            ]

        # braced quoted strings
        for lbrace, rbrace, bracecc, name in \
                ('\\{', '\\}', '{}', 'cb'), \
                ('\\[', '\\]', '\\[\\]', 'sb'), \
                ('\\(', '\\)', '()', 'pa'), \
                ('<', '>', '<>', 'ab'):
            states[name + '-intp-string'] = [
                (r'\\[\\' + bracecc + ']', String.Other),
                (lbrace, String.Other, '#push'),
                (rbrace, String.Other, '#pop'),
                include('string-intp-escaped'),
                (r'[\\#' + bracecc + ']', String.Other),
                (r'[^\\#' + bracecc + ']+', String.Other),
            ]
            states['strings'].append(
                (r'%[QWx]?' + lbrace, String.Other, name + '-intp-string'))
            states[name + '-string'] = [
                (r'\\[\\' + bracecc + ']', String.Other),
                (lbrace, String.Other, '#push'),
                (rbrace, String.Other, '#pop'),
                (r'[\\#' + bracecc + ']', String.Other),
                (r'[^\\#' + bracecc + ']+', String.Other),
            ]
            states['strings'].append(
                (r'%[qsw]' + lbrace, String.Other, name + '-string'))
            states[name + '-regex'] = [
                (r'\\[\\' + bracecc + ']', String.Regex),
                (lbrace, String.Regex, '#push'),
                (rbrace + '[mixounse]*', String.Regex, '#pop'),
                include('string-intp'),
                (r'[\\#' + bracecc + ']', String.Regex),
                (r'[^\\#' + bracecc + ']+', String.Regex),
            ]
            states['strings'].append(
                (r'%r' + lbrace, String.Regex, name + '-regex'))

        # these must come after %<brace>!
        states['strings'] += [
            # %r regex
            (r'(%r([\W_]))((?:\\\2|(?!\2).)*)(\2[mixounse]*)',
             intp_regex_callback),
            # regular fancy strings with qsw
            (r'%[qsw]([\W_])((?:\\\1|(?!\1).)*)\1', String.Other),
            (r'(%[QWx]([\W_]))((?:\\\2|(?!\2).)*)(\2)', intp_string_callback),
            # special forms of fancy strings after operators or
            # in method calls with braces
            (r'(?<=[-+/*%=<>&!^|~,(])(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
             bygroups(Text, String.Other, None)),
            # and because of fixed width lookbehinds the whole thing a
            # second time for line startings...
            (r'^(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
             bygroups(Text, String.Other, None)),
            # all regular fancy strings without qsw
            (r'(%([^a-zA-Z0-9\s]))((?:\\\2|(?!\2).)*)(\2)',
             intp_string_callback),
        ]

        return states

    tokens = {
        'root': [
            (r'\A#!.+?$', Comment.Hashbang),
            (r'#.*?$', Comment.Single),
            (r'=begin\s.*?\n=end.*?$', Comment.Multiline),
            # keywords
            (words(
                ('BEGIN', 'END', 'alias', 'begin', 'break', 'case', 'defined?',
                 'do', 'else', 'elsif', 'end', 'ensure', 'for', 'if', 'in',
                 'next', 'redo', 'rescue', 'raise', 'retry', 'return', 'super',
                 'then', 'undef', 'unless', 'until', 'when', 'while', 'yield'),
                suffix=r'\b'), Keyword),
            # start of function, class and module names
            (r'(module)(\s+)([a-zA-Z_]\w*'
             r'(?:::[a-zA-Z_]\w*)*)', bygroups(Keyword, Text, Name.Namespace)),
            (r'(def)(\s+)', bygroups(Keyword, Text), 'funcname'),
            (r'def(?=[*%&^`~+-/\[<>=])', Keyword, 'funcname'),
            (r'(class)(\s+)', bygroups(Keyword, Text), 'classname'),
            # special methods
            (words(('initialize', 'new', 'loop', 'include', 'extend', 'raise',
                    'attr_reader', 'attr_writer', 'attr_accessor', 'attr',
                    'catch', 'throw', 'private', 'module_function', 'public',
                    'protected', 'true', 'false', 'nil'),
                   suffix=r'\b'), Keyword.Pseudo),
            (r'(not|and|or)\b', Operator.Word),
            (words(('autoload', 'block_given', 'const_defined', 'eql', 'equal',
                    'frozen', 'include', 'instance_of', 'is_a', 'iterator',
                    'kind_of', 'method_defined', 'nil',
                    'private_method_defined', 'protected_method_defined',
                    'public_method_defined', 'respond_to', 'tainted'),
                   suffix=r'\?'), Name.Builtin),
            (r'(chomp|chop|exit|gsub|sub)!', Name.Builtin),
            (words(
                ('Array', 'Float', 'Integer', 'String', '__id__', '__send__',
                 'abort', 'ancestors', 'at_exit', 'autoload', 'binding',
                 'callcc', 'caller', 'catch', 'chomp', 'chop', 'class_eval',
                 'class_variables', 'clone', 'const_defined?', 'const_get',
                 'const_missing', 'const_set', 'constants', 'display', 'dup',
                 'eval', 'exec', 'exit', 'extend', 'fail', 'fork', 'format',
                 'freeze', 'getc', 'gets', 'global_variables', 'gsub', 'hash',
                 'id', 'included_modules', 'inspect', 'instance_eval',
                 'instance_method', 'instance_methods',
                 'instance_variable_get', 'instance_variable_set',
                 'instance_variables', 'lambda', 'load', 'local_variables',
                 'loop', 'method', 'method_missing', 'methods', 'module_eval',
                 'name', 'object_id', 'open', 'p', 'print', 'printf',
                 'private_class_method', 'private_instance_methods',
                 'private_methods', 'proc', 'protected_instance_methods',
                 'protected_methods', 'public_class_method',
                 'public_instance_methods', 'public_methods', 'putc', 'puts',
                 'raise', 'rand', 'readline', 'readlines', 'require', 'scan',
                 'select', 'self', 'send', 'set_trace_func',
                 'singleton_methods', 'sleep', 'split', 'sprintf', 'srand',
                 'sub', 'syscall', 'system', 'taint', 'test', 'throw', 'to_a',
                 'to_s', 'trace_var', 'trap', 'untaint', 'untrace_var',
                 'warn'),
                prefix=r'(?<!\.)',
                suffix=r'\b'), Name.Builtin),
            (r'__(FILE|LINE)__\b', Name.Builtin.Pseudo),
            # normal heredocs
            (r'(?<!\w)(<<-?)(["`\']?)([a-zA-Z_]\w*)(\2)(.*?\n)',
             heredoc_callback),
            # empty string heredocs
            (r'(<<-?)("|\')()(\2)(.*?\n)', heredoc_callback),
            (r'__END__', Comment.Preproc, 'end-part'),
            # multiline regex (after keywords or assignments)
            (r'(?:^|(?<=[=<>~!:])|'
             r'(?<=(?:\s|;)when\s)|'
             r'(?<=(?:\s|;)or\s)|'
             r'(?<=(?:\s|;)and\s)|'
             r'(?<=\.index\s)|'
             r'(?<=\.scan\s)|'
             r'(?<=\.sub\s)|'
             r'(?<=\.sub!\s)|'
             r'(?<=\.gsub\s)|'
             r'(?<=\.gsub!\s)|'
             r'(?<=\.match\s)|'
             r'(?<=(?:\s|;)if\s)|'
             r'(?<=(?:\s|;)elsif\s)|'
             r'(?<=^when\s)|'
             r'(?<=^index\s)|'
             r'(?<=^scan\s)|'
             r'(?<=^sub\s)|'
             r'(?<=^gsub\s)|'
             r'(?<=^sub!\s)|'
             r'(?<=^gsub!\s)|'
             r'(?<=^match\s)|'
             r'(?<=^if\s)|'
             r'(?<=^elsif\s)'
             r')(\s*)(/)', bygroups(Text, String.Regex), 'multiline-regex'),
            # multiline regex (in method calls or subscripts)
            (r'(?<=\(|,|\[)/', String.Regex, 'multiline-regex'),
            # multiline regex (this time the funny no whitespace rule)
            (r'(\s+)(/)(?![\s=])', bygroups(Text,
                                            String.Regex), 'multiline-regex'),
            # lex numbers and ignore following regular expressions which
            # are division operators in fact (grrrr. i hate that. any
            # better ideas?)
            # since pygments 0.7 we also eat a "?" operator after numbers
            # so that the char operator does not work. Chars are not allowed
            # there so that you can use the ternary operator.
            # stupid example:
            #   x>=0?n[x]:""
            (r'(0_?[0-7]+(?:_[0-7]+)*)(\s*)([/?])?',
             bygroups(Number.Oct, Text, Operator)),
            (r'(0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*)(\s*)([/?])?',
             bygroups(Number.Hex, Text, Operator)),
            (r'(0b[01]+(?:_[01]+)*)(\s*)([/?])?',
             bygroups(Number.Bin, Text, Operator)),
            (r'([\d]+(?:_\d+)*)(\s*)([/?])?',
             bygroups(Number.Integer, Text, Operator)),
            # Names
            (r'@@[a-zA-Z_]\w*', Name.Variable.Class),
            (r'@[a-zA-Z_]\w*', Name.Variable.Instance),
            (r'\$\w+', Name.Variable.Global),
            (r'\$[!@&`\'+~=/\\,;.<>_*$?:"^-]', Name.Variable.Global),
            (r'\$-[0adFiIlpvw]', Name.Variable.Global),
            (r'::', Operator),
            include('strings'),
            # chars
            (
                r'\?(\\[MC]-)*'  # modifiers
                r'(\\([\\abefnrstv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})|\S)'
                r'(?!\w)',
                String.Char),
            (r'[A-Z]\w+', Name.Constant),
            # this is needed because ruby attributes can look
            # like keywords (class) or like this: ` ?!?
            (words(RUBY_OPERATORS,
                   prefix=r'(\.|::)'), bygroups(Operator, Name.Operator)),
            (r'(\.|::)([a-zA-Z_]\w*[!?]?|[*%&^`~+\-/\[<>=])',
             bygroups(Operator, Name)),
            (r'[a-zA-Z_]\w*[!?]?', Name),
            (r'(\[|\]|\*\*|<<?|>>?|>=|<=|<=>|=~|={3}|'
             r'!~|&&?|\|\||\.{1,3})', Operator),
            (r'[-+/*%=<>&!^|~]=?', Operator),
            (r'[(){};,/?:\\]', Punctuation),
            (r'\s+', Text)
        ],
        'funcname': [(r'\(', Punctuation, 'defexpr'),
                     (r'(?:([a-zA-Z_]\w*)(\.))?'
                      r'([a-zA-Z_]\w*[!?]?|\*\*?|[-+]@?|'
                      r'[/%&|^`~]|\[\]=?|<<|>>|<=?>|>=?|===?)',
                      bygroups(Name.Class, Operator, Name.Function), '#pop'),
                     default('#pop')],
        'classname': [(r'\(', Punctuation, 'defexpr'),
                      (r'<<', Operator, '#pop'),
                      (r'[A-Z_]\w*', Name.Class, '#pop'),
                      default('#pop')],
        'defexpr': [(r'(\))(\.|::)?', bygroups(Punctuation, Operator), '#pop'),
                    (r'\(', Operator, '#push'),
                    include('root')],
        'in-intp': [
            (r'\{', String.Interpol, '#push'),
            (r'\}', String.Interpol, '#pop'),
            include('root'),
        ],
        'string-intp': [(r'#\{', String.Interpol, 'in-intp'),
                        (r'#@@?[a-zA-Z_]\w*', String.Interpol),
                        (r'#\$[a-zA-Z_]\w*', String.Interpol)],
        'string-intp-escaped': [
            include('string-intp'),
            (r'\\([\\abefnrstv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})',
             String.Escape)
        ],
        'interpolated-regex': [
            include('string-intp'),
            (r'[\\#]', String.Regex),
            (r'[^\\#]+', String.Regex),
        ],
        'interpolated-string': [
            include('string-intp'),
            (r'[\\#]', String.Other),
            (r'[^\\#]+', String.Other),
        ],
        'multiline-regex': [
            include('string-intp'),
            (r'\\\\', String.Regex),
            (r'\\/', String.Regex),
            (r'[\\#]', String.Regex),
            (r'[^\\/#]+', String.Regex),
            (r'/[mixounse]*', String.Regex, '#pop'),
        ],
        'end-part': [(r'.+', Comment.Preproc, '#pop')]
    }
    tokens.update(gen_rubystrings_rules())

    def analyse_text(text):
        return shebang_matches(text, r'ruby(1\.\d)?')
예제 #13
0
class TclLexer(RegexLexer):
    """
    For Tcl source code.

    .. versionadded:: 0.10
    """

    keyword_cmds_re = words((
        'after', 'apply', 'array', 'break', 'catch', 'continue', 'elseif', 'else', 'error',
        'eval', 'expr', 'for', 'foreach', 'global', 'if', 'namespace', 'proc', 'rename', 'return',
        'set', 'switch', 'then', 'trace', 'unset', 'update', 'uplevel', 'upvar', 'variable',
        'vwait', 'while'), prefix=r'\b', suffix=r'\b')

    builtin_cmds_re = words((
        'append', 'bgerror', 'binary', 'cd', 'chan', 'clock', 'close', 'concat', 'dde', 'dict',
        'encoding', 'eof', 'exec', 'exit', 'fblocked', 'fconfigure', 'fcopy', 'file',
        'fileevent', 'flush', 'format', 'gets', 'glob', 'history', 'http', 'incr', 'info', 'interp',
        'join', 'lappend', 'lassign', 'lindex', 'linsert', 'list', 'llength', 'load', 'loadTk',
        'lrange', 'lrepeat', 'lreplace', 'lreverse', 'lsearch', 'lset', 'lsort', 'mathfunc',
        'mathop', 'memory', 'msgcat', 'open', 'package', 'pid', 'pkg::create', 'pkg_mkIndex',
        'platform', 'platform::shell', 'puts', 'pwd', 're_syntax', 'read', 'refchan',
        'regexp', 'registry', 'regsub', 'scan', 'seek', 'socket', 'source', 'split', 'string',
        'subst', 'tell', 'time', 'tm', 'unknown', 'unload'), prefix=r'\b', suffix=r'\b')

    name = 'Tcl'
    aliases = ['tcl']
    filenames = ['*.tcl', '*.rvt']
    mimetypes = ['text/x-tcl', 'text/x-script.tcl', 'application/x-tcl']

    def _gen_command_rules(keyword_cmds_re, builtin_cmds_re, context=""):
        return [
            (keyword_cmds_re, Keyword, 'params' + context),
            (builtin_cmds_re, Name.Builtin, 'params' + context),
            (r'([\w.-]+)', Name.Variable, 'params' + context),
            (r'#', Comment, 'comment'),
        ]

    tokens = {
        'root': [
            include('command'),
            include('basic'),
            include('data'),
            (r'\}', Keyword),  # HACK: somehow we miscounted our braces
        ],
        'command': _gen_command_rules(keyword_cmds_re, builtin_cmds_re),
        'command-in-brace': _gen_command_rules(keyword_cmds_re,
                                               builtin_cmds_re,
                                               "-in-brace"),
        'command-in-bracket': _gen_command_rules(keyword_cmds_re,
                                                 builtin_cmds_re,
                                                 "-in-bracket"),
        'command-in-paren': _gen_command_rules(keyword_cmds_re,
                                               builtin_cmds_re,
                                               "-in-paren"),
        'basic': [
            (r'\(', Keyword, 'paren'),
            (r'\[', Keyword, 'bracket'),
            (r'\{', Keyword, 'brace'),
            (r'"', String.Double, 'string'),
            (r'(eq|ne|in|ni)\b', Operator.Word),
            (r'!=|==|<<|>>|<=|>=|&&|\|\||\*\*|[-+~!*/%<>&^|?:]', Operator),
        ],
        'data': [
            (r'\s+', Text),
            (r'0x[a-fA-F0-9]+', Number.Hex),
            (r'0[0-7]+', Number.Oct),
            (r'\d+\.\d+', Number.Float),
            (r'\d+', Number.Integer),
            (r'\$([\w.:-]+)', Name.Variable),
            (r'([\w.:-]+)', Text),
        ],
        'params': [
            (r';', Keyword, '#pop'),
            (r'\n', Text, '#pop'),
            (r'(else|elseif|then)\b', Keyword),
            include('basic'),
            include('data'),
        ],
        'params-in-brace': [
            (r'\}', Keyword, ('#pop', '#pop')),
            include('params')
        ],
        'params-in-paren': [
            (r'\)', Keyword, ('#pop', '#pop')),
            include('params')
        ],
        'params-in-bracket': [
            (r'\]', Keyword, ('#pop', '#pop')),
            include('params')
        ],
        'string': [
            (r'\[', String.Double, 'string-square'),
            (r'(?s)(\\\\|\\[0-7]+|\\.|[^"\\])', String.Double),
            (r'"', String.Double, '#pop')
        ],
        'string-square': [
            (r'\[', String.Double, 'string-square'),
            (r'(?s)(\\\\|\\[0-7]+|\\.|\\\n|[^\]\\])', String.Double),
            (r'\]', String.Double, '#pop')
        ],
        'brace': [
            (r'\}', Keyword, '#pop'),
            include('command-in-brace'),
            include('basic'),
            include('data'),
        ],
        'paren': [
            (r'\)', Keyword, '#pop'),
            include('command-in-paren'),
            include('basic'),
            include('data'),
        ],
        'bracket': [
            (r'\]', Keyword, '#pop'),
            include('command-in-bracket'),
            include('basic'),
            include('data'),
        ],
        'comment': [
            (r'.*[^\\]\n', Comment, '#pop'),
            (r'.*\\\n', Comment),
        ],
    }

    def analyse_text(text):
        return shebang_matches(text, r'(tcl)')
예제 #14
0
파일: crystal.py 프로젝트: Jenyay/outwiker
    def gen_crystalstrings_rules():
        def intp_regex_callback(self, match, ctx):
            yield match.start(1), String.Regex, match.group(1)  # begin
            nctx = LexerContext(match.group(3), 0, ['interpolated-regex'])
            for i, t, v in self.get_tokens_unprocessed(context=nctx):
                yield match.start(3)+i, t, v
            yield match.start(4), String.Regex, match.group(4)  # end[imsx]*
            ctx.pos = match.end()

        def intp_string_callback(self, match, ctx):
            yield match.start(1), String.Other, match.group(1)
            nctx = LexerContext(match.group(3), 0, ['interpolated-string'])
            for i, t, v in self.get_tokens_unprocessed(context=nctx):
                yield match.start(3)+i, t, v
            yield match.start(4), String.Other, match.group(4)  # end
            ctx.pos = match.end()

        states = {}
        states['strings'] = [
            (r'\:@{0,2}[a-zA-Z_]\w*[!?]?', String.Symbol),
            (words(CRYSTAL_OPERATORS, prefix=r'\:@{0,2}'), String.Symbol),
            (r":'(\\\\|\\'|[^'])*'", String.Symbol),
            # This allows arbitrary text after '\ for simplicity
            (r"'(\\\\|\\'|[^']|\\[^'\\]+)'", String.Char),
            (r':"', String.Symbol, 'simple-sym'),
            # Crystal doesn't have "symbol:"s but this simplifies function args
            (r'([a-zA-Z_]\w*)(:)(?!:)', bygroups(String.Symbol, Punctuation)),
            (r'"', String.Double, 'simple-string'),
            (r'(?<!\.)`', String.Backtick, 'simple-backtick'),
        ]

        # double-quoted string and symbol
        for name, ttype, end in ('string', String.Double, '"'), \
                                ('sym', String.Symbol, '"'), \
                                ('backtick', String.Backtick, '`'):
            states['simple-'+name] = [
                include('string-escaped' if name == 'sym' else 'string-intp-escaped'),
                (r'[^\\%s#]+' % end, ttype),
                (r'[\\#]', ttype),
                (end, ttype, '#pop'),
            ]

        # braced quoted strings
        for lbrace, rbrace, bracecc, name in \
                ('\\{', '\\}', '{}', 'cb'), \
                ('\\[', '\\]', '\\[\\]', 'sb'), \
                ('\\(', '\\)', '()', 'pa'), \
                ('<', '>', '<>', 'ab'):
            states[name+'-intp-string'] = [
                (r'\\[' + lbrace + ']', String.Other),
                (lbrace, String.Other, '#push'),
                (rbrace, String.Other, '#pop'),
                include('string-intp-escaped'),
                (r'[\\#' + bracecc + ']', String.Other),
                (r'[^\\#' + bracecc + ']+', String.Other),
            ]
            states['strings'].append((r'%' + lbrace, String.Other,
                                      name+'-intp-string'))
            states[name+'-string'] = [
                (r'\\[\\' + bracecc + ']', String.Other),
                (lbrace, String.Other, '#push'),
                (rbrace, String.Other, '#pop'),
                (r'[\\#' + bracecc + ']', String.Other),
                (r'[^\\#' + bracecc + ']+', String.Other),
            ]
            # http://crystal-lang.org/docs/syntax_and_semantics/literals/array.html
            states['strings'].append((r'%[wi]' + lbrace, String.Other,
                                      name+'-string'))
            states[name+'-regex'] = [
                (r'\\[\\' + bracecc + ']', String.Regex),
                (lbrace, String.Regex, '#push'),
                (rbrace + '[imsx]*', String.Regex, '#pop'),
                include('string-intp'),
                (r'[\\#' + bracecc + ']', String.Regex),
                (r'[^\\#' + bracecc + ']+', String.Regex),
            ]
            states['strings'].append((r'%r' + lbrace, String.Regex,
                                      name+'-regex'))

        # these must come after %<brace>!
        states['strings'] += [
            # %r regex
            (r'(%r([\W_]))((?:\\\2|(?!\2).)*)(\2[imsx]*)',
             intp_regex_callback),
            # regular fancy strings with qsw
            (r'(%[wi]([\W_]))((?:\\\2|(?!\2).)*)(\2)',
             intp_string_callback),
            # special forms of fancy strings after operators or
            # in method calls with braces
            (r'(?<=[-+/*%=<>&!^|~,(])(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
             bygroups(Text, String.Other, None)),
            # and because of fixed width lookbehinds the whole thing a
            # second time for line startings...
            (r'^(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
             bygroups(Text, String.Other, None)),
            # all regular fancy strings without qsw
            (r'(%([\[{(<]))((?:\\\2|(?!\2).)*)(\2)',
             intp_string_callback),
        ]

        return states
예제 #15
0
파일: pascal.py 프로젝트: th0/test2
class AdaLexer(RegexLexer):
    """
    For Ada source code.

    .. versionadded:: 1.3
    """

    name = 'Ada'
    aliases = ['ada', 'ada95', 'ada2005']
    filenames = ['*.adb', '*.ads', '*.ada']
    mimetypes = ['text/x-ada']

    flags = re.MULTILINE | re.IGNORECASE

    tokens = {
        'root': [
            (r'[^\S\n]+', Text),
            (r'--.*?\n', Comment.Single),
            (r'[^\S\n]+', Text),
            (r'function|procedure|entry', Keyword.Declaration, 'subprogram'),
            (r'(subtype|type)(\s+)(\w+)',
             bygroups(Keyword.Declaration, Text, Keyword.Type), 'type_def'),
            (r'task|protected', Keyword.Declaration),
            (r'(subtype)(\s+)', bygroups(Keyword.Declaration, Text)),
            (r'(end)(\s+)', bygroups(Keyword.Reserved, Text), 'end'),
            (r'(pragma)(\s+)(\w+)',
             bygroups(Keyword.Reserved, Text, Comment.Preproc)),
            (r'(true|false|null)\b', Keyword.Constant),
            (words(
                ('Address', 'Byte', 'Boolean', 'Character', 'Controlled',
                 'Count', 'Cursor', 'Duration', 'File_Mode', 'File_Type',
                 'Float', 'Generator', 'Integer', 'Long_Float', 'Long_Integer',
                 'Long_Long_Float', 'Long_Long_Integer', 'Natural', 'Positive',
                 'Reference_Type', 'Short_Float', 'Short_Integer',
                 'Short_Short_Float', 'Short_Short_Integer', 'String',
                 'Wide_Character', 'Wide_String'),
                suffix=r'\b'), Keyword.Type),
            (r'(and(\s+then)?|in|mod|not|or(\s+else)|rem)\b', Operator.Word),
            (r'generic|private', Keyword.Declaration),
            (r'package', Keyword.Declaration, 'package'),
            (r'array\b', Keyword.Reserved, 'array_def'),
            (r'(with|use)(\s+)', bygroups(Keyword.Namespace, Text), 'import'),
            (r'(\w+)(\s*)(:)(\s*)(constant)',
             bygroups(Name.Constant, Text, Punctuation, Text,
                      Keyword.Reserved)),
            (r'<<\w+>>', Name.Label),
            (r'(\w+)(\s*)(:)(\s*)(declare|begin|loop|for|while)',
             bygroups(Name.Label, Text, Punctuation, Text, Keyword.Reserved)),
            (words(
                ('abort', 'abs', 'abstract', 'accept', 'access', 'aliased',
                 'all', 'array', 'at', 'begin', 'body', 'case', 'constant',
                 'declare', 'delay', 'delta', 'digits', 'do', 'else', 'elsif',
                 'end', 'entry', 'exception', 'exit', 'interface', 'for',
                 'goto', 'if', 'is', 'limited', 'loop', 'new', 'null', 'of',
                 'or', 'others', 'out', 'overriding', 'pragma', 'protected',
                 'raise', 'range', 'record', 'renames', 'requeue', 'return',
                 'reverse', 'select', 'separate', 'subtype', 'synchronized',
                 'task', 'tagged', 'terminate', 'then', 'type', 'until',
                 'when', 'while', 'xor'),
                prefix=r'\b',
                suffix=r'\b'), Keyword.Reserved),
            (r'"[^"]*"', String),
            include('attribute'),
            include('numbers'),
            (r"'[^']'", String.Character),
            (r'(\w+)(\s*|[(,])', bygroups(Name, using(this))),
            (r"(<>|=>|:=|[()|:;,.'])", Punctuation),
            (r'[*<>+=/&-]', Operator),
            (r'\n+', Text),
        ],
        'numbers': [
            (r'[0-9_]+#[0-9a-f]+#', Number.Hex),
            (r'[0-9_]+\.[0-9_]*', Number.Float),
            (r'[0-9_]+', Number.Integer),
        ],
        'attribute': [
            (r"(')(\w+)", bygroups(Punctuation, Name.Attribute)),
        ],
        'subprogram': [
            (r'\(', Punctuation, ('#pop', 'formal_part')),
            (r';', Punctuation, '#pop'),
            (r'is\b', Keyword.Reserved, '#pop'),
            (r'"[^"]+"|\w+', Name.Function),
            include('root'),
        ],
        'end': [
            ('(if|case|record|loop|select)', Keyword.Reserved),
            ('"[^"]+"|[\w.]+', Name.Function),
            ('\s+', Text),
            (';', Punctuation, '#pop'),
        ],
        'type_def': [
            (r';', Punctuation, '#pop'),
            (r'\(', Punctuation, 'formal_part'),
            (r'with|and|use', Keyword.Reserved),
            (r'array\b', Keyword.Reserved, ('#pop', 'array_def')),
            (r'record\b', Keyword.Reserved, ('record_def')),
            (r'(null record)(;)', bygroups(Keyword.Reserved,
                                           Punctuation), '#pop'),
            include('root'),
        ],
        'array_def': [
            (r';', Punctuation, '#pop'),
            (r'(\w+)(\s+)(range)',
             bygroups(Keyword.Type, Text, Keyword.Reserved)),
            include('root'),
        ],
        'record_def': [
            (r'end record', Keyword.Reserved, '#pop'),
            include('root'),
        ],
        'import': [
            (r'[\w.]+', Name.Namespace, '#pop'),
            default('#pop'),
        ],
        'formal_part': [
            (r'\)', Punctuation, '#pop'),
            (r'\w+', Name.Variable),
            (r',|:[^=]', Punctuation),
            (r'(in|not|null|out|access)\b', Keyword.Reserved),
            include('root'),
        ],
        'package': [
            ('body', Keyword.Declaration),
            ('is\s+new|renames', Keyword.Reserved),
            ('is', Keyword.Reserved, '#pop'),
            (';', Punctuation, '#pop'),
            ('\(', Punctuation, 'package_instantiation'),
            ('([\w.]+)', Name.Class),
            include('root'),
        ],
        'package_instantiation': [
            (r'("[^"]+"|\w+)(\s+)(=>)',
             bygroups(Name.Variable, Text, Punctuation)),
            (r'[\w.\'"]', Text),
            (r'\)', Punctuation, '#pop'),
            include('root'),
        ],
    }
예제 #16
0
파일: repl_utils.py 프로젝트: sminez/ripl
 def _multi_escape(entries):
     return words(entries, suffix=' ')
예제 #17
0
class BoaLexer(RegexLexer):
    """
    Lexer for the `Boa <http://boa.cs.iastate.edu/docs/>`_ language.

    .. versionadded:: 2.4
    """
    name = 'Boa'
    aliases = ['boa']
    filenames = ['*.boa']

    reserved = words(
        ('input', 'output', 'of', 'weight', 'before', 'after', 'stop',
         'ifall', 'foreach', 'exists', 'function', 'break', 'switch', 'case',
         'visitor', 'default', 'return', 'visit', 'while', 'if', 'else'),
        suffix=r'\b', prefix=r'\b')
    keywords = words(
        ('bottom', 'collection', 'maximum', 'mean', 'minimum', 'set', 'sum',
         'top', 'string', 'int', 'bool', 'float', 'time', 'false', 'true',
         'array', 'map', 'stack', 'enum', 'type'), suffix=r'\b', prefix=r'\b')
    classes = words(
        ('Project', 'ForgeKind', 'CodeRepository', 'Revision', 'RepositoryKind',
         'ChangedFile', 'FileKind', 'ASTRoot', 'Namespace', 'Declaration', 'Type',
         'Method', 'Variable', 'Statement', 'Expression', 'Modifier',
         'StatementKind', 'ExpressionKind', 'ModifierKind', 'Visibility',
         'TypeKind', 'Person', 'ChangeKind'),
        suffix=r'\b', prefix=r'\b')
    operators = ('->', ':=', ':', '=', '<<', '!', '++', '||',
                 '&&', '+', '-', '*', ">", "<")
    string_sep = ('`', '\"')
    built_in_functions = words(
        (
            # Array functions
            'new', 'sort',
            # Date & Time functions
            'yearof', 'dayofyear', 'hourof', 'minuteof', 'secondof', 'now',
            'addday', 'addmonth', 'addweek', 'addyear', 'dayofmonth', 'dayofweek',
            'dayofyear', 'formattime', 'trunctoday', 'trunctohour', 'trunctominute',
            'trunctomonth', 'trunctosecond', 'trunctoyear',
            # Map functions
            'clear', 'haskey', 'keys', 'lookup', 'remove', 'values',
            # Math functions
            'abs', 'acos', 'acosh', 'asin', 'asinh', 'atan', 'atan2', 'atanh',
            'ceil', 'cos', 'cosh', 'exp', 'floor', 'highbit', 'isfinite', 'isinf',
            'isnan', 'isnormal', 'log', 'log10', 'max', 'min', 'nrand', 'pow',
            'rand', 'round', 'sin', 'sinh', 'sqrt', 'tan', 'tanh', 'trunc',
            # Other functions
            'def', 'hash', 'len',
            # Set functions
            'add', 'contains', 'remove',
            # String functions
            'format', 'lowercase', 'match', 'matchposns', 'matchstrs', 'regex',
            'split', 'splitall', 'splitn', 'strfind', 'strreplace', 'strrfind',
            'substring', 'trim', 'uppercase',
            # Type Conversion functions
            'bool', 'float', 'int', 'string', 'time',
            # Domain-Specific functions
            'getast', 'getsnapshot', 'hasfiletype', 'isfixingrevision', 'iskind',
            'isliteral',
        ),
        prefix=r'\b',
        suffix=r'\(')

    tokens = {
        'root': [
            (r'#.*?$', Comment.Single),
            (r'/\*.*?\*/', Comment.Multiline),
            (reserved, Keyword.Reserved),
            (built_in_functions, Name.Function),
            (keywords, Keyword.Type),
            (classes, Name.Classes),
            (words(operators), Operator),
            (r'[][(),;{}\\.]', Punctuation),
            (r'"(\\\\|\\"|[^"])*"', String),
            (r'`(\\\\|\\`|[^`])*`', String),
            (words(string_sep), String.Delimeter),
            (r'[a-zA-Z_]+', Name.Variable),
            (r'[0-9]+', Number.Integer),
            (r'\s+?', Text),  # Whitespace
        ]
    }
예제 #18
0
파일: css.py 프로젝트: Jenyay/outwiker
 (r'[a-z_-][\w-]*(?=\()', Name.Function),
 (words(_css_properties + (
     'above', 'absolute', 'always', 'armenian', 'aural', 'auto', 'avoid', 'baseline',
     'behind', 'below', 'bidi-override', 'blink', 'block', 'bold', 'bolder', 'both',
     'capitalize', 'center-left', 'center-right', 'center', 'circle',
     'cjk-ideographic', 'close-quote', 'collapse', 'condensed', 'continuous',
     'crop', 'crosshair', 'cross', 'cursive', 'dashed', 'decimal-leading-zero',
     'decimal', 'default', 'digits', 'disc', 'dotted', 'double', 'e-resize', 'embed',
     'extra-condensed', 'extra-expanded', 'expanded', 'fantasy', 'far-left',
     'far-right', 'faster', 'fast', 'fixed', 'georgian', 'groove', 'hebrew', 'help',
     'hidden', 'hide', 'higher', 'high', 'hiragana-iroha', 'hiragana', 'icon',
     'inherit', 'inline-table', 'inline', 'inset', 'inside', 'invert', 'italic',
     'justify', 'katakana-iroha', 'katakana', 'landscape', 'larger', 'large',
     'left-side', 'leftwards', 'level', 'lighter', 'line-through', 'list-item',
     'loud', 'lower-alpha', 'lower-greek', 'lower-roman', 'lowercase', 'ltr',
     'lower', 'low', 'medium', 'message-box', 'middle', 'mix', 'monospace',
     'n-resize', 'narrower', 'ne-resize', 'no-close-quote', 'no-open-quote',
     'no-repeat', 'none', 'normal', 'nowrap', 'nw-resize', 'oblique', 'once',
     'open-quote', 'outset', 'outside', 'overline', 'pointer', 'portrait', 'px',
     'relative', 'repeat-x', 'repeat-y', 'repeat', 'rgb', 'ridge', 'right-side',
     'rightwards', 's-resize', 'sans-serif', 'scroll', 'se-resize',
     'semi-condensed', 'semi-expanded', 'separate', 'serif', 'show', 'silent',
     'slow', 'slower', 'small-caps', 'small-caption', 'smaller', 'soft', 'solid',
     'spell-out', 'square', 'static', 'status-bar', 'super', 'sw-resize',
     'table-caption', 'table-cell', 'table-column', 'table-column-group',
     'table-footer-group', 'table-header-group', 'table-row',
     'table-row-group', 'text', 'text-bottom', 'text-top', 'thick', 'thin',
     'transparent', 'ultra-condensed', 'ultra-expanded', 'underline',
     'upper-alpha', 'upper-latin', 'upper-roman', 'uppercase', 'url',
     'visible', 'w-resize', 'wait', 'wider', 'x-fast', 'x-high', 'x-large', 'x-loud',
     'x-low', 'x-small', 'x-soft', 'xx-large', 'xx-small', 'yes'), suffix=r'\b'),
  Name.Constant),
예제 #19
0
파일: elm.py 프로젝트: amitkummer/pygments
class ElmLexer(RegexLexer):
    """
    For Elm source code.

    .. versionadded:: 2.1
    """

    name = 'Elm'
    url = 'http://elm-lang.org/'
    aliases = ['elm']
    filenames = ['*.elm']
    mimetypes = ['text/x-elm']

    validName = r'[a-z_][a-zA-Z0-9_\']*'

    specialName = r'^main '

    builtinOps = (
        '~',
        '||',
        '|>',
        '|',
        '`',
        '^',
        '\\',
        '\'',
        '>>',
        '>=',
        '>',
        '==',
        '=',
        '<~',
        '<|',
        '<=',
        '<<',
        '<-',
        '<',
        '::',
        ':',
        '/=',
        '//',
        '/',
        '..',
        '.',
        '->',
        '-',
        '++',
        '+',
        '*',
        '&&',
        '%',
    )

    reservedWords = words((
        'alias',
        'as',
        'case',
        'else',
        'if',
        'import',
        'in',
        'let',
        'module',
        'of',
        'port',
        'then',
        'type',
        'where',
    ),
                          suffix=r'\b')

    tokens = {
        'root': [

            # Comments
            (r'\{-', Comment.Multiline, 'comment'),
            (r'--.*', Comment.Single),

            # Whitespace
            (r'\s+', Whitespace),

            # Strings
            (r'"', String, 'doublequote'),

            # Modules
            (r'^(\s*)(module)(\s*)',
             bygroups(Whitespace, Keyword.Namespace, Whitespace), 'imports'),

            # Imports
            (r'^(\s*)(import)(\s*)',
             bygroups(Whitespace, Keyword.Namespace, Whitespace), 'imports'),

            # Shaders
            (r'\[glsl\|.*', Name.Entity, 'shader'),

            # Keywords
            (reservedWords, Keyword.Reserved),

            # Types
            (r'[A-Z][a-zA-Z0-9_]*', Keyword.Type),

            # Main
            (specialName, Keyword.Reserved),

            # Prefix Operators
            (words((builtinOps), prefix=r'\(', suffix=r'\)'), Name.Function),

            # Infix Operators
            (words(builtinOps), Name.Function),

            # Numbers
            include('numbers'),

            # Variable Names
            (validName, Name.Variable),

            # Parens
            (r'[,()\[\]{}]', Punctuation),
        ],
        'comment': [
            (r'-(?!\})', Comment.Multiline),
            (r'\{-', Comment.Multiline, 'comment'),
            (r'[^-}]', Comment.Multiline),
            (r'-\}', Comment.Multiline, '#pop'),
        ],
        'doublequote': [
            (r'\\u[0-9a-fA-F]{4}', String.Escape),
            (r'\\[nrfvb\\"]', String.Escape),
            (r'[^"]', String),
            (r'"', String, '#pop'),
        ],
        'imports': [
            (r'\w+(\.\w+)*', Name.Class, '#pop'),
        ],
        'numbers': [
            (r'_?\d+\.(?=\d+)', Number.Float),
            (r'_?\d+', Number.Integer),
        ],
        'shader': [
            (r'\|(?!\])', Name.Entity),
            (r'\|\]', Name.Entity, '#pop'),
            (r'(.*)(\n)', bygroups(Name.Entity, Whitespace)),
        ],
    }
예제 #20
0
class FactorLexer(RegexLexer):
    """
    Lexer for the `Factor <http://factorcode.org>`_ language.

    .. versionadded:: 1.4
    """
    name = 'Factor'
    aliases = ['factor']
    filenames = ['*.factor']
    mimetypes = ['text/x-factor']

    flags = re.MULTILINE | re.UNICODE

    builtin_kernel = words(
        ('-rot', '2bi', '2bi@', '2bi*', '2curry', '2dip', '2drop', '2dup',
         '2keep', '2nip', '2over', '2tri', '2tri@', '2tri*', '3bi', '3curry',
         '3dip', '3drop', '3dup', '3keep', '3tri', '4dip', '4drop', '4dup',
         '4keep', '<wrapper>', '=', '>boolean', 'clone', '?', '?execute',
         '?if', 'and', 'assert', 'assert=', 'assert?', 'bi', 'bi-curry',
         'bi-curry@', 'bi-curry*', 'bi@', 'bi*', 'boa', 'boolean', 'boolean?',
         'both?', 'build', 'call', 'callstack', 'callstack>array',
         'callstack?', 'clear', '(clone)', 'compose', 'compose?', 'curry',
         'curry?', 'datastack', 'die', 'dip', 'do', 'drop', 'dup', 'dupd',
         'either?', 'eq?', 'equal?', 'execute', 'hashcode', 'hashcode*',
         'identity-hashcode', 'identity-tuple', 'identity-tuple?', 'if', 'if*',
         'keep', 'loop', 'most', 'new', 'nip', 'not', 'null', 'object', 'or',
         'over', 'pick', 'prepose', 'retainstack', 'rot', 'same?', 'swap',
         'swapd', 'throw', 'tri', 'tri-curry', 'tri-curry@', 'tri-curry*',
         'tri@', 'tri*', 'tuple', 'tuple?', 'unless', 'unless*', 'until',
         'when', 'when*', 'while', 'with', 'wrapper', 'wrapper?', 'xor'),
        suffix=r'\s')

    builtin_assocs = words(
        ('2cache', '<enum>', '>alist', '?at', '?of', 'assoc', 'assoc-all?',
         'assoc-any?', 'assoc-clone-like', 'assoc-combine', 'assoc-diff',
         'assoc-diff!', 'assoc-differ', 'assoc-each', 'assoc-empty?',
         'assoc-filter', 'assoc-filter!', 'assoc-filter-as', 'assoc-find',
         'assoc-hashcode', 'assoc-intersect', 'assoc-like', 'assoc-map',
         'assoc-map-as', 'assoc-partition', 'assoc-refine', 'assoc-size',
         'assoc-stack', 'assoc-subset?', 'assoc-union', 'assoc-union!',
         'assoc=', 'assoc>map', 'assoc?', 'at', 'at+', 'at*', 'cache',
         'change-at', 'clear-assoc', 'delete-at', 'delete-at*', 'enum',
         'enum?', 'extract-keys', 'inc-at', 'key?', 'keys', 'map>assoc',
         'maybe-set-at', 'new-assoc', 'of', 'push-at', 'rename-at', 'set-at',
         'sift-keys', 'sift-values', 'substitute', 'unzip', 'value-at',
         'value-at*', 'value?', 'values', 'zip'),
        suffix=r'\s')

    builtin_combinators = words(
        ('2cleave', '2cleave>quot', '3cleave', '3cleave>quot', '4cleave',
         '4cleave>quot', 'alist>quot', 'call-effect', 'case', 'case-find',
         'case>quot', 'cleave', 'cleave>quot', 'cond', 'cond>quot',
         'deep-spread>quot', 'execute-effect', 'linear-case-quot', 'no-case',
         'no-case?', 'no-cond', 'no-cond?', 'recursive-hashcode',
         'shallow-spread>quot', 'spread', 'to-fixed-point', 'wrong-values',
         'wrong-values?'),
        suffix=r'\s')

    builtin_math = words(
        ('-', '/', '/f', '/i', '/mod', '2/', '2^', '<', '<=', '<fp-nan>', '>',
         '>=', '>bignum', '>fixnum', '>float', '>integer', '(all-integers?)',
         '(each-integer)', '(find-integer)', '*', '+', '?1+', 'abs', 'align',
         'all-integers?', 'bignum', 'bignum?', 'bit?', 'bitand', 'bitnot',
         'bitor', 'bits>double', 'bits>float', 'bitxor', 'complex', 'complex?',
         'denominator', 'double>bits', 'each-integer', 'even?', 'find-integer',
         'find-last-integer', 'fixnum', 'fixnum?', 'float', 'float>bits',
         'float?', 'fp-bitwise=', 'fp-infinity?', 'fp-nan-payload', 'fp-nan?',
         'fp-qnan?', 'fp-sign', 'fp-snan?', 'fp-special?', 'if-zero',
         'imaginary-part', 'integer', 'integer>fixnum',
         'integer>fixnum-strict', 'integer?', 'log2', 'log2-expects-positive',
         'log2-expects-positive?', 'mod', 'neg', 'neg?', 'next-float',
         'next-power-of-2', 'number', 'number=', 'number?', 'numerator',
         'odd?', 'out-of-fixnum-range', 'out-of-fixnum-range?', 'power-of-2?',
         'prev-float', 'ratio', 'ratio?', 'rational', 'rational?', 'real',
         'real-part', 'real?', 'recip', 'rem', 'sgn', 'shift', 'sq', 'times',
         'u<', 'u<=', 'u>', 'u>=', 'unless-zero', 'unordered?', 'when-zero',
         'zero?'),
        suffix=r'\s')

    builtin_sequences = words(
        ('1sequence', '2all?', '2each', '2map', '2map-as', '2map-reduce',
         '2reduce', '2selector', '2sequence', '3append', '3append-as', '3each',
         '3map', '3map-as', '3sequence', '4sequence', '<repetition>',
         '<reversed>', '<slice>', '?first', '?last', '?nth', '?second',
         '?set-nth', 'accumulate', 'accumulate!', 'accumulate-as', 'all?',
         'any?', 'append', 'append!', 'append-as', 'assert-sequence',
         'assert-sequence=', 'assert-sequence?', 'binary-reduce',
         'bounds-check', 'bounds-check?', 'bounds-error', 'bounds-error?',
         'but-last', 'but-last-slice', 'cartesian-each', 'cartesian-map',
         'cartesian-product', 'change-nth', 'check-slice', 'check-slice-error',
         'clone-like', 'collapse-slice', 'collector', 'collector-for',
         'concat', 'concat-as', 'copy', 'count', 'cut', 'cut-slice', 'cut*',
         'delete-all', 'delete-slice', 'drop-prefix', 'each', 'each-from',
         'each-index', 'empty?', 'exchange', 'filter', 'filter!', 'filter-as',
         'find', 'find-from', 'find-index', 'find-index-from', 'find-last',
         'find-last-from', 'first', 'first2', 'first3', 'first4', 'flip',
         'follow', 'fourth', 'glue', 'halves', 'harvest', 'head', 'head-slice',
         'head-slice*', 'head*', 'head?', 'if-empty', 'immutable',
         'immutable-sequence', 'immutable-sequence?', 'immutable?', 'index',
         'index-from', 'indices', 'infimum', 'infimum-by', 'insert-nth',
         'interleave', 'iota', 'iota-tuple', 'iota-tuple?', 'join', 'join-as',
         'last', 'last-index', 'last-index-from', 'length', 'lengthen', 'like',
         'longer', 'longer?', 'longest', 'map', 'map!', 'map-as', 'map-find',
         'map-find-last', 'map-index', 'map-integers', 'map-reduce', 'map-sum',
         'max-length', 'member-eq?', 'member?', 'midpoint@', 'min-length',
         'mismatch', 'move', 'new-like', 'new-resizable', 'new-sequence',
         'non-negative-integer-expected', 'non-negative-integer-expected?',
         'nth', 'nths', 'pad-head', 'pad-tail', 'padding', 'partition', 'pop',
         'pop*', 'prefix', 'prepend', 'prepend-as', 'produce', 'produce-as',
         'product', 'push', 'push-all', 'push-either', 'push-if', 'reduce',
         'reduce-index', 'remove', 'remove!', 'remove-eq', 'remove-eq!',
         'remove-nth', 'remove-nth!', 'repetition', 'repetition?',
         'replace-slice', 'replicate', 'replicate-as', 'rest', 'rest-slice',
         'reverse', 'reverse!', 'reversed', 'reversed?', 'second', 'selector',
         'selector-for', 'sequence', 'sequence-hashcode', 'sequence=',
         'sequence?', 'set-first', 'set-fourth', 'set-last', 'set-length',
         'set-nth', 'set-second', 'set-third', 'short', 'shorten', 'shorter',
         'shorter?', 'shortest', 'sift', 'slice', 'slice-error',
         'slice-error?', 'slice?', 'snip', 'snip-slice', 'start', 'start*',
         'subseq', 'subseq?', 'suffix', 'suffix!', 'sum', 'sum-lengths',
         'supremum', 'supremum-by', 'surround', 'tail', 'tail-slice',
         'tail-slice*', 'tail*', 'tail?', 'third', 'trim', 'trim-head',
         'trim-head-slice', 'trim-slice', 'trim-tail', 'trim-tail-slice',
         'unclip', 'unclip-last', 'unclip-last-slice', 'unclip-slice',
         'unless-empty', 'virtual-exemplar', 'virtual-sequence',
         'virtual-sequence?', 'virtual@', 'when-empty'),
        suffix=r'\s')

    builtin_namespaces = words(
        ('+@', 'change', 'change-global', 'counter', 'dec', 'get',
         'get-global', 'global', 'inc', 'init-namespaces', 'initialize',
         'is-global', 'make-assoc', 'namespace', 'namestack', 'off', 'on',
         'set', 'set-global', 'set-namestack', 'toggle', 'with-global',
         'with-scope', 'with-variable', 'with-variables'),
        suffix=r'\s')

    builtin_arrays = words(
        ('1array', '2array', '3array', '4array', '<array>', '>array', 'array',
         'array?', 'pair', 'pair?', 'resize-array'),
        suffix=r'\s')

    builtin_io = words(
        ('(each-stream-block-slice)', '(each-stream-block)',
         '(stream-contents-by-block)', '(stream-contents-by-element)',
         '(stream-contents-by-length-or-block)', '(stream-contents-by-length)',
         '+byte+', '+character+', 'bad-seek-type', 'bad-seek-type?', 'bl',
         'contents', 'each-block', 'each-block-size', 'each-block-slice',
         'each-line', 'each-morsel', 'each-stream-block',
         'each-stream-block-slice', 'each-stream-line', 'error-stream',
         'flush', 'input-stream', 'input-stream?', 'invalid-read-buffer',
         'invalid-read-buffer?', 'lines', 'nl', 'output-stream',
         'output-stream?', 'print', 'read', 'read-into', 'read-partial',
         'read-partial-into', 'read-until', 'read1', 'readln', 'seek-absolute',
         'seek-absolute?', 'seek-end', 'seek-end?', 'seek-input',
         'seek-output', 'seek-relative', 'seek-relative?', 'stream-bl',
         'stream-contents', 'stream-contents*', 'stream-copy', 'stream-copy*',
         'stream-element-type', 'stream-flush', 'stream-length',
         'stream-lines', 'stream-nl', 'stream-print', 'stream-read',
         'stream-read-into', 'stream-read-partial', 'stream-read-partial-into',
         'stream-read-partial-unsafe', 'stream-read-unsafe',
         'stream-read-until', 'stream-read1', 'stream-readln', 'stream-seek',
         'stream-seekable?', 'stream-tell', 'stream-write', 'stream-write1',
         'tell-input', 'tell-output', 'with-error-stream',
         'with-error-stream*', 'with-error>output',
         'with-input-output+error-streams', 'with-input-output+error-streams*',
         'with-input-stream', 'with-input-stream*', 'with-output-stream',
         'with-output-stream*', 'with-output>error',
         'with-output+error-stream', 'with-output+error-stream*',
         'with-streams', 'with-streams*', 'write', 'write1'),
        suffix=r'\s')

    builtin_strings = words(('1string', '<string>', '>string', 'resize-string',
                             'string', 'string?'),
                            suffix=r'\s')

    builtin_vectors = words(
        ('1vector', '<vector>', '>vector', '?push', 'vector', 'vector?'),
        suffix=r'\s')

    builtin_continuations = words(
        ('<condition>', '<continuation>', '<restart>', 'attempt-all',
         'attempt-all-error', 'attempt-all-error?', 'callback-error-hook',
         'callcc0', 'callcc1', 'cleanup', 'compute-restarts', 'condition',
         'condition?', 'continuation', 'continuation?', 'continue',
         'continue-restart', 'continue-with', 'current-continuation', 'error',
         'error-continuation', 'error-in-thread', 'error-thread', 'ifcc',
         'ignore-errors', 'in-callback?', 'original-error', 'recover',
         'restart', 'restart?', 'restarts', 'rethrow', 'rethrow-restarts',
         'return', 'return-continuation', 'thread-error-hook',
         'throw-continue', 'throw-restarts', 'with-datastack', 'with-return'),
        suffix=r'\s')

    tokens = {
        'root': [
            # factor allows a file to start with a shebang
            (r'#!.*$', Comment.Preproc),
            default('base'),
        ],
        'base': [
            (r'\s+', Text),

            # defining words
            (r'((?:MACRO|MEMO|TYPED)?:[:]?)(\s+)(\S+)',
             bygroups(Keyword, Text, Name.Function)),
            (r'(M:[:]?)(\s+)(\S+)(\s+)(\S+)',
             bygroups(Keyword, Text, Name.Class, Text, Name.Function)),
            (r'(C:)(\s+)(\S+)(\s+)(\S+)',
             bygroups(Keyword, Text, Name.Function, Text, Name.Class)),
            (r'(GENERIC:)(\s+)(\S+)', bygroups(Keyword, Text, Name.Function)),
            (r'(HOOK:|GENERIC#)(\s+)(\S+)(\s+)(\S+)',
             bygroups(Keyword, Text, Name.Function, Text, Name.Function)),
            (r'\(\s', Name.Function, 'stackeffect'),
            (r';\s', Keyword),

            # imports and namespaces
            (r'(USING:)(\s+)', bygroups(Keyword.Namespace, Text), 'vocabs'),
            (r'(USE:|UNUSE:|IN:|QUALIFIED:)(\s+)(\S+)',
             bygroups(Keyword.Namespace, Text, Name.Namespace)),
            (r'(QUALIFIED-WITH:)(\s+)(\S+)(\s+)(\S+)',
             bygroups(Keyword.Namespace, Text, Name.Namespace, Text,
                      Name.Namespace)),
            (r'(FROM:|EXCLUDE:)(\s+)(\S+)(\s+=>\s)',
             bygroups(Keyword.Namespace, Text, Name.Namespace, Text), 'words'),
            (r'(RENAME:)(\s+)(\S+)(\s+)(\S+)(\s+=>\s+)(\S+)',
             bygroups(Keyword.Namespace, Text, Name.Function, Text,
                      Name.Namespace, Text, Name.Function)),
            (r'(ALIAS:|TYPEDEF:)(\s+)(\S+)(\s+)(\S+)',
             bygroups(Keyword.Namespace, Text, Name.Function, Text,
                      Name.Function)),
            (r'(DEFER:|FORGET:|POSTPONE:)(\s+)(\S+)',
             bygroups(Keyword.Namespace, Text, Name.Function)),

            # tuples and classes
            (r'(TUPLE:|ERROR:)(\s+)(\S+)(\s+<\s+)(\S+)',
             bygroups(Keyword, Text, Name.Class, Text, Name.Class), 'slots'),
            (r'(TUPLE:|ERROR:|BUILTIN:)(\s+)(\S+)',
             bygroups(Keyword, Text, Name.Class), 'slots'),
            (r'(MIXIN:|UNION:|INTERSECTION:)(\s+)(\S+)',
             bygroups(Keyword, Text, Name.Class)),
            (r'(PREDICATE:)(\s+)(\S+)(\s+<\s+)(\S+)',
             bygroups(Keyword, Text, Name.Class, Text, Name.Class)),
            (r'(C:)(\s+)(\S+)(\s+)(\S+)',
             bygroups(Keyword, Text, Name.Function, Text, Name.Class)),
            (r'(INSTANCE:)(\s+)(\S+)(\s+)(\S+)',
             bygroups(Keyword, Text, Name.Class, Text, Name.Class)),
            (r'(SLOT:)(\s+)(\S+)', bygroups(Keyword, Text, Name.Function)),
            (r'(SINGLETON:)(\s+)(\S+)', bygroups(Keyword, Text, Name.Class)),
            (r'SINGLETONS:', Keyword, 'classes'),

            # other syntax
            (r'(CONSTANT:|SYMBOL:|MAIN:|HELP:)(\s+)(\S+)',
             bygroups(Keyword, Text, Name.Function)),
            (r'SYMBOLS:\s', Keyword, 'words'),
            (r'SYNTAX:\s', Keyword),
            (r'ALIEN:\s', Keyword),
            (r'(STRUCT:)(\s+)(\S+)', bygroups(Keyword, Text, Name.Class)),
            (r'(FUNCTION:)(\s+\S+\s+)(\S+)(\s+\(\s+[^\)]+\)\s)',
             bygroups(Keyword.Namespace, Text, Name.Function, Text)),
            (r'(FUNCTION-ALIAS:)(\s+)(\S+)(\s+\S+\s+)(\S+)(\s+\(\s+[^\)]+\)\s)',
             bygroups(Keyword.Namespace, Text, Name.Function, Text,
                      Name.Function, Text)),

            # vocab.private
            (r'(?:<PRIVATE|PRIVATE>)\s', Keyword.Namespace),

            # strings
            (r'"""\s+(?:.|\n)*?\s+"""', String),
            (r'"(?:\\\\|\\"|[^"])*"', String),
            (r'\S+"\s+(?:\\\\|\\"|[^"])*"', String),
            (r'CHAR:\s+(?:\\[\\abfnrstv]|[^\\]\S*)\s', String.Char),

            # comments
            (r'!\s+.*$', Comment),
            (r'#!\s+.*$', Comment),
            (r'/\*\s+(?:.|\n)*?\s\*/\s', Comment),

            # boolean constants
            (r'[tf]\s', Name.Constant),

            # symbols and literals
            (r'[\\$]\s+\S+', Name.Constant),
            (r'M\\\s+\S+\s+\S+', Name.Constant),

            # numbers
            (r'[+-]?(?:[\d,]*\d)?\.(?:\d([\d,]*\d)?)?(?:[eE][+-]?\d+)?\s',
             Number),
            (r'[+-]?\d(?:[\d,]*\d)?(?:[eE][+-]?\d+)?\s', Number),
            (r'0x[a-fA-F\d](?:[a-fA-F\d,]*[a-fA-F\d])?(?:p\d([\d,]*\d)?)?\s',
             Number),
            (r'NAN:\s+[a-fA-F\d](?:[a-fA-F\d,]*[a-fA-F\d])?(?:p\d([\d,]*\d)?)?\s',
             Number),
            (r'0b[01]+\s', Number.Bin),
            (r'0o[0-7]+\s', Number.Oct),
            (r'(?:\d([\d,]*\d)?)?\+\d(?:[\d,]*\d)?/\d(?:[\d,]*\d)?\s', Number),
            (r'(?:\-\d([\d,]*\d)?)?\-\d(?:[\d,]*\d)?/\d(?:[\d,]*\d)?\s',
             Number),

            # keywords
            (r'(?:deprecated|final|foldable|flushable|inline|recursive)\s',
             Keyword),

            # builtins
            (builtin_kernel, Name.Builtin),
            (builtin_assocs, Name.Builtin),
            (builtin_combinators, Name.Builtin),
            (builtin_math, Name.Builtin),
            (builtin_sequences, Name.Builtin),
            (builtin_namespaces, Name.Builtin),
            (builtin_arrays, Name.Builtin),
            (builtin_io, Name.Builtin),
            (builtin_strings, Name.Builtin),
            (builtin_vectors, Name.Builtin),
            (builtin_continuations, Name.Builtin),

            # everything else is text
            (r'\S+', Text),
        ],
        'stackeffect': [
            (r'\s+', Text),
            (r'\(\s+', Name.Function, 'stackeffect'),
            (r'\)\s', Name.Function, '#pop'),
            (r'--\s', Name.Function),
            (r'\S+', Name.Variable),
        ],
        'slots': [
            (r'\s+', Text),
            (r';\s', Keyword, '#pop'),
            (r'(\{\s+)(\S+)(\s+[^}]+\s+\}\s)',
             bygroups(Text, Name.Variable, Text)),
            (r'\S+', Name.Variable),
        ],
        'vocabs': [
            (r'\s+', Text),
            (r';\s', Keyword, '#pop'),
            (r'\S+', Name.Namespace),
        ],
        'classes': [
            (r'\s+', Text),
            (r';\s', Keyword, '#pop'),
            (r'\S+', Name.Class),
        ],
        'words': [
            (r'\s+', Text),
            (r';\s', Keyword, '#pop'),
            (r'\S+', Name.Function),
        ],
    }
예제 #21
0
class NedLexer(RegexLexer):
    name = 'ned'
    filenames = ['*.ned']

    #: optional Comment or Whitespace
    _ws = r'(?:\s|//.*?\n|/[*].*?[*]/)+'

    # The trailing ?, rather than *, avoids a geometric performance drop here.
    #: only one /* */ style comment
    _ws1 = r'\s*(?:/[*].*?[*]/\s*)?'

    tokens = {
        'whitespace': [
            (r'\n', Text),
            (r'\s+', Text),
            (r'\\\n', Text),  # line continuation
            (r'//(\n|[\w\W]*?[^\\]\n)', Comment.Single),
            (r'/(\\\n)?[*][\w\W]*?[*](\\\n)?/', Comment.Multiline),
            # Open until EOF, so no ending delimeter
            (r'/(\\\n)?[*][\w\W]*', Comment.Multiline),
        ],
        'statements': [
            (r'(L?)(")', bygroups(String.Affix, String), 'string'),
            (r"(L?)(')(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])(')",
             bygroups(String.Affix, String.Char, String.Char, String.Char)),
            (r'(true|false)\b', Name.Builtin),
            (r'(<-->|-->|<--|\.\.)', Keyword),
            (r'(bool|double|int|xml)\b', Keyword.Type),
            (r'(inout|input|output)\b', Keyword.Type),
            (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*', Number.Float),
            (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
            (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex),
            (r'#[0-9a-fA-F]+[LlUu]*', Number.Hex),
            (r'0[0-7]+[LlUu]*', Number.Oct),
            (r'\d+[LlUu]*', Number.Integer),
            (r'\*/', Error),
            (r'[~!%^&*+=|?:<>/-]', Operator),
            (r'[()\[\],.]', Punctuation),
            (words(("channel", "channelinterface", "simple", "module",
                    "network", "moduleinterface"),
                   suffix=r'\b'), Keyword),
            (words(
                ("parameters", "gates", "types", "submodules", "connections"),
                suffix=r'\b'), Keyword),
            (words(("volatile", "allowunconnected", "extends", "for", "if",
                    "import", "like", "package", "property"),
                   suffix=r'\b'), Keyword),
            (words(("sizeof", "const", "default", "ask", "this", "index",
                    "typename", "xmldoc"),
                   suffix=r'\b'), Keyword),
            (words(("acos", "asin", "atan", "atan2", "bernoulli", "beta",
                    "binomial", "cauchy", "ceil", "chi_square", "cos",
                    "erlang_k", "exp", "exponential", "fabs", "floor", "fmod",
                    "gamma_d", "genk_exponential", "genk_intuniform",
                    "genk_normal", "genk_truncnormal", "genk_uniform",
                    "geometric", "hypergeometric", "hypot", "intuniform",
                    "log", "log10", "lognormal", "max", "min", "negbinomial",
                    "normal", "pareto_shifted", "poisson", "pow", "simTime",
                    "sin", "sqrt", "student_t", "tan", "triang", "truncnormal",
                    "uniform", "weibull", "xml", "xmldoc"),
                   suffix=r'\b'), Name.Builtin),
            ('@[a-zA-Z_]\w*', Name.Builtin),
            ('[a-zA-Z_]\w*', Name),
        ],
        'root': [
            include('whitespace'),
            # functions
            (
                r'((?:[\w*\s])+?(?:\s|[*]))'  # return arguments
                r'([a-zA-Z_]\w*)'  # method name
                r'(\s*\([^;]*?\))'  # signature
                r'([^;{]*)(\{)',
                bygroups(using(this), Name.Function, using(this), using(this),
                         Punctuation),
                'function'),
            # function declarations
            (
                r'((?:[\w*\s])+?(?:\s|[*]))'  # return arguments
                r'([a-zA-Z_]\w*)'  # method name
                r'(\s*\([^;]*?\))'  # signature
                r'([^;]*)(;)',
                bygroups(using(this), Name.Function, using(this), using(this),
                         Punctuation)),
            default('statement'),
        ],
        'statement': [
            include('whitespace'),
            include('statements'),
            ('[{}]', Punctuation),
            (';', Punctuation, '#pop'),
        ],
        'function': [
            include('whitespace'),
            include('statements'),
            (';', Punctuation),
            (r'\{', Punctuation, '#push'),
            (r'\}', Punctuation, '#pop'),
        ],
        'string': [
            (r'"', String, '#pop'),
            (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|'
             r'u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|[0-7]{1,3})', String.Escape),
            (r'[^\\"\n]+', String),  # all other characters
            (r'\\\n', String),  # line continuation
            (r'\\', String),  # stray backslash
        ]
    }
예제 #22
0
class CobolLexer(RegexLexer):
    """
    Lexer for OpenCOBOL code.

    .. versionadded:: 1.6
    """
    name = 'COBOL'
    aliases = ['cobol']
    filenames = ['*.cob', '*.COB', '*.cpy', '*.CPY']
    mimetypes = ['text/x-cobol']
    flags = re.IGNORECASE | re.MULTILINE

    # Data Types: by PICTURE and USAGE
    # Operators: **, *, +, -, /, <, >, <=, >=, =, <>
    # Logical (?): NOT, AND, OR

    # Reserved words:
    # http://opencobol.add1tocobol.com/#reserved-words
    # Intrinsics:
    # http://opencobol.add1tocobol.com/#does-opencobol-implement-any-intrinsic-functions

    tokens = {
        'root': [
            include('comment'),
            include('strings'),
            include('core'),
            include('nums'),
            (r'[a-z0-9]([\w\-]*[a-z0-9]+)?', Name.Variable),
            # (r'[\s]+', Text),
            (r'[ \t]+', Text),
        ],
        'comment': [
            (r'(^.{6}[*/].*\n|^.{6}|\*>.*\n)', Comment),
        ],
        'core': [
            # Figurative constants
            (r'(^|(?<=[^\w\-]))(ALL\s+)?'
             r'((ZEROES)|(HIGH-VALUE|LOW-VALUE|QUOTE|SPACE|ZERO)(S)?)'
             r'\s*($|(?=[^\w\-]))', Name.Constant),

            # Reserved words STATEMENTS and other bolds
            (words(
                ('ACCEPT', 'ADD', 'ALLOCATE', 'CALL', 'CANCEL', 'CLOSE',
                 'COMPUTE', 'CONFIGURATION', 'CONTINUE', 'DATA', 'DELETE',
                 'DISPLAY', 'DIVIDE', 'DIVISION', 'ELSE', 'END', 'END-ACCEPT',
                 'END-ADD', 'END-CALL', 'END-COMPUTE', 'END-DELETE',
                 'END-DISPLAY', 'END-DIVIDE', 'END-EVALUATE', 'END-IF',
                 'END-MULTIPLY', 'END-OF-PAGE', 'END-PERFORM', 'END-READ',
                 'END-RETURN', 'END-REWRITE', 'END-SEARCH', 'END-START',
                 'END-STRING', 'END-SUBTRACT', 'END-UNSTRING', 'END-WRITE',
                 'ENVIRONMENT', 'EVALUATE', 'EXIT', 'FD', 'FILE',
                 'FILE-CONTROL', 'FOREVER', 'FREE', 'GENERATE', 'GO', 'GOBACK',
                 'IDENTIFICATION', 'IF', 'INITIALIZE', 'INITIATE',
                 'INPUT-OUTPUT', 'INSPECT', 'INVOKE', 'I-O-CONTROL', 'LINKAGE',
                 'LOCAL-STORAGE', 'MERGE', 'MOVE', 'MULTIPLY', 'OPEN',
                 'PERFORM', 'PROCEDURE', 'PROGRAM-ID', 'RAISE', 'READ',
                 'RELEASE', 'RESUME', 'RETURN', 'REWRITE', 'SCREEN', 'SD',
                 'SEARCH', 'SECTION', 'SET', 'SORT', 'START', 'STOP', 'STRING',
                 'SUBTRACT', 'SUPPRESS', 'TERMINATE', 'THEN', 'UNLOCK',
                 'UNSTRING', 'USE', 'VALIDATE', 'WORKING-STORAGE', 'WRITE'),
                prefix=r'(^|(?<=[^\w\-]))',
                suffix=r'\s*($|(?=[^\w\-]))'), Keyword.Reserved),

            # Reserved words
            (words(
                ('ACCESS', 'ADDRESS', 'ADVANCING', 'AFTER', 'ALL', 'ALPHABET',
                 'ALPHABETIC', 'ALPHABETIC-LOWER', 'ALPHABETIC-UPPER',
                 'ALPHANUMERIC', 'ALPHANUMERIC-EDITED', 'ALSO', 'ALTER',
                 'ALTERNATE'
                 'ANY', 'ARE', 'AREA', 'AREAS', 'ARGUMENT-NUMBER',
                 'ARGUMENT-VALUE', 'AS', 'ASCENDING', 'ASSIGN', 'AT', 'AUTO',
                 'AUTO-SKIP', 'AUTOMATIC', 'AUTOTERMINATE', 'BACKGROUND-COLOR',
                 'BASED', 'BEEP', 'BEFORE', 'BELL', 'BLANK', 'BLINK', 'BLOCK',
                 'BOTTOM', 'BY', 'BYTE-LENGTH', 'CHAINING', 'CHARACTER',
                 'CHARACTERS', 'CLASS', 'CODE', 'CODE-SET', 'COL', 'COLLATING',
                 'COLS', 'COLUMN', 'COLUMNS', 'COMMA', 'COMMAND-LINE',
                 'COMMIT', 'COMMON', 'CONSTANT', 'CONTAINS', 'CONTENT',
                 'CONTROL', 'CONTROLS', 'CONVERTING', 'COPY', 'CORR',
                 'CORRESPONDING', 'COUNT', 'CRT', 'CURRENCY', 'CURSOR',
                 'CYCLE', 'DATE', 'DAY', 'DAY-OF-WEEK', 'DE', 'DEBUGGING',
                 'DECIMAL-POINT', 'DECLARATIVES', 'DEFAULT', 'DELIMITED',
                 'DELIMITER', 'DEPENDING', 'DESCENDING', 'DETAIL', 'DISK',
                 'DOWN', 'DUPLICATES', 'DYNAMIC', 'EBCDIC', 'ENTRY',
                 'ENVIRONMENT-NAME', 'ENVIRONMENT-VALUE', 'EOL', 'EOP', 'EOS',
                 'ERASE', 'ERROR', 'ESCAPE', 'EXCEPTION', 'EXCLUSIVE',
                 'EXTEND', 'EXTERNAL', 'FILE-ID', 'FILLER', 'FINAL', 'FIRST',
                 'FIXED', 'FLOAT-LONG', 'FLOAT-SHORT', 'FOOTING', 'FOR',
                 'FOREGROUND-COLOR', 'FORMAT', 'FROM', 'FULL', 'FUNCTION',
                 'FUNCTION-ID', 'GIVING', 'GLOBAL', 'GROUP', 'HEADING',
                 'HIGHLIGHT', 'I-O', 'ID', 'IGNORE', 'IGNORING', 'IN', 'INDEX',
                 'INDEXED', 'INDICATE', 'INITIAL', 'INITIALIZED', 'INPUT',
                 'INTO', 'INTRINSIC', 'INVALID', 'IS', 'JUST', 'JUSTIFIED',
                 'KEY', 'LABEL', 'LAST', 'LEADING', 'LEFT', 'LENGTH', 'LIMIT',
                 'LIMITS', 'LINAGE', 'LINAGE-COUNTER', 'LINE', 'LINES',
                 'LOCALE', 'LOCK', 'LOWLIGHT', 'MANUAL', 'MEMORY', 'MINUS',
                 'MODE', 'MULTIPLE', 'NATIONAL', 'NATIONAL-EDITED', 'NATIVE',
                 'NEGATIVE', 'NEXT', 'NO', 'NULL', 'NULLS', 'NUMBER',
                 'NUMBERS', 'NUMERIC', 'NUMERIC-EDITED', 'OBJECT-COMPUTER',
                 'OCCURS', 'OF', 'OFF', 'OMITTED', 'ON', 'ONLY', 'OPTIONAL',
                 'ORDER', 'ORGANIZATION', 'OTHER', 'OUTPUT', 'OVERFLOW',
                 'OVERLINE', 'PACKED-DECIMAL', 'PADDING', 'PAGE', 'PARAGRAPH',
                 'PLUS', 'POINTER', 'POSITION', 'POSITIVE', 'PRESENT',
                 'PREVIOUS', 'PRINTER', 'PRINTING', 'PROCEDURE-POINTER',
                 'PROCEDURES', 'PROCEED', 'PROGRAM', 'PROGRAM-POINTER',
                 'PROMPT', 'QUOTE', 'QUOTES', 'RANDOM', 'RD', 'RECORD',
                 'RECORDING', 'RECORDS', 'RECURSIVE', 'REDEFINES', 'REEL',
                 'REFERENCE', 'RELATIVE', 'REMAINDER', 'REMOVAL', 'RENAMES',
                 'REPLACING', 'REPORT', 'REPORTING', 'REPORTS', 'REPOSITORY',
                 'REQUIRED', 'RESERVE', 'RETURNING', 'REVERSE-VIDEO', 'REWIND',
                 'RIGHT', 'ROLLBACK', 'ROUNDED', 'RUN', 'SAME', 'SCROLL',
                 'SECURE', 'SEGMENT-LIMIT', 'SELECT', 'SENTENCE', 'SEPARATE',
                 'SEQUENCE', 'SEQUENTIAL', 'SHARING', 'SIGN', 'SIGNED',
                 'SIGNED-INT', 'SIGNED-LONG', 'SIGNED-SHORT', 'SIZE',
                 'SORT-MERGE', 'SOURCE', 'SOURCE-COMPUTER', 'SPECIAL-NAMES',
                 'STANDARD', 'STANDARD-1', 'STANDARD-2', 'STATUS', 'SUM',
                 'SYMBOLIC', 'SYNC', 'SYNCHRONIZED', 'TALLYING', 'TAPE',
                 'TEST', 'THROUGH', 'THRU', 'TIME', 'TIMES', 'TO', 'TOP',
                 'TRAILING', 'TRANSFORM', 'TYPE', 'UNDERLINE', 'UNIT',
                 'UNSIGNED', 'UNSIGNED-INT', 'UNSIGNED-LONG', 'UNSIGNED-SHORT',
                 'UNTIL', 'UP', 'UPDATE', 'UPON', 'USAGE', 'USING', 'VALUE',
                 'VALUES', 'VARYING', 'WAIT', 'WHEN', 'WITH', 'WORDS',
                 'YYYYDDD', 'YYYYMMDD'),
                prefix=r'(^|(?<=[^\w\-]))',
                suffix=r'\s*($|(?=[^\w\-]))'), Keyword.Pseudo),

            # inactive reserved words
            (words(
                ('ACTIVE-CLASS', 'ALIGNED', 'ANYCASE', 'ARITHMETIC',
                 'ATTRIBUTE', 'B-AND', 'B-NOT', 'B-OR', 'B-XOR', 'BIT',
                 'BOOLEAN', 'CD', 'CENTER', 'CF', 'CH', 'CHAIN', 'CLASS-ID',
                 'CLASSIFICATION', 'COMMUNICATION', 'CONDITION',
                 'DATA-POINTER', 'DESTINATION', 'DISABLE', 'EC', 'EGI', 'EMI',
                 'ENABLE', 'END-RECEIVE', 'ENTRY-CONVENTION', 'EO', 'ESI',
                 'EXCEPTION-OBJECT', 'EXPANDS', 'FACTORY', 'FLOAT-BINARY-16',
                 'FLOAT-BINARY-34', 'FLOAT-BINARY-7', 'FLOAT-DECIMAL-16',
                 'FLOAT-DECIMAL-34', 'FLOAT-EXTENDED', 'FORMAT',
                 'FUNCTION-POINTER', 'GET', 'GROUP-USAGE', 'IMPLEMENTS',
                 'INFINITY', 'INHERITS', 'INTERFACE', 'INTERFACE-ID', 'INVOKE',
                 'LC_ALL', 'LC_COLLATE', 'LC_CTYPE', 'LC_MESSAGES',
                 'LC_MONETARY', 'LC_NUMERIC', 'LC_TIME', 'LINE-COUNTER',
                 'MESSAGE', 'METHOD', 'METHOD-ID', 'NESTED', 'NONE', 'NORMAL',
                 'OBJECT', 'OBJECT-REFERENCE', 'OPTIONS', 'OVERRIDE',
                 'PAGE-COUNTER', 'PF', 'PH', 'PROPERTY', 'PROTOTYPE', 'PURGE',
                 'QUEUE', 'RAISE', 'RAISING', 'RECEIVE', 'RELATION', 'REPLACE',
                 'REPRESENTS-NOT-A-NUMBER', 'RESET', 'RESUME', 'RETRY', 'RF',
                 'RH', 'SECONDS', 'SEGMENT', 'SELF', 'SEND', 'SOURCES',
                 'STATEMENT', 'STEP', 'STRONG', 'SUB-QUEUE-1', 'SUB-QUEUE-2',
                 'SUB-QUEUE-3', 'SUPER', 'SYMBOL', 'SYSTEM-DEFAULT', 'TABLE',
                 'TERMINAL', 'TEXT', 'TYPEDEF', 'UCS-4', 'UNIVERSAL',
                 'USER-DEFAULT', 'UTF-16', 'UTF-8', 'VAL-STATUS', 'VALID',
                 'VALIDATE', 'VALIDATE-STATUS'),
                prefix=r'(^|(?<=[^\w\-]))',
                suffix=r'\s*($|(?=[^\w\-]))'), Error),

            # Data Types
            (r'(^|(?<=[^\w\-]))'
             r'(PIC\s+.+?(?=(\s|\.\s))|PICTURE\s+.+?(?=(\s|\.\s))|'
             r'(COMPUTATIONAL)(-[1-5X])?|(COMP)(-[1-5X])?|'
             r'BINARY-C-LONG|'
             r'BINARY-CHAR|BINARY-DOUBLE|BINARY-LONG|BINARY-SHORT|'
             r'BINARY)\s*($|(?=[^\w\-]))', Keyword.Type),

            # Operators
            (r'(\*\*|\*|\+|-|/|<=|>=|<|>|==|/=|=)', Operator),

            # (r'(::)', Keyword.Declaration),
            (r'([(),;:&%.])', Punctuation),

            # Intrinsics
            (r'(^|(?<=[^\w\-]))(ABS|ACOS|ANNUITY|ASIN|ATAN|BYTE-LENGTH|'
             r'CHAR|COMBINED-DATETIME|CONCATENATE|COS|CURRENT-DATE|'
             r'DATE-OF-INTEGER|DATE-TO-YYYYMMDD|DAY-OF-INTEGER|DAY-TO-YYYYDDD|'
             r'EXCEPTION-(?:FILE|LOCATION|STATEMENT|STATUS)|EXP10|EXP|E|'
             r'FACTORIAL|FRACTION-PART|INTEGER-OF-(?:DATE|DAY|PART)|INTEGER|'
             r'LENGTH|LOCALE-(?:DATE|TIME(?:-FROM-SECONDS)?)|LOG(?:10)?|'
             r'LOWER-CASE|MAX|MEAN|MEDIAN|MIDRANGE|MIN|MOD|NUMVAL(?:-C)?|'
             r'ORD(?:-MAX|-MIN)?|PI|PRESENT-VALUE|RANDOM|RANGE|REM|REVERSE|'
             r'SECONDS-FROM-FORMATTED-TIME|SECONDS-PAST-MIDNIGHT|SIGN|SIN|SQRT|'
             r'STANDARD-DEVIATION|STORED-CHAR-LENGTH|SUBSTITUTE(?:-CASE)?|'
             r'SUM|TAN|TEST-DATE-YYYYMMDD|TEST-DAY-YYYYDDD|TRIM|'
             r'UPPER-CASE|VARIANCE|WHEN-COMPILED|YEAR-TO-YYYY)\s*'
             r'($|(?=[^\w\-]))', Name.Function),

            # Booleans
            (r'(^|(?<=[^\w\-]))(true|false)\s*($|(?=[^\w\-]))', Name.Builtin),
            # Comparing Operators
            (r'(^|(?<=[^\w\-]))(equal|equals|ne|lt|le|gt|ge|'
             r'greater|less|than|not|and|or)\s*($|(?=[^\w\-]))', Operator.Word
             ),
        ],

        # \"[^\"\n]*\"|\'[^\'\n]*\'
        'strings': [
            # apparently strings can be delimited by EOL if they are continued
            # in the next line
            (r'"[^"\n]*("|\n)', String.Double),
            (r"'[^'\n]*('|\n)", String.Single),
        ],
        'nums': [
            (r'\d+(\s*|\.$|$)', Number.Integer),
            (r'[+-]?\d*\.\d+(E[-+]?\d+)?', Number.Float),
            (r'[+-]?\d+\.\d*(E[-+]?\d+)?', Number.Float),
        ],
    }
예제 #23
0
class MonteLexer(RegexLexer):
    """
    Lexer for the `Monte <https://monte.readthedocs.io/>`_ programming language.

    .. versionadded:: 2.2
    """
    name = 'Monte'
    aliases = ['monte']
    filenames = ['*.mt']

    tokens = {
        'root': [
            # Comments
            (r'#[^\n]*\n', Comment),

            # Docstrings
            # Apologies for the non-greedy matcher here.
            (r'/\*\*.*?\*/', String.Doc),

            # `var` declarations
            (r'\bvar\b', Keyword.Declaration, 'var'),

            # `interface` declarations
            (r'\binterface\b', Keyword.Declaration, 'interface'),

            # method declarations
            (words(_methods, prefix='\\b', suffix='\\b'), Keyword, 'method'),

            # All other declarations
            (words(_declarations, prefix='\\b',
                   suffix='\\b'), Keyword.Declaration),

            # Keywords
            (words(_keywords, prefix='\\b', suffix='\\b'), Keyword),

            # Literals
            ('[+-]?0x[_0-9a-fA-F]+', Number.Hex),
            (r'[+-]?[_0-9]+\.[_0-9]*([eE][+-]?[_0-9]+)?', Number.Float),
            ('[+-]?[_0-9]+', Number.Integer),
            ("'", String.Double, 'char'),
            ('"', String.Double, 'string'),

            # Quasiliterals
            ('`', String.Backtick, 'ql'),

            # Operators
            (words(_operators), Operator),

            # Verb operators
            (_identifier + '=', Operator.Word),

            # Safe scope constants
            (words(_constants, prefix='\\b', suffix='\\b'), Keyword.Pseudo),

            # Safe scope guards
            (words(_guards, prefix='\\b', suffix='\\b'), Keyword.Type),

            # All other safe scope names
            (words(_safeScope, prefix='\\b', suffix='\\b'), Name.Builtin),

            # Identifiers
            (_identifier, Name),

            # Punctuation
            (r'\(|\)|\{|\}|\[|\]|:|,', Punctuation),

            # Whitespace
            (' +', Whitespace),

            # Definite lexer errors
            ('=', Error),
        ],
        'char': [
            # It is definitely an error to have a char of width == 0.
            ("'", Error, 'root'),
            (_escape_pattern, String.Escape, 'charEnd'),
            ('.', String.Char, 'charEnd'),
        ],
        'charEnd': [
            ("'", String.Char, '#pop:2'),
            # It is definitely an error to have a char of width > 1.
            ('.', Error),
        ],
        # The state of things coming into an interface.
        'interface': [
            (' +', Whitespace),
            (_identifier, Name.Class, '#pop'),
            include('root'),
        ],
        # The state of things coming into a method.
        'method': [
            (' +', Whitespace),
            (_identifier, Name.Function, '#pop'),
            include('root'),
        ],
        'string': [
            ('"', String.Double, 'root'),
            (_escape_pattern, String.Escape),
            (r'\n', String.Double),
            ('.', String.Double),
        ],
        'ql': [
            ('`', String.Backtick, 'root'),
            (r'\$' + _escape_pattern, String.Escape),
            (r'\$\$', String.Escape),
            (r'@@', String.Escape),
            (r'\$\{', String.Interpol, 'qlNest'),
            (r'@\{', String.Interpol, 'qlNest'),
            (r'\$' + _identifier, Name),
            ('@' + _identifier, Name),
            ('.', String.Backtick),
        ],
        'qlNest': [
            (r'\}', String.Interpol, '#pop'),
            include('root'),
        ],
        # The state of things immediately following `var`.
        'var': [
            (' +', Whitespace),
            (_identifier, Name.Variable, '#pop'),
            include('root'),
        ],
    }
예제 #24
0
class VerilogLexer(RegexLexer):
    """
    For verilog source code with preprocessor directives.

    .. versionadded:: 1.4
    """
    name = 'verilog'
    aliases = ['verilog', 'v']
    filenames = ['*.v']
    mimetypes = ['text/x-verilog']

    # : optional Comment or Whitespace
    _ws = r'(?:\s|//.*?\n|/[*].*?[*]/)+'

    tokens = {
        'root': [
            (r'^\s*`define', Comment.Preproc, 'macro'),
            (r'\n', Text),
            (r'\s+', Text),
            (r'\\\n', Text),  # line continuation
            (r'/(\\\n)?/(\n|(.|\n)*?[^\\]\n)', Comment.Single),
            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
            (r'[{}#@]', Punctuation),
            (r'L?"', String, 'string'),
            (r"L?'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'",
             String.Char),
            (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[lL]?', Number.Float),
            (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
            (r'([0-9]+)|(\'h)[0-9a-fA-F]+', Number.Hex),
            (r'([0-9]+)|(\'b)[01]+', Number.Bin),
            (r'([0-9]+)|(\'d)[0-9]+', Number.Integer),
            (r'([0-9]+)|(\'o)[0-7]+', Number.Oct),
            (r'\'[01xz]', Number),
            (r'\d+[Ll]?', Number.Integer),
            (r'\*/', Error),
            (r'[~!%^&*+=|?:<>/-]', Operator),
            (r'[()\[\],.;\']', Punctuation),
            (r'`[a-zA-Z_]\w*', Name.Constant),
            (r'^(\s*)(package)(\s+)', bygroups(Text, Keyword.Namespace, Text)),
            (r'^(\s*)(import)(\s+)', bygroups(Text, Keyword.Namespace,
                                              Text), 'import'),
            (words(
                ('always', 'always_comb', 'always_ff', 'always_latch', 'and',
                 'assign', 'automatic', 'begin', 'break', 'buf', 'bufif0',
                 'bufif1', 'case', 'casex', 'casez', 'cmos', 'const',
                 'continue', 'deassign', 'default', 'defparam', 'disable',
                 'do', 'edge', 'else', 'end', 'endcase', 'endfunction',
                 'endgenerate', 'endmodule', 'endpackage', 'endprimitive',
                 'endspecify', 'endtable', 'endtask', 'enum', 'event', 'final',
                 'for', 'force', 'forever', 'fork', 'function', 'generate',
                 'genvar', 'highz0', 'highz1', 'if', 'initial', 'inout',
                 'input', 'integer', 'join', 'large', 'localparam',
                 'macromodule', 'medium', 'module', 'nand', 'negedge', 'nmos',
                 'nor', 'not', 'notif0', 'notif1', 'or', 'output', 'packed',
                 'parameter', 'pmos', 'posedge', 'primitive', 'pull0', 'pull1',
                 'pulldown', 'pullup', 'rcmos', 'ref', 'release', 'repeat',
                 'return', 'rnmos', 'rpmos', 'rtran', 'rtranif0', 'rtranif1',
                 'scalared', 'signed', 'small', 'specify', 'specparam',
                 'strength', 'string', 'strong0', 'strong1', 'struct', 'table',
                 'task', 'tran', 'tranif0', 'tranif1', 'type', 'typedef',
                 'unsigned', 'var', 'vectored', 'void', 'wait', 'weak0',
                 'weak1', 'while', 'xnor', 'xor'),
                suffix=r'\b'), Keyword),
            (words(('accelerate', 'autoexpand_vectornets', 'celldefine',
                    'default_nettype', 'else', 'elsif', 'endcelldefine',
                    'endif', 'endprotect', 'endprotected', 'expand_vectornets',
                    'ifdef', 'ifndef', 'include', 'noaccelerate',
                    'noexpand_vectornets', 'noremove_gatenames',
                    'noremove_netnames', 'nounconnected_drive', 'protect',
                    'protected', 'remove_gatenames', 'remove_netnames',
                    'resetall', 'timescale', 'unconnected_drive', 'undef'),
                   prefix=r'`',
                   suffix=r'\b'), Comment.Preproc),
            (words(('bits', 'bitstoreal', 'bitstoshortreal', 'countdrivers',
                    'display', 'fclose', 'fdisplay', 'finish', 'floor',
                    'fmonitor', 'fopen', 'fstrobe', 'fwrite', 'getpattern',
                    'history', 'incsave', 'input', 'itor', 'key', 'list',
                    'log', 'monitor', 'monitoroff', 'monitoron', 'nokey',
                    'nolog', 'printtimescale', 'random', 'readmemb',
                    'readmemh', 'realtime', 'realtobits', 'reset',
                    'reset_count', 'reset_value', 'restart', 'rtoi', 'save',
                    'scale', 'scope', 'shortrealtobits', 'showscopes',
                    'showvariables', 'showvars', 'sreadmemb', 'sreadmemh',
                    'stime', 'stop', 'strobe', 'time', 'timeformat', 'write'),
                   prefix=r'\$',
                   suffix=r'\b'), Name.Builtin),
            (words(('byte', 'shortint', 'int', 'longint', 'integer', 'time',
                    'bit', 'logic', 'reg', 'supply0', 'supply1', 'tri',
                    'triand', 'trior', 'tri0', 'tri1', 'trireg', 'uwire',
                    'wire', 'wand', 'wo'
                    'shortreal', 'real', 'realtime'),
                   suffix=r'\b'), Keyword.Type),
            ('[a-zA-Z_]\w*:(?!:)', Name.Label),
            ('[a-zA-Z_]\w*', Name),
        ],
        'string': [
            (r'"', String, '#pop'),
            (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})',
             String.Escape),
            (r'[^\\"\n]+', String),  # all other characters
            (r'\\\n', String),  # line continuation
            (r'\\', String),  # stray backslash
        ],
        'macro': [
            (r'[^/\n]+', Comment.Preproc),
            (r'/[*](.|\n)*?[*]/', Comment.Multiline),
            (r'//.*?\n', Comment.Single, '#pop'),
            (r'/', Comment.Preproc),
            (r'(?<=\\)\n', Comment.Preproc),
            (r'\n', Comment.Preproc, '#pop'),
        ],
        'import': [(r'[\w:]+\*?', Name.Namespace, '#pop')]
    }

    def get_tokens_unprocessed(self, text):
        for index, token, value in \
                RegexLexer.get_tokens_unprocessed(self, text):
            # Convention: mark all upper case names as constants
            if token is Name:
                if value.isupper():
                    token = Name.Constant
            yield index, token, value
예제 #25
0
class FireLexer(CFamilyLexer):
    """
    For `Fire Script <https://github.com/rbrich/xcikit>`_.

    .. versionadded:: 2.0
    """
    name = 'Fire'
    aliases = ('fire', )
    filenames = ('*.fire', )
    mimetypes = ('text/x-fire-script', )

    flags = re.MULTILINE | re.UNICODE

    tokens = {
        'keywords': [(words(('void', 'false', 'true'),
                            suffix=r'\b'), Keyword.Constant),
                     (words(('stdin', 'stdout', 'stderr', 'null'),
                            suffix=r'\b'), Name.Builtin),
                     (words((
                         'catch',
                         'class',
                         'else',
                         'fun',
                         'if',
                         'import',
                         'instance',
                         'in',
                         'match',
                         'module',
                         'then',
                         'try',
                         'type',
                         'with',
                     ),
                            suffix=r'\b'), Keyword.Reserved)],
        'types': [(words(('Void', 'Bool', 'Byte', 'Char', 'Int', 'Int32',
                          'Int64', 'Float', 'Float32', 'Float64', 'String'),
                         suffix=r'\b'), Keyword.Type)],
        'raw_string': [
            (r'"""', String.Double, '#pop'),
            (r'\\"""+', String.Escape),
            (r'\\', String.Double),  # backslash must be parsed one at a time
            (r'[^\\"]+',
             String.Double),  # other characters, including newlines
        ],
        'statements': [
            include('keywords'),
            include('types'),
            (r'(b?)(""")', bygroups(String.Affix,
                                    String.Double), 'raw_string'),
            (r'(b?)(")', bygroups(String.Affix, String.Double), 'string'),
            (r"(b?)(')(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])(')",
             bygroups(String.Affix, String.Char, String.Char, String.Char)),
            (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*', Number.Float),
            (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
            (r'0x[0-9a-fA-F]+[LlUuBb]*', Number.Hex),
            (r'0o[0-7]+[LlUuBb]*', Number.Oct),
            (r'0b[01]+[LlUuBb]*', Number.Bin),
            (r'\d+[LlUuBb]*', Number.Integer),
            (r'\*/', Error),
            (r'[~!%^&*+=|?:<>/-]', Operator),
            (r'[()\[\],.]', Punctuation),
            (r'([a-zA-Z_]\w*)(\s*)(:)(?!:)',
             bygroups(Name.Label, Text, Punctuation)),
            (r'[a-z_]\w*', Name),
            (r'[A-Z]\w*', Name.Class),
        ],
    }
예제 #26
0
파일: asm.py 프로젝트: Cold5nap/sobolIter
class HsailLexer(RegexLexer):
    """
    For HSAIL assembly code.

    .. versionadded:: 2.2
    """
    name = 'HSAIL'
    aliases = ['hsail', 'hsa']
    filenames = ['*.hsail']
    mimetypes = ['text/x-hsail']

    string = r'"[^"]*?"'
    identifier = r'[a-zA-Z_][\w.]*'
    # Registers
    register_number = r'[0-9]+'
    register = r'(\$(c|s|d|q)' + register_number + ')'
    # Qualifiers
    alignQual = r'(align\(\d+\))'
    widthQual = r'(width\((\d+|all)\))'
    allocQual = r'(alloc\(agent\))'
    # Instruction Modifiers
    roundingMod = (r'((_ftz)?(_up|_down|_zero|_near))')
    datatypeMod = (
        r'_('
        # packedTypes
        r'u8x4|s8x4|u16x2|s16x2|u8x8|s8x8|u16x4|s16x4|u32x2|s32x2|'
        r'u8x16|s8x16|u16x8|s16x8|u32x4|s32x4|u64x2|s64x2|'
        r'f16x2|f16x4|f16x8|f32x2|f32x4|f64x2|'
        # baseTypes
        r'u8|s8|u16|s16|u32|s32|u64|s64|'
        r'b128|b8|b16|b32|b64|b1|'
        r'f16|f32|f64|'
        # opaqueType
        r'roimg|woimg|rwimg|samp|sig32|sig64)')

    # Numeric Constant
    float = r'((\d+\.)|(\d*\.\d+))[eE][+-]?\d+'
    hexfloat = r'0[xX](([0-9a-fA-F]+\.[0-9a-fA-F]*)|([0-9a-fA-F]*\.[0-9a-fA-F]+))[pP][+-]?\d+'
    ieeefloat = r'0((h|H)[0-9a-fA-F]{4}|(f|F)[0-9a-fA-F]{8}|(d|D)[0-9a-fA-F]{16})'

    tokens = {
        'root': [
            include('whitespace'),
            include('comments'), (string, String),
            (r'@' + identifier + ':?', Name.Label),
            (register, Name.Variable.Anonymous),
            include('keyword'), (r'&' + identifier, Name.Variable.Global),
            (r'%' + identifier, Name.Variable), (hexfloat, Number.Hex),
            (r'0[xX][a-fA-F0-9]+', Number.Hex), (ieeefloat, Number.Float),
            (float, Number.Float), (r'\d+', Number.Integer),
            (r'[=<>{}\[\]()*.,:;!]|x\b', Punctuation)
        ],
        'whitespace': [
            (r'(\n|\s)+', Text),
        ],
        'comments': [
            (r'/\*.*?\*/', Comment.Multiline),
            (r'//.*?\n', Comment.Single),
        ],
        'keyword': [
            # Types
            (r'kernarg' + datatypeMod, Keyword.Type),

            # Regular keywords
            (r'\$(full|base|small|large|default|zero|near)', Keyword),
            (words(('module', 'extension', 'pragma', 'prog', 'indirect',
                    'signature', 'decl', 'kernel', 'function',
                    'enablebreakexceptions', 'enabledetectexceptions',
                    'maxdynamicgroupsize', 'maxflatgridsize',
                    'maxflatworkgroupsize', 'requireddim', 'requiredgridsize',
                    'requiredworkgroupsize', 'requirenopartialworkgroups'),
                   suffix=r'\b'), Keyword),

            # instructions
            (roundingMod, Keyword),
            (datatypeMod, Keyword),
            (r'_(' + alignQual + '|' + widthQual + ')', Keyword),
            (r'_kernarg', Keyword),
            (r'(nop|imagefence)\b', Keyword),
            (words(
                ('cleardetectexcept', 'clock', 'cuid', 'debugtrap', 'dim',
                 'getdetectexcept', 'groupbaseptr', 'kernargbaseptr', 'laneid',
                 'maxcuid', 'maxwaveid', 'packetid', 'setdetectexcept',
                 'waveid', 'workitemflatabsid', 'workitemflatid', 'nullptr',
                 'abs', 'bitrev', 'currentworkgroupsize',
                 'currentworkitemflatid', 'fract', 'ncos', 'neg', 'nexp2',
                 'nlog2', 'nrcp', 'nrsqrt', 'nsin', 'nsqrt', 'gridgroups',
                 'gridsize', 'not', 'sqrt', 'workgroupid', 'workgroupsize',
                 'workitemabsid', 'workitemid', 'ceil', 'floor', 'rint',
                 'trunc', 'add', 'bitmask', 'borrow', 'carry', 'copysign',
                 'div', 'rem', 'sub', 'shl', 'shr', 'and', 'or', 'xor',
                 'unpackhi', 'unpacklo', 'max', 'min', 'fma', 'mad',
                 'bitextract', 'bitselect', 'shuffle', 'cmov', 'bitalign',
                 'bytealign', 'lerp', 'nfma', 'mul', 'mulhi', 'mul24hi',
                 'mul24', 'mad24', 'mad24hi', 'bitinsert', 'combine', 'expand',
                 'lda', 'mov', 'pack', 'unpack', 'packcvt', 'unpackcvt', 'sad',
                 'sementp', 'ftos', 'stof', 'cmp', 'ld', 'st', '_eq', '_ne',
                 '_lt', '_le', '_gt', '_ge', '_equ', '_neu', '_ltu', '_leu',
                 '_gtu', '_geu', '_num', '_nan', '_seq', '_sne', '_slt',
                 '_sle', '_sgt', '_sge', '_snum', '_snan', '_sequ', '_sneu',
                 '_sltu', '_sleu', '_sgtu', '_sgeu', 'atomic', '_ld', '_st',
                 '_cas', '_add', '_and', '_exch', '_max', '_min', '_or',
                 '_sub', '_wrapdec', '_wrapinc', '_xor', 'ret', 'cvt',
                 '_readonly', '_kernarg', '_global', 'br', 'cbr', 'sbr',
                 '_scacq', '_screl', '_scar', '_rlx', '_wave', '_wg', '_agent',
                 '_system', 'ldimage', 'stimage', '_v2', '_v3', '_v4', '_1d',
                 '_2d', '_3d', '_1da', '_2da', '_1db', '_2ddepth', '_2dadepth',
                 '_width', '_height', '_depth', '_array', '_channelorder',
                 '_channeltype', 'querysampler', '_coord', '_filter',
                 '_addressing', 'barrier', 'wavebarrier', 'initfbar',
                 'joinfbar', 'waitfbar', 'arrivefbar', 'leavefbar',
                 'releasefbar', 'ldf', 'activelaneid', 'activelanecount',
                 'activelanemask', 'activelanepermute', 'call', 'scall',
                 'icall', 'alloca', 'packetcompletionsig',
                 'addqueuewriteindex', 'casqueuewriteindex',
                 'ldqueuereadindex', 'stqueuereadindex', 'readonly', 'global',
                 'private', 'group', 'spill', 'arg', '_upi', '_downi',
                 '_zeroi', '_neari', '_upi_sat', '_downi_sat', '_zeroi_sat',
                 '_neari_sat', '_supi', '_sdowni', '_szeroi', '_sneari',
                 '_supi_sat', '_sdowni_sat', '_szeroi_sat', '_sneari_sat',
                 '_pp', '_ps', '_sp', '_ss', '_s', '_p', '_pp_sat', '_ps_sat',
                 '_sp_sat', '_ss_sat', '_s_sat', '_p_sat')), Keyword),

            # Integer types
            (r'i[1-9]\d*', Keyword)
        ]
    }
예제 #27
0
    class GeneratedObjectiveCVariant(baselexer):
        """
        Implements Objective-C syntax on top of an existing C family lexer.
        """

        tokens = {
            'statements': [
                (r'@"', String, 'string'),
                (r'@(YES|NO)', Number),
                (r"@'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'",
                 String.Char),
                (r'@(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[lL]?', Number.Float),
                (r'@(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
                (r'@0x[0-9a-fA-F]+[Ll]?', Number.Hex),
                (r'@0[0-7]+[Ll]?', Number.Oct),
                (r'@\d+[Ll]?', Number.Integer),
                (r'@\(', Literal, 'literal_number'),
                (r'@\[', Literal, 'literal_array'),
                (r'@\{', Literal, 'literal_dictionary'),
                (words(
                    ('@selector', '@private', '@protected', '@public',
                     '@encode', '@synchronized', '@try', '@throw', '@catch',
                     '@finally', '@end', '@property', '@synthesize',
                     '__bridge', '__bridge_transfer', '__autoreleasing',
                     '__block', '__weak', '__strong', 'weak', 'strong', 'copy',
                     'retain', 'assign', 'unsafe_unretained', 'atomic',
                     'nonatomic', 'readonly', 'readwrite', 'setter', 'getter',
                     'typeof', 'in', 'out', 'inout', 'release', 'class',
                     '@dynamic', '@optional', '@required', '@autoreleasepool'),
                    suffix=r'\b'), Keyword),
                (words(('id', 'instancetype', 'Class', 'IMP', 'SEL', 'BOOL',
                        'IBOutlet', 'IBAction', 'unichar'),
                       suffix=r'\b'), Keyword.Type),
                (r'@(true|false|YES|NO)\n', Name.Builtin),
                (r'(YES|NO|nil|self|super)\b', Name.Builtin),
                # Carbon types
                (r'(Boolean|UInt8|SInt8|UInt16|SInt16|UInt32|SInt32)\b',
                 Keyword.Type),
                # Carbon built-ins
                (r'(TRUE|FALSE)\b', Name.Builtin),
                (r'(@interface|@implementation)(\s+)', bygroups(Keyword, Text),
                 ('#pop', 'oc_classname')),
                (r'(@class|@protocol)(\s+)', bygroups(Keyword, Text),
                 ('#pop', 'oc_forward_classname')),
                # @ can also prefix other expressions like @{...} or @(...)
                (r'@', Punctuation),
                inherit,
            ],
            'oc_classname': [
                # interface definition that inherits
                ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?(\s*)(\{)',
                 bygroups(Name.Class, Text, Name.Class, Text,
                          Punctuation), ('#pop', 'oc_ivars')),
                ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?',
                 bygroups(Name.Class, Text, Name.Class), '#pop'),
                # interface definition for a category
                ('([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))(\s*)(\{)',
                 bygroups(Name.Class, Text, Name.Label, Text,
                          Punctuation), ('#pop', 'oc_ivars')),
                ('([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))',
                 bygroups(Name.Class, Text, Name.Label), '#pop'),
                # simple interface / implementation
                ('([a-zA-Z$_][\w$]*)(\s*)(\{)',
                 bygroups(Name.Class, Text,
                          Punctuation), ('#pop', 'oc_ivars')),
                ('([a-zA-Z$_][\w$]*)', Name.Class, '#pop')
            ],
            'oc_forward_classname':
            [('([a-zA-Z$_][\w$]*)(\s*,\s*)', bygroups(Name.Class, Text),
              'oc_forward_classname'),
             ('([a-zA-Z$_][\w$]*)(\s*;?)', bygroups(Name.Class,
                                                    Text), '#pop')],
            'oc_ivars': [
                include('whitespace'),
                include('statements'),
                (';', Punctuation),
                (r'\{', Punctuation, '#push'),
                (r'\}', Punctuation, '#pop'),
            ],
            'root': [
                # methods
                (
                    r'^([-+])(\s*)'  # method marker
                    r'(\(.*?\))?(\s*)'  # return type
                    r'([a-zA-Z$_][\w$]*:?)',  # begin of method name
                    bygroups(Punctuation, Text, using(this), Text,
                             Name.Function),
                    'method'),
                inherit,
            ],
            'method': [
                include('whitespace'),
                # TODO unsure if ellipses are allowed elsewhere, see
                # discussion in Issue 789
                (r',', Punctuation),
                (r'\.\.\.', Punctuation),
                (r'(\(.*?\))(\s*)([a-zA-Z$_][\w$]*)',
                 bygroups(using(this), Text, Name.Variable)),
                (r'[a-zA-Z$_][\w$]*:', Name.Function),
                (';', Punctuation, '#pop'),
                (r'\{', Punctuation, 'function'),
                default('#pop'),
            ],
            'literal_number': [
                (r'\(', Punctuation, 'literal_number_inner'),
                (r'\)', Literal, '#pop'),
                include('statement'),
            ],
            'literal_number_inner': [
                (r'\(', Punctuation, '#push'),
                (r'\)', Punctuation, '#pop'),
                include('statement'),
            ],
            'literal_array': [
                (r'\[', Punctuation, 'literal_array_inner'),
                (r'\]', Literal, '#pop'),
                include('statement'),
            ],
            'literal_array_inner': [
                (r'\[', Punctuation, '#push'),
                (r'\]', Punctuation, '#pop'),
                include('statement'),
            ],
            'literal_dictionary': [
                (r'\}', Literal, '#pop'),
                include('statement'),
            ],
        }

        def analyse_text(text):
            if _oc_keywords.search(text):
                return 1.0
            elif '@"' in text:  # strings
                return 0.8
            elif re.search('@[0-9]+', text):
                return 0.7
            elif _oc_message.search(text):
                return 0.8
            return 0

        def get_tokens_unprocessed(self, text):
            from pygments.lexers._cocoa_builtins import COCOA_INTERFACES, \
                COCOA_PROTOCOLS, COCOA_PRIMITIVES

            for index, token, value in \
                    baselexer.get_tokens_unprocessed(self, text):
                if token is Name or token is Name.Class:
                    if value in COCOA_INTERFACES or value in COCOA_PROTOCOLS \
                       or value in COCOA_PRIMITIVES:
                        token = Name.Builtin.Pseudo

                yield index, token, value
예제 #28
0
파일: asm.py 프로젝트: Cold5nap/sobolIter
class LlvmMirBodyLexer(RegexLexer):
    """
    For LLVM MIR examples without the YAML wrapper.

    For more information on LLVM MIR see https://llvm.org/docs/MIRLangRef.html.

    .. versionadded:: 2.6
    """
    name = 'LLVM-MIR Body'
    aliases = ['llvm-mir-body']
    filenames = []
    mimetypes = []

    tokens = {
        'root': [
            # Attributes on basic blocks
            (words(('liveins', 'successors'), suffix=':'), Keyword),
            # Basic Block Labels
            (r'bb\.[0-9]+(\.[a-zA-Z0-9_.-]+)?( \(address-taken\))?:',
             Name.Label),
            (r'bb\.[0-9]+ \(%[a-zA-Z0-9_.-]+\)( \(address-taken\))?:',
             Name.Label),
            (r'%bb\.[0-9]+(\.\w+)?', Name.Label),
            # Stack references
            (r'%stack\.[0-9]+(\.\w+\.addr)?', Name),
            # Subreg indices
            (r'%subreg\.\w+', Name),
            # Virtual registers
            (r'%[a-zA-Z0-9_]+ *', Name.Variable, 'vreg'),
            # Reference to LLVM-IR global
            include('global'),
            # Reference to Intrinsic
            (r'intrinsic\(\@[a-zA-Z0-9_.]+\)', Name.Variable.Global),
            # Comparison predicates
            (words(('eq', 'ne', 'sgt', 'sge', 'slt', 'sle', 'ugt', 'uge',
                    'ult', 'ule'),
                   prefix=r'intpred\(',
                   suffix=r'\)'), Name.Builtin),
            (words(('oeq', 'one', 'ogt', 'oge', 'olt', 'ole', 'ugt', 'uge',
                    'ult', 'ule'),
                   prefix=r'floatpred\(',
                   suffix=r'\)'), Name.Builtin),
            # Physical registers
            (r'\$\w+', String.Single),
            # Assignment operator
            (r'=', Operator),
            # gMIR Opcodes
            (r'(G_ANYEXT|G_[SZ]EXT|G_SEXT_INREG|G_TRUNC|G_IMPLICIT_DEF|G_PHI|'
             r'G_FRAME_INDEX|G_GLOBAL_VALUE|G_INTTOPTR|G_PTRTOINT|G_BITCAST|'
             r'G_CONSTANT|G_FCONSTANT|G_VASTART|G_VAARG|G_CTLZ|G_CTLZ_ZERO_UNDEF|'
             r'G_CTTZ|G_CTTZ_ZERO_UNDEF|G_CTPOP|G_BSWAP|G_BITREVERSE|'
             r'G_ADDRSPACE_CAST|G_BLOCK_ADDR|G_JUMP_TABLE|G_DYN_STACKALLOC|'
             r'G_ADD|G_SUB|G_MUL|G_[SU]DIV|G_[SU]REM|G_AND|G_OR|G_XOR|G_SHL|'
             r'G_[LA]SHR|G_[IF]CMP|G_SELECT|G_GEP|G_PTR_MASK|G_SMIN|G_SMAX|'
             r'G_UMIN|G_UMAX|G_[US]ADDO|G_[US]ADDE|G_[US]SUBO|G_[US]SUBE|'
             r'G_[US]MULO|G_[US]MULH|G_FNEG|G_FPEXT|G_FPTRUNC|G_FPTO[US]I|'
             r'G_[US]ITOFP|G_FABS|G_FCOPYSIGN|G_FCANONICALIZE|G_FMINNUM|'
             r'G_FMAXNUM|G_FMINNUM_IEEE|G_FMAXNUM_IEEE|G_FMINIMUM|G_FMAXIMUM|'
             r'G_FADD|G_FSUB|G_FMUL|G_FMA|G_FMAD|G_FDIV|G_FREM|G_FPOW|G_FEXP|'
             r'G_FEXP2|G_FLOG|G_FLOG2|G_FLOG10|G_FCEIL|G_FCOS|G_FSIN|G_FSQRT|'
             r'G_FFLOOR|G_FRINT|G_FNEARBYINT|G_INTRINSIC_TRUNC|'
             r'G_INTRINSIC_ROUND|G_LOAD|G_[ZS]EXTLOAD|G_INDEXED_LOAD|'
             r'G_INDEXED_[ZS]EXTLOAD|G_STORE|G_INDEXED_STORE|'
             r'G_ATOMIC_CMPXCHG_WITH_SUCCESS|G_ATOMIC_CMPXCHG|'
             r'G_ATOMICRMW_(XCHG|ADD|SUB|AND|NAND|OR|XOR|MAX|MIN|UMAX|UMIN|FADD|'
             r'FSUB)'
             r'|G_FENCE|G_EXTRACT|G_UNMERGE_VALUES|G_INSERT|G_MERGE_VALUES|'
             r'G_BUILD_VECTOR|G_BUILD_VECTOR_TRUNC|G_CONCAT_VECTORS|'
             r'G_INTRINSIC|G_INTRINSIC_W_SIDE_EFFECTS|G_BR|G_BRCOND|'
             r'G_BRINDIRECT|G_BRJT|G_INSERT_VECTOR_ELT|G_EXTRACT_VECTOR_ELT|'
             r'G_SHUFFLE_VECTOR)\b', Name.Builtin),
            # Target independent opcodes
            (r'(COPY|PHI|INSERT_SUBREG|EXTRACT_SUBREG|REG_SEQUENCE)\b',
             Name.Builtin),
            # Flags
            (words(('killed', 'implicit')), Keyword),
            # ConstantInt values
            (r'i[0-9]+ +', Keyword.Type, 'constantint'),
            # ConstantFloat values
            (r'(half|float|double) +', Keyword.Type, 'constantfloat'),
            # Bare immediates
            include('integer'),
            # MMO's
            (r':: *', Operator, 'mmo'),
            # MIR Comments
            (r';.*', Comment),
            # If we get here, assume it's a target instruction
            (r'[a-zA-Z0-9_]+', Name),
            # Everything else that isn't highlighted
            (r'[(), \n]+', Text),
        ],
        # The integer constant from a ConstantInt value
        'constantint': [
            include('integer'),
            (r'(?=.)', Text, '#pop'),
        ],
        # The floating point constant from a ConstantFloat value
        'constantfloat': [
            include('float'),
            (r'(?=.)', Text, '#pop'),
        ],
        'vreg': [
            # The bank or class if there is one
            (r' *:(?!:)', Keyword, ('#pop', 'vreg_bank_or_class')),
            # The LLT if there is one
            (r' *\(', Text, 'vreg_type'),
            (r'(?=.)', Text, '#pop'),
        ],
        'vreg_bank_or_class': [
            # The unassigned bank/class
            (r' *_', Name.Variable.Magic),
            (r' *[a-zA-Z0-9_]+', Name.Variable),
            # The LLT if there is one
            (r' *\(', Text, 'vreg_type'),
            (r'(?=.)', Text, '#pop'),
        ],
        'vreg_type': [
            # Scalar and pointer types
            (r' *[sp][0-9]+', Keyword.Type),
            (r' *<[0-9]+ *x *[sp][0-9]+>', Keyword.Type),
            (r'\)', Text, '#pop'),
            (r'(?=.)', Text, '#pop'),
        ],
        'mmo': [
            (r'\(', Text),
            (r' +', Text),
            (words(
                ('load', 'store', 'on', 'into', 'from', 'align', 'monotonic',
                 'acquire', 'release', 'acq_rel', 'seq_cst')), Keyword),
            # IR references
            (r'%ir\.[a-zA-Z0-9_.-]+', Name),
            (r'%ir-block\.[a-zA-Z0-9_.-]+', Name),
            (r'[-+]', Operator),
            include('integer'),
            include('global'),
            (r',', Punctuation),
            (r'\), \(', Text),
            (r'\)', Text, '#pop'),
        ],
        'integer': [
            (r'-?[0-9]+', Number.Integer),
        ],
        'float': [(r'-?[0-9]+\.[0-9]+(e[+-][0-9]+)?', Number.Float)],
        'global': [(r'\@[a-zA-Z0-9_.]+', Name.Variable.Global)],
    }
예제 #29
0
class WhileyLexer(RegexLexer):
    """
    Lexer for the Whiley programming language.

    .. versionadded:: 2.2
    """
    name = 'Whiley'
    filenames = ['*.whiley']
    aliases = ['whiley']
    mimetypes = ['text/x-whiley']

    # See the language specification:
    # http://whiley.org/download/WhileyLanguageSpec.pdf

    tokens = {
        'root': [
            # Whitespace
            (r'\s+', Text),

            # Comments
            (r'//.*', Comment.Single),
            # don't parse empty comment as doc comment
            (r'/\*\*/', Comment.Multiline),
            (r'(?s)/\*\*.*?\*/', String.Doc),
            (r'(?s)/\*.*?\*/', Comment.Multiline),

            # Keywords
            (words(('if', 'else', 'while', 'for', 'do', 'return', 'switch',
                    'case', 'default', 'break', 'continue', 'requires',
                    'ensures', 'where', 'assert', 'assume', 'all', 'no',
                    'some', 'in', 'is', 'new', 'throw', 'try', 'catch',
                    'debug', 'skip', 'fail', 'finite', 'total'),
                   suffix=r'\b'), Keyword.Reserved),
            (words(('function', 'method', 'public', 'private', 'protected',
                    'export', 'native'),
                   suffix=r'\b'), Keyword.Declaration),
            # "constant" & "type" are not keywords unless used in declarations
            (r'(constant|type)(\s+)([a-zA-Z_]\w*)(\s+)(is)\b',
             bygroups(Keyword.Declaration, Text, Name, Text,
                      Keyword.Reserved)),
            (r'(true|false|null)\b', Keyword.Constant),
            (r'(bool|byte|int|real|any|void)\b', Keyword.Type),
            # "from" is not a keyword unless used with import
            (r'(import)(\s+)(\*)([^\S\n]+)(from)\b',
             bygroups(Keyword.Namespace, Text, Punctuation, Text,
                      Keyword.Namespace)),
            (r'(import)(\s+)([a-zA-Z_]\w*)([^\S\n]+)(from)\b',
             bygroups(Keyword.Namespace, Text, Name, Text, Keyword.Namespace)),
            (r'(package|import)\b', Keyword.Namespace),

            # standard library: https://github.com/Whiley/WhileyLibs/
            (
                words(
                    (
                        # types defined in whiley.lang.Int
                        'i8',
                        'i16',
                        'i32',
                        'i64',
                        'u8',
                        'u16',
                        'u32',
                        'u64',
                        'uint',
                        'nat',

                        # whiley.lang.Any
                        'toString'),
                    suffix=r'\b'),
                Name.Builtin),

            # byte literal
            (r'[01]+b', Number.Bin),

            # decimal literal
            (r'[0-9]+\.[0-9]+', Number.Float),
            # match "1." but not ranges like "3..5"
            (r'[0-9]+\.(?!\.)', Number.Float),

            # integer literal
            (r'0x[0-9a-fA-F]+', Number.Hex),
            (r'[0-9]+', Number.Integer),

            # character literal
            (r"""'[^\\]'""", String.Char),
            (r"""(')(\\['"\\btnfr])(')""",
             bygroups(String.Char, String.Escape, String.Char)),

            # string literal
            (r'"', String, 'string'),

            # operators and punctuation
            (r'[{}()\[\],.;]', Punctuation),
            (
                r'[+\-*/%&|<>^!~@=:?'
                # unicode operators
                r'\u2200\u2203\u2205\u2282\u2286\u2283\u2287'
                r'\u222A\u2229\u2264\u2265\u2208\u2227\u2228'
                r']',
                Operator),

            # identifier
            (r'[a-zA-Z_]\w*', Name),
        ],
        'string': [
            (r'"', String, '#pop'),
            (r'\\[btnfr]', String.Escape),
            (r'\\u[0-9a-fA-F]{4}', String.Escape),
            (r'\\.', String),
            (r'[^\\"]+', String),
        ],
    }
예제 #30
0
class CoqLexer(RegexLexer):
    """
    For the `Coq <http://coq.inria.fr/>`_ theorem prover.

    .. versionadded:: 1.5
    """

    name = 'Coq'
    aliases = ['coq']
    filenames = ['*.v']
    mimetypes = ['text/x-coq']

    keywords1 = (
        # Vernacular commands
        'Section',
        'Module',
        'End',
        'Require',
        'Import',
        'Export',
        'Variable',
        'Variables',
        'Parameter',
        'Parameters',
        'Axiom',
        'Hypothesis',
        'Hypotheses',
        'Notation',
        'Local',
        'Tactic',
        'Reserved',
        'Scope',
        'Open',
        'Close',
        'Bind',
        'Delimit',
        'Definition',
        'Let',
        'Ltac',
        'Fixpoint',
        'CoFixpoint',
        'Morphism',
        'Relation',
        'Implicit',
        'Arguments',
        'Set',
        'Unset',
        'Contextual',
        'Strict',
        'Prenex',
        'Implicits',
        'Inductive',
        'CoInductive',
        'Record',
        'Structure',
        'Canonical',
        'Coercion',
        'Theorem',
        'Lemma',
        'Corollary',
        'Proposition',
        'Fact',
        'Remark',
        'Example',
        'Proof',
        'Goal',
        'Save',
        'Qed',
        'Defined',
        'Hint',
        'Resolve',
        'Rewrite',
        'View',
        'Search',
        'Show',
        'Print',
        'Printing',
        'All',
        'Graph',
        'Projections',
        'inside',
        'outside',
        'Check',
        'Global',
        'Instance',
        'Class',
        'Existing',
        'Universe',
        'Polymorphic',
        'Monomorphic',
        'Context')
    keywords2 = (
        # Gallina
        'forall',
        'exists',
        'exists2',
        'fun',
        'fix',
        'cofix',
        'struct',
        'match',
        'end',
        'in',
        'return',
        'let',
        'if',
        'is',
        'then',
        'else',
        'for',
        'of',
        'nosimpl',
        'with',
        'as',
    )
    keywords3 = (
        # Sorts
        'Type',
        'Prop',
    )
    keywords4 = (
        # Tactics
        'pose',
        'set',
        'move',
        'case',
        'elim',
        'apply',
        'clear',
        'hnf',
        'intro',
        'intros',
        'generalize',
        'rename',
        'pattern',
        'after',
        'destruct',
        'induction',
        'using',
        'refine',
        'inversion',
        'injection',
        'rewrite',
        'congr',
        'unlock',
        'compute',
        'ring',
        'field',
        'replace',
        'fold',
        'unfold',
        'change',
        'cutrewrite',
        'simpl',
        'have',
        'suff',
        'wlog',
        'suffices',
        'without',
        'loss',
        'nat_norm',
        'assert',
        'cut',
        'trivial',
        'revert',
        'bool_congr',
        'nat_congr',
        'symmetry',
        'transitivity',
        'auto',
        'split',
        'left',
        'right',
        'autorewrite',
        'tauto',
        'setoid_rewrite',
        'intuition',
        'eauto',
        'eapply',
        'econstructor',
        'etransitivity',
        'constructor',
        'erewrite',
        'red',
        'cbv',
        'lazy',
        'vm_compute',
        'native_compute',
        'subst',
    )
    keywords5 = (
        # Terminators
        'by',
        'done',
        'exact',
        'reflexivity',
        'tauto',
        'romega',
        'omega',
        'assumption',
        'solve',
        'contradiction',
        'discriminate',
        'congruence',
    )
    keywords6 = (
        # Control
        'do',
        'last',
        'first',
        'try',
        'idtac',
        'repeat',
    )
    # 'as', 'assert', 'begin', 'class', 'constraint', 'do', 'done',
    # 'downto', 'else', 'end', 'exception', 'external', 'false',
    # 'for', 'fun', 'function', 'functor', 'if', 'in', 'include',
    # 'inherit', 'initializer', 'lazy', 'let', 'match', 'method',
    # 'module', 'mutable', 'new', 'object', 'of', 'open', 'private',
    # 'raise', 'rec', 'sig', 'struct', 'then', 'to', 'true', 'try',
    # 'type', 'val', 'virtual', 'when', 'while', 'with'
    keyopts = (
        '!=',
        '#',
        '&',
        '&&',
        r'\(',
        r'\)',
        r'\*',
        r'\+',
        ',',
        '-',
        r'-\.',
        '->',
        r'\.',
        r'\.\.',
        ':',
        '::',
        ':=',
        ':>',
        ';',
        ';;',
        '<',
        '<-',
        '<->',
        '=',
        '>',
        '>]',
        r'>\}',
        r'\?',
        r'\?\?',
        r'\[',
        r'\[<',
        r'\[>',
        r'\[\|',
        ']',
        '_',
        '`',
        r'\{',
        r'\{<',
        r'\|',
        r'\|]',
        r'\}',
        '~',
        '=>',
        r'/\\',
        r'\\/',
        r'\{\|',
        r'\|\}',
        u'Π',
        u'λ',
    )
    operators = r'[!$%&*+\./:<=>?@^|~-]'
    prefix_syms = r'[!?~]'
    infix_syms = r'[=<>@^|&+\*/$%-]'
    primitives = ('unit', 'nat', 'bool', 'string', 'ascii', 'list')

    tokens = {
        'root': [
            (r'\s+', Text),
            (r'false|true|\(\)|\[\]', Name.Builtin.Pseudo),
            (r'\(\*', Comment, 'comment'),
            (words(keywords1, prefix=r'\b', suffix=r'\b'), Keyword.Namespace),
            (words(keywords2, prefix=r'\b', suffix=r'\b'), Keyword),
            (words(keywords3, prefix=r'\b', suffix=r'\b'), Keyword.Type),
            (words(keywords4, prefix=r'\b', suffix=r'\b'), Keyword),
            (words(keywords5, prefix=r'\b', suffix=r'\b'), Keyword.Pseudo),
            (words(keywords6, prefix=r'\b', suffix=r'\b'), Keyword.Reserved),
            # (r'\b([A-Z][\w\']*)(\.)', Name.Namespace, 'dotted'),
            (r'\b([A-Z][\w\']*)', Name),
            (r'(%s)' % '|'.join(keyopts[::-1]), Operator),
            (r'(%s|%s)?%s' % (infix_syms, prefix_syms, operators), Operator),
            (r'\b(%s)\b' % '|'.join(primitives), Keyword.Type),
            (r"[^\W\d][\w']*", Name),
            (r'\d[\d_]*', Number.Integer),
            (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex),
            (r'0[oO][0-7][0-7_]*', Number.Oct),
            (r'0[bB][01][01_]*', Number.Bin),
            (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)', Number.Float),
            (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'",
             String.Char),
            (r"'.'", String.Char),
            (r"'", Keyword),  # a stray quote is another syntax element
            (r'"', String.Double, 'string'),
            (r'[~?][a-z][\w\']*:', Name),
        ],
        'comment': [
            (r'[^(*)]+', Comment),
            (r'\(\*', Comment, '#push'),
            (r'\*\)', Comment, '#pop'),
            (r'[(*)]', Comment),
        ],
        'string': [
            (r'[^"]+', String.Double),
            (r'""', String.Double),
            (r'"', String.Double, '#pop'),
        ],
        'dotted': [(r'\s+', Text), (r'\.', Punctuation),
                   (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace),
                   (r'[A-Z][\w\']*', Name.Class, '#pop'),
                   (r'[a-z][a-z0-9_\']*', Name, '#pop'),
                   default('#pop')],
    }

    def analyse_text(text):
        if text.startswith('(*'):
            return True
예제 #31
0
파일: gdscript.py 프로젝트: bruvzg/godot_tl
class GDScriptLexer(RegexLexer):
    """
    For `Godot source code <https://www.godotengine.org>`_ source code.
    """

    name = 'GDScript'
    aliases = ['gdscript', 'gd']
    filenames = ['*.gd']
    mimetypes = ['text/x-gdscript', 'application/x-gdscript']

    def innerstring_rules(ttype):
        return [
            # the old style '%s' % (...) string formatting
            (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
             '[hlL]?[E-GXc-giorsux%]', String.Interpol),
            # backslashes, quotes and formatting signs must be parsed one at a time
            (r'[^\\\'"%\n]+', ttype),
            (r'[\'"\\]', ttype),
            # unhandled string formatting sign
            (r'%', ttype),
            # newlines are an error (use "nl" state)
        ]

    tokens = {
        'root': [
            (r'\n', Text),
            (r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")',
             bygroups(Text, String.Affix, String.Doc)),
            (r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')",
             bygroups(Text, String.Affix, String.Doc)),
            (r'[^\S\n]+', Text),
            (r'#.*$', Comment.Single),
            (r'[]{}:(),;[]', Punctuation),
            (r'\\\n', Text),
            (r'\\', Text),
            (r'(in|and|or|not)\b', Operator.Word),
            (r'!=|==|<<|>>|&&|\+=|-=|\*=|/=|%=|&=|\|=|\|\||[-~+/*%=<>&^.!|$]',
             Operator),
            include('keywords'),
            (r'(func)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'funcname'),
            (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'classname'),
            include('builtins'),
            ('([rR]|[uUbB][rR]|[rR][uUbB])(""")',
             bygroups(String.Affix, String.Double), 'tdqs'),
            ("([rR]|[uUbB][rR]|[rR][uUbB])(''')",
             bygroups(String.Affix, String.Single), 'tsqs'),
            ('([rR]|[uUbB][rR]|[rR][uUbB])(")',
             bygroups(String.Affix, String.Double), 'dqs'),
            ("([rR]|[uUbB][rR]|[rR][uUbB])(')",
             bygroups(String.Affix, String.Single), 'sqs'),
            ('([uUbB]?)(""")', bygroups(String.Affix, String.Double),
             combined('stringescape', 'tdqs')),
            ("([uUbB]?)(''')", bygroups(String.Affix, String.Single),
             combined('stringescape', 'tsqs')),
            ('([uUbB]?)(")', bygroups(String.Affix, String.Double),
             combined('stringescape', 'dqs')),
            ("([uUbB]?)(')", bygroups(String.Affix, String.Single),
             combined('stringescape', 'sqs')),
            include('name'),
            include('numbers'),
        ],
        'keywords': [
            (words(('and', 'in', 'not', 'or', 'as', 'breakpoint', 'class',
                    'class_name', 'extends', 'is', 'func', 'setget', 'signal',
                    'tool', 'const', 'enum', 'export', 'onready', 'static',
                    'var', 'break', 'continue', 'if', 'elif', 'else', 'for',
                    'pass', 'return', 'match', 'while', 'remote', 'master',
                    'puppet', 'remotesync', 'mastersync', 'puppetsync'),
                   suffix=r'\b'), Keyword),
        ],
        'builtins': [
            (words(('Color8', 'ColorN', 'abs', 'acos', 'asin', 'assert',
                    'atan', 'atan2', 'bytes2var', 'ceil', 'char', 'clamp',
                    'convert', 'cos', 'cosh', 'db2linear', 'decimals',
                    'dectime', 'deg2rad', 'dict2inst', 'ease', 'exp', 'floor',
                    'fmod', 'fposmod', 'funcref', 'hash', 'inst2dict',
                    'instance_from_id', 'is_inf', 'is_nan', 'lerp',
                    'linear2db', 'load', 'log', 'max', 'min', 'nearest_po2',
                    'pow', 'preload', 'print', 'print_stack', 'printerr',
                    'printraw', 'prints', 'printt', 'rad2deg', 'rand_range',
                    'rand_seed', 'randf', 'randi', 'randomize', 'range',
                    'round', 'seed', 'sign', 'sin', 'sinh', 'sqrt', 'stepify',
                    'str', 'str2var', 'tan', 'tan', 'tanh', 'type_exist',
                    'typeof', 'var2bytes', 'var2str', 'weakref', 'yield'),
                   prefix=r'(?<!\.)',
                   suffix=r'\b'), Name.Builtin),
            (r'((?<!\.)(self|false|true)|(PI|TAU|NAN|INF)'
             r')\b', Name.Builtin.Pseudo),
            (words((
                'bool',
                'int',
                'float',
                'String',
                'NodePath'
                'Vector2',
                'Rect2',
                'Transform2D',
                'Vector3',
                'Rect3',
                'Plane',
                'Quat',
                'Basis',
                'Transform',
                'Color',
                "RID",
                'Object',
                'NodePath',
                'Dictionary',
                'Array',
                'PoolByteArray',
                'PoolIntArray',
                'PoolRealArray',
                'PoolStringArray',
                'PoolVector2Array',
                'PoolVector3Array',
                'PoolColorArray',
                'null',
            ),
                   prefix=r'(?<!\.)',
                   suffix=r'\b'), Name.Builtin.Type),
        ],
        'numbers': [(r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float),
                    (r'\d+[eE][+-]?[0-9]+j?', Number.Float),
                    (r'0[xX][a-fA-F0-9]+', Number.Hex),
                    (r'\d+j?', Number.Integer)],
        'name': [
            ('[a-zA-Z_]\w*', Name),
        ],
        'funcname': [
            ('[a-zA-Z_]\w*', Name.Function, '#pop'),
            default('#pop'),
        ],
        'classname': [('[a-zA-Z_]\w*', Name.Class, '#pop')],
        'stringescape':
        [(r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
          r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)],
        'strings-single':
        innerstring_rules(String.Single),
        'strings-double':
        innerstring_rules(String.Double),
        'dqs': [
            (r'"', String.Double, '#pop'),
            (r'\\\\|\\"|\\\n', String.Escape),  # included here for raw strings
            include('strings-double')
        ],
        'sqs': [
            (r"'", String.Single, '#pop'),
            (r"\\\\|\\'|\\\n", String.Escape),  # included here for raw strings
            include('strings-single')
        ],
        'tdqs': [(r'"""', String.Double, '#pop'),
                 include('strings-double'), (r'\n', String.Double)],
        'tsqs': [(r"'''", String.Single, '#pop'),
                 include('strings-single'), (r'\n', String.Single)],
    }
예제 #32
0
def keywords(*args):
    return words(args, prefix=r'\b', suffix=r'\b')
예제 #33
0
class SASLexer(RegexLexer):
    """
    For `SAS <http://www.sas.com/>`_ files.

    .. versionadded:: 2.2
    """
    # Syntax from syntax/sas.vim by James Kidd <*****@*****.**>

    name = 'SAS'
    aliases = ['sas']
    filenames = ['*.SAS', '*.sas']
    mimetypes = ['text/x-sas', 'text/sas', 'application/x-sas']
    flags = re.IGNORECASE | re.MULTILINE

    builtins_macros = ("bquote", "nrbquote", "cmpres", "qcmpres", "compstor",
                       "datatyp", "display", "do", "else", "end", "eval",
                       "global", "goto", "if", "index", "input", "keydef",
                       "label", "left", "length", "let", "local", "lowcase",
                       "macro", "mend", "nrquote", "nrstr", "put", "qleft",
                       "qlowcase", "qscan", "qsubstr", "qsysfunc", "qtrim",
                       "quote", "qupcase", "scan", "str", "substr", "superq",
                       "syscall", "sysevalf", "sysexec", "sysfunc", "sysget",
                       "syslput", "sysprod", "sysrc", "sysrput", "then", "to",
                       "trim", "unquote", "until", "upcase", "verify", "while",
                       "window")

    builtins_conditionals = ("do", "if", "then", "else", "end", "until",
                             "while")

    builtins_statements = (
        "abort", "array", "attrib", "by", "call", "cards", "cards4", "catname",
        "continue", "datalines", "datalines4", "delete", "delim", "delimiter",
        "display", "dm", "drop", "endsas", "error", "file", "filename",
        "footnote", "format", "goto", "in", "infile", "informat", "input",
        "keep", "label", "leave", "length", "libname", "link", "list",
        "lostcard", "merge", "missing", "modify", "options", "output", "out",
        "page", "put", "redirect", "remove", "rename", "replace", "retain",
        "return", "select", "set", "skip", "startsas", "stop", "title",
        "update", "waitsas", "where", "window", "x", "systask")

    builtins_sql = ("add", "and", "alter", "as", "cascade", "check", "create",
                    "delete", "describe", "distinct", "drop", "foreign",
                    "from", "group", "having", "index", "insert", "into", "in",
                    "key", "like", "message", "modify", "msgtype", "not",
                    "null", "on", "or", "order", "primary", "references",
                    "reset", "restrict", "select", "set", "table", "unique",
                    "update", "validate", "view", "where")

    builtins_functions = (
        "abs", "addr", "airy", "arcos", "arsin", "atan", "attrc", "attrn",
        "band", "betainv", "blshift", "bnot", "bor", "brshift", "bxor", "byte",
        "cdf", "ceil", "cexist", "cinv", "close", "cnonct", "collate",
        "compbl", "compound", "compress", "cos", "cosh", "css", "curobs", "cv",
        "daccdb", "daccdbsl", "daccsl", "daccsyd", "dacctab", "dairy", "date",
        "datejul", "datepart", "datetime", "day", "dclose", "depdb", "depdbsl",
        "depsl", "depsyd", "deptab", "dequote", "dhms", "dif", "digamma",
        "dim", "dinfo", "dnum", "dopen", "doptname", "doptnum", "dread",
        "dropnote", "dsname", "erf", "erfc", "exist", "exp", "fappend",
        "fclose", "fcol", "fdelete", "fetch", "fetchobs", "fexist", "fget",
        "fileexist", "filename", "fileref", "finfo", "finv", "fipname",
        "fipnamel", "fipstate", "floor", "fnonct", "fnote", "fopen",
        "foptname", "foptnum", "fpoint", "fpos", "fput", "fread", "frewind",
        "frlen", "fsep", "fuzz", "fwrite", "gaminv", "gamma", "getoption",
        "getvarc", "getvarn", "hbound", "hms", "hosthelp", "hour", "ibessel",
        "index", "indexc", "indexw", "input", "inputc", "inputn", "int",
        "intck", "intnx", "intrr", "irr", "jbessel", "juldate", "kurtosis",
        "lag", "lbound", "left", "length", "lgamma", "libname", "libref",
        "log", "log10", "log2", "logpdf", "logpmf", "logsdf", "lowcase", "max",
        "mdy", "mean", "min", "minute", "mod", "month", "mopen", "mort", "n",
        "netpv", "nmiss", "normal", "note", "npv", "open", "ordinal",
        "pathname", "pdf", "peek", "peekc", "pmf", "point", "poisson", "poke",
        "probbeta", "probbnml", "probchi", "probf", "probgam", "probhypr",
        "probit", "probnegb", "probnorm", "probt", "put", "putc", "putn",
        "qtr", "quote", "ranbin", "rancau", "ranexp", "rangam", "range",
        "rank", "rannor", "ranpoi", "rantbl", "rantri", "ranuni", "repeat",
        "resolve", "reverse", "rewind", "right", "round", "saving", "scan",
        "sdf", "second", "sign", "sin", "sinh", "skewness", "soundex",
        "spedis", "sqrt", "std", "stderr", "stfips", "stname", "stnamel",
        "substr", "sum", "symget", "sysget", "sysmsg", "sysprod", "sysrc",
        "system", "tan", "tanh", "time", "timepart", "tinv", "tnonct", "today",
        "translate", "tranwrd", "trigamma", "trim", "trimn", "trunc",
        "uniform", "upcase", "uss", "var", "varfmt", "varinfmt", "varlabel",
        "varlen", "varname", "varnum", "varray", "varrayx", "vartype",
        "verify", "vformat", "vformatd", "vformatdx", "vformatn", "vformatnx",
        "vformatw", "vformatwx", "vformatx", "vinarray", "vinarrayx",
        "vinformat", "vinformatd", "vinformatdx", "vinformatn", "vinformatnx",
        "vinformatw", "vinformatwx", "vinformatx", "vlabel", "vlabelx",
        "vlength", "vlengthx", "vname", "vnamex", "vtype", "vtypex", "weekday",
        "year", "yyq", "zipfips", "zipname", "zipnamel", "zipstate")

    tokens = {
        'root': [
            include('comments'),
            include('proc-data'),
            include('cards-datalines'),
            include('logs'),
            include('general'),
            (r'.', Text),
        ],
        # SAS is multi-line regardless, but * is ended by ;
        'comments': [
            (r'^\s*\*.*?;', Comment),
            (r'/\*.*?\*/', Comment),
            (r'^\s*\*(.|\n)*?;', Comment.Multiline),
            (r'/[*](.|\n)*?[*]/', Comment.Multiline),
        ],
        # Special highlight for proc, data, quit, run
        'proc-data': [
            (r'(^|;)\s*(proc \w+|data|run|quit)[\s;]', Keyword.Reserved),
        ],
        # Special highlight cards and datalines
        'cards-datalines': [
            (r'^\s*(datalines|cards)\s*;\s*$', Keyword, 'data'),
        ],
        'data': [
            (r'(.|\n)*^\s*;\s*$', Other, '#pop'),
        ],
        # Special highlight for put NOTE|ERROR|WARNING (order matters)
        'logs': [
            (r'\n?^\s*%?put ', Keyword, 'log-messages'),
        ],
        'log-messages': [
            (r'NOTE(:|-).*', Generic, '#pop'),
            (r'WARNING(:|-).*', Generic.Emph, '#pop'),
            (r'ERROR(:|-).*', Generic.Error, '#pop'),
            include('general'),
        ],
        'general': [
            include('keywords'),
            include('vars-strings'),
            include('special'),
            include('numbers'),
        ],
        # Keywords, statements, functions, macros
        'keywords': [
            (words(builtins_statements, prefix=r'\b', suffix=r'\b'), Keyword),
            (words(builtins_sql, prefix=r'\b', suffix=r'\b'), Keyword),
            (words(builtins_conditionals, prefix=r'\b',
                   suffix=r'\b'), Keyword),
            (words(builtins_macros, prefix=r'%', suffix=r'\b'), Name.Builtin),
            (words(builtins_functions, prefix=r'\b',
                   suffix=r'\('), Name.Builtin),
        ],
        # Strings and user-defined variables and macros (order matters)
        'vars-strings': [
            (r'&[a-z_]\w{0,31}\.?', Name.Variable),
            (r'%[a-z_]\w{0,31}', Name.Function),
            (r'\'', String, 'string_squote'),
            (r'"', String, 'string_dquote'),
        ],
        'string_squote': [
            ('\'', String, '#pop'),
            (r'\\\\|\\"|\\\n', String.Escape),
            # AFAIK, macro variables are not evaluated in single quotes
            # (r'&', Name.Variable, 'validvar'),
            (r'[^$\'\\]+', String),
            (r'[$\'\\]', String),
        ],
        'string_dquote': [
            (r'"', String, '#pop'),
            (r'\\\\|\\"|\\\n', String.Escape),
            (r'&', Name.Variable, 'validvar'),
            (r'[^$&"\\]+', String),
            (r'[$"\\]', String),
        ],
        'validvar': [
            (r'[a-z_]\w{0,31}\.?', Name.Variable, '#pop'),
        ],
        # SAS numbers and special variables
        'numbers': [
            (r'\b[+-]?([0-9]+(\.[0-9]+)?|\.[0-9]+|\.)(E[+-]?[0-9]+)?i?\b',
             Number),
        ],
        'special': [
            (r'(null|missing|_all_|_automatic_|_character_|_n_|'
             r'_infile_|_name_|_null_|_numeric_|_user_|_webout_)',
             Keyword.Constant),
        ],
        # 'operators': [
        #     (r'(-|=|<=|>=|<|>|<>|&|!=|'
        #      r'\||\*|\+|\^|/|!|~|~=)', Operator)
        # ],
    }
예제 #34
0
파일: shell.py 프로젝트: LihMeh/outwiker
 def _make_begin_state(compound, _core_token=_core_token,
                       _core_token_compound=_core_token_compound,
                       _keyword_terminator=_keyword_terminator,
                       _nl=_nl, _punct=_punct, _string=_string,
                       _space=_space, _start_label=_start_label,
                       _stoken=_stoken, _token_terminator=_token_terminator,
                       _variable=_variable, _ws=_ws):
     rest = '(?:%s|%s|[^"%%%s%s%s])*' % (_string, _variable, _nl, _punct,
                                         ')' if compound else '')
     rest_of_line = r'(?:(?:[^%s^]|\^[%s]?[\w\W])*)' % (_nl, _nl)
     rest_of_line_compound = r'(?:(?:[^%s^)]|\^[%s]?[^)])*)' % (_nl, _nl)
     set_space = r'((?:(?:\^[%s]?)?[^\S\n])*)' % _nl
     suffix = ''
     if compound:
         _keyword_terminator = r'(?:(?=\))|%s)' % _keyword_terminator
         _token_terminator = r'(?:(?=\))|%s)' % _token_terminator
         suffix = '/compound'
     return [
         ((r'\)', Punctuation, '#pop') if compound else
          (r'\)((?=\()|%s)%s' % (_token_terminator, rest_of_line),
           Comment.Single)),
         (r'(?=%s)' % _start_label, Text, 'follow%s' % suffix),
         (_space, using(this, state='text')),
         include('redirect%s' % suffix),
         (r'[%s]+' % _nl, Text),
         (r'\(', Punctuation, 'root/compound'),
         (r'@+', Punctuation),
         (r'((?:for|if|rem)(?:(?=(?:\^[%s]?)?/)|(?:(?!\^)|'
          r'(?<=m))(?:(?=\()|%s)))(%s?%s?(?:\^[%s]?)?/(?:\^[%s]?)?\?)' %
          (_nl, _token_terminator, _space,
           _core_token_compound if compound else _core_token, _nl, _nl),
          bygroups(Keyword, using(this, state='text')),
          'follow%s' % suffix),
         (r'(goto%s)(%s(?:\^[%s]?)?/(?:\^[%s]?)?\?%s)' %
          (_keyword_terminator, rest, _nl, _nl, rest),
          bygroups(Keyword, using(this, state='text')),
          'follow%s' % suffix),
         (words(('assoc', 'break', 'cd', 'chdir', 'cls', 'color', 'copy',
                 'date', 'del', 'dir', 'dpath', 'echo', 'endlocal', 'erase',
                 'exit', 'ftype', 'keys', 'md', 'mkdir', 'mklink', 'move',
                 'path', 'pause', 'popd', 'prompt', 'pushd', 'rd', 'ren',
                 'rename', 'rmdir', 'setlocal', 'shift', 'start', 'time',
                 'title', 'type', 'ver', 'verify', 'vol'),
                suffix=_keyword_terminator), Keyword, 'follow%s' % suffix),
         (r'(call)(%s?)(:)' % _space,
          bygroups(Keyword, using(this, state='text'), Punctuation),
          'call%s' % suffix),
         (r'call%s' % _keyword_terminator, Keyword),
         (r'(for%s(?!\^))(%s)(/f%s)' %
          (_token_terminator, _space, _token_terminator),
          bygroups(Keyword, using(this, state='text'), Keyword),
          ('for/f', 'for')),
         (r'(for%s(?!\^))(%s)(/l%s)' %
          (_token_terminator, _space, _token_terminator),
          bygroups(Keyword, using(this, state='text'), Keyword),
          ('for/l', 'for')),
         (r'for%s(?!\^)' % _token_terminator, Keyword, ('for2', 'for')),
         (r'(goto%s)(%s?)(:?)' % (_keyword_terminator, _space),
          bygroups(Keyword, using(this, state='text'), Punctuation),
          'label%s' % suffix),
         (r'(if(?:(?=\()|%s)(?!\^))(%s?)((?:/i%s)?)(%s?)((?:not%s)?)(%s?)' %
          (_token_terminator, _space, _token_terminator, _space,
           _token_terminator, _space),
          bygroups(Keyword, using(this, state='text'), Keyword,
                   using(this, state='text'), Keyword,
                   using(this, state='text')), ('(?', 'if')),
         (r'rem(((?=\()|%s)%s?%s?.*|%s%s)' %
          (_token_terminator, _space, _stoken, _keyword_terminator,
           rest_of_line_compound if compound else rest_of_line),
          Comment.Single, 'follow%s' % suffix),
         (r'(set%s)%s(/a)' % (_keyword_terminator, set_space),
          bygroups(Keyword, using(this, state='text'), Keyword),
          'arithmetic%s' % suffix),
         (r'(set%s)%s((?:/p)?)%s((?:(?:(?:\^[%s]?)?[^"%s%s^=%s]|'
          r'\^[%s]?[^"=])+)?)((?:(?:\^[%s]?)?=)?)' %
          (_keyword_terminator, set_space, set_space, _nl, _nl, _punct,
           ')' if compound else '', _nl, _nl),
          bygroups(Keyword, using(this, state='text'), Keyword,
                   using(this, state='text'), using(this, state='variable'),
                   Punctuation),
          'follow%s' % suffix),
         default('follow%s' % suffix)
     ]
예제 #35
0
파일: erlang.py 프로젝트: 0x008800/Sandbox
class ErlangLexer(RegexLexer):
    """
    For the Erlang functional programming language.

    Blame Jeremy Thurgood (http://jerith.za.net/).

    .. versionadded:: 0.9
    """

    name = 'Erlang'
    aliases = ['erlang']
    filenames = ['*.erl', '*.hrl', '*.es', '*.escript']
    mimetypes = ['text/x-erlang']

    keywords = (
        'after',
        'begin',
        'case',
        'catch',
        'cond',
        'end',
        'fun',
        'if',
        'let',
        'of',
        'query',
        'receive',
        'try',
        'when',
    )

    builtins = (  # See erlang(3) man page
        'abs', 'append_element', 'apply', 'atom_to_list', 'binary_to_list',
        'bitstring_to_list', 'binary_to_term', 'bit_size', 'bump_reductions',
        'byte_size', 'cancel_timer', 'check_process_code', 'delete_module',
        'demonitor', 'disconnect_node', 'display', 'element', 'erase', 'exit',
        'float', 'float_to_list', 'fun_info', 'fun_to_list',
        'function_exported', 'garbage_collect', 'get', 'get_keys',
        'group_leader', 'hash', 'hd', 'integer_to_list', 'iolist_to_binary',
        'iolist_size', 'is_atom', 'is_binary', 'is_bitstring', 'is_boolean',
        'is_builtin', 'is_float', 'is_function', 'is_integer', 'is_list',
        'is_number', 'is_pid', 'is_port', 'is_process_alive', 'is_record',
        'is_reference', 'is_tuple', 'length', 'link', 'list_to_atom',
        'list_to_binary', 'list_to_bitstring', 'list_to_existing_atom',
        'list_to_float', 'list_to_integer', 'list_to_pid', 'list_to_tuple',
        'load_module', 'localtime_to_universaltime', 'make_tuple', 'md5',
        'md5_final', 'md5_update', 'memory', 'module_loaded', 'monitor',
        'monitor_node', 'node', 'nodes', 'open_port', 'phash', 'phash2',
        'pid_to_list', 'port_close', 'port_command', 'port_connect',
        'port_control', 'port_call', 'port_info', 'port_to_list',
        'process_display', 'process_flag', 'process_info', 'purge_module',
        'put', 'read_timer', 'ref_to_list', 'register', 'resume_process',
        'round', 'send', 'send_after', 'send_nosuspend', 'set_cookie',
        'setelement', 'size', 'spawn', 'spawn_link', 'spawn_monitor',
        'spawn_opt', 'split_binary', 'start_timer', 'statistics',
        'suspend_process', 'system_flag', 'system_info', 'system_monitor',
        'system_profile', 'term_to_binary', 'tl', 'trace', 'trace_delivered',
        'trace_info', 'trace_pattern', 'trunc', 'tuple_size', 'tuple_to_list',
        'universaltime_to_localtime', 'unlink', 'unregister', 'whereis')

    operators = r'(\+\+?|--?|\*|/|<|>|/=|=:=|=/=|=<|>=|==?|<-|!|\?)'
    word_operators = ('and', 'andalso', 'band', 'bnot', 'bor', 'bsl', 'bsr',
                      'bxor', 'div', 'not', 'or', 'orelse', 'rem', 'xor')

    atom_re = r"(?:[a-z]\w*|'[^\n']*[^\\]')"

    variable_re = r'(?:[A-Z_]\w*)'

    esc_char_re = r'[bdefnrstv\'"\\]'
    esc_octal_re = r'[0-7][0-7]?[0-7]?'
    esc_hex_re = r'(?:x[0-9a-fA-F]{2}|x\{[0-9a-fA-F]+\})'
    esc_ctrl_re = r'\^[a-zA-Z]'
    escape_re = r'(?:\\(?:' + esc_char_re + r'|' + esc_octal_re + r'|' + esc_hex_re + r'|' + esc_ctrl_re + r'))'

    macro_re = r'(?:' + variable_re + r'|' + atom_re + r')'

    base_re = r'(?:[2-9]|[12][0-9]|3[0-6])'

    tokens = {
        'root': [
            (r'\s+', Text),
            (r'%.*\n', Comment),
            (words(keywords, suffix=r'\b'), Keyword),
            (words(builtins, suffix=r'\b'), Name.Builtin),
            (words(word_operators, suffix=r'\b'), Operator.Word),
            (r'^-', Punctuation, 'directive'),
            (operators, Operator),
            (r'"', String, 'string'),
            (r'<<', Name.Label),
            (r'>>', Name.Label),
            ('(' + atom_re + ')(:)', bygroups(Name.Namespace, Punctuation)),
            ('(?:^|(?<=:))(' + atom_re + r')(\s*)(\()',
             bygroups(Name.Function, Text, Punctuation)),
            (r'[+-]?' + base_re + r'#[0-9a-zA-Z]+', Number.Integer),
            (r'[+-]?\d+', Number.Integer),
            (r'[+-]?\d+.\d+', Number.Float),
            (r'[]\[:_@\".{}()|;,]', Punctuation),
            (variable_re, Name.Variable),
            (atom_re, Name),
            (r'\?' + macro_re, Name.Constant),
            (r'\$(?:' + escape_re + r'|\\[ %]|[^\\])', String.Char),
            (r'#' + atom_re + r'(:?\.' + atom_re + r')?', Name.Label),

            # Erlang script shebang
            (r'\A#!.+\n', Comment.Hashbang),

            # EEP 43: Maps
            # http://www.erlang.org/eeps/eep-0043.html
            (r'#\{', Punctuation, 'map_key'),
        ],
        'string': [
            (escape_re, String.Escape),
            (r'"', String, '#pop'),
            (r'~[0-9.*]*[~#+BPWXb-ginpswx]', String.Interpol),
            (r'[^"\\~]+', String),
            (r'~', String),
        ],
        'directive': [
            (r'(define)(\s*)(\()(' + macro_re + r')',
             bygroups(Name.Entity, Text, Punctuation, Name.Constant), '#pop'),
            (r'(record)(\s*)(\()(' + macro_re + r')',
             bygroups(Name.Entity, Text, Punctuation, Name.Label), '#pop'),
            (atom_re, Name.Entity, '#pop'),
        ],
        'map_key': [
            include('root'),
            (r'=>', Punctuation, 'map_val'),
            (r':=', Punctuation, 'map_val'),
            (r'\}', Punctuation, '#pop'),
        ],
        'map_val': [
            include('root'),
            (r',', Punctuation, '#pop'),
            (r'(?=\})', Punctuation, '#pop'),
        ],
    }
예제 #36
0
파일: ruby.py 프로젝트: bkerler/PythonQt
    def gen_rubystrings_rules():
        def intp_regex_callback(self, match, ctx):
            yield match.start(1), String.Regex, match.group(1)  # begin
            nctx = LexerContext(match.group(3), 0, ['interpolated-regex'])
            for i, t, v in self.get_tokens_unprocessed(context=nctx):
                yield match.start(3)+i, t, v
            yield match.start(4), String.Regex, match.group(4)  # end[mixounse]*
            ctx.pos = match.end()

        def intp_string_callback(self, match, ctx):
            yield match.start(1), String.Other, match.group(1)
            nctx = LexerContext(match.group(3), 0, ['interpolated-string'])
            for i, t, v in self.get_tokens_unprocessed(context=nctx):
                yield match.start(3)+i, t, v
            yield match.start(4), String.Other, match.group(4)  # end
            ctx.pos = match.end()

        states = {}
        states['strings'] = [
            # easy ones
            (r'\:@{0,2}[a-zA-Z_]\w*[\!\?]?', String.Symbol),
            (words(RUBY_OPERATORS, prefix=r'\:@{0,2}'), String.Symbol),
            (r":'(\\\\|\\'|[^'])*'", String.Symbol),
            (r"'(\\\\|\\'|[^'])*'", String.Single),
            (r':"', String.Symbol, 'simple-sym'),
            (r'([a-zA-Z_]\w*)(:)(?!:)',
             bygroups(String.Symbol, Punctuation)),  # Since Ruby 1.9
            (r'"', String.Double, 'simple-string'),
            (r'(?<!\.)`', String.Backtick, 'simple-backtick'),
        ]

        # double-quoted string and symbol
        for name, ttype, end in ('string', String.Double, '"'), \
                                ('sym', String.Symbol, '"'), \
                                ('backtick', String.Backtick, '`'):
            states['simple-'+name] = [
                include('string-intp-escaped'),
                (r'[^\\%s#]+' % end, ttype),
                (r'[\\#]', ttype),
                (end, ttype, '#pop'),
            ]

        # braced quoted strings
        for lbrace, rbrace, name in ('\\{', '\\}', 'cb'), \
                                    ('\\[', '\\]', 'sb'), \
                                    ('\\(', '\\)', 'pa'), \
                                    ('<', '>', 'ab'):
            states[name+'-intp-string'] = [
                (r'\\[\\' + lbrace + rbrace + ']', String.Other),
                (lbrace, String.Other, '#push'),
                (rbrace, String.Other, '#pop'),
                include('string-intp-escaped'),
                (r'[\\#' + lbrace + rbrace + ']', String.Other),
                (r'[^\\#' + lbrace + rbrace + ']+', String.Other),
            ]
            states['strings'].append((r'%[QWx]?' + lbrace, String.Other,
                                      name+'-intp-string'))
            states[name+'-string'] = [
                (r'\\[\\' + lbrace + rbrace + ']', String.Other),
                (lbrace, String.Other, '#push'),
                (rbrace, String.Other, '#pop'),
                (r'[\\#' + lbrace + rbrace + ']', String.Other),
                (r'[^\\#' + lbrace + rbrace + ']+', String.Other),
            ]
            states['strings'].append((r'%[qsw]' + lbrace, String.Other,
                                      name+'-string'))
            states[name+'-regex'] = [
                (r'\\[\\' + lbrace + rbrace + ']', String.Regex),
                (lbrace, String.Regex, '#push'),
                (rbrace + '[mixounse]*', String.Regex, '#pop'),
                include('string-intp'),
                (r'[\\#' + lbrace + rbrace + ']', String.Regex),
                (r'[^\\#' + lbrace + rbrace + ']+', String.Regex),
            ]
            states['strings'].append((r'%r' + lbrace, String.Regex,
                                      name+'-regex'))

        # these must come after %<brace>!
        states['strings'] += [
            # %r regex
            (r'(%r([^a-zA-Z0-9]))((?:\\\2|(?!\2).)*)(\2[mixounse]*)',
             intp_regex_callback),
            # regular fancy strings with qsw
            (r'%[qsw]([^a-zA-Z0-9])((?:\\\1|(?!\1).)*)\1', String.Other),
            (r'(%[QWx]([^a-zA-Z0-9]))((?:\\\2|(?!\2).)*)(\2)',
             intp_string_callback),
            # special forms of fancy strings after operators or
            # in method calls with braces
            (r'(?<=[-+/*%=<>&!^|~,(])(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
             bygroups(Text, String.Other, None)),
            # and because of fixed width lookbehinds the whole thing a
            # second time for line startings...
            (r'^(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
             bygroups(Text, String.Other, None)),
            # all regular fancy strings without qsw
            (r'(%([^a-zA-Z0-9\s]))((?:\\\2|(?!\2).)*)(\2)',
             intp_string_callback),
        ]

        return states
예제 #37
0
class SolidityLexer(RegexLexer):
    """
    For Solidity source code.

    .. versionadded:: 2.5
    """

    name = 'Solidity'
    aliases = ['solidity']
    filenames = ['*.sol']
    mimetypes = []

    flags = re.MULTILINE | re.UNICODE

    datatype = (
        r'\b(address|bool|(?:(?:bytes|hash|int|string|uint)(?:8|16|24|32|40|48|56|64'
        r'|72|80|88|96|104|112|120|128|136|144|152|160|168|176|184|192|200|208'
        r'|216|224|232|240|248|256)?))\b')

    tokens = {
        'root': [
            include('whitespace'),
            include('comments'), (r'\bpragma\s+solidity\b', Keyword, 'pragma'),
            (r'\b(contract)(\s+)([a-zA-Z_]\w*)',
             bygroups(Keyword, Whitespace, Name.Entity)),
            (datatype + r'(\s+)((?:external|public|internal|private)\s+)?' +
             r'([a-zA-Z_]\w*)',
             bygroups(Keyword.Type, Whitespace, Keyword, Name.Variable)),
            (r'\b(enum|event|function|struct)(\s+)([a-zA-Z_]\w*)',
             bygroups(Keyword.Type, Whitespace, Name.Variable)),
            (r'\b(msg|block|tx)\.([A-Za-z_][a-zA-Z0-9_]*)\b', Keyword),
            (words(('block', 'break', 'constant', 'constructor', 'continue',
                    'contract', 'do', 'else', 'external', 'false', 'for',
                    'function', 'if', 'import', 'inherited', 'internal', 'is',
                    'library', 'mapping', 'memory', 'modifier', 'msg', 'new',
                    'payable', 'private', 'public', 'require', 'return',
                    'returns', 'struct', 'suicide', 'throw', 'this', 'true',
                    'tx', 'var', 'while'),
                   prefix=r'\b',
                   suffix=r'\b'), Keyword.Type),
            (words(('keccak256', ), prefix=r'\b', suffix=r'\b'), Name.Builtin),
            (datatype, Keyword.Type),
            include('constants'), (r'[a-zA-Z_]\w*', Text),
            (r'[!<=>+*/-]', Operator), (r'[.;:{}(),\[\]]', Punctuation)
        ],
        'comments': [(r'//(\n|[\w\W]*?[^\\]\n)', Comment.Single),
                     (r'/(\\\n)?[*][\w\W]*?[*](\\\n)?/', Comment.Multiline),
                     (r'/(\\\n)?[*][\w\W]*', Comment.Multiline)],
        'constants': [
            (r'("(\\"|.)*?")', String.Double),
            (r"('(\\'|.)*?')", String.Single),
            (r'\b0[xX][0-9a-fA-F]+\b', Number.Hex),
            (r'\b\d+\b', Number.Decimal),
        ],
        'pragma': [
            include('whitespace'),
            include('comments'),
            (r'(\^|>=|<)(\s*)(\d+\.\d+\.\d+)',
             bygroups(Operator, Whitespace, Keyword)),
            (r';', Punctuation, '#pop')
        ],
        'whitespace': [(r'\s+', Whitespace), (r'\n', Whitespace)]
    }
예제 #38
0
파일: crystal.py 프로젝트: tmm1/pygments.rb
    def gen_crystalstrings_rules():
        def intp_regex_callback(self, match, ctx):
            yield match.start(1), String.Regex, match.group(1)  # begin
            nctx = LexerContext(match.group(3), 0, ["interpolated-regex"])
            for i, t, v in self.get_tokens_unprocessed(context=nctx):
                yield match.start(3) + i, t, v
            yield match.start(4), String.Regex, match.group(4)  # end[imsx]*
            ctx.pos = match.end()

        def intp_string_callback(self, match, ctx):
            yield match.start(1), String.Other, match.group(1)
            nctx = LexerContext(match.group(3), 0, ["interpolated-string"])
            for i, t, v in self.get_tokens_unprocessed(context=nctx):
                yield match.start(3) + i, t, v
            yield match.start(4), String.Other, match.group(4)  # end
            ctx.pos = match.end()

        states = {}
        states["strings"] = [
            (r"\:@{0,2}[a-zA-Z_]\w*[!?]?", String.Symbol),
            (words(CRYSTAL_OPERATORS, prefix=r"\:@{0,2}"), String.Symbol),
            (r":'(\\\\|\\'|[^'])*'", String.Symbol),
            # This allows arbitrary text after '\ for simplicity
            (r"'(\\\\|\\'|[^']|\\[^'\\]+)'", String.Char),
            (r':"', String.Symbol, "simple-sym"),
            # Crystal doesn't have "symbol:"s but this simplifies function args
            (r"([a-zA-Z_]\w*)(:)(?!:)", bygroups(String.Symbol, Punctuation)),
            (r'"', String.Double, "simple-string"),
            (r"(?<!\.)`", String.Backtick, "simple-backtick"),
        ]

        # double-quoted string and symbol
        for name, ttype, end in (
            ("string", String.Double, '"'),
            ("sym", String.Symbol, '"'),
            ("backtick", String.Backtick, "`"),
        ):
            states["simple-" + name] = [
                include("string-escaped" if name == "sym" else "string-intp-escaped"),
                (r"[^\\%s#]+" % end, ttype),
                (r"[\\#]", ttype),
                (end, ttype, "#pop"),
            ]

        # braced quoted strings
        for lbrace, rbrace, bracecc, name in (
            ("\\{", "\\}", "{}", "cb"),
            ("\\[", "\\]", "\\[\\]", "sb"),
            ("\\(", "\\)", "()", "pa"),
            ("<", ">", "<>", "ab"),
        ):
            states[name + "-intp-string"] = [
                (r"\\[" + lbrace + "]", String.Other),
                (lbrace, String.Other, "#push"),
                (rbrace, String.Other, "#pop"),
                include("string-intp-escaped"),
                (r"[\\#" + bracecc + "]", String.Other),
                (r"[^\\#" + bracecc + "]+", String.Other),
            ]
            states["strings"].append((r"%" + lbrace, String.Other, name + "-intp-string"))
            states[name + "-string"] = [
                (r"\\[\\" + bracecc + "]", String.Other),
                (lbrace, String.Other, "#push"),
                (rbrace, String.Other, "#pop"),
                (r"[\\#" + bracecc + "]", String.Other),
                (r"[^\\#" + bracecc + "]+", String.Other),
            ]
            # http://crystal-lang.org/docs/syntax_and_semantics/literals/array.html
            states["strings"].append((r"%[wi]" + lbrace, String.Other, name + "-string"))
            states[name + "-regex"] = [
                (r"\\[\\" + bracecc + "]", String.Regex),
                (lbrace, String.Regex, "#push"),
                (rbrace + "[imsx]*", String.Regex, "#pop"),
                include("string-intp"),
                (r"[\\#" + bracecc + "]", String.Regex),
                (r"[^\\#" + bracecc + "]+", String.Regex),
            ]
            states["strings"].append((r"%r" + lbrace, String.Regex, name + "-regex"))

        # these must come after %<brace>!
        states["strings"] += [
            # %r regex
            (r"(%r([\W_]))((?:\\\2|(?!\2).)*)(\2[imsx]*)", intp_regex_callback),
            # regular fancy strings with qsw
            (r"(%[wi]([\W_]))((?:\\\2|(?!\2).)*)(\2)", intp_string_callback),
            # special forms of fancy strings after operators or
            # in method calls with braces
            (r"(?<=[-+/*%=<>&!^|~,(])(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)", bygroups(Text, String.Other, None)),
            # and because of fixed width lookbehinds the whole thing a
            # second time for line startings...
            (r"^(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)", bygroups(Text, String.Other, None)),
            # all regular fancy strings without qsw
            (r"(%([\[{(<]))((?:\\\2|(?!\2).)*)(\2)", intp_string_callback),
        ]

        return states
예제 #39
0
파일: css.py 프로젝트: NMLOR10N/termsaver
 words(
     (
         "azimuth",
         "background-attachment",
         "background-color",
         "background-image",
         "background-position",
         "background-repeat",
         "background",
         "border-bottom-color",
         "border-bottom-style",
         "border-bottom-width",
         "border-left-color",
         "border-left-style",
         "border-left-width",
         "border-right",
         "border-right-color",
         "border-right-style",
         "border-right-width",
         "border-top-color",
         "border-top-style",
         "border-top-width",
         "border-bottom",
         "border-collapse",
         "border-left",
         "border-width",
         "border-color",
         "border-spacing",
         "border-style",
         "border-top",
         "border",
         "caption-side",
         "clear",
         "clip",
         "color",
         "content",
         "counter-increment",
         "counter-reset",
         "cue-after",
         "cue-before",
         "cue",
         "cursor",
         "direction",
         "display",
         "elevation",
         "empty-cells",
         "float",
         "font-family",
         "font-size",
         "font-size-adjust",
         "font-stretch",
         "font-style",
         "font-variant",
         "font-weight",
         "font",
         "height",
         "letter-spacing",
         "line-height",
         "list-style-type",
         "list-style-image",
         "list-style-position",
         "list-style",
         "margin-bottom",
         "margin-left",
         "margin-right",
         "margin-top",
         "margin",
         "marker-offset",
         "marks",
         "max-height",
         "max-width",
         "min-height",
         "min-width",
         "opacity",
         "orphans",
         "outline",
         "outline-color",
         "outline-style",
         "outline-width",
         "overflow",
         "padding-bottom",
         "padding-left",
         "padding-right",
         "padding-top",
         "padding",
         "page",
         "page-break-after",
         "page-break-before",
         "page-break-inside",
         "pause-after",
         "pause-before",
         "pause",
         "pitch",
         "pitch-range",
         "play-during",
         "position",
         "quotes",
         "richness",
         "right",
         "size",
         "speak-header",
         "speak-numeral",
         "speak-punctuation",
         "speak",
         "speech-rate",
         "stress",
         "table-layout",
         "text-align",
         "text-decoration",
         "text-indent",
         "text-shadow",
         "text-transform",
         "top",
         "unicode-bidi",
         "vertical-align",
         "visibility",
         "voice-family",
         "volume",
         "white-space",
         "widows",
         "width",
         "word-spacing",
         "z-index",
         "bottom",
         "left",
         "above",
         "absolute",
         "always",
         "armenian",
         "aural",
         "auto",
         "avoid",
         "baseline",
         "behind",
         "below",
         "bidi-override",
         "blink",
         "block",
         "bold",
         "bolder",
         "both",
         "capitalize",
         "center-left",
         "center-right",
         "center",
         "circle",
         "cjk-ideographic",
         "close-quote",
         "collapse",
         "condensed",
         "continuous",
         "crop",
         "crosshair",
         "cross",
         "cursive",
         "dashed",
         "decimal-leading-zero",
         "decimal",
         "default",
         "digits",
         "disc",
         "dotted",
         "double",
         "e-resize",
         "embed",
         "extra-condensed",
         "extra-expanded",
         "expanded",
         "fantasy",
         "far-left",
         "far-right",
         "faster",
         "fast",
         "fixed",
         "georgian",
         "groove",
         "hebrew",
         "help",
         "hidden",
         "hide",
         "higher",
         "high",
         "hiragana-iroha",
         "hiragana",
         "icon",
         "inherit",
         "inline-table",
         "inline",
         "inset",
         "inside",
         "invert",
         "italic",
         "justify",
         "katakana-iroha",
         "katakana",
         "landscape",
         "larger",
         "large",
         "left-side",
         "leftwards",
         "level",
         "lighter",
         "line-through",
         "list-item",
         "loud",
         "lower-alpha",
         "lower-greek",
         "lower-roman",
         "lowercase",
         "ltr",
         "lower",
         "low",
         "medium",
         "message-box",
         "middle",
         "mix",
         "monospace",
         "n-resize",
         "narrower",
         "ne-resize",
         "no-close-quote",
         "no-open-quote",
         "no-repeat",
         "none",
         "normal",
         "nowrap",
         "nw-resize",
         "oblique",
         "once",
         "open-quote",
         "outset",
         "outside",
         "overline",
         "pointer",
         "portrait",
         "px",
         "relative",
         "repeat-x",
         "repeat-y",
         "repeat",
         "rgb",
         "ridge",
         "right-side",
         "rightwards",
         "s-resize",
         "sans-serif",
         "scroll",
         "se-resize",
         "semi-condensed",
         "semi-expanded",
         "separate",
         "serif",
         "show",
         "silent",
         "slow",
         "slower",
         "small-caps",
         "small-caption",
         "smaller",
         "soft",
         "solid",
         "spell-out",
         "square",
         "static",
         "status-bar",
         "super",
         "sw-resize",
         "table-caption",
         "table-cell",
         "table-column",
         "table-column-group",
         "table-footer-group",
         "table-header-group",
         "table-row",
         "table-row-group",
         "text",
         "text-bottom",
         "text-top",
         "thick",
         "thin",
         "transparent",
         "ultra-condensed",
         "ultra-expanded",
         "underline",
         "upper-alpha",
         "upper-latin",
         "upper-roman",
         "uppercase",
         "url",
         "visible",
         "w-resize",
         "wait",
         "wider",
         "x-fast",
         "x-high",
         "x-large",
         "x-loud",
         "x-low",
         "x-small",
         "x-soft",
         "xx-large",
         "xx-small",
         "yes",
     ),
     suffix=r"\b",
 ),