示例#1
0
    def citdl_expr_from_trg(self, buf, trg):
        """Return a Python CITDL expression preceding the given trigger.
        
        The expression drops newlines, whitespace, and function call
        arguments -- basically any stuff that is not used by the codeintel
        database system for determining the resultant object type of the
        expression. For example (in which <|> represents the given position):
        
            GIVEN                       RETURN
            -----                       ------
            foo<|>.                     foo
            foo(bar<|>.                 bar
            foo(bar,blam)<|>.           foo()
            foo(bar,                    foo()
                blam)<|>.
            @foo<|>(                    foo

        If (trg.form == TRG_FORM_DEFN), then it's similar to above, except it
        looks forward to grab additional characters.

            GIVEN                       RETURN
            -----                       ------
            foo<|>.                     foo
            f<|>oo.bar                  foo.bar
            foo(bar<|>.                 bar
            foo(bar,blam)<|>.           foo()
            foo(bar,                    foo().bar
                blam).b<|>ar
        """
        DEBUG = False
        if DEBUG:
            print banner("Python-style citdl_expr_from_trg @ %d" % trg.pos)
        if trg.form == TRG_FORM_DEFN:
            pos = trg.pos
            expr = self._citdl_expr_from_pos(buf,
                                             pos,
                                             implicit=True,
                                             trg=trg,
                                             include_forwards=True,
                                             DEBUG=DEBUG)
            if expr:
                # Chop off any trailing "." characters
                return expr.rstrip(".")
            return expr
        else:
            if trg.type == 'array-members':
                # Get everything before the bracket position.
                pos = trg.extra.get('bracket_pos') - 1
            else:
                pos = trg.pos - 2  # skip ahead of the trigger char
            return self._citdl_expr_from_pos(buf,
                                             pos,
                                             implicit=trg.implicit,
                                             DEBUG=DEBUG,
                                             trg=trg)
    def citdl_expr_from_trg(self, buf, trg):
        """Return a Python CITDL expression preceding the given trigger.
        
        The expression drops newlines, whitespace, and function call
        arguments -- basically any stuff that is not used by the codeintel
        database system for determining the resultant object type of the
        expression. For example (in which <|> represents the given position):
        
            GIVEN                       RETURN
            -----                       ------
            foo<|>.                     foo
            foo(bar<|>.                 bar
            foo(bar,blam)<|>.           foo()
            foo(bar,                    foo()
                blam)<|>.
            @foo<|>(                    foo

        If (trg.form == TRG_FORM_DEFN), then it's similar to above, except it
        looks forward to grab additional characters.

            GIVEN                       RETURN
            -----                       ------
            foo<|>.                     foo
            f<|>oo.bar                  foo.bar
            foo(bar<|>.                 bar
            foo(bar,blam)<|>.           foo()
            foo(bar,                    foo().bar
                blam).b<|>ar
        """
        DEBUG = False
        if DEBUG:
            print banner("Python-style citdl_expr_from_trg @ %d" % trg.pos)
        if trg.form == TRG_FORM_DEFN:
            pos = trg.pos
            expr = self._citdl_expr_from_pos(buf, pos, implicit=True, trg=trg,
                                             include_forwards=True, DEBUG=DEBUG)
            if expr:
                # Chop off any trailing "." characters
                return expr.rstrip(".")
            return expr
        else:
            if trg.type == 'array-members':
                # Get everything before the bracket position.
                pos = trg.extra.get('bracket_pos') - 1
            else:
                pos = trg.pos - 2   # skip ahead of the trigger char
            return self._citdl_expr_from_pos(buf, pos, implicit=trg.implicit,
                                             DEBUG=DEBUG, trg=trg)
示例#3
0
    def do_play(self, subcmd, opts):
        """Run my current play/dev code.

        ${cmd_usage}
        ${cmd_option_list}
        """
        import pprint
        import random
        import ciElementTree as ET
        from codeintel2.manager import Manager
        from codeintel2.tree import pretty_tree_from_tree
        from codeintel2.common import LogEvalController, Error
        from codeintel2.util import tree_from_cix, dedent, unmark_text, banner
        from ci2 import _escaped_text_from_text

        if False:
            lang = "CSS"
            markedup_content = dedent("""
                /* http://www.w3.org/TR/REC-CSS2/fonts.html#propdef-font-weight */
                h1 {
                    border: 1px solid black;
                    font-weight /* hi */: <|> !important
                }
            """)
            content, data = unmark_text(markedup_content)
            pos = data["pos"]
            mgr = Manager()
            # mgr.upgrade() # Don't need it for just CSS usage.
            mgr.initialize()
            try:
                buf = mgr.buf_from_content(content, lang=lang, path="play.css")
                trg = buf.trg_from_pos(pos)
                if trg is None:
                    raise Error("unexpected trigger: %r" % trg)
                completions = buf.cplns_from_trg(trg)
                print("COMPLETIONS: %r" % completions)
            finally:
                mgr.finalize()

        elif False:
            lang = "Python"
            path = os.path.join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
            markedup_content = dedent("""
                import sys, os

                class Foo:
                    def bar(self):
                        pass

                sys.<|>path    # should have path in completion list
                f = Foo()
                """)
            content, data = unmark_text(markedup_content)
            print(banner(path))
            print(_escaped_text_from_text(content, "whitespace"))
            pos = data["pos"]
            mgr = Manager()
            mgr.upgrade()
            mgr.initialize()
            try:
                buf = mgr.buf_from_content(content, lang=lang, path=path)
                print(banner("cix", '-'))
                print(buf.cix)

                trg = buf.trg_from_pos(pos)
                if trg is None:
                    raise Error("unexpected trigger: %r" % trg)
                print(banner("completions", '-'))
                ctlr = LogEvalController(self.log)
                buf.async_eval_at_trg(trg, ctlr)
                ctlr.wait(2)  # XXX
                if not ctlr.is_done():
                    ctlr.abort()
                    raise Error("XXX async eval timed out")
                pprint.pprint(ctlr.cplns)
                print(banner(None))
            finally:
                mgr.finalize()
        elif False:
            lang = "Ruby"
            path = os.path.join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
            markedup_content = dedent("""\
            r<1>equire 'net/http'
            include Net
            req = HTTPRequest.new
            req.<2>get()
            """)
            content, data = unmark_text(markedup_content)
            print(banner(path))
            print(_escaped_text_from_text(content, "whitespace"))
            pos = data[1]
            mgr = Manager()
            mgr.upgrade()
            mgr.initialize()
            try:
                buf = mgr.buf_from_content(content, lang=lang, path=path)
                print(banner("cix", '-'))
                cix = buf.cix
                print(ET.tostring(pretty_tree_from_tree(tree_from_cix(cix))))

                trg = buf.trg_from_pos(pos, implicit=False)
                if trg is None:
                    raise Error("unexpected trigger: %r" % trg)
                print(banner("completions", '-'))
                ctlr = LogEvalController(self.log)
                buf.async_eval_at_trg(trg, ctlr)
                ctlr.wait(30)  # XXX
                if not ctlr.is_done():
                    ctlr.abort()
                    raise Error("XXX async eval timed out")
                pprint.pprint(ctlr.cplns)
                print(banner(None))
            finally:
                mgr.finalize()
示例#4
0
    def _citdl_expr_from_pos(self,
                             buf,
                             pos,
                             implicit=False,
                             include_forwards=False,
                             DEBUG=False,
                             trg=None,
                             array_as_attr=False):

        #PERF: Would dicts be faster for all of these?
        WHITESPACE = tuple(" \t\n\r\v\f")
        EOL = tuple("\r\n")
        BLOCKCLOSES = tuple(")}]")
        STOPOPS = tuple("({[,&+-=!^|%/<>;:#@")
        EXTRA_STOPOPS_PRECEDING_IDENT = BLOCKCLOSES  # Might be others.

        #TODO: clean this up for LangIntel-usage
        if implicit:
            skip_styles = buf.implicit_completion_skip_styles
        else:
            skip_styles = buf.completion_skip_styles
        string_styles = buf.string_styles()
        comment_styles = buf.comment_styles()

        #XXX Add sentinel num chars?
        citdl_expr = []
        accessor = buf.accessor
        i = pos

        is_udl_buffer = False
        from codeintel2.udl import UDLBuffer
        if isinstance(buf, UDLBuffer):
            # We need to check for udl transition points and not go beyond the
            # current sub-language, bug 95946.
            is_udl_buffer = True
            udl_lang = buf.lang_from_pos(pos)

        # Move ahead to include forward chars as well
        # We stop when we go out of the expression or when the expression is
        # becomes a multiple fragment, i.e.
        #  'sys.pa<|>th.expanduser' -> 'sys.path'
        if include_forwards:
            buf_length = accessor.length()
            if i < buf_length:
                max_look_ahead = min(buf_length, i + 100)
                lastch_was_whitespace = False
                while i < max_look_ahead:
                    ch = accessor.char_at_pos(i)
                    style = accessor.style_at_pos(i)
                    if is_udl_buffer and buf.lang_from_style(
                            style) != udl_lang:
                        if DEBUG:
                            print "UDL boundary at pos %d, changed from %r to %r" % (
                                i, udl_lang, buf.lang_from_style(style))
                        break
                    if ch in WHITESPACE:
                        lastch_was_whitespace = True
                    elif ch in ".)}]" or ch in STOPOPS:
                        break
                    elif lastch_was_whitespace:
                        break
                    else:
                        lastch_was_whitespace = False
                    i += 1
                # Move back to last valid char
                i -= 1
            else:
                i = buf_length - 1
            if DEBUG:
                if i > pos:
                    print "Including chars from pos %d up to %d" % (pos, i)
                else:
                    print "No valid chars forward from pos %d, i now: %d" % (
                        pos, i)

        # Be careful here, we cannot move from code into a comment, but we
        # can be in a comment to begin with.
        first_citdl_expr_style = None
        first_citdl_expr_style_is_comment = False
        while i >= 0:
            ch = accessor.char_at_pos(i)
            style = accessor.style_at_pos(i)
            if is_udl_buffer and buf.lang_from_style(style) != udl_lang:
                if DEBUG:
                    print "UDL boundary at pos %d, changed from %r to %r" % (
                        i, udl_lang, buf.lang_from_style(style))
                break
            if ch in WHITESPACE:
                # drop all whitespace
                while i >= 0:
                    ch = accessor.char_at_pos(i)
                    if ch in WHITESPACE \
                       or (ch == '\\' and accessor.char_at_pos(i+1) in EOL):
                        if DEBUG:
                            print "drop whitespace: %r" % accessor.char_at_pos(
                                i)
                    else:
                        break
                    i -= 1
                # If there are two whitespace-separated words with no .
                # in between we're changing expressions:
                #   if foo<|> and ...
                #   def foo<|>(...
                if i >= 0 and citdl_expr and isident(citdl_expr[-1]) \
                   and ch != '.':
                    if DEBUG:
                        print "stop at non-dot: %r" % ch
                    break
            elif style in string_styles:  # Convert to string
                citdl_type = self.citdl_from_literal_type.get("string")
                if DEBUG:
                    print "found string style, converting to: %s and now " \
                          "finished" % (citdl_type)
                if citdl_type:
                    citdl_expr += reversed(citdl_type)
                break
            elif style in skip_styles:  # drop styles to ignore
                while i >= 0 and accessor.style_at_pos(i) in skip_styles:
                    if DEBUG:
                        print "drop char of style to ignore: %r"\
                              % accessor.char_at_pos(i)
                    i -= 1
            elif ch in STOPOPS or (
                    # This check ensures that, for example, we get "foo" instead
                    # of "bar()foo" in the following:
                    #      bar()
                    #      foo<|>.
                    citdl_expr and citdl_expr[-1] != '.' and ch
                    in EXTRA_STOPOPS_PRECEDING_IDENT):
                if DEBUG:
                    print "stop at stop-operator %d: %r" % (i, ch)
                break
            elif ch in BLOCKCLOSES:
                if DEBUG:
                    print "found block at %d: %r" % (i, ch)
                citdl_expr.append(ch)

                BLOCKS = { # map block close char to block open char
                    ')': '(',
                    ']': '[',
                    '}': '{',
                }
                stack = []  # stack of blocks: (<block close char>, <style>)
                stack.append((ch, style, BLOCKS[ch], i))
                i -= 1
                num_lines = 0
                content_styles = []
                while i >= 0:
                    ch = accessor.char_at_pos(i)
                    style = accessor.style_at_pos(i)
                    content_styles.append(style)
                    if DEBUG:
                        print "finding matching brace: ch %r (%s), stack %r"\
                              % (ch, ', '.join(buf.style_names_from_style_num(style)), stack)
                    if ch in EOL:
                        num_lines += 1
                    if num_lines >= 3:
                        if DEBUG:
                            print "stop search for matching brace at 3 line sentinel"
                        break
                    elif ch in BLOCKS and style not in skip_styles:
                        stack.append((ch, style, BLOCKS[ch]))
                    elif ch == stack[-1][2] and style not in skip_styles:
                        #XXX Replace the second test with the following
                        #    when LexPython+SilverCity styling bugs are fixed
                        #    (spurious 'stderr' problem):
                        #       and style == stack[-1][1]:
                        last_frame = stack.pop()
                        if not stack:
                            content_styles.pop()  # Drop the thing that matched
                            if array_as_attr and \
                                    all(style in string_styles for style in content_styles):
                                prop = accessor.text_range(
                                    i + 1, last_frame[3])
                                if DEBUG:
                                    print "Injecting %s" % (prop, )
                                citdl_expr.append(prop)
                            if DEBUG:
                                print "jump to matching brace at %d: %r" % (i,
                                                                            ch)
                            citdl_expr.append(ch)
                            if trg and ch == "(":
                                # save the text in params, in case the completion
                                # needs to special-case things depending on the
                                # argument.
                                params = trg.extra.setdefault("_params", [])
                                params.insert(
                                    0,
                                    accessor.text_range(i + 1, last_frame[3]))
                            i -= 1
                            break
                    i -= 1
                else:
                    # Didn't find the matching brace.
                    if DEBUG:
                        print "couldn't find matching brace"
                    raise EvalError("could not find matching brace for "
                                    "'%s' at position %d" %
                                    (stack[-1][0], stack[-1][3]))

            else:
                if DEBUG:
                    style_names = buf.style_names_from_style_num(style)
                    print "add char: %r (%s)" % (ch, ', '.join(style_names))
                if first_citdl_expr_style is None:
                    # Remember the first citdl style we found
                    first_citdl_expr_style = style
                    first_citdl_expr_style_is_comment = style in comment_styles
                elif first_citdl_expr_style != style and \
                     (first_citdl_expr_style_is_comment or
                      style in comment_styles):
                    # We've moved into or out of a comment, let's leave now
                    # Fixes: http://bugs.activestate.com/show_bug.cgi?id=65672
                    break
                citdl_expr.append(ch)
                i -= 1

        citdl_expr.reverse()
        citdl_expr = ''.join(citdl_expr)
        if DEBUG:
            print "return: %r" % citdl_expr
            print banner("done")
        return citdl_expr
示例#5
0
    def preceding_trg_from_pos(self,
                               buf,
                               pos,
                               curr_pos,
                               preceding_trg_terminators=None,
                               DEBUG=False):
        accessor = buf.accessor
        if preceding_trg_terminators is None:
            preceding_trg_terminators = self.preceding_trg_terminators
        if DEBUG:
            print banner("preceding_trg_from_pos(pos=%r, curr_pos=%r)" %
                         (pos, curr_pos))
            print indent(
                markup_text(accessor.text, pos=curr_pos, start_pos=pos))
            print banner(None, '-')

        # Skip over comments and strings in our checking, unless we are
        # in one of these styles for the whole range. This is so an explicit
        # trigger in a comment (or, e.g., a doc string) will work, but
        # the appearance of small comments or strings in code will not mess
        # things up.
        comment_and_string_styles = dict(
            (s, True) for s in buf.comment_styles() + buf.string_styles())
        skip_styles = {}
        start_style = accessor.style_at_pos(pos - 1)
        EOL_CHARS = tuple("\n\r")

        # Limiting simplification: Only backtrack a max of 200 chars.
        # Can increase that if necessary. The problem is detecting a
        # statement boundary backwards in langs like Python and Ruby
        # where you can't rely on ';' (actually
        # `preceding_trg_terminators').
        limit = max(1, pos - 200)

        # First stage. We only consider autocomplete trigger (i.e.
        # trg.form==TRG_FORM_COMPLETION) if within range of the
        # curr_pos. Here "within range" means you don't have to more the
        # cursor to show the autocomplete UI.
        first_stage_limit = curr_pos
        for (char, style) in accessor.gen_char_and_style_back(
                curr_pos - 1, limit - 1):
            if not isident(char):
                break
            first_stage_limit -= 1
        if DEBUG:
            print "[stage 1] first_stage_limit=%d (prev_ch=%r)"\
                  % (first_stage_limit,
                     (first_stage_limit > 0
                      and accessor.char_at_pos(first_stage_limit-1)
                      or None))
        p = pos
        if p >= first_stage_limit:
            for (prev_ch, prev_style) in accessor.gen_char_and_style_back(
                    p - 1, first_stage_limit - 2):
                if (not skip_styles and prev_style != start_style
                        # EOLs in comments seem to always be style 0. Don't count
                        # them.
                        and prev_ch not in EOL_CHARS):
                    if DEBUG:
                        print "[stage 1] have seen a style change (%d -> %d), " \
                              "now skipping strings and comments" \
                              % (start_style, prev_style)
                    skip_styles = comment_and_string_styles
                if DEBUG:
                    print "[stage 1] consider pos %2d: prev_ch=%r (%d) --"\
                          % (p, prev_ch, prev_style),
                if prev_style in skip_styles:
                    if DEBUG: print "comment or string, skip it"
                elif self._is_terminating_char(prev_ch, prev_style,
                                               preceding_trg_terminators):
                    if DEBUG: print "in `preceding_trg_terminators': break"
                    return None
                elif prev_ch in self.trg_chars:
                    if DEBUG: print "trigger char, try it"
                    trg = buf.trg_from_pos(p, implicit=False)
                    if trg:
                        if DEBUG: print "[stage 1] %s" % trg
                        return trg
                    p -= 1
                    break
                elif DEBUG:
                    print "not a trigger char, skip it"
                p -= 1
        if DEBUG:
            print "[stage 1] end of possible autocomplete trigger range"

        # Second stage. We only consider calltip triggers now
        # (self.calltip_trg_chars).
        #
        # As well, ignore enclosed paren sections to make sure we are
        # in-range. For example, we shouldn't trigger on "bar(" here:
        #   foo(bar("skip", "this", "arg", "list"), <|>)
        close_paren_count = 0
        for (prev_ch,
             prev_style) in accessor.gen_char_and_style_back(p - 1, limit - 2):
            if (not skip_styles and prev_style != start_style
                    # EOLs in comments seem to always be style 0. Don't count
                    # them.
                    and prev_ch not in EOL_CHARS):
                if DEBUG:
                    print "[stage 2] seen a style change (%d -> %d), now " \
                          "skipping strings and comments" \
                          % (start_style, prev_style)
                skip_styles = comment_and_string_styles

            if DEBUG:
                print "[stage 2] consider pos %2d: prev_ch=%r (%d) --"\
                      % (p, prev_ch, prev_style),
            if prev_style in skip_styles:
                if DEBUG: print "comment or string, skip it"
            elif prev_ch == ')':
                close_paren_count += 1
                if DEBUG: print "close paren: count=%d" % close_paren_count
            elif close_paren_count and prev_ch == '(':
                close_paren_count -= 1
                if DEBUG: print "open paren: count=%d" % close_paren_count
            elif self._is_terminating_char(prev_ch, prev_style,
                                           preceding_trg_terminators):
                if DEBUG: print "in `preceding_trg_terminators': break"
                return None
            elif prev_ch in self.calltip_trg_chars:
                if DEBUG: print "trigger char, try it"
                trg = buf.trg_from_pos(p, implicit=False)
                if trg:
                    if DEBUG: print "[stage 2] %s" % trg
                    return trg
            elif DEBUG:
                print "not a trigger char, skip it"
            p -= 1

        return None
示例#6
0
    def curr_calltip_arg_range(self,
                               buf,
                               trg_pos,
                               calltip,
                               curr_pos,
                               DEBUG=False):
        """Return that range in the calltip of the "current" arg.
        I.e. what argument is currently being entered.
        
            "buf" is the buffer object on which this is being done.
            "trg_pos" is the trigger position.
            "calltip" is the full calltip text.
            "curr_pos" is the current position in the buffer.
            
        Returns a range: (start, end)
        Set `start == -1` to cancel the calltip, i.e. if the entered text
        has closed the call region.

        The default implementation uses:
            self.calltip_region_terminators
        to handle languages with calltip signatures with the following
        characteristics:
        - uses '(' and ')' to bound the argument list (though because of
          support for ';' statement termination, this isn't absolutely
          required)
        - uses a comma to separate arguments
        - basic block delimiters are {}, (), and []

        For example:
            foo()
            blam(a, b)
            range([start,] stop[, step]) -> list of integers
            bar(arg1, *args, **kwargs)
            flash(boom, bang=42)
        """
        # Dev Notes:
        # - Eventually should pass in the trigger to aid in processing.
        # - TODO figure out dependence on buf.comment_styles() and
        #   buf.string_styles()
        accessor = buf.accessor
        if DEBUG:
            print banner("curr_calltip_arg_range")
            print "calltip:\n%s" % indent(calltip)
            print "buffer:\n%s" % indent(
                markup_text(accessor.text, trg_pos=trg_pos, pos=curr_pos))

        # Start from the trigger position and walk forward to the current
        # pos: counting args and looking for termination of the calltip
        # region.
        skip_styles = dict(
            (s, True) for s in buf.comment_styles() + buf.string_styles())
        if accessor.style_at_pos(trg_pos - 1) in skip_styles:
            skip_styles = {}
        comma_count = 0
        blocks = {
            # Map a block start token to its block end token.
            '(': ')',
            '[': ']',
            '{': '}',
        }
        block_stack = []
        p = trg_pos
        for ch, style in accessor.gen_char_and_style(trg_pos, curr_pos):
            if DEBUG: print "pos %2d: %r (%2s) --" % (p, ch, style),
            if style in skip_styles:
                if DEBUG: print "skip"
            elif ch in blocks:
                if DEBUG: print "open block"
                block_stack.append(blocks[ch])
            elif block_stack:
                if ch == block_stack[-1]:
                    if DEBUG: print "close block"
                    block_stack.pop()
                elif ch in self.calltip_region_terminators:
                    if DEBUG: print "end of call region: (-1, -1)"
                    return (-1, -1)
                elif DEBUG:
                    print "ignore (in block)"
            elif ch == ',':
                if DEBUG: print "next arg"
                comma_count += 1
            elif ch in self.calltip_region_terminators and \
                 self.calltip_verify_termination(accessor, ch, trg_pos, curr_pos):
                if DEBUG: print "end of call region: (-1, -1)"
                return (-1, -1)
            elif DEBUG:
                print "ignore"
            p += 1

        # Parse the signature from the calltip. If there is no signature
        # then we default to not indicating any arg range.
        if self._parsed_calltip_cache[0] == calltip:
            parsed = self._parsed_calltip_cache[1]
        else:
            parsed = _parse_calltip(calltip, DEBUG)
            self._parsed_calltip_cache = (calltip, parsed)
        if parsed is None:
            if DEBUG: print "couldn't parse any calltip: (0, 0)"
            return (0, 0)
        signature, name, args = parsed
        if DEBUG:
            print "parsed calltip:\n  signature:\n%s\n  name:\n%s\n  args:\n%s"\
                  % (indent(signature), indent(name), indent(pformat(args)))

        if not args:
            if DEBUG: print "no args in signature: (0, 0)"
            return (0, 0)
        elif comma_count >= len(args):
            #XXX ellipsis
            if DEBUG: print "more commas than args: ellipsis?"
            span = args[-1].span  # default to last arg
        else:
            span = args[comma_count].span

        if DEBUG:
            print "curr calltip range (%s, %s):" % (span[0], span[1])
            print indent(signature)
            print "    %s%s" % (' ' * span[0], '-' * (span[1] - span[0]))
        return span
示例#7
0
    def post_process_cplns(self, cplns):
        DEBUG = False
        if DEBUG:
            print banner("Perl post_process_cplns (before)")
            pprint(cplns)

        trg_type = self.trg.type
        if trg_type in ("package-subs", "object-subs"):
            #TODO: This may not be necessary if current evalr only
            #      generates the function.
            cplns = [
                c for c in cplns if c[0] == "function"
                if c[1] not in self._special_names_to_skip
            ]
        elif trg_type == "package-members":
            if self.prefix_filter in ('', '&'):  # filter out variables
                cplns = [
                    c for c in cplns if c[0] != "variable"
                    if c[1] not in self._special_names_to_skip
                ]
            elif self.prefix_filter in ('$', '@', '%'):  # filter out funcs
                cplns = [
                    c for c in cplns if c[0] != "function"
                    if c[1] not in self._special_names_to_skip
                ]
            else:
                cplns = [
                    c for c in cplns if c[1] not in self._special_names_to_skip
                ]

            # Morph type and value of variable based on the prefix.
            # For example the completions for: `$HTTP::Message::` include
            # ("variable", "$VERSION"). The actual correct completion is
            # "VERSION" (no '$'-prefix). We morph this to ("$variable",
            # "VERSION") so that:
            # 1. the proper completion will be used, and
            # 2. the different type string can be used for a custom
            #    autocomplete image.
            # Ditto for '@' and '%'.
            morphed_cplns = []
            for type, value in cplns:
                if type == "variable":
                    match = self._perl_var_tokenizer.match(value)
                    if not match:
                        self.warn(
                            "could not parse Perl var '%s': "
                            "pattern='%s'", value,
                            self._perl_var_tokenizer.pattern)
                        continue
                    prefix, name = match.groups()
                    if DEBUG:
                        print "tokenize perl var: %r -> %r %r"\
                              % (value, prefix, name)
                    if prefix:
                        prefix = prefix[-1]  # only last char is relevant

                    if self.prefix_filter in (None, '*', '$'):
                        # '*': pass all
                        # '$': pass all because arrays and hashes can have
                        #      a '$' prefix for subsequent indexing
                        pass
                    elif self.prefix_filter and self.prefix_filter != prefix:
                        # If the filter is '%' or '@', then filter out vars
                        # not of that persuasion.
                        continue

                    #TODO: Test cases for these and review by Perl guy.
                    if prefix in ('$', '%', '@'):
                        # Don't yet support '*' special a/c image.
                        type = prefix + type
                    value = name
                morphed_cplns.append((type, value))
            cplns = morphed_cplns

        cplns = CandidatesForTreeEvaluator.post_process_cplns(self, cplns)
        if DEBUG:
            print banner("(after)", '-')
            pprint(cplns)
            print banner(None, '-')
        return cplns
    def _citdl_expr_from_pos(self, buf, pos, implicit=False,
                             include_forwards=False, DEBUG=False, trg=None):

        #PERF: Would dicts be faster for all of these?
        WHITESPACE = tuple(" \t\n\r\v\f")
        EOL = tuple("\r\n")
        BLOCKCLOSES = tuple(")}]")
        STOPOPS = tuple("({[,&+-=!^|%/<>;:#@")
        EXTRA_STOPOPS_PRECEDING_IDENT = BLOCKCLOSES # Might be others.

        #TODO: clean this up for LangIntel-usage
        if implicit:
            skip_styles = buf.implicit_completion_skip_styles
        else:
            skip_styles = buf.completion_skip_styles
        string_styles = buf.string_styles()
        comment_styles = buf.comment_styles()

        #XXX Add sentinel num chars?
        citdl_expr = []
        accessor = buf.accessor
        i = pos

        # Move ahead to include forward chars as well
        # We stop when we go out of the expression or when the expression is
        # becomes a multiple fragment, i.e.
        #  'sys.pa<|>th.expanduser' -> 'sys.path'
        if include_forwards:
            buf_length = accessor.length()
            if i < buf_length:
                max_look_ahead = min(buf_length, i+100)
                lastch_was_whitespace = False
                while i < max_look_ahead:
                    ch = accessor.char_at_pos(i)
                    style = accessor.style_at_pos(i)
                    if ch in WHITESPACE:
                        lastch_was_whitespace = True
                    elif ch in ".)}]" or ch in STOPOPS:
                        break
                    elif lastch_was_whitespace:
                        break
                    else:
                        lastch_was_whitespace = False
                    i += 1
                # Move back to last valid char
                i -= 1
            else:
                i = buf_length - 1
            if DEBUG:
                if i > pos:
                    print "Including chars from pos %d up to %d" % (pos, i)
                else:
                    print "No valid chars forward from pos %d, i now: %d" % (pos, i)

        # Be careful here, we cannot move from code into a comment, but we
        # can be in a comment to begin with.
        first_citdl_expr_style = None
        first_citdl_expr_style_is_comment = False
        while i >= 0:
            ch = accessor.char_at_pos(i)
            style = accessor.style_at_pos(i)
            if ch in WHITESPACE:
                # drop all whitespace
                while i >= 0:
                    ch = accessor.char_at_pos(i)
                    if ch in WHITESPACE \
                       or (ch == '\\' and accessor.char_at_pos(i+1) in EOL):
                        if DEBUG:
                            print "drop whitespace: %r" % accessor.char_at_pos(i)
                    else:
                        break
                    i -= 1
                # If there are two whitespace-separated words with no .
                # in between we're changing expressions:
                #   if foo<|> and ...
                #   def foo<|>(...
                if i >= 0 and citdl_expr and isident(citdl_expr[-1]) \
                   and ch != '.':
                    if DEBUG: 
                        print "stop at non-dot: %r" % ch
                    break
            elif style in string_styles: # Convert to string
                citdl_type = self.citdl_from_literal_type.get("string")
                if DEBUG:
                    print "found string style, converting to: %s and now " \
                          "finished" % (citdl_type)
                if citdl_type:
                    citdl_expr += reversed(citdl_type)
                break
            elif style in skip_styles: # drop styles to ignore
                while i >= 0 and accessor.style_at_pos(i) in skip_styles:
                    if DEBUG:
                        print "drop char of style to ignore: %r"\
                              % accessor.char_at_pos(i)
                    i -= 1
            elif ch in STOPOPS or (
                 # This check ensures that, for example, we get "foo" instead
                 # of "bar()foo" in the following:
                 #      bar()
                 #      foo<|>.
                 citdl_expr and citdl_expr[-1] != '.'
                 and ch in EXTRA_STOPOPS_PRECEDING_IDENT):
                if DEBUG:
                    print "stop at stop-operator %d: %r" % (i, ch)
                break
            elif ch in BLOCKCLOSES:
                if DEBUG:
                    print "found block at %d: %r" % (i, ch)
                citdl_expr.append(ch)
    
                BLOCKS = { # map block close char to block open char
                    ')': '(',
                    ']': '[',
                    '}': '{',
                }
                stack = [] # stack of blocks: (<block close char>, <style>)
                stack.append( (ch, style, BLOCKS[ch], i) )
                i -= 1
                num_lines = 0
                while i >= 0:
                    ch = accessor.char_at_pos(i)
                    style = accessor.style_at_pos(i)
                    if DEBUG:
                        print "finding matching brace: ch %r (%s), stack %r"\
                              % (ch, ', '.join(buf.style_names_from_style_num(style)), stack)
                    if ch in EOL:
                        num_lines += 1
                    if num_lines >= 3:
                        if DEBUG: print "stop search for matching brace at 3 line sentinel"
                        break
                    elif ch in BLOCKS and style not in skip_styles:
                        stack.append( (ch, style, BLOCKS[ch]) )
                    elif ch == stack[-1][2] and style not in skip_styles:
                        #XXX Replace the second test with the following
                        #    when LexPython+SilverCity styling bugs are fixed
                        #    (spurious 'stderr' problem):
                        #       and style == stack[-1][1]:
                        last_frame = stack.pop()
                        if not stack:
                            if DEBUG:
                                print "jump to matching brace at %d: %r" % (i, ch)
                            citdl_expr.append(ch)
                            if trg:
                                # save the text in params, in case the completion
                                # needs to special-case things depending on the
                                # argument.
                                trg.extra["_params"] = accessor.text_range(i + 1, last_frame[3])
                            i -= 1
                            break
                    i -= 1
                else:
                    # Didn't find the matching brace.
                    if DEBUG:
                        print "couldn't find matching brace"
                    raise EvalError("could not find matching brace for "
                                    "'%s' at position %d"
                                    % (stack[-1][0], stack[-1][3]))
    
            else:
                if DEBUG:
                    style_names = buf.style_names_from_style_num(style)
                    print "add char: %r (%s)" % (ch, ', '.join(style_names))
                if first_citdl_expr_style is None:
                    # Remember the first citdl style we found
                    first_citdl_expr_style = style
                    first_citdl_expr_style_is_comment = style in comment_styles
                elif first_citdl_expr_style != style and \
                     (first_citdl_expr_style_is_comment or
                      style in comment_styles):
                    # We've moved into or out of a comment, let's leave now
                    # Fixes: http://bugs.activestate.com/show_bug.cgi?id=65672
                    break
                citdl_expr.append(ch)
                i -= 1

        citdl_expr.reverse()
        citdl_expr = ''.join(citdl_expr)
        if DEBUG:
            print "return: %r" % citdl_expr
            print banner("done")
        return citdl_expr
    def preceding_trg_from_pos(self, buf, pos, curr_pos,
                               preceding_trg_terminators=None, DEBUG=False):
        accessor = buf.accessor
        if preceding_trg_terminators is None:
            preceding_trg_terminators = self.preceding_trg_terminators
        if DEBUG:
            print banner("preceding_trg_from_pos(pos=%r, curr_pos=%r)"
                          % (pos, curr_pos))
            print indent(markup_text(accessor.text, pos=curr_pos,
                                     start_pos=pos))
            print banner(None, '-')

        # Skip over comments and strings in our checking, unless we are
        # in one of these styles for the whole range. This is so an explicit
        # trigger in a comment (or, e.g., a doc string) will work, but
        # the appearance of small comments or strings in code will not mess
        # things up.
        comment_and_string_styles = dict(
            (s, True) for s in buf.comment_styles() + buf.string_styles())
        skip_styles = {}
        start_style = accessor.style_at_pos(pos-1)
        EOL_CHARS = tuple("\n\r")

        # Limiting simplification: Only backtrack a max of 200 chars.
        # Can increase that if necessary. The problem is detecting a
        # statement boundary backwards in langs like Python and Ruby
        # where you can't rely on ';' (actually
        # `preceding_trg_terminators').
        limit = max(1, pos - 200)
        
        # First stage. We only consider autocomplete trigger (i.e.
        # trg.form==TRG_FORM_COMPLETION) if within range of the
        # curr_pos. Here "within range" means you don't have to more the
        # cursor to show the autocomplete UI.
        first_stage_limit = curr_pos
        for (char, style) in accessor.gen_char_and_style_back(curr_pos-1,
                                                              limit-1):
            if not isident(char):
                break
            first_stage_limit -= 1
        if DEBUG:
            print "[stage 1] first_stage_limit=%d (prev_ch=%r)"\
                  % (first_stage_limit,
                     (first_stage_limit > 0
                      and accessor.char_at_pos(first_stage_limit-1)
                      or None))
        p = pos
        if p >= first_stage_limit:
            for (prev_ch, prev_style) in accessor.gen_char_and_style_back(p-1,
                                                          first_stage_limit-2):
                if (not skip_styles and prev_style != start_style
                    # EOLs in comments seem to always be style 0. Don't count
                    # them.
                    and prev_ch not in EOL_CHARS):
                    if DEBUG:
                        print "[stage 1] have seen a style change (%d -> %d), " \
                              "now skipping strings and comments" \
                              % (start_style, prev_style)
                    skip_styles = comment_and_string_styles
                if DEBUG:
                    print "[stage 1] consider pos %2d: prev_ch=%r (%d) --"\
                          % (p, prev_ch, prev_style),
                if prev_style in skip_styles:
                    if DEBUG: print "comment or string, skip it"
                elif self._is_terminating_char(prev_ch, prev_style,
                                               preceding_trg_terminators):
                    if DEBUG: print "in `preceding_trg_terminators': break"
                    return None
                elif prev_ch in self.trg_chars:
                    if DEBUG: print "trigger char, try it"
                    trg = buf.trg_from_pos(p, implicit=False)
                    if trg:
                        if DEBUG: print "[stage 1] %s" % trg
                        return trg
                    p -= 1
                    break
                elif DEBUG:
                    print "not a trigger char, skip it"
                p -= 1
        if DEBUG:
            print "[stage 1] end of possible autocomplete trigger range"

        # Second stage. We only consider calltip triggers now
        # (self.calltip_trg_chars).
        # 
        # As well, ignore enclosed paren sections to make sure we are
        # in-range. For example, we shouldn't trigger on "bar(" here:
        #   foo(bar("skip", "this", "arg", "list"), <|>)
        close_paren_count = 0
        for (prev_ch, prev_style) in accessor.gen_char_and_style_back(p-1, limit-2):
            if (not skip_styles and prev_style != start_style
                # EOLs in comments seem to always be style 0. Don't count
                # them.
                and prev_ch not in EOL_CHARS):
                if DEBUG:
                    print "[stage 2] seen a style change (%d -> %d), now " \
                          "skipping strings and comments" \
                          % (start_style, prev_style)
                skip_styles = comment_and_string_styles

            if DEBUG:
                print "[stage 2] consider pos %2d: prev_ch=%r (%d) --"\
                      % (p, prev_ch, prev_style),
            if prev_style in skip_styles:
                if DEBUG: print "comment or string, skip it"
            elif prev_ch == ')':
                close_paren_count += 1
                if DEBUG: print "close paren: count=%d" % close_paren_count
            elif close_paren_count and prev_ch == '(':
                close_paren_count -= 1
                if DEBUG: print "open paren: count=%d" % close_paren_count
            elif self._is_terminating_char(prev_ch, prev_style,
                                           preceding_trg_terminators):
                if DEBUG: print "in `preceding_trg_terminators': break"
                return None
            elif prev_ch in self.calltip_trg_chars:
                if DEBUG: print "trigger char, try it"
                trg = buf.trg_from_pos(p, implicit=False)
                if trg:
                    if DEBUG: print "[stage 2] %s" % trg
                    return trg
            elif DEBUG:
                print "not a trigger char, skip it"
            p -= 1

        return None
    def curr_calltip_arg_range(self, buf, trg_pos, calltip, curr_pos,
                               DEBUG=False):
        """Return that range in the calltip of the "current" arg.
        I.e. what argument is currently being entered.
        
            "buf" is the buffer object on which this is being done.
            "trg_pos" is the trigger position.
            "calltip" is the full calltip text.
            "curr_pos" is the current position in the buffer.
            
        Returns a range: (start, end)
        Set `start == -1` to cancel the calltip, i.e. if the entered text
        has closed the call region.

        The default implementation uses:
            self.calltip_region_terminators
        to handle languages with calltip signatures with the following
        characteristics:
        - uses '(' and ')' to bound the argument list (though because of
          support for ';' statement termination, this isn't absolutely
          required)
        - uses a comma to separate arguments
        - basic block delimiters are {}, (), and []

        For example:
            foo()
            blam(a, b)
            range([start,] stop[, step]) -> list of integers
            bar(arg1, *args, **kwargs)
            flash(boom, bang=42)
        """
        # Dev Notes:
        # - Eventually should pass in the trigger to aid in processing.
        # - TODO figure out dependence on buf.comment_styles() and
        #   buf.string_styles()
        accessor = buf.accessor
        if DEBUG:
            print banner("curr_calltip_arg_range")
            print "calltip:\n%s" % indent(calltip)
            print "buffer:\n%s" % indent(markup_text(accessor.text,
                                                     trg_pos=trg_pos,
                                                     pos=curr_pos))
            
        # Start from the trigger position and walk forward to the current
        # pos: counting args and looking for termination of the calltip
        # region.
        skip_styles = dict(
            (s, True) for s in buf.comment_styles() + buf.string_styles())
        if accessor.style_at_pos(trg_pos-1) in skip_styles:
            skip_styles = {}
        comma_count = 0
        blocks = {
            # Map a block start token to its block end token.
            '(': ')', '[': ']', '{': '}',
        }
        block_stack = []
        p = trg_pos
        for ch, style in accessor.gen_char_and_style(trg_pos, curr_pos):
            if DEBUG: print "pos %2d: %r (%2s) --" % (p, ch, style),
            if style in skip_styles:
                if DEBUG: print "skip"
            elif ch in blocks:
                if DEBUG: print "open block"
                block_stack.append(blocks[ch])
            elif block_stack:
                if ch == block_stack[-1]:
                    if DEBUG: print "close block"
                    block_stack.pop()
                elif ch in self.calltip_region_terminators:
                    if DEBUG: print "end of call region: (-1, -1)"
                    return (-1, -1)
                elif DEBUG:
                    print "ignore (in block)"
            elif ch == ',':
                if DEBUG: print "next arg"
                comma_count += 1
            elif ch in self.calltip_region_terminators and \
                 self.calltip_verify_termination(accessor, ch, trg_pos, curr_pos):
                if DEBUG: print "end of call region: (-1, -1)"
                return (-1, -1)
            elif DEBUG:
                print "ignore"
            p += 1

        # Parse the signature from the calltip. If there is no signature
        # then we default to not indicating any arg range.
        if self._parsed_calltip_cache[0] == calltip:
            parsed = self._parsed_calltip_cache[1]
        else:
            parsed = _parse_calltip(calltip, DEBUG)
            self._parsed_calltip_cache = (calltip, parsed)
        if parsed is None:
            if DEBUG: print "couldn't parse any calltip: (0, 0)"
            return (0, 0)
        signature, name, args = parsed
        if DEBUG:
            print "parsed calltip:\n  signature:\n%s\n  name:\n%s\n  args:\n%s"\
                  % (indent(signature), indent(name), indent(pformat(args)))

        if not args:
            if DEBUG: print "no args in signature: (0, 0)"
            return (0, 0)
        elif comma_count >= len(args):
            #XXX ellipsis
            if DEBUG: print "more commas than args: ellipsis?"
            span = args[-1].span # default to last arg
        else:
            span = args[comma_count].span

        if DEBUG:
            print "curr calltip range (%s, %s):" % (span[0], span[1])
            print indent(signature)
            print "    %s%s" % (' '*span[0], '-'*(span[1]-span[0]))
        return span
示例#11
0
文件: ci2.py 项目: p4p3r/CodeIntel
    def do_play(self, subcmd, opts):
        """Run my current play/dev code.

        ${cmd_usage}
        ${cmd_option_list}
        """
        if False:
            lang = "CSS"
            markedup_content = dedent("""
                /* http://www.w3.org/TR/REC-CSS2/fonts.html#propdef-font-weight */
                h1 {
                    border: 1px solid black;
                    font-weight /* hi */: <|> !important
                }
            """)
            content, data = unmark_text(markedup_content)
            pos = data["pos"]
            mgr = Manager()
            #mgr.upgrade() # Don't need it for just CSS usage.
            mgr.initialize()
            try:
                buf = mgr.buf_from_content(content, lang=lang, path="play.css")
                trg = buf.trg_from_pos(pos)
                if trg is None:
                    raise Error("unexpected trigger: %r" % trg)
                completions = buf.cplns_from_trg(trg)
                print("COMPLETIONS: %r" % completions)
            finally:
                mgr.finalize()

        elif False:
            lang = "Python"
            path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
            markedup_content = dedent("""
                import sys, os

                class Foo:
                    def bar(self):
                        pass

                sys.<|>path    # should have path in completion list
                f = Foo()
                """)
            content, data = unmark_text(markedup_content)
            print(banner(path))
            print(_escaped_text_from_text(content, "whitespace"))
            pos = data["pos"]
            mgr = Manager()
            mgr.upgrade()
            mgr.initialize()
            try:
                buf = mgr.buf_from_content(content, lang=lang, path=path)
                print(banner("cix", '-'))
                print(buf.cix)

                trg = buf.trg_from_pos(pos)
                if trg is None:
                    raise Error("unexpected trigger: %r" % trg)
                print(banner("completions", '-'))
                ctlr = LogEvalController(log)
                buf.async_eval_at_trg(trg, ctlr)
                ctlr.wait(2)  #XXX
                if not ctlr.is_done():
                    ctlr.abort()
                    raise Error("XXX async eval timed out")
                pprint(ctlr.cplns)
                print(banner(None))
            finally:
                mgr.finalize()
        elif False:
            lang = "Ruby"
            path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
            markedup_content = dedent("""\
            r<1>equire 'net/http'
            include Net
            req = HTTPRequest.new
            req.<2>get()
            """)
            content, data = unmark_text(markedup_content)
            print(banner(path))
            print(_escaped_text_from_text(content, "whitespace"))
            pos = data[1]
            mgr = Manager()
            mgr.upgrade()
            mgr.initialize()
            try:
                buf = mgr.buf_from_content(content, lang=lang, path=path)
                print(banner("cix", '-'))
                cix = buf.cix
                print(
                    ET.tostring(pretty_tree_from_tree(tree_from_cix(buf.cix))))

                trg = buf.trg_from_pos(pos, implicit=False)
                if trg is None:
                    raise Error("unexpected trigger: %r" % trg)
                print(banner("completions", '-'))
                ctlr = LogEvalController(log)
                buf.async_eval_at_trg(trg, ctlr)
                ctlr.wait(30)  #XXX
                if not ctlr.is_done():
                    ctlr.abort()
                    raise Error("XXX async eval timed out")
                pprint(ctlr.cplns)
                print(banner(None))
            finally:
                mgr.finalize()
    def post_process_cplns(self, cplns):
        DEBUG = False
        if DEBUG:
            print banner("Perl post_process_cplns (before)")
            pprint(cplns)
        
        trg_type = self.trg.type
        if trg_type in ("package-subs", "object-subs"):
            #TODO: This may not be necessary if current evalr only
            #      generates the function.
            cplns = [c for c in cplns
                     if c[0] == "function"
                     if c[1] not in self._special_names_to_skip]
        elif trg_type == "package-members":
            if self.prefix_filter in ('', '&'): # filter out variables
                cplns = [c for c in cplns
                         if c[0] != "variable"
                         if c[1] not in self._special_names_to_skip]
            elif self.prefix_filter in ('$', '@', '%'): # filter out funcs
                cplns = [c for c in cplns
                         if c[0] != "function"
                         if c[1] not in self._special_names_to_skip]
            else:
                cplns = [c for c in cplns 
                         if c[1] not in self._special_names_to_skip]
                
            # Morph type and value of variable based on the prefix.
            # For example the completions for: `$HTTP::Message::` include
            # ("variable", "$VERSION"). The actual correct completion is
            # "VERSION" (no '$'-prefix). We morph this to ("$variable",
            # "VERSION") so that:
            # 1. the proper completion will be used, and
            # 2. the different type string can be used for a custom
            #    autocomplete image.
            # Ditto for '@' and '%'.
            morphed_cplns = []
            for type, value in cplns:
                if type == "variable":
                    match = self._perl_var_tokenizer.match(value)
                    if not match:
                        self.warn("could not parse Perl var '%s': "
                                  "pattern='%s'", value,
                                  self._perl_var_tokenizer.pattern)
                        continue
                    prefix, name = match.groups()
                    if DEBUG:
                        print "tokenize perl var: %r -> %r %r"\
                              % (value, prefix, name)
                    if prefix:
                        prefix = prefix[-1] # only last char is relevant
                    
                    if self.prefix_filter in (None, '*', '$'):
                        # '*': pass all
                        # '$': pass all because arrays and hashes can have
                        #      a '$' prefix for subsequent indexing
                        pass
                    elif self.prefix_filter and self.prefix_filter != prefix:
                        # If the filter is '%' or '@', then filter out vars
                        # not of that persuasion.
                        continue
                    
                    #TODO: Test cases for these and review by Perl guy.
                    if prefix in ('$', '%', '@'):
                        # Don't yet support '*' special a/c image.
                        type = prefix+type
                    value = name
                morphed_cplns.append((type, value))
            cplns = morphed_cplns

        cplns = CandidatesForTreeEvaluator.post_process_cplns(self, cplns)
        if DEBUG:
            print banner("(after)", '-')
            pprint(cplns)
            print banner(None, '-')
        return cplns