def citdl_expr_from_trg(self, buf, trg):
"""Return a Python CITDL expression preceding the given trigger.
The expression drops newlines, whitespace, and function call
arguments -- basically any stuff that is not used by the codeintel
database system for determining the resultant object type of the
expression. For example (in which <|> represents the given position):
GIVEN RETURN
----- ------
foo<|>. foo
foo(bar<|>. bar
foo(bar,blam)<|>. foo()
foo(bar, foo()
blam)<|>.
@foo<|>( foo
If (trg.form == TRG_FORM_DEFN), then it's similar to above, except it
looks forward to grab additional characters.
GIVEN RETURN
----- ------
foo<|>. foo
f<|>oo.bar foo.bar
foo(bar<|>. bar
foo(bar,blam)<|>. foo()
foo(bar, foo().bar
blam).b<|>ar
"""
DEBUG = False
if DEBUG:
print banner("Python-style citdl_expr_from_trg @ %d" % trg.pos)
if trg.form == TRG_FORM_DEFN:
pos = trg.pos
expr = self._citdl_expr_from_pos(buf,
pos,
implicit=True,
trg=trg,
include_forwards=True,
DEBUG=DEBUG)
if expr:
# Chop off any trailing "." characters
return expr.rstrip(".")
return expr
else:
if trg.type == 'array-members':
# Get everything before the bracket position.
pos = trg.extra.get('bracket_pos') - 1
else:
pos = trg.pos - 2 # skip ahead of the trigger char
return self._citdl_expr_from_pos(buf,
pos,
implicit=trg.implicit,
DEBUG=DEBUG,
trg=trg)
def citdl_expr_from_trg(self, buf, trg):
"""Return a Python CITDL expression preceding the given trigger.
The expression drops newlines, whitespace, and function call
arguments -- basically any stuff that is not used by the codeintel
database system for determining the resultant object type of the
expression. For example (in which <|> represents the given position):
GIVEN RETURN
----- ------
foo<|>. foo
foo(bar<|>. bar
foo(bar,blam)<|>. foo()
foo(bar, foo()
blam)<|>.
@foo<|>( foo
If (trg.form == TRG_FORM_DEFN), then it's similar to above, except it
looks forward to grab additional characters.
GIVEN RETURN
----- ------
foo<|>. foo
f<|>oo.bar foo.bar
foo(bar<|>. bar
foo(bar,blam)<|>. foo()
foo(bar, foo().bar
blam).b<|>ar
"""
DEBUG = False
if DEBUG:
print banner("Python-style citdl_expr_from_trg @ %d" % trg.pos)
if trg.form == TRG_FORM_DEFN:
pos = trg.pos
expr = self._citdl_expr_from_pos(buf, pos, implicit=True, trg=trg,
include_forwards=True, DEBUG=DEBUG)
if expr:
# Chop off any trailing "." characters
return expr.rstrip(".")
return expr
else:
if trg.type == 'array-members':
# Get everything before the bracket position.
pos = trg.extra.get('bracket_pos') - 1
else:
pos = trg.pos - 2 # skip ahead of the trigger char
return self._citdl_expr_from_pos(buf, pos, implicit=trg.implicit,
DEBUG=DEBUG, trg=trg)
def do_play(self, subcmd, opts):
"""Run my current play/dev code.
${cmd_usage}
${cmd_option_list}
"""
import pprint
import random
import ciElementTree as ET
from codeintel2.manager import Manager
from codeintel2.tree import pretty_tree_from_tree
from codeintel2.common import LogEvalController, Error
from codeintel2.util import tree_from_cix, dedent, unmark_text, banner
from ci2 import _escaped_text_from_text
if False:
lang = "CSS"
markedup_content = dedent("""
/* http://www.w3.org/TR/REC-CSS2/fonts.html#propdef-font-weight */
h1 {
border: 1px solid black;
font-weight /* hi */: <|> !important
}
""")
content, data = unmark_text(markedup_content)
pos = data["pos"]
mgr = Manager()
# mgr.upgrade() # Don't need it for just CSS usage.
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path="play.css")
trg = buf.trg_from_pos(pos)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
completions = buf.cplns_from_trg(trg)
print("COMPLETIONS: %r" % completions)
finally:
mgr.finalize()
elif False:
lang = "Python"
path = os.path.join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
markedup_content = dedent("""
import sys, os
class Foo:
def bar(self):
pass
sys.<|>path # should have path in completion list
f = Foo()
""")
content, data = unmark_text(markedup_content)
print(banner(path))
print(_escaped_text_from_text(content, "whitespace"))
pos = data["pos"]
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path=path)
print(banner("cix", '-'))
print(buf.cix)
trg = buf.trg_from_pos(pos)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
print(banner("completions", '-'))
ctlr = LogEvalController(self.log)
buf.async_eval_at_trg(trg, ctlr)
ctlr.wait(2) # XXX
if not ctlr.is_done():
ctlr.abort()
raise Error("XXX async eval timed out")
pprint.pprint(ctlr.cplns)
print(banner(None))
finally:
mgr.finalize()
elif False:
lang = "Ruby"
path = os.path.join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
markedup_content = dedent("""\
r<1>equire 'net/http'
include Net
req = HTTPRequest.new
req.<2>get()
""")
content, data = unmark_text(markedup_content)
print(banner(path))
print(_escaped_text_from_text(content, "whitespace"))
pos = data[1]
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path=path)
print(banner("cix", '-'))
cix = buf.cix
print(ET.tostring(pretty_tree_from_tree(tree_from_cix(cix))))
trg = buf.trg_from_pos(pos, implicit=False)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
print(banner("completions", '-'))
ctlr = LogEvalController(self.log)
buf.async_eval_at_trg(trg, ctlr)
ctlr.wait(30) # XXX
if not ctlr.is_done():
ctlr.abort()
raise Error("XXX async eval timed out")
pprint.pprint(ctlr.cplns)
print(banner(None))
finally:
mgr.finalize()
def _citdl_expr_from_pos(self,
buf,
pos,
implicit=False,
include_forwards=False,
DEBUG=False,
trg=None,
array_as_attr=False):
#PERF: Would dicts be faster for all of these?
WHITESPACE = tuple(" \t\n\r\v\f")
EOL = tuple("\r\n")
BLOCKCLOSES = tuple(")}]")
STOPOPS = tuple("({[,&+-=!^|%/<>;:#@")
EXTRA_STOPOPS_PRECEDING_IDENT = BLOCKCLOSES # Might be others.
#TODO: clean this up for LangIntel-usage
if implicit:
skip_styles = buf.implicit_completion_skip_styles
else:
skip_styles = buf.completion_skip_styles
string_styles = buf.string_styles()
comment_styles = buf.comment_styles()
#XXX Add sentinel num chars?
citdl_expr = []
accessor = buf.accessor
i = pos
is_udl_buffer = False
from codeintel2.udl import UDLBuffer
if isinstance(buf, UDLBuffer):
# We need to check for udl transition points and not go beyond the
# current sub-language, bug 95946.
is_udl_buffer = True
udl_lang = buf.lang_from_pos(pos)
# Move ahead to include forward chars as well
# We stop when we go out of the expression or when the expression is
# becomes a multiple fragment, i.e.
# 'sys.pa<|>th.expanduser' -> 'sys.path'
if include_forwards:
buf_length = accessor.length()
if i < buf_length:
max_look_ahead = min(buf_length, i + 100)
lastch_was_whitespace = False
while i < max_look_ahead:
ch = accessor.char_at_pos(i)
style = accessor.style_at_pos(i)
if is_udl_buffer and buf.lang_from_style(
style) != udl_lang:
if DEBUG:
print "UDL boundary at pos %d, changed from %r to %r" % (
i, udl_lang, buf.lang_from_style(style))
break
if ch in WHITESPACE:
lastch_was_whitespace = True
elif ch in ".)}]" or ch in STOPOPS:
break
elif lastch_was_whitespace:
break
else:
lastch_was_whitespace = False
i += 1
# Move back to last valid char
i -= 1
else:
i = buf_length - 1
if DEBUG:
if i > pos:
print "Including chars from pos %d up to %d" % (pos, i)
else:
print "No valid chars forward from pos %d, i now: %d" % (
pos, i)
# Be careful here, we cannot move from code into a comment, but we
# can be in a comment to begin with.
first_citdl_expr_style = None
first_citdl_expr_style_is_comment = False
while i >= 0:
ch = accessor.char_at_pos(i)
style = accessor.style_at_pos(i)
if is_udl_buffer and buf.lang_from_style(style) != udl_lang:
if DEBUG:
print "UDL boundary at pos %d, changed from %r to %r" % (
i, udl_lang, buf.lang_from_style(style))
break
if ch in WHITESPACE:
# drop all whitespace
while i >= 0:
ch = accessor.char_at_pos(i)
if ch in WHITESPACE \
or (ch == '\\' and accessor.char_at_pos(i+1) in EOL):
if DEBUG:
print "drop whitespace: %r" % accessor.char_at_pos(
i)
else:
break
i -= 1
# If there are two whitespace-separated words with no .
# in between we're changing expressions:
# if foo<|> and ...
# def foo<|>(...
if i >= 0 and citdl_expr and isident(citdl_expr[-1]) \
and ch != '.':
if DEBUG:
print "stop at non-dot: %r" % ch
break
elif style in string_styles: # Convert to string
citdl_type = self.citdl_from_literal_type.get("string")
if DEBUG:
print "found string style, converting to: %s and now " \
"finished" % (citdl_type)
if citdl_type:
citdl_expr += reversed(citdl_type)
break
elif style in skip_styles: # drop styles to ignore
while i >= 0 and accessor.style_at_pos(i) in skip_styles:
if DEBUG:
print "drop char of style to ignore: %r"\
% accessor.char_at_pos(i)
i -= 1
elif ch in STOPOPS or (
# This check ensures that, for example, we get "foo" instead
# of "bar()foo" in the following:
# bar()
# foo<|>.
citdl_expr and citdl_expr[-1] != '.' and ch
in EXTRA_STOPOPS_PRECEDING_IDENT):
if DEBUG:
print "stop at stop-operator %d: %r" % (i, ch)
break
elif ch in BLOCKCLOSES:
if DEBUG:
print "found block at %d: %r" % (i, ch)
citdl_expr.append(ch)
BLOCKS = { # map block close char to block open char
')': '(',
']': '[',
'}': '{',
}
stack = [] # stack of blocks: (<block close char>, <style>)
stack.append((ch, style, BLOCKS[ch], i))
i -= 1
num_lines = 0
content_styles = []
while i >= 0:
ch = accessor.char_at_pos(i)
style = accessor.style_at_pos(i)
content_styles.append(style)
if DEBUG:
print "finding matching brace: ch %r (%s), stack %r"\
% (ch, ', '.join(buf.style_names_from_style_num(style)), stack)
if ch in EOL:
num_lines += 1
if num_lines >= 3:
if DEBUG:
print "stop search for matching brace at 3 line sentinel"
break
elif ch in BLOCKS and style not in skip_styles:
stack.append((ch, style, BLOCKS[ch]))
elif ch == stack[-1][2] and style not in skip_styles:
#XXX Replace the second test with the following
# when LexPython+SilverCity styling bugs are fixed
# (spurious 'stderr' problem):
# and style == stack[-1][1]:
last_frame = stack.pop()
if not stack:
content_styles.pop() # Drop the thing that matched
if array_as_attr and \
all(style in string_styles for style in content_styles):
prop = accessor.text_range(
i + 1, last_frame[3])
if DEBUG:
print "Injecting %s" % (prop, )
citdl_expr.append(prop)
if DEBUG:
print "jump to matching brace at %d: %r" % (i,
ch)
citdl_expr.append(ch)
if trg and ch == "(":
# save the text in params, in case the completion
# needs to special-case things depending on the
# argument.
params = trg.extra.setdefault("_params", [])
params.insert(
0,
accessor.text_range(i + 1, last_frame[3]))
i -= 1
break
i -= 1
else:
# Didn't find the matching brace.
if DEBUG:
print "couldn't find matching brace"
raise EvalError("could not find matching brace for "
"'%s' at position %d" %
(stack[-1][0], stack[-1][3]))
else:
if DEBUG:
style_names = buf.style_names_from_style_num(style)
print "add char: %r (%s)" % (ch, ', '.join(style_names))
if first_citdl_expr_style is None:
# Remember the first citdl style we found
first_citdl_expr_style = style
first_citdl_expr_style_is_comment = style in comment_styles
elif first_citdl_expr_style != style and \
(first_citdl_expr_style_is_comment or
style in comment_styles):
# We've moved into or out of a comment, let's leave now
# Fixes: http://bugs.activestate.com/show_bug.cgi?id=65672
break
citdl_expr.append(ch)
i -= 1
citdl_expr.reverse()
citdl_expr = ''.join(citdl_expr)
if DEBUG:
print "return: %r" % citdl_expr
print banner("done")
return citdl_expr
def preceding_trg_from_pos(self,
buf,
pos,
curr_pos,
preceding_trg_terminators=None,
DEBUG=False):
accessor = buf.accessor
if preceding_trg_terminators is None:
preceding_trg_terminators = self.preceding_trg_terminators
if DEBUG:
print banner("preceding_trg_from_pos(pos=%r, curr_pos=%r)" %
(pos, curr_pos))
print indent(
markup_text(accessor.text, pos=curr_pos, start_pos=pos))
print banner(None, '-')
# Skip over comments and strings in our checking, unless we are
# in one of these styles for the whole range. This is so an explicit
# trigger in a comment (or, e.g., a doc string) will work, but
# the appearance of small comments or strings in code will not mess
# things up.
comment_and_string_styles = dict(
(s, True) for s in buf.comment_styles() + buf.string_styles())
skip_styles = {}
start_style = accessor.style_at_pos(pos - 1)
EOL_CHARS = tuple("\n\r")
# Limiting simplification: Only backtrack a max of 200 chars.
# Can increase that if necessary. The problem is detecting a
# statement boundary backwards in langs like Python and Ruby
# where you can't rely on ';' (actually
# `preceding_trg_terminators').
limit = max(1, pos - 200)
# First stage. We only consider autocomplete trigger (i.e.
# trg.form==TRG_FORM_COMPLETION) if within range of the
# curr_pos. Here "within range" means you don't have to more the
# cursor to show the autocomplete UI.
first_stage_limit = curr_pos
for (char, style) in accessor.gen_char_and_style_back(
curr_pos - 1, limit - 1):
if not isident(char):
break
first_stage_limit -= 1
if DEBUG:
print "[stage 1] first_stage_limit=%d (prev_ch=%r)"\
% (first_stage_limit,
(first_stage_limit > 0
and accessor.char_at_pos(first_stage_limit-1)
or None))
p = pos
if p >= first_stage_limit:
for (prev_ch, prev_style) in accessor.gen_char_and_style_back(
p - 1, first_stage_limit - 2):
if (not skip_styles and prev_style != start_style
# EOLs in comments seem to always be style 0. Don't count
# them.
and prev_ch not in EOL_CHARS):
if DEBUG:
print "[stage 1] have seen a style change (%d -> %d), " \
"now skipping strings and comments" \
% (start_style, prev_style)
skip_styles = comment_and_string_styles
if DEBUG:
print "[stage 1] consider pos %2d: prev_ch=%r (%d) --"\
% (p, prev_ch, prev_style),
if prev_style in skip_styles:
if DEBUG: print "comment or string, skip it"
elif self._is_terminating_char(prev_ch, prev_style,
preceding_trg_terminators):
if DEBUG: print "in `preceding_trg_terminators': break"
return None
elif prev_ch in self.trg_chars:
if DEBUG: print "trigger char, try it"
trg = buf.trg_from_pos(p, implicit=False)
if trg:
if DEBUG: print "[stage 1] %s" % trg
return trg
p -= 1
break
elif DEBUG:
print "not a trigger char, skip it"
p -= 1
if DEBUG:
print "[stage 1] end of possible autocomplete trigger range"
# Second stage. We only consider calltip triggers now
# (self.calltip_trg_chars).
#
# As well, ignore enclosed paren sections to make sure we are
# in-range. For example, we shouldn't trigger on "bar(" here:
# foo(bar("skip", "this", "arg", "list"), <|>)
close_paren_count = 0
for (prev_ch,
prev_style) in accessor.gen_char_and_style_back(p - 1, limit - 2):
if (not skip_styles and prev_style != start_style
# EOLs in comments seem to always be style 0. Don't count
# them.
and prev_ch not in EOL_CHARS):
if DEBUG:
print "[stage 2] seen a style change (%d -> %d), now " \
"skipping strings and comments" \
% (start_style, prev_style)
skip_styles = comment_and_string_styles
if DEBUG:
print "[stage 2] consider pos %2d: prev_ch=%r (%d) --"\
% (p, prev_ch, prev_style),
if prev_style in skip_styles:
if DEBUG: print "comment or string, skip it"
elif prev_ch == ')':
close_paren_count += 1
if DEBUG: print "close paren: count=%d" % close_paren_count
elif close_paren_count and prev_ch == '(':
close_paren_count -= 1
if DEBUG: print "open paren: count=%d" % close_paren_count
elif self._is_terminating_char(prev_ch, prev_style,
preceding_trg_terminators):
if DEBUG: print "in `preceding_trg_terminators': break"
return None
elif prev_ch in self.calltip_trg_chars:
if DEBUG: print "trigger char, try it"
trg = buf.trg_from_pos(p, implicit=False)
if trg:
if DEBUG: print "[stage 2] %s" % trg
return trg
elif DEBUG:
print "not a trigger char, skip it"
p -= 1
return None
def curr_calltip_arg_range(self,
buf,
trg_pos,
calltip,
curr_pos,
DEBUG=False):
"""Return that range in the calltip of the "current" arg.
I.e. what argument is currently being entered.
"buf" is the buffer object on which this is being done.
"trg_pos" is the trigger position.
"calltip" is the full calltip text.
"curr_pos" is the current position in the buffer.
Returns a range: (start, end)
Set `start == -1` to cancel the calltip, i.e. if the entered text
has closed the call region.
The default implementation uses:
self.calltip_region_terminators
to handle languages with calltip signatures with the following
characteristics:
- uses '(' and ')' to bound the argument list (though because of
support for ';' statement termination, this isn't absolutely
required)
- uses a comma to separate arguments
- basic block delimiters are {}, (), and []
For example:
foo()
blam(a, b)
range([start,] stop[, step]) -> list of integers
bar(arg1, *args, **kwargs)
flash(boom, bang=42)
"""
# Dev Notes:
# - Eventually should pass in the trigger to aid in processing.
# - TODO figure out dependence on buf.comment_styles() and
# buf.string_styles()
accessor = buf.accessor
if DEBUG:
print banner("curr_calltip_arg_range")
print "calltip:\n%s" % indent(calltip)
print "buffer:\n%s" % indent(
markup_text(accessor.text, trg_pos=trg_pos, pos=curr_pos))
# Start from the trigger position and walk forward to the current
# pos: counting args and looking for termination of the calltip
# region.
skip_styles = dict(
(s, True) for s in buf.comment_styles() + buf.string_styles())
if accessor.style_at_pos(trg_pos - 1) in skip_styles:
skip_styles = {}
comma_count = 0
blocks = {
# Map a block start token to its block end token.
'(': ')',
'[': ']',
'{': '}',
}
block_stack = []
p = trg_pos
for ch, style in accessor.gen_char_and_style(trg_pos, curr_pos):
if DEBUG: print "pos %2d: %r (%2s) --" % (p, ch, style),
if style in skip_styles:
if DEBUG: print "skip"
elif ch in blocks:
if DEBUG: print "open block"
block_stack.append(blocks[ch])
elif block_stack:
if ch == block_stack[-1]:
if DEBUG: print "close block"
block_stack.pop()
elif ch in self.calltip_region_terminators:
if DEBUG: print "end of call region: (-1, -1)"
return (-1, -1)
elif DEBUG:
print "ignore (in block)"
elif ch == ',':
if DEBUG: print "next arg"
comma_count += 1
elif ch in self.calltip_region_terminators and \
self.calltip_verify_termination(accessor, ch, trg_pos, curr_pos):
if DEBUG: print "end of call region: (-1, -1)"
return (-1, -1)
elif DEBUG:
print "ignore"
p += 1
# Parse the signature from the calltip. If there is no signature
# then we default to not indicating any arg range.
if self._parsed_calltip_cache[0] == calltip:
parsed = self._parsed_calltip_cache[1]
else:
parsed = _parse_calltip(calltip, DEBUG)
self._parsed_calltip_cache = (calltip, parsed)
if parsed is None:
if DEBUG: print "couldn't parse any calltip: (0, 0)"
return (0, 0)
signature, name, args = parsed
if DEBUG:
print "parsed calltip:\n signature:\n%s\n name:\n%s\n args:\n%s"\
% (indent(signature), indent(name), indent(pformat(args)))
if not args:
if DEBUG: print "no args in signature: (0, 0)"
return (0, 0)
elif comma_count >= len(args):
#XXX ellipsis
if DEBUG: print "more commas than args: ellipsis?"
span = args[-1].span # default to last arg
else:
span = args[comma_count].span
if DEBUG:
print "curr calltip range (%s, %s):" % (span[0], span[1])
print indent(signature)
print " %s%s" % (' ' * span[0], '-' * (span[1] - span[0]))
return span
def post_process_cplns(self, cplns):
DEBUG = False
if DEBUG:
print banner("Perl post_process_cplns (before)")
pprint(cplns)
trg_type = self.trg.type
if trg_type in ("package-subs", "object-subs"):
#TODO: This may not be necessary if current evalr only
# generates the function.
cplns = [
c for c in cplns if c[0] == "function"
if c[1] not in self._special_names_to_skip
]
elif trg_type == "package-members":
if self.prefix_filter in ('', '&'): # filter out variables
cplns = [
c for c in cplns if c[0] != "variable"
if c[1] not in self._special_names_to_skip
]
elif self.prefix_filter in ('$', '@', '%'): # filter out funcs
cplns = [
c for c in cplns if c[0] != "function"
if c[1] not in self._special_names_to_skip
]
else:
cplns = [
c for c in cplns if c[1] not in self._special_names_to_skip
]
# Morph type and value of variable based on the prefix.
# For example the completions for: `$HTTP::Message::` include
# ("variable", "$VERSION"). The actual correct completion is
# "VERSION" (no '$'-prefix). We morph this to ("$variable",
# "VERSION") so that:
# 1. the proper completion will be used, and
# 2. the different type string can be used for a custom
# autocomplete image.
# Ditto for '@' and '%'.
morphed_cplns = []
for type, value in cplns:
if type == "variable":
match = self._perl_var_tokenizer.match(value)
if not match:
self.warn(
"could not parse Perl var '%s': "
"pattern='%s'", value,
self._perl_var_tokenizer.pattern)
continue
prefix, name = match.groups()
if DEBUG:
print "tokenize perl var: %r -> %r %r"\
% (value, prefix, name)
if prefix:
prefix = prefix[-1] # only last char is relevant
if self.prefix_filter in (None, '*', '$'):
# '*': pass all
# '$': pass all because arrays and hashes can have
# a '$' prefix for subsequent indexing
pass
elif self.prefix_filter and self.prefix_filter != prefix:
# If the filter is '%' or '@', then filter out vars
# not of that persuasion.
continue
#TODO: Test cases for these and review by Perl guy.
if prefix in ('$', '%', '@'):
# Don't yet support '*' special a/c image.
type = prefix + type
value = name
morphed_cplns.append((type, value))
cplns = morphed_cplns
cplns = CandidatesForTreeEvaluator.post_process_cplns(self, cplns)
if DEBUG:
print banner("(after)", '-')
pprint(cplns)
print banner(None, '-')
return cplns
def _citdl_expr_from_pos(self, buf, pos, implicit=False,
include_forwards=False, DEBUG=False, trg=None):
#PERF: Would dicts be faster for all of these?
WHITESPACE = tuple(" \t\n\r\v\f")
EOL = tuple("\r\n")
BLOCKCLOSES = tuple(")}]")
STOPOPS = tuple("({[,&+-=!^|%/<>;:#@")
EXTRA_STOPOPS_PRECEDING_IDENT = BLOCKCLOSES # Might be others.
#TODO: clean this up for LangIntel-usage
if implicit:
skip_styles = buf.implicit_completion_skip_styles
else:
skip_styles = buf.completion_skip_styles
string_styles = buf.string_styles()
comment_styles = buf.comment_styles()
#XXX Add sentinel num chars?
citdl_expr = []
accessor = buf.accessor
i = pos
# Move ahead to include forward chars as well
# We stop when we go out of the expression or when the expression is
# becomes a multiple fragment, i.e.
# 'sys.pa<|>th.expanduser' -> 'sys.path'
if include_forwards:
buf_length = accessor.length()
if i < buf_length:
max_look_ahead = min(buf_length, i+100)
lastch_was_whitespace = False
while i < max_look_ahead:
ch = accessor.char_at_pos(i)
style = accessor.style_at_pos(i)
if ch in WHITESPACE:
lastch_was_whitespace = True
elif ch in ".)}]" or ch in STOPOPS:
break
elif lastch_was_whitespace:
break
else:
lastch_was_whitespace = False
i += 1
# Move back to last valid char
i -= 1
else:
i = buf_length - 1
if DEBUG:
if i > pos:
print "Including chars from pos %d up to %d" % (pos, i)
else:
print "No valid chars forward from pos %d, i now: %d" % (pos, i)
# Be careful here, we cannot move from code into a comment, but we
# can be in a comment to begin with.
first_citdl_expr_style = None
first_citdl_expr_style_is_comment = False
while i >= 0:
ch = accessor.char_at_pos(i)
style = accessor.style_at_pos(i)
if ch in WHITESPACE:
# drop all whitespace
while i >= 0:
ch = accessor.char_at_pos(i)
if ch in WHITESPACE \
or (ch == '\\' and accessor.char_at_pos(i+1) in EOL):
if DEBUG:
print "drop whitespace: %r" % accessor.char_at_pos(i)
else:
break
i -= 1
# If there are two whitespace-separated words with no .
# in between we're changing expressions:
# if foo<|> and ...
# def foo<|>(...
if i >= 0 and citdl_expr and isident(citdl_expr[-1]) \
and ch != '.':
if DEBUG:
print "stop at non-dot: %r" % ch
break
elif style in string_styles: # Convert to string
citdl_type = self.citdl_from_literal_type.get("string")
if DEBUG:
print "found string style, converting to: %s and now " \
"finished" % (citdl_type)
if citdl_type:
citdl_expr += reversed(citdl_type)
break
elif style in skip_styles: # drop styles to ignore
while i >= 0 and accessor.style_at_pos(i) in skip_styles:
if DEBUG:
print "drop char of style to ignore: %r"\
% accessor.char_at_pos(i)
i -= 1
elif ch in STOPOPS or (
# This check ensures that, for example, we get "foo" instead
# of "bar()foo" in the following:
# bar()
# foo<|>.
citdl_expr and citdl_expr[-1] != '.'
and ch in EXTRA_STOPOPS_PRECEDING_IDENT):
if DEBUG:
print "stop at stop-operator %d: %r" % (i, ch)
break
elif ch in BLOCKCLOSES:
if DEBUG:
print "found block at %d: %r" % (i, ch)
citdl_expr.append(ch)
BLOCKS = { # map block close char to block open char
')': '(',
']': '[',
'}': '{',
}
stack = [] # stack of blocks: (<block close char>, <style>)
stack.append( (ch, style, BLOCKS[ch], i) )
i -= 1
num_lines = 0
while i >= 0:
ch = accessor.char_at_pos(i)
style = accessor.style_at_pos(i)
if DEBUG:
print "finding matching brace: ch %r (%s), stack %r"\
% (ch, ', '.join(buf.style_names_from_style_num(style)), stack)
if ch in EOL:
num_lines += 1
if num_lines >= 3:
if DEBUG: print "stop search for matching brace at 3 line sentinel"
break
elif ch in BLOCKS and style not in skip_styles:
stack.append( (ch, style, BLOCKS[ch]) )
elif ch == stack[-1][2] and style not in skip_styles:
#XXX Replace the second test with the following
# when LexPython+SilverCity styling bugs are fixed
# (spurious 'stderr' problem):
# and style == stack[-1][1]:
last_frame = stack.pop()
if not stack:
if DEBUG:
print "jump to matching brace at %d: %r" % (i, ch)
citdl_expr.append(ch)
if trg:
# save the text in params, in case the completion
# needs to special-case things depending on the
# argument.
trg.extra["_params"] = accessor.text_range(i + 1, last_frame[3])
i -= 1
break
i -= 1
else:
# Didn't find the matching brace.
if DEBUG:
print "couldn't find matching brace"
raise EvalError("could not find matching brace for "
"'%s' at position %d"
% (stack[-1][0], stack[-1][3]))
else:
if DEBUG:
style_names = buf.style_names_from_style_num(style)
print "add char: %r (%s)" % (ch, ', '.join(style_names))
if first_citdl_expr_style is None:
# Remember the first citdl style we found
first_citdl_expr_style = style
first_citdl_expr_style_is_comment = style in comment_styles
elif first_citdl_expr_style != style and \
(first_citdl_expr_style_is_comment or
style in comment_styles):
# We've moved into or out of a comment, let's leave now
# Fixes: http://bugs.activestate.com/show_bug.cgi?id=65672
break
citdl_expr.append(ch)
i -= 1
citdl_expr.reverse()
citdl_expr = ''.join(citdl_expr)
if DEBUG:
print "return: %r" % citdl_expr
print banner("done")
return citdl_expr
def preceding_trg_from_pos(self, buf, pos, curr_pos,
preceding_trg_terminators=None, DEBUG=False):
accessor = buf.accessor
if preceding_trg_terminators is None:
preceding_trg_terminators = self.preceding_trg_terminators
if DEBUG:
print banner("preceding_trg_from_pos(pos=%r, curr_pos=%r)"
% (pos, curr_pos))
print indent(markup_text(accessor.text, pos=curr_pos,
start_pos=pos))
print banner(None, '-')
# Skip over comments and strings in our checking, unless we are
# in one of these styles for the whole range. This is so an explicit
# trigger in a comment (or, e.g., a doc string) will work, but
# the appearance of small comments or strings in code will not mess
# things up.
comment_and_string_styles = dict(
(s, True) for s in buf.comment_styles() + buf.string_styles())
skip_styles = {}
start_style = accessor.style_at_pos(pos-1)
EOL_CHARS = tuple("\n\r")
# Limiting simplification: Only backtrack a max of 200 chars.
# Can increase that if necessary. The problem is detecting a
# statement boundary backwards in langs like Python and Ruby
# where you can't rely on ';' (actually
# `preceding_trg_terminators').
limit = max(1, pos - 200)
# First stage. We only consider autocomplete trigger (i.e.
# trg.form==TRG_FORM_COMPLETION) if within range of the
# curr_pos. Here "within range" means you don't have to more the
# cursor to show the autocomplete UI.
first_stage_limit = curr_pos
for (char, style) in accessor.gen_char_and_style_back(curr_pos-1,
limit-1):
if not isident(char):
break
first_stage_limit -= 1
if DEBUG:
print "[stage 1] first_stage_limit=%d (prev_ch=%r)"\
% (first_stage_limit,
(first_stage_limit > 0
and accessor.char_at_pos(first_stage_limit-1)
or None))
p = pos
if p >= first_stage_limit:
for (prev_ch, prev_style) in accessor.gen_char_and_style_back(p-1,
first_stage_limit-2):
if (not skip_styles and prev_style != start_style
# EOLs in comments seem to always be style 0. Don't count
# them.
and prev_ch not in EOL_CHARS):
if DEBUG:
print "[stage 1] have seen a style change (%d -> %d), " \
"now skipping strings and comments" \
% (start_style, prev_style)
skip_styles = comment_and_string_styles
if DEBUG:
print "[stage 1] consider pos %2d: prev_ch=%r (%d) --"\
% (p, prev_ch, prev_style),
if prev_style in skip_styles:
if DEBUG: print "comment or string, skip it"
elif self._is_terminating_char(prev_ch, prev_style,
preceding_trg_terminators):
if DEBUG: print "in `preceding_trg_terminators': break"
return None
elif prev_ch in self.trg_chars:
if DEBUG: print "trigger char, try it"
trg = buf.trg_from_pos(p, implicit=False)
if trg:
if DEBUG: print "[stage 1] %s" % trg
return trg
p -= 1
break
elif DEBUG:
print "not a trigger char, skip it"
p -= 1
if DEBUG:
print "[stage 1] end of possible autocomplete trigger range"
# Second stage. We only consider calltip triggers now
# (self.calltip_trg_chars).
#
# As well, ignore enclosed paren sections to make sure we are
# in-range. For example, we shouldn't trigger on "bar(" here:
# foo(bar("skip", "this", "arg", "list"), <|>)
close_paren_count = 0
for (prev_ch, prev_style) in accessor.gen_char_and_style_back(p-1, limit-2):
if (not skip_styles and prev_style != start_style
# EOLs in comments seem to always be style 0. Don't count
# them.
and prev_ch not in EOL_CHARS):
if DEBUG:
print "[stage 2] seen a style change (%d -> %d), now " \
"skipping strings and comments" \
% (start_style, prev_style)
skip_styles = comment_and_string_styles
if DEBUG:
print "[stage 2] consider pos %2d: prev_ch=%r (%d) --"\
% (p, prev_ch, prev_style),
if prev_style in skip_styles:
if DEBUG: print "comment or string, skip it"
elif prev_ch == ')':
close_paren_count += 1
if DEBUG: print "close paren: count=%d" % close_paren_count
elif close_paren_count and prev_ch == '(':
close_paren_count -= 1
if DEBUG: print "open paren: count=%d" % close_paren_count
elif self._is_terminating_char(prev_ch, prev_style,
preceding_trg_terminators):
if DEBUG: print "in `preceding_trg_terminators': break"
return None
elif prev_ch in self.calltip_trg_chars:
if DEBUG: print "trigger char, try it"
trg = buf.trg_from_pos(p, implicit=False)
if trg:
if DEBUG: print "[stage 2] %s" % trg
return trg
elif DEBUG:
print "not a trigger char, skip it"
p -= 1
return None
def curr_calltip_arg_range(self, buf, trg_pos, calltip, curr_pos,
DEBUG=False):
"""Return that range in the calltip of the "current" arg.
I.e. what argument is currently being entered.
"buf" is the buffer object on which this is being done.
"trg_pos" is the trigger position.
"calltip" is the full calltip text.
"curr_pos" is the current position in the buffer.
Returns a range: (start, end)
Set `start == -1` to cancel the calltip, i.e. if the entered text
has closed the call region.
The default implementation uses:
self.calltip_region_terminators
to handle languages with calltip signatures with the following
characteristics:
- uses '(' and ')' to bound the argument list (though because of
support for ';' statement termination, this isn't absolutely
required)
- uses a comma to separate arguments
- basic block delimiters are {}, (), and []
For example:
foo()
blam(a, b)
range([start,] stop[, step]) -> list of integers
bar(arg1, *args, **kwargs)
flash(boom, bang=42)
"""
# Dev Notes:
# - Eventually should pass in the trigger to aid in processing.
# - TODO figure out dependence on buf.comment_styles() and
# buf.string_styles()
accessor = buf.accessor
if DEBUG:
print banner("curr_calltip_arg_range")
print "calltip:\n%s" % indent(calltip)
print "buffer:\n%s" % indent(markup_text(accessor.text,
trg_pos=trg_pos,
pos=curr_pos))
# Start from the trigger position and walk forward to the current
# pos: counting args and looking for termination of the calltip
# region.
skip_styles = dict(
(s, True) for s in buf.comment_styles() + buf.string_styles())
if accessor.style_at_pos(trg_pos-1) in skip_styles:
skip_styles = {}
comma_count = 0
blocks = {
# Map a block start token to its block end token.
'(': ')', '[': ']', '{': '}',
}
block_stack = []
p = trg_pos
for ch, style in accessor.gen_char_and_style(trg_pos, curr_pos):
if DEBUG: print "pos %2d: %r (%2s) --" % (p, ch, style),
if style in skip_styles:
if DEBUG: print "skip"
elif ch in blocks:
if DEBUG: print "open block"
block_stack.append(blocks[ch])
elif block_stack:
if ch == block_stack[-1]:
if DEBUG: print "close block"
block_stack.pop()
elif ch in self.calltip_region_terminators:
if DEBUG: print "end of call region: (-1, -1)"
return (-1, -1)
elif DEBUG:
print "ignore (in block)"
elif ch == ',':
if DEBUG: print "next arg"
comma_count += 1
elif ch in self.calltip_region_terminators and \
self.calltip_verify_termination(accessor, ch, trg_pos, curr_pos):
if DEBUG: print "end of call region: (-1, -1)"
return (-1, -1)
elif DEBUG:
print "ignore"
p += 1
# Parse the signature from the calltip. If there is no signature
# then we default to not indicating any arg range.
if self._parsed_calltip_cache[0] == calltip:
parsed = self._parsed_calltip_cache[1]
else:
parsed = _parse_calltip(calltip, DEBUG)
self._parsed_calltip_cache = (calltip, parsed)
if parsed is None:
if DEBUG: print "couldn't parse any calltip: (0, 0)"
return (0, 0)
signature, name, args = parsed
if DEBUG:
print "parsed calltip:\n signature:\n%s\n name:\n%s\n args:\n%s"\
% (indent(signature), indent(name), indent(pformat(args)))
if not args:
if DEBUG: print "no args in signature: (0, 0)"
return (0, 0)
elif comma_count >= len(args):
#XXX ellipsis
if DEBUG: print "more commas than args: ellipsis?"
span = args[-1].span # default to last arg
else:
span = args[comma_count].span
if DEBUG:
print "curr calltip range (%s, %s):" % (span[0], span[1])
print indent(signature)
print " %s%s" % (' '*span[0], '-'*(span[1]-span[0]))
return span
def do_play(self, subcmd, opts):
"""Run my current play/dev code.
${cmd_usage}
${cmd_option_list}
"""
if False:
lang = "CSS"
markedup_content = dedent("""
/* http://www.w3.org/TR/REC-CSS2/fonts.html#propdef-font-weight */
h1 {
border: 1px solid black;
font-weight /* hi */: <|> !important
}
""")
content, data = unmark_text(markedup_content)
pos = data["pos"]
mgr = Manager()
#mgr.upgrade() # Don't need it for just CSS usage.
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path="play.css")
trg = buf.trg_from_pos(pos)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
completions = buf.cplns_from_trg(trg)
print("COMPLETIONS: %r" % completions)
finally:
mgr.finalize()
elif False:
lang = "Python"
path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
markedup_content = dedent("""
import sys, os
class Foo:
def bar(self):
pass
sys.<|>path # should have path in completion list
f = Foo()
""")
content, data = unmark_text(markedup_content)
print(banner(path))
print(_escaped_text_from_text(content, "whitespace"))
pos = data["pos"]
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path=path)
print(banner("cix", '-'))
print(buf.cix)
trg = buf.trg_from_pos(pos)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
print(banner("completions", '-'))
ctlr = LogEvalController(log)
buf.async_eval_at_trg(trg, ctlr)
ctlr.wait(2) #XXX
if not ctlr.is_done():
ctlr.abort()
raise Error("XXX async eval timed out")
pprint(ctlr.cplns)
print(banner(None))
finally:
mgr.finalize()
elif False:
lang = "Ruby"
path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
markedup_content = dedent("""\
r<1>equire 'net/http'
include Net
req = HTTPRequest.new
req.<2>get()
""")
content, data = unmark_text(markedup_content)
print(banner(path))
print(_escaped_text_from_text(content, "whitespace"))
pos = data[1]
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path=path)
print(banner("cix", '-'))
cix = buf.cix
print(
ET.tostring(pretty_tree_from_tree(tree_from_cix(buf.cix))))
trg = buf.trg_from_pos(pos, implicit=False)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
print(banner("completions", '-'))
ctlr = LogEvalController(log)
buf.async_eval_at_trg(trg, ctlr)
ctlr.wait(30) #XXX
if not ctlr.is_done():
ctlr.abort()
raise Error("XXX async eval timed out")
pprint(ctlr.cplns)
print(banner(None))
finally:
mgr.finalize()
def post_process_cplns(self, cplns):
DEBUG = False
if DEBUG:
print banner("Perl post_process_cplns (before)")
pprint(cplns)
trg_type = self.trg.type
if trg_type in ("package-subs", "object-subs"):
#TODO: This may not be necessary if current evalr only
# generates the function.
cplns = [c for c in cplns
if c[0] == "function"
if c[1] not in self._special_names_to_skip]
elif trg_type == "package-members":
if self.prefix_filter in ('', '&'): # filter out variables
cplns = [c for c in cplns
if c[0] != "variable"
if c[1] not in self._special_names_to_skip]
elif self.prefix_filter in ('$', '@', '%'): # filter out funcs
cplns = [c for c in cplns
if c[0] != "function"
if c[1] not in self._special_names_to_skip]
else:
cplns = [c for c in cplns
if c[1] not in self._special_names_to_skip]
# Morph type and value of variable based on the prefix.
# For example the completions for: `$HTTP::Message::` include
# ("variable", "$VERSION"). The actual correct completion is
# "VERSION" (no '$'-prefix). We morph this to ("$variable",
# "VERSION") so that:
# 1. the proper completion will be used, and
# 2. the different type string can be used for a custom
# autocomplete image.
# Ditto for '@' and '%'.
morphed_cplns = []
for type, value in cplns:
if type == "variable":
match = self._perl_var_tokenizer.match(value)
if not match:
self.warn("could not parse Perl var '%s': "
"pattern='%s'", value,
self._perl_var_tokenizer.pattern)
continue
prefix, name = match.groups()
if DEBUG:
print "tokenize perl var: %r -> %r %r"\
% (value, prefix, name)
if prefix:
prefix = prefix[-1] # only last char is relevant
if self.prefix_filter in (None, '*', '$'):
# '*': pass all
# '$': pass all because arrays and hashes can have
# a '$' prefix for subsequent indexing
pass
elif self.prefix_filter and self.prefix_filter != prefix:
# If the filter is '%' or '@', then filter out vars
# not of that persuasion.
continue
#TODO: Test cases for these and review by Perl guy.
if prefix in ('$', '%', '@'):
# Don't yet support '*' special a/c image.
type = prefix+type
value = name
morphed_cplns.append((type, value))
cplns = morphed_cplns
cplns = CandidatesForTreeEvaluator.post_process_cplns(self, cplns)
if DEBUG:
print banner("(after)", '-')
pprint(cplns)
print banner(None, '-')
return cplns