def do_play(self, subcmd, opts):
"""Run my current play/dev code.
${cmd_usage}
${cmd_option_list}
"""
import pprint
import random
import ciElementTree as ET
from codeintel2.manager import Manager
from codeintel2.tree import pretty_tree_from_tree
from codeintel2.common import LogEvalController, Error
from codeintel2.util import tree_from_cix, dedent, unmark_text, banner
from ci2 import _escaped_text_from_text
if False:
lang = "CSS"
markedup_content = dedent("""
/* http://www.w3.org/TR/REC-CSS2/fonts.html#propdef-font-weight */
h1 {
border: 1px solid black;
font-weight /* hi */: <|> !important
}
""")
content, data = unmark_text(markedup_content)
pos = data["pos"]
mgr = Manager()
# mgr.upgrade() # Don't need it for just CSS usage.
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path="play.css")
trg = buf.trg_from_pos(pos)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
completions = buf.cplns_from_trg(trg)
print("COMPLETIONS: %r" % completions)
finally:
mgr.finalize()
elif False:
lang = "Python"
path = os.path.join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
markedup_content = dedent("""
import sys, os
class Foo:
def bar(self):
pass
sys.<|>path # should have path in completion list
f = Foo()
""")
content, data = unmark_text(markedup_content)
print(banner(path))
print(_escaped_text_from_text(content, "whitespace"))
pos = data["pos"]
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path=path)
print(banner("cix", '-'))
print(buf.cix)
trg = buf.trg_from_pos(pos)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
print(banner("completions", '-'))
ctlr = LogEvalController(self.log)
buf.async_eval_at_trg(trg, ctlr)
ctlr.wait(2) # XXX
if not ctlr.is_done():
ctlr.abort()
raise Error("XXX async eval timed out")
pprint.pprint(ctlr.cplns)
print(banner(None))
finally:
mgr.finalize()
elif False:
lang = "Ruby"
path = os.path.join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
markedup_content = dedent("""\
r<1>equire 'net/http'
include Net
req = HTTPRequest.new
req.<2>get()
""")
content, data = unmark_text(markedup_content)
print(banner(path))
print(_escaped_text_from_text(content, "whitespace"))
pos = data[1]
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path=path)
print(banner("cix", '-'))
cix = buf.cix
print(ET.tostring(pretty_tree_from_tree(tree_from_cix(cix))))
trg = buf.trg_from_pos(pos, implicit=False)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
print(banner("completions", '-'))
ctlr = LogEvalController(self.log)
buf.async_eval_at_trg(trg, ctlr)
ctlr.wait(30) # XXX
if not ctlr.is_done():
ctlr.abort()
raise Error("XXX async eval timed out")
pprint.pprint(ctlr.cplns)
print(banner(None))
finally:
mgr.finalize()
class Driver(threading.Thread):
"""
Out-of-process codeintel2 driver
This class implements the main loop of the codeintel worker process
This should be a singleton object
"""
# static
_instance = None
_command_handler_map = collections.defaultdict(list)
"""Registered command handlers; the key is the command name (a str), and the
value is a list of command handler instances or (for lazy handlers) a
single-element tuple containing the callable to get the real handler."""
_default_handler = None
"""The default (core) command handler; instance of a CoreHandler"""
_builtin_commands = {}
"""Built-in commands that cannot be overridden"""
def __init__(self, db_base_dir=None, fd_in=sys.stdin, fd_out=sys.stdout):
threading.Thread.__init__(self, name="CodeIntel OOP Driver")
assert Driver._instance is None, "Driver should be a singleton"
Driver._instance = self
logging.root.addHandler(LoggingHandler(self))
self.daemon = True
self.fd_in = fd_in
self.fd_out = fd_out
self.abort = None
self.quit = False
self.buffers = {} # path to Buffer objects
self.next_buffer = 0
self.active_request = None
self.send_queue = Queue.Queue()
self.send_thread = threading.Thread(name="Codeintel OOP Driver Send Thread",
target=self._send_proc)
self.send_thread.daemon = True
self.send_thread.start()
self.queue = collections.deque()
self.queue_cv = threading.Condition()
self.env = Environment(name="global",
send_fn=functools.partial(self.send, request=None))
# Fill out the non-overridable build-in commands
self._builtin_commands = {}
for attr in dir(self):
# Note that we check startswith first to avoid getters etc.
if attr.startswith("do_") and callable(getattr(self, attr)):
command = attr[len("do_"):].replace("_", "-")
self._builtin_commands[command] = getattr(self, attr)
from codeintel2.manager import Manager
log.debug("using db base dir %s", db_base_dir)
self.mgr = Manager(db_base_dir=db_base_dir,
db_catalog_dirs=[],
db_event_reporter=self._DBEventReporter(self),
env=self.env,
on_scan_complete=self._on_scan_complete)
self.mgr.initialize()
def _on_scan_complete(self, scan_request):
if scan_request.status in ("changed", "skipped"):
# Send unsolicited response about the completed scan
buf = scan_request.buf
self.send(request=None,
path=buf.path,
language=buf.lang,
command="scan-complete")
class _DBEventReporter(object):
def __init__(self, driver):
self.driver = driver
self.log = log.getChild("DBEventReporter")
self.debug = self.log.debug
# directories being scanned (completed or not)
# key is unicode path, value is number of times it's active
self._dirs = collections.defaultdict(int)
# directories which are complete (unicode path)
self._completed_dirs = set()
def send(self, **kwargs):
self.driver.send(request=None, command="report-message", **kwargs)
def __call__(self, message):
"""Old-style status messages before long-running jobs
@param msg {str or None} The message to display
"""
if len(self._dirs):
return # ignore old-style db events if we're doing a scan
self.debug("db event: %s", message)
self.send(message=message)
def onScanStarted(self, description, dirs=set()):
"""Called when a directory scan is about to start
@param description {unicode} A string suitable for showing the user
about the upcoming operation
@param dirs {set of unicode} The directories about to be scanned
"""
self.debug("scan started: %s (%s dirs)", description, len(dirs))
assert dirs, "onScanStarted expects non-empty directories"
if not dirs: # empty set - we shouldn't have gotten here, but be nice
return
for dir_path in dirs:
self._dirs[dir_path] += 1
self.send(type="scan-progress", message=description,
completed=len(self._completed_dirs),
total=len(self._dirs))
def onScanDirectory(self, description, dir_path, current=None, total=None):
"""Called when a directory is being scanned (out of possibly many)
@param description {unicode} A string suitable for showing the user
regarding the progress
@param dir {unicode} The directory currently being scanned
@param current {int} The current progress
@param total {int} The total number of directories to scan in this
request
"""
self.debug("scan directory: %s (%s %s/%s)",
description, dir_path, current, total)
assert dir_path, "onScanDirectory got no directory"
if dir_path:
self._completed_dirs.add(dir_path)
self.send(type="scan-progress", message=description,
completed=len(self._completed_dirs),
total=len(self._dirs))
def onScanComplete(self, dirs=set(), scanned=set()):
"""Called when a scan operation is complete
@param dirs {set of unicode} The directories that were intially
requested to be scanned (as pass in onScanStarted)
@param scanned {set of unicode} Directories which were successfully
scanned. This may be a subset of dirs if the scan was
aborted.
"""
self.debug("scan complete: scanned %r/%r dirs",
len(scanned), len(dirs))
for dir_path in dirs:
self._dirs[dir_path] -= 1
if not self._dirs[dir_path]:
del self._dirs[dir_path]
self._completed_dirs.discard(dir_path)
self.send(type="scan-progress", completed=len(self._completed_dirs),
total=len(self._dirs))
REQUEST_DEFAULT = object()
def send(self, request=REQUEST_DEFAULT, **kwargs):
"""
Send a response
"""
data = dict(kwargs)
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if request:
data["req_id"] = request.id
if "success" not in data:
data["success"] = True
elif data["success"] is None:
del data["success"]
buf = json.dumps(data, separators=(',',':'))
buf_len = str(len(buf))
log.debug("sending: %s:[%s]", buf_len, buf)
self.send_queue.put(buf)
def _send_proc(self):
while True:
buf = self.send_queue.get()
try:
buf_len = str(len(buf))
self.fd_out.write(buf_len)
self.fd_out.write(buf)
finally:
self.send_queue.task_done()
def fail(self, request=REQUEST_DEFAULT, **kwargs):
kwargs = kwargs.copy()
if not "command" in kwargs and request:
try:
kwargs["command"] = request["command"]
except KeyError:
pass
return self.send(request=request, success=False, **kwargs)
def exception(self, request=REQUEST_DEFAULT, **kwargs):
kwargs["command"] = "report-error"
kwargs["type"] = "logging"
kwargs["name"] = log.name
kwargs["level"] = logging.ERROR
return self.fail(request=request, stack=traceback.format_exc(), **kwargs)
@staticmethod
def normpath(path):
"""Routine to normalize the path used for codeintel buffers
This is annoying because it needs to handle unsaved files, as well as
urls.
@note See also koCodeIntel.py::KoCodeIntelBuffer.normpath
"""
if path.startswith("<Unsaved>"):
return path # Don't munge unsaved file paths
scheme = path.split(":", 1)[0]
if len(scheme) == len(path):
# didn't find a scheme at all; assume this is a local path
return os.path.normcase(path)
if len(scheme) == 1:
# single-character scheme; assume this is actually a drive letter
# (for a Windows-style path)
return os.path.normcase(path)
scheme_chars = string.ascii_letters + string.digits + "-"
try:
scheme = scheme.encode("ascii")
except UnicodeEncodeError:
# scheme has a non-ascii character; assume local path
return os.path.normcase(path)
if scheme.translate(None, scheme_chars):
# has a non-scheme character: this is not a valid scheme
# assume this is a local file path
return os.path.normcase(path)
if scheme != "file":
return path # non-file scheme
path = path[len(scheme) + 1:]
return os.path.normcase(path)
def get_buffer(self, request=REQUEST_DEFAULT, path=None):
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if path is None:
if not "path" in request:
raise RequestFailure(message="No path given to locate buffer")
path = request.path
path = self.normpath(path)
try:
buf = self.buffers[path]
except KeyError:
buf = None
else:
if "language" in request and buf.lang != request.language:
buf = None # language changed, re-scan
if not buf:
# Need to construct a new buffer
lang = request.get("language")
env = Environment(request=request, name=os.path.basename(path))
if request.get("text") is not None:
# pass no content; we'll reset it later
buf = self.mgr.buf_from_content("", lang, path=path, env=env)
else:
# read from file
try:
buf = self.mgr.buf_from_path(path, lang, env=env,
encoding=request.get("encoding"))
except OSError:
# Can't read the file
buf = self.mgr.buf_from_content("", lang, path=path, env=env)
assert not request.path.startswith("<"), \
"Can't create an unsaved buffer with no text"
if request.get("text") is not None:
# overwrite the buffer contents if we have new ones
buf.accessor.reset_content(request.text)
buf.encoding = "utf-8"
try:
env = request["env"]
except KeyError:
pass # no environment, use current
else:
buf._env.update(env)
#log.debug("Got buffer %r: [%s]", buf, buf.accessor.content)
log.debug("Got buffer %r", buf)
self.buffers[path] = buf
return buf
def do_abort(self, request):
try:
req_id = request["id"]
with self.queue_cv:
for item in self.queue:
if item.get("req_id") == req_id:
self.queue.remove(item)
self.send(request=request)
break
else:
self.abort = req_id
if self.active_request and self.active_request.id == req_id:
# need to wait a bit...
self.send(request=request)
else:
self.fail(request=request,
message="Request %s not found" % (req_id,))
except RequestFailure as e:
self.fail(request=request, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
def do_add_dirs(self, request):
catalog_dirs = request.get("catalog-dirs", None)
if catalog_dirs is not None:
self.mgr.db.catalog_dirs.extend(catalog_dirs)
catalog_dirs = self.mgr.db.catalog_dirs
lexer_dirs = request.get("lexer-dirs", [])
codeintel2.udl.UDLLexer.add_extra_lexer_dirs(lexer_dirs)
module_dirs = request.get("module-dirs", [])
if module_dirs:
self.mgr._register_modules(module_dirs)
if catalog_dirs is not None:
self.mgr.db.get_catalogs_zone().catalog_dirs = catalog_dirs
self.send(request=request)
def do_load_extension(self, request):
"""Load an extension that, for example, might provide additional
command handlers"""
name = request.get("module-name", None)
if not name:
raise RequestFailure(msg="load-extension requires a module-name")
path = request.get("module-path", None)
names = name.split(".")
if len(names) > 1:
# It's inside a package - go look for the package(s).
for package_name in names[:-1]:
iinfo = imp.find_module(package_name, [path] if path else None)
if not iinfo:
raise RequestFailure(msg="load-extension could not find "
"package %r for given name %r"
% (package_name, name))
path = iinfo[1]
name = names[-1]
iinfo = imp.find_module(name, [path] if path else None)
try:
module = imp.load_module(name, *iinfo)
finally:
if iinfo and iinfo[0]:
iinfo[0].close()
callback = getattr(module, "registerExtension", None)
if not callback:
raise RequestFailure(msg="load-extension module %s should "
"have a 'registerExtension' method "
"taking no arguments" % (name,))
callback()
self.send() # success, finally
def do_quit(self, request):
self.quit = True
self.send(command="quit")
def report_error(self, message):
self.send(request=None,
command="report-error",
message=unicode(message))
def start(self):
"""Start reading from the socket and dump requests into the queue"""
log.info("Running codeintel driver...")
buf = ""
self.send(success=None)
self.daemon = True
threading.Thread.start(self)
while not self.quit:
try:
ch = self.fd_in.read(1)
except IOError:
log.debug("Failed to read frame length, assuming connection died")
self.quit = True
break
if len(ch) == 0:
log.debug("Input was closed")
self.quit = True
break
if ch == "{":
size = int(buf, 10)
try:
buf = ch + self.fd_in.read(size - 1) # exclude already-read {
except IOError:
log.debug("Failed to read frame data, assuming connection died")
self.quit = True
break
try:
data = json.loads(buf)
request = Request(data)
except Exception as e:
log.exception(e)
self.exception(message=e.message, request=None)
continue
finally:
buf = ""
if request.get("command") == "abort":
self.do_abort(request=request)
else:
log.debug("queuing request %r", request)
with self.queue_cv:
self.queue.appendleft(request)
self.queue_cv.notify()
elif ch in "0123456789":
buf += ch
else:
raise ValueError("Invalid request data: " + ch.encode("hex"))
def run(self):
"""Evaluate and send results back"""
log.info("Running codeintel eval thread...")
buf = ""
log.debug("default supported commands: %s",
", ".join(self._default_handler.supportedCommands))
while True:
with self.queue_cv:
try:
request = self.queue.pop()
except IndexError:
self.queue_cv.wait()
continue
log.debug("doing request %r", request)
try:
self.active_request = request
command = request.command
# First, check abort and quit; don't allow those to be overridden
try:
builtin = self._builtin_commands[command]
except KeyError:
pass
else:
builtin(request)
continue
handlers = self._command_handler_map.get(command, [])[:]
if command in self._default_handler.supportedCommands:
# The default handler can deal with this, put it at the end
handlers.append(self._default_handler)
for handler in handlers:
if isinstance(handler, tuple):
try:
real_handler = handler[0]()
except Exception as ex:
log.exception("Failed to get lazy handler for %s", command)
real_handler = None
if real_handler is None:
# Handler failed to instantiate, drop it
try:
self._command_handler_map[command].remove(handler)
except ValueError:
pass # ... shouldn't happen, but tolerate it
continue
for handlers in self._command_handler_map.values():
try:
handlers[handlers.index(handler)] = real_handler
except ValueError:
pass # handler not in this list
handler = real_handler
if handler.canHandleRequest(request):
handler.handleRequest(request, self)
break
else:
self.fail(request=request,
message="Don't know how to handle command %s" % (command,))
except RequestFailure as e:
self.fail(request=request, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
finally:
self.active_request = None
@classmethod
def registerCommandHandler(cls, handlerInstance):
"""Register a command handler"""
for command in handlerInstance.supportedCommands:
cls._command_handler_map[command].append(handlerInstance)
@classmethod
def registerLazyCommandHandler(cls, supported_commands, constructor):
"""Register a lazy command handler
@param supported_commands {iterable} The commands to handle; each
element should be a str of the command name.
@param constructor {callable} Function to be called to get the real
command handler; it should take no arguments and return a command
handler instance. It may return None if the command is not
available; it will not be asked again.
"""
for command in supported_commands:
cls._command_handler_map[command].append((constructor,))
@classmethod
def getInstance(cls):
"""Get the singleton instance of the driver"""
return Driver._instance
def do_play(self, subcmd, opts):
"""Run my current play/dev code.
${cmd_usage}
${cmd_option_list}
"""
if False:
lang = "CSS"
markedup_content = dedent("""
/* http://www.w3.org/TR/REC-CSS2/fonts.html#propdef-font-weight */
h1 {
border: 1px solid black;
font-weight /* hi */: <|> !important
}
""")
content, data = unmark_text(markedup_content)
pos = data["pos"]
mgr = Manager()
#mgr.upgrade() # Don't need it for just CSS usage.
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path="play.css")
trg = buf.trg_from_pos(pos)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
completions = buf.cplns_from_trg(trg)
print("COMPLETIONS: %r" % completions)
finally:
mgr.finalize()
elif False:
lang = "Python"
path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
markedup_content = dedent("""
import sys, os
class Foo:
def bar(self):
pass
sys.<|>path # should have path in completion list
f = Foo()
""")
content, data = unmark_text(markedup_content)
print(banner(path))
print(_escaped_text_from_text(content, "whitespace"))
pos = data["pos"]
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path=path)
print(banner("cix", '-'))
print(buf.cix)
trg = buf.trg_from_pos(pos)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
print(banner("completions", '-'))
ctlr = LogEvalController(log)
buf.async_eval_at_trg(trg, ctlr)
ctlr.wait(2) #XXX
if not ctlr.is_done():
ctlr.abort()
raise Error("XXX async eval timed out")
pprint(ctlr.cplns)
print(banner(None))
finally:
mgr.finalize()
elif False:
lang = "Ruby"
path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
markedup_content = dedent("""\
r<1>equire 'net/http'
include Net
req = HTTPRequest.new
req.<2>get()
""")
content, data = unmark_text(markedup_content)
print(banner(path))
print(_escaped_text_from_text(content, "whitespace"))
pos = data[1]
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path=path)
print(banner("cix", '-'))
cix = buf.cix
print(
ET.tostring(pretty_tree_from_tree(tree_from_cix(buf.cix))))
trg = buf.trg_from_pos(pos, implicit=False)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
print(banner("completions", '-'))
ctlr = LogEvalController(log)
buf.async_eval_at_trg(trg, ctlr)
ctlr.wait(30) #XXX
if not ctlr.is_done():
ctlr.abort()
raise Error("XXX async eval timed out")
pprint(ctlr.cplns)
print(banner(None))
finally:
mgr.finalize()
class Driver(threading.Thread):
"""
Out-of-process codeintel2 driver
This class implements the main loop of the codeintel worker process
This should be a singleton object
"""
# static
_instance = None
_command_handler_map = collections.defaultdict(list)
"""Registered command handlers; the key is the command name (a str), and the
value is a list of command handler instances or (for lazy handlers) a
single-element tuple containing the callable to get the real handler."""
_default_handler = None
"""The default (core) command handler; instance of a CoreHandler"""
_builtin_commands = {}
"""Built-in commands that cannot be overridden"""
def __init__(self, db_base_dir=None, fd_in=sys.stdin, fd_out=sys.stdout):
threading.Thread.__init__(self, name="Codeintel OOP Driver")
assert Driver._instance is None, "Driver should be a singleton"
Driver._instance = self
logging.root.addHandler(LoggingHandler(self))
self.daemon = True
self.fd_in = fd_in
self.fd_out = fd_out
self.abort = None
self.quit = False
self.buffers = {} # path to Buffer objects
self.next_buffer = 0
self.active_request = None
self.send_queue = Queue.Queue()
self.send_thread = threading.Thread(target=self._send_proc)
self.send_thread.daemon = True
self.send_thread.start()
self.queue = collections.deque()
self.queue_cv = threading.Condition()
self.env = Environment(name="global",
send_fn=functools.partial(self.send,
request=None))
# Fill out the non-overridable build-in commands
self._builtin_commands = {}
for attr in dir(self):
# Note that we check startswith first to avoid getters etc.
if attr.startswith("do_") and callable(getattr(self, attr)):
command = attr[len("do_"):].replace("_", "-")
self._builtin_commands[command] = getattr(self, attr)
from codeintel2.manager import Manager
log.debug("using db base dir %s", db_base_dir)
self.mgr = Manager(db_base_dir=db_base_dir,
db_catalog_dirs=[],
db_event_reporter=self._DBEventReporter(self),
env=self.env,
on_scan_complete=self._on_scan_complete)
self.mgr.initialize()
def _on_scan_complete(self, scan_request):
if scan_request.status in ("changed", "skipped"):
# Send unsolicited response about the completed scan
buf = scan_request.buf
self.send(request=None,
path=buf.path,
language=buf.lang,
command="scan-complete")
class _DBEventReporter(object):
def __init__(self, driver):
self.driver = driver
self.log = log.getChild("DBEventReporter")
self.debug = self.log.debug
# directories being scanned (completed or not)
# key is unicode path, value is number of times it's active
self._dirs = collections.defaultdict(int)
# directories which are complete (unicode path)
self._completed_dirs = set()
def send(self, **kwargs):
self.driver.send(request=None, command="report-message", **kwargs)
def __call__(self, message):
"""Old-style status messages before long-running jobs
@param msg {str or None} The message to display
"""
if len(self._dirs):
return # ignore old-style db events if we're doing a scan
self.debug("db event: %s", message)
self.send(message=message)
def onScanStarted(self, description, dirs=set()):
"""Called when a directory scan is about to start
@param description {unicode} A string suitable for showing the user
about the upcoming operation
@param dirs {set of unicode} The directories about to be scanned
"""
self.debug("scan started: %s (%s dirs)", description, len(dirs))
assert dirs, "onScanStarted expects non-empty directories"
if not dirs: # empty set - we shouldn't have gotten here, but be nice
return
for dir_path in dirs:
self._dirs[dir_path] += 1
self.send(type="scan-progress",
message=description,
completed=len(self._completed_dirs),
total=len(self._dirs))
def onScanDirectory(self,
description,
dir_path,
current=None,
total=None):
"""Called when a directory is being scanned (out of possibly many)
@param description {unicode} A string suitable for showing the user
regarding the progress
@param dir {unicode} The directory currently being scanned
@param current {int} The current progress
@param total {int} The total number of directories to scan in this
request
"""
self.debug("scan directory: %s (%s %s/%s)", description, dir_path,
current, total)
assert dir_path, "onScanDirectory got no directory"
if dir_path:
self._completed_dirs.add(dir_path)
self.send(type="scan-progress",
message=description,
completed=len(self._completed_dirs),
total=len(self._dirs))
def onScanComplete(self, dirs=set(), scanned=set()):
"""Called when a scan operation is complete
@param dirs {set of unicode} The directories that were intially
requested to be scanned (as pass in onScanStarted)
@param scanned {set of unicode} Directories which were successfully
scanned. This may be a subset of dirs if the scan was
aborted.
"""
self.debug("scan complete: scanned %r/%r dirs", len(scanned),
len(dirs))
for dir_path in dirs:
self._dirs[dir_path] -= 1
if not self._dirs[dir_path]:
del self._dirs[dir_path]
self._completed_dirs.discard(dir_path)
self.send(type="scan-progress",
completed=len(self._completed_dirs),
total=len(self._dirs))
REQUEST_DEFAULT = object()
def send(self, request=REQUEST_DEFAULT, **kwargs):
"""
Send a response
"""
data = dict(kwargs)
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if request:
data["req_id"] = request.id
if "success" not in data:
data["success"] = True
elif data["success"] is None:
del data["success"]
buf = json.dumps(data, separators=(',', ':'))
buf_len = str(len(buf))
log.debug("sending: %s:[%s]", buf_len, buf)
self.send_queue.put(buf)
def _send_proc(self):
while True:
buf = self.send_queue.get()
try:
buf_len = str(len(buf))
self.fd_out.write(buf_len)
self.fd_out.write(buf)
finally:
self.send_queue.task_done()
def fail(self, request=REQUEST_DEFAULT, **kwargs):
kwargs = kwargs.copy()
if not "command" in kwargs and request:
try:
kwargs["command"] = request["command"]
except KeyError:
pass
return self.send(request=request, success=False, **kwargs)
def exception(self, request=REQUEST_DEFAULT, **kwargs):
return self.fail(request=request,
stack=traceback.format_exc(),
**kwargs)
def get_buffer(self, request=REQUEST_DEFAULT, path=None):
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if path is None:
if not "path" in request:
raise RequestFailure(message="No path given to locate buffer")
path = request.path
if abspath(path) == path:
# this is an actual file path, not a URL or whatever
path = normcase(normpath(path))
try:
buf = self.buffers[path]
except KeyError:
buf = None
else:
if "language" in request and buf.lang != request.language:
buf = None # language changed, re-scan
if not buf:
# Need to construct a new buffer
lang = request.get("language")
if request.get("text") is not None:
# pass no content; we'll reset it later
buf = self.mgr.buf_from_content("", lang, path=path)
else:
# read from file
try:
buf = self.mgr.buf_from_path(
path, lang, encoding=request.get("encoding"))
except OSError:
# Can't read the file
buf = self.mgr.buf_from_content("", lang, path=path)
assert not request.path.startswith("<"), \
"Can't create an unsaved buffer with no text"
if request.get("text") is not None:
# overwrite the buffer contents if we have new ones
buf.accessor.reset_content(request.text)
try:
env = request["env"]
except KeyError:
pass # no environment set, use current environment
else:
if env.get("env", {}) is None and env.get("prefs", []) is None:
buf._env = None # explicitly clearing environment
elif buf._env:
buf._env.update(env)
else:
buf._env = Environment(env, name=os.path.basename(path))
# log.debug("Got buffer %r: [%s]", buf, buf.accessor.content)
log.debug("Got buffer %r", buf)
self.buffers[path] = buf
return buf
def do_abort(self, request):
try:
req_id = request["id"]
with self.queue_cv:
for item in self.queue:
if queue.get("req_id") == req_id:
self.queue.remove(queue)
self.send(request=request)
break
else:
self.abort = req_id
if self.active_request and self.active_request.id == req_id:
# need to wait a bit...
self.send(request=request)
else:
self.fail(request=request,
message="Request %s not found" % (req_id, ))
except RequestFailure as e:
self.fail(request=request, *e.args, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
def do_add_dirs(self, request):
catalog_dirs = request.get("catalog-dirs", None)
if catalog_dirs is not None:
self.mgr.db.catalog_dirs.extend(catalog_dirs)
catalog_dirs = self.mgr.db.catalog_dirs
lexer_dirs = request.get("lexer-dirs", [])
codeintel2.udl.UDLLexer.add_extra_lexer_dirs(lexer_dirs)
module_dirs = request.get("module-dirs", [])
if module_dirs:
self.mgr._register_modules(module_dirs)
if catalog_dirs is not None:
self.mgr.db.get_catalogs_zone().catalog_dirs = catalog_dirs
def do_load_extension(self, request):
"""Load an extension that, for example, might provide additional
command handlers"""
path = request.get("module-path", None)
if not path:
raise RequestFailure(msg="load-extension requires a module-path")
name = request.get("module-name", None)
if not name:
raise RequestFailure(msg="load-extension requires a module-name")
iinfo = imp.find_module(name, [path])
try:
module = imp.load_module(name, *iinfo)
finally:
if iinfo and iinfo[0]:
iinfo[0].close()
callback = getattr(module, "registerExtension", None)
if not callback:
raise RequestFailure(msg="load-extension module %s should "
"have a 'registerExtension' method "
"taking no arguments" % (name, ))
callback()
self.send() # success, finally
def do_quit(self, request):
self.quit = True
self.send(command="quit")
def report_error(self, message):
self.send(request=None,
command="report-error",
message=unicode(message))
def start(self):
"""Start reading from the socket and dump requests into the queue"""
log.info("Running codeintel driver...")
buf = ""
self.send(success=None)
self.daemon = True
threading.Thread.start(self)
while not self.quit:
try:
ch = self.fd_in.read(1)
except IOError:
log.debug(
"Failed to read frame length, assuming connection died")
self.quit = True
break
if len(ch) == 0:
log.debug("Input was closed")
self.quit = True
break
if ch == "{":
size = int(buf, 10)
try:
buf = ch + self.fd_in.read(
size - 1) # exclude already-read {
except IOError:
log.debug(
"Failed to read frame data, assuming connection died")
self.quit = True
break
try:
buf = buf.decode("utf-8")
except UnicodeDecodeError:
pass # what :(
try:
data = json.loads(buf)
request = Request(data)
except Exception as e:
log.exception(e)
self.exception(message=e.message, request=None)
continue
finally:
buf = ""
if request.get("command") == "abort":
self.do_abort(request=request)
else:
log.debug("queuing request %r", request)
with self.queue_cv:
self.queue.appendleft(request)
self.queue_cv.notify()
elif ch in "0123456789":
buf += ch
else:
raise ValueError("Invalid request data: " + ch.encode("hex"))
def run(self):
"""Evaluate and send results back"""
log.info("Running codeintel eval thread...")
buf = ""
log.debug("default supported commands: %s",
", ".join(self._default_handler.supportedCommands))
while True:
with self.queue_cv:
try:
request = self.queue.pop()
except IndexError:
self.queue_cv.wait()
continue
log.debug("doing request %r", request)
try:
self.active_request = request
command = request.command
# First, check abort and quit; don't allow those to be
# overridden
try:
builtin = self._builtin_commands[command]
except KeyError:
pass
else:
builtin(request)
continue
handlers = self._command_handler_map.get(command, [])[:]
if command in self._default_handler.supportedCommands:
# The default handler can deal with this, put it at the end
handlers.append(self._default_handler)
for handler in handlers:
if isinstance(handler, tuple):
try:
real_handler = handler[0]()
except Exception as ex:
log.exception("Failed to get lazy handler for %s",
command)
real_handler = None
if real_handler is None:
# Handler failed to instantiate, drop it
try:
self._command_handler_map["command"].remove(
handler)
except ValueError:
pass # ... shouldn't happen, but tolerate it
continue
for handlers in self._command_handler_map.values():
try:
handlers[handlers.index(
handler)] = real_handler
except ValueError:
pass # handler not in this list
handler = real_handler
if handler.canHandleRequest(request):
handler.handleRequest(request, self)
break
else:
self.fail(request=request,
msg="Don't know how to handle command %s" %
(command, ))
except RequestFailure as e:
self.fail(request=request, *e.args, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
finally:
self.active_request = None
@classmethod
def registerCommandHandler(cls, handlerInstance):
"""Register a command handler"""
for command in handlerInstance.supportedCommands:
cls._command_handler_map[command].append(handlerInstance)
@classmethod
def registerLazyCommandHandler(cls, supported_commands, constructor):
"""Register a lazy command handler
@param supported_commands {iterable} The commands to handle; each
element should be a str of the command name.
@param constructor {callable} Function to be called to get the real
command handler; it should take no arguments and return a command
handler instance. It may return None if the command is not
available; it will not be asked again.
"""
for command in supported_commands:
cls._command_handler_map[command].append((constructor, ))
@classmethod
def getInstance(cls):
"""Get the singleton instance of the driver"""
return Driver._instance
class Driver(threading.Thread):
"""
Out-of-process codeintel2 driver
This class implements the main loop of the codeintel worker process
This should be a singleton object
"""
# static
_command_handler_map = collections.defaultdict(list)
"""Registered command handlers"""
_default_handler = None
"""The default (core) command handler; instance of a CoreHandler"""
_builtin_commands = {}
"""Built-in commands that cannot be overridden"""
def __init__(self, db_base_dir=None, fd_in=sys.stdin, fd_out=sys.stdout):
threading.Thread.__init__(self, name="Codeintel OOP Driver")
self.daemon = True
self.fd_in = fd_in
self.fd_out = fd_out
self.abort = None
self.quit = False
self.buffers = {} # path to Buffer objects
self.next_buffer = 0
self.active_request = None
self.queue = collections.deque()
self.queue_cv = threading.Condition()
self.env = Environment(name="global",
send_fn=functools.partial(self.send, request=None))
# Fill out the non-overridable build-in commands
self._builtin_commands = {}
for attr in dir(self):
# Note that we check startswith first to avoid getters etc.
if attr.startswith("do_") and callable(getattr(self, attr)):
command = attr[len("do_"):].replace("_", "-")
self._builtin_commands[command] = getattr(self, attr)
from codeintel2.manager import Manager
log.debug("using db base dir %s", db_base_dir)
self.mgr = Manager(db_base_dir=db_base_dir,
db_catalog_dirs=[],
env=self.env,
on_scan_complete=self._on_scan_complete)
self.mgr.initialize()
def _on_scan_complete(self, scan_request):
if scan_request.status == "changed":
# Send unsolicited response about the completed scan
buf = scan_request.buf
self.send(request=None,
path=buf.path,
language=buf.lang,
command="scan-complete")
REQUEST_DEFAULT = object()
def send(self, request=REQUEST_DEFAULT, **kwargs):
"""
Send a response
"""
data = dict(kwargs)
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if request:
data["req_id"] = request.id
if "success" not in data:
data["success"] = True
elif data["success"] is None:
del data["success"]
buf = json.dumps(data, separators=(',', ':'))
log.debug("sending: [%s]", buf)
self.fd_out.write(str(len(buf)) + buf)
def fail(self, request=REQUEST_DEFAULT, **kwargs):
kwargs = kwargs.copy()
if not "command" in kwargs and request:
try:
kwargs["command"] = request["command"]
except KeyError:
pass
return self.send(request=request, success=False, **kwargs)
def exception(self, request=REQUEST_DEFAULT, **kwargs):
return self.fail(request=request, stack=traceback.format_exc(), **kwargs)
def get_buffer(self, request=REQUEST_DEFAULT, path=None):
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if path is None:
if not "path" in request:
raise RequestFailure(message="No path given to locate buffer")
path = request.path
if abspath(path) == path:
# this is an actual file path, not a URL or whatever
path = normcase(normpath(path))
try:
buf = self.buffers[path]
except KeyError:
buf = None
else:
if "language" in request and buf.lang != request.language:
buf = None # language changed, re-scan
if not buf:
# Need to construct a new buffer
lang = request.get("language")
if request.get("text") is not None:
# pass no content; we'll reset it later
buf = self.mgr.buf_from_content("", lang, path=path)
else:
# read from file
try:
buf = self.mgr.buf_from_path(path, lang,
encoding=request.get("encoding"))
except OSError:
# Can't read the file
buf = self.mgr.buf_from_content("", lang, path=path)
assert not request.path.startswith("<")
if request.get("text") is not None:
# overwrite the buffer contents if we have new ones
buf.accessor.reset_content(request.text)
try:
env = request["env"]
except KeyError:
pass # no environment set, use current environment
else:
if env.get("env", {}) is None and env.get("prefs", []) is None:
buf._env = None # explicitly clearing environment
elif buf._env:
buf._env.update(env)
else:
buf._env = Environment(env, name=os.path.basename(path))
# log.debug("Got buffer %r: [%s]", buf, buf.accessor.content)
log.debug("Got buffer %r", buf)
self.buffers[path] = buf
return buf
def do_abort(self, request):
try:
req_id = request["id"]
with self.queue_cv:
for item in self.queue:
if queue.get("req_id") == req_id:
self.queue.remove(queue)
self.send(request=request)
break
else:
self.abort = req_id
if self.active_request and self.active_request.id == req_id:
# need to wait a bit...
self.send(request=request)
else:
self.fail(request=request,
message="Request %s not found" % (req_id,))
except RequestFailure as e:
self.fail(request=request, *e.args, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
def do_add_dirs(self, request):
catalog_dirs = request.get("catalog-dirs", None)
if catalog_dirs is not None:
self.mgr.db.catalog_dirs.extend(catalog_dirs)
lexer_dirs = request.get("lexer-dirs", [])
codeintel2.udl.UDLLexer.add_extra_lexer_dirs(lexer_dirs)
module_dirs = request.get("module-dirs", [])
if module_dirs:
self.mgr._register_modules(module_dirs)
if catalog_dirs is not None:
self.mgr.db.get_catalogs_zone().update()
def do_load_extension(self, request):
"""Load an extension that, for example, might provide additional
command handlers"""
path = request.get("module-path", None)
if not path:
raise RequestFailure(msg="load-extension requires a module-path")
name = request.get("module-name", None)
if not name:
raise RequestFailure(msg="load-extension requires a module-name")
iinfo = imp.find_module(name, [path])
try:
module = imp.load_module(name, *iinfo)
finally:
if iinfo and iinfo[0]:
iinfo[0].close()
callback = getattr(module, "registerExtension", None)
if not callback:
raise RequestFailure(msg="load-extension module %s should "
"have a 'registerExtension' method "
"taking no arguments" % (name,))
callback()
self.send() # success, finally
def do_quit(self, request):
self.quit = True
self.send(command="quit")
def report_error(self, message):
self.send(request=None,
command="report-error",
message=unicode(message))
def start(self):
"""Start reading from the socket and dump requests into the queue"""
log.info("Running codeintel driver...")
buf = ""
self.send(success=None)
self.daemon = True
threading.Thread.start(self)
while not self.quit:
try:
ch = self.fd_in.read(1)
except IOError:
log.debug(
"Failed to read frame length, assuming connection died")
self.quit = True
break
if len(ch) == 0:
log.debug("Input was closed")
self.quit = True
break
if ch == "{":
size = int(buf, 10)
try:
buf = ch + self.fd_in.read(
size - 1) # exclude already-read {
except IOError:
log.debug(
"Failed to read frame data, assuming connection died")
self.quit = True
break
try:
buf = buf.decode("utf-8")
except UnicodeDecodeError:
pass # what :(
try:
data = json.loads(buf)
request = Request(data)
except Exception as e:
log.exception(e)
self.exception(message=e.message, request=None)
continue
finally:
buf = ""
if request.get("command") == "abort":
self.do_abort(request=request)
else:
log.debug("queuing request %r", request)
with self.queue_cv:
self.queue.appendleft(request)
self.queue_cv.notify()
elif ch in "0123456789":
buf += ch
else:
raise ValueError("Invalid request data: " + ch.encode("hex"))
def run(self):
"""Evaluate and send results back"""
log.info("Running codeintel eval thread...")
buf = ""
log.debug("default supported commands: %s",
", ".join(self._default_handler.supportedCommands))
while True:
with self.queue_cv:
try:
request = self.queue.pop()
except IndexError:
self.queue_cv.wait()
continue
log.debug("doing request %r", request)
try:
self.active_request = request
command = request.command
# First, check abort and quit; don't allow those to be
# overridden
try:
builtin = self._builtin_commands[command]
except KeyError:
pass
else:
builtin(request)
continue
handlers = self._command_handler_map.get(command, [])[:]
if command in self._default_handler.supportedCommands:
# The default handler can deal with this, put it at the end
handlers.append(self._default_handler)
for handler in handlers:
if handler.canHandleRequest(request):
handler.handleRequest(request, self)
break
else:
self.fail(request=request,
msg="Don't know how to handle command %s" % (command,))
except RequestFailure as e:
self.fail(request=request, *e.args, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
finally:
self.active_request = None
@classmethod
def registerCommandHandler(cls, handlerInstance):
"""Register a command handler"""
for command in handlerInstance.supportedCommands:
cls._command_handler_map[command].append(handlerInstance)
class CodeIntelTestCase(unittest.TestCase):
# Subclasses can override this to have setUp pass these settings to the
# codeintel.Manager.
_ci_db_base_dir_ = None
_ci_db_catalog_dirs_ = []
_ci_db_import_everything_langs = None
_ci_env_prefs_ = None
# A test case can set this to false to have the setUp/tearDown *not*
# create a `self.mgr'.
_ci_test_setup_mgr_ = True
_ci_extra_module_dirs_ = None
def setUp(self):
if _xpcom_:
# The tests are run outside of Komodo. If run with PyXPCOM up
# parts codeintel will try to use the nsIDirectoryService and
# will query dirs only provided by nsXREDirProvider -- which
# isn't registered outside of Komodo (XRE_main() isn't called).
# The KoTestService provides a backup.
koTestSvc = components.classes["@activestate.com/koTestService;1"] \
.getService(components.interfaces.koITestService)
koTestSvc.init()
if self._ci_test_setup_mgr_:
env = None
if self._ci_env_prefs_ is not None:
env = SimplePrefsEnvironment(**self._ci_env_prefs_)
self.mgr = Manager(
extra_module_dirs=self._ci_extra_module_dirs_,
db_base_dir=self._ci_db_base_dir_ or test_db_base_dir,
db_catalog_dirs=self._ci_db_catalog_dirs_,
db_import_everything_langs=self._ci_db_import_everything_langs,
env=env)
self.mgr.upgrade()
self.mgr.initialize()
def tearDown(self):
if self._ci_test_setup_mgr_:
self.mgr.finalize()
self.mgr = None
def adjust_content(self, content):
"""A hook for subclasses to modify markedup_content before use in
test cases. This is useful for sharing test cases between pure-
and multi-lang uses of a given lang.
"""
return content
def adjust_pos(self, pos):
"""A accompanying hook for `adjust_content' to adjust trigger
pos values accordingly.
"""
return pos
def _get_buf_and_data(self, markedup_content, lang, path=None, env=None):
if path is None:
# Try to ensure no accidental re-use of the same buffer name
# across the whole test suite. Also try to keep the buffer
# names relatively short (keeps tracebacks cleaner).
name = "buf-" + md5(markedup_content).hexdigest()[:16]
path = os.path.join("<Unsaved>", name)
content, data = unmark_text(self.adjust_content(markedup_content))
#print banner(path)
#sys.stdout.write(content)
#print banner(None)
buf = self.mgr.buf_from_content(content, lang=lang, path=path, env=env)
return buf, data
def _get_buf_and_trg(self,
markedup_content,
lang,
path=None,
implicit=True,
env=None):
buf, data = self._get_buf_and_data(markedup_content, lang, path, env)
trg = buf.trg_from_pos(data["pos"], implicit=implicit)
return buf, trg
def assertCITDLExprUnderPosIs(self,
markedup_content,
citdl_expr,
lang=None,
prefix_filter=None,
implicit=True,
trigger_name=None,
**fields):
"""Assert that the CITDL expression at the current position
is as expected.
This uses buf.citdl_expr_under_pos() -- or, for Perl,
buf.citdl_expr_and_prefix_filter_from_trg().
Note: This API is a mess right now. C.f. bug 65776.
The "prefix_filter" optional argument can be used for Perl to test
the value its relevant function returns.
"""
if lang is None:
lang = self.lang
content, data = unmark_text(self.adjust_content(markedup_content))
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
langintel = self.mgr.langintel_from_lang(lang)
if trigger_name is None:
trigger_name = "fakey-completion-type"
if lang == "Perl":
trg = Trigger(lang,
TRG_FORM_DEFN,
trigger_name,
data["pos"],
implicit=implicit,
length=0,
**fields)
actual_citdl_expr, actual_prefix_filter \
= langintel.citdl_expr_and_prefix_filter_from_trg(buf, trg)
else:
#actual_citdl_expr = langintel.citdl_expr_under_pos(buf, data["pos"])
trg = Trigger(lang,
TRG_FORM_DEFN,
trigger_name,
data["pos"],
implicit=implicit,
**fields)
actual_citdl_expr = langintel.citdl_expr_from_trg(buf, trg)
self.assertEqual(
actual_citdl_expr, citdl_expr,
"unexpected actual %s CITDL expr under pos:\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s" %
(lang, citdl_expr, actual_citdl_expr, indent(markedup_content)))
if prefix_filter is not None:
XXX #TODO: compare prefix_filter to given value
def _assertDefnMatches(self, buf, pos, lang=None, **fields):
ctlr = _CaptureEvalController()
trg = buf.defn_trg_from_pos(pos)
defns = buf.defns_from_trg(trg, ctlr=ctlr)
if not defns:
self.fail(
"unexpectedly did not find a definition in %r at pos %d\n"
" eval log\n%s\n"
" buffer:\n%s" % (buf, pos,
indent('\n'.join('%5s: %s' % (lvl, m)
for lvl, m in ctlr.log)),
indent(buf.accessor.text)))
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
defn = defns[0]
for name, value in fields.items():
try:
actual_value = getattr(defn, name)
except AttributeError:
actual_value = None
self.assertEqual(
actual_value, value,
"%s definition, unexpected value for field %r\n"
" defn: %r\n"
" expected: %r\n"
" got: %r\n"
" eval log\n%s\n"
" buffer:\n%s" % (buf.lang, name, defn, value, actual_value,
indent('\n'.join('%5s: %s' % (lvl, m)
for lvl, m in ctlr.log)),
indent(buf.accessor.text)))
def assertDefnMatches(self, markedup_content, lang=None, **fields):
if lang is None:
lang = self.lang
buf, data = self._get_buf_and_data(markedup_content, lang)
self._assertDefnMatches(buf, data["pos"], lang, **fields)
def assertDefnMatches2(self, buf, pos, lang=None, **fields):
self._assertDefnMatches(buf, pos, lang, **fields)
def assertNoDuplicateDefns(self, markedup_content, lang=None):
if lang is None:
lang = self.lang
buf, data = self._get_buf_and_data(markedup_content, lang)
self.assertNoDuplicateDefns2(buf, data["pos"])
def assertNoDuplicateDefns2(self, buf, pos):
markedup_content = markup_text(buf.accessor.text, pos=pos)
ctlr = _CaptureEvalController()
trg = buf.defn_trg_from_pos(pos)
actual_defns = buf.defns_from_trg(trg, ctlr=ctlr)
if not actual_defns:
self.fail("%s trigger resulted in no definitions when expecting "
"to check for duplicate definitions:\n%s" %
(buf.lang, indent(markedup_content)))
count_from_defn_repr = {}
for defn_repr in (repr(d) for d in actual_defns):
if defn_repr not in count_from_defn_repr:
count_from_defn_repr[defn_repr] = 0
count_from_defn_repr[defn_repr] += 1
defn_dupes = [(count, defn_repr)
for defn_repr, count in count_from_defn_repr.items()
if count > 1]
self.failIf(
defn_dupes,
"unexpectedly got duplicate completions at the given position\n"
" duplicates:\n%s\n"
" eval log\n%s\n"
" buffer:\n%s" % (indent('\n'.join('%d of %s' % d
for d in defn_dupes)),
indent('\n'.join('%5s: %s' % (lvl, m)
for lvl, m in ctlr.log)),
indent(markedup_content)))
def assertTriggerMatches(self,
markedup_content,
lang=None,
implicit=True,
env=None,
**fields):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
if trg is None:
self.fail("unexpectedly did not find a %s trigger, buffer:\n%s" %
(lang, indent(markedup_content)))
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
for name, value in fields.items():
try:
actual_value = getattr(trg, name)
except AttributeError:
actual_value = trg.extra[name]
self.assertEqual(
actual_value, value,
"unexpected %s trigger '%s' value: expected %r, "
"got %r, buffer:\n%s" %
(lang, name, value, actual_value, indent(markedup_content)))
# Used when a position generates a trigger, but it's not the one specified
def assertTriggerDoesNotMatch(self,
markedup_content,
lang=None,
implicit=True,
env=None,
**fields):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
if trg is None:
# No trigger is as good as
return
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
for name, value in fields.items():
try:
actual_value = getattr(trg, name)
except AttributeError:
actual_value = trg.extra[name]
self.assertNotEqual(
actual_value, value,
"unexpected %s trigger '%s' value: expected not %r, "
"got %r, buffer:\n%s" %
(lang, name, value, actual_value, indent(markedup_content)))
def assertNoTrigger(self,
markedup_content,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
if trg is not None:
self.fail("unexpectedly found a %s trigger %r when didn't expect "
"one, buffer:\n%s" %
(lang, trg, indent(markedup_content)))
def assertPrecedingTriggerMatches(self,
markedup_content,
lang=None,
**fields):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(self.adjust_content(markedup_content))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
trg = buf.preceding_trg_from_pos(data["start_pos"], data["pos"])
if trg is None:
self.fail("unexpectedly did not find a preceding %s trigger, "
"buffer:\n%s" % (lang, indent(markedup_content)))
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
for name, value in fields.items():
actual_value = getattr(trg, name)
self.assertEqual(
actual_value, value,
"unexpected preceding %s trigger '%s' value: expected %r, "
"got %r, buffer:\n%s" %
(lang, name, value, actual_value, indent(markedup_content)))
def assertNoPrecedingTrigger(self, markedup_content, lang=None):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(self.adjust_content(markedup_content))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
trg = buf.preceding_trg_from_pos(data["start_pos"], data["pos"])
if trg is not None:
self.fail("unexpectedly found a preceding %s trigger '%s' when "
"didn't expect one, buffer:\n%s" %
(lang, trg.name, indent(markedup_content)))
def assertScopeLpathIs(self, markedup_content, lpath, lang=None):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(self.adjust_content(markedup_content))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
buf.scan(skip_scan_time_check=True)
actual_blob, actual_lpath = buf.scoperef_from_pos(data["pos"])
self.failUnlessEqual(
lpath, actual_lpath,
"unexpected %s scope lookup path (lpath) at the given position\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s" %
(self.lang, lpath, actual_lpath, indent(markedup_content)))
def assertCompletionsAre2(self,
buf,
pos,
completions,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
markedup_content = markup_text(buf.accessor.text, pos=pos)
trg = buf.trg_from_pos(pos, implicit=implicit)
self._assertCompletionsAre(markedup_content, buf, trg, completions,
lang, implicit)
def assertCompletionsAre(self,
markedup_content,
completions,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
self._assertCompletionsAre(markedup_content, buf, trg, completions,
lang, implicit)
def _assertCompletionsAre(self, markedup_content, buf, trg, completions,
lang, implicit):
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected completions to be %r:\n%s" %
(lang, completions, indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
self.assertEqual(
completions, actual_completions,
"unexpected %s completions at the given position\n"
" expected: %r\n"
" got: %r\n"
" extra: %r\n"
" missing: %r\n"
" eval log\n%s\n"
" buffer:\n%s" %
(lang, completions, actual_completions,
list(set(actual_completions or []).difference(completions or [])),
list(set(completions or []).difference(actual_completions or [])),
indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)),
indent(markedup_content)))
def assertNoDuplicateCompletions(self,
markedup_content,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected there to be completions:\n%s" %
(lang, indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
if actual_completions is None:
self.fail("%s trigger resulted in no completions when expecting "
"to check for duplicate completions:\n%s" %
(lang, indent(markedup_content)))
count_from_cpln = {}
for cpln in actual_completions:
if cpln not in count_from_cpln:
count_from_cpln[cpln] = 0
count_from_cpln[cpln] += 1
cpln_dupes = [(count, cpln) for cpln, count in count_from_cpln.items()
if count > 1]
self.failIf(
cpln_dupes,
"unexpectedly got duplicate completions at the given position\n"
" duplicates:\n%s\n"
" eval log\n%s\n"
" buffer:\n%s" % (indent('\n'.join('%d of %r' % d
for d in cpln_dupes)),
indent('\n'.join('%5s: %s' % (lvl, m)
for lvl, m in ctlr.log)),
indent(markedup_content)))
def _assertCompletionsInclude(self, buf, trg, completions):
markedup_content = markup_text(buf.accessor.text, pos=trg.pos)
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
missing_completions = [
c for c in completions if c not in (actual_completions or [])
]
self.failIf(
missing_completions,
"%s completions at the given position did not "
"include all expected values\n"
" missing: %r\n"
" expected all of: %r\n"
" got: %r\n"
" eval log:\n%s\n"
" buffer:\n%s" %
(buf.lang, missing_completions, completions, actual_completions,
indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)),
indent(markedup_content)))
def assertCompletionsInclude(self,
markedup_content,
completions,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected completions to include %r:\n%s" %
(lang, completions, indent(markedup_content)))
self._assertCompletionsInclude(buf, trg, completions)
def assertCompletionsInclude2(self, buf, pos, completions, implicit=True):
"""A version of assertCompletionsInclude() where you pass in
a Buffer instance instead of marked up content. Sometimes
this is more convenient.
"""
trg = buf.trg_from_pos(pos, implicit=implicit)
if trg is None:
markedup_content = markup_text(buf.accessor.text, pos=pos)
self.fail("given position is not a %s trigger point, "
"expected completions to include %r:\n%s" %
(buf.lang, completions, indent(markedup_content)))
self._assertCompletionsInclude(buf, trg, completions)
def _assertCompletionsDoNotInclude(self, buf, trg, completions):
markedup_content = markup_text(buf.accessor.text, pos=trg.pos)
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
completions_that_shouldnt_be_there = [
c for c in (actual_completions or []) if c in completions
]
self.failIf(
completions_that_shouldnt_be_there,
"%s completions at the given position included "
"some unexpected values\n"
" shouldn't have had these: %r\n"
" expected none of: %r\n"
" got: %r\n"
" eval log:\n%s\n"
" buffer:\n%s" % (buf.lang, completions_that_shouldnt_be_there,
completions, actual_completions,
indent('\n'.join('%5s: %s' % (lvl, m)
for lvl, m in ctlr.log)),
indent(markedup_content)))
def assertCompletionsDoNotInclude(self,
markedup_content,
completions,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected completions to exclude %r:\n%s" %
(lang, completions, indent(markedup_content)))
self._assertCompletionsDoNotInclude(buf, trg, completions)
def assertCompletionsDoNotInclude2(self,
buf,
pos,
completions,
implicit=True):
"""A version of assertCompletionsDoNotInclude() where you pass in
a Buffer instance instead of marked up content. Sometimes
this is more convenient.
"""
trg = buf.trg_from_pos(pos, implicit=implicit)
if trg is None:
markedup_content = markup_text(buf.accessor.text, pos=pos)
self.fail("given position is not a %s trigger point, "
"expected completions to exclude %r:\n%s" %
(buf.lang, completions, indent(markedup_content)))
self._assertCompletionsDoNotInclude(buf, trg, completions)
def assertCalltipIs2(self, buf, pos, calltip, implicit=True):
"""A variant of assertCalltipIs() where you pass in
a Buffer instance instead of marked up content. Sometimes
this is more convenient.
"""
trg = buf.trg_from_pos(pos, implicit=implicit)
markedup_content = markup_text(buf.accessor.text, pos=trg.pos)
self._assertCalltipIs(buf, trg, markedup_content, calltip, self.lang)
def assertCalltipIs(self,
markedup_content,
calltip,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
self._assertCalltipIs(buf, trg, markedup_content, calltip, lang)
def _assertCalltipIs(self, buf, trg, markedup_content, calltip, lang):
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected the following calltip:\n"
" calltip:\n%s\n"
" buffer:\n%s" %
(lang, indent(calltip), indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_calltips = buf.calltips_from_trg(trg, ctlr=ctlr)
if actual_calltips and actual_calltips[0]:
actual_calltip = actual_calltips[0]
else:
actual_calltip = None
self.assertEqual(
calltip, actual_calltip,
"unexpected %s calltip at the given position\n"
" expected:\n%s\n"
" got:\n%s\n"
" eval log:\n%s\n"
" buffer:\n%s" %
(trg.name, indent(calltip and calltip or "(none)"),
indent(actual_calltip and actual_calltip or "(none)"),
indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)),
indent(markedup_content)))
def assertCalltipMatches(self,
markedup_content,
calltip,
lang=None,
implicit=True,
env=None,
flags=0):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
self._assertCalltipMatches(buf, trg, markedup_content, calltip, lang,
flags)
def _assertCalltipMatches(self, buf, trg, markedup_content, expr, lang,
flags):
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected the calltip to match the following:\n"
" exression:\n%s\n"
" buffer:\n%s" %
(lang, indent(expr), indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_calltips = buf.calltips_from_trg(trg, ctlr=ctlr)
if actual_calltips and actual_calltips[0]:
actual_calltip = actual_calltips[0]
else:
actual_calltip = None
self.assertNotEquals(
re.search(expr, actual_calltip, flags), None,
"unexpected %s calltip at the given position\n"
" expression:\n%s\n"
" got:\n%s\n"
" eval log:\n%s\n"
" buffer:\n%s" %
(trg.name, indent(expr and expr or "(none)"),
indent(actual_calltip and actual_calltip or "(none)"),
indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)),
indent(markedup_content)))
def assertCurrCalltipArgRange(self,
markedup_content,
calltip,
expected_range,
lang=None,
implicit=True):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(self.adjust_content(markedup_content))
pos = data["pos"]
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
trg = buf.trg_from_pos(data["trg_pos"], implicit=implicit)
actual_range = buf.curr_calltip_arg_range(trg.pos,
calltip,
curr_pos=data["pos"])
self.assertEqual(
actual_range, expected_range,
"unexpected current calltip arg range\n"
" expected: %s\n"
" got: %s\n"
" calltip:\n%s\n"
" buffer:\n%s" % (expected_range, actual_range, indent(calltip),
indent(markedup_content)))
#def assertCompletionRaises(self, markedup_content, exception, lang=None,
# **kwargs):
# """Assert that the given completion raises the given exception.
#
# You may also specify either of the "exc_args" or "exc_pattern"
# keyword args to match the exception's "args" attribute or match
# the stringified exception against a regex pattern.
#
# c.f. http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/307970
# """
# if lang is None:
# lang = self.lang
# if "exc_args" in kwargs:
# exc_args = kwargs["exc_args"]
# del kwargs["exc_args"]
# else:
# exc_args = None
# if "exc_pattern" in kwargs:
# exc_pattern = kwargs["exc_pattern"]
# del kwargs["exc_pattern"]
# else:
# exc_pattern = None
#
# callsig = "the given %s completion" % lang
# try:
# buf, trg = self._get_buf_and_trg(markedup_content, lang)
# if trg is None:
# self.fail("given position is not a %s trigger point, "
# "no completion can be done to see if it raises"
# % self.lang)
# callsig = "completion at %s" % trg
# cplns = buf.cplns_from_trg(trg)
# except exception, exc:
# if exc_args is not None:
# self.failIf(exc.args != exc_args,
# "%s raised %s with unexpected args: "\
# "expected=%r, actual=%r"\
# % (callsig, exc.__class__, exc_args, exc.args))
# if exc_pattern is not None:
# self.failUnless(exc_pattern.search(str(exc)),
# "%s raised %s, but the exception "\
# "does not match '%s': %r"\
# % (callsig, exc.__class__, exc_pattern.pattern,
# str(exc)))
# except:
# exc_info = sys.exc_info()
# print exc_info
# self.fail("%s raised an unexpected exception type: "\
# "expected=%s, actual=%s"\
# % (callsig, exception, exc_info[0]))
# else:
# self.fail("%s did not raise %s" % (callsig, exception))
def assertEvalError(self,
markedup_content,
log_pattern,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"no completion can be done to see if errors" % self.lang)
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
class TestEvalController(EvalController):
"""A completion controller that captures all eval logging."""
def __init__(self):
EvalController.__init__(self)
self.log = []
def debug(self, msg, *args):
self.log.append(("debug", msg % args))
def info(self, msg, *args):
self.log.append(("info", msg % args))
def warn(self, msg, *args):
self.log.append(("warn", msg % args))
def error(self, msg, *args):
self.log.append(("error", msg % args))
ctlr = TestEvalController()
buf.async_eval_at_trg(trg, ctlr=ctlr)
ctlr.wait()
if not ctlr.is_done():
self.fail("evaluation is not 'done': didn't expect that")
if trg.form == TRG_FORM_CPLN and ctlr.cplns:
self.fail("evalution had results: didn't expect that: %r" %
ctlr.cplns)
elif trg.form == TRG_FORM_CALLTIP and ctlr.calltips:
self.fail("evalution had results: didn't expect that: %r" %
ctlr.cplns)
if log_pattern:
#pprint(ctlr.log)
matching_logs = [(level, msg) for level, msg in ctlr.log
if log_pattern.search(msg)]
self.failUnless(
matching_logs,
"the given completion failed but no logs matched the given pattern:\n"
" log_pattern: /%s/\n"
" log messages:\n%s\n"
" buffer:\n%s" %
(log_pattern.pattern,
indent('\n'.join(['%s: %s' % lg for lg in ctlr.log
])), indent(markedup_content)))
#pprint(matching_logs)
def assertCITDLExprIs(self,
markedup_content,
citdl_expr,
lang=None,
prefix_filter=None,
implicit=True,
trigger_name=None,
**fields):
"""Assert that the preceding CITDL expression at the current position
is as expected.
This uses buf.citdl_expr_from_trg() -- or, for Perl,
buf.citdl_expr_and_prefix_filter_from_trg().
The "prefix_filter" optional argument can be used for Perl to test
the value its relevant function returns.
"""
if lang is None:
lang = self.lang
content, data = unmark_text(self.adjust_content(markedup_content))
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
langintel = self.mgr.langintel_from_lang(lang)
if trigger_name is None:
trigger_name = "fakey-completion-type"
if lang == "Perl":
# Bit of a hack to fake the trigger length.
if content[data["pos"] - 1] in ('>', ':'): # '->' or '::' triggers
length = 2
else:
length = 1
trg = Trigger(lang,
TRG_FORM_CPLN,
trigger_name,
data["pos"],
implicit=implicit,
length=length,
**fields)
actual_citdl_expr, actual_prefix_filter \
= langintel.citdl_expr_and_prefix_filter_from_trg(buf, trg)
else:
trg = Trigger(lang,
TRG_FORM_CPLN,
trigger_name,
data["pos"],
implicit=implicit,
**fields)
actual_citdl_expr = langintel.citdl_expr_from_trg(buf, trg)
self.assertEqual(
actual_citdl_expr, citdl_expr,
"unexpected actual %s CITDL expr preceding trigger:\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s" %
(lang, citdl_expr, actual_citdl_expr, indent(markedup_content)))
if lang == "Perl" and prefix_filter is not None:
self.assertEqual(
actual_prefix_filter, prefix_filter,
"unexpected actual %s variable prefix filter "
"preceding trigger:\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s" % (lang, prefix_filter, actual_prefix_filter,
indent(markedup_content)))
def _unmark_lex_text(self, markedup_text):
from SilverCity import ScintillaConstants
tokenizer = re.compile(r'(<(SCE_\w+)>(.*?)</\2>)')
tokens = []
text = ''
while markedup_text:
match = tokenizer.search(markedup_text)
if match:
text += markedup_text[:match.start()]
token = {
'end_index': len(text) + len(match.group(3)) - 1,
'start_index': len(text),
'style': getattr(ScintillaConstants, match.group(2)),
'text': match.group(3),
#'end_column': ???,
#'end_line': ???,
#'start_column': ???,
#'start_line': ???,
}
tokens.append(token)
text += match.group(3)
markedup_text = markedup_text[match.end():]
else:
text += markedup_text
markedup_text = ''
return text, tokens
def assertLex(self, markedup_content, lang=None):
"""Lex the given content and assert that the lexed tokens are as
expected.
What is "expected" is given via pseudo-xml markup like this:
fuzzy wuzzy <SCE_UDL_SSL_COMMENTBLOCK>wuzza</SCE_UDL_SSL_COMMENTBLOCK> bear
This example expects that "wuzza" will be a token with style
SCE_UDL_SSL_COMMENTBLOCK.
"""
from codeintel2.accessor import SilverCityAccessor
if lang is None:
lang = self.lang
content, tokens = self._unmark_lex_text(markedup_content)
# Do lexing of this content via the codeintel Buffer's, because
# they already handle all the SilverCity lexer hookup.
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
assert isinstance(buf.accessor, SilverCityAccessor)
actual_tokens = buf.accessor.tokens # cheating
for actual_token in actual_tokens:
# There are a few SilverCity token dict keys that we
# don't bother checking.
del actual_token["end_column"]
del actual_token["end_line"]
del actual_token["start_column"]
del actual_token["start_line"]
unmatched_tokens = [t for t in tokens if t not in actual_tokens]
if unmatched_tokens:
self.fail("not all expected %s lex tokens were found in the "
"actual lexer output:\n"
" buffer:\n%s\n"
" actual lexer tokens:\n%s\n"
" unmatched tokens:\n%s\n" %
(lang, indent(content), indent(pformat(actual_tokens)),
indent(pformat(unmatched_tokens))))
class Driver(threading.Thread):
"""
Out-of-process codeintel2 driver
This class implements the main loop of the codeintel worker process
This should be a singleton object
"""
# static
_command_handler_map = collections.defaultdict(list)
"""Registered command handlers"""
_default_handler = None
"""The default (core) command handler; instance of a CoreHandler"""
_builtin_commands = {}
"""Built-in commands that cannot be overridden"""
def __init__(self, db_base_dir=None, fd_in=sys.stdin, fd_out=sys.stdout):
threading.Thread.__init__(self, name="Codeintel OOP Driver")
self.daemon = True
self.fd_in = fd_in
self.fd_out = fd_out
self.abort = None
self.quit = False
self.buffers = {} # path to Buffer objects
self.next_buffer = 0
self.active_request = None
self.queue = collections.deque()
self.queue_cv = threading.Condition()
self.env = Environment(name="global",
send_fn=functools.partial(self.send,
request=None))
# Fill out the non-overridable build-in commands
self._builtin_commands = {}
for attr in dir(self):
# Note that we check startswith first to avoid getters etc.
if attr.startswith("do_") and callable(getattr(self, attr)):
command = attr[len("do_"):].replace("_", "-")
self._builtin_commands[command] = getattr(self, attr)
from codeintel2.manager import Manager
log.debug("using db base dir %s", db_base_dir)
self.mgr = Manager(db_base_dir=db_base_dir,
db_catalog_dirs=[],
env=self.env,
on_scan_complete=self._on_scan_complete)
self.mgr.initialize()
def _on_scan_complete(self, scan_request):
if scan_request.status == "changed":
# Send unsolicited response about the completed scan
buf = scan_request.buf
self.send(request=None,
path=buf.path,
language=buf.lang,
command="scan-complete")
REQUEST_DEFAULT = object()
def send(self, request=REQUEST_DEFAULT, **kwargs):
"""
Send a response
"""
data = dict(kwargs)
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if request:
data["req_id"] = request.id
if "success" not in data:
data["success"] = True
elif data["success"] is None:
del data["success"]
buf = json.dumps(data, separators=(',', ':'))
log.debug("sending: [%s]", buf)
self.fd_out.write(str(len(buf)) + buf)
def fail(self, request=REQUEST_DEFAULT, **kwargs):
kwargs = kwargs.copy()
if not "command" in kwargs and request:
try:
kwargs["command"] = request["command"]
except KeyError:
pass
return self.send(request=request, success=False, **kwargs)
def exception(self, request=REQUEST_DEFAULT, **kwargs):
return self.fail(request=request,
stack=traceback.format_exc(),
**kwargs)
def get_buffer(self, request=REQUEST_DEFAULT, path=None):
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if path is None:
if not "path" in request:
raise RequestFailure(message="No path given to locate buffer")
path = request.path
if abspath(path) == path:
# this is an actual file path, not a URL or whatever
path = normcase(normpath(path))
try:
buf = self.buffers[path]
except KeyError:
buf = None
else:
if "language" in request and buf.lang != request.language:
buf = None # language changed, re-scan
if not buf:
# Need to construct a new buffer
lang = request.get("language")
if request.get("text") is not None:
# pass no content; we'll reset it later
buf = self.mgr.buf_from_content("", lang, path=path)
else:
# read from file
try:
buf = self.mgr.buf_from_path(
path, lang, encoding=request.get("encoding"))
except OSError:
# Can't read the file
buf = self.mgr.buf_from_content("", lang, path=path)
assert not request.path.startswith("<")
if request.get("text") is not None:
# overwrite the buffer contents if we have new ones
buf.accessor.reset_content(request.text)
try:
env = request["env"]
except KeyError:
pass # no environment set, use current environment
else:
if env.get("env", {}) is None and env.get("prefs", []) is None:
buf._env = None # explicitly clearing environment
elif buf._env:
buf._env.update(env)
else:
buf._env = Environment(env, name=os.path.basename(path))
# log.debug("Got buffer %r: [%s]", buf, buf.accessor.content)
log.debug("Got buffer %r", buf)
self.buffers[path] = buf
return buf
def do_abort(self, request):
try:
req_id = request["id"]
with self.queue_cv:
for item in self.queue:
if queue.get("req_id") == req_id:
self.queue.remove(queue)
self.send(request=request)
break
else:
self.abort = req_id
if self.active_request and self.active_request.id == req_id:
# need to wait a bit...
self.send(request=request)
else:
self.fail(request=request,
message="Request %s not found" % (req_id, ))
except RequestFailure as e:
self.fail(request=request, *e.args, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
def do_add_dirs(self, request):
catalog_dirs = request.get("catalog-dirs", None)
if catalog_dirs is not None:
self.mgr.db.catalog_dirs.extend(catalog_dirs)
lexer_dirs = request.get("lexer-dirs", [])
codeintel2.udl.UDLLexer.add_extra_lexer_dirs(lexer_dirs)
module_dirs = request.get("module-dirs", [])
if module_dirs:
self.mgr._register_modules(module_dirs)
if catalog_dirs is not None:
self.mgr.db.get_catalogs_zone().update()
def do_load_extension(self, request):
"""Load an extension that, for example, might provide additional
command handlers"""
path = request.get("module-path", None)
if not path:
raise RequestFailure(msg="load-extension requires a module-path")
name = request.get("module-name", None)
if not name:
raise RequestFailure(msg="load-extension requires a module-name")
iinfo = imp.find_module(name, [path])
try:
module = imp.load_module(name, *iinfo)
finally:
if iinfo and iinfo[0]:
iinfo[0].close()
callback = getattr(module, "registerExtension", None)
if not callback:
raise RequestFailure(msg="load-extension module %s should "
"have a 'registerExtension' method "
"taking no arguments" % (name, ))
callback()
self.send() # success, finally
def do_quit(self, request):
self.quit = True
self.send(command="quit")
def report_error(self, message):
self.send(request=None,
command="report-error",
message=unicode(message))
def start(self):
"""Start reading from the socket and dump requests into the queue"""
log.info("Running codeintel driver...")
buf = ""
self.send(success=None)
self.daemon = True
threading.Thread.start(self)
while not self.quit:
try:
ch = self.fd_in.read(1)
except IOError:
log.debug(
"Failed to read frame length, assuming connection died")
self.quit = True
break
if len(ch) == 0:
log.debug("Input was closed")
self.quit = True
break
if ch == "{":
size = int(buf, 10)
try:
buf = ch + self.fd_in.read(
size - 1) # exclude already-read {
except IOError:
log.debug(
"Failed to read frame data, assuming connection died")
self.quit = True
break
try:
buf = buf.decode("utf-8")
except UnicodeDecodeError:
pass # what :(
try:
data = json.loads(buf)
request = Request(data)
except Exception as e:
log.exception(e)
self.exception(message=e.message, request=None)
continue
finally:
buf = ""
if request.get("command") == "abort":
self.do_abort(request=request)
else:
log.debug("queuing request %r", request)
with self.queue_cv:
self.queue.appendleft(request)
self.queue_cv.notify()
elif ch in "0123456789":
buf += ch
else:
raise ValueError("Invalid request data: " + ch.encode("hex"))
def run(self):
"""Evaluate and send results back"""
log.info("Running codeintel eval thread...")
buf = ""
log.debug("default supported commands: %s",
", ".join(self._default_handler.supportedCommands))
while True:
with self.queue_cv:
try:
request = self.queue.pop()
except IndexError:
self.queue_cv.wait()
continue
log.debug("doing request %r", request)
try:
self.active_request = request
command = request.command
# First, check abort and quit; don't allow those to be
# overridden
try:
builtin = self._builtin_commands[command]
except KeyError:
pass
else:
builtin(request)
continue
handlers = self._command_handler_map.get(command, [])[:]
if command in self._default_handler.supportedCommands:
# The default handler can deal with this, put it at the end
handlers.append(self._default_handler)
for handler in handlers:
if handler.canHandleRequest(request):
handler.handleRequest(request, self)
break
else:
self.fail(request=request,
msg="Don't know how to handle command %s" %
(command, ))
except RequestFailure as e:
self.fail(request=request, *e.args, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
finally:
self.active_request = None
@classmethod
def registerCommandHandler(cls, handlerInstance):
"""Register a command handler"""
for command in handlerInstance.supportedCommands:
cls._command_handler_map[command].append(handlerInstance)
class CodeIntelTestCase(unittest.TestCase):
# Subclasses can override this to have setUp pass these settings to the
# codeintel.Manager.
_ci_db_base_dir_ = None
_ci_db_catalog_dirs_ = []
_ci_db_import_everything_langs = None
_ci_env_prefs_ = None
# A test case can set this to false to have the setUp/tearDown *not*
# create a `self.mgr'.
_ci_test_setup_mgr_ = True
_ci_extra_module_dirs_ = None
def setUp(self):
if _xpcom_:
# The tests are run outside of Komodo. If run with PyXPCOM up
# parts codeintel will try to use the nsIDirectoryService and
# will query dirs only provided by nsXREDirProvider -- which
# isn't registered outside of Komodo (XRE_main() isn't called).
# The KoTestService provides a backup.
koTestSvc = components.classes["@activestate.com/koTestService;1"] \
.getService(components.interfaces.koITestService)
koTestSvc.init()
if self._ci_test_setup_mgr_:
env = None
if self._ci_env_prefs_ is not None:
env = SimplePrefsEnvironment(**self._ci_env_prefs_)
self.mgr = Manager(
extra_module_dirs=self._ci_extra_module_dirs_,
db_base_dir=self._ci_db_base_dir_ or test_db_base_dir,
db_catalog_dirs=self._ci_db_catalog_dirs_,
db_import_everything_langs=self._ci_db_import_everything_langs,
env=env)
self.mgr.upgrade()
self.mgr.initialize()
def tearDown(self):
if self._ci_test_setup_mgr_:
self.mgr.finalize()
self.mgr = None
def adjust_content(self, content):
"""A hook for subclasses to modify markedup_content before use in
test cases. This is useful for sharing test cases between pure-
and multi-lang uses of a given lang.
"""
return content
def adjust_pos(self, pos):
"""A accompanying hook for `adjust_content' to adjust trigger
pos values accordingly.
"""
return pos
def _get_buf_and_data(self, markedup_content, lang, path=None, env=None):
if path is None:
# Try to ensure no accidental re-use of the same buffer name
# across the whole test suite. Also try to keep the buffer
# names relatively short (keeps tracebacks cleaner).
name = "buf-" + md5(markedup_content).hexdigest()[:16]
path = os.path.join("<Unsaved>", name)
content, data = unmark_text(self.adjust_content(markedup_content))
#print banner(path)
#sys.stdout.write(content)
#print banner(None)
buf = self.mgr.buf_from_content(content, lang=lang, path=path,
env=env)
return buf, data
def _get_buf_and_trg(self, markedup_content, lang, path=None,
implicit=True, env=None):
buf, data = self._get_buf_and_data(markedup_content, lang, path, env)
trg = buf.trg_from_pos(data["pos"], implicit=implicit)
return buf, trg
def assertCITDLExprUnderPosIs(self, markedup_content, citdl_expr, lang=None,
prefix_filter=None, implicit=True, trigger_name=None,
**fields):
"""Assert that the CITDL expression at the current position
is as expected.
This uses buf.citdl_expr_under_pos() -- or, for Perl,
buf.citdl_expr_and_prefix_filter_from_trg().
Note: This API is a mess right now. C.f. bug 65776.
The "prefix_filter" optional argument can be used for Perl to test
the value its relevant function returns.
"""
if lang is None:
lang = self.lang
content, data = unmark_text(
self.adjust_content(markedup_content))
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
langintel = self.mgr.langintel_from_lang(lang)
if trigger_name is None:
trigger_name = "fakey-completion-type"
if lang == "Perl":
trg = Trigger(lang, TRG_FORM_DEFN, trigger_name,
data["pos"], implicit=implicit, length=0,
**fields)
actual_citdl_expr, actual_prefix_filter \
= langintel.citdl_expr_and_prefix_filter_from_trg(buf, trg)
else:
#actual_citdl_expr = langintel.citdl_expr_under_pos(buf, data["pos"])
trg = Trigger(lang, TRG_FORM_DEFN, trigger_name,
data["pos"], implicit=implicit,
**fields)
actual_citdl_expr = langintel.citdl_expr_from_trg(buf, trg)
self.assertEqual(actual_citdl_expr, citdl_expr,
"unexpected actual %s CITDL expr under pos:\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s"
% (lang, citdl_expr, actual_citdl_expr,
indent(markedup_content)))
if prefix_filter is not None:
XXX #TODO: compare prefix_filter to given value
def _assertDefnMatches(self, buf, pos, lang=None, **fields):
ctlr = _CaptureEvalController()
trg = buf.defn_trg_from_pos(pos)
defns = buf.defns_from_trg(trg, ctlr=ctlr)
if not defns:
self.fail("unexpectedly did not find a definition in %r at pos %d\n"
" eval log\n%s\n"
" buffer:\n%s"
% (buf, pos,
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(buf.accessor.text)))
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
defn = defns[0]
for name, value in fields.items():
try:
actual_value = getattr(defn, name)
except AttributeError:
actual_value = None
self.assertEqual(actual_value, value,
"%s definition, unexpected value for field %r\n"
" defn: %r\n"
" expected: %r\n"
" got: %r\n"
" eval log\n%s\n"
" buffer:\n%s"
% (buf.lang, name, defn, value, actual_value,
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(buf.accessor.text)))
def assertDefnMatches(self, markedup_content, lang=None, **fields):
if lang is None:
lang = self.lang
buf, data = self._get_buf_and_data(markedup_content, lang)
self._assertDefnMatches(buf, data["pos"], lang, **fields)
def assertDefnMatches2(self, buf, pos, lang=None, **fields):
self._assertDefnMatches(buf, pos, lang, **fields)
def assertNoDuplicateDefns(self, markedup_content, lang=None):
if lang is None:
lang = self.lang
buf, data = self._get_buf_and_data(markedup_content, lang)
self.assertNoDuplicateDefns2(buf, data["pos"])
def assertNoDuplicateDefns2(self, buf, pos):
markedup_content = markup_text(buf.accessor.text, pos=pos)
ctlr = _CaptureEvalController()
trg = buf.defn_trg_from_pos(pos)
actual_defns = buf.defns_from_trg(trg, ctlr=ctlr)
if not actual_defns:
self.fail("%s trigger resulted in no definitions when expecting "
"to check for duplicate definitions:\n%s"
% (buf.lang, indent(markedup_content)))
count_from_defn_repr = {}
for defn_repr in (repr(d) for d in actual_defns):
if defn_repr not in count_from_defn_repr:
count_from_defn_repr[defn_repr] = 0
count_from_defn_repr[defn_repr] += 1
defn_dupes = [(count, defn_repr)
for defn_repr, count in count_from_defn_repr.items()
if count > 1]
self.failIf(defn_dupes,
"unexpectedly got duplicate completions at the given position\n"
" duplicates:\n%s\n"
" eval log\n%s\n"
" buffer:\n%s"
% (indent('\n'.join('%d of %s' % d for d in defn_dupes)),
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(markedup_content)))
def assertTriggerMatches(self, markedup_content, lang=None,
implicit=True, env=None, **fields):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
if trg is None:
self.fail("unexpectedly did not find a %s trigger, buffer:\n%s"
% (lang, indent(markedup_content)))
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
for name, value in fields.items():
try:
actual_value = getattr(trg, name)
except AttributeError:
actual_value = trg.extra[name]
self.assertEqual(actual_value, value,
"unexpected %s trigger '%s' value: expected %r, "
"got %r, buffer:\n%s"
% (lang, name, value, actual_value,
indent(markedup_content)))
# Used when a position generates a trigger, but it's not the one specified
def assertTriggerDoesNotMatch(self, markedup_content, lang=None,
implicit=True, env=None, **fields):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
if trg is None:
# No trigger is as good as
return
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
for name, value in fields.items():
try:
actual_value = getattr(trg, name)
except AttributeError:
actual_value = trg.extra[name]
self.assertNotEqual(actual_value, value,
"unexpected %s trigger '%s' value: expected not %r, "
"got %r, buffer:\n%s"
% (lang, name, value, actual_value,
indent(markedup_content)))
def assertNoTrigger(self, markedup_content, lang=None, implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
if trg is not None:
self.fail("unexpectedly found a %s trigger %r when didn't expect "
"one, buffer:\n%s"
% (lang, trg, indent(markedup_content)))
def assertPrecedingTriggerMatches(self, markedup_content, lang=None,
**fields):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(
self.adjust_content(markedup_content))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
trg = buf.preceding_trg_from_pos(data["start_pos"], data["pos"])
if trg is None:
self.fail("unexpectedly did not find a preceding %s trigger, "
"buffer:\n%s" % (lang, indent(markedup_content)))
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
for name, value in fields.items():
actual_value = getattr(trg, name)
self.assertEqual(actual_value, value,
"unexpected preceding %s trigger '%s' value: expected %r, "
"got %r, buffer:\n%s"
% (lang, name, value, actual_value,
indent(markedup_content)))
def assertNoPrecedingTrigger(self, markedup_content, lang=None):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(
self.adjust_content(markedup_content))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
trg = buf.preceding_trg_from_pos(data["start_pos"], data["pos"])
if trg is not None:
self.fail("unexpectedly found a preceding %s trigger '%s' when "
"didn't expect one, buffer:\n%s"
% (lang, trg.name, indent(markedup_content)))
def assertScopeLpathIs(self, markedup_content, lpath, lang=None):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(
self.adjust_content(markedup_content))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
buf.scan(skip_scan_time_check=True)
actual_blob, actual_lpath = buf.scoperef_from_pos(data["pos"])
self.failUnlessEqual(lpath, actual_lpath,
"unexpected %s scope lookup path (lpath) at the given position\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s"
% (self.lang, lpath,
actual_lpath,
indent(markedup_content)))
def assertCompletionsAre2(self, buf, pos, completions, lang=None, implicit=True, env=None):
if lang is None:
lang = self.lang
markedup_content = markup_text(buf.accessor.text, pos=pos)
trg = buf.trg_from_pos(pos, implicit=implicit)
self._assertCompletionsAre(markedup_content, buf, trg, completions, lang, implicit)
def assertCompletionsAre(self, markedup_content, completions,
lang=None, implicit=True, env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
self._assertCompletionsAre(markedup_content, buf, trg, completions, lang, implicit)
def _assertCompletionsAre(self, markedup_content, buf, trg, completions,
lang, implicit):
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected completions to be %r:\n%s"
% (lang, completions, indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
self.assertEqual(completions, actual_completions,
"unexpected %s completions at the given position\n"
" expected: %r\n"
" got: %r\n"
" extra: %r\n"
" missing: %r\n"
" eval log\n%s\n"
" buffer:\n%s"
% (lang, completions,
actual_completions,
list(set(actual_completions or []).difference(completions or [])),
list(set(completions or []).difference(actual_completions or [])),
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(markedup_content)))
def assertNoDuplicateCompletions(self, markedup_content, lang=None,
implicit=True, env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected there to be completions:\n%s"
% (lang, indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
if actual_completions is None:
self.fail("%s trigger resulted in no completions when expecting "
"to check for duplicate completions:\n%s"
% (lang, indent(markedup_content)))
count_from_cpln = {}
for cpln in actual_completions:
if cpln not in count_from_cpln:
count_from_cpln[cpln] = 0
count_from_cpln[cpln] += 1
cpln_dupes = [(count, cpln) for cpln, count in count_from_cpln.items()
if count > 1]
self.failIf(cpln_dupes,
"unexpectedly got duplicate completions at the given position\n"
" duplicates:\n%s\n"
" eval log\n%s\n"
" buffer:\n%s"
% (indent('\n'.join('%d of %r' % d for d in cpln_dupes)),
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(markedup_content)))
def _assertCompletionsInclude(self, buf, trg, completions):
markedup_content = markup_text(buf.accessor.text, pos=trg.pos)
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
missing_completions = [c for c in completions
if c not in (actual_completions or [])]
self.failIf(missing_completions,
"%s completions at the given position did not "
"include all expected values\n"
" missing: %r\n"
" expected all of: %r\n"
" got: %r\n"
" eval log:\n%s\n"
" buffer:\n%s"
% (buf.lang, missing_completions, completions,
actual_completions,
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(markedup_content)))
def assertCompletionsInclude(self, markedup_content, completions,
lang=None, implicit=True, env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected completions to include %r:\n%s"
% (lang, completions, indent(markedup_content)))
self._assertCompletionsInclude(buf, trg, completions)
def assertCompletionsInclude2(self, buf, pos, completions, implicit=True):
"""A version of assertCompletionsInclude() where you pass in
a Buffer instance instead of marked up content. Sometimes
this is more convenient.
"""
trg = buf.trg_from_pos(pos, implicit=implicit)
if trg is None:
markedup_content = markup_text(buf.accessor.text, pos=pos)
self.fail("given position is not a %s trigger point, "
"expected completions to include %r:\n%s"
% (buf.lang, completions, indent(markedup_content)))
self._assertCompletionsInclude(buf, trg, completions)
def _assertCompletionsDoNotInclude(self, buf, trg, completions):
markedup_content = markup_text(buf.accessor.text, pos=trg.pos)
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
completions_that_shouldnt_be_there = [
c for c in (actual_completions or []) if c in completions
]
self.failIf(completions_that_shouldnt_be_there,
"%s completions at the given position included "
"some unexpected values\n"
" shouldn't have had these: %r\n"
" expected none of: %r\n"
" got: %r\n"
" eval log:\n%s\n"
" buffer:\n%s"
% (buf.lang, completions_that_shouldnt_be_there, completions,
actual_completions,
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(markedup_content)))
def assertCompletionsDoNotInclude(self, markedup_content, completions,
lang=None, implicit=True, env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected completions to exclude %r:\n%s"
% (lang, completions, indent(markedup_content)))
self._assertCompletionsDoNotInclude(buf, trg, completions)
def assertCompletionsDoNotInclude2(self, buf, pos, completions, implicit=True):
"""A version of assertCompletionsDoNotInclude() where you pass in
a Buffer instance instead of marked up content. Sometimes
this is more convenient.
"""
trg = buf.trg_from_pos(pos, implicit=implicit)
if trg is None:
markedup_content = markup_text(buf.accessor.text, pos=pos)
self.fail("given position is not a %s trigger point, "
"expected completions to exclude %r:\n%s"
% (buf.lang, completions, indent(markedup_content)))
self._assertCompletionsDoNotInclude(buf, trg, completions)
def assertCalltipIs2(self, buf, pos, calltip, implicit=True):
"""A variant of assertCalltipIs() where you pass in
a Buffer instance instead of marked up content. Sometimes
this is more convenient.
"""
trg = buf.trg_from_pos(pos, implicit=implicit)
markedup_content = markup_text(buf.accessor.text, pos=trg.pos)
self._assertCalltipIs(buf, trg, markedup_content, calltip, self.lang)
def assertCalltipIs(self, markedup_content, calltip, lang=None,
implicit=True, env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
self._assertCalltipIs(buf, trg, markedup_content, calltip, lang)
def _assertCalltipIs(self, buf, trg, markedup_content, calltip, lang):
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected the following calltip:\n"
" calltip:\n%s\n"
" buffer:\n%s"
% (lang, indent(calltip),
indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_calltips = buf.calltips_from_trg(trg, ctlr=ctlr)
if actual_calltips and actual_calltips[0]:
actual_calltip = actual_calltips[0]
else:
actual_calltip = None
self.assertEqual(calltip, actual_calltip,
"unexpected %s calltip at the given position\n"
" expected:\n%s\n"
" got:\n%s\n"
" eval log:\n%s\n"
" buffer:\n%s"
% (trg.name, indent(calltip and calltip or "(none)"),
indent(actual_calltip and actual_calltip or "(none)"),
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(markedup_content)))
def assertCalltipMatches(self, markedup_content, calltip, lang=None,
implicit=True, env=None, flags=0):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
self._assertCalltipMatches(buf, trg, markedup_content, calltip, lang, flags)
def _assertCalltipMatches(self, buf, trg, markedup_content, expr, lang, flags):
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected the calltip to match the following:\n"
" exression:\n%s\n"
" buffer:\n%s"
% (lang, indent(expr),
indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_calltips = buf.calltips_from_trg(trg, ctlr=ctlr)
if actual_calltips and actual_calltips[0]:
actual_calltip = actual_calltips[0]
else:
actual_calltip = None
self.assertNotEquals(re.search(expr, actual_calltip, flags), None,
"unexpected %s calltip at the given position\n"
" expression:\n%s\n"
" got:\n%s\n"
" eval log:\n%s\n"
" buffer:\n%s"
% (trg.name, indent(expr and expr or "(none)"),
indent(actual_calltip and actual_calltip or "(none)"),
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(markedup_content)))
def assertCurrCalltipArgRange(self, markedup_content, calltip,
expected_range, lang=None, implicit=True):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(
self.adjust_content(markedup_content))
pos = data["pos"]
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
trg = buf.trg_from_pos(data["trg_pos"], implicit=implicit)
actual_range = buf.curr_calltip_arg_range(trg.pos, calltip,
curr_pos=data["pos"])
self.assertEqual(actual_range, expected_range,
"unexpected current calltip arg range\n"
" expected: %s\n"
" got: %s\n"
" calltip:\n%s\n"
" buffer:\n%s"
% (expected_range, actual_range, indent(calltip),
indent(markedup_content)))
#def assertCompletionRaises(self, markedup_content, exception, lang=None,
# **kwargs):
# """Assert that the given completion raises the given exception.
#
# You may also specify either of the "exc_args" or "exc_pattern"
# keyword args to match the exception's "args" attribute or match
# the stringified exception against a regex pattern.
#
# c.f. http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/307970
# """
# if lang is None:
# lang = self.lang
# if "exc_args" in kwargs:
# exc_args = kwargs["exc_args"]
# del kwargs["exc_args"]
# else:
# exc_args = None
# if "exc_pattern" in kwargs:
# exc_pattern = kwargs["exc_pattern"]
# del kwargs["exc_pattern"]
# else:
# exc_pattern = None
#
# callsig = "the given %s completion" % lang
# try:
# buf, trg = self._get_buf_and_trg(markedup_content, lang)
# if trg is None:
# self.fail("given position is not a %s trigger point, "
# "no completion can be done to see if it raises"
# % self.lang)
# callsig = "completion at %s" % trg
# cplns = buf.cplns_from_trg(trg)
# except exception, exc:
# if exc_args is not None:
# self.failIf(exc.args != exc_args,
# "%s raised %s with unexpected args: "\
# "expected=%r, actual=%r"\
# % (callsig, exc.__class__, exc_args, exc.args))
# if exc_pattern is not None:
# self.failUnless(exc_pattern.search(str(exc)),
# "%s raised %s, but the exception "\
# "does not match '%s': %r"\
# % (callsig, exc.__class__, exc_pattern.pattern,
# str(exc)))
# except:
# exc_info = sys.exc_info()
# print exc_info
# self.fail("%s raised an unexpected exception type: "\
# "expected=%s, actual=%s"\
# % (callsig, exception, exc_info[0]))
# else:
# self.fail("%s did not raise %s" % (callsig, exception))
def assertEvalError(self, markedup_content, log_pattern, lang=None,
implicit=True, env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"no completion can be done to see if errors"
% self.lang)
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
class TestEvalController(EvalController):
"""A completion controller that captures all eval logging."""
def __init__(self):
EvalController.__init__(self)
self.log = []
def debug(self, msg, *args):
self.log.append(("debug", msg % args))
def info(self, msg, *args):
self.log.append(("info", msg % args))
def warn(self, msg, *args):
self.log.append(("warn", msg % args))
def error(self, msg, *args):
self.log.append(("error", msg % args))
ctlr = TestEvalController()
buf.async_eval_at_trg(trg, ctlr=ctlr)
ctlr.wait()
if not ctlr.is_done():
self.fail("evaluation is not 'done': didn't expect that")
if trg.form == TRG_FORM_CPLN and ctlr.cplns:
self.fail("evalution had results: didn't expect that: %r"
% ctlr.cplns)
elif trg.form == TRG_FORM_CALLTIP and ctlr.calltips:
self.fail("evalution had results: didn't expect that: %r"
% ctlr.cplns)
if log_pattern:
#pprint(ctlr.log)
matching_logs = [(level, msg) for level, msg in ctlr.log
if log_pattern.search(msg)]
self.failUnless(matching_logs,
"the given completion failed but no logs matched the given pattern:\n"
" log_pattern: /%s/\n"
" log messages:\n%s\n"
" buffer:\n%s"
% (log_pattern.pattern,
indent('\n'.join(['%s: %s' % lg for lg in ctlr.log])),
indent(markedup_content)))
#pprint(matching_logs)
def assertCITDLExprIs(self, markedup_content, citdl_expr, lang=None,
prefix_filter=None, implicit=True, trigger_name=None,
**fields):
"""Assert that the preceding CITDL expression at the current position
is as expected.
This uses buf.citdl_expr_from_trg() -- or, for Perl,
buf.citdl_expr_and_prefix_filter_from_trg().
The "prefix_filter" optional argument can be used for Perl to test
the value its relevant function returns.
"""
if lang is None:
lang = self.lang
content, data = unmark_text(
self.adjust_content(markedup_content))
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
langintel = self.mgr.langintel_from_lang(lang)
if trigger_name is None:
trigger_name = "fakey-completion-type"
if lang == "Perl":
# Bit of a hack to fake the trigger length.
if content[data["pos"]-1] in ('>', ':'): # '->' or '::' triggers
length = 2
else:
length = 1
trg = Trigger(lang, TRG_FORM_CPLN, trigger_name,
data["pos"], implicit=implicit, length=length,
**fields)
actual_citdl_expr, actual_prefix_filter \
= langintel.citdl_expr_and_prefix_filter_from_trg(buf, trg)
else:
trg = Trigger(lang, TRG_FORM_CPLN, trigger_name,
data["pos"], implicit=implicit,
**fields)
actual_citdl_expr = langintel.citdl_expr_from_trg(buf, trg)
self.assertEqual(actual_citdl_expr, citdl_expr,
"unexpected actual %s CITDL expr preceding trigger:\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s"
% (lang, citdl_expr, actual_citdl_expr,
indent(markedup_content)))
if lang == "Perl" and prefix_filter is not None:
self.assertEqual(actual_prefix_filter, prefix_filter,
"unexpected actual %s variable prefix filter "
"preceding trigger:\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s"
% (lang, prefix_filter, actual_prefix_filter,
indent(markedup_content)))
def _unmark_lex_text(self, markedup_text):
from SilverCity import ScintillaConstants
tokenizer = re.compile(r'(<(SCE_\w+)>(.*?)</\2>)')
tokens = []
text = ''
while markedup_text:
match = tokenizer.search(markedup_text)
if match:
text += markedup_text[:match.start()]
token = {
'end_index': len(text) + len(match.group(3)) - 1,
'start_index': len(text),
'style': getattr(ScintillaConstants, match.group(2)),
'text': match.group(3),
#'end_column': ???,
#'end_line': ???,
#'start_column': ???,
#'start_line': ???,
}
tokens.append(token)
text += match.group(3)
markedup_text = markedup_text[match.end():]
else:
text += markedup_text
markedup_text = ''
return text, tokens
def assertLex(self, markedup_content, lang=None):
"""Lex the given content and assert that the lexed tokens are as
expected.
What is "expected" is given via pseudo-xml markup like this:
fuzzy wuzzy <SCE_UDL_SSL_COMMENTBLOCK>wuzza</SCE_UDL_SSL_COMMENTBLOCK> bear
This example expects that "wuzza" will be a token with style
SCE_UDL_SSL_COMMENTBLOCK.
"""
from codeintel2.accessor import SilverCityAccessor
if lang is None:
lang = self.lang
content, tokens = self._unmark_lex_text(markedup_content)
# Do lexing of this content via the codeintel Buffer's, because
# they already handle all the SilverCity lexer hookup.
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
assert isinstance(buf.accessor, SilverCityAccessor)
actual_tokens = buf.accessor.tokens # cheating
for actual_token in actual_tokens:
# There are a few SilverCity token dict keys that we
# don't bother checking.
del actual_token["end_column"]
del actual_token["end_line"]
del actual_token["start_column"]
del actual_token["start_line"]
unmatched_tokens = [t for t in tokens if t not in actual_tokens]
if unmatched_tokens:
self.fail("not all expected %s lex tokens were found in the "
"actual lexer output:\n"
" buffer:\n%s\n"
" actual lexer tokens:\n%s\n"
" unmatched tokens:\n%s\n"
% (lang,
indent(content),
indent(pformat(actual_tokens)),
indent(pformat(unmatched_tokens))))
