def codeintel_manager():
global _ci_mgr_, condeintel_log_filename, condeintel_log_file
if _ci_mgr_:
mgr = _ci_mgr_
else:
for thread in threading.enumerate():
if thread.name == "CodeIntel Manager":
thread.finalize() # this finalizes the index, citadel and the manager and waits them to end (join)
mgr = Manager(
extra_module_dirs=_ci_extra_module_dirs_,
db_base_dir=_ci_db_base_dir_,
db_catalog_dirs=_ci_db_catalog_dirs_,
db_import_everything_langs=_ci_db_import_everything_langs,
)
mgr.upgrade()
mgr.initialize()
# Connect the logging file to the handler
condeintel_log_filename = os.path.join(mgr.db.base_dir, "codeintel.log")
condeintel_log_file = open(condeintel_log_filename, "w", 1)
codeintel_log.handlers = [logging.StreamHandler(condeintel_log_file)]
msg = "Starting logging SublimeCodeIntel rev %s (%s) on %s" % (
get_revision()[:12],
os.stat(__file__)[stat.ST_MTIME],
datetime.datetime.now().ctime(),
)
print >> condeintel_log_file, "%s\n%s" % (msg, "=" * len(msg))
_ci_mgr_ = mgr
return mgr
def codeintel_manager(folders_id):
folders_id = None
global _ci_mgr_, condeintel_log_filename, condeintel_log_file
mgr = _ci_mgr_.get(folders_id)
if mgr is None:
for thread in threading.enumerate():
if thread.name == "CodeIntel Manager":
thread.finalize() # this finalizes the index, citadel and the manager and waits them to end (join)
mgr = Manager(
extra_module_dirs=None,
db_base_dir=None, # os.path.expanduser(os.path.join('~', '.codeintel', 'databases', folders_id)),
db_catalog_dirs=[],
db_import_everything_langs=None,
)
mgr.upgrade()
mgr.initialize()
# Connect the logging file to the handler
condeintel_log_filename = os.path.join(mgr.db.base_dir, 'codeintel.log')
condeintel_log_file = open(condeintel_log_filename, 'w', 1)
codeintel_log.handlers = [logging.StreamHandler(condeintel_log_file)]
msg = "Starting logging SublimeCodeIntel v%s rev %s (%s) on %s" % (VERSION, get_revision()[:12], os.stat(__file__)[stat.ST_MTIME], datetime.datetime.now().ctime())
print("%s\n%s" % (msg, "=" * len(msg)), file=condeintel_log_file)
_ci_mgr_[folders_id] = mgr
return mgr
def do_html(self, subcmd, opts, path):
"""Convert the given path to styled HTML.
${cmd_usage}
${cmd_option_list}
The output includes trigger info and other stats. I.e. this is
primarily a debugging tool.
"""
from codeintel2.manager import Manager
from codeintel2.common import Error
from ci2 import _url_from_local_path
mgr = Manager()
try:
if opts.browse:
htmls = []
buf = mgr.buf_from_path(path, lang=opts.lang)
html = buf.to_html(True, True, title=path,
do_trg=opts.do_trg,
do_eval=opts.do_eval)
finally:
mgr.finalize()
if opts.output == '-':
output_path = None
output_file = sys.stdout
else:
if opts.output:
output_path = opts.output
else:
output_path = path + ".html"
if os.path.exists(output_path):
if opts.force:
os.remove(output_path)
else:
raise Error("`%s' exists: use -f|--force option to allow overwrite" % output_path)
output_file = open(output_path, 'w')
# else:
# output_path = None
# output_file = sys.stdout
# #XXX Disable writing t
# output_file = None
if output_file:
output_file.write(html)
if output_path:
output_file.close()
if opts.browse:
if not output_path:
raise Error("cannot open in browser if stdout used "
"for output")
import webbrowser
url = _url_from_local_path(output_path)
webbrowser.open_new(url)
def do_html(self, subcmd, opts, path):
"""Convert the given path to styled HTML.
${cmd_usage}
${cmd_option_list}
The generated HTML provides a good tool for seeing how Komodo's
lexer lexes the given content. This is the same tokenization that
you get from "buf.accessor.*" methods -- which you can use for
writing the CILE, trigger handling, and completion evaluation
for this language.
Use the "-t" and "-e" option to also exercise the current
trigger handling and completion evaluation (i.e. determining
the appropriate completions and calltips at a given trigger
point).
"""
extra_lang_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_lang_module_dirs = [sys.path[-1]]
mgr = Manager(extra_lang_module_dirs=extra_lang_module_dirs)
try:
if opts.browse:
htmls = []
buf = mgr.buf_from_path(path, lang=opts.lang)
html = buf.to_html(True, True, title=path, do_trg=opts.do_trg, do_eval=opts.do_eval)
finally:
mgr.finalize()
if opts.output == "-":
output_path = None
output_file = sys.stdout
else:
if opts.output:
output_path = opts.output
else:
output_path = path + ".html"
if exists(output_path):
os.remove(output_path)
output_file = open(output_path, "w")
if output_file:
output_file.write(html)
if output_path:
output_file.close()
if opts.browse:
if not output_path:
raise CodeIntelError("cannot open in browser if stdout " "used for output")
import webbrowser
url = _url_from_local_path(output_path)
webbrowser.open_new(url)
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
extra_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_module_dirs = [sys.path[-1]]
mgr = Manager(extra_module_dirs=extra_module_dirs)
mgr.upgrade()
mgr.initialize()
try:
tree = None
for path in _paths_from_path_patterns(path_patterns):
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info("skip `%s': couldn't determine language " "(use --lang option)", path)
continue
buf = mgr.buf_from_path(path, lang=opts.lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang))
buf.scan() # force a fresh scan
tree = buf.tree
for severity, msg in check_tree(tree):
dump = {"warning": log.warn, "error": log.error}[severity]
dump(msg)
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
ET.dump(tree)
finally:
mgr.finalize()
def do_dbcheck(self, subcmd, opts):
"""Run an internal consistency check on the database.
${cmd_usage}
${cmd_option_list}
Any errors will be printed. Returns the number of errors (i.e.
exit value is 0 if there are no consistency problems).
"""
mgr = Manager(opts.db_base_dir)
try:
errors = mgr.db.check()
finally:
mgr.finalize()
for error in errors:
print error
return len(errors)
def __init__(self, db_base_dir=None, fd_in=sys.stdin, fd_out=sys.stdout):
threading.Thread.__init__(self, name="Codeintel OOP Driver")
self.daemon = True
self.fd_in = fd_in
self.fd_out = fd_out
self.abort = None
self.quit = False
self.buffers = {} # path to Buffer objects
self.next_buffer = 0
self.active_request = None
self.queue = collections.deque()
self.queue_cv = threading.Condition()
self.env = Environment(name="global",
send_fn=functools.partial(self.send, request=None))
# Fill out the non-overridable build-in commands
self._builtin_commands = {}
for attr in dir(self):
# Note that we check startswith first to avoid getters etc.
if attr.startswith("do_") and callable(getattr(self, attr)):
command = attr[len("do_"):].replace("_", "-")
self._builtin_commands[command] = getattr(self, attr)
from codeintel2.manager import Manager
log.debug("using db base dir %s", db_base_dir)
self.mgr = Manager(db_base_dir=db_base_dir,
db_catalog_dirs=[],
env=self.env,
on_scan_complete=self._on_scan_complete)
self.mgr.initialize()
def do_json(self, subcmd, opts, path):
"""Convert cix XML file into json format.
${cmd_usage}
${cmd_option_list}
"""
import json
if opts.output == '-':
output_path = None
output_file = sys.stdout
else:
if opts.output:
output_path = opts.output
else:
output_path = splitext(path)[0]+".json"
if exists(output_path):
if opts.force:
os.remove(output_path)
else:
raise Error("`%s' exists: use -f|--force option to "
"allow overwrite" % output_path)
output_file = open(output_path, 'w')
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
tree = buf.tree
result = {}
ci = result["codeintel"] = defaultdict(list)
def _elemToDict(parent, elem):
data = defaultdict(list)
name = elem.get("name")
if name is not None:
data["name"] = name
data["tag"] = elem.tag
for attr_name, attr in elem.attrib.items():
data[attr_name] = attr
parent["children"].append(data)
for child in elem:
_elemToDict(data, child)
for child in tree:
_elemToDict(ci, child)
json.dump(result, output_file, indent=2)
finally:
mgr.finalize()
def worker(queue, lock, cix_dir, root="/"):
"""
worker procedure
"""
fix_module_path()
import dxr.mime
from codeintel2.citadel import CitadelBuffer
from codeintel2.common import CodeIntelError
from codeintel2.manager import Manager
from codeintel2.util import guess_lang_from_path
logging.getLogger("codeintel").setLevel(logging.ERROR)
log.info("starting indexing using %s", multiprocessing.current_process().name)
mgr = Manager(db_base_dir=cix_dir)
#mgr.upgrade()
mgr.initialize()
while True:
file_path = queue.get()
if file_path is None:
# marker for end of list
queue.put(None) # put it back so others can quit too
break
rel_path = os.path.relpath(file_path, root)
try:
lang = guess_lang_from_path(file_path)
except CodeIntelError:
log.info("%s: Cannot determine language, skipping", rel_path)
continue
# the file
with open(file_path, "r") as source_file:
data = source_file.read()
# Discard non-text files
if not dxr.mime.is_text(file_path, data):
continue
try:
buf = mgr.buf_from_path(file_path, lang=lang)
except CodeIntelError as ex:
if ex.message.startswith("File too big."):
log.info("%s: %s", file_path, ex.message)
continue # Nothing we can do about that, and the user can't
# fix this ever.
raise
if not isinstance(buf, CitadelBuffer):
log.info("%s: language %s does not have CIX, skipping",
rel_path, lang)
continue
log.info("%s: Using language %s", rel_path, lang)
buf.scan()
mgr.finalize()
def casper_tests():
from codeintel2.manager import Manager
db_base_dir = join(dirname(__file__), "tmp", "db")
mgr = Manager(db_base_dir)
mgr.upgrade()
mgr.initialize()
try:
for testpath in testpaths():
buf = mgr.buf_from_path(testpath, lang="JavaScript")
# Ensure the test is up to date.
if buf.scan_time < os.stat(testpath).st_mtime:
buf.scan()
for test in casper_tests_from_citree(buf.tree):
yield test
finally:
mgr.finalize()
def setUp(self):
if _xpcom_:
# The tests are run outside of Komodo. If run with PyXPCOM up
# parts codeintel will try to use the nsIDirectoryService and
# will query dirs only provided by nsXREDirProvider -- which
# isn't registered outside of Komodo (XRE_main() isn't called).
# The KoTestService provides a backup.
koTestSvc = components.classes["@activestate.com/koTestService;1"] \
.getService(components.interfaces.koITestService)
koTestSvc.init()
if self._ci_test_setup_mgr_:
env = None
if self._ci_env_prefs_ is not None:
env = SimplePrefsEnvironment(**self._ci_env_prefs_)
self.mgr = Manager(
extra_module_dirs=self._ci_extra_module_dirs_,
db_base_dir=self._ci_db_base_dir_ or test_db_base_dir,
db_catalog_dirs=self._ci_db_catalog_dirs_,
db_import_everything_langs=self._ci_db_import_everything_langs,
env=env)
self.mgr.upgrade()
self.mgr.initialize()
def ci_setUpClass(cls):
if _xpcom_:
# The tests are run outside of Komodo. If run with PyXPCOM up
# parts codeintel will try to use the nsIDirectoryService and
# will query dirs only provided by nsXREDirProvider -- which
# isn't registered outside of Komodo (XRE_main() isn't called).
# The KoTestService provides a backup.
koTestSvc = components.classes["@activestate.com/koTestService;1"] \
.getService(components.interfaces.koITestService)
koTestSvc.init()
if cls._ci_test_setup_mgr_:
env = None
if cls._ci_env_prefs_ is not None:
env = SimplePrefsEnvironment(**cls._ci_env_prefs_)
def get_extra_module_dirs():
spec = join(dirname(
__file__), "..", "..", "udl", "skel", "*", "pylib")
for d in glob(spec):
if glob(join(spec, "lang_*.py")):
yield d
for d in cls._ci_extra_module_dirs_ or []:
yield d
cls.mgr = Manager(
extra_module_dirs=get_extra_module_dirs(),
db_base_dir=cls._ci_db_base_dir_ or test_db_base_dir,
db_catalog_dirs=cls._ci_db_catalog_dirs_,
db_import_everything_langs=cls._ci_db_import_everything_langs,
env=env)
cls.mgr.upgrade()
cls.mgr.initialize()
init_xml_catalogs()
class Driver(threading.Thread):
"""
Out-of-process codeintel2 driver
This class implements the main loop of the codeintel worker process
This should be a singleton object
"""
# static
_instance = None
_command_handler_map = collections.defaultdict(list)
"""Registered command handlers; the key is the command name (a str), and the
value is a list of command handler instances or (for lazy handlers) a
single-element tuple containing the callable to get the real handler."""
_default_handler = None
"""The default (core) command handler; instance of a CoreHandler"""
_builtin_commands = {}
"""Built-in commands that cannot be overridden"""
def __init__(self, db_base_dir=None, fd_in=sys.stdin, fd_out=sys.stdout):
threading.Thread.__init__(self, name="CodeIntel OOP Driver")
assert Driver._instance is None, "Driver should be a singleton"
Driver._instance = self
logging.root.addHandler(LoggingHandler(self))
self.daemon = True
self.fd_in = fd_in
self.fd_out = fd_out
self.abort = None
self.quit = False
self.buffers = {} # path to Buffer objects
self.next_buffer = 0
self.active_request = None
self.send_queue = Queue.Queue()
self.send_thread = threading.Thread(name="Codeintel OOP Driver Send Thread",
target=self._send_proc)
self.send_thread.daemon = True
self.send_thread.start()
self.queue = collections.deque()
self.queue_cv = threading.Condition()
self.env = Environment(name="global",
send_fn=functools.partial(self.send, request=None))
# Fill out the non-overridable build-in commands
self._builtin_commands = {}
for attr in dir(self):
# Note that we check startswith first to avoid getters etc.
if attr.startswith("do_") and callable(getattr(self, attr)):
command = attr[len("do_"):].replace("_", "-")
self._builtin_commands[command] = getattr(self, attr)
from codeintel2.manager import Manager
log.debug("using db base dir %s", db_base_dir)
self.mgr = Manager(db_base_dir=db_base_dir,
db_catalog_dirs=[],
db_event_reporter=self._DBEventReporter(self),
env=self.env,
on_scan_complete=self._on_scan_complete)
self.mgr.initialize()
def _on_scan_complete(self, scan_request):
if scan_request.status in ("changed", "skipped"):
# Send unsolicited response about the completed scan
buf = scan_request.buf
self.send(request=None,
path=buf.path,
language=buf.lang,
command="scan-complete")
class _DBEventReporter(object):
def __init__(self, driver):
self.driver = driver
self.log = log.getChild("DBEventReporter")
self.debug = self.log.debug
# directories being scanned (completed or not)
# key is unicode path, value is number of times it's active
self._dirs = collections.defaultdict(int)
# directories which are complete (unicode path)
self._completed_dirs = set()
def send(self, **kwargs):
self.driver.send(request=None, command="report-message", **kwargs)
def __call__(self, message):
"""Old-style status messages before long-running jobs
@param msg {str or None} The message to display
"""
if len(self._dirs):
return # ignore old-style db events if we're doing a scan
self.debug("db event: %s", message)
self.send(message=message)
def onScanStarted(self, description, dirs=set()):
"""Called when a directory scan is about to start
@param description {unicode} A string suitable for showing the user
about the upcoming operation
@param dirs {set of unicode} The directories about to be scanned
"""
self.debug("scan started: %s (%s dirs)", description, len(dirs))
assert dirs, "onScanStarted expects non-empty directories"
if not dirs: # empty set - we shouldn't have gotten here, but be nice
return
for dir_path in dirs:
self._dirs[dir_path] += 1
self.send(type="scan-progress", message=description,
completed=len(self._completed_dirs),
total=len(self._dirs))
def onScanDirectory(self, description, dir_path, current=None, total=None):
"""Called when a directory is being scanned (out of possibly many)
@param description {unicode} A string suitable for showing the user
regarding the progress
@param dir {unicode} The directory currently being scanned
@param current {int} The current progress
@param total {int} The total number of directories to scan in this
request
"""
self.debug("scan directory: %s (%s %s/%s)",
description, dir_path, current, total)
assert dir_path, "onScanDirectory got no directory"
if dir_path:
self._completed_dirs.add(dir_path)
self.send(type="scan-progress", message=description,
completed=len(self._completed_dirs),
total=len(self._dirs))
def onScanComplete(self, dirs=set(), scanned=set()):
"""Called when a scan operation is complete
@param dirs {set of unicode} The directories that were intially
requested to be scanned (as pass in onScanStarted)
@param scanned {set of unicode} Directories which were successfully
scanned. This may be a subset of dirs if the scan was
aborted.
"""
self.debug("scan complete: scanned %r/%r dirs",
len(scanned), len(dirs))
for dir_path in dirs:
self._dirs[dir_path] -= 1
if not self._dirs[dir_path]:
del self._dirs[dir_path]
self._completed_dirs.discard(dir_path)
self.send(type="scan-progress", completed=len(self._completed_dirs),
total=len(self._dirs))
REQUEST_DEFAULT = object()
def send(self, request=REQUEST_DEFAULT, **kwargs):
"""
Send a response
"""
data = dict(kwargs)
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if request:
data["req_id"] = request.id
if "success" not in data:
data["success"] = True
elif data["success"] is None:
del data["success"]
buf = json.dumps(data, separators=(',',':'))
buf_len = str(len(buf))
log.debug("sending: %s:[%s]", buf_len, buf)
self.send_queue.put(buf)
def _send_proc(self):
while True:
buf = self.send_queue.get()
try:
buf_len = str(len(buf))
self.fd_out.write(buf_len)
self.fd_out.write(buf)
finally:
self.send_queue.task_done()
def fail(self, request=REQUEST_DEFAULT, **kwargs):
kwargs = kwargs.copy()
if not "command" in kwargs and request:
try:
kwargs["command"] = request["command"]
except KeyError:
pass
return self.send(request=request, success=False, **kwargs)
def exception(self, request=REQUEST_DEFAULT, **kwargs):
kwargs["command"] = "report-error"
kwargs["type"] = "logging"
kwargs["name"] = log.name
kwargs["level"] = logging.ERROR
return self.fail(request=request, stack=traceback.format_exc(), **kwargs)
@staticmethod
def normpath(path):
"""Routine to normalize the path used for codeintel buffers
This is annoying because it needs to handle unsaved files, as well as
urls.
@note See also koCodeIntel.py::KoCodeIntelBuffer.normpath
"""
if path.startswith("<Unsaved>"):
return path # Don't munge unsaved file paths
scheme = path.split(":", 1)[0]
if len(scheme) == len(path):
# didn't find a scheme at all; assume this is a local path
return os.path.normcase(path)
if len(scheme) == 1:
# single-character scheme; assume this is actually a drive letter
# (for a Windows-style path)
return os.path.normcase(path)
scheme_chars = string.ascii_letters + string.digits + "-"
try:
scheme = scheme.encode("ascii")
except UnicodeEncodeError:
# scheme has a non-ascii character; assume local path
return os.path.normcase(path)
if scheme.translate(None, scheme_chars):
# has a non-scheme character: this is not a valid scheme
# assume this is a local file path
return os.path.normcase(path)
if scheme != "file":
return path # non-file scheme
path = path[len(scheme) + 1:]
return os.path.normcase(path)
def get_buffer(self, request=REQUEST_DEFAULT, path=None):
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if path is None:
if not "path" in request:
raise RequestFailure(message="No path given to locate buffer")
path = request.path
path = self.normpath(path)
try:
buf = self.buffers[path]
except KeyError:
buf = None
else:
if "language" in request and buf.lang != request.language:
buf = None # language changed, re-scan
if not buf:
# Need to construct a new buffer
lang = request.get("language")
env = Environment(request=request, name=os.path.basename(path))
if request.get("text") is not None:
# pass no content; we'll reset it later
buf = self.mgr.buf_from_content("", lang, path=path, env=env)
else:
# read from file
try:
buf = self.mgr.buf_from_path(path, lang, env=env,
encoding=request.get("encoding"))
except OSError:
# Can't read the file
buf = self.mgr.buf_from_content("", lang, path=path, env=env)
assert not request.path.startswith("<"), \
"Can't create an unsaved buffer with no text"
if request.get("text") is not None:
# overwrite the buffer contents if we have new ones
buf.accessor.reset_content(request.text)
buf.encoding = "utf-8"
try:
env = request["env"]
except KeyError:
pass # no environment, use current
else:
buf._env.update(env)
#log.debug("Got buffer %r: [%s]", buf, buf.accessor.content)
log.debug("Got buffer %r", buf)
self.buffers[path] = buf
return buf
def do_abort(self, request):
try:
req_id = request["id"]
with self.queue_cv:
for item in self.queue:
if item.get("req_id") == req_id:
self.queue.remove(item)
self.send(request=request)
break
else:
self.abort = req_id
if self.active_request and self.active_request.id == req_id:
# need to wait a bit...
self.send(request=request)
else:
self.fail(request=request,
message="Request %s not found" % (req_id,))
except RequestFailure as e:
self.fail(request=request, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
def do_add_dirs(self, request):
catalog_dirs = request.get("catalog-dirs", None)
if catalog_dirs is not None:
self.mgr.db.catalog_dirs.extend(catalog_dirs)
catalog_dirs = self.mgr.db.catalog_dirs
lexer_dirs = request.get("lexer-dirs", [])
codeintel2.udl.UDLLexer.add_extra_lexer_dirs(lexer_dirs)
module_dirs = request.get("module-dirs", [])
if module_dirs:
self.mgr._register_modules(module_dirs)
if catalog_dirs is not None:
self.mgr.db.get_catalogs_zone().catalog_dirs = catalog_dirs
self.send(request=request)
def do_load_extension(self, request):
"""Load an extension that, for example, might provide additional
command handlers"""
name = request.get("module-name", None)
if not name:
raise RequestFailure(msg="load-extension requires a module-name")
path = request.get("module-path", None)
names = name.split(".")
if len(names) > 1:
# It's inside a package - go look for the package(s).
for package_name in names[:-1]:
iinfo = imp.find_module(package_name, [path] if path else None)
if not iinfo:
raise RequestFailure(msg="load-extension could not find "
"package %r for given name %r"
% (package_name, name))
path = iinfo[1]
name = names[-1]
iinfo = imp.find_module(name, [path] if path else None)
try:
module = imp.load_module(name, *iinfo)
finally:
if iinfo and iinfo[0]:
iinfo[0].close()
callback = getattr(module, "registerExtension", None)
if not callback:
raise RequestFailure(msg="load-extension module %s should "
"have a 'registerExtension' method "
"taking no arguments" % (name,))
callback()
self.send() # success, finally
def do_quit(self, request):
self.quit = True
self.send(command="quit")
def report_error(self, message):
self.send(request=None,
command="report-error",
message=unicode(message))
def start(self):
"""Start reading from the socket and dump requests into the queue"""
log.info("Running codeintel driver...")
buf = ""
self.send(success=None)
self.daemon = True
threading.Thread.start(self)
while not self.quit:
try:
ch = self.fd_in.read(1)
except IOError:
log.debug("Failed to read frame length, assuming connection died")
self.quit = True
break
if len(ch) == 0:
log.debug("Input was closed")
self.quit = True
break
if ch == "{":
size = int(buf, 10)
try:
buf = ch + self.fd_in.read(size - 1) # exclude already-read {
except IOError:
log.debug("Failed to read frame data, assuming connection died")
self.quit = True
break
try:
data = json.loads(buf)
request = Request(data)
except Exception as e:
log.exception(e)
self.exception(message=e.message, request=None)
continue
finally:
buf = ""
if request.get("command") == "abort":
self.do_abort(request=request)
else:
log.debug("queuing request %r", request)
with self.queue_cv:
self.queue.appendleft(request)
self.queue_cv.notify()
elif ch in "0123456789":
buf += ch
else:
raise ValueError("Invalid request data: " + ch.encode("hex"))
def run(self):
"""Evaluate and send results back"""
log.info("Running codeintel eval thread...")
buf = ""
log.debug("default supported commands: %s",
", ".join(self._default_handler.supportedCommands))
while True:
with self.queue_cv:
try:
request = self.queue.pop()
except IndexError:
self.queue_cv.wait()
continue
log.debug("doing request %r", request)
try:
self.active_request = request
command = request.command
# First, check abort and quit; don't allow those to be overridden
try:
builtin = self._builtin_commands[command]
except KeyError:
pass
else:
builtin(request)
continue
handlers = self._command_handler_map.get(command, [])[:]
if command in self._default_handler.supportedCommands:
# The default handler can deal with this, put it at the end
handlers.append(self._default_handler)
for handler in handlers:
if isinstance(handler, tuple):
try:
real_handler = handler[0]()
except Exception as ex:
log.exception("Failed to get lazy handler for %s", command)
real_handler = None
if real_handler is None:
# Handler failed to instantiate, drop it
try:
self._command_handler_map[command].remove(handler)
except ValueError:
pass # ... shouldn't happen, but tolerate it
continue
for handlers in self._command_handler_map.values():
try:
handlers[handlers.index(handler)] = real_handler
except ValueError:
pass # handler not in this list
handler = real_handler
if handler.canHandleRequest(request):
handler.handleRequest(request, self)
break
else:
self.fail(request=request,
message="Don't know how to handle command %s" % (command,))
except RequestFailure as e:
self.fail(request=request, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
finally:
self.active_request = None
@classmethod
def registerCommandHandler(cls, handlerInstance):
"""Register a command handler"""
for command in handlerInstance.supportedCommands:
cls._command_handler_map[command].append(handlerInstance)
@classmethod
def registerLazyCommandHandler(cls, supported_commands, constructor):
"""Register a lazy command handler
@param supported_commands {iterable} The commands to handle; each
element should be a str of the command name.
@param constructor {callable} Function to be called to get the real
command handler; it should take no arguments and return a command
handler instance. It may return None if the command is not
available; it will not be asked again.
"""
for command in supported_commands:
cls._command_handler_map[command].append((constructor,))
@classmethod
def getInstance(cls):
"""Get the singleton instance of the driver"""
return Driver._instance
class Driver(threading.Thread):
"""
Out-of-process codeintel2 driver
This class implements the main loop of the codeintel worker process
This should be a singleton object
"""
# static
_instance = None
_command_handler_map = collections.defaultdict(list)
"""Registered command handlers; the key is the command name (a str), and the
value is a list of command handler instances or (for lazy handlers) a
single-element tuple containing the callable to get the real handler."""
_default_handler = None
"""The default (core) command handler; instance of a CoreHandler"""
_builtin_commands = {}
"""Built-in commands that cannot be overridden"""
def __init__(self, db_base_dir=None, fd_in=sys.stdin, fd_out=sys.stdout):
threading.Thread.__init__(self, name="Codeintel OOP Driver")
assert Driver._instance is None, "Driver should be a singleton"
Driver._instance = self
logging.root.addHandler(LoggingHandler(self))
self.daemon = True
self.fd_in = fd_in
self.fd_out = fd_out
self.abort = None
self.quit = False
self.buffers = {} # path to Buffer objects
self.next_buffer = 0
self.active_request = None
self.send_queue = Queue.Queue()
self.send_thread = threading.Thread(target=self._send_proc)
self.send_thread.daemon = True
self.send_thread.start()
self.queue = collections.deque()
self.queue_cv = threading.Condition()
self.env = Environment(name="global",
send_fn=functools.partial(self.send,
request=None))
# Fill out the non-overridable build-in commands
self._builtin_commands = {}
for attr in dir(self):
# Note that we check startswith first to avoid getters etc.
if attr.startswith("do_") and callable(getattr(self, attr)):
command = attr[len("do_"):].replace("_", "-")
self._builtin_commands[command] = getattr(self, attr)
from codeintel2.manager import Manager
log.debug("using db base dir %s", db_base_dir)
self.mgr = Manager(db_base_dir=db_base_dir,
db_catalog_dirs=[],
db_event_reporter=self._DBEventReporter(self),
env=self.env,
on_scan_complete=self._on_scan_complete)
self.mgr.initialize()
def _on_scan_complete(self, scan_request):
if scan_request.status in ("changed", "skipped"):
# Send unsolicited response about the completed scan
buf = scan_request.buf
self.send(request=None,
path=buf.path,
language=buf.lang,
command="scan-complete")
class _DBEventReporter(object):
def __init__(self, driver):
self.driver = driver
self.log = log.getChild("DBEventReporter")
self.debug = self.log.debug
# directories being scanned (completed or not)
# key is unicode path, value is number of times it's active
self._dirs = collections.defaultdict(int)
# directories which are complete (unicode path)
self._completed_dirs = set()
def send(self, **kwargs):
self.driver.send(request=None, command="report-message", **kwargs)
def __call__(self, message):
"""Old-style status messages before long-running jobs
@param msg {str or None} The message to display
"""
if len(self._dirs):
return # ignore old-style db events if we're doing a scan
self.debug("db event: %s", message)
self.send(message=message)
def onScanStarted(self, description, dirs=set()):
"""Called when a directory scan is about to start
@param description {unicode} A string suitable for showing the user
about the upcoming operation
@param dirs {set of unicode} The directories about to be scanned
"""
self.debug("scan started: %s (%s dirs)", description, len(dirs))
assert dirs, "onScanStarted expects non-empty directories"
if not dirs: # empty set - we shouldn't have gotten here, but be nice
return
for dir_path in dirs:
self._dirs[dir_path] += 1
self.send(type="scan-progress",
message=description,
completed=len(self._completed_dirs),
total=len(self._dirs))
def onScanDirectory(self,
description,
dir_path,
current=None,
total=None):
"""Called when a directory is being scanned (out of possibly many)
@param description {unicode} A string suitable for showing the user
regarding the progress
@param dir {unicode} The directory currently being scanned
@param current {int} The current progress
@param total {int} The total number of directories to scan in this
request
"""
self.debug("scan directory: %s (%s %s/%s)", description, dir_path,
current, total)
assert dir_path, "onScanDirectory got no directory"
if dir_path:
self._completed_dirs.add(dir_path)
self.send(type="scan-progress",
message=description,
completed=len(self._completed_dirs),
total=len(self._dirs))
def onScanComplete(self, dirs=set(), scanned=set()):
"""Called when a scan operation is complete
@param dirs {set of unicode} The directories that were intially
requested to be scanned (as pass in onScanStarted)
@param scanned {set of unicode} Directories which were successfully
scanned. This may be a subset of dirs if the scan was
aborted.
"""
self.debug("scan complete: scanned %r/%r dirs", len(scanned),
len(dirs))
for dir_path in dirs:
self._dirs[dir_path] -= 1
if not self._dirs[dir_path]:
del self._dirs[dir_path]
self._completed_dirs.discard(dir_path)
self.send(type="scan-progress",
completed=len(self._completed_dirs),
total=len(self._dirs))
REQUEST_DEFAULT = object()
def send(self, request=REQUEST_DEFAULT, **kwargs):
"""
Send a response
"""
data = dict(kwargs)
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if request:
data["req_id"] = request.id
if "success" not in data:
data["success"] = True
elif data["success"] is None:
del data["success"]
buf = json.dumps(data, separators=(',', ':'))
buf_len = str(len(buf))
log.debug("sending: %s:[%s]", buf_len, buf)
self.send_queue.put(buf)
def _send_proc(self):
while True:
buf = self.send_queue.get()
try:
buf_len = str(len(buf))
self.fd_out.write(buf_len)
self.fd_out.write(buf)
finally:
self.send_queue.task_done()
def fail(self, request=REQUEST_DEFAULT, **kwargs):
kwargs = kwargs.copy()
if not "command" in kwargs and request:
try:
kwargs["command"] = request["command"]
except KeyError:
pass
return self.send(request=request, success=False, **kwargs)
def exception(self, request=REQUEST_DEFAULT, **kwargs):
return self.fail(request=request,
stack=traceback.format_exc(),
**kwargs)
def get_buffer(self, request=REQUEST_DEFAULT, path=None):
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if path is None:
if not "path" in request:
raise RequestFailure(message="No path given to locate buffer")
path = request.path
if abspath(path) == path:
# this is an actual file path, not a URL or whatever
path = normcase(normpath(path))
try:
buf = self.buffers[path]
except KeyError:
buf = None
else:
if "language" in request and buf.lang != request.language:
buf = None # language changed, re-scan
if not buf:
# Need to construct a new buffer
lang = request.get("language")
if request.get("text") is not None:
# pass no content; we'll reset it later
buf = self.mgr.buf_from_content("", lang, path=path)
else:
# read from file
try:
buf = self.mgr.buf_from_path(
path, lang, encoding=request.get("encoding"))
except OSError:
# Can't read the file
buf = self.mgr.buf_from_content("", lang, path=path)
assert not request.path.startswith("<"), \
"Can't create an unsaved buffer with no text"
if request.get("text") is not None:
# overwrite the buffer contents if we have new ones
buf.accessor.reset_content(request.text)
try:
env = request["env"]
except KeyError:
pass # no environment set, use current environment
else:
if env.get("env", {}) is None and env.get("prefs", []) is None:
buf._env = None # explicitly clearing environment
elif buf._env:
buf._env.update(env)
else:
buf._env = Environment(env, name=os.path.basename(path))
# log.debug("Got buffer %r: [%s]", buf, buf.accessor.content)
log.debug("Got buffer %r", buf)
self.buffers[path] = buf
return buf
def do_abort(self, request):
try:
req_id = request["id"]
with self.queue_cv:
for item in self.queue:
if queue.get("req_id") == req_id:
self.queue.remove(queue)
self.send(request=request)
break
else:
self.abort = req_id
if self.active_request and self.active_request.id == req_id:
# need to wait a bit...
self.send(request=request)
else:
self.fail(request=request,
message="Request %s not found" % (req_id, ))
except RequestFailure as e:
self.fail(request=request, *e.args, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
def do_add_dirs(self, request):
catalog_dirs = request.get("catalog-dirs", None)
if catalog_dirs is not None:
self.mgr.db.catalog_dirs.extend(catalog_dirs)
catalog_dirs = self.mgr.db.catalog_dirs
lexer_dirs = request.get("lexer-dirs", [])
codeintel2.udl.UDLLexer.add_extra_lexer_dirs(lexer_dirs)
module_dirs = request.get("module-dirs", [])
if module_dirs:
self.mgr._register_modules(module_dirs)
if catalog_dirs is not None:
self.mgr.db.get_catalogs_zone().catalog_dirs = catalog_dirs
def do_load_extension(self, request):
"""Load an extension that, for example, might provide additional
command handlers"""
path = request.get("module-path", None)
if not path:
raise RequestFailure(msg="load-extension requires a module-path")
name = request.get("module-name", None)
if not name:
raise RequestFailure(msg="load-extension requires a module-name")
iinfo = imp.find_module(name, [path])
try:
module = imp.load_module(name, *iinfo)
finally:
if iinfo and iinfo[0]:
iinfo[0].close()
callback = getattr(module, "registerExtension", None)
if not callback:
raise RequestFailure(msg="load-extension module %s should "
"have a 'registerExtension' method "
"taking no arguments" % (name, ))
callback()
self.send() # success, finally
def do_quit(self, request):
self.quit = True
self.send(command="quit")
def report_error(self, message):
self.send(request=None,
command="report-error",
message=unicode(message))
def start(self):
"""Start reading from the socket and dump requests into the queue"""
log.info("Running codeintel driver...")
buf = ""
self.send(success=None)
self.daemon = True
threading.Thread.start(self)
while not self.quit:
try:
ch = self.fd_in.read(1)
except IOError:
log.debug(
"Failed to read frame length, assuming connection died")
self.quit = True
break
if len(ch) == 0:
log.debug("Input was closed")
self.quit = True
break
if ch == "{":
size = int(buf, 10)
try:
buf = ch + self.fd_in.read(
size - 1) # exclude already-read {
except IOError:
log.debug(
"Failed to read frame data, assuming connection died")
self.quit = True
break
try:
buf = buf.decode("utf-8")
except UnicodeDecodeError:
pass # what :(
try:
data = json.loads(buf)
request = Request(data)
except Exception as e:
log.exception(e)
self.exception(message=e.message, request=None)
continue
finally:
buf = ""
if request.get("command") == "abort":
self.do_abort(request=request)
else:
log.debug("queuing request %r", request)
with self.queue_cv:
self.queue.appendleft(request)
self.queue_cv.notify()
elif ch in "0123456789":
buf += ch
else:
raise ValueError("Invalid request data: " + ch.encode("hex"))
def run(self):
"""Evaluate and send results back"""
log.info("Running codeintel eval thread...")
buf = ""
log.debug("default supported commands: %s",
", ".join(self._default_handler.supportedCommands))
while True:
with self.queue_cv:
try:
request = self.queue.pop()
except IndexError:
self.queue_cv.wait()
continue
log.debug("doing request %r", request)
try:
self.active_request = request
command = request.command
# First, check abort and quit; don't allow those to be
# overridden
try:
builtin = self._builtin_commands[command]
except KeyError:
pass
else:
builtin(request)
continue
handlers = self._command_handler_map.get(command, [])[:]
if command in self._default_handler.supportedCommands:
# The default handler can deal with this, put it at the end
handlers.append(self._default_handler)
for handler in handlers:
if isinstance(handler, tuple):
try:
real_handler = handler[0]()
except Exception as ex:
log.exception("Failed to get lazy handler for %s",
command)
real_handler = None
if real_handler is None:
# Handler failed to instantiate, drop it
try:
self._command_handler_map["command"].remove(
handler)
except ValueError:
pass # ... shouldn't happen, but tolerate it
continue
for handlers in self._command_handler_map.values():
try:
handlers[handlers.index(
handler)] = real_handler
except ValueError:
pass # handler not in this list
handler = real_handler
if handler.canHandleRequest(request):
handler.handleRequest(request, self)
break
else:
self.fail(request=request,
msg="Don't know how to handle command %s" %
(command, ))
except RequestFailure as e:
self.fail(request=request, *e.args, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
finally:
self.active_request = None
@classmethod
def registerCommandHandler(cls, handlerInstance):
"""Register a command handler"""
for command in handlerInstance.supportedCommands:
cls._command_handler_map[command].append(handlerInstance)
@classmethod
def registerLazyCommandHandler(cls, supported_commands, constructor):
"""Register a lazy command handler
@param supported_commands {iterable} The commands to handle; each
element should be a str of the command name.
@param constructor {callable} Function to be called to get the real
command handler; it should take no arguments and return a command
handler instance. It may return None if the command is not
available; it will not be asked again.
"""
for command in supported_commands:
cls._command_handler_map[command].append((constructor, ))
@classmethod
def getInstance(cls):
"""Get the singleton instance of the driver"""
return Driver._instance
def log_message(self, format, *args):
return # log silently
logStream = logging.StreamHandler(sys.stderr)
logStream.setFormatter(
logging.Formatter("%(name)s: %(levelname)s: %(message)s"))
logging.getLogger("codeintel").addHandler(logStream)
logger = logging.getLogger("codeintel_server")
logger.addHandler(logStream)
logger.setLevel(logging.WARNING)
manager = Manager(
# db_base_dir = path.join(CI_DIR, 'db'),
# extra_module_dirs = [path.join(CI_DIR, 'codeintel2'),],
# db_import_everything_langs = None,
# db_catalog_dirs = []
)
manager.upgrade()
manager.initialize()
scanned_langs = []
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Run codeintel commands as a daemon or via stdin")
parser.add_argument("mode",
help="Mode of operation",
choices=[
"daemon", "completions", "definitions", "calltips",
"catalogs"
])
self.send_response(200)
self.end_headers()
self.wfile.write(process(source, dict(args)))
def log_message(self, format, *args):
return # log silently
logStream = logging.StreamHandler(sys.stderr)
logStream.setFormatter(logging.Formatter("%(name)s: %(levelname)s: %(message)s"))
logging.getLogger("codeintel").addHandler(logStream)
logger = logging.getLogger("codeintel_server")
logger.addHandler(logStream)
logger.setLevel(logging.WARNING)
manager = Manager(
# db_base_dir = path.join(CI_DIR, 'db'),
# extra_module_dirs = [path.join(CI_DIR, 'codeintel2'),],
# db_import_everything_langs = None,
# db_catalog_dirs = []
)
manager.upgrade()
manager.initialize()
scanned_langs = []
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Run codeintel commands as a daemon or via stdin")
parser.add_argument("mode", help="Mode of operation", choices=["daemon", "completions", "definitions", "calltips", "catalogs"])
parser.add_argument("--row", type=int, help="The row to read from")
parser.add_argument("--column", type=int, help="The column to read from")
parser.add_argument("--path", help="The path of the file")
parser.add_argument("--basedir", help="The basedir of the file")
parser.add_argument("--language", help="The language of the file")
parser.add_argument("--port", type=int, help="The port for the daemon to listen on")
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if opts.time_it:
start = time.time()
quiet = opts.quiet
if opts.time_it or opts.time_details:
opts.force = True
scan_count = 0
lang_warnings = set()
tree = None
for path in _paths_from_path_patterns(path_patterns,
recursive=opts.recursive,
includes=opts.includes):
if opts.time_it:
sys.stderr.write(path + "\n")
if opts.time_details:
start1 = time.time()
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info("skip `%s': couldn't determine language", path)
continue
try:
buf = mgr.buf_from_path(path, lang=lang)
except OSError as ex:
# Couldn't access the file.
if not opts.recursive:
raise
# Ignore files we don't really care about.
log.warn("%r - %r", ex, path)
continue
if not isinstance(buf, CitadelBuffer):
if opts.recursive:
# Ignore files that scanning isn't provided for.
continue
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
scan_count += 1
if scan_count % 10 == 0:
log.info("%d scanning %r", scan_count, path)
try:
if opts.force:
buf.scan()
if tree is None:
tree = ET.Element("codeintel", version="2.0")
file_elem = ET.SubElement(tree,
"file",
lang=buf.lang,
mtime=str(int(time.time())),
path=os.path.basename(path))
for lang, blob in sorted(buf.blob_from_lang.items()):
blob = buf.blob_from_lang[lang]
file_elem.append(blob)
except KeyError as ex:
# Unknown cile language.
if not opts.recursive:
raise
message = str(ex)
if message not in lang_warnings:
lang_warnings.add(message)
log.warn("Skipping unhandled language %s", message)
if opts.time_details:
delta = time.time() - start1
sys.stderr.write("%.3f %s\n" % (delta, path))
sys.stderr.flush()
if tree is not None:
if opts.stripfuncvars:
# For stdlibs, we don't care about variables inside of
# functions and they take up a lot of space.
for function in tree.getiterator('scope'):
if function.get('ilk') == 'function':
function[:] = [
child for child in function
if child.tag != 'variable'
]
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
if not quiet:
sys.stdout.write(
'<?xml version="1.0" encoding="UTF-8"?>\n')
ET.dump(tree)
if opts.time_it:
end = time.time()
sys.stderr.write("scan took %.3fs\n" % (end - start))
finally:
mgr.finalize()
def do_play(self, subcmd, opts):
"""Run my current play/dev code.
${cmd_usage}
${cmd_option_list}
"""
if False:
lang = "CSS"
markedup_content = dedent("""
/* http://www.w3.org/TR/REC-CSS2/fonts.html#propdef-font-weight */
h1 {
border: 1px solid black;
font-weight /* hi */: <|> !important
}
""")
content, data = unmark_text(markedup_content)
pos = data["pos"]
mgr = Manager()
#mgr.upgrade() # Don't need it for just CSS usage.
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path="play.css")
trg = buf.trg_from_pos(pos)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
completions = buf.cplns_from_trg(trg)
print("COMPLETIONS: %r" % completions)
finally:
mgr.finalize()
elif False:
lang = "Python"
path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
markedup_content = dedent("""
import sys, os
class Foo:
def bar(self):
pass
sys.<|>path # should have path in completion list
f = Foo()
""")
content, data = unmark_text(markedup_content)
print(banner(path))
print(_escaped_text_from_text(content, "whitespace"))
pos = data["pos"]
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path=path)
print(banner("cix", '-'))
print(buf.cix)
trg = buf.trg_from_pos(pos)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
print(banner("completions", '-'))
ctlr = LogEvalController(log)
buf.async_eval_at_trg(trg, ctlr)
ctlr.wait(2) #XXX
if not ctlr.is_done():
ctlr.abort()
raise Error("XXX async eval timed out")
pprint(ctlr.cplns)
print(banner(None))
finally:
mgr.finalize()
elif False:
lang = "Ruby"
path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
markedup_content = dedent("""\
r<1>equire 'net/http'
include Net
req = HTTPRequest.new
req.<2>get()
""")
content, data = unmark_text(markedup_content)
print(banner(path))
print(_escaped_text_from_text(content, "whitespace"))
pos = data[1]
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path=path)
print(banner("cix", '-'))
cix = buf.cix
print(
ET.tostring(pretty_tree_from_tree(tree_from_cix(buf.cix))))
trg = buf.trg_from_pos(pos, implicit=False)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
print(banner("completions", '-'))
ctlr = LogEvalController(log)
buf.async_eval_at_trg(trg, ctlr)
ctlr.wait(30) #XXX
if not ctlr.is_done():
ctlr.abort()
raise Error("XXX async eval timed out")
pprint(ctlr.cplns)
print(banner(None))
finally:
mgr.finalize()
class CodeIntelTestCase(unittest.TestCase):
# Subclasses can override this to have setUp pass these settings to the
# codeintel.Manager.
_ci_db_base_dir_ = None
_ci_db_catalog_dirs_ = []
_ci_db_import_everything_langs = None
_ci_env_prefs_ = None
# A test case can set this to false to have the setUp/tearDown *not*
# create a `self.mgr'.
_ci_test_setup_mgr_ = True
_ci_extra_module_dirs_ = None
def setUp(self):
if _xpcom_:
# The tests are run outside of Komodo. If run with PyXPCOM up
# parts codeintel will try to use the nsIDirectoryService and
# will query dirs only provided by nsXREDirProvider -- which
# isn't registered outside of Komodo (XRE_main() isn't called).
# The KoTestService provides a backup.
koTestSvc = components.classes["@activestate.com/koTestService;1"] \
.getService(components.interfaces.koITestService)
koTestSvc.init()
if self._ci_test_setup_mgr_:
env = None
if self._ci_env_prefs_ is not None:
env = SimplePrefsEnvironment(**self._ci_env_prefs_)
self.mgr = Manager(
extra_module_dirs=self._ci_extra_module_dirs_,
db_base_dir=self._ci_db_base_dir_ or test_db_base_dir,
db_catalog_dirs=self._ci_db_catalog_dirs_,
db_import_everything_langs=self._ci_db_import_everything_langs,
env=env)
self.mgr.upgrade()
self.mgr.initialize()
def tearDown(self):
if self._ci_test_setup_mgr_:
self.mgr.finalize()
self.mgr = None
def adjust_content(self, content):
"""A hook for subclasses to modify markedup_content before use in
test cases. This is useful for sharing test cases between pure-
and multi-lang uses of a given lang.
"""
return content
def adjust_pos(self, pos):
"""A accompanying hook for `adjust_content' to adjust trigger
pos values accordingly.
"""
return pos
def _get_buf_and_data(self, markedup_content, lang, path=None, env=None):
if path is None:
# Try to ensure no accidental re-use of the same buffer name
# across the whole test suite. Also try to keep the buffer
# names relatively short (keeps tracebacks cleaner).
name = "buf-" + md5(markedup_content).hexdigest()[:16]
path = os.path.join("<Unsaved>", name)
content, data = unmark_text(self.adjust_content(markedup_content))
#print banner(path)
#sys.stdout.write(content)
#print banner(None)
buf = self.mgr.buf_from_content(content, lang=lang, path=path,
env=env)
return buf, data
def _get_buf_and_trg(self, markedup_content, lang, path=None,
implicit=True, env=None):
buf, data = self._get_buf_and_data(markedup_content, lang, path, env)
trg = buf.trg_from_pos(data["pos"], implicit=implicit)
return buf, trg
def assertCITDLExprUnderPosIs(self, markedup_content, citdl_expr, lang=None,
prefix_filter=None, implicit=True, trigger_name=None,
**fields):
"""Assert that the CITDL expression at the current position
is as expected.
This uses buf.citdl_expr_under_pos() -- or, for Perl,
buf.citdl_expr_and_prefix_filter_from_trg().
Note: This API is a mess right now. C.f. bug 65776.
The "prefix_filter" optional argument can be used for Perl to test
the value its relevant function returns.
"""
if lang is None:
lang = self.lang
content, data = unmark_text(
self.adjust_content(markedup_content))
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
langintel = self.mgr.langintel_from_lang(lang)
if trigger_name is None:
trigger_name = "fakey-completion-type"
if lang == "Perl":
trg = Trigger(lang, TRG_FORM_DEFN, trigger_name,
data["pos"], implicit=implicit, length=0,
**fields)
actual_citdl_expr, actual_prefix_filter \
= langintel.citdl_expr_and_prefix_filter_from_trg(buf, trg)
else:
#actual_citdl_expr = langintel.citdl_expr_under_pos(buf, data["pos"])
trg = Trigger(lang, TRG_FORM_DEFN, trigger_name,
data["pos"], implicit=implicit,
**fields)
actual_citdl_expr = langintel.citdl_expr_from_trg(buf, trg)
self.assertEqual(actual_citdl_expr, citdl_expr,
"unexpected actual %s CITDL expr under pos:\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s"
% (lang, citdl_expr, actual_citdl_expr,
indent(markedup_content)))
if prefix_filter is not None:
XXX #TODO: compare prefix_filter to given value
def _assertDefnMatches(self, buf, pos, lang=None, **fields):
ctlr = _CaptureEvalController()
trg = buf.defn_trg_from_pos(pos)
defns = buf.defns_from_trg(trg, ctlr=ctlr)
if not defns:
self.fail("unexpectedly did not find a definition in %r at pos %d\n"
" eval log\n%s\n"
" buffer:\n%s"
% (buf, pos,
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(buf.accessor.text)))
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
defn = defns[0]
for name, value in fields.items():
try:
actual_value = getattr(defn, name)
except AttributeError:
actual_value = None
self.assertEqual(actual_value, value,
"%s definition, unexpected value for field %r\n"
" defn: %r\n"
" expected: %r\n"
" got: %r\n"
" eval log\n%s\n"
" buffer:\n%s"
% (buf.lang, name, defn, value, actual_value,
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(buf.accessor.text)))
def assertDefnMatches(self, markedup_content, lang=None, **fields):
if lang is None:
lang = self.lang
buf, data = self._get_buf_and_data(markedup_content, lang)
self._assertDefnMatches(buf, data["pos"], lang, **fields)
def assertDefnMatches2(self, buf, pos, lang=None, **fields):
self._assertDefnMatches(buf, pos, lang, **fields)
def assertNoDuplicateDefns(self, markedup_content, lang=None):
if lang is None:
lang = self.lang
buf, data = self._get_buf_and_data(markedup_content, lang)
self.assertNoDuplicateDefns2(buf, data["pos"])
def assertNoDuplicateDefns2(self, buf, pos):
markedup_content = markup_text(buf.accessor.text, pos=pos)
ctlr = _CaptureEvalController()
trg = buf.defn_trg_from_pos(pos)
actual_defns = buf.defns_from_trg(trg, ctlr=ctlr)
if not actual_defns:
self.fail("%s trigger resulted in no definitions when expecting "
"to check for duplicate definitions:\n%s"
% (buf.lang, indent(markedup_content)))
count_from_defn_repr = {}
for defn_repr in (repr(d) for d in actual_defns):
if defn_repr not in count_from_defn_repr:
count_from_defn_repr[defn_repr] = 0
count_from_defn_repr[defn_repr] += 1
defn_dupes = [(count, defn_repr)
for defn_repr, count in count_from_defn_repr.items()
if count > 1]
self.failIf(defn_dupes,
"unexpectedly got duplicate completions at the given position\n"
" duplicates:\n%s\n"
" eval log\n%s\n"
" buffer:\n%s"
% (indent('\n'.join('%d of %s' % d for d in defn_dupes)),
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(markedup_content)))
def assertTriggerMatches(self, markedup_content, lang=None,
implicit=True, env=None, **fields):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
if trg is None:
self.fail("unexpectedly did not find a %s trigger, buffer:\n%s"
% (lang, indent(markedup_content)))
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
for name, value in fields.items():
try:
actual_value = getattr(trg, name)
except AttributeError:
actual_value = trg.extra[name]
self.assertEqual(actual_value, value,
"unexpected %s trigger '%s' value: expected %r, "
"got %r, buffer:\n%s"
% (lang, name, value, actual_value,
indent(markedup_content)))
# Used when a position generates a trigger, but it's not the one specified
def assertTriggerDoesNotMatch(self, markedup_content, lang=None,
implicit=True, env=None, **fields):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
if trg is None:
# No trigger is as good as
return
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
for name, value in fields.items():
try:
actual_value = getattr(trg, name)
except AttributeError:
actual_value = trg.extra[name]
self.assertNotEqual(actual_value, value,
"unexpected %s trigger '%s' value: expected not %r, "
"got %r, buffer:\n%s"
% (lang, name, value, actual_value,
indent(markedup_content)))
def assertNoTrigger(self, markedup_content, lang=None, implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
if trg is not None:
self.fail("unexpectedly found a %s trigger %r when didn't expect "
"one, buffer:\n%s"
% (lang, trg, indent(markedup_content)))
def assertPrecedingTriggerMatches(self, markedup_content, lang=None,
**fields):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(
self.adjust_content(markedup_content))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
trg = buf.preceding_trg_from_pos(data["start_pos"], data["pos"])
if trg is None:
self.fail("unexpectedly did not find a preceding %s trigger, "
"buffer:\n%s" % (lang, indent(markedup_content)))
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
for name, value in fields.items():
actual_value = getattr(trg, name)
self.assertEqual(actual_value, value,
"unexpected preceding %s trigger '%s' value: expected %r, "
"got %r, buffer:\n%s"
% (lang, name, value, actual_value,
indent(markedup_content)))
def assertNoPrecedingTrigger(self, markedup_content, lang=None):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(
self.adjust_content(markedup_content))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
trg = buf.preceding_trg_from_pos(data["start_pos"], data["pos"])
if trg is not None:
self.fail("unexpectedly found a preceding %s trigger '%s' when "
"didn't expect one, buffer:\n%s"
% (lang, trg.name, indent(markedup_content)))
def assertScopeLpathIs(self, markedup_content, lpath, lang=None):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(
self.adjust_content(markedup_content))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
buf.scan(skip_scan_time_check=True)
actual_blob, actual_lpath = buf.scoperef_from_pos(data["pos"])
self.failUnlessEqual(lpath, actual_lpath,
"unexpected %s scope lookup path (lpath) at the given position\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s"
% (self.lang, lpath,
actual_lpath,
indent(markedup_content)))
def assertCompletionsAre2(self, buf, pos, completions, lang=None, implicit=True, env=None):
if lang is None:
lang = self.lang
markedup_content = markup_text(buf.accessor.text, pos=pos)
trg = buf.trg_from_pos(pos, implicit=implicit)
self._assertCompletionsAre(markedup_content, buf, trg, completions, lang, implicit)
def assertCompletionsAre(self, markedup_content, completions,
lang=None, implicit=True, env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
self._assertCompletionsAre(markedup_content, buf, trg, completions, lang, implicit)
def _assertCompletionsAre(self, markedup_content, buf, trg, completions,
lang, implicit):
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected completions to be %r:\n%s"
% (lang, completions, indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
self.assertEqual(completions, actual_completions,
"unexpected %s completions at the given position\n"
" expected: %r\n"
" got: %r\n"
" extra: %r\n"
" missing: %r\n"
" eval log\n%s\n"
" buffer:\n%s"
% (lang, completions,
actual_completions,
list(set(actual_completions or []).difference(completions or [])),
list(set(completions or []).difference(actual_completions or [])),
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(markedup_content)))
def assertNoDuplicateCompletions(self, markedup_content, lang=None,
implicit=True, env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected there to be completions:\n%s"
% (lang, indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
if actual_completions is None:
self.fail("%s trigger resulted in no completions when expecting "
"to check for duplicate completions:\n%s"
% (lang, indent(markedup_content)))
count_from_cpln = {}
for cpln in actual_completions:
if cpln not in count_from_cpln:
count_from_cpln[cpln] = 0
count_from_cpln[cpln] += 1
cpln_dupes = [(count, cpln) for cpln, count in count_from_cpln.items()
if count > 1]
self.failIf(cpln_dupes,
"unexpectedly got duplicate completions at the given position\n"
" duplicates:\n%s\n"
" eval log\n%s\n"
" buffer:\n%s"
% (indent('\n'.join('%d of %r' % d for d in cpln_dupes)),
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(markedup_content)))
def _assertCompletionsInclude(self, buf, trg, completions):
markedup_content = markup_text(buf.accessor.text, pos=trg.pos)
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
missing_completions = [c for c in completions
if c not in (actual_completions or [])]
self.failIf(missing_completions,
"%s completions at the given position did not "
"include all expected values\n"
" missing: %r\n"
" expected all of: %r\n"
" got: %r\n"
" eval log:\n%s\n"
" buffer:\n%s"
% (buf.lang, missing_completions, completions,
actual_completions,
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(markedup_content)))
def assertCompletionsInclude(self, markedup_content, completions,
lang=None, implicit=True, env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected completions to include %r:\n%s"
% (lang, completions, indent(markedup_content)))
self._assertCompletionsInclude(buf, trg, completions)
def assertCompletionsInclude2(self, buf, pos, completions, implicit=True):
"""A version of assertCompletionsInclude() where you pass in
a Buffer instance instead of marked up content. Sometimes
this is more convenient.
"""
trg = buf.trg_from_pos(pos, implicit=implicit)
if trg is None:
markedup_content = markup_text(buf.accessor.text, pos=pos)
self.fail("given position is not a %s trigger point, "
"expected completions to include %r:\n%s"
% (buf.lang, completions, indent(markedup_content)))
self._assertCompletionsInclude(buf, trg, completions)
def _assertCompletionsDoNotInclude(self, buf, trg, completions):
markedup_content = markup_text(buf.accessor.text, pos=trg.pos)
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
completions_that_shouldnt_be_there = [
c for c in (actual_completions or []) if c in completions
]
self.failIf(completions_that_shouldnt_be_there,
"%s completions at the given position included "
"some unexpected values\n"
" shouldn't have had these: %r\n"
" expected none of: %r\n"
" got: %r\n"
" eval log:\n%s\n"
" buffer:\n%s"
% (buf.lang, completions_that_shouldnt_be_there, completions,
actual_completions,
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(markedup_content)))
def assertCompletionsDoNotInclude(self, markedup_content, completions,
lang=None, implicit=True, env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected completions to exclude %r:\n%s"
% (lang, completions, indent(markedup_content)))
self._assertCompletionsDoNotInclude(buf, trg, completions)
def assertCompletionsDoNotInclude2(self, buf, pos, completions, implicit=True):
"""A version of assertCompletionsDoNotInclude() where you pass in
a Buffer instance instead of marked up content. Sometimes
this is more convenient.
"""
trg = buf.trg_from_pos(pos, implicit=implicit)
if trg is None:
markedup_content = markup_text(buf.accessor.text, pos=pos)
self.fail("given position is not a %s trigger point, "
"expected completions to exclude %r:\n%s"
% (buf.lang, completions, indent(markedup_content)))
self._assertCompletionsDoNotInclude(buf, trg, completions)
def assertCalltipIs2(self, buf, pos, calltip, implicit=True):
"""A variant of assertCalltipIs() where you pass in
a Buffer instance instead of marked up content. Sometimes
this is more convenient.
"""
trg = buf.trg_from_pos(pos, implicit=implicit)
markedup_content = markup_text(buf.accessor.text, pos=trg.pos)
self._assertCalltipIs(buf, trg, markedup_content, calltip, self.lang)
def assertCalltipIs(self, markedup_content, calltip, lang=None,
implicit=True, env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
self._assertCalltipIs(buf, trg, markedup_content, calltip, lang)
def _assertCalltipIs(self, buf, trg, markedup_content, calltip, lang):
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected the following calltip:\n"
" calltip:\n%s\n"
" buffer:\n%s"
% (lang, indent(calltip),
indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_calltips = buf.calltips_from_trg(trg, ctlr=ctlr)
if actual_calltips and actual_calltips[0]:
actual_calltip = actual_calltips[0]
else:
actual_calltip = None
self.assertEqual(calltip, actual_calltip,
"unexpected %s calltip at the given position\n"
" expected:\n%s\n"
" got:\n%s\n"
" eval log:\n%s\n"
" buffer:\n%s"
% (trg.name, indent(calltip and calltip or "(none)"),
indent(actual_calltip and actual_calltip or "(none)"),
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(markedup_content)))
def assertCalltipMatches(self, markedup_content, calltip, lang=None,
implicit=True, env=None, flags=0):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
self._assertCalltipMatches(buf, trg, markedup_content, calltip, lang, flags)
def _assertCalltipMatches(self, buf, trg, markedup_content, expr, lang, flags):
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected the calltip to match the following:\n"
" exression:\n%s\n"
" buffer:\n%s"
% (lang, indent(expr),
indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_calltips = buf.calltips_from_trg(trg, ctlr=ctlr)
if actual_calltips and actual_calltips[0]:
actual_calltip = actual_calltips[0]
else:
actual_calltip = None
self.assertNotEquals(re.search(expr, actual_calltip, flags), None,
"unexpected %s calltip at the given position\n"
" expression:\n%s\n"
" got:\n%s\n"
" eval log:\n%s\n"
" buffer:\n%s"
% (trg.name, indent(expr and expr or "(none)"),
indent(actual_calltip and actual_calltip or "(none)"),
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(markedup_content)))
def assertCurrCalltipArgRange(self, markedup_content, calltip,
expected_range, lang=None, implicit=True):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(
self.adjust_content(markedup_content))
pos = data["pos"]
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
trg = buf.trg_from_pos(data["trg_pos"], implicit=implicit)
actual_range = buf.curr_calltip_arg_range(trg.pos, calltip,
curr_pos=data["pos"])
self.assertEqual(actual_range, expected_range,
"unexpected current calltip arg range\n"
" expected: %s\n"
" got: %s\n"
" calltip:\n%s\n"
" buffer:\n%s"
% (expected_range, actual_range, indent(calltip),
indent(markedup_content)))
#def assertCompletionRaises(self, markedup_content, exception, lang=None,
# **kwargs):
# """Assert that the given completion raises the given exception.
#
# You may also specify either of the "exc_args" or "exc_pattern"
# keyword args to match the exception's "args" attribute or match
# the stringified exception against a regex pattern.
#
# c.f. http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/307970
# """
# if lang is None:
# lang = self.lang
# if "exc_args" in kwargs:
# exc_args = kwargs["exc_args"]
# del kwargs["exc_args"]
# else:
# exc_args = None
# if "exc_pattern" in kwargs:
# exc_pattern = kwargs["exc_pattern"]
# del kwargs["exc_pattern"]
# else:
# exc_pattern = None
#
# callsig = "the given %s completion" % lang
# try:
# buf, trg = self._get_buf_and_trg(markedup_content, lang)
# if trg is None:
# self.fail("given position is not a %s trigger point, "
# "no completion can be done to see if it raises"
# % self.lang)
# callsig = "completion at %s" % trg
# cplns = buf.cplns_from_trg(trg)
# except exception, exc:
# if exc_args is not None:
# self.failIf(exc.args != exc_args,
# "%s raised %s with unexpected args: "\
# "expected=%r, actual=%r"\
# % (callsig, exc.__class__, exc_args, exc.args))
# if exc_pattern is not None:
# self.failUnless(exc_pattern.search(str(exc)),
# "%s raised %s, but the exception "\
# "does not match '%s': %r"\
# % (callsig, exc.__class__, exc_pattern.pattern,
# str(exc)))
# except:
# exc_info = sys.exc_info()
# print exc_info
# self.fail("%s raised an unexpected exception type: "\
# "expected=%s, actual=%s"\
# % (callsig, exception, exc_info[0]))
# else:
# self.fail("%s did not raise %s" % (callsig, exception))
def assertEvalError(self, markedup_content, log_pattern, lang=None,
implicit=True, env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"no completion can be done to see if errors"
% self.lang)
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
class TestEvalController(EvalController):
"""A completion controller that captures all eval logging."""
def __init__(self):
EvalController.__init__(self)
self.log = []
def debug(self, msg, *args):
self.log.append(("debug", msg % args))
def info(self, msg, *args):
self.log.append(("info", msg % args))
def warn(self, msg, *args):
self.log.append(("warn", msg % args))
def error(self, msg, *args):
self.log.append(("error", msg % args))
ctlr = TestEvalController()
buf.async_eval_at_trg(trg, ctlr=ctlr)
ctlr.wait()
if not ctlr.is_done():
self.fail("evaluation is not 'done': didn't expect that")
if trg.form == TRG_FORM_CPLN and ctlr.cplns:
self.fail("evalution had results: didn't expect that: %r"
% ctlr.cplns)
elif trg.form == TRG_FORM_CALLTIP and ctlr.calltips:
self.fail("evalution had results: didn't expect that: %r"
% ctlr.cplns)
if log_pattern:
#pprint(ctlr.log)
matching_logs = [(level, msg) for level, msg in ctlr.log
if log_pattern.search(msg)]
self.failUnless(matching_logs,
"the given completion failed but no logs matched the given pattern:\n"
" log_pattern: /%s/\n"
" log messages:\n%s\n"
" buffer:\n%s"
% (log_pattern.pattern,
indent('\n'.join(['%s: %s' % lg for lg in ctlr.log])),
indent(markedup_content)))
#pprint(matching_logs)
def assertCITDLExprIs(self, markedup_content, citdl_expr, lang=None,
prefix_filter=None, implicit=True, trigger_name=None,
**fields):
"""Assert that the preceding CITDL expression at the current position
is as expected.
This uses buf.citdl_expr_from_trg() -- or, for Perl,
buf.citdl_expr_and_prefix_filter_from_trg().
The "prefix_filter" optional argument can be used for Perl to test
the value its relevant function returns.
"""
if lang is None:
lang = self.lang
content, data = unmark_text(
self.adjust_content(markedup_content))
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
langintel = self.mgr.langintel_from_lang(lang)
if trigger_name is None:
trigger_name = "fakey-completion-type"
if lang == "Perl":
# Bit of a hack to fake the trigger length.
if content[data["pos"]-1] in ('>', ':'): # '->' or '::' triggers
length = 2
else:
length = 1
trg = Trigger(lang, TRG_FORM_CPLN, trigger_name,
data["pos"], implicit=implicit, length=length,
**fields)
actual_citdl_expr, actual_prefix_filter \
= langintel.citdl_expr_and_prefix_filter_from_trg(buf, trg)
else:
trg = Trigger(lang, TRG_FORM_CPLN, trigger_name,
data["pos"], implicit=implicit,
**fields)
actual_citdl_expr = langintel.citdl_expr_from_trg(buf, trg)
self.assertEqual(actual_citdl_expr, citdl_expr,
"unexpected actual %s CITDL expr preceding trigger:\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s"
% (lang, citdl_expr, actual_citdl_expr,
indent(markedup_content)))
if lang == "Perl" and prefix_filter is not None:
self.assertEqual(actual_prefix_filter, prefix_filter,
"unexpected actual %s variable prefix filter "
"preceding trigger:\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s"
% (lang, prefix_filter, actual_prefix_filter,
indent(markedup_content)))
def _unmark_lex_text(self, markedup_text):
from SilverCity import ScintillaConstants
tokenizer = re.compile(r'(<(SCE_\w+)>(.*?)</\2>)')
tokens = []
text = ''
while markedup_text:
match = tokenizer.search(markedup_text)
if match:
text += markedup_text[:match.start()]
token = {
'end_index': len(text) + len(match.group(3)) - 1,
'start_index': len(text),
'style': getattr(ScintillaConstants, match.group(2)),
'text': match.group(3),
#'end_column': ???,
#'end_line': ???,
#'start_column': ???,
#'start_line': ???,
}
tokens.append(token)
text += match.group(3)
markedup_text = markedup_text[match.end():]
else:
text += markedup_text
markedup_text = ''
return text, tokens
def assertLex(self, markedup_content, lang=None):
"""Lex the given content and assert that the lexed tokens are as
expected.
What is "expected" is given via pseudo-xml markup like this:
fuzzy wuzzy <SCE_UDL_SSL_COMMENTBLOCK>wuzza</SCE_UDL_SSL_COMMENTBLOCK> bear
This example expects that "wuzza" will be a token with style
SCE_UDL_SSL_COMMENTBLOCK.
"""
from codeintel2.accessor import SilverCityAccessor
if lang is None:
lang = self.lang
content, tokens = self._unmark_lex_text(markedup_content)
# Do lexing of this content via the codeintel Buffer's, because
# they already handle all the SilverCity lexer hookup.
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
assert isinstance(buf.accessor, SilverCityAccessor)
actual_tokens = buf.accessor.tokens # cheating
for actual_token in actual_tokens:
# There are a few SilverCity token dict keys that we
# don't bother checking.
del actual_token["end_column"]
del actual_token["end_line"]
del actual_token["start_column"]
del actual_token["start_line"]
unmatched_tokens = [t for t in tokens if t not in actual_tokens]
if unmatched_tokens:
self.fail("not all expected %s lex tokens were found in the "
"actual lexer output:\n"
" buffer:\n%s\n"
" actual lexer tokens:\n%s\n"
" unmatched tokens:\n%s\n"
% (lang,
indent(content),
indent(pformat(actual_tokens)),
indent(pformat(unmatched_tokens))))
class CodeIntelTestCase(unittest.TestCase):
# Subclasses can override this to have setUp pass these settings to the
# codeintel.Manager.
_ci_db_base_dir_ = None
_ci_db_catalog_dirs_ = []
_ci_db_import_everything_langs = None
_ci_env_prefs_ = None
# A test case can set this to false to have the setUp/tearDown *not*
# create a `self.mgr'.
_ci_test_setup_mgr_ = True
_ci_extra_module_dirs_ = None
def setUp(self):
if _xpcom_:
# The tests are run outside of Komodo. If run with PyXPCOM up
# parts codeintel will try to use the nsIDirectoryService and
# will query dirs only provided by nsXREDirProvider -- which
# isn't registered outside of Komodo (XRE_main() isn't called).
# The KoTestService provides a backup.
koTestSvc = components.classes["@activestate.com/koTestService;1"] \
.getService(components.interfaces.koITestService)
koTestSvc.init()
if self._ci_test_setup_mgr_:
env = None
if self._ci_env_prefs_ is not None:
env = SimplePrefsEnvironment(**self._ci_env_prefs_)
self.mgr = Manager(
extra_module_dirs=self._ci_extra_module_dirs_,
db_base_dir=self._ci_db_base_dir_ or test_db_base_dir,
db_catalog_dirs=self._ci_db_catalog_dirs_,
db_import_everything_langs=self._ci_db_import_everything_langs,
env=env)
self.mgr.upgrade()
self.mgr.initialize()
def tearDown(self):
if self._ci_test_setup_mgr_:
self.mgr.finalize()
self.mgr = None
def adjust_content(self, content):
"""A hook for subclasses to modify markedup_content before use in
test cases. This is useful for sharing test cases between pure-
and multi-lang uses of a given lang.
"""
return content
def adjust_pos(self, pos):
"""A accompanying hook for `adjust_content' to adjust trigger
pos values accordingly.
"""
return pos
def _get_buf_and_data(self, markedup_content, lang, path=None, env=None):
if path is None:
# Try to ensure no accidental re-use of the same buffer name
# across the whole test suite. Also try to keep the buffer
# names relatively short (keeps tracebacks cleaner).
name = "buf-" + md5(markedup_content).hexdigest()[:16]
path = os.path.join("<Unsaved>", name)
content, data = unmark_text(self.adjust_content(markedup_content))
#print banner(path)
#sys.stdout.write(content)
#print banner(None)
buf = self.mgr.buf_from_content(content, lang=lang, path=path, env=env)
return buf, data
def _get_buf_and_trg(self,
markedup_content,
lang,
path=None,
implicit=True,
env=None):
buf, data = self._get_buf_and_data(markedup_content, lang, path, env)
trg = buf.trg_from_pos(data["pos"], implicit=implicit)
return buf, trg
def assertCITDLExprUnderPosIs(self,
markedup_content,
citdl_expr,
lang=None,
prefix_filter=None,
implicit=True,
trigger_name=None,
**fields):
"""Assert that the CITDL expression at the current position
is as expected.
This uses buf.citdl_expr_under_pos() -- or, for Perl,
buf.citdl_expr_and_prefix_filter_from_trg().
Note: This API is a mess right now. C.f. bug 65776.
The "prefix_filter" optional argument can be used for Perl to test
the value its relevant function returns.
"""
if lang is None:
lang = self.lang
content, data = unmark_text(self.adjust_content(markedup_content))
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
langintel = self.mgr.langintel_from_lang(lang)
if trigger_name is None:
trigger_name = "fakey-completion-type"
if lang == "Perl":
trg = Trigger(lang,
TRG_FORM_DEFN,
trigger_name,
data["pos"],
implicit=implicit,
length=0,
**fields)
actual_citdl_expr, actual_prefix_filter \
= langintel.citdl_expr_and_prefix_filter_from_trg(buf, trg)
else:
#actual_citdl_expr = langintel.citdl_expr_under_pos(buf, data["pos"])
trg = Trigger(lang,
TRG_FORM_DEFN,
trigger_name,
data["pos"],
implicit=implicit,
**fields)
actual_citdl_expr = langintel.citdl_expr_from_trg(buf, trg)
self.assertEqual(
actual_citdl_expr, citdl_expr,
"unexpected actual %s CITDL expr under pos:\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s" %
(lang, citdl_expr, actual_citdl_expr, indent(markedup_content)))
if prefix_filter is not None:
XXX #TODO: compare prefix_filter to given value
def _assertDefnMatches(self, buf, pos, lang=None, **fields):
ctlr = _CaptureEvalController()
trg = buf.defn_trg_from_pos(pos)
defns = buf.defns_from_trg(trg, ctlr=ctlr)
if not defns:
self.fail(
"unexpectedly did not find a definition in %r at pos %d\n"
" eval log\n%s\n"
" buffer:\n%s" % (buf, pos,
indent('\n'.join('%5s: %s' % (lvl, m)
for lvl, m in ctlr.log)),
indent(buf.accessor.text)))
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
defn = defns[0]
for name, value in fields.items():
try:
actual_value = getattr(defn, name)
except AttributeError:
actual_value = None
self.assertEqual(
actual_value, value,
"%s definition, unexpected value for field %r\n"
" defn: %r\n"
" expected: %r\n"
" got: %r\n"
" eval log\n%s\n"
" buffer:\n%s" % (buf.lang, name, defn, value, actual_value,
indent('\n'.join('%5s: %s' % (lvl, m)
for lvl, m in ctlr.log)),
indent(buf.accessor.text)))
def assertDefnMatches(self, markedup_content, lang=None, **fields):
if lang is None:
lang = self.lang
buf, data = self._get_buf_and_data(markedup_content, lang)
self._assertDefnMatches(buf, data["pos"], lang, **fields)
def assertDefnMatches2(self, buf, pos, lang=None, **fields):
self._assertDefnMatches(buf, pos, lang, **fields)
def assertNoDuplicateDefns(self, markedup_content, lang=None):
if lang is None:
lang = self.lang
buf, data = self._get_buf_and_data(markedup_content, lang)
self.assertNoDuplicateDefns2(buf, data["pos"])
def assertNoDuplicateDefns2(self, buf, pos):
markedup_content = markup_text(buf.accessor.text, pos=pos)
ctlr = _CaptureEvalController()
trg = buf.defn_trg_from_pos(pos)
actual_defns = buf.defns_from_trg(trg, ctlr=ctlr)
if not actual_defns:
self.fail("%s trigger resulted in no definitions when expecting "
"to check for duplicate definitions:\n%s" %
(buf.lang, indent(markedup_content)))
count_from_defn_repr = {}
for defn_repr in (repr(d) for d in actual_defns):
if defn_repr not in count_from_defn_repr:
count_from_defn_repr[defn_repr] = 0
count_from_defn_repr[defn_repr] += 1
defn_dupes = [(count, defn_repr)
for defn_repr, count in count_from_defn_repr.items()
if count > 1]
self.failIf(
defn_dupes,
"unexpectedly got duplicate completions at the given position\n"
" duplicates:\n%s\n"
" eval log\n%s\n"
" buffer:\n%s" % (indent('\n'.join('%d of %s' % d
for d in defn_dupes)),
indent('\n'.join('%5s: %s' % (lvl, m)
for lvl, m in ctlr.log)),
indent(markedup_content)))
def assertTriggerMatches(self,
markedup_content,
lang=None,
implicit=True,
env=None,
**fields):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
if trg is None:
self.fail("unexpectedly did not find a %s trigger, buffer:\n%s" %
(lang, indent(markedup_content)))
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
for name, value in fields.items():
try:
actual_value = getattr(trg, name)
except AttributeError:
actual_value = trg.extra[name]
self.assertEqual(
actual_value, value,
"unexpected %s trigger '%s' value: expected %r, "
"got %r, buffer:\n%s" %
(lang, name, value, actual_value, indent(markedup_content)))
# Used when a position generates a trigger, but it's not the one specified
def assertTriggerDoesNotMatch(self,
markedup_content,
lang=None,
implicit=True,
env=None,
**fields):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
if trg is None:
# No trigger is as good as
return
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
for name, value in fields.items():
try:
actual_value = getattr(trg, name)
except AttributeError:
actual_value = trg.extra[name]
self.assertNotEqual(
actual_value, value,
"unexpected %s trigger '%s' value: expected not %r, "
"got %r, buffer:\n%s" %
(lang, name, value, actual_value, indent(markedup_content)))
def assertNoTrigger(self,
markedup_content,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
if trg is not None:
self.fail("unexpectedly found a %s trigger %r when didn't expect "
"one, buffer:\n%s" %
(lang, trg, indent(markedup_content)))
def assertPrecedingTriggerMatches(self,
markedup_content,
lang=None,
**fields):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(self.adjust_content(markedup_content))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
trg = buf.preceding_trg_from_pos(data["start_pos"], data["pos"])
if trg is None:
self.fail("unexpectedly did not find a preceding %s trigger, "
"buffer:\n%s" % (lang, indent(markedup_content)))
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
for name, value in fields.items():
actual_value = getattr(trg, name)
self.assertEqual(
actual_value, value,
"unexpected preceding %s trigger '%s' value: expected %r, "
"got %r, buffer:\n%s" %
(lang, name, value, actual_value, indent(markedup_content)))
def assertNoPrecedingTrigger(self, markedup_content, lang=None):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(self.adjust_content(markedup_content))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
trg = buf.preceding_trg_from_pos(data["start_pos"], data["pos"])
if trg is not None:
self.fail("unexpectedly found a preceding %s trigger '%s' when "
"didn't expect one, buffer:\n%s" %
(lang, trg.name, indent(markedup_content)))
def assertScopeLpathIs(self, markedup_content, lpath, lang=None):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(self.adjust_content(markedup_content))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
buf.scan(skip_scan_time_check=True)
actual_blob, actual_lpath = buf.scoperef_from_pos(data["pos"])
self.failUnlessEqual(
lpath, actual_lpath,
"unexpected %s scope lookup path (lpath) at the given position\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s" %
(self.lang, lpath, actual_lpath, indent(markedup_content)))
def assertCompletionsAre2(self,
buf,
pos,
completions,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
markedup_content = markup_text(buf.accessor.text, pos=pos)
trg = buf.trg_from_pos(pos, implicit=implicit)
self._assertCompletionsAre(markedup_content, buf, trg, completions,
lang, implicit)
def assertCompletionsAre(self,
markedup_content,
completions,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
self._assertCompletionsAre(markedup_content, buf, trg, completions,
lang, implicit)
def _assertCompletionsAre(self, markedup_content, buf, trg, completions,
lang, implicit):
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected completions to be %r:\n%s" %
(lang, completions, indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
self.assertEqual(
completions, actual_completions,
"unexpected %s completions at the given position\n"
" expected: %r\n"
" got: %r\n"
" extra: %r\n"
" missing: %r\n"
" eval log\n%s\n"
" buffer:\n%s" %
(lang, completions, actual_completions,
list(set(actual_completions or []).difference(completions or [])),
list(set(completions or []).difference(actual_completions or [])),
indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)),
indent(markedup_content)))
def assertNoDuplicateCompletions(self,
markedup_content,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected there to be completions:\n%s" %
(lang, indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
if actual_completions is None:
self.fail("%s trigger resulted in no completions when expecting "
"to check for duplicate completions:\n%s" %
(lang, indent(markedup_content)))
count_from_cpln = {}
for cpln in actual_completions:
if cpln not in count_from_cpln:
count_from_cpln[cpln] = 0
count_from_cpln[cpln] += 1
cpln_dupes = [(count, cpln) for cpln, count in count_from_cpln.items()
if count > 1]
self.failIf(
cpln_dupes,
"unexpectedly got duplicate completions at the given position\n"
" duplicates:\n%s\n"
" eval log\n%s\n"
" buffer:\n%s" % (indent('\n'.join('%d of %r' % d
for d in cpln_dupes)),
indent('\n'.join('%5s: %s' % (lvl, m)
for lvl, m in ctlr.log)),
indent(markedup_content)))
def _assertCompletionsInclude(self, buf, trg, completions):
markedup_content = markup_text(buf.accessor.text, pos=trg.pos)
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
missing_completions = [
c for c in completions if c not in (actual_completions or [])
]
self.failIf(
missing_completions,
"%s completions at the given position did not "
"include all expected values\n"
" missing: %r\n"
" expected all of: %r\n"
" got: %r\n"
" eval log:\n%s\n"
" buffer:\n%s" %
(buf.lang, missing_completions, completions, actual_completions,
indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)),
indent(markedup_content)))
def assertCompletionsInclude(self,
markedup_content,
completions,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected completions to include %r:\n%s" %
(lang, completions, indent(markedup_content)))
self._assertCompletionsInclude(buf, trg, completions)
def assertCompletionsInclude2(self, buf, pos, completions, implicit=True):
"""A version of assertCompletionsInclude() where you pass in
a Buffer instance instead of marked up content. Sometimes
this is more convenient.
"""
trg = buf.trg_from_pos(pos, implicit=implicit)
if trg is None:
markedup_content = markup_text(buf.accessor.text, pos=pos)
self.fail("given position is not a %s trigger point, "
"expected completions to include %r:\n%s" %
(buf.lang, completions, indent(markedup_content)))
self._assertCompletionsInclude(buf, trg, completions)
def _assertCompletionsDoNotInclude(self, buf, trg, completions):
markedup_content = markup_text(buf.accessor.text, pos=trg.pos)
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
completions_that_shouldnt_be_there = [
c for c in (actual_completions or []) if c in completions
]
self.failIf(
completions_that_shouldnt_be_there,
"%s completions at the given position included "
"some unexpected values\n"
" shouldn't have had these: %r\n"
" expected none of: %r\n"
" got: %r\n"
" eval log:\n%s\n"
" buffer:\n%s" % (buf.lang, completions_that_shouldnt_be_there,
completions, actual_completions,
indent('\n'.join('%5s: %s' % (lvl, m)
for lvl, m in ctlr.log)),
indent(markedup_content)))
def assertCompletionsDoNotInclude(self,
markedup_content,
completions,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected completions to exclude %r:\n%s" %
(lang, completions, indent(markedup_content)))
self._assertCompletionsDoNotInclude(buf, trg, completions)
def assertCompletionsDoNotInclude2(self,
buf,
pos,
completions,
implicit=True):
"""A version of assertCompletionsDoNotInclude() where you pass in
a Buffer instance instead of marked up content. Sometimes
this is more convenient.
"""
trg = buf.trg_from_pos(pos, implicit=implicit)
if trg is None:
markedup_content = markup_text(buf.accessor.text, pos=pos)
self.fail("given position is not a %s trigger point, "
"expected completions to exclude %r:\n%s" %
(buf.lang, completions, indent(markedup_content)))
self._assertCompletionsDoNotInclude(buf, trg, completions)
def assertCalltipIs2(self, buf, pos, calltip, implicit=True):
"""A variant of assertCalltipIs() where you pass in
a Buffer instance instead of marked up content. Sometimes
this is more convenient.
"""
trg = buf.trg_from_pos(pos, implicit=implicit)
markedup_content = markup_text(buf.accessor.text, pos=trg.pos)
self._assertCalltipIs(buf, trg, markedup_content, calltip, self.lang)
def assertCalltipIs(self,
markedup_content,
calltip,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
self._assertCalltipIs(buf, trg, markedup_content, calltip, lang)
def _assertCalltipIs(self, buf, trg, markedup_content, calltip, lang):
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected the following calltip:\n"
" calltip:\n%s\n"
" buffer:\n%s" %
(lang, indent(calltip), indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_calltips = buf.calltips_from_trg(trg, ctlr=ctlr)
if actual_calltips and actual_calltips[0]:
actual_calltip = actual_calltips[0]
else:
actual_calltip = None
self.assertEqual(
calltip, actual_calltip,
"unexpected %s calltip at the given position\n"
" expected:\n%s\n"
" got:\n%s\n"
" eval log:\n%s\n"
" buffer:\n%s" %
(trg.name, indent(calltip and calltip or "(none)"),
indent(actual_calltip and actual_calltip or "(none)"),
indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)),
indent(markedup_content)))
def assertCalltipMatches(self,
markedup_content,
calltip,
lang=None,
implicit=True,
env=None,
flags=0):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
self._assertCalltipMatches(buf, trg, markedup_content, calltip, lang,
flags)
def _assertCalltipMatches(self, buf, trg, markedup_content, expr, lang,
flags):
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected the calltip to match the following:\n"
" exression:\n%s\n"
" buffer:\n%s" %
(lang, indent(expr), indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_calltips = buf.calltips_from_trg(trg, ctlr=ctlr)
if actual_calltips and actual_calltips[0]:
actual_calltip = actual_calltips[0]
else:
actual_calltip = None
self.assertNotEquals(
re.search(expr, actual_calltip, flags), None,
"unexpected %s calltip at the given position\n"
" expression:\n%s\n"
" got:\n%s\n"
" eval log:\n%s\n"
" buffer:\n%s" %
(trg.name, indent(expr and expr or "(none)"),
indent(actual_calltip and actual_calltip or "(none)"),
indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)),
indent(markedup_content)))
def assertCurrCalltipArgRange(self,
markedup_content,
calltip,
expected_range,
lang=None,
implicit=True):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(self.adjust_content(markedup_content))
pos = data["pos"]
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
trg = buf.trg_from_pos(data["trg_pos"], implicit=implicit)
actual_range = buf.curr_calltip_arg_range(trg.pos,
calltip,
curr_pos=data["pos"])
self.assertEqual(
actual_range, expected_range,
"unexpected current calltip arg range\n"
" expected: %s\n"
" got: %s\n"
" calltip:\n%s\n"
" buffer:\n%s" % (expected_range, actual_range, indent(calltip),
indent(markedup_content)))
#def assertCompletionRaises(self, markedup_content, exception, lang=None,
# **kwargs):
# """Assert that the given completion raises the given exception.
#
# You may also specify either of the "exc_args" or "exc_pattern"
# keyword args to match the exception's "args" attribute or match
# the stringified exception against a regex pattern.
#
# c.f. http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/307970
# """
# if lang is None:
# lang = self.lang
# if "exc_args" in kwargs:
# exc_args = kwargs["exc_args"]
# del kwargs["exc_args"]
# else:
# exc_args = None
# if "exc_pattern" in kwargs:
# exc_pattern = kwargs["exc_pattern"]
# del kwargs["exc_pattern"]
# else:
# exc_pattern = None
#
# callsig = "the given %s completion" % lang
# try:
# buf, trg = self._get_buf_and_trg(markedup_content, lang)
# if trg is None:
# self.fail("given position is not a %s trigger point, "
# "no completion can be done to see if it raises"
# % self.lang)
# callsig = "completion at %s" % trg
# cplns = buf.cplns_from_trg(trg)
# except exception, exc:
# if exc_args is not None:
# self.failIf(exc.args != exc_args,
# "%s raised %s with unexpected args: "\
# "expected=%r, actual=%r"\
# % (callsig, exc.__class__, exc_args, exc.args))
# if exc_pattern is not None:
# self.failUnless(exc_pattern.search(str(exc)),
# "%s raised %s, but the exception "\
# "does not match '%s': %r"\
# % (callsig, exc.__class__, exc_pattern.pattern,
# str(exc)))
# except:
# exc_info = sys.exc_info()
# print exc_info
# self.fail("%s raised an unexpected exception type: "\
# "expected=%s, actual=%s"\
# % (callsig, exception, exc_info[0]))
# else:
# self.fail("%s did not raise %s" % (callsig, exception))
def assertEvalError(self,
markedup_content,
log_pattern,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"no completion can be done to see if errors" % self.lang)
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
class TestEvalController(EvalController):
"""A completion controller that captures all eval logging."""
def __init__(self):
EvalController.__init__(self)
self.log = []
def debug(self, msg, *args):
self.log.append(("debug", msg % args))
def info(self, msg, *args):
self.log.append(("info", msg % args))
def warn(self, msg, *args):
self.log.append(("warn", msg % args))
def error(self, msg, *args):
self.log.append(("error", msg % args))
ctlr = TestEvalController()
buf.async_eval_at_trg(trg, ctlr=ctlr)
ctlr.wait()
if not ctlr.is_done():
self.fail("evaluation is not 'done': didn't expect that")
if trg.form == TRG_FORM_CPLN and ctlr.cplns:
self.fail("evalution had results: didn't expect that: %r" %
ctlr.cplns)
elif trg.form == TRG_FORM_CALLTIP and ctlr.calltips:
self.fail("evalution had results: didn't expect that: %r" %
ctlr.cplns)
if log_pattern:
#pprint(ctlr.log)
matching_logs = [(level, msg) for level, msg in ctlr.log
if log_pattern.search(msg)]
self.failUnless(
matching_logs,
"the given completion failed but no logs matched the given pattern:\n"
" log_pattern: /%s/\n"
" log messages:\n%s\n"
" buffer:\n%s" %
(log_pattern.pattern,
indent('\n'.join(['%s: %s' % lg for lg in ctlr.log
])), indent(markedup_content)))
#pprint(matching_logs)
def assertCITDLExprIs(self,
markedup_content,
citdl_expr,
lang=None,
prefix_filter=None,
implicit=True,
trigger_name=None,
**fields):
"""Assert that the preceding CITDL expression at the current position
is as expected.
This uses buf.citdl_expr_from_trg() -- or, for Perl,
buf.citdl_expr_and_prefix_filter_from_trg().
The "prefix_filter" optional argument can be used for Perl to test
the value its relevant function returns.
"""
if lang is None:
lang = self.lang
content, data = unmark_text(self.adjust_content(markedup_content))
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
langintel = self.mgr.langintel_from_lang(lang)
if trigger_name is None:
trigger_name = "fakey-completion-type"
if lang == "Perl":
# Bit of a hack to fake the trigger length.
if content[data["pos"] - 1] in ('>', ':'): # '->' or '::' triggers
length = 2
else:
length = 1
trg = Trigger(lang,
TRG_FORM_CPLN,
trigger_name,
data["pos"],
implicit=implicit,
length=length,
**fields)
actual_citdl_expr, actual_prefix_filter \
= langintel.citdl_expr_and_prefix_filter_from_trg(buf, trg)
else:
trg = Trigger(lang,
TRG_FORM_CPLN,
trigger_name,
data["pos"],
implicit=implicit,
**fields)
actual_citdl_expr = langintel.citdl_expr_from_trg(buf, trg)
self.assertEqual(
actual_citdl_expr, citdl_expr,
"unexpected actual %s CITDL expr preceding trigger:\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s" %
(lang, citdl_expr, actual_citdl_expr, indent(markedup_content)))
if lang == "Perl" and prefix_filter is not None:
self.assertEqual(
actual_prefix_filter, prefix_filter,
"unexpected actual %s variable prefix filter "
"preceding trigger:\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s" % (lang, prefix_filter, actual_prefix_filter,
indent(markedup_content)))
def _unmark_lex_text(self, markedup_text):
from SilverCity import ScintillaConstants
tokenizer = re.compile(r'(<(SCE_\w+)>(.*?)</\2>)')
tokens = []
text = ''
while markedup_text:
match = tokenizer.search(markedup_text)
if match:
text += markedup_text[:match.start()]
token = {
'end_index': len(text) + len(match.group(3)) - 1,
'start_index': len(text),
'style': getattr(ScintillaConstants, match.group(2)),
'text': match.group(3),
#'end_column': ???,
#'end_line': ???,
#'start_column': ???,
#'start_line': ???,
}
tokens.append(token)
text += match.group(3)
markedup_text = markedup_text[match.end():]
else:
text += markedup_text
markedup_text = ''
return text, tokens
def assertLex(self, markedup_content, lang=None):
"""Lex the given content and assert that the lexed tokens are as
expected.
What is "expected" is given via pseudo-xml markup like this:
fuzzy wuzzy <SCE_UDL_SSL_COMMENTBLOCK>wuzza</SCE_UDL_SSL_COMMENTBLOCK> bear
This example expects that "wuzza" will be a token with style
SCE_UDL_SSL_COMMENTBLOCK.
"""
from codeintel2.accessor import SilverCityAccessor
if lang is None:
lang = self.lang
content, tokens = self._unmark_lex_text(markedup_content)
# Do lexing of this content via the codeintel Buffer's, because
# they already handle all the SilverCity lexer hookup.
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
assert isinstance(buf.accessor, SilverCityAccessor)
actual_tokens = buf.accessor.tokens # cheating
for actual_token in actual_tokens:
# There are a few SilverCity token dict keys that we
# don't bother checking.
del actual_token["end_column"]
del actual_token["end_line"]
del actual_token["start_column"]
del actual_token["start_line"]
unmatched_tokens = [t for t in tokens if t not in actual_tokens]
if unmatched_tokens:
self.fail("not all expected %s lex tokens were found in the "
"actual lexer output:\n"
" buffer:\n%s\n"
" actual lexer tokens:\n%s\n"
" unmatched tokens:\n%s\n" %
(lang, indent(content), indent(pformat(actual_tokens)),
indent(pformat(unmatched_tokens))))
def do_outline(self, subcmd, opts, path):
"""Print code outline of the given file.
You can specify a lookup path into the file code outline to
display via URL-anchor syntax, e.g.:
ci2 outline path/to/foo.py#AClass.amethod
${cmd_usage}
${cmd_option_list}
"""
import re
from ci2 import _outline_ci_elem
from codeintel2.manager import Manager
from codeintel2.util import tree_from_cix
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if '#' in path:
path, anchor = path.rsplit('#', 1)
else:
anchor = None
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
# buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
tree = buf.tree
if anchor is not None:
# Lookup the anchor in the codeintel CIX tree.
lpath = re.split(r'\.|::', anchor)
def blobs_from_tree(tree):
for file_elem in tree:
for blob in file_elem:
yield blob
for elem in blobs_from_tree(tree):
# Generally have 3 types of codeintel trees:
# 1. single-lang file: one <file>, one <blob>
# 2. multi-lang file: one <file>, one or two <blob>'s
# 3. CIX stdlib/catalog file: possibly multiple
# <file>'s, likely multiple <blob>'s
# Allow the first token to be the blob name or lang.
# (This can sometimes be weird, but seems the most
# convenient solution.)
if lpath[0] in (elem.get("name"), elem.get("lang")):
remaining_lpath = lpath[1:]
else:
remaining_lpath = lpath
for name in remaining_lpath:
try:
elem = elem.names[name]
except KeyError:
elem = None
break # try next lang blob
if elem is not None:
break # found one
else:
self.log.error("could not find `%s' definition (or blob) in `%s'",
anchor, path)
return 1
else:
elem = tree
try:
_outline_ci_elem(elem, brief=opts.brief, doSort=opts.doSort)
except IOError as ex:
if ex.errno == 0:
# Ignore this error from aborting 'less' of 'ci2 outline'
# output:
# IOError: (0, 'Error')
pass
else:
raise
finally:
mgr.finalize()
def _codeintel_scan():
global _ci_mgr_, despair, despaired
env = None
mtime = None
catalogs = []
now = time.time()
try:
env = _ci_envs_[path]
if env._folders != folders:
raise KeyError
mgr = _ci_mgr_
if now > env._time:
mtime = max(tryGetMTime(env._config_file), tryGetMTime(env._config_default_file))
if env._mtime < mtime:
raise KeyError
except KeyError:
if env is not None:
config_default_file = env._config_default_file
project_dir = env._project_dir
project_base_dir = env._project_base_dir
config_file = env._config_file
else:
config_default_file = os.path.join(CODEINTEL_HOME_DIR, "config")
if not (config_default_file and os.path.exists(config_default_file)):
config_default_file = None
project_dir = None
project_base_dir = None
if path:
# Try to find a suitable project directory (or best guess):
for folder in [".codeintel", ".git", ".hg", "trunk"]:
project_dir = find_folder(path, folder)
if project_dir and (
folder != ".codeintel" or not os.path.exists(os.path.join(project_dir, "db"))
):
if folder.startswith("."):
project_base_dir = os.path.abspath(os.path.join(project_dir, ".."))
else:
project_base_dir = project_dir
break
if not (project_dir and os.path.exists(project_dir)):
project_dir = None
config_file = project_dir and folder == ".codeintel" and os.path.join(project_dir, "config")
if not (config_file and os.path.exists(config_file)):
config_file = None
if _ci_mgr_:
mgr = _ci_mgr_
else:
for thread in threading.enumerate():
if thread.name == "CodeIntel Manager":
thread.finalize() # this finalizes the index, citadel and the manager and waits them to end (join)
mgr = Manager(
extra_module_dirs=_ci_extra_module_dirs_,
db_base_dir=_ci_db_base_dir_,
db_catalog_dirs=_ci_db_catalog_dirs_,
db_import_everything_langs=_ci_db_import_everything_langs,
db_event_reporter=lambda m: logger(view, "event", m),
)
mgr.upgrade()
mgr.initialize()
# Connect the logging file to the handler
condeintel_log_file = os.path.join(mgr.db.base_dir, "codeintel.log")
codeintel_log.handlers = [logging.StreamHandler(open(condeintel_log_file, "w", 1))]
msg = "Starting logging SublimeCodeIntel rev %s (%s) on %s" % (
get_git_revision()[:12],
os.stat(__file__)[stat.ST_MTIME],
datetime.datetime.now().ctime(),
)
codeintel_log.info("%s\n%s" % (msg, "=" * len(msg)))
_ci_mgr_ = mgr
# Load configuration files:
for catalog in mgr.db.get_catalogs_zone().avail_catalogs():
if catalog["lang"] == lang:
catalogs.append(catalog["name"])
config = {
"codeintel_selected_catalogs": catalogs,
"codeintel_max_recursive_dir_depth": 10,
"codeintel_scan_files_in_project": True,
}
_config = {}
try:
tryReadDict(config_default_file, _config)
except Exception, e:
msg = "Malformed configuration file '%s': %s" % (config_default_file, e)
log.error(msg)
codeintel_log.error(msg)
try:
tryReadDict(config_file, _config)
except Exception, e:
msg = "Malformed configuration file '%s': %s" % (config_default_file, e)
log.error(msg)
codeintel_log.error(msg)
def cix2html(opts, path):
"""Turn cix file into html API documentation.
Example:
cix2html path/to/foo.cix#AClass.amethod
cix2html path/to/foo.cix -o file.html
${cmd_usage}
${cmd_option_list}
"""
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
def blobs_from_tree(tree):
for file_elem in tree:
for blob in file_elem:
yield blob
if '#' in path:
path, anchor = path.rsplit('#', 1)
else:
anchor = None
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
# buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
tree = buf.tree
if anchor is not None:
# Lookup the anchor in the codeintel CIX tree.
lpath = re.split(r'\.|::', anchor)
for elem in blobs_from_tree(tree):
# Generally have 3 types of codeintel trees:
# 1. single-lang file: one <file>, one <blob>
# 2. multi-lang file: one <file>, one or two <blob>'s
# 3. CIX stdlib/catalog file: possibly multiple
# <file>'s, likely multiple <blob>'s
# Allow the first token to be the blob name or lang.
# (This can sometimes be weird, but seems the most
# convenient solution.)
if lpath[0] in (elem.get("name"), elem.get("lang")):
remaining_lpath = lpath[1:]
else:
remaining_lpath = lpath
for name in remaining_lpath:
try:
elem = elem.names[name]
except KeyError:
elem = None
break # try next lang blob
if elem is not None:
break # found one
else:
log.error("could not find `%s' definition (or blob) in `%s'",
anchor, path)
return 1
else:
elem = tree
try:
if elem.tag == "codeintel":
_html_ci_elem(opts, elem.getchildren()[0])
else:
_html_ci_elem(opts, elem)
except IOError, ex:
if ex.errno == 0:
# Ignore this error from aborting 'less' of 'ci2 outline'
# output:
# IOError: (0, 'Error')
pass
else:
raise
except Exception, e:
import traceback
traceback.print_exc()
class Driver(threading.Thread):
"""
Out-of-process codeintel2 driver
This class implements the main loop of the codeintel worker process
This should be a singleton object
"""
# static
_command_handler_map = collections.defaultdict(list)
"""Registered command handlers"""
_default_handler = None
"""The default (core) command handler; instance of a CoreHandler"""
_builtin_commands = {}
"""Built-in commands that cannot be overridden"""
def __init__(self, db_base_dir=None, fd_in=sys.stdin, fd_out=sys.stdout):
threading.Thread.__init__(self, name="Codeintel OOP Driver")
self.daemon = True
self.fd_in = fd_in
self.fd_out = fd_out
self.abort = None
self.quit = False
self.buffers = {} # path to Buffer objects
self.next_buffer = 0
self.active_request = None
self.queue = collections.deque()
self.queue_cv = threading.Condition()
self.env = Environment(name="global",
send_fn=functools.partial(self.send,
request=None))
# Fill out the non-overridable build-in commands
self._builtin_commands = {}
for attr in dir(self):
# Note that we check startswith first to avoid getters etc.
if attr.startswith("do_") and callable(getattr(self, attr)):
command = attr[len("do_"):].replace("_", "-")
self._builtin_commands[command] = getattr(self, attr)
from codeintel2.manager import Manager
log.debug("using db base dir %s", db_base_dir)
self.mgr = Manager(db_base_dir=db_base_dir,
db_catalog_dirs=[],
env=self.env,
on_scan_complete=self._on_scan_complete)
self.mgr.initialize()
def _on_scan_complete(self, scan_request):
if scan_request.status == "changed":
# Send unsolicited response about the completed scan
buf = scan_request.buf
self.send(request=None,
path=buf.path,
language=buf.lang,
command="scan-complete")
REQUEST_DEFAULT = object()
def send(self, request=REQUEST_DEFAULT, **kwargs):
"""
Send a response
"""
data = dict(kwargs)
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if request:
data["req_id"] = request.id
if "success" not in data:
data["success"] = True
elif data["success"] is None:
del data["success"]
buf = json.dumps(data, separators=(',', ':'))
log.debug("sending: [%s]", buf)
self.fd_out.write(str(len(buf)) + buf)
def fail(self, request=REQUEST_DEFAULT, **kwargs):
kwargs = kwargs.copy()
if not "command" in kwargs and request:
try:
kwargs["command"] = request["command"]
except KeyError:
pass
return self.send(request=request, success=False, **kwargs)
def exception(self, request=REQUEST_DEFAULT, **kwargs):
return self.fail(request=request,
stack=traceback.format_exc(),
**kwargs)
def get_buffer(self, request=REQUEST_DEFAULT, path=None):
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if path is None:
if not "path" in request:
raise RequestFailure(message="No path given to locate buffer")
path = request.path
if abspath(path) == path:
# this is an actual file path, not a URL or whatever
path = normcase(normpath(path))
try:
buf = self.buffers[path]
except KeyError:
buf = None
else:
if "language" in request and buf.lang != request.language:
buf = None # language changed, re-scan
if not buf:
# Need to construct a new buffer
lang = request.get("language")
if request.get("text") is not None:
# pass no content; we'll reset it later
buf = self.mgr.buf_from_content("", lang, path=path)
else:
# read from file
try:
buf = self.mgr.buf_from_path(
path, lang, encoding=request.get("encoding"))
except OSError:
# Can't read the file
buf = self.mgr.buf_from_content("", lang, path=path)
assert not request.path.startswith("<")
if request.get("text") is not None:
# overwrite the buffer contents if we have new ones
buf.accessor.reset_content(request.text)
try:
env = request["env"]
except KeyError:
pass # no environment set, use current environment
else:
if env.get("env", {}) is None and env.get("prefs", []) is None:
buf._env = None # explicitly clearing environment
elif buf._env:
buf._env.update(env)
else:
buf._env = Environment(env, name=os.path.basename(path))
# log.debug("Got buffer %r: [%s]", buf, buf.accessor.content)
log.debug("Got buffer %r", buf)
self.buffers[path] = buf
return buf
def do_abort(self, request):
try:
req_id = request["id"]
with self.queue_cv:
for item in self.queue:
if queue.get("req_id") == req_id:
self.queue.remove(queue)
self.send(request=request)
break
else:
self.abort = req_id
if self.active_request and self.active_request.id == req_id:
# need to wait a bit...
self.send(request=request)
else:
self.fail(request=request,
message="Request %s not found" % (req_id, ))
except RequestFailure as e:
self.fail(request=request, *e.args, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
def do_add_dirs(self, request):
catalog_dirs = request.get("catalog-dirs", None)
if catalog_dirs is not None:
self.mgr.db.catalog_dirs.extend(catalog_dirs)
lexer_dirs = request.get("lexer-dirs", [])
codeintel2.udl.UDLLexer.add_extra_lexer_dirs(lexer_dirs)
module_dirs = request.get("module-dirs", [])
if module_dirs:
self.mgr._register_modules(module_dirs)
if catalog_dirs is not None:
self.mgr.db.get_catalogs_zone().update()
def do_load_extension(self, request):
"""Load an extension that, for example, might provide additional
command handlers"""
path = request.get("module-path", None)
if not path:
raise RequestFailure(msg="load-extension requires a module-path")
name = request.get("module-name", None)
if not name:
raise RequestFailure(msg="load-extension requires a module-name")
iinfo = imp.find_module(name, [path])
try:
module = imp.load_module(name, *iinfo)
finally:
if iinfo and iinfo[0]:
iinfo[0].close()
callback = getattr(module, "registerExtension", None)
if not callback:
raise RequestFailure(msg="load-extension module %s should "
"have a 'registerExtension' method "
"taking no arguments" % (name, ))
callback()
self.send() # success, finally
def do_quit(self, request):
self.quit = True
self.send(command="quit")
def report_error(self, message):
self.send(request=None,
command="report-error",
message=unicode(message))
def start(self):
"""Start reading from the socket and dump requests into the queue"""
log.info("Running codeintel driver...")
buf = ""
self.send(success=None)
self.daemon = True
threading.Thread.start(self)
while not self.quit:
try:
ch = self.fd_in.read(1)
except IOError:
log.debug(
"Failed to read frame length, assuming connection died")
self.quit = True
break
if len(ch) == 0:
log.debug("Input was closed")
self.quit = True
break
if ch == "{":
size = int(buf, 10)
try:
buf = ch + self.fd_in.read(
size - 1) # exclude already-read {
except IOError:
log.debug(
"Failed to read frame data, assuming connection died")
self.quit = True
break
try:
buf = buf.decode("utf-8")
except UnicodeDecodeError:
pass # what :(
try:
data = json.loads(buf)
request = Request(data)
except Exception as e:
log.exception(e)
self.exception(message=e.message, request=None)
continue
finally:
buf = ""
if request.get("command") == "abort":
self.do_abort(request=request)
else:
log.debug("queuing request %r", request)
with self.queue_cv:
self.queue.appendleft(request)
self.queue_cv.notify()
elif ch in "0123456789":
buf += ch
else:
raise ValueError("Invalid request data: " + ch.encode("hex"))
def run(self):
"""Evaluate and send results back"""
log.info("Running codeintel eval thread...")
buf = ""
log.debug("default supported commands: %s",
", ".join(self._default_handler.supportedCommands))
while True:
with self.queue_cv:
try:
request = self.queue.pop()
except IndexError:
self.queue_cv.wait()
continue
log.debug("doing request %r", request)
try:
self.active_request = request
command = request.command
# First, check abort and quit; don't allow those to be
# overridden
try:
builtin = self._builtin_commands[command]
except KeyError:
pass
else:
builtin(request)
continue
handlers = self._command_handler_map.get(command, [])[:]
if command in self._default_handler.supportedCommands:
# The default handler can deal with this, put it at the end
handlers.append(self._default_handler)
for handler in handlers:
if handler.canHandleRequest(request):
handler.handleRequest(request, self)
break
else:
self.fail(request=request,
msg="Don't know how to handle command %s" %
(command, ))
except RequestFailure as e:
self.fail(request=request, *e.args, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
finally:
self.active_request = None
@classmethod
def registerCommandHandler(cls, handlerInstance):
"""Register a command handler"""
for command in handlerInstance.supportedCommands:
cls._command_handler_map[command].append(handlerInstance)
def cix2html(opts, path):
"""Turn cix file into html API documentation.
Example:
cix2html path/to/foo.cix#AClass.amethod
cix2html path/to/foo.cix -o file.html
${cmd_usage}
${cmd_option_list}
"""
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
def blobs_from_tree(tree):
for file_elem in tree:
for blob in file_elem:
yield blob
if '#' in path:
path, anchor = path.rsplit('#', 1)
else:
anchor = None
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
# buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
tree = buf.tree
if anchor is not None:
# Lookup the anchor in the codeintel CIX tree.
lpath = re.split(r'\.|::', anchor)
for elem in blobs_from_tree(tree):
# Generally have 3 types of codeintel trees:
# 1. single-lang file: one <file>, one <blob>
# 2. multi-lang file: one <file>, one or two <blob>'s
# 3. CIX stdlib/catalog file: possibly multiple
# <file>'s, likely multiple <blob>'s
# Allow the first token to be the blob name or lang.
# (This can sometimes be weird, but seems the most
# convenient solution.)
if lpath[0] in (elem.get("name"), elem.get("lang")):
remaining_lpath = lpath[1:]
else:
remaining_lpath = lpath
for name in remaining_lpath:
try:
elem = elem.names[name]
except KeyError:
elem = None
break # try next lang blob
if elem is not None:
break # found one
else:
log.error("could not find `%s' definition (or blob) in `%s'",
anchor, path)
return 1
else:
elem = tree
try:
if elem.tag == "codeintel":
_html_ci_elem(opts, elem.getchildren()[0])
else:
_html_ci_elem(opts, elem)
except IOError, ex:
if ex.errno == 0:
# Ignore this error from aborting 'less' of 'ci2 outline'
# output:
# IOError: (0, 'Error')
pass
else:
raise
except Exception, e:
import traceback
traceback.print_exc()
def do_outline(self, subcmd, opts, path):
"""Scan and outline the structure of the given path.
${cmd_usage}
${cmd_option_list}
"""
extra_lang_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_lang_module_dirs = [sys.path[-1]]
mgr = Manager(extra_lang_module_dirs=extra_lang_module_dirs)
mgr.upgrade()
mgr.initialize()
try:
if '#' in path:
path, anchor = path.rsplit('#', 1)
else:
anchor = None
tree = None
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info(
"skip `%s': couldn't determine language "
"(use --lang option)", path)
return
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
#buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
tree = buf.tree
if anchor is not None:
# Lookup the anchor in the codeintel CIX tree.
lpath = re.split(r'\.|::', anchor)
def blobs_from_tree(tree):
for file_elem in tree:
for blob in file_elem:
yield blob
for elem in blobs_from_tree(tree):
# Generally have 3 types of codeintel trees:
# 1. single-lang file: one <file>, one <blob>
# 2. multi-lang file: one <file>, one or two <blob>'s
# 3. CIX stdlib/catalog file: possibly multiple
# <file>'s, likely multiple <blob>'s
# Allow the first token to be the blob name or lang.
# (This can sometimes be weird, but seems the most
# convenient solution.)
if lpath[0] in (elem.get("name"), elem.get("lang")):
remaining_lpath = lpath[1:]
else:
remaining_lpath = lpath
for name in remaining_lpath:
try:
elem = elem.names[name]
except KeyError:
elem = None
break # try next lang blob
if elem is not None:
break # found one
else:
log.error(
"could not find `%s' definition (or blob) in `%s'",
anchor, path)
return 1
else:
elem = tree
try:
_outline_ci_elem(mgr, elem, quiet=opts.quiet)
except IOError, ex:
if ex.errno == 0:
# Ignore this error from aborting 'less' of this
# command:
# IOError: (0, 'Error')
pass
else:
raise
finally:
mgr.finalize()
def do_json(self, subcmd, opts, path):
"""Convert cix XML file into json format.
${cmd_usage}
${cmd_option_list}
"""
import json
from collections import defaultdict
from codeintel2.manager import Manager
from codeintel2.util import tree_from_cix
from codeintel2.common import Error
if opts.output == '-':
output_path = None
output_file = sys.stdout
else:
if opts.output:
output_path = opts.output
else:
output_path = os.path.splitext(path)[0] + ".json"
if os.path.exists(output_path):
if opts.force:
os.remove(output_path)
else:
raise Error("`%s' exists: use -f|--force option to "
"allow overwrite" % output_path)
output_file = open(output_path, 'w')
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
tree = buf.tree
result = {}
ci = result["codeintel"] = defaultdict(list)
def _elemToDict(parent, elem):
data = defaultdict(list)
name = elem.get("name")
if name is not None:
data["name"] = name
data["tag"] = elem.tag
for attr_name, attr in elem.attrib.items():
data[attr_name] = attr
parent["children"].append(data)
for child in elem:
_elemToDict(data, child)
for child in tree:
_elemToDict(ci, child)
json.dump(result, output_file, indent=2)
finally:
mgr.finalize()
class Driver(threading.Thread):
"""
Out-of-process codeintel2 driver
This class implements the main loop of the codeintel worker process
This should be a singleton object
"""
# static
_command_handler_map = collections.defaultdict(list)
"""Registered command handlers"""
_default_handler = None
"""The default (core) command handler; instance of a CoreHandler"""
_builtin_commands = {}
"""Built-in commands that cannot be overridden"""
def __init__(self, db_base_dir=None, fd_in=sys.stdin, fd_out=sys.stdout):
threading.Thread.__init__(self, name="Codeintel OOP Driver")
self.daemon = True
self.fd_in = fd_in
self.fd_out = fd_out
self.abort = None
self.quit = False
self.buffers = {} # path to Buffer objects
self.next_buffer = 0
self.active_request = None
self.queue = collections.deque()
self.queue_cv = threading.Condition()
self.env = Environment(name="global",
send_fn=functools.partial(self.send, request=None))
# Fill out the non-overridable build-in commands
self._builtin_commands = {}
for attr in dir(self):
# Note that we check startswith first to avoid getters etc.
if attr.startswith("do_") and callable(getattr(self, attr)):
command = attr[len("do_"):].replace("_", "-")
self._builtin_commands[command] = getattr(self, attr)
from codeintel2.manager import Manager
log.debug("using db base dir %s", db_base_dir)
self.mgr = Manager(db_base_dir=db_base_dir,
db_catalog_dirs=[],
env=self.env,
on_scan_complete=self._on_scan_complete)
self.mgr.initialize()
def _on_scan_complete(self, scan_request):
if scan_request.status == "changed":
# Send unsolicited response about the completed scan
buf = scan_request.buf
self.send(request=None,
path=buf.path,
language=buf.lang,
command="scan-complete")
REQUEST_DEFAULT = object()
def send(self, request=REQUEST_DEFAULT, **kwargs):
"""
Send a response
"""
data = dict(kwargs)
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if request:
data["req_id"] = request.id
if "success" not in data:
data["success"] = True
elif data["success"] is None:
del data["success"]
buf = json.dumps(data, separators=(',', ':'))
log.debug("sending: [%s]", buf)
self.fd_out.write(str(len(buf)) + buf)
def fail(self, request=REQUEST_DEFAULT, **kwargs):
kwargs = kwargs.copy()
if not "command" in kwargs and request:
try:
kwargs["command"] = request["command"]
except KeyError:
pass
return self.send(request=request, success=False, **kwargs)
def exception(self, request=REQUEST_DEFAULT, **kwargs):
return self.fail(request=request, stack=traceback.format_exc(), **kwargs)
def get_buffer(self, request=REQUEST_DEFAULT, path=None):
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if path is None:
if not "path" in request:
raise RequestFailure(message="No path given to locate buffer")
path = request.path
if abspath(path) == path:
# this is an actual file path, not a URL or whatever
path = normcase(normpath(path))
try:
buf = self.buffers[path]
except KeyError:
buf = None
else:
if "language" in request and buf.lang != request.language:
buf = None # language changed, re-scan
if not buf:
# Need to construct a new buffer
lang = request.get("language")
if request.get("text") is not None:
# pass no content; we'll reset it later
buf = self.mgr.buf_from_content("", lang, path=path)
else:
# read from file
try:
buf = self.mgr.buf_from_path(path, lang,
encoding=request.get("encoding"))
except OSError:
# Can't read the file
buf = self.mgr.buf_from_content("", lang, path=path)
assert not request.path.startswith("<")
if request.get("text") is not None:
# overwrite the buffer contents if we have new ones
buf.accessor.reset_content(request.text)
try:
env = request["env"]
except KeyError:
pass # no environment set, use current environment
else:
if env.get("env", {}) is None and env.get("prefs", []) is None:
buf._env = None # explicitly clearing environment
elif buf._env:
buf._env.update(env)
else:
buf._env = Environment(env, name=os.path.basename(path))
# log.debug("Got buffer %r: [%s]", buf, buf.accessor.content)
log.debug("Got buffer %r", buf)
self.buffers[path] = buf
return buf
def do_abort(self, request):
try:
req_id = request["id"]
with self.queue_cv:
for item in self.queue:
if queue.get("req_id") == req_id:
self.queue.remove(queue)
self.send(request=request)
break
else:
self.abort = req_id
if self.active_request and self.active_request.id == req_id:
# need to wait a bit...
self.send(request=request)
else:
self.fail(request=request,
message="Request %s not found" % (req_id,))
except RequestFailure as e:
self.fail(request=request, *e.args, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
def do_add_dirs(self, request):
catalog_dirs = request.get("catalog-dirs", None)
if catalog_dirs is not None:
self.mgr.db.catalog_dirs.extend(catalog_dirs)
lexer_dirs = request.get("lexer-dirs", [])
codeintel2.udl.UDLLexer.add_extra_lexer_dirs(lexer_dirs)
module_dirs = request.get("module-dirs", [])
if module_dirs:
self.mgr._register_modules(module_dirs)
if catalog_dirs is not None:
self.mgr.db.get_catalogs_zone().update()
def do_load_extension(self, request):
"""Load an extension that, for example, might provide additional
command handlers"""
path = request.get("module-path", None)
if not path:
raise RequestFailure(msg="load-extension requires a module-path")
name = request.get("module-name", None)
if not name:
raise RequestFailure(msg="load-extension requires a module-name")
iinfo = imp.find_module(name, [path])
try:
module = imp.load_module(name, *iinfo)
finally:
if iinfo and iinfo[0]:
iinfo[0].close()
callback = getattr(module, "registerExtension", None)
if not callback:
raise RequestFailure(msg="load-extension module %s should "
"have a 'registerExtension' method "
"taking no arguments" % (name,))
callback()
self.send() # success, finally
def do_quit(self, request):
self.quit = True
self.send(command="quit")
def report_error(self, message):
self.send(request=None,
command="report-error",
message=unicode(message))
def start(self):
"""Start reading from the socket and dump requests into the queue"""
log.info("Running codeintel driver...")
buf = ""
self.send(success=None)
self.daemon = True
threading.Thread.start(self)
while not self.quit:
try:
ch = self.fd_in.read(1)
except IOError:
log.debug(
"Failed to read frame length, assuming connection died")
self.quit = True
break
if len(ch) == 0:
log.debug("Input was closed")
self.quit = True
break
if ch == "{":
size = int(buf, 10)
try:
buf = ch + self.fd_in.read(
size - 1) # exclude already-read {
except IOError:
log.debug(
"Failed to read frame data, assuming connection died")
self.quit = True
break
try:
buf = buf.decode("utf-8")
except UnicodeDecodeError:
pass # what :(
try:
data = json.loads(buf)
request = Request(data)
except Exception as e:
log.exception(e)
self.exception(message=e.message, request=None)
continue
finally:
buf = ""
if request.get("command") == "abort":
self.do_abort(request=request)
else:
log.debug("queuing request %r", request)
with self.queue_cv:
self.queue.appendleft(request)
self.queue_cv.notify()
elif ch in "0123456789":
buf += ch
else:
raise ValueError("Invalid request data: " + ch.encode("hex"))
def run(self):
"""Evaluate and send results back"""
log.info("Running codeintel eval thread...")
buf = ""
log.debug("default supported commands: %s",
", ".join(self._default_handler.supportedCommands))
while True:
with self.queue_cv:
try:
request = self.queue.pop()
except IndexError:
self.queue_cv.wait()
continue
log.debug("doing request %r", request)
try:
self.active_request = request
command = request.command
# First, check abort and quit; don't allow those to be
# overridden
try:
builtin = self._builtin_commands[command]
except KeyError:
pass
else:
builtin(request)
continue
handlers = self._command_handler_map.get(command, [])[:]
if command in self._default_handler.supportedCommands:
# The default handler can deal with this, put it at the end
handlers.append(self._default_handler)
for handler in handlers:
if handler.canHandleRequest(request):
handler.handleRequest(request, self)
break
else:
self.fail(request=request,
msg="Don't know how to handle command %s" % (command,))
except RequestFailure as e:
self.fail(request=request, *e.args, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
finally:
self.active_request = None
@classmethod
def registerCommandHandler(cls, handlerInstance):
"""Register a command handler"""
for command in handlerInstance.supportedCommands:
cls._command_handler_map[command].append(handlerInstance)
def do_outline(self, subcmd, opts, path):
"""Print code outline of the given file.
You can specify a lookup path into the file code outline to
display via URL-anchor syntax, e.g.:
ci2 outline path/to/foo.py#AClass.amethod
${cmd_usage}
${cmd_option_list}
"""
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if '#' in path:
path, anchor = path.rsplit('#', 1)
else:
anchor = None
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
#buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
tree = buf.tree
if anchor is not None:
# Lookup the anchor in the codeintel CIX tree.
lpath = re.split(r'\.|::', anchor)
def blobs_from_tree(tree):
for file_elem in tree:
for blob in file_elem:
yield blob
for elem in blobs_from_tree(tree):
# Generally have 3 types of codeintel trees:
# 1. single-lang file: one <file>, one <blob>
# 2. multi-lang file: one <file>, one or two <blob>'s
# 3. CIX stdlib/catalog file: possibly multiple
# <file>'s, likely multiple <blob>'s
# Allow the first token to be the blob name or lang.
# (This can sometimes be weird, but seems the most
# convenient solution.)
if lpath[0] in (elem.get("name"), elem.get("lang")):
remaining_lpath = lpath[1:]
else:
remaining_lpath = lpath
for name in remaining_lpath:
try:
elem = elem.names[name]
except KeyError:
elem = None
break # try next lang blob
if elem is not None:
break # found one
else:
log.error(
"could not find `%s' definition (or blob) in `%s'",
anchor, path)
return 1
else:
elem = tree
try:
_outline_ci_elem(elem, brief=opts.brief, doSort=opts.doSort)
except IOError as ex:
if ex.errno == 0:
# Ignore this error from aborting 'less' of 'ci2 outline'
# output:
# IOError: (0, 'Error')
pass
else:
raise
finally:
mgr.finalize()
def do_outline(self, subcmd, opts, path):
"""Scan and outline the structure of the given path.
${cmd_usage}
${cmd_option_list}
"""
extra_lang_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_lang_module_dirs = [sys.path[-1]]
mgr = Manager(extra_lang_module_dirs=extra_lang_module_dirs)
mgr.upgrade()
mgr.initialize()
try:
if "#" in path:
path, anchor = path.rsplit("#", 1)
else:
anchor = None
tree = None
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info("skip `%s': couldn't determine language " "(use --lang option)", path)
return
if path.endswith(".cix"):
tree = tree_from_cix(open(path, "r").read())
# buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang))
tree = buf.tree
if anchor is not None:
# Lookup the anchor in the codeintel CIX tree.
lpath = re.split(r"\.|::", anchor)
def blobs_from_tree(tree):
for file_elem in tree:
for blob in file_elem:
yield blob
for elem in blobs_from_tree(tree):
# Generally have 3 types of codeintel trees:
# 1. single-lang file: one <file>, one <blob>
# 2. multi-lang file: one <file>, one or two <blob>'s
# 3. CIX stdlib/catalog file: possibly multiple
# <file>'s, likely multiple <blob>'s
# Allow the first token to be the blob name or lang.
# (This can sometimes be weird, but seems the most
# convenient solution.)
if lpath[0] in (elem.get("name"), elem.get("lang")):
remaining_lpath = lpath[1:]
else:
remaining_lpath = lpath
for name in remaining_lpath:
try:
elem = elem.names[name]
except KeyError:
elem = None
break # try next lang blob
if elem is not None:
break # found one
else:
log.error("could not find `%s' definition (or blob) in `%s'", anchor, path)
return 1
else:
elem = tree
try:
_outline_ci_elem(mgr, elem, quiet=opts.quiet)
except IOError, ex:
if ex.errno == 0:
# Ignore this error from aborting 'less' of this
# command:
# IOError: (0, 'Error')
pass
else:
raise
finally:
mgr.finalize()
_fn = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'path.ini')
if os.path.isfile(_fn):
with open(_fn, 'r') as f:
paths = f.readline()
print 'Paths:\n' + '\n'.join(paths.split(os.pathsep))
d = {}
d['phpExtraPaths'] = paths
d['pythonExtraPaths'] = paths
d['perlExtraPaths'] = paths
d['javascriptExtraPaths'] = paths
d['rubyExtraPaths'] = paths
env = SimplePrefsEnvironment(**d)
mgr = Manager()
mgr.upgrade()
mgr.initialize()
#------------------------
def file_text(fn):
try:
with open(fn, 'r') as f:
text = f.readlines()
return ''.join(text)
except IOError as e:
if e.errno == errno.ENOENT: # No such file or directory
logging.error(e.strerror + ': "' + fn + '"')
else:
logging.error(e.strerror)
def _codeintel_scan():
global _ci_mgr_, despair, despaired
env = None
mtime = None
catalogs = []
now = time.time()
try:
env = _ci_envs_[path]
if env._folders != folders:
raise KeyError
mgr = _ci_mgr_
if now > env._time:
mtime = max(tryGetMTime(env._config_file),
tryGetMTime(env._config_default_file))
if env._mtime < mtime:
raise KeyError
except KeyError:
if env is not None:
config_default_file = env._config_default_file
project_dir = env._project_dir
project_base_dir = env._project_base_dir
config_file = env._config_file
else:
config_default_file = os.path.join(CODEINTEL_HOME_DIR,
'config')
if not (config_default_file
and os.path.exists(config_default_file)):
config_default_file = None
project_dir = None
project_base_dir = None
if path:
# Try to find a suitable project directory (or best guess):
for folder in ['.codeintel', '.git', '.hg', 'trunk']:
project_dir = find_folder(path, folder)
if project_dir and (
folder != '.codeintel' or not os.path.exists(
os.path.join(project_dir, 'db'))):
if folder.startswith('.'):
project_base_dir = os.path.abspath(
os.path.join(project_dir, '..'))
else:
project_base_dir = project_dir
break
if not (project_dir and os.path.exists(project_dir)):
project_dir = None
config_file = project_dir and folder == '.codeintel' and os.path.join(
project_dir, 'config')
if not (config_file and os.path.exists(config_file)):
config_file = None
if _ci_mgr_:
mgr = _ci_mgr_
else:
for thread in threading.enumerate():
if thread.name == "CodeIntel Manager":
thread.finalize(
) # this finalizes the index, citadel and the manager and waits them to end (join)
mgr = Manager(
extra_module_dirs=_ci_extra_module_dirs_,
db_base_dir=_ci_db_base_dir_,
db_catalog_dirs=_ci_db_catalog_dirs_,
db_import_everything_langs=_ci_db_import_everything_langs,
db_event_reporter=lambda m: logger(view, 'event', m),
)
mgr.upgrade()
mgr.initialize()
# Connect the logging file to the handler
condeintel_log_file = os.path.join(mgr.db.base_dir,
'codeintel.log')
codeintel_log.handlers = [
logging.StreamHandler(open(condeintel_log_file, 'w', 1))
]
msg = "Starting logging SublimeCodeIntel rev %s (%s) on %s" % (
get_git_revision()[:12], os.stat(__file__)[stat.ST_MTIME],
datetime.datetime.now().ctime())
codeintel_log.info("%s\n%s" % (msg, "=" * len(msg)))
_ci_mgr_ = mgr
# Load configuration files:
for catalog in mgr.db.get_catalogs_zone().avail_catalogs():
if catalog['lang'] == lang:
catalogs.append(catalog['name'])
config = {
"codeintel_selected_catalogs": catalogs,
"codeintel_max_recursive_dir_depth": 10,
"codeintel_scan_files_in_project": True,
}
_config = {}
try:
tryReadDict(config_default_file, _config)
except Exception, e:
msg = "Malformed configuration file '%s': %s" % (
config_default_file, e)
log.error(msg)
codeintel_log.error(msg)
try:
tryReadDict(config_file, _config)
except Exception, e:
msg = "Malformed configuration file '%s': %s" % (
config_default_file, e)
log.error(msg)
codeintel_log.error(msg)
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
import time
import ciElementTree as ET
from ci2 import _paths_from_path_patterns
from codeintel2.manager import Manager
from codeintel2.citadel import CitadelBuffer
from codeintel2.common import CodeIntelError
from codeintel2.tree import pretty_tree_from_tree
from codeintel2.util import guess_lang_from_path
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if opts.time_it:
start = time.time()
quiet = opts.quiet
if opts.time_it or opts.time_details:
opts.force = True
scan_count = 0
lang_warnings = set()
tree = None
for path in _paths_from_path_patterns(path_patterns,
recursive=opts.recursive,
includes=opts.includes):
if opts.time_it:
sys.stderr.write(path + "\n")
if opts.time_details:
start1 = time.time()
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
self.log.info("skip `%s': couldn't determine language", path)
continue
try:
buf = mgr.buf_from_path(path, lang=lang)
except OSError as ex:
# Couldn't access the file.
if not opts.recursive:
raise
# Ignore files we don't really care about.
self.log.warn("%r - %r", ex, path)
continue
if not isinstance(buf, CitadelBuffer):
if opts.recursive:
# Ignore files that scanning isn't provided for.
continue
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
scan_count += 1
if scan_count % 10 == 0:
self.log.info("%d scanning %r", scan_count, path)
try:
if opts.force:
buf.scan()
if tree is None:
tree = ET.Element("codeintel", version="2.0")
file_elem = ET.SubElement(tree, "file",
lang=buf.lang,
mtime=str(int(time.time())),
path=os.path.basename(path))
for lang, blob in sorted(buf.blob_from_lang.items()):
blob = buf.blob_from_lang[lang]
file_elem.append(blob)
except KeyError as ex:
# Unknown cile language.
if not opts.recursive:
raise
message = str(ex)
if message not in lang_warnings:
lang_warnings.add(message)
self.log.warn("Skipping unhandled language %s", message)
if opts.time_details:
delta = time.time() - start1
sys.stderr.write("%.3f %s\n" % (delta, path))
sys.stderr.flush()
if tree is not None:
if opts.stripfuncvars:
# For stdlibs, we don't care about variables inside of
# functions and they take up a lot of space.
for function in tree.getiterator('scope'):
if function.get('ilk') == 'function':
function[:] = [child for child in function
if child.tag != 'variable']
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
if not quiet:
sys.stdout.write('<?xml version="1.0" encoding="UTF-8"?>\n')
ET.dump(tree)
if opts.time_it:
end = time.time()
sys.stderr.write("scan took %.3fs\n" % (end - start))
finally:
mgr.finalize()
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if opts.time_it:
start = time.time()
quiet = opts.quiet
if opts.time_it or opts.time_details:
opts.force = True
tree = None
for path in _paths_from_path_patterns(path_patterns,
recursive=opts.recursive,
includes=opts.includes):
if opts.time_it:
sys.stderr.write(path+"\n")
if opts.time_details:
start1 = time.time()
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info("skip `%s': couldn't determine language", path)
continue
buf = mgr.buf_from_path(path, lang=lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
if opts.force:
buf.scan()
if tree is None:
tree = ET.Element("codeintel", version="2.0")
file_elem = ET.SubElement(tree, "file",
lang=buf.lang,
mtime=str(int(time.time())),
path=os.path.basename(path))
for lang, blob in sorted(buf.blob_from_lang.items()):
blob = buf.blob_from_lang[lang]
file_elem.append(blob)
if opts.time_details:
delta = time.time() - start1
sys.stderr.write("%.3f %s\n" % (delta, path))
sys.stderr.flush()
if tree is not None:
if opts.stripfuncvars:
# For stdlibs, we don't care about variables inside of
# functions and they take up a lot of space.
for function in tree.getiterator('scope'):
if function.get('ilk') == 'function':
function[:] = [child for child in function
if child.tag != 'variable']
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
if not quiet:
sys.stdout.write('<?xml version="1.0" encoding="UTF-8"?>\n')
ET.dump(tree)
if opts.time_it:
end = time.time()
sys.stderr.write("scan took %.3fs\n" % (end-start))
finally:
mgr.finalize()
def do_play(self, subcmd, opts):
"""Run my current play/dev code.
${cmd_usage}
${cmd_option_list}
"""
import pprint
import random
import ciElementTree as ET
from codeintel2.manager import Manager
from codeintel2.tree import pretty_tree_from_tree
from codeintel2.common import LogEvalController, Error
from codeintel2.util import tree_from_cix, dedent, unmark_text, banner
from ci2 import _escaped_text_from_text
if False:
lang = "CSS"
markedup_content = dedent("""
/* http://www.w3.org/TR/REC-CSS2/fonts.html#propdef-font-weight */
h1 {
border: 1px solid black;
font-weight /* hi */: <|> !important
}
""")
content, data = unmark_text(markedup_content)
pos = data["pos"]
mgr = Manager()
# mgr.upgrade() # Don't need it for just CSS usage.
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path="play.css")
trg = buf.trg_from_pos(pos)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
completions = buf.cplns_from_trg(trg)
print("COMPLETIONS: %r" % completions)
finally:
mgr.finalize()
elif False:
lang = "Python"
path = os.path.join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
markedup_content = dedent("""
import sys, os
class Foo:
def bar(self):
pass
sys.<|>path # should have path in completion list
f = Foo()
""")
content, data = unmark_text(markedup_content)
print(banner(path))
print(_escaped_text_from_text(content, "whitespace"))
pos = data["pos"]
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path=path)
print(banner("cix", '-'))
print(buf.cix)
trg = buf.trg_from_pos(pos)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
print(banner("completions", '-'))
ctlr = LogEvalController(self.log)
buf.async_eval_at_trg(trg, ctlr)
ctlr.wait(2) # XXX
if not ctlr.is_done():
ctlr.abort()
raise Error("XXX async eval timed out")
pprint.pprint(ctlr.cplns)
print(banner(None))
finally:
mgr.finalize()
elif False:
lang = "Ruby"
path = os.path.join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
markedup_content = dedent("""\
r<1>equire 'net/http'
include Net
req = HTTPRequest.new
req.<2>get()
""")
content, data = unmark_text(markedup_content)
print(banner(path))
print(_escaped_text_from_text(content, "whitespace"))
pos = data[1]
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path=path)
print(banner("cix", '-'))
cix = buf.cix
print(ET.tostring(pretty_tree_from_tree(tree_from_cix(cix))))
trg = buf.trg_from_pos(pos, implicit=False)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
print(banner("completions", '-'))
ctlr = LogEvalController(self.log)
buf.async_eval_at_trg(trg, ctlr)
ctlr.wait(30) # XXX
if not ctlr.is_done():
ctlr.abort()
raise Error("XXX async eval timed out")
pprint.pprint(ctlr.cplns)
print(banner(None))
finally:
mgr.finalize()
_fn = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'path.ini')
if os.path.isfile(_fn):
with open(_fn, 'r') as f:
paths = f.readline()
print 'Paths:\n' + '\n'.join(paths.split(os.pathsep))
d = {}
d['phpExtraPaths'] = paths
d['pythonExtraPaths'] = paths
d['perlExtraPaths'] = paths
d['javascriptExtraPaths'] = paths
d['rubyExtraPaths'] = paths
env = SimplePrefsEnvironment(**d)
mgr = Manager()
mgr.upgrade()
mgr.initialize()
#------------------------
def file_text(fn):
try:
with open(fn, 'r') as f:
text = f.readlines()
return ''.join(text)
except IOError as e:
if e.errno == errno.ENOENT: # No such file or directory
logging.error(e.strerror + ': "' + fn + '"')
else:
logging.error(e.strerror)
return None
