def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
extra_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_module_dirs = [sys.path[-1]]
mgr = Manager(extra_module_dirs=extra_module_dirs)
mgr.upgrade()
mgr.initialize()
try:
tree = None
for path in _paths_from_path_patterns(path_patterns):
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info("skip `%s': couldn't determine language " "(use --lang option)", path)
continue
buf = mgr.buf_from_path(path, lang=opts.lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang))
buf.scan() # force a fresh scan
tree = buf.tree
for severity, msg in check_tree(tree):
dump = {"warning": log.warn, "error": log.error}[severity]
dump(msg)
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
ET.dump(tree)
finally:
mgr.finalize()
def do_html(self, subcmd, opts, path):
"""Convert the given path to styled HTML.
${cmd_usage}
${cmd_option_list}
The output includes trigger info and other stats. I.e. this is
primarily a debugging tool.
"""
from codeintel2.manager import Manager
from codeintel2.common import Error
from ci2 import _url_from_local_path
mgr = Manager()
try:
if opts.browse:
htmls = []
buf = mgr.buf_from_path(path, lang=opts.lang)
html = buf.to_html(True,
True,
title=path,
do_trg=opts.do_trg,
do_eval=opts.do_eval)
finally:
mgr.finalize()
if opts.output == '-':
output_path = None
output_file = sys.stdout
else:
if opts.output:
output_path = opts.output
else:
output_path = path + ".html"
if os.path.exists(output_path):
if opts.force:
os.remove(output_path)
else:
raise Error(
"`%s' exists: use -f|--force option to allow overwrite"
% output_path)
output_file = open(output_path, 'w')
# else:
# output_path = None
# output_file = sys.stdout
# #XXX Disable writing t
# output_file = None
if output_file:
output_file.write(html)
if output_path:
output_file.close()
if opts.browse:
if not output_path:
raise Error("cannot open in browser if stdout used "
"for output")
import webbrowser
url = _url_from_local_path(output_path)
webbrowser.open_new(url)
def worker(queue, lock, cix_dir, root="/"):
"""
worker procedure
"""
fix_module_path()
import dxr.mime
from codeintel2.citadel import CitadelBuffer
from codeintel2.common import CodeIntelError
from codeintel2.manager import Manager
from codeintel2.util import guess_lang_from_path
logging.getLogger("codeintel").setLevel(logging.ERROR)
log.info("starting indexing using %s", multiprocessing.current_process().name)
mgr = Manager(db_base_dir=cix_dir)
#mgr.upgrade()
mgr.initialize()
while True:
file_path = queue.get()
if file_path is None:
# marker for end of list
queue.put(None) # put it back so others can quit too
break
rel_path = os.path.relpath(file_path, root)
try:
lang = guess_lang_from_path(file_path)
except CodeIntelError:
log.info("%s: Cannot determine language, skipping", rel_path)
continue
# the file
with open(file_path, "r") as source_file:
data = source_file.read()
# Discard non-text files
if not dxr.mime.is_text(file_path, data):
continue
try:
buf = mgr.buf_from_path(file_path, lang=lang)
except CodeIntelError as ex:
if ex.message.startswith("File too big."):
log.info("%s: %s", file_path, ex.message)
continue # Nothing we can do about that, and the user can't
# fix this ever.
raise
if not isinstance(buf, CitadelBuffer):
log.info("%s: language %s does not have CIX, skipping",
rel_path, lang)
continue
log.info("%s: Using language %s", rel_path, lang)
buf.scan()
mgr.finalize()
def do_html(self, subcmd, opts, path):
"""Convert the given path to styled HTML.
${cmd_usage}
${cmd_option_list}
The generated HTML provides a good tool for seeing how Komodo's
lexer lexes the given content. This is the same tokenization that
you get from "buf.accessor.*" methods -- which you can use for
writing the CILE, trigger handling, and completion evaluation
for this language.
Use the "-t" and "-e" option to also exercise the current
trigger handling and completion evaluation (i.e. determining
the appropriate completions and calltips at a given trigger
point).
"""
extra_lang_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_lang_module_dirs = [sys.path[-1]]
mgr = Manager(extra_lang_module_dirs=extra_lang_module_dirs)
try:
if opts.browse:
htmls = []
buf = mgr.buf_from_path(path, lang=opts.lang)
html = buf.to_html(True,
True,
title=path,
do_trg=opts.do_trg,
do_eval=opts.do_eval)
finally:
mgr.finalize()
if opts.output == '-':
output_path = None
output_file = sys.stdout
else:
if opts.output:
output_path = opts.output
else:
output_path = path + ".html"
if exists(output_path):
os.remove(output_path)
output_file = open(output_path, 'w')
if output_file:
output_file.write(html)
if output_path:
output_file.close()
if opts.browse:
if not output_path:
raise CodeIntelError("cannot open in browser if stdout "
"used for output")
import webbrowser
url = _url_from_local_path(output_path)
webbrowser.open_new(url)
def do_json(self, subcmd, opts, path):
"""Convert cix XML file into json format.
${cmd_usage}
${cmd_option_list}
"""
import json
if opts.output == '-':
output_path = None
output_file = sys.stdout
else:
if opts.output:
output_path = opts.output
else:
output_path = splitext(path)[0]+".json"
if exists(output_path):
if opts.force:
os.remove(output_path)
else:
raise Error("`%s' exists: use -f|--force option to "
"allow overwrite" % output_path)
output_file = open(output_path, 'w')
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
tree = buf.tree
result = {}
ci = result["codeintel"] = defaultdict(list)
def _elemToDict(parent, elem):
data = defaultdict(list)
name = elem.get("name")
if name is not None:
data["name"] = name
data["tag"] = elem.tag
for attr_name, attr in elem.attrib.items():
data[attr_name] = attr
parent["children"].append(data)
for child in elem:
_elemToDict(data, child)
for child in tree:
_elemToDict(ci, child)
json.dump(result, output_file, indent=2)
finally:
mgr.finalize()
def do_json(self, subcmd, opts, path):
"""Convert cix XML file into json format.
${cmd_usage}
${cmd_option_list}
"""
import json
if opts.output == '-':
output_path = None
output_file = sys.stdout
else:
if opts.output:
output_path = opts.output
else:
output_path = splitext(path)[0] + ".json"
if exists(output_path):
if opts.force:
os.remove(output_path)
else:
raise Error("`%s' exists: use -f|--force option to "
"allow overwrite" % output_path)
output_file = open(output_path, 'w')
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
tree = buf.tree
result = {}
ci = result["codeintel"] = defaultdict(list)
def _elemToDict(parent, elem):
data = defaultdict(list)
name = elem.get("name")
if name is not None:
data["name"] = name
data["tag"] = elem.tag
for attr_name, attr in elem.attrib.items():
data[attr_name] = attr
parent["children"].append(data)
for child in elem:
_elemToDict(data, child)
for child in tree:
_elemToDict(ci, child)
json.dump(result, output_file, indent=2)
finally:
mgr.finalize()
def do_html(self, subcmd, opts, path):
"""Convert the given path to styled HTML.
${cmd_usage}
${cmd_option_list}
The output includes trigger info and other stats. I.e. this is
primarily a debugging tool.
"""
from codeintel2.manager import Manager
from codeintel2.common import Error
from ci2 import _url_from_local_path
mgr = Manager()
try:
if opts.browse:
htmls = []
buf = mgr.buf_from_path(path, lang=opts.lang)
html = buf.to_html(True, True, title=path,
do_trg=opts.do_trg,
do_eval=opts.do_eval)
finally:
mgr.finalize()
if opts.output == '-':
output_path = None
output_file = sys.stdout
else:
if opts.output:
output_path = opts.output
else:
output_path = path + ".html"
if os.path.exists(output_path):
if opts.force:
os.remove(output_path)
else:
raise Error("`%s' exists: use -f|--force option to allow overwrite" % output_path)
output_file = open(output_path, 'w')
# else:
# output_path = None
# output_file = sys.stdout
# #XXX Disable writing t
# output_file = None
if output_file:
output_file.write(html)
if output_path:
output_file.close()
if opts.browse:
if not output_path:
raise Error("cannot open in browser if stdout used "
"for output")
import webbrowser
url = _url_from_local_path(output_path)
webbrowser.open_new(url)
def do_html(self, subcmd, opts, path):
"""Convert the given path to styled HTML.
${cmd_usage}
${cmd_option_list}
The generated HTML provides a good tool for seeing how Komodo's
lexer lexes the given content. This is the same tokenization that
you get from "buf.accessor.*" methods -- which you can use for
writing the CILE, trigger handling, and completion evaluation
for this language.
Use the "-t" and "-e" option to also exercise the current
trigger handling and completion evaluation (i.e. determining
the appropriate completions and calltips at a given trigger
point).
"""
extra_lang_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_lang_module_dirs = [sys.path[-1]]
mgr = Manager(extra_lang_module_dirs=extra_lang_module_dirs)
try:
if opts.browse:
htmls = []
buf = mgr.buf_from_path(path, lang=opts.lang)
html = buf.to_html(True, True, title=path, do_trg=opts.do_trg, do_eval=opts.do_eval)
finally:
mgr.finalize()
if opts.output == "-":
output_path = None
output_file = sys.stdout
else:
if opts.output:
output_path = opts.output
else:
output_path = path + ".html"
if exists(output_path):
os.remove(output_path)
output_file = open(output_path, "w")
if output_file:
output_file.write(html)
if output_path:
output_file.close()
if opts.browse:
if not output_path:
raise CodeIntelError("cannot open in browser if stdout " "used for output")
import webbrowser
url = _url_from_local_path(output_path)
webbrowser.open_new(url)
def casper_tests():
from codeintel2.manager import Manager
db_base_dir = join(dirname(__file__), "tmp", "db")
mgr = Manager(db_base_dir)
mgr.upgrade()
mgr.initialize()
try:
for testpath in testpaths():
buf = mgr.buf_from_path(testpath, lang="JavaScript")
# Ensure the test is up to date.
if buf.scan_time < os.stat(testpath).st_mtime:
buf.scan()
for test in casper_tests_from_citree(buf.tree):
yield test
finally:
mgr.finalize()
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
extra_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_module_dirs = [sys.path[-1]]
mgr = Manager(extra_module_dirs=extra_module_dirs)
mgr.upgrade()
mgr.initialize()
try:
tree = None
for path in _paths_from_path_patterns(path_patterns):
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info(
"skip `%s': couldn't determine language "
"(use --lang option)", path)
continue
buf = mgr.buf_from_path(path, lang=opts.lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
buf.scan() # force a fresh scan
tree = buf.tree
for severity, msg in check_tree(tree):
dump = {"warning": log.warn, "error": log.error}[severity]
dump(msg)
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
ET.dump(tree)
finally:
mgr.finalize()
def do_outline(self, subcmd, opts, path):
"""Print code outline of the given file.
You can specify a lookup path into the file code outline to
display via URL-anchor syntax, e.g.:
ci2 outline path/to/foo.py#AClass.amethod
${cmd_usage}
${cmd_option_list}
"""
import re
from ci2 import _outline_ci_elem
from codeintel2.manager import Manager
from codeintel2.util import tree_from_cix
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if '#' in path:
path, anchor = path.rsplit('#', 1)
else:
anchor = None
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
# buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
tree = buf.tree
if anchor is not None:
# Lookup the anchor in the codeintel CIX tree.
lpath = re.split(r'\.|::', anchor)
def blobs_from_tree(tree):
for file_elem in tree:
for blob in file_elem:
yield blob
for elem in blobs_from_tree(tree):
# Generally have 3 types of codeintel trees:
# 1. single-lang file: one <file>, one <blob>
# 2. multi-lang file: one <file>, one or two <blob>'s
# 3. CIX stdlib/catalog file: possibly multiple
# <file>'s, likely multiple <blob>'s
# Allow the first token to be the blob name or lang.
# (This can sometimes be weird, but seems the most
# convenient solution.)
if lpath[0] in (elem.get("name"), elem.get("lang")):
remaining_lpath = lpath[1:]
else:
remaining_lpath = lpath
for name in remaining_lpath:
try:
elem = elem.names[name]
except KeyError:
elem = None
break # try next lang blob
if elem is not None:
break # found one
else:
self.log.error("could not find `%s' definition (or blob) in `%s'",
anchor, path)
return 1
else:
elem = tree
try:
_outline_ci_elem(elem, brief=opts.brief, doSort=opts.doSort)
except IOError as ex:
if ex.errno == 0:
# Ignore this error from aborting 'less' of 'ci2 outline'
# output:
# IOError: (0, 'Error')
pass
else:
raise
finally:
mgr.finalize()
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if opts.time_it:
start = time.time()
quiet = opts.quiet
if opts.time_it or opts.time_details:
opts.force = True
scan_count = 0
lang_warnings = set()
tree = None
for path in _paths_from_path_patterns(path_patterns,
recursive=opts.recursive,
includes=opts.includes):
if opts.time_it:
sys.stderr.write(path + "\n")
if opts.time_details:
start1 = time.time()
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info("skip `%s': couldn't determine language", path)
continue
try:
buf = mgr.buf_from_path(path, lang=lang)
except OSError as ex:
# Couldn't access the file.
if not opts.recursive:
raise
# Ignore files we don't really care about.
log.warn("%r - %r", ex, path)
continue
if not isinstance(buf, CitadelBuffer):
if opts.recursive:
# Ignore files that scanning isn't provided for.
continue
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
scan_count += 1
if scan_count % 10 == 0:
log.info("%d scanning %r", scan_count, path)
try:
if opts.force:
buf.scan()
if tree is None:
tree = ET.Element("codeintel", version="2.0")
file_elem = ET.SubElement(tree,
"file",
lang=buf.lang,
mtime=str(int(time.time())),
path=os.path.basename(path))
for lang, blob in sorted(buf.blob_from_lang.items()):
blob = buf.blob_from_lang[lang]
file_elem.append(blob)
except KeyError as ex:
# Unknown cile language.
if not opts.recursive:
raise
message = str(ex)
if message not in lang_warnings:
lang_warnings.add(message)
log.warn("Skipping unhandled language %s", message)
if opts.time_details:
delta = time.time() - start1
sys.stderr.write("%.3f %s\n" % (delta, path))
sys.stderr.flush()
if tree is not None:
if opts.stripfuncvars:
# For stdlibs, we don't care about variables inside of
# functions and they take up a lot of space.
for function in tree.getiterator('scope'):
if function.get('ilk') == 'function':
function[:] = [
child for child in function
if child.tag != 'variable'
]
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
if not quiet:
sys.stdout.write(
'<?xml version="1.0" encoding="UTF-8"?>\n')
ET.dump(tree)
if opts.time_it:
end = time.time()
sys.stderr.write("scan took %.3fs\n" % (end - start))
finally:
mgr.finalize()
def do_outline(self, subcmd, opts, path):
"""Print code outline of the given file.
You can specify a lookup path into the file code outline to
display via URL-anchor syntax, e.g.:
ci2 outline path/to/foo.py#AClass.amethod
${cmd_usage}
${cmd_option_list}
"""
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if '#' in path:
path, anchor = path.rsplit('#', 1)
else:
anchor = None
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
#buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
tree = buf.tree
if anchor is not None:
# Lookup the anchor in the codeintel CIX tree.
lpath = re.split(r'\.|::', anchor)
def blobs_from_tree(tree):
for file_elem in tree:
for blob in file_elem:
yield blob
for elem in blobs_from_tree(tree):
# Generally have 3 types of codeintel trees:
# 1. single-lang file: one <file>, one <blob>
# 2. multi-lang file: one <file>, one or two <blob>'s
# 3. CIX stdlib/catalog file: possibly multiple
# <file>'s, likely multiple <blob>'s
# Allow the first token to be the blob name or lang.
# (This can sometimes be weird, but seems the most
# convenient solution.)
if lpath[0] in (elem.get("name"), elem.get("lang")):
remaining_lpath = lpath[1:]
else:
remaining_lpath = lpath
for name in remaining_lpath:
try:
elem = elem.names[name]
except KeyError:
elem = None
break # try next lang blob
if elem is not None:
break # found one
else:
log.error(
"could not find `%s' definition (or blob) in `%s'",
anchor, path)
return 1
else:
elem = tree
try:
_outline_ci_elem(elem, brief=opts.brief, doSort=opts.doSort)
except IOError as ex:
if ex.errno == 0:
# Ignore this error from aborting 'less' of 'ci2 outline'
# output:
# IOError: (0, 'Error')
pass
else:
raise
finally:
mgr.finalize()
def do_outline(self, subcmd, opts, path):
"""Scan and outline the structure of the given path.
${cmd_usage}
${cmd_option_list}
"""
extra_lang_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_lang_module_dirs = [sys.path[-1]]
mgr = Manager(extra_lang_module_dirs=extra_lang_module_dirs)
mgr.upgrade()
mgr.initialize()
try:
if '#' in path:
path, anchor = path.rsplit('#', 1)
else:
anchor = None
tree = None
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info(
"skip `%s': couldn't determine language "
"(use --lang option)", path)
return
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
#buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
tree = buf.tree
if anchor is not None:
# Lookup the anchor in the codeintel CIX tree.
lpath = re.split(r'\.|::', anchor)
def blobs_from_tree(tree):
for file_elem in tree:
for blob in file_elem:
yield blob
for elem in blobs_from_tree(tree):
# Generally have 3 types of codeintel trees:
# 1. single-lang file: one <file>, one <blob>
# 2. multi-lang file: one <file>, one or two <blob>'s
# 3. CIX stdlib/catalog file: possibly multiple
# <file>'s, likely multiple <blob>'s
# Allow the first token to be the blob name or lang.
# (This can sometimes be weird, but seems the most
# convenient solution.)
if lpath[0] in (elem.get("name"), elem.get("lang")):
remaining_lpath = lpath[1:]
else:
remaining_lpath = lpath
for name in remaining_lpath:
try:
elem = elem.names[name]
except KeyError:
elem = None
break # try next lang blob
if elem is not None:
break # found one
else:
log.error(
"could not find `%s' definition (or blob) in `%s'",
anchor, path)
return 1
else:
elem = tree
try:
_outline_ci_elem(mgr, elem, quiet=opts.quiet)
except IOError, ex:
if ex.errno == 0:
# Ignore this error from aborting 'less' of this
# command:
# IOError: (0, 'Error')
pass
else:
raise
finally:
mgr.finalize()
def cix2html(opts, path):
"""Turn cix file into html API documentation.
Example:
cix2html path/to/foo.cix#AClass.amethod
cix2html path/to/foo.cix -o file.html
${cmd_usage}
${cmd_option_list}
"""
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
def blobs_from_tree(tree):
for file_elem in tree:
for blob in file_elem:
yield blob
if '#' in path:
path, anchor = path.rsplit('#', 1)
else:
anchor = None
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
# buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
tree = buf.tree
if anchor is not None:
# Lookup the anchor in the codeintel CIX tree.
lpath = re.split(r'\.|::', anchor)
for elem in blobs_from_tree(tree):
# Generally have 3 types of codeintel trees:
# 1. single-lang file: one <file>, one <blob>
# 2. multi-lang file: one <file>, one or two <blob>'s
# 3. CIX stdlib/catalog file: possibly multiple
# <file>'s, likely multiple <blob>'s
# Allow the first token to be the blob name or lang.
# (This can sometimes be weird, but seems the most
# convenient solution.)
if lpath[0] in (elem.get("name"), elem.get("lang")):
remaining_lpath = lpath[1:]
else:
remaining_lpath = lpath
for name in remaining_lpath:
try:
elem = elem.names[name]
except KeyError:
elem = None
break # try next lang blob
if elem is not None:
break # found one
else:
log.error("could not find `%s' definition (or blob) in `%s'",
anchor, path)
return 1
else:
elem = tree
try:
if elem.tag == "codeintel":
_html_ci_elem(opts, elem.getchildren()[0])
else:
_html_ci_elem(opts, elem)
except IOError, ex:
if ex.errno == 0:
# Ignore this error from aborting 'less' of 'ci2 outline'
# output:
# IOError: (0, 'Error')
pass
else:
raise
except Exception, e:
import traceback
traceback.print_exc()
class Driver(threading.Thread):
"""
Out-of-process codeintel2 driver
This class implements the main loop of the codeintel worker process
This should be a singleton object
"""
# static
_instance = None
_command_handler_map = collections.defaultdict(list)
"""Registered command handlers; the key is the command name (a str), and the
value is a list of command handler instances or (for lazy handlers) a
single-element tuple containing the callable to get the real handler."""
_default_handler = None
"""The default (core) command handler; instance of a CoreHandler"""
_builtin_commands = {}
"""Built-in commands that cannot be overridden"""
def __init__(self, db_base_dir=None, fd_in=sys.stdin, fd_out=sys.stdout):
threading.Thread.__init__(self, name="Codeintel OOP Driver")
assert Driver._instance is None, "Driver should be a singleton"
Driver._instance = self
logging.root.addHandler(LoggingHandler(self))
self.daemon = True
self.fd_in = fd_in
self.fd_out = fd_out
self.abort = None
self.quit = False
self.buffers = {} # path to Buffer objects
self.next_buffer = 0
self.active_request = None
self.send_queue = Queue.Queue()
self.send_thread = threading.Thread(target=self._send_proc)
self.send_thread.daemon = True
self.send_thread.start()
self.queue = collections.deque()
self.queue_cv = threading.Condition()
self.env = Environment(name="global",
send_fn=functools.partial(self.send,
request=None))
# Fill out the non-overridable build-in commands
self._builtin_commands = {}
for attr in dir(self):
# Note that we check startswith first to avoid getters etc.
if attr.startswith("do_") and callable(getattr(self, attr)):
command = attr[len("do_"):].replace("_", "-")
self._builtin_commands[command] = getattr(self, attr)
from codeintel2.manager import Manager
log.debug("using db base dir %s", db_base_dir)
self.mgr = Manager(db_base_dir=db_base_dir,
db_catalog_dirs=[],
db_event_reporter=self._DBEventReporter(self),
env=self.env,
on_scan_complete=self._on_scan_complete)
self.mgr.initialize()
def _on_scan_complete(self, scan_request):
if scan_request.status in ("changed", "skipped"):
# Send unsolicited response about the completed scan
buf = scan_request.buf
self.send(request=None,
path=buf.path,
language=buf.lang,
command="scan-complete")
class _DBEventReporter(object):
def __init__(self, driver):
self.driver = driver
self.log = log.getChild("DBEventReporter")
self.debug = self.log.debug
# directories being scanned (completed or not)
# key is unicode path, value is number of times it's active
self._dirs = collections.defaultdict(int)
# directories which are complete (unicode path)
self._completed_dirs = set()
def send(self, **kwargs):
self.driver.send(request=None, command="report-message", **kwargs)
def __call__(self, message):
"""Old-style status messages before long-running jobs
@param msg {str or None} The message to display
"""
if len(self._dirs):
return # ignore old-style db events if we're doing a scan
self.debug("db event: %s", message)
self.send(message=message)
def onScanStarted(self, description, dirs=set()):
"""Called when a directory scan is about to start
@param description {unicode} A string suitable for showing the user
about the upcoming operation
@param dirs {set of unicode} The directories about to be scanned
"""
self.debug("scan started: %s (%s dirs)", description, len(dirs))
assert dirs, "onScanStarted expects non-empty directories"
if not dirs: # empty set - we shouldn't have gotten here, but be nice
return
for dir_path in dirs:
self._dirs[dir_path] += 1
self.send(type="scan-progress",
message=description,
completed=len(self._completed_dirs),
total=len(self._dirs))
def onScanDirectory(self,
description,
dir_path,
current=None,
total=None):
"""Called when a directory is being scanned (out of possibly many)
@param description {unicode} A string suitable for showing the user
regarding the progress
@param dir {unicode} The directory currently being scanned
@param current {int} The current progress
@param total {int} The total number of directories to scan in this
request
"""
self.debug("scan directory: %s (%s %s/%s)", description, dir_path,
current, total)
assert dir_path, "onScanDirectory got no directory"
if dir_path:
self._completed_dirs.add(dir_path)
self.send(type="scan-progress",
message=description,
completed=len(self._completed_dirs),
total=len(self._dirs))
def onScanComplete(self, dirs=set(), scanned=set()):
"""Called when a scan operation is complete
@param dirs {set of unicode} The directories that were intially
requested to be scanned (as pass in onScanStarted)
@param scanned {set of unicode} Directories which were successfully
scanned. This may be a subset of dirs if the scan was
aborted.
"""
self.debug("scan complete: scanned %r/%r dirs", len(scanned),
len(dirs))
for dir_path in dirs:
self._dirs[dir_path] -= 1
if not self._dirs[dir_path]:
del self._dirs[dir_path]
self._completed_dirs.discard(dir_path)
self.send(type="scan-progress",
completed=len(self._completed_dirs),
total=len(self._dirs))
REQUEST_DEFAULT = object()
def send(self, request=REQUEST_DEFAULT, **kwargs):
"""
Send a response
"""
data = dict(kwargs)
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if request:
data["req_id"] = request.id
if "success" not in data:
data["success"] = True
elif data["success"] is None:
del data["success"]
buf = json.dumps(data, separators=(',', ':'))
buf_len = str(len(buf))
log.debug("sending: %s:[%s]", buf_len, buf)
self.send_queue.put(buf)
def _send_proc(self):
while True:
buf = self.send_queue.get()
try:
buf_len = str(len(buf))
self.fd_out.write(buf_len)
self.fd_out.write(buf)
finally:
self.send_queue.task_done()
def fail(self, request=REQUEST_DEFAULT, **kwargs):
kwargs = kwargs.copy()
if not "command" in kwargs and request:
try:
kwargs["command"] = request["command"]
except KeyError:
pass
return self.send(request=request, success=False, **kwargs)
def exception(self, request=REQUEST_DEFAULT, **kwargs):
return self.fail(request=request,
stack=traceback.format_exc(),
**kwargs)
def get_buffer(self, request=REQUEST_DEFAULT, path=None):
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if path is None:
if not "path" in request:
raise RequestFailure(message="No path given to locate buffer")
path = request.path
if abspath(path) == path:
# this is an actual file path, not a URL or whatever
path = normcase(normpath(path))
try:
buf = self.buffers[path]
except KeyError:
buf = None
else:
if "language" in request and buf.lang != request.language:
buf = None # language changed, re-scan
if not buf:
# Need to construct a new buffer
lang = request.get("language")
if request.get("text") is not None:
# pass no content; we'll reset it later
buf = self.mgr.buf_from_content("", lang, path=path)
else:
# read from file
try:
buf = self.mgr.buf_from_path(
path, lang, encoding=request.get("encoding"))
except OSError:
# Can't read the file
buf = self.mgr.buf_from_content("", lang, path=path)
assert not request.path.startswith("<"), \
"Can't create an unsaved buffer with no text"
if request.get("text") is not None:
# overwrite the buffer contents if we have new ones
buf.accessor.reset_content(request.text)
try:
env = request["env"]
except KeyError:
pass # no environment set, use current environment
else:
if env.get("env", {}) is None and env.get("prefs", []) is None:
buf._env = None # explicitly clearing environment
elif buf._env:
buf._env.update(env)
else:
buf._env = Environment(env, name=os.path.basename(path))
# log.debug("Got buffer %r: [%s]", buf, buf.accessor.content)
log.debug("Got buffer %r", buf)
self.buffers[path] = buf
return buf
def do_abort(self, request):
try:
req_id = request["id"]
with self.queue_cv:
for item in self.queue:
if queue.get("req_id") == req_id:
self.queue.remove(queue)
self.send(request=request)
break
else:
self.abort = req_id
if self.active_request and self.active_request.id == req_id:
# need to wait a bit...
self.send(request=request)
else:
self.fail(request=request,
message="Request %s not found" % (req_id, ))
except RequestFailure as e:
self.fail(request=request, *e.args, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
def do_add_dirs(self, request):
catalog_dirs = request.get("catalog-dirs", None)
if catalog_dirs is not None:
self.mgr.db.catalog_dirs.extend(catalog_dirs)
catalog_dirs = self.mgr.db.catalog_dirs
lexer_dirs = request.get("lexer-dirs", [])
codeintel2.udl.UDLLexer.add_extra_lexer_dirs(lexer_dirs)
module_dirs = request.get("module-dirs", [])
if module_dirs:
self.mgr._register_modules(module_dirs)
if catalog_dirs is not None:
self.mgr.db.get_catalogs_zone().catalog_dirs = catalog_dirs
def do_load_extension(self, request):
"""Load an extension that, for example, might provide additional
command handlers"""
path = request.get("module-path", None)
if not path:
raise RequestFailure(msg="load-extension requires a module-path")
name = request.get("module-name", None)
if not name:
raise RequestFailure(msg="load-extension requires a module-name")
iinfo = imp.find_module(name, [path])
try:
module = imp.load_module(name, *iinfo)
finally:
if iinfo and iinfo[0]:
iinfo[0].close()
callback = getattr(module, "registerExtension", None)
if not callback:
raise RequestFailure(msg="load-extension module %s should "
"have a 'registerExtension' method "
"taking no arguments" % (name, ))
callback()
self.send() # success, finally
def do_quit(self, request):
self.quit = True
self.send(command="quit")
def report_error(self, message):
self.send(request=None,
command="report-error",
message=unicode(message))
def start(self):
"""Start reading from the socket and dump requests into the queue"""
log.info("Running codeintel driver...")
buf = ""
self.send(success=None)
self.daemon = True
threading.Thread.start(self)
while not self.quit:
try:
ch = self.fd_in.read(1)
except IOError:
log.debug(
"Failed to read frame length, assuming connection died")
self.quit = True
break
if len(ch) == 0:
log.debug("Input was closed")
self.quit = True
break
if ch == "{":
size = int(buf, 10)
try:
buf = ch + self.fd_in.read(
size - 1) # exclude already-read {
except IOError:
log.debug(
"Failed to read frame data, assuming connection died")
self.quit = True
break
try:
buf = buf.decode("utf-8")
except UnicodeDecodeError:
pass # what :(
try:
data = json.loads(buf)
request = Request(data)
except Exception as e:
log.exception(e)
self.exception(message=e.message, request=None)
continue
finally:
buf = ""
if request.get("command") == "abort":
self.do_abort(request=request)
else:
log.debug("queuing request %r", request)
with self.queue_cv:
self.queue.appendleft(request)
self.queue_cv.notify()
elif ch in "0123456789":
buf += ch
else:
raise ValueError("Invalid request data: " + ch.encode("hex"))
def run(self):
"""Evaluate and send results back"""
log.info("Running codeintel eval thread...")
buf = ""
log.debug("default supported commands: %s",
", ".join(self._default_handler.supportedCommands))
while True:
with self.queue_cv:
try:
request = self.queue.pop()
except IndexError:
self.queue_cv.wait()
continue
log.debug("doing request %r", request)
try:
self.active_request = request
command = request.command
# First, check abort and quit; don't allow those to be
# overridden
try:
builtin = self._builtin_commands[command]
except KeyError:
pass
else:
builtin(request)
continue
handlers = self._command_handler_map.get(command, [])[:]
if command in self._default_handler.supportedCommands:
# The default handler can deal with this, put it at the end
handlers.append(self._default_handler)
for handler in handlers:
if isinstance(handler, tuple):
try:
real_handler = handler[0]()
except Exception as ex:
log.exception("Failed to get lazy handler for %s",
command)
real_handler = None
if real_handler is None:
# Handler failed to instantiate, drop it
try:
self._command_handler_map["command"].remove(
handler)
except ValueError:
pass # ... shouldn't happen, but tolerate it
continue
for handlers in self._command_handler_map.values():
try:
handlers[handlers.index(
handler)] = real_handler
except ValueError:
pass # handler not in this list
handler = real_handler
if handler.canHandleRequest(request):
handler.handleRequest(request, self)
break
else:
self.fail(request=request,
msg="Don't know how to handle command %s" %
(command, ))
except RequestFailure as e:
self.fail(request=request, *e.args, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
finally:
self.active_request = None
@classmethod
def registerCommandHandler(cls, handlerInstance):
"""Register a command handler"""
for command in handlerInstance.supportedCommands:
cls._command_handler_map[command].append(handlerInstance)
@classmethod
def registerLazyCommandHandler(cls, supported_commands, constructor):
"""Register a lazy command handler
@param supported_commands {iterable} The commands to handle; each
element should be a str of the command name.
@param constructor {callable} Function to be called to get the real
command handler; it should take no arguments and return a command
handler instance. It may return None if the command is not
available; it will not be asked again.
"""
for command in supported_commands:
cls._command_handler_map[command].append((constructor, ))
@classmethod
def getInstance(cls):
"""Get the singleton instance of the driver"""
return Driver._instance
class Driver(threading.Thread):
"""
Out-of-process codeintel2 driver
This class implements the main loop of the codeintel worker process
This should be a singleton object
"""
# static
_command_handler_map = collections.defaultdict(list)
"""Registered command handlers"""
_default_handler = None
"""The default (core) command handler; instance of a CoreHandler"""
_builtin_commands = {}
"""Built-in commands that cannot be overridden"""
def __init__(self, db_base_dir=None, fd_in=sys.stdin, fd_out=sys.stdout):
threading.Thread.__init__(self, name="Codeintel OOP Driver")
self.daemon = True
self.fd_in = fd_in
self.fd_out = fd_out
self.abort = None
self.quit = False
self.buffers = {} # path to Buffer objects
self.next_buffer = 0
self.active_request = None
self.queue = collections.deque()
self.queue_cv = threading.Condition()
self.env = Environment(name="global",
send_fn=functools.partial(self.send, request=None))
# Fill out the non-overridable build-in commands
self._builtin_commands = {}
for attr in dir(self):
# Note that we check startswith first to avoid getters etc.
if attr.startswith("do_") and callable(getattr(self, attr)):
command = attr[len("do_"):].replace("_", "-")
self._builtin_commands[command] = getattr(self, attr)
from codeintel2.manager import Manager
log.debug("using db base dir %s", db_base_dir)
self.mgr = Manager(db_base_dir=db_base_dir,
db_catalog_dirs=[],
env=self.env,
on_scan_complete=self._on_scan_complete)
self.mgr.initialize()
def _on_scan_complete(self, scan_request):
if scan_request.status == "changed":
# Send unsolicited response about the completed scan
buf = scan_request.buf
self.send(request=None,
path=buf.path,
language=buf.lang,
command="scan-complete")
REQUEST_DEFAULT = object()
def send(self, request=REQUEST_DEFAULT, **kwargs):
"""
Send a response
"""
data = dict(kwargs)
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if request:
data["req_id"] = request.id
if "success" not in data:
data["success"] = True
elif data["success"] is None:
del data["success"]
buf = json.dumps(data, separators=(',', ':'))
log.debug("sending: [%s]", buf)
self.fd_out.write(str(len(buf)) + buf)
def fail(self, request=REQUEST_DEFAULT, **kwargs):
kwargs = kwargs.copy()
if not "command" in kwargs and request:
try:
kwargs["command"] = request["command"]
except KeyError:
pass
return self.send(request=request, success=False, **kwargs)
def exception(self, request=REQUEST_DEFAULT, **kwargs):
return self.fail(request=request, stack=traceback.format_exc(), **kwargs)
def get_buffer(self, request=REQUEST_DEFAULT, path=None):
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if path is None:
if not "path" in request:
raise RequestFailure(message="No path given to locate buffer")
path = request.path
if abspath(path) == path:
# this is an actual file path, not a URL or whatever
path = normcase(normpath(path))
try:
buf = self.buffers[path]
except KeyError:
buf = None
else:
if "language" in request and buf.lang != request.language:
buf = None # language changed, re-scan
if not buf:
# Need to construct a new buffer
lang = request.get("language")
if request.get("text") is not None:
# pass no content; we'll reset it later
buf = self.mgr.buf_from_content("", lang, path=path)
else:
# read from file
try:
buf = self.mgr.buf_from_path(path, lang,
encoding=request.get("encoding"))
except OSError:
# Can't read the file
buf = self.mgr.buf_from_content("", lang, path=path)
assert not request.path.startswith("<")
if request.get("text") is not None:
# overwrite the buffer contents if we have new ones
buf.accessor.reset_content(request.text)
try:
env = request["env"]
except KeyError:
pass # no environment set, use current environment
else:
if env.get("env", {}) is None and env.get("prefs", []) is None:
buf._env = None # explicitly clearing environment
elif buf._env:
buf._env.update(env)
else:
buf._env = Environment(env, name=os.path.basename(path))
# log.debug("Got buffer %r: [%s]", buf, buf.accessor.content)
log.debug("Got buffer %r", buf)
self.buffers[path] = buf
return buf
def do_abort(self, request):
try:
req_id = request["id"]
with self.queue_cv:
for item in self.queue:
if queue.get("req_id") == req_id:
self.queue.remove(queue)
self.send(request=request)
break
else:
self.abort = req_id
if self.active_request and self.active_request.id == req_id:
# need to wait a bit...
self.send(request=request)
else:
self.fail(request=request,
message="Request %s not found" % (req_id,))
except RequestFailure as e:
self.fail(request=request, *e.args, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
def do_add_dirs(self, request):
catalog_dirs = request.get("catalog-dirs", None)
if catalog_dirs is not None:
self.mgr.db.catalog_dirs.extend(catalog_dirs)
lexer_dirs = request.get("lexer-dirs", [])
codeintel2.udl.UDLLexer.add_extra_lexer_dirs(lexer_dirs)
module_dirs = request.get("module-dirs", [])
if module_dirs:
self.mgr._register_modules(module_dirs)
if catalog_dirs is not None:
self.mgr.db.get_catalogs_zone().update()
def do_load_extension(self, request):
"""Load an extension that, for example, might provide additional
command handlers"""
path = request.get("module-path", None)
if not path:
raise RequestFailure(msg="load-extension requires a module-path")
name = request.get("module-name", None)
if not name:
raise RequestFailure(msg="load-extension requires a module-name")
iinfo = imp.find_module(name, [path])
try:
module = imp.load_module(name, *iinfo)
finally:
if iinfo and iinfo[0]:
iinfo[0].close()
callback = getattr(module, "registerExtension", None)
if not callback:
raise RequestFailure(msg="load-extension module %s should "
"have a 'registerExtension' method "
"taking no arguments" % (name,))
callback()
self.send() # success, finally
def do_quit(self, request):
self.quit = True
self.send(command="quit")
def report_error(self, message):
self.send(request=None,
command="report-error",
message=unicode(message))
def start(self):
"""Start reading from the socket and dump requests into the queue"""
log.info("Running codeintel driver...")
buf = ""
self.send(success=None)
self.daemon = True
threading.Thread.start(self)
while not self.quit:
try:
ch = self.fd_in.read(1)
except IOError:
log.debug(
"Failed to read frame length, assuming connection died")
self.quit = True
break
if len(ch) == 0:
log.debug("Input was closed")
self.quit = True
break
if ch == "{":
size = int(buf, 10)
try:
buf = ch + self.fd_in.read(
size - 1) # exclude already-read {
except IOError:
log.debug(
"Failed to read frame data, assuming connection died")
self.quit = True
break
try:
buf = buf.decode("utf-8")
except UnicodeDecodeError:
pass # what :(
try:
data = json.loads(buf)
request = Request(data)
except Exception as e:
log.exception(e)
self.exception(message=e.message, request=None)
continue
finally:
buf = ""
if request.get("command") == "abort":
self.do_abort(request=request)
else:
log.debug("queuing request %r", request)
with self.queue_cv:
self.queue.appendleft(request)
self.queue_cv.notify()
elif ch in "0123456789":
buf += ch
else:
raise ValueError("Invalid request data: " + ch.encode("hex"))
def run(self):
"""Evaluate and send results back"""
log.info("Running codeintel eval thread...")
buf = ""
log.debug("default supported commands: %s",
", ".join(self._default_handler.supportedCommands))
while True:
with self.queue_cv:
try:
request = self.queue.pop()
except IndexError:
self.queue_cv.wait()
continue
log.debug("doing request %r", request)
try:
self.active_request = request
command = request.command
# First, check abort and quit; don't allow those to be
# overridden
try:
builtin = self._builtin_commands[command]
except KeyError:
pass
else:
builtin(request)
continue
handlers = self._command_handler_map.get(command, [])[:]
if command in self._default_handler.supportedCommands:
# The default handler can deal with this, put it at the end
handlers.append(self._default_handler)
for handler in handlers:
if handler.canHandleRequest(request):
handler.handleRequest(request, self)
break
else:
self.fail(request=request,
msg="Don't know how to handle command %s" % (command,))
except RequestFailure as e:
self.fail(request=request, *e.args, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
finally:
self.active_request = None
@classmethod
def registerCommandHandler(cls, handlerInstance):
"""Register a command handler"""
for command in handlerInstance.supportedCommands:
cls._command_handler_map[command].append(handlerInstance)
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if opts.time_it:
start = time.time()
quiet = opts.quiet
if opts.time_it or opts.time_details:
opts.force = True
tree = None
for path in _paths_from_path_patterns(path_patterns,
recursive=opts.recursive,
includes=opts.includes):
if opts.time_it:
sys.stderr.write(path+"\n")
if opts.time_details:
start1 = time.time()
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info("skip `%s': couldn't determine language", path)
continue
buf = mgr.buf_from_path(path, lang=lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
if opts.force:
buf.scan()
if tree is None:
tree = ET.Element("codeintel", version="2.0")
file_elem = ET.SubElement(tree, "file",
lang=buf.lang,
mtime=str(int(time.time())),
path=os.path.basename(path))
for lang, blob in sorted(buf.blob_from_lang.items()):
blob = buf.blob_from_lang[lang]
file_elem.append(blob)
if opts.time_details:
delta = time.time() - start1
sys.stderr.write("%.3f %s\n" % (delta, path))
sys.stderr.flush()
if tree is not None:
if opts.stripfuncvars:
# For stdlibs, we don't care about variables inside of
# functions and they take up a lot of space.
for function in tree.getiterator('scope'):
if function.get('ilk') == 'function':
function[:] = [child for child in function
if child.tag != 'variable']
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
if not quiet:
sys.stdout.write('<?xml version="1.0" encoding="UTF-8"?>\n')
ET.dump(tree)
if opts.time_it:
end = time.time()
sys.stderr.write("scan took %.3fs\n" % (end-start))
finally:
mgr.finalize()
def cix2html(opts, path):
"""Turn cix file into html API documentation.
Example:
cix2html path/to/foo.cix#AClass.amethod
cix2html path/to/foo.cix -o file.html
${cmd_usage}
${cmd_option_list}
"""
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
def blobs_from_tree(tree):
for file_elem in tree:
for blob in file_elem:
yield blob
if '#' in path:
path, anchor = path.rsplit('#', 1)
else:
anchor = None
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
# buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
tree = buf.tree
if anchor is not None:
# Lookup the anchor in the codeintel CIX tree.
lpath = re.split(r'\.|::', anchor)
for elem in blobs_from_tree(tree):
# Generally have 3 types of codeintel trees:
# 1. single-lang file: one <file>, one <blob>
# 2. multi-lang file: one <file>, one or two <blob>'s
# 3. CIX stdlib/catalog file: possibly multiple
# <file>'s, likely multiple <blob>'s
# Allow the first token to be the blob name or lang.
# (This can sometimes be weird, but seems the most
# convenient solution.)
if lpath[0] in (elem.get("name"), elem.get("lang")):
remaining_lpath = lpath[1:]
else:
remaining_lpath = lpath
for name in remaining_lpath:
try:
elem = elem.names[name]
except KeyError:
elem = None
break # try next lang blob
if elem is not None:
break # found one
else:
log.error("could not find `%s' definition (or blob) in `%s'",
anchor, path)
return 1
else:
elem = tree
try:
if elem.tag == "codeintel":
_html_ci_elem(opts, elem.getchildren()[0])
else:
_html_ci_elem(opts, elem)
except IOError, ex:
if ex.errno == 0:
# Ignore this error from aborting 'less' of 'ci2 outline'
# output:
# IOError: (0, 'Error')
pass
else:
raise
except Exception, e:
import traceback
traceback.print_exc()
class Driver(threading.Thread):
"""
Out-of-process codeintel2 driver
This class implements the main loop of the codeintel worker process
This should be a singleton object
"""
# static
_command_handler_map = collections.defaultdict(list)
"""Registered command handlers"""
_default_handler = None
"""The default (core) command handler; instance of a CoreHandler"""
_builtin_commands = {}
"""Built-in commands that cannot be overridden"""
def __init__(self, db_base_dir=None, fd_in=sys.stdin, fd_out=sys.stdout):
threading.Thread.__init__(self, name="Codeintel OOP Driver")
self.daemon = True
self.fd_in = fd_in
self.fd_out = fd_out
self.abort = None
self.quit = False
self.buffers = {} # path to Buffer objects
self.next_buffer = 0
self.active_request = None
self.queue = collections.deque()
self.queue_cv = threading.Condition()
self.env = Environment(name="global",
send_fn=functools.partial(self.send,
request=None))
# Fill out the non-overridable build-in commands
self._builtin_commands = {}
for attr in dir(self):
# Note that we check startswith first to avoid getters etc.
if attr.startswith("do_") and callable(getattr(self, attr)):
command = attr[len("do_"):].replace("_", "-")
self._builtin_commands[command] = getattr(self, attr)
from codeintel2.manager import Manager
log.debug("using db base dir %s", db_base_dir)
self.mgr = Manager(db_base_dir=db_base_dir,
db_catalog_dirs=[],
env=self.env,
on_scan_complete=self._on_scan_complete)
self.mgr.initialize()
def _on_scan_complete(self, scan_request):
if scan_request.status == "changed":
# Send unsolicited response about the completed scan
buf = scan_request.buf
self.send(request=None,
path=buf.path,
language=buf.lang,
command="scan-complete")
REQUEST_DEFAULT = object()
def send(self, request=REQUEST_DEFAULT, **kwargs):
"""
Send a response
"""
data = dict(kwargs)
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if request:
data["req_id"] = request.id
if "success" not in data:
data["success"] = True
elif data["success"] is None:
del data["success"]
buf = json.dumps(data, separators=(',', ':'))
log.debug("sending: [%s]", buf)
self.fd_out.write(str(len(buf)) + buf)
def fail(self, request=REQUEST_DEFAULT, **kwargs):
kwargs = kwargs.copy()
if not "command" in kwargs and request:
try:
kwargs["command"] = request["command"]
except KeyError:
pass
return self.send(request=request, success=False, **kwargs)
def exception(self, request=REQUEST_DEFAULT, **kwargs):
return self.fail(request=request,
stack=traceback.format_exc(),
**kwargs)
def get_buffer(self, request=REQUEST_DEFAULT, path=None):
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if path is None:
if not "path" in request:
raise RequestFailure(message="No path given to locate buffer")
path = request.path
if abspath(path) == path:
# this is an actual file path, not a URL or whatever
path = normcase(normpath(path))
try:
buf = self.buffers[path]
except KeyError:
buf = None
else:
if "language" in request and buf.lang != request.language:
buf = None # language changed, re-scan
if not buf:
# Need to construct a new buffer
lang = request.get("language")
if request.get("text") is not None:
# pass no content; we'll reset it later
buf = self.mgr.buf_from_content("", lang, path=path)
else:
# read from file
try:
buf = self.mgr.buf_from_path(
path, lang, encoding=request.get("encoding"))
except OSError:
# Can't read the file
buf = self.mgr.buf_from_content("", lang, path=path)
assert not request.path.startswith("<")
if request.get("text") is not None:
# overwrite the buffer contents if we have new ones
buf.accessor.reset_content(request.text)
try:
env = request["env"]
except KeyError:
pass # no environment set, use current environment
else:
if env.get("env", {}) is None and env.get("prefs", []) is None:
buf._env = None # explicitly clearing environment
elif buf._env:
buf._env.update(env)
else:
buf._env = Environment(env, name=os.path.basename(path))
# log.debug("Got buffer %r: [%s]", buf, buf.accessor.content)
log.debug("Got buffer %r", buf)
self.buffers[path] = buf
return buf
def do_abort(self, request):
try:
req_id = request["id"]
with self.queue_cv:
for item in self.queue:
if queue.get("req_id") == req_id:
self.queue.remove(queue)
self.send(request=request)
break
else:
self.abort = req_id
if self.active_request and self.active_request.id == req_id:
# need to wait a bit...
self.send(request=request)
else:
self.fail(request=request,
message="Request %s not found" % (req_id, ))
except RequestFailure as e:
self.fail(request=request, *e.args, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
def do_add_dirs(self, request):
catalog_dirs = request.get("catalog-dirs", None)
if catalog_dirs is not None:
self.mgr.db.catalog_dirs.extend(catalog_dirs)
lexer_dirs = request.get("lexer-dirs", [])
codeintel2.udl.UDLLexer.add_extra_lexer_dirs(lexer_dirs)
module_dirs = request.get("module-dirs", [])
if module_dirs:
self.mgr._register_modules(module_dirs)
if catalog_dirs is not None:
self.mgr.db.get_catalogs_zone().update()
def do_load_extension(self, request):
"""Load an extension that, for example, might provide additional
command handlers"""
path = request.get("module-path", None)
if not path:
raise RequestFailure(msg="load-extension requires a module-path")
name = request.get("module-name", None)
if not name:
raise RequestFailure(msg="load-extension requires a module-name")
iinfo = imp.find_module(name, [path])
try:
module = imp.load_module(name, *iinfo)
finally:
if iinfo and iinfo[0]:
iinfo[0].close()
callback = getattr(module, "registerExtension", None)
if not callback:
raise RequestFailure(msg="load-extension module %s should "
"have a 'registerExtension' method "
"taking no arguments" % (name, ))
callback()
self.send() # success, finally
def do_quit(self, request):
self.quit = True
self.send(command="quit")
def report_error(self, message):
self.send(request=None,
command="report-error",
message=unicode(message))
def start(self):
"""Start reading from the socket and dump requests into the queue"""
log.info("Running codeintel driver...")
buf = ""
self.send(success=None)
self.daemon = True
threading.Thread.start(self)
while not self.quit:
try:
ch = self.fd_in.read(1)
except IOError:
log.debug(
"Failed to read frame length, assuming connection died")
self.quit = True
break
if len(ch) == 0:
log.debug("Input was closed")
self.quit = True
break
if ch == "{":
size = int(buf, 10)
try:
buf = ch + self.fd_in.read(
size - 1) # exclude already-read {
except IOError:
log.debug(
"Failed to read frame data, assuming connection died")
self.quit = True
break
try:
buf = buf.decode("utf-8")
except UnicodeDecodeError:
pass # what :(
try:
data = json.loads(buf)
request = Request(data)
except Exception as e:
log.exception(e)
self.exception(message=e.message, request=None)
continue
finally:
buf = ""
if request.get("command") == "abort":
self.do_abort(request=request)
else:
log.debug("queuing request %r", request)
with self.queue_cv:
self.queue.appendleft(request)
self.queue_cv.notify()
elif ch in "0123456789":
buf += ch
else:
raise ValueError("Invalid request data: " + ch.encode("hex"))
def run(self):
"""Evaluate and send results back"""
log.info("Running codeintel eval thread...")
buf = ""
log.debug("default supported commands: %s",
", ".join(self._default_handler.supportedCommands))
while True:
with self.queue_cv:
try:
request = self.queue.pop()
except IndexError:
self.queue_cv.wait()
continue
log.debug("doing request %r", request)
try:
self.active_request = request
command = request.command
# First, check abort and quit; don't allow those to be
# overridden
try:
builtin = self._builtin_commands[command]
except KeyError:
pass
else:
builtin(request)
continue
handlers = self._command_handler_map.get(command, [])[:]
if command in self._default_handler.supportedCommands:
# The default handler can deal with this, put it at the end
handlers.append(self._default_handler)
for handler in handlers:
if handler.canHandleRequest(request):
handler.handleRequest(request, self)
break
else:
self.fail(request=request,
msg="Don't know how to handle command %s" %
(command, ))
except RequestFailure as e:
self.fail(request=request, *e.args, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
finally:
self.active_request = None
@classmethod
def registerCommandHandler(cls, handlerInstance):
"""Register a command handler"""
for command in handlerInstance.supportedCommands:
cls._command_handler_map[command].append(handlerInstance)
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
import time
import ciElementTree as ET
from ci2 import _paths_from_path_patterns
from codeintel2.manager import Manager
from codeintel2.citadel import CitadelBuffer
from codeintel2.common import CodeIntelError
from codeintel2.tree import pretty_tree_from_tree
from codeintel2.util import guess_lang_from_path
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if opts.time_it:
start = time.time()
quiet = opts.quiet
if opts.time_it or opts.time_details:
opts.force = True
scan_count = 0
lang_warnings = set()
tree = None
for path in _paths_from_path_patterns(path_patterns,
recursive=opts.recursive,
includes=opts.includes):
if opts.time_it:
sys.stderr.write(path + "\n")
if opts.time_details:
start1 = time.time()
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
self.log.info("skip `%s': couldn't determine language", path)
continue
try:
buf = mgr.buf_from_path(path, lang=lang)
except OSError as ex:
# Couldn't access the file.
if not opts.recursive:
raise
# Ignore files we don't really care about.
self.log.warn("%r - %r", ex, path)
continue
if not isinstance(buf, CitadelBuffer):
if opts.recursive:
# Ignore files that scanning isn't provided for.
continue
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
scan_count += 1
if scan_count % 10 == 0:
self.log.info("%d scanning %r", scan_count, path)
try:
if opts.force:
buf.scan()
if tree is None:
tree = ET.Element("codeintel", version="2.0")
file_elem = ET.SubElement(tree, "file",
lang=buf.lang,
mtime=str(int(time.time())),
path=os.path.basename(path))
for lang, blob in sorted(buf.blob_from_lang.items()):
blob = buf.blob_from_lang[lang]
file_elem.append(blob)
except KeyError as ex:
# Unknown cile language.
if not opts.recursive:
raise
message = str(ex)
if message not in lang_warnings:
lang_warnings.add(message)
self.log.warn("Skipping unhandled language %s", message)
if opts.time_details:
delta = time.time() - start1
sys.stderr.write("%.3f %s\n" % (delta, path))
sys.stderr.flush()
if tree is not None:
if opts.stripfuncvars:
# For stdlibs, we don't care about variables inside of
# functions and they take up a lot of space.
for function in tree.getiterator('scope'):
if function.get('ilk') == 'function':
function[:] = [child for child in function
if child.tag != 'variable']
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
if not quiet:
sys.stdout.write('<?xml version="1.0" encoding="UTF-8"?>\n')
ET.dump(tree)
if opts.time_it:
end = time.time()
sys.stderr.write("scan took %.3fs\n" % (end - start))
finally:
mgr.finalize()
class Driver(threading.Thread):
"""
Out-of-process codeintel2 driver
This class implements the main loop of the codeintel worker process
This should be a singleton object
"""
# static
_instance = None
_command_handler_map = collections.defaultdict(list)
"""Registered command handlers; the key is the command name (a str), and the
value is a list of command handler instances or (for lazy handlers) a
single-element tuple containing the callable to get the real handler."""
_default_handler = None
"""The default (core) command handler; instance of a CoreHandler"""
_builtin_commands = {}
"""Built-in commands that cannot be overridden"""
def __init__(self, db_base_dir=None, fd_in=sys.stdin, fd_out=sys.stdout):
threading.Thread.__init__(self, name="CodeIntel OOP Driver")
assert Driver._instance is None, "Driver should be a singleton"
Driver._instance = self
logging.root.addHandler(LoggingHandler(self))
self.daemon = True
self.fd_in = fd_in
self.fd_out = fd_out
self.abort = None
self.quit = False
self.buffers = {} # path to Buffer objects
self.next_buffer = 0
self.active_request = None
self.send_queue = Queue.Queue()
self.send_thread = threading.Thread(name="Codeintel OOP Driver Send Thread",
target=self._send_proc)
self.send_thread.daemon = True
self.send_thread.start()
self.queue = collections.deque()
self.queue_cv = threading.Condition()
self.env = Environment(name="global",
send_fn=functools.partial(self.send, request=None))
# Fill out the non-overridable build-in commands
self._builtin_commands = {}
for attr in dir(self):
# Note that we check startswith first to avoid getters etc.
if attr.startswith("do_") and callable(getattr(self, attr)):
command = attr[len("do_"):].replace("_", "-")
self._builtin_commands[command] = getattr(self, attr)
from codeintel2.manager import Manager
log.debug("using db base dir %s", db_base_dir)
self.mgr = Manager(db_base_dir=db_base_dir,
db_catalog_dirs=[],
db_event_reporter=self._DBEventReporter(self),
env=self.env,
on_scan_complete=self._on_scan_complete)
self.mgr.initialize()
def _on_scan_complete(self, scan_request):
if scan_request.status in ("changed", "skipped"):
# Send unsolicited response about the completed scan
buf = scan_request.buf
self.send(request=None,
path=buf.path,
language=buf.lang,
command="scan-complete")
class _DBEventReporter(object):
def __init__(self, driver):
self.driver = driver
self.log = log.getChild("DBEventReporter")
self.debug = self.log.debug
# directories being scanned (completed or not)
# key is unicode path, value is number of times it's active
self._dirs = collections.defaultdict(int)
# directories which are complete (unicode path)
self._completed_dirs = set()
def send(self, **kwargs):
self.driver.send(request=None, command="report-message", **kwargs)
def __call__(self, message):
"""Old-style status messages before long-running jobs
@param msg {str or None} The message to display
"""
if len(self._dirs):
return # ignore old-style db events if we're doing a scan
self.debug("db event: %s", message)
self.send(message=message)
def onScanStarted(self, description, dirs=set()):
"""Called when a directory scan is about to start
@param description {unicode} A string suitable for showing the user
about the upcoming operation
@param dirs {set of unicode} The directories about to be scanned
"""
self.debug("scan started: %s (%s dirs)", description, len(dirs))
assert dirs, "onScanStarted expects non-empty directories"
if not dirs: # empty set - we shouldn't have gotten here, but be nice
return
for dir_path in dirs:
self._dirs[dir_path] += 1
self.send(type="scan-progress", message=description,
completed=len(self._completed_dirs),
total=len(self._dirs))
def onScanDirectory(self, description, dir_path, current=None, total=None):
"""Called when a directory is being scanned (out of possibly many)
@param description {unicode} A string suitable for showing the user
regarding the progress
@param dir {unicode} The directory currently being scanned
@param current {int} The current progress
@param total {int} The total number of directories to scan in this
request
"""
self.debug("scan directory: %s (%s %s/%s)",
description, dir_path, current, total)
assert dir_path, "onScanDirectory got no directory"
if dir_path:
self._completed_dirs.add(dir_path)
self.send(type="scan-progress", message=description,
completed=len(self._completed_dirs),
total=len(self._dirs))
def onScanComplete(self, dirs=set(), scanned=set()):
"""Called when a scan operation is complete
@param dirs {set of unicode} The directories that were intially
requested to be scanned (as pass in onScanStarted)
@param scanned {set of unicode} Directories which were successfully
scanned. This may be a subset of dirs if the scan was
aborted.
"""
self.debug("scan complete: scanned %r/%r dirs",
len(scanned), len(dirs))
for dir_path in dirs:
self._dirs[dir_path] -= 1
if not self._dirs[dir_path]:
del self._dirs[dir_path]
self._completed_dirs.discard(dir_path)
self.send(type="scan-progress", completed=len(self._completed_dirs),
total=len(self._dirs))
REQUEST_DEFAULT = object()
def send(self, request=REQUEST_DEFAULT, **kwargs):
"""
Send a response
"""
data = dict(kwargs)
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if request:
data["req_id"] = request.id
if "success" not in data:
data["success"] = True
elif data["success"] is None:
del data["success"]
buf = json.dumps(data, separators=(',',':'))
buf_len = str(len(buf))
log.debug("sending: %s:[%s]", buf_len, buf)
self.send_queue.put(buf)
def _send_proc(self):
while True:
buf = self.send_queue.get()
try:
buf_len = str(len(buf))
self.fd_out.write(buf_len)
self.fd_out.write(buf)
finally:
self.send_queue.task_done()
def fail(self, request=REQUEST_DEFAULT, **kwargs):
kwargs = kwargs.copy()
if not "command" in kwargs and request:
try:
kwargs["command"] = request["command"]
except KeyError:
pass
return self.send(request=request, success=False, **kwargs)
def exception(self, request=REQUEST_DEFAULT, **kwargs):
kwargs["command"] = "report-error"
kwargs["type"] = "logging"
kwargs["name"] = log.name
kwargs["level"] = logging.ERROR
return self.fail(request=request, stack=traceback.format_exc(), **kwargs)
@staticmethod
def normpath(path):
"""Routine to normalize the path used for codeintel buffers
This is annoying because it needs to handle unsaved files, as well as
urls.
@note See also koCodeIntel.py::KoCodeIntelBuffer.normpath
"""
if path.startswith("<Unsaved>"):
return path # Don't munge unsaved file paths
scheme = path.split(":", 1)[0]
if len(scheme) == len(path):
# didn't find a scheme at all; assume this is a local path
return os.path.normcase(path)
if len(scheme) == 1:
# single-character scheme; assume this is actually a drive letter
# (for a Windows-style path)
return os.path.normcase(path)
scheme_chars = string.ascii_letters + string.digits + "-"
try:
scheme = scheme.encode("ascii")
except UnicodeEncodeError:
# scheme has a non-ascii character; assume local path
return os.path.normcase(path)
if scheme.translate(None, scheme_chars):
# has a non-scheme character: this is not a valid scheme
# assume this is a local file path
return os.path.normcase(path)
if scheme != "file":
return path # non-file scheme
path = path[len(scheme) + 1:]
return os.path.normcase(path)
def get_buffer(self, request=REQUEST_DEFAULT, path=None):
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if path is None:
if not "path" in request:
raise RequestFailure(message="No path given to locate buffer")
path = request.path
path = self.normpath(path)
try:
buf = self.buffers[path]
except KeyError:
buf = None
else:
if "language" in request and buf.lang != request.language:
buf = None # language changed, re-scan
if not buf:
# Need to construct a new buffer
lang = request.get("language")
env = Environment(request=request, name=os.path.basename(path))
if request.get("text") is not None:
# pass no content; we'll reset it later
buf = self.mgr.buf_from_content("", lang, path=path, env=env)
else:
# read from file
try:
buf = self.mgr.buf_from_path(path, lang, env=env,
encoding=request.get("encoding"))
except OSError:
# Can't read the file
buf = self.mgr.buf_from_content("", lang, path=path, env=env)
assert not request.path.startswith("<"), \
"Can't create an unsaved buffer with no text"
if request.get("text") is not None:
# overwrite the buffer contents if we have new ones
buf.accessor.reset_content(request.text)
buf.encoding = "utf-8"
try:
env = request["env"]
except KeyError:
pass # no environment, use current
else:
buf._env.update(env)
#log.debug("Got buffer %r: [%s]", buf, buf.accessor.content)
log.debug("Got buffer %r", buf)
self.buffers[path] = buf
return buf
def do_abort(self, request):
try:
req_id = request["id"]
with self.queue_cv:
for item in self.queue:
if item.get("req_id") == req_id:
self.queue.remove(item)
self.send(request=request)
break
else:
self.abort = req_id
if self.active_request and self.active_request.id == req_id:
# need to wait a bit...
self.send(request=request)
else:
self.fail(request=request,
message="Request %s not found" % (req_id,))
except RequestFailure as e:
self.fail(request=request, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
def do_add_dirs(self, request):
catalog_dirs = request.get("catalog-dirs", None)
if catalog_dirs is not None:
self.mgr.db.catalog_dirs.extend(catalog_dirs)
catalog_dirs = self.mgr.db.catalog_dirs
lexer_dirs = request.get("lexer-dirs", [])
codeintel2.udl.UDLLexer.add_extra_lexer_dirs(lexer_dirs)
module_dirs = request.get("module-dirs", [])
if module_dirs:
self.mgr._register_modules(module_dirs)
if catalog_dirs is not None:
self.mgr.db.get_catalogs_zone().catalog_dirs = catalog_dirs
self.send(request=request)
def do_load_extension(self, request):
"""Load an extension that, for example, might provide additional
command handlers"""
name = request.get("module-name", None)
if not name:
raise RequestFailure(msg="load-extension requires a module-name")
path = request.get("module-path", None)
names = name.split(".")
if len(names) > 1:
# It's inside a package - go look for the package(s).
for package_name in names[:-1]:
iinfo = imp.find_module(package_name, [path] if path else None)
if not iinfo:
raise RequestFailure(msg="load-extension could not find "
"package %r for given name %r"
% (package_name, name))
path = iinfo[1]
name = names[-1]
iinfo = imp.find_module(name, [path] if path else None)
try:
module = imp.load_module(name, *iinfo)
finally:
if iinfo and iinfo[0]:
iinfo[0].close()
callback = getattr(module, "registerExtension", None)
if not callback:
raise RequestFailure(msg="load-extension module %s should "
"have a 'registerExtension' method "
"taking no arguments" % (name,))
callback()
self.send() # success, finally
def do_quit(self, request):
self.quit = True
self.send(command="quit")
def report_error(self, message):
self.send(request=None,
command="report-error",
message=unicode(message))
def start(self):
"""Start reading from the socket and dump requests into the queue"""
log.info("Running codeintel driver...")
buf = ""
self.send(success=None)
self.daemon = True
threading.Thread.start(self)
while not self.quit:
try:
ch = self.fd_in.read(1)
except IOError:
log.debug("Failed to read frame length, assuming connection died")
self.quit = True
break
if len(ch) == 0:
log.debug("Input was closed")
self.quit = True
break
if ch == "{":
size = int(buf, 10)
try:
buf = ch + self.fd_in.read(size - 1) # exclude already-read {
except IOError:
log.debug("Failed to read frame data, assuming connection died")
self.quit = True
break
try:
data = json.loads(buf)
request = Request(data)
except Exception as e:
log.exception(e)
self.exception(message=e.message, request=None)
continue
finally:
buf = ""
if request.get("command") == "abort":
self.do_abort(request=request)
else:
log.debug("queuing request %r", request)
with self.queue_cv:
self.queue.appendleft(request)
self.queue_cv.notify()
elif ch in "0123456789":
buf += ch
else:
raise ValueError("Invalid request data: " + ch.encode("hex"))
def run(self):
"""Evaluate and send results back"""
log.info("Running codeintel eval thread...")
buf = ""
log.debug("default supported commands: %s",
", ".join(self._default_handler.supportedCommands))
while True:
with self.queue_cv:
try:
request = self.queue.pop()
except IndexError:
self.queue_cv.wait()
continue
log.debug("doing request %r", request)
try:
self.active_request = request
command = request.command
# First, check abort and quit; don't allow those to be overridden
try:
builtin = self._builtin_commands[command]
except KeyError:
pass
else:
builtin(request)
continue
handlers = self._command_handler_map.get(command, [])[:]
if command in self._default_handler.supportedCommands:
# The default handler can deal with this, put it at the end
handlers.append(self._default_handler)
for handler in handlers:
if isinstance(handler, tuple):
try:
real_handler = handler[0]()
except Exception as ex:
log.exception("Failed to get lazy handler for %s", command)
real_handler = None
if real_handler is None:
# Handler failed to instantiate, drop it
try:
self._command_handler_map[command].remove(handler)
except ValueError:
pass # ... shouldn't happen, but tolerate it
continue
for handlers in self._command_handler_map.values():
try:
handlers[handlers.index(handler)] = real_handler
except ValueError:
pass # handler not in this list
handler = real_handler
if handler.canHandleRequest(request):
handler.handleRequest(request, self)
break
else:
self.fail(request=request,
message="Don't know how to handle command %s" % (command,))
except RequestFailure as e:
self.fail(request=request, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
finally:
self.active_request = None
@classmethod
def registerCommandHandler(cls, handlerInstance):
"""Register a command handler"""
for command in handlerInstance.supportedCommands:
cls._command_handler_map[command].append(handlerInstance)
@classmethod
def registerLazyCommandHandler(cls, supported_commands, constructor):
"""Register a lazy command handler
@param supported_commands {iterable} The commands to handle; each
element should be a str of the command name.
@param constructor {callable} Function to be called to get the real
command handler; it should take no arguments and return a command
handler instance. It may return None if the command is not
available; it will not be asked again.
"""
for command in supported_commands:
cls._command_handler_map[command].append((constructor,))
@classmethod
def getInstance(cls):
"""Get the singleton instance of the driver"""
return Driver._instance
def do_outline(self, subcmd, opts, path):
"""Scan and outline the structure of the given path.
${cmd_usage}
${cmd_option_list}
"""
extra_lang_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_lang_module_dirs = [sys.path[-1]]
mgr = Manager(extra_lang_module_dirs=extra_lang_module_dirs)
mgr.upgrade()
mgr.initialize()
try:
if "#" in path:
path, anchor = path.rsplit("#", 1)
else:
anchor = None
tree = None
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info("skip `%s': couldn't determine language " "(use --lang option)", path)
return
if path.endswith(".cix"):
tree = tree_from_cix(open(path, "r").read())
# buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang))
tree = buf.tree
if anchor is not None:
# Lookup the anchor in the codeintel CIX tree.
lpath = re.split(r"\.|::", anchor)
def blobs_from_tree(tree):
for file_elem in tree:
for blob in file_elem:
yield blob
for elem in blobs_from_tree(tree):
# Generally have 3 types of codeintel trees:
# 1. single-lang file: one <file>, one <blob>
# 2. multi-lang file: one <file>, one or two <blob>'s
# 3. CIX stdlib/catalog file: possibly multiple
# <file>'s, likely multiple <blob>'s
# Allow the first token to be the blob name or lang.
# (This can sometimes be weird, but seems the most
# convenient solution.)
if lpath[0] in (elem.get("name"), elem.get("lang")):
remaining_lpath = lpath[1:]
else:
remaining_lpath = lpath
for name in remaining_lpath:
try:
elem = elem.names[name]
except KeyError:
elem = None
break # try next lang blob
if elem is not None:
break # found one
else:
log.error("could not find `%s' definition (or blob) in `%s'", anchor, path)
return 1
else:
elem = tree
try:
_outline_ci_elem(mgr, elem, quiet=opts.quiet)
except IOError, ex:
if ex.errno == 0:
# Ignore this error from aborting 'less' of this
# command:
# IOError: (0, 'Error')
pass
else:
raise
finally:
mgr.finalize()
