def codeintel_manager(folders_id):
folders_id = None
global _ci_mgr_, condeintel_log_filename, condeintel_log_file
mgr = _ci_mgr_.get(folders_id)
if mgr is None:
for thread in threading.enumerate():
if thread.name == "CodeIntel Manager":
thread.finalize() # this finalizes the index, citadel and the manager and waits them to end (join)
mgr = Manager(
extra_module_dirs=None,
db_base_dir=None, # os.path.expanduser(os.path.join('~', '.codeintel', 'databases', folders_id)),
db_catalog_dirs=[],
db_import_everything_langs=None,
)
mgr.upgrade()
mgr.initialize()
# Connect the logging file to the handler
condeintel_log_filename = os.path.join(mgr.db.base_dir, 'codeintel.log')
condeintel_log_file = open(condeintel_log_filename, 'w', 1)
codeintel_log.handlers = [logging.StreamHandler(condeintel_log_file)]
msg = "Starting logging SublimeCodeIntel v%s rev %s (%s) on %s" % (VERSION, get_revision()[:12], os.stat(__file__)[stat.ST_MTIME], datetime.datetime.now().ctime())
print("%s\n%s" % (msg, "=" * len(msg)), file=condeintel_log_file)
_ci_mgr_[folders_id] = mgr
return mgr
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
extra_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_module_dirs = [sys.path[-1]]
mgr = Manager(extra_module_dirs=extra_module_dirs)
mgr.upgrade()
mgr.initialize()
try:
tree = None
for path in _paths_from_path_patterns(path_patterns):
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info("skip `%s': couldn't determine language " "(use --lang option)", path)
continue
buf = mgr.buf_from_path(path, lang=opts.lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang))
buf.scan() # force a fresh scan
tree = buf.tree
for severity, msg in check_tree(tree):
dump = {"warning": log.warn, "error": log.error}[severity]
dump(msg)
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
ET.dump(tree)
finally:
mgr.finalize()
def codeintel_manager():
global _ci_mgr_, condeintel_log_filename, condeintel_log_file
if _ci_mgr_:
mgr = _ci_mgr_
else:
for thread in threading.enumerate():
if thread.name == "CodeIntel Manager":
thread.finalize() # this finalizes the index, citadel and the manager and waits them to end (join)
mgr = Manager(
extra_module_dirs=_ci_extra_module_dirs_,
db_base_dir=_ci_db_base_dir_,
db_catalog_dirs=_ci_db_catalog_dirs_,
db_import_everything_langs=_ci_db_import_everything_langs,
)
mgr.upgrade()
mgr.initialize()
# Connect the logging file to the handler
condeintel_log_filename = os.path.join(mgr.db.base_dir, "codeintel.log")
condeintel_log_file = open(condeintel_log_filename, "w", 1)
codeintel_log.handlers = [logging.StreamHandler(condeintel_log_file)]
msg = "Starting logging SublimeCodeIntel rev %s (%s) on %s" % (
get_revision()[:12],
os.stat(__file__)[stat.ST_MTIME],
datetime.datetime.now().ctime(),
)
print >> condeintel_log_file, "%s\n%s" % (msg, "=" * len(msg))
_ci_mgr_ = mgr
return mgr
def codeintel_manager():
global _ci_mgr_, condeintel_log_filename, condeintel_log_file
if _ci_mgr_:
mgr = _ci_mgr_
else:
for thread in threading.enumerate():
if thread.name == "CodeIntel Manager":
thread.finalize(
) # this finalizes the index, citadel and the manager and waits them to end (join)
mgr = Manager(
extra_module_dirs=_ci_extra_module_dirs_,
db_base_dir=_ci_db_base_dir_,
db_catalog_dirs=_ci_db_catalog_dirs_,
db_import_everything_langs=_ci_db_import_everything_langs,
)
mgr.upgrade()
mgr.initialize()
# Connect the logging file to the handler
condeintel_log_filename = os.path.join(mgr.db.base_dir,
'codeintel.log')
condeintel_log_file = open(condeintel_log_filename, 'w', 1)
codeintel_log.handlers = [logging.StreamHandler(condeintel_log_file)]
msg = "Starting logging SublimeCodeIntel rev %s (%s) on %s" % (
get_revision()[:12], os.stat(__file__)[stat.ST_MTIME],
datetime.datetime.now().ctime())
print >> condeintel_log_file, "%s\n%s" % (msg, "=" * len(msg))
_ci_mgr_ = mgr
return mgr
def codeintel_manager(folders_id):
folders_id = None
global _ci_mgr_, condeintel_log_filename, condeintel_log_file
mgr = _ci_mgr_.get(folders_id)
if mgr is None:
for thread in threading.enumerate():
if thread.name == "CodeIntel Manager":
thread.finalize() # this finalizes the index, citadel and the manager and waits them to end (join)
mgr = Manager(
extra_module_dirs=None,
db_base_dir=None, # os.path.expanduser(os.path.join('~', '.codeintel', 'databases', folders_id)),
db_catalog_dirs=[],
db_import_everything_langs=None,
)
mgr.upgrade()
mgr.initialize()
# Connect the logging file to the handler
condeintel_log_filename = os.path.join(mgr.db.base_dir, 'codeintel.log')
condeintel_log_file = open(condeintel_log_filename, 'w', 1)
codeintel_log.handlers = [logging.StreamHandler(condeintel_log_file)]
msg = "Starting logging SublimeCodeIntel v%s rev %s (%s) on %s" % (VERSION, get_revision()[:12], os.stat(__file__)[stat.ST_MTIME], datetime.datetime.now().ctime())
print >>condeintel_log_file, "%s\n%s" % (msg, "=" * len(msg))
_ci_mgr_[folders_id] = mgr
return mgr
def do_json(self, subcmd, opts, path):
"""Convert cix XML file into json format.
${cmd_usage}
${cmd_option_list}
"""
import json
if opts.output == '-':
output_path = None
output_file = sys.stdout
else:
if opts.output:
output_path = opts.output
else:
output_path = splitext(path)[0]+".json"
if exists(output_path):
if opts.force:
os.remove(output_path)
else:
raise Error("`%s' exists: use -f|--force option to "
"allow overwrite" % output_path)
output_file = open(output_path, 'w')
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
tree = buf.tree
result = {}
ci = result["codeintel"] = defaultdict(list)
def _elemToDict(parent, elem):
data = defaultdict(list)
name = elem.get("name")
if name is not None:
data["name"] = name
data["tag"] = elem.tag
for attr_name, attr in elem.attrib.items():
data[attr_name] = attr
parent["children"].append(data)
for child in elem:
_elemToDict(data, child)
for child in tree:
_elemToDict(ci, child)
json.dump(result, output_file, indent=2)
finally:
mgr.finalize()
def do_json(self, subcmd, opts, path):
"""Convert cix XML file into json format.
${cmd_usage}
${cmd_option_list}
"""
import json
if opts.output == '-':
output_path = None
output_file = sys.stdout
else:
if opts.output:
output_path = opts.output
else:
output_path = splitext(path)[0] + ".json"
if exists(output_path):
if opts.force:
os.remove(output_path)
else:
raise Error("`%s' exists: use -f|--force option to "
"allow overwrite" % output_path)
output_file = open(output_path, 'w')
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
tree = buf.tree
result = {}
ci = result["codeintel"] = defaultdict(list)
def _elemToDict(parent, elem):
data = defaultdict(list)
name = elem.get("name")
if name is not None:
data["name"] = name
data["tag"] = elem.tag
for attr_name, attr in elem.attrib.items():
data[attr_name] = attr
parent["children"].append(data)
for child in elem:
_elemToDict(data, child)
for child in tree:
_elemToDict(ci, child)
json.dump(result, output_file, indent=2)
finally:
mgr.finalize()
def casper_tests():
from codeintel2.manager import Manager
db_base_dir = join(dirname(__file__), "tmp", "db")
mgr = Manager(db_base_dir)
mgr.upgrade()
mgr.initialize()
try:
for testpath in testpaths():
buf = mgr.buf_from_path(testpath, lang="JavaScript")
# Ensure the test is up to date.
if buf.scan_time < os.stat(testpath).st_mtime:
buf.scan()
for test in casper_tests_from_citree(buf.tree):
yield test
finally:
mgr.finalize()
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
extra_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_module_dirs = [sys.path[-1]]
mgr = Manager(extra_module_dirs=extra_module_dirs)
mgr.upgrade()
mgr.initialize()
try:
tree = None
for path in _paths_from_path_patterns(path_patterns):
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info(
"skip `%s': couldn't determine language "
"(use --lang option)", path)
continue
buf = mgr.buf_from_path(path, lang=opts.lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
buf.scan() # force a fresh scan
tree = buf.tree
for severity, msg in check_tree(tree):
dump = {"warning": log.warn, "error": log.error}[severity]
dump(msg)
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
ET.dump(tree)
finally:
mgr.finalize()
class CodeIntelTestCase(unittest.TestCase):
# Subclasses can override this to have setUp pass these settings to the
# codeintel.Manager.
_ci_db_base_dir_ = None
_ci_db_catalog_dirs_ = []
_ci_db_import_everything_langs = None
_ci_env_prefs_ = None
# A test case can set this to false to have the setUp/tearDown *not*
# create a `self.mgr'.
_ci_test_setup_mgr_ = True
_ci_extra_module_dirs_ = None
def setUp(self):
if _xpcom_:
# The tests are run outside of Komodo. If run with PyXPCOM up
# parts codeintel will try to use the nsIDirectoryService and
# will query dirs only provided by nsXREDirProvider -- which
# isn't registered outside of Komodo (XRE_main() isn't called).
# The KoTestService provides a backup.
koTestSvc = components.classes["@activestate.com/koTestService;1"] \
.getService(components.interfaces.koITestService)
koTestSvc.init()
if self._ci_test_setup_mgr_:
env = None
if self._ci_env_prefs_ is not None:
env = SimplePrefsEnvironment(**self._ci_env_prefs_)
self.mgr = Manager(
extra_module_dirs=self._ci_extra_module_dirs_,
db_base_dir=self._ci_db_base_dir_ or test_db_base_dir,
db_catalog_dirs=self._ci_db_catalog_dirs_,
db_import_everything_langs=self._ci_db_import_everything_langs,
env=env)
self.mgr.upgrade()
self.mgr.initialize()
def tearDown(self):
if self._ci_test_setup_mgr_:
self.mgr.finalize()
self.mgr = None
def adjust_content(self, content):
"""A hook for subclasses to modify markedup_content before use in
test cases. This is useful for sharing test cases between pure-
and multi-lang uses of a given lang.
"""
return content
def adjust_pos(self, pos):
"""A accompanying hook for `adjust_content' to adjust trigger
pos values accordingly.
"""
return pos
def _get_buf_and_data(self, markedup_content, lang, path=None, env=None):
if path is None:
# Try to ensure no accidental re-use of the same buffer name
# across the whole test suite. Also try to keep the buffer
# names relatively short (keeps tracebacks cleaner).
name = "buf-" + md5(markedup_content).hexdigest()[:16]
path = os.path.join("<Unsaved>", name)
content, data = unmark_text(self.adjust_content(markedup_content))
#print banner(path)
#sys.stdout.write(content)
#print banner(None)
buf = self.mgr.buf_from_content(content, lang=lang, path=path, env=env)
return buf, data
def _get_buf_and_trg(self,
markedup_content,
lang,
path=None,
implicit=True,
env=None):
buf, data = self._get_buf_and_data(markedup_content, lang, path, env)
trg = buf.trg_from_pos(data["pos"], implicit=implicit)
return buf, trg
def assertCITDLExprUnderPosIs(self,
markedup_content,
citdl_expr,
lang=None,
prefix_filter=None,
implicit=True,
trigger_name=None,
**fields):
"""Assert that the CITDL expression at the current position
is as expected.
This uses buf.citdl_expr_under_pos() -- or, for Perl,
buf.citdl_expr_and_prefix_filter_from_trg().
Note: This API is a mess right now. C.f. bug 65776.
The "prefix_filter" optional argument can be used for Perl to test
the value its relevant function returns.
"""
if lang is None:
lang = self.lang
content, data = unmark_text(self.adjust_content(markedup_content))
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
langintel = self.mgr.langintel_from_lang(lang)
if trigger_name is None:
trigger_name = "fakey-completion-type"
if lang == "Perl":
trg = Trigger(lang,
TRG_FORM_DEFN,
trigger_name,
data["pos"],
implicit=implicit,
length=0,
**fields)
actual_citdl_expr, actual_prefix_filter \
= langintel.citdl_expr_and_prefix_filter_from_trg(buf, trg)
else:
#actual_citdl_expr = langintel.citdl_expr_under_pos(buf, data["pos"])
trg = Trigger(lang,
TRG_FORM_DEFN,
trigger_name,
data["pos"],
implicit=implicit,
**fields)
actual_citdl_expr = langintel.citdl_expr_from_trg(buf, trg)
self.assertEqual(
actual_citdl_expr, citdl_expr,
"unexpected actual %s CITDL expr under pos:\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s" %
(lang, citdl_expr, actual_citdl_expr, indent(markedup_content)))
if prefix_filter is not None:
XXX #TODO: compare prefix_filter to given value
def _assertDefnMatches(self, buf, pos, lang=None, **fields):
ctlr = _CaptureEvalController()
trg = buf.defn_trg_from_pos(pos)
defns = buf.defns_from_trg(trg, ctlr=ctlr)
if not defns:
self.fail(
"unexpectedly did not find a definition in %r at pos %d\n"
" eval log\n%s\n"
" buffer:\n%s" % (buf, pos,
indent('\n'.join('%5s: %s' % (lvl, m)
for lvl, m in ctlr.log)),
indent(buf.accessor.text)))
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
defn = defns[0]
for name, value in fields.items():
try:
actual_value = getattr(defn, name)
except AttributeError:
actual_value = None
self.assertEqual(
actual_value, value,
"%s definition, unexpected value for field %r\n"
" defn: %r\n"
" expected: %r\n"
" got: %r\n"
" eval log\n%s\n"
" buffer:\n%s" % (buf.lang, name, defn, value, actual_value,
indent('\n'.join('%5s: %s' % (lvl, m)
for lvl, m in ctlr.log)),
indent(buf.accessor.text)))
def assertDefnMatches(self, markedup_content, lang=None, **fields):
if lang is None:
lang = self.lang
buf, data = self._get_buf_and_data(markedup_content, lang)
self._assertDefnMatches(buf, data["pos"], lang, **fields)
def assertDefnMatches2(self, buf, pos, lang=None, **fields):
self._assertDefnMatches(buf, pos, lang, **fields)
def assertNoDuplicateDefns(self, markedup_content, lang=None):
if lang is None:
lang = self.lang
buf, data = self._get_buf_and_data(markedup_content, lang)
self.assertNoDuplicateDefns2(buf, data["pos"])
def assertNoDuplicateDefns2(self, buf, pos):
markedup_content = markup_text(buf.accessor.text, pos=pos)
ctlr = _CaptureEvalController()
trg = buf.defn_trg_from_pos(pos)
actual_defns = buf.defns_from_trg(trg, ctlr=ctlr)
if not actual_defns:
self.fail("%s trigger resulted in no definitions when expecting "
"to check for duplicate definitions:\n%s" %
(buf.lang, indent(markedup_content)))
count_from_defn_repr = {}
for defn_repr in (repr(d) for d in actual_defns):
if defn_repr not in count_from_defn_repr:
count_from_defn_repr[defn_repr] = 0
count_from_defn_repr[defn_repr] += 1
defn_dupes = [(count, defn_repr)
for defn_repr, count in count_from_defn_repr.items()
if count > 1]
self.failIf(
defn_dupes,
"unexpectedly got duplicate completions at the given position\n"
" duplicates:\n%s\n"
" eval log\n%s\n"
" buffer:\n%s" % (indent('\n'.join('%d of %s' % d
for d in defn_dupes)),
indent('\n'.join('%5s: %s' % (lvl, m)
for lvl, m in ctlr.log)),
indent(markedup_content)))
def assertTriggerMatches(self,
markedup_content,
lang=None,
implicit=True,
env=None,
**fields):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
if trg is None:
self.fail("unexpectedly did not find a %s trigger, buffer:\n%s" %
(lang, indent(markedup_content)))
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
for name, value in fields.items():
try:
actual_value = getattr(trg, name)
except AttributeError:
actual_value = trg.extra[name]
self.assertEqual(
actual_value, value,
"unexpected %s trigger '%s' value: expected %r, "
"got %r, buffer:\n%s" %
(lang, name, value, actual_value, indent(markedup_content)))
# Used when a position generates a trigger, but it's not the one specified
def assertTriggerDoesNotMatch(self,
markedup_content,
lang=None,
implicit=True,
env=None,
**fields):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
if trg is None:
# No trigger is as good as
return
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
for name, value in fields.items():
try:
actual_value = getattr(trg, name)
except AttributeError:
actual_value = trg.extra[name]
self.assertNotEqual(
actual_value, value,
"unexpected %s trigger '%s' value: expected not %r, "
"got %r, buffer:\n%s" %
(lang, name, value, actual_value, indent(markedup_content)))
def assertNoTrigger(self,
markedup_content,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
if trg is not None:
self.fail("unexpectedly found a %s trigger %r when didn't expect "
"one, buffer:\n%s" %
(lang, trg, indent(markedup_content)))
def assertPrecedingTriggerMatches(self,
markedup_content,
lang=None,
**fields):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(self.adjust_content(markedup_content))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
trg = buf.preceding_trg_from_pos(data["start_pos"], data["pos"])
if trg is None:
self.fail("unexpectedly did not find a preceding %s trigger, "
"buffer:\n%s" % (lang, indent(markedup_content)))
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
for name, value in fields.items():
actual_value = getattr(trg, name)
self.assertEqual(
actual_value, value,
"unexpected preceding %s trigger '%s' value: expected %r, "
"got %r, buffer:\n%s" %
(lang, name, value, actual_value, indent(markedup_content)))
def assertNoPrecedingTrigger(self, markedup_content, lang=None):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(self.adjust_content(markedup_content))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
trg = buf.preceding_trg_from_pos(data["start_pos"], data["pos"])
if trg is not None:
self.fail("unexpectedly found a preceding %s trigger '%s' when "
"didn't expect one, buffer:\n%s" %
(lang, trg.name, indent(markedup_content)))
def assertScopeLpathIs(self, markedup_content, lpath, lang=None):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(self.adjust_content(markedup_content))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
buf.scan(skip_scan_time_check=True)
actual_blob, actual_lpath = buf.scoperef_from_pos(data["pos"])
self.failUnlessEqual(
lpath, actual_lpath,
"unexpected %s scope lookup path (lpath) at the given position\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s" %
(self.lang, lpath, actual_lpath, indent(markedup_content)))
def assertCompletionsAre2(self,
buf,
pos,
completions,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
markedup_content = markup_text(buf.accessor.text, pos=pos)
trg = buf.trg_from_pos(pos, implicit=implicit)
self._assertCompletionsAre(markedup_content, buf, trg, completions,
lang, implicit)
def assertCompletionsAre(self,
markedup_content,
completions,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
self._assertCompletionsAre(markedup_content, buf, trg, completions,
lang, implicit)
def _assertCompletionsAre(self, markedup_content, buf, trg, completions,
lang, implicit):
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected completions to be %r:\n%s" %
(lang, completions, indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
self.assertEqual(
completions, actual_completions,
"unexpected %s completions at the given position\n"
" expected: %r\n"
" got: %r\n"
" extra: %r\n"
" missing: %r\n"
" eval log\n%s\n"
" buffer:\n%s" %
(lang, completions, actual_completions,
list(set(actual_completions or []).difference(completions or [])),
list(set(completions or []).difference(actual_completions or [])),
indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)),
indent(markedup_content)))
def assertNoDuplicateCompletions(self,
markedup_content,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected there to be completions:\n%s" %
(lang, indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
if actual_completions is None:
self.fail("%s trigger resulted in no completions when expecting "
"to check for duplicate completions:\n%s" %
(lang, indent(markedup_content)))
count_from_cpln = {}
for cpln in actual_completions:
if cpln not in count_from_cpln:
count_from_cpln[cpln] = 0
count_from_cpln[cpln] += 1
cpln_dupes = [(count, cpln) for cpln, count in count_from_cpln.items()
if count > 1]
self.failIf(
cpln_dupes,
"unexpectedly got duplicate completions at the given position\n"
" duplicates:\n%s\n"
" eval log\n%s\n"
" buffer:\n%s" % (indent('\n'.join('%d of %r' % d
for d in cpln_dupes)),
indent('\n'.join('%5s: %s' % (lvl, m)
for lvl, m in ctlr.log)),
indent(markedup_content)))
def _assertCompletionsInclude(self, buf, trg, completions):
markedup_content = markup_text(buf.accessor.text, pos=trg.pos)
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
missing_completions = [
c for c in completions if c not in (actual_completions or [])
]
self.failIf(
missing_completions,
"%s completions at the given position did not "
"include all expected values\n"
" missing: %r\n"
" expected all of: %r\n"
" got: %r\n"
" eval log:\n%s\n"
" buffer:\n%s" %
(buf.lang, missing_completions, completions, actual_completions,
indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)),
indent(markedup_content)))
def assertCompletionsInclude(self,
markedup_content,
completions,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected completions to include %r:\n%s" %
(lang, completions, indent(markedup_content)))
self._assertCompletionsInclude(buf, trg, completions)
def assertCompletionsInclude2(self, buf, pos, completions, implicit=True):
"""A version of assertCompletionsInclude() where you pass in
a Buffer instance instead of marked up content. Sometimes
this is more convenient.
"""
trg = buf.trg_from_pos(pos, implicit=implicit)
if trg is None:
markedup_content = markup_text(buf.accessor.text, pos=pos)
self.fail("given position is not a %s trigger point, "
"expected completions to include %r:\n%s" %
(buf.lang, completions, indent(markedup_content)))
self._assertCompletionsInclude(buf, trg, completions)
def _assertCompletionsDoNotInclude(self, buf, trg, completions):
markedup_content = markup_text(buf.accessor.text, pos=trg.pos)
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
completions_that_shouldnt_be_there = [
c for c in (actual_completions or []) if c in completions
]
self.failIf(
completions_that_shouldnt_be_there,
"%s completions at the given position included "
"some unexpected values\n"
" shouldn't have had these: %r\n"
" expected none of: %r\n"
" got: %r\n"
" eval log:\n%s\n"
" buffer:\n%s" % (buf.lang, completions_that_shouldnt_be_there,
completions, actual_completions,
indent('\n'.join('%5s: %s' % (lvl, m)
for lvl, m in ctlr.log)),
indent(markedup_content)))
def assertCompletionsDoNotInclude(self,
markedup_content,
completions,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected completions to exclude %r:\n%s" %
(lang, completions, indent(markedup_content)))
self._assertCompletionsDoNotInclude(buf, trg, completions)
def assertCompletionsDoNotInclude2(self,
buf,
pos,
completions,
implicit=True):
"""A version of assertCompletionsDoNotInclude() where you pass in
a Buffer instance instead of marked up content. Sometimes
this is more convenient.
"""
trg = buf.trg_from_pos(pos, implicit=implicit)
if trg is None:
markedup_content = markup_text(buf.accessor.text, pos=pos)
self.fail("given position is not a %s trigger point, "
"expected completions to exclude %r:\n%s" %
(buf.lang, completions, indent(markedup_content)))
self._assertCompletionsDoNotInclude(buf, trg, completions)
def assertCalltipIs2(self, buf, pos, calltip, implicit=True):
"""A variant of assertCalltipIs() where you pass in
a Buffer instance instead of marked up content. Sometimes
this is more convenient.
"""
trg = buf.trg_from_pos(pos, implicit=implicit)
markedup_content = markup_text(buf.accessor.text, pos=trg.pos)
self._assertCalltipIs(buf, trg, markedup_content, calltip, self.lang)
def assertCalltipIs(self,
markedup_content,
calltip,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
self._assertCalltipIs(buf, trg, markedup_content, calltip, lang)
def _assertCalltipIs(self, buf, trg, markedup_content, calltip, lang):
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected the following calltip:\n"
" calltip:\n%s\n"
" buffer:\n%s" %
(lang, indent(calltip), indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_calltips = buf.calltips_from_trg(trg, ctlr=ctlr)
if actual_calltips and actual_calltips[0]:
actual_calltip = actual_calltips[0]
else:
actual_calltip = None
self.assertEqual(
calltip, actual_calltip,
"unexpected %s calltip at the given position\n"
" expected:\n%s\n"
" got:\n%s\n"
" eval log:\n%s\n"
" buffer:\n%s" %
(trg.name, indent(calltip and calltip or "(none)"),
indent(actual_calltip and actual_calltip or "(none)"),
indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)),
indent(markedup_content)))
def assertCalltipMatches(self,
markedup_content,
calltip,
lang=None,
implicit=True,
env=None,
flags=0):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
self._assertCalltipMatches(buf, trg, markedup_content, calltip, lang,
flags)
def _assertCalltipMatches(self, buf, trg, markedup_content, expr, lang,
flags):
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected the calltip to match the following:\n"
" exression:\n%s\n"
" buffer:\n%s" %
(lang, indent(expr), indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_calltips = buf.calltips_from_trg(trg, ctlr=ctlr)
if actual_calltips and actual_calltips[0]:
actual_calltip = actual_calltips[0]
else:
actual_calltip = None
self.assertNotEquals(
re.search(expr, actual_calltip, flags), None,
"unexpected %s calltip at the given position\n"
" expression:\n%s\n"
" got:\n%s\n"
" eval log:\n%s\n"
" buffer:\n%s" %
(trg.name, indent(expr and expr or "(none)"),
indent(actual_calltip and actual_calltip or "(none)"),
indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)),
indent(markedup_content)))
def assertCurrCalltipArgRange(self,
markedup_content,
calltip,
expected_range,
lang=None,
implicit=True):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(self.adjust_content(markedup_content))
pos = data["pos"]
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
trg = buf.trg_from_pos(data["trg_pos"], implicit=implicit)
actual_range = buf.curr_calltip_arg_range(trg.pos,
calltip,
curr_pos=data["pos"])
self.assertEqual(
actual_range, expected_range,
"unexpected current calltip arg range\n"
" expected: %s\n"
" got: %s\n"
" calltip:\n%s\n"
" buffer:\n%s" % (expected_range, actual_range, indent(calltip),
indent(markedup_content)))
#def assertCompletionRaises(self, markedup_content, exception, lang=None,
# **kwargs):
# """Assert that the given completion raises the given exception.
#
# You may also specify either of the "exc_args" or "exc_pattern"
# keyword args to match the exception's "args" attribute or match
# the stringified exception against a regex pattern.
#
# c.f. http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/307970
# """
# if lang is None:
# lang = self.lang
# if "exc_args" in kwargs:
# exc_args = kwargs["exc_args"]
# del kwargs["exc_args"]
# else:
# exc_args = None
# if "exc_pattern" in kwargs:
# exc_pattern = kwargs["exc_pattern"]
# del kwargs["exc_pattern"]
# else:
# exc_pattern = None
#
# callsig = "the given %s completion" % lang
# try:
# buf, trg = self._get_buf_and_trg(markedup_content, lang)
# if trg is None:
# self.fail("given position is not a %s trigger point, "
# "no completion can be done to see if it raises"
# % self.lang)
# callsig = "completion at %s" % trg
# cplns = buf.cplns_from_trg(trg)
# except exception, exc:
# if exc_args is not None:
# self.failIf(exc.args != exc_args,
# "%s raised %s with unexpected args: "\
# "expected=%r, actual=%r"\
# % (callsig, exc.__class__, exc_args, exc.args))
# if exc_pattern is not None:
# self.failUnless(exc_pattern.search(str(exc)),
# "%s raised %s, but the exception "\
# "does not match '%s': %r"\
# % (callsig, exc.__class__, exc_pattern.pattern,
# str(exc)))
# except:
# exc_info = sys.exc_info()
# print exc_info
# self.fail("%s raised an unexpected exception type: "\
# "expected=%s, actual=%s"\
# % (callsig, exception, exc_info[0]))
# else:
# self.fail("%s did not raise %s" % (callsig, exception))
def assertEvalError(self,
markedup_content,
log_pattern,
lang=None,
implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content,
lang,
implicit=implicit,
env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"no completion can be done to see if errors" % self.lang)
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
class TestEvalController(EvalController):
"""A completion controller that captures all eval logging."""
def __init__(self):
EvalController.__init__(self)
self.log = []
def debug(self, msg, *args):
self.log.append(("debug", msg % args))
def info(self, msg, *args):
self.log.append(("info", msg % args))
def warn(self, msg, *args):
self.log.append(("warn", msg % args))
def error(self, msg, *args):
self.log.append(("error", msg % args))
ctlr = TestEvalController()
buf.async_eval_at_trg(trg, ctlr=ctlr)
ctlr.wait()
if not ctlr.is_done():
self.fail("evaluation is not 'done': didn't expect that")
if trg.form == TRG_FORM_CPLN and ctlr.cplns:
self.fail("evalution had results: didn't expect that: %r" %
ctlr.cplns)
elif trg.form == TRG_FORM_CALLTIP and ctlr.calltips:
self.fail("evalution had results: didn't expect that: %r" %
ctlr.cplns)
if log_pattern:
#pprint(ctlr.log)
matching_logs = [(level, msg) for level, msg in ctlr.log
if log_pattern.search(msg)]
self.failUnless(
matching_logs,
"the given completion failed but no logs matched the given pattern:\n"
" log_pattern: /%s/\n"
" log messages:\n%s\n"
" buffer:\n%s" %
(log_pattern.pattern,
indent('\n'.join(['%s: %s' % lg for lg in ctlr.log
])), indent(markedup_content)))
#pprint(matching_logs)
def assertCITDLExprIs(self,
markedup_content,
citdl_expr,
lang=None,
prefix_filter=None,
implicit=True,
trigger_name=None,
**fields):
"""Assert that the preceding CITDL expression at the current position
is as expected.
This uses buf.citdl_expr_from_trg() -- or, for Perl,
buf.citdl_expr_and_prefix_filter_from_trg().
The "prefix_filter" optional argument can be used for Perl to test
the value its relevant function returns.
"""
if lang is None:
lang = self.lang
content, data = unmark_text(self.adjust_content(markedup_content))
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
langintel = self.mgr.langintel_from_lang(lang)
if trigger_name is None:
trigger_name = "fakey-completion-type"
if lang == "Perl":
# Bit of a hack to fake the trigger length.
if content[data["pos"] - 1] in ('>', ':'): # '->' or '::' triggers
length = 2
else:
length = 1
trg = Trigger(lang,
TRG_FORM_CPLN,
trigger_name,
data["pos"],
implicit=implicit,
length=length,
**fields)
actual_citdl_expr, actual_prefix_filter \
= langintel.citdl_expr_and_prefix_filter_from_trg(buf, trg)
else:
trg = Trigger(lang,
TRG_FORM_CPLN,
trigger_name,
data["pos"],
implicit=implicit,
**fields)
actual_citdl_expr = langintel.citdl_expr_from_trg(buf, trg)
self.assertEqual(
actual_citdl_expr, citdl_expr,
"unexpected actual %s CITDL expr preceding trigger:\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s" %
(lang, citdl_expr, actual_citdl_expr, indent(markedup_content)))
if lang == "Perl" and prefix_filter is not None:
self.assertEqual(
actual_prefix_filter, prefix_filter,
"unexpected actual %s variable prefix filter "
"preceding trigger:\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s" % (lang, prefix_filter, actual_prefix_filter,
indent(markedup_content)))
def _unmark_lex_text(self, markedup_text):
from SilverCity import ScintillaConstants
tokenizer = re.compile(r'(<(SCE_\w+)>(.*?)</\2>)')
tokens = []
text = ''
while markedup_text:
match = tokenizer.search(markedup_text)
if match:
text += markedup_text[:match.start()]
token = {
'end_index': len(text) + len(match.group(3)) - 1,
'start_index': len(text),
'style': getattr(ScintillaConstants, match.group(2)),
'text': match.group(3),
#'end_column': ???,
#'end_line': ???,
#'start_column': ???,
#'start_line': ???,
}
tokens.append(token)
text += match.group(3)
markedup_text = markedup_text[match.end():]
else:
text += markedup_text
markedup_text = ''
return text, tokens
def assertLex(self, markedup_content, lang=None):
"""Lex the given content and assert that the lexed tokens are as
expected.
What is "expected" is given via pseudo-xml markup like this:
fuzzy wuzzy <SCE_UDL_SSL_COMMENTBLOCK>wuzza</SCE_UDL_SSL_COMMENTBLOCK> bear
This example expects that "wuzza" will be a token with style
SCE_UDL_SSL_COMMENTBLOCK.
"""
from codeintel2.accessor import SilverCityAccessor
if lang is None:
lang = self.lang
content, tokens = self._unmark_lex_text(markedup_content)
# Do lexing of this content via the codeintel Buffer's, because
# they already handle all the SilverCity lexer hookup.
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
assert isinstance(buf.accessor, SilverCityAccessor)
actual_tokens = buf.accessor.tokens # cheating
for actual_token in actual_tokens:
# There are a few SilverCity token dict keys that we
# don't bother checking.
del actual_token["end_column"]
del actual_token["end_line"]
del actual_token["start_column"]
del actual_token["start_line"]
unmatched_tokens = [t for t in tokens if t not in actual_tokens]
if unmatched_tokens:
self.fail("not all expected %s lex tokens were found in the "
"actual lexer output:\n"
" buffer:\n%s\n"
" actual lexer tokens:\n%s\n"
" unmatched tokens:\n%s\n" %
(lang, indent(content), indent(pformat(actual_tokens)),
indent(pformat(unmatched_tokens))))
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if opts.time_it:
start = time.time()
quiet = opts.quiet
if opts.time_it or opts.time_details:
opts.force = True
scan_count = 0
lang_warnings = set()
tree = None
for path in _paths_from_path_patterns(path_patterns,
recursive=opts.recursive,
includes=opts.includes):
if opts.time_it:
sys.stderr.write(path + "\n")
if opts.time_details:
start1 = time.time()
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info("skip `%s': couldn't determine language", path)
continue
try:
buf = mgr.buf_from_path(path, lang=lang)
except OSError as ex:
# Couldn't access the file.
if not opts.recursive:
raise
# Ignore files we don't really care about.
log.warn("%r - %r", ex, path)
continue
if not isinstance(buf, CitadelBuffer):
if opts.recursive:
# Ignore files that scanning isn't provided for.
continue
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
scan_count += 1
if scan_count % 10 == 0:
log.info("%d scanning %r", scan_count, path)
try:
if opts.force:
buf.scan()
if tree is None:
tree = ET.Element("codeintel", version="2.0")
file_elem = ET.SubElement(tree,
"file",
lang=buf.lang,
mtime=str(int(time.time())),
path=os.path.basename(path))
for lang, blob in sorted(buf.blob_from_lang.items()):
blob = buf.blob_from_lang[lang]
file_elem.append(blob)
except KeyError as ex:
# Unknown cile language.
if not opts.recursive:
raise
message = str(ex)
if message not in lang_warnings:
lang_warnings.add(message)
log.warn("Skipping unhandled language %s", message)
if opts.time_details:
delta = time.time() - start1
sys.stderr.write("%.3f %s\n" % (delta, path))
sys.stderr.flush()
if tree is not None:
if opts.stripfuncvars:
# For stdlibs, we don't care about variables inside of
# functions and they take up a lot of space.
for function in tree.getiterator('scope'):
if function.get('ilk') == 'function':
function[:] = [
child for child in function
if child.tag != 'variable'
]
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
if not quiet:
sys.stdout.write(
'<?xml version="1.0" encoding="UTF-8"?>\n')
ET.dump(tree)
if opts.time_it:
end = time.time()
sys.stderr.write("scan took %.3fs\n" % (end - start))
finally:
mgr.finalize()
def do_outline(self, subcmd, opts, path):
"""Print code outline of the given file.
You can specify a lookup path into the file code outline to
display via URL-anchor syntax, e.g.:
ci2 outline path/to/foo.py#AClass.amethod
${cmd_usage}
${cmd_option_list}
"""
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if '#' in path:
path, anchor = path.rsplit('#', 1)
else:
anchor = None
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
#buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
tree = buf.tree
if anchor is not None:
# Lookup the anchor in the codeintel CIX tree.
lpath = re.split(r'\.|::', anchor)
def blobs_from_tree(tree):
for file_elem in tree:
for blob in file_elem:
yield blob
for elem in blobs_from_tree(tree):
# Generally have 3 types of codeintel trees:
# 1. single-lang file: one <file>, one <blob>
# 2. multi-lang file: one <file>, one or two <blob>'s
# 3. CIX stdlib/catalog file: possibly multiple
# <file>'s, likely multiple <blob>'s
# Allow the first token to be the blob name or lang.
# (This can sometimes be weird, but seems the most
# convenient solution.)
if lpath[0] in (elem.get("name"), elem.get("lang")):
remaining_lpath = lpath[1:]
else:
remaining_lpath = lpath
for name in remaining_lpath:
try:
elem = elem.names[name]
except KeyError:
elem = None
break # try next lang blob
if elem is not None:
break # found one
else:
log.error(
"could not find `%s' definition (or blob) in `%s'",
anchor, path)
return 1
else:
elem = tree
try:
_outline_ci_elem(elem, brief=opts.brief, doSort=opts.doSort)
except IOError as ex:
if ex.errno == 0:
# Ignore this error from aborting 'less' of 'ci2 outline'
# output:
# IOError: (0, 'Error')
pass
else:
raise
finally:
mgr.finalize()
def do_play(self, subcmd, opts):
"""Run my current play/dev code.
${cmd_usage}
${cmd_option_list}
"""
if False:
lang = "CSS"
markedup_content = dedent("""
/* http://www.w3.org/TR/REC-CSS2/fonts.html#propdef-font-weight */
h1 {
border: 1px solid black;
font-weight /* hi */: <|> !important
}
""")
content, data = unmark_text(markedup_content)
pos = data["pos"]
mgr = Manager()
#mgr.upgrade() # Don't need it for just CSS usage.
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path="play.css")
trg = buf.trg_from_pos(pos)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
completions = buf.cplns_from_trg(trg)
print("COMPLETIONS: %r" % completions)
finally:
mgr.finalize()
elif False:
lang = "Python"
path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
markedup_content = dedent("""
import sys, os
class Foo:
def bar(self):
pass
sys.<|>path # should have path in completion list
f = Foo()
""")
content, data = unmark_text(markedup_content)
print(banner(path))
print(_escaped_text_from_text(content, "whitespace"))
pos = data["pos"]
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path=path)
print(banner("cix", '-'))
print(buf.cix)
trg = buf.trg_from_pos(pos)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
print(banner("completions", '-'))
ctlr = LogEvalController(log)
buf.async_eval_at_trg(trg, ctlr)
ctlr.wait(2) #XXX
if not ctlr.is_done():
ctlr.abort()
raise Error("XXX async eval timed out")
pprint(ctlr.cplns)
print(banner(None))
finally:
mgr.finalize()
elif False:
lang = "Ruby"
path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
markedup_content = dedent("""\
r<1>equire 'net/http'
include Net
req = HTTPRequest.new
req.<2>get()
""")
content, data = unmark_text(markedup_content)
print(banner(path))
print(_escaped_text_from_text(content, "whitespace"))
pos = data[1]
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path=path)
print(banner("cix", '-'))
cix = buf.cix
print(
ET.tostring(pretty_tree_from_tree(tree_from_cix(buf.cix))))
trg = buf.trg_from_pos(pos, implicit=False)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
print(banner("completions", '-'))
ctlr = LogEvalController(log)
buf.async_eval_at_trg(trg, ctlr)
ctlr.wait(30) #XXX
if not ctlr.is_done():
ctlr.abort()
raise Error("XXX async eval timed out")
pprint(ctlr.cplns)
print(banner(None))
finally:
mgr.finalize()
def cix2html(opts, path):
"""Turn cix file into html API documentation.
Example:
cix2html path/to/foo.cix#AClass.amethod
cix2html path/to/foo.cix -o file.html
${cmd_usage}
${cmd_option_list}
"""
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
def blobs_from_tree(tree):
for file_elem in tree:
for blob in file_elem:
yield blob
if '#' in path:
path, anchor = path.rsplit('#', 1)
else:
anchor = None
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
# buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
tree = buf.tree
if anchor is not None:
# Lookup the anchor in the codeintel CIX tree.
lpath = re.split(r'\.|::', anchor)
for elem in blobs_from_tree(tree):
# Generally have 3 types of codeintel trees:
# 1. single-lang file: one <file>, one <blob>
# 2. multi-lang file: one <file>, one or two <blob>'s
# 3. CIX stdlib/catalog file: possibly multiple
# <file>'s, likely multiple <blob>'s
# Allow the first token to be the blob name or lang.
# (This can sometimes be weird, but seems the most
# convenient solution.)
if lpath[0] in (elem.get("name"), elem.get("lang")):
remaining_lpath = lpath[1:]
else:
remaining_lpath = lpath
for name in remaining_lpath:
try:
elem = elem.names[name]
except KeyError:
elem = None
break # try next lang blob
if elem is not None:
break # found one
else:
log.error("could not find `%s' definition (or blob) in `%s'",
anchor, path)
return 1
else:
elem = tree
try:
if elem.tag == "codeintel":
_html_ci_elem(opts, elem.getchildren()[0])
else:
_html_ci_elem(opts, elem)
except IOError, ex:
if ex.errno == 0:
# Ignore this error from aborting 'less' of 'ci2 outline'
# output:
# IOError: (0, 'Error')
pass
else:
raise
except Exception, e:
import traceback
traceback.print_exc()
def _codeintel_scan():
global _ci_mgr_, despair, despaired
env = None
mtime = None
catalogs = []
now = time.time()
try:
env = _ci_envs_[path]
if env._folders != folders:
raise KeyError
mgr = _ci_mgr_
if now > env._time:
mtime = max(tryGetMTime(env._config_file),
tryGetMTime(env._config_default_file))
if env._mtime < mtime:
raise KeyError
except KeyError:
if env is not None:
config_default_file = env._config_default_file
project_dir = env._project_dir
project_base_dir = env._project_base_dir
config_file = env._config_file
else:
config_default_file = os.path.join(CODEINTEL_HOME_DIR,
'config')
if not (config_default_file
and os.path.exists(config_default_file)):
config_default_file = None
project_dir = None
project_base_dir = None
if path:
# Try to find a suitable project directory (or best guess):
for folder in ['.codeintel', '.git', '.hg', 'trunk']:
project_dir = find_folder(path, folder)
if project_dir and (
folder != '.codeintel' or not os.path.exists(
os.path.join(project_dir, 'db'))):
if folder.startswith('.'):
project_base_dir = os.path.abspath(
os.path.join(project_dir, '..'))
else:
project_base_dir = project_dir
break
if not (project_dir and os.path.exists(project_dir)):
project_dir = None
config_file = project_dir and folder == '.codeintel' and os.path.join(
project_dir, 'config')
if not (config_file and os.path.exists(config_file)):
config_file = None
if _ci_mgr_:
mgr = _ci_mgr_
else:
for thread in threading.enumerate():
if thread.name == "CodeIntel Manager":
thread.finalize(
) # this finalizes the index, citadel and the manager and waits them to end (join)
mgr = Manager(
extra_module_dirs=_ci_extra_module_dirs_,
db_base_dir=_ci_db_base_dir_,
db_catalog_dirs=_ci_db_catalog_dirs_,
db_import_everything_langs=_ci_db_import_everything_langs,
db_event_reporter=lambda m: logger(view, 'event', m),
)
mgr.upgrade()
mgr.initialize()
# Connect the logging file to the handler
condeintel_log_file = os.path.join(mgr.db.base_dir,
'codeintel.log')
codeintel_log.handlers = [
logging.StreamHandler(open(condeintel_log_file, 'w', 1))
]
msg = "Starting logging SublimeCodeIntel rev %s (%s) on %s" % (
get_git_revision()[:12], os.stat(__file__)[stat.ST_MTIME],
datetime.datetime.now().ctime())
codeintel_log.info("%s\n%s" % (msg, "=" * len(msg)))
_ci_mgr_ = mgr
# Load configuration files:
for catalog in mgr.db.get_catalogs_zone().avail_catalogs():
if catalog['lang'] == lang:
catalogs.append(catalog['name'])
config = {
"codeintel_selected_catalogs": catalogs,
"codeintel_max_recursive_dir_depth": 10,
"codeintel_scan_files_in_project": True,
}
_config = {}
try:
tryReadDict(config_default_file, _config)
except Exception, e:
msg = "Malformed configuration file '%s': %s" % (
config_default_file, e)
log.error(msg)
codeintel_log.error(msg)
try:
tryReadDict(config_file, _config)
except Exception, e:
msg = "Malformed configuration file '%s': %s" % (
config_default_file, e)
log.error(msg)
codeintel_log.error(msg)
_fn = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'path.ini')
if os.path.isfile(_fn):
with open(_fn, 'r') as f:
paths = f.readline()
print 'Paths:\n' + '\n'.join(paths.split(os.pathsep))
d = {}
d['phpExtraPaths'] = paths
d['pythonExtraPaths'] = paths
d['perlExtraPaths'] = paths
d['javascriptExtraPaths'] = paths
d['rubyExtraPaths'] = paths
env = SimplePrefsEnvironment(**d)
mgr = Manager()
mgr.upgrade()
mgr.initialize()
#------------------------
def file_text(fn):
try:
with open(fn, 'r') as f:
text = f.readlines()
return ''.join(text)
except IOError as e:
if e.errno == errno.ENOENT: # No such file or directory
logging.error(e.strerror + ': "' + fn + '"')
else:
logging.error(e.strerror)
return None
logStream = logging.StreamHandler(sys.stderr)
logStream.setFormatter(
logging.Formatter("%(name)s: %(levelname)s: %(message)s"))
logging.getLogger("codeintel").addHandler(logStream)
logger = logging.getLogger("codeintel_server")
logger.addHandler(logStream)
logger.setLevel(logging.WARNING)
manager = Manager(
# db_base_dir = path.join(CI_DIR, 'db'),
# extra_module_dirs = [path.join(CI_DIR, 'codeintel2'),],
# db_import_everything_langs = None,
# db_catalog_dirs = []
)
manager.upgrade()
manager.initialize()
scanned_langs = []
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Run codeintel commands as a daemon or via stdin")
parser.add_argument("mode",
help="Mode of operation",
choices=[
"daemon", "completions", "definitions", "calltips",
"catalogs"
])
parser.add_argument("--row", type=int, help="The row to read from")
parser.add_argument("--column", type=int, help="The column to read from")
parser.add_argument("--path", help="The path of the file")
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
import time
import ciElementTree as ET
from ci2 import _paths_from_path_patterns
from codeintel2.manager import Manager
from codeintel2.citadel import CitadelBuffer
from codeintel2.common import CodeIntelError
from codeintel2.tree import pretty_tree_from_tree
from codeintel2.util import guess_lang_from_path
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if opts.time_it:
start = time.time()
quiet = opts.quiet
if opts.time_it or opts.time_details:
opts.force = True
scan_count = 0
lang_warnings = set()
tree = None
for path in _paths_from_path_patterns(path_patterns,
recursive=opts.recursive,
includes=opts.includes):
if opts.time_it:
sys.stderr.write(path + "\n")
if opts.time_details:
start1 = time.time()
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
self.log.info("skip `%s': couldn't determine language", path)
continue
try:
buf = mgr.buf_from_path(path, lang=lang)
except OSError as ex:
# Couldn't access the file.
if not opts.recursive:
raise
# Ignore files we don't really care about.
self.log.warn("%r - %r", ex, path)
continue
if not isinstance(buf, CitadelBuffer):
if opts.recursive:
# Ignore files that scanning isn't provided for.
continue
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
scan_count += 1
if scan_count % 10 == 0:
self.log.info("%d scanning %r", scan_count, path)
try:
if opts.force:
buf.scan()
if tree is None:
tree = ET.Element("codeintel", version="2.0")
file_elem = ET.SubElement(tree, "file",
lang=buf.lang,
mtime=str(int(time.time())),
path=os.path.basename(path))
for lang, blob in sorted(buf.blob_from_lang.items()):
blob = buf.blob_from_lang[lang]
file_elem.append(blob)
except KeyError as ex:
# Unknown cile language.
if not opts.recursive:
raise
message = str(ex)
if message not in lang_warnings:
lang_warnings.add(message)
self.log.warn("Skipping unhandled language %s", message)
if opts.time_details:
delta = time.time() - start1
sys.stderr.write("%.3f %s\n" % (delta, path))
sys.stderr.flush()
if tree is not None:
if opts.stripfuncvars:
# For stdlibs, we don't care about variables inside of
# functions and they take up a lot of space.
for function in tree.getiterator('scope'):
if function.get('ilk') == 'function':
function[:] = [child for child in function
if child.tag != 'variable']
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
if not quiet:
sys.stdout.write('<?xml version="1.0" encoding="UTF-8"?>\n')
ET.dump(tree)
if opts.time_it:
end = time.time()
sys.stderr.write("scan took %.3fs\n" % (end - start))
finally:
mgr.finalize()
class CodeIntelTestCase(unittest.TestCase):
# Subclasses can override this to have setUp pass these settings to the
# codeintel.Manager.
_ci_db_base_dir_ = None
_ci_db_catalog_dirs_ = []
_ci_db_import_everything_langs = None
_ci_env_prefs_ = None
# A test case can set this to false to have the setUp/tearDown *not*
# create a `self.mgr'.
_ci_test_setup_mgr_ = True
_ci_extra_module_dirs_ = None
def setUp(self):
if _xpcom_:
# The tests are run outside of Komodo. If run with PyXPCOM up
# parts codeintel will try to use the nsIDirectoryService and
# will query dirs only provided by nsXREDirProvider -- which
# isn't registered outside of Komodo (XRE_main() isn't called).
# The KoTestService provides a backup.
koTestSvc = components.classes["@activestate.com/koTestService;1"] \
.getService(components.interfaces.koITestService)
koTestSvc.init()
if self._ci_test_setup_mgr_:
env = None
if self._ci_env_prefs_ is not None:
env = SimplePrefsEnvironment(**self._ci_env_prefs_)
self.mgr = Manager(
extra_module_dirs=self._ci_extra_module_dirs_,
db_base_dir=self._ci_db_base_dir_ or test_db_base_dir,
db_catalog_dirs=self._ci_db_catalog_dirs_,
db_import_everything_langs=self._ci_db_import_everything_langs,
env=env)
self.mgr.upgrade()
self.mgr.initialize()
def tearDown(self):
if self._ci_test_setup_mgr_:
self.mgr.finalize()
self.mgr = None
def adjust_content(self, content):
"""A hook for subclasses to modify markedup_content before use in
test cases. This is useful for sharing test cases between pure-
and multi-lang uses of a given lang.
"""
return content
def adjust_pos(self, pos):
"""A accompanying hook for `adjust_content' to adjust trigger
pos values accordingly.
"""
return pos
def _get_buf_and_data(self, markedup_content, lang, path=None, env=None):
if path is None:
# Try to ensure no accidental re-use of the same buffer name
# across the whole test suite. Also try to keep the buffer
# names relatively short (keeps tracebacks cleaner).
name = "buf-" + md5(markedup_content).hexdigest()[:16]
path = os.path.join("<Unsaved>", name)
content, data = unmark_text(self.adjust_content(markedup_content))
#print banner(path)
#sys.stdout.write(content)
#print banner(None)
buf = self.mgr.buf_from_content(content, lang=lang, path=path,
env=env)
return buf, data
def _get_buf_and_trg(self, markedup_content, lang, path=None,
implicit=True, env=None):
buf, data = self._get_buf_and_data(markedup_content, lang, path, env)
trg = buf.trg_from_pos(data["pos"], implicit=implicit)
return buf, trg
def assertCITDLExprUnderPosIs(self, markedup_content, citdl_expr, lang=None,
prefix_filter=None, implicit=True, trigger_name=None,
**fields):
"""Assert that the CITDL expression at the current position
is as expected.
This uses buf.citdl_expr_under_pos() -- or, for Perl,
buf.citdl_expr_and_prefix_filter_from_trg().
Note: This API is a mess right now. C.f. bug 65776.
The "prefix_filter" optional argument can be used for Perl to test
the value its relevant function returns.
"""
if lang is None:
lang = self.lang
content, data = unmark_text(
self.adjust_content(markedup_content))
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
langintel = self.mgr.langintel_from_lang(lang)
if trigger_name is None:
trigger_name = "fakey-completion-type"
if lang == "Perl":
trg = Trigger(lang, TRG_FORM_DEFN, trigger_name,
data["pos"], implicit=implicit, length=0,
**fields)
actual_citdl_expr, actual_prefix_filter \
= langintel.citdl_expr_and_prefix_filter_from_trg(buf, trg)
else:
#actual_citdl_expr = langintel.citdl_expr_under_pos(buf, data["pos"])
trg = Trigger(lang, TRG_FORM_DEFN, trigger_name,
data["pos"], implicit=implicit,
**fields)
actual_citdl_expr = langintel.citdl_expr_from_trg(buf, trg)
self.assertEqual(actual_citdl_expr, citdl_expr,
"unexpected actual %s CITDL expr under pos:\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s"
% (lang, citdl_expr, actual_citdl_expr,
indent(markedup_content)))
if prefix_filter is not None:
XXX #TODO: compare prefix_filter to given value
def _assertDefnMatches(self, buf, pos, lang=None, **fields):
ctlr = _CaptureEvalController()
trg = buf.defn_trg_from_pos(pos)
defns = buf.defns_from_trg(trg, ctlr=ctlr)
if not defns:
self.fail("unexpectedly did not find a definition in %r at pos %d\n"
" eval log\n%s\n"
" buffer:\n%s"
% (buf, pos,
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(buf.accessor.text)))
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
defn = defns[0]
for name, value in fields.items():
try:
actual_value = getattr(defn, name)
except AttributeError:
actual_value = None
self.assertEqual(actual_value, value,
"%s definition, unexpected value for field %r\n"
" defn: %r\n"
" expected: %r\n"
" got: %r\n"
" eval log\n%s\n"
" buffer:\n%s"
% (buf.lang, name, defn, value, actual_value,
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(buf.accessor.text)))
def assertDefnMatches(self, markedup_content, lang=None, **fields):
if lang is None:
lang = self.lang
buf, data = self._get_buf_and_data(markedup_content, lang)
self._assertDefnMatches(buf, data["pos"], lang, **fields)
def assertDefnMatches2(self, buf, pos, lang=None, **fields):
self._assertDefnMatches(buf, pos, lang, **fields)
def assertNoDuplicateDefns(self, markedup_content, lang=None):
if lang is None:
lang = self.lang
buf, data = self._get_buf_and_data(markedup_content, lang)
self.assertNoDuplicateDefns2(buf, data["pos"])
def assertNoDuplicateDefns2(self, buf, pos):
markedup_content = markup_text(buf.accessor.text, pos=pos)
ctlr = _CaptureEvalController()
trg = buf.defn_trg_from_pos(pos)
actual_defns = buf.defns_from_trg(trg, ctlr=ctlr)
if not actual_defns:
self.fail("%s trigger resulted in no definitions when expecting "
"to check for duplicate definitions:\n%s"
% (buf.lang, indent(markedup_content)))
count_from_defn_repr = {}
for defn_repr in (repr(d) for d in actual_defns):
if defn_repr not in count_from_defn_repr:
count_from_defn_repr[defn_repr] = 0
count_from_defn_repr[defn_repr] += 1
defn_dupes = [(count, defn_repr)
for defn_repr, count in count_from_defn_repr.items()
if count > 1]
self.failIf(defn_dupes,
"unexpectedly got duplicate completions at the given position\n"
" duplicates:\n%s\n"
" eval log\n%s\n"
" buffer:\n%s"
% (indent('\n'.join('%d of %s' % d for d in defn_dupes)),
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(markedup_content)))
def assertTriggerMatches(self, markedup_content, lang=None,
implicit=True, env=None, **fields):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
if trg is None:
self.fail("unexpectedly did not find a %s trigger, buffer:\n%s"
% (lang, indent(markedup_content)))
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
for name, value in fields.items():
try:
actual_value = getattr(trg, name)
except AttributeError:
actual_value = trg.extra[name]
self.assertEqual(actual_value, value,
"unexpected %s trigger '%s' value: expected %r, "
"got %r, buffer:\n%s"
% (lang, name, value, actual_value,
indent(markedup_content)))
# Used when a position generates a trigger, but it's not the one specified
def assertTriggerDoesNotMatch(self, markedup_content, lang=None,
implicit=True, env=None, **fields):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
if trg is None:
# No trigger is as good as
return
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
for name, value in fields.items():
try:
actual_value = getattr(trg, name)
except AttributeError:
actual_value = trg.extra[name]
self.assertNotEqual(actual_value, value,
"unexpected %s trigger '%s' value: expected not %r, "
"got %r, buffer:\n%s"
% (lang, name, value, actual_value,
indent(markedup_content)))
def assertNoTrigger(self, markedup_content, lang=None, implicit=True,
env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
if trg is not None:
self.fail("unexpectedly found a %s trigger %r when didn't expect "
"one, buffer:\n%s"
% (lang, trg, indent(markedup_content)))
def assertPrecedingTriggerMatches(self, markedup_content, lang=None,
**fields):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(
self.adjust_content(markedup_content))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
trg = buf.preceding_trg_from_pos(data["start_pos"], data["pos"])
if trg is None:
self.fail("unexpectedly did not find a preceding %s trigger, "
"buffer:\n%s" % (lang, indent(markedup_content)))
if "pos" in fields:
fields["pos"] = self.adjust_pos(fields["pos"])
for name, value in fields.items():
actual_value = getattr(trg, name)
self.assertEqual(actual_value, value,
"unexpected preceding %s trigger '%s' value: expected %r, "
"got %r, buffer:\n%s"
% (lang, name, value, actual_value,
indent(markedup_content)))
def assertNoPrecedingTrigger(self, markedup_content, lang=None):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(
self.adjust_content(markedup_content))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
trg = buf.preceding_trg_from_pos(data["start_pos"], data["pos"])
if trg is not None:
self.fail("unexpectedly found a preceding %s trigger '%s' when "
"didn't expect one, buffer:\n%s"
% (lang, trg.name, indent(markedup_content)))
def assertScopeLpathIs(self, markedup_content, lpath, lang=None):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(
self.adjust_content(markedup_content))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
buf.scan(skip_scan_time_check=True)
actual_blob, actual_lpath = buf.scoperef_from_pos(data["pos"])
self.failUnlessEqual(lpath, actual_lpath,
"unexpected %s scope lookup path (lpath) at the given position\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s"
% (self.lang, lpath,
actual_lpath,
indent(markedup_content)))
def assertCompletionsAre2(self, buf, pos, completions, lang=None, implicit=True, env=None):
if lang is None:
lang = self.lang
markedup_content = markup_text(buf.accessor.text, pos=pos)
trg = buf.trg_from_pos(pos, implicit=implicit)
self._assertCompletionsAre(markedup_content, buf, trg, completions, lang, implicit)
def assertCompletionsAre(self, markedup_content, completions,
lang=None, implicit=True, env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
self._assertCompletionsAre(markedup_content, buf, trg, completions, lang, implicit)
def _assertCompletionsAre(self, markedup_content, buf, trg, completions,
lang, implicit):
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected completions to be %r:\n%s"
% (lang, completions, indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
self.assertEqual(completions, actual_completions,
"unexpected %s completions at the given position\n"
" expected: %r\n"
" got: %r\n"
" extra: %r\n"
" missing: %r\n"
" eval log\n%s\n"
" buffer:\n%s"
% (lang, completions,
actual_completions,
list(set(actual_completions or []).difference(completions or [])),
list(set(completions or []).difference(actual_completions or [])),
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(markedup_content)))
def assertNoDuplicateCompletions(self, markedup_content, lang=None,
implicit=True, env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected there to be completions:\n%s"
% (lang, indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
if actual_completions is None:
self.fail("%s trigger resulted in no completions when expecting "
"to check for duplicate completions:\n%s"
% (lang, indent(markedup_content)))
count_from_cpln = {}
for cpln in actual_completions:
if cpln not in count_from_cpln:
count_from_cpln[cpln] = 0
count_from_cpln[cpln] += 1
cpln_dupes = [(count, cpln) for cpln, count in count_from_cpln.items()
if count > 1]
self.failIf(cpln_dupes,
"unexpectedly got duplicate completions at the given position\n"
" duplicates:\n%s\n"
" eval log\n%s\n"
" buffer:\n%s"
% (indent('\n'.join('%d of %r' % d for d in cpln_dupes)),
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(markedup_content)))
def _assertCompletionsInclude(self, buf, trg, completions):
markedup_content = markup_text(buf.accessor.text, pos=trg.pos)
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
missing_completions = [c for c in completions
if c not in (actual_completions or [])]
self.failIf(missing_completions,
"%s completions at the given position did not "
"include all expected values\n"
" missing: %r\n"
" expected all of: %r\n"
" got: %r\n"
" eval log:\n%s\n"
" buffer:\n%s"
% (buf.lang, missing_completions, completions,
actual_completions,
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(markedup_content)))
def assertCompletionsInclude(self, markedup_content, completions,
lang=None, implicit=True, env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected completions to include %r:\n%s"
% (lang, completions, indent(markedup_content)))
self._assertCompletionsInclude(buf, trg, completions)
def assertCompletionsInclude2(self, buf, pos, completions, implicit=True):
"""A version of assertCompletionsInclude() where you pass in
a Buffer instance instead of marked up content. Sometimes
this is more convenient.
"""
trg = buf.trg_from_pos(pos, implicit=implicit)
if trg is None:
markedup_content = markup_text(buf.accessor.text, pos=pos)
self.fail("given position is not a %s trigger point, "
"expected completions to include %r:\n%s"
% (buf.lang, completions, indent(markedup_content)))
self._assertCompletionsInclude(buf, trg, completions)
def _assertCompletionsDoNotInclude(self, buf, trg, completions):
markedup_content = markup_text(buf.accessor.text, pos=trg.pos)
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr)
completions_that_shouldnt_be_there = [
c for c in (actual_completions or []) if c in completions
]
self.failIf(completions_that_shouldnt_be_there,
"%s completions at the given position included "
"some unexpected values\n"
" shouldn't have had these: %r\n"
" expected none of: %r\n"
" got: %r\n"
" eval log:\n%s\n"
" buffer:\n%s"
% (buf.lang, completions_that_shouldnt_be_there, completions,
actual_completions,
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(markedup_content)))
def assertCompletionsDoNotInclude(self, markedup_content, completions,
lang=None, implicit=True, env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected completions to exclude %r:\n%s"
% (lang, completions, indent(markedup_content)))
self._assertCompletionsDoNotInclude(buf, trg, completions)
def assertCompletionsDoNotInclude2(self, buf, pos, completions, implicit=True):
"""A version of assertCompletionsDoNotInclude() where you pass in
a Buffer instance instead of marked up content. Sometimes
this is more convenient.
"""
trg = buf.trg_from_pos(pos, implicit=implicit)
if trg is None:
markedup_content = markup_text(buf.accessor.text, pos=pos)
self.fail("given position is not a %s trigger point, "
"expected completions to exclude %r:\n%s"
% (buf.lang, completions, indent(markedup_content)))
self._assertCompletionsDoNotInclude(buf, trg, completions)
def assertCalltipIs2(self, buf, pos, calltip, implicit=True):
"""A variant of assertCalltipIs() where you pass in
a Buffer instance instead of marked up content. Sometimes
this is more convenient.
"""
trg = buf.trg_from_pos(pos, implicit=implicit)
markedup_content = markup_text(buf.accessor.text, pos=trg.pos)
self._assertCalltipIs(buf, trg, markedup_content, calltip, self.lang)
def assertCalltipIs(self, markedup_content, calltip, lang=None,
implicit=True, env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
self._assertCalltipIs(buf, trg, markedup_content, calltip, lang)
def _assertCalltipIs(self, buf, trg, markedup_content, calltip, lang):
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected the following calltip:\n"
" calltip:\n%s\n"
" buffer:\n%s"
% (lang, indent(calltip),
indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_calltips = buf.calltips_from_trg(trg, ctlr=ctlr)
if actual_calltips and actual_calltips[0]:
actual_calltip = actual_calltips[0]
else:
actual_calltip = None
self.assertEqual(calltip, actual_calltip,
"unexpected %s calltip at the given position\n"
" expected:\n%s\n"
" got:\n%s\n"
" eval log:\n%s\n"
" buffer:\n%s"
% (trg.name, indent(calltip and calltip or "(none)"),
indent(actual_calltip and actual_calltip or "(none)"),
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(markedup_content)))
def assertCalltipMatches(self, markedup_content, calltip, lang=None,
implicit=True, env=None, flags=0):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
self._assertCalltipMatches(buf, trg, markedup_content, calltip, lang, flags)
def _assertCalltipMatches(self, buf, trg, markedup_content, expr, lang, flags):
if trg is None:
self.fail("given position is not a %s trigger point, "
"expected the calltip to match the following:\n"
" exression:\n%s\n"
" buffer:\n%s"
% (lang, indent(expr),
indent(markedup_content)))
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
ctlr = _CaptureEvalController()
actual_calltips = buf.calltips_from_trg(trg, ctlr=ctlr)
if actual_calltips and actual_calltips[0]:
actual_calltip = actual_calltips[0]
else:
actual_calltip = None
self.assertNotEquals(re.search(expr, actual_calltip, flags), None,
"unexpected %s calltip at the given position\n"
" expression:\n%s\n"
" got:\n%s\n"
" eval log:\n%s\n"
" buffer:\n%s"
% (trg.name, indent(expr and expr or "(none)"),
indent(actual_calltip and actual_calltip or "(none)"),
indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)),
indent(markedup_content)))
def assertCurrCalltipArgRange(self, markedup_content, calltip,
expected_range, lang=None, implicit=True):
if lang is None:
lang = self.lang
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
content, data = unmark_text(
self.adjust_content(markedup_content))
pos = data["pos"]
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
trg = buf.trg_from_pos(data["trg_pos"], implicit=implicit)
actual_range = buf.curr_calltip_arg_range(trg.pos, calltip,
curr_pos=data["pos"])
self.assertEqual(actual_range, expected_range,
"unexpected current calltip arg range\n"
" expected: %s\n"
" got: %s\n"
" calltip:\n%s\n"
" buffer:\n%s"
% (expected_range, actual_range, indent(calltip),
indent(markedup_content)))
#def assertCompletionRaises(self, markedup_content, exception, lang=None,
# **kwargs):
# """Assert that the given completion raises the given exception.
#
# You may also specify either of the "exc_args" or "exc_pattern"
# keyword args to match the exception's "args" attribute or match
# the stringified exception against a regex pattern.
#
# c.f. http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/307970
# """
# if lang is None:
# lang = self.lang
# if "exc_args" in kwargs:
# exc_args = kwargs["exc_args"]
# del kwargs["exc_args"]
# else:
# exc_args = None
# if "exc_pattern" in kwargs:
# exc_pattern = kwargs["exc_pattern"]
# del kwargs["exc_pattern"]
# else:
# exc_pattern = None
#
# callsig = "the given %s completion" % lang
# try:
# buf, trg = self._get_buf_and_trg(markedup_content, lang)
# if trg is None:
# self.fail("given position is not a %s trigger point, "
# "no completion can be done to see if it raises"
# % self.lang)
# callsig = "completion at %s" % trg
# cplns = buf.cplns_from_trg(trg)
# except exception, exc:
# if exc_args is not None:
# self.failIf(exc.args != exc_args,
# "%s raised %s with unexpected args: "\
# "expected=%r, actual=%r"\
# % (callsig, exc.__class__, exc_args, exc.args))
# if exc_pattern is not None:
# self.failUnless(exc_pattern.search(str(exc)),
# "%s raised %s, but the exception "\
# "does not match '%s': %r"\
# % (callsig, exc.__class__, exc_pattern.pattern,
# str(exc)))
# except:
# exc_info = sys.exc_info()
# print exc_info
# self.fail("%s raised an unexpected exception type: "\
# "expected=%s, actual=%s"\
# % (callsig, exception, exc_info[0]))
# else:
# self.fail("%s did not raise %s" % (callsig, exception))
def assertEvalError(self, markedup_content, log_pattern, lang=None,
implicit=True, env=None):
if lang is None:
lang = self.lang
buf, trg = self._get_buf_and_trg(markedup_content, lang,
implicit=implicit, env=env)
if trg is None:
self.fail("given position is not a %s trigger point, "
"no completion can be done to see if errors"
% self.lang)
if isinstance(buf, CitadelBuffer):
buf.unload() # remove any entry from CIDB to ensure clean test
class TestEvalController(EvalController):
"""A completion controller that captures all eval logging."""
def __init__(self):
EvalController.__init__(self)
self.log = []
def debug(self, msg, *args):
self.log.append(("debug", msg % args))
def info(self, msg, *args):
self.log.append(("info", msg % args))
def warn(self, msg, *args):
self.log.append(("warn", msg % args))
def error(self, msg, *args):
self.log.append(("error", msg % args))
ctlr = TestEvalController()
buf.async_eval_at_trg(trg, ctlr=ctlr)
ctlr.wait()
if not ctlr.is_done():
self.fail("evaluation is not 'done': didn't expect that")
if trg.form == TRG_FORM_CPLN and ctlr.cplns:
self.fail("evalution had results: didn't expect that: %r"
% ctlr.cplns)
elif trg.form == TRG_FORM_CALLTIP and ctlr.calltips:
self.fail("evalution had results: didn't expect that: %r"
% ctlr.cplns)
if log_pattern:
#pprint(ctlr.log)
matching_logs = [(level, msg) for level, msg in ctlr.log
if log_pattern.search(msg)]
self.failUnless(matching_logs,
"the given completion failed but no logs matched the given pattern:\n"
" log_pattern: /%s/\n"
" log messages:\n%s\n"
" buffer:\n%s"
% (log_pattern.pattern,
indent('\n'.join(['%s: %s' % lg for lg in ctlr.log])),
indent(markedup_content)))
#pprint(matching_logs)
def assertCITDLExprIs(self, markedup_content, citdl_expr, lang=None,
prefix_filter=None, implicit=True, trigger_name=None,
**fields):
"""Assert that the preceding CITDL expression at the current position
is as expected.
This uses buf.citdl_expr_from_trg() -- or, for Perl,
buf.citdl_expr_and_prefix_filter_from_trg().
The "prefix_filter" optional argument can be used for Perl to test
the value its relevant function returns.
"""
if lang is None:
lang = self.lang
content, data = unmark_text(
self.adjust_content(markedup_content))
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
langintel = self.mgr.langintel_from_lang(lang)
if trigger_name is None:
trigger_name = "fakey-completion-type"
if lang == "Perl":
# Bit of a hack to fake the trigger length.
if content[data["pos"]-1] in ('>', ':'): # '->' or '::' triggers
length = 2
else:
length = 1
trg = Trigger(lang, TRG_FORM_CPLN, trigger_name,
data["pos"], implicit=implicit, length=length,
**fields)
actual_citdl_expr, actual_prefix_filter \
= langintel.citdl_expr_and_prefix_filter_from_trg(buf, trg)
else:
trg = Trigger(lang, TRG_FORM_CPLN, trigger_name,
data["pos"], implicit=implicit,
**fields)
actual_citdl_expr = langintel.citdl_expr_from_trg(buf, trg)
self.assertEqual(actual_citdl_expr, citdl_expr,
"unexpected actual %s CITDL expr preceding trigger:\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s"
% (lang, citdl_expr, actual_citdl_expr,
indent(markedup_content)))
if lang == "Perl" and prefix_filter is not None:
self.assertEqual(actual_prefix_filter, prefix_filter,
"unexpected actual %s variable prefix filter "
"preceding trigger:\n"
" expected: %r\n"
" got: %r\n"
" buffer:\n%s"
% (lang, prefix_filter, actual_prefix_filter,
indent(markedup_content)))
def _unmark_lex_text(self, markedup_text):
from SilverCity import ScintillaConstants
tokenizer = re.compile(r'(<(SCE_\w+)>(.*?)</\2>)')
tokens = []
text = ''
while markedup_text:
match = tokenizer.search(markedup_text)
if match:
text += markedup_text[:match.start()]
token = {
'end_index': len(text) + len(match.group(3)) - 1,
'start_index': len(text),
'style': getattr(ScintillaConstants, match.group(2)),
'text': match.group(3),
#'end_column': ???,
#'end_line': ???,
#'start_column': ???,
#'start_line': ???,
}
tokens.append(token)
text += match.group(3)
markedup_text = markedup_text[match.end():]
else:
text += markedup_text
markedup_text = ''
return text, tokens
def assertLex(self, markedup_content, lang=None):
"""Lex the given content and assert that the lexed tokens are as
expected.
What is "expected" is given via pseudo-xml markup like this:
fuzzy wuzzy <SCE_UDL_SSL_COMMENTBLOCK>wuzza</SCE_UDL_SSL_COMMENTBLOCK> bear
This example expects that "wuzza" will be a token with style
SCE_UDL_SSL_COMMENTBLOCK.
"""
from codeintel2.accessor import SilverCityAccessor
if lang is None:
lang = self.lang
content, tokens = self._unmark_lex_text(markedup_content)
# Do lexing of this content via the codeintel Buffer's, because
# they already handle all the SilverCity lexer hookup.
path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100))
buf = self.mgr.buf_from_content(content, lang=lang, path=path)
assert isinstance(buf.accessor, SilverCityAccessor)
actual_tokens = buf.accessor.tokens # cheating
for actual_token in actual_tokens:
# There are a few SilverCity token dict keys that we
# don't bother checking.
del actual_token["end_column"]
del actual_token["end_line"]
del actual_token["start_column"]
del actual_token["start_line"]
unmatched_tokens = [t for t in tokens if t not in actual_tokens]
if unmatched_tokens:
self.fail("not all expected %s lex tokens were found in the "
"actual lexer output:\n"
" buffer:\n%s\n"
" actual lexer tokens:\n%s\n"
" unmatched tokens:\n%s\n"
% (lang,
indent(content),
indent(pformat(actual_tokens)),
indent(pformat(unmatched_tokens))))
def do_outline(self, subcmd, opts, path):
"""Scan and outline the structure of the given path.
${cmd_usage}
${cmd_option_list}
"""
extra_lang_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_lang_module_dirs = [sys.path[-1]]
mgr = Manager(extra_lang_module_dirs=extra_lang_module_dirs)
mgr.upgrade()
mgr.initialize()
try:
if '#' in path:
path, anchor = path.rsplit('#', 1)
else:
anchor = None
tree = None
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info(
"skip `%s': couldn't determine language "
"(use --lang option)", path)
return
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
#buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
tree = buf.tree
if anchor is not None:
# Lookup the anchor in the codeintel CIX tree.
lpath = re.split(r'\.|::', anchor)
def blobs_from_tree(tree):
for file_elem in tree:
for blob in file_elem:
yield blob
for elem in blobs_from_tree(tree):
# Generally have 3 types of codeintel trees:
# 1. single-lang file: one <file>, one <blob>
# 2. multi-lang file: one <file>, one or two <blob>'s
# 3. CIX stdlib/catalog file: possibly multiple
# <file>'s, likely multiple <blob>'s
# Allow the first token to be the blob name or lang.
# (This can sometimes be weird, but seems the most
# convenient solution.)
if lpath[0] in (elem.get("name"), elem.get("lang")):
remaining_lpath = lpath[1:]
else:
remaining_lpath = lpath
for name in remaining_lpath:
try:
elem = elem.names[name]
except KeyError:
elem = None
break # try next lang blob
if elem is not None:
break # found one
else:
log.error(
"could not find `%s' definition (or blob) in `%s'",
anchor, path)
return 1
else:
elem = tree
try:
_outline_ci_elem(mgr, elem, quiet=opts.quiet)
except IOError, ex:
if ex.errno == 0:
# Ignore this error from aborting 'less' of this
# command:
# IOError: (0, 'Error')
pass
else:
raise
finally:
mgr.finalize()
def _codeintel_scan():
global _ci_mgr_, despair, despaired
env = None
mtime = None
catalogs = []
now = time.time()
try:
env = _ci_envs_[path]
if env._folders != folders:
raise KeyError
mgr = _ci_mgr_
if now > env._time:
mtime = max(tryGetMTime(env._config_file), tryGetMTime(env._config_default_file))
if env._mtime < mtime:
raise KeyError
except KeyError:
if env is not None:
config_default_file = env._config_default_file
project_dir = env._project_dir
project_base_dir = env._project_base_dir
config_file = env._config_file
else:
config_default_file = os.path.join(CODEINTEL_HOME_DIR, "config")
if not (config_default_file and os.path.exists(config_default_file)):
config_default_file = None
project_dir = None
project_base_dir = None
if path:
# Try to find a suitable project directory (or best guess):
for folder in [".codeintel", ".git", ".hg", "trunk"]:
project_dir = find_folder(path, folder)
if project_dir and (
folder != ".codeintel" or not os.path.exists(os.path.join(project_dir, "db"))
):
if folder.startswith("."):
project_base_dir = os.path.abspath(os.path.join(project_dir, ".."))
else:
project_base_dir = project_dir
break
if not (project_dir and os.path.exists(project_dir)):
project_dir = None
config_file = project_dir and folder == ".codeintel" and os.path.join(project_dir, "config")
if not (config_file and os.path.exists(config_file)):
config_file = None
if _ci_mgr_:
mgr = _ci_mgr_
else:
for thread in threading.enumerate():
if thread.name == "CodeIntel Manager":
thread.finalize() # this finalizes the index, citadel and the manager and waits them to end (join)
mgr = Manager(
extra_module_dirs=_ci_extra_module_dirs_,
db_base_dir=_ci_db_base_dir_,
db_catalog_dirs=_ci_db_catalog_dirs_,
db_import_everything_langs=_ci_db_import_everything_langs,
db_event_reporter=lambda m: logger(view, "event", m),
)
mgr.upgrade()
mgr.initialize()
# Connect the logging file to the handler
condeintel_log_file = os.path.join(mgr.db.base_dir, "codeintel.log")
codeintel_log.handlers = [logging.StreamHandler(open(condeintel_log_file, "w", 1))]
msg = "Starting logging SublimeCodeIntel rev %s (%s) on %s" % (
get_git_revision()[:12],
os.stat(__file__)[stat.ST_MTIME],
datetime.datetime.now().ctime(),
)
codeintel_log.info("%s\n%s" % (msg, "=" * len(msg)))
_ci_mgr_ = mgr
# Load configuration files:
for catalog in mgr.db.get_catalogs_zone().avail_catalogs():
if catalog["lang"] == lang:
catalogs.append(catalog["name"])
config = {
"codeintel_selected_catalogs": catalogs,
"codeintel_max_recursive_dir_depth": 10,
"codeintel_scan_files_in_project": True,
}
_config = {}
try:
tryReadDict(config_default_file, _config)
except Exception, e:
msg = "Malformed configuration file '%s': %s" % (config_default_file, e)
log.error(msg)
codeintel_log.error(msg)
try:
tryReadDict(config_file, _config)
except Exception, e:
msg = "Malformed configuration file '%s': %s" % (config_default_file, e)
log.error(msg)
codeintel_log.error(msg)
def cix2html(opts, path):
"""Turn cix file into html API documentation.
Example:
cix2html path/to/foo.cix#AClass.amethod
cix2html path/to/foo.cix -o file.html
${cmd_usage}
${cmd_option_list}
"""
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
def blobs_from_tree(tree):
for file_elem in tree:
for blob in file_elem:
yield blob
if '#' in path:
path, anchor = path.rsplit('#', 1)
else:
anchor = None
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
# buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
tree = buf.tree
if anchor is not None:
# Lookup the anchor in the codeintel CIX tree.
lpath = re.split(r'\.|::', anchor)
for elem in blobs_from_tree(tree):
# Generally have 3 types of codeintel trees:
# 1. single-lang file: one <file>, one <blob>
# 2. multi-lang file: one <file>, one or two <blob>'s
# 3. CIX stdlib/catalog file: possibly multiple
# <file>'s, likely multiple <blob>'s
# Allow the first token to be the blob name or lang.
# (This can sometimes be weird, but seems the most
# convenient solution.)
if lpath[0] in (elem.get("name"), elem.get("lang")):
remaining_lpath = lpath[1:]
else:
remaining_lpath = lpath
for name in remaining_lpath:
try:
elem = elem.names[name]
except KeyError:
elem = None
break # try next lang blob
if elem is not None:
break # found one
else:
log.error("could not find `%s' definition (or blob) in `%s'",
anchor, path)
return 1
else:
elem = tree
try:
if elem.tag == "codeintel":
_html_ci_elem(opts, elem.getchildren()[0])
else:
_html_ci_elem(opts, elem)
except IOError, ex:
if ex.errno == 0:
# Ignore this error from aborting 'less' of 'ci2 outline'
# output:
# IOError: (0, 'Error')
pass
else:
raise
except Exception, e:
import traceback
traceback.print_exc()
return # log silently
logStream = logging.StreamHandler(sys.stderr)
logStream.setFormatter(logging.Formatter("%(name)s: %(levelname)s: %(message)s"))
logging.getLogger("codeintel").addHandler(logStream)
logger = logging.getLogger("codeintel_server")
logger.addHandler(logStream)
logger.setLevel(logging.WARNING)
manager = Manager(
# db_base_dir = path.join(CI_DIR, 'db'),
# extra_module_dirs = [path.join(CI_DIR, 'codeintel2'),],
# db_import_everything_langs = None,
# db_catalog_dirs = []
)
manager.upgrade()
manager.initialize()
scanned_langs = []
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Run codeintel commands as a daemon or via stdin")
parser.add_argument("mode", help="Mode of operation", choices=["daemon", "completions", "definitions", "calltips", "catalogs"])
parser.add_argument("--row", type=int, help="The row to read from")
parser.add_argument("--column", type=int, help="The column to read from")
parser.add_argument("--path", help="The path of the file")
parser.add_argument("--basedir", help="The basedir of the file")
parser.add_argument("--language", help="The language of the file")
parser.add_argument("--port", type=int, help="The port for the daemon to listen on")
parser.add_argument("--nodoc", help="Don't include docstrings in output")
parser.add_argument("--catalogs", help="Catalogs to include (comma-separated)")
args = parser.parse_args()
def do_outline(self, subcmd, opts, path):
"""Scan and outline the structure of the given path.
${cmd_usage}
${cmd_option_list}
"""
extra_lang_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_lang_module_dirs = [sys.path[-1]]
mgr = Manager(extra_lang_module_dirs=extra_lang_module_dirs)
mgr.upgrade()
mgr.initialize()
try:
if "#" in path:
path, anchor = path.rsplit("#", 1)
else:
anchor = None
tree = None
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info("skip `%s': couldn't determine language " "(use --lang option)", path)
return
if path.endswith(".cix"):
tree = tree_from_cix(open(path, "r").read())
# buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang))
tree = buf.tree
if anchor is not None:
# Lookup the anchor in the codeintel CIX tree.
lpath = re.split(r"\.|::", anchor)
def blobs_from_tree(tree):
for file_elem in tree:
for blob in file_elem:
yield blob
for elem in blobs_from_tree(tree):
# Generally have 3 types of codeintel trees:
# 1. single-lang file: one <file>, one <blob>
# 2. multi-lang file: one <file>, one or two <blob>'s
# 3. CIX stdlib/catalog file: possibly multiple
# <file>'s, likely multiple <blob>'s
# Allow the first token to be the blob name or lang.
# (This can sometimes be weird, but seems the most
# convenient solution.)
if lpath[0] in (elem.get("name"), elem.get("lang")):
remaining_lpath = lpath[1:]
else:
remaining_lpath = lpath
for name in remaining_lpath:
try:
elem = elem.names[name]
except KeyError:
elem = None
break # try next lang blob
if elem is not None:
break # found one
else:
log.error("could not find `%s' definition (or blob) in `%s'", anchor, path)
return 1
else:
elem = tree
try:
_outline_ci_elem(mgr, elem, quiet=opts.quiet)
except IOError, ex:
if ex.errno == 0:
# Ignore this error from aborting 'less' of this
# command:
# IOError: (0, 'Error')
pass
else:
raise
finally:
mgr.finalize()
def do_outline(self, subcmd, opts, path):
"""Print code outline of the given file.
You can specify a lookup path into the file code outline to
display via URL-anchor syntax, e.g.:
ci2 outline path/to/foo.py#AClass.amethod
${cmd_usage}
${cmd_option_list}
"""
import re
from ci2 import _outline_ci_elem
from codeintel2.manager import Manager
from codeintel2.util import tree_from_cix
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if '#' in path:
path, anchor = path.rsplit('#', 1)
else:
anchor = None
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
# buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
tree = buf.tree
if anchor is not None:
# Lookup the anchor in the codeintel CIX tree.
lpath = re.split(r'\.|::', anchor)
def blobs_from_tree(tree):
for file_elem in tree:
for blob in file_elem:
yield blob
for elem in blobs_from_tree(tree):
# Generally have 3 types of codeintel trees:
# 1. single-lang file: one <file>, one <blob>
# 2. multi-lang file: one <file>, one or two <blob>'s
# 3. CIX stdlib/catalog file: possibly multiple
# <file>'s, likely multiple <blob>'s
# Allow the first token to be the blob name or lang.
# (This can sometimes be weird, but seems the most
# convenient solution.)
if lpath[0] in (elem.get("name"), elem.get("lang")):
remaining_lpath = lpath[1:]
else:
remaining_lpath = lpath
for name in remaining_lpath:
try:
elem = elem.names[name]
except KeyError:
elem = None
break # try next lang blob
if elem is not None:
break # found one
else:
self.log.error("could not find `%s' definition (or blob) in `%s'",
anchor, path)
return 1
else:
elem = tree
try:
_outline_ci_elem(elem, brief=opts.brief, doSort=opts.doSort)
except IOError as ex:
if ex.errno == 0:
# Ignore this error from aborting 'less' of 'ci2 outline'
# output:
# IOError: (0, 'Error')
pass
else:
raise
finally:
mgr.finalize()
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if opts.time_it:
start = time.time()
quiet = opts.quiet
if opts.time_it or opts.time_details:
opts.force = True
tree = None
for path in _paths_from_path_patterns(path_patterns,
recursive=opts.recursive,
includes=opts.includes):
if opts.time_it:
sys.stderr.write(path+"\n")
if opts.time_details:
start1 = time.time()
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info("skip `%s': couldn't determine language", path)
continue
buf = mgr.buf_from_path(path, lang=lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
if opts.force:
buf.scan()
if tree is None:
tree = ET.Element("codeintel", version="2.0")
file_elem = ET.SubElement(tree, "file",
lang=buf.lang,
mtime=str(int(time.time())),
path=os.path.basename(path))
for lang, blob in sorted(buf.blob_from_lang.items()):
blob = buf.blob_from_lang[lang]
file_elem.append(blob)
if opts.time_details:
delta = time.time() - start1
sys.stderr.write("%.3f %s\n" % (delta, path))
sys.stderr.flush()
if tree is not None:
if opts.stripfuncvars:
# For stdlibs, we don't care about variables inside of
# functions and they take up a lot of space.
for function in tree.getiterator('scope'):
if function.get('ilk') == 'function':
function[:] = [child for child in function
if child.tag != 'variable']
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
if not quiet:
sys.stdout.write('<?xml version="1.0" encoding="UTF-8"?>\n')
ET.dump(tree)
if opts.time_it:
end = time.time()
sys.stderr.write("scan took %.3fs\n" % (end-start))
finally:
mgr.finalize()
def do_play(self, subcmd, opts):
"""Run my current play/dev code.
${cmd_usage}
${cmd_option_list}
"""
import pprint
import random
import ciElementTree as ET
from codeintel2.manager import Manager
from codeintel2.tree import pretty_tree_from_tree
from codeintel2.common import LogEvalController, Error
from codeintel2.util import tree_from_cix, dedent, unmark_text, banner
from ci2 import _escaped_text_from_text
if False:
lang = "CSS"
markedup_content = dedent("""
/* http://www.w3.org/TR/REC-CSS2/fonts.html#propdef-font-weight */
h1 {
border: 1px solid black;
font-weight /* hi */: <|> !important
}
""")
content, data = unmark_text(markedup_content)
pos = data["pos"]
mgr = Manager()
# mgr.upgrade() # Don't need it for just CSS usage.
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path="play.css")
trg = buf.trg_from_pos(pos)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
completions = buf.cplns_from_trg(trg)
print("COMPLETIONS: %r" % completions)
finally:
mgr.finalize()
elif False:
lang = "Python"
path = os.path.join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
markedup_content = dedent("""
import sys, os
class Foo:
def bar(self):
pass
sys.<|>path # should have path in completion list
f = Foo()
""")
content, data = unmark_text(markedup_content)
print(banner(path))
print(_escaped_text_from_text(content, "whitespace"))
pos = data["pos"]
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path=path)
print(banner("cix", '-'))
print(buf.cix)
trg = buf.trg_from_pos(pos)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
print(banner("completions", '-'))
ctlr = LogEvalController(self.log)
buf.async_eval_at_trg(trg, ctlr)
ctlr.wait(2) # XXX
if not ctlr.is_done():
ctlr.abort()
raise Error("XXX async eval timed out")
pprint.pprint(ctlr.cplns)
print(banner(None))
finally:
mgr.finalize()
elif False:
lang = "Ruby"
path = os.path.join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
markedup_content = dedent("""\
r<1>equire 'net/http'
include Net
req = HTTPRequest.new
req.<2>get()
""")
content, data = unmark_text(markedup_content)
print(banner(path))
print(_escaped_text_from_text(content, "whitespace"))
pos = data[1]
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path=path)
print(banner("cix", '-'))
cix = buf.cix
print(ET.tostring(pretty_tree_from_tree(tree_from_cix(cix))))
trg = buf.trg_from_pos(pos, implicit=False)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
print(banner("completions", '-'))
ctlr = LogEvalController(self.log)
buf.async_eval_at_trg(trg, ctlr)
ctlr.wait(30) # XXX
if not ctlr.is_done():
ctlr.abort()
raise Error("XXX async eval timed out")
pprint.pprint(ctlr.cplns)
print(banner(None))
finally:
mgr.finalize()
_fn = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'path.ini')
if os.path.isfile(_fn):
with open(_fn, 'r') as f:
paths = f.readline()
print 'Paths:\n' + '\n'.join(paths.split(os.pathsep))
d = {}
d['phpExtraPaths'] = paths
d['pythonExtraPaths'] = paths
d['perlExtraPaths'] = paths
d['javascriptExtraPaths'] = paths
d['rubyExtraPaths'] = paths
env = SimplePrefsEnvironment(**d)
mgr = Manager()
mgr.upgrade()
mgr.initialize()
#------------------------
def file_text(fn):
try:
with open(fn, 'r') as f:
text = f.readlines()
return ''.join(text)
except IOError as e:
if e.errno == errno.ENOENT: # No such file or directory
logging.error(e.strerror + ': "' + fn + '"')
else:
logging.error(e.strerror)
return None
