def htmlify(rel_path, text):
from codeintel2.citadel import CitadelBuffer
from codeintel2.common import CodeIntelError
from codeintel2.util import guess_lang_from_path
path = os.path.join(opts.root, rel_path)
try:
lang = guess_lang_from_path(path)
except CodeIntelError:
return None
try:
buf = opts.mgr.buf_from_path(path, lang=lang)
except CodeIntelError as ex:
if ex.message.startswith("File too big."):
log.info("%s: %s", rel_path, ex.message)
return None # Nothing we can do about that, and the user can't
# fix this ever.
raise
if not isinstance(buf, CitadelBuffer):
log.info("%s: language %s does not have CIX, not htmlifying",
rel_path, lang)
return None
if lang == "Tcl":
# Tcl is busted, it has no async_eval_at_trg; Komodo just plain has a
# different system for handling tcl...
return None
log.info("htmlifying %s as %s", rel_path, lang)
return CiHtmlifier(buf)
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
extra_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_module_dirs = [sys.path[-1]]
mgr = Manager(extra_module_dirs=extra_module_dirs)
mgr.upgrade()
mgr.initialize()
try:
tree = None
for path in _paths_from_path_patterns(path_patterns):
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info("skip `%s': couldn't determine language " "(use --lang option)", path)
continue
buf = mgr.buf_from_path(path, lang=opts.lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang))
buf.scan() # force a fresh scan
tree = buf.tree
for severity, msg in check_tree(tree):
dump = {"warning": log.warn, "error": log.error}[severity]
dump(msg)
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
ET.dump(tree)
finally:
mgr.finalize()
def on_modified(self, view):
path = view.file_name()
lang, _ = os.path.splitext(os.path.basename(view.settings().get("syntax")))
try:
lang = lang or guess_lang_from_path(path)
except CodeIntelError:
pass
live = view.settings().get("codeintel_live", True)
pos = view.sel()[0].end()
text = view.substr(sublime.Region(pos - 1, pos))
_sentinel = sentinel.get(path)
is_fill_char = text and text[-1] in cpln_fillup_chars.get(lang, "")
sentinel[path] = _sentinel if _sentinel is not None else pos if is_fill_char else None
if not live and text and sentinel[path] is not None:
live = True
if live:
if not hasattr(view, "command_history") or view.command_history(0)[0] == "insert":
autocomplete(view, 0 if is_fill_char else 200, 50 if is_fill_char else 600, is_fill_char, args=[path])
else:
view.run_command("hide_auto_complete")
else:
def _scan_callback(view, path):
content = view.substr(sublime.Region(0, view.size()))
lang, _ = os.path.splitext(os.path.basename(view.settings().get("syntax")))
codeintel_scan(view, path, content, lang)
queue(view, _scan_callback, 3000, args=[path])
def guess_lang(view=None, path=None):
if not view.settings().get('codeintel', True):
return None
syntax = None
if view:
syntax = os.path.splitext(
os.path.basename(view.settings().get('syntax')))[0]
id = view.id()
_k_ = '%s::%s' % (syntax, path)
try:
return languages[id][_k_]
except KeyError:
pass
languages.setdefault(id, {})
lang = None
_codeintel_syntax_map = dict(
(k.lower(), v)
for k, v in view.settings().get('codeintel_syntax_map', {}).items())
_lang = lang = syntax and _codeintel_syntax_map.get(syntax.lower(), syntax)
mgr = codeintel_manager()
if not mgr.is_citadel_lang(lang) and not mgr.is_cpln_lang(lang):
lang = None
if mgr.is_citadel_lang(syntax) or mgr.is_cpln_lang(syntax):
_lang = lang = syntax
else:
if view and not path:
path = view.file_name()
if path:
try:
_lang = lang = guess_lang_from_path(path)
except CodeIntelError:
languages[id][_k_] = None
return
_codeintel_disabled_languages = [
l.lower()
for l in view.settings().get('codeintel_disabled_languages', [])
]
if lang and lang.lower() in _codeintel_disabled_languages:
logger(view, 'info', "skip `%s': disabled language" % lang)
languages[id][_k_] = None
return
if not lang and _lang and _lang not in ('Console', ):
if mgr:
logger(
view, 'info',
"Invalid language: %s. Available: %s" % (_lang, ', '.join(
set(mgr.get_citadel_langs() + mgr.get_cpln_langs()))))
else:
logger(view, 'info', "Invalid language: %s" % _lang)
languages[id][_k_] = lang
return lang
def worker(queue, lock, cix_dir, root="/"):
"""
worker procedure
"""
fix_module_path()
import dxr.mime
from codeintel2.citadel import CitadelBuffer
from codeintel2.common import CodeIntelError
from codeintel2.manager import Manager
from codeintel2.util import guess_lang_from_path
logging.getLogger("codeintel").setLevel(logging.ERROR)
log.info("starting indexing using %s", multiprocessing.current_process().name)
mgr = Manager(db_base_dir=cix_dir)
#mgr.upgrade()
mgr.initialize()
while True:
file_path = queue.get()
if file_path is None:
# marker for end of list
queue.put(None) # put it back so others can quit too
break
rel_path = os.path.relpath(file_path, root)
try:
lang = guess_lang_from_path(file_path)
except CodeIntelError:
log.info("%s: Cannot determine language, skipping", rel_path)
continue
# the file
with open(file_path, "r") as source_file:
data = source_file.read()
# Discard non-text files
if not dxr.mime.is_text(file_path, data):
continue
try:
buf = mgr.buf_from_path(file_path, lang=lang)
except CodeIntelError as ex:
if ex.message.startswith("File too big."):
log.info("%s: %s", file_path, ex.message)
continue # Nothing we can do about that, and the user can't
# fix this ever.
raise
if not isinstance(buf, CitadelBuffer):
log.info("%s: language %s does not have CIX, skipping",
rel_path, lang)
continue
log.info("%s: Using language %s", rel_path, lang)
buf.scan()
mgr.finalize()
def guess_lang(view=None, path=None):
if not view.settings().get("codeintel", True):
return None
syntax = None
if view:
syntax = os.path.splitext(os.path.basename(view.settings().get("syntax")))[0]
id = view.id()
_k_ = "%s::%s" % (syntax, path)
try:
return languages[id][_k_]
except KeyError:
pass
languages.setdefault(id, {})
lang = None
_codeintel_syntax_map = dict((k.lower(), v) for k, v in view.settings().get("codeintel_syntax_map", {}).items())
_lang = lang = syntax and _codeintel_syntax_map.get(syntax.lower(), syntax)
mgr = codeintel_manager()
if not mgr.is_citadel_lang(lang) and not mgr.is_cpln_lang(lang):
lang = None
if mgr.is_citadel_lang(syntax) or mgr.is_cpln_lang(syntax):
_lang = lang = syntax
else:
if view and not path:
path = view.file_name()
if path:
try:
_lang = lang = guess_lang_from_path(path)
except CodeIntelError:
languages[id][_k_] = None
return
_codeintel_disabled_languages = [l.lower() for l in view.settings().get("codeintel_disabled_languages", [])]
if lang and lang.lower() in _codeintel_disabled_languages:
logger(view, "info", "skip `%s': disabled language" % lang)
languages[id][_k_] = None
return
if not lang and _lang and _lang not in ("Console",):
if mgr:
logger(
view,
"info",
"Invalid language: %s. Available: %s"
% (_lang, ", ".join(set(mgr.get_citadel_langs() + mgr.get_cpln_langs()))),
)
else:
logger(view, "info", "Invalid language: %s" % _lang)
languages[id][_k_] = lang
return lang
def _fillScanInputsTestCase():
for dpath, dnames, fnames in os.walk(gInputsDir):
# Don't descend into SCC control dirs.
scc_dirs = [".svn", "CVS", ".hg", ".git"]
for scc_dir in scc_dirs:
if scc_dir in dnames:
dnames.remove(scc_dir)
if dpath == gInputsDir and "unicode" in dnames:
# The scan_inputs/unicode is where the unicode test files
# are placed. Don't descend into here. They are handled elsewhere.
dnames.remove("unicode")
if ".svn" in dpath.split(os.sep):
# Skip subversion directories.
continue
for fname in fnames:
fpath = os.path.join(dpath, fname)[len(gInputsDir) + len(os.sep):]
if not isfile(join(dpath, fname)):
# With our Unicode testing we may have a directory that
# Python's os.walk() doesn't recognize as a dir, defaults to
# a file and hands it to us here. Skip those.
continue
if fname == ".DS_Store": continue
if fpath.endswith(".swp"): continue
if fpath.endswith("~"): continue
if fpath.endswith("__pycache__"): continue
if fpath.endswith(".pyc"): continue
if fpath.endswith(".pyo"): continue
if fpath.endswith(".pod"): continue
if fpath.endswith(".options"): continue # skip input option files
if fpath.endswith(".tags"): continue # skip tags files
lang = guess_lang_from_path(fpath)
# Manual hack to detect as Python 3.
if lang == "Python" and "py3" in fpath:
lang = "Python3"
safe_lang = safe_lang_from_lang(lang)
# Set tags for this test case.
tags = [safe_lang]
tagspath = join(dpath, fname + ".tags") # ws-separate set of tags
if exists(tagspath):
tags += open(tagspath, 'r').read().split()
def makeTestFunction(fpath_, tags_):
testFunction \
= lambda self, fpath=fpath_: _testOneInputFile(self, fpath_, tags=tags_)
testFunction.tags = tags_
return testFunction
name = "test_path:" + fpath
setattr(ScanInputsTestCase, name, makeTestFunction(fpath, tags))
_addUnicodeScanInputTests()
def _fillScanInputsTestCase():
for dpath, dnames, fnames in os.walk(gInputsDir):
# Don't descend into SCC control dirs.
scc_dirs = [".svn", "CVS", ".hg", ".git"]
for scc_dir in scc_dirs:
if scc_dir in dnames:
dnames.remove(scc_dir)
if dpath == gInputsDir and "unicode" in dnames:
# The scan_inputs/unicode is where the unicode test files
# are placed. Don't descend into here. They are handled elsewhere.
dnames.remove("unicode")
if ".svn" in dpath.split(os.sep):
# Skip subversion directories.
continue
for fname in fnames:
fpath = os.path.join(dpath, fname)[len(gInputsDir)+len(os.sep):]
if not isfile(join(dpath, fname)):
# With our Unicode testing we may have a directory that
# Python's os.walk() doesn't recognize as a dir, defaults to
# a file and hands it to us here. Skip those.
continue
if fname == ".DS_Store": continue
if fpath.endswith(".swp"): continue
if fpath.endswith("~"): continue
if fpath.endswith("__pycache__"): continue
if fpath.endswith(".pyc"): continue
if fpath.endswith(".pyo"): continue
if fpath.endswith(".pod"): continue
if fpath.endswith(".options"): continue # skip input option files
if fpath.endswith(".tags"): continue # skip tags files
lang = guess_lang_from_path(fpath)
# Manual hack to detect as Python 3.
if lang == "Python" and "py3" in fpath:
lang = "Python3"
safe_lang = safe_lang_from_lang(lang)
# Set tags for this test case.
tags = [safe_lang]
tagspath = join(dpath, fname + ".tags") # ws-separate set of tags
if exists(tagspath):
tags += open(tagspath, 'r').read().split()
def makeTestFunction(fpath_, tags_):
testFunction \
= lambda self, fpath=fpath_: _testOneInputFile(self, fpath_, tags=tags_)
testFunction.tags = tags_
return testFunction
name = "test_path:"+fpath
setattr(ScanInputsTestCase, name, makeTestFunction(fpath, tags))
_addUnicodeScanInputTests()
def guess_lang(view=None, path=None):
if not view or not view.settings().get('codeintel', True):
return None
syntax = None
if view:
syntax = os.path.splitext(os.path.basename(view.settings().get('syntax')))[0]
vid = view.id()
_k_ = '%s::%s' % (syntax, path)
try:
return languages[vid][_k_]
except KeyError:
pass
languages.setdefault(vid, {})
lang = None
_codeintel_syntax_map = dict((k.lower(), v) for k, v in view.settings().get('codeintel_syntax_map', {}).items())
_lang = lang = syntax and _codeintel_syntax_map.get(syntax.lower(), syntax)
folders = getattr(view.window(), 'folders', lambda: [])() # FIXME: it's like this for backward compatibility (<= 2060)
folders_id = str(hash(frozenset(folders)))
mgr = codeintel_manager(folders_id)
if not mgr.is_citadel_lang(lang) and not mgr.is_cpln_lang(lang):
lang = None
if mgr.is_citadel_lang(syntax) or mgr.is_cpln_lang(syntax):
_lang = lang = syntax
else:
if view and not path:
path = view.file_name()
if path:
try:
_lang = lang = guess_lang_from_path(path)
except CodeIntelError:
languages[vid][_k_] = None
return
_codeintel_disabled_languages = [l.lower() for l in view.settings().get('codeintel_disabled_languages', [])]
if lang and lang.lower() in _codeintel_disabled_languages:
logger(view, 'debug', "skip `%s': disabled language" % lang)
languages[vid][_k_] = None
return
if not lang and _lang and _lang in ('Console', 'Plain text'):
if mgr:
logger(view, 'debug', "Invalid language: %s. Available: %s" % (_lang, ', '.join(set(mgr.get_citadel_langs() + mgr.get_cpln_langs()))))
else:
logger(view, 'debug', "Invalid language: %s" % _lang)
languages[vid][_k_] = lang
return lang
def guess_lang(view=None, path=None):
if not view or not view.settings().get('codeintel', True):
return None
syntax = None
if view:
syntax = os.path.splitext(os.path.basename(view.settings().get('syntax')))[0]
vid = view.id()
_k_ = '%s::%s' % (syntax, path)
try:
return languages[vid][_k_]
except KeyError:
pass
languages.setdefault(vid, {})
lang = None
_codeintel_syntax_map = dict((k.lower(), v) for k, v in view.settings().get('codeintel_syntax_map', {}).items())
_lang = lang = syntax and _codeintel_syntax_map.get(syntax.lower(), syntax)
folders = getattr(view.window(), 'folders', lambda: [])() # FIXME: it's like this for backward compatibility (<= 2060)
folders_id = str(hash(frozenset(folders)))
mgr = codeintel_manager(folders_id)
if not mgr.is_citadel_lang(lang) and not mgr.is_cpln_lang(lang):
lang = None
if mgr.is_citadel_lang(syntax) or mgr.is_cpln_lang(syntax):
_lang = lang = syntax
else:
if view and not path:
path = view.file_name()
if path:
try:
_lang = lang = guess_lang_from_path(path)
except CodeIntelError:
languages[vid][_k_] = None
return
_codeintel_enabled_languages = [l.lower() for l in view.settings().get('codeintel_enabled_languages', [])]
if lang and lang.lower() not in _codeintel_enabled_languages:
languages[vid][_k_] = None
return None
if not lang and _lang and _lang in ('Console', 'Plain text'):
if mgr:
logger(view, 'debug', "Invalid language: %s. Available: %s" % (_lang, ', '.join(set(mgr.get_citadel_langs() + mgr.get_cpln_langs()))))
else:
logger(view, 'debug', "Invalid language: %s" % _lang)
languages[vid][_k_] = lang
return lang
def guess_lang(view=None, path=None):
if not view.settings().get('codeintel', True):
return None
mgr = codeintel_manager()
_codeintel_syntax_map = dict(
(k.lower(), v)
for k, v in view.settings().get('codeintel_syntax_map', {}).items())
_codeintel_disabled_languages = [
l.lower()
for l in view.settings().get('codeintel_disabled_languages', [])
]
lang = None
syntax = None
if view:
syntax = os.path.splitext(
os.path.basename(view.settings().get('syntax')))[0]
lang = syntax and _codeintel_syntax_map.get(syntax.lower(), syntax)
if not mgr.is_citadel_lang(lang) and not mgr.is_cpln_lang(lang):
lang = None
if mgr.is_citadel_lang(syntax) or mgr.is_cpln_lang(syntax):
lang = syntax
else:
if view and not path:
path = view.file_name()
if path:
try:
lang = guess_lang_from_path(path)
except CodeIntelError:
logger(view, 'warning',
"skip `%s': couldn't determine language" % path)
return
if lang and lang.lower() in _codeintel_disabled_languages:
logger(view, 'info', "skip `%s': disabled language" % lang)
return
if not lang:
if mgr:
logger(
view, 'info',
"Invalid language: %s. Available: %s" % (lang, ', '.join(
set(mgr.get_citadel_langs() + mgr.get_cpln_langs()))))
else:
logger(view, 'info', "Invalid language: %s" % lang)
return lang
def guess_lang(view=None, path=None):
if not view.settings().get('codeintel', True):
return None
mgr = codeintel_manager()
_codeintel_syntax_map = dict((k.lower(), v) for k, v in view.settings().get('codeintel_syntax_map', {}).items())
_codeintel_disabled_languages = [l.lower() for l in view.settings().get('codeintel_disabled_languages', [])]
lang = None
syntax = None
if view:
syntax = os.path.splitext(os.path.basename(view.settings().get('syntax')))[0]
lang = syntax and _codeintel_syntax_map.get(syntax.lower(), syntax)
if not mgr.is_citadel_lang(lang) and not mgr.is_cpln_lang(lang):
lang = None
if mgr.is_citadel_lang(syntax) or mgr.is_cpln_lang(syntax):
lang = syntax
else:
if view and not path:
path = view.file_name()
if path:
try:
lang = guess_lang_from_path(path)
except CodeIntelError:
logger(view, 'warning', "skip `%s': couldn't determine language" % path)
return
if lang and lang.lower() in _codeintel_disabled_languages:
logger(view, 'info', "skip `%s': disabled language" % lang)
return
if not lang:
if mgr:
logger(view, 'info', "Invalid language: %s. Available: %s" % (lang, ', '.join(set(mgr.get_citadel_langs() + mgr.get_cpln_langs()))))
else:
logger(view, 'info', "Invalid language: %s" % lang)
return lang
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
extra_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_module_dirs = [sys.path[-1]]
mgr = Manager(extra_module_dirs=extra_module_dirs)
mgr.upgrade()
mgr.initialize()
try:
tree = None
for path in _paths_from_path_patterns(path_patterns):
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info(
"skip `%s': couldn't determine language "
"(use --lang option)", path)
continue
buf = mgr.buf_from_path(path, lang=opts.lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
buf.scan() # force a fresh scan
tree = buf.tree
for severity, msg in check_tree(tree):
dump = {"warning": log.warn, "error": log.error}[severity]
dump(msg)
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
ET.dump(tree)
finally:
mgr.finalize()
def on_modified(self, view):
path = view.file_name()
lang, _ = os.path.splitext(
os.path.basename(view.settings().get('syntax')))
try:
lang = lang or guess_lang_from_path(path)
except CodeIntelError:
pass
live = view.settings().get('codeintel_live', True)
pos = view.sel()[0].end()
text = view.substr(sublime.Region(pos - 1, pos))
_sentinel = sentinel.get(path)
is_fill_char = (text and text[-1] in cpln_fillup_chars.get(lang, ''))
sentinel[
path] = _sentinel if _sentinel is not None else pos if is_fill_char else None
if not live and text and sentinel[path] is not None:
live = True
if live:
if not hasattr(view, 'command_history') or view.command_history(
0)[0] == 'insert':
autocomplete(view,
0 if is_fill_char else 200,
50 if is_fill_char else 600,
is_fill_char,
args=[path])
else:
view.run_command('hide_auto_complete')
else:
def _scan_callback(view, path):
content = view.substr(sublime.Region(0, view.size()))
lang, _ = os.path.splitext(
os.path.basename(view.settings().get('syntax')))
codeintel_scan(view, path, content, lang)
queue(view, _scan_callback, 3000, args=[path])
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
import time
import ciElementTree as ET
from ci2 import _paths_from_path_patterns
from codeintel2.manager import Manager
from codeintel2.citadel import CitadelBuffer
from codeintel2.common import CodeIntelError
from codeintel2.tree import pretty_tree_from_tree
from codeintel2.util import guess_lang_from_path
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if opts.time_it:
start = time.time()
quiet = opts.quiet
if opts.time_it or opts.time_details:
opts.force = True
scan_count = 0
lang_warnings = set()
tree = None
for path in _paths_from_path_patterns(path_patterns,
recursive=opts.recursive,
includes=opts.includes):
if opts.time_it:
sys.stderr.write(path + "\n")
if opts.time_details:
start1 = time.time()
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
self.log.info("skip `%s': couldn't determine language", path)
continue
try:
buf = mgr.buf_from_path(path, lang=lang)
except OSError as ex:
# Couldn't access the file.
if not opts.recursive:
raise
# Ignore files we don't really care about.
self.log.warn("%r - %r", ex, path)
continue
if not isinstance(buf, CitadelBuffer):
if opts.recursive:
# Ignore files that scanning isn't provided for.
continue
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
scan_count += 1
if scan_count % 10 == 0:
self.log.info("%d scanning %r", scan_count, path)
try:
if opts.force:
buf.scan()
if tree is None:
tree = ET.Element("codeintel", version="2.0")
file_elem = ET.SubElement(tree, "file",
lang=buf.lang,
mtime=str(int(time.time())),
path=os.path.basename(path))
for lang, blob in sorted(buf.blob_from_lang.items()):
blob = buf.blob_from_lang[lang]
file_elem.append(blob)
except KeyError as ex:
# Unknown cile language.
if not opts.recursive:
raise
message = str(ex)
if message not in lang_warnings:
lang_warnings.add(message)
self.log.warn("Skipping unhandled language %s", message)
if opts.time_details:
delta = time.time() - start1
sys.stderr.write("%.3f %s\n" % (delta, path))
sys.stderr.flush()
if tree is not None:
if opts.stripfuncvars:
# For stdlibs, we don't care about variables inside of
# functions and they take up a lot of space.
for function in tree.getiterator('scope'):
if function.get('ilk') == 'function':
function[:] = [child for child in function
if child.tag != 'variable']
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
if not quiet:
sys.stdout.write('<?xml version="1.0" encoding="UTF-8"?>\n')
ET.dump(tree)
if opts.time_it:
end = time.time()
sys.stderr.write("scan took %.3fs\n" % (end - start))
finally:
mgr.finalize()
def codeintel_scan(view,
path,
content,
lang,
callback=None,
pos=None,
forms=None):
global despair
for thread in threading.enumerate():
if thread.isAlive() and thread.name == "scanning thread":
logger(
view,
'info',
"Updating indexes... The first time this can take a while. Do not despair!",
timeout=20000,
delay=despair)
despair = 0
return
try:
lang = lang or guess_lang_from_path(path)
except CodeIntelError:
logger(view, 'warning',
"skip `%s': couldn't determine language" % path)
return
if lang.lower() in [
l.lower()
for l in view.settings().get('codeintel_disabled_languages', [])
]:
return
is_scratch = view.is_scratch()
is_dirty = view.is_dirty()
folders = getattr(view.window(), 'folders', lambda: [])(
) # FIXME: it's like this for backward compatibility (<= 2060)
def _codeintel_scan():
global _ci_mgr_, despair, despaired
env = None
mtime = None
catalogs = []
now = time.time()
try:
env = _ci_envs_[path]
if env._folders != folders:
raise KeyError
mgr = _ci_mgr_
if now > env._time:
mtime = max(tryGetMTime(env._config_file),
tryGetMTime(env._config_default_file))
if env._mtime < mtime:
raise KeyError
except KeyError:
if env is not None:
config_default_file = env._config_default_file
project_dir = env._project_dir
project_base_dir = env._project_base_dir
config_file = env._config_file
else:
config_default_file = os.path.join(CODEINTEL_HOME_DIR,
'config')
if not (config_default_file
and os.path.exists(config_default_file)):
config_default_file = None
project_dir = None
project_base_dir = None
if path:
# Try to find a suitable project directory (or best guess):
for folder in ['.codeintel', '.git', '.hg', 'trunk']:
project_dir = find_folder(path, folder)
if project_dir and (
folder != '.codeintel' or not os.path.exists(
os.path.join(project_dir, 'db'))):
if folder.startswith('.'):
project_base_dir = os.path.abspath(
os.path.join(project_dir, '..'))
else:
project_base_dir = project_dir
break
if not (project_dir and os.path.exists(project_dir)):
project_dir = None
config_file = project_dir and folder == '.codeintel' and os.path.join(
project_dir, 'config')
if not (config_file and os.path.exists(config_file)):
config_file = None
if _ci_mgr_:
mgr = _ci_mgr_
else:
for thread in threading.enumerate():
if thread.name == "CodeIntel Manager":
thread.finalize(
) # this finalizes the index, citadel and the manager and waits them to end (join)
mgr = Manager(
extra_module_dirs=_ci_extra_module_dirs_,
db_base_dir=_ci_db_base_dir_,
db_catalog_dirs=_ci_db_catalog_dirs_,
db_import_everything_langs=_ci_db_import_everything_langs,
db_event_reporter=lambda m: logger(view, 'event', m),
)
mgr.upgrade()
mgr.initialize()
# Connect the logging file to the handler
condeintel_log_file = os.path.join(mgr.db.base_dir,
'codeintel.log')
codeintel_log.handlers = [
logging.StreamHandler(open(condeintel_log_file, 'w', 1))
]
msg = "Starting logging SublimeCodeIntel rev %s (%s) on %s" % (
get_git_revision()[:12], os.stat(__file__)[stat.ST_MTIME],
datetime.datetime.now().ctime())
codeintel_log.info("%s\n%s" % (msg, "=" * len(msg)))
_ci_mgr_ = mgr
# Load configuration files:
for catalog in mgr.db.get_catalogs_zone().avail_catalogs():
if catalog['lang'] == lang:
catalogs.append(catalog['name'])
config = {
"codeintel_selected_catalogs": catalogs,
"codeintel_max_recursive_dir_depth": 10,
"codeintel_scan_files_in_project": True,
}
_config = {}
try:
tryReadDict(config_default_file, _config)
except Exception, e:
msg = "Malformed configuration file '%s': %s" % (
config_default_file, e)
log.error(msg)
codeintel_log.error(msg)
try:
tryReadDict(config_file, _config)
except Exception, e:
msg = "Malformed configuration file '%s': %s" % (
config_default_file, e)
log.error(msg)
codeintel_log.error(msg)
config.update(_config.get(lang, {}))
for conf in [
'pythonExtraPaths', 'rubyExtraPaths', 'perlExtraPaths',
'javascriptExtraPaths', 'phpExtraPaths'
]:
v = [
p.strip() for p in config.get(conf, []) + folders
if p.strip()
]
config[conf] = os.pathsep.join(
set(p if p.startswith('/') else os.path.expanduser(p) if p.
startswith('~') else os.path.
abspath(os.path.join(project_base_dir, p)
) if project_base_dir else p for p in v
if p.strip()))
env = SimplePrefsEnvironment(**config)
env._mtime = mtime or max(tryGetMTime(config_file),
tryGetMTime(config_default_file))
env._folders = folders
env._config_default_file = config_default_file
env._project_dir = project_dir
env._project_base_dir = project_base_dir
env._config_file = config_file
env.__class__.get_proj_base_dir = lambda self: project_base_dir
_ci_envs_[path] = env
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if opts.time_it:
start = time.time()
quiet = opts.quiet
if opts.time_it or opts.time_details:
opts.force = True
scan_count = 0
lang_warnings = set()
tree = None
for path in _paths_from_path_patterns(path_patterns,
recursive=opts.recursive,
includes=opts.includes):
if opts.time_it:
sys.stderr.write(path + "\n")
if opts.time_details:
start1 = time.time()
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info("skip `%s': couldn't determine language", path)
continue
try:
buf = mgr.buf_from_path(path, lang=lang)
except OSError as ex:
# Couldn't access the file.
if not opts.recursive:
raise
# Ignore files we don't really care about.
log.warn("%r - %r", ex, path)
continue
if not isinstance(buf, CitadelBuffer):
if opts.recursive:
# Ignore files that scanning isn't provided for.
continue
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
scan_count += 1
if scan_count % 10 == 0:
log.info("%d scanning %r", scan_count, path)
try:
if opts.force:
buf.scan()
if tree is None:
tree = ET.Element("codeintel", version="2.0")
file_elem = ET.SubElement(tree,
"file",
lang=buf.lang,
mtime=str(int(time.time())),
path=os.path.basename(path))
for lang, blob in sorted(buf.blob_from_lang.items()):
blob = buf.blob_from_lang[lang]
file_elem.append(blob)
except KeyError as ex:
# Unknown cile language.
if not opts.recursive:
raise
message = str(ex)
if message not in lang_warnings:
lang_warnings.add(message)
log.warn("Skipping unhandled language %s", message)
if opts.time_details:
delta = time.time() - start1
sys.stderr.write("%.3f %s\n" % (delta, path))
sys.stderr.flush()
if tree is not None:
if opts.stripfuncvars:
# For stdlibs, we don't care about variables inside of
# functions and they take up a lot of space.
for function in tree.getiterator('scope'):
if function.get('ilk') == 'function':
function[:] = [
child for child in function
if child.tag != 'variable'
]
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
if not quiet:
sys.stdout.write(
'<?xml version="1.0" encoding="UTF-8"?>\n')
ET.dump(tree)
if opts.time_it:
end = time.time()
sys.stderr.write("scan took %.3fs\n" % (end - start))
finally:
mgr.finalize()
def do_outline(self, subcmd, opts, path):
"""Scan and outline the structure of the given path.
${cmd_usage}
${cmd_option_list}
"""
extra_lang_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_lang_module_dirs = [sys.path[-1]]
mgr = Manager(extra_lang_module_dirs=extra_lang_module_dirs)
mgr.upgrade()
mgr.initialize()
try:
if '#' in path:
path, anchor = path.rsplit('#', 1)
else:
anchor = None
tree = None
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info(
"skip `%s': couldn't determine language "
"(use --lang option)", path)
return
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
#buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
tree = buf.tree
if anchor is not None:
# Lookup the anchor in the codeintel CIX tree.
lpath = re.split(r'\.|::', anchor)
def blobs_from_tree(tree):
for file_elem in tree:
for blob in file_elem:
yield blob
for elem in blobs_from_tree(tree):
# Generally have 3 types of codeintel trees:
# 1. single-lang file: one <file>, one <blob>
# 2. multi-lang file: one <file>, one or two <blob>'s
# 3. CIX stdlib/catalog file: possibly multiple
# <file>'s, likely multiple <blob>'s
# Allow the first token to be the blob name or lang.
# (This can sometimes be weird, but seems the most
# convenient solution.)
if lpath[0] in (elem.get("name"), elem.get("lang")):
remaining_lpath = lpath[1:]
else:
remaining_lpath = lpath
for name in remaining_lpath:
try:
elem = elem.names[name]
except KeyError:
elem = None
break # try next lang blob
if elem is not None:
break # found one
else:
log.error(
"could not find `%s' definition (or blob) in `%s'",
anchor, path)
return 1
else:
elem = tree
try:
_outline_ci_elem(mgr, elem, quiet=opts.quiet)
except IOError, ex:
if ex.errno == 0:
# Ignore this error from aborting 'less' of this
# command:
# IOError: (0, 'Error')
pass
else:
raise
finally:
mgr.finalize()
def do_outline(self, subcmd, opts, path):
"""Scan and outline the structure of the given path.
${cmd_usage}
${cmd_option_list}
"""
extra_lang_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_lang_module_dirs = [sys.path[-1]]
mgr = Manager(extra_lang_module_dirs=extra_lang_module_dirs)
mgr.upgrade()
mgr.initialize()
try:
if "#" in path:
path, anchor = path.rsplit("#", 1)
else:
anchor = None
tree = None
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info("skip `%s': couldn't determine language " "(use --lang option)", path)
return
if path.endswith(".cix"):
tree = tree_from_cix(open(path, "r").read())
# buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang))
tree = buf.tree
if anchor is not None:
# Lookup the anchor in the codeintel CIX tree.
lpath = re.split(r"\.|::", anchor)
def blobs_from_tree(tree):
for file_elem in tree:
for blob in file_elem:
yield blob
for elem in blobs_from_tree(tree):
# Generally have 3 types of codeintel trees:
# 1. single-lang file: one <file>, one <blob>
# 2. multi-lang file: one <file>, one or two <blob>'s
# 3. CIX stdlib/catalog file: possibly multiple
# <file>'s, likely multiple <blob>'s
# Allow the first token to be the blob name or lang.
# (This can sometimes be weird, but seems the most
# convenient solution.)
if lpath[0] in (elem.get("name"), elem.get("lang")):
remaining_lpath = lpath[1:]
else:
remaining_lpath = lpath
for name in remaining_lpath:
try:
elem = elem.names[name]
except KeyError:
elem = None
break # try next lang blob
if elem is not None:
break # found one
else:
log.error("could not find `%s' definition (or blob) in `%s'", anchor, path)
return 1
else:
elem = tree
try:
_outline_ci_elem(mgr, elem, quiet=opts.quiet)
except IOError, ex:
if ex.errno == 0:
# Ignore this error from aborting 'less' of this
# command:
# IOError: (0, 'Error')
pass
else:
raise
finally:
mgr.finalize()
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if opts.time_it:
start = time.time()
quiet = opts.quiet
if opts.time_it or opts.time_details:
opts.force = True
tree = None
for path in _paths_from_path_patterns(path_patterns,
recursive=opts.recursive,
includes=opts.includes):
if opts.time_it:
sys.stderr.write(path+"\n")
if opts.time_details:
start1 = time.time()
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info("skip `%s': couldn't determine language", path)
continue
buf = mgr.buf_from_path(path, lang=lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
if opts.force:
buf.scan()
if tree is None:
tree = ET.Element("codeintel", version="2.0")
file_elem = ET.SubElement(tree, "file",
lang=buf.lang,
mtime=str(int(time.time())),
path=os.path.basename(path))
for lang, blob in sorted(buf.blob_from_lang.items()):
blob = buf.blob_from_lang[lang]
file_elem.append(blob)
if opts.time_details:
delta = time.time() - start1
sys.stderr.write("%.3f %s\n" % (delta, path))
sys.stderr.flush()
if tree is not None:
if opts.stripfuncvars:
# For stdlibs, we don't care about variables inside of
# functions and they take up a lot of space.
for function in tree.getiterator('scope'):
if function.get('ilk') == 'function':
function[:] = [child for child in function
if child.tag != 'variable']
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
if not quiet:
sys.stdout.write('<?xml version="1.0" encoding="UTF-8"?>\n')
ET.dump(tree)
if opts.time_it:
end = time.time()
sys.stderr.write("scan took %.3fs\n" % (end-start))
finally:
mgr.finalize()
def codeintel_scan(view, path, content, lang, callback=None, pos=None, forms=None):
global despair
for thread in threading.enumerate():
if thread.isAlive() and thread.name == "scanning thread":
logger(
view,
"info",
"Updating indexes... The first time this can take a while. Do not despair!",
timeout=20000,
delay=despair,
)
despair = 0
return
try:
lang = lang or guess_lang_from_path(path)
except CodeIntelError:
logger(view, "warning", "skip `%s': couldn't determine language" % path)
return
if lang.lower() in [l.lower() for l in view.settings().get("codeintel_disabled_languages", [])]:
return
is_scratch = view.is_scratch()
is_dirty = view.is_dirty()
folders = getattr(
view.window(), "folders", lambda: []
)() # FIXME: it's like this for backward compatibility (<= 2060)
def _codeintel_scan():
global _ci_mgr_, despair, despaired
env = None
mtime = None
catalogs = []
now = time.time()
try:
env = _ci_envs_[path]
if env._folders != folders:
raise KeyError
mgr = _ci_mgr_
if now > env._time:
mtime = max(tryGetMTime(env._config_file), tryGetMTime(env._config_default_file))
if env._mtime < mtime:
raise KeyError
except KeyError:
if env is not None:
config_default_file = env._config_default_file
project_dir = env._project_dir
project_base_dir = env._project_base_dir
config_file = env._config_file
else:
config_default_file = os.path.join(CODEINTEL_HOME_DIR, "config")
if not (config_default_file and os.path.exists(config_default_file)):
config_default_file = None
project_dir = None
project_base_dir = None
if path:
# Try to find a suitable project directory (or best guess):
for folder in [".codeintel", ".git", ".hg", "trunk"]:
project_dir = find_folder(path, folder)
if project_dir and (
folder != ".codeintel" or not os.path.exists(os.path.join(project_dir, "db"))
):
if folder.startswith("."):
project_base_dir = os.path.abspath(os.path.join(project_dir, ".."))
else:
project_base_dir = project_dir
break
if not (project_dir and os.path.exists(project_dir)):
project_dir = None
config_file = project_dir and folder == ".codeintel" and os.path.join(project_dir, "config")
if not (config_file and os.path.exists(config_file)):
config_file = None
if _ci_mgr_:
mgr = _ci_mgr_
else:
for thread in threading.enumerate():
if thread.name == "CodeIntel Manager":
thread.finalize() # this finalizes the index, citadel and the manager and waits them to end (join)
mgr = Manager(
extra_module_dirs=_ci_extra_module_dirs_,
db_base_dir=_ci_db_base_dir_,
db_catalog_dirs=_ci_db_catalog_dirs_,
db_import_everything_langs=_ci_db_import_everything_langs,
db_event_reporter=lambda m: logger(view, "event", m),
)
mgr.upgrade()
mgr.initialize()
# Connect the logging file to the handler
condeintel_log_file = os.path.join(mgr.db.base_dir, "codeintel.log")
codeintel_log.handlers = [logging.StreamHandler(open(condeintel_log_file, "w", 1))]
msg = "Starting logging SublimeCodeIntel rev %s (%s) on %s" % (
get_git_revision()[:12],
os.stat(__file__)[stat.ST_MTIME],
datetime.datetime.now().ctime(),
)
codeintel_log.info("%s\n%s" % (msg, "=" * len(msg)))
_ci_mgr_ = mgr
# Load configuration files:
for catalog in mgr.db.get_catalogs_zone().avail_catalogs():
if catalog["lang"] == lang:
catalogs.append(catalog["name"])
config = {
"codeintel_selected_catalogs": catalogs,
"codeintel_max_recursive_dir_depth": 10,
"codeintel_scan_files_in_project": True,
}
_config = {}
try:
tryReadDict(config_default_file, _config)
except Exception, e:
msg = "Malformed configuration file '%s': %s" % (config_default_file, e)
log.error(msg)
codeintel_log.error(msg)
try:
tryReadDict(config_file, _config)
except Exception, e:
msg = "Malformed configuration file '%s': %s" % (config_default_file, e)
log.error(msg)
codeintel_log.error(msg)
config.update(_config.get(lang, {}))
for conf in [
"pythonExtraPaths",
"rubyExtraPaths",
"perlExtraPaths",
"javascriptExtraPaths",
"phpExtraPaths",
]:
v = [p.strip() for p in config.get(conf, []) + folders if p.strip()]
config[conf] = os.pathsep.join(
set(
p
if p.startswith("/")
else os.path.expanduser(p)
if p.startswith("~")
else os.path.abspath(os.path.join(project_base_dir, p))
if project_base_dir
else p
for p in v
if p.strip()
)
)
env = SimplePrefsEnvironment(**config)
env._mtime = mtime or max(tryGetMTime(config_file), tryGetMTime(config_default_file))
env._folders = folders
env._config_default_file = config_default_file
env._project_dir = project_dir
env._project_base_dir = project_base_dir
env._config_file = config_file
env.__class__.get_proj_base_dir = lambda self: project_base_dir
_ci_envs_[path] = env
