def compile_gooutline(self, buf):
go_exe = buf.langintel.get_go_exe(buf.env)
if not go_exe:
raise CodeIntelError("Unable to locate go executable")
if self._gooutline_executable_and_error is None or go_exe != self._gooutline_executable_and_error[0]:
self._gooutline_executable_and_error = (go_exe, None, "Unknown Error")
import tempfile
tempdir = tempfile.gettempdir()
outline_exe = os.path.join(tempdir, "outline")
if sys.platform.startswith("win"):
outline_exe += ".exe"
outline_src = os.path.join(self.golib_dir, "outline.go")
cmd = [go_exe, "build", outline_src]
cwd = tempdir
env = buf.env.get_all_envvars()
try:
# Compile the executable.
p = process.ProcessOpen(cmd, cwd=cwd, env=env, stdin=None)
output, error = p.communicate()
if error:
log.warn("'%s' stderr: [%s]", cmd, error)
# Remember the executable.
self._gooutline_executable_and_error = (go_exe, outline_exe, None)
except Exception as ex:
error_message = "Unable to compile 'outline.go'" + str(ex)
self._gooutline_executable_and_error = (go_exe, None, error_message)
if self._gooutline_executable_and_error[1]:
return self._gooutline_executable_and_error[1]
raise CodeIntelError(self._gooutline_executable_and_error[2])
def available_imports(self, buf, trg, ctlr):
env = buf.env
go_exe = self.get_go_exe(env)
if not go_exe:
raise CodeIntelError("Unable to locate go executable")
if go_exe not in self._packages_from_exe:
cmd = [go_exe, 'list', 'std']
cwd = None
if buf.path != "<Unsaved>":
cwd = os.path.dirname(buf.path)
env_vars = env.get_all_envvars()
log.debug("running cmd %r", cmd)
try:
p = process.ProcessOpen(cmd, cwd=cwd, env=env_vars)
except OSError as e:
raise CodeIntelError("Error executing '%s': %s" % (cmd, e))
output, error = p.communicate()
if error:
log.warn("cmd %r error [%s]", cmd, error)
raise CodeIntelError(error)
package_names = [x.strip() for x in output.splitlines() if x]
log.debug("retrieved %d package names", len(package_names))
self._packages_from_exe[go_exe] = package_names
stdlib_imports = self._packages_from_exe.get(go_exe, [])
ctlr.start(buf, trg)
ctlr.set_cplns([("import", name) for name in stdlib_imports])
ctlr.done("success")
def guess_lang_from_path(path):
lang_from_ext = {
".py": "Python",
".pl": "Perl",
".pm": "Perl",
".tcl": "Tcl",
".php": "PHP",
".inc": "PHP",
".rb": "Ruby",
".rhtml": "RHTML",
".html.erb": "RHTML",
".js": "JavaScript",
".java": "Java",
".css": "CSS",
".xul": "XUL",
".xbl": "XBL",
".html": "HTML",
".xml": "XML",
".tpl": "Smarty",
".django.html": "Django",
".mason.html": "Mason",
".ttkt.html": "TemplateToolkit",
".cxx": "C++",
}
idx = 0
base = basename(path)
while base.find('.', idx) != -1:
idx = base.find('.', idx)
if idx == -1:
break
ext = base[idx:]
if ext in lang_from_ext:
return lang_from_ext[ext]
idx += 1
raise CodeIntelError("couldn't guess lang for `%s'" % path)
def do_html(self, subcmd, opts, path):
"""Convert the given path to styled HTML.
${cmd_usage}
${cmd_option_list}
The generated HTML provides a good tool for seeing how Komodo's
lexer lexes the given content. This is the same tokenization that
you get from "buf.accessor.*" methods -- which you can use for
writing the CILE, trigger handling, and completion evaluation
for this language.
Use the "-t" and "-e" option to also exercise the current
trigger handling and completion evaluation (i.e. determining
the appropriate completions and calltips at a given trigger
point).
"""
extra_lang_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_lang_module_dirs = [sys.path[-1]]
mgr = Manager(extra_lang_module_dirs=extra_lang_module_dirs)
try:
if opts.browse:
htmls = []
buf = mgr.buf_from_path(path, lang=opts.lang)
html = buf.to_html(True,
True,
title=path,
do_trg=opts.do_trg,
do_eval=opts.do_eval)
finally:
mgr.finalize()
if opts.output == '-':
output_path = None
output_file = sys.stdout
else:
if opts.output:
output_path = opts.output
else:
output_path = path + ".html"
if exists(output_path):
os.remove(output_path)
output_file = open(output_path, 'w')
if output_file:
output_file.write(html)
if output_path:
output_file.close()
if opts.browse:
if not output_path:
raise CodeIntelError("cannot open in browser if stdout "
"used for output")
import webbrowser
url = _url_from_local_path(output_path)
webbrowser.open_new(url)
def lookup_defn(self, buf, trg, ctlr):
env = buf.env
godef_path = self._get_gotool('godef', env)
# We can't store the path and watch prefs because this isn't a
# true xpcom object.
if godef_path is None:
godef_path = 'godef'
cmd = [
godef_path, '-i=true', '-t=true',
'-f=%s' % buf.path,
'-o=%s' % trg.pos
]
log.debug("running [%s]", cmd)
p = process.ProcessOpen(cmd, env=buf.env.get_all_envvars())
output, error = p.communicate(buf.accessor.text)
if error:
log.debug("'gocode' stderr: [%s]", error)
raise CodeIntelError(error)
lines = output.splitlines()
log.debug(output)
defparts = lines[0].rsplit(":", 2)
if len(defparts) == 2:
# current file
path = buf.path
line = defparts[0]
else:
# other file
path = defparts[0]
line = defparts[1]
name, typeDesc = lines[1].split(' ', 1)
d = Definition(
"Go",
path,
blobname=None,
lpath=None,
name=name,
line=line,
ilk='function' if typeDesc.startswith('func') else typeDesc,
citdl=None,
signature=typeDesc,
doc='\n'.join(lines[1:]),
)
log.debug(d)
ctlr.start(buf, trg)
ctlr.set_defns([d])
ctlr.done("success")
def _hit_from_tokens(self,
expr,
tokens,
scoperef,
variable=None,
defn_only=False):
args_scoperef = scoperef
# First part...
hit, nconsumed = self._hit_from_first_part(tokens,
scoperef,
variable=variable,
defn_only=defn_only)
if not hit:
# TODO: Add the fallback Buffer-specific near-by hunt
# for a symbol for the first token. See my spiral-bound
# book for some notes.
raise CodeIntelError("could not resolve first part of '%s'" % expr)
self.debug("_hit_from_citdl: first part: %r -> %r", tokens[:nconsumed],
hit)
# ...the remainder.
remaining_tokens = tokens[nconsumed:]
while remaining_tokens:
elem, scoperef = hit
self.debug("_hit_from_citdl: resolve %r on %r in %r",
remaining_tokens, elem, scoperef)
if remaining_tokens[0][0] == "(":
new_hit = self._hit_from_call(elem,
scoperef,
remaining_tokens[0],
args_scoperef,
defn_only=defn_only)
nconsumed = 1
else:
new_hit, nconsumed = self._hit_from_getattr(
remaining_tokens, elem, scoperef, defn_only=defn_only)
remaining_tokens = remaining_tokens[nconsumed:]
hit = new_hit
# Resolve any variable type inferences.
elem, scoperef = hit
while elem.tag == "variable" and (not defn_only
or "__no_defn__" in elem.get(
"attributes", "").split()):
elem, scoperef = self._hit_from_variable_type_inference(
elem, scoperef, defn_only=defn_only)
self.info("'%s' is %s on %s", expr, elem, scoperef)
return (elem, scoperef), len(tokens) - len(remaining_tokens)
def _hit_from_variable_type_inference(self,
elem,
scoperef,
defn_only=False):
"""Resolve the type inference for 'elem' at 'scoperef'."""
citdl = elem.get("citdl")
if not citdl:
raise CodeIntelError("no type-inference info for %r" % elem)
self.log("resolve '%s' type inference for %r:", citdl, elem)
return self._hit_from_type_inference(citdl,
scoperef,
variable=elem,
defn_only=defn_only)
def _hit_from_call(self,
elem,
scoperef,
args,
args_scoperef,
defn_only=False):
"""Resolve the function call inference for 'elem' at 'scoperef'."""
# This might be a variable, in that case we keep resolving the variable
# until we get to the final function/class element that is to be called.
while elem.tag == "variable":
elem, scoperef = self._hit_from_variable_type_inference(
elem, scoperef, defn_only=defn_only)
ilk = elem.get("ilk")
if ilk == "class":
# Return the class element.
self.log("_hit_from_call: resolved to class instance '%s'",
elem.get("name"))
return (ClassInstance(elem), scoperef)
if ilk == "function":
citdl = elem.get("returns")
if citdl:
self.log("_hit_from_call: function with citdl %r", citdl)
if citdl.startswith("__arg"):
args = args[1:-1].split(",")
try:
arg = args[int(citdl[5:]) - 1]
if not arg.startswith("__arg"):
citdl = arg
scoperef = args_scoperef
except (ValueError, IndexError):
pass
return self._hit_from_type_inference(citdl,
scoperef,
defn_only=defn_only)
raise CodeIntelError("no return type info for %r" % elem)
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
extra_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_module_dirs = [sys.path[-1]]
mgr = Manager(extra_module_dirs=extra_module_dirs)
mgr.upgrade()
mgr.initialize()
try:
tree = None
for path in _paths_from_path_patterns(path_patterns):
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info(
"skip `%s': couldn't determine language "
"(use --lang option)", path)
continue
buf = mgr.buf_from_path(path, lang=opts.lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
buf.scan() # force a fresh scan
tree = buf.tree
for severity, msg in check_tree(tree):
dump = {"warning": log.warn, "error": log.error}[severity]
dump(msg)
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
ET.dump(tree)
finally:
mgr.finalize()
def _hit_from_require(self,
tokens,
scoperef,
variable=None,
defn_only=False):
# Node.js / CommonJS hack: try to resolve things via require()
if len(tokens) > 1 and tokens[1][0] == "(":
if variable is None:
variable, _ = self._elem_from_scoperef(scoperef)
requirename = variable.get(
"required_library_name",
getattr(self.trg, "required_library_name", None))
if requirename:
self.log(
"_hit_from_variable_type_inference: resolving require(%r)",
requirename)
module_name = requirename.lstrip("./")
if len(tokens) > 2:
symbol = tokens[2]
remaining_tokens = tokens[3:]
_nconsumed = 2
else:
symbol = None
remaining_tokens = tokens[2:]
_nconsumed = 1
require = FakeImport(variable,
module=requirename,
symbol=symbol,
alias=None)
hit, nconsumed = self._hit_from_elem_imports(
[module_name] + remaining_tokens,
require,
defn_only=defn_only)
if hit is not None:
return hit, nconsumed + _nconsumed
raise CodeIntelError("could not resolve require(%r)" % requirename)
return None, None
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
import time
import ciElementTree as ET
from ci2 import _paths_from_path_patterns
from codeintel2.manager import Manager
from codeintel2.citadel import CitadelBuffer
from codeintel2.common import CodeIntelError
from codeintel2.tree import pretty_tree_from_tree
from codeintel2.util import guess_lang_from_path
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if opts.time_it:
start = time.time()
quiet = opts.quiet
if opts.time_it or opts.time_details:
opts.force = True
scan_count = 0
lang_warnings = set()
tree = None
for path in _paths_from_path_patterns(path_patterns,
recursive=opts.recursive,
includes=opts.includes):
if opts.time_it:
sys.stderr.write(path + "\n")
if opts.time_details:
start1 = time.time()
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
self.log.info("skip `%s': couldn't determine language",
path)
continue
try:
buf = mgr.buf_from_path(path, lang=lang)
except OSError as ex:
# Couldn't access the file.
if not opts.recursive:
raise
# Ignore files we don't really care about.
self.log.warn("%r - %r", ex, path)
continue
if not isinstance(buf, CitadelBuffer):
if opts.recursive:
# Ignore files that scanning isn't provided for.
continue
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
scan_count += 1
if scan_count % 10 == 0:
self.log.info("%d scanning %r", scan_count, path)
try:
if opts.force:
buf.scan()
if tree is None:
tree = ET.Element("codeintel", version="2.0")
file_elem = ET.SubElement(tree,
"file",
lang=buf.lang,
mtime=str(int(time.time())),
path=os.path.basename(path))
for lang, blob in sorted(buf.blob_from_lang.items()):
blob = buf.blob_from_lang[lang]
file_elem.append(blob)
except KeyError as ex:
# Unknown cile language.
if not opts.recursive:
raise
message = str(ex)
if message not in lang_warnings:
lang_warnings.add(message)
self.log.warn("Skipping unhandled language %s",
message)
if opts.time_details:
delta = time.time() - start1
sys.stderr.write("%.3f %s\n" % (delta, path))
sys.stderr.flush()
if tree is not None:
if opts.stripfuncvars:
# For stdlibs, we don't care about variables inside of
# functions and they take up a lot of space.
for function in tree.getiterator('scope'):
if function.get('ilk') == 'function':
function[:] = [
child for child in function
if child.tag != 'variable'
]
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
if not quiet:
sys.stdout.write(
'<?xml version="1.0" encoding="UTF-8"?>\n')
ET.dump(tree)
if opts.time_it:
end = time.time()
sys.stderr.write("scan took %.3fs\n" % (end - start))
finally:
mgr.finalize()
def _hit_from_getattr(self,
tokens,
elem,
scoperef,
variable=None,
defn_only=False):
"""Return a hit for a getattr on the given element.
Returns (<hit>, <num-tokens-consumed>) or raises an CodeIntelError.
Typically this just does a getattr of tokens[0], but handling
some multi-level imports can result in multiple tokens being
consumed.
"""
# TODO: On failure, call a hook to make an educated guess. Some
# attribute names are strong signals as to the object type
# -- typically those for common built-in classes.
first_token = tokens[0]
self.log("resolve getattr '%s' on %r in %r:", first_token, elem,
scoperef)
ilk = elem.get("ilk")
refs = {
"blob": None,
"block": None,
"function": None,
"class": "classrefs",
"instance": "classrefs",
"interface": "interfacerefs"
}.get(ilk, "objectrefs")
attr = elem.names.get(first_token)
if attr is not None and attr is not variable:
self.log("attr is %r in %r", attr, elem)
if ilk != "blob":
# update the scoperef, we are now inside the element.
scoperef = (scoperef[0], scoperef[1] + [elem.get("name")])
return (attr, scoperef), 1
self.debug("look for %r from imports in %r", tokens, elem)
hit, nconsumed = self._hit_from_elem_imports(tokens,
elem,
defn_only=defn_only)
if hit is not None:
return hit, nconsumed
for ref in elem.get(refs, "").split():
try:
self.log("is '%s' from base %s: %r?", first_token, ilk
or elem.tag, ref)
hit, nconsumed = self._hit_from_citdl(ref,
tokens,
scoperef,
variable=elem,
defn_only=defn_only)
if hit:
return hit, nconsumed
except CodeIntelError as ex:
self.log("could not resolve ref '%s' on scoperef %r", ref,
scoperef)
# Was not available, try the next object then.
# If there's a citdl type:
citdl = elem.get("citdl")
if citdl:
hit, nconsumed = self._hit_from_citdl(citdl,
tokens,
scoperef,
variable=elem,
defn_only=defn_only)
if hit:
return hit, nconsumed
raise CodeIntelError("could not resolve '%s' getattr on %r in %r" %
(first_token, elem, scoperef))
def do_outline(self, subcmd, opts, path):
"""Scan and outline the structure of the given path.
${cmd_usage}
${cmd_option_list}
"""
extra_lang_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_lang_module_dirs = [sys.path[-1]]
mgr = Manager(extra_lang_module_dirs=extra_lang_module_dirs)
mgr.upgrade()
mgr.initialize()
try:
if '#' in path:
path, anchor = path.rsplit('#', 1)
else:
anchor = None
tree = None
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info(
"skip `%s': couldn't determine language "
"(use --lang option)", path)
return
if path.endswith(".cix"):
tree = tree_from_cix(open(path, 'r').read())
#buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)
else:
buf = mgr.buf_from_path(path, lang=opts.lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
tree = buf.tree
if anchor is not None:
# Lookup the anchor in the codeintel CIX tree.
lpath = re.split(r'\.|::', anchor)
def blobs_from_tree(tree):
for file_elem in tree:
for blob in file_elem:
yield blob
for elem in blobs_from_tree(tree):
# Generally have 3 types of codeintel trees:
# 1. single-lang file: one <file>, one <blob>
# 2. multi-lang file: one <file>, one or two <blob>'s
# 3. CIX stdlib/catalog file: possibly multiple
# <file>'s, likely multiple <blob>'s
# Allow the first token to be the blob name or lang.
# (This can sometimes be weird, but seems the most
# convenient solution.)
if lpath[0] in (elem.get("name"), elem.get("lang")):
remaining_lpath = lpath[1:]
else:
remaining_lpath = lpath
for name in remaining_lpath:
try:
elem = elem.names[name]
except KeyError:
elem = None
break # try next lang blob
if elem is not None:
break # found one
else:
log.error(
"could not find `%s' definition (or blob) in `%s'",
anchor, path)
return 1
else:
elem = tree
try:
_outline_ci_elem(mgr, elem, quiet=opts.quiet)
except IOError, ex:
if ex.errno == 0:
# Ignore this error from aborting 'less' of this
# command:
# IOError: (0, 'Error')
pass
else:
raise
finally:
mgr.finalize()