def scan_binary(self, buf): from codeintel2 import pybinary python = buf.langintel.interpreter_from_env(buf.env) if not python: raise CodeIntelError("cannot find a usable Python interpreter") cix = pybinary.safe_scan(buf.path, python) return tree_from_cix(cix)
def scan_purelang(self, buf): print("scan_purelang(%s)" % buf.path) log.info("scan '%s'", buf.path) filename = buf.path env = buf.env flags = env.get_pref('cppFlags', []) extra_dirs = [] for pref in env.get_all_prefs(self.extraPathsPrefName): if not pref: continue extra_dirs.extend( d.strip() for d in pref.split(os.pathsep) if os.path.exists(d.strip()) and d.strip() not in extra_dirs) if extra_dirs: flags = flags + [ '-I{}'.format(extra_dir) for extra_dir in extra_dirs ] flags = flags + ['-I{}'.format(os.path.dirname(filename)), '-I.'] content = buf.accessor.text completer.warmupCache(buf.path, fileBuffer=content, flags=flags) output = '<file path="%s" lang="%s"></file>' % (filename, lang) xml = '<codeintel version="2.0">\n' + output + '</codeintel>' return tree_from_cix(xml)
def update_cix_file(mgr, path): log.info("convert `%s' to pretty CIX 2.0", path) cix = open(path, 'r').read() tree = tree_from_cix(cix) # converts to CIX 2.0 tree = pretty_tree_from_tree(tree) new_cix = ET.tostring(tree) _run("p4 edit %s" % path) open(path, 'w').write(new_cix)
def do_json(self, subcmd, opts, path): """Convert cix XML file into json format. ${cmd_usage} ${cmd_option_list} """ import json if opts.output == '-': output_path = None output_file = sys.stdout else: if opts.output: output_path = opts.output else: output_path = splitext(path)[0]+".json" if exists(output_path): if opts.force: os.remove(output_path) else: raise Error("`%s' exists: use -f|--force option to " "allow overwrite" % output_path) output_file = open(output_path, 'w') mgr = Manager() mgr.upgrade() mgr.initialize() try: if path.endswith(".cix"): tree = tree_from_cix(open(path, 'r').read()) else: buf = mgr.buf_from_path(path, lang=opts.lang) tree = buf.tree result = {} ci = result["codeintel"] = defaultdict(list) def _elemToDict(parent, elem): data = defaultdict(list) name = elem.get("name") if name is not None: data["name"] = name data["tag"] = elem.tag for attr_name, attr in elem.attrib.items(): data[attr_name] = attr parent["children"].append(data) for child in elem: _elemToDict(data, child) for child in tree: _elemToDict(ci, child) json.dump(result, output_file, indent=2) finally: mgr.finalize()
def do_json(self, subcmd, opts, path): """Convert cix XML file into json format. ${cmd_usage} ${cmd_option_list} """ import json if opts.output == '-': output_path = None output_file = sys.stdout else: if opts.output: output_path = opts.output else: output_path = splitext(path)[0] + ".json" if exists(output_path): if opts.force: os.remove(output_path) else: raise Error("`%s' exists: use -f|--force option to " "allow overwrite" % output_path) output_file = open(output_path, 'w') mgr = Manager() mgr.upgrade() mgr.initialize() try: if path.endswith(".cix"): tree = tree_from_cix(open(path, 'r').read()) else: buf = mgr.buf_from_path(path, lang=opts.lang) tree = buf.tree result = {} ci = result["codeintel"] = defaultdict(list) def _elemToDict(parent, elem): data = defaultdict(list) name = elem.get("name") if name is not None: data["name"] = name data["tag"] = elem.tag for attr_name, attr in elem.attrib.items(): data[attr_name] = attr parent["children"].append(data) for child in elem: _elemToDict(data, child) for child in tree: _elemToDict(ci, child) json.dump(result, output_file, indent=2) finally: mgr.finalize()
def scan_purelang(self, buf, mtime=None, lang="Go"): """Scan the given GoBuffer return an ElementTree (conforming to the CIX schema) giving a summary of its code elements. @param buf {GoBuffer} is the Go buffer to scan @param mtime {int} is a modified time for the file (in seconds since the "epoch"). If it is not specified the _current_ time is used. Note that the default is not to stat() the file and use that because the given content might not reflect the saved file state. """ # Dev Notes: # - This stub implementation of the Go CILE return an "empty" # summary for the given content, i.e. CIX content that says "there # are no code elements in this Go content". # - Use the following command (in the extension source dir) to # debug/test your scanner: # codeintel scan -p -l Go <example-Go-file> # "codeintel" is a script available in the Komodo SDK. log.info("scan '%s'", buf.path) if mtime is None: mtime = int(time.time()) # The 'path' attribute must use normalized dir separators. if sys.platform.startswith("win"): path = buf.path.replace('\\', '/') else: path = buf.path try: gooutline_exe_path = self.compile_gooutline(buf) except Exception as e: log.error("Error compiling outline: %s", e) raise cmd = [gooutline_exe_path, buf.path] env = buf.env.get_all_envvars() log.debug("running [%s]", cmd) try: p = process.ProcessOpen(cmd, env=env) except OSError as e: log.error("Error executing '%s': %s", cmd, e) return output, error = p.communicate() if error: log.warn("'%s' stderr: [%s]", cmd[0], error) xml = '<codeintel version="2.0">\n' + output + '</codeintel>' return tree_from_cix(xml)
def scan_purelang(self, buf): print "scan_purelang(%s)" % buf.path log.info("scan '%s'", buf.path) if sys.platform.startswith("win"): path = buf.path.replace('\\', '/') else: path = buf.path # update_translation_unit(buf.path, flags) # FIXME output = '<file path="%s" lang="%s"></file>' % (path, lang) xml = '<codeintel version="2.0">\n' + output + '</codeintel>' return tree_from_cix(xml)
def do_cix(self, subcmd, opts, *path_patterns): """Read in and print a CIX file (possibly converting to CIX 2.0 and prettifying in the process. ${cmd_usage} ${cmd_option_list} """ if opts.pretty_print: opts.convert = True for path in _paths_from_path_patterns(path_patterns): tree = None cix = open(path, 'r').read() if opts.convert: tree = tree_from_cix(cix) if opts.pretty_print: tree = pretty_tree_from_tree(tree) ET.dump(tree) else: sys.stdout.write(cix)
def do_cix_check(self, subcmd, opts, *path_patterns): """Check the given CIX file(s) for warnings, errors. ${cmd_usage} ${cmd_option_list} Eventually this should include an XML validity check against the RelaxNG schema for CIX. However, currently it just checks for some common errors. Returns the number of warnings/errors generated. """ num_results = 0 for path in _paths_from_path_patterns(path_patterns): tree = None cix = open(path, 'r').read() tree = tree_from_cix(cix) for sev, msg in check_tree(tree): num_results += 1 print("%s: %s: %s" % (path, sev, msg)) return num_results
def do_cix_check(self, subcmd, opts, *path_patterns): """Check the given CIX file(s) for warnings, errors. ${cmd_usage} ${cmd_option_list} Eventually this should include an XML validity check against the RelaxNG schema for CIX. However, currently it just checks for some common errors. Returns the number of warnings/errors generated. """ num_results = 0 for path in _paths_from_path_patterns(path_patterns): tree = None cix = open(path, 'r').read() tree = tree_from_cix(cix) for sev, msg in check_tree(tree): num_results += 1 print "%s: %s: %s" % (path, sev, msg) return num_results
def scan_purelang(self, buf): print("scan_purelang(%s)" % buf.path) log.info("scan '%s'", buf.path) filename = buf.path env = buf.env flags = env.get_pref('cppFlags', []) extra_dirs = [] for pref in env.get_all_prefs(self.extraPathsPrefName): if not pref: continue extra_dirs.extend(d.strip() for d in pref.split(os.pathsep) if os.path.exists(d.strip()) and d.strip() not in extra_dirs) if extra_dirs: flags = flags + ['-I{}'.format(extra_dir) for extra_dir in extra_dirs] flags = flags + ['-I{}'.format(os.path.dirname(filename)), '-I.'] content = buf.accessor.text completer.warmupCache(buf.path, fileBuffer=content, flags=flags) output = '<file path="%s" lang="%s"></file>' % (filename, lang) xml = '<codeintel version="2.0">\n' + output + '</codeintel>' return tree_from_cix(xml)
def do_outline(self, subcmd, opts, path): """Print code outline of the given file. You can specify a lookup path into the file code outline to display via URL-anchor syntax, e.g.: ci2 outline path/to/foo.py#AClass.amethod ${cmd_usage} ${cmd_option_list} """ mgr = Manager() mgr.upgrade() mgr.initialize() try: if '#' in path: path, anchor = path.rsplit('#', 1) else: anchor = None if path.endswith(".cix"): tree = tree_from_cix(open(path, 'r').read()) #buf = mgr.buf_from_content("", tree[0].get("lang"), path=path) else: buf = mgr.buf_from_path(path, lang=opts.lang) tree = buf.tree if anchor is not None: # Lookup the anchor in the codeintel CIX tree. lpath = re.split(r'\.|::', anchor) def blobs_from_tree(tree): for file_elem in tree: for blob in file_elem: yield blob for elem in blobs_from_tree(tree): # Generally have 3 types of codeintel trees: # 1. single-lang file: one <file>, one <blob> # 2. multi-lang file: one <file>, one or two <blob>'s # 3. CIX stdlib/catalog file: possibly multiple # <file>'s, likely multiple <blob>'s # Allow the first token to be the blob name or lang. # (This can sometimes be weird, but seems the most # convenient solution.) if lpath[0] in (elem.get("name"), elem.get("lang")): remaining_lpath = lpath[1:] else: remaining_lpath = lpath for name in remaining_lpath: try: elem = elem.names[name] except KeyError: elem = None break # try next lang blob if elem is not None: break # found one else: log.error( "could not find `%s' definition (or blob) in `%s'", anchor, path) return 1 else: elem = tree try: _outline_ci_elem(elem, brief=opts.brief, doSort=opts.doSort) except IOError as ex: if ex.errno == 0: # Ignore this error from aborting 'less' of 'ci2 outline' # output: # IOError: (0, 'Error') pass else: raise finally: mgr.finalize()
def do_play(self, subcmd, opts): """Run my current play/dev code. ${cmd_usage} ${cmd_option_list} """ if False: lang = "CSS" markedup_content = dedent(""" /* http://www.w3.org/TR/REC-CSS2/fonts.html#propdef-font-weight */ h1 { border: 1px solid black; font-weight /* hi */: <|> !important } """) content, data = unmark_text(markedup_content) pos = data["pos"] mgr = Manager() #mgr.upgrade() # Don't need it for just CSS usage. mgr.initialize() try: buf = mgr.buf_from_content(content, lang=lang, path="play.css") trg = buf.trg_from_pos(pos) if trg is None: raise Error("unexpected trigger: %r" % trg) completions = buf.cplns_from_trg(trg) print("COMPLETIONS: %r" % completions) finally: mgr.finalize() elif False: lang = "Python" path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100)) markedup_content = dedent(""" import sys, os class Foo: def bar(self): pass sys.<|>path # should have path in completion list f = Foo() """) content, data = unmark_text(markedup_content) print(banner(path)) print(_escaped_text_from_text(content, "whitespace")) pos = data["pos"] mgr = Manager() mgr.upgrade() mgr.initialize() try: buf = mgr.buf_from_content(content, lang=lang, path=path) print(banner("cix", '-')) print(buf.cix) trg = buf.trg_from_pos(pos) if trg is None: raise Error("unexpected trigger: %r" % trg) print(banner("completions", '-')) ctlr = LogEvalController(log) buf.async_eval_at_trg(trg, ctlr) ctlr.wait(2) #XXX if not ctlr.is_done(): ctlr.abort() raise Error("XXX async eval timed out") pprint(ctlr.cplns) print(banner(None)) finally: mgr.finalize() elif False: lang = "Ruby" path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100)) markedup_content = dedent("""\ r<1>equire 'net/http' include Net req = HTTPRequest.new req.<2>get() """) content, data = unmark_text(markedup_content) print(banner(path)) print(_escaped_text_from_text(content, "whitespace")) pos = data[1] mgr = Manager() mgr.upgrade() mgr.initialize() try: buf = mgr.buf_from_content(content, lang=lang, path=path) print(banner("cix", '-')) cix = buf.cix print( ET.tostring(pretty_tree_from_tree(tree_from_cix(buf.cix)))) trg = buf.trg_from_pos(pos, implicit=False) if trg is None: raise Error("unexpected trigger: %r" % trg) print(banner("completions", '-')) ctlr = LogEvalController(log) buf.async_eval_at_trg(trg, ctlr) ctlr.wait(30) #XXX if not ctlr.is_done(): ctlr.abort() raise Error("XXX async eval timed out") pprint(ctlr.cplns) print(banner(None)) finally: mgr.finalize()
cmd = [gooutline_exe_path, buf.path] env = buf.env.get_all_envvars() log.debug("running [%s]", cmd) try: p = process.ProcessOpen(cmd, env=env) except OSError, e: log.error("Error executing '%s': %s", cmd, e) return output, error = p.communicate() if error: log.warn("'%s' stderr: [%s]", cmd[0], error) xml = '<codeintel version="2.0">\n' + output + "</codeintel>" return tree_from_cix(xml) # ---- registration def register(mgr): """Register language support with the Manager.""" mgr.set_lang_info( lang, silvercity_lexer=GoLexer(), buf_class=GoBuffer, langintel_class=GoLangIntel, import_handler_class=None, cile_driver_class=GoCILEDriver, is_cpln_lang=True,
def cix2html(opts, path): """Turn cix file into html API documentation. Example: cix2html path/to/foo.cix#AClass.amethod cix2html path/to/foo.cix -o file.html ${cmd_usage} ${cmd_option_list} """ mgr = Manager() mgr.upgrade() mgr.initialize() try: def blobs_from_tree(tree): for file_elem in tree: for blob in file_elem: yield blob if '#' in path: path, anchor = path.rsplit('#', 1) else: anchor = None if path.endswith(".cix"): tree = tree_from_cix(open(path, 'r').read()) # buf = mgr.buf_from_content("", tree[0].get("lang"), path=path) else: buf = mgr.buf_from_path(path, lang=opts.lang) tree = buf.tree if anchor is not None: # Lookup the anchor in the codeintel CIX tree. lpath = re.split(r'\.|::', anchor) for elem in blobs_from_tree(tree): # Generally have 3 types of codeintel trees: # 1. single-lang file: one <file>, one <blob> # 2. multi-lang file: one <file>, one or two <blob>'s # 3. CIX stdlib/catalog file: possibly multiple # <file>'s, likely multiple <blob>'s # Allow the first token to be the blob name or lang. # (This can sometimes be weird, but seems the most # convenient solution.) if lpath[0] in (elem.get("name"), elem.get("lang")): remaining_lpath = lpath[1:] else: remaining_lpath = lpath for name in remaining_lpath: try: elem = elem.names[name] except KeyError: elem = None break # try next lang blob if elem is not None: break # found one else: log.error("could not find `%s' definition (or blob) in `%s'", anchor, path) return 1 else: elem = tree try: if elem.tag == "codeintel": _html_ci_elem(opts, elem.getchildren()[0]) else: _html_ci_elem(opts, elem) except IOError, ex: if ex.errno == 0: # Ignore this error from aborting 'less' of 'ci2 outline' # output: # IOError: (0, 'Error') pass else: raise except Exception, e: import traceback traceback.print_exc()
cmd = [gooutline_exe_path, buf.path] env = buf.env.get_all_envvars() log.debug("running [%s]", cmd) try: p = process.ProcessOpen(cmd, env=env) except OSError, e: log.error("Error executing '%s': %s", cmd, e) return output, error = p.communicate() if error: log.warn("'%s' stderr: [%s]", cmd[0], error) xml = '<codeintel version="2.0">\n' + output + '</codeintel>' return tree_from_cix(xml) #---- registration def register(mgr): """Register language support with the Manager.""" mgr.set_lang_info(lang, silvercity_lexer=GoLexer(), buf_class=GoBuffer, langintel_class=GoLangIntel, import_handler_class=None, cile_driver_class=GoCILEDriver, is_cpln_lang=True)
def do_outline(self, subcmd, opts, path): """Print code outline of the given file. You can specify a lookup path into the file code outline to display via URL-anchor syntax, e.g.: ci2 outline path/to/foo.py#AClass.amethod ${cmd_usage} ${cmd_option_list} """ mgr = Manager() mgr.upgrade() mgr.initialize() try: if '#' in path: path, anchor = path.rsplit('#', 1) else: anchor = None if path.endswith(".cix"): tree = tree_from_cix(open(path, 'r').read()) #buf = mgr.buf_from_content("", tree[0].get("lang"), path=path) else: buf = mgr.buf_from_path(path, lang=opts.lang) tree = buf.tree if anchor is not None: # Lookup the anchor in the codeintel CIX tree. lpath = re.split(r'\.|::', anchor) def blobs_from_tree(tree): for file_elem in tree: for blob in file_elem: yield blob for elem in blobs_from_tree(tree): # Generally have 3 types of codeintel trees: # 1. single-lang file: one <file>, one <blob> # 2. multi-lang file: one <file>, one or two <blob>'s # 3. CIX stdlib/catalog file: possibly multiple # <file>'s, likely multiple <blob>'s # Allow the first token to be the blob name or lang. # (This can sometimes be weird, but seems the most # convenient solution.) if lpath[0] in (elem.get("name"), elem.get("lang")): remaining_lpath = lpath[1:] else: remaining_lpath = lpath for name in remaining_lpath: try: elem = elem.names[name] except KeyError: elem = None break # try next lang blob if elem is not None: break # found one else: log.error("could not find `%s' definition (or blob) in `%s'", anchor, path) return 1 else: elem = tree try: _outline_ci_elem(elem, brief=opts.brief, doSort=opts.doSort) except IOError, ex: if ex.errno == 0: # Ignore this error from aborting 'less' of 'ci2 outline' # output: # IOError: (0, 'Error') pass else: raise finally: mgr.finalize()
def do_play(self, subcmd, opts): """Run my current play/dev code. ${cmd_usage} ${cmd_option_list} """ if False: lang = "CSS" markedup_content = dedent(""" /* http://www.w3.org/TR/REC-CSS2/fonts.html#propdef-font-weight */ h1 { border: 1px solid black; font-weight /* hi */: <|> !important } """) content, data = unmark_text(markedup_content) pos = data["pos"] mgr = Manager() #mgr.upgrade() # Don't need it for just CSS usage. mgr.initialize() try: buf = mgr.buf_from_content(content, lang=lang, path="play.css") trg = buf.trg_from_pos(pos) if trg is None: raise Error("unexpected trigger: %r" % trg) completions = buf.cplns_from_trg(trg) print "COMPLETIONS: %r" % completions finally: mgr.finalize() elif False: lang = "Python" path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100)) markedup_content = dedent(""" import sys, os class Foo: def bar(self): pass sys.<|>path # should have path in completion list f = Foo() """) content, data = unmark_text(markedup_content) print banner(path) print _escaped_text_from_text(content, "whitespace") pos = data["pos"] mgr = Manager() mgr.upgrade() mgr.initialize() try: buf = mgr.buf_from_content(content, lang=lang, path=path) print banner("cix", '-') print buf.cix trg = buf.trg_from_pos(pos) if trg is None: raise Error("unexpected trigger: %r" % trg) print banner("completions", '-') ctlr = LogEvalController(log) buf.async_eval_at_trg(trg, ctlr) ctlr.wait(2) #XXX if not ctlr.is_done(): ctlr.abort() raise Error("XXX async eval timed out") pprint(ctlr.cplns) print banner(None) finally: mgr.finalize() elif False: lang = "Ruby" path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100)) markedup_content = dedent("""\ r<1>equire 'net/http' include Net req = HTTPRequest.new req.<2>get() """) content, data = unmark_text(markedup_content) print banner(path) print _escaped_text_from_text(content, "whitespace") pos = data[1] mgr = Manager() mgr.upgrade() mgr.initialize() try: buf = mgr.buf_from_content(content, lang=lang, path=path) print banner("cix", '-') cix = buf.cix print ET.tostring(pretty_tree_from_tree(tree_from_cix(buf.cix))) trg = buf.trg_from_pos(pos, implicit=False) if trg is None: raise Error("unexpected trigger: %r" % trg) print banner("completions", '-') ctlr = LogEvalController(log) buf.async_eval_at_trg(trg, ctlr) ctlr.wait(30) #XXX if not ctlr.is_done(): ctlr.abort() raise Error("XXX async eval timed out") pprint(ctlr.cplns) print banner(None) finally: mgr.finalize()