def do_dbcheck(self, subcmd, opts): """Run an internal consistency check on the database. ${cmd_usage} ${cmd_option_list} Any errors will be printed. Returns the number of errors (i.e. exit value is 0 if there are no consistency problems). """ from codeintel2.manager import Manager mgr = Manager(opts.db_base_dir) try: errors = mgr.db.check() finally: mgr.finalize() for error in errors: print(error) return len(errors)
def __init__(self, db_base_dir=None, fd_in=sys.stdin, fd_out=sys.stdout, buffer_max=20): threading.Thread.__init__(self, name="CodeIntel OOP Driver") assert Driver._instance is None, "Driver should be a singleton" Driver._instance = self logging.root.addHandler(LoggingHandler(self)) self.daemon = True self.fd_in = fd_in self.fd_out = fd_out self.abort = None self.quit = False self.buffers = DriverBufferCache(buffer_max) self.next_buffer = 0 self.active_request = None self.send_queue = Queue.Queue() self.send_thread = threading.Thread(name="Codeintel OOP Driver Send Thread", target=self._send_proc) self.send_thread.daemon = True self.send_thread.start() self.queue = collections.deque() self.queue_cv = threading.Condition() self.env = Environment(name="global", send_fn=functools.partial(self.send, request=None)) # Fill out the non-overridable build-in commands self._builtin_commands = {} for attr in dir(self): # Note that we check startswith first to avoid getters etc. if attr.startswith("do_") and callable(getattr(self, attr)): command = attr[len("do_"):].replace("_", "-") self._builtin_commands[command] = getattr(self, attr) from codeintel2.manager import Manager log.debug("using db base dir %s", db_base_dir) self.mgr = Manager(db_base_dir=db_base_dir, db_catalog_dirs=[], db_event_reporter=self._DBEventReporter(self), env=self.env, on_scan_complete=self._on_scan_complete) self.mgr.initialize()
def main(cix_filename, toolkit_jar_file, updatePerforce=False): cix_komodo = createCixRoot(name="Mozilla Toolkit", description="Mozilla Toolkit API - version 1.8") #cix_yui_file = createCixFile(cix_yui, "yui", lang="JavaScript") #cix_yui_module = createCixModule(cix_yui_file, "*", lang="JavaScript") print "cix_filename: %r" % (cix_filename, ) filenames_and_content = getMozillaToolkitFilenamesAndContent( toolkit_jar_file) jscile = JavaScriptCiler(Manager(), "Mozilla Toolkit") for filename, content in filenames_and_content.items(): jscile.scan_puretext(content) jscile.convertToElementTreeFile(cix_komodo, "JavaScript") #mergeElementTreeScopes(cix_yui_module) #remove_cix_line_numbers_from_tree(cix_komodo) #remove_private_elements(cix_komodo) # Write out the tree updateCix(cix_filename, get_cix_string(cix_komodo), updatePerforce)
def do_scan(self, subcmd, opts, *path_patterns): """Scan and print the CIX for the given path(s). ${cmd_usage} ${cmd_option_list} """ extra_module_dirs = [] if koextlib.is_ext_dir() and exists("pylib"): sys.path.append(abspath("pylib")) extra_module_dirs = [sys.path[-1]] mgr = Manager(extra_module_dirs=extra_module_dirs) mgr.upgrade() mgr.initialize() try: tree = None for path in _paths_from_path_patterns(path_patterns): try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: log.info( "skip `%s': couldn't determine language " "(use --lang option)", path) continue buf = mgr.buf_from_path(path, lang=opts.lang) if not isinstance(buf, CitadelBuffer): raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) buf.scan() # force a fresh scan tree = buf.tree for severity, msg in check_tree(tree): dump = {"warning": log.warn, "error": log.error}[severity] dump(msg) if opts.pretty_print: tree = pretty_tree_from_tree(tree) ET.dump(tree) finally: mgr.finalize()
def setUp(self): if _xpcom_: # The tests are run outside of Komodo. If run with PyXPCOM up # parts codeintel will try to use the nsIDirectoryService and # will query dirs only provided by nsXREDirProvider -- which # isn't registered outside of Komodo (XRE_main() isn't called). # The KoTestService provides a backup. koTestSvc = components.classes["@activestate.com/koTestService;1"] \ .getService(components.interfaces.koITestService) koTestSvc.init() if self._ci_test_setup_mgr_: env = None if self._ci_env_prefs_ is not None: env = SimplePrefsEnvironment(**self._ci_env_prefs_) self.mgr = Manager( extra_module_dirs=self._ci_extra_module_dirs_, db_base_dir=self._ci_db_base_dir_ or test_db_base_dir, db_catalog_dirs=self._ci_db_catalog_dirs_, db_import_everything_langs=self._ci_db_import_everything_langs, env=env) self.mgr.upgrade() self.mgr.initialize()
def main(cix_filename): cix = createCixRoot(name="%s_%s" % (library_name, library_version.replace(".", "")), description="%s JavaScript framework - version %s" % (library_name, library_version)) files = getFilesFromWebpage() jscile = JavaScriptCiler(Manager(), "extjs") for path, (dirname, filename, content) in files["source"].items(): dir_split = dirname.split("/") if ("source" in dir_split and not filename.startswith("ext-lang-")) or \ ("src" in dir_split and not "adapter" in dir_split): print "filename: %r" % (filename) jscile.path = filename jscile.scan_puretext(content.decode("utf-8"), updateAllScopeNames=False) jscile.cile.updateAllScopeNames() jscile.cile.name = "%s_%s" % (library_name.lower(), library_version.replace(".", "")) # Convert the Javascript to CIX, content goes into cix element jscile.convertToElementTreeFile(cix, "JavaScript") # Write out the tree updateCix(cix_filename, get_cix_string(cix))
def ci_setUpClass(cls): if _xpcom_: # The tests are run outside of Komodo. If run with PyXPCOM up # parts codeintel will try to use the nsIDirectoryService and # will query dirs only provided by nsXREDirProvider -- which # isn't registered outside of Komodo (XRE_main() isn't called). # The KoTestService provides a backup. koTestSvc = components.classes["@activestate.com/koTestService;1"] \ .getService(components.interfaces.koITestService) koTestSvc.init() if cls._ci_test_setup_mgr_: env = None if cls._ci_env_prefs_ is not None: env = SimplePrefsEnvironment(**cls._ci_env_prefs_) def get_extra_module_dirs(): spec = join(dirname( __file__), "..", "..", "udl", "skel", "*", "pylib") for d in glob(spec): if glob(join(spec, "lang_*.py")): yield d for d in cls._ci_extra_module_dirs_ or []: yield d cls.mgr = Manager( extra_module_dirs=get_extra_module_dirs(), db_base_dir=cls._ci_db_base_dir_ or test_db_base_dir, db_catalog_dirs=cls._ci_db_catalog_dirs_, db_import_everything_langs=cls._ci_db_import_everything_langs, env=env) cls.mgr.upgrade() cls.mgr.initialize() init_xml_catalogs()
def do_scan(self, subcmd, opts, *path_patterns): """Scan and print the CIX for the given path(s). ${cmd_usage} ${cmd_option_list} """ mgr = Manager() mgr.upgrade() mgr.initialize() try: if opts.time_it: start = time.time() quiet = opts.quiet if opts.time_it or opts.time_details: opts.force = True scan_count = 0 lang_warnings = set() tree = None for path in _paths_from_path_patterns(path_patterns, recursive=opts.recursive, includes=opts.includes): if opts.time_it: sys.stderr.write(path + "\n") if opts.time_details: start1 = time.time() try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: log.info("skip `%s': couldn't determine language", path) continue try: buf = mgr.buf_from_path(path, lang=lang) except OSError as ex: # Couldn't access the file. if not opts.recursive: raise # Ignore files we don't really care about. log.warn("%r - %r", ex, path) continue if not isinstance(buf, CitadelBuffer): if opts.recursive: # Ignore files that scanning isn't provided for. continue raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) scan_count += 1 if scan_count % 10 == 0: log.info("%d scanning %r", scan_count, path) try: if opts.force: buf.scan() if tree is None: tree = ET.Element("codeintel", version="2.0") file_elem = ET.SubElement(tree, "file", lang=buf.lang, mtime=str(int(time.time())), path=os.path.basename(path)) for lang, blob in sorted(buf.blob_from_lang.items()): blob = buf.blob_from_lang[lang] file_elem.append(blob) except KeyError as ex: # Unknown cile language. if not opts.recursive: raise message = str(ex) if message not in lang_warnings: lang_warnings.add(message) log.warn("Skipping unhandled language %s", message) if opts.time_details: delta = time.time() - start1 sys.stderr.write("%.3f %s\n" % (delta, path)) sys.stderr.flush() if tree is not None: if opts.stripfuncvars: # For stdlibs, we don't care about variables inside of # functions and they take up a lot of space. for function in tree.getiterator('scope'): if function.get('ilk') == 'function': function[:] = [ child for child in function if child.tag != 'variable' ] if opts.pretty_print: tree = pretty_tree_from_tree(tree) if not quiet: sys.stdout.write( '<?xml version="1.0" encoding="UTF-8"?>\n') ET.dump(tree) if opts.time_it: end = time.time() sys.stderr.write("scan took %.3fs\n" % (end - start)) finally: mgr.finalize()
def do_outline(self, subcmd, opts, path): """Print code outline of the given file. You can specify a lookup path into the file code outline to display via URL-anchor syntax, e.g.: ci2 outline path/to/foo.py#AClass.amethod ${cmd_usage} ${cmd_option_list} """ mgr = Manager() mgr.upgrade() mgr.initialize() try: if '#' in path: path, anchor = path.rsplit('#', 1) else: anchor = None if path.endswith(".cix"): tree = tree_from_cix(open(path, 'r').read()) #buf = mgr.buf_from_content("", tree[0].get("lang"), path=path) else: buf = mgr.buf_from_path(path, lang=opts.lang) tree = buf.tree if anchor is not None: # Lookup the anchor in the codeintel CIX tree. lpath = re.split(r'\.|::', anchor) def blobs_from_tree(tree): for file_elem in tree: for blob in file_elem: yield blob for elem in blobs_from_tree(tree): # Generally have 3 types of codeintel trees: # 1. single-lang file: one <file>, one <blob> # 2. multi-lang file: one <file>, one or two <blob>'s # 3. CIX stdlib/catalog file: possibly multiple # <file>'s, likely multiple <blob>'s # Allow the first token to be the blob name or lang. # (This can sometimes be weird, but seems the most # convenient solution.) if lpath[0] in (elem.get("name"), elem.get("lang")): remaining_lpath = lpath[1:] else: remaining_lpath = lpath for name in remaining_lpath: try: elem = elem.names[name] except KeyError: elem = None break # try next lang blob if elem is not None: break # found one else: log.error( "could not find `%s' definition (or blob) in `%s'", anchor, path) return 1 else: elem = tree try: _outline_ci_elem(elem, brief=opts.brief, doSort=opts.doSort) except IOError as ex: if ex.errno == 0: # Ignore this error from aborting 'less' of 'ci2 outline' # output: # IOError: (0, 'Error') pass else: raise finally: mgr.finalize()
def do_play(self, subcmd, opts): """Run my current play/dev code. ${cmd_usage} ${cmd_option_list} """ if False: lang = "CSS" markedup_content = dedent(""" /* http://www.w3.org/TR/REC-CSS2/fonts.html#propdef-font-weight */ h1 { border: 1px solid black; font-weight /* hi */: <|> !important } """) content, data = unmark_text(markedup_content) pos = data["pos"] mgr = Manager() #mgr.upgrade() # Don't need it for just CSS usage. mgr.initialize() try: buf = mgr.buf_from_content(content, lang=lang, path="play.css") trg = buf.trg_from_pos(pos) if trg is None: raise Error("unexpected trigger: %r" % trg) completions = buf.cplns_from_trg(trg) print("COMPLETIONS: %r" % completions) finally: mgr.finalize() elif False: lang = "Python" path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100)) markedup_content = dedent(""" import sys, os class Foo: def bar(self): pass sys.<|>path # should have path in completion list f = Foo() """) content, data = unmark_text(markedup_content) print(banner(path)) print(_escaped_text_from_text(content, "whitespace")) pos = data["pos"] mgr = Manager() mgr.upgrade() mgr.initialize() try: buf = mgr.buf_from_content(content, lang=lang, path=path) print(banner("cix", '-')) print(buf.cix) trg = buf.trg_from_pos(pos) if trg is None: raise Error("unexpected trigger: %r" % trg) print(banner("completions", '-')) ctlr = LogEvalController(log) buf.async_eval_at_trg(trg, ctlr) ctlr.wait(2) #XXX if not ctlr.is_done(): ctlr.abort() raise Error("XXX async eval timed out") pprint(ctlr.cplns) print(banner(None)) finally: mgr.finalize() elif False: lang = "Ruby" path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100)) markedup_content = dedent("""\ r<1>equire 'net/http' include Net req = HTTPRequest.new req.<2>get() """) content, data = unmark_text(markedup_content) print(banner(path)) print(_escaped_text_from_text(content, "whitespace")) pos = data[1] mgr = Manager() mgr.upgrade() mgr.initialize() try: buf = mgr.buf_from_content(content, lang=lang, path=path) print(banner("cix", '-')) cix = buf.cix print( ET.tostring(pretty_tree_from_tree(tree_from_cix(buf.cix)))) trg = buf.trg_from_pos(pos, implicit=False) if trg is None: raise Error("unexpected trigger: %r" % trg) print(banner("completions", '-')) ctlr = LogEvalController(log) buf.async_eval_at_trg(trg, ctlr) ctlr.wait(30) #XXX if not ctlr.is_done(): ctlr.abort() raise Error("XXX async eval timed out") pprint(ctlr.cplns) print(banner(None)) finally: mgr.finalize()
def do_json(self, subcmd, opts, path): """Convert cix XML file into json format. ${cmd_usage} ${cmd_option_list} """ import json from collections import defaultdict from codeintel2.manager import Manager from codeintel2.util import tree_from_cix from codeintel2.common import Error if opts.output == '-': output_path = None output_file = sys.stdout else: if opts.output: output_path = opts.output else: output_path = os.path.splitext(path)[0] + ".json" if os.path.exists(output_path): if opts.force: os.remove(output_path) else: raise Error("`%s' exists: use -f|--force option to " "allow overwrite" % output_path) output_file = open(output_path, 'w') mgr = Manager() mgr.upgrade() mgr.initialize() try: if path.endswith(".cix"): tree = tree_from_cix(open(path, 'r').read()) else: buf = mgr.buf_from_path(path, lang=opts.lang) tree = buf.tree result = {} ci = result["codeintel"] = defaultdict(list) def _elemToDict(parent, elem): data = defaultdict(list) name = elem.get("name") if name is not None: data["name"] = name data["tag"] = elem.tag for attr_name, attr in elem.attrib.items(): data[attr_name] = attr parent["children"].append(data) for child in elem: _elemToDict(data, child) for child in tree: _elemToDict(ci, child) json.dump(result, output_file, indent=2) finally: mgr.finalize()
def cix2html(opts, path): """Turn cix file into html API documentation. Example: cix2html path/to/foo.cix#AClass.amethod cix2html path/to/foo.cix -o file.html ${cmd_usage} ${cmd_option_list} """ mgr = Manager() mgr.upgrade() mgr.initialize() try: def blobs_from_tree(tree): for file_elem in tree: for blob in file_elem: yield blob if '#' in path: path, anchor = path.rsplit('#', 1) else: anchor = None if path.endswith(".cix"): tree = tree_from_cix(open(path, 'r').read()) # buf = mgr.buf_from_content("", tree[0].get("lang"), path=path) else: buf = mgr.buf_from_path(path, lang=opts.lang) tree = buf.tree if anchor is not None: # Lookup the anchor in the codeintel CIX tree. lpath = re.split(r'\.|::', anchor) for elem in blobs_from_tree(tree): # Generally have 3 types of codeintel trees: # 1. single-lang file: one <file>, one <blob> # 2. multi-lang file: one <file>, one or two <blob>'s # 3. CIX stdlib/catalog file: possibly multiple # <file>'s, likely multiple <blob>'s # Allow the first token to be the blob name or lang. # (This can sometimes be weird, but seems the most # convenient solution.) if lpath[0] in (elem.get("name"), elem.get("lang")): remaining_lpath = lpath[1:] else: remaining_lpath = lpath for name in remaining_lpath: try: elem = elem.names[name] except KeyError: elem = None break # try next lang blob if elem is not None: break # found one else: log.error("could not find `%s' definition (or blob) in `%s'", anchor, path) return 1 else: elem = tree try: if elem.tag == "codeintel": _html_ci_elem(opts, elem.getchildren()[0]) else: _html_ci_elem(opts, elem) except IOError, ex: if ex.errno == 0: # Ignore this error from aborting 'less' of 'ci2 outline' # output: # IOError: (0, 'Error') pass else: raise except Exception, e: import traceback traceback.print_exc()
def log_message(self, format, *args): return # log silently logStream = logging.StreamHandler(sys.stderr) logStream.setFormatter( logging.Formatter("%(name)s: %(levelname)s: %(message)s")) logging.getLogger("codeintel").addHandler(logStream) logger = logging.getLogger("codeintel_server") logger.addHandler(logStream) logger.setLevel(logging.WARNING) manager = Manager( # db_base_dir = path.join(CI_DIR, 'db'), # extra_module_dirs = [path.join(CI_DIR, 'codeintel2'),], # db_import_everything_langs = None, # db_catalog_dirs = [] ) manager.upgrade() manager.initialize() scanned_langs = [] if __name__ == "__main__": parser = argparse.ArgumentParser( description="Run codeintel commands as a daemon or via stdin") parser.add_argument("mode", help="Mode of operation", choices=[ "daemon", "completions", "definitions", "calltips", "catalogs" ])
def do_outline(self, subcmd, opts, path): """Scan and outline the structure of the given path. ${cmd_usage} ${cmd_option_list} """ extra_lang_module_dirs = [] if koextlib.is_ext_dir() and exists("pylib"): sys.path.append(abspath("pylib")) extra_lang_module_dirs = [sys.path[-1]] mgr = Manager(extra_lang_module_dirs=extra_lang_module_dirs) mgr.upgrade() mgr.initialize() try: if '#' in path: path, anchor = path.rsplit('#', 1) else: anchor = None tree = None try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: log.info( "skip `%s': couldn't determine language " "(use --lang option)", path) return if path.endswith(".cix"): tree = tree_from_cix(open(path, 'r').read()) #buf = mgr.buf_from_content("", tree[0].get("lang"), path=path) else: buf = mgr.buf_from_path(path, lang=opts.lang) if not isinstance(buf, CitadelBuffer): raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) tree = buf.tree if anchor is not None: # Lookup the anchor in the codeintel CIX tree. lpath = re.split(r'\.|::', anchor) def blobs_from_tree(tree): for file_elem in tree: for blob in file_elem: yield blob for elem in blobs_from_tree(tree): # Generally have 3 types of codeintel trees: # 1. single-lang file: one <file>, one <blob> # 2. multi-lang file: one <file>, one or two <blob>'s # 3. CIX stdlib/catalog file: possibly multiple # <file>'s, likely multiple <blob>'s # Allow the first token to be the blob name or lang. # (This can sometimes be weird, but seems the most # convenient solution.) if lpath[0] in (elem.get("name"), elem.get("lang")): remaining_lpath = lpath[1:] else: remaining_lpath = lpath for name in remaining_lpath: try: elem = elem.names[name] except KeyError: elem = None break # try next lang blob if elem is not None: break # found one else: log.error( "could not find `%s' definition (or blob) in `%s'", anchor, path) return 1 else: elem = tree try: _outline_ci_elem(mgr, elem, quiet=opts.quiet) except IOError, ex: if ex.errno == 0: # Ignore this error from aborting 'less' of this # command: # IOError: (0, 'Error') pass else: raise finally: mgr.finalize()
_fn = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'path.ini') if os.path.isfile(_fn): with open(_fn, 'r') as f: paths = f.readline() print 'Paths:\n' + '\n'.join(paths.split(os.pathsep)) d = {} d['phpExtraPaths'] = paths d['pythonExtraPaths'] = paths d['perlExtraPaths'] = paths d['javascriptExtraPaths'] = paths d['rubyExtraPaths'] = paths env = SimplePrefsEnvironment(**d) mgr = Manager() mgr.upgrade() mgr.initialize() #------------------------ def file_text(fn): try: with open(fn, 'r') as f: text = f.readlines() return ''.join(text) except IOError as e: if e.errno == errno.ENOENT: # No such file or directory logging.error(e.strerror + ': "' + fn + '"') else: logging.error(e.strerror)
def _codeintel_scan(): global _ci_mgr_, despair, despaired env = None mtime = None catalogs = [] now = time.time() try: env = _ci_envs_[path] if env._folders != folders: raise KeyError mgr = _ci_mgr_ if now > env._time: mtime = max(tryGetMTime(env._config_file), tryGetMTime(env._config_default_file)) if env._mtime < mtime: raise KeyError except KeyError: if env is not None: config_default_file = env._config_default_file project_dir = env._project_dir project_base_dir = env._project_base_dir config_file = env._config_file else: config_default_file = os.path.join(CODEINTEL_HOME_DIR, 'config') if not (config_default_file and os.path.exists(config_default_file)): config_default_file = None project_dir = None project_base_dir = None if path: # Try to find a suitable project directory (or best guess): for folder in ['.codeintel', '.git', '.hg', 'trunk']: project_dir = find_folder(path, folder) if project_dir and ( folder != '.codeintel' or not os.path.exists( os.path.join(project_dir, 'db'))): if folder.startswith('.'): project_base_dir = os.path.abspath( os.path.join(project_dir, '..')) else: project_base_dir = project_dir break if not (project_dir and os.path.exists(project_dir)): project_dir = None config_file = project_dir and folder == '.codeintel' and os.path.join( project_dir, 'config') if not (config_file and os.path.exists(config_file)): config_file = None if _ci_mgr_: mgr = _ci_mgr_ else: for thread in threading.enumerate(): if thread.name == "CodeIntel Manager": thread.finalize( ) # this finalizes the index, citadel and the manager and waits them to end (join) mgr = Manager( extra_module_dirs=_ci_extra_module_dirs_, db_base_dir=_ci_db_base_dir_, db_catalog_dirs=_ci_db_catalog_dirs_, db_import_everything_langs=_ci_db_import_everything_langs, db_event_reporter=lambda m: logger(view, 'event', m), ) mgr.upgrade() mgr.initialize() # Connect the logging file to the handler condeintel_log_file = os.path.join(mgr.db.base_dir, 'codeintel.log') codeintel_log.handlers = [ logging.StreamHandler(open(condeintel_log_file, 'w', 1)) ] msg = "Starting logging SublimeCodeIntel rev %s (%s) on %s" % ( get_git_revision()[:12], os.stat(__file__)[stat.ST_MTIME], datetime.datetime.now().ctime()) codeintel_log.info("%s\n%s" % (msg, "=" * len(msg))) _ci_mgr_ = mgr # Load configuration files: for catalog in mgr.db.get_catalogs_zone().avail_catalogs(): if catalog['lang'] == lang: catalogs.append(catalog['name']) config = { "codeintel_selected_catalogs": catalogs, "codeintel_max_recursive_dir_depth": 10, "codeintel_scan_files_in_project": True, } _config = {} try: tryReadDict(config_default_file, _config) except Exception, e: msg = "Malformed configuration file '%s': %s" % ( config_default_file, e) log.error(msg) codeintel_log.error(msg) try: tryReadDict(config_file, _config) except Exception, e: msg = "Malformed configuration file '%s': %s" % ( config_default_file, e) log.error(msg) codeintel_log.error(msg)