def htmlify(rel_path, text): from codeintel2.citadel import CitadelBuffer from codeintel2.common import CodeIntelError from codeintel2.util import guess_lang_from_path path = os.path.join(opts.root, rel_path) try: lang = guess_lang_from_path(path) except CodeIntelError: return None try: buf = opts.mgr.buf_from_path(path, lang=lang) except CodeIntelError as ex: if ex.message.startswith("File too big."): log.info("%s: %s", rel_path, ex.message) return None # Nothing we can do about that, and the user can't # fix this ever. raise if not isinstance(buf, CitadelBuffer): log.info("%s: language %s does not have CIX, not htmlifying", rel_path, lang) return None if lang == "Tcl": # Tcl is busted, it has no async_eval_at_trg; Komodo just plain has a # different system for handling tcl... return None log.info("htmlifying %s as %s", rel_path, lang) return CiHtmlifier(buf)
def do_scan(self, subcmd, opts, *path_patterns): """Scan and print the CIX for the given path(s). ${cmd_usage} ${cmd_option_list} """ extra_module_dirs = [] if koextlib.is_ext_dir() and exists("pylib"): sys.path.append(abspath("pylib")) extra_module_dirs = [sys.path[-1]] mgr = Manager(extra_module_dirs=extra_module_dirs) mgr.upgrade() mgr.initialize() try: tree = None for path in _paths_from_path_patterns(path_patterns): try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: log.info("skip `%s': couldn't determine language " "(use --lang option)", path) continue buf = mgr.buf_from_path(path, lang=opts.lang) if not isinstance(buf, CitadelBuffer): raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) buf.scan() # force a fresh scan tree = buf.tree for severity, msg in check_tree(tree): dump = {"warning": log.warn, "error": log.error}[severity] dump(msg) if opts.pretty_print: tree = pretty_tree_from_tree(tree) ET.dump(tree) finally: mgr.finalize()
def on_modified(self, view): path = view.file_name() lang, _ = os.path.splitext(os.path.basename(view.settings().get("syntax"))) try: lang = lang or guess_lang_from_path(path) except CodeIntelError: pass live = view.settings().get("codeintel_live", True) pos = view.sel()[0].end() text = view.substr(sublime.Region(pos - 1, pos)) _sentinel = sentinel.get(path) is_fill_char = text and text[-1] in cpln_fillup_chars.get(lang, "") sentinel[path] = _sentinel if _sentinel is not None else pos if is_fill_char else None if not live and text and sentinel[path] is not None: live = True if live: if not hasattr(view, "command_history") or view.command_history(0)[0] == "insert": autocomplete(view, 0 if is_fill_char else 200, 50 if is_fill_char else 600, is_fill_char, args=[path]) else: view.run_command("hide_auto_complete") else: def _scan_callback(view, path): content = view.substr(sublime.Region(0, view.size())) lang, _ = os.path.splitext(os.path.basename(view.settings().get("syntax"))) codeintel_scan(view, path, content, lang) queue(view, _scan_callback, 3000, args=[path])
def guess_lang(view=None, path=None): if not view.settings().get('codeintel', True): return None syntax = None if view: syntax = os.path.splitext( os.path.basename(view.settings().get('syntax')))[0] id = view.id() _k_ = '%s::%s' % (syntax, path) try: return languages[id][_k_] except KeyError: pass languages.setdefault(id, {}) lang = None _codeintel_syntax_map = dict( (k.lower(), v) for k, v in view.settings().get('codeintel_syntax_map', {}).items()) _lang = lang = syntax and _codeintel_syntax_map.get(syntax.lower(), syntax) mgr = codeintel_manager() if not mgr.is_citadel_lang(lang) and not mgr.is_cpln_lang(lang): lang = None if mgr.is_citadel_lang(syntax) or mgr.is_cpln_lang(syntax): _lang = lang = syntax else: if view and not path: path = view.file_name() if path: try: _lang = lang = guess_lang_from_path(path) except CodeIntelError: languages[id][_k_] = None return _codeintel_disabled_languages = [ l.lower() for l in view.settings().get('codeintel_disabled_languages', []) ] if lang and lang.lower() in _codeintel_disabled_languages: logger(view, 'info', "skip `%s': disabled language" % lang) languages[id][_k_] = None return if not lang and _lang and _lang not in ('Console', ): if mgr: logger( view, 'info', "Invalid language: %s. Available: %s" % (_lang, ', '.join( set(mgr.get_citadel_langs() + mgr.get_cpln_langs())))) else: logger(view, 'info', "Invalid language: %s" % _lang) languages[id][_k_] = lang return lang
def worker(queue, lock, cix_dir, root="/"): """ worker procedure """ fix_module_path() import dxr.mime from codeintel2.citadel import CitadelBuffer from codeintel2.common import CodeIntelError from codeintel2.manager import Manager from codeintel2.util import guess_lang_from_path logging.getLogger("codeintel").setLevel(logging.ERROR) log.info("starting indexing using %s", multiprocessing.current_process().name) mgr = Manager(db_base_dir=cix_dir) #mgr.upgrade() mgr.initialize() while True: file_path = queue.get() if file_path is None: # marker for end of list queue.put(None) # put it back so others can quit too break rel_path = os.path.relpath(file_path, root) try: lang = guess_lang_from_path(file_path) except CodeIntelError: log.info("%s: Cannot determine language, skipping", rel_path) continue # the file with open(file_path, "r") as source_file: data = source_file.read() # Discard non-text files if not dxr.mime.is_text(file_path, data): continue try: buf = mgr.buf_from_path(file_path, lang=lang) except CodeIntelError as ex: if ex.message.startswith("File too big."): log.info("%s: %s", file_path, ex.message) continue # Nothing we can do about that, and the user can't # fix this ever. raise if not isinstance(buf, CitadelBuffer): log.info("%s: language %s does not have CIX, skipping", rel_path, lang) continue log.info("%s: Using language %s", rel_path, lang) buf.scan() mgr.finalize()
def guess_lang(view=None, path=None): if not view.settings().get("codeintel", True): return None syntax = None if view: syntax = os.path.splitext(os.path.basename(view.settings().get("syntax")))[0] id = view.id() _k_ = "%s::%s" % (syntax, path) try: return languages[id][_k_] except KeyError: pass languages.setdefault(id, {}) lang = None _codeintel_syntax_map = dict((k.lower(), v) for k, v in view.settings().get("codeintel_syntax_map", {}).items()) _lang = lang = syntax and _codeintel_syntax_map.get(syntax.lower(), syntax) mgr = codeintel_manager() if not mgr.is_citadel_lang(lang) and not mgr.is_cpln_lang(lang): lang = None if mgr.is_citadel_lang(syntax) or mgr.is_cpln_lang(syntax): _lang = lang = syntax else: if view and not path: path = view.file_name() if path: try: _lang = lang = guess_lang_from_path(path) except CodeIntelError: languages[id][_k_] = None return _codeintel_disabled_languages = [l.lower() for l in view.settings().get("codeintel_disabled_languages", [])] if lang and lang.lower() in _codeintel_disabled_languages: logger(view, "info", "skip `%s': disabled language" % lang) languages[id][_k_] = None return if not lang and _lang and _lang not in ("Console",): if mgr: logger( view, "info", "Invalid language: %s. Available: %s" % (_lang, ", ".join(set(mgr.get_citadel_langs() + mgr.get_cpln_langs()))), ) else: logger(view, "info", "Invalid language: %s" % _lang) languages[id][_k_] = lang return lang
def _fillScanInputsTestCase(): for dpath, dnames, fnames in os.walk(gInputsDir): # Don't descend into SCC control dirs. scc_dirs = [".svn", "CVS", ".hg", ".git"] for scc_dir in scc_dirs: if scc_dir in dnames: dnames.remove(scc_dir) if dpath == gInputsDir and "unicode" in dnames: # The scan_inputs/unicode is where the unicode test files # are placed. Don't descend into here. They are handled elsewhere. dnames.remove("unicode") if ".svn" in dpath.split(os.sep): # Skip subversion directories. continue for fname in fnames: fpath = os.path.join(dpath, fname)[len(gInputsDir) + len(os.sep):] if not isfile(join(dpath, fname)): # With our Unicode testing we may have a directory that # Python's os.walk() doesn't recognize as a dir, defaults to # a file and hands it to us here. Skip those. continue if fname == ".DS_Store": continue if fpath.endswith(".swp"): continue if fpath.endswith("~"): continue if fpath.endswith("__pycache__"): continue if fpath.endswith(".pyc"): continue if fpath.endswith(".pyo"): continue if fpath.endswith(".pod"): continue if fpath.endswith(".options"): continue # skip input option files if fpath.endswith(".tags"): continue # skip tags files lang = guess_lang_from_path(fpath) # Manual hack to detect as Python 3. if lang == "Python" and "py3" in fpath: lang = "Python3" safe_lang = safe_lang_from_lang(lang) # Set tags for this test case. tags = [safe_lang] tagspath = join(dpath, fname + ".tags") # ws-separate set of tags if exists(tagspath): tags += open(tagspath, 'r').read().split() def makeTestFunction(fpath_, tags_): testFunction \ = lambda self, fpath=fpath_: _testOneInputFile(self, fpath_, tags=tags_) testFunction.tags = tags_ return testFunction name = "test_path:" + fpath setattr(ScanInputsTestCase, name, makeTestFunction(fpath, tags)) _addUnicodeScanInputTests()
def _fillScanInputsTestCase(): for dpath, dnames, fnames in os.walk(gInputsDir): # Don't descend into SCC control dirs. scc_dirs = [".svn", "CVS", ".hg", ".git"] for scc_dir in scc_dirs: if scc_dir in dnames: dnames.remove(scc_dir) if dpath == gInputsDir and "unicode" in dnames: # The scan_inputs/unicode is where the unicode test files # are placed. Don't descend into here. They are handled elsewhere. dnames.remove("unicode") if ".svn" in dpath.split(os.sep): # Skip subversion directories. continue for fname in fnames: fpath = os.path.join(dpath, fname)[len(gInputsDir)+len(os.sep):] if not isfile(join(dpath, fname)): # With our Unicode testing we may have a directory that # Python's os.walk() doesn't recognize as a dir, defaults to # a file and hands it to us here. Skip those. continue if fname == ".DS_Store": continue if fpath.endswith(".swp"): continue if fpath.endswith("~"): continue if fpath.endswith("__pycache__"): continue if fpath.endswith(".pyc"): continue if fpath.endswith(".pyo"): continue if fpath.endswith(".pod"): continue if fpath.endswith(".options"): continue # skip input option files if fpath.endswith(".tags"): continue # skip tags files lang = guess_lang_from_path(fpath) # Manual hack to detect as Python 3. if lang == "Python" and "py3" in fpath: lang = "Python3" safe_lang = safe_lang_from_lang(lang) # Set tags for this test case. tags = [safe_lang] tagspath = join(dpath, fname + ".tags") # ws-separate set of tags if exists(tagspath): tags += open(tagspath, 'r').read().split() def makeTestFunction(fpath_, tags_): testFunction \ = lambda self, fpath=fpath_: _testOneInputFile(self, fpath_, tags=tags_) testFunction.tags = tags_ return testFunction name = "test_path:"+fpath setattr(ScanInputsTestCase, name, makeTestFunction(fpath, tags)) _addUnicodeScanInputTests()
def guess_lang(view=None, path=None): if not view or not view.settings().get('codeintel', True): return None syntax = None if view: syntax = os.path.splitext(os.path.basename(view.settings().get('syntax')))[0] vid = view.id() _k_ = '%s::%s' % (syntax, path) try: return languages[vid][_k_] except KeyError: pass languages.setdefault(vid, {}) lang = None _codeintel_syntax_map = dict((k.lower(), v) for k, v in view.settings().get('codeintel_syntax_map', {}).items()) _lang = lang = syntax and _codeintel_syntax_map.get(syntax.lower(), syntax) folders = getattr(view.window(), 'folders', lambda: [])() # FIXME: it's like this for backward compatibility (<= 2060) folders_id = str(hash(frozenset(folders))) mgr = codeintel_manager(folders_id) if not mgr.is_citadel_lang(lang) and not mgr.is_cpln_lang(lang): lang = None if mgr.is_citadel_lang(syntax) or mgr.is_cpln_lang(syntax): _lang = lang = syntax else: if view and not path: path = view.file_name() if path: try: _lang = lang = guess_lang_from_path(path) except CodeIntelError: languages[vid][_k_] = None return _codeintel_disabled_languages = [l.lower() for l in view.settings().get('codeintel_disabled_languages', [])] if lang and lang.lower() in _codeintel_disabled_languages: logger(view, 'debug', "skip `%s': disabled language" % lang) languages[vid][_k_] = None return if not lang and _lang and _lang in ('Console', 'Plain text'): if mgr: logger(view, 'debug', "Invalid language: %s. Available: %s" % (_lang, ', '.join(set(mgr.get_citadel_langs() + mgr.get_cpln_langs())))) else: logger(view, 'debug', "Invalid language: %s" % _lang) languages[vid][_k_] = lang return lang
def guess_lang(view=None, path=None): if not view or not view.settings().get('codeintel', True): return None syntax = None if view: syntax = os.path.splitext(os.path.basename(view.settings().get('syntax')))[0] vid = view.id() _k_ = '%s::%s' % (syntax, path) try: return languages[vid][_k_] except KeyError: pass languages.setdefault(vid, {}) lang = None _codeintel_syntax_map = dict((k.lower(), v) for k, v in view.settings().get('codeintel_syntax_map', {}).items()) _lang = lang = syntax and _codeintel_syntax_map.get(syntax.lower(), syntax) folders = getattr(view.window(), 'folders', lambda: [])() # FIXME: it's like this for backward compatibility (<= 2060) folders_id = str(hash(frozenset(folders))) mgr = codeintel_manager(folders_id) if not mgr.is_citadel_lang(lang) and not mgr.is_cpln_lang(lang): lang = None if mgr.is_citadel_lang(syntax) or mgr.is_cpln_lang(syntax): _lang = lang = syntax else: if view and not path: path = view.file_name() if path: try: _lang = lang = guess_lang_from_path(path) except CodeIntelError: languages[vid][_k_] = None return _codeintel_enabled_languages = [l.lower() for l in view.settings().get('codeintel_enabled_languages', [])] if lang and lang.lower() not in _codeintel_enabled_languages: languages[vid][_k_] = None return None if not lang and _lang and _lang in ('Console', 'Plain text'): if mgr: logger(view, 'debug', "Invalid language: %s. Available: %s" % (_lang, ', '.join(set(mgr.get_citadel_langs() + mgr.get_cpln_langs())))) else: logger(view, 'debug', "Invalid language: %s" % _lang) languages[vid][_k_] = lang return lang
def guess_lang(view=None, path=None): if not view.settings().get('codeintel', True): return None mgr = codeintel_manager() _codeintel_syntax_map = dict( (k.lower(), v) for k, v in view.settings().get('codeintel_syntax_map', {}).items()) _codeintel_disabled_languages = [ l.lower() for l in view.settings().get('codeintel_disabled_languages', []) ] lang = None syntax = None if view: syntax = os.path.splitext( os.path.basename(view.settings().get('syntax')))[0] lang = syntax and _codeintel_syntax_map.get(syntax.lower(), syntax) if not mgr.is_citadel_lang(lang) and not mgr.is_cpln_lang(lang): lang = None if mgr.is_citadel_lang(syntax) or mgr.is_cpln_lang(syntax): lang = syntax else: if view and not path: path = view.file_name() if path: try: lang = guess_lang_from_path(path) except CodeIntelError: logger(view, 'warning', "skip `%s': couldn't determine language" % path) return if lang and lang.lower() in _codeintel_disabled_languages: logger(view, 'info', "skip `%s': disabled language" % lang) return if not lang: if mgr: logger( view, 'info', "Invalid language: %s. Available: %s" % (lang, ', '.join( set(mgr.get_citadel_langs() + mgr.get_cpln_langs())))) else: logger(view, 'info', "Invalid language: %s" % lang) return lang
def guess_lang(view=None, path=None): if not view.settings().get('codeintel', True): return None mgr = codeintel_manager() _codeintel_syntax_map = dict((k.lower(), v) for k, v in view.settings().get('codeintel_syntax_map', {}).items()) _codeintel_disabled_languages = [l.lower() for l in view.settings().get('codeintel_disabled_languages', [])] lang = None syntax = None if view: syntax = os.path.splitext(os.path.basename(view.settings().get('syntax')))[0] lang = syntax and _codeintel_syntax_map.get(syntax.lower(), syntax) if not mgr.is_citadel_lang(lang) and not mgr.is_cpln_lang(lang): lang = None if mgr.is_citadel_lang(syntax) or mgr.is_cpln_lang(syntax): lang = syntax else: if view and not path: path = view.file_name() if path: try: lang = guess_lang_from_path(path) except CodeIntelError: logger(view, 'warning', "skip `%s': couldn't determine language" % path) return if lang and lang.lower() in _codeintel_disabled_languages: logger(view, 'info', "skip `%s': disabled language" % lang) return if not lang: if mgr: logger(view, 'info', "Invalid language: %s. Available: %s" % (lang, ', '.join(set(mgr.get_citadel_langs() + mgr.get_cpln_langs())))) else: logger(view, 'info', "Invalid language: %s" % lang) return lang
def do_scan(self, subcmd, opts, *path_patterns): """Scan and print the CIX for the given path(s). ${cmd_usage} ${cmd_option_list} """ extra_module_dirs = [] if koextlib.is_ext_dir() and exists("pylib"): sys.path.append(abspath("pylib")) extra_module_dirs = [sys.path[-1]] mgr = Manager(extra_module_dirs=extra_module_dirs) mgr.upgrade() mgr.initialize() try: tree = None for path in _paths_from_path_patterns(path_patterns): try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: log.info( "skip `%s': couldn't determine language " "(use --lang option)", path) continue buf = mgr.buf_from_path(path, lang=opts.lang) if not isinstance(buf, CitadelBuffer): raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) buf.scan() # force a fresh scan tree = buf.tree for severity, msg in check_tree(tree): dump = {"warning": log.warn, "error": log.error}[severity] dump(msg) if opts.pretty_print: tree = pretty_tree_from_tree(tree) ET.dump(tree) finally: mgr.finalize()
def on_modified(self, view): path = view.file_name() lang, _ = os.path.splitext( os.path.basename(view.settings().get('syntax'))) try: lang = lang or guess_lang_from_path(path) except CodeIntelError: pass live = view.settings().get('codeintel_live', True) pos = view.sel()[0].end() text = view.substr(sublime.Region(pos - 1, pos)) _sentinel = sentinel.get(path) is_fill_char = (text and text[-1] in cpln_fillup_chars.get(lang, '')) sentinel[ path] = _sentinel if _sentinel is not None else pos if is_fill_char else None if not live and text and sentinel[path] is not None: live = True if live: if not hasattr(view, 'command_history') or view.command_history( 0)[0] == 'insert': autocomplete(view, 0 if is_fill_char else 200, 50 if is_fill_char else 600, is_fill_char, args=[path]) else: view.run_command('hide_auto_complete') else: def _scan_callback(view, path): content = view.substr(sublime.Region(0, view.size())) lang, _ = os.path.splitext( os.path.basename(view.settings().get('syntax'))) codeintel_scan(view, path, content, lang) queue(view, _scan_callback, 3000, args=[path])
def do_scan(self, subcmd, opts, *path_patterns): """Scan and print the CIX for the given path(s). ${cmd_usage} ${cmd_option_list} """ import time import ciElementTree as ET from ci2 import _paths_from_path_patterns from codeintel2.manager import Manager from codeintel2.citadel import CitadelBuffer from codeintel2.common import CodeIntelError from codeintel2.tree import pretty_tree_from_tree from codeintel2.util import guess_lang_from_path mgr = Manager() mgr.upgrade() mgr.initialize() try: if opts.time_it: start = time.time() quiet = opts.quiet if opts.time_it or opts.time_details: opts.force = True scan_count = 0 lang_warnings = set() tree = None for path in _paths_from_path_patterns(path_patterns, recursive=opts.recursive, includes=opts.includes): if opts.time_it: sys.stderr.write(path + "\n") if opts.time_details: start1 = time.time() try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: self.log.info("skip `%s': couldn't determine language", path) continue try: buf = mgr.buf_from_path(path, lang=lang) except OSError as ex: # Couldn't access the file. if not opts.recursive: raise # Ignore files we don't really care about. self.log.warn("%r - %r", ex, path) continue if not isinstance(buf, CitadelBuffer): if opts.recursive: # Ignore files that scanning isn't provided for. continue raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) scan_count += 1 if scan_count % 10 == 0: self.log.info("%d scanning %r", scan_count, path) try: if opts.force: buf.scan() if tree is None: tree = ET.Element("codeintel", version="2.0") file_elem = ET.SubElement(tree, "file", lang=buf.lang, mtime=str(int(time.time())), path=os.path.basename(path)) for lang, blob in sorted(buf.blob_from_lang.items()): blob = buf.blob_from_lang[lang] file_elem.append(blob) except KeyError as ex: # Unknown cile language. if not opts.recursive: raise message = str(ex) if message not in lang_warnings: lang_warnings.add(message) self.log.warn("Skipping unhandled language %s", message) if opts.time_details: delta = time.time() - start1 sys.stderr.write("%.3f %s\n" % (delta, path)) sys.stderr.flush() if tree is not None: if opts.stripfuncvars: # For stdlibs, we don't care about variables inside of # functions and they take up a lot of space. for function in tree.getiterator('scope'): if function.get('ilk') == 'function': function[:] = [child for child in function if child.tag != 'variable'] if opts.pretty_print: tree = pretty_tree_from_tree(tree) if not quiet: sys.stdout.write('<?xml version="1.0" encoding="UTF-8"?>\n') ET.dump(tree) if opts.time_it: end = time.time() sys.stderr.write("scan took %.3fs\n" % (end - start)) finally: mgr.finalize()
def codeintel_scan(view, path, content, lang, callback=None, pos=None, forms=None): global despair for thread in threading.enumerate(): if thread.isAlive() and thread.name == "scanning thread": logger( view, 'info', "Updating indexes... The first time this can take a while. Do not despair!", timeout=20000, delay=despair) despair = 0 return try: lang = lang or guess_lang_from_path(path) except CodeIntelError: logger(view, 'warning', "skip `%s': couldn't determine language" % path) return if lang.lower() in [ l.lower() for l in view.settings().get('codeintel_disabled_languages', []) ]: return is_scratch = view.is_scratch() is_dirty = view.is_dirty() folders = getattr(view.window(), 'folders', lambda: [])( ) # FIXME: it's like this for backward compatibility (<= 2060) def _codeintel_scan(): global _ci_mgr_, despair, despaired env = None mtime = None catalogs = [] now = time.time() try: env = _ci_envs_[path] if env._folders != folders: raise KeyError mgr = _ci_mgr_ if now > env._time: mtime = max(tryGetMTime(env._config_file), tryGetMTime(env._config_default_file)) if env._mtime < mtime: raise KeyError except KeyError: if env is not None: config_default_file = env._config_default_file project_dir = env._project_dir project_base_dir = env._project_base_dir config_file = env._config_file else: config_default_file = os.path.join(CODEINTEL_HOME_DIR, 'config') if not (config_default_file and os.path.exists(config_default_file)): config_default_file = None project_dir = None project_base_dir = None if path: # Try to find a suitable project directory (or best guess): for folder in ['.codeintel', '.git', '.hg', 'trunk']: project_dir = find_folder(path, folder) if project_dir and ( folder != '.codeintel' or not os.path.exists( os.path.join(project_dir, 'db'))): if folder.startswith('.'): project_base_dir = os.path.abspath( os.path.join(project_dir, '..')) else: project_base_dir = project_dir break if not (project_dir and os.path.exists(project_dir)): project_dir = None config_file = project_dir and folder == '.codeintel' and os.path.join( project_dir, 'config') if not (config_file and os.path.exists(config_file)): config_file = None if _ci_mgr_: mgr = _ci_mgr_ else: for thread in threading.enumerate(): if thread.name == "CodeIntel Manager": thread.finalize( ) # this finalizes the index, citadel and the manager and waits them to end (join) mgr = Manager( extra_module_dirs=_ci_extra_module_dirs_, db_base_dir=_ci_db_base_dir_, db_catalog_dirs=_ci_db_catalog_dirs_, db_import_everything_langs=_ci_db_import_everything_langs, db_event_reporter=lambda m: logger(view, 'event', m), ) mgr.upgrade() mgr.initialize() # Connect the logging file to the handler condeintel_log_file = os.path.join(mgr.db.base_dir, 'codeintel.log') codeintel_log.handlers = [ logging.StreamHandler(open(condeintel_log_file, 'w', 1)) ] msg = "Starting logging SublimeCodeIntel rev %s (%s) on %s" % ( get_git_revision()[:12], os.stat(__file__)[stat.ST_MTIME], datetime.datetime.now().ctime()) codeintel_log.info("%s\n%s" % (msg, "=" * len(msg))) _ci_mgr_ = mgr # Load configuration files: for catalog in mgr.db.get_catalogs_zone().avail_catalogs(): if catalog['lang'] == lang: catalogs.append(catalog['name']) config = { "codeintel_selected_catalogs": catalogs, "codeintel_max_recursive_dir_depth": 10, "codeintel_scan_files_in_project": True, } _config = {} try: tryReadDict(config_default_file, _config) except Exception, e: msg = "Malformed configuration file '%s': %s" % ( config_default_file, e) log.error(msg) codeintel_log.error(msg) try: tryReadDict(config_file, _config) except Exception, e: msg = "Malformed configuration file '%s': %s" % ( config_default_file, e) log.error(msg) codeintel_log.error(msg) config.update(_config.get(lang, {})) for conf in [ 'pythonExtraPaths', 'rubyExtraPaths', 'perlExtraPaths', 'javascriptExtraPaths', 'phpExtraPaths' ]: v = [ p.strip() for p in config.get(conf, []) + folders if p.strip() ] config[conf] = os.pathsep.join( set(p if p.startswith('/') else os.path.expanduser(p) if p. startswith('~') else os.path. abspath(os.path.join(project_base_dir, p) ) if project_base_dir else p for p in v if p.strip())) env = SimplePrefsEnvironment(**config) env._mtime = mtime or max(tryGetMTime(config_file), tryGetMTime(config_default_file)) env._folders = folders env._config_default_file = config_default_file env._project_dir = project_dir env._project_base_dir = project_base_dir env._config_file = config_file env.__class__.get_proj_base_dir = lambda self: project_base_dir _ci_envs_[path] = env
def do_scan(self, subcmd, opts, *path_patterns): """Scan and print the CIX for the given path(s). ${cmd_usage} ${cmd_option_list} """ mgr = Manager() mgr.upgrade() mgr.initialize() try: if opts.time_it: start = time.time() quiet = opts.quiet if opts.time_it or opts.time_details: opts.force = True scan_count = 0 lang_warnings = set() tree = None for path in _paths_from_path_patterns(path_patterns, recursive=opts.recursive, includes=opts.includes): if opts.time_it: sys.stderr.write(path + "\n") if opts.time_details: start1 = time.time() try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: log.info("skip `%s': couldn't determine language", path) continue try: buf = mgr.buf_from_path(path, lang=lang) except OSError as ex: # Couldn't access the file. if not opts.recursive: raise # Ignore files we don't really care about. log.warn("%r - %r", ex, path) continue if not isinstance(buf, CitadelBuffer): if opts.recursive: # Ignore files that scanning isn't provided for. continue raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) scan_count += 1 if scan_count % 10 == 0: log.info("%d scanning %r", scan_count, path) try: if opts.force: buf.scan() if tree is None: tree = ET.Element("codeintel", version="2.0") file_elem = ET.SubElement(tree, "file", lang=buf.lang, mtime=str(int(time.time())), path=os.path.basename(path)) for lang, blob in sorted(buf.blob_from_lang.items()): blob = buf.blob_from_lang[lang] file_elem.append(blob) except KeyError as ex: # Unknown cile language. if not opts.recursive: raise message = str(ex) if message not in lang_warnings: lang_warnings.add(message) log.warn("Skipping unhandled language %s", message) if opts.time_details: delta = time.time() - start1 sys.stderr.write("%.3f %s\n" % (delta, path)) sys.stderr.flush() if tree is not None: if opts.stripfuncvars: # For stdlibs, we don't care about variables inside of # functions and they take up a lot of space. for function in tree.getiterator('scope'): if function.get('ilk') == 'function': function[:] = [ child for child in function if child.tag != 'variable' ] if opts.pretty_print: tree = pretty_tree_from_tree(tree) if not quiet: sys.stdout.write( '<?xml version="1.0" encoding="UTF-8"?>\n') ET.dump(tree) if opts.time_it: end = time.time() sys.stderr.write("scan took %.3fs\n" % (end - start)) finally: mgr.finalize()
def do_outline(self, subcmd, opts, path): """Scan and outline the structure of the given path. ${cmd_usage} ${cmd_option_list} """ extra_lang_module_dirs = [] if koextlib.is_ext_dir() and exists("pylib"): sys.path.append(abspath("pylib")) extra_lang_module_dirs = [sys.path[-1]] mgr = Manager(extra_lang_module_dirs=extra_lang_module_dirs) mgr.upgrade() mgr.initialize() try: if '#' in path: path, anchor = path.rsplit('#', 1) else: anchor = None tree = None try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: log.info( "skip `%s': couldn't determine language " "(use --lang option)", path) return if path.endswith(".cix"): tree = tree_from_cix(open(path, 'r').read()) #buf = mgr.buf_from_content("", tree[0].get("lang"), path=path) else: buf = mgr.buf_from_path(path, lang=opts.lang) if not isinstance(buf, CitadelBuffer): raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) tree = buf.tree if anchor is not None: # Lookup the anchor in the codeintel CIX tree. lpath = re.split(r'\.|::', anchor) def blobs_from_tree(tree): for file_elem in tree: for blob in file_elem: yield blob for elem in blobs_from_tree(tree): # Generally have 3 types of codeintel trees: # 1. single-lang file: one <file>, one <blob> # 2. multi-lang file: one <file>, one or two <blob>'s # 3. CIX stdlib/catalog file: possibly multiple # <file>'s, likely multiple <blob>'s # Allow the first token to be the blob name or lang. # (This can sometimes be weird, but seems the most # convenient solution.) if lpath[0] in (elem.get("name"), elem.get("lang")): remaining_lpath = lpath[1:] else: remaining_lpath = lpath for name in remaining_lpath: try: elem = elem.names[name] except KeyError: elem = None break # try next lang blob if elem is not None: break # found one else: log.error( "could not find `%s' definition (or blob) in `%s'", anchor, path) return 1 else: elem = tree try: _outline_ci_elem(mgr, elem, quiet=opts.quiet) except IOError, ex: if ex.errno == 0: # Ignore this error from aborting 'less' of this # command: # IOError: (0, 'Error') pass else: raise finally: mgr.finalize()
def do_outline(self, subcmd, opts, path): """Scan and outline the structure of the given path. ${cmd_usage} ${cmd_option_list} """ extra_lang_module_dirs = [] if koextlib.is_ext_dir() and exists("pylib"): sys.path.append(abspath("pylib")) extra_lang_module_dirs = [sys.path[-1]] mgr = Manager(extra_lang_module_dirs=extra_lang_module_dirs) mgr.upgrade() mgr.initialize() try: if "#" in path: path, anchor = path.rsplit("#", 1) else: anchor = None tree = None try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: log.info("skip `%s': couldn't determine language " "(use --lang option)", path) return if path.endswith(".cix"): tree = tree_from_cix(open(path, "r").read()) # buf = mgr.buf_from_content("", tree[0].get("lang"), path=path) else: buf = mgr.buf_from_path(path, lang=opts.lang) if not isinstance(buf, CitadelBuffer): raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) tree = buf.tree if anchor is not None: # Lookup the anchor in the codeintel CIX tree. lpath = re.split(r"\.|::", anchor) def blobs_from_tree(tree): for file_elem in tree: for blob in file_elem: yield blob for elem in blobs_from_tree(tree): # Generally have 3 types of codeintel trees: # 1. single-lang file: one <file>, one <blob> # 2. multi-lang file: one <file>, one or two <blob>'s # 3. CIX stdlib/catalog file: possibly multiple # <file>'s, likely multiple <blob>'s # Allow the first token to be the blob name or lang. # (This can sometimes be weird, but seems the most # convenient solution.) if lpath[0] in (elem.get("name"), elem.get("lang")): remaining_lpath = lpath[1:] else: remaining_lpath = lpath for name in remaining_lpath: try: elem = elem.names[name] except KeyError: elem = None break # try next lang blob if elem is not None: break # found one else: log.error("could not find `%s' definition (or blob) in `%s'", anchor, path) return 1 else: elem = tree try: _outline_ci_elem(mgr, elem, quiet=opts.quiet) except IOError, ex: if ex.errno == 0: # Ignore this error from aborting 'less' of this # command: # IOError: (0, 'Error') pass else: raise finally: mgr.finalize()
def do_scan(self, subcmd, opts, *path_patterns): """Scan and print the CIX for the given path(s). ${cmd_usage} ${cmd_option_list} """ mgr = Manager() mgr.upgrade() mgr.initialize() try: if opts.time_it: start = time.time() quiet = opts.quiet if opts.time_it or opts.time_details: opts.force = True tree = None for path in _paths_from_path_patterns(path_patterns, recursive=opts.recursive, includes=opts.includes): if opts.time_it: sys.stderr.write(path+"\n") if opts.time_details: start1 = time.time() try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: log.info("skip `%s': couldn't determine language", path) continue buf = mgr.buf_from_path(path, lang=lang) if not isinstance(buf, CitadelBuffer): raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) if opts.force: buf.scan() if tree is None: tree = ET.Element("codeintel", version="2.0") file_elem = ET.SubElement(tree, "file", lang=buf.lang, mtime=str(int(time.time())), path=os.path.basename(path)) for lang, blob in sorted(buf.blob_from_lang.items()): blob = buf.blob_from_lang[lang] file_elem.append(blob) if opts.time_details: delta = time.time() - start1 sys.stderr.write("%.3f %s\n" % (delta, path)) sys.stderr.flush() if tree is not None: if opts.stripfuncvars: # For stdlibs, we don't care about variables inside of # functions and they take up a lot of space. for function in tree.getiterator('scope'): if function.get('ilk') == 'function': function[:] = [child for child in function if child.tag != 'variable'] if opts.pretty_print: tree = pretty_tree_from_tree(tree) if not quiet: sys.stdout.write('<?xml version="1.0" encoding="UTF-8"?>\n') ET.dump(tree) if opts.time_it: end = time.time() sys.stderr.write("scan took %.3fs\n" % (end-start)) finally: mgr.finalize()
def codeintel_scan(view, path, content, lang, callback=None, pos=None, forms=None): global despair for thread in threading.enumerate(): if thread.isAlive() and thread.name == "scanning thread": logger( view, "info", "Updating indexes... The first time this can take a while. Do not despair!", timeout=20000, delay=despair, ) despair = 0 return try: lang = lang or guess_lang_from_path(path) except CodeIntelError: logger(view, "warning", "skip `%s': couldn't determine language" % path) return if lang.lower() in [l.lower() for l in view.settings().get("codeintel_disabled_languages", [])]: return is_scratch = view.is_scratch() is_dirty = view.is_dirty() folders = getattr( view.window(), "folders", lambda: [] )() # FIXME: it's like this for backward compatibility (<= 2060) def _codeintel_scan(): global _ci_mgr_, despair, despaired env = None mtime = None catalogs = [] now = time.time() try: env = _ci_envs_[path] if env._folders != folders: raise KeyError mgr = _ci_mgr_ if now > env._time: mtime = max(tryGetMTime(env._config_file), tryGetMTime(env._config_default_file)) if env._mtime < mtime: raise KeyError except KeyError: if env is not None: config_default_file = env._config_default_file project_dir = env._project_dir project_base_dir = env._project_base_dir config_file = env._config_file else: config_default_file = os.path.join(CODEINTEL_HOME_DIR, "config") if not (config_default_file and os.path.exists(config_default_file)): config_default_file = None project_dir = None project_base_dir = None if path: # Try to find a suitable project directory (or best guess): for folder in [".codeintel", ".git", ".hg", "trunk"]: project_dir = find_folder(path, folder) if project_dir and ( folder != ".codeintel" or not os.path.exists(os.path.join(project_dir, "db")) ): if folder.startswith("."): project_base_dir = os.path.abspath(os.path.join(project_dir, "..")) else: project_base_dir = project_dir break if not (project_dir and os.path.exists(project_dir)): project_dir = None config_file = project_dir and folder == ".codeintel" and os.path.join(project_dir, "config") if not (config_file and os.path.exists(config_file)): config_file = None if _ci_mgr_: mgr = _ci_mgr_ else: for thread in threading.enumerate(): if thread.name == "CodeIntel Manager": thread.finalize() # this finalizes the index, citadel and the manager and waits them to end (join) mgr = Manager( extra_module_dirs=_ci_extra_module_dirs_, db_base_dir=_ci_db_base_dir_, db_catalog_dirs=_ci_db_catalog_dirs_, db_import_everything_langs=_ci_db_import_everything_langs, db_event_reporter=lambda m: logger(view, "event", m), ) mgr.upgrade() mgr.initialize() # Connect the logging file to the handler condeintel_log_file = os.path.join(mgr.db.base_dir, "codeintel.log") codeintel_log.handlers = [logging.StreamHandler(open(condeintel_log_file, "w", 1))] msg = "Starting logging SublimeCodeIntel rev %s (%s) on %s" % ( get_git_revision()[:12], os.stat(__file__)[stat.ST_MTIME], datetime.datetime.now().ctime(), ) codeintel_log.info("%s\n%s" % (msg, "=" * len(msg))) _ci_mgr_ = mgr # Load configuration files: for catalog in mgr.db.get_catalogs_zone().avail_catalogs(): if catalog["lang"] == lang: catalogs.append(catalog["name"]) config = { "codeintel_selected_catalogs": catalogs, "codeintel_max_recursive_dir_depth": 10, "codeintel_scan_files_in_project": True, } _config = {} try: tryReadDict(config_default_file, _config) except Exception, e: msg = "Malformed configuration file '%s': %s" % (config_default_file, e) log.error(msg) codeintel_log.error(msg) try: tryReadDict(config_file, _config) except Exception, e: msg = "Malformed configuration file '%s': %s" % (config_default_file, e) log.error(msg) codeintel_log.error(msg) config.update(_config.get(lang, {})) for conf in [ "pythonExtraPaths", "rubyExtraPaths", "perlExtraPaths", "javascriptExtraPaths", "phpExtraPaths", ]: v = [p.strip() for p in config.get(conf, []) + folders if p.strip()] config[conf] = os.pathsep.join( set( p if p.startswith("/") else os.path.expanduser(p) if p.startswith("~") else os.path.abspath(os.path.join(project_base_dir, p)) if project_base_dir else p for p in v if p.strip() ) ) env = SimplePrefsEnvironment(**config) env._mtime = mtime or max(tryGetMTime(config_file), tryGetMTime(config_default_file)) env._folders = folders env._config_default_file = config_default_file env._project_dir = project_dir env._project_base_dir = project_base_dir env._config_file = config_file env.__class__.get_proj_base_dir = lambda self: project_base_dir _ci_envs_[path] = env