def worker(queue, lock, cix_dir, root="/"): """ worker procedure """ fix_module_path() import dxr.mime from codeintel2.citadel import CitadelBuffer from codeintel2.common import CodeIntelError from codeintel2.manager import Manager from codeintel2.util import guess_lang_from_path logging.getLogger("codeintel").setLevel(logging.ERROR) log.info("starting indexing using %s", multiprocessing.current_process().name) mgr = Manager(db_base_dir=cix_dir) #mgr.upgrade() mgr.initialize() while True: file_path = queue.get() if file_path is None: # marker for end of list queue.put(None) # put it back so others can quit too break rel_path = os.path.relpath(file_path, root) try: lang = guess_lang_from_path(file_path) except CodeIntelError: log.info("%s: Cannot determine language, skipping", rel_path) continue # the file with open(file_path, "r") as source_file: data = source_file.read() # Discard non-text files if not dxr.mime.is_text(file_path, data): continue try: buf = mgr.buf_from_path(file_path, lang=lang) except CodeIntelError as ex: if ex.message.startswith("File too big."): log.info("%s: %s", file_path, ex.message) continue # Nothing we can do about that, and the user can't # fix this ever. raise if not isinstance(buf, CitadelBuffer): log.info("%s: language %s does not have CIX, skipping", rel_path, lang) continue log.info("%s: Using language %s", rel_path, lang) buf.scan() mgr.finalize()
def load(tree, conn): fix_module_path() opts.db_path = os.path.join(tree.temp_folder, "plugins", "komodo_codeintel") opts.root = tree.source_folder opts.wwwroot = tree.config.wwwroot opts.name = tree.name log.info("reading from db %s", opts.db_path) from codeintel2.manager import Manager opts.mgr = Manager(db_base_dir=opts.db_path) opts.mgr.initialize()