def worker(queue, lock, cix_dir, root="/"):
"""
worker procedure
"""
fix_module_path()
import dxr.mime
from codeintel2.citadel import CitadelBuffer
from codeintel2.common import CodeIntelError
from codeintel2.manager import Manager
from codeintel2.util import guess_lang_from_path
logging.getLogger("codeintel").setLevel(logging.ERROR)
log.info("starting indexing using %s", multiprocessing.current_process().name)
mgr = Manager(db_base_dir=cix_dir)
#mgr.upgrade()
mgr.initialize()
while True:
file_path = queue.get()
if file_path is None:
# marker for end of list
queue.put(None) # put it back so others can quit too
break
rel_path = os.path.relpath(file_path, root)
try:
lang = guess_lang_from_path(file_path)
except CodeIntelError:
log.info("%s: Cannot determine language, skipping", rel_path)
continue
# the file
with open(file_path, "r") as source_file:
data = source_file.read()
# Discard non-text files
if not dxr.mime.is_text(file_path, data):
continue
try:
buf = mgr.buf_from_path(file_path, lang=lang)
except CodeIntelError as ex:
if ex.message.startswith("File too big."):
log.info("%s: %s", file_path, ex.message)
continue # Nothing we can do about that, and the user can't
# fix this ever.
raise
if not isinstance(buf, CitadelBuffer):
log.info("%s: language %s does not have CIX, skipping",
rel_path, lang)
continue
log.info("%s: Using language %s", rel_path, lang)
buf.scan()
mgr.finalize()
def load(tree, conn):
fix_module_path()
opts.db_path = os.path.join(tree.temp_folder, "plugins", "komodo_codeintel")
opts.root = tree.source_folder
opts.wwwroot = tree.config.wwwroot
opts.name = tree.name
log.info("reading from db %s", opts.db_path)
from codeintel2.manager import Manager
opts.mgr = Manager(db_base_dir=opts.db_path)
opts.mgr.initialize()