class Driver(threading.Thread):
"""
Out-of-process codeintel2 driver
This class implements the main loop of the codeintel worker process
This should be a singleton object
"""
# static
_instance = None
_command_handler_map = collections.defaultdict(list)
"""Registered command handlers; the key is the command name (a str), and the
value is a list of command handler instances or (for lazy handlers) a
single-element tuple containing the callable to get the real handler."""
_default_handler = None
"""The default (core) command handler; instance of a CoreHandler"""
_builtin_commands = {}
"""Built-in commands that cannot be overridden"""
def __init__(self, db_base_dir=None, fd_in=sys.stdin, fd_out=sys.stdout):
threading.Thread.__init__(self, name="Codeintel OOP Driver")
assert Driver._instance is None, "Driver should be a singleton"
Driver._instance = self
logging.root.addHandler(LoggingHandler(self))
self.daemon = True
self.fd_in = fd_in
self.fd_out = fd_out
self.abort = None
self.quit = False
self.buffers = {} # path to Buffer objects
self.next_buffer = 0
self.active_request = None
self.send_queue = Queue.Queue()
self.send_thread = threading.Thread(name="Codeintel OOP Driver Send Thread",
target=self._send_proc)
self.send_thread.daemon = True
self.send_thread.start()
self.queue = collections.deque()
self.queue_cv = threading.Condition()
self.env = Environment(name="global",
send_fn=functools.partial(self.send, request=None))
# Fill out the non-overridable build-in commands
self._builtin_commands = {}
for attr in dir(self):
# Note that we check startswith first to avoid getters etc.
if attr.startswith("do_") and callable(getattr(self, attr)):
command = attr[len("do_"):].replace("_", "-")
self._builtin_commands[command] = getattr(self, attr)
from codeintel2.manager import Manager
log.debug("using db base dir %s", db_base_dir)
self.mgr = Manager(db_base_dir=db_base_dir,
db_catalog_dirs=[],
db_event_reporter=self._DBEventReporter(self),
env=self.env,
on_scan_complete=self._on_scan_complete)
self.mgr.initialize()
def _on_scan_complete(self, scan_request):
if scan_request.status in ("changed", "skipped"):
# Send unsolicited response about the completed scan
buf = scan_request.buf
self.send(request=None,
path=buf.path,
language=buf.lang,
command="scan-complete")
class _DBEventReporter(object):
def __init__(self, driver):
self.driver = driver
self.log = log.getChild("DBEventReporter")
self.debug = self.log.debug
# directories being scanned (completed or not)
# key is unicode path, value is number of times it's active
self._dirs = collections.defaultdict(int)
# directories which are complete (unicode path)
self._completed_dirs = set()
def send(self, **kwargs):
self.driver.send(request=None, command="report-message", **kwargs)
def __call__(self, message):
"""Old-style status messages before long-running jobs
@param msg {str or None} The message to display
"""
if len(self._dirs):
return # ignore old-style db events if we're doing a scan
self.debug("db event: %s", message)
self.send(message=message)
def onScanStarted(self, description, dirs=set()):
"""Called when a directory scan is about to start
@param description {unicode} A string suitable for showing the user
about the upcoming operation
@param dirs {set of unicode} The directories about to be scanned
"""
self.debug("scan started: %s (%s dirs)", description, len(dirs))
assert dirs, "onScanStarted expects non-empty directories"
if not dirs: # empty set - we shouldn't have gotten here, but be nice
return
for dir_path in dirs:
self._dirs[dir_path] += 1
self.send(type="scan-progress", message=description,
completed=len(self._completed_dirs),
total=len(self._dirs))
def onScanDirectory(self, description, dir_path, current=None, total=None):
"""Called when a directory is being scanned (out of possibly many)
@param description {unicode} A string suitable for showing the user
regarding the progress
@param dir {unicode} The directory currently being scanned
@param current {int} The current progress
@param total {int} The total number of directories to scan in this
request
"""
self.debug("scan directory: %s (%s %s/%s)",
description, dir_path, current, total)
assert dir_path, "onScanDirectory got no directory"
if dir_path:
self._completed_dirs.add(dir_path)
self.send(type="scan-progress", message=description,
completed=len(self._completed_dirs),
total=len(self._dirs))
def onScanComplete(self, dirs=set(), scanned=set()):
"""Called when a scan operation is complete
@param dirs {set of unicode} The directories that were intially
requested to be scanned (as pass in onScanStarted)
@param scanned {set of unicode} Directories which were successfully
scanned. This may be a subset of dirs if the scan was
aborted.
"""
self.debug("scan complete: scanned %r/%r dirs",
len(scanned), len(dirs))
for dir_path in dirs:
self._dirs[dir_path] -= 1
if not self._dirs[dir_path]:
del self._dirs[dir_path]
self._completed_dirs.discard(dir_path)
self.send(type="scan-progress", completed=len(self._completed_dirs),
total=len(self._dirs))
REQUEST_DEFAULT = object()
def send(self, request=REQUEST_DEFAULT, **kwargs):
"""
Send a response
"""
data = dict(kwargs)
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if request:
data["req_id"] = request.id
if "success" not in data:
data["success"] = True
elif data["success"] is None:
del data["success"]
buf = json.dumps(data, separators=(',',':'))
buf_len = str(len(buf))
log.debug("sending: %s:[%s]", buf_len, buf)
self.send_queue.put(buf)
def _send_proc(self):
while True:
buf = self.send_queue.get()
try:
buf_len = str(len(buf))
self.fd_out.write(buf_len)
self.fd_out.write(buf)
finally:
self.send_queue.task_done()
def fail(self, request=REQUEST_DEFAULT, **kwargs):
kwargs = kwargs.copy()
if not "command" in kwargs and request:
try:
kwargs["command"] = request["command"]
except KeyError:
pass
return self.send(request=request, success=False, **kwargs)
def exception(self, request=REQUEST_DEFAULT, **kwargs):
return self.fail(request=request, stack=traceback.format_exc(), **kwargs)
@staticmethod
def normpath(path):
"""Routine to normalize the path used for codeintel buffers
This is annoying because it needs to handle unsaved files, as well as
urls.
@note See also koCodeIntel.py::KoCodeIntelBuffer.normpath
"""
if path.startswith("<Unsaved>"):
return path # Don't munge unsaved file paths
scheme = path.split(":", 1)[0]
if len(scheme) == len(path):
# didn't find a scheme at all; assume this is a local path
return os.path.normcase(path)
if len(scheme) == 1:
# single-character scheme; assume this is actually a drive letter
# (for a Windows-style path)
return os.path.normcase(path)
scheme_chars = string.ascii_letters + string.digits + "-"
try:
scheme = scheme.encode("ascii")
except UnicodeEncodeError:
# scheme has a non-ascii character; assume local path
return os.path.normcase(path)
if scheme.translate(None, scheme_chars):
# has a non-scheme character: this is not a valid scheme
# assume this is a local file path
return os.path.normcase(path)
if scheme != "file":
return path # non-file scheme
path = path[len(scheme) + 1:]
return os.path.normcase(path)
def get_buffer(self, request=REQUEST_DEFAULT, path=None):
if request is Driver.REQUEST_DEFAULT:
request = self.active_request
if path is None:
if not "path" in request:
raise RequestFailure(message="No path given to locate buffer")
path = request.path
path = self.normpath(path)
try:
buf = self.buffers[path]
except KeyError:
buf = None
else:
if "language" in request and buf.lang != request.language:
buf = None # language changed, re-scan
if not buf:
# Need to construct a new buffer
lang = request.get("language")
env = Environment(request=request, name=os.path.basename(path))
lexer = self.mgr.silvercity_lexer_from_lang.get(lang)
accessor = codeintel2.accessor.OOPAccessor(lexer, self)
buf = self.mgr.buf_from_accessor(accessor, lang, path=path, env=env)
accessor.buf = buf
if "checksum" in request:
buf.accessor.checksum = request.get("checksum")
if request.get("text") is not None:
# text came with the buffer; most likely it's a unit test...
buf.accessor.reset_content(request.get("text"))
buf.encoding = "utf-8"
try:
env = request["env"]
except KeyError:
pass # no environment, use current
else:
buf._env.update(env)
#log.debug("Got buffer %r: [%s]", buf, buf.accessor.content)
log.debug("Got buffer %r", buf)
self.buffers[path] = buf
return buf
def sync_get_buffer_contents(self, path, checksum):
"""Synchronously attempt to get the buffer contents from the parent
process. This will block the code intel process until the response has
been obtained.
@param path {str or unicode} The path of the buffer
@param checksum {str} The checksum of the current contents (md5)
@return {dict} The response from the parent process
"""
self.send(request=None,
command="get-buffer-contents",
checksum=checksum,
path=path)
# Wait synchronously for the response to come back
buf_req = None
with self.queue_cv:
while buf_req is None:
for i, buf_req in enumerate(self.queue):
if (buf_req.get("command") == "get-buffer-contents"
and buf_req.get("path") == path):
del self.queue[i]
break
else:
buf_req = None
log.debug("Skipping wake, waiting for more...")
self.queue_cv.wait()
if not buf_req.get("success", False):
log.debug("Failed to get text for %s", path)
raise RequestFailure(msg="Failed to get buffer %s" % (path,))
log.debug("Got text for %s", path)
return buf_req
def do_abort(self, request):
try:
req_id = request["id"]
with self.queue_cv:
for item in self.queue:
if queue.get("req_id") == req_id:
self.queue.remove(queue)
self.send(request=request)
break
else:
self.abort = req_id
if self.active_request and self.active_request.id == req_id:
# need to wait a bit...
self.send(request=request)
else:
self.fail(request=request,
message="Request %s not found" % (req_id,))
except RequestFailure as e:
self.fail(request=request, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
def do_add_dirs(self, request):
catalog_dirs = request.get("catalog-dirs", None)
if catalog_dirs is not None:
self.mgr.db.catalog_dirs.extend(catalog_dirs)
catalog_dirs = self.mgr.db.catalog_dirs
lexer_dirs = request.get("lexer-dirs", [])
codeintel2.udl.UDLLexer.add_extra_lexer_dirs(lexer_dirs)
module_dirs = request.get("module-dirs", [])
if module_dirs:
self.mgr._register_modules(module_dirs)
if catalog_dirs is not None:
self.mgr.db.get_catalogs_zone().catalog_dirs = catalog_dirs
self.send(request=request)
def do_load_extension(self, request):
"""Load an extension that, for example, might provide additional
command handlers"""
name = request.get("module-name", None)
if not name:
raise RequestFailure(msg="load-extension requires a module-name")
path = request.get("module-path", None)
names = name.split(".")
if len(names) > 1:
# It's inside a package - go look for the package(s).
for package_name in names[:-1]:
iinfo = imp.find_module(package_name, [path] if path else None)
if not iinfo:
raise RequestFailure(msg="load-extension could not find "
"package %r for given name %r"
% (package_name, name))
path = iinfo[1]
name = names[-1]
iinfo = imp.find_module(name, [path] if path else None)
try:
module = imp.load_module(name, *iinfo)
finally:
if iinfo and iinfo[0]:
iinfo[0].close()
callback = getattr(module, "registerExtension", None)
if not callback:
raise RequestFailure(msg="load-extension module %s should "
"have a 'registerExtension' method "
"taking no arguments" % (name,))
callback()
self.send() # success, finally
def do_quit(self, request):
self.quit = True
self.send(command="quit")
def report_error(self, message):
self.send(request=None,
command="report-error",
message=unicode(message))
def start(self):
"""Start reading from the socket and dump requests into the queue"""
log.info("Running codeintel driver...")
buf = ""
self.send(success=None)
self.daemon = True
threading.Thread.start(self)
while not self.quit:
try:
ch = self.fd_in.read(1)
except IOError:
log.debug("Failed to read frame length, assuming connection died")
self.quit = True
break
if len(ch) == 0:
log.debug("Input was closed")
self.quit = True
break
if ch == "{":
size = int(buf, 10)
try:
buf = ch + self.fd_in.read(size - 1) # exclude already-read {
except IOError:
log.debug("Failed to read frame data, assuming connection died")
self.quit = True
break
try:
data = json.loads(buf)
request = Request(data)
except Exception as e:
log.exception(e)
self.exception(message=e.message, request=None)
continue
finally:
buf = ""
if request.get("command") == "abort":
self.do_abort(request=request)
else:
log.debug("queuing request %r", request)
with self.queue_cv:
self.queue.appendleft(request)
self.queue_cv.notify()
elif ch in "0123456789":
buf += ch
else:
raise ValueError("Invalid request data: " + ch.encode("hex"))
def run(self):
"""Evaluate and send results back"""
log.info("Running codeintel eval thread...")
buf = ""
log.debug("default supported commands: %s",
", ".join(self._default_handler.supportedCommands))
while True:
with self.queue_cv:
try:
request = self.queue.pop()
except IndexError:
self.queue_cv.wait()
continue
log.debug("doing request %r", request)
try:
self.active_request = request
command = request.command
# First, check abort and quit; don't allow those to be overridden
try:
builtin = self._builtin_commands[command]
except KeyError:
pass
else:
builtin(request)
continue
handlers = self._command_handler_map.get(command, [])[:]
if command in self._default_handler.supportedCommands:
# The default handler can deal with this, put it at the end
handlers.append(self._default_handler)
for handler in handlers:
if isinstance(handler, tuple):
try:
real_handler = handler[0]()
except Exception as ex:
log.exception("Failed to get lazy handler for %s", command)
real_handler = None
if real_handler is None:
# Handler failed to instantiate, drop it
try:
self._command_handler_map[command].remove(handler)
except ValueError:
pass # ... shouldn't happen, but tolerate it
continue
for handlers in self._command_handler_map.values():
try:
handlers[handlers.index(handler)] = real_handler
except ValueError:
pass # handler not in this list
handler = real_handler
if handler.canHandleRequest(request):
handler.handleRequest(request, self)
break
else:
self.fail(request=request,
msg="Don't know how to handle command %s" % (command,))
except RequestFailure as e:
self.fail(request=request, **e.kwargs)
except Exception as e:
log.exception(e.message)
self.exception(request=request, message=e.message)
finally:
self.active_request = None
@classmethod
def registerCommandHandler(cls, handlerInstance):
"""Register a command handler"""
for command in handlerInstance.supportedCommands:
cls._command_handler_map[command].append(handlerInstance)
@classmethod
def registerLazyCommandHandler(cls, supported_commands, constructor):
"""Register a lazy command handler
@param supported_commands {iterable} The commands to handle; each
element should be a str of the command name.
@param constructor {callable} Function to be called to get the real
command handler; it should take no arguments and return a command
handler instance. It may return None if the command is not
available; it will not be asked again.
"""
for command in supported_commands:
cls._command_handler_map[command].append((constructor,))
@classmethod
def getInstance(cls):
"""Get the singleton instance of the driver"""
return Driver._instance
