def __init__(self):
"""Constructor; also constructs ZMQ stream."""
super(QObject, self).__init__()
self.context = zmq.Context()
self.socket = self.context.socket(zmq.PAIR)
self.port = self.socket.bind_to_random_port('tcp://*')
fid = self.socket.getsockopt(zmq.FD)
self.notifier = QSocketNotifier(fid, QSocketNotifier.Read, self)
self.notifier.activated.connect(self.received_message)
def start(self):
self.zmq_out_socket = self.context.socket(zmq.PAIR)
self.zmq_out_port = self.zmq_out_socket.bind_to_random_port('tcp://*')
self.zmq_in_socket = self.context.socket(zmq.PAIR)
self.zmq_in_socket.set_hwm(0)
self.zmq_in_port = self.zmq_in_socket.bind_to_random_port('tcp://*')
self.transport_args += [
'--zmq-in-port', self.zmq_out_port, '--zmq-out-port',
self.zmq_in_port
]
self.lsp_server_log = subprocess.PIPE
if get_debug_level() > 0:
lsp_server_file = 'lsp_server_{0}.log'.format(self.language)
log_file = get_conf_path(osp.join('lsp_logs', lsp_server_file))
if not osp.exists(osp.dirname(log_file)):
os.makedirs(osp.dirname(log_file))
self.lsp_server_log = open(log_file, 'w')
if not self.external_server:
logger.info('Starting server: {0}'.format(' '.join(
self.server_args)))
creation_flags = 0
if WINDOWS:
creation_flags = (subprocess.CREATE_NEW_PROCESS_GROUP
| 0x08000000) # CREATE_NO_WINDOW
self.lsp_server = subprocess.Popen(self.server_args,
stdout=self.lsp_server_log,
stderr=subprocess.STDOUT,
creationflags=creation_flags)
# self.transport_args += self.server_args
self.stdout_log = subprocess.PIPE
self.stderr_log = subprocess.PIPE
if get_debug_level() > 0:
stderr_log_file = 'lsp_transport_{0}_err.log'.format(self.language)
log_file = get_conf_path(osp.join('lsp_logs', stderr_log_file))
if not osp.exists(osp.dirname(log_file)):
os.makedirs(osp.dirname(log_file))
self.stderr_log = open(log_file, 'w')
new_env = dict(os.environ)
python_path = os.pathsep.join(sys.path)[1:]
new_env['PYTHONPATH'] = python_path
self.transport_args = list(map(str, self.transport_args))
logger.info('Starting transport: {0}'.format(' '.join(
self.transport_args)))
self.transport_client = subprocess.Popen(self.transport_args,
stdout=self.stdout_log,
stderr=self.stderr_log,
env=new_env)
fid = self.zmq_in_socket.getsockopt(zmq.FD)
self.notifier = QSocketNotifier(fid, QSocketNotifier.Read, self)
# self.notifier.activated.connect(self.debug_print)
self.notifier.activated.connect(self.on_msg_received)
def _maybe_allow_interrupt(qapp):
"""
This manager allows to terminate a plot by sending a SIGINT. It is
necessary because the running Qt backend prevents Python interpreter to
run and process signals (i.e., to raise KeyboardInterrupt exception). To
solve this one needs to somehow wake up the interpreter and make it close
the plot window. We do this by using the signal.set_wakeup_fd() function
which organizes a write of the signal number into a socketpair connected
to the QSocketNotifier (since it is part of the Qt backend, it can react
to that write event). Afterwards, the Qt handler empties the socketpair
by a recv() command to re-arm it (we need this if a signal different from
SIGINT was caught by set_wakeup_fd() and we shall continue waiting). If
the SIGINT was caught indeed, after exiting the on_signal() function the
interpreter reacts to the SIGINT according to the handle() function which
had been set up by a signal.signal() call: it causes the qt_object to
exit by calling its quit() method. Finally, we call the old SIGINT
handler with the same arguments that were given to our custom handle()
handler.
We do this only if the old handler for SIGINT was not None, which means
that a non-python handler was installed, i.e. in Julia, and not SIG_IGN
which means we should ignore the interrupts.
code from https://github.com/matplotlib/matplotlib/pull/13306
"""
old_sigint_handler = signal.getsignal(signal.SIGINT)
handler_args = None
skip = False
if old_sigint_handler in (None, signal.SIG_IGN, signal.SIG_DFL):
skip = True
else:
wsock, rsock = socket.socketpair()
wsock.setblocking(False)
old_wakeup_fd = signal.set_wakeup_fd(wsock.fileno())
sn = QSocketNotifier(rsock.fileno(), QSocketNotifier.Type.Read)
# Clear the socket to re-arm the notifier.
sn.activated.connect(lambda *args: rsock.recv(1))
def handle(*args):
nonlocal handler_args
handler_args = args
qapp.exit()
signal.signal(signal.SIGINT, handle)
try:
yield
finally:
if not skip:
wsock.close()
rsock.close()
sn.setEnabled(False)
signal.set_wakeup_fd(old_wakeup_fd)
signal.signal(signal.SIGINT, old_sigint_handler)
if handler_args is not None:
old_sigint_handler(*handler_args)
def __init__(self, parent, reactor, watcher, socketType):
QObject.__init__(self, parent)
self.reactor = reactor
self.watcher = watcher
fd = watcher.fileno()
self.notifier = QSocketNotifier(fd, socketType, parent)
self.notifier.setEnabled(True)
if socketType == QSocketNotifier.Read:
self.fn = self.read
else:
self.fn = self.write
self.notifier.activated.connect(self.fn)
def run(self):
"""Handle the connection with the server.
"""
# Set up the zmq port.
self.socket = self.context.socket(zmq.PAIR)
self.port = self.socket.bind_to_random_port('tcp://*')
# Set up the process.
self.process = QProcess(self)
if self.cwd:
self.process.setWorkingDirectory(self.cwd)
p_args = ['-u', self.target, str(self.port)]
if self.extra_args is not None:
p_args += self.extra_args
# Set up environment variables.
processEnvironment = QProcessEnvironment()
env = self.process.systemEnvironment()
if (self.env and 'PYTHONPATH' not in self.env) or self.env is None:
python_path = osp.dirname(get_module_path('spyder'))
# Add the libs to the python path.
for lib in self.libs:
try:
path = osp.dirname(imp.find_module(lib)[1])
python_path = osp.pathsep.join([python_path, path])
except ImportError:
pass
if self.extra_path:
try:
python_path = osp.pathsep.join([python_path] +
self.extra_path)
except Exception as e:
debug_print("Error when adding extra_path to plugin env")
debug_print(e)
env.append("PYTHONPATH=%s" % python_path)
if self.env:
env.update(self.env)
for envItem in env:
envName, separator, envValue = envItem.partition('=')
processEnvironment.insert(envName, envValue)
self.process.setProcessEnvironment(processEnvironment)
# Start the process and wait for started.
self.process.start(self.executable, p_args)
self.process.finished.connect(self._on_finished)
running = self.process.waitForStarted()
if not running:
raise IOError('Could not start %s' % self)
# Set up the socket notifer.
fid = self.socket.getsockopt(zmq.FD)
self.notifier = QSocketNotifier(fid, QSocketNotifier.Read, self)
self.notifier.activated.connect(self._on_msg_received)
def start(self):
"""Start client."""
# NOTE: DO NOT change the order in which these methods are called.
self.create_transport_sockets()
self.start_server()
self.start_transport()
# Create notifier
fid = self.zmq_in_socket.getsockopt(zmq.FD)
self.notifier = QSocketNotifier(fid, QSocketNotifier.Read, self)
self.notifier.activated.connect(self.on_msg_received)
# This is necessary for tests to pass locally!
logger.debug('LSP {} client started!'.format(self.language))
def __init__(self):
"""Constructor; also constructs ZMQ stream."""
super(QObject, self).__init__()
self.context = zmq.Context()
self.socket = self.context.socket(zmq.PAIR)
self.port = self.socket.bind_to_random_port('tcp://*')
fid = self.socket.getsockopt(zmq.FD)
self.notifier = QSocketNotifier(fid, QSocketNotifier.Read, self)
self.notifier.activated.connect(self.received_message)
def start(self):
self.zmq_out_socket = self.context.socket(zmq.PAIR)
self.zmq_out_port = self.zmq_out_socket.bind_to_random_port('tcp://*')
self.zmq_in_socket = self.context.socket(zmq.PAIR)
self.zmq_in_socket.set_hwm(0)
self.zmq_in_port = self.zmq_in_socket.bind_to_random_port('tcp://*')
self.transport_args += ['--zmq-in-port', self.zmq_out_port,
'--zmq-out-port', self.zmq_in_port]
self.lsp_server_log = subprocess.PIPE
if get_debug_level() > 0:
lsp_server_file = 'lsp_server_logfile.log'
log_file = get_conf_path(osp.join('lsp_logs', lsp_server_file))
if not osp.exists(osp.dirname(log_file)):
os.makedirs(osp.dirname(log_file))
self.lsp_server_log = open(log_file, 'w')
if not self.external_server:
logger.info('Starting server: {0}'.format(
' '.join(self.server_args)))
creation_flags = 0
if WINDOWS:
creation_flags = subprocess.CREATE_NEW_PROCESS_GROUP
self.lsp_server = subprocess.Popen(
self.server_args,
stdout=self.lsp_server_log,
stderr=subprocess.STDOUT,
creationflags=creation_flags)
# self.transport_args += self.server_args
self.stdout_log = subprocess.PIPE
self.stderr_log = subprocess.PIPE
if get_debug_level() > 0:
stderr_log_file = 'lsp_client_{0}.log'.format(self.language)
log_file = get_conf_path(osp.join('lsp_logs', stderr_log_file))
if not osp.exists(osp.dirname(log_file)):
os.makedirs(osp.dirname(log_file))
self.stderr_log = open(log_file, 'w')
new_env = dict(os.environ)
python_path = os.pathsep.join(sys.path)[1:]
new_env['PYTHONPATH'] = python_path
self.transport_args = map(str, self.transport_args)
self.transport_client = subprocess.Popen(self.transport_args,
stdout=self.stdout_log,
stderr=self.stderr_log,
env=new_env)
fid = self.zmq_in_socket.getsockopt(zmq.FD)
self.notifier = QSocketNotifier(fid, QSocketNotifier.Read, self)
# self.notifier.activated.connect(self.debug_print)
self.notifier.activated.connect(self.on_msg_received)
def run(self):
"""Handle the connection with the server.
"""
# Set up the zmq port.
self.socket = self.context.socket(zmq.PAIR)
self.port = self.socket.bind_to_random_port('tcp://*')
# Set up the process.
self.process = QProcess(self)
if self.cwd:
self.process.setWorkingDirectory(self.cwd)
p_args = ['-u', self.target, str(self.port)]
if self.extra_args is not None:
p_args += self.extra_args
# Set up environment variables.
processEnvironment = QProcessEnvironment()
env = self.process.systemEnvironment()
if (self.env and 'PYTHONPATH' not in self.env) or DEV:
python_path = osp.dirname(get_module_path('spyderlib'))
# Add the libs to the python path.
for lib in self.libs:
try:
path = osp.dirname(imp.find_module(lib)[1])
python_path = osp.pathsep.join([python_path, path])
except ImportError:
pass
env.append("PYTHONPATH=%s" % python_path)
if self.env:
env.update(self.env)
for envItem in env:
envName, separator, envValue = envItem.partition('=')
processEnvironment.insert(envName, envValue)
self.process.setProcessEnvironment(processEnvironment)
# Start the process and wait for started.
self.process.start(self.executable, p_args)
self.process.finished.connect(self._on_finished)
running = self.process.waitForStarted()
if not running:
raise IOError('Could not start %s' % self)
# Set up the socket notifer.
fid = self.socket.getsockopt(zmq.FD)
self.notifier = QSocketNotifier(fid, QSocketNotifier.Read, self)
self.notifier.activated.connect(self._on_msg_received)
class ZmqStreamReader(QObject):
"""
Reader for receiving stream of Python objects via a ZMQ stream.
Attributes
----------
port : int
TCP port number used for the stream.
Signals
-------
sig_received(list)
Emitted when objects are received; argument is list of received
objects.
"""
sig_received = Signal(object)
def __init__(self):
"""Constructor; also constructs ZMQ stream."""
super(QObject, self).__init__()
self.context = zmq.Context()
self.socket = self.context.socket(zmq.PAIR)
self.port = self.socket.bind_to_random_port('tcp://*')
fid = self.socket.getsockopt(zmq.FD)
self.notifier = QSocketNotifier(fid, QSocketNotifier.Read, self)
self.notifier.activated.connect(self.received_message)
def received_message(self):
"""Called when a message is received."""
self.notifier.setEnabled(False)
messages = []
try:
while 1:
message = self.socket.recv_pyobj(flags=zmq.NOBLOCK)
messages.append(message)
except zmq.ZMQError:
pass
finally:
self.notifier.setEnabled(True)
if messages:
self.sig_received.emit(messages)
def close(self):
"""Read any remaining messages and close stream."""
self.received_message() # Flush remaining messages
self.notifier.setEnabled(False)
self.socket.close()
self.context.destroy()
class ZmqStreamReader(QObject):
"""
Reader for receiving stream of Python objects via a ZMQ stream.
Attributes
----------
port : int
TCP port number used for the stream.
Signals
-------
sig_received(list)
Emitted when objects are received; argument is list of received
objects.
"""
sig_received = Signal(object)
def __init__(self):
"""Constructor; also constructs ZMQ stream."""
super(QObject, self).__init__()
self.context = zmq.Context()
self.socket = self.context.socket(zmq.PAIR)
self.port = self.socket.bind_to_random_port('tcp://*')
fid = self.socket.getsockopt(zmq.FD)
self.notifier = QSocketNotifier(fid, QSocketNotifier.Read, self)
self.notifier.activated.connect(self.received_message)
def received_message(self):
"""Called when a message is received."""
self.notifier.setEnabled(False)
messages = []
try:
while 1:
message = self.socket.recv_pyobj(flags=zmq.NOBLOCK)
messages.append(message)
except zmq.ZMQError:
pass
finally:
self.notifier.setEnabled(True)
if messages:
self.sig_received.emit(messages)
def close(self):
"""Read any remaining messages and close stream."""
self.received_message() # Flush remaining messages
self.notifier.setEnabled(False)
self.socket.close()
self.context.destroy()
class AsyncClient(QObject):
"""
A class which handles a connection to a client through a QProcess.
"""
# Emitted when the client has initialized.
initialized = Signal()
# Emitted when the client errors.
errored = Signal()
# Emitted when a request response is received.
received = Signal(object)
def __init__(self,
target,
executable=None,
name=None,
extra_args=None,
libs=None,
cwd=None,
env=None):
super(AsyncClient, self).__init__()
self.executable = executable or sys.executable
self.extra_args = extra_args
self.target = target
self.name = name or self
self.libs = libs
self.cwd = cwd
self.env = env
self.is_initialized = False
self.closing = False
self.context = zmq.Context()
QApplication.instance().aboutToQuit.connect(self.close)
# Set up the heartbeat timer.
self.timer = QTimer(self)
self.timer.timeout.connect(self._heartbeat)
def run(self):
"""Handle the connection with the server.
"""
# Set up the zmq port.
self.socket = self.context.socket(zmq.PAIR)
self.port = self.socket.bind_to_random_port('tcp://*')
# Set up the process.
self.process = QProcess(self)
if self.cwd:
self.process.setWorkingDirectory(self.cwd)
p_args = ['-u', self.target, str(self.port)]
if self.extra_args is not None:
p_args += self.extra_args
# Set up environment variables.
processEnvironment = QProcessEnvironment()
env = self.process.systemEnvironment()
if (self.env and 'PYTHONPATH' not in self.env) or DEV:
python_path = osp.dirname(get_module_path('spyderlib'))
# Add the libs to the python path.
for lib in self.libs:
try:
path = osp.dirname(imp.find_module(lib)[1])
python_path = osp.pathsep.join([python_path, path])
except ImportError:
pass
env.append("PYTHONPATH=%s" % python_path)
if self.env:
env.update(self.env)
for envItem in env:
envName, separator, envValue = envItem.partition('=')
processEnvironment.insert(envName, envValue)
self.process.setProcessEnvironment(processEnvironment)
# Start the process and wait for started.
self.process.start(self.executable, p_args)
self.process.finished.connect(self._on_finished)
running = self.process.waitForStarted()
if not running:
raise IOError('Could not start %s' % self)
# Set up the socket notifer.
fid = self.socket.getsockopt(zmq.FD)
self.notifier = QSocketNotifier(fid, QSocketNotifier.Read, self)
self.notifier.activated.connect(self._on_msg_received)
def request(self, func_name, *args, **kwargs):
"""Send a request to the server.
The response will be a dictionary the 'request_id' and the
'func_name' as well as a 'result' field with the object returned by
the function call or or an 'error' field with a traceback.
"""
if not self.is_initialized:
return
request_id = uuid.uuid4().hex
request = dict(func_name=func_name,
args=args,
kwargs=kwargs,
request_id=request_id)
self._send(request)
return request_id
def close(self):
"""Cleanly close the connection to the server.
"""
self.closing = True
self.is_initialized = False
self.timer.stop()
self.notifier.activated.disconnect(self._on_msg_received)
self.notifier.setEnabled(False)
del self.notifier
self.request('server_quit')
self.process.waitForFinished(1000)
self.process.close()
self.context.destroy()
def _on_finished(self):
"""Handle a finished signal from the process.
"""
if self.closing:
return
if self.is_initialized:
debug_print('Restarting %s' % self.name)
debug_print(self.process.readAllStandardOutput())
debug_print(self.process.readAllStandardError())
self.is_initialized = False
self.notifier.setEnabled(False)
self.run()
else:
debug_print('Errored %s' % self.name)
debug_print(self.process.readAllStandardOutput())
debug_print(self.process.readAllStandardError())
self.errored.emit()
def _on_msg_received(self):
"""Handle a message trigger from the socket.
"""
self.notifier.setEnabled(False)
while 1:
try:
resp = self.socket.recv_pyobj(flags=zmq.NOBLOCK)
except zmq.ZMQError:
self.notifier.setEnabled(True)
return
if not self.is_initialized:
self.is_initialized = True
debug_print('Initialized %s' % self.name)
self.initialized.emit()
self.timer.start(HEARTBEAT)
continue
resp['name'] = self.name
self.received.emit(resp)
def _heartbeat(self):
"""Send a heartbeat to keep the server alive.
"""
self._send(dict(func_name='server_heartbeat'))
def _send(self, obj):
"""Send an object to the server.
"""
try:
self.socket.send_pyobj(obj)
except Exception as e:
debug_print(e)
self.is_initialized = False
self._on_finished()
def start(self):
self.zmq_out_socket = self.context.socket(zmq.PAIR)
self.zmq_out_port = self.zmq_out_socket.bind_to_random_port('tcp://*')
self.zmq_in_socket = self.context.socket(zmq.PAIR)
self.zmq_in_socket.set_hwm(0)
self.zmq_in_port = self.zmq_in_socket.bind_to_random_port('tcp://*')
self.transport_args += ['--zmq-in-port', self.zmq_out_port,
'--zmq-out-port', self.zmq_in_port]
server_log = subprocess.PIPE
if get_debug_level() > 0:
# Create server log file
server_log_fname = 'server_{0}.log'.format(self.language)
server_log_file = get_conf_path(osp.join('lsp_logs',
server_log_fname))
if not osp.exists(osp.dirname(server_log_file)):
os.makedirs(osp.dirname(server_log_file))
server_log = open(server_log_file, 'w')
# Start server with logging options
if get_debug_level() == 2:
self.server_args.append('-v')
elif get_debug_level() == 3:
self.server_args.append('-vv')
if not self.external_server:
logger.info('Starting server: {0}'.format(
' '.join(self.server_args)))
creation_flags = 0
if os.name == 'nt':
creation_flags = (subprocess.CREATE_NEW_PROCESS_GROUP
| 0x08000000) # CREATE_NO_WINDOW
if os.environ.get('CI') and os.name == 'nt':
# The following patching avoids:
#
# OSError: [WinError 6] The handle is invalid
#
# while running our tests in CI services on Windows
# (they run fine locally).
# See this comment for an explanation:
# https://stackoverflow.com/q/43966523/
# 438386#comment74964124_43966523
def patched_cleanup():
pass
subprocess._cleanup = patched_cleanup
self.lsp_server = subprocess.Popen(
self.server_args,
stdout=server_log,
stderr=subprocess.STDOUT,
creationflags=creation_flags)
client_log = subprocess.PIPE
if get_debug_level() > 0:
# Client log file
client_log_fname = 'client_{0}.log'.format(self.language)
client_log_file = get_conf_path(osp.join('lsp_logs',
client_log_fname))
if not osp.exists(osp.dirname(client_log_file)):
os.makedirs(osp.dirname(client_log_file))
client_log = open(client_log_file, 'w')
new_env = dict(os.environ)
python_path = os.pathsep.join(sys.path)[1:]
new_env['PYTHONPATH'] = python_path
self.transport_args = list(map(str, self.transport_args))
logger.info('Starting transport: {0}'
.format(' '.join(self.transport_args)))
self.transport_client = subprocess.Popen(self.transport_args,
stdout=subprocess.PIPE,
stderr=client_log,
env=new_env)
fid = self.zmq_in_socket.getsockopt(zmq.FD)
self.notifier = QSocketNotifier(fid, QSocketNotifier.Read, self)
self.notifier.activated.connect(self.on_msg_received)
# This is necessary for tests to pass locally!
logger.debug('LSP {} client started!'.format(self.language))
class LSPClient(QObject, LSPMethodProviderMixIn):
"""Language Server Protocol v3.0 client implementation."""
#: Signal to inform the editor plugin that the client has
# started properly and it's ready to be used.
sig_initialize = Signal(dict, str)
#: Signal to report internal server errors through Spyder's
# facilities.
sig_server_error = Signal(str)
#: Signal to warn the user when either the transport layer or the
# server went down
sig_went_down = Signal(str)
def __init__(self, parent,
server_settings={},
folder=getcwd_or_home(),
language='python'):
QObject.__init__(self)
self.manager = parent
self.zmq_in_socket = None
self.zmq_out_socket = None
self.zmq_in_port = None
self.zmq_out_port = None
self.transport = None
self.server = None
self.stdio_pid = None
self.notifier = None
self.language = language
self.initialized = False
self.ready_to_close = False
self.request_seq = 1
self.req_status = {}
self.watched_files = {}
self.watched_folders = {}
self.req_reply = {}
self.server_unresponsive = False
self.transport_unresponsive = False
# Select a free port to start the server.
# NOTE: Don't use the new value to set server_setttings['port']!!
# That's not required because this doesn't really correspond to a
# change in the config settings of the server. Else a server
# restart would be generated when doing a
# workspace/didChangeConfiguration request.
if not server_settings['external']:
self.server_port = select_port(
default_port=server_settings['port'])
else:
self.server_port = server_settings['port']
self.server_host = server_settings['host']
self.external_server = server_settings.get('external', False)
self.stdio = server_settings.get('stdio', False)
# Setting stdio on implies that external_server is off
if self.stdio and self.external_server:
error = ('If server is set to use stdio communication, '
'then it cannot be an external server')
logger.error(error)
raise AssertionError(error)
self.folder = folder
self.configurations = server_settings.get('configurations', {})
self.client_capabilites = CLIENT_CAPABILITES
self.server_capabilites = SERVER_CAPABILITES
self.context = zmq.Context()
# To set server args
self._server_args = server_settings.get('args', '')
self._server_cmd = server_settings['cmd']
# Save requests name and id. This is only necessary for testing.
self._requests = []
def _get_log_filename(self, kind):
"""
Get filename to redirect server or transport logs to in
debugging mode.
Parameters
----------
kind: str
It can be "server" or "transport".
"""
if get_debug_level() == 0:
return None
fname = '{0}_{1}_{2}.log'.format(kind, self.language, os.getpid())
location = get_conf_path(osp.join('lsp_logs', fname))
# Create directory that contains the file, in case it doesn't
# exist
if not osp.exists(osp.dirname(location)):
os.makedirs(osp.dirname(location))
return location
@property
def server_log_file(self):
"""
Filename to redirect the server process stdout/stderr output.
"""
return self._get_log_filename('server')
@property
def transport_log_file(self):
"""
Filename to redirect the transport process stdout/stderr
output.
"""
return self._get_log_filename('transport')
@property
def server_args(self):
"""Arguments for the server process."""
args = []
if self.language == 'python':
args += [sys.executable, '-m']
args += [self._server_cmd]
# Replace host and port placeholders
host_and_port = self._server_args.format(
host=self.server_host,
port=self.server_port)
if len(host_and_port) > 0:
args += host_and_port.split(' ')
if self.language == 'python' and get_debug_level() > 0:
args += ['--log-file', self.server_log_file]
if get_debug_level() == 2:
args.append('-v')
elif get_debug_level() == 3:
args.append('-vv')
return args
@property
def transport_args(self):
"""Arguments for the transport process."""
args = [
sys.executable,
'-u',
osp.join(LOCATION, 'transport', 'main.py'),
'--folder', self.folder,
'--transport-debug', str(get_debug_level())
]
# Replace host and port placeholders
host_and_port = '--server-host {host} --server-port {port} '.format(
host=self.server_host,
port=self.server_port)
args += host_and_port.split(' ')
# Add socket ports
args += ['--zmq-in-port', str(self.zmq_out_port),
'--zmq-out-port', str(self.zmq_in_port)]
# Adjustments for stdio/tcp
if self.stdio:
args += ['--stdio-server']
if get_debug_level() > 0:
args += ['--server-log-file', self.server_log_file]
args += self.server_args
else:
args += ['--external-server']
return args
def create_transport_sockets(self):
"""Create PyZMQ sockets for transport."""
self.zmq_out_socket = self.context.socket(zmq.PAIR)
self.zmq_out_port = self.zmq_out_socket.bind_to_random_port(
'tcp://{}'.format(LOCALHOST))
self.zmq_in_socket = self.context.socket(zmq.PAIR)
self.zmq_in_socket.set_hwm(0)
self.zmq_in_port = self.zmq_in_socket.bind_to_random_port(
'tcp://{}'.format(LOCALHOST))
@Slot(QProcess.ProcessError)
def handle_process_errors(self, error):
"""Handle errors with the transport layer or server processes."""
self.sig_went_down.emit(self.language)
def start_server(self):
"""Start server."""
# This is not necessary if we're trying to connect to an
# external server
if self.external_server or self.stdio:
return
logger.info('Starting server: {0}'.format(' '.join(self.server_args)))
# Create server process
self.server = QProcess(self)
env = self.server.processEnvironment()
# Use local PyLS instead of site-packages one.
if DEV or running_under_pytest():
running_in_ci = bool(os.environ.get('CI'))
if os.name != 'nt' or os.name == 'nt' and not running_in_ci:
env.insert('PYTHONPATH', os.pathsep.join(sys.path)[:])
# Adjustments for the Python language server.
if self.language == 'python':
# Set the PyLS current working to an empty dir inside
# our config one. This avoids the server to pick up user
# files such as random.py or string.py instead of the
# standard library modules named the same.
cwd = osp.join(get_conf_path(), 'lsp_paths', 'cwd')
if not osp.exists(cwd):
os.makedirs(cwd)
# On Windows, some modules (notably Matplotlib)
# cause exceptions if they cannot get the user home.
# So, we need to pass the USERPROFILE env variable to
# the PyLS.
if os.name == "nt" and "USERPROFILE" in os.environ:
env.insert("USERPROFILE", os.environ["USERPROFILE"])
else:
# There's no need to define a cwd for other servers.
cwd = None
# Most LSP servers spawn other processes, which may require
# some environment variables.
for var in os.environ:
env.insert(var, os.environ[var])
logger.info('Server process env variables: {0}'.format(env.keys()))
# Setup server
self.server.setProcessEnvironment(env)
self.server.errorOccurred.connect(self.handle_process_errors)
self.server.setWorkingDirectory(cwd)
self.server.setProcessChannelMode(QProcess.MergedChannels)
if self.server_log_file is not None:
self.server.setStandardOutputFile(self.server_log_file)
# Start server
self.server.start(self.server_args[0], self.server_args[1:])
def start_transport(self):
"""Start transport layer."""
logger.info('Starting transport for {1}: {0}'
.format(' '.join(self.transport_args), self.language))
# Create transport process
self.transport = QProcess(self)
env = self.transport.processEnvironment()
# Most LSP servers spawn other processes other than Python, which may
# require some environment variables
if self.language != 'python' and self.stdio:
for var in os.environ:
env.insert(var, os.environ[var])
logger.info('Transport process env variables: {0}'.format(
env.keys()))
self.transport.setProcessEnvironment(env)
# Modifying PYTHONPATH to run transport in development mode or
# tests
if DEV or running_under_pytest():
if running_under_pytest():
env.insert('PYTHONPATH', os.pathsep.join(sys.path)[:])
else:
env.insert('PYTHONPATH', os.pathsep.join(sys.path)[1:])
self.transport.setProcessEnvironment(env)
# Set up transport
self.transport.errorOccurred.connect(self.handle_process_errors)
if self.stdio:
self.transport.setProcessChannelMode(QProcess.SeparateChannels)
if self.transport_log_file is not None:
self.transport.setStandardErrorFile(self.transport_log_file)
else:
self.transport.setProcessChannelMode(QProcess.MergedChannels)
if self.transport_log_file is not None:
self.transport.setStandardOutputFile(self.transport_log_file)
# Start transport
self.transport.start(self.transport_args[0], self.transport_args[1:])
def start(self):
"""Start client."""
# NOTE: DO NOT change the order in which these methods are called.
self.create_transport_sockets()
self.start_server()
self.start_transport()
# Create notifier
fid = self.zmq_in_socket.getsockopt(zmq.FD)
self.notifier = QSocketNotifier(fid, QSocketNotifier.Read, self)
self.notifier.activated.connect(self.on_msg_received)
# This is necessary for tests to pass locally!
logger.debug('LSP {} client started!'.format(self.language))
def stop(self):
"""Stop transport and server."""
logger.info('Stopping {} client...'.format(self.language))
if self.notifier is not None:
self.notifier.activated.disconnect(self.on_msg_received)
self.notifier.setEnabled(False)
self.notifier = None
if self.transport is not None:
self.transport.kill()
self.context.destroy()
if self.server is not None:
self.server.kill()
def is_transport_alive(self):
"""Detect if transport layer is alive."""
state = self.transport.state()
return state != QProcess.NotRunning
def is_stdio_alive(self):
"""Check if an stdio server is alive."""
alive = True
if not psutil.pid_exists(self.stdio_pid):
alive = False
else:
try:
pid_status = psutil.Process(self.stdio_pid).status()
except psutil.NoSuchProcess:
pid_status = ''
if pid_status == psutil.STATUS_ZOMBIE:
alive = False
return alive
def is_server_alive(self):
"""Detect if a tcp server is alive."""
state = self.server.state()
return state != QProcess.NotRunning
def is_down(self):
"""
Detect if the transport layer or server are down to inform our
users about it.
"""
is_down = False
if self.transport and not self.is_transport_alive():
logger.debug(
"Transport layer for {} is down!!".format(self.language))
if not self.transport_unresponsive:
self.transport_unresponsive = True
self.sig_went_down.emit(self.language)
is_down = True
if self.server and not self.is_server_alive():
logger.debug("LSP server for {} is down!!".format(self.language))
if not self.server_unresponsive:
self.server_unresponsive = True
self.sig_went_down.emit(self.language)
is_down = True
if self.stdio_pid and not self.is_stdio_alive():
logger.debug("LSP server for {} is down!!".format(self.language))
if not self.server_unresponsive:
self.server_unresponsive = True
self.sig_went_down.emit(self.language)
is_down = True
return is_down
def send(self, method, params, kind):
"""Send message to transport."""
if self.is_down():
return
if ClientConstants.CANCEL in params:
return
_id = self.request_seq
if kind == MessageKind.REQUEST:
msg = {
'id': self.request_seq,
'method': method,
'params': params
}
self.req_status[self.request_seq] = method
elif kind == MessageKind.RESPONSE:
msg = {
'id': self.request_seq,
'result': params
}
elif kind == MessageKind.NOTIFICATION:
msg = {
'method': method,
'params': params
}
logger.debug('Perform request {0} with id {1}'.format(method, _id))
# Save requests to check their ordering.
if running_under_pytest():
self._requests.append((_id, method))
# Try sending a message. If the send queue is full, keep trying for a
# a second before giving up.
timeout = 1
start_time = time.time()
timeout_time = start_time + timeout
while True:
try:
self.zmq_out_socket.send_pyobj(msg, flags=zmq.NOBLOCK)
self.request_seq += 1
return int(_id)
except zmq.error.Again:
if time.time() > timeout_time:
self.sig_went_down.emit(self.language)
return
# The send queue is full! wait 0.1 seconds before retrying.
if self.initialized:
logger.warning("The send queue is full! Retrying...")
time.sleep(.1)
@Slot()
def on_msg_received(self):
"""Process received messages."""
self.notifier.setEnabled(False)
while True:
try:
# events = self.zmq_in_socket.poll(1500)
resp = self.zmq_in_socket.recv_pyobj(flags=zmq.NOBLOCK)
try:
method = resp['method']
logger.debug(
'{} response: {}'.format(self.language, method))
except KeyError:
pass
if 'error' in resp:
logger.debug('{} Response error: {}'
.format(self.language, repr(resp['error'])))
if self.language == 'python':
# Show PyLS errors in our error report dialog only in
# debug or development modes
if get_debug_level() > 0 or DEV:
message = resp['error'].get('message', '')
traceback = (resp['error'].get('data', {}).
get('traceback'))
if traceback is not None:
traceback = ''.join(traceback)
traceback = traceback + '\n' + message
self.sig_server_error.emit(traceback)
req_id = resp['id']
if req_id in self.req_reply:
self.req_reply[req_id](None, {'params': []})
elif 'method' in resp:
if resp['method'][0] != '$':
if 'id' in resp:
self.request_seq = int(resp['id'])
if resp['method'] in self.handler_registry:
handler_name = (
self.handler_registry[resp['method']])
handler = getattr(self, handler_name)
handler(resp['params'])
elif 'result' in resp:
if resp['result'] is not None:
req_id = resp['id']
if req_id in self.req_status:
req_type = self.req_status[req_id]
if req_type in self.handler_registry:
handler_name = self.handler_registry[req_type]
handler = getattr(self, handler_name)
handler(resp['result'], req_id)
self.req_status.pop(req_id)
if req_id in self.req_reply:
self.req_reply.pop(req_id)
except RuntimeError:
# This is triggered when a codeeditor instance has been
# removed before the response can be processed.
pass
except zmq.ZMQError:
self.notifier.setEnabled(True)
return
def perform_request(self, method, params):
if method in self.sender_registry:
handler_name = self.sender_registry[method]
handler = getattr(self, handler_name)
_id = handler(params)
if 'response_callback' in params:
if params['requires_response']:
self.req_reply[_id] = params['response_callback']
return _id
# ------ LSP initialization methods --------------------------------
@handles(SERVER_READY)
@send_request(method=LSPRequestTypes.INITIALIZE)
def initialize(self, params, *args, **kwargs):
self.stdio_pid = params['pid']
pid = self.transport.processId() if not self.external_server else None
params = {
'processId': pid,
'rootUri': pathlib.Path(osp.abspath(self.folder)).as_uri(),
'capabilities': self.client_capabilites,
'trace': TRACE
}
return params
@send_request(method=LSPRequestTypes.SHUTDOWN)
def shutdown(self):
params = {}
return params
@handles(LSPRequestTypes.SHUTDOWN)
def handle_shutdown(self, response, *args):
self.ready_to_close = True
@send_notification(method=LSPRequestTypes.EXIT)
def exit(self):
params = {}
return params
@handles(LSPRequestTypes.INITIALIZE)
def process_server_capabilities(self, server_capabilites, *args):
"""
Register server capabilities and inform other plugins that it's
available.
"""
# Update server capabilities with the info sent by the server.
server_capabilites = server_capabilites['capabilities']
if isinstance(server_capabilites['textDocumentSync'], int):
kind = server_capabilites['textDocumentSync']
server_capabilites['textDocumentSync'] = TEXT_DOCUMENT_SYNC_OPTIONS
server_capabilites['textDocumentSync']['change'] = kind
if server_capabilites['textDocumentSync'] is None:
server_capabilites.pop('textDocumentSync')
self.server_capabilites.update(server_capabilites)
# The initialized notification needs to be the first request sent by
# the client according to the protocol.
self.initialized = True
self.initialized_call()
# This sends a DidChangeConfiguration request to pass to the server
# the configurations set by the user in our config system.
self.send_configurations(self.configurations)
# Inform other plugins that the server is up.
self.sig_initialize.emit(self.server_capabilites, self.language)
@send_notification(method=LSPRequestTypes.INITIALIZED)
def initialized_call(self):
params = {}
return params
# ------ Settings queries --------------------------------
@property
def support_multiple_workspaces(self):
workspace_settings = self.server_capabilites['workspace']
return workspace_settings['workspaceFolders']['supported']
@property
def support_workspace_update(self):
workspace_settings = self.server_capabilites['workspace']
return workspace_settings['workspaceFolders']['changeNotifications']
class LSPClient(QObject, LSPMethodProviderMixIn):
"""Language Server Protocol v3.0 client implementation."""
# Signals
sig_initialize = Signal(dict, str)
# Constants
external_server_fmt = ('--server-host %(host)s '
'--server-port %(port)s '
'--external-server')
local_server_fmt = ('--server-host %(host)s '
'--server-port %(port)s '
'--server %(cmd)s')
def __init__(self, parent,
server_settings={},
folder=getcwd_or_home(),
language='python'):
QObject.__init__(self)
# LSPMethodProviderMixIn.__init__(self)
self.manager = parent
self.zmq_in_socket = None
self.zmq_out_socket = None
self.zmq_in_port = None
self.zmq_out_port = None
self.transport_client = None
self.language = language
self.initialized = False
self.ready_to_close = False
self.request_seq = 1
self.req_status = {}
self.watched_files = {}
self.req_reply = {}
self.transport_args = [sys.executable, '-u',
osp.join(LOCATION, 'transport', 'main.py')]
self.external_server = server_settings.get('external', False)
self.folder = folder
self.plugin_configurations = server_settings.get('configurations', {})
self.client_capabilites = CLIENT_CAPABILITES
self.server_capabilites = SERVER_CAPABILITES
self.context = zmq.Context()
server_args_fmt = server_settings.get('args', '')
server_args = server_args_fmt.format(**server_settings)
# transport_args = self.local_server_fmt % (server_settings)
# if self.external_server:
transport_args = self.external_server_fmt % (server_settings)
self.server_args = [sys.executable, '-m', server_settings['cmd']]
self.server_args += server_args.split(' ')
self.transport_args += transport_args.split(' ')
self.transport_args += ['--folder', folder]
self.transport_args += ['--transport-debug', str(get_debug_level())]
def start(self):
self.zmq_out_socket = self.context.socket(zmq.PAIR)
self.zmq_out_port = self.zmq_out_socket.bind_to_random_port('tcp://*')
self.zmq_in_socket = self.context.socket(zmq.PAIR)
self.zmq_in_socket.set_hwm(0)
self.zmq_in_port = self.zmq_in_socket.bind_to_random_port('tcp://*')
self.transport_args += ['--zmq-in-port', self.zmq_out_port,
'--zmq-out-port', self.zmq_in_port]
server_log = subprocess.PIPE
if get_debug_level() > 0:
# Create server log file
server_log_fname = 'server_{0}.log'.format(self.language)
server_log_file = get_conf_path(osp.join('lsp_logs',
server_log_fname))
if not osp.exists(osp.dirname(server_log_file)):
os.makedirs(osp.dirname(server_log_file))
server_log = open(server_log_file, 'w')
# Start server with logging options
if get_debug_level() == 2:
self.server_args.append('-v')
elif get_debug_level() == 3:
self.server_args.append('-vv')
if not self.external_server:
logger.info('Starting server: {0}'.format(
' '.join(self.server_args)))
creation_flags = 0
if os.name == 'nt':
creation_flags = (subprocess.CREATE_NEW_PROCESS_GROUP
| 0x08000000) # CREATE_NO_WINDOW
if os.environ.get('CI') and os.name == 'nt':
# The following patching avoids:
#
# OSError: [WinError 6] The handle is invalid
#
# while running our tests in CI services on Windows
# (they run fine locally).
# See this comment for an explanation:
# https://stackoverflow.com/q/43966523/
# 438386#comment74964124_43966523
def patched_cleanup():
pass
subprocess._cleanup = patched_cleanup
self.lsp_server = subprocess.Popen(
self.server_args,
stdout=server_log,
stderr=subprocess.STDOUT,
creationflags=creation_flags)
client_log = subprocess.PIPE
if get_debug_level() > 0:
# Client log file
client_log_fname = 'client_{0}.log'.format(self.language)
client_log_file = get_conf_path(osp.join('lsp_logs',
client_log_fname))
if not osp.exists(osp.dirname(client_log_file)):
os.makedirs(osp.dirname(client_log_file))
client_log = open(client_log_file, 'w')
new_env = dict(os.environ)
python_path = os.pathsep.join(sys.path)[1:]
new_env['PYTHONPATH'] = python_path
self.transport_args = list(map(str, self.transport_args))
logger.info('Starting transport: {0}'
.format(' '.join(self.transport_args)))
self.transport_client = subprocess.Popen(self.transport_args,
stdout=subprocess.PIPE,
stderr=client_log,
env=new_env)
fid = self.zmq_in_socket.getsockopt(zmq.FD)
self.notifier = QSocketNotifier(fid, QSocketNotifier.Read, self)
self.notifier.activated.connect(self.on_msg_received)
# This is necessary for tests to pass locally!
logger.debug('LSP {} client started!'.format(self.language))
def stop(self):
# self.shutdown()
# self.exit()
logger.info('Stopping {} client...'.format(self.language))
if self.notifier is not None:
self.notifier.activated.disconnect(self.on_msg_received)
self.notifier.setEnabled(False)
self.notifier = None
# if os.name == 'nt':
# self.transport_client.send_signal(signal.CTRL_BREAK_EVENT)
# else:
self.transport_client.kill()
self.context.destroy()
if not self.external_server:
self.lsp_server.kill()
def send(self, method, params, requires_response):
if ClientConstants.CANCEL in params:
return
msg = {
'id': self.request_seq,
'method': method,
'params': params
}
if requires_response:
self.req_status[self.request_seq] = method
logger.debug('{} request: {}'.format(self.language, method))
self.zmq_out_socket.send_pyobj(msg)
self.request_seq += 1
return int(msg['id'])
@Slot()
def on_msg_received(self):
self.notifier.setEnabled(False)
while True:
try:
# events = self.zmq_in_socket.poll(1500)
resp = self.zmq_in_socket.recv_pyobj(flags=zmq.NOBLOCK)
try:
method = resp['method']
logger.debug(
'{} response: {}'.format(self.language, method))
except KeyError:
pass
if 'error' in resp:
logger.debug('{} Response error: {}'
.format(self.language, repr(resp['error'])))
if 'method' in resp:
if resp['method'][0] != '$':
if resp['method'] in self.handler_registry:
handler_name = (
self.handler_registry[resp['method']])
handler = getattr(self, handler_name)
handler(resp['params'])
if 'id' in resp:
self.request_seq = resp['id']
elif 'result' in resp:
if resp['result'] is not None:
req_id = resp['id']
if req_id in self.req_status:
req_type = self.req_status[req_id]
if req_type in self.handler_registry:
handler_name = self.handler_registry[req_type]
handler = getattr(self, handler_name)
handler(resp['result'], req_id)
self.req_status.pop(req_id)
if req_id in self.req_reply:
self.req_reply.pop(req_id)
except zmq.ZMQError:
self.notifier.setEnabled(True)
return
def perform_request(self, method, params):
if method in self.sender_registry:
handler_name = self.sender_registry[method]
handler = getattr(self, handler_name)
_id = handler(params)
if 'response_codeeditor' in params:
if params['requires_response']:
self.req_reply[_id] = params['response_codeeditor']
return _id
# ------ LSP initialization methods --------------------------------
@handles(SERVER_READY)
@send_request(method=LSPRequestTypes.INITIALIZE)
def initialize(self, *args, **kwargs):
params = {
'processId': self.transport_client.pid,
'rootUri': pathlib.Path(osp.abspath(self.folder)).as_uri(),
'capabilities': self.client_capabilites,
'trace': TRACE
}
return params
@send_request(method=LSPRequestTypes.SHUTDOWN)
def shutdown(self):
params = {}
return params
@handles(LSPRequestTypes.SHUTDOWN)
def handle_shutdown(self, response, *args):
self.ready_to_close = True
@send_request(method=LSPRequestTypes.EXIT, requires_response=False)
def exit(self):
params = {}
return params
@handles(LSPRequestTypes.INITIALIZE)
def process_server_capabilities(self, server_capabilites, *args):
self.send_plugin_configurations(self.plugin_configurations)
self.initialized = True
server_capabilites = server_capabilites['capabilities']
if isinstance(server_capabilites['textDocumentSync'], int):
kind = server_capabilites['textDocumentSync']
server_capabilites['textDocumentSync'] = TEXT_DOCUMENT_SYNC_OPTIONS
server_capabilites['textDocumentSync']['change'] = kind
if server_capabilites['textDocumentSync'] is None:
server_capabilites.pop('textDocumentSync')
self.server_capabilites.update(server_capabilites)
self.sig_initialize.emit(self.server_capabilites, self.language)
@send_request(method=LSPRequestTypes.WORKSPACE_CONFIGURATION_CHANGE,
requires_response=False)
def send_plugin_configurations(self, configurations, *args):
self.plugin_configurations = configurations
params = {
'settings': configurations
}
return params
class TwistedSocketNotifier(QObject):
"""Connection between an fd event and reader/writer callbacks."""
activated = Signal(int)
def __init__(self, parent, reactor, watcher, socketType):
QObject.__init__(self, parent)
self.reactor = reactor
self.watcher = watcher
fd = watcher.fileno()
self.notifier = QSocketNotifier(fd, socketType, parent)
self.notifier.setEnabled(True)
if socketType == QSocketNotifier.Read:
self.fn = self.read
else:
self.fn = self.write
self.notifier.activated.connect(self.fn)
def shutdown(self):
self.notifier.setEnabled(False)
self.notifier.activated.disconnect(self.fn)
self.fn = self.watcher = None
self.notifier.deleteLater()
self.deleteLater()
def read(self, fd):
if not self.watcher:
return
w = self.watcher
# doRead can cause self.shutdown to be called so keep
# a reference to self.watcher
def _read():
# Don't call me again, until the data has been read
self.notifier.setEnabled(False)
why = None
try:
why = w.doRead()
inRead = True
except:
inRead = False
log.err()
why = sys.exc_info()[1]
if why:
self.reactor._disconnectSelectable(w, why, inRead)
elif self.watcher:
self.notifier.setEnabled(True)
# Re enable notification following sucessfull read
self.reactor._iterate(fromqt=True)
log.callWithLogger(w, _read)
def write(self, sock):
if not self.watcher:
return
w = self.watcher
def _write():
why = None
self.notifier.setEnabled(False)
try:
why = w.doWrite()
except:
log.err()
why = sys.exc_info()[1]
if why:
self.reactor._disconnectSelectable(w, why, False)
elif self.watcher:
self.notifier.setEnabled(True)
self.reactor._iterate(fromqt=True)
log.callWithLogger(w, _write)
class LSPClient(QObject, LSPMethodProviderMixIn):
"""Language Server Protocol v3.0 client implementation."""
initialized = Signal()
external_server_fmt = ('--server-host %(host)s '
'--server-port %(port)s '
'--external-server')
local_server_fmt = ('--server-host %(host)s '
'--server-port %(port)s '
'--server %(cmd)s')
def __init__(self,
parent,
server_args_fmt='',
server_settings={},
external_server=False,
folder=getcwd(),
language='python',
plugin_configurations={}):
QObject.__init__(self)
# LSPMethodProviderMixIn.__init__(self)
self.manager = parent
self.zmq_in_socket = None
self.zmq_out_socket = None
self.zmq_in_port = None
self.zmq_out_port = None
self.transport_client = None
self.language = language
self.initialized = False
self.ready_to_close = False
self.request_seq = 1
self.req_status = {}
self.plugin_registry = {}
self.watched_files = {}
self.req_reply = {}
self.transport_args = [
sys.executable, '-u',
osp.join(LOCATION, 'transport', 'main.py')
]
self.external_server = external_server
self.folder = folder
self.plugin_configurations = plugin_configurations
self.client_capabilites = CLIENT_CAPABILITES
self.server_capabilites = SERVER_CAPABILITES
self.context = zmq.Context()
server_args = server_args_fmt % (server_settings)
# transport_args = self.local_server_fmt % (server_settings)
# if self.external_server:
transport_args = self.external_server_fmt % (server_settings)
self.server_args = [server_settings['cmd']]
self.server_args += server_args.split(' ')
self.transport_args += transport_args.split(' ')
self.transport_args += ['--folder', folder]
self.transport_args += ['--transport-debug', str(get_debug_level())]
def start(self):
self.zmq_out_socket = self.context.socket(zmq.PAIR)
self.zmq_out_port = self.zmq_out_socket.bind_to_random_port('tcp://*')
self.zmq_in_socket = self.context.socket(zmq.PAIR)
self.zmq_in_socket.set_hwm(0)
self.zmq_in_port = self.zmq_in_socket.bind_to_random_port('tcp://*')
self.transport_args += [
'--zmq-in-port', self.zmq_out_port, '--zmq-out-port',
self.zmq_in_port
]
self.lsp_server_log = subprocess.PIPE
if get_debug_level() > 0:
lsp_server_file = 'lsp_server_logfile.log'
log_file = get_conf_path(osp.join('lsp_logs', lsp_server_file))
if not osp.exists(osp.dirname(log_file)):
os.makedirs(osp.dirname(log_file))
self.lsp_server_log = open(log_file, 'w')
if not self.external_server:
logger.info('Starting server: {0}'.format(' '.join(
self.server_args)))
creation_flags = 0
if WINDOWS:
creation_flags = subprocess.CREATE_NEW_PROCESS_GROUP
self.lsp_server = subprocess.Popen(self.server_args,
stdout=self.lsp_server_log,
stderr=subprocess.STDOUT,
creationflags=creation_flags)
# self.transport_args += self.server_args
self.stdout_log = subprocess.PIPE
self.stderr_log = subprocess.PIPE
if get_debug_level() > 0:
stderr_log_file = 'lsp_client_{0}.log'.format(self.language)
log_file = get_conf_path(osp.join('lsp_logs', stderr_log_file))
if not osp.exists(osp.dirname(log_file)):
os.makedirs(osp.dirname(log_file))
self.stderr_log = open(log_file, 'w')
new_env = dict(os.environ)
python_path = os.pathsep.join(sys.path)[1:]
new_env['PYTHONPATH'] = python_path
self.transport_args = map(str, self.transport_args)
self.transport_client = subprocess.Popen(self.transport_args,
stdout=self.stdout_log,
stderr=self.stderr_log,
env=new_env)
fid = self.zmq_in_socket.getsockopt(zmq.FD)
self.notifier = QSocketNotifier(fid, QSocketNotifier.Read, self)
# self.notifier.activated.connect(self.debug_print)
self.notifier.activated.connect(self.on_msg_received)
# self.initialize()
def stop(self):
# self.shutdown()
# self.exit()
logger.info('Stopping client...')
if self.notifier is not None:
self.notifier.activated.disconnect(self.on_msg_received)
self.notifier.setEnabled(False)
self.notifier = None
# if WINDOWS:
# self.transport_client.send_signal(signal.CTRL_BREAK_EVENT)
# else:
self.transport_client.kill()
self.context.destroy()
if not self.external_server:
self.lsp_server.kill()
def send(self, method, params, requires_response):
if ClientConstants.CANCEL in params:
return
msg = {'id': self.request_seq, 'method': method, 'params': params}
if requires_response:
self.req_status[self.request_seq] = method
logger.debug('{} request: {}'.format(self.language, method))
self.zmq_out_socket.send_pyobj(msg)
self.request_seq += 1
return int(msg['id'])
@Slot()
def on_msg_received(self):
self.notifier.setEnabled(False)
while True:
try:
# events = self.zmq_in_socket.poll(1500)
resp = self.zmq_in_socket.recv_pyobj(flags=zmq.NOBLOCK)
try:
method = resp['method']
logger.debug('{} response: {}'.format(
self.language, method))
except KeyError:
pass
if 'method' in resp:
if resp['method'][0] != '$':
if resp['method'] in self.handler_registry:
handler_name = (
self.handler_registry[resp['method']])
handler = getattr(self, handler_name)
handler(resp['params'])
if 'id' in resp:
self.request_seq = resp['id']
elif 'result' in resp:
if resp['result'] is not None:
req_id = resp['id']
if req_id in self.req_status:
req_type = self.req_status[req_id]
if req_type in self.handler_registry:
handler_name = self.handler_registry[req_type]
handler = getattr(self, handler_name)
handler(resp['result'], req_id)
self.req_status.pop(req_id)
if req_id in self.req_reply:
self.req_reply.pop(req_id)
except zmq.ZMQError as e:
self.notifier.setEnabled(True)
return
def perform_request(self, method, params):
if method in self.sender_registry:
handler_name = self.sender_registry[method]
handler = getattr(self, handler_name)
_id = handler(params)
if 'response_codeeditor' in params:
if params['requires_response']:
self.req_reply[_id] = params['response_codeeditor']
return _id
# ------ Spyder plugin registration --------------------------------
def register_plugin_type(self, plugin_type, notification_sig):
if plugin_type not in self.plugin_registry:
self.plugin_registry[plugin_type] = []
self.plugin_registry[plugin_type].append(notification_sig)
# ------ LSP initialization methods --------------------------------
@handles(SERVER_READY)
@send_request(method=LSPRequestTypes.INITIALIZE)
def initialize(self, *args, **kwargs):
params = {
'processId': self.transport_client.pid,
'rootUri': pathlib.Path(osp.abspath(self.folder)).as_uri(),
'capabilities': self.client_capabilites,
'trace': TRACE
}
return params
@send_request(method=LSPRequestTypes.SHUTDOWN)
def shutdown(self):
params = {}
return params
@handles(LSPRequestTypes.SHUTDOWN)
def handle_shutdown(self, response, *args):
self.ready_to_close = True
@send_request(method=LSPRequestTypes.EXIT, requires_response=False)
def exit(self):
params = {}
return params
@handles(LSPRequestTypes.INITIALIZE)
def process_server_capabilities(self, server_capabilites, *args):
self.send_plugin_configurations(self.plugin_configurations)
self.initialized = True
server_capabilites = server_capabilites['capabilities']
if isinstance(server_capabilites['textDocumentSync'], int):
kind = server_capabilites['textDocumentSync']
server_capabilites['textDocumentSync'] = TEXT_DOCUMENT_SYNC_OPTIONS
server_capabilites['textDocumentSync']['change'] = kind
if server_capabilites['textDocumentSync'] is None:
server_capabilites.pop('textDocumentSync')
self.server_capabilites.update(server_capabilites)
for sig in self.plugin_registry[LSPEventTypes.DOCUMENT]:
sig.emit(self.server_capabilites, self.language)
@send_request(method=LSPRequestTypes.WORKSPACE_CONFIGURATION_CHANGE,
requires_response=False)
def send_plugin_configurations(self, configurations, *args):
self.plugin_configurations = configurations
params = {'settings': configurations}
return params
def start(self):
self.zmq_out_socket = self.context.socket(zmq.PAIR)
self.zmq_out_port = self.zmq_out_socket.bind_to_random_port(
'tcp://{}'.format(LOCALHOST))
self.zmq_in_socket = self.context.socket(zmq.PAIR)
self.zmq_in_socket.set_hwm(0)
self.zmq_in_port = self.zmq_in_socket.bind_to_random_port(
'tcp://{}'.format(LOCALHOST))
self.transport_args += [
'--zmq-in-port', self.zmq_out_port, '--zmq-out-port',
self.zmq_in_port
]
server_log = subprocess.PIPE
pid = os.getpid()
if get_debug_level() > 0:
# Create server log file
server_log_fname = 'server_{0}_{1}.log'.format(self.language, pid)
server_log_file = get_conf_path(
osp.join('lsp_logs', server_log_fname))
if not osp.exists(osp.dirname(server_log_file)):
os.makedirs(osp.dirname(server_log_file))
server_log = open(server_log_file, 'w')
if self.stdio:
server_log.close()
if self.language == 'python':
self.server_args += ['--log-file', server_log_file]
self.transport_args += ['--server-log-file', server_log_file]
# Start server with logging options
if self.language != 'cpp':
if get_debug_level() == 2:
self.server_args.append('-v')
elif get_debug_level() == 3:
self.server_args.append('-vv')
server_stdin = subprocess.PIPE
server_stdout = server_log
server_stderr = subprocess.STDOUT
if not self.external_server:
logger.info('Starting server: {0}'.format(' '.join(
self.server_args)))
creation_flags = 0
if os.name == 'nt':
creation_flags = (subprocess.CREATE_NEW_PROCESS_GROUP
| 0x08000000) # CREATE_NO_WINDOW
if os.environ.get('CI') and os.name == 'nt':
# The following patching avoids:
#
# OSError: [WinError 6] The handle is invalid
#
# while running our tests in CI services on Windows
# (they run fine locally).
# See this comment for an explanation:
# https://stackoverflow.com/q/43966523/
# 438386#comment74964124_43966523
def patched_cleanup():
pass
subprocess._cleanup = patched_cleanup
self.lsp_server = subprocess.Popen(self.server_args,
stdout=server_stdout,
stdin=server_stdin,
stderr=server_stderr,
creationflags=creation_flags)
client_log = subprocess.PIPE
if get_debug_level() > 0:
# Client log file
client_log_fname = 'client_{0}_{1}.log'.format(self.language, pid)
client_log_file = get_conf_path(
osp.join('lsp_logs', client_log_fname))
if not osp.exists(osp.dirname(client_log_file)):
os.makedirs(osp.dirname(client_log_file))
client_log = open(client_log_file, 'w')
new_env = dict(os.environ)
python_path = os.pathsep.join(sys.path)[1:]
new_env['PYTHONPATH'] = python_path
# This allows running LSP tests directly without having to install
# Spyder
if running_under_pytest():
new_env['PYTHONPATH'] = os.pathsep.join(sys.path)[:]
# On some CI systems there are unicode characters inside PYTHOPATH
# which raise errors if not removed
if PY2:
new_env = clean_env(new_env)
self.transport_args = list(map(str, self.transport_args))
logger.info('Starting transport: {0}'.format(' '.join(
self.transport_args)))
if self.stdio:
transport_stdin = subprocess.PIPE
transport_stdout = subprocess.PIPE
transport_stderr = client_log
self.transport_args += self.server_args
else:
transport_stdout = client_log
transport_stdin = subprocess.PIPE
transport_stderr = subprocess.STDOUT
self.transport_client = subprocess.Popen(self.transport_args,
stdout=transport_stdout,
stdin=transport_stdin,
stderr=transport_stderr,
env=new_env)
fid = self.zmq_in_socket.getsockopt(zmq.FD)
self.notifier = QSocketNotifier(fid, QSocketNotifier.Read, self)
self.notifier.activated.connect(self.on_msg_received)
# This is necessary for tests to pass locally!
logger.debug('LSP {} client started!'.format(self.language))
def start(self):
self.zmq_out_socket = self.context.socket(zmq.PAIR)
self.zmq_out_port = self.zmq_out_socket.bind_to_random_port('tcp://*')
self.zmq_in_socket = self.context.socket(zmq.PAIR)
self.zmq_in_socket.set_hwm(0)
self.zmq_in_port = self.zmq_in_socket.bind_to_random_port('tcp://*')
self.transport_args += [
'--zmq-in-port', self.zmq_out_port, '--zmq-out-port',
self.zmq_in_port
]
server_log = subprocess.PIPE
if get_debug_level() > 0:
# Create server log file
server_log_fname = 'server_{0}.log'.format(self.language)
server_log_file = get_conf_path(
osp.join('lsp_logs', server_log_fname))
if not osp.exists(osp.dirname(server_log_file)):
os.makedirs(osp.dirname(server_log_file))
server_log = open(server_log_file, 'w')
# Start server with logging options
if get_debug_level() == 2:
self.server_args.append('-v')
elif get_debug_level() == 3:
self.server_args.append('-vv')
if not self.external_server:
logger.info('Starting server: {0}'.format(' '.join(
self.server_args)))
creation_flags = 0
if os.name == 'nt':
creation_flags = (subprocess.CREATE_NEW_PROCESS_GROUP
| 0x08000000) # CREATE_NO_WINDOW
if os.environ.get('CI') and os.name == 'nt':
# The following patching avoids:
#
# OSError: [WinError 6] The handle is invalid
#
# while running our tests in CI services on Windows
# (they run fine locally).
# See this comment for an explanation:
# https://stackoverflow.com/q/43966523/
# 438386#comment74964124_43966523
def patched_cleanup():
pass
subprocess._cleanup = patched_cleanup
self.lsp_server = subprocess.Popen(self.server_args,
stdout=server_log,
stderr=subprocess.STDOUT,
creationflags=creation_flags)
client_log = subprocess.PIPE
if get_debug_level() > 0:
# Client log file
client_log_fname = 'client_{0}.log'.format(self.language)
client_log_file = get_conf_path(
osp.join('lsp_logs', client_log_fname))
if not osp.exists(osp.dirname(client_log_file)):
os.makedirs(osp.dirname(client_log_file))
client_log = open(client_log_file, 'w')
new_env = dict(os.environ)
python_path = os.pathsep.join(sys.path)[1:]
new_env['PYTHONPATH'] = python_path
self.transport_args = list(map(str, self.transport_args))
logger.info('Starting transport: {0}'.format(' '.join(
self.transport_args)))
self.transport_client = subprocess.Popen(self.transport_args,
stdout=subprocess.PIPE,
stderr=client_log,
env=new_env)
fid = self.zmq_in_socket.getsockopt(zmq.FD)
self.notifier = QSocketNotifier(fid, QSocketNotifier.Read, self)
self.notifier.activated.connect(self.on_msg_received)
# This is necessary for tests to pass locally!
logger.debug('LSP {} client started!'.format(self.language))
class LSPClient(QObject, LSPMethodProviderMixIn):
"""Language Server Protocol v3.0 client implementation."""
#: Signal to inform the editor plugin that the client has
# started properly and it's ready to be used.
sig_initialize = Signal(dict, str)
#: Signal to report internal server errors through Spyder's
# facilities.
sig_server_error = Signal(str)
#: Signal to warn the user when either the transport layer or the
# server are down
sig_lsp_down = Signal(str)
def __init__(self,
parent,
server_settings={},
folder=getcwd_or_home(),
language='python'):
QObject.__init__(self)
self.manager = parent
self.zmq_in_socket = None
self.zmq_out_socket = None
self.zmq_in_port = None
self.zmq_out_port = None
self.transport_client = None
self.lsp_server = None
self.notifier = None
self.language = language
self.initialized = False
self.ready_to_close = False
self.request_seq = 1
self.req_status = {}
self.watched_files = {}
self.watched_folders = {}
self.req_reply = {}
# Select a free port to start the server.
# NOTE: Don't use the new value to set server_setttings['port']!!
# That's not required because this doesn't really correspond to a
# change in the config settings of the server. Else a server
# restart would be generated when doing a
# workspace/didChangeConfiguration request.
if not server_settings['external']:
self.server_port = select_port(
default_port=server_settings['port'])
else:
self.server_port = server_settings['port']
self.server_host = server_settings['host']
self.transport_args = [
sys.executable, '-u',
osp.join(LOCATION, 'transport', 'main.py')
]
self.external_server = server_settings.get('external', False)
self.stdio = server_settings.get('stdio', False)
# Setting stdio on implies that external_server is off
if self.stdio and self.external_server:
error = ('If server is set to use stdio communication, '
'then it cannot be an external server')
logger.error(error)
raise AssertionError(error)
self.folder = folder
self.plugin_configurations = server_settings.get('configurations', {})
self.client_capabilites = CLIENT_CAPABILITES
self.server_capabilites = SERVER_CAPABILITES
self.context = zmq.Context()
server_args_fmt = server_settings.get('args', '')
server_args = server_args_fmt.format(host=self.server_host,
port=self.server_port)
transport_args_fmt = '--server-host {host} --server-port {port} '
transport_args = transport_args_fmt.format(host=self.server_host,
port=self.server_port)
self.server_args = []
if self.language == 'python':
self.server_args += [sys.executable, '-m']
self.server_args += [server_settings['cmd']]
if len(server_args) > 0:
self.server_args += server_args.split(' ')
self.server_unresponsive = False
self.transport_args += transport_args.split(' ')
self.transport_args += ['--folder', folder]
self.transport_args += ['--transport-debug', str(get_debug_level())]
if not self.stdio:
self.transport_args += ['--external-server']
else:
self.transport_args += ['--stdio-server']
self.external_server = True
self.transport_unresponsive = False
def start(self):
self.zmq_out_socket = self.context.socket(zmq.PAIR)
self.zmq_out_port = self.zmq_out_socket.bind_to_random_port(
'tcp://{}'.format(LOCALHOST))
self.zmq_in_socket = self.context.socket(zmq.PAIR)
self.zmq_in_socket.set_hwm(0)
self.zmq_in_port = self.zmq_in_socket.bind_to_random_port(
'tcp://{}'.format(LOCALHOST))
self.transport_args += [
'--zmq-in-port', self.zmq_out_port, '--zmq-out-port',
self.zmq_in_port
]
server_log = subprocess.PIPE
pid = os.getpid()
if get_debug_level() > 0:
# Create server log file
server_log_fname = 'server_{0}_{1}.log'.format(self.language, pid)
server_log_file = get_conf_path(
osp.join('lsp_logs', server_log_fname))
if not osp.exists(osp.dirname(server_log_file)):
os.makedirs(osp.dirname(server_log_file))
server_log = open(server_log_file, 'w')
if self.stdio:
server_log.close()
if self.language == 'python':
self.server_args += ['--log-file', server_log_file]
self.transport_args += ['--server-log-file', server_log_file]
# Start server with logging options
if self.language != 'cpp':
if get_debug_level() == 2:
self.server_args.append('-v')
elif get_debug_level() == 3:
self.server_args.append('-vv')
server_stdin = subprocess.PIPE
server_stdout = server_log
server_stderr = subprocess.STDOUT
if not self.external_server:
logger.info('Starting server: {0}'.format(' '.join(
self.server_args)))
creation_flags = 0
if os.name == 'nt':
creation_flags = (subprocess.CREATE_NEW_PROCESS_GROUP
| 0x08000000) # CREATE_NO_WINDOW
if os.environ.get('CI') and os.name == 'nt':
# The following patching avoids:
#
# OSError: [WinError 6] The handle is invalid
#
# while running our tests in CI services on Windows
# (they run fine locally).
# See this comment for an explanation:
# https://stackoverflow.com/q/43966523/
# 438386#comment74964124_43966523
def patched_cleanup():
pass
subprocess._cleanup = patched_cleanup
self.lsp_server = subprocess.Popen(self.server_args,
stdout=server_stdout,
stdin=server_stdin,
stderr=server_stderr,
creationflags=creation_flags)
client_log = subprocess.PIPE
if get_debug_level() > 0:
# Client log file
client_log_fname = 'client_{0}_{1}.log'.format(self.language, pid)
client_log_file = get_conf_path(
osp.join('lsp_logs', client_log_fname))
if not osp.exists(osp.dirname(client_log_file)):
os.makedirs(osp.dirname(client_log_file))
client_log = open(client_log_file, 'w')
new_env = dict(os.environ)
python_path = os.pathsep.join(sys.path)[1:]
new_env['PYTHONPATH'] = python_path
# This allows running LSP tests directly without having to install
# Spyder
if running_under_pytest():
new_env['PYTHONPATH'] = os.pathsep.join(sys.path)[:]
# On some CI systems there are unicode characters inside PYTHOPATH
# which raise errors if not removed
if PY2:
new_env = clean_env(new_env)
self.transport_args = list(map(str, self.transport_args))
logger.info('Starting transport: {0}'.format(' '.join(
self.transport_args)))
if self.stdio:
transport_stdin = subprocess.PIPE
transport_stdout = subprocess.PIPE
transport_stderr = client_log
self.transport_args += self.server_args
else:
transport_stdout = client_log
transport_stdin = subprocess.PIPE
transport_stderr = subprocess.STDOUT
self.transport_client = subprocess.Popen(self.transport_args,
stdout=transport_stdout,
stdin=transport_stdin,
stderr=transport_stderr,
env=new_env)
fid = self.zmq_in_socket.getsockopt(zmq.FD)
self.notifier = QSocketNotifier(fid, QSocketNotifier.Read, self)
self.notifier.activated.connect(self.on_msg_received)
# This is necessary for tests to pass locally!
logger.debug('LSP {} client started!'.format(self.language))
def stop(self):
logger.info('Stopping {} client...'.format(self.language))
if self.notifier is not None:
self.notifier.activated.disconnect(self.on_msg_received)
self.notifier.setEnabled(False)
self.notifier = None
if self.transport_client is not None:
self.transport_client.kill()
self.context.destroy()
if not self.external_server:
self.lsp_server.kill()
def send(self, method, params, kind):
# Detect when the transport layer is down to show a message
# to our users about it.
if (self.transport_client is not None
and self.transport_client.poll() is not None):
logger.debug("Transport layer for {} is down!!".format(
self.language))
if not self.transport_unresponsive:
self.transport_unresponsive = True
self.sig_lsp_down.emit(self.language)
return
if (self.lsp_server is not None
and self.lsp_server.poll() is not None):
logger.debug("LSP server for {} is down!!".format(self.language))
if not self.server_unresponsive:
self.server_unresponsive = True
self.sig_lsp_down.emit(self.language)
return
if ClientConstants.CANCEL in params:
return
_id = self.request_seq
if kind == MessageKind.REQUEST:
msg = {'id': self.request_seq, 'method': method, 'params': params}
self.req_status[self.request_seq] = method
elif kind == MessageKind.RESPONSE:
msg = {'id': self.request_seq, 'result': params}
elif kind == MessageKind.NOTIFICATION:
msg = {'method': method, 'params': params}
logger.debug('{} request: {}'.format(self.language, method))
self.zmq_out_socket.send_pyobj(msg)
self.request_seq += 1
return int(_id)
@Slot()
def on_msg_received(self):
self.notifier.setEnabled(False)
while True:
try:
# events = self.zmq_in_socket.poll(1500)
resp = self.zmq_in_socket.recv_pyobj(flags=zmq.NOBLOCK)
try:
method = resp['method']
logger.debug('{} response: {}'.format(
self.language, method))
except KeyError:
pass
if 'error' in resp:
logger.debug('{} Response error: {}'.format(
self.language, repr(resp['error'])))
if self.language == 'python':
# Show PyLS errors in our error report dialog only in
# debug or development modes
if get_debug_level() > 0 or DEV:
message = resp['error'].get('message', '')
traceback = (resp['error'].get(
'data', {}).get('traceback'))
if traceback is not None:
traceback = ''.join(traceback)
traceback = traceback + '\n' + message
self.sig_server_error.emit(traceback)
req_id = resp['id']
if req_id in self.req_reply:
self.req_reply[req_id](None, {'params': []})
elif 'method' in resp:
if resp['method'][0] != '$':
if 'id' in resp:
self.request_seq = int(resp['id'])
if resp['method'] in self.handler_registry:
handler_name = (
self.handler_registry[resp['method']])
handler = getattr(self, handler_name)
handler(resp['params'])
elif 'result' in resp:
if resp['result'] is not None:
req_id = resp['id']
if req_id in self.req_status:
req_type = self.req_status[req_id]
if req_type in self.handler_registry:
handler_name = self.handler_registry[req_type]
handler = getattr(self, handler_name)
handler(resp['result'], req_id)
self.req_status.pop(req_id)
if req_id in self.req_reply:
self.req_reply.pop(req_id)
except RuntimeError:
# This is triggered when a codeeditor instance has been
# removed before the response can be processed.
pass
except zmq.ZMQError:
self.notifier.setEnabled(True)
return
def perform_request(self, method, params):
if method in self.sender_registry:
handler_name = self.sender_registry[method]
handler = getattr(self, handler_name)
_id = handler(params)
if 'response_callback' in params:
if params['requires_response']:
self.req_reply[_id] = params['response_callback']
return _id
# ------ LSP initialization methods --------------------------------
@handles(SERVER_READY)
@send_request(method=LSPRequestTypes.INITIALIZE)
def initialize(self, *args, **kwargs):
pid = self.transport_client.pid if not self.external_server else None
params = {
'processId': pid,
'rootUri': pathlib.Path(osp.abspath(self.folder)).as_uri(),
'capabilities': self.client_capabilites,
'trace': TRACE
}
return params
@send_request(method=LSPRequestTypes.SHUTDOWN)
def shutdown(self):
params = {}
return params
@handles(LSPRequestTypes.SHUTDOWN)
def handle_shutdown(self, response, *args):
self.ready_to_close = True
@send_notification(method=LSPRequestTypes.EXIT)
def exit(self):
params = {}
return params
@handles(LSPRequestTypes.INITIALIZE)
def process_server_capabilities(self, server_capabilites, *args):
self.send_plugin_configurations(self.plugin_configurations)
self.initialized = True
server_capabilites = server_capabilites['capabilities']
if isinstance(server_capabilites['textDocumentSync'], int):
kind = server_capabilites['textDocumentSync']
server_capabilites['textDocumentSync'] = TEXT_DOCUMENT_SYNC_OPTIONS
server_capabilites['textDocumentSync']['change'] = kind
if server_capabilites['textDocumentSync'] is None:
server_capabilites.pop('textDocumentSync')
self.server_capabilites.update(server_capabilites)
self.sig_initialize.emit(self.server_capabilites, self.language)
self.initialized_call()
@send_notification(method=LSPRequestTypes.INITIALIZED)
def initialized_call(self):
params = {}
return params
class LSPClient(QObject, LSPMethodProviderMixIn):
"""Language Server Protocol v3.0 client implementation."""
initialized = Signal()
external_server_fmt = ('--server-host %(host)s '
'--server-port %(port)s '
'--external-server')
local_server_fmt = ('--server-host %(host)s '
'--server-port %(port)s '
'--server %(cmd)s')
def __init__(self, parent, server_args_fmt='',
server_settings={}, external_server=False,
folder=getcwd(), language='python',
plugin_configurations={}):
QObject.__init__(self)
# LSPMethodProviderMixIn.__init__(self)
self.manager = parent
self.zmq_in_socket = None
self.zmq_out_socket = None
self.zmq_in_port = None
self.zmq_out_port = None
self.transport_client = None
self.language = language
self.initialized = False
self.ready_to_close = False
self.request_seq = 1
self.req_status = {}
self.plugin_registry = {}
self.watched_files = {}
self.req_reply = {}
self.transport_args = [sys.executable, '-u',
osp.join(LOCATION, 'transport', 'main.py')]
self.external_server = external_server
self.folder = folder
self.plugin_configurations = plugin_configurations
self.client_capabilites = CLIENT_CAPABILITES
self.server_capabilites = SERVER_CAPABILITES
self.context = zmq.Context()
server_args = server_args_fmt % (server_settings)
# transport_args = self.local_server_fmt % (server_settings)
# if self.external_server:
transport_args = self.external_server_fmt % (server_settings)
self.server_args = [server_settings['cmd']]
self.server_args += server_args.split(' ')
self.transport_args += transport_args.split(' ')
self.transport_args += ['--folder', folder]
self.transport_args += ['--transport-debug', str(get_debug_level())]
def start(self):
self.zmq_out_socket = self.context.socket(zmq.PAIR)
self.zmq_out_port = self.zmq_out_socket.bind_to_random_port('tcp://*')
self.zmq_in_socket = self.context.socket(zmq.PAIR)
self.zmq_in_socket.set_hwm(0)
self.zmq_in_port = self.zmq_in_socket.bind_to_random_port('tcp://*')
self.transport_args += ['--zmq-in-port', self.zmq_out_port,
'--zmq-out-port', self.zmq_in_port]
self.lsp_server_log = subprocess.PIPE
if get_debug_level() > 0:
lsp_server_file = 'lsp_server_logfile.log'
log_file = get_conf_path(osp.join('lsp_logs', lsp_server_file))
if not osp.exists(osp.dirname(log_file)):
os.makedirs(osp.dirname(log_file))
self.lsp_server_log = open(log_file, 'w')
if not self.external_server:
logger.info('Starting server: {0}'.format(
' '.join(self.server_args)))
creation_flags = 0
if WINDOWS:
creation_flags = subprocess.CREATE_NEW_PROCESS_GROUP
self.lsp_server = subprocess.Popen(
self.server_args,
stdout=self.lsp_server_log,
stderr=subprocess.STDOUT,
creationflags=creation_flags)
# self.transport_args += self.server_args
self.stdout_log = subprocess.PIPE
self.stderr_log = subprocess.PIPE
if get_debug_level() > 0:
stderr_log_file = 'lsp_client_{0}.log'.format(self.language)
log_file = get_conf_path(osp.join('lsp_logs', stderr_log_file))
if not osp.exists(osp.dirname(log_file)):
os.makedirs(osp.dirname(log_file))
self.stderr_log = open(log_file, 'w')
new_env = dict(os.environ)
python_path = os.pathsep.join(sys.path)[1:]
new_env['PYTHONPATH'] = python_path
self.transport_args = map(str, self.transport_args)
self.transport_client = subprocess.Popen(self.transport_args,
stdout=self.stdout_log,
stderr=self.stderr_log,
env=new_env)
fid = self.zmq_in_socket.getsockopt(zmq.FD)
self.notifier = QSocketNotifier(fid, QSocketNotifier.Read, self)
# self.notifier.activated.connect(self.debug_print)
self.notifier.activated.connect(self.on_msg_received)
# self.initialize()
def stop(self):
# self.shutdown()
# self.exit()
logger.info('Stopping client...')
if self.notifier is not None:
self.notifier.activated.disconnect(self.on_msg_received)
self.notifier.setEnabled(False)
self.notifier = None
# if WINDOWS:
# self.transport_client.send_signal(signal.CTRL_BREAK_EVENT)
# else:
self.transport_client.kill()
self.context.destroy()
if not self.external_server:
self.lsp_server.kill()
def send(self, method, params, requires_response):
if ClientConstants.CANCEL in params:
return
msg = {
'id': self.request_seq,
'method': method,
'params': params
}
if requires_response:
self.req_status[self.request_seq] = method
logger.debug('{} request: {}'.format(self.language, method))
self.zmq_out_socket.send_pyobj(msg)
self.request_seq += 1
return int(msg['id'])
@Slot()
def on_msg_received(self):
self.notifier.setEnabled(False)
while True:
try:
# events = self.zmq_in_socket.poll(1500)
resp = self.zmq_in_socket.recv_pyobj(flags=zmq.NOBLOCK)
try:
method = resp['method']
logger.debug(
'{} response: {}'.format(self.language, method))
except KeyError:
pass
if 'method' in resp:
if resp['method'][0] != '$':
if resp['method'] in self.handler_registry:
handler_name = (
self.handler_registry[resp['method']])
handler = getattr(self, handler_name)
handler(resp['params'])
if 'id' in resp:
self.request_seq = resp['id']
elif 'result' in resp:
if resp['result'] is not None:
req_id = resp['id']
if req_id in self.req_status:
req_type = self.req_status[req_id]
if req_type in self.handler_registry:
handler_name = self.handler_registry[req_type]
handler = getattr(self, handler_name)
handler(resp['result'], req_id)
self.req_status.pop(req_id)
if req_id in self.req_reply:
self.req_reply.pop(req_id)
except zmq.ZMQError as e:
self.notifier.setEnabled(True)
return
def perform_request(self, method, params):
if method in self.sender_registry:
handler_name = self.sender_registry[method]
handler = getattr(self, handler_name)
_id = handler(params)
if 'response_codeeditor' in params:
if params['requires_response']:
self.req_reply[_id] = params['response_codeeditor']
return _id
# ------ Spyder plugin registration --------------------------------
def register_plugin_type(self, plugin_type, notification_sig):
if plugin_type not in self.plugin_registry:
self.plugin_registry[plugin_type] = []
self.plugin_registry[plugin_type].append(notification_sig)
# ------ LSP initialization methods --------------------------------
@handles(SERVER_READY)
@send_request(method=LSPRequestTypes.INITIALIZE)
def initialize(self, *args, **kwargs):
params = {
'processId': self.transport_client.pid,
'rootUri': pathlib.Path(osp.abspath(self.folder)).as_uri(),
'capabilities': self.client_capabilites,
'trace': TRACE
}
return params
@send_request(method=LSPRequestTypes.SHUTDOWN)
def shutdown(self):
params = {}
return params
@handles(LSPRequestTypes.SHUTDOWN)
def handle_shutdown(self, response, *args):
self.ready_to_close = True
@send_request(method=LSPRequestTypes.EXIT, requires_response=False)
def exit(self):
params = {}
return params
@handles(LSPRequestTypes.INITIALIZE)
def process_server_capabilities(self, server_capabilites, *args):
self.send_plugin_configurations(self.plugin_configurations)
self.initialized = True
server_capabilites = server_capabilites['capabilities']
if isinstance(server_capabilites['textDocumentSync'], int):
kind = server_capabilites['textDocumentSync']
server_capabilites['textDocumentSync'] = TEXT_DOCUMENT_SYNC_OPTIONS
server_capabilites['textDocumentSync']['change'] = kind
if server_capabilites['textDocumentSync'] is None:
server_capabilites.pop('textDocumentSync')
self.server_capabilites.update(server_capabilites)
for sig in self.plugin_registry[LSPEventTypes.DOCUMENT]:
sig.emit(self.server_capabilites, self.language)
@send_request(method=LSPRequestTypes.WORKSPACE_CONFIGURATION_CHANGE,
requires_response=False)
def send_plugin_configurations(self, configurations, *args):
self.plugin_configurations = configurations
params = {
'settings': configurations
}
return params
class LSPClient(QObject, LSPMethodProviderMixIn):
"""Language Server Protocol v3.0 client implementation."""
# Signals
sig_initialize = Signal(dict, str)
# Constants
external_server_fmt = ('--server-host %(host)s '
'--server-port %(port)s '
'--external-server')
local_server_fmt = ('--server-host %(host)s '
'--server-port %(port)s '
'--server %(cmd)s')
def __init__(self,
parent,
server_settings={},
folder=getcwd_or_home(),
language='python'):
QObject.__init__(self)
# LSPMethodProviderMixIn.__init__(self)
self.manager = parent
self.zmq_in_socket = None
self.zmq_out_socket = None
self.zmq_in_port = None
self.zmq_out_port = None
self.transport_client = None
self.language = language
self.initialized = False
self.ready_to_close = False
self.request_seq = 1
self.req_status = {}
self.watched_files = {}
self.req_reply = {}
self.transport_args = [
sys.executable, '-u',
osp.join(LOCATION, 'transport', 'main.py')
]
self.external_server = server_settings.get('external', False)
self.folder = folder
self.plugin_configurations = server_settings.get('configurations', {})
self.client_capabilites = CLIENT_CAPABILITES
self.server_capabilites = SERVER_CAPABILITES
self.context = zmq.Context()
server_args_fmt = server_settings.get('args', '')
server_args = server_args_fmt.format(**server_settings)
# transport_args = self.local_server_fmt % (server_settings)
# if self.external_server:
transport_args = self.external_server_fmt % (server_settings)
self.server_args = [sys.executable, '-m', server_settings['cmd']]
self.server_args += server_args.split(' ')
self.transport_args += transport_args.split(' ')
self.transport_args += ['--folder', folder]
self.transport_args += ['--transport-debug', str(get_debug_level())]
def start(self):
self.zmq_out_socket = self.context.socket(zmq.PAIR)
self.zmq_out_port = self.zmq_out_socket.bind_to_random_port('tcp://*')
self.zmq_in_socket = self.context.socket(zmq.PAIR)
self.zmq_in_socket.set_hwm(0)
self.zmq_in_port = self.zmq_in_socket.bind_to_random_port('tcp://*')
self.transport_args += [
'--zmq-in-port', self.zmq_out_port, '--zmq-out-port',
self.zmq_in_port
]
server_log = subprocess.PIPE
if get_debug_level() > 0:
# Create server log file
server_log_fname = 'server_{0}.log'.format(self.language)
server_log_file = get_conf_path(
osp.join('lsp_logs', server_log_fname))
if not osp.exists(osp.dirname(server_log_file)):
os.makedirs(osp.dirname(server_log_file))
server_log = open(server_log_file, 'w')
# Start server with logging options
if get_debug_level() == 2:
self.server_args.append('-v')
elif get_debug_level() == 3:
self.server_args.append('-vv')
if not self.external_server:
logger.info('Starting server: {0}'.format(' '.join(
self.server_args)))
creation_flags = 0
if os.name == 'nt':
creation_flags = (subprocess.CREATE_NEW_PROCESS_GROUP
| 0x08000000) # CREATE_NO_WINDOW
if os.environ.get('CI') and os.name == 'nt':
# The following patching avoids:
#
# OSError: [WinError 6] The handle is invalid
#
# while running our tests in CI services on Windows
# (they run fine locally).
# See this comment for an explanation:
# https://stackoverflow.com/q/43966523/
# 438386#comment74964124_43966523
def patched_cleanup():
pass
subprocess._cleanup = patched_cleanup
self.lsp_server = subprocess.Popen(self.server_args,
stdout=server_log,
stderr=subprocess.STDOUT,
creationflags=creation_flags)
client_log = subprocess.PIPE
if get_debug_level() > 0:
# Client log file
client_log_fname = 'client_{0}.log'.format(self.language)
client_log_file = get_conf_path(
osp.join('lsp_logs', client_log_fname))
if not osp.exists(osp.dirname(client_log_file)):
os.makedirs(osp.dirname(client_log_file))
client_log = open(client_log_file, 'w')
new_env = dict(os.environ)
python_path = os.pathsep.join(sys.path)[1:]
new_env['PYTHONPATH'] = python_path
self.transport_args = list(map(str, self.transport_args))
logger.info('Starting transport: {0}'.format(' '.join(
self.transport_args)))
self.transport_client = subprocess.Popen(self.transport_args,
stdout=subprocess.PIPE,
stderr=client_log,
env=new_env)
fid = self.zmq_in_socket.getsockopt(zmq.FD)
self.notifier = QSocketNotifier(fid, QSocketNotifier.Read, self)
self.notifier.activated.connect(self.on_msg_received)
# This is necessary for tests to pass locally!
logger.debug('LSP {} client started!'.format(self.language))
def stop(self):
# self.shutdown()
# self.exit()
logger.info('Stopping {} client...'.format(self.language))
if self.notifier is not None:
self.notifier.activated.disconnect(self.on_msg_received)
self.notifier.setEnabled(False)
self.notifier = None
# if os.name == 'nt':
# self.transport_client.send_signal(signal.CTRL_BREAK_EVENT)
# else:
self.transport_client.kill()
self.context.destroy()
if not self.external_server:
self.lsp_server.kill()
def send(self, method, params, requires_response):
if ClientConstants.CANCEL in params:
return
msg = {'id': self.request_seq, 'method': method, 'params': params}
if requires_response:
self.req_status[self.request_seq] = method
logger.debug('{} request: {}'.format(self.language, method))
self.zmq_out_socket.send_pyobj(msg)
self.request_seq += 1
return int(msg['id'])
@Slot()
def on_msg_received(self):
self.notifier.setEnabled(False)
while True:
try:
# events = self.zmq_in_socket.poll(1500)
resp = self.zmq_in_socket.recv_pyobj(flags=zmq.NOBLOCK)
try:
method = resp['method']
logger.debug('{} response: {}'.format(
self.language, method))
except KeyError:
pass
if 'error' in resp:
logger.debug('{} Response error: {}'.format(
self.language, repr(resp['error'])))
if 'method' in resp:
if resp['method'][0] != '$':
if resp['method'] in self.handler_registry:
handler_name = (
self.handler_registry[resp['method']])
handler = getattr(self, handler_name)
handler(resp['params'])
if 'id' in resp:
self.request_seq = resp['id']
elif 'result' in resp:
if resp['result'] is not None:
req_id = resp['id']
if req_id in self.req_status:
req_type = self.req_status[req_id]
if req_type in self.handler_registry:
handler_name = self.handler_registry[req_type]
handler = getattr(self, handler_name)
handler(resp['result'], req_id)
self.req_status.pop(req_id)
if req_id in self.req_reply:
self.req_reply.pop(req_id)
except zmq.ZMQError:
self.notifier.setEnabled(True)
return
def perform_request(self, method, params):
if method in self.sender_registry:
handler_name = self.sender_registry[method]
handler = getattr(self, handler_name)
_id = handler(params)
if 'response_codeeditor' in params:
if params['requires_response']:
self.req_reply[_id] = params['response_codeeditor']
return _id
# ------ LSP initialization methods --------------------------------
@handles(SERVER_READY)
@send_request(method=LSPRequestTypes.INITIALIZE)
def initialize(self, *args, **kwargs):
params = {
'processId': self.transport_client.pid,
'rootUri': pathlib.Path(osp.abspath(self.folder)).as_uri(),
'capabilities': self.client_capabilites,
'trace': TRACE
}
return params
@send_request(method=LSPRequestTypes.SHUTDOWN)
def shutdown(self):
params = {}
return params
@handles(LSPRequestTypes.SHUTDOWN)
def handle_shutdown(self, response, *args):
self.ready_to_close = True
@send_request(method=LSPRequestTypes.EXIT, requires_response=False)
def exit(self):
params = {}
return params
@handles(LSPRequestTypes.INITIALIZE)
def process_server_capabilities(self, server_capabilites, *args):
self.send_plugin_configurations(self.plugin_configurations)
self.initialized = True
server_capabilites = server_capabilites['capabilities']
if isinstance(server_capabilites['textDocumentSync'], int):
kind = server_capabilites['textDocumentSync']
server_capabilites['textDocumentSync'] = TEXT_DOCUMENT_SYNC_OPTIONS
server_capabilites['textDocumentSync']['change'] = kind
if server_capabilites['textDocumentSync'] is None:
server_capabilites.pop('textDocumentSync')
self.server_capabilites.update(server_capabilites)
self.sig_initialize.emit(self.server_capabilites, self.language)
@send_request(method=LSPRequestTypes.WORKSPACE_CONFIGURATION_CHANGE,
requires_response=False)
def send_plugin_configurations(self, configurations, *args):
self.plugin_configurations = configurations
params = {'settings': configurations}
return params
class AsyncClient(QObject):
"""
A class which handles a connection to a client through a QProcess.
"""
# Emitted when the client has initialized.
initialized = Signal()
# Emitted when the client errors.
errored = Signal()
# Emitted when a request response is received.
received = Signal(object)
def __init__(self, target, executable=None, name=None,
extra_args=None, libs=None, cwd=None, env=None):
super(AsyncClient, self).__init__()
self.executable = executable or sys.executable
self.extra_args = extra_args
self.target = target
self.name = name or self
self.libs = libs
self.cwd = cwd
self.env = env
self.is_initialized = False
self.closing = False
self.context = zmq.Context()
QApplication.instance().aboutToQuit.connect(self.close)
# Set up the heartbeat timer.
self.timer = QTimer(self)
self.timer.timeout.connect(self._heartbeat)
def run(self):
"""Handle the connection with the server.
"""
# Set up the zmq port.
self.socket = self.context.socket(zmq.PAIR)
self.port = self.socket.bind_to_random_port('tcp://*')
# Set up the process.
self.process = QProcess(self)
if self.cwd:
self.process.setWorkingDirectory(self.cwd)
p_args = ['-u', self.target, str(self.port)]
if self.extra_args is not None:
p_args += self.extra_args
# Set up environment variables.
processEnvironment = QProcessEnvironment()
env = self.process.systemEnvironment()
if (self.env and 'PYTHONPATH' not in self.env) or DEV:
python_path = osp.dirname(get_module_path('spyderlib'))
# Add the libs to the python path.
for lib in self.libs:
try:
path = osp.dirname(imp.find_module(lib)[1])
python_path = osp.pathsep.join([python_path, path])
except ImportError:
pass
env.append("PYTHONPATH=%s" % python_path)
if self.env:
env.update(self.env)
for envItem in env:
envName, separator, envValue = envItem.partition('=')
processEnvironment.insert(envName, envValue)
self.process.setProcessEnvironment(processEnvironment)
# Start the process and wait for started.
self.process.start(self.executable, p_args)
self.process.finished.connect(self._on_finished)
running = self.process.waitForStarted()
if not running:
raise IOError('Could not start %s' % self)
# Set up the socket notifer.
fid = self.socket.getsockopt(zmq.FD)
self.notifier = QSocketNotifier(fid, QSocketNotifier.Read, self)
self.notifier.activated.connect(self._on_msg_received)
def request(self, func_name, *args, **kwargs):
"""Send a request to the server.
The response will be a dictionary the 'request_id' and the
'func_name' as well as a 'result' field with the object returned by
the function call or or an 'error' field with a traceback.
"""
if not self.is_initialized:
return
request_id = uuid.uuid4().hex
request = dict(func_name=func_name,
args=args,
kwargs=kwargs,
request_id=request_id)
self._send(request)
return request_id
def close(self):
"""Cleanly close the connection to the server.
"""
self.closing = True
self.is_initialized = False
self.timer.stop()
self.notifier.activated.disconnect(self._on_msg_received)
self.notifier.setEnabled(False)
del self.notifier
self.request('server_quit')
self.process.waitForFinished(1000)
self.process.close()
self.context.destroy()
def _on_finished(self):
"""Handle a finished signal from the process.
"""
if self.closing:
return
if self.is_initialized:
debug_print('Restarting %s' % self.name)
debug_print(self.process.readAllStandardOutput())
debug_print(self.process.readAllStandardError())
self.is_initialized = False
self.notifier.setEnabled(False)
self.run()
else:
debug_print('Errored %s' % self.name)
debug_print(self.process.readAllStandardOutput())
debug_print(self.process.readAllStandardError())
self.errored.emit()
def _on_msg_received(self):
"""Handle a message trigger from the socket.
"""
self.notifier.setEnabled(False)
while 1:
try:
resp = self.socket.recv_pyobj(flags=zmq.NOBLOCK)
except zmq.ZMQError:
self.notifier.setEnabled(True)
return
if not self.is_initialized:
self.is_initialized = True
debug_print('Initialized %s' % self.name)
self.initialized.emit()
self.timer.start(HEARTBEAT)
continue
resp['name'] = self.name
self.received.emit(resp)
def _heartbeat(self):
"""Send a heartbeat to keep the server alive.
"""
self._send(dict(func_name='server_heartbeat'))
def _send(self, obj):
"""Send an object to the server.
"""
try:
self.socket.send_pyobj(obj)
except Exception as e:
debug_print(e)
self.is_initialized = False
self._on_finished()
