def launch_process(self, kernel_cmd, **kw):
""" Launches the kernel process.
"""
super(ConductorClusterProcessProxy, self).launch_process(kernel_cmd, **kw)
# Get cred from process env
env_dict = dict(os.environ.copy())
if env_dict and 'EGO_SERVICE_CREDENTIAL' in env_dict:
self.rest_credential = env_dict['EGO_SERVICE_CREDENTIAL']
else:
error_message = "ConductorClusterProcessProxy failed to obtain the Conductor credential."
self.log_and_raise(http_status_code=500, reason=error_message)
# dynamically update Spark submit parameters
self.update_launch_info(kernel_cmd, **kw)
# Enable stderr PIPE for the run command
kw.update({'stderr': subprocess.PIPE})
self.local_proc = launch_kernel(kernel_cmd, **kw)
self.pid = self.local_proc.pid
self.ip = local_ip
self.env = kw.get('env')
self.log.debug("Conductor cluster kernel launched using Conductor endpoint: {}, pid: {}, Kernel ID: {}, cmd: '{}'"
.format(self.conductor_endpoint, self.local_proc.pid, self.kernel_id, kernel_cmd))
self.confirm_remote_startup(kernel_cmd, **kw)
return self
def launch_process(self, kernel_cmd, **kw):
super(LocalProcessProxy, self).launch_process(kernel_cmd, **kw)
# launch the local run.sh
self.local_proc = launch_kernel(kernel_cmd, **kw)
self.pid = self.local_proc.pid
if hasattr(os, "getpgid"):
try:
self.pgid = os.getpgid(self.pid)
except OSError:
pass
self.ip = local_ip
self.log.info("Local kernel launched on '{}', pid: {}, pgid: {}, KernelID: {}, cmd: '{}'"
.format(self.ip, self.pid, self.pgid, self.kernel_id, kernel_cmd))
return self
def launch_process(self, kernel_cmd, **kw):
super(LocalProcessProxy, self).launch_process(kernel_cmd, **kw)
# launch the local run.sh
self.local_proc = launch_kernel(kernel_cmd, **kw)
self.pid = self.local_proc.pid
if hasattr(os, "getpgid"):
try:
self.pgid = os.getpgid(self.pid)
except OSError:
pass
self.ip = local_ip
self.log.info(
"Local kernel launched on '{}', pid: {}, pgid: {}, KernelID: {}, cmd: '{}'"
.format(self.ip, self.pid, self.pgid, self.kernel_id, kernel_cmd))
return self
def launch_process(self, kernel_cmd, **kw):
""" Launches the Yarn process. Prior to invocation, connection files will be distributed to each applicable
Yarn node so that its in place when the kernel is started.
"""
super(YarnClusterProcessProxy, self).launch_process(kernel_cmd, **kw)
# launch the local run.sh - which is configured for yarn-cluster...
self.local_proc = launch_kernel(kernel_cmd, **kw)
self.pid = self.local_proc.pid
self.ip = local_ip
self.log.debug("Yarn cluster kernel launched using YARN endpoint: {}, pid: {}, Kernel ID: {}, cmd: '{}'"
.format(self.yarn_endpoint, self.local_proc.pid, self.kernel_id, kernel_cmd))
self.confirm_remote_startup(kernel_cmd, **kw)
return self
def launch_process(self, kernel_cmd, **kw):
""" Launches the Yarn process. Prior to invocation, connection files will be distributed to each applicable
Yarn node so that its in place when the kernel is started.
"""
super(YarnClusterProcessProxy, self).launch_process(kernel_cmd, **kw)
# launch the local run.sh - which is configured for yarn-cluster...
self.local_proc = launch_kernel(kernel_cmd, **kw)
self.pid = self.local_proc.pid
self.ip = local_ip
self.log.debug("Yarn cluster kernel launched using YARN endpoint: {}, pid: {}, Kernel ID: {}, cmd: '{}'"
.format(self.yarn_endpoint, self.local_proc.pid, self.kernel_id, kernel_cmd))
self.confirm_remote_startup(kernel_cmd, **kw)
return self
def launch_process(self, kernel_cmd, **kwargs):
# checks to see if the queue resource is available
# if not kernel startup is not tried
self.confirm_yarn_queue_availability(**kwargs)
"""Launches the specified process within a YARN cluster environment."""
super(YarnClusterProcessProxy, self).launch_process(kernel_cmd, **kwargs)
# launch the local run.sh - which is configured for yarn-cluster...
self.local_proc = launch_kernel(kernel_cmd, **kwargs)
self.pid = self.local_proc.pid
self.ip = local_ip
self.log.debug("Yarn cluster kernel launched using YARN RM address: {}, pid: {}, Kernel ID: {}, cmd: '{}'"
.format(self.rm_addr, self.local_proc.pid, self.kernel_id, kernel_cmd))
self.confirm_remote_startup()
return self
def launch_process(self, kernel_cmd, **kwargs):
"""Launches the specified process within a YARN cluster environment."""
super(YarnClusterProcessProxy,
self).launch_process(kernel_cmd, **kwargs)
# launch the local run.sh - which is configured for yarn-cluster...
self.local_proc = launch_kernel(kernel_cmd, **kwargs)
self.pid = self.local_proc.pid
self.ip = local_ip
self.log.debug(
"Yarn cluster kernel launched using YARN endpoint: {}, pid: {}, Kernel ID: {}, cmd: '{}'"
.format(self.yarn_endpoint, self.local_proc.pid, self.kernel_id,
kernel_cmd))
self.confirm_remote_startup()
return self
def launch_process(self, kernel_cmd, **kw):
# Set env before superclass call so we see these in the debug output
kw['env']['KERNEL_IMAGE'] = self.kernel_image
kw['env']['KERNEL_EXECUTOR_IMAGE'] = self.kernel_executor_image
super(ContainerProcessProxy, self).launch_process(kernel_cmd, **kw)
self.local_proc = launch_kernel(kernel_cmd, **kw)
self.pid = self.local_proc.pid
self.ip = local_ip
self.log.info("{}: kernel launched. Kernel image: {}, KernelID: {}, cmd: '{}'"
.format(self.__class__.__name__, self.kernel_image, self.kernel_id, kernel_cmd))
self.confirm_remote_startup(kernel_cmd, **kw)
return self
def launch_process(self, kernel_cmd, **kw):
""" Launches the kernel process. Prior to invocation, connection files will be distributed to each applicable
Conductor node so that its in place when the kernel is started. This step is skipped if pull or socket modes
are configured, which results in the kernel process determining ports and generating encoding key.
Once started, the method will poll the application (after discovering the application ID via the
kernel ID) until host is known. Note that this polling may timeout and result in a 503 Http error (Service
unavailable).
Once the host is determined the connection file is retrieved. If pull mode is configured, the remote file is
copied locally and member variables are loaded based on its contents. If socket mode is configured, the
kernel launcher sends the connection information - which is then written out upon its reception. If push
mode is configured, the kernel manager's IP is updated to the selected node.
"""
super(ConductorClusterProcessProxy,
self).launch_process(kernel_cmd, **kw)
# Get cred from process env
env_dict = dict(os.environ.copy())
if env_dict is not None and 'EGO_SERVICE_CREDENTIAL' in env_dict:
self.rest_credential = env_dict['EGO_SERVICE_CREDENTIAL']
else:
self.log.error(
"ConductorClusterProcessProxy failed to obtain the Conductor credential."
)
raise tornado.web.HTTPError(
500,
"ConductorClusterProcessProxy failed to obtain the Conductor credential."
)
# dynamically update Spark submit parameters
self.update_launch_info(kernel_cmd, **kw)
# launch the local run.sh - which is configured for Conductor-cluster...
self.local_proc = launch_kernel(kernel_cmd, **kw)
self.pid = self.local_proc.pid
self.ip = local_ip
self.env = kw.get('env')
self.log.debug(
"Conductor cluster kernel launched using Conductor endpoint: {}, pid: {}, Kernel ID: {}, cmd: '{}'"
.format(self.conductor_endpoint, self.local_proc.pid,
self.kernel_id, kernel_cmd))
self.confirm_remote_startup(kernel_cmd, **kw)
return self
def _launch_remote_process(self, kernel_cmd, **kwargs):
"""
Launch the kernel as indicated by the argv stanza in the kernelspec. Note that this method
will bypass use of ssh if the remote host is also the local machine.
"""
cmd = self._build_startup_command(kernel_cmd, **kwargs)
self.log.debug("Invoking cmd: '{}' on host: {}".format(cmd, self.assigned_host))
result_pid = 'bad_pid' # purposely initialize to bad int value
if BaseProcessProxyABC.ip_is_local(self.ip):
# launch the local command with redirection in place
self.local_proc = launch_kernel(cmd, stdout=open(self.kernel_log, mode='w'), stderr=STDOUT, **kwargs)
result_pid = str(self.local_proc.pid)
else:
# launch remote command via ssh
result = self.rsh(self.ip, cmd)
for line in result:
result_pid = line.strip()
return result_pid
def _launch_remote_process(self, kernel_cmd, **kw):
"""
Launch the kernel as indicated by the argv stanza in the kernelspec. Note that this method
will bypass use of ssh if the remote host is also the local machine.
"""
cmd = self._build_startup_command(kernel_cmd, **kw)
self.log.debug("Invoking cmd: '{}' on host: {}".format(cmd, self.assigned_host))
result_pid = 'bad_pid' # purposely initialize to bad int value
if BaseProcessProxyABC.ip_is_local(self.ip):
# launch the local command with redirection in place
self.local_proc = launch_kernel(cmd, stdout=open(self.kernel_log, mode='w'), stderr=STDOUT, **kw)
result_pid = str(self.local_proc.pid)
else:
# launch remote command via ssh
result = self.rsh(self.ip, cmd)
for line in result:
result_pid = line.strip()
return result_pid
def launch_process(self, kernel_cmd, **kwargs):
"""Launches the specified process within the container environment."""
# Set env before superclass call so we see these in the debug output
kwargs['env']['KERNEL_IMAGE'] = self.kernel_image
kwargs['env']['KERNEL_EXECUTOR_IMAGE'] = self.kernel_executor_image
super(ContainerProcessProxy, self).launch_process(kernel_cmd, **kwargs)
self.local_proc = launch_kernel(kernel_cmd, **kwargs)
self.pid = self.local_proc.pid
self.ip = local_ip
self.log.info(
"{}: kernel launched. Kernel image: {}, KernelID: {}, cmd: '{}'".
format(self.__class__.__name__, self.kernel_image, self.kernel_id,
kernel_cmd))
self.confirm_remote_startup()
return self
def launch_process(self, kernel_cmd, **kw):
""" Launches the Yarn process. Prior to invocation, connection files will be distributed to each applicable
Yarn node so that its in place when the kernel is started. This step is skipped if pull or socket modes
are configured, which results in the kernel process determining ports and generating encoding key.
Once started, the method will poll the Yarn application (after discovering the application ID via the
kernel ID) until host is known. Note that this polling may timeout and result in a 503 Http error (Service
unavailable).
Once the host is determined the connection file is retrieved. If pull mode is configured, the remote file is
copied locally and member variables are loaded based on its contents. If socket mode is configured, the
kernel launcher sends the connection information - which is then written out upon its reception. If push
mode is configured, the kernel manager's IP is updated to the selected node.
"""
super(YarnClusterProcessProxy, self).launch_process(kernel_cmd, **kw)
# launch the local run.sh - which is configured for yarn-cluster...
self.local_proc = launch_kernel(kernel_cmd, **kw)
self.pid = self.local_proc.pid
self.ip = local_ip
self.log.debug("Yarn cluster kernel launched using YARN endpoint: {}, pid: {}, Kernel ID: {}, cmd: '{}'"
.format(self.yarn_endpoint, self.local_proc.pid, self.kernel_id, kernel_cmd))
self.confirm_remote_startup(kernel_cmd, **kw)
return self
def launch_process(self, kernel_cmd, **kwargs):
"""Launches the specified process within a Conductor cluster environment."""
super(ConductorClusterProcessProxy, self).launch_process(kernel_cmd, **kwargs)
# Get cred from process env
env_dict = dict(os.environ.copy())
if env_dict and 'EGO_SERVICE_CREDENTIAL' in env_dict:
self.rest_credential = env_dict['EGO_SERVICE_CREDENTIAL']
else:
error_message = "ConductorClusterProcessProxy failed to obtain the Conductor credential."
self.log_and_raise(http_status_code=500, reason=error_message)
# dynamically update Spark submit parameters
self._update_launch_info(kernel_cmd, **kwargs)
# Enable stderr PIPE for the run command
kwargs.update({'stderr': subprocess.PIPE})
self.local_proc = launch_kernel(kernel_cmd, **kwargs)
self.pid = self.local_proc.pid
self.ip = local_ip
self.env = kwargs.get('env')
self.log.debug("Conductor cluster kernel launched using Conductor endpoint: {}, pid: {}, Kernel ID: {}, "
"cmd: '{}'".format(self.conductor_endpoint, self.local_proc.pid, self.kernel_id, kernel_cmd))
self.confirm_remote_startup()
return self
def _launch_kernel(self, kernel_cmd, **kw):
"""actually launch the kernel
override in a subclass to launch kernel subprocesses differently
"""
return launch_kernel(kernel_cmd, **kw)
def _launch_kernel(self, kernel_cmd, **kw):
"""actually launch the kernel
override in a subclass to launch kernel subprocesses differently
"""
return launch_kernel(kernel_cmd, **kw)
async def _async_launch_kernel(self, kernel_cmd: t.List[str], **kw) -> Popen:
"""actually launch the kernel
override in a subclass to launch kernel subprocesses differently
"""
return launch_kernel(kernel_cmd, **kw)