def start_channels(self,
shell=True,
iopub=True,
stdin=True,
hb=True,
control=True):
"""Starts the channels for this kernel.
For this class, we establish a websocket connection to the destination
and setup the channel-based queues on which applicable messages will
be posted.
"""
ws_url = url_path_join(GatewayClient.instance().ws_url,
GatewayClient.instance().kernels_endpoint,
url_escape(self.kernel_id), 'channels')
self.channel_socket = websocket.create_connection(
ws_url, timeout=60, enable_multithread=True)
self.response_router = Thread(target=self._route_responses)
self.response_router.start()
super().start_channels(shell=shell,
iopub=iopub,
stdin=stdin,
hb=hb,
control=control)
async def start_kernel(self, **kwargs):
"""Starts a kernel via HTTP in an asynchronous manner.
Parameters
----------
`**kwargs` : optional
keyword arguments that are passed down to build the kernel_cmd
and launching the kernel (e.g. Popen kwargs).
"""
kernel_id = kwargs.get('kernel_id')
if kernel_id is None:
kernel_name = kwargs.get('kernel_name', 'python3')
kernel_url = self._get_kernel_endpoint_url()
self.log.debug("Request new kernel at: %s" % kernel_url)
# Let KERNEL_USERNAME take precedent over http_user config option.
if os.environ.get('KERNEL_USERNAME'
) is None and GatewayClient.instance().http_user:
os.environ['KERNEL_USERNAME'] = GatewayClient.instance(
).http_user
kernel_env = {
k: v
for (k, v) in dict(os.environ).items()
if k.startswith('KERNEL_')
or k in GatewayClient.instance().env_whitelist.split(",")
}
# Add any env entries in this request
kernel_env.update(kwargs.get('env'))
# Convey the full path to where this notebook file is located.
if kwargs.get('cwd') is not None and kernel_env.get(
'KERNEL_WORKING_DIR') is None:
kernel_env['KERNEL_WORKING_DIR'] = kwargs['cwd']
json_body = json_encode({'name': kernel_name, 'env': kernel_env})
response = await gateway_request(kernel_url,
method='POST',
body=json_body)
self.kernel = json_decode(response.body)
self.kernel_id = self.kernel['id']
self.log.info(
"HTTPKernelManager started kernel: {}, args: {}".format(
self.kernel_id, kwargs))
else:
self.kernel = await self.get_kernel(kernel_id)
self.kernel_id = self.kernel['id']
self.log.info("HTTPKernelManager using existing kernel: {}".format(
self.kernel_id))
def process(self, operation: Operation):
filepath = self.get_valid_filepath(operation.filename)
file_dir = os.path.dirname(filepath)
file_name = os.path.basename(filepath)
self.log.debug(f'Processing notebook: {filepath}')
# We'll always use the ElyraEngine. This engine is essentially the default Papermill engine
# but allows for environment variables to be passed to the kernel process (via 'kernel_env').
# If the current notebook server is running with Enterprise Gateway configured, we will also
# point the 'kernel_manager_class' to our HTTPKernelManager so that notebooks run as they
# would outside of Elyra. Current working directory (cwd) is specified both for where papermill
# runs the notebook (cwd) and where the directory of the kernel process (kernel_cwd). The latter
# of which is important when EG is configured.
additional_kwargs = dict()
additional_kwargs['engine_name'] = "ElyraEngine"
additional_kwargs[
'cwd'] = file_dir # For local operations, papermill runs from this dir
additional_kwargs['kernel_cwd'] = file_dir
additional_kwargs['kernel_env'] = operation.env_vars_as_dict()
if GatewayClient.instance().gateway_enabled:
additional_kwargs[
'kernel_manager_class'] = 'elyra.pipeline.http_kernel_manager.HTTPKernelManager'
t0 = time.time()
try:
papermill.execute_notebook(filepath, filepath, **additional_kwargs)
except Exception as ex:
raise RuntimeError(
f'Internal error executing {filepath}: {ex}') from ex
t1 = time.time()
duration = (t1 - t0)
self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def run_cells(kernel_id, cid, cells):
print(f"running {cid} on {kernel_id}")
gw_client = GatewayClient.instance()
url = url_path_join(
gw_client.ws_url,
gw_client.kernels_endpoint,
url_escape(kernel_id),
"channels",
)
outputs = {}
ws = create_connection(url)
for idx, cell in enumerate(cells):
if cell["cell_type"] == "code":
ws.send(
json_encode({
"header": {
"username": cid,
"version": "5.3",
"session": "",
"msg_id": uuid4().hex,
"msg_type": "execute_request",
},
"parent_header": {},
"channel": "shell",
"content": {
"code": cell["source"],
"silent": False,
"store_history": False,
"user_expressions": {},
"allow_stdin": False,
"stop_on_error": True,
},
"metadata": {},
"buffers": {},
}))
outputs[idx] = []
status = None
while status is None or status == "busy":
msg = ws.recv()
msg = json_decode(msg)
msg_type = msg["msg_type"]
if msg_type == "status":
status = msg["content"]["execution_state"]
elif msg_type in ["stream", "display_data", "execute_result"]:
# display_data/execute_result required fields:
# "output_type", "data", "metadata"
# stream required fields: "output_type", "name", "text"
output = msg["content"]
output.pop("transient", None)
output["output_type"] = msg_type
outputs[idx].append(output)
elif msg_type == "error":
tb = msg["content"]["traceback"]
raise ValueError(tb)
ws.close()
return outputs
def get_kernel_endpoint(kernel_id=None, ws=False):
gw_client = GatewayClient.instance()
base_url = gw_client.ws_url if ws else gw_client.url
base_endpoint = url_path_join(base_url, gw_client.kernels_endpoint)
if isinstance(kernel_id, str) and kernel_id:
return url_path_join(base_endpoint, kernel_id)
return base_endpoint
def connect_kernel():
run_sync(manager.list_kernels())
kernels = {
kernel_id: dateparser.parse(kernel["last_activity"])
for kernel_id, kernel in manager._kernels.items()
}
kernel_id = url_escape(sorted(kernels, key=kernels.get)[0])
client = GatewayClient.instance()
url = url_path_join(client.ws_url, client.kernels_endpoint, kernel_id,
"channels")
ws_req = HTTPRequest(url=url)
return run_sync(websocket_connect(ws_req))
async def start_channels(self,
shell=True,
iopub=True,
stdin=True,
hb=True,
control=True):
"""Starts the channels for this kernel.
For this class, we establish a websocket connection to the destination
and setup the channel-based queues on which applicable messages will
be posted.
"""
ws_url = url_path_join(GatewayClient.instance().ws_url,
GatewayClient.instance().kernels_endpoint,
url_escape(self.kernel_id), 'channels')
# Gather cert info in case where ssl is desired...
ssl_options = dict()
ssl_options['ca_certs'] = GatewayClient.instance().ca_certs
ssl_options['certfile'] = GatewayClient.instance().client_cert
ssl_options['keyfile'] = GatewayClient.instance().client_key
self.channel_socket = websocket.create_connection(
ws_url,
timeout=GatewayClient.instance().KERNEL_LAUNCH_TIMEOUT,
enable_multithread=True,
sslopt=ssl_options)
self.response_router = Thread(target=self._route_responses)
self.response_router.start()
await maybe_future(super().start_channels(shell=shell,
iopub=iopub,
stdin=stdin,
hb=hb,
control=control))
def get_kernels():
"""retrieve list of kernels from KernelGateway service"""
gw_client = GatewayClient.instance()
base_endpoint = url_path_join(gw_client.url, gw_client.kernels_endpoint)
try:
r = requests.get(base_endpoint)
except ConnectionError:
print("WARNING: Kernel Gateway NOT AVAILABLE")
return None
kernels = r.json()
return {kernel["id"]: None for kernel in kernels}
async def execute_cells(cid, cells, loop=None):
gw_client = GatewayClient.instance()
base_endpoint = url_path_join(gw_client.url, gw_client.kernels_endpoint)
response = await loop.run_in_executor(None, requests.get, base_endpoint)
kernels = response.json()
while 1:
for kernel in kernels:
if kernel["execution_state"] == "idle":
kernel_id = kernel["id"]
print(f"{kernel_id} AVAILABLE")
return await loop.run_in_executor(
None, run_cells, kernel_id, cid, cells
)
else:
print("WAITING for kernel")
await asyncio.sleep(5, loop=loop)
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.base_endpoint = url_path_join(
GatewayClient.instance().url,
GatewayClient.instance().kernels_endpoint)
self.kernel = None
def setUp(self):
kwargs = dict()
GatewayClient.instance().load_connection_args(**kwargs)
super(TestGateway, self).setUp()
