def start_channels(self, shell=True, iopub=True, stdin=True, hb=True, control=True): """Starts the channels for this kernel. For this class, we establish a websocket connection to the destination and setup the channel-based queues on which applicable messages will be posted. """ ws_url = url_path_join(GatewayClient.instance().ws_url, GatewayClient.instance().kernels_endpoint, url_escape(self.kernel_id), 'channels') self.channel_socket = websocket.create_connection( ws_url, timeout=60, enable_multithread=True) self.response_router = Thread(target=self._route_responses) self.response_router.start() super().start_channels(shell=shell, iopub=iopub, stdin=stdin, hb=hb, control=control)
async def start_kernel(self, **kwargs): """Starts a kernel via HTTP in an asynchronous manner. Parameters ---------- `**kwargs` : optional keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ kernel_id = kwargs.get('kernel_id') if kernel_id is None: kernel_name = kwargs.get('kernel_name', 'python3') kernel_url = self._get_kernel_endpoint_url() self.log.debug("Request new kernel at: %s" % kernel_url) # Let KERNEL_USERNAME take precedent over http_user config option. if os.environ.get('KERNEL_USERNAME' ) is None and GatewayClient.instance().http_user: os.environ['KERNEL_USERNAME'] = GatewayClient.instance( ).http_user kernel_env = { k: v for (k, v) in dict(os.environ).items() if k.startswith('KERNEL_') or k in GatewayClient.instance().env_whitelist.split(",") } # Add any env entries in this request kernel_env.update(kwargs.get('env')) # Convey the full path to where this notebook file is located. if kwargs.get('cwd') is not None and kernel_env.get( 'KERNEL_WORKING_DIR') is None: kernel_env['KERNEL_WORKING_DIR'] = kwargs['cwd'] json_body = json_encode({'name': kernel_name, 'env': kernel_env}) response = await gateway_request(kernel_url, method='POST', body=json_body) self.kernel = json_decode(response.body) self.kernel_id = self.kernel['id'] self.log.info( "HTTPKernelManager started kernel: {}, args: {}".format( self.kernel_id, kwargs)) else: self.kernel = await self.get_kernel(kernel_id) self.kernel_id = self.kernel['id'] self.log.info("HTTPKernelManager using existing kernel: {}".format( self.kernel_id))
def process(self, operation: Operation): filepath = self.get_valid_filepath(operation.filename) file_dir = os.path.dirname(filepath) file_name = os.path.basename(filepath) self.log.debug(f'Processing notebook: {filepath}') # We'll always use the ElyraEngine. This engine is essentially the default Papermill engine # but allows for environment variables to be passed to the kernel process (via 'kernel_env'). # If the current notebook server is running with Enterprise Gateway configured, we will also # point the 'kernel_manager_class' to our HTTPKernelManager so that notebooks run as they # would outside of Elyra. Current working directory (cwd) is specified both for where papermill # runs the notebook (cwd) and where the directory of the kernel process (kernel_cwd). The latter # of which is important when EG is configured. additional_kwargs = dict() additional_kwargs['engine_name'] = "ElyraEngine" additional_kwargs[ 'cwd'] = file_dir # For local operations, papermill runs from this dir additional_kwargs['kernel_cwd'] = file_dir additional_kwargs['kernel_env'] = operation.env_vars_as_dict() if GatewayClient.instance().gateway_enabled: additional_kwargs[ 'kernel_manager_class'] = 'elyra.pipeline.http_kernel_manager.HTTPKernelManager' t0 = time.time() try: papermill.execute_notebook(filepath, filepath, **additional_kwargs) except Exception as ex: raise RuntimeError( f'Internal error executing {filepath}: {ex}') from ex t1 = time.time() duration = (t1 - t0) self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def run_cells(kernel_id, cid, cells): print(f"running {cid} on {kernel_id}") gw_client = GatewayClient.instance() url = url_path_join( gw_client.ws_url, gw_client.kernels_endpoint, url_escape(kernel_id), "channels", ) outputs = {} ws = create_connection(url) for idx, cell in enumerate(cells): if cell["cell_type"] == "code": ws.send( json_encode({ "header": { "username": cid, "version": "5.3", "session": "", "msg_id": uuid4().hex, "msg_type": "execute_request", }, "parent_header": {}, "channel": "shell", "content": { "code": cell["source"], "silent": False, "store_history": False, "user_expressions": {}, "allow_stdin": False, "stop_on_error": True, }, "metadata": {}, "buffers": {}, })) outputs[idx] = [] status = None while status is None or status == "busy": msg = ws.recv() msg = json_decode(msg) msg_type = msg["msg_type"] if msg_type == "status": status = msg["content"]["execution_state"] elif msg_type in ["stream", "display_data", "execute_result"]: # display_data/execute_result required fields: # "output_type", "data", "metadata" # stream required fields: "output_type", "name", "text" output = msg["content"] output.pop("transient", None) output["output_type"] = msg_type outputs[idx].append(output) elif msg_type == "error": tb = msg["content"]["traceback"] raise ValueError(tb) ws.close() return outputs
def get_kernel_endpoint(kernel_id=None, ws=False): gw_client = GatewayClient.instance() base_url = gw_client.ws_url if ws else gw_client.url base_endpoint = url_path_join(base_url, gw_client.kernels_endpoint) if isinstance(kernel_id, str) and kernel_id: return url_path_join(base_endpoint, kernel_id) return base_endpoint
def connect_kernel(): run_sync(manager.list_kernels()) kernels = { kernel_id: dateparser.parse(kernel["last_activity"]) for kernel_id, kernel in manager._kernels.items() } kernel_id = url_escape(sorted(kernels, key=kernels.get)[0]) client = GatewayClient.instance() url = url_path_join(client.ws_url, client.kernels_endpoint, kernel_id, "channels") ws_req = HTTPRequest(url=url) return run_sync(websocket_connect(ws_req))
async def start_channels(self, shell=True, iopub=True, stdin=True, hb=True, control=True): """Starts the channels for this kernel. For this class, we establish a websocket connection to the destination and setup the channel-based queues on which applicable messages will be posted. """ ws_url = url_path_join(GatewayClient.instance().ws_url, GatewayClient.instance().kernels_endpoint, url_escape(self.kernel_id), 'channels') # Gather cert info in case where ssl is desired... ssl_options = dict() ssl_options['ca_certs'] = GatewayClient.instance().ca_certs ssl_options['certfile'] = GatewayClient.instance().client_cert ssl_options['keyfile'] = GatewayClient.instance().client_key self.channel_socket = websocket.create_connection( ws_url, timeout=GatewayClient.instance().KERNEL_LAUNCH_TIMEOUT, enable_multithread=True, sslopt=ssl_options) self.response_router = Thread(target=self._route_responses) self.response_router.start() await maybe_future(super().start_channels(shell=shell, iopub=iopub, stdin=stdin, hb=hb, control=control))
def get_kernels(): """retrieve list of kernels from KernelGateway service""" gw_client = GatewayClient.instance() base_endpoint = url_path_join(gw_client.url, gw_client.kernels_endpoint) try: r = requests.get(base_endpoint) except ConnectionError: print("WARNING: Kernel Gateway NOT AVAILABLE") return None kernels = r.json() return {kernel["id"]: None for kernel in kernels}
async def execute_cells(cid, cells, loop=None): gw_client = GatewayClient.instance() base_endpoint = url_path_join(gw_client.url, gw_client.kernels_endpoint) response = await loop.run_in_executor(None, requests.get, base_endpoint) kernels = response.json() while 1: for kernel in kernels: if kernel["execution_state"] == "idle": kernel_id = kernel["id"] print(f"{kernel_id} AVAILABLE") return await loop.run_in_executor( None, run_cells, kernel_id, cid, cells ) else: print("WAITING for kernel") await asyncio.sleep(5, loop=loop)
def __init__(self, **kwargs): super().__init__(**kwargs) self.base_endpoint = url_path_join( GatewayClient.instance().url, GatewayClient.instance().kernels_endpoint) self.kernel = None
def teardown_class(cls): GatewayClient.clear_instance() super(TestGateway, cls).teardown_class()
def setup_class(cls): GatewayClient.clear_instance() super(TestGateway, cls).setup_class()
def setUp(self): kwargs = dict() GatewayClient.instance().load_connection_args(**kwargs) super(TestGateway, self).setUp()