def setup_server_listener(conn_filename, parent_pid, lower_port, upper_port, response_addr, kernel_id, public_key): ip = "0.0.0.0" key = str(uuid.uuid4()).encode() # convert to bytes ports = _select_ports(5, lower_port, upper_port) write_connection_file(fname=conn_filename, ip=ip, key=key, shell_port=ports[0], iopub_port=ports[1], stdin_port=ports[2], hb_port=ports[3], control_port=ports[4]) if response_addr: comm_socket = return_connection_info(conn_filename, response_addr, int(lower_port), int(upper_port), kernel_id, public_key, int(parent_pid)) if comm_socket: # socket in use, start server listener thread server_listener_thread = Thread(target=server_listener, args=( comm_socket, int(parent_pid), )) server_listener_thread.start() return
def setup_gateway_listener(conn_filename, parent_pid, lower_port, upper_port, response_addr, kernel_id, public_key): ip = "0.0.0.0" key = str_to_bytes(str(uuid.uuid4())) ports = _select_ports(5, lower_port, upper_port) write_connection_file(fname=conn_filename, ip=ip, key=key, shell_port=ports[0], iopub_port=ports[1], stdin_port=ports[2], hb_port=ports[3], control_port=ports[4]) if response_addr: gateway_socket = return_connection_info(conn_filename, response_addr, int(lower_port), int(upper_port), kernel_id, public_key, int(parent_pid)) if gateway_socket: # socket in use, start gateway listener thread gateway_listener_thread = Thread(target=gateway_listener, args=( gateway_socket, int(parent_pid), )) gateway_listener_thread.start() return
def test_write_connection_file(): with TemporaryDirectory() as d: cf = os.path.join(d, 'kernel.json') connect.write_connection_file(cf, **sample_info) assert os.path.exists(cf) with open(cf, 'r') as f: info = json.load(f) info['key'] = str_to_bytes(info['key']) assert info == sample_info
def test_write_connection_file(): with TemporaryDirectory() as d: cf = os.path.join(d, 'kernel.json') connect.write_connection_file(cf, **sample_info) nt.assert_true(os.path.exists(cf)) with open(cf, 'r') as f: info = json.load(f) info['key'] = str_to_bytes(info['key']) nt.assert_equal(info, sample_info)
def test_write_connection_file(): with TemporaryDirectory() as d: cf = os.path.join(d, "kernel.json") connect.write_connection_file(cf, **sample_info) assert os.path.exists(cf) with open(cf, "r") as f: info = json.load(f) info["key"] = info["key"].encode() assert info == sample_info
def test_app_load_connection_file(): """test `ipython console --existing` loads a connection file""" with TemporaryDirectory() as d: cf = os.path.join(d, 'kernel.json') connect.write_connection_file(cf, **sample_info) app = DummyConsoleApp(connection_file=cf) app.initialize(argv=[]) for attr, expected in sample_info.items(): if attr in ('key', 'signature_scheme'): continue value = getattr(app, attr) assert value == expected, "app.%s = %s != %s" % (attr, value, expected)
def test_app_load_connection_file(): """test `ipython console --existing` loads a connection file""" with TemporaryDirectory() as d: cf = os.path.join(d, 'kernel.json') connect.write_connection_file(cf, **sample_info) app = DummyConsoleApp(connection_file=cf) app.initialize(argv=[]) for attr, expected in sample_info.items(): if attr in ('key', 'signature_scheme'): continue value = getattr(app, attr) nt.assert_equal(value, expected, "app.%s = %s != %s" % (attr, value, expected))
def test_get_connection_info(): with TemporaryDirectory() as d: cf = os.path.join(d, 'kernel.json') connect.write_connection_file(cf, **sample_info) json_info = connect.get_connection_info(cf) info = connect.get_connection_info(cf, unpack=True) nt.assert_equal(type(json_info), type("")) nt.assert_equal(info, sample_info) info2 = json.loads(json_info) info2['key'] = str_to_bytes(info2['key']) nt.assert_equal(info2, sample_info)
def test_load_connection_file_session_with_kn(): """test load_connection_file() after """ session = Session() app = DummyConsoleApp(session=Session()) app.initialize(argv=[]) session = app.session with TemporaryDirectory() as d: cf = os.path.join(d, 'kernel.json') connect.write_connection_file(cf, **sample_info_kn) app.connection_file = cf app.load_connection_file() assert session.key == sample_info_kn['key'] assert session.signature_scheme == sample_info_kn['signature_scheme']
def test_load_connection_file_session(): """test load_connection_file() after """ session = Session() app = DummyConsoleApp(session=Session()) app.initialize(argv=[]) session = app.session with TemporaryDirectory() as d: cf = os.path.join(d, 'kernel.json') connect.write_connection_file(cf, **sample_info) app.connection_file = cf app.load_connection_file() nt.assert_equal(session.key, sample_info['key']) nt.assert_equal(session.signature_scheme, sample_info['signature_scheme'])
def setup_gateway_listener(fname, parent_pid, lower_port, upper_port): ip = "0.0.0.0" key = str_to_bytes(str(uuid.uuid4())) gateway_socket = prepare_gateway_socket(lower_port, upper_port) gateway_listener_thread = Thread(target=gateway_listener, args=( gateway_socket, parent_pid, )) gateway_listener_thread.start() basename = os.path.splitext(os.path.basename(fname))[0] fd, conn_file = tempfile.mkstemp(suffix=".json", prefix=basename + "_") os.close(fd) ports = _select_ports(5, lower_port, upper_port) conn_file, config = write_connection_file(fname=conn_file, ip=ip, key=key, shell_port=ports[0], iopub_port=ports[1], stdin_port=ports[2], hb_port=ports[3], control_port=ports[4]) try: os.remove(conn_file) except: pass # Add in the gateway_socket and parent_pid fields... config['comm_port'] = gateway_socket.getsockname()[1] config['pid'] = parent_pid with open(fname, 'w') as f: f.write(json.dumps(config, indent=2)) return
if create_spark_context: spark = WaitingForSparkSessionToBeInitialized(global_variable_name='spark') sc = WaitingForSparkSessionToBeInitialized(global_variable_name='sc') sqlContext = WaitingForSparkSessionToBeInitialized(global_variable_name='sqlContext') sqlCtx = WaitingForSparkSessionToBeInitialized(global_variable_name='sqlCtx') thread_to_initialize_spark_session = Thread(target=initialize_spark_session) # If the connection file doesn't exist, then create it. if not os.path.isfile(connection_file): key = str_to_bytes(str(uuid.uuid4())) connection_file = determine_connection_file(connection_file) ports = _select_ports(5, lower_port, upper_port) write_connection_file(fname=connection_file, ip=ip, key=key, shell_port=ports[0], iopub_port=ports[1], stdin_port=ports[2], hb_port=ports[3], control_port=ports[4]) if response_addr: gateway_socket = return_connection_info(connection_file, response_addr, disable_gateway_socket, lower_port, upper_port) if gateway_socket: # socket in use, start gateway listener thread gateway_listener_thread = Thread(target=gateway_listener, args=(gateway_socket,)) gateway_listener_thread.start() # start to initialize the Spark session in the background if create_spark_context: thread_to_initialize_spark_session.start() # launch the IPython kernel instance embed_kernel(connection_file=connection_file, ip=ip) try:
dest='disable_gateway_socket', action='store_true', help='Disable use of gateway socket for extended communications', default=False) arguments = vars(parser.parse_args()) connection_file = arguments['connection_file'] response_addr = arguments['response_address'] disable_gateway_socket = arguments['disable_gateway_socket'] ip = "0.0.0.0" # If the connection file doesn't exist, then create it. if not os.path.isfile(connection_file): key = str_to_bytes(str(uuid.uuid4())) connection_file = determine_connection_file(connection_file) write_connection_file(fname=connection_file, ip=ip, key=key) if response_addr: gateway_socket = return_connection_info(connection_file, ip, response_addr, disable_gateway_socket) if gateway_socket: # socket in use, start gateway listener thread gateway_listener_thread = Thread(target=gateway_listener, args=(gateway_socket, )) gateway_listener_thread.start() # start to initialize the Spark session in the background thread_to_initialize_spark_session.start() # launch the IPython kernel instance embed_kernel(connection_file=connection_file, ip=ip)
raise RuntimeError( "At least one of the parameters: 'connection_file' or " "'--RemoteProcessProxy.kernel-id' must be provided!") # If the connection file doesn't exist, then create it. if (connection_file and not os.path.isfile(connection_file)) or kernel_id is not None: key = str_to_bytes(str(uuid.uuid4())) connection_file = determine_connection_file(connection_file, kernel_id) ports = _select_ports(5, lower_port, upper_port) write_connection_file(fname=connection_file, ip=ip, key=key, shell_port=ports[0], iopub_port=ports[1], stdin_port=ports[2], hb_port=ports[3], control_port=ports[4]) if response_addr: gateway_socket = return_connection_info(connection_file, response_addr, lower_port, upper_port) if gateway_socket: # socket in use, start gateway listener thread gateway_listener_process = Process(target=gateway_listener, args=( gateway_socket, os.getpid(), )) gateway_listener_process.start()
def initialize(self): relay_host = self.remote_connection_file['relayHost'] relay_port = self.remote_connection_file['relayPort'] signaling_port = self.remote_connection_file['signalPort'] # generate the local file from jupyter_client.connect import write_connection_file local_connection_file_name, local_connection_file = write_connection_file() local_connection_file['remoteHost'] = socket.gethostname() local_connection_file['ip'] = '0.0.0.0' local_connection_file['key'] = self.remote_connection_file['key'] # otherwise no auth logging.info('Use connection file %s' % json.dumps(local_connection_file)) with open(local_connection_file_name, 'w') as f: f.write(json.dumps(local_connection_file)) # notify the launcher context = zmq.Context() self.signaling_socket = context.socket(zmq.SUB) self.signaling_socket.setsockopt(zmq.SUBSCRIBE, b'') self.signaling_socket.connect('tcp://%s:%s' % (relay_host, signaling_port)) self.callback_socket = context.socket(zmq.REQ) self.callback_socket.connect('tcp://%s:%s' % (relay_host, relay_port)) self.callback_socket.send(json.dumps(local_connection_file).encode('utf8')) message = self.callback_socket.recv() logging.info("Got %s" % message) # proceed with heartbeating heart_failure = False hb_thread = threading.Thread(name="forwarder-watcher", target=self.hb_forwarder) hb_thread.daemon = True hb_thread.start() signal_thread = threading.Thread(name="signaling-watcher", target=self.signaling_handler) signal_thread.daemon = True signal_thread.start() # start relaying the sockets in the connection file port_pairs = [] for port_type in ['shell_port', 'iopub_port', 'stdin_port', 'control_port', 'hb_port']: local_port = local_connection_file.get(port_type, None) remote_port = self.remote_connection_file.get(port_type, None) if local_port is None or local_port == 0: continue if remote_port is None or remote_port == 0: continue port_pairs.append([local_port, remote_port, port_type[:-5]]) def printout(x): logging.info(x) # bind on 127.0.0.1 for the jupyter-server-facing side and on all interfaces for the kernel-facing side def forward_ROUTER_DEALER(local_port, remote_port, port_type): return ROUTER_DEALER_Forwarder(relay_host, remote_port, '127.0.0.1', local_port, port_type, printout, False, False) def forward_PUB_SUB(local_port, remote_port, port_type): return PUB_SUB_Forwarder(relay_host, remote_port, '127.0.0.1', local_port, port_type, printout, False, False) def forward_REP_REQ(local_port, remote_port, port_type): return REQ_REP_Forwarder(relay_host, remote_port, '127.0.0.1', local_port, port_type, printout, False, False) socket_forwarders = {'hb' : forward_REP_REQ, 'shell' : forward_ROUTER_DEALER, 'iopub' : forward_PUB_SUB, 'stdin' : forward_ROUTER_DEALER, 'control': forward_ROUTER_DEALER} for port_pair in port_pairs: local_port = port_pair[0] remote_port = port_pair[1] port_type = port_pair[2] logging.info("Relay port %s to %s on type %s" % (local_port, remote_port, port_type)) socket_forwarders[port_type](local_port, remote_port, port_type) return local_connection_file_name