def disambiguate_ip_address(ip, location=None): """turn multi-ip interfaces '0.0.0.0' and '*' into a connectable address Explicit IP addresses are returned unmodified. Parameters ---------- ip : IP address An IP address, or the special values 0.0.0.0, or * location: IP address, optional A public IP of the target machine. If location is an IP of the current machine, localhost will be returned, otherwise location will be returned. """ if ip in {'0.0.0.0', '*'}: if not location: # unspecified location, localhost is the only choice ip = localhost() elif is_public_ip(location): # location is a public IP on this machine, use localhost ip = localhost() elif not public_ips(): # this machine's public IPs cannot be determined, # assume `location` is not this machine warnings.warn("IPython could not determine public IPs", RuntimeWarning) ip = location else: # location is not this machine, do not use loopback ip = location return ip
def test_disambiguate_ip(): # garbage in, garbage out public_ip = public_ips()[0] assert util.disambiguate_ip_address('garbage') == 'garbage' assert util.disambiguate_ip_address('0.0.0.0', socket.gethostname()) == localhost() wontresolve = 'this.wontresolve.dns' assert util.disambiguate_ip_address('0.0.0.0', wontresolve) == wontresolve assert util.disambiguate_ip_address('0.0.0.0', public_ip) == localhost()
def test_disambiguate_ip(warn_mock): # garbage in, garbage out public_ip = public_ips()[0] assert util.disambiguate_ip_address('garbage') == 'garbage' assert util.disambiguate_ip_address('0.0.0.0', socket.gethostname()) == localhost() wontresolve = 'this.wontresolve.dns' assert util.disambiguate_ip_address('0.0.0.0', wontresolve) == wontresolve assert warn_mock.called_once_with( 'IPython could not determine IPs for {}: ' '[Errno -2] Name or service not known'.format(wontresolve), RuntimeWarning) assert util.disambiguate_ip_address('0.0.0.0', public_ip) == localhost()
def init_ssh(self): """set up ssh tunnels, if needed.""" if not self.existing or (not self.sshserver and not self.sshkey): return self.load_connection_file() transport = self.transport ip = self.ip if transport != 'tcp': self.log.error("Can only use ssh tunnels with TCP sockets, \ not %s", transport) sys.exit(-1) if self.sshkey and not self.sshserver: # specifying just the key implies that we are connecting directly self.sshserver = ip ip = localhost() # build connection dict for tunnels: info = dict( ip=ip, shell_port=self.shell_port, iopub_port=self.iopub_port, stdin_port=self.stdin_port, hb_port=self.hb_port ) self.log.info("Forwarding connections \ to %s via %s" % (ip, self.sshserver)) # tunnels return a new set of ports, which will be on localhost: self.ip = localhost() try: newports = tunnel_to_kernel(info, self.sshserver, self.sshkey) except IOError: # even catch KeyboardInterrupt self.log.error("Could not setup tunnels", exc_info=True) self.exit(1) self.shell_port, self.iopub_port, \ self.stdin_port, self.hb_port = newports cf = self.connection_file root, ext = os.path.splitext(cf) self.connection_file = root + '-ssh' + ext self.write_connection_file() # write the new connection file self.log.info("To connect another client via this tunnel, use:") self.log.info("--existing %s" % os.path.basename(self.connection_file))
def test_stop(self, mocked_logger): url = 'tcp://{}:20201'.format(localhost()) watcher = ema.start_logwatcher(url) watcher.stop() time.sleep(3) mocked_logger.warning.assert_called_once_with('shutting down log watcher')
def get_free_port(): """Find a free port on the local machine.""" sock = socket.socket() sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, b'\0' * 8) sock.bind((localhost(), 0)) port = sock.getsockname()[1] sock.close() return port
def start_heartbeat(self, hb_ping, hb_pong, hb_period, identity): """Start our heart beating""" hb_monitor = None if self.max_heartbeat_misses > 0: # Add a monitor socket which will record the last time a ping was seen mon = self.context.socket(zmq.SUB) if self.curve_serverkey: mon.setsockopt(zmq.CURVE_SERVER, 1) mon.setsockopt(zmq.CURVE_SECRETKEY, self.curve_secretkey) mport = mon.bind_to_random_port('tcp://%s' % localhost()) mon.setsockopt(zmq.SUBSCRIBE, b"") self._hb_listener = zmqstream.ZMQStream(mon, self.loop) self._hb_listener.on_recv(self._report_ping) hb_monitor = "tcp://%s:%i" % (localhost(), mport) heart = Heart( hb_ping, hb_pong, hb_monitor, heart_id=identity, curve_serverkey=self.curve_serverkey, curve_secretkey=self.curve_secretkey, curve_publickey=self.curve_publickey, ) heart.start() # periodically check the heartbeat pings of the controller # Should be started here and not in "start()" so that the right period can be taken # from the hubs HeartBeatMonitor.period if self.max_heartbeat_misses > 0: # Use a slightly bigger check period than the hub signal period to not warn unnecessary self.hb_check_period = hb_period + 500 self.log.info( "Starting to monitor the heartbeat signal from the hub every %i ms.", self.hb_check_period, ) self._hb_reporter = ioloop.PeriodicCallback( self._hb_monitor, self.hb_check_period ) self._hb_reporter.start() else: self.log.info( "Monitoring of the heartbeat signal from the hub is not enabled." )
def setUpClass(cls): logger = ema_logging.get_logger() mocked_logger = mock.Mock(spec=logger) mocked_logger.handlers = [] ema_logging._logger = mocked_logger cls.client = ipyparallel.Client(profile='default') cls.url = 'tcp://{}:20202'.format(localhost()) cls.watcher = ema.start_logwatcher(cls.url)
def save_connection_dict(self, fname, cdict): """save a connection dict to json file.""" c = self.config url = cdict['registration'] location = cdict['location'] if not location: if public_ips(): location = public_ips()[-1] else: self.log.warn("Could not identify this machine's IP, assuming %s." " You may need to specify '--location=<external_ip_address>' to help" " IPython decide when to connect via loopback." % localhost() ) location = localhost() cdict['location'] = location fname = os.path.join(self.profile_dir.security_dir, fname) self.log.info("writing connection info to %s", fname) with open(fname, 'w') as f: f.write(json.dumps(cdict, indent=2)) os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)
def __init__(self, context, addr=None): if addr is None: addr = ('tcp', localhost(), 0) Thread.__init__(self) self.context = context self.transport, self.ip, self.port = addr self.original_port = self.port if self.original_port == 0: self.pick_port() self.addr = (self.ip, self.port) self.daemon = True
def save_connection_dict(self, fname, cdict): """save a connection dict to json file.""" c = self.config url = cdict['registration'] location = cdict['location'] if not location: if public_ips(): location = public_ips()[-1] else: self.log.warn( "Could not identify this machine's IP, assuming %s." " You may need to specify '--location=<external_ip_address>' to help" " IPython decide when to connect via loopback." % localhost()) location = localhost() cdict['location'] = location fname = os.path.join(self.profile_dir.security_dir, fname) self.log.info("writing connection info to %s", fname) with open(fname, 'w') as f: f.write(json.dumps(cdict, indent=2)) os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR)
def __init__(self, context, addr=None): if addr is None: addr = ('tcp', localhost(), 0) Thread.__init__(self, name="Heartbeat") self.context = context self.transport, self.ip, self.port = addr self.original_port = self.port if self.original_port == 0: self.pick_port() self.addr = (self.ip, self.port) self.daemon = True self.pydev_do_not_trace = True self.is_pydev_daemon_thread = True self.name = "Heartbeat"
def __init__(self, context, addr=None): if addr is None: addr = ('tcp', localhost(), 0) Thread.__init__(self) self.context = context self.transport, self.ip, self.port = addr if self.port == 0: if addr[0] == 'tcp': s = socket.socket() # '*' means all interfaces to 0MQ, which is '' to socket.socket s.bind(('' if self.ip == '*' else self.ip, 0)) self.port = s.getsockname()[1] s.close() elif addr[0] == 'ipc': self.port = 1 while os.path.exists("%s-%s" % (self.ip, self.port)): self.port = self.port + 1 else: raise ValueError("Unrecognized zmq transport: %s" % addr[0]) self.addr = (self.ip, self.port) self.daemon = True
async def test_tcp_cinfo(self): async with self._get_tcp_km() as km: await self._run_cinfo(km, 'tcp', localhost())
def complete_registration(self, msg, connect, maybe_tunnel): # print msg self.loop.remove_timeout(self._abort_timeout) ctx = self.context loop = self.loop identity = self.bident idents, msg = self.session.feed_identities(msg) msg = self.session.deserialize(msg) content = msg['content'] info = self.connection_info def url(key): """get zmq url for given channel""" return str(info["interface"] + ":%i" % info[key]) if content['status'] == 'ok': self.id = int(content['id']) # launch heartbeat # possibly forward hb ports with tunnels hb_ping = maybe_tunnel(url('hb_ping')) hb_pong = maybe_tunnel(url('hb_pong')) hb_monitor = None if self.max_heartbeat_misses > 0: # Add a monitor socket which will record the last time a ping was seen mon = self.context.socket(zmq.SUB) mport = mon.bind_to_random_port('tcp://%s' % localhost()) mon.setsockopt(zmq.SUBSCRIBE, b"") self._hb_listener = zmqstream.ZMQStream(mon, self.loop) self._hb_listener.on_recv(self._report_ping) hb_monitor = "tcp://%s:%i" % (localhost(), mport) heart = Heart(hb_ping, hb_pong, hb_monitor, heart_id=identity) heart.start() # create Shell Connections (MUX, Task, etc.): shell_addrs = url('mux'), url('task') # Use only one shell stream for mux and tasks stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) stream.setsockopt(zmq.IDENTITY, identity) shell_streams = [stream] for addr in shell_addrs: connect(stream, addr) # control stream: control_addr = url('control') control_stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) control_stream.setsockopt(zmq.IDENTITY, identity) connect(control_stream, control_addr) # create iopub stream: iopub_addr = url('iopub') iopub_socket = ctx.socket(zmq.PUB) iopub_socket.setsockopt(zmq.IDENTITY, identity) connect(iopub_socket, iopub_addr) # disable history: self.config.HistoryManager.hist_file = ':memory:' # Redirect input streams and set a display hook. if self.out_stream_factory: sys.stdout = self.out_stream_factory(self.session, iopub_socket, u'stdout') sys.stdout.topic = cast_bytes('engine.%i.stdout' % self.id) sys.stderr = self.out_stream_factory(self.session, iopub_socket, u'stderr') sys.stderr.topic = cast_bytes('engine.%i.stderr' % self.id) if self.display_hook_factory: sys.displayhook = self.display_hook_factory( self.session, iopub_socket) sys.displayhook.topic = cast_bytes('engine.%i.execute_result' % self.id) self.kernel = Kernel(parent=self, int_id=self.id, ident=self.ident, session=self.session, control_stream=control_stream, shell_streams=shell_streams, iopub_socket=iopub_socket, loop=loop, user_ns=self.user_ns, log=self.log) self.kernel.shell.display_pub.topic = cast_bytes( 'engine.%i.displaypub' % self.id) # periodically check the heartbeat pings of the controller # Should be started here and not in "start()" so that the right period can be taken # from the hubs HeartBeatMonitor.period if self.max_heartbeat_misses > 0: # Use a slightly bigger check period than the hub signal period to not warn unnecessary self.hb_check_period = int(content['hb_period']) + 10 self.log.info( "Starting to monitor the heartbeat signal from the hub every %i ms.", self.hb_check_period) self._hb_reporter = ioloop.PeriodicCallback( self._hb_monitor, self.hb_check_period, self.loop) self._hb_reporter.start() else: self.log.info( "Monitoring of the heartbeat signal from the hub is not enabled." ) # FIXME: This is a hack until IPKernelApp and IPEngineApp can be fully merged app = IPKernelApp(parent=self, shell=self.kernel.shell, kernel=self.kernel, log=self.log) app.init_profile_dir() app.init_code() self.kernel.start() else: self.log.fatal("Registration Failed: %s" % msg) raise Exception("Registration Failed: %s" % msg) self.log.info("Completed registration with id %i" % self.id)
def complete_registration(self, msg, connect, maybe_tunnel): # print msg self.loop.remove_timeout(self._abort_timeout) ctx = self.context loop = self.loop identity = self.bident idents,msg = self.session.feed_identities(msg) msg = self.session.deserialize(msg) content = msg['content'] info = self.connection_info def url(key): """get zmq url for given channel""" return str(info["interface"] + ":%i" % info[key]) if content['status'] == 'ok': self.id = int(content['id']) # launch heartbeat # possibly forward hb ports with tunnels hb_ping = maybe_tunnel(url('hb_ping')) hb_pong = maybe_tunnel(url('hb_pong')) hb_monitor = None if self.max_heartbeat_misses > 0: # Add a monitor socket which will record the last time a ping was seen mon = self.context.socket(zmq.SUB) mport = mon.bind_to_random_port('tcp://%s' % localhost()) mon.setsockopt(zmq.SUBSCRIBE, b"") self._hb_listener = zmqstream.ZMQStream(mon, self.loop) self._hb_listener.on_recv(self._report_ping) hb_monitor = "tcp://%s:%i" % (localhost(), mport) heart = Heart(hb_ping, hb_pong, hb_monitor , heart_id=identity) heart.start() # create Shell Connections (MUX, Task, etc.): shell_addrs = url('mux'), url('task') # Use only one shell stream for mux and tasks stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) stream.setsockopt(zmq.IDENTITY, identity) shell_streams = [stream] for addr in shell_addrs: connect(stream, addr) # control stream: control_addr = url('control') control_stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) control_stream.setsockopt(zmq.IDENTITY, identity) connect(control_stream, control_addr) # create iopub stream: iopub_addr = url('iopub') iopub_socket = ctx.socket(zmq.PUB) iopub_socket.setsockopt(zmq.IDENTITY, identity) connect(iopub_socket, iopub_addr) # disable history: self.config.HistoryManager.hist_file = ':memory:' # Redirect input streams and set a display hook. if self.out_stream_factory: sys.stdout = self.out_stream_factory(self.session, iopub_socket, u'stdout') sys.stdout.topic = cast_bytes('engine.%i.stdout' % self.id) sys.stderr = self.out_stream_factory(self.session, iopub_socket, u'stderr') sys.stderr.topic = cast_bytes('engine.%i.stderr' % self.id) if self.display_hook_factory: sys.displayhook = self.display_hook_factory(self.session, iopub_socket) sys.displayhook.topic = cast_bytes('engine.%i.execute_result' % self.id) self.kernel = Kernel(parent=self, int_id=self.id, ident=self.ident, session=self.session, control_stream=control_stream, shell_streams=shell_streams, iopub_socket=iopub_socket, loop=loop, user_ns=self.user_ns, log=self.log) self.kernel.shell.display_pub.topic = cast_bytes('engine.%i.displaypub' % self.id) # periodically check the heartbeat pings of the controller # Should be started here and not in "start()" so that the right period can be taken # from the hubs HeartBeatMonitor.period if self.max_heartbeat_misses > 0: # Use a slightly bigger check period than the hub signal period to not warn unnecessary self.hb_check_period = int(content['hb_period'])+10 self.log.info("Starting to monitor the heartbeat signal from the hub every %i ms." , self.hb_check_period) self._hb_reporter = ioloop.PeriodicCallback(self._hb_monitor, self.hb_check_period, self.loop) self._hb_reporter.start() else: self.log.info("Monitoring of the heartbeat signal from the hub is not enabled.") # FIXME: This is a hack until IPKernelApp and IPEngineApp can be fully merged app = IPKernelApp(parent=self, shell=self.kernel.shell, kernel=self.kernel, log=self.log) app.init_profile_dir() app.init_code() self.kernel.start() else: self.log.fatal("Registration Failed: %s"%msg) raise Exception("Registration Failed: %s"%msg) self.log.info("Completed registration with id %i"%self.id)
def test_tcp_cinfo(self): with self._get_tcp_km() as km: self._run_cinfo(km, 'tcp', localhost())
def _url_default(self): return 'tcp://%s:20202' % localhost()
def _ip_default(self): return localhost()
def _engine_ip_default(self): return localhost()