def disambiguate_ip_address(ip, location=None):
"""turn multi-ip interfaces '0.0.0.0' and '*' into a connectable address
Explicit IP addresses are returned unmodified.
Parameters
----------
ip : IP address
An IP address, or the special values 0.0.0.0, or *
location: IP address, optional
A public IP of the target machine.
If location is an IP of the current machine,
localhost will be returned,
otherwise location will be returned.
"""
if ip in {'0.0.0.0', '*'}:
if not location:
# unspecified location, localhost is the only choice
ip = localhost()
elif is_public_ip(location):
# location is a public IP on this machine, use localhost
ip = localhost()
elif not public_ips():
# this machine's public IPs cannot be determined,
# assume `location` is not this machine
warnings.warn("IPython could not determine public IPs",
RuntimeWarning)
ip = location
else:
# location is not this machine, do not use loopback
ip = location
return ip
def disambiguate_ip_address(ip, location=None):
"""turn multi-ip interfaces '0.0.0.0' and '*' into a connectable address
Explicit IP addresses are returned unmodified.
Parameters
----------
ip : IP address
An IP address, or the special values 0.0.0.0, or *
location: IP address, optional
A public IP of the target machine.
If location is an IP of the current machine,
localhost will be returned,
otherwise location will be returned.
"""
if ip in {'0.0.0.0', '*'}:
if not location:
# unspecified location, localhost is the only choice
ip = localhost()
elif is_public_ip(location):
# location is a public IP on this machine, use localhost
ip = localhost()
elif not public_ips():
# this machine's public IPs cannot be determined,
# assume `location` is not this machine
warnings.warn("IPython could not determine public IPs", RuntimeWarning)
ip = location
else:
# location is not this machine, do not use loopback
ip = location
return ip
def test_disambiguate_ip():
# garbage in, garbage out
public_ip = public_ips()[0]
assert util.disambiguate_ip_address('garbage') == 'garbage'
assert util.disambiguate_ip_address('0.0.0.0', socket.gethostname()) == localhost()
wontresolve = 'this.wontresolve.dns'
assert util.disambiguate_ip_address('0.0.0.0', wontresolve) == wontresolve
assert util.disambiguate_ip_address('0.0.0.0', public_ip) == localhost()
def test_disambiguate_ip():
# garbage in, garbage out
public_ip = public_ips()[0]
assert util.disambiguate_ip_address('garbage') == 'garbage'
assert util.disambiguate_ip_address('0.0.0.0',
socket.gethostname()) == localhost()
wontresolve = 'this.wontresolve.dns'
assert util.disambiguate_ip_address('0.0.0.0', wontresolve) == wontresolve
assert util.disambiguate_ip_address('0.0.0.0', public_ip) == localhost()
def test_disambiguate_ip(warn_mock):
# garbage in, garbage out
public_ip = public_ips()[0]
assert util.disambiguate_ip_address('garbage') == 'garbage'
assert util.disambiguate_ip_address('0.0.0.0',
socket.gethostname()) == localhost()
wontresolve = 'this.wontresolve.dns'
assert util.disambiguate_ip_address('0.0.0.0', wontresolve) == wontresolve
assert warn_mock.called_once_with(
'IPython could not determine IPs for {}: '
'[Errno -2] Name or service not known'.format(wontresolve),
RuntimeWarning)
assert util.disambiguate_ip_address('0.0.0.0', public_ip) == localhost()
def init_ssh(self):
"""set up ssh tunnels, if needed."""
if not self.existing or (not self.sshserver and not self.sshkey):
return
self.load_connection_file()
transport = self.transport
ip = self.ip
if transport != 'tcp':
self.log.error("Can only use ssh tunnels with TCP sockets, \
not %s", transport)
sys.exit(-1)
if self.sshkey and not self.sshserver:
# specifying just the key implies that we are connecting directly
self.sshserver = ip
ip = localhost()
# build connection dict for tunnels:
info = dict(
ip=ip,
shell_port=self.shell_port,
iopub_port=self.iopub_port,
stdin_port=self.stdin_port,
hb_port=self.hb_port
)
self.log.info("Forwarding connections \
to %s via %s" % (ip, self.sshserver))
# tunnels return a new set of ports, which will be on localhost:
self.ip = localhost()
try:
newports = tunnel_to_kernel(info, self.sshserver, self.sshkey)
except IOError:
# even catch KeyboardInterrupt
self.log.error("Could not setup tunnels", exc_info=True)
self.exit(1)
self.shell_port, self.iopub_port, \
self.stdin_port, self.hb_port = newports
cf = self.connection_file
root, ext = os.path.splitext(cf)
self.connection_file = root + '-ssh' + ext
self.write_connection_file() # write the new connection file
self.log.info("To connect another client via this tunnel, use:")
self.log.info("--existing %s" % os.path.basename(self.connection_file))
def test_stop(self, mocked_logger):
url = 'tcp://{}:20201'.format(localhost())
watcher = ema.start_logwatcher(url)
watcher.stop()
time.sleep(3)
mocked_logger.warning.assert_called_once_with('shutting down log watcher')
def get_free_port():
"""Find a free port on the local machine."""
sock = socket.socket()
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, b'\0' * 8)
sock.bind((localhost(), 0))
port = sock.getsockname()[1]
sock.close()
return port
def start_heartbeat(self, hb_ping, hb_pong, hb_period, identity):
"""Start our heart beating"""
hb_monitor = None
if self.max_heartbeat_misses > 0:
# Add a monitor socket which will record the last time a ping was seen
mon = self.context.socket(zmq.SUB)
if self.curve_serverkey:
mon.setsockopt(zmq.CURVE_SERVER, 1)
mon.setsockopt(zmq.CURVE_SECRETKEY, self.curve_secretkey)
mport = mon.bind_to_random_port('tcp://%s' % localhost())
mon.setsockopt(zmq.SUBSCRIBE, b"")
self._hb_listener = zmqstream.ZMQStream(mon, self.loop)
self._hb_listener.on_recv(self._report_ping)
hb_monitor = "tcp://%s:%i" % (localhost(), mport)
heart = Heart(
hb_ping,
hb_pong,
hb_monitor,
heart_id=identity,
curve_serverkey=self.curve_serverkey,
curve_secretkey=self.curve_secretkey,
curve_publickey=self.curve_publickey,
)
heart.start()
# periodically check the heartbeat pings of the controller
# Should be started here and not in "start()" so that the right period can be taken
# from the hubs HeartBeatMonitor.period
if self.max_heartbeat_misses > 0:
# Use a slightly bigger check period than the hub signal period to not warn unnecessary
self.hb_check_period = hb_period + 500
self.log.info(
"Starting to monitor the heartbeat signal from the hub every %i ms.",
self.hb_check_period,
)
self._hb_reporter = ioloop.PeriodicCallback(
self._hb_monitor, self.hb_check_period
)
self._hb_reporter.start()
else:
self.log.info(
"Monitoring of the heartbeat signal from the hub is not enabled."
)
def setUpClass(cls):
logger = ema_logging.get_logger()
mocked_logger = mock.Mock(spec=logger)
mocked_logger.handlers = []
ema_logging._logger = mocked_logger
cls.client = ipyparallel.Client(profile='default')
cls.url = 'tcp://{}:20202'.format(localhost())
cls.watcher = ema.start_logwatcher(cls.url)
def save_connection_dict(self, fname, cdict):
"""save a connection dict to json file."""
c = self.config
url = cdict['registration']
location = cdict['location']
if not location:
if public_ips():
location = public_ips()[-1]
else:
self.log.warn("Could not identify this machine's IP, assuming %s."
" You may need to specify '--location=<external_ip_address>' to help"
" IPython decide when to connect via loopback." % localhost() )
location = localhost()
cdict['location'] = location
fname = os.path.join(self.profile_dir.security_dir, fname)
self.log.info("writing connection info to %s", fname)
with open(fname, 'w') as f:
f.write(json.dumps(cdict, indent=2))
os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)
def __init__(self, context, addr=None):
if addr is None:
addr = ('tcp', localhost(), 0)
Thread.__init__(self)
self.context = context
self.transport, self.ip, self.port = addr
self.original_port = self.port
if self.original_port == 0:
self.pick_port()
self.addr = (self.ip, self.port)
self.daemon = True
def save_connection_dict(self, fname, cdict):
"""save a connection dict to json file."""
c = self.config
url = cdict['registration']
location = cdict['location']
if not location:
if public_ips():
location = public_ips()[-1]
else:
self.log.warn(
"Could not identify this machine's IP, assuming %s."
" You may need to specify '--location=<external_ip_address>' to help"
" IPython decide when to connect via loopback." %
localhost())
location = localhost()
cdict['location'] = location
fname = os.path.join(self.profile_dir.security_dir, fname)
self.log.info("writing connection info to %s", fname)
with open(fname, 'w') as f:
f.write(json.dumps(cdict, indent=2))
os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR)
def __init__(self, context, addr=None):
if addr is None:
addr = ('tcp', localhost(), 0)
Thread.__init__(self, name="Heartbeat")
self.context = context
self.transport, self.ip, self.port = addr
self.original_port = self.port
if self.original_port == 0:
self.pick_port()
self.addr = (self.ip, self.port)
self.daemon = True
self.pydev_do_not_trace = True
self.is_pydev_daemon_thread = True
self.name = "Heartbeat"
def __init__(self, context, addr=None):
if addr is None:
addr = ('tcp', localhost(), 0)
Thread.__init__(self)
self.context = context
self.transport, self.ip, self.port = addr
if self.port == 0:
if addr[0] == 'tcp':
s = socket.socket()
# '*' means all interfaces to 0MQ, which is '' to socket.socket
s.bind(('' if self.ip == '*' else self.ip, 0))
self.port = s.getsockname()[1]
s.close()
elif addr[0] == 'ipc':
self.port = 1
while os.path.exists("%s-%s" % (self.ip, self.port)):
self.port = self.port + 1
else:
raise ValueError("Unrecognized zmq transport: %s" % addr[0])
self.addr = (self.ip, self.port)
self.daemon = True
def __init__(self, context, addr=None):
if addr is None:
addr = ('tcp', localhost(), 0)
Thread.__init__(self)
self.context = context
self.transport, self.ip, self.port = addr
if self.port == 0:
if addr[0] == 'tcp':
s = socket.socket()
# '*' means all interfaces to 0MQ, which is '' to socket.socket
s.bind(('' if self.ip == '*' else self.ip, 0))
self.port = s.getsockname()[1]
s.close()
elif addr[0] == 'ipc':
self.port = 1
while os.path.exists("%s-%s" % (self.ip, self.port)):
self.port = self.port + 1
else:
raise ValueError("Unrecognized zmq transport: %s" % addr[0])
self.addr = (self.ip, self.port)
self.daemon = True
async def test_tcp_cinfo(self):
async with self._get_tcp_km() as km:
await self._run_cinfo(km, 'tcp', localhost())
def complete_registration(self, msg, connect, maybe_tunnel):
# print msg
self.loop.remove_timeout(self._abort_timeout)
ctx = self.context
loop = self.loop
identity = self.bident
idents, msg = self.session.feed_identities(msg)
msg = self.session.deserialize(msg)
content = msg['content']
info = self.connection_info
def url(key):
"""get zmq url for given channel"""
return str(info["interface"] + ":%i" % info[key])
if content['status'] == 'ok':
self.id = int(content['id'])
# launch heartbeat
# possibly forward hb ports with tunnels
hb_ping = maybe_tunnel(url('hb_ping'))
hb_pong = maybe_tunnel(url('hb_pong'))
hb_monitor = None
if self.max_heartbeat_misses > 0:
# Add a monitor socket which will record the last time a ping was seen
mon = self.context.socket(zmq.SUB)
mport = mon.bind_to_random_port('tcp://%s' % localhost())
mon.setsockopt(zmq.SUBSCRIBE, b"")
self._hb_listener = zmqstream.ZMQStream(mon, self.loop)
self._hb_listener.on_recv(self._report_ping)
hb_monitor = "tcp://%s:%i" % (localhost(), mport)
heart = Heart(hb_ping, hb_pong, hb_monitor, heart_id=identity)
heart.start()
# create Shell Connections (MUX, Task, etc.):
shell_addrs = url('mux'), url('task')
# Use only one shell stream for mux and tasks
stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop)
stream.setsockopt(zmq.IDENTITY, identity)
shell_streams = [stream]
for addr in shell_addrs:
connect(stream, addr)
# control stream:
control_addr = url('control')
control_stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop)
control_stream.setsockopt(zmq.IDENTITY, identity)
connect(control_stream, control_addr)
# create iopub stream:
iopub_addr = url('iopub')
iopub_socket = ctx.socket(zmq.PUB)
iopub_socket.setsockopt(zmq.IDENTITY, identity)
connect(iopub_socket, iopub_addr)
# disable history:
self.config.HistoryManager.hist_file = ':memory:'
# Redirect input streams and set a display hook.
if self.out_stream_factory:
sys.stdout = self.out_stream_factory(self.session,
iopub_socket, u'stdout')
sys.stdout.topic = cast_bytes('engine.%i.stdout' % self.id)
sys.stderr = self.out_stream_factory(self.session,
iopub_socket, u'stderr')
sys.stderr.topic = cast_bytes('engine.%i.stderr' % self.id)
if self.display_hook_factory:
sys.displayhook = self.display_hook_factory(
self.session, iopub_socket)
sys.displayhook.topic = cast_bytes('engine.%i.execute_result' %
self.id)
self.kernel = Kernel(parent=self,
int_id=self.id,
ident=self.ident,
session=self.session,
control_stream=control_stream,
shell_streams=shell_streams,
iopub_socket=iopub_socket,
loop=loop,
user_ns=self.user_ns,
log=self.log)
self.kernel.shell.display_pub.topic = cast_bytes(
'engine.%i.displaypub' % self.id)
# periodically check the heartbeat pings of the controller
# Should be started here and not in "start()" so that the right period can be taken
# from the hubs HeartBeatMonitor.period
if self.max_heartbeat_misses > 0:
# Use a slightly bigger check period than the hub signal period to not warn unnecessary
self.hb_check_period = int(content['hb_period']) + 10
self.log.info(
"Starting to monitor the heartbeat signal from the hub every %i ms.",
self.hb_check_period)
self._hb_reporter = ioloop.PeriodicCallback(
self._hb_monitor, self.hb_check_period, self.loop)
self._hb_reporter.start()
else:
self.log.info(
"Monitoring of the heartbeat signal from the hub is not enabled."
)
# FIXME: This is a hack until IPKernelApp and IPEngineApp can be fully merged
app = IPKernelApp(parent=self,
shell=self.kernel.shell,
kernel=self.kernel,
log=self.log)
app.init_profile_dir()
app.init_code()
self.kernel.start()
else:
self.log.fatal("Registration Failed: %s" % msg)
raise Exception("Registration Failed: %s" % msg)
self.log.info("Completed registration with id %i" % self.id)
def complete_registration(self, msg, connect, maybe_tunnel):
# print msg
self.loop.remove_timeout(self._abort_timeout)
ctx = self.context
loop = self.loop
identity = self.bident
idents,msg = self.session.feed_identities(msg)
msg = self.session.deserialize(msg)
content = msg['content']
info = self.connection_info
def url(key):
"""get zmq url for given channel"""
return str(info["interface"] + ":%i" % info[key])
if content['status'] == 'ok':
self.id = int(content['id'])
# launch heartbeat
# possibly forward hb ports with tunnels
hb_ping = maybe_tunnel(url('hb_ping'))
hb_pong = maybe_tunnel(url('hb_pong'))
hb_monitor = None
if self.max_heartbeat_misses > 0:
# Add a monitor socket which will record the last time a ping was seen
mon = self.context.socket(zmq.SUB)
mport = mon.bind_to_random_port('tcp://%s' % localhost())
mon.setsockopt(zmq.SUBSCRIBE, b"")
self._hb_listener = zmqstream.ZMQStream(mon, self.loop)
self._hb_listener.on_recv(self._report_ping)
hb_monitor = "tcp://%s:%i" % (localhost(), mport)
heart = Heart(hb_ping, hb_pong, hb_monitor , heart_id=identity)
heart.start()
# create Shell Connections (MUX, Task, etc.):
shell_addrs = url('mux'), url('task')
# Use only one shell stream for mux and tasks
stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop)
stream.setsockopt(zmq.IDENTITY, identity)
shell_streams = [stream]
for addr in shell_addrs:
connect(stream, addr)
# control stream:
control_addr = url('control')
control_stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop)
control_stream.setsockopt(zmq.IDENTITY, identity)
connect(control_stream, control_addr)
# create iopub stream:
iopub_addr = url('iopub')
iopub_socket = ctx.socket(zmq.PUB)
iopub_socket.setsockopt(zmq.IDENTITY, identity)
connect(iopub_socket, iopub_addr)
# disable history:
self.config.HistoryManager.hist_file = ':memory:'
# Redirect input streams and set a display hook.
if self.out_stream_factory:
sys.stdout = self.out_stream_factory(self.session, iopub_socket, u'stdout')
sys.stdout.topic = cast_bytes('engine.%i.stdout' % self.id)
sys.stderr = self.out_stream_factory(self.session, iopub_socket, u'stderr')
sys.stderr.topic = cast_bytes('engine.%i.stderr' % self.id)
if self.display_hook_factory:
sys.displayhook = self.display_hook_factory(self.session, iopub_socket)
sys.displayhook.topic = cast_bytes('engine.%i.execute_result' % self.id)
self.kernel = Kernel(parent=self, int_id=self.id, ident=self.ident, session=self.session,
control_stream=control_stream, shell_streams=shell_streams, iopub_socket=iopub_socket,
loop=loop, user_ns=self.user_ns, log=self.log)
self.kernel.shell.display_pub.topic = cast_bytes('engine.%i.displaypub' % self.id)
# periodically check the heartbeat pings of the controller
# Should be started here and not in "start()" so that the right period can be taken
# from the hubs HeartBeatMonitor.period
if self.max_heartbeat_misses > 0:
# Use a slightly bigger check period than the hub signal period to not warn unnecessary
self.hb_check_period = int(content['hb_period'])+10
self.log.info("Starting to monitor the heartbeat signal from the hub every %i ms." , self.hb_check_period)
self._hb_reporter = ioloop.PeriodicCallback(self._hb_monitor, self.hb_check_period, self.loop)
self._hb_reporter.start()
else:
self.log.info("Monitoring of the heartbeat signal from the hub is not enabled.")
# FIXME: This is a hack until IPKernelApp and IPEngineApp can be fully merged
app = IPKernelApp(parent=self, shell=self.kernel.shell, kernel=self.kernel, log=self.log)
app.init_profile_dir()
app.init_code()
self.kernel.start()
else:
self.log.fatal("Registration Failed: %s"%msg)
raise Exception("Registration Failed: %s"%msg)
self.log.info("Completed registration with id %i"%self.id)
def test_tcp_cinfo(self):
with self._get_tcp_km() as km:
self._run_cinfo(km, 'tcp', localhost())
def _url_default(self):
return 'tcp://%s:20202' % localhost()
def _ip_default(self):
return localhost()
def _engine_ip_default(self):
return localhost()
def _url_default(self):
return 'tcp://%s:20202' % localhost()
def _ip_default(self):
return localhost()