def get_connection_info(self, session=False):
info = super(RemoteKernelManager, self).get_connection_info(session)
# Convert bytes to string for persistence. Will reverse operation in load_connection_info
info_key = info.get('key')
if info_key:
info['key'] = bytes_to_str(info_key)
return info
def get_connection_info(self, session=False):
info = super(RemoteKernelManager, self).get_connection_info(session)
# Convert bytes to string for persistence. Will reverse operation in load_connection_info
info_key = info.get('key')
if info_key:
info['key'] = bytes_to_str(info_key)
return info
def run_python(parameters, ignore_return_code=False, stdin=None):
"""
Run python as a shell command, listening for both Errors and
non-zero return codes. Returns the tuple (stdout, stderr) of
output produced during the nbconvert run.
Parameters
----------
parameters : str, list(str)
List of parameters to pass to IPython.
ignore_return_code : optional bool (default False)
Throw an OSError if the return code
"""
cmd = [sys.executable]
if sys.platform == 'win32':
if isinstance(parameters, string_types):
cmd = ' '.join(cmd) + ' ' + parameters
else:
cmd = ' '.join(cmd + parameters)
else:
if isinstance(parameters, string_types):
parameters = shlex.split(parameters)
cmd += parameters
p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE)
stdout, stderr = p.communicate(input=stdin)
if not (p.returncode == 0 or ignore_return_code):
raise OSError(bytes_to_str(stderr))
return stdout.decode('utf8', 'replace'), stderr.decode('utf8', 'replace')
def pre_save_transformation(sessions):
sessions_copy = copy.deepcopy(sessions)
for kernel_id, session in sessions_copy.items():
if session.get('connection_info'):
info = session['connection_info']
key = info.get('key')
if key:
info['key'] = bytes_to_str(key)
return sessions_copy
def pre_save_transformation(session):
kernel_id = list(session.keys())[0]
session_info = session[kernel_id]
if session_info.get('connection_info'):
info = session_info['connection_info']
key = info.get('key')
if key:
info['key'] = bytes_to_str(key)
return session
def run(self):
self.started = True
while not self.stop.is_set():
chunk = os.read(self.readfd, 1024)
with self.buffer_lock:
self.buffer.write(chunk)
if self.echo:
sys.stdout.write(bytes_to_str(chunk))
os.close(self.readfd)
os.close(self.writefd)
def wait(self, *pargs, **kwargs):
"""Wait for the JSController to finish"""
ret = super(JSController, self).wait(*pargs, **kwargs)
# If this is a SlimerJS controller, check the captured stdout for
# errors. Otherwise, just return the return code.
if self.engine == 'slimerjs':
stdout = bytes_to_str(self.stdout)
if ret != 0:
# This could still happen e.g. if it's stopped by SIGINT
return ret
return bool(self.slimer_failure.search(stdout))
else:
return ret
def run(self):
self.started = True
while not self.stop.is_set():
chunk = os.read(self.readfd, 1024)
with self.buffer_lock:
self.buffer.write(chunk)
if self.echo:
sys.stdout.write(bytes_to_str(chunk))
os.close(self.readfd)
os.close(self.writefd)
def wait(self, *pargs, **kwargs):
"""Wait for the JSController to finish"""
ret = super(JSController, self).wait(*pargs, **kwargs)
# If this is a SlimerJS controller, check the captured stdout for
# errors. Otherwise, just return the return code.
if self.engine == 'slimerjs':
stdout = bytes_to_str(self.stdout)
if ret != 0:
# This could still happen e.g. if it's stopped by SIGINT
return ret
return bool(self.slimer_failure.search(stdout))
else:
return ret
def nbconvert(self, parameters, ignore_return_code=False):
"""
Run nbconvert a, IPython shell command, listening for both Errors and non-zero
return codes.
Parameters
----------
parameters : str, list(str)
List of parameters to pass to IPython.
ignore_return_code : optional bool (default False)
Throw an OSError if the return code
"""
if isinstance(parameters, string_types):
parameters = shlex.split(parameters)
cmd = [sys.executable, '-m', 'jupyter_nbconvert'] + parameters
p = Popen(cmd, stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate()
if not (p.returncode == 0 or ignore_return_code):
raise OSError(bytes_to_str(stderr))
return stdout.decode('utf8', 'replace'), stderr.decode('utf8', 'replace')
def nbconvert(self, parameters, ignore_return_code=False):
"""
Run nbconvert a, IPython shell command, listening for both Errors and non-zero
return codes.
Parameters
----------
parameters : str, list(str)
List of parameters to pass to IPython.
ignore_return_code : optional bool (default False)
Throw an OSError if the return code
"""
if isinstance(parameters, string_types):
parameters = shlex.split(parameters)
cmd = [sys.executable, '-m', 'jupyter_nbconvert'] + parameters
p = Popen(cmd, stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate()
if not (p.returncode == 0 or ignore_return_code):
raise OSError(bytes_to_str(stderr))
return stdout.decode('utf8',
'replace'), stderr.decode('utf8', 'replace')
def run_jstestall(options):
"""Run the entire Javascript test suite.
This function constructs TestControllers and runs them in subprocesses.
Parameters
----------
All parameters are passed as attributes of the options object.
testgroups : list of str
Run only these sections of the test suite. If empty, run all the available
sections.
fast : int or None
Run the test suite in parallel, using n simultaneous processes. If None
is passed, one process is used per CPU core. Default 1 (i.e. sequential)
inc_slow : bool
Include slow tests. By default, these tests aren't run.
slimerjs : bool
Use slimerjs if it's installed instead of phantomjs for casperjs tests.
url : unicode
Address:port to use when running the JS tests.
xunit : bool
Produce Xunit XML output. This is written to multiple foo.xunit.xml files.
extra_args : list
Extra arguments to pass to the test subprocesses, e.g. '-v'
"""
to_run, not_run = prepare_controllers(options)
def justify(ltext, rtext, width=70, fill='-'):
ltext += ' '
rtext = (' ' + rtext).rjust(width - len(ltext), fill)
return ltext + rtext
# Run all test runners, tracking execution time
failed = []
t_start = time.time()
print()
if options.fast == 1:
# This actually means sequential, i.e. with 1 job
for controller in to_run:
print('Test group:', controller.section)
sys.stdout.flush() # Show in correct order when output is piped
controller, res = do_run(controller, buffer_output=False)
if res:
failed.append(controller)
if res == -signal.SIGINT:
print("Interrupted")
break
print()
else:
# Run tests concurrently
try:
pool = multiprocessing.pool.ThreadPool(options.fast)
for (controller, res) in pool.imap_unordered(do_run, to_run):
res_string = 'OK' if res == 0 else 'FAILED'
print(justify('Test group: ' + controller.section, res_string))
if res:
controller.print_extra_info()
print(bytes_to_str(controller.stdout))
failed.append(controller)
if res == -signal.SIGINT:
print("Interrupted")
break
except KeyboardInterrupt:
return
for controller in not_run:
print(justify('Test group: ' + controller.section, 'NOT RUN'))
t_end = time.time()
t_tests = t_end - t_start
nrunners = len(to_run)
nfail = len(failed)
# summarize results
print('_'*70)
print('Test suite completed for system with the following information:')
print(report())
took = "Took %.3fs." % t_tests
print('Status: ', end='')
if not failed:
print('OK (%d test groups).' % nrunners, took)
else:
# If anything went wrong, point out what command to rerun manually to
# see the actual errors and individual summary
failed_sections = [c.section for c in failed]
print('ERROR - {} out of {} test groups failed ({}).'.format(nfail,
nrunners, ', '.join(failed_sections)), took)
print()
print('You may wish to rerun these, with:')
print(' python -m notebook.jstest', *failed_sections)
print()
if failed:
# Ensure that our exit code indicates failure
sys.exit(1)
def write_connection_file(fname=None,
shell_port=0,
iopub_port=0,
stdin_port=0,
hb_port=0,
control_port=0,
ip='',
key=b'',
transport='tcp',
signature_scheme='hmac-sha256',
kernel_name=''):
"""Generates a JSON config file, including the selection of random ports.
Parameters
----------
fname : unicode
The path to the file to write
shell_port : int, optional
The port to use for ROUTER (shell) channel.
iopub_port : int, optional
The port to use for the SUB channel.
stdin_port : int, optional
The port to use for the ROUTER (raw input) channel.
control_port : int, optional
The port to use for the ROUTER (control) channel.
hb_port : int, optional
The port to use for the heartbeat REP channel.
ip : str, optional
The ip address the kernel will bind to.
key : str, optional
The Session key used for message authentication.
signature_scheme : str, optional
The scheme used for message authentication.
This has the form 'digest-hash', where 'digest'
is the scheme used for digests, and 'hash' is the name of the hash function
used by the digest scheme.
Currently, 'hmac' is the only supported digest scheme,
and 'sha256' is the default hash function.
kernel_name : str, optional
The name of the kernel currently connected to.
"""
if not ip:
ip = localhost()
# default to temporary connector file
if not fname:
fd, fname = tempfile.mkstemp('.json')
os.close(fd)
# Find open ports as necessary.
ports = []
ports_needed = int(shell_port <= 0) + \
int(iopub_port <= 0) + \
int(stdin_port <= 0) + \
int(control_port <= 0) + \
int(hb_port <= 0)
if transport == 'tcp':
for i in range(ports_needed):
sock = socket.socket()
# struct.pack('ii', (0,0)) is 8 null bytes
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, b'\0' * 8)
sock.bind((ip, 0))
ports.append(sock)
for i, sock in enumerate(ports):
port = sock.getsockname()[1]
sock.close()
ports[i] = port
else:
N = 1
for i in range(ports_needed):
while os.path.exists("%s-%s" % (ip, str(N))):
N += 1
ports.append(N)
N += 1
if shell_port <= 0:
shell_port = ports.pop(0)
if iopub_port <= 0:
iopub_port = ports.pop(0)
if stdin_port <= 0:
stdin_port = ports.pop(0)
if control_port <= 0:
control_port = ports.pop(0)
if hb_port <= 0:
hb_port = ports.pop(0)
cfg = dict(
shell_port=shell_port,
iopub_port=iopub_port,
stdin_port=stdin_port,
control_port=control_port,
hb_port=hb_port,
)
cfg['ip'] = ip
cfg['key'] = bytes_to_str(key)
cfg['transport'] = transport
cfg['signature_scheme'] = signature_scheme
cfg['kernel_name'] = kernel_name
# Only ever write this file as user read/writeable
# This would otherwise introduce a vulnerability as a file has secrets
# which would let others execute arbitrarily code as you
with secure_write(fname) as f:
f.write(json.dumps(cfg, indent=2))
if hasattr(stat, 'S_ISVTX'):
# set the sticky bit on the file and its parent directory
# to avoid periodic cleanup
paths = [fname]
runtime_dir = os.path.dirname(fname)
if runtime_dir:
paths.append(runtime_dir)
for path in paths:
permissions = os.stat(path).st_mode
new_permissions = permissions | stat.S_ISVTX
if new_permissions != permissions:
try:
os.chmod(path, new_permissions)
except OSError as e:
if e.errno == errno.EPERM and path == runtime_dir:
# suppress permission errors setting sticky bit on runtime_dir,
# which we may not own.
pass
else:
# failed to set sticky bit, probably not a big deal
warnings.warn(
"Failed to set sticky bit on %r: %s"
"\nProbably not a big deal, but runtime files may be cleaned up periodically."
% (path, e),
RuntimeWarning,
)
return fname, cfg
def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0,
control_port=0, ip='', key=b'', transport='tcp',
signature_scheme='hmac-sha256',
):
"""Generates a JSON config file, including the selection of random ports.
Parameters
----------
fname : unicode
The path to the file to write
shell_port : int, optional
The port to use for ROUTER (shell) channel.
iopub_port : int, optional
The port to use for the SUB channel.
stdin_port : int, optional
The port to use for the ROUTER (raw input) channel.
control_port : int, optional
The port to use for the ROUTER (control) channel.
hb_port : int, optional
The port to use for the heartbeat REP channel.
ip : str, optional
The ip address the kernel will bind to.
key : str, optional
The Session key used for message authentication.
signature_scheme : str, optional
The scheme used for message authentication.
This has the form 'digest-hash', where 'digest'
is the scheme used for digests, and 'hash' is the name of the hash function
used by the digest scheme.
Currently, 'hmac' is the only supported digest scheme,
and 'sha256' is the default hash function.
"""
if not ip:
ip = localhost()
# default to temporary connector file
if not fname:
fd, fname = tempfile.mkstemp('.json')
os.close(fd)
# Find open ports as necessary.
ports = []
ports_needed = int(shell_port <= 0) + \
int(iopub_port <= 0) + \
int(stdin_port <= 0) + \
int(control_port <= 0) + \
int(hb_port <= 0)
if transport == 'tcp':
for i in range(ports_needed):
sock = socket.socket()
# struct.pack('ii', (0,0)) is 8 null bytes
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, b'\0' * 8)
sock.bind(('', 0))
ports.append(sock)
for i, sock in enumerate(ports):
port = sock.getsockname()[1]
sock.close()
ports[i] = port
else:
N = 1
for i in range(ports_needed):
while os.path.exists("%s-%s" % (ip, str(N))):
N += 1
ports.append(N)
N += 1
if shell_port <= 0:
shell_port = ports.pop(0)
if iopub_port <= 0:
iopub_port = ports.pop(0)
if stdin_port <= 0:
stdin_port = ports.pop(0)
if control_port <= 0:
control_port = ports.pop(0)
if hb_port <= 0:
hb_port = ports.pop(0)
cfg = dict( shell_port=shell_port,
iopub_port=iopub_port,
stdin_port=stdin_port,
control_port=control_port,
hb_port=hb_port,
)
cfg['ip'] = ip
cfg['key'] = bytes_to_str(key)
cfg['transport'] = transport
cfg['signature_scheme'] = signature_scheme
with open(fname, 'w') as f:
f.write(json.dumps(cfg, indent=2))
return fname, cfg
def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0,
control_port=0, ip='', key=b'', transport='tcp',
signature_scheme='hmac-sha256',
):
"""Generates a JSON config file, including the selection of random ports.
Parameters
----------
fname : unicode
The path to the file to write
shell_port : int, optional
The port to use for ROUTER (shell) channel.
iopub_port : int, optional
The port to use for the SUB channel.
stdin_port : int, optional
The port to use for the ROUTER (raw input) channel.
control_port : int, optional
The port to use for the ROUTER (control) channel.
hb_port : int, optional
The port to use for the heartbeat REP channel.
ip : str, optional
The ip address the kernel will bind to.
key : str, optional
The Session key used for message authentication.
signature_scheme : str, optional
The scheme used for message authentication.
This has the form 'digest-hash', where 'digest'
is the scheme used for digests, and 'hash' is the name of the hash function
used by the digest scheme.
Currently, 'hmac' is the only supported digest scheme,
and 'sha256' is the default hash function.
"""
if not ip:
ip = localhost()
# default to temporary connector file
if not fname:
fd, fname = tempfile.mkstemp('.json')
os.close(fd)
# Find open ports as necessary.
ports = []
ports_needed = int(shell_port <= 0) + \
int(iopub_port <= 0) + \
int(stdin_port <= 0) + \
int(control_port <= 0) + \
int(hb_port <= 0)
if transport == 'tcp':
for i in range(ports_needed):
sock = socket.socket()
# struct.pack('ii', (0,0)) is 8 null bytes
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, b'\0' * 8)
sock.bind(('', 0))
ports.append(sock)
for i, sock in enumerate(ports):
port = sock.getsockname()[1]
sock.close()
ports[i] = port
else:
N = 1
for i in range(ports_needed):
while os.path.exists("%s-%s" % (ip, str(N))):
N += 1
ports.append(N)
N += 1
if shell_port <= 0:
shell_port = ports.pop(0)
if iopub_port <= 0:
iopub_port = ports.pop(0)
if stdin_port <= 0:
stdin_port = ports.pop(0)
if control_port <= 0:
control_port = ports.pop(0)
if hb_port <= 0:
hb_port = ports.pop(0)
cfg = dict( shell_port=shell_port,
iopub_port=iopub_port,
stdin_port=stdin_port,
control_port=control_port,
hb_port=hb_port,
)
cfg['ip'] = ip
cfg['key'] = bytes_to_str(key)
cfg['transport'] = transport
cfg['signature_scheme'] = signature_scheme
with open(fname, 'w') as f:
f.write(json.dumps(cfg, indent=2))
return fname, cfg
def run_jstestall(options):
"""Run the entire Javascript test suite.
This function constructs TestControllers and runs them in subprocesses.
Parameters
----------
All parameters are passed as attributes of the options object.
testgroups : list of str
Run only these sections of the test suite. If empty, run all the available
sections.
fast : int or None
Run the test suite in parallel, using n simultaneous processes. If None
is passed, one process is used per CPU core. Default 1 (i.e. sequential)
inc_slow : bool
Include slow tests. By default, these tests aren't run.
slimerjs : bool
Use slimerjs if it's installed instead of phantomjs for casperjs tests.
url : unicode
Address:port to use when running the JS tests.
xunit : bool
Produce Xunit XML output. This is written to multiple foo.xunit.xml files.
extra_args : list
Extra arguments to pass to the test subprocesses, e.g. '-v'
"""
to_run, not_run = prepare_controllers(options)
def justify(ltext, rtext, width=70, fill='-'):
ltext += ' '
rtext = (' ' + rtext).rjust(width - len(ltext), fill)
return ltext + rtext
# Run all test runners, tracking execution time
failed = []
t_start = time.time()
print()
if options.fast == 1:
# This actually means sequential, i.e. with 1 job
for controller in to_run:
print('Test group:', controller.section)
sys.stdout.flush() # Show in correct order when output is piped
controller, res = do_run(controller, buffer_output=False)
if res:
failed.append(controller)
if res == -signal.SIGINT:
print("Interrupted")
break
print()
else:
# Run tests concurrently
try:
pool = multiprocessing.pool.ThreadPool(options.fast)
for (controller, res) in pool.imap_unordered(do_run, to_run):
res_string = 'OK' if res == 0 else 'FAILED'
print(justify('Test group: ' + controller.section, res_string))
if res:
controller.print_extra_info()
print(bytes_to_str(controller.stdout))
failed.append(controller)
if res == -signal.SIGINT:
print("Interrupted")
break
except KeyboardInterrupt:
return
for controller in not_run:
print(justify('Test group: ' + controller.section, 'NOT RUN'))
t_end = time.time()
t_tests = t_end - t_start
nrunners = len(to_run)
nfail = len(failed)
# summarize results
print('_' * 70)
print('Test suite completed for system with the following information:')
print(report())
took = "Took %.3fs." % t_tests
print('Status: ', end='')
if not failed:
print('OK (%d test groups).' % nrunners, took)
else:
# If anything went wrong, point out what command to rerun manually to
# see the actual errors and individual summary
failed_sections = [c.section for c in failed]
print(
'ERROR - {} out of {} test groups failed ({}).'.format(
nfail, nrunners, ', '.join(failed_sections)), took)
print()
print('You may wish to rerun these, with:')
print(' python -m notebook.jstest', *failed_sections)
print()
if failed:
# Ensure that our exit code indicates failure
sys.exit(1)
def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0,
control_port=0, ip='', key=b'', transport='tcp',
signature_scheme='hmac-sha256', kernel_name=''
):
"""Generates a JSON config file, including the selection of random ports.
Parameters
----------
fname : unicode
The path to the file to write
shell_port : int, optional
The port to use for ROUTER (shell) channel.
iopub_port : int, optional
The port to use for the SUB channel.
stdin_port : int, optional
The port to use for the ROUTER (raw input) channel.
control_port : int, optional
The port to use for the ROUTER (control) channel.
hb_port : int, optional
The port to use for the heartbeat REP channel.
ip : str, optional
The ip address the kernel will bind to.
key : str, optional
The Session key used for message authentication.
signature_scheme : str, optional
The scheme used for message authentication.
This has the form 'digest-hash', where 'digest'
is the scheme used for digests, and 'hash' is the name of the hash function
used by the digest scheme.
Currently, 'hmac' is the only supported digest scheme,
and 'sha256' is the default hash function.
kernel_name : str, optional
The name of the kernel currently connected to.
"""
if not ip:
ip = localhost()
# default to temporary connector file
if not fname:
fd, fname = tempfile.mkstemp('.json')
os.close(fd)
# Find open ports as necessary.
ports = []
ports_needed = int(shell_port <= 0) + \
int(iopub_port <= 0) + \
int(stdin_port <= 0) + \
int(control_port <= 0) + \
int(hb_port <= 0)
if transport == 'tcp':
for i in range(ports_needed):
sock = socket.socket()
# struct.pack('ii', (0,0)) is 8 null bytes
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, b'\0' * 8)
sock.bind((ip, 0))
ports.append(sock)
for i, sock in enumerate(ports):
port = sock.getsockname()[1]
sock.close()
ports[i] = port
else:
N = 1
for i in range(ports_needed):
while os.path.exists("%s-%s" % (ip, str(N))):
N += 1
ports.append(N)
N += 1
if shell_port <= 0:
shell_port = ports.pop(0)
if iopub_port <= 0:
iopub_port = ports.pop(0)
if stdin_port <= 0:
stdin_port = ports.pop(0)
if control_port <= 0:
control_port = ports.pop(0)
if hb_port <= 0:
hb_port = ports.pop(0)
cfg = dict( shell_port=shell_port,
iopub_port=iopub_port,
stdin_port=stdin_port,
control_port=control_port,
hb_port=hb_port,
)
cfg['ip'] = ip
cfg['key'] = bytes_to_str(key)
cfg['transport'] = transport
cfg['signature_scheme'] = signature_scheme
cfg['kernel_name'] = kernel_name
with open(fname, 'w') as f:
f.write(json.dumps(cfg, indent=2))
if hasattr(stat, 'S_ISVTX'):
# set the sticky bit on the file and its parent directory
# to avoid periodic cleanup
paths = [fname]
runtime_dir = os.path.dirname(fname)
if runtime_dir:
paths.append(runtime_dir)
for path in paths:
permissions = os.stat(path).st_mode
new_permissions = permissions | stat.S_ISVTX
if new_permissions != permissions:
try:
os.chmod(path, permissions)
except OSError as e:
# failed to set sticky bit,
# probably not a big deal
warnings.warn(
"Failed to set sticky bit on %r: %s" % (path, e),
RuntimeWarning,
)
return fname, cfg
