def get_connection_info(self, session=False): info = super(RemoteKernelManager, self).get_connection_info(session) # Convert bytes to string for persistence. Will reverse operation in load_connection_info info_key = info.get('key') if info_key: info['key'] = bytes_to_str(info_key) return info
def run_python(parameters, ignore_return_code=False, stdin=None): """ Run python as a shell command, listening for both Errors and non-zero return codes. Returns the tuple (stdout, stderr) of output produced during the nbconvert run. Parameters ---------- parameters : str, list(str) List of parameters to pass to IPython. ignore_return_code : optional bool (default False) Throw an OSError if the return code """ cmd = [sys.executable] if sys.platform == 'win32': if isinstance(parameters, string_types): cmd = ' '.join(cmd) + ' ' + parameters else: cmd = ' '.join(cmd + parameters) else: if isinstance(parameters, string_types): parameters = shlex.split(parameters) cmd += parameters p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE) stdout, stderr = p.communicate(input=stdin) if not (p.returncode == 0 or ignore_return_code): raise OSError(bytes_to_str(stderr)) return stdout.decode('utf8', 'replace'), stderr.decode('utf8', 'replace')
def pre_save_transformation(sessions): sessions_copy = copy.deepcopy(sessions) for kernel_id, session in sessions_copy.items(): if session.get('connection_info'): info = session['connection_info'] key = info.get('key') if key: info['key'] = bytes_to_str(key) return sessions_copy
def pre_save_transformation(session): kernel_id = list(session.keys())[0] session_info = session[kernel_id] if session_info.get('connection_info'): info = session_info['connection_info'] key = info.get('key') if key: info['key'] = bytes_to_str(key) return session
def run(self): self.started = True while not self.stop.is_set(): chunk = os.read(self.readfd, 1024) with self.buffer_lock: self.buffer.write(chunk) if self.echo: sys.stdout.write(bytes_to_str(chunk)) os.close(self.readfd) os.close(self.writefd)
def wait(self, *pargs, **kwargs): """Wait for the JSController to finish""" ret = super(JSController, self).wait(*pargs, **kwargs) # If this is a SlimerJS controller, check the captured stdout for # errors. Otherwise, just return the return code. if self.engine == 'slimerjs': stdout = bytes_to_str(self.stdout) if ret != 0: # This could still happen e.g. if it's stopped by SIGINT return ret return bool(self.slimer_failure.search(stdout)) else: return ret
def nbconvert(self, parameters, ignore_return_code=False): """ Run nbconvert a, IPython shell command, listening for both Errors and non-zero return codes. Parameters ---------- parameters : str, list(str) List of parameters to pass to IPython. ignore_return_code : optional bool (default False) Throw an OSError if the return code """ if isinstance(parameters, string_types): parameters = shlex.split(parameters) cmd = [sys.executable, '-m', 'jupyter_nbconvert'] + parameters p = Popen(cmd, stdout=PIPE, stderr=PIPE) stdout, stderr = p.communicate() if not (p.returncode == 0 or ignore_return_code): raise OSError(bytes_to_str(stderr)) return stdout.decode('utf8', 'replace'), stderr.decode('utf8', 'replace')
def run_jstestall(options): """Run the entire Javascript test suite. This function constructs TestControllers and runs them in subprocesses. Parameters ---------- All parameters are passed as attributes of the options object. testgroups : list of str Run only these sections of the test suite. If empty, run all the available sections. fast : int or None Run the test suite in parallel, using n simultaneous processes. If None is passed, one process is used per CPU core. Default 1 (i.e. sequential) inc_slow : bool Include slow tests. By default, these tests aren't run. slimerjs : bool Use slimerjs if it's installed instead of phantomjs for casperjs tests. url : unicode Address:port to use when running the JS tests. xunit : bool Produce Xunit XML output. This is written to multiple foo.xunit.xml files. extra_args : list Extra arguments to pass to the test subprocesses, e.g. '-v' """ to_run, not_run = prepare_controllers(options) def justify(ltext, rtext, width=70, fill='-'): ltext += ' ' rtext = (' ' + rtext).rjust(width - len(ltext), fill) return ltext + rtext # Run all test runners, tracking execution time failed = [] t_start = time.time() print() if options.fast == 1: # This actually means sequential, i.e. with 1 job for controller in to_run: print('Test group:', controller.section) sys.stdout.flush() # Show in correct order when output is piped controller, res = do_run(controller, buffer_output=False) if res: failed.append(controller) if res == -signal.SIGINT: print("Interrupted") break print() else: # Run tests concurrently try: pool = multiprocessing.pool.ThreadPool(options.fast) for (controller, res) in pool.imap_unordered(do_run, to_run): res_string = 'OK' if res == 0 else 'FAILED' print(justify('Test group: ' + controller.section, res_string)) if res: controller.print_extra_info() print(bytes_to_str(controller.stdout)) failed.append(controller) if res == -signal.SIGINT: print("Interrupted") break except KeyboardInterrupt: return for controller in not_run: print(justify('Test group: ' + controller.section, 'NOT RUN')) t_end = time.time() t_tests = t_end - t_start nrunners = len(to_run) nfail = len(failed) # summarize results print('_'*70) print('Test suite completed for system with the following information:') print(report()) took = "Took %.3fs." % t_tests print('Status: ', end='') if not failed: print('OK (%d test groups).' % nrunners, took) else: # If anything went wrong, point out what command to rerun manually to # see the actual errors and individual summary failed_sections = [c.section for c in failed] print('ERROR - {} out of {} test groups failed ({}).'.format(nfail, nrunners, ', '.join(failed_sections)), took) print() print('You may wish to rerun these, with:') print(' python -m notebook.jstest', *failed_sections) print() if failed: # Ensure that our exit code indicates failure sys.exit(1)
def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0, control_port=0, ip='', key=b'', transport='tcp', signature_scheme='hmac-sha256', kernel_name=''): """Generates a JSON config file, including the selection of random ports. Parameters ---------- fname : unicode The path to the file to write shell_port : int, optional The port to use for ROUTER (shell) channel. iopub_port : int, optional The port to use for the SUB channel. stdin_port : int, optional The port to use for the ROUTER (raw input) channel. control_port : int, optional The port to use for the ROUTER (control) channel. hb_port : int, optional The port to use for the heartbeat REP channel. ip : str, optional The ip address the kernel will bind to. key : str, optional The Session key used for message authentication. signature_scheme : str, optional The scheme used for message authentication. This has the form 'digest-hash', where 'digest' is the scheme used for digests, and 'hash' is the name of the hash function used by the digest scheme. Currently, 'hmac' is the only supported digest scheme, and 'sha256' is the default hash function. kernel_name : str, optional The name of the kernel currently connected to. """ if not ip: ip = localhost() # default to temporary connector file if not fname: fd, fname = tempfile.mkstemp('.json') os.close(fd) # Find open ports as necessary. ports = [] ports_needed = int(shell_port <= 0) + \ int(iopub_port <= 0) + \ int(stdin_port <= 0) + \ int(control_port <= 0) + \ int(hb_port <= 0) if transport == 'tcp': for i in range(ports_needed): sock = socket.socket() # struct.pack('ii', (0,0)) is 8 null bytes sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, b'\0' * 8) sock.bind((ip, 0)) ports.append(sock) for i, sock in enumerate(ports): port = sock.getsockname()[1] sock.close() ports[i] = port else: N = 1 for i in range(ports_needed): while os.path.exists("%s-%s" % (ip, str(N))): N += 1 ports.append(N) N += 1 if shell_port <= 0: shell_port = ports.pop(0) if iopub_port <= 0: iopub_port = ports.pop(0) if stdin_port <= 0: stdin_port = ports.pop(0) if control_port <= 0: control_port = ports.pop(0) if hb_port <= 0: hb_port = ports.pop(0) cfg = dict( shell_port=shell_port, iopub_port=iopub_port, stdin_port=stdin_port, control_port=control_port, hb_port=hb_port, ) cfg['ip'] = ip cfg['key'] = bytes_to_str(key) cfg['transport'] = transport cfg['signature_scheme'] = signature_scheme cfg['kernel_name'] = kernel_name # Only ever write this file as user read/writeable # This would otherwise introduce a vulnerability as a file has secrets # which would let others execute arbitrarily code as you with secure_write(fname) as f: f.write(json.dumps(cfg, indent=2)) if hasattr(stat, 'S_ISVTX'): # set the sticky bit on the file and its parent directory # to avoid periodic cleanup paths = [fname] runtime_dir = os.path.dirname(fname) if runtime_dir: paths.append(runtime_dir) for path in paths: permissions = os.stat(path).st_mode new_permissions = permissions | stat.S_ISVTX if new_permissions != permissions: try: os.chmod(path, new_permissions) except OSError as e: if e.errno == errno.EPERM and path == runtime_dir: # suppress permission errors setting sticky bit on runtime_dir, # which we may not own. pass else: # failed to set sticky bit, probably not a big deal warnings.warn( "Failed to set sticky bit on %r: %s" "\nProbably not a big deal, but runtime files may be cleaned up periodically." % (path, e), RuntimeWarning, ) return fname, cfg
def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0, control_port=0, ip='', key=b'', transport='tcp', signature_scheme='hmac-sha256', ): """Generates a JSON config file, including the selection of random ports. Parameters ---------- fname : unicode The path to the file to write shell_port : int, optional The port to use for ROUTER (shell) channel. iopub_port : int, optional The port to use for the SUB channel. stdin_port : int, optional The port to use for the ROUTER (raw input) channel. control_port : int, optional The port to use for the ROUTER (control) channel. hb_port : int, optional The port to use for the heartbeat REP channel. ip : str, optional The ip address the kernel will bind to. key : str, optional The Session key used for message authentication. signature_scheme : str, optional The scheme used for message authentication. This has the form 'digest-hash', where 'digest' is the scheme used for digests, and 'hash' is the name of the hash function used by the digest scheme. Currently, 'hmac' is the only supported digest scheme, and 'sha256' is the default hash function. """ if not ip: ip = localhost() # default to temporary connector file if not fname: fd, fname = tempfile.mkstemp('.json') os.close(fd) # Find open ports as necessary. ports = [] ports_needed = int(shell_port <= 0) + \ int(iopub_port <= 0) + \ int(stdin_port <= 0) + \ int(control_port <= 0) + \ int(hb_port <= 0) if transport == 'tcp': for i in range(ports_needed): sock = socket.socket() # struct.pack('ii', (0,0)) is 8 null bytes sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, b'\0' * 8) sock.bind(('', 0)) ports.append(sock) for i, sock in enumerate(ports): port = sock.getsockname()[1] sock.close() ports[i] = port else: N = 1 for i in range(ports_needed): while os.path.exists("%s-%s" % (ip, str(N))): N += 1 ports.append(N) N += 1 if shell_port <= 0: shell_port = ports.pop(0) if iopub_port <= 0: iopub_port = ports.pop(0) if stdin_port <= 0: stdin_port = ports.pop(0) if control_port <= 0: control_port = ports.pop(0) if hb_port <= 0: hb_port = ports.pop(0) cfg = dict( shell_port=shell_port, iopub_port=iopub_port, stdin_port=stdin_port, control_port=control_port, hb_port=hb_port, ) cfg['ip'] = ip cfg['key'] = bytes_to_str(key) cfg['transport'] = transport cfg['signature_scheme'] = signature_scheme with open(fname, 'w') as f: f.write(json.dumps(cfg, indent=2)) return fname, cfg
def run_jstestall(options): """Run the entire Javascript test suite. This function constructs TestControllers and runs them in subprocesses. Parameters ---------- All parameters are passed as attributes of the options object. testgroups : list of str Run only these sections of the test suite. If empty, run all the available sections. fast : int or None Run the test suite in parallel, using n simultaneous processes. If None is passed, one process is used per CPU core. Default 1 (i.e. sequential) inc_slow : bool Include slow tests. By default, these tests aren't run. slimerjs : bool Use slimerjs if it's installed instead of phantomjs for casperjs tests. url : unicode Address:port to use when running the JS tests. xunit : bool Produce Xunit XML output. This is written to multiple foo.xunit.xml files. extra_args : list Extra arguments to pass to the test subprocesses, e.g. '-v' """ to_run, not_run = prepare_controllers(options) def justify(ltext, rtext, width=70, fill='-'): ltext += ' ' rtext = (' ' + rtext).rjust(width - len(ltext), fill) return ltext + rtext # Run all test runners, tracking execution time failed = [] t_start = time.time() print() if options.fast == 1: # This actually means sequential, i.e. with 1 job for controller in to_run: print('Test group:', controller.section) sys.stdout.flush() # Show in correct order when output is piped controller, res = do_run(controller, buffer_output=False) if res: failed.append(controller) if res == -signal.SIGINT: print("Interrupted") break print() else: # Run tests concurrently try: pool = multiprocessing.pool.ThreadPool(options.fast) for (controller, res) in pool.imap_unordered(do_run, to_run): res_string = 'OK' if res == 0 else 'FAILED' print(justify('Test group: ' + controller.section, res_string)) if res: controller.print_extra_info() print(bytes_to_str(controller.stdout)) failed.append(controller) if res == -signal.SIGINT: print("Interrupted") break except KeyboardInterrupt: return for controller in not_run: print(justify('Test group: ' + controller.section, 'NOT RUN')) t_end = time.time() t_tests = t_end - t_start nrunners = len(to_run) nfail = len(failed) # summarize results print('_' * 70) print('Test suite completed for system with the following information:') print(report()) took = "Took %.3fs." % t_tests print('Status: ', end='') if not failed: print('OK (%d test groups).' % nrunners, took) else: # If anything went wrong, point out what command to rerun manually to # see the actual errors and individual summary failed_sections = [c.section for c in failed] print( 'ERROR - {} out of {} test groups failed ({}).'.format( nfail, nrunners, ', '.join(failed_sections)), took) print() print('You may wish to rerun these, with:') print(' python -m notebook.jstest', *failed_sections) print() if failed: # Ensure that our exit code indicates failure sys.exit(1)
def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0, control_port=0, ip='', key=b'', transport='tcp', signature_scheme='hmac-sha256', kernel_name='' ): """Generates a JSON config file, including the selection of random ports. Parameters ---------- fname : unicode The path to the file to write shell_port : int, optional The port to use for ROUTER (shell) channel. iopub_port : int, optional The port to use for the SUB channel. stdin_port : int, optional The port to use for the ROUTER (raw input) channel. control_port : int, optional The port to use for the ROUTER (control) channel. hb_port : int, optional The port to use for the heartbeat REP channel. ip : str, optional The ip address the kernel will bind to. key : str, optional The Session key used for message authentication. signature_scheme : str, optional The scheme used for message authentication. This has the form 'digest-hash', where 'digest' is the scheme used for digests, and 'hash' is the name of the hash function used by the digest scheme. Currently, 'hmac' is the only supported digest scheme, and 'sha256' is the default hash function. kernel_name : str, optional The name of the kernel currently connected to. """ if not ip: ip = localhost() # default to temporary connector file if not fname: fd, fname = tempfile.mkstemp('.json') os.close(fd) # Find open ports as necessary. ports = [] ports_needed = int(shell_port <= 0) + \ int(iopub_port <= 0) + \ int(stdin_port <= 0) + \ int(control_port <= 0) + \ int(hb_port <= 0) if transport == 'tcp': for i in range(ports_needed): sock = socket.socket() # struct.pack('ii', (0,0)) is 8 null bytes sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, b'\0' * 8) sock.bind((ip, 0)) ports.append(sock) for i, sock in enumerate(ports): port = sock.getsockname()[1] sock.close() ports[i] = port else: N = 1 for i in range(ports_needed): while os.path.exists("%s-%s" % (ip, str(N))): N += 1 ports.append(N) N += 1 if shell_port <= 0: shell_port = ports.pop(0) if iopub_port <= 0: iopub_port = ports.pop(0) if stdin_port <= 0: stdin_port = ports.pop(0) if control_port <= 0: control_port = ports.pop(0) if hb_port <= 0: hb_port = ports.pop(0) cfg = dict( shell_port=shell_port, iopub_port=iopub_port, stdin_port=stdin_port, control_port=control_port, hb_port=hb_port, ) cfg['ip'] = ip cfg['key'] = bytes_to_str(key) cfg['transport'] = transport cfg['signature_scheme'] = signature_scheme cfg['kernel_name'] = kernel_name with open(fname, 'w') as f: f.write(json.dumps(cfg, indent=2)) if hasattr(stat, 'S_ISVTX'): # set the sticky bit on the file and its parent directory # to avoid periodic cleanup paths = [fname] runtime_dir = os.path.dirname(fname) if runtime_dir: paths.append(runtime_dir) for path in paths: permissions = os.stat(path).st_mode new_permissions = permissions | stat.S_ISVTX if new_permissions != permissions: try: os.chmod(path, permissions) except OSError as e: # failed to set sticky bit, # probably not a big deal warnings.warn( "Failed to set sticky bit on %r: %s" % (path, e), RuntimeWarning, ) return fname, cfg