def restore_bytes(nb): """Restore bytes of image data from unicode-only formats. Base64 encoding is handled elsewhere. Bytes objects in the notebook are always b64-encoded. We DO NOT encode/decode around file formats. """ for ws in nb.worksheets: for cell in ws.cells: if cell.cell_type == 'code': for output in cell.outputs: if 'png' in output: output.png = str_to_bytes(output.png, 'ascii') if 'jpeg' in output: output.jpeg = str_to_bytes(output.jpeg, 'ascii') return nb
def handle_pong(self, msg): "a heart just beat" current = str_to_bytes(str(self.lifetime)) last = str_to_bytes(str(self.last_ping)) if msg[1] == current: delta = time.time()-self.tic if self.debug: self.log.debug("heartbeat::heart %r took %.2f ms to respond", msg[0], 1000*delta) self.responses.add(msg[0]) elif msg[1] == last: delta = time.time()-self.tic + (self.lifetime-self.last_ping) self.log.warn("heartbeat::heart %r missed a beat, and took %.2f ms to respond", msg[0], 1000*delta) self.responses.add(msg[0]) else: self.log.warn("heartbeat::got bad heartbeat (possibly old?): %s (current=%.3f)", msg[1], self.lifetime)
def get_connection_info(connection_file=None, unpack=False, profile=None): """Return the connection information for the current Kernel. Parameters ---------- connection_file : str [optional] The connection file to be used. Can be given by absolute path, or IPython will search in the security directory of a given profile. If run from IPython, If unspecified, the connection file for the currently running IPython Kernel will be used, which is only allowed from inside a kernel. unpack : bool [default: False] if True, return the unpacked dict, otherwise just the string contents of the file. profile : DEPRECATED Returns ------- The connection dictionary of the current kernel, as string or dict, depending on `unpack`. """ cf = _find_connection_file(connection_file, profile) with open(cf) as f: info = f.read() if unpack: info = json.loads(info) # ensure key is bytes: info['key'] = str_to_bytes(info.get('key', '')) return info
def beat(self): self.pongstream.flush() self.last_ping = self.lifetime toc = time.time() self.lifetime += toc-self.tic self.tic = toc if self.debug: self.log.debug("heartbeat::sending %s", self.lifetime) goodhearts = self.hearts.intersection(self.responses) missed_beats = self.hearts.difference(goodhearts) newhearts = self.responses.difference(goodhearts) for heart in newhearts: self.handle_new_heart(heart) heartfailures, on_probation = self._check_missed(missed_beats, self.on_probation, self.hearts) for failure in heartfailures: self.handle_heart_failure(failure) self.on_probation = on_probation self.responses = set() #print self.on_probation, self.hearts # self.log.debug("heartbeat::beat %.3f, %i beating hearts", self.lifetime, len(self.hearts)) self.pingstream.send(str_to_bytes(str(self.lifetime))) # flush stream to force immediate socket send self.pingstream.flush()
def test_encode_images(): # invalid data, but the header and footer are from real files pngdata = b'\x89PNG\r\n\x1a\nblahblahnotactuallyvalidIEND\xaeB`\x82' jpegdata = b'\xff\xd8\xff\xe0\x00\x10JFIFblahblahjpeg(\xa0\x0f\xff\xd9' pdfdata = b'%PDF-1.\ntrailer<</Root<</Pages<</Kids[<</MediaBox[0 0 3 3]>>]>>>>>>' fmt = { 'image/png' : pngdata, 'image/jpeg' : jpegdata, 'application/pdf' : pdfdata } encoded = encode_images(fmt) for key, value in iteritems(fmt): # encoded has unicode, want bytes decoded = decodestring(encoded[key].encode('ascii')) nt.assert_equal(decoded, value) encoded2 = encode_images(encoded) nt.assert_equal(encoded, encoded2) b64_str = {} for key, encoded in iteritems(encoded): b64_str[key] = unicode_to_str(encoded) encoded3 = encode_images(b64_str) nt.assert_equal(encoded3, b64_str) for key, value in iteritems(fmt): # encoded3 has str, want bytes decoded = decodestring(str_to_bytes(encoded3[key])) nt.assert_equal(decoded, value)
def test_write_connection_file(): with TemporaryDirectory() as d: cf = os.path.join(d, 'kernel.json') connect.write_connection_file(cf, **sample_info) nt.assert_true(os.path.exists(cf)) with open(cf, 'r') as f: info = json.load(f) info['key'] = str_to_bytes(info['key']) nt.assert_equal(info, sample_info)
def handle_pong(self, msg): "a heart just beat" current = str_to_bytes(str(self.lifetime)) last = str_to_bytes(str(self.last_ping)) if msg[1] == current: delta = time.time() - self.tic if self.debug: self.log.debug("heartbeat::heart %r took %.2f ms to respond", msg[0], 1000 * delta) self.responses.add(msg[0]) elif msg[1] == last: delta = time.time() - self.tic + (self.lifetime - self.last_ping) self.log.warn( "heartbeat::heart %r missed a beat, and took %.2f ms to respond", msg[0], 1000 * delta) self.responses.add(msg[0]) else: self.log.warn( "heartbeat::got bad heartbeat (possibly old?): %s (current=%.3f)", msg[1], self.lifetime)
def passwd(passphrase=None, algorithm='argon2'): """Generate hashed password and salt for use in notebook configuration. In the notebook configuration, set `c.NotebookApp.password` to the generated string. Parameters ---------- passphrase : str Password to hash. If unspecified, the user is asked to input and verify a password. algorithm : str Hashing algorithm to use (e.g, 'sha1' or any argument supported by :func:`hashlib.new`, or 'argon2'). Returns ------- hashed_passphrase : str Hashed password, in the format 'hash_algorithm:salt:passphrase_hash'. Examples -------- >>> passwd('mypassword') 'sha1:7cf3:b7d6da294ea9592a9480c8f52e63cd42cfb9dd12' """ if passphrase is None: for i in range(3): p0 = getpass.getpass('Enter password: '******'Verify password: '******'Passwords do not match.') else: raise ValueError('No matching passwords found. Giving up.') if algorithm == 'argon2': from argon2 import PasswordHasher ph = PasswordHasher( memory_cost=10240, time_cost=10, parallelism=8, ) h = ph.hash(passphrase) return ':'.join((algorithm, cast_unicode(h, 'ascii'))) else: h = hashlib.new(algorithm) salt = ('%0' + str(salt_len) + 'x') % random.getrandbits(4 * salt_len) h.update(cast_bytes(passphrase, 'utf-8') + str_to_bytes(salt, 'ascii')) return ':'.join((algorithm, salt, h.hexdigest()))
def default_secure(cfg): """Set the default behavior for a config environment to be secure. If Session.key/keyfile have not been set, set Session.key to a new random UUID. """ warnings.warn("default_secure is deprecated", DeprecationWarning) if 'Session' in cfg: if 'key' in cfg.Session or 'keyfile' in cfg.Session: return # key/keyfile not specified, generate new UUID: cfg.Session.key = str_to_bytes(str(uuid.uuid4()))
def test_get_connection_info(): with TemporaryDirectory() as d: cf = os.path.join(d, 'kernel.json') connect.write_connection_file(cf, **sample_info) json_info = connect.get_connection_info(cf) info = connect.get_connection_info(cf, unpack=True) nt.assert_equal(type(json_info), type("")) nt.assert_equal(info, sample_info) info2 = json.loads(json_info) info2['key'] = str_to_bytes(info2['key']) nt.assert_equal(info2, sample_info)
def sign(self, msg_list): """Sign a message with HMAC digest. If no auth, return b''. Parameters ---------- msg_list : list The [p_header,p_parent,p_content] part of the message list. """ if self.auth is None: return b'' h = self.auth.copy() for m in msg_list: h.update(m) return str_to_bytes(h.hexdigest())
def test_get_connection_info(): with TemporaryDirectory() as d: cf = os.path.join(d, 'kernel.json') connect.write_connection_file(cf, **sample_info) json_info = connect.get_connection_info(cf) info = connect.get_connection_info(cf, unpack=True) assert isinstance(json_info, str) sub_info = {k:v for k,v in info.items() if k in sample_info} assert sub_info == sample_info info2 = json.loads(json_info) info2['key'] = str_to_bytes(info2['key']) sub_info2 = {k:v for k,v in info.items() if k in sample_info} assert sub_info2 == sample_info
def jupyter_password(passphrase='',salt='0'*12,algorithm='sha1'): import hashlib from ipython_genutils.py3compat import cast_bytes,str_to_bytes if py.isbytes(salt): bsalt=salt salt=bsalt.decode('ascii') elif py.istr(salt): bsalt=str_to_bytes(salt, 'ascii') else: raise py.ArgumentError(salt) h=hashlib.new(algorithm) h.update(cast_bytes(passphrase, 'utf-8') + bsalt) return ':'.join((algorithm, salt, h.hexdigest()))
def test_get_connection_info(): with TemporaryDirectory() as d: cf = os.path.join(d, 'kernel.json') connect.write_connection_file(cf, **sample_info) json_info = connect.get_connection_info(cf) info = connect.get_connection_info(cf, unpack=True) assert isinstance(json_info, str) sub_info = {k: v for k, v in info.items() if k in sample_info} assert sub_info == sample_info info2 = json.loads(json_info) info2['key'] = str_to_bytes(info2['key']) sub_info2 = {k: v for k, v in info.items() if k in sample_info} assert sub_info2 == sample_info
def load_connection_file(self): """Load connection info from JSON dict in self.connection_file.""" self.log.debug(u"Loading connection file %s", self.connection_file) with open(self.connection_file) as f: cfg = json.load(f) self.transport = cfg.get('transport', self.transport) self.ip = cfg.get('ip', self._ip_default()) for name in port_names: if getattr(self, name) == 0 and name in cfg: # not overridden by config or cl_args setattr(self, name, cfg[name]) if 'key' in cfg: self.session.key = str_to_bytes(cfg['key']) if 'signature_scheme' in cfg: self.session.signature_scheme = cfg['signature_scheme']
def passwd(passphrase=None, algorithm='sha1'): """Generate hashed password and salt for use in notebook configuration. In the notebook configuration, set `c.NotebookApp.password` to the generated string. Parameters ---------- passphrase : str Password to hash. If unspecified, the user is asked to input and verify a password. algorithm : str Hashing algorithm to use (e.g, 'sha1' or any argument supported by :func:`hashlib.new`). Returns ------- hashed_passphrase : str Hashed password, in the format 'hash_algorithm:salt:passphrase_hash'. Examples -------- >>> passwd('mypassword') 'sha1:7cf3:b7d6da294ea9592a9480c8f52e63cd42cfb9dd12' """ if passphrase is None: for i in range(3): p0 = getpass.getpass('Enter password: '******'Verify password: '******'Passwords do not match.') else: raise ValueError('No matching passwords found. Giving up.') h = hashlib.new(algorithm) salt = ('%0' + str(salt_len) + 'x') % random.getrandbits(4 * salt_len) h.update(cast_bytes(passphrase, 'utf-8') + str_to_bytes(salt, 'ascii')) return ':'.join((algorithm, salt, h.hexdigest()))
def passwd(passphrase=None, algorithm="sha1"): """Generate hashed password and salt for use in server configuration. In the server configuration, set `c.ServerApp.password` to the generated string. Parameters ---------- passphrase : str Password to hash. If unspecified, the user is asked to input and verify a password. algorithm : str Hashing algorithm to use (e.g, 'sha1' or any argument supported by :func:`hashlib.new`). Returns ------- hashed_passphrase : str Hashed password, in the format 'hash_algorithm:salt:passphrase_hash'. Examples -------- >>> passwd('mypassword') 'sha1:7cf3:b7d6da294ea9592a9480c8f52e63cd42cfb9dd12' """ if passphrase is None: for i in range(3): p0 = getpass.getpass("Enter password: "******"Verify password: "******"Passwords do not match.") else: raise ValueError("No matching passwords found. Giving up.") h = hashlib.new(algorithm) salt = ("%0" + str(salt_len) + "x") % random.getrandbits(4 * salt_len) h.update(cast_bytes(passphrase, "utf-8") + str_to_bytes(salt, "ascii")) return ":".join((algorithm, salt, h.hexdigest()))
def get_connection_info(connection_file=None, unpack=False, profile=None): """Return the connection information for the current Kernel. Parameters ---------- connection_file : str [optional] The connection file to be used. Can be given by absolute path, or IPython will search in the security directory of a given profile. If run from IPython, If unspecified, the connection file for the currently running IPython Kernel will be used, which is only allowed from inside a kernel. unpack : bool [default: False] if True, return the unpacked dict, otherwise just the string contents of the file. profile : str [optional] The name of the profile to use when searching for the connection file, if different from the current IPython session or 'default'. Returns ------- The connection dictionary of the current kernel, as string or dict, depending on `unpack`. """ if connection_file is None: # get connection file from current kernel cf = get_connection_file() else: # connection file specified, allow shortnames: cf = find_connection_file(connection_file, profile=profile) with open(cf) as f: info = f.read() if unpack: info = json.loads(info) # ensure key is bytes: info['key'] = str_to_bytes(info.get('key', '')) return info
def setup_gateway_listener(fname, parent_pid, lower_port, upper_port): ip = "0.0.0.0" key = str_to_bytes(str(uuid.uuid4())) gateway_socket = prepare_gateway_socket(lower_port, upper_port) gateway_listener_thread = Thread(target=gateway_listener, args=( gateway_socket, parent_pid, )) gateway_listener_thread.start() basename = os.path.splitext(os.path.basename(fname))[0] fd, conn_file = tempfile.mkstemp(suffix=".json", prefix=basename + "_") os.close(fd) ports = _select_ports(5, lower_port, upper_port) conn_file, config = write_connection_file(fname=conn_file, ip=ip, key=key, shell_port=ports[0], iopub_port=ports[1], stdin_port=ports[2], hb_port=ports[3], control_port=ports[4]) try: os.remove(conn_file) except: pass # Add in the gateway_socket and parent_pid fields... config['comm_port'] = gateway_socket.getsockname()[1] config['pid'] = parent_pid with open(fname, 'w') as f: f.write(json.dumps(config, indent=2)) return
def _key_default(self): return str_to_bytes(str(uuid.uuid4()))
# -*- encoding: utf-8 -*- import os import fileinput import hashlib import random from ipython_genutils.py3compat import cast_bytes, str_to_bytes # Get the password from the environment password_environment_variable = os.environ.get('JUPYTER_PASSWORD') # Hash the password, this is taken from https://github.com/jupyter/notebook/blob/master/notebook/auth/security.py salt_len = 12 algorithm = 'sha1' h = hashlib.new(algorithm) salt = ('%0' + str(salt_len) + 'x') % random.getrandbits(4 * salt_len) h.update(cast_bytes(password_environment_variable, 'utf-8') + str_to_bytes(salt, 'ascii')) password = '******'.join((algorithm, salt, h.hexdigest())) # Store the password in the configuration setup_line = "#c.NotebookApp.password = ''" new_setup_line = setup_line.replace("''", "u'" + password + "'") new_setup_line = new_setup_line.replace("#", "") setup_file = os.getenv("HOME") + "/.jupyter/jupyter_notebook_config.py" for line in fileinput.input(setup_file, inplace=True): print line.replace(setup_line, new_setup_line), for line in fileinput.input(setup_file, inplace=True): print line.replace("#c.NotebookApp.password_required = False", "c.NotebookApp.password_required = True"),
DELIM = b"<IDS|MSG>" # singleton dummy tracker, which will always report as done DONE = zmq.MessageTracker() #----------------------------------------------------------------------------- # Mixin tools for apps that use Sessions #----------------------------------------------------------------------------- session_aliases = dict( ident = 'Session.session', user = '******', keyfile = 'Session.keyfile', ) session_flags = { 'secure' : ({'Session' : { 'key' : str_to_bytes(str(uuid.uuid4())), 'keyfile' : '' }}, """Use HMAC digests for authentication of messages. Setting this flag will generate a new UUID to use as the HMAC key. """), 'no-secure' : ({'Session' : { 'key' : b'', 'keyfile' : '' }}, """Don't authenticate messages."""), } def default_secure(cfg): """Set the default behavior for a config environment to be secure. If Session.key/keyfile have not been set, set Session.key to a new random UUID. """ warnings.warn("default_secure is deprecated", DeprecationWarning)
def _hash(self, filename): """Compute the hash of a file.""" md5 = hashlib.md5() with open(filename) as f: md5.update(str_to_bytes(f.read())) return md5.digest()
# since 'lazy' and 'eager' behave the same, just check if not none create_spark_context = can_create_spark_context and init_mode != 'none' # placeholder objects for Spark session variables which are initialized in a background thread if create_spark_context: spark = WaitingForSparkSessionToBeInitialized(global_variable_name='spark') sc = WaitingForSparkSessionToBeInitialized(global_variable_name='sc') sqlContext = WaitingForSparkSessionToBeInitialized(global_variable_name='sqlContext') sqlCtx = WaitingForSparkSessionToBeInitialized(global_variable_name='sqlCtx') thread_to_initialize_spark_session = Thread(target=initialize_spark_session) # If the connection file doesn't exist, then create it. if not os.path.isfile(connection_file): key = str_to_bytes(str(uuid.uuid4())) connection_file = determine_connection_file(connection_file) ports = _select_ports(5, lower_port, upper_port) write_connection_file(fname=connection_file, ip=ip, key=key, shell_port=ports[0], iopub_port=ports[1], stdin_port=ports[2], hb_port=ports[3], control_port=ports[4]) if response_addr: gateway_socket = return_connection_info(connection_file, response_addr, disable_gateway_socket, lower_port, upper_port) if gateway_socket: # socket in use, start gateway listener thread gateway_listener_thread = Thread(target=gateway_listener, args=(gateway_socket,)) gateway_listener_thread.start() # start to initialize the Spark session in the background if create_spark_context:
#!/bin/env python3 # -*- encoding: utf-8 -*- import os import fileinput import hashlib import random from ipython_genutils.py3compat import cast_bytes, str_to_bytes # Get the password from the environment env_pass = os.environ.get('JUP_PASSWD') if env_pass == "": exit() # Hash the password, this is taken from # https://github.com/jupyter/notebook/blob/master/notebook/auth/security.py salt_len = 12 algorithm = 'sha1' h = hashlib.new(algorithm) salt = ('%0' + str(salt_len) + 'x') % random.getrandbits(4 * salt_len) h.update(cast_bytes(env_pass, 'utf-8') + str_to_bytes(salt, 'ascii')) password = '******'.join((algorithm, salt, h.hexdigest())) print(password)
DONE = zmq.MessageTracker() #----------------------------------------------------------------------------- # Mixin tools for apps that use Sessions #----------------------------------------------------------------------------- session_aliases = dict( ident='Session.session', user='******', keyfile='Session.keyfile', ) session_flags = { 'secure': ({ 'Session': { 'key': str_to_bytes(str(uuid.uuid4())), 'keyfile': '' } }, """Use HMAC digests for authentication of messages. Setting this flag will generate a new UUID to use as the HMAC key. """), 'no-secure': ({ 'Session': { 'key': b'', 'keyfile': '' } }, """Don't authenticate messages."""), } def default_secure(cfg):
def load_connection_info(self, info): # get the key back to bytes... info_key = info.get('key') if info_key: info['key'] = str_to_bytes(info_key) return super(RemoteKernelManager, self).load_connection_info(info)
parser.add_argument( '--RemoteProcessProxy.disable-gateway-socket', dest='disable_gateway_socket', action='store_true', help='Disable use of gateway socket for extended communications', default=False) arguments = vars(parser.parse_args()) connection_file = arguments['connection_file'] response_addr = arguments['response_address'] disable_gateway_socket = arguments['disable_gateway_socket'] ip = "0.0.0.0" # If the connection file doesn't exist, then create it. if not os.path.isfile(connection_file): key = str_to_bytes(str(uuid.uuid4())) connection_file = determine_connection_file(connection_file) write_connection_file(fname=connection_file, ip=ip, key=key) if response_addr: gateway_socket = return_connection_info(connection_file, ip, response_addr, disable_gateway_socket) if gateway_socket: # socket in use, start gateway listener thread gateway_listener_thread = Thread(target=gateway_listener, args=(gateway_socket, )) gateway_listener_thread.start() # start to initialize the Spark session in the background thread_to_initialize_spark_session.start()
import hashlib import random import re from ipython_genutils.py3compat import cast_bytes, str_to_bytes # Get the password from the environment password_environment_variable = sys.argv[1] # Hash the password, this is taken from https://github.com/jupyter/notebook/blob/master/notebook/auth/security.py salt_len = 12 algorithm = 'sha1' h = hashlib.new(algorithm) salt = ('%0' + str(salt_len) + 'x') % random.getrandbits(4 * salt_len) h.update( cast_bytes(password_environment_variable, 'utf-8') + str_to_bytes(salt, 'ascii')) password = '******'.join((algorithm, salt, h.hexdigest())) # Store the password in the configuration setup_line = "c.ServerApp.password = "******"u'" + password + "'" new_setup_line = new_setup_line.replace("# ", "") setup_file = os.getenv("HOME") + "/.jupyter/jupyter_server_config.py" if not os.path.exists(setup_file): os.system('jupyter server --generate-config') for line in fileinput.input(setup_file, inplace=True): m = re.search(setup_line, line) if m: print(new_setup_line)