def perm_to_403(self, os_path=''): """context manager for turning permission errors into 403.""" try: yield except (OSError, IOError) as e: if e.errno in {errno.EPERM, errno.EACCES}: # make 403 error message without root prefix # this may not work perfectly on unicode paths on Python 2, # but nobody should be doing that anyway. if not os_path: os_path = str_to_unicode(e.filename or 'unknown file') path = to_api_path(os_path, root=self.root_dir) # iOS: better error message import sys if (sys.platform == "darwin" and os.uname().machine.startswith("iP")): raise HTTPError( 403, u'Permission denied: %s (maybe you need to grand permission on the folder)' % path) from e else: raise HTTPError(403, u'Permission denied: %s' % path) from e else: raise
def complete_request(self, text): line = str_to_unicode(readline.get_line_buffer()) byte_cursor_pos = readline.get_endidx() # get_endidx is a byte offset # account for multi-byte characters to get correct cursor_pos bytes_before_cursor = cast_bytes(line)[:byte_cursor_pos] cursor_pos = len(cast_unicode(bytes_before_cursor)) # send completion request to kernel # Give the kernel up to 5s to respond msg_id = self.client.complete( code=line, cursor_pos=cursor_pos, ) msg = self.client.shell_channel.get_msg(timeout=self.timeout) if msg['parent_header']['msg_id'] == msg_id: content = msg['content'] cursor_start = content['cursor_start'] matches = [line[:cursor_start] + m for m in content['matches']] if content["cursor_end"] < cursor_pos: extra = line[content["cursor_end"]:cursor_pos] matches = [m + extra for m in matches] matches = [unicode_to_str(m) for m in matches] return matches return []
def write(self, nb, fp, **kwargs): """Write a notebook to a file like object""" nbs = self.writes(nb, **kwargs) if not py3compat.PY3 and not isinstance(nbs, unicode_type): # this branch is likely only taken for JSON on Python 2 nbs = py3compat.str_to_unicode(nbs) return fp.write(nbs)
def write(self, nb, fp, **kwargs): """Write a notebook to a file like object""" nbs = self.writes(nb,**kwargs) if not py3compat.PY3 and not isinstance(nbs, unicode_type): # this branch is likely only taken for JSON on Python 2 nbs = py3compat.str_to_unicode(nbs) return fp.write(nbs)
def complete_request(self, text): line = str_to_unicode(readline.get_line_buffer()) byte_cursor_pos = readline.get_endidx() # get_endidx is a byte offset # account for multi-byte characters to get correct cursor_pos bytes_before_cursor = cast_bytes(line)[:byte_cursor_pos] cursor_pos = len(cast_unicode(bytes_before_cursor)) # send completion request to kernel # Give the kernel up to 5s to respond msg_id = self.client.complete( code=line, cursor_pos=cursor_pos, ) msg = self.client.shell_channel.get_msg(timeout=self.timeout) if msg['parent_header']['msg_id'] == msg_id: content = msg['content'] cursor_start = content['cursor_start'] matches = [ line[:cursor_start] + m for m in content['matches'] ] if content["cursor_end"] < cursor_pos: extra = line[content["cursor_end"]: cursor_pos] matches = [m + extra for m in matches] matches = [ unicode_to_str(m) for m in matches ] return matches return []
def get_home_dir(): """Get the real path of the home directory""" homedir = os.path.expanduser('~') # Next line will make things work even when /home/ is a symlink to # /usr/home as it is on FreeBSD, for example homedir = os.path.realpath(homedir) homedir = py3compat.str_to_unicode(homedir, encoding=sys.getfilesystemencoding()) return homedir
def url_unescape(path): """Unescape special characters in a URL path Turns '/foo%20bar/' into '/foo bar/' """ return u'/'.join([ py3compat.str_to_unicode(unquote(p), encoding='utf8') for p in py3compat.unicode_to_str(path, encoding='utf8').split('/') ])
def _check_created(self, resp, path, type='notebook'): self.assertEqual(resp.status_code, 201) location_header = py3compat.str_to_unicode(resp.headers['Location']) self.assertEqual(location_header, url_path_join(self.url_prefix, u'api/contents', url_escape(path))) rjson = resp.json() self.assertEqual(rjson['name'], path.rsplit('/', 1)[-1]) self.assertEqual(rjson['path'], path) self.assertEqual(rjson['type'], type) isright = self.isdir if type == 'directory' else self.isfile assert isright(path)
def writes(self, nb, **kwargs): kwargs['cls'] = BytesEncoder kwargs['indent'] = 1 kwargs['sort_keys'] = True kwargs['separators'] = (',', ': ') nb = copy.deepcopy(nb) nb = strip_transient(nb) if kwargs.pop('split_lines', True): nb = split_lines(nb) return py3compat.str_to_unicode(json.dumps(nb, **kwargs), 'utf-8')
def writes(self, nb, **kwargs): kwargs["cls"] = BytesEncoder kwargs["indent"] = 1 kwargs["sort_keys"] = True kwargs["separators"] = (",", ": ") nb = copy.deepcopy(nb) nb = strip_transient(nb) if kwargs.pop("split_lines", True): nb = split_lines(nb) return py3compat.str_to_unicode(json.dumps(nb, **kwargs), "utf-8")
def perm_to_403(self, os_path=''): """context manager for turning permission errors into 403.""" try: yield except (OSError, IOError) as e: if e.errno in {errno.EPERM, errno.EACCES}: # make 403 error message without root prefix # this may not work perfectly on unicode paths on Python 2, # but nobody should be doing that anyway. if not os_path: os_path = str_to_unicode(e.filename or 'unknown file') path = to_api_path(os_path, root=self.root_dir) raise HTTPError(403, u'Permission denied: %s' % path) else: raise
def rlcomplete(self, text, state): if state == 0: line = str_to_unicode(readline.get_line_buffer()) byte_cursor_pos = readline.get_endidx() # get_endidx is a byte offset # account for multi-byte characters to get correct cursor_pos bytes_before_cursor = cast_bytes(line)[:byte_cursor_pos] cursor_pos = len(cast_unicode(bytes_before_cursor)) try: content = self.complete_request(line, cursor_pos) self.matches = _construct_readline_matches(line, cursor_pos, content) except Empty: #print('WARNING: Kernel timeout on tab completion.') pass try: return self.matches[state] except IndexError: return None
def rlcomplete(self, text, state): if state == 0: line = str_to_unicode(readline.get_line_buffer()) byte_cursor_pos = readline.get_endidx() # get_endidx is a byte offset # account for multi-byte characters to get correct cursor_pos bytes_before_cursor = cast_bytes(line)[:byte_cursor_pos] cursor_pos = len(cast_unicode(bytes_before_cursor)) try: content = self.complete_request(line, cursor_pos) self.matches = _construct_readline_matches( line, cursor_pos, content) except Empty: #print('WARNING: Kernel timeout on tab completion.') pass try: return self.matches[state] except IndexError: return None
def from_filename(self, filename, resources=None, **kw): """ Convert a notebook from a notebook file. Parameters ---------- filename : str Full filename of the notebook file to open and convert. resources : dict Additional resources that can be accessed read/write by preprocessors and filters. `**kw` Ignored """ # Convert full filename string to unicode # In python 2.7.x if filename comes as unicode string, # just skip converting it. if isinstance(filename, str): filename = py3compat.str_to_unicode(filename) # Pull the metadata from the filesystem. if resources is None: resources = ResourcesDict() if not 'metadata' in resources or resources['metadata'] == '': resources['metadata'] = ResourcesDict() path, basename = os.path.split(filename) notebook_name = basename[:basename.rfind('.')] resources['metadata']['name'] = notebook_name resources['metadata']['path'] = path modified_date = datetime.datetime.fromtimestamp( os.path.getmtime(filename)) resources['metadata']['modified_date'] = modified_date.strftime( text.date_format) with io.open(filename, encoding='utf-8') as f: return self.from_file(f, resources=resources, **kw)
def from_filename(self, filename, resources=None, **kw): """ Convert a notebook from a notebook file. Parameters ---------- filename : str Full filename of the notebook file to open and convert. resources : dict Additional resources that can be accessed read/write by preprocessors and filters. `**kw` Ignored """ # Convert full filename string to unicode # In python 2.7.x if filename comes as unicode string, # just skip converting it. if isinstance(filename, str): filename = py3compat.str_to_unicode(filename) # Pull the metadata from the filesystem. if resources is None: resources = ResourcesDict() if not 'metadata' in resources or resources['metadata'] == '': resources['metadata'] = ResourcesDict() path, basename = os.path.split(filename) notebook_name = basename[:basename.rfind('.')] resources['metadata']['name'] = notebook_name resources['metadata']['path'] = path modified_date = datetime.datetime.fromtimestamp(os.path.getmtime(filename)) resources['metadata']['modified_date'] = modified_date.strftime(text.date_format) with io.open(filename, encoding='utf-8') as f: return self.from_file(f, resources=resources, **kw)
class Session(Configurable): """Object for handling serialization and sending of messages. The Session object handles building messages and sending them with ZMQ sockets or ZMQStream objects. Objects can communicate with each other over the network via Session objects, and only need to work with the dict-based IPython message spec. The Session will handle serialization/deserialization, security, and metadata. Sessions support configurable serialization via packer/unpacker traits, and signing with HMAC digests via the key/keyfile traits. Parameters ---------- debug : bool whether to trigger extra debugging statements packer/unpacker : str : 'json', 'pickle' or import_string importstrings for methods to serialize message parts. If just 'json' or 'pickle', predefined JSON and pickle packers will be used. Otherwise, the entire importstring must be used. The functions must accept at least valid JSON input, and output *bytes*. For example, to use msgpack: packer = 'msgpack.packb', unpacker='msgpack.unpackb' pack/unpack : callables You can also set the pack/unpack callables for serialization directly. session : bytes the ID of this Session object. The default is to generate a new UUID. username : unicode username added to message headers. The default is to ask the OS. key : bytes The key used to initialize an HMAC signature. If unset, messages will not be signed or checked. keyfile : filepath The file containing a key. If this is set, `key` will be initialized to the contents of the file. """ debug = Bool(False, config=True, help="""Debug output in the Session""") check_pid = Bool(True, config=True, help="""Whether to check PID to protect against calls after fork. This check can be disabled if fork-safety is handled elsewhere. """) packer = DottedObjectName('json',config=True, help="""The name of the packer for serializing messages. Should be one of 'json', 'pickle', or an import name for a custom callable serializer.""") @observe('packer') def _packer_changed(self, change): new = change['new'] if new.lower() == 'json': self.pack = json_packer self.unpack = json_unpacker self.unpacker = new elif new.lower() == 'pickle': self.pack = pickle_packer self.unpack = pickle_unpacker self.unpacker = new else: self.pack = import_item(str(new)) unpacker = DottedObjectName('json', config=True, help="""The name of the unpacker for unserializing messages. Only used with custom functions for `packer`.""") @observe('unpacker') def _unpacker_changed(self, change): new = change['new'] if new.lower() == 'json': self.pack = json_packer self.unpack = json_unpacker self.packer = new elif new.lower() == 'pickle': self.pack = pickle_packer self.unpack = pickle_unpacker self.packer = new else: self.unpack = import_item(str(new)) session = CUnicode(u'', config=True, help="""The UUID identifying this session.""") def _session_default(self): u = new_id() self.bsession = u.encode('ascii') return u @observe('session') def _session_changed(self, change): self.bsession = self.session.encode('ascii') # bsession is the session as bytes bsession = CBytes(b'') username = Unicode(str_to_unicode(os.environ.get('USER', 'username')), help="""Username for the Session. Default is your system username.""", config=True) metadata = Dict({}, config=True, help="""Metadata dictionary, which serves as the default top-level metadata dict for each message.""") # if 0, no adapting to do. adapt_version = Integer(0) # message signature related traits: key = CBytes(config=True, help="""execution key, for signing messages.""") def _key_default(self): return new_id_bytes() @observe('key') def _key_changed(self, change): self._new_auth() signature_scheme = Unicode('hmac-sha256', config=True, help="""The digest scheme used to construct the message signatures. Must have the form 'hmac-HASH'.""") @observe('signature_scheme') def _signature_scheme_changed(self, change): new = change['new'] if not new.startswith('hmac-'): raise TraitError("signature_scheme must start with 'hmac-', got %r" % new) hash_name = new.split('-', 1)[1] try: self.digest_mod = getattr(hashlib, hash_name) except AttributeError: raise TraitError("hashlib has no such attribute: %s" % hash_name) self._new_auth() digest_mod = Any() def _digest_mod_default(self): return hashlib.sha256 auth = Instance(hmac.HMAC, allow_none=True) def _new_auth(self): if self.key: self.auth = hmac.HMAC(self.key, digestmod=self.digest_mod) else: self.auth = None digest_history = Set() digest_history_size = Integer(2**16, config=True, help="""The maximum number of digests to remember. The digest history will be culled when it exceeds this value. """ ) keyfile = Unicode('', config=True, help="""path to file containing execution key.""") @observe('keyfile') def _keyfile_changed(self, change): with open(change['new'], 'rb') as f: self.key = f.read().strip() # for protecting against sends from forks pid = Integer() # serialization traits: pack = Any(default_packer) # the actual packer function @observe('pack') def _pack_changed(self, change): new = change['new'] if not callable(new): raise TypeError("packer must be callable, not %s"%type(new)) unpack = Any(default_unpacker) # the actual packer function @observe('unpack') def _unpack_changed(self, change): # unpacker is not checked - it is assumed to be new = change['new'] if not callable(new): raise TypeError("unpacker must be callable, not %s"%type(new)) # thresholds: copy_threshold = Integer(2**16, config=True, help="Threshold (in bytes) beyond which a buffer should be sent without copying.") buffer_threshold = Integer(MAX_BYTES, config=True, help="Threshold (in bytes) beyond which an object's buffer should be extracted to avoid pickling.") item_threshold = Integer(MAX_ITEMS, config=True, help="""The maximum number of items for a container to be introspected for custom serialization. Containers larger than this are pickled outright. """ ) def __init__(self, **kwargs): """create a Session object Parameters ---------- debug : bool whether to trigger extra debugging statements packer/unpacker : str : 'json', 'pickle' or import_string importstrings for methods to serialize message parts. If just 'json' or 'pickle', predefined JSON and pickle packers will be used. Otherwise, the entire importstring must be used. The functions must accept at least valid JSON input, and output *bytes*. For example, to use msgpack: packer = 'msgpack.packb', unpacker='msgpack.unpackb' pack/unpack : callables You can also set the pack/unpack callables for serialization directly. session : unicode (must be ascii) the ID of this Session object. The default is to generate a new UUID. bsession : bytes The session as bytes username : unicode username added to message headers. The default is to ask the OS. key : bytes The key used to initialize an HMAC signature. If unset, messages will not be signed or checked. signature_scheme : str The message digest scheme. Currently must be of the form 'hmac-HASH', where 'HASH' is a hashing function available in Python's hashlib. The default is 'hmac-sha256'. This is ignored if 'key' is empty. keyfile : filepath The file containing a key. If this is set, `key` will be initialized to the contents of the file. """ super(Session, self).__init__(**kwargs) self._check_packers() self.none = self.pack({}) # ensure self._session_default() if necessary, so bsession is defined: self.session self.pid = os.getpid() self._new_auth() if not self.key: get_logger().warning("Message signing is disabled. This is insecure and not recommended!") def clone(self): """Create a copy of this Session Useful when connecting multiple times to a given kernel. This prevents a shared digest_history warning about duplicate digests due to multiple connections to IOPub in the same process. .. versionadded:: 5.1 """ # make a copy new_session = type(self)() for name in self.traits(): setattr(new_session, name, getattr(self, name)) # fork digest_history new_session.digest_history = set() new_session.digest_history.update(self.digest_history) return new_session @property def msg_id(self): """always return new uuid""" return new_id() def _check_packers(self): """check packers for datetime support.""" pack = self.pack unpack = self.unpack # check simple serialization msg = dict(a=[1,'hi']) try: packed = pack(msg) except Exception as e: msg = "packer '{packer}' could not serialize a simple message: {e}{jsonmsg}" if self.packer == 'json': jsonmsg = "\nzmq.utils.jsonapi.jsonmod = %s" % jsonapi.jsonmod else: jsonmsg = "" raise ValueError( msg.format(packer=self.packer, e=e, jsonmsg=jsonmsg) ) # ensure packed message is bytes if not isinstance(packed, bytes): raise ValueError("message packed to %r, but bytes are required"%type(packed)) # check that unpack is pack's inverse try: unpacked = unpack(packed) assert unpacked == msg except Exception as e: msg = "unpacker '{unpacker}' could not handle output from packer '{packer}': {e}{jsonmsg}" if self.packer == 'json': jsonmsg = "\nzmq.utils.jsonapi.jsonmod = %s" % jsonapi.jsonmod else: jsonmsg = "" raise ValueError( msg.format(packer=self.packer, unpacker=self.unpacker, e=e, jsonmsg=jsonmsg) ) # check datetime support msg = dict(t=utcnow()) try: unpacked = unpack(pack(msg)) if isinstance(unpacked['t'], datetime): raise ValueError("Shouldn't deserialize to datetime") except Exception: self.pack = lambda o: pack(squash_dates(o)) self.unpack = lambda s: unpack(s) def msg_header(self, msg_type): return msg_header(self.msg_id, msg_type, self.username, self.session) def msg(self, msg_type, content=None, parent=None, header=None, metadata=None): """Return the nested message dict. This format is different from what is sent over the wire. The serialize/deserialize methods converts this nested message dict to the wire format, which is a list of message parts. """ msg = {} header = self.msg_header(msg_type) if header is None else header msg['header'] = header msg['msg_id'] = header['msg_id'] msg['msg_type'] = header['msg_type'] msg['parent_header'] = {} if parent is None else extract_header(parent) msg['content'] = {} if content is None else content msg['metadata'] = self.metadata.copy() if metadata is not None: msg['metadata'].update(metadata) return msg def sign(self, msg_list): """Sign a message with HMAC digest. If no auth, return b''. Parameters ---------- msg_list : list The [p_header,p_parent,p_content] part of the message list. """ if self.auth is None: return b'' h = self.auth.copy() for m in msg_list: h.update(m) return str_to_bytes(h.hexdigest()) def serialize(self, msg, ident=None): """Serialize the message components to bytes. This is roughly the inverse of deserialize. The serialize/deserialize methods work with full message lists, whereas pack/unpack work with the individual message parts in the message list. Parameters ---------- msg : dict or Message The next message dict as returned by the self.msg method. Returns ------- msg_list : list The list of bytes objects to be sent with the format:: [ident1, ident2, ..., DELIM, HMAC, p_header, p_parent, p_metadata, p_content, buffer1, buffer2, ...] In this list, the ``p_*`` entities are the packed or serialized versions, so if JSON is used, these are utf8 encoded JSON strings. """ content = msg.get('content', {}) if content is None: content = self.none elif isinstance(content, dict): content = self.pack(content) elif isinstance(content, bytes): # content is already packed, as in a relayed message pass elif isinstance(content, unicode_type): # should be bytes, but JSON often spits out unicode content = content.encode('utf8') else: raise TypeError("Content incorrect type: %s"%type(content)) real_message = [self.pack(msg['header']), self.pack(msg['parent_header']), self.pack(msg['metadata']), content, ] to_send = [] if isinstance(ident, list): # accept list of idents to_send.extend(ident) elif ident is not None: to_send.append(ident) to_send.append(DELIM) signature = self.sign(real_message) to_send.append(signature) to_send.extend(real_message) return to_send def send(self, stream, msg_or_type, content=None, parent=None, ident=None, buffers=None, track=False, header=None, metadata=None): """Build and send a message via stream or socket. The message format used by this function internally is as follows: [ident1,ident2,...,DELIM,HMAC,p_header,p_parent,p_content, buffer1,buffer2,...] The serialize/deserialize methods convert the nested message dict into this format. Parameters ---------- stream : zmq.Socket or ZMQStream The socket-like object used to send the data. msg_or_type : str or Message/dict Normally, msg_or_type will be a msg_type unless a message is being sent more than once. If a header is supplied, this can be set to None and the msg_type will be pulled from the header. content : dict or None The content of the message (ignored if msg_or_type is a message). header : dict or None The header dict for the message (ignored if msg_to_type is a message). parent : Message or dict or None The parent or parent header describing the parent of this message (ignored if msg_or_type is a message). ident : bytes or list of bytes The zmq.IDENTITY routing path. metadata : dict or None The metadata describing the message buffers : list or None The already-serialized buffers to be appended to the message. track : bool Whether to track. Only for use with Sockets, because ZMQStream objects cannot track messages. Returns ------- msg : dict The constructed message. """ if not isinstance(stream, zmq.Socket): # ZMQStreams and dummy sockets do not support tracking. track = False if isinstance(msg_or_type, (Message, dict)): # We got a Message or message dict, not a msg_type so don't # build a new Message. msg = msg_or_type buffers = buffers or msg.get('buffers', []) else: msg = self.msg(msg_or_type, content=content, parent=parent, header=header, metadata=metadata) if self.check_pid and not os.getpid() == self.pid: get_logger().warning("WARNING: attempted to send message from fork\n%s", msg ) return buffers = [] if buffers is None else buffers for idx, buf in enumerate(buffers): if isinstance(buf, memoryview): view = buf else: try: # check to see if buf supports the buffer protocol. view = memoryview(buf) except TypeError: raise TypeError("Buffer objects must support the buffer protocol.") # memoryview.contiguous is new in 3.3, # just skip the check on Python 2 if hasattr(view, 'contiguous') and not view.contiguous: # zmq requires memoryviews to be contiguous raise ValueError("Buffer %i (%r) is not contiguous" % (idx, buf)) if self.adapt_version: msg = adapt(msg, self.adapt_version) to_send = self.serialize(msg, ident) to_send.extend(buffers) longest = max([ len(s) for s in to_send ]) copy = (longest < self.copy_threshold) if buffers and track and not copy: # only really track when we are doing zero-copy buffers tracker = stream.send_multipart(to_send, copy=False, track=True) else: # use dummy tracker, which will be done immediately tracker = DONE stream.send_multipart(to_send, copy=copy) if self.debug: pprint.pprint(msg) pprint.pprint(to_send) pprint.pprint(buffers) msg['tracker'] = tracker return msg def send_raw(self, stream, msg_list, flags=0, copy=True, ident=None): """Send a raw message via ident path. This method is used to send a already serialized message. Parameters ---------- stream : ZMQStream or Socket The ZMQ stream or socket to use for sending the message. msg_list : list The serialized list of messages to send. This only includes the [p_header,p_parent,p_metadata,p_content,buffer1,buffer2,...] portion of the message. ident : ident or list A single ident or a list of idents to use in sending. """ to_send = [] if isinstance(ident, bytes): ident = [ident] if ident is not None: to_send.extend(ident) to_send.append(DELIM) # Don't include buffers in signature (per spec). to_send.append(self.sign(msg_list[0:4])) to_send.extend(msg_list) stream.send_multipart(to_send, flags, copy=copy) def recv(self, socket, mode=zmq.NOBLOCK, content=True, copy=True): """Receive and unpack a message. Parameters ---------- socket : ZMQStream or Socket The socket or stream to use in receiving. Returns ------- [idents], msg [idents] is a list of idents and msg is a nested message dict of same format as self.msg returns. """ if isinstance(socket, ZMQStream): socket = socket.socket try: msg_list = socket.recv_multipart(mode, copy=copy) except zmq.ZMQError as e: if e.errno == zmq.EAGAIN: # We can convert EAGAIN to None as we know in this case # recv_multipart won't return None. return None,None else: raise # split multipart message into identity list and message dict # invalid large messages can cause very expensive string comparisons idents, msg_list = self.feed_identities(msg_list, copy) try: return idents, self.deserialize(msg_list, content=content, copy=copy) except Exception as e: # TODO: handle it raise e def feed_identities(self, msg_list, copy=True): """Split the identities from the rest of the message. Feed until DELIM is reached, then return the prefix as idents and remainder as msg_list. This is easily broken by setting an IDENT to DELIM, but that would be silly. Parameters ---------- msg_list : a list of Message or bytes objects The message to be split. copy : bool flag determining whether the arguments are bytes or Messages Returns ------- (idents, msg_list) : two lists idents will always be a list of bytes, each of which is a ZMQ identity. msg_list will be a list of bytes or zmq.Messages of the form [HMAC,p_header,p_parent,p_content,buffer1,buffer2,...] and should be unpackable/unserializable via self.deserialize at this point. """ if copy: idx = msg_list.index(DELIM) return msg_list[:idx], msg_list[idx+1:] else: failed = True for idx,m in enumerate(msg_list): if m.bytes == DELIM: failed = False break if failed: raise ValueError("DELIM not in msg_list") idents, msg_list = msg_list[:idx], msg_list[idx+1:] return [m.bytes for m in idents], msg_list def _add_digest(self, signature): """add a digest to history to protect against replay attacks""" if self.digest_history_size == 0: # no history, never add digests return self.digest_history.add(signature) if len(self.digest_history) > self.digest_history_size: # threshold reached, cull 10% self._cull_digest_history() def _cull_digest_history(self): """cull the digest history Removes a randomly selected 10% of the digest history """ current = len(self.digest_history) n_to_cull = max(int(current // 10), current - self.digest_history_size) if n_to_cull >= current: self.digest_history = set() return to_cull = random.sample(self.digest_history, n_to_cull) self.digest_history.difference_update(to_cull) def deserialize(self, msg_list, content=True, copy=True): """Unserialize a msg_list to a nested message dict. This is roughly the inverse of serialize. The serialize/deserialize methods work with full message lists, whereas pack/unpack work with the individual message parts in the message list. Parameters ---------- msg_list : list of bytes or Message objects The list of message parts of the form [HMAC,p_header,p_parent, p_metadata,p_content,buffer1,buffer2,...]. content : bool (True) Whether to unpack the content dict (True), or leave it packed (False). copy : bool (True) Whether msg_list contains bytes (True) or the non-copying Message objects in each place (False). Returns ------- msg : dict The nested message dict with top-level keys [header, parent_header, content, buffers]. The buffers are returned as memoryviews. """ minlen = 5 message = {} if not copy: # pyzmq didn't copy the first parts of the message, so we'll do it for i in range(minlen): msg_list[i] = msg_list[i].bytes if self.auth is not None: signature = msg_list[0] if not signature: raise ValueError("Unsigned Message") if signature in self.digest_history: raise ValueError("Duplicate Signature: %r" % signature) if content: # Only store signature if we are unpacking content, don't store if just peeking. self._add_digest(signature) check = self.sign(msg_list[1:5]) if not compare_digest(signature, check): raise ValueError("Invalid Signature: %r" % signature) if not len(msg_list) >= minlen: raise TypeError("malformed message, must have at least %i elements"%minlen) header = self.unpack(msg_list[1]) message['header'] = extract_dates(header) message['msg_id'] = header['msg_id'] message['msg_type'] = header['msg_type'] message['parent_header'] = extract_dates(self.unpack(msg_list[2])) message['metadata'] = self.unpack(msg_list[3]) if content: message['content'] = self.unpack(msg_list[4]) else: message['content'] = msg_list[4] buffers = [memoryview(b) for b in msg_list[5:]] if buffers and buffers[0].shape is None: # force copy to workaround pyzmq #646 buffers = [memoryview(b.bytes) for b in msg_list[5:]] message['buffers'] = buffers if self.debug: pprint.pprint(message) # adapt to the current version return adapt(message) def unserialize(self, *args, **kwargs): warnings.warn( "Session.unserialize is deprecated. Use Session.deserialize.", DeprecationWarning, ) return self.deserialize(*args, **kwargs)
async def mock_gateway_request(url, **kwargs): method = "GET" if kwargs["method"]: method = kwargs["method"] request = HTTPRequest(url=url, **kwargs) endpoint = str(url) # Fetch all kernelspecs if endpoint.endswith("/api/kernelspecs") and method == "GET": response_buf = StringIO(str_to_unicode(json.dumps(kernelspecs))) response = await gen.maybe_future( HTTPResponse(request, 200, buffer=response_buf) ) return response # Fetch named kernelspec if endpoint.rfind("/api/kernelspecs/") >= 0 and method == "GET": requested_kernelspec = endpoint.rpartition("/")[2] kspecs = kernelspecs.get("kernelspecs") if requested_kernelspec in kspecs: response_buf = StringIO( str_to_unicode(json.dumps(kspecs.get(requested_kernelspec))) ) response = await gen.maybe_future( HTTPResponse(request, 200, buffer=response_buf) ) return response else: raise HTTPError( 404, message="Kernelspec does not exist: %s" % requested_kernelspec ) # Create kernel if endpoint.endswith("/api/kernels") and method == "POST": json_body = json.loads(kwargs["body"]) name = json_body.get("name") env = json_body.get("env") kspec_name = env.get("KERNEL_KSPEC_NAME") assert name == kspec_name # Ensure that KERNEL_ env values get propagated model = generate_model(name) running_kernels[model.get("id")] = model # Register model as a running kernel response_buf = StringIO(str_to_unicode(json.dumps(model))) response = await gen.maybe_future( HTTPResponse(request, 201, buffer=response_buf) ) return response # Fetch list of running kernels if endpoint.endswith("/api/kernels") and method == "GET": kernels = [] for kernel_id in running_kernels.keys(): model = running_kernels.get(kernel_id) kernels.append(model) response_buf = StringIO(str_to_unicode(json.dumps(kernels))) response = await gen.maybe_future( HTTPResponse(request, 200, buffer=response_buf) ) return response # Interrupt or restart existing kernel if endpoint.rfind("/api/kernels/") >= 0 and method == "POST": requested_kernel_id, sep, action = endpoint.rpartition("/api/kernels/")[ 2 ].rpartition("/") if action == "interrupt": if requested_kernel_id in running_kernels: response = await gen.maybe_future(HTTPResponse(request, 204)) return response else: raise HTTPError( 404, message="Kernel does not exist: %s" % requested_kernel_id ) elif action == "restart": if requested_kernel_id in running_kernels: response_buf = StringIO( str_to_unicode(json.dumps(running_kernels.get(requested_kernel_id))) ) response = await gen.maybe_future( HTTPResponse(request, 204, buffer=response_buf) ) return response else: raise HTTPError( 404, message="Kernel does not exist: %s" % requested_kernel_id ) else: raise HTTPError(404, message="Bad action detected: %s" % action) # Shutdown existing kernel if endpoint.rfind("/api/kernels/") >= 0 and method == "DELETE": requested_kernel_id = endpoint.rpartition("/")[2] running_kernels.pop( requested_kernel_id ) # Simulate shutdown by removing kernel from running set response = await gen.maybe_future(HTTPResponse(request, 204)) return response # Fetch existing kernel if endpoint.rfind("/api/kernels/") >= 0 and method == "GET": requested_kernel_id = endpoint.rpartition("/")[2] if requested_kernel_id in running_kernels: response_buf = StringIO( str_to_unicode(json.dumps(running_kernels.get(requested_kernel_id))) ) response = await gen.maybe_future( HTTPResponse(request, 200, buffer=response_buf) ) return response else: raise HTTPError( 404, message="Kernel does not exist: %s" % requested_kernel_id )
def read(self, fp, **kwargs): """Read a notebook from a file like object""" nbs = fp.read() if not py3compat.PY3 and not isinstance(nbs, unicode_type): nbs = py3compat.str_to_unicode(nbs) return self.reads(nbs, **kwargs)
async def mock_gateway_request(url, **kwargs): method = 'GET' if kwargs['method']: method = kwargs['method'] request = HTTPRequest(url=url, **kwargs) endpoint = str(url) # Fetch all kernelspecs if endpoint.endswith('/api/kernelspecs') and method == 'GET': response_buf = StringIO(str_to_unicode(json.dumps(kernelspecs))) response = await ensure_async( HTTPResponse(request, 200, buffer=response_buf)) return response # Fetch named kernelspec if endpoint.rfind('/api/kernelspecs/') >= 0 and method == 'GET': requested_kernelspec = endpoint.rpartition('/')[2] kspecs = kernelspecs.get('kernelspecs') if requested_kernelspec in kspecs: response_buf = StringIO( str_to_unicode(json.dumps(kspecs.get(requested_kernelspec)))) response = await ensure_async( HTTPResponse(request, 200, buffer=response_buf)) return response else: raise HTTPError(404, message='Kernelspec does not exist: %s' % requested_kernelspec) # Create kernel if endpoint.endswith('/api/kernels') and method == 'POST': json_body = json.loads(kwargs['body']) name = json_body.get('name') env = json_body.get('env') kspec_name = env.get('KERNEL_KSPEC_NAME') assert name == kspec_name # Ensure that KERNEL_ env values get propagated model = generate_model(name) running_kernels[model.get( 'id')] = model # Register model as a running kernel response_buf = StringIO(str_to_unicode(json.dumps(model))) response = await ensure_async( HTTPResponse(request, 201, buffer=response_buf)) return response # Fetch list of running kernels if endpoint.endswith('/api/kernels') and method == 'GET': kernels = [] for kernel_id in running_kernels.keys(): model = running_kernels.get(kernel_id) kernels.append(model) response_buf = StringIO(str_to_unicode(json.dumps(kernels))) response = await ensure_async( HTTPResponse(request, 200, buffer=response_buf)) return response # Interrupt or restart existing kernel if endpoint.rfind('/api/kernels/') >= 0 and method == 'POST': requested_kernel_id, sep, action = endpoint.rpartition( '/api/kernels/')[2].rpartition('/') if action == 'interrupt': if requested_kernel_id in running_kernels: response = await ensure_async(HTTPResponse(request, 204)) return response else: raise HTTPError(404, message='Kernel does not exist: %s' % requested_kernel_id) elif action == 'restart': if requested_kernel_id in running_kernels: response_buf = StringIO( str_to_unicode( json.dumps(running_kernels.get(requested_kernel_id)))) response = await ensure_async( HTTPResponse(request, 204, buffer=response_buf)) return response else: raise HTTPError(404, message='Kernel does not exist: %s' % requested_kernel_id) else: raise HTTPError(404, message='Bad action detected: %s' % action) # Shutdown existing kernel if endpoint.rfind('/api/kernels/') >= 0 and method == 'DELETE': requested_kernel_id = endpoint.rpartition('/')[2] running_kernels.pop( requested_kernel_id ) # Simulate shutdown by removing kernel from running set response = await ensure_async(HTTPResponse(request, 204)) return response # Fetch existing kernel if endpoint.rfind('/api/kernels/') >= 0 and method == 'GET': requested_kernel_id = endpoint.rpartition('/')[2] if requested_kernel_id in running_kernels: response_buf = StringIO( str_to_unicode( json.dumps(running_kernels.get(requested_kernel_id)))) response = await ensure_async( HTTPResponse(request, 200, buffer=response_buf)) return response else: raise HTTPError(404, message='Kernel does not exist: %s' % requested_kernel_id)