def decode_all(data, codec_options=DEFAULT_CODEC_OPTIONS): if not isinstance(codec_options, CodecOptions): raise bson._CODEC_OPTIONS_TYPE_ERROR docs = [] position = 0 end = len(data) - 1 use_raw = _raw_document_class(codec_options.document_class) try: while position < end: obj_size = bson._UNPACK_INT(data[position:position + 4])[0] if len(data) - position < obj_size: raise bson.InvalidBSON("invalid object size") obj_end = position + obj_size - 1 if data[obj_end:position + obj_size] != b"\x00": raise bson.InvalidBSON("bad eoo") if use_raw: docs.append( codec_options.document_class(data[position:obj_end + 1], codec_options)) else: docs.append( _elements_to_dict(data, position + 4, obj_end, codec_options)) position += obj_size return docs except bson.InvalidBSON: raise except Exception: # Change exception type to InvalidBSON but preserve traceback. _, exc_value, exc_tb = sys.exc_info() bson.reraise(bson.InvalidBSON, exc_value, exc_tb)
def getvals(data, opts): import ipdb; ipdb.set_trace(); obj_size = bson._UNPACK_INT(data[:4])[0] obj_end = obj_size - 1 pos = 4 while pos < obj_size - 1: ename, value, pos = bson._element_to_dict(data, pos, obj_end, opts) yield ename, value
def getvals(data, opts): import ipdb ipdb.set_trace() obj_size = bson._UNPACK_INT(data[:4])[0] obj_end = obj_size - 1 pos = 4 while pos < obj_size - 1: ename, value, pos = bson._element_to_dict(data, pos, obj_end, opts) yield ename, value
def _get_object(data, position, obj_end, opts, dummy): """Decode a BSON subdocument to opts.document_class or bson.dbref.DBRef.""" obj_size = bson._UNPACK_INT(data[position:position + 4])[0] end = position + obj_size - 1 if data[end:position + obj_size] != b"\x00": raise bson.InvalidBSON("bad eoo") if end >= obj_end: raise bson.InvalidBSON("invalid object length") return data[position:end + 1], end + 1
def getval(data, opts, name): obj_size = bson._UNPACK_INT(data[:4])[0] obj_end = obj_size - 1 pos = 4 while pos < obj_size - 1: ename, value, pos = bson._element_to_dict(data, pos, obj_end, opts) if ename == name: return value raise KeyError(name)
def __inflated(self): if self.__inflated_doc is None: # We already validated the object's size when this document was # created, so no need to do that again. self.__inflated_doc = dict( element for element in _iterate_elements( self.__raw, 4, _UNPACK_INT(self.__raw[:4])[0] - 1, self.__codec_options)) return self.__inflated_doc
def _get_object(data, position, obj_end, opts, dummy): """Decode a BSON subdocument to opts.document_class or bson.dbref.DBRef.""" obj_size = bson._UNPACK_INT(data[position:position + 4])[0] end = position + obj_size - 1 if data[end:position + obj_size] != b"\x00": raise bson.InvalidBSON("bad eoo") if end >= obj_end: raise bson.InvalidBSON("invalid object length") return data[position:end + 1], end + 1
def __inflated(self): if self.__inflated_doc is None: # We already validated the object's size when this document was # created, so no need to do that again. self.__inflated_doc = dict( element for element in _iterate_elements( self.__raw, 4, _UNPACK_INT(self.__raw[:4])[0] - 1, self.__codec_options)) return self.__inflated_doc
def getval(data, opts, name): obj_size = bson._UNPACK_INT(data[:4])[0] obj_end = obj_size - 1 pos = 4 while pos < obj_size - 1: ename, value, pos = bson._element_to_dict(data, pos, obj_end, opts) if ename == name: return value raise KeyError(name)
def _get_string(data, position, obj_end, opts, dummy): """Decode a BSON string to python unicode string.""" length = bson._UNPACK_INT(data[position:position + 4])[0] position += 4 if length < 1 or obj_end - position < length: raise bson.InvalidBSON("invalid string length") end = position + length - 1 if data[end:end + 1] != b"\x00": raise bson.InvalidBSON("invalid end of string") return data[position:end].decode('utf8'), end + 1
def _get_string(data, position, obj_end, opts, dummy): """Decode a BSON string to python unicode string.""" length = bson._UNPACK_INT(data[position:position + 4])[0] position += 4 if length < 1 or obj_end - position < length: raise bson.InvalidBSON("invalid string length") end = position + length - 1 if data[end:end + 1] != b"\x00": raise bson.InvalidBSON("invalid end of string") return data[position:end].decode('utf8'), end + 1
def __inflated(self): if self.__inflated_doc is None: # We already validated the object's size when this document was # created, so no need to do that again. We still need to check the # size of all the elements and compare to the document size. object_size = _UNPACK_INT(self.__raw[:4])[0] - 1 position = 0 self.__inflated_doc = {} for key, value, position in _iterate_elements( self.__raw, 4, object_size, self.__codec_options): self.__inflated_doc[key] = value if position != object_size: raise InvalidBSON('bad object or element length') return self.__inflated_doc
def __inflated(self): if self.__inflated_doc is None: # We already validated the object's size when this document was # created, so no need to do that again. We still need to check the # size of all the elements and compare to the document size. object_size = _UNPACK_INT(self.__raw[:4])[0] - 1 position = 0 self.__inflated_doc = {} for key, value, position in _iterate_elements( self.__raw, 4, object_size, self.__codec_options): self.__inflated_doc[key] = value if position != object_size: raise InvalidBSON('bad object or element length') return self.__inflated_doc
def _iter_elements(data, opts, recurser=None): obj_size = bson._UNPACK_INT(data[:4])[0] obj_end = obj_size - 1 pos = 4 while pos < obj_size - 1: etype = data[pos] pos += 1 ename, pos = _get_c_string(data, pos) value, pos = _ELEMENT_GETTER[etype](data, pos, obj_end, opts, ename) if recurser and etype == bson.BSONOBJ: if ename in recurser: for k, v in _iter_elements(value, opts, recurser(ename)): yield [ename] + k, v else: yield [ename], value else: yield [ename], value
def _iter_elements(data, opts, recurser=None): obj_size = bson._UNPACK_INT(data[:4])[0] obj_end = obj_size - 1 pos = 4 while pos < obj_size - 1: etype = data[pos] pos += 1 ename, pos = _get_c_string(data, pos) value, pos = _ELEMENT_GETTER[etype]( data, pos, obj_end, opts, ename) if recurser and etype == bson.BSONOBJ: if ename in recurser: for k, v in _iter_elements(value, opts, recurser(ename)): yield [ename] + k, v else: yield [ename], value else: yield [ename], value
def _get_object(data, position, obj_end, opts, dummy): """Decode a BSON subdocument to opts.document_class or bson.dbref.DBRef.""" obj_size = bson._UNPACK_INT(data[position:position + 4])[0] end = position + obj_size - 1 if data[end:position + obj_size] != b"\x00": raise bson.InvalidBSON("bad eoo") if end >= obj_end: raise bson.InvalidBSON("invalid object length") if _raw_document_class(opts.document_class): return (opts.document_class(data[position:end + 1], opts), position + obj_size) obj = _elements_to_dict(data, position + 4, end, opts, subdocument=True) position += obj_size if "$ref" in obj: return (bson.DBRef(obj.pop("$ref"), obj.pop("$id", None), obj.pop("$db", None), obj), position) return obj, position
def get_fields(data, opts, trie): """Return list of (name, data) tuples where name is in the trie.""" obj_size = bson._UNPACK_INT(data[:4])[0] obj_end = obj_size - 1 pos = 4 result = [] while pos < obj_size - 1: etype = data[pos] pos += 1 ename, pos = _get_c_string(data, pos) value, pos = _ELEMENT_GETTER[etype](data, pos, obj_end, opts, ename) if ename in trie: result.append(([ename], value)) continue new_trie = trie.get(ename) if new_trie: result += [([ename] + key, v) for key, v in get_fields(value, opts, new_trie)] return result
def _decode_batch(batch): """ NOTE: This implementation is the same as the bson.decode_iter implementation, with some minor differences. This methods decodes a raw bson data batch, and yields dict items for every valid batch item. Batch items that cannot be decoded from their bson format will be skipped. """ position = 0 end = len(batch) - 1 while position < end: obj_size = bson._UNPACK_INT(batch[position:position + 4])[0] elements = batch[position:position + obj_size] position += obj_size try: yield bson._bson_to_dict(elements, codec_options.DEFAULT_CODEC_OPTIONS) except bson.InvalidBSON as err: logging.warning("ignored invalid record: {}".format(str(err))) continue
def get_fields(data, opts, trie): """Return list of (name, data) tuples where name is in the trie.""" obj_size = bson._UNPACK_INT(data[:4])[0] obj_end = obj_size - 1 pos = 4 result = [] while pos < obj_size - 1: etype = data[pos] pos += 1 ename, pos = _get_c_string(data, pos) value, pos = _ELEMENT_GETTER[etype]( data, pos, obj_end, opts, ename) if ename in trie: result.append(([ename], value)) continue new_trie = trie.get(ename) if new_trie: result += [ ([ename] + key, v) for key, v in get_fields(value, opts, new_trie) ] return result
def _obj_end(self): obj_size = bson._UNPACK_INT(self._raw[:4])[0] return obj_size - 1
def _obj_end(self): obj_size = bson._UNPACK_INT(self._raw[:4])[0] return obj_size - 1