def test_basic_decode(self):
self.assertEqual({"test": "hello world"},
decode(b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74\x00\x0C"
b"\x00\x00\x00\x68\x65\x6C\x6C\x6F\x20\x77\x6F"
b"\x72\x6C\x64\x00\x00"))
self.assertEqual([{"test": "hello world"}, {}],
decode_all(b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
b"\x05\x00\x00\x00\x00"))
self.assertEqual([{"test": "hello world"}, {}],
list(decode_iter(
b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
b"\x05\x00\x00\x00\x00")))
self.assertEqual([{"test": "hello world"}, {}],
list(decode_file_iter(BytesIO(
b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
b"\x05\x00\x00\x00\x00"))))
def test_subclasses(self):
# make sure we can serialize subclasses of native Python types.
class _myint(int):
pass
class _myfloat(float):
pass
class _myunicode(text_type):
pass
d = {
'a': _myint(42),
'b': _myfloat(63.9),
'c': _myunicode('hello world')
}
d2 = decode(encode(d))
for key, value in iteritems(d2):
orig_value = d[key]
orig_type = orig_value.__class__.__bases__[0]
self.assertEqual(type(value), orig_type)
self.assertEqual(value, orig_type(value))
def encrypt(
self,
value: Any,
algorithm: str,
key_id: Optional[Binary] = None,
key_alt_name: Optional[str] = None,
) -> Binary:
"""Encrypt a BSON value with a given key and algorithm.
Note that exactly one of ``key_id`` or ``key_alt_name`` must be
provided.
:Parameters:
- `value`: The BSON value to encrypt.
- `algorithm` (string): The encryption algorithm to use. See
:class:`Algorithm` for some valid options.
- `key_id`: Identifies a data key by ``_id`` which must be a
:class:`~bson.binary.Binary` with subtype 4 (
:attr:`~bson.binary.UUID_SUBTYPE`).
- `key_alt_name`: Identifies a key vault document by 'keyAltName'.
:Returns:
The encrypted value, a :class:`~bson.binary.Binary` with subtype 6.
"""
self._check_closed()
if key_id is not None and not (isinstance(key_id, Binary)
and key_id.subtype == UUID_SUBTYPE):
raise TypeError(
"key_id must be a bson.binary.Binary with subtype 4")
doc = encode({"v": value}, codec_options=self._codec_options)
with _wrap_encryption_errors():
encrypted_doc = self._encryption.encrypt(doc,
algorithm,
key_id=key_id,
key_alt_name=key_alt_name)
return decode(encrypted_doc)["v"] # type: ignore[index]
def from_bson_partial(data):
snapshot_doc = bson.decode(data)
return Snapshot(**snapshot_doc)
def test_ordered_dict(self):
d = OrderedDict([("one", 1), ("two", 2), ("three", 3), ("four", 4)])
self.assertEqual(
d, decode(encode(d), CodecOptions(document_class=OrderedDict)))
def test_tuple(self):
self.assertEqual({"tuple": [1, 2]}, decode(encode({"tuple": (1, 2)})))
def test_large_datetime_truncation(self):
# Ensure that a large datetime is truncated correctly.
dt1 = datetime.datetime(9999, 1, 1, 1, 1, 1, 999999)
dt2 = decode(encode({"date": dt1}))["date"]
self.assertEqual(dt2.microsecond, 999000)
self.assertEqual(dt2.second, dt1.second)
def test_invalid_field_name(self):
# Decode a truncated field
with self.assertRaises(InvalidBSON) as ctx:
decode(b'\x0b\x00\x00\x00\x02field\x00')
# Assert that the InvalidBSON error message is not empty.
self.assertTrue(str(ctx.exception))
def roundtrip(self, doc):
bsonbytes = encode(doc, codec_options=self.codecopts)
rt_document = decode(bsonbytes, codec_options=self.codecopts)
self.assertEqual(doc, rt_document)
def test_decoding_1_2_3(self):
for doc in [
# 1, Valid documents MUST be decoded to a DBRef:
{
"$ref": "coll0",
"$id": ObjectId("60a6fe9a54f4180c86309efa")
},
{
"$ref": "coll0",
"$id": 1
},
{
"$ref": "coll0",
"$id": None
},
{
"$ref": "coll0",
"$id": 1,
"$db": "db0"
},
# 2, Valid documents with extra fields:
{
"$ref": "coll0",
"$id": 1,
"$db": "db0",
"foo": "bar"
},
{
"$ref": "coll0",
"$id": 1,
"foo": True,
"bar": False
},
{
"$ref": "coll0",
"$id": 1,
"meta": {
"foo": 1,
"bar": 2
}
},
{
"$ref": "coll0",
"$id": 1,
"$foo": "bar"
},
{
"$ref": "coll0",
"$id": 1,
"foo.bar": 0
},
# 3, Valid documents with out of order fields:
{
"$id": 1,
"$ref": "coll0"
},
{
"$db": "db0",
"$ref": "coll0",
"$id": 1
},
{
"foo": 1,
"$id": 1,
"$ref": "coll0"
},
{
"foo": 1,
"$ref": "coll0",
"$id": 1,
"$db": "db0"
},
{
"foo": 1,
"$ref": "coll0",
"$id": 1,
"$db": "db0",
"bar": 1
},
]:
with self.subTest(doc=doc):
decoded = decode(encode({'dbref': doc}))
dbref = decoded['dbref']
self.assertIsInstance(dbref, DBRef)
self.assertEqual(dbref.collection, doc['$ref'])
self.assertEqual(dbref.id, doc['$id'])
self.assertEqual(dbref.database, doc.get('$db'))
for extra in set(doc.keys()) - {"$ref", "$id", "$db"}:
self.assertEqual(getattr(dbref, extra), doc[extra])
import logging
import bson
from pymongo_inmemory import MongoClient
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
with MongoClient() as client:
db = client["test-db"]
collection = db["my-collection"]
data = {"some": "data"}
inserted_id = collection.insert_one(data).inserted_id
mongo_dump = bson.decode(
client.pim_mongodump("test-db", "my-collection"))
assert mongo_dump["some"] == "data"
assert mongo_dump["_id"] == inserted_id
def get_doc(self, collection, doc_id):
obj = self._cache[collection].get(str(doc_id))
if self._strict and obj:
return bson.decode(obj)
return obj
import logging
import bson
from pymongo_inmemory import MongoClient
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
with MongoClient() as client1:
with MongoClient() as client2:
db1 = client1["test-db"]
db2 = client2["test-db"]
collection1 = db1["my-collection"]
collection2 = db2["my-collection"]
data = {"some": "data"}
inserted_id1 = collection1.insert_one(data).inserted_id
inserted_id2 = collection2.insert_one(data).inserted_id
mongo_dump1 = bson.decode(
client1.pim_mongodump("test-db", "my-collection"))
mongo_dump2 = bson.decode(
client2.pim_mongodump("test-db", "my-collection"))
assert mongo_dump1["some"] == "data"
assert mongo_dump1["_id"] == inserted_id1
assert mongo_dump2["some"] == "data"
assert mongo_dump2["_id"] == inserted_id2
def test_bad_dbref(self):
ref_only = {'ref': {'$ref': 'collection'}}
id_only = {'ref': {'$id': ObjectId()}}
self.assertEqual(ref_only, decode(encode(ref_only)))
self.assertEqual(id_only, decode(encode(id_only)))
def bson_decode(self, data):
"""Decode BSON to a dictionary."""
return bson.decode(data)
def toJsonBody(self):
return bson.decode(self.Body)
# See the file 'docs/LICENSE' for copying permission.
import datetime
import logging
import struct
try:
import bson
HAVE_BSON = True
except ImportError:
HAVE_BSON = False
else:
# The BSON module provided by pymongo works through its "BSON" class.
if hasattr(bson, "BSON"):
bson_decode = lambda d: bson.decode(d)
# The BSON module provided by "pip3 install bson" works through the "loads" function (just like pickle etc.)
elif hasattr(bson, "loads"):
bson_decode = lambda d: bson.loads(d)
else:
HAVE_BSON = False
from lib.cuckoo.common.logtbl import table as LOGTBL
from lib.cuckoo.common.utils import default_converter, get_filename_from_path
log = logging.getLogger(__name__)
###############################################################################
# Generic BSON based protocol - by rep
# Allows all kinds of languages / sources to generate input for Cuckoo,
# thus we can reuse report generation / signatures for other API trace sources.
def test_encoding_1_2(self):
for doc in [
# 1, Encoding DBRefs with basic fields:
{
"$ref": "coll0",
"$id": ObjectId("60a6fe9a54f4180c86309efa")
},
{
"$ref": "coll0",
"$id": 1
},
{
"$ref": "coll0",
"$id": None
},
{
"$ref": "coll0",
"$id": 1,
"$db": "db0"
},
# 2, Encoding DBRefs with extra, optional fields:
{
"$ref": "coll0",
"$id": 1,
"$db": "db0",
"foo": "bar"
},
{
"$ref": "coll0",
"$id": 1,
"foo": True,
"bar": False
},
{
"$ref": "coll0",
"$id": 1,
"meta": {
"foo": 1,
"bar": 2
}
},
{
"$ref": "coll0",
"$id": 1,
"$foo": "bar"
},
{
"$ref": "coll0",
"$id": 1,
"foo.bar": 0
},
]:
with self.subTest(doc=doc):
# Decode the test input to a DBRef via a BSON roundtrip.
encoded_doc = encode({'dbref': doc})
decoded = decode(encoded_doc)
dbref = decoded['dbref']
self.assertIsInstance(dbref, DBRef)
# Encode the DBRef.
encoded_dbref = encode(decoded)
self.assertEqual(encoded_dbref, encoded_doc)
# Ensure extra fields are present.
for extra in set(doc.keys()) - {"$ref", "$id", "$db"}:
self.assertEqual(getattr(dbref, extra), doc[extra])
def test_unicode_decode_error_handler(self):
enc = encode({"keystr": "foobar"})
# Test handling of bad key value.
invalid_key = enc[:7] + b'\xe9' + enc[8:]
replaced_key = b'ke\xe9str'.decode('utf-8', 'replace')
ignored_key = b'ke\xe9str'.decode('utf-8', 'ignore')
dec = decode(invalid_key,
CodecOptions(unicode_decode_error_handler="replace"))
self.assertEqual(dec, {replaced_key: "foobar"})
dec = decode(invalid_key,
CodecOptions(unicode_decode_error_handler="ignore"))
self.assertEqual(dec, {ignored_key: "foobar"})
self.assertRaises(InvalidBSON, decode, invalid_key,
CodecOptions(unicode_decode_error_handler="strict"))
self.assertRaises(InvalidBSON, decode, invalid_key, CodecOptions())
self.assertRaises(InvalidBSON, decode, invalid_key)
# Test handing of bad string value.
invalid_val = BSON(enc[:18] + b'\xe9' + enc[19:])
replaced_val = b'fo\xe9bar'.decode('utf-8', 'replace')
ignored_val = b'fo\xe9bar'.decode('utf-8', 'ignore')
dec = decode(invalid_val,
CodecOptions(unicode_decode_error_handler="replace"))
self.assertEqual(dec, {"keystr": replaced_val})
dec = decode(invalid_val,
CodecOptions(unicode_decode_error_handler="ignore"))
self.assertEqual(dec, {"keystr": ignored_val})
self.assertRaises(InvalidBSON, decode, invalid_val,
CodecOptions(unicode_decode_error_handler="strict"))
self.assertRaises(InvalidBSON, decode, invalid_val, CodecOptions())
self.assertRaises(InvalidBSON, decode, invalid_val)
# Test handing bad key + bad value.
invalid_both = enc[:7] + b'\xe9' + enc[8:18] + b'\xe9' + enc[19:]
dec = decode(invalid_both,
CodecOptions(unicode_decode_error_handler="replace"))
self.assertEqual(dec, {replaced_key: replaced_val})
dec = decode(invalid_both,
CodecOptions(unicode_decode_error_handler="ignore"))
self.assertEqual(dec, {ignored_key: ignored_val})
self.assertRaises(InvalidBSON, decode, invalid_both,
CodecOptions(unicode_decode_error_handler="strict"))
self.assertRaises(InvalidBSON, decode, invalid_both, CodecOptions())
self.assertRaises(InvalidBSON, decode, invalid_both)
# Test handling bad error mode.
dec = decode(enc, CodecOptions(unicode_decode_error_handler="junk"))
self.assertEqual(dec, {"keystr": "foobar"})
self.assertRaises(InvalidBSON, decode, invalid_both,
CodecOptions(unicode_decode_error_handler="junk"))
def _bsonDecode(self, value):
res = bson.decode(value)
return res.get("data")
def test_data_timestamp(self):
self.assertEqual({"test": Timestamp(4, 20)},
decode(b"\x13\x00\x00\x00\x11\x74\x65\x73\x74\x00\x14"
b"\x00\x00\x00\x04\x00\x00\x00\x00"))
def from_bson(cls, bson_bytes: bytes) -> "ASETDocumentBase":
"""
Deserialize a document base from a BSON byte string.
https://pymongo.readthedocs.io/en/stable/api/bson/index.html
:return: document base created from the BSON byte string
"""
tick: float = time.time()
serialized_base: Dict[str, Any] = bson.decode(bson_bytes)
# deserialize the document base
document_base: "ASETDocumentBase" = cls([], [])
for serialized_attribute in serialized_base["attributes"]:
# deserialize the attribute
attribute: ASETAttribute = ASETAttribute(
name=serialized_attribute["name"])
# deserialize the signals
for signal_str, serialized_signal in serialized_attribute[
"signals"].items():
attribute.signals[signal_str] = BaseSignal.from_serializable(
serialized_signal, signal_str)
document_base.attributes.append(attribute)
for serialized_document in serialized_base["documents"]:
# deserialize the document
document: ASETDocument = ASETDocument(
name=serialized_document["name"],
text=serialized_document["text"])
for serialized_nugget in serialized_document["nuggets"]:
# deserialize the nugget
nugget: ASETNugget = ASETNugget(
document=document,
start_char=serialized_nugget["start_char"],
end_char=serialized_nugget["end_char"],
extractor_str=serialized_nugget["extractor_str"],
type_str=serialized_nugget["type_str"],
value=serialized_nugget["value"])
# deserialize the signals
for signal_str, serialized_signal in serialized_nugget[
"signals"].items():
nugget.signals[signal_str] = BaseSignal.from_serializable(
serialized_signal, signal_str)
document.nuggets.append(nugget)
# deserialize the attribute mappings
for name, indices in serialized_document[
"attribute_mappings"].items():
document.attribute_mappings[name] = [
document.nuggets[idx] for idx in indices
]
# deserialize the annotations
for annotation_str, serialized_annotation in serialized_document[
"annotations"].items():
annotation: BaseAnnotation = BaseAnnotation.from_serializable(
serialized_annotation, annotation_str)
document.annotations[annotation_str] = annotation
document_base.documents.append(document)
tack: float = time.time()
logger.info(f"Deserialized document base in {tack - tick} seconds.")
return document_base
def test_dst(self):
d = {"x": datetime.datetime(1993, 4, 4, 2)}
self.assertEqual(d, decode(encode(d)))
def disconnect_task(msg):
payload = bson.decode(msg.body)
print("disconnect... msg: %s" % (payload))
Device.remove_connection(payload)
def test_unicode_regex(self):
regex = re.compile('revisi\xf3n')
decode(encode({"regex": regex}))
def connect_task(msg):
payload = bson.decode(msg.body)
print("connected... msg: %s" % (payload))
Device.add_connection(payload)
def decode_raw(val):
"""Decode RawBSONDocuments in the given container."""
if isinstance(val, (list, abc.Mapping)):
return decode(encode({'v': val}, codec_options=OPTS), OPTS)['v']
return val
json_options_uuid_04 = json_util.JSONOptions(json_mode=JSONMode.CANONICAL,
uuid_representation=STANDARD)
json_options_iso8601 = json_util.JSONOptions(
datetime_representation=json_util.DatetimeRepresentation.ISO8601)
to_extjson = functools.partial(json_util.dumps,
json_options=json_util.CANONICAL_JSON_OPTIONS)
to_extjson_uuid_04 = functools.partial(json_util.dumps,
json_options=json_options_uuid_04)
to_extjson_iso8601 = functools.partial(json_util.dumps,
json_options=json_options_iso8601)
to_relaxed_extjson = functools.partial(
json_util.dumps, json_options=json_util.RELAXED_JSON_OPTIONS)
to_bson_uuid_04 = functools.partial(encode,
codec_options=codec_options_uuid_04)
to_bson = functools.partial(encode, codec_options=codec_options)
decode_bson = lambda bbytes: decode(bbytes, codec_options=codec_options)
decode_extjson = functools.partial(json_util.loads,
json_options=json_util.JSONOptions(
json_mode=JSONMode.CANONICAL,
document_class=SON))
loads = functools.partial(json.loads, object_pairs_hook=SON)
class TestBSONCorpus(unittest.TestCase):
def assertJsonEqual(self, first, second, msg=None):
"""Fail if the two json strings are unequal.
Normalize json by parsing it with the built-in json library. This
accounts for discrepancies in spacing.
"""
self.assertEqual(loads(first), loads(second), msg=msg)
def do_task(self):
for _ in range(NUM_DOCS):
decode(self.document)
def bson_encode(x):
r = bson.decode(x.read())['data']
return r