def test_backports(self):
doc = BSON.encode({"tuple": (1, 2)})
exp = {"tuple": [1, 2]}
options = CodecOptions(uuid_representation=ALL_UUID_REPRESENTATIONS[0],
tz_aware=False, document_class=dict)
self.assertEqual(
{"tuple": [1, 2]},
BSON.encode(
{"tuple": (1, 2)}, codec_options=options,
uuid_subtype=ALL_UUID_REPRESENTATIONS[1]).decode())
self.assertEqual(exp, doc.decode(
as_class=SON,
tz_aware=True,
uuid_subtype=ALL_UUID_REPRESENTATIONS[1],
codec_options=options))
self.assertEqual([exp], list(decode_iter(
doc,
as_class=SON,
tz_aware=True,
uuid_subtype=ALL_UUID_REPRESENTATIONS[1],
codec_options=options)))
self.assertEqual([exp], list(decode_file_iter(
StringIO(doc),
as_class=SON,
tz_aware=True,
uuid_subtype=ALL_UUID_REPRESENTATIONS[1],
codec_options=options)))
self.assertEqual([exp], decode_all(
doc, SON, True, ALL_UUID_REPRESENTATIONS[1], True, options))
def setUp(self):
# prepare the app
settings_file = os.path.join(SETTINGS_DIR, 'settings.py')
self.app = eve.Eve(settings=settings_file,
url_converters=None,
auth=TokenAuth,
validator=ConsentValidatorEve)
# load normalization.
with open(BSON_FILE) as fin:
mappings = list(bson.decode_iter(fin.read()))
# add normalization.
self.db = get_db()
for mapping in mappings:
self.db['normalize'].drop()
self.db['normalize'].insert(mapping)
# create the validator.
resource_def = self.app.config['DOMAIN']['trial']
schema = resource_def['schema']
#print self.app.config['SOURCES']
with self.app.app_context():
self.v = self.app.validator(schema=schema, resource='trial')
def test_basic_decode(self):
self.assertEqual({"test": u"hello world"},
BSON(b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74\x00\x0C"
b"\x00\x00\x00\x68\x65\x6C\x6C\x6F\x20\x77\x6F"
b"\x72\x6C\x64\x00\x00").decode())
self.assertEqual([{
"test": u"hello world"
}, {}],
decode_all(b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
b"\x05\x00\x00\x00\x00"))
self.assertEqual([{
"test": u"hello world"
}, {}],
list(
decode_iter(
b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
b"\x05\x00\x00\x00\x00")))
self.assertEqual([{
"test": u"hello world"
}, {}],
list(
decode_file_iter(
StringIO(
b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
b"\x05\x00\x00\x00\x00"))))
def setUp(self):
# prepare the app
settings_file = os.path.join(SETTINGS_DIR, 'settings.py')
self.app = eve.Eve(settings=settings_file,
url_converters = None,
auth = TokenAuth,
validator = ConsentValidatorEve)
# load normalization.
with open(BSON_FILE) as fin:
mappings = list(bson.decode_iter(fin.read()))
# add normalization.
self.db = get_db()
for mapping in mappings:
self.db['normalize'].drop()
self.db['normalize'].insert(mapping)
# create the validator.
resource_def = self.app.config['DOMAIN']['trial']
schema = resource_def['schema']
#print self.app.config['SOURCES']
with self.app.app_context():
self.v = self.app.validator(schema=schema, resource='trial')
def children(self):
"""children."""
memory = gdb.selected_inferior().read_memory(self.ptr,
self.size).tobytes()
bsonobj = next(bson.decode_iter(memory)) # pylint: disable=stop-iteration-return
for key, value in list(bsonobj.items()):
yield 'key', key
yield 'value', bson.json_util.dumps(value)
def parse_msg(msg_bytes):
""" Return a command from a binary mongo db message or None if we shoudln't
trace it. The protocol is documented here:
http://docs.mongodb.com/manual/reference/mongodb-wire-protocol
"""
# NOTE[matt] this is used for queries in pymongo <= 3.0.0 and for inserts
# in up to date versions.
msg_len = len(msg_bytes)
if msg_len <= 0:
return None
header = header_struct.unpack_from(msg_bytes, 0)
(length, req_id, response_to, op_code) = header
op = OP_CODES.get(op_code)
if not op:
log.debug("unknown op code: %s", op_code)
return None
db = None
coll = None
offset = header_struct.size
cmd = None
if op == "query":
# NOTE[matt] inserts, updates and queries can all use this opcode
offset += 4 # skip flags
ns = _cstring(msg_bytes[offset:])
offset += len(ns) + 1 # include null terminator
# note: here coll could be '$cmd' because it can be overridden in the
# query itself (like {"insert":"songs"})
db, coll = _split_namespace(ns)
offset += 8 # skip numberToSkip & numberToReturn
if msg_len <= MAX_MSG_PARSE_LEN:
# FIXME[matt] don't try to parse large messages for performance
# reasons. ideally we'd just peek at the first bytes to get
# the critical info (op type, collection, query, # of docs)
# rather than parse the whole thing. i suspect only massive
# inserts will be affected.
codec = CodecOptions(SON)
spec = next(
bson.decode_iter(msg_bytes[offset:], codec_options=codec))
cmd = parse_spec(spec, db)
else:
# let's still note that a command happened.
cmd = Command("command", db, "untraced_message_too_large")
# If the command didn't contain namespace info, set it here.
if not cmd.coll:
cmd.coll = coll
cmd.metrics[netx.BYTES_OUT] = msg_len
return cmd
def parse_msg(msg_bytes):
""" Return a command from a binary mongo db message or None if we shoudln't
trace it. The protocol is documented here:
http://docs.mongodb.com/manual/reference/mongodb-wire-protocol
"""
# NOTE[matt] this is used for queries in pymongo <= 3.0.0 and for inserts
# in up to date versions.
msg_len = len(msg_bytes)
if msg_len <= 0:
return None
header = header_struct.unpack_from(msg_bytes, 0)
(length, req_id, response_to, op_code) = header
op = OP_CODES.get(op_code)
if not op:
log.debug("unknown op code: %s", op_code)
return None
db = None
coll = None
offset = header_struct.size
cmd = None
if op == "query":
# NOTE[matt] inserts, updates and queries can all use this opcode
offset += 4 # skip flags
ns = _cstring(msg_bytes[offset:])
offset += len(ns) + 1 # include null terminator
# note: here coll could be '$cmd' because it can be overridden in the
# query itself (like {"insert":"songs"})
db, coll = _split_namespace(ns)
offset += 8 # skip numberToSkip & numberToReturn
if msg_len <= MAX_MSG_PARSE_LEN:
# FIXME[matt] don't try to parse large messages for performance
# reasons. ideally we'd just peek at the first bytes to get
# the critical info (op type, collection, query, # of docs)
# rather than parse the whole thing. i suspect only massive
# inserts will be affected.
codec = CodecOptions(SON)
spec = next(bson.decode_iter(msg_bytes[offset:], codec_options=codec))
cmd = parse_spec(spec, db)
else:
# let's still note that a command happened.
cmd = Command("command", db, "untraced_message_too_large")
# If the command didn't contain namespace info, set it here.
if not cmd.coll:
cmd.coll = coll
cmd.metrics[netx.BYTES_OUT] = msg_len
return cmd
def test_invalid_decodes(self):
# Invalid object size (not enough bytes in document for even
# an object size of first object.
# NOTE: decode_all and decode_iter don't care, not sure if they should?
self.assertRaises(InvalidBSON, list,
decode_file_iter(StringIO(b"\x1B")))
# An object size that's too small to even include the object size,
# but is correctly encoded, along with a correct EOO (and no data).
data = b"\x01\x00\x00\x00\x00"
self.assertRaises(InvalidBSON, decode_all, data)
self.assertRaises(InvalidBSON, list, decode_iter(data))
self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))
# One object, but with object size listed smaller than it is in the
# data.
data = (b"\x1A\x00\x00\x00\x0E\x74\x65\x73\x74"
b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
b"\x05\x00\x00\x00\x00")
self.assertRaises(InvalidBSON, decode_all, data)
self.assertRaises(InvalidBSON, list, decode_iter(data))
self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))
# One object, missing the EOO at the end.
data = (b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
b"\x05\x00\x00\x00")
self.assertRaises(InvalidBSON, decode_all, data)
self.assertRaises(InvalidBSON, list, decode_iter(data))
self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))
# One object, sized correctly, with a spot for an EOO, but the EOO
# isn't 0x00.
data = (b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
b"\x05\x00\x00\x00\xFF")
self.assertRaises(InvalidBSON, decode_all, data)
self.assertRaises(InvalidBSON, list, decode_iter(data))
self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))
def test_invalid_decodes(self):
# Invalid object size (not enough bytes in document for even
# an object size of first object.
# NOTE: decode_all and decode_iter don't care, not sure if they should?
self.assertRaises(InvalidBSON, list,
decode_file_iter(StringIO(b"\x1B")))
# An object size that's too small to even include the object size,
# but is correctly encoded, along with a correct EOO (and no data).
data = b"\x01\x00\x00\x00\x00"
self.assertRaises(InvalidBSON, decode_all, data)
self.assertRaises(InvalidBSON, list, decode_iter(data))
self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))
# One object, but with object size listed smaller than it is in the
# data.
data = (b"\x1A\x00\x00\x00\x0E\x74\x65\x73\x74"
b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
b"\x05\x00\x00\x00\x00")
self.assertRaises(InvalidBSON, decode_all, data)
self.assertRaises(InvalidBSON, list, decode_iter(data))
self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))
# One object, missing the EOO at the end.
data = (b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
b"\x05\x00\x00\x00")
self.assertRaises(InvalidBSON, decode_all, data)
self.assertRaises(InvalidBSON, list, decode_iter(data))
self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))
# One object, sized correctly, with a spot for an EOO, but the EOO
# isn't 0x00.
data = (b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
b"\x05\x00\x00\x00\xFF")
self.assertRaises(InvalidBSON, decode_all, data)
self.assertRaises(InvalidBSON, list, decode_iter(data))
self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))
def invoke(self, args, _from_tty): # pylint: disable=no-self-use
"""Invoke."""
args = args.split(' ')
if len(args) == 0 or (len(args) == 1 and len(args[0]) == 0):
print("Usage: mongodb-pprint-bson <ptr> <optional length>")
return
ptr = eval(args[0]) # pylint: disable=eval-used
size = 20 * 1024
if len(args) >= 2:
size = int(args[1])
print("Pretty printing bson object at %s (%d bytes)" % (ptr, size))
memory = gdb.selected_inferior().read_memory(ptr, size).tobytes()
bsonobj = next(bson.decode_iter(memory))
from pprint import pprint
pprint(bsonobj)
def children(self):
"""children."""
if self.val['type'] != 3:
# Type 3 is a "normal" update. Notably type 4 is a deletion and type 1 represents a
# delta relative to the previous committed version in the update chain. Only attempt
# to parse type 3 as bson.
return
memory = gdb.selected_inferior().read_memory(self.ptr,
self.size).tobytes()
bsonobj = None
try:
bsonobj = next(bson.decode_iter(memory)) # pylint: disable=stop-iteration-return
except bson.errors.InvalidBSON:
return
for key, value in list(bsonobj.items()):
yield 'key', key
yield 'value', bson.json_util.dumps(value)
def test_invalid_decodes(self):
# Invalid object size (not enough bytes in document for even
# an object size of first object.
# NOTE: decode_all and decode_iter don't care, not sure if they should?
self.assertRaises(InvalidBSON, list,
decode_file_iter(BytesIO(b"\x1B")))
bad_bsons = [
# An object size that's too small to even include the object size,
# but is correctly encoded, along with a correct EOO (and no data).
b"\x01\x00\x00\x00\x00",
# One object, but with object size listed smaller than it is in the
# data.
(b"\x1A\x00\x00\x00\x0E\x74\x65\x73\x74"
b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
b"\x05\x00\x00\x00\x00"),
# One object, missing the EOO at the end.
(b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
b"\x05\x00\x00\x00"),
# One object, sized correctly, with a spot for an EOO, but the EOO
# isn't 0x00.
(b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
b"\x05\x00\x00\x00\xFF"),
]
for i, data in enumerate(bad_bsons):
msg = "bad_bson[{}]".format(i)
with self.assertRaises(InvalidBSON, msg=msg):
decode_all(data)
with self.assertRaises(InvalidBSON, msg=msg):
list(decode_iter(data))
with self.assertRaises(InvalidBSON, msg=msg):
list(decode_file_iter(BytesIO(data)))
with tempfile.TemporaryFile() as scratch:
scratch.write(data)
scratch.seek(0, os.SEEK_SET)
with self.assertRaises(InvalidBSON, msg=msg):
list(decode_file_iter(scratch))
def test_basic_decode(self):
self.assertEqual({"test": u("hello world")},
BSON(b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74\x00\x0C"
b"\x00\x00\x00\x68\x65\x6C\x6C\x6F\x20\x77\x6F"
b"\x72\x6C\x64\x00\x00").decode())
self.assertEqual([{"test": u("hello world")}, {}],
decode_all(b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
b"\x05\x00\x00\x00\x00"))
self.assertEqual([{"test": u("hello world")}, {}],
list(decode_iter(
b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
b"\x05\x00\x00\x00\x00")))
self.assertEqual([{"test": u("hello world")}, {}],
list(decode_file_iter(StringIO(
b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
b"\x05\x00\x00\x00\x00"))))
def test_backport_codec_options_uuid(self):
if not should_test_uuid:
raise SkipTest("No uuid module")
# Generated by the Java driver
from_java = b('bAAAAAdfaWQAUCBQxkVm+XdxJ9tOBW5ld2d1aWQAEAAAAAMIQkfACFu'
'Z/0RustLOU/G6Am5ld2d1aWRzdHJpbmcAJQAAAGZmOTk1YjA4LWMwND'
'ctNDIwOC1iYWYxLTUzY2VkMmIyNmU0NAAAbAAAAAdfaWQAUCBQxkVm+'
'XdxJ9tPBW5ld2d1aWQAEAAAAANgS/xhRXXv8kfIec+dYdyCAm5ld2d1'
'aWRzdHJpbmcAJQAAAGYyZWY3NTQ1LTYxZmMtNGI2MC04MmRjLTYxOWR'
'jZjc5Yzg0NwAAbAAAAAdfaWQAUCBQxkVm+XdxJ9tQBW5ld2d1aWQAEA'
'AAAAPqREIbhZPUJOSdHCJIgaqNAm5ld2d1aWRzdHJpbmcAJQAAADI0Z'
'DQ5Mzg1LTFiNDItNDRlYS04ZGFhLTgxNDgyMjFjOWRlNAAAbAAAAAdf'
'aWQAUCBQxkVm+XdxJ9tRBW5ld2d1aWQAEAAAAANjQBn/aQuNfRyfNyx'
'29COkAm5ld2d1aWRzdHJpbmcAJQAAADdkOGQwYjY5LWZmMTktNDA2My'
'1hNDIzLWY0NzYyYzM3OWYxYwAAbAAAAAdfaWQAUCBQxkVm+XdxJ9tSB'
'W5ld2d1aWQAEAAAAAMtSv/Et1cAQUFHUYevqxaLAm5ld2d1aWRzdHJp'
'bmcAJQAAADQxMDA1N2I3LWM0ZmYtNGEyZC04YjE2LWFiYWY4NzUxNDc'
'0MQAA')
data = base64.b64decode(from_java)
# Test decode_all.
docs = bson.decode_all(
data, dict, True, JAVA_LEGACY, True,
CodecOptions(SON, False, STANDARD))
for d in docs:
self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring']))
encoded = [bson.BSON.encode(
doc, uuid_subtype=JAVA_LEGACY,
codec_options=CodecOptions(uuid_representation=STANDARD))
for doc in docs]
# Test decode.
docs2 = [e.decode(
uuid_subtype=JAVA_LEGACY, as_class=dict, tz_aware=True,
codec_options=CodecOptions(SON, False, STANDARD))
for e in encoded]
for d in docs2:
self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring']))
# Test encode.
for i in range(len(docs)):
self.assertEqual(docs2[i]['newguid'], docs[i]['newguid'])
self.assertEqual(uuid.UUID(docs2[i]['newguidstring']),
uuid.UUID(docs[i]['newguidstring']))
# Test decode_iter
docs = bson.decode_iter(
data, dict, True, JAVA_LEGACY,
True, CodecOptions(SON, False, STANDARD))
for d in docs:
self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring']))
# Test decode_file_iter
docs = bson.decode_file_iter(
StringIO(data), dict, True, JAVA_LEGACY,
True, CodecOptions(SON, False, STANDARD))
for d in docs:
self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring']))
def test_decode_iter(self):
expected, bson_data = self._generate_multidocument_bson_stream()
for expected_doc, decoded_doc in zip(
expected, decode_iter(bson_data, self.codecopts)):
self.assertEqual(expected_doc, decoded_doc)
def test_decode_iter(self):
expected, bson_data = self._generate_multidocument_bson_stream()
for expected_doc, decoded_doc in zip(
expected, decode_iter(bson_data, self.codecopts)):
self.assertEqual(expected_doc, decoded_doc)
#urlloc = "moschetti.org:7779"
headers = {"Accept": "application/" + fmt}
conn = http.client.HTTPConnection(urlloc)
params = None
conn.request("GET", "/func1", params, headers)
response = conn.getresponse()
#print response.status, response.reason
data = response.read()
if fmt == "json":
doc = json.loads(data)
print(doc)
elif fmt == "bson":
options = CodecOptions(document_class=collections.OrderedDict)
for doc in bson.decode_iter(data, codec_options=options):
print(doc)
# Important to get things back to native decimal...
z2 = doc['bson128_amt'].to_decimal()
print("mult:", z2 * doc['num'])
print("escaped:", doc['CR'])
print("quotes:", doc['quotes'])
conn.close()