def test_backports(self):
        doc = BSON.encode({"tuple": (1, 2)})
        exp = {"tuple": [1, 2]}
        options = CodecOptions(uuid_representation=ALL_UUID_REPRESENTATIONS[0],
                               tz_aware=False, document_class=dict)

        self.assertEqual(
            {"tuple": [1, 2]},
            BSON.encode(
                {"tuple": (1, 2)}, codec_options=options,
                uuid_subtype=ALL_UUID_REPRESENTATIONS[1]).decode())
        self.assertEqual(exp, doc.decode(
            as_class=SON,
            tz_aware=True,
            uuid_subtype=ALL_UUID_REPRESENTATIONS[1],
            codec_options=options))
        self.assertEqual([exp], list(decode_iter(
            doc,
            as_class=SON,
            tz_aware=True,
            uuid_subtype=ALL_UUID_REPRESENTATIONS[1],
            codec_options=options)))
        self.assertEqual([exp], list(decode_file_iter(
            StringIO(doc),
            as_class=SON,
            tz_aware=True,
            uuid_subtype=ALL_UUID_REPRESENTATIONS[1],
            codec_options=options)))
        self.assertEqual([exp], decode_all(
            doc, SON, True, ALL_UUID_REPRESENTATIONS[1], True, options))
Exemple #2
0
    def setUp(self):

        # prepare the app
        settings_file = os.path.join(SETTINGS_DIR, 'settings.py')
        self.app = eve.Eve(settings=settings_file,
                           url_converters=None,
                           auth=TokenAuth,
                           validator=ConsentValidatorEve)

        # load normalization.
        with open(BSON_FILE) as fin:
            mappings = list(bson.decode_iter(fin.read()))

        # add normalization.
        self.db = get_db()
        for mapping in mappings:
            self.db['normalize'].drop()
            self.db['normalize'].insert(mapping)

        # create the validator.
        resource_def = self.app.config['DOMAIN']['trial']
        schema = resource_def['schema']

        #print self.app.config['SOURCES']
        with self.app.app_context():
            self.v = self.app.validator(schema=schema, resource='trial')
 def test_basic_decode(self):
     self.assertEqual({"test": u"hello world"},
                      BSON(b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74\x00\x0C"
                           b"\x00\x00\x00\x68\x65\x6C\x6C\x6F\x20\x77\x6F"
                           b"\x72\x6C\x64\x00\x00").decode())
     self.assertEqual([{
         "test": u"hello world"
     }, {}],
                      decode_all(b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
                                 b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
                                 b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
                                 b"\x05\x00\x00\x00\x00"))
     self.assertEqual([{
         "test": u"hello world"
     }, {}],
                      list(
                          decode_iter(
                              b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
                              b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
                              b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
                              b"\x05\x00\x00\x00\x00")))
     self.assertEqual([{
         "test": u"hello world"
     }, {}],
                      list(
                          decode_file_iter(
                              StringIO(
                                  b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
                                  b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
                                  b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
                                  b"\x05\x00\x00\x00\x00"))))
Exemple #4
0
    def setUp(self):

        # prepare the app
        settings_file = os.path.join(SETTINGS_DIR, 'settings.py')
        self.app = eve.Eve(settings=settings_file,
                            url_converters = None,
                            auth = TokenAuth,
                            validator = ConsentValidatorEve)

        # load normalization.
        with open(BSON_FILE) as fin:
            mappings = list(bson.decode_iter(fin.read()))

        # add normalization.
        self.db = get_db()
        for mapping in mappings:
            self.db['normalize'].drop()
            self.db['normalize'].insert(mapping)

        # create the validator.
        resource_def = self.app.config['DOMAIN']['trial']
        schema = resource_def['schema']

        #print self.app.config['SOURCES']
        with self.app.app_context():
            self.v = self.app.validator(schema=schema, resource='trial')
Exemple #5
0
    def children(self):
        """children."""
        memory = gdb.selected_inferior().read_memory(self.ptr,
                                                     self.size).tobytes()
        bsonobj = next(bson.decode_iter(memory))  # pylint: disable=stop-iteration-return

        for key, value in list(bsonobj.items()):
            yield 'key', key
            yield 'value', bson.json_util.dumps(value)
Exemple #6
0
def parse_msg(msg_bytes):
    """ Return a command from a binary mongo db message or None if we shoudln't
        trace it. The protocol is documented here:
        http://docs.mongodb.com/manual/reference/mongodb-wire-protocol
    """
    # NOTE[matt] this is used for queries in pymongo <= 3.0.0 and for inserts
    # in up to date versions.
    msg_len = len(msg_bytes)
    if msg_len <= 0:
        return None

    header = header_struct.unpack_from(msg_bytes, 0)
    (length, req_id, response_to, op_code) = header

    op = OP_CODES.get(op_code)
    if not op:
        log.debug("unknown op code: %s", op_code)
        return None

    db = None
    coll = None

    offset = header_struct.size
    cmd = None
    if op == "query":
        # NOTE[matt] inserts, updates and queries can all use this opcode

        offset += 4  # skip flags
        ns = _cstring(msg_bytes[offset:])
        offset += len(ns) + 1  # include null terminator

        # note: here coll could be '$cmd' because it can be overridden in the
        # query itself (like {"insert":"songs"})
        db, coll = _split_namespace(ns)

        offset += 8  # skip numberToSkip & numberToReturn
        if msg_len <= MAX_MSG_PARSE_LEN:
            # FIXME[matt] don't try to parse large messages for performance
            # reasons. ideally we'd just peek at the first bytes to get
            # the critical info (op type, collection, query, # of docs)
            # rather than parse the whole thing. i suspect only massive
            # inserts will be affected.
            codec = CodecOptions(SON)
            spec = next(
                bson.decode_iter(msg_bytes[offset:], codec_options=codec))
            cmd = parse_spec(spec, db)
        else:
            # let's still note that a command happened.
            cmd = Command("command", db, "untraced_message_too_large")

        # If the command didn't contain namespace info, set it here.
        if not cmd.coll:
            cmd.coll = coll

    cmd.metrics[netx.BYTES_OUT] = msg_len
    return cmd
Exemple #7
0
def parse_msg(msg_bytes):
    """ Return a command from a binary mongo db message or None if we shoudln't
        trace it. The protocol is documented here:
        http://docs.mongodb.com/manual/reference/mongodb-wire-protocol
    """
    # NOTE[matt] this is used for queries in pymongo <= 3.0.0 and for inserts
    # in up to date versions.
    msg_len = len(msg_bytes)
    if msg_len <= 0:
        return None

    header = header_struct.unpack_from(msg_bytes, 0)
    (length, req_id, response_to, op_code) = header

    op = OP_CODES.get(op_code)
    if not op:
        log.debug("unknown op code: %s", op_code)
        return None

    db = None
    coll = None

    offset = header_struct.size
    cmd = None
    if op == "query":
        # NOTE[matt] inserts, updates and queries can all use this opcode

        offset += 4  # skip flags
        ns = _cstring(msg_bytes[offset:])
        offset += len(ns) + 1  # include null terminator

        # note: here coll could be '$cmd' because it can be overridden in the
        # query itself (like {"insert":"songs"})
        db, coll = _split_namespace(ns)

        offset += 8  # skip numberToSkip & numberToReturn
        if msg_len <= MAX_MSG_PARSE_LEN:
            # FIXME[matt] don't try to parse large messages for performance
            # reasons. ideally we'd just peek at the first bytes to get
            # the critical info (op type, collection, query, # of docs)
            # rather than parse the whole thing. i suspect only massive
            # inserts will be affected.
            codec = CodecOptions(SON)
            spec = next(bson.decode_iter(msg_bytes[offset:], codec_options=codec))
            cmd = parse_spec(spec, db)
        else:
            # let's still note that a command happened.
            cmd = Command("command", db, "untraced_message_too_large")

        # If the command didn't contain namespace info, set it here.
        if not cmd.coll:
            cmd.coll = coll

    cmd.metrics[netx.BYTES_OUT] = msg_len
    return cmd
    def test_invalid_decodes(self):
        # Invalid object size (not enough bytes in document for even
        # an object size of first object.
        # NOTE: decode_all and decode_iter don't care, not sure if they should?
        self.assertRaises(InvalidBSON, list,
                          decode_file_iter(StringIO(b"\x1B")))

        # An object size that's too small to even include the object size,
        # but is correctly encoded, along with a correct EOO (and no data).
        data = b"\x01\x00\x00\x00\x00"
        self.assertRaises(InvalidBSON, decode_all, data)
        self.assertRaises(InvalidBSON, list, decode_iter(data))
        self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))

        # One object, but with object size listed smaller than it is in the
        # data.
        data = (b"\x1A\x00\x00\x00\x0E\x74\x65\x73\x74"
                b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
                b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
                b"\x05\x00\x00\x00\x00")
        self.assertRaises(InvalidBSON, decode_all, data)
        self.assertRaises(InvalidBSON, list, decode_iter(data))
        self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))

        # One object, missing the EOO at the end.
        data = (b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
                b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
                b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
                b"\x05\x00\x00\x00")
        self.assertRaises(InvalidBSON, decode_all, data)
        self.assertRaises(InvalidBSON, list, decode_iter(data))
        self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))

        # One object, sized correctly, with a spot for an EOO, but the EOO
        # isn't 0x00.
        data = (b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
                b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
                b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
                b"\x05\x00\x00\x00\xFF")
        self.assertRaises(InvalidBSON, decode_all, data)
        self.assertRaises(InvalidBSON, list, decode_iter(data))
        self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))
    def test_invalid_decodes(self):
        # Invalid object size (not enough bytes in document for even
        # an object size of first object.
        # NOTE: decode_all and decode_iter don't care, not sure if they should?
        self.assertRaises(InvalidBSON, list,
                          decode_file_iter(StringIO(b"\x1B")))

        # An object size that's too small to even include the object size,
        # but is correctly encoded, along with a correct EOO (and no data).
        data = b"\x01\x00\x00\x00\x00"
        self.assertRaises(InvalidBSON, decode_all, data)
        self.assertRaises(InvalidBSON, list, decode_iter(data))
        self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))

        # One object, but with object size listed smaller than it is in the
        # data.
        data = (b"\x1A\x00\x00\x00\x0E\x74\x65\x73\x74"
                b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
                b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
                b"\x05\x00\x00\x00\x00")
        self.assertRaises(InvalidBSON, decode_all, data)
        self.assertRaises(InvalidBSON, list, decode_iter(data))
        self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))

        # One object, missing the EOO at the end.
        data = (b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
                b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
                b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
                b"\x05\x00\x00\x00")
        self.assertRaises(InvalidBSON, decode_all, data)
        self.assertRaises(InvalidBSON, list, decode_iter(data))
        self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))

        # One object, sized correctly, with a spot for an EOO, but the EOO
        # isn't 0x00.
        data = (b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
                b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
                b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
                b"\x05\x00\x00\x00\xFF")
        self.assertRaises(InvalidBSON, decode_all, data)
        self.assertRaises(InvalidBSON, list, decode_iter(data))
        self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))
Exemple #10
0
    def invoke(self, args, _from_tty):  # pylint: disable=no-self-use
        """Invoke."""
        args = args.split(' ')
        if len(args) == 0 or (len(args) == 1 and len(args[0]) == 0):
            print("Usage: mongodb-pprint-bson <ptr> <optional length>")
            return

        ptr = eval(args[0])  # pylint: disable=eval-used
        size = 20 * 1024
        if len(args) >= 2:
            size = int(args[1])
        print("Pretty printing bson object at %s (%d bytes)" % (ptr, size))

        memory = gdb.selected_inferior().read_memory(ptr, size).tobytes()
        bsonobj = next(bson.decode_iter(memory))

        from pprint import pprint
        pprint(bsonobj)
Exemple #11
0
    def children(self):
        """children."""
        if self.val['type'] != 3:
            # Type 3 is a "normal" update. Notably type 4 is a deletion and type 1 represents a
            # delta relative to the previous committed version in the update chain. Only attempt
            # to parse type 3 as bson.
            return

        memory = gdb.selected_inferior().read_memory(self.ptr,
                                                     self.size).tobytes()
        bsonobj = None
        try:
            bsonobj = next(bson.decode_iter(memory))  # pylint: disable=stop-iteration-return
        except bson.errors.InvalidBSON:
            return

        for key, value in list(bsonobj.items()):
            yield 'key', key
            yield 'value', bson.json_util.dumps(value)
    def test_invalid_decodes(self):
        # Invalid object size (not enough bytes in document for even
        # an object size of first object.
        # NOTE: decode_all and decode_iter don't care, not sure if they should?
        self.assertRaises(InvalidBSON, list,
                          decode_file_iter(BytesIO(b"\x1B")))

        bad_bsons = [
            # An object size that's too small to even include the object size,
            # but is correctly encoded, along with a correct EOO (and no data).
            b"\x01\x00\x00\x00\x00",
            # One object, but with object size listed smaller than it is in the
            # data.
            (b"\x1A\x00\x00\x00\x0E\x74\x65\x73\x74"
             b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
             b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
             b"\x05\x00\x00\x00\x00"),
            # One object, missing the EOO at the end.
            (b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
             b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
             b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
             b"\x05\x00\x00\x00"),
            # One object, sized correctly, with a spot for an EOO, but the EOO
            # isn't 0x00.
            (b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
             b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
             b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
             b"\x05\x00\x00\x00\xFF"),
        ]
        for i, data in enumerate(bad_bsons):
            msg = "bad_bson[{}]".format(i)
            with self.assertRaises(InvalidBSON, msg=msg):
                decode_all(data)
            with self.assertRaises(InvalidBSON, msg=msg):
                list(decode_iter(data))
            with self.assertRaises(InvalidBSON, msg=msg):
                list(decode_file_iter(BytesIO(data)))
            with tempfile.TemporaryFile() as scratch:
                scratch.write(data)
                scratch.seek(0, os.SEEK_SET)
                with self.assertRaises(InvalidBSON, msg=msg):
                    list(decode_file_iter(scratch))
 def test_basic_decode(self):
     self.assertEqual({"test": u("hello world")},
                      BSON(b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74\x00\x0C"
                           b"\x00\x00\x00\x68\x65\x6C\x6C\x6F\x20\x77\x6F"
                           b"\x72\x6C\x64\x00\x00").decode())
     self.assertEqual([{"test": u("hello world")}, {}],
                      decode_all(b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
                                 b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
                                 b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
                                 b"\x05\x00\x00\x00\x00"))
     self.assertEqual([{"test": u("hello world")}, {}],
                      list(decode_iter(
                         b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
                         b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
                         b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
                         b"\x05\x00\x00\x00\x00")))
     self.assertEqual([{"test": u("hello world")}, {}],
                      list(decode_file_iter(StringIO(
                         b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
                         b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
                         b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
                         b"\x05\x00\x00\x00\x00"))))
    def test_backport_codec_options_uuid(self):
        if not should_test_uuid:
            raise SkipTest("No uuid module")

        # Generated by the Java driver
        from_java = b('bAAAAAdfaWQAUCBQxkVm+XdxJ9tOBW5ld2d1aWQAEAAAAAMIQkfACFu'
                      'Z/0RustLOU/G6Am5ld2d1aWRzdHJpbmcAJQAAAGZmOTk1YjA4LWMwND'
                      'ctNDIwOC1iYWYxLTUzY2VkMmIyNmU0NAAAbAAAAAdfaWQAUCBQxkVm+'
                      'XdxJ9tPBW5ld2d1aWQAEAAAAANgS/xhRXXv8kfIec+dYdyCAm5ld2d1'
                      'aWRzdHJpbmcAJQAAAGYyZWY3NTQ1LTYxZmMtNGI2MC04MmRjLTYxOWR'
                      'jZjc5Yzg0NwAAbAAAAAdfaWQAUCBQxkVm+XdxJ9tQBW5ld2d1aWQAEA'
                      'AAAAPqREIbhZPUJOSdHCJIgaqNAm5ld2d1aWRzdHJpbmcAJQAAADI0Z'
                      'DQ5Mzg1LTFiNDItNDRlYS04ZGFhLTgxNDgyMjFjOWRlNAAAbAAAAAdf'
                      'aWQAUCBQxkVm+XdxJ9tRBW5ld2d1aWQAEAAAAANjQBn/aQuNfRyfNyx'
                      '29COkAm5ld2d1aWRzdHJpbmcAJQAAADdkOGQwYjY5LWZmMTktNDA2My'
                      '1hNDIzLWY0NzYyYzM3OWYxYwAAbAAAAAdfaWQAUCBQxkVm+XdxJ9tSB'
                      'W5ld2d1aWQAEAAAAAMtSv/Et1cAQUFHUYevqxaLAm5ld2d1aWRzdHJp'
                      'bmcAJQAAADQxMDA1N2I3LWM0ZmYtNGEyZC04YjE2LWFiYWY4NzUxNDc'
                      '0MQAA')

        data = base64.b64decode(from_java)

        # Test decode_all.
        docs = bson.decode_all(
            data, dict, True, JAVA_LEGACY, True,
            CodecOptions(SON, False, STANDARD))
        for d in docs:
            self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring']))

        encoded = [bson.BSON.encode(
            doc, uuid_subtype=JAVA_LEGACY,
            codec_options=CodecOptions(uuid_representation=STANDARD))
                   for doc in docs]

        # Test decode.
        docs2 = [e.decode(
            uuid_subtype=JAVA_LEGACY, as_class=dict, tz_aware=True,
            codec_options=CodecOptions(SON, False, STANDARD))
                 for e in encoded]
        for d in docs2:
            self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring']))

        # Test encode.
        for i in range(len(docs)):
            self.assertEqual(docs2[i]['newguid'], docs[i]['newguid'])
            self.assertEqual(uuid.UUID(docs2[i]['newguidstring']),
                             uuid.UUID(docs[i]['newguidstring']))

        # Test decode_iter
        docs = bson.decode_iter(
            data, dict, True, JAVA_LEGACY,
            True, CodecOptions(SON, False, STANDARD))
        for d in docs:
            self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring']))

        # Test decode_file_iter
        docs = bson.decode_file_iter(
            StringIO(data), dict, True, JAVA_LEGACY,
            True, CodecOptions(SON, False, STANDARD))
        for d in docs:
            self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring']))
 def test_decode_iter(self):
     expected, bson_data = self._generate_multidocument_bson_stream()
     for expected_doc, decoded_doc in zip(
             expected, decode_iter(bson_data, self.codecopts)):
         self.assertEqual(expected_doc, decoded_doc)
 def test_decode_iter(self):
     expected, bson_data = self._generate_multidocument_bson_stream()
     for expected_doc, decoded_doc in zip(
             expected, decode_iter(bson_data, self.codecopts)):
         self.assertEqual(expected_doc, decoded_doc)
Exemple #17
0
#urlloc = "moschetti.org:7779"

headers = {"Accept": "application/" + fmt}

conn = http.client.HTTPConnection(urlloc)

params = None

conn.request("GET", "/func1", params, headers)
response = conn.getresponse()
#print response.status, response.reason

data = response.read()

if fmt == "json":
    doc = json.loads(data)
    print(doc)

elif fmt == "bson":
    options = CodecOptions(document_class=collections.OrderedDict)
    for doc in bson.decode_iter(data, codec_options=options):
        print(doc)

        # Important to get things back to native decimal...
        z2 = doc['bson128_amt'].to_decimal()
        print("mult:", z2 * doc['num'])
        print("escaped:", doc['CR'])
        print("quotes:", doc['quotes'])

conn.close()