Example #1
0
    def test_complex(self):
        field = {
            "k1.k2": [{
                "l1.l2": "123"
            }, {
                "l3.l4": "456"
            }],
            "k3": [{
                "l5.l6": "789"
            }],
            "k4.k5": [1, 2, 3],
            "k6": ["a", "b"],
        }

        expected = {
            "k1\uff0ek2": [{
                "l1\uff0el2": "123"
            }, {
                "l3\uff0el4": "456"
            }],
            "k3": [{
                "l5\uff0el6": "789"
            }],
            "k4\uff0ek5": [1, 2, 3],
            "k6": ["a", "b"],
        }

        escaped = mongoescape.escape_chars(field)
        self.assertDictEqual(expected, escaped)

        unescaped = mongoescape.unescape_chars(escaped)
        self.assertDictEqual(field, unescaped)
Example #2
0
    def _from_model(cls, model, mask_secrets=False):
        doc = model.to_mongo()

        if "_id" in doc:
            doc["id"] = str(doc.pop("_id"))

        # Special case for models which utilize JSONDictField - there is no need to escape those
        # fields since it contains a JSON string and not a dictionary which doesn't need to be
        # mongo escaped. Skipping this step here substantially speeds things up for that field.

        # Right now we do this here manually for all those fields types but eventually we should
        # refactor the code to just call unescape chars on escaped fields - more generic and
        # faster.
        raw_values = {}

        for field_name in cls.skip_unescape_field_names:
            if isinstance(doc.get(field_name, None), bytes):
                raw_values[field_name] = doc.pop(field_name)

        # TODO (Tomaz): In general we really shouldn't need to call unescape chars on the whole doc,
        # but just on the EscapedDict and EscapedDynamicField fields - doing it on the whole doc
        # level is slow and not necessary!
        doc = util_mongodb.unescape_chars(doc)

        # Now add the JSON string field value which shouldn't be escaped back.
        # We don't JSON parse the field value here because that happens inside the model specific
        # "from_model()" method where we also parse and convert all the other field values.
        for field_name, field_value in raw_values.items():
            doc[field_name] = field_value

        if mask_secrets and cfg.CONF.log.mask_secrets:
            doc = model.mask_secrets(value=doc)

        return doc
Example #3
0
    def test_complex(self):
        field = {
            'k1.k2': [{
                'l1.l2': '123'
            }, {
                'l3.l4': '456'
            }],
            'k3': [{
                'l5.l6': '789'
            }],
            'k4.k5': [1, 2, 3],
            'k6': ['a', 'b']
        }

        expected = {
            u'k1\uff0ek2': [{
                u'l1\uff0el2': '123'
            }, {
                u'l3\uff0el4': '456'
            }],
            'k3': [{
                u'l5\uff0el6': '789'
            }],
            u'k4\uff0ek5': [1, 2, 3],
            'k6': ['a', 'b']
        }

        escaped = mongoescape.escape_chars(field)
        self.assertDictEqual(expected, escaped)

        unescaped = mongoescape.unescape_chars(escaped)
        self.assertDictEqual(field, unescaped)
Example #4
0
 def test_unnested(self):
     field = {'k1.k1.k1': 'v1', 'k2$': 'v2', '$k3.': 'v3'}
     escaped = mongoescape.escape_chars(field)
     self.assertEqual(escaped, {u'k1\uff0ek1\uff0ek1': 'v1',
                                u'k2\uff04': 'v2',
                                u'\uff04k3\uff0e': 'v3'}, 'Escaping failed.')
     unescaped = mongoescape.unescape_chars(escaped)
     self.assertEqual(unescaped, field, 'Unescaping failed.')
Example #5
0
 def test_unnested(self):
     field = {'k1.k1.k1': 'v1', 'k2$': 'v2', '$k3.': 'v3'}
     escaped = mongoescape.escape_chars(field)
     self.assertEqual(escaped, {u'k1\uff0ek1\uff0ek1': 'v1',
                                u'k2\uff04': 'v2',
                                u'\uff04k3\uff0e': 'v3'}, 'Escaping failed.')
     unescaped = mongoescape.unescape_chars(escaped)
     self.assertEqual(unescaped, field, 'Unescaping failed.')
Example #6
0
    def test_unescaping_of_rule_criteria(self):
        # Verify that dot escaped in rule criteria is correctly escaped.
        # Note: In the past we used different character to escape dot in the
        # rule criteria.
        escaped = {"k1\u2024k1\u2024k1": "v1", "k2$": "v2", "$k3\u2024": "v3"}
        unescaped = {"k1.k1.k1": "v1", "k2$": "v2", "$k3.": "v3"}

        result = mongoescape.unescape_chars(escaped)
        self.assertEqual(result, unescaped)
Example #7
0
    def _from_model(cls, model, mask_secrets=False):
        doc = util_mongodb.unescape_chars(model.to_mongo())

        if '_id' in doc:
            doc['id'] = str(doc.pop('_id'))

        if mask_secrets and cfg.CONF.log.mask_secrets:
            doc = model.mask_secrets(value=doc)

        return doc
Example #8
0
    def test_original_value(self):
        field = {'k1.k2.k3': 'v1'}

        escaped = mongoescape.escape_chars(field)
        self.assertIn('k1.k2.k3', field.keys())
        self.assertIn(u'k1\uff0ek2\uff0ek3', escaped.keys())

        unescaped = mongoescape.unescape_chars(escaped)
        self.assertIn('k1.k2.k3', unescaped.keys())
        self.assertIn(u'k1\uff0ek2\uff0ek3', escaped.keys())
Example #9
0
    def test_original_value(self):
        field = {'k1.k2.k3': 'v1'}

        escaped = mongoescape.escape_chars(field)
        self.assertIn('k1.k2.k3', list(field.keys()))
        self.assertIn(u'k1\uff0ek2\uff0ek3', list(escaped.keys()))

        unescaped = mongoescape.unescape_chars(escaped)
        self.assertIn('k1.k2.k3', list(unescaped.keys()))
        self.assertIn(u'k1\uff0ek2\uff0ek3', list(escaped.keys()))
Example #10
0
    def _from_model(cls, model, mask_secrets=False):
        doc = util_mongodb.unescape_chars(model.to_mongo())

        if '_id' in doc:
            doc['id'] = str(doc.pop('_id'))

        if mask_secrets and cfg.CONF.log.mask_secrets:
            doc = model.mask_secrets(value=doc)

        return doc
Example #11
0
    def test_original_value(self):
        field = {"k1.k2.k3": "v1"}

        escaped = mongoescape.escape_chars(field)
        self.assertIn("k1.k2.k3", list(field.keys()))
        self.assertIn("k1\uff0ek2\uff0ek3", list(escaped.keys()))

        unescaped = mongoescape.unescape_chars(escaped)
        self.assertIn("k1.k2.k3", list(unescaped.keys()))
        self.assertIn("k1\uff0ek2\uff0ek3", list(escaped.keys()))
Example #12
0
 def test_nested(self):
     nested_field = {'nk1.nk1.nk1': 'v1', 'nk2$': 'v2', '$nk3.': 'v3'}
     field = {'k1.k1.k1': nested_field, 'k2$': 'v2', '$k3.': 'v3'}
     escaped = mongoescape.escape_chars(field)
     self.assertEqual(escaped, {u'k1\uff0ek1\uff0ek1': {u'\uff04nk3\uff0e': 'v3',
                                                        u'nk1\uff0enk1\uff0enk1': 'v1',
                                                        u'nk2\uff04': 'v2'},
                                u'k2\uff04': 'v2',
                                u'\uff04k3\uff0e': 'v3'}, 'un-escaping failed.')
     unescaped = mongoescape.unescape_chars(escaped)
     self.assertEqual(unescaped, field, 'Unescaping failed.')
Example #13
0
 def test_nested(self):
     nested_field = {'nk1.nk1.nk1': 'v1', 'nk2$': 'v2', '$nk3.': 'v3'}
     field = {'k1.k1.k1': nested_field, 'k2$': 'v2', '$k3.': 'v3'}
     escaped = mongoescape.escape_chars(field)
     self.assertEqual(escaped, {u'k1\uff0ek1\uff0ek1': {u'\uff04nk3\uff0e': 'v3',
                                                        u'nk1\uff0enk1\uff0enk1': 'v1',
                                                        u'nk2\uff04': 'v2'},
                                u'k2\uff04': 'v2',
                                u'\uff04k3\uff0e': 'v3'}, 'un-escaping failed.')
     unescaped = mongoescape.unescape_chars(escaped)
     self.assertEqual(unescaped, field, 'Unescaping failed.')
Example #14
0
    def _from_model(cls, model, mask_secrets=False):
        doc = model.to_mongo()

        if "_id" in doc:
            doc["id"] = str(doc.pop("_id"))

        doc = util_mongodb.unescape_chars(doc)

        if mask_secrets and cfg.CONF.log.mask_secrets:
            doc = model.mask_secrets(value=doc)

        return doc
Example #15
0
    def to_python(self, value):
        if isinstance(value, dict):
            # Old format which used a native dict with escaped special characters
            # TODO: We can remove that once we assume there is no more old style data in the
            # database and save quite some time.
            value = mongoescape.unescape_chars(value)
            return value

        if isinstance(value, bytes):
            return self.parse_field_value(value)

        return value
Example #16
0
    def test_unescaping_of_rule_criteria(self):
        # Verify that dot escaped in rule criteria is correctly escaped.
        # Note: In the past we used different character to escape dot in the
        # rule criteria.
        escaped = {
            u'k1\u2024k1\u2024k1': 'v1',
            u'k2$': 'v2',
            u'$k3\u2024': 'v3'
        }
        unescaped = {'k1.k1.k1': 'v1', 'k2$': 'v2', '$k3.': 'v3'}

        result = mongoescape.unescape_chars(escaped)
        self.assertEqual(result, unescaped)
Example #17
0
 def test_unnested(self):
     field = {"k1.k1.k1": "v1", "k2$": "v2", "$k3.": "v3"}
     escaped = mongoescape.escape_chars(field)
     self.assertEqual(
         escaped,
         {
             "k1\uff0ek1\uff0ek1": "v1",
             "k2\uff04": "v2",
             "\uff04k3\uff0e": "v3"
         },
         "Escaping failed.",
     )
     unescaped = mongoescape.unescape_chars(escaped)
     self.assertEqual(unescaped, field, "Unescaping failed.")
Example #18
0
def test_escape_chars(benchmark, fixture_file: str) -> None:
    with open(os.path.join(FIXTURES_DIR, fixture_file), "r") as fp:
        content = fp.read()

    data = json.loads(content)

    def run_benchmark():
        result = mongoescape.escape_chars(data)
        return result

    escaped_data = benchmark(run_benchmark)
    unescaped_data = mongoescape.unescape_chars(escaped_data)
    assert escaped_data != data
    assert unescaped_data == data
Example #19
0
    def test_unescaping_of_rule_criteria(self):
        # Verify that dot escaped in rule criteria is correctly escaped.
        # Note: In the past we used different character to escape dot in the
        # rule criteria.
        escaped = {
            u'k1\u2024k1\u2024k1': 'v1',
            u'k2$': 'v2',
            u'$k3\u2024': 'v3'
        }
        unescaped = {
            'k1.k1.k1': 'v1',
            'k2$': 'v2',
            '$k3.': 'v3'
        }

        result = mongoescape.unescape_chars(escaped)
        self.assertEqual(result, unescaped)
Example #20
0
 def test_nested(self):
     nested_field = {"nk1.nk1.nk1": "v1", "nk2$": "v2", "$nk3.": "v3"}
     field = {"k1.k1.k1": nested_field, "k2$": "v2", "$k3.": "v3"}
     escaped = mongoescape.escape_chars(field)
     self.assertEqual(
         escaped,
         {
             "k1\uff0ek1\uff0ek1": {
                 "\uff04nk3\uff0e": "v3",
                 "nk1\uff0enk1\uff0enk1": "v1",
                 "nk2\uff04": "v2",
             },
             "k2\uff04": "v2",
             "\uff04k3\uff0e": "v3",
         },
         "un-escaping failed.",
     )
     unescaped = mongoescape.unescape_chars(escaped)
     self.assertEqual(unescaped, field, "Unescaping failed.")
Example #21
0
    def test_complex(self):
        field = {
            'k1.k2': [{'l1.l2': '123'}, {'l3.l4': '456'}],
            'k3': [{'l5.l6': '789'}],
            'k4.k5': [1, 2, 3],
            'k6': ['a', 'b']
        }

        expected = {
            u'k1\uff0ek2': [{u'l1\uff0el2': '123'}, {u'l3\uff0el4': '456'}],
            'k3': [{u'l5\uff0el6': '789'}],
            u'k4\uff0ek5': [1, 2, 3],
            'k6': ['a', 'b']
        }

        escaped = mongoescape.escape_chars(field)
        self.assertDictEqual(expected, escaped)

        unescaped = mongoescape.unescape_chars(escaped)
        self.assertDictEqual(field, unescaped)
Example #22
0
 def _from_model(cls, model):
     doc = util_mongodb.unescape_chars(model.to_mongo())
     if '_id' in doc:
         doc['id'] = str(doc.pop('_id'))
     return doc
Example #23
0
 def to_python(self, value):
     value = super(EscapedDynamicField, self).to_python(value)
     return mongoescape.unescape_chars(value)
Example #24
0
 def run_benchmark():
     result = mongoescape.unescape_chars(escaped_data)
     return result
Example #25
0
 def _from_model(cls, model):
     doc = util_mongodb.unescape_chars(model.to_mongo())
     if '_id' in doc:
         doc['id'] = str(doc.pop('_id'))
     return doc
Example #26
0
 def to_python(self, value):
     value = super(EscapedDictField, self).to_python(value)
     return mongoescape.unescape_chars(value)