def test_complex(self):
field = {
"k1.k2": [{
"l1.l2": "123"
}, {
"l3.l4": "456"
}],
"k3": [{
"l5.l6": "789"
}],
"k4.k5": [1, 2, 3],
"k6": ["a", "b"],
}
expected = {
"k1\uff0ek2": [{
"l1\uff0el2": "123"
}, {
"l3\uff0el4": "456"
}],
"k3": [{
"l5\uff0el6": "789"
}],
"k4\uff0ek5": [1, 2, 3],
"k6": ["a", "b"],
}
escaped = mongoescape.escape_chars(field)
self.assertDictEqual(expected, escaped)
unescaped = mongoescape.unescape_chars(escaped)
self.assertDictEqual(field, unescaped)
def _from_model(cls, model, mask_secrets=False):
doc = model.to_mongo()
if "_id" in doc:
doc["id"] = str(doc.pop("_id"))
# Special case for models which utilize JSONDictField - there is no need to escape those
# fields since it contains a JSON string and not a dictionary which doesn't need to be
# mongo escaped. Skipping this step here substantially speeds things up for that field.
# Right now we do this here manually for all those fields types but eventually we should
# refactor the code to just call unescape chars on escaped fields - more generic and
# faster.
raw_values = {}
for field_name in cls.skip_unescape_field_names:
if isinstance(doc.get(field_name, None), bytes):
raw_values[field_name] = doc.pop(field_name)
# TODO (Tomaz): In general we really shouldn't need to call unescape chars on the whole doc,
# but just on the EscapedDict and EscapedDynamicField fields - doing it on the whole doc
# level is slow and not necessary!
doc = util_mongodb.unescape_chars(doc)
# Now add the JSON string field value which shouldn't be escaped back.
# We don't JSON parse the field value here because that happens inside the model specific
# "from_model()" method where we also parse and convert all the other field values.
for field_name, field_value in raw_values.items():
doc[field_name] = field_value
if mask_secrets and cfg.CONF.log.mask_secrets:
doc = model.mask_secrets(value=doc)
return doc
def test_complex(self):
field = {
'k1.k2': [{
'l1.l2': '123'
}, {
'l3.l4': '456'
}],
'k3': [{
'l5.l6': '789'
}],
'k4.k5': [1, 2, 3],
'k6': ['a', 'b']
}
expected = {
u'k1\uff0ek2': [{
u'l1\uff0el2': '123'
}, {
u'l3\uff0el4': '456'
}],
'k3': [{
u'l5\uff0el6': '789'
}],
u'k4\uff0ek5': [1, 2, 3],
'k6': ['a', 'b']
}
escaped = mongoescape.escape_chars(field)
self.assertDictEqual(expected, escaped)
unescaped = mongoescape.unescape_chars(escaped)
self.assertDictEqual(field, unescaped)
def test_unnested(self):
field = {'k1.k1.k1': 'v1', 'k2$': 'v2', '$k3.': 'v3'}
escaped = mongoescape.escape_chars(field)
self.assertEqual(escaped, {u'k1\uff0ek1\uff0ek1': 'v1',
u'k2\uff04': 'v2',
u'\uff04k3\uff0e': 'v3'}, 'Escaping failed.')
unescaped = mongoescape.unescape_chars(escaped)
self.assertEqual(unescaped, field, 'Unescaping failed.')
def test_unnested(self):
field = {'k1.k1.k1': 'v1', 'k2$': 'v2', '$k3.': 'v3'}
escaped = mongoescape.escape_chars(field)
self.assertEqual(escaped, {u'k1\uff0ek1\uff0ek1': 'v1',
u'k2\uff04': 'v2',
u'\uff04k3\uff0e': 'v3'}, 'Escaping failed.')
unescaped = mongoescape.unescape_chars(escaped)
self.assertEqual(unescaped, field, 'Unescaping failed.')
def test_unescaping_of_rule_criteria(self):
# Verify that dot escaped in rule criteria is correctly escaped.
# Note: In the past we used different character to escape dot in the
# rule criteria.
escaped = {"k1\u2024k1\u2024k1": "v1", "k2$": "v2", "$k3\u2024": "v3"}
unescaped = {"k1.k1.k1": "v1", "k2$": "v2", "$k3.": "v3"}
result = mongoescape.unescape_chars(escaped)
self.assertEqual(result, unescaped)
def _from_model(cls, model, mask_secrets=False):
doc = util_mongodb.unescape_chars(model.to_mongo())
if '_id' in doc:
doc['id'] = str(doc.pop('_id'))
if mask_secrets and cfg.CONF.log.mask_secrets:
doc = model.mask_secrets(value=doc)
return doc
def test_original_value(self):
field = {'k1.k2.k3': 'v1'}
escaped = mongoescape.escape_chars(field)
self.assertIn('k1.k2.k3', field.keys())
self.assertIn(u'k1\uff0ek2\uff0ek3', escaped.keys())
unescaped = mongoescape.unescape_chars(escaped)
self.assertIn('k1.k2.k3', unescaped.keys())
self.assertIn(u'k1\uff0ek2\uff0ek3', escaped.keys())
def test_original_value(self):
field = {'k1.k2.k3': 'v1'}
escaped = mongoescape.escape_chars(field)
self.assertIn('k1.k2.k3', list(field.keys()))
self.assertIn(u'k1\uff0ek2\uff0ek3', list(escaped.keys()))
unescaped = mongoescape.unescape_chars(escaped)
self.assertIn('k1.k2.k3', list(unescaped.keys()))
self.assertIn(u'k1\uff0ek2\uff0ek3', list(escaped.keys()))
def _from_model(cls, model, mask_secrets=False):
doc = util_mongodb.unescape_chars(model.to_mongo())
if '_id' in doc:
doc['id'] = str(doc.pop('_id'))
if mask_secrets and cfg.CONF.log.mask_secrets:
doc = model.mask_secrets(value=doc)
return doc
def test_original_value(self):
field = {"k1.k2.k3": "v1"}
escaped = mongoescape.escape_chars(field)
self.assertIn("k1.k2.k3", list(field.keys()))
self.assertIn("k1\uff0ek2\uff0ek3", list(escaped.keys()))
unescaped = mongoescape.unescape_chars(escaped)
self.assertIn("k1.k2.k3", list(unescaped.keys()))
self.assertIn("k1\uff0ek2\uff0ek3", list(escaped.keys()))
def test_nested(self):
nested_field = {'nk1.nk1.nk1': 'v1', 'nk2$': 'v2', '$nk3.': 'v3'}
field = {'k1.k1.k1': nested_field, 'k2$': 'v2', '$k3.': 'v3'}
escaped = mongoescape.escape_chars(field)
self.assertEqual(escaped, {u'k1\uff0ek1\uff0ek1': {u'\uff04nk3\uff0e': 'v3',
u'nk1\uff0enk1\uff0enk1': 'v1',
u'nk2\uff04': 'v2'},
u'k2\uff04': 'v2',
u'\uff04k3\uff0e': 'v3'}, 'un-escaping failed.')
unescaped = mongoescape.unescape_chars(escaped)
self.assertEqual(unescaped, field, 'Unescaping failed.')
def test_nested(self):
nested_field = {'nk1.nk1.nk1': 'v1', 'nk2$': 'v2', '$nk3.': 'v3'}
field = {'k1.k1.k1': nested_field, 'k2$': 'v2', '$k3.': 'v3'}
escaped = mongoescape.escape_chars(field)
self.assertEqual(escaped, {u'k1\uff0ek1\uff0ek1': {u'\uff04nk3\uff0e': 'v3',
u'nk1\uff0enk1\uff0enk1': 'v1',
u'nk2\uff04': 'v2'},
u'k2\uff04': 'v2',
u'\uff04k3\uff0e': 'v3'}, 'un-escaping failed.')
unescaped = mongoescape.unescape_chars(escaped)
self.assertEqual(unescaped, field, 'Unescaping failed.')
def _from_model(cls, model, mask_secrets=False):
doc = model.to_mongo()
if "_id" in doc:
doc["id"] = str(doc.pop("_id"))
doc = util_mongodb.unescape_chars(doc)
if mask_secrets and cfg.CONF.log.mask_secrets:
doc = model.mask_secrets(value=doc)
return doc
def to_python(self, value):
if isinstance(value, dict):
# Old format which used a native dict with escaped special characters
# TODO: We can remove that once we assume there is no more old style data in the
# database and save quite some time.
value = mongoescape.unescape_chars(value)
return value
if isinstance(value, bytes):
return self.parse_field_value(value)
return value
def test_unescaping_of_rule_criteria(self):
# Verify that dot escaped in rule criteria is correctly escaped.
# Note: In the past we used different character to escape dot in the
# rule criteria.
escaped = {
u'k1\u2024k1\u2024k1': 'v1',
u'k2$': 'v2',
u'$k3\u2024': 'v3'
}
unescaped = {'k1.k1.k1': 'v1', 'k2$': 'v2', '$k3.': 'v3'}
result = mongoescape.unescape_chars(escaped)
self.assertEqual(result, unescaped)
def test_unnested(self):
field = {"k1.k1.k1": "v1", "k2$": "v2", "$k3.": "v3"}
escaped = mongoescape.escape_chars(field)
self.assertEqual(
escaped,
{
"k1\uff0ek1\uff0ek1": "v1",
"k2\uff04": "v2",
"\uff04k3\uff0e": "v3"
},
"Escaping failed.",
)
unescaped = mongoescape.unescape_chars(escaped)
self.assertEqual(unescaped, field, "Unescaping failed.")
def test_escape_chars(benchmark, fixture_file: str) -> None:
with open(os.path.join(FIXTURES_DIR, fixture_file), "r") as fp:
content = fp.read()
data = json.loads(content)
def run_benchmark():
result = mongoescape.escape_chars(data)
return result
escaped_data = benchmark(run_benchmark)
unescaped_data = mongoescape.unescape_chars(escaped_data)
assert escaped_data != data
assert unescaped_data == data
def test_unescaping_of_rule_criteria(self):
# Verify that dot escaped in rule criteria is correctly escaped.
# Note: In the past we used different character to escape dot in the
# rule criteria.
escaped = {
u'k1\u2024k1\u2024k1': 'v1',
u'k2$': 'v2',
u'$k3\u2024': 'v3'
}
unescaped = {
'k1.k1.k1': 'v1',
'k2$': 'v2',
'$k3.': 'v3'
}
result = mongoescape.unescape_chars(escaped)
self.assertEqual(result, unescaped)
def test_nested(self):
nested_field = {"nk1.nk1.nk1": "v1", "nk2$": "v2", "$nk3.": "v3"}
field = {"k1.k1.k1": nested_field, "k2$": "v2", "$k3.": "v3"}
escaped = mongoescape.escape_chars(field)
self.assertEqual(
escaped,
{
"k1\uff0ek1\uff0ek1": {
"\uff04nk3\uff0e": "v3",
"nk1\uff0enk1\uff0enk1": "v1",
"nk2\uff04": "v2",
},
"k2\uff04": "v2",
"\uff04k3\uff0e": "v3",
},
"un-escaping failed.",
)
unescaped = mongoescape.unescape_chars(escaped)
self.assertEqual(unescaped, field, "Unescaping failed.")
def test_complex(self):
field = {
'k1.k2': [{'l1.l2': '123'}, {'l3.l4': '456'}],
'k3': [{'l5.l6': '789'}],
'k4.k5': [1, 2, 3],
'k6': ['a', 'b']
}
expected = {
u'k1\uff0ek2': [{u'l1\uff0el2': '123'}, {u'l3\uff0el4': '456'}],
'k3': [{u'l5\uff0el6': '789'}],
u'k4\uff0ek5': [1, 2, 3],
'k6': ['a', 'b']
}
escaped = mongoescape.escape_chars(field)
self.assertDictEqual(expected, escaped)
unescaped = mongoescape.unescape_chars(escaped)
self.assertDictEqual(field, unescaped)
def _from_model(cls, model):
doc = util_mongodb.unescape_chars(model.to_mongo())
if '_id' in doc:
doc['id'] = str(doc.pop('_id'))
return doc
def to_python(self, value):
value = super(EscapedDynamicField, self).to_python(value)
return mongoescape.unescape_chars(value)
def run_benchmark():
result = mongoescape.unescape_chars(escaped_data)
return result
def _from_model(cls, model):
doc = util_mongodb.unescape_chars(model.to_mongo())
if '_id' in doc:
doc['id'] = str(doc.pop('_id'))
return doc
def to_python(self, value):
value = super(EscapedDictField, self).to_python(value)
return mongoescape.unescape_chars(value)
