def test_non_ascii_data_non_utf_locale_coding_orjson(self):
util.set_json_lib("orjson")
original_data = "čććžšđ"
# Default (UTF-8) locale
result = util.json_encode(original_data)
self.assertEqual(result, '"%s"' % (original_data))
loaded = util.json_decode(result)
self.assertEqual(loaded, original_data)
# Non UTF-8 Locale
os.environ["LC_ALL"] = "invalid"
result = util.json_encode(original_data)
self.assertEqual(result, '"%s"' % (original_data))
loaded = util.json_decode(result)
self.assertEqual(loaded, original_data)
# Invalid UTF-8, should fall back to standard json implementation
original_data = "\ud800"
result = util.json_encode(original_data)
self.assertEqual(result, '"\\ud800"')
loaded = util.json_decode('"\ud800"')
self.assertEqual(loaded, original_data)
def test_json_encode_with_custom_options(benchmark, json_lib, keys_count,
sort_keys):
# NOTE: orjson doesn't support sort_keys=True
if json_lib == "orjson":
if not six.PY3:
pytest.skip(
"Skipping under Python 2, orjson is only available for Python 3"
)
elif sort_keys is True:
pytest.skip("orjson doesn't support sort_keys=True")
set_json_lib(json_lib)
scalyr_agent.util.SORT_KEYS = sort_keys
data = generate_random_dict(keys_count=keys_count)
def run_benchmark():
return json_encode(data)
result = benchmark.pedantic(run_benchmark, iterations=50, rounds=100)
assert get_json_lib() == json_lib
assert scalyr_agent.util.SORT_KEYS == sort_keys
assert isinstance(result, six.text_type)
assert json_decode(result) == data
def _test_json_decode(benchmark, json_lib, log_tuple):
"""
:param json_lib: JSON library to use.
:param log_tuple: Tuple with (log_filename, log_bytes_to_use).
"""
set_json_lib(json_lib)
file_name, bytes_to_read = log_tuple
if log_tuple not in CACHED_TEST_DATA["decode"]:
data = read_bytes_from_log_fixture_file(file_name,
bytes_to_read).strip()
obj = {"lines": []}
for line in data.split(b"\n"):
line_decoded = json.loads(six.ensure_text(line))
obj["lines"].append(line_decoded)
data = json.dumps(obj)
CACHED_TEST_DATA["decode"][log_tuple] = six.ensure_text(data)
data = CACHED_TEST_DATA["decode"][log_tuple]
def run_benchmark():
return json_decode(data)
result = benchmark.pedantic(run_benchmark, iterations=20, rounds=50)
assert get_json_lib() == json_lib
assert isinstance(result, dict)
def setUp(self):
# We need to reset the log destinations here because it is only at this point is stdout replaced with
# whatever object is capturing stdout for this test case.
scalyr_logging.set_log_destination(use_stdout=True)
self.__setup_invoked = True
# Enable keys sort for json.dumps to make it easier to assert on the serialized output
scalyr_util.SORT_KEYS = True
# NOTE: orjson doesn't support sort_keys so we fallback to implementation which supports it
scalyr_util.set_json_lib("json")
def __runtest(library):
original_lib = util.get_json_lib()
self._setlib(library)
try:
text2 = util.json_encode(obj)
self.assertEquals(
sorted(six.ensure_text(text)),
sorted(text2),
"%s != %s" % (str(text), str(text2)),
)
obj2 = util.json_decode(text2)
text3 = util.json_encode(obj2)
self.assertEquals(
sorted(six.ensure_text(text)),
sorted(text3),
"%s != %s" % (str(text), str(text3)),
)
obj3 = util.json_decode(text)
self.assertEquals(obj3, obj)
# Sanity test to ensure curly brace is always the last character when serializing
# a dict.
# Our "rewind to last curly brace" logic in scalyr_agent/scalyr_client.py relies on
# this behavior.
values = [
{},
{
"a": "b"
},
{
"a": 1,
"b": 2
},
]
for value in values:
result = util.json_encode(value)
self.assertEqual(result[-1], "}")
finally:
util.set_json_lib(original_lib)
def test_non_ascii_data_non_utf_locale_coding_default_json_lib(self):
util.set_json_lib("json")
original_data = "čććžšđ"
# Default (UTF-8) locale
result = util.json_encode(original_data)
self.assertEqual(result,
'"\\u010d\\u0107\\u0107\\u017e\\u0161\\u0111"')
loaded = util.json_decode(result)
self.assertEqual(loaded, original_data)
# Non UTF-8 Locale
os.environ["LC_ALL"] = "invalid"
result = util.json_encode(original_data)
self.assertEqual(result,
'"\\u010d\\u0107\\u0107\\u017e\\u0161\\u0111"')
loaded = util.json_decode(result)
self.assertEqual(loaded, original_data)
def _setlib(self, library):
if library == JSON:
util.set_json_lib("json")
elif library == UJSON:
util.set_json_lib("ujson")
elif library == ORJSON:
util.set_json_lib("orjson")
else:
raise ValueError("Invalid library name: %s" % (library))
def _test_json_encode(benchmark, json_lib, log_tuple):
"""
:param json_lib: JSON library to use.
:param log_tuple: Tuple with (log_filename, log_bytes_to_use).
"""
set_json_lib(json_lib)
file_name, bytes_to_read = log_tuple
if log_tuple not in CACHED_TEST_DATA["encode"]:
data = read_bytes_from_log_fixture_file(file_name, bytes_to_read)
data = six.ensure_text(data)
CACHED_TEST_DATA["encode"][log_tuple] = data
data = CACHED_TEST_DATA["encode"][log_tuple]
def run_benchmark():
return json_encode(data)
result = benchmark.pedantic(run_benchmark, iterations=20, rounds=50)
assert get_json_lib() == json_lib
assert isinstance(result, six.text_type)
def test_invalid_lib(self):
self.assertRaises(
ValueError, lambda: util.set_json_lib("BAD JSON LIBRARY NAME")
)
