def test_non_ascii_data_non_utf_locale_coding_orjson(self): util.set_json_lib("orjson") original_data = "čććžšđ" # Default (UTF-8) locale result = util.json_encode(original_data) self.assertEqual(result, '"%s"' % (original_data)) loaded = util.json_decode(result) self.assertEqual(loaded, original_data) # Non UTF-8 Locale os.environ["LC_ALL"] = "invalid" result = util.json_encode(original_data) self.assertEqual(result, '"%s"' % (original_data)) loaded = util.json_decode(result) self.assertEqual(loaded, original_data) # Invalid UTF-8, should fall back to standard json implementation original_data = "\ud800" result = util.json_encode(original_data) self.assertEqual(result, '"\\ud800"') loaded = util.json_decode('"\ud800"') self.assertEqual(loaded, original_data)
def test_json_encode_with_custom_options(benchmark, json_lib, keys_count, sort_keys): # NOTE: orjson doesn't support sort_keys=True if json_lib == "orjson": if not six.PY3: pytest.skip( "Skipping under Python 2, orjson is only available for Python 3" ) elif sort_keys is True: pytest.skip("orjson doesn't support sort_keys=True") set_json_lib(json_lib) scalyr_agent.util.SORT_KEYS = sort_keys data = generate_random_dict(keys_count=keys_count) def run_benchmark(): return json_encode(data) result = benchmark.pedantic(run_benchmark, iterations=50, rounds=100) assert get_json_lib() == json_lib assert scalyr_agent.util.SORT_KEYS == sort_keys assert isinstance(result, six.text_type) assert json_decode(result) == data
def _test_json_decode(benchmark, json_lib, log_tuple): """ :param json_lib: JSON library to use. :param log_tuple: Tuple with (log_filename, log_bytes_to_use). """ set_json_lib(json_lib) file_name, bytes_to_read = log_tuple if log_tuple not in CACHED_TEST_DATA["decode"]: data = read_bytes_from_log_fixture_file(file_name, bytes_to_read).strip() obj = {"lines": []} for line in data.split(b"\n"): line_decoded = json.loads(six.ensure_text(line)) obj["lines"].append(line_decoded) data = json.dumps(obj) CACHED_TEST_DATA["decode"][log_tuple] = six.ensure_text(data) data = CACHED_TEST_DATA["decode"][log_tuple] def run_benchmark(): return json_decode(data) result = benchmark.pedantic(run_benchmark, iterations=20, rounds=50) assert get_json_lib() == json_lib assert isinstance(result, dict)
def setUp(self): # We need to reset the log destinations here because it is only at this point is stdout replaced with # whatever object is capturing stdout for this test case. scalyr_logging.set_log_destination(use_stdout=True) self.__setup_invoked = True # Enable keys sort for json.dumps to make it easier to assert on the serialized output scalyr_util.SORT_KEYS = True # NOTE: orjson doesn't support sort_keys so we fallback to implementation which supports it scalyr_util.set_json_lib("json")
def __runtest(library): original_lib = util.get_json_lib() self._setlib(library) try: text2 = util.json_encode(obj) self.assertEquals( sorted(six.ensure_text(text)), sorted(text2), "%s != %s" % (str(text), str(text2)), ) obj2 = util.json_decode(text2) text3 = util.json_encode(obj2) self.assertEquals( sorted(six.ensure_text(text)), sorted(text3), "%s != %s" % (str(text), str(text3)), ) obj3 = util.json_decode(text) self.assertEquals(obj3, obj) # Sanity test to ensure curly brace is always the last character when serializing # a dict. # Our "rewind to last curly brace" logic in scalyr_agent/scalyr_client.py relies on # this behavior. values = [ {}, { "a": "b" }, { "a": 1, "b": 2 }, ] for value in values: result = util.json_encode(value) self.assertEqual(result[-1], "}") finally: util.set_json_lib(original_lib)
def test_non_ascii_data_non_utf_locale_coding_default_json_lib(self): util.set_json_lib("json") original_data = "čććžšđ" # Default (UTF-8) locale result = util.json_encode(original_data) self.assertEqual(result, '"\\u010d\\u0107\\u0107\\u017e\\u0161\\u0111"') loaded = util.json_decode(result) self.assertEqual(loaded, original_data) # Non UTF-8 Locale os.environ["LC_ALL"] = "invalid" result = util.json_encode(original_data) self.assertEqual(result, '"\\u010d\\u0107\\u0107\\u017e\\u0161\\u0111"') loaded = util.json_decode(result) self.assertEqual(loaded, original_data)
def _setlib(self, library): if library == JSON: util.set_json_lib("json") elif library == UJSON: util.set_json_lib("ujson") elif library == ORJSON: util.set_json_lib("orjson") else: raise ValueError("Invalid library name: %s" % (library))
def _test_json_encode(benchmark, json_lib, log_tuple): """ :param json_lib: JSON library to use. :param log_tuple: Tuple with (log_filename, log_bytes_to_use). """ set_json_lib(json_lib) file_name, bytes_to_read = log_tuple if log_tuple not in CACHED_TEST_DATA["encode"]: data = read_bytes_from_log_fixture_file(file_name, bytes_to_read) data = six.ensure_text(data) CACHED_TEST_DATA["encode"][log_tuple] = data data = CACHED_TEST_DATA["encode"][log_tuple] def run_benchmark(): return json_encode(data) result = benchmark.pedantic(run_benchmark, iterations=20, rounds=50) assert get_json_lib() == json_lib assert isinstance(result, six.text_type)
def test_invalid_lib(self): self.assertRaises( ValueError, lambda: util.set_json_lib("BAD JSON LIBRARY NAME") )