def pytest_generate_tests(metafunc):
if 'dumps' in metafunc.fixturenames and 'loads' in metafunc.fixturenames:
metafunc.parametrize('dumps,loads',
(((rj.dumps, rj.loads),
(lambda o, **opts: rj.Encoder(**opts)
(o), lambda j, **opts: rj.Decoder(**opts)
(j)))),
ids=('func[string]', 'class[string]'))
elif 'dumps' in metafunc.fixturenames:
metafunc.parametrize('dumps', (
rj.dumps,
binary_streaming_dumps,
text_streaming_dumps,
lambda o, **opts: rj.Encoder(**opts)(o),
binary_streaming_encoder,
text_streaming_encoder,
),
ids=('func[string]', 'func[bytestream]',
'func[textstream]', 'class[string]',
'class[binarystream]', 'class[textstream]'))
elif 'loads' in metafunc.fixturenames:
metafunc.parametrize('loads', (
rj.loads,
lambda j, **opts: rj.load(
io.BytesIO(j.encode('utf-8')
if isinstance(j, str) else j), **opts),
lambda j, **opts: rj.load(io.StringIO(j), **opts),
lambda j, **opts: rj.Decoder(**opts)(j),
lambda j, **opts: rj.Decoder(**opts)
(io.BytesIO(j.encode('utf-8') if isinstance(j, str) else j)),
lambda j, **opts: rj.Decoder(**opts)(io.StringIO(j)),
),
ids=('func[string]', 'func[bytestream]',
'func[textstream]', 'class[string]',
'class[bytestream]', 'class[textstream]'))
def test_skip_invalid_keys():
o = {True: False, -1: 1, 1.1: 1.1, (1, 2): "foo", b"asdf": 1, None: None}
with pytest.raises(TypeError):
rj.Encoder()(o)
with pytest.raises(TypeError):
rj.Encoder(skip_invalid_keys=False)(o)
assert rj.Encoder(skip_invalid_keys=True)(o) == '{}'
def test_skip_invalid_keys():
o = {True: False, -1: 1, 1.1: 1.1, (1, 2): "foo", b"asdf": 1, None: None}
with pytest.raises(TypeError):
rj.Encoder()(o)
with pytest.raises(TypeError):
rj.Encoder(skip_invalid_keys=False)(o)
assert rj.Encoder(skip_invalid_keys=True)(o) == '{}'
assert rj.Encoder(mapping_mode=rj.MM_SKIP_NON_STRING_KEYS)(o) == '{}'
def pytest_generate_tests(metafunc):
if 'dumps' in metafunc.fixturenames and 'loads' in metafunc.fixturenames:
metafunc.parametrize('dumps,loads', (
((rj.dumps, rj.loads),
(lambda o,**opts: rj.Encoder(**opts)(o), lambda j,**opts: rj.Decoder(**opts)(j)))))
elif 'dumps' in metafunc.fixturenames:
metafunc.parametrize('dumps', (
rj.dumps, lambda o,**opts: rj.Encoder(**opts)(o)))
elif 'loads' in metafunc.fixturenames:
metafunc.parametrize('loads', (
rj.loads, lambda j,**opts: rj.Decoder(**opts)(j)))
def write(self, kind: FileType) -> None:
fn = self._path(kind)
if kind == FileType.PICKLE:
# serialize as TreeNode
with open(fn, "wb") as f:
pickle.dump(self.treenode, f, protocol=-1)
elif kind == FileType.CSV:
# serialize as id_dict
with open(fn, "w") as f:
w = csv.DictWriter(f, Node._fields)
w.writeheader()
for item in self.treenode.node_iter():
w.writerow(item._asdict())
elif kind == FileType.MSGPACK:
# https://msgpack-python.readthedocs.io/en/latest/api.html
with open(fn, "wb") as f:
# Doesn't improve speed
# msgpack.pack(self._to_dict(), f, use_bin_type=True)
msgpack.pack(self.to_dict_list(), f)
elif kind == FileType.JSON:
self._json_dump(fn, json.dump)
elif kind == FileType.UJSON:
self._json_dump(fn, ujson.dump)
elif kind == FileType.SIMPLEJSON:
# NOTE: simplejson includes key names when serializing NamedTuples
with open(fn, "w") as f:
if self.json_dict_list:
simplejson.dump(list(self.id_dict.values()), f, ensure_ascii=True)
else:
simplejson.dump(self.id_dict, f, ensure_ascii=True)
elif kind == FileType.CBOR2:
with open(fn, "wb") as f:
cbor2.dump(self.to_dict_list(), f)
elif kind == FileType.CBOR:
with open(fn, "wb") as f:
cbor.dump(self.to_dict_list(), f)
elif kind == FileType.RAPIDJSON:
# https://python-rapidjson.readthedocs.io/en/latest/benchmarks.html
# TODO: See this example for possible speed improvement - deeper integration with Node
# https://python-rapidjson.readthedocs.io/en/latest/encoder.html
# NOTE: can't use id_dict - keys must be strings
# can't use self.id_dict.values() - not serializable
# list(self.id_dict.values()) produces a list of lists - no keys - very fragile
with open(fn, "w") as f:
if self.json_dict_list:
rapidjson.Encoder(number_mode=rapidjson.NM_NATIVE, ensure_ascii=False)(self.to_dict_list(), f)
else:
rapidjson.Encoder(number_mode=rapidjson.NM_NATIVE, ensure_ascii=False)(list(self.id_dict.values()), f)
elif kind == FileType.BSON:
with open(fn, "wb") as f:
co = CodecOptions(document_class=RawBSONDocument)
for node in self.treenode.node_iter():
f.write(BSON.encode(node._asdict(), codec_options=co))
def test_encoder_call():
o = {'foo': 'bar'}
result = rj.Encoder()(o)
assert result == '{"foo":"bar"}'
assert rj.Encoder()(o, None) == result
stream = io.StringIO()
assert rj.Encoder()(o, stream) is None
assert stream.getvalue() == result
stream = io.StringIO()
assert rj.Encoder()(o, stream=stream) is None
assert stream.getvalue() == result
class RapidJSONEncoder(json.JSONEncoder):
encode = rapidjson.Encoder(skip_invalid_keys=False,
ensure_ascii=False,
indent=None,
sort_keys=False,
number_mode=rapidjson.NM_NATIVE,
datetime_mode=rapidjson.DM_ISO8601,
uuid_mode=rapidjson.UM_CANONICAL)
def test_explicit_defaults_encoder():
assert rj.Encoder(skip_invalid_keys=False,
ensure_ascii=True,
indent=None,
sort_keys=False,
number_mode=None,
datetime_mode=None,
uuid_mode=None)({
'foo': 'bar'
}) == '{"foo":"bar"}'
def test_negative_infinity_c():
inf = float("-infinity")
dumped = rj.Encoder()(inf)
loaded = rj.Decoder()(dumped)
assert loaded == inf
with pytest.raises(ValueError):
rj.Encoder(number_mode=None)(inf)
d = Decimal(inf)
assert d.is_infinite()
with pytest.raises(ValueError):
rj.Encoder(number_mode=rj.NM_DECIMAL)(d)
dumped = rj.Encoder(number_mode=rj.NM_DECIMAL|rj.NM_NAN)(d)
loaded = rj.Decoder(number_mode=rj.NM_DECIMAL|rj.NM_NAN)(dumped)
assert loaded == inf
assert loaded.is_infinite()
def test_nan_c():
nan = float("nan")
dumped = rj.Encoder()(nan)
loaded = rj.Decoder()(dumped)
assert math.isnan(nan)
assert math.isnan(loaded)
with pytest.raises(ValueError):
rj.Encoder(number_mode=None)(nan)
d = Decimal(nan)
assert d.is_nan()
with pytest.raises(ValueError):
rj.Encoder(number_mode=rj.NM_DECIMAL)(d)
dumped = rj.Encoder(number_mode=rj.NM_DECIMAL|rj.NM_NAN)(d)
loaded = rj.Decoder(number_mode=rj.NM_DECIMAL|rj.NM_NAN)(dumped)
assert loaded.is_nan()
def test_serialize_sets_encoder():
class SetsEncoder(rj.Encoder):
def default(self, obj):
if isinstance(obj, set):
return list(obj)
raise TypeError(repr(obj) + " is not JSON serializable")
dumps = SetsEncoder()
dumps([set()])
with pytest.raises(TypeError):
rj.Encoder()([set()])
def test_encoder_attrs():
e = rj.Encoder(skip_invalid_keys=True,
ensure_ascii=True,
indent='\n',
sort_keys=True,
number_mode=rj.NM_NAN,
datetime_mode=rj.DM_ISO8601,
uuid_mode=rj.UM_CANONICAL,
bytes_mode=rj.BM_UTF8)
assert e.skip_invalid_keys
assert e.ensure_ascii
assert e.indent_char == '\n'
assert e.indent_count == 1
assert e.write_mode == rj.WM_PRETTY
assert e.sort_keys
assert e.number_mode == rj.NM_NAN
assert e.datetime_mode == rj.DM_ISO8601
assert e.uuid_mode == rj.UM_CANONICAL
assert e.bytes_mode == rj.BM_UTF8
e = rj.Encoder(mapping_mode=rj.MM_SKIP_NON_STRING_KEYS | rj.MM_SORT_KEYS)
assert e.skip_invalid_keys
assert e.sort_keys
def __init__(self, *args, **kwargs):
rjkwargs = dict(
skip_invalid_keys=kwargs.get("skipkeys", False),
ensure_ascii=kwargs.get("ensure_ascii", True),
write_mode=json.WM_COMPACT,
indent=kwargs.get("indent", None),
sort_keys=kwargs.get("sort_keys", False),
number_mode=json.NM_DECIMAL,
datetime_mode=json.DM_ISO8601,
uuid_mode=json.UM_CANONICAL,
# bytes_mode=None,
# iterable_mode=IM_ANY_ITERABLE,
# mapping_mode=MM_ANY_MAPPING,
)
encoder = json.Encoder(**rjkwargs)
self.encode = encoder.__call__
def test_coerce_keys():
o = {True: False, -1: 1, 1.1: 1.1, (1, 2): "foo", b"asdf": 1, None: None}
expected = '{"True":false,"-1":1,"1.1":1.1,"(1, 2)":"foo","b\'asdf\'":1,"None":null}'
assert rj.dumps(o, mapping_mode=rj.MM_COERCE_KEYS_TO_STRINGS) == expected
assert rj.Encoder(mapping_mode=rj.MM_COERCE_KEYS_TO_STRINGS)(o) == expected
if 'data' in benchmark['stats']:
benchmark['stats'].pop('data')
def pytest_addoption(parser):
parser.addoption('--compare-other-engines',
action='store_true',
help='compare against other JSON engines')
contenders = []
import rapidjson as rj
contenders.append(Contender('rapidjson_f', rj.dumps, rj.loads))
contenders.append(Contender('rapidjson_c', rj.Encoder(), rj.Decoder()))
contenders.append(
Contender('rapidjson_nn_f', partial(rj.dumps, number_mode=rj.NM_NATIVE),
partial(rj.loads, number_mode=rj.NM_NATIVE)))
contenders.append(
Contender('rapidjson_nn_c', rj.Encoder(number_mode=rj.NM_NATIVE),
rj.Decoder(number_mode=rj.NM_NATIVE)))
numbers_contenders = [
Contender('Wide numbers', rj.dumps, rj.loads),
Contender('Native numbers', partial(rj.dumps, number_mode=rj.NM_NATIVE),
partial(rj.loads, number_mode=rj.NM_NATIVE)),
]
string_contenders = [
Contender('rapidjson utf8', partial(rj.dumps, ensure_ascii=False),
def pytest_addoption(parser):
parser.addoption('--compare-other-engines', action='store_true',
help='compare against other JSON engines')
contenders = []
import rapidjson as rj
contenders.append(Contender('rapidjson_f',
rj.dumps,
rj.loads))
contenders.append(Contender('rapidjson_c',
rj.Encoder(),
rj.Decoder()))
contenders.append(Contender('rapidjson_nn_f',
partial(rj.dumps, number_mode=rj.NM_NATIVE),
partial(rj.loads, number_mode=rj.NM_NATIVE)))
contenders.append(Contender('rapidjson_nn_c',
rj.Encoder(number_mode=rj.NM_NATIVE),
rj.Decoder(number_mode=rj.NM_NATIVE)))
numbers_contenders = [
Contender('Wide numbers', rj.dumps, rj.loads),
Contender('Native numbers',
partial(rj.dumps, number_mode=rj.NM_NATIVE),
partial(rj.loads, number_mode=rj.NM_NATIVE)),
]
import rapidjson # Dump datetime in ISO format # treat "native" datetime objects as UTC DATETIME_MODE = rapidjson.DM_ISO8601 | rapidjson.DM_NAIVE_IS_UTC dumps = rapidjson.Encoder(datetime_mode=DATETIME_MODE) loads = rapidjson.Decoder(datetime_mode=DATETIME_MODE)
import rapidjson
# Dump datetime in ISO format
# treat "native" datetime objects as UTC
DATETIME_MODE = rapidjson.DM_ISO8601 | rapidjson.DM_NAIVE_IS_UTC
dumps = rapidjson.Encoder(datetime_mode=DATETIME_MODE)
dumps_notascii = rapidjson.Encoder(datetime_mode=DATETIME_MODE,
ensure_ascii=False)
loads = rapidjson.Decoder(datetime_mode=DATETIME_MODE)
def text_streaming_encoder(o, **opts):
stream = io.StringIO()
rj.Encoder(**opts)(o, stream=stream)
return stream.getvalue()
def binary_streaming_encoder(o, **opts):
stream = io.BytesIO()
rj.Encoder(**opts)(o, stream=stream)
return stream.getvalue().decode('utf-8')
