def pytest_generate_tests(metafunc): if 'dumps' in metafunc.fixturenames and 'loads' in metafunc.fixturenames: metafunc.parametrize('dumps,loads', (((rj.dumps, rj.loads), (lambda o, **opts: rj.Encoder(**opts) (o), lambda j, **opts: rj.Decoder(**opts) (j)))), ids=('func[string]', 'class[string]')) elif 'dumps' in metafunc.fixturenames: metafunc.parametrize('dumps', ( rj.dumps, binary_streaming_dumps, text_streaming_dumps, lambda o, **opts: rj.Encoder(**opts)(o), binary_streaming_encoder, text_streaming_encoder, ), ids=('func[string]', 'func[bytestream]', 'func[textstream]', 'class[string]', 'class[binarystream]', 'class[textstream]')) elif 'loads' in metafunc.fixturenames: metafunc.parametrize('loads', ( rj.loads, lambda j, **opts: rj.load( io.BytesIO(j.encode('utf-8') if isinstance(j, str) else j), **opts), lambda j, **opts: rj.load(io.StringIO(j), **opts), lambda j, **opts: rj.Decoder(**opts)(j), lambda j, **opts: rj.Decoder(**opts) (io.BytesIO(j.encode('utf-8') if isinstance(j, str) else j)), lambda j, **opts: rj.Decoder(**opts)(io.StringIO(j)), ), ids=('func[string]', 'func[bytestream]', 'func[textstream]', 'class[string]', 'class[bytestream]', 'class[textstream]'))
def test_skip_invalid_keys(): o = {True: False, -1: 1, 1.1: 1.1, (1, 2): "foo", b"asdf": 1, None: None} with pytest.raises(TypeError): rj.Encoder()(o) with pytest.raises(TypeError): rj.Encoder(skip_invalid_keys=False)(o) assert rj.Encoder(skip_invalid_keys=True)(o) == '{}'
def test_skip_invalid_keys(): o = {True: False, -1: 1, 1.1: 1.1, (1, 2): "foo", b"asdf": 1, None: None} with pytest.raises(TypeError): rj.Encoder()(o) with pytest.raises(TypeError): rj.Encoder(skip_invalid_keys=False)(o) assert rj.Encoder(skip_invalid_keys=True)(o) == '{}' assert rj.Encoder(mapping_mode=rj.MM_SKIP_NON_STRING_KEYS)(o) == '{}'
def pytest_generate_tests(metafunc): if 'dumps' in metafunc.fixturenames and 'loads' in metafunc.fixturenames: metafunc.parametrize('dumps,loads', ( ((rj.dumps, rj.loads), (lambda o,**opts: rj.Encoder(**opts)(o), lambda j,**opts: rj.Decoder(**opts)(j))))) elif 'dumps' in metafunc.fixturenames: metafunc.parametrize('dumps', ( rj.dumps, lambda o,**opts: rj.Encoder(**opts)(o))) elif 'loads' in metafunc.fixturenames: metafunc.parametrize('loads', ( rj.loads, lambda j,**opts: rj.Decoder(**opts)(j)))
def write(self, kind: FileType) -> None: fn = self._path(kind) if kind == FileType.PICKLE: # serialize as TreeNode with open(fn, "wb") as f: pickle.dump(self.treenode, f, protocol=-1) elif kind == FileType.CSV: # serialize as id_dict with open(fn, "w") as f: w = csv.DictWriter(f, Node._fields) w.writeheader() for item in self.treenode.node_iter(): w.writerow(item._asdict()) elif kind == FileType.MSGPACK: # https://msgpack-python.readthedocs.io/en/latest/api.html with open(fn, "wb") as f: # Doesn't improve speed # msgpack.pack(self._to_dict(), f, use_bin_type=True) msgpack.pack(self.to_dict_list(), f) elif kind == FileType.JSON: self._json_dump(fn, json.dump) elif kind == FileType.UJSON: self._json_dump(fn, ujson.dump) elif kind == FileType.SIMPLEJSON: # NOTE: simplejson includes key names when serializing NamedTuples with open(fn, "w") as f: if self.json_dict_list: simplejson.dump(list(self.id_dict.values()), f, ensure_ascii=True) else: simplejson.dump(self.id_dict, f, ensure_ascii=True) elif kind == FileType.CBOR2: with open(fn, "wb") as f: cbor2.dump(self.to_dict_list(), f) elif kind == FileType.CBOR: with open(fn, "wb") as f: cbor.dump(self.to_dict_list(), f) elif kind == FileType.RAPIDJSON: # https://python-rapidjson.readthedocs.io/en/latest/benchmarks.html # TODO: See this example for possible speed improvement - deeper integration with Node # https://python-rapidjson.readthedocs.io/en/latest/encoder.html # NOTE: can't use id_dict - keys must be strings # can't use self.id_dict.values() - not serializable # list(self.id_dict.values()) produces a list of lists - no keys - very fragile with open(fn, "w") as f: if self.json_dict_list: rapidjson.Encoder(number_mode=rapidjson.NM_NATIVE, ensure_ascii=False)(self.to_dict_list(), f) else: rapidjson.Encoder(number_mode=rapidjson.NM_NATIVE, ensure_ascii=False)(list(self.id_dict.values()), f) elif kind == FileType.BSON: with open(fn, "wb") as f: co = CodecOptions(document_class=RawBSONDocument) for node in self.treenode.node_iter(): f.write(BSON.encode(node._asdict(), codec_options=co))
def test_encoder_call(): o = {'foo': 'bar'} result = rj.Encoder()(o) assert result == '{"foo":"bar"}' assert rj.Encoder()(o, None) == result stream = io.StringIO() assert rj.Encoder()(o, stream) is None assert stream.getvalue() == result stream = io.StringIO() assert rj.Encoder()(o, stream=stream) is None assert stream.getvalue() == result
class RapidJSONEncoder(json.JSONEncoder): encode = rapidjson.Encoder(skip_invalid_keys=False, ensure_ascii=False, indent=None, sort_keys=False, number_mode=rapidjson.NM_NATIVE, datetime_mode=rapidjson.DM_ISO8601, uuid_mode=rapidjson.UM_CANONICAL)
def test_explicit_defaults_encoder(): assert rj.Encoder(skip_invalid_keys=False, ensure_ascii=True, indent=None, sort_keys=False, number_mode=None, datetime_mode=None, uuid_mode=None)({ 'foo': 'bar' }) == '{"foo":"bar"}'
def test_negative_infinity_c(): inf = float("-infinity") dumped = rj.Encoder()(inf) loaded = rj.Decoder()(dumped) assert loaded == inf with pytest.raises(ValueError): rj.Encoder(number_mode=None)(inf) d = Decimal(inf) assert d.is_infinite() with pytest.raises(ValueError): rj.Encoder(number_mode=rj.NM_DECIMAL)(d) dumped = rj.Encoder(number_mode=rj.NM_DECIMAL|rj.NM_NAN)(d) loaded = rj.Decoder(number_mode=rj.NM_DECIMAL|rj.NM_NAN)(dumped) assert loaded == inf assert loaded.is_infinite()
def test_nan_c(): nan = float("nan") dumped = rj.Encoder()(nan) loaded = rj.Decoder()(dumped) assert math.isnan(nan) assert math.isnan(loaded) with pytest.raises(ValueError): rj.Encoder(number_mode=None)(nan) d = Decimal(nan) assert d.is_nan() with pytest.raises(ValueError): rj.Encoder(number_mode=rj.NM_DECIMAL)(d) dumped = rj.Encoder(number_mode=rj.NM_DECIMAL|rj.NM_NAN)(d) loaded = rj.Decoder(number_mode=rj.NM_DECIMAL|rj.NM_NAN)(dumped) assert loaded.is_nan()
def test_serialize_sets_encoder(): class SetsEncoder(rj.Encoder): def default(self, obj): if isinstance(obj, set): return list(obj) raise TypeError(repr(obj) + " is not JSON serializable") dumps = SetsEncoder() dumps([set()]) with pytest.raises(TypeError): rj.Encoder()([set()])
def test_encoder_attrs(): e = rj.Encoder(skip_invalid_keys=True, ensure_ascii=True, indent='\n', sort_keys=True, number_mode=rj.NM_NAN, datetime_mode=rj.DM_ISO8601, uuid_mode=rj.UM_CANONICAL, bytes_mode=rj.BM_UTF8) assert e.skip_invalid_keys assert e.ensure_ascii assert e.indent_char == '\n' assert e.indent_count == 1 assert e.write_mode == rj.WM_PRETTY assert e.sort_keys assert e.number_mode == rj.NM_NAN assert e.datetime_mode == rj.DM_ISO8601 assert e.uuid_mode == rj.UM_CANONICAL assert e.bytes_mode == rj.BM_UTF8 e = rj.Encoder(mapping_mode=rj.MM_SKIP_NON_STRING_KEYS | rj.MM_SORT_KEYS) assert e.skip_invalid_keys assert e.sort_keys
def __init__(self, *args, **kwargs): rjkwargs = dict( skip_invalid_keys=kwargs.get("skipkeys", False), ensure_ascii=kwargs.get("ensure_ascii", True), write_mode=json.WM_COMPACT, indent=kwargs.get("indent", None), sort_keys=kwargs.get("sort_keys", False), number_mode=json.NM_DECIMAL, datetime_mode=json.DM_ISO8601, uuid_mode=json.UM_CANONICAL, # bytes_mode=None, # iterable_mode=IM_ANY_ITERABLE, # mapping_mode=MM_ANY_MAPPING, ) encoder = json.Encoder(**rjkwargs) self.encode = encoder.__call__
def test_coerce_keys(): o = {True: False, -1: 1, 1.1: 1.1, (1, 2): "foo", b"asdf": 1, None: None} expected = '{"True":false,"-1":1,"1.1":1.1,"(1, 2)":"foo","b\'asdf\'":1,"None":null}' assert rj.dumps(o, mapping_mode=rj.MM_COERCE_KEYS_TO_STRINGS) == expected assert rj.Encoder(mapping_mode=rj.MM_COERCE_KEYS_TO_STRINGS)(o) == expected
if 'data' in benchmark['stats']: benchmark['stats'].pop('data') def pytest_addoption(parser): parser.addoption('--compare-other-engines', action='store_true', help='compare against other JSON engines') contenders = [] import rapidjson as rj contenders.append(Contender('rapidjson_f', rj.dumps, rj.loads)) contenders.append(Contender('rapidjson_c', rj.Encoder(), rj.Decoder())) contenders.append( Contender('rapidjson_nn_f', partial(rj.dumps, number_mode=rj.NM_NATIVE), partial(rj.loads, number_mode=rj.NM_NATIVE))) contenders.append( Contender('rapidjson_nn_c', rj.Encoder(number_mode=rj.NM_NATIVE), rj.Decoder(number_mode=rj.NM_NATIVE))) numbers_contenders = [ Contender('Wide numbers', rj.dumps, rj.loads), Contender('Native numbers', partial(rj.dumps, number_mode=rj.NM_NATIVE), partial(rj.loads, number_mode=rj.NM_NATIVE)), ] string_contenders = [ Contender('rapidjson utf8', partial(rj.dumps, ensure_ascii=False),
def pytest_addoption(parser): parser.addoption('--compare-other-engines', action='store_true', help='compare against other JSON engines') contenders = [] import rapidjson as rj contenders.append(Contender('rapidjson_f', rj.dumps, rj.loads)) contenders.append(Contender('rapidjson_c', rj.Encoder(), rj.Decoder())) contenders.append(Contender('rapidjson_nn_f', partial(rj.dumps, number_mode=rj.NM_NATIVE), partial(rj.loads, number_mode=rj.NM_NATIVE))) contenders.append(Contender('rapidjson_nn_c', rj.Encoder(number_mode=rj.NM_NATIVE), rj.Decoder(number_mode=rj.NM_NATIVE))) numbers_contenders = [ Contender('Wide numbers', rj.dumps, rj.loads), Contender('Native numbers', partial(rj.dumps, number_mode=rj.NM_NATIVE), partial(rj.loads, number_mode=rj.NM_NATIVE)), ]
import rapidjson # Dump datetime in ISO format # treat "native" datetime objects as UTC DATETIME_MODE = rapidjson.DM_ISO8601 | rapidjson.DM_NAIVE_IS_UTC dumps = rapidjson.Encoder(datetime_mode=DATETIME_MODE) loads = rapidjson.Decoder(datetime_mode=DATETIME_MODE)
import rapidjson # Dump datetime in ISO format # treat "native" datetime objects as UTC DATETIME_MODE = rapidjson.DM_ISO8601 | rapidjson.DM_NAIVE_IS_UTC dumps = rapidjson.Encoder(datetime_mode=DATETIME_MODE) dumps_notascii = rapidjson.Encoder(datetime_mode=DATETIME_MODE, ensure_ascii=False) loads = rapidjson.Decoder(datetime_mode=DATETIME_MODE)
def text_streaming_encoder(o, **opts): stream = io.StringIO() rj.Encoder(**opts)(o, stream=stream) return stream.getvalue()
def binary_streaming_encoder(o, **opts): stream = io.BytesIO() rj.Encoder(**opts)(o, stream=stream) return stream.getvalue().decode('utf-8')