def test_next_nested_array(self):
json = '{"name1": "value1", "name2": [10, 20, true, "some value"]}'
stream = io.StringIO(json)
tokenizer = JsonTokenizer(stream)
self.assertEqual(tokenizer.next(), ('{', 0))
self.assertEqual(tokenizer.next(), ('name1', 1))
self.assertEqual(tokenizer.next(), ('value1', 1))
self.assertEqual(tokenizer.next(), ('name2', 1))
self.assertEqual(tokenizer.next(), ('[', 0))
self.assertEqual(tokenizer.next(), ('10', 2))
self.assertEqual(tokenizer.next(), ('20', 2))
self.assertEqual(tokenizer.next(), ('true', 2))
self.assertEqual(tokenizer.next(), ('some value', 1))
self.assertEqual(tokenizer.next(), (']', 0))
self.assertEqual(tokenizer.next(), ('}', 0))
self.assertEqual(tokenizer.has_next(), False)
def test_next_nested(self):
json = '{"name1": "value1", "name2": {"name22": "value22", "name23": "value23"}}'
stream = io.StringIO(json)
tokenizer = JsonTokenizer(stream)
self.assertEqual(tokenizer.next(), ('{', 0))
self.assertEqual(tokenizer.next(), ('name1', 1))
self.assertEqual(tokenizer.next(), ('value1', 1))
self.assertEqual(tokenizer.next(), ('name2', 1))
self.assertEqual(tokenizer.next(), ('{', 0))
self.assertEqual(tokenizer.next(), ('name22', 1))
self.assertEqual(tokenizer.next(), ('value22', 1))
self.assertEqual(tokenizer.next(), ('name23', 1))
self.assertEqual(tokenizer.next(), ('value23', 1))
self.assertEqual(tokenizer.next(), ('}', 0))
self.assertEqual(tokenizer.next(), ('}', 0))
self.assertEqual(tokenizer.has_next(), False)
def test_next_with_different_value_types(self):
json = '{"name1": "value1", "name2": null, "name3": true, "name4": false, "name5": 10.67}'
stream = io.StringIO(json)
tokenizer = JsonTokenizer(stream)
self.assertEqual(tokenizer.next(), ('{', 0))
self.assertEqual(tokenizer.next(), ('name1', 1))
self.assertEqual(tokenizer.next(), ('value1', 1))
self.assertEqual(tokenizer.next(), ('name2', 1))
self.assertEqual(tokenizer.next(), ('null', 2))
self.assertEqual(tokenizer.next(), ('name3', 1))
self.assertEqual(tokenizer.next(), ('true', 2))
self.assertEqual(tokenizer.next(), ('name4', 1))
self.assertEqual(tokenizer.next(), ('false', 2))
self.assertEqual(tokenizer.next(), ('name5', 1))
self.assertEqual(tokenizer.next(), ('10.67', 2))
self.assertEqual(tokenizer.next(), ('}', 0))
self.assertEqual(tokenizer.has_next(), False)
def test_next_simple_array(self):
json = '["value1", "value2", "value3"]'
stream = io.StringIO(json)
tokenizer = JsonTokenizer(stream)
self.assertEqual(tokenizer.next(), ('[', 0))
self.assertEqual(tokenizer.next(), ('value1', 1))
self.assertEqual(tokenizer.next(), ('value2', 1))
self.assertEqual(tokenizer.next(), ('value3', 1))
self.assertEqual(tokenizer.next(), (']', 0))
self.assertEqual(tokenizer.has_next(), False)
def test_decode(self):
json = '{"name1": "value1", "name2": "value2"}'
result = list()
stream = io.StringIO(json)
tokenizer = JsonTokenizer(stream)
JsonDecoder() \
.tokenizer(tokenizer) \
.root_class_name('Example') \
.event_handler(lambda e, p: result.append((e, p))) \
.decode()
self.assertEqual(len(result), 1)
(obj, path) = result[0]
self.assertEqual(path, '')
self.assertEqual(type(obj).__name__, 'Example')
self.assertEqual(obj.name1, 'value1')
self.assertEqual(obj.name2, 'value2')
def test_decode_array_of_arrays(self):
json = """
{
"id": "123",
"type": "car",
"data":
[
["red", "blue", "green"],
[1,2,3],
[true, false, true]
]
}"""
result = list()
stream = io.StringIO(json)
tokenizer = JsonTokenizer(stream)
JsonDecoder() \
.tokenizer(tokenizer) \
.root_class_name('Example') \
.event_handler(lambda e, p: result.append(e)) \
.decode()
self.assertEqual(len(result), 1)
self.assertEqual(result[0].id, '123')
self.assertEqual(result[0].type, 'car')
self.assertEqual(len(result[0].data), 3)
self.assertEqual(len(result[0].data[0]), 3)
self.assertEqual(result[0].data[0][0], 'red')
self.assertEqual(result[0].data[0][1], 'blue')
self.assertEqual(result[0].data[0][2], 'green')
self.assertEqual(len(result[0].data[1]), 3)
self.assertEqual(result[0].data[1][0], '1')
self.assertEqual(result[0].data[1][1], '2')
self.assertEqual(result[0].data[1][2], '3')
self.assertEqual(len(result[0].data[2]), 3)
self.assertEqual(result[0].data[2][0], True)
self.assertEqual(result[0].data[2][1], False)
self.assertEqual(result[0].data[2][2], True)
def test_next_nested_mixedarray(self):
json = """
{
"name1": "value1",
"name2": [
10,
20,
{
"name3": "value3",
"name4": "value4"
},
30]
}"""
stream = io.StringIO(json)
tokenizer = JsonTokenizer(stream)
self.assertEqual(tokenizer.next(), ('{', 0))
self.assertEqual(tokenizer.next(), ('name1', 1))
self.assertEqual(tokenizer.next(), ('value1', 1))
self.assertEqual(tokenizer.next(), ('name2', 1))
self.assertEqual(tokenizer.next(), ('[', 0))
self.assertEqual(tokenizer.next(), ('10', 2))
self.assertEqual(tokenizer.next(), ('20', 2))
self.assertEqual(tokenizer.next(), ('{', 0))
self.assertEqual(tokenizer.next(), ('name3', 1))
self.assertEqual(tokenizer.next(), ('value3', 1))
self.assertEqual(tokenizer.next(), ('name4', 1))
self.assertEqual(tokenizer.next(), ('value4', 1))
self.assertEqual(tokenizer.next(), ('}', 0))
self.assertEqual(tokenizer.next(), ('30', 2))
self.assertEqual(tokenizer.next(), (']', 0))
self.assertEqual(tokenizer.next(), ('}', 0))
self.assertEqual(tokenizer.has_next(), False)
def test_next_array_of_arrays(self):
json = """
{
"id": "123",
"type": "car",
"data":
[
["red", "blue", "green"],
[1,2,3],
[true, false, true]
]
}"""
stream = io.StringIO(json)
tokenizer = JsonTokenizer(stream)
self.assertEqual(tokenizer.next(), ('{', 0))
self.assertEqual(tokenizer.next(), ('id', 1))
self.assertEqual(tokenizer.next(), ('123', 1))
self.assertEqual(tokenizer.next(), ('type', 1))
self.assertEqual(tokenizer.next(), ('car', 1))
self.assertEqual(tokenizer.next(), ('data', 1))
self.assertEqual(tokenizer.next(), ('[', 0))
self.assertEqual(tokenizer.next(), ('[', 0))
self.assertEqual(tokenizer.next(), ('red', 1))
self.assertEqual(tokenizer.next(), ('blue', 1))
self.assertEqual(tokenizer.next(), ('green', 1))
self.assertEqual(tokenizer.next(), (']', 0))
self.assertEqual(tokenizer.next(), ('[', 0))
self.assertEqual(tokenizer.next(), ('1', 2))
self.assertEqual(tokenizer.next(), ('2', 2))
self.assertEqual(tokenizer.next(), ('3', 2))
self.assertEqual(tokenizer.next(), (']', 0))
self.assertEqual(tokenizer.next(), ('[', 0))
self.assertEqual(tokenizer.next(), ('true', 2))
self.assertEqual(tokenizer.next(), ('false', 2))
self.assertEqual(tokenizer.next(), ('true', 2))
self.assertEqual(tokenizer.next(), (']', 0))
self.assertEqual(tokenizer.next(), (']', 0))
self.assertEqual(tokenizer.next(), ('}', 0))
self.assertEqual(tokenizer.has_next(), False)
def test_next_deeply_nested(self):
json = """
{
"id": "0001",
"type": "donut",
"name": "Cake",
"ppu": 0.55,
"batters":
{
"batter":
[
{ "id": "1001", "type": "Regular" },
{ "id": "1002", "type": "Chocolate" },
{ "id": "1003", "type": "Blueberry" },
{ "id": "1004", "type": "Devils Food" }
]
},
"topping":
[
{ "id": "5001", "type": "None" },
{ "id": "5002", "type": "Glazed" },
{ "id": "5005", "type": "Sugar" },
{ "id": "5007", "type": "Powdered Sugar" },
{ "id": "5006", "type": "Chocolate with Sprinkles" },
{ "id": "5003", "type": "Chocolate" },
{ "id": "5004", "type": "Maple" }
]
}"""
stream = io.StringIO(json)
tokenizer = JsonTokenizer(stream)
self.assertEqual(tokenizer.next(), ('{', 0))
self.assertEqual(tokenizer.next(), ('id', 1))
self.assertEqual(tokenizer.next(), ('0001', 1))
self.assertEqual(tokenizer.next(), ('type', 1))
self.assertEqual(tokenizer.next(), ('donut', 1))
self.assertEqual(tokenizer.next(), ('name', 1))
self.assertEqual(tokenizer.next(), ('Cake', 1))
self.assertEqual(tokenizer.next(), ('ppu', 1))
self.assertEqual(tokenizer.next(), ('0.55', 2))
self.assertEqual(tokenizer.next(), ('batters', 1))
self.assertEqual(tokenizer.next(), ('{', 0))
self.assertEqual(tokenizer.next(), ('batter', 1))
self.assertEqual(tokenizer.next(), ('[', 0))
self.assertEqual(tokenizer.next(), ('{', 0))
self.assertEqual(tokenizer.next(), ('id', 1))
self.assertEqual(tokenizer.next(), ('1001', 1))
self.assertEqual(tokenizer.next(), ('type', 1))
self.assertEqual(tokenizer.next(), ('Regular', 1))
self.assertEqual(tokenizer.next(), ('}', 0))
self.assertEqual(tokenizer.next(), ('{', 0))
self.assertEqual(tokenizer.next(), ('id', 1))
self.assertEqual(tokenizer.next(), ('1002', 1))
self.assertEqual(tokenizer.next(), ('type', 1))
self.assertEqual(tokenizer.next(), ('Chocolate', 1))
self.assertEqual(tokenizer.next(), ('}', 0))
self.assertEqual(tokenizer.next(), ('{', 0))
self.assertEqual(tokenizer.next(), ('id', 1))
self.assertEqual(tokenizer.next(), ('1003', 1))
self.assertEqual(tokenizer.next(), ('type', 1))
self.assertEqual(tokenizer.next(), ('Blueberry', 1))
self.assertEqual(tokenizer.next(), ('}', 0))
self.assertEqual(tokenizer.next(), ('{', 0))
self.assertEqual(tokenizer.next(), ('id', 1))
self.assertEqual(tokenizer.next(), ('1004', 1))
self.assertEqual(tokenizer.next(), ('type', 1))
self.assertEqual(tokenizer.next(), ('Devils Food', 1))
self.assertEqual(tokenizer.next(), ('}', 0))
self.assertEqual(tokenizer.next(), (']', 0))
self.assertEqual(tokenizer.next(), ('}', 0))
self.assertEqual(tokenizer.next(), ('topping', 1))
self.assertEqual(tokenizer.next(), ('[', 0))
self.assertEqual(tokenizer.next(), ('{', 0))
self.assertEqual(tokenizer.next(), ('id', 1))
self.assertEqual(tokenizer.next(), ('5001', 1))
self.assertEqual(tokenizer.next(), ('type', 1))
self.assertEqual(tokenizer.next(), ('None', 1))
self.assertEqual(tokenizer.next(), ('}', 0))
self.assertEqual(tokenizer.next(), ('{', 0))
self.assertEqual(tokenizer.next(), ('id', 1))
self.assertEqual(tokenizer.next(), ('5002', 1))
self.assertEqual(tokenizer.next(), ('type', 1))
self.assertEqual(tokenizer.next(), ('Glazed', 1))
self.assertEqual(tokenizer.next(), ('}', 0))
self.assertEqual(tokenizer.next(), ('{', 0))
self.assertEqual(tokenizer.next(), ('id', 1))
self.assertEqual(tokenizer.next(), ('5005', 1))
self.assertEqual(tokenizer.next(), ('type', 1))
self.assertEqual(tokenizer.next(), ('Sugar', 1))
self.assertEqual(tokenizer.next(), ('}', 0))
self.assertEqual(tokenizer.next(), ('{', 0))
self.assertEqual(tokenizer.next(), ('id', 1))
self.assertEqual(tokenizer.next(), ('5007', 1))
self.assertEqual(tokenizer.next(), ('type', 1))
self.assertEqual(tokenizer.next(), ('Powdered Sugar', 1))
self.assertEqual(tokenizer.next(), ('}', 0))
self.assertEqual(tokenizer.next(), ('{', 0))
self.assertEqual(tokenizer.next(), ('id', 1))
self.assertEqual(tokenizer.next(), ('5006', 1))
self.assertEqual(tokenizer.next(), ('type', 1))
self.assertEqual(tokenizer.next(), ('Chocolate with Sprinkles', 1))
self.assertEqual(tokenizer.next(), ('}', 0))
self.assertEqual(tokenizer.next(), ('{', 0))
self.assertEqual(tokenizer.next(), ('id', 1))
self.assertEqual(tokenizer.next(), ('5003', 1))
self.assertEqual(tokenizer.next(), ('type', 1))
self.assertEqual(tokenizer.next(), ('Chocolate', 1))
self.assertEqual(tokenizer.next(), ('}', 0))
self.assertEqual(tokenizer.next(), ('{', 0))
self.assertEqual(tokenizer.next(), ('id', 1))
self.assertEqual(tokenizer.next(), ('5004', 1))
self.assertEqual(tokenizer.next(), ('type', 1))
self.assertEqual(tokenizer.next(), ('Maple', 1))
self.assertEqual(tokenizer.next(), ('}', 0))
self.assertEqual(tokenizer.next(), (']', 0))
self.assertEqual(tokenizer.next(), ('}', 0))
self.assertEqual(tokenizer.has_next(), False)
def test_decode_with_translators(self):
class MyBatter:
__slots__ = ('my_id', 'my_type')
def __init__(self, id, type):
self.my_id = id
self.my_type = type
class MyBatters:
__slots__ = ('my_batter')
def __init__(self, batter):
self.my_batter = batter
json = """
[
{
"id": "0001",
"type": "donut",
"name": "Cake",
"ppu": 0.55,
"batters":
{
"batter":
[
{ "id": "1001", "type": "Regular" },
{ "id": "1002", "type": "Chocolate" },
{ "id": "1003", "type": "Blueberry" },
{ "id": "1004", "type": "Devil's Food" }
]
},
"topping":
[
{ "id": "5001", "type": "None" },
{ "id": "5002", "type": "Glazed" },
{ "id": "5005", "type": "Sugar" },
{ "id": "5007", "type": "Powdered Sugar" },
{ "id": "5006", "type": "Chocolate with Sprinkles" },
{ "id": "5003", "type": "Chocolate" },
{ "id": "5004", "type": "Maple" }
]
},
{
"id": "0002",
"type": "donut",
"name": "Raised",
"ppu": 0.56,
"batters":
{
"batter":
[
{ "id": "1001", "type": "Regular" }
]
},
"topping":
[
{ "id": "5001", "type": "None" },
{ "id": "5002", "type": "Glazed" },
{ "id": "5005", "type": "Sugar" },
{ "id": "5003", "type": "Chocolate" },
{ "id": "5004", "type": "Maple" }
]
},
{
"id": "0003",
"type": "donut",
"name": "Old Fashioned",
"ppu": 0.57,
"batters":
{
"batter":
[
{ "id": "1001", "type": "Regular" },
{ "id": "1002", "type": "Chocolate" }
]
},
"topping":
[
{ "id": "5001", "type": "None" },
{ "id": "5002", "type": "Glazed" },
{ "id": "5003", "type": "Chocolate" },
{ "id": "5004", "type": "Maple" }
]
}
]"""
result = list()
stream = io.StringIO(json)
tokenizer = JsonTokenizer(stream)
JsonDecoder() \
.tokenizer(tokenizer) \
.root_class_name('Example') \
.translator('batters', lambda o: MyBatters(o.batter)) \
.translator('batters.batter', lambda o: MyBatter(o.id, o.type)) \
.event_handler(lambda e, p: result.append(e)) \
.decode()
self.assertEqual(len(result), 3)
self.assertEqual(type(result[0].batters).__name__, 'MyBatters')
self.assertEqual(
type(result[0].batters.my_batter[0]).__name__, 'MyBatter')
self.assertEqual(result[0].batters.my_batter[0].my_id, '1001')
self.assertEqual(result[0].batters.my_batter[0].my_type, 'Regular')
def test_decode_subscribe_on_nested_object(self):
json = """
[
{
"id": "0001",
"type": "donut",
"name": "Cake",
"ppu": 0.55,
"batters":
{
"batter":
[
{ "id": "1001", "type": "Regular" },
{ "id": "1002", "type": "Chocolate" },
{ "id": "1003", "type": "Blueberry" },
{ "id": "1004", "type": "Devil's Food" }
]
},
"topping":
[
{ "id": "5001", "type": "None" },
{ "id": "5002", "type": "Glazed" },
{ "id": "5005", "type": "Sugar" },
{ "id": "5007", "type": "Powdered Sugar" },
{ "id": "5006", "type": "Chocolate with Sprinkles" },
{ "id": "5003", "type": "Chocolate" },
{ "id": "5004", "type": "Maple" }
]
},
{
"id": "0002",
"type": "donut",
"name": "Raised",
"ppu": 0.55,
"batters":
{
"batter":
[
{ "id": "1001", "type": "Regular" }
]
},
"topping":
[
{ "id": "5001", "type": "None" },
{ "id": "5002", "type": "Glazed" },
{ "id": "5005", "type": "Sugar" },
{ "id": "5003", "type": "Chocolate" },
{ "id": "5004", "type": "Maple" }
]
},
{
"id": "0003",
"type": "donut",
"name": "Old Fashioned",
"ppu": 0.55,
"batters":
{
"batter":
[
{ "id": "1001", "type": "Regular" },
{ "id": "1002", "type": "Chocolate" }
]
},
"topping":
[
{ "id": "5001", "type": "None" },
{ "id": "5002", "type": "Glazed" },
{ "id": "5003", "type": "Chocolate" },
{ "id": "5004", "type": "Maple" }
]
}
]"""
result = list()
stream = io.StringIO(json)
tokenizer = JsonTokenizer(stream)
JsonDecoder() \
.tokenizer(tokenizer) \
.root_class_name('Example') \
.predicate('batters') \
.event_handler(lambda e, p: result.append((e, p))) \
.decode()
self.assertEqual(len(result), 3)
self.assertEqual(len(result[0][0].batter), 4)
self.assertEqual(len(result[1][0].batter), 1)
self.assertEqual(len(result[2][0].batter), 2)
(obj, path) = result[0]
self.assertEqual(path, 'batters')
self.assertEqual(type(obj).__name__, 'Batters')
self.assertEqual(type(obj.batter).__name__, 'list')
self.assertEqual(len(obj.batter), 4)
self.assertEqual(type(obj.batter[0]).__name__, 'Batter')
self.assertEqual(obj.batter[0].id, '1001')
self.assertEqual(obj.batter[0].type, 'Regular')
self.assertEqual(obj.batter[1].id, '1002')
self.assertEqual(obj.batter[1].type, 'Chocolate')
self.assertEqual(obj.batter[2].id, '1003')
self.assertEqual(obj.batter[2].type, 'Blueberry')
self.assertEqual(obj.batter[3].id, '1004')
self.assertEqual(obj.batter[3].type, "Devil's Food")
def test_decode_subscribe_on_simple_value(self):
json = """
[
{
"id": "0001",
"type": "donut",
"name": "Cake",
"ppu": 0.55,
"batters":
{
"batter":
[
{ "id": "1001", "type": "Regular" },
{ "id": "1002", "type": "Chocolate" },
{ "id": "1003", "type": "Blueberry" },
{ "id": "1004", "type": "Devil's Food" }
]
},
"topping":
[
{ "id": "5001", "type": "None" },
{ "id": "5002", "type": "Glazed" },
{ "id": "5005", "type": "Sugar" },
{ "id": "5007", "type": "Powdered Sugar" },
{ "id": "5006", "type": "Chocolate with Sprinkles" },
{ "id": "5003", "type": "Chocolate" },
{ "id": "5004", "type": "Maple" }
]
},
{
"id": "0002",
"type": "donut",
"name": "Raised",
"ppu": 0.56,
"batters":
{
"batter":
[
{ "id": "1001", "type": "Regular" }
]
},
"topping":
[
{ "id": "5001", "type": "None" },
{ "id": "5002", "type": "Glazed" },
{ "id": "5005", "type": "Sugar" },
{ "id": "5003", "type": "Chocolate" },
{ "id": "5004", "type": "Maple" }
]
},
{
"id": "0003",
"type": "donut",
"name": "Old Fashioned",
"ppu": 0.57,
"batters":
{
"batter":
[
{ "id": "1001", "type": "Regular" },
{ "id": "1002", "type": "Chocolate" }
]
},
"topping":
[
{ "id": "5001", "type": "None" },
{ "id": "5002", "type": "Glazed" },
{ "id": "5003", "type": "Chocolate" },
{ "id": "5004", "type": "Maple" }
]
}
]"""
result = list()
stream = io.StringIO(json)
tokenizer = JsonTokenizer(stream)
JsonDecoder() \
.tokenizer(tokenizer) \
.root_class_name('Example') \
.predicate('ppu') \
.event_handler(lambda e, p: result.append((e, p))) \
.decode()
self.assertEqual(len(result), 3)
self.assertEqual(result[0][0], '0.55')
self.assertEqual(result[1][0], '0.56')
self.assertEqual(result[2][0], '0.57')
print('processed ' + str(counter[0]) + ' objects')
counter[0] += 1
if __name__ == '__main__':
url = '/prust/wikipedia-movie-data/master/movies.json'
endpoint = HTTPSConnection('raw.githubusercontent.com',
'443',
context=ssl._create_unverified_context())
try:
ts1 = datetime.datetime.now().timestamp()
print('start time=' + str(ts1))
endpoint.request('GET', url)
response = endpoint.getresponse()
tokenizer = JsonTokenizer(response, 'ISO-8859-1', 65536)
JsonDecoder()\
.tokenizer(tokenizer)\
.root_class_name('Data')\
.event_handler(lambda e, p: handle_event(e, p, counter))\
.predicate('genres') \
.with_snake_cased_props()\
.decode()
ts2 = datetime.datetime.now().timestamp()
print('end time=' + str(ts2))
print('delta=' + str(ts2 - ts1))
finally:
endpoint.close()
print('total events count=' + str(counter[0]))