def test_next_nested_mixedarray(self): json = """ { "name1": "value1", "name2": [ 10, 20, { "name3": "value3", "name4": "value4" }, 30] }""" stream = io.StringIO(json) tokenizer = JsonTokenizer(stream) self.assertEqual(tokenizer.next(), ('{', 0)) self.assertEqual(tokenizer.next(), ('name1', 1)) self.assertEqual(tokenizer.next(), ('value1', 1)) self.assertEqual(tokenizer.next(), ('name2', 1)) self.assertEqual(tokenizer.next(), ('[', 0)) self.assertEqual(tokenizer.next(), ('10', 2)) self.assertEqual(tokenizer.next(), ('20', 2)) self.assertEqual(tokenizer.next(), ('{', 0)) self.assertEqual(tokenizer.next(), ('name3', 1)) self.assertEqual(tokenizer.next(), ('value3', 1)) self.assertEqual(tokenizer.next(), ('name4', 1)) self.assertEqual(tokenizer.next(), ('value4', 1)) self.assertEqual(tokenizer.next(), ('}', 0)) self.assertEqual(tokenizer.next(), ('30', 2)) self.assertEqual(tokenizer.next(), (']', 0)) self.assertEqual(tokenizer.next(), ('}', 0)) self.assertEqual(tokenizer.has_next(), False)
def test_next_simple_array(self): json = '["value1", "value2", "value3"]' stream = io.StringIO(json) tokenizer = JsonTokenizer(stream) self.assertEqual(tokenizer.next(), ('[', 0)) self.assertEqual(tokenizer.next(), ('value1', 1)) self.assertEqual(tokenizer.next(), ('value2', 1)) self.assertEqual(tokenizer.next(), ('value3', 1)) self.assertEqual(tokenizer.next(), (']', 0)) self.assertEqual(tokenizer.has_next(), False)
def test_next(self): json = '{"name1": "value1", "name2": "value2"}' stream = io.StringIO(json) tokenizer = JsonTokenizer(stream) self.assertEqual(tokenizer.next(), ('{', 0)) self.assertEqual(tokenizer.next(), ('name1', 1)) self.assertEqual(tokenizer.next(), ('value1', 1)) self.assertEqual(tokenizer.next(), ('name2', 1)) self.assertEqual(tokenizer.next(), ('value2', 1)) self.assertEqual(tokenizer.next(), ('}', 0)) self.assertEqual(tokenizer.has_next(), False)
def test_next_nested_array(self): json = '{"name1": "value1", "name2": [10, 20, true, "some value"]}' stream = io.StringIO(json) tokenizer = JsonTokenizer(stream) self.assertEqual(tokenizer.next(), ('{', 0)) self.assertEqual(tokenizer.next(), ('name1', 1)) self.assertEqual(tokenizer.next(), ('value1', 1)) self.assertEqual(tokenizer.next(), ('name2', 1)) self.assertEqual(tokenizer.next(), ('[', 0)) self.assertEqual(tokenizer.next(), ('10', 2)) self.assertEqual(tokenizer.next(), ('20', 2)) self.assertEqual(tokenizer.next(), ('true', 2)) self.assertEqual(tokenizer.next(), ('some value', 1)) self.assertEqual(tokenizer.next(), (']', 0)) self.assertEqual(tokenizer.next(), ('}', 0)) self.assertEqual(tokenizer.has_next(), False)
def test_next_with_different_value_types(self): json = '{"name1": "value1", "name2": null, "name3": true, "name4": false, "name5": 10.67}' stream = io.StringIO(json) tokenizer = JsonTokenizer(stream) self.assertEqual(tokenizer.next(), ('{', 0)) self.assertEqual(tokenizer.next(), ('name1', 1)) self.assertEqual(tokenizer.next(), ('value1', 1)) self.assertEqual(tokenizer.next(), ('name2', 1)) self.assertEqual(tokenizer.next(), ('null', 2)) self.assertEqual(tokenizer.next(), ('name3', 1)) self.assertEqual(tokenizer.next(), ('true', 2)) self.assertEqual(tokenizer.next(), ('name4', 1)) self.assertEqual(tokenizer.next(), ('false', 2)) self.assertEqual(tokenizer.next(), ('name5', 1)) self.assertEqual(tokenizer.next(), ('10.67', 2)) self.assertEqual(tokenizer.next(), ('}', 0)) self.assertEqual(tokenizer.has_next(), False)
def test_decode(self): json = '{"name1": "value1", "name2": "value2"}' result = list() stream = io.StringIO(json) tokenizer = JsonTokenizer(stream) JsonDecoder() \ .tokenizer(tokenizer) \ .root_class_name('Example') \ .event_handler(lambda e, p: result.append((e, p))) \ .decode() self.assertEqual(len(result), 1) (obj, path) = result[0] self.assertEqual(path, '') self.assertEqual(type(obj).__name__, 'Example') self.assertEqual(obj.name1, 'value1') self.assertEqual(obj.name2, 'value2')
def test_next_array_of_arrays(self): json = """ { "id": "123", "type": "car", "data": [ ["red", "blue", "green"], [1,2,3], [true, false, true] ] }""" stream = io.StringIO(json) tokenizer = JsonTokenizer(stream) self.assertEqual(tokenizer.next(), ('{', 0)) self.assertEqual(tokenizer.next(), ('id', 1)) self.assertEqual(tokenizer.next(), ('123', 1)) self.assertEqual(tokenizer.next(), ('type', 1)) self.assertEqual(tokenizer.next(), ('car', 1)) self.assertEqual(tokenizer.next(), ('data', 1)) self.assertEqual(tokenizer.next(), ('[', 0)) self.assertEqual(tokenizer.next(), ('[', 0)) self.assertEqual(tokenizer.next(), ('red', 1)) self.assertEqual(tokenizer.next(), ('blue', 1)) self.assertEqual(tokenizer.next(), ('green', 1)) self.assertEqual(tokenizer.next(), (']', 0)) self.assertEqual(tokenizer.next(), ('[', 0)) self.assertEqual(tokenizer.next(), ('1', 2)) self.assertEqual(tokenizer.next(), ('2', 2)) self.assertEqual(tokenizer.next(), ('3', 2)) self.assertEqual(tokenizer.next(), (']', 0)) self.assertEqual(tokenizer.next(), ('[', 0)) self.assertEqual(tokenizer.next(), ('true', 2)) self.assertEqual(tokenizer.next(), ('false', 2)) self.assertEqual(tokenizer.next(), ('true', 2)) self.assertEqual(tokenizer.next(), (']', 0)) self.assertEqual(tokenizer.next(), (']', 0)) self.assertEqual(tokenizer.next(), ('}', 0)) self.assertEqual(tokenizer.has_next(), False)
def test_decode_array_of_arrays(self): json = """ { "id": "123", "type": "car", "data": [ ["red", "blue", "green"], [1,2,3], [true, false, true] ] }""" result = list() stream = io.StringIO(json) tokenizer = JsonTokenizer(stream) JsonDecoder() \ .tokenizer(tokenizer) \ .root_class_name('Example') \ .event_handler(lambda e, p: result.append(e)) \ .decode() self.assertEqual(len(result), 1) self.assertEqual(result[0].id, '123') self.assertEqual(result[0].type, 'car') self.assertEqual(len(result[0].data), 3) self.assertEqual(len(result[0].data[0]), 3) self.assertEqual(result[0].data[0][0], 'red') self.assertEqual(result[0].data[0][1], 'blue') self.assertEqual(result[0].data[0][2], 'green') self.assertEqual(len(result[0].data[1]), 3) self.assertEqual(result[0].data[1][0], '1') self.assertEqual(result[0].data[1][1], '2') self.assertEqual(result[0].data[1][2], '3') self.assertEqual(len(result[0].data[2]), 3) self.assertEqual(result[0].data[2][0], True) self.assertEqual(result[0].data[2][1], False) self.assertEqual(result[0].data[2][2], True)
def test_next_deeply_nested(self): json = """ { "id": "0001", "type": "donut", "name": "Cake", "ppu": 0.55, "batters": { "batter": [ { "id": "1001", "type": "Regular" }, { "id": "1002", "type": "Chocolate" }, { "id": "1003", "type": "Blueberry" }, { "id": "1004", "type": "Devils Food" } ] }, "topping": [ { "id": "5001", "type": "None" }, { "id": "5002", "type": "Glazed" }, { "id": "5005", "type": "Sugar" }, { "id": "5007", "type": "Powdered Sugar" }, { "id": "5006", "type": "Chocolate with Sprinkles" }, { "id": "5003", "type": "Chocolate" }, { "id": "5004", "type": "Maple" } ] }""" stream = io.StringIO(json) tokenizer = JsonTokenizer(stream) self.assertEqual(tokenizer.next(), ('{', 0)) self.assertEqual(tokenizer.next(), ('id', 1)) self.assertEqual(tokenizer.next(), ('0001', 1)) self.assertEqual(tokenizer.next(), ('type', 1)) self.assertEqual(tokenizer.next(), ('donut', 1)) self.assertEqual(tokenizer.next(), ('name', 1)) self.assertEqual(tokenizer.next(), ('Cake', 1)) self.assertEqual(tokenizer.next(), ('ppu', 1)) self.assertEqual(tokenizer.next(), ('0.55', 2)) self.assertEqual(tokenizer.next(), ('batters', 1)) self.assertEqual(tokenizer.next(), ('{', 0)) self.assertEqual(tokenizer.next(), ('batter', 1)) self.assertEqual(tokenizer.next(), ('[', 0)) self.assertEqual(tokenizer.next(), ('{', 0)) self.assertEqual(tokenizer.next(), ('id', 1)) self.assertEqual(tokenizer.next(), ('1001', 1)) self.assertEqual(tokenizer.next(), ('type', 1)) self.assertEqual(tokenizer.next(), ('Regular', 1)) self.assertEqual(tokenizer.next(), ('}', 0)) self.assertEqual(tokenizer.next(), ('{', 0)) self.assertEqual(tokenizer.next(), ('id', 1)) self.assertEqual(tokenizer.next(), ('1002', 1)) self.assertEqual(tokenizer.next(), ('type', 1)) self.assertEqual(tokenizer.next(), ('Chocolate', 1)) self.assertEqual(tokenizer.next(), ('}', 0)) self.assertEqual(tokenizer.next(), ('{', 0)) self.assertEqual(tokenizer.next(), ('id', 1)) self.assertEqual(tokenizer.next(), ('1003', 1)) self.assertEqual(tokenizer.next(), ('type', 1)) self.assertEqual(tokenizer.next(), ('Blueberry', 1)) self.assertEqual(tokenizer.next(), ('}', 0)) self.assertEqual(tokenizer.next(), ('{', 0)) self.assertEqual(tokenizer.next(), ('id', 1)) self.assertEqual(tokenizer.next(), ('1004', 1)) self.assertEqual(tokenizer.next(), ('type', 1)) self.assertEqual(tokenizer.next(), ('Devils Food', 1)) self.assertEqual(tokenizer.next(), ('}', 0)) self.assertEqual(tokenizer.next(), (']', 0)) self.assertEqual(tokenizer.next(), ('}', 0)) self.assertEqual(tokenizer.next(), ('topping', 1)) self.assertEqual(tokenizer.next(), ('[', 0)) self.assertEqual(tokenizer.next(), ('{', 0)) self.assertEqual(tokenizer.next(), ('id', 1)) self.assertEqual(tokenizer.next(), ('5001', 1)) self.assertEqual(tokenizer.next(), ('type', 1)) self.assertEqual(tokenizer.next(), ('None', 1)) self.assertEqual(tokenizer.next(), ('}', 0)) self.assertEqual(tokenizer.next(), ('{', 0)) self.assertEqual(tokenizer.next(), ('id', 1)) self.assertEqual(tokenizer.next(), ('5002', 1)) self.assertEqual(tokenizer.next(), ('type', 1)) self.assertEqual(tokenizer.next(), ('Glazed', 1)) self.assertEqual(tokenizer.next(), ('}', 0)) self.assertEqual(tokenizer.next(), ('{', 0)) self.assertEqual(tokenizer.next(), ('id', 1)) self.assertEqual(tokenizer.next(), ('5005', 1)) self.assertEqual(tokenizer.next(), ('type', 1)) self.assertEqual(tokenizer.next(), ('Sugar', 1)) self.assertEqual(tokenizer.next(), ('}', 0)) self.assertEqual(tokenizer.next(), ('{', 0)) self.assertEqual(tokenizer.next(), ('id', 1)) self.assertEqual(tokenizer.next(), ('5007', 1)) self.assertEqual(tokenizer.next(), ('type', 1)) self.assertEqual(tokenizer.next(), ('Powdered Sugar', 1)) self.assertEqual(tokenizer.next(), ('}', 0)) self.assertEqual(tokenizer.next(), ('{', 0)) self.assertEqual(tokenizer.next(), ('id', 1)) self.assertEqual(tokenizer.next(), ('5006', 1)) self.assertEqual(tokenizer.next(), ('type', 1)) self.assertEqual(tokenizer.next(), ('Chocolate with Sprinkles', 1)) self.assertEqual(tokenizer.next(), ('}', 0)) self.assertEqual(tokenizer.next(), ('{', 0)) self.assertEqual(tokenizer.next(), ('id', 1)) self.assertEqual(tokenizer.next(), ('5003', 1)) self.assertEqual(tokenizer.next(), ('type', 1)) self.assertEqual(tokenizer.next(), ('Chocolate', 1)) self.assertEqual(tokenizer.next(), ('}', 0)) self.assertEqual(tokenizer.next(), ('{', 0)) self.assertEqual(tokenizer.next(), ('id', 1)) self.assertEqual(tokenizer.next(), ('5004', 1)) self.assertEqual(tokenizer.next(), ('type', 1)) self.assertEqual(tokenizer.next(), ('Maple', 1)) self.assertEqual(tokenizer.next(), ('}', 0)) self.assertEqual(tokenizer.next(), (']', 0)) self.assertEqual(tokenizer.next(), ('}', 0)) self.assertEqual(tokenizer.has_next(), False)
def test_decode_with_translators(self): class MyBatter: __slots__ = ('my_id', 'my_type') def __init__(self, id, type): self.my_id = id self.my_type = type class MyBatters: __slots__ = ('my_batter') def __init__(self, batter): self.my_batter = batter json = """ [ { "id": "0001", "type": "donut", "name": "Cake", "ppu": 0.55, "batters": { "batter": [ { "id": "1001", "type": "Regular" }, { "id": "1002", "type": "Chocolate" }, { "id": "1003", "type": "Blueberry" }, { "id": "1004", "type": "Devil's Food" } ] }, "topping": [ { "id": "5001", "type": "None" }, { "id": "5002", "type": "Glazed" }, { "id": "5005", "type": "Sugar" }, { "id": "5007", "type": "Powdered Sugar" }, { "id": "5006", "type": "Chocolate with Sprinkles" }, { "id": "5003", "type": "Chocolate" }, { "id": "5004", "type": "Maple" } ] }, { "id": "0002", "type": "donut", "name": "Raised", "ppu": 0.56, "batters": { "batter": [ { "id": "1001", "type": "Regular" } ] }, "topping": [ { "id": "5001", "type": "None" }, { "id": "5002", "type": "Glazed" }, { "id": "5005", "type": "Sugar" }, { "id": "5003", "type": "Chocolate" }, { "id": "5004", "type": "Maple" } ] }, { "id": "0003", "type": "donut", "name": "Old Fashioned", "ppu": 0.57, "batters": { "batter": [ { "id": "1001", "type": "Regular" }, { "id": "1002", "type": "Chocolate" } ] }, "topping": [ { "id": "5001", "type": "None" }, { "id": "5002", "type": "Glazed" }, { "id": "5003", "type": "Chocolate" }, { "id": "5004", "type": "Maple" } ] } ]""" result = list() stream = io.StringIO(json) tokenizer = JsonTokenizer(stream) JsonDecoder() \ .tokenizer(tokenizer) \ .root_class_name('Example') \ .translator('batters', lambda o: MyBatters(o.batter)) \ .translator('batters.batter', lambda o: MyBatter(o.id, o.type)) \ .event_handler(lambda e, p: result.append(e)) \ .decode() self.assertEqual(len(result), 3) self.assertEqual(type(result[0].batters).__name__, 'MyBatters') self.assertEqual( type(result[0].batters.my_batter[0]).__name__, 'MyBatter') self.assertEqual(result[0].batters.my_batter[0].my_id, '1001') self.assertEqual(result[0].batters.my_batter[0].my_type, 'Regular')
def test_decode_subscribe_on_nested_object(self): json = """ [ { "id": "0001", "type": "donut", "name": "Cake", "ppu": 0.55, "batters": { "batter": [ { "id": "1001", "type": "Regular" }, { "id": "1002", "type": "Chocolate" }, { "id": "1003", "type": "Blueberry" }, { "id": "1004", "type": "Devil's Food" } ] }, "topping": [ { "id": "5001", "type": "None" }, { "id": "5002", "type": "Glazed" }, { "id": "5005", "type": "Sugar" }, { "id": "5007", "type": "Powdered Sugar" }, { "id": "5006", "type": "Chocolate with Sprinkles" }, { "id": "5003", "type": "Chocolate" }, { "id": "5004", "type": "Maple" } ] }, { "id": "0002", "type": "donut", "name": "Raised", "ppu": 0.55, "batters": { "batter": [ { "id": "1001", "type": "Regular" } ] }, "topping": [ { "id": "5001", "type": "None" }, { "id": "5002", "type": "Glazed" }, { "id": "5005", "type": "Sugar" }, { "id": "5003", "type": "Chocolate" }, { "id": "5004", "type": "Maple" } ] }, { "id": "0003", "type": "donut", "name": "Old Fashioned", "ppu": 0.55, "batters": { "batter": [ { "id": "1001", "type": "Regular" }, { "id": "1002", "type": "Chocolate" } ] }, "topping": [ { "id": "5001", "type": "None" }, { "id": "5002", "type": "Glazed" }, { "id": "5003", "type": "Chocolate" }, { "id": "5004", "type": "Maple" } ] } ]""" result = list() stream = io.StringIO(json) tokenizer = JsonTokenizer(stream) JsonDecoder() \ .tokenizer(tokenizer) \ .root_class_name('Example') \ .predicate('batters') \ .event_handler(lambda e, p: result.append((e, p))) \ .decode() self.assertEqual(len(result), 3) self.assertEqual(len(result[0][0].batter), 4) self.assertEqual(len(result[1][0].batter), 1) self.assertEqual(len(result[2][0].batter), 2) (obj, path) = result[0] self.assertEqual(path, 'batters') self.assertEqual(type(obj).__name__, 'Batters') self.assertEqual(type(obj.batter).__name__, 'list') self.assertEqual(len(obj.batter), 4) self.assertEqual(type(obj.batter[0]).__name__, 'Batter') self.assertEqual(obj.batter[0].id, '1001') self.assertEqual(obj.batter[0].type, 'Regular') self.assertEqual(obj.batter[1].id, '1002') self.assertEqual(obj.batter[1].type, 'Chocolate') self.assertEqual(obj.batter[2].id, '1003') self.assertEqual(obj.batter[2].type, 'Blueberry') self.assertEqual(obj.batter[3].id, '1004') self.assertEqual(obj.batter[3].type, "Devil's Food")
def test_decode_subscribe_on_simple_value(self): json = """ [ { "id": "0001", "type": "donut", "name": "Cake", "ppu": 0.55, "batters": { "batter": [ { "id": "1001", "type": "Regular" }, { "id": "1002", "type": "Chocolate" }, { "id": "1003", "type": "Blueberry" }, { "id": "1004", "type": "Devil's Food" } ] }, "topping": [ { "id": "5001", "type": "None" }, { "id": "5002", "type": "Glazed" }, { "id": "5005", "type": "Sugar" }, { "id": "5007", "type": "Powdered Sugar" }, { "id": "5006", "type": "Chocolate with Sprinkles" }, { "id": "5003", "type": "Chocolate" }, { "id": "5004", "type": "Maple" } ] }, { "id": "0002", "type": "donut", "name": "Raised", "ppu": 0.56, "batters": { "batter": [ { "id": "1001", "type": "Regular" } ] }, "topping": [ { "id": "5001", "type": "None" }, { "id": "5002", "type": "Glazed" }, { "id": "5005", "type": "Sugar" }, { "id": "5003", "type": "Chocolate" }, { "id": "5004", "type": "Maple" } ] }, { "id": "0003", "type": "donut", "name": "Old Fashioned", "ppu": 0.57, "batters": { "batter": [ { "id": "1001", "type": "Regular" }, { "id": "1002", "type": "Chocolate" } ] }, "topping": [ { "id": "5001", "type": "None" }, { "id": "5002", "type": "Glazed" }, { "id": "5003", "type": "Chocolate" }, { "id": "5004", "type": "Maple" } ] } ]""" result = list() stream = io.StringIO(json) tokenizer = JsonTokenizer(stream) JsonDecoder() \ .tokenizer(tokenizer) \ .root_class_name('Example') \ .predicate('ppu') \ .event_handler(lambda e, p: result.append((e, p))) \ .decode() self.assertEqual(len(result), 3) self.assertEqual(result[0][0], '0.55') self.assertEqual(result[1][0], '0.56') self.assertEqual(result[2][0], '0.57')
print('processed ' + str(counter[0]) + ' objects') counter[0] += 1 if __name__ == '__main__': url = '/prust/wikipedia-movie-data/master/movies.json' endpoint = HTTPSConnection('raw.githubusercontent.com', '443', context=ssl._create_unverified_context()) try: ts1 = datetime.datetime.now().timestamp() print('start time=' + str(ts1)) endpoint.request('GET', url) response = endpoint.getresponse() tokenizer = JsonTokenizer(response, 'ISO-8859-1', 65536) JsonDecoder()\ .tokenizer(tokenizer)\ .root_class_name('Data')\ .event_handler(lambda e, p: handle_event(e, p, counter))\ .predicate('genres') \ .with_snake_cased_props()\ .decode() ts2 = datetime.datetime.now().timestamp() print('end time=' + str(ts2)) print('delta=' + str(ts2 - ts1)) finally: endpoint.close() print('total events count=' + str(counter[0]))