def test_nullable_list_depths_strings(self):
with ParquetFile(
open("tests/samples/nullable-list-depths-strings.parquet",
"rb")) as f:
self.assertEqual(tojson(f()),
[{
"list0": "one",
"list1": [],
"list2": [],
"list3": []
}, {
"list0": "two",
"list1": ["two"],
"list2": [[]],
"list3": [[]]
}, {
"list0": "three",
"list1": ["two", "three"],
"list2": [["three"]],
"list3": [[[]]]
}, {
"list0": "four",
"list1": ["two", "three", "four"],
"list2": [["three", "four"]],
"list3": [[["four"]]]
}, {
"list0": "five",
"list1": ["two", "three", "four", "five"],
"list2": [["three", "four", "five"]],
"list3": [[["four", "five"]]]
}])
def test_nullable_depths(self):
with ParquetFile(open("tests/samples/nullable-depths.parquet",
"rb")) as f:
# for line in f():
# print tojson(line)
self.assertEqual(tojson(f()), [{
"whatever": {
"r0": [{
"r1": [{
"r2": [0, 1, 2, 3]
}]
}]
}
}, {
"whatever": {
"r0": [{
"r1": [{
"r2": []
}]
}]
}
}, {
"whatever": {
"r0": [{
"r1": []
}]
}
}, {
"whatever": {
"r0": []
}
}, {
"whatever": None
}, {
"whatever": {
"r0": []
}
}, {
"whatever": {
"r0": [{
"r1": []
}]
}
}, {
"whatever": {
"r0": [{
"r1": [{
"r2": []
}]
}]
}
}, {
"whatever": {
"r0": [{
"r1": [{
"r2": [0, 1, 2, 3]
}]
}]
}
}])
def test_nullable_list_depths(self):
with ParquetFile(
open("tests/samples/nullable-list-depths.parquet", "rb")) as f:
self.assertEqual(tojson(f()), [{
"list0": 1,
"list1": [],
"list2": [],
"list3": []
}, {
"list0": 2,
"list1": [2],
"list2": [[]],
"list3": [[]]
}, {
"list0": 3,
"list1": [2, 3],
"list2": [[3]],
"list3": [[[]]]
}, {
"list0": 4,
"list1": [2, 3, 4],
"list2": [[3, 4]],
"list3": [[[4]]]
}, {
"list0": 5,
"list1": [2, 3, 4, 5],
"list2": [[3, 4, 5]],
"list3": [[[4, 5]]]
}])
def test_nonnullable_depths(self):
with ParquetFile(open("tests/samples/nonnullable-depths.parquet",
"rb")) as f:
self.assertEqual(tojson(f()), [{
"whatever": {
"r0": [{
"r1": [{
"r2": [0, 1, 2, 3]
}]
}]
}
}, {
"whatever": {
"r0": [{
"r1": [{
"r2": []
}]
}]
}
}, {
"whatever": {
"r0": [{
"r1": []
}]
}
}, {
"whatever": {
"r0": []
}
}, {
"whatever": {
"r0": []
}
}, {
"whatever": {
"r0": [{
"r1": []
}]
}
}, {
"whatever": {
"r0": [{
"r1": [{
"r2": []
}]
}]
}
}, {
"whatever": {
"r0": [{
"r1": [{
"r2": [0, 1, 2, 3]
}]
}]
}
}])
def test_list_lengths(self):
with ParquetFile(open("tests/samples/list-lengths.parquet",
"rb")) as f:
self.assertEqual(tojson(f()), [{
"list3": [[[0, 1, 2], [], [], [3, 4]]]
}, {
"list3": [[[5, 6]], [], [], [[7, 8]]]
}, {
"list3": [[[9, 10, 11], []], []]
}])
def test_nullable_record_primitives(self):
with ParquetFile(
open("tests/samples/nullable-record-primitives.parquet",
"rb")) as f:
self.assertEqual(tojson(f()), [{
"u1": None,
"u4": 1,
"u8": None,
"f4": 1.100000023841858,
"f8": None,
"raw": b"one",
"utf8": "one"
}, {
"u1": True,
"u4": None,
"u8": 2,
"f4": 2.200000047683716,
"f8": None,
"raw": None,
"utf8": None
}, {
"u1": None,
"u4": None,
"u8": 3,
"f4": None,
"f8": None,
"raw": b"three",
"utf8": None
}, {
"u1": False,
"u4": None,
"u8": 4,
"f4": None,
"f8": 4.4,
"raw": None,
"utf8": None
}, {
"u1": None,
"u4": 5,
"u8": None,
"f4": None,
"f8": 5.5,
"raw": None,
"utf8": "five"
}])
def test_record_primitives(self):
with ParquetFile(open("tests/samples/record-primitives.parquet",
"rb")) as f:
self.assertEqual(tojson(f()), [{
"u1": False,
"u4": 1,
"u8": 1,
"f4": 1.100000023841858,
"f8": 1.1,
"raw": b"one",
"utf8": "one"
}, {
"u1": True,
"u4": 2,
"u8": 2,
"f4": 2.200000047683716,
"f8": 2.2,
"raw": b"two",
"utf8": "two"
}, {
"u1": True,
"u4": 3,
"u8": 3,
"f4": 3.299999952316284,
"f8": 3.3,
"raw": b"three",
"utf8": "three"
}, {
"u1": False,
"u4": 4,
"u8": 4,
"f4": 4.400000095367432,
"f8": 4.4,
"raw": b"four",
"utf8": "four"
}, {
"u1": False,
"u4": 5,
"u8": 5,
"f4": 5.5,
"f8": 5.5,
"raw": b"five",
"utf8": "five"
}])
def test_list_depths_simple(self):
with ParquetFile(open("tests/samples/list-depths-simple.parquet",
"rb")) as f:
self.assertEqual(tojson(f()), [{
"list0": 1,
"list1": [1]
}, {
"list0": 2,
"list1": [1, 2]
}, {
"list0": 3,
"list1": [1, 2, 3]
}, {
"list0": 4,
"list1": [1, 2, 3, 4]
}, {
"list0": 5,
"list1": [1, 2, 3, 4, 5]
}])
def test_nullable_record_primitives_simple(self):
with ParquetFile(
open("tests/samples/nullable-record-primitives-simple.parquet",
"rb")) as f:
self.assertEqual(tojson(f()), [{
"u4": None,
"u8": 1
}, {
"u4": None,
"u8": 2
}, {
"u4": None,
"u8": 3
}, {
"u4": None,
"u8": 4
}, {
"u4": None,
"u8": 5
}])
def test_nullable_list_depths_records_list(self):
with ParquetFile(
open("tests/samples/nullable-list-depths-records-list.parquet",
"rb")) as f:
self.assertEqual(tojson(f()), [{
"list0": {
"one": 1,
"two": 1.100000023841858,
"three": []
},
"list1": [],
"list2": [],
"list3": []
}, {
"list0": {
"one": 2,
"two": 2.200000047683716,
"three": [2]
},
"list1": [{
"one": 2,
"two": 2.200000047683716,
"three": [2]
}],
"list2": [[]],
"list3": [[]]
}, {
"list0": {
"one": 3,
"two": 3.299999952316284,
"three": [3]
},
"list1": [{
"one": 2,
"two": 2.200000047683716,
"three": [2]
}, {
"one": 3,
"two": 3.299999952316284,
"three": [3]
}],
"list2": [[{
"one": 3,
"two": 3.299999952316284,
"three": [3]
}]],
"list3": [[[]]]
}, {
"list0": {
"one": 4,
"two": 4.400000095367432,
"three": [4, 4, 4]
},
"list1": [{
"one": 2,
"two": 2.200000047683716,
"three": [2]
}, {
"one": 3,
"two": 3.299999952316284,
"three": [3]
}, {
"one": 4,
"two": 4.400000095367432,
"three": [4, 4, 4]
}],
"list2": [[{
"one": 3,
"two": 3.299999952316284,
"three": [3]
}, {
"one": 4,
"two": 4.400000095367432,
"three": [4, 4, 4]
}]],
"list3": [[[{
"one": 4,
"two": 4.400000095367432,
"three": [4, 4, 4]
}]]]
}, {
"list0": {
"one": 5,
"two": 5.5,
"three": [5, 5, 5, 5]
},
"list1": [{
"one": 2,
"two": 2.200000047683716,
"three": [2]
}, {
"one": 3,
"two": 3.299999952316284,
"three": [3]
}, {
"one": 4,
"two": 4.400000095367432,
"three": [4, 4, 4]
}, {
"one": 5,
"two": 5.5,
"three": [5, 5, 5, 5]
}],
"list2": [[{
"one": 3,
"two": 3.299999952316284,
"three": [3]
}, {
"one": 4,
"two": 4.400000095367432,
"three": [4, 4, 4]
}, {
"one": 5,
"two": 5.5,
"three": [5, 5, 5, 5]
}]],
"list3": [[[{
"one": 4,
"two": 4.400000095367432,
"three": [4, 4, 4]
}, {
"one": 5,
"two": 5.5,
"three": [5, 5, 5, 5]
}]]]
}])
def test_list_depths_records(self):
with ParquetFile(
open("tests/samples/list-depths-records.parquet", "rb")) as f:
self.assertEqual(tojson(f()), [{
"list0": {
"one": 1,
"two": 1.100000023841858,
"three": "one"
},
"list1": [],
"list2": [],
"list3": []
}, {
"list0": {
"one": 2,
"two": 2.200000047683716,
"three": "two"
},
"list1": [{
"one": 2,
"two": 2.200000047683716,
"three": "two"
}],
"list2": [[]],
"list3": [[]]
}, {
"list0": {
"one": 3,
"two": 3.299999952316284,
"three": "three"
},
"list1": [{
"one": 2,
"two": 2.200000047683716,
"three": "two"
}, {
"one": 3,
"two": 3.299999952316284,
"three": "three"
}],
"list2": [[{
"one": 3,
"two": 3.299999952316284,
"three": "three"
}]],
"list3": [[[]]]
}, {
"list0": {
"one": 4,
"two": 4.400000095367432,
"three": "four"
},
"list1": [{
"one": 2,
"two": 2.200000047683716,
"three": "two"
}, {
"one": 3,
"two": 3.299999952316284,
"three": "three"
}, {
"one": 4,
"two": 4.400000095367432,
"three": "four"
}],
"list2": [[{
"one": 3,
"two": 3.299999952316284,
"three": "three"
}, {
"one": 4,
"two": 4.400000095367432,
"three": "four"
}]],
"list3": [[[{
"one": 4,
"two": 4.400000095367432,
"three": "four"
}]]]
}, {
"list0": {
"one": 5,
"two": 5.5,
"three": "five"
},
"list1": [{
"one": 2,
"two": 2.200000047683716,
"three": "two"
}, {
"one": 3,
"two": 3.299999952316284,
"three": "three"
}, {
"one": 4,
"two": 4.400000095367432,
"three": "four"
}, {
"one": 5,
"two": 5.5,
"three": "five"
}],
"list2": [[{
"one": 3,
"two": 3.299999952316284,
"three": "three"
}, {
"one": 4,
"two": 4.400000095367432,
"three": "four"
}, {
"one": 5,
"two": 5.5,
"three": "five"
}]],
"list3": [[[{
"one": 4,
"two": 4.400000095367432,
"three": "four"
}, {
"one": 5,
"two": 5.5,
"three": "five"
}]]]
}])
def test_nullable_levels(self):
with ParquetFile(open("tests/samples/nullable-levels.parquet",
"rb")) as f:
self.assertEqual(tojson(f()), [{
"whatever": {
"r0": {
"r1": {
"r2": {
"r3": 1
}
}
}
}
}, {
"whatever": {
"r0": {
"r1": {
"r2": {
"r3": None
}
}
}
}
}, {
"whatever": {
"r0": {
"r1": {
"r2": None
}
}
}
}, {
"whatever": {
"r0": None
}
}, {
"whatever": None
}, {
"whatever": {
"r0": None
}
}, {
"whatever": {
"r0": {
"r1": {
"r2": None
}
}
}
}, {
"whatever": {
"r0": {
"r1": {
"r2": {
"r3": None
}
}
}
}
}, {
"whatever": {
"r0": {
"r1": {
"r2": {
"r3": 1
}
}
}
}
}])
