def test_arrow_nomask(): array = awkward1.Array([1.1, 2.2, 3.3, 4.4, None]) assert str(awkward1.type(awkward1.from_arrow( awkward1.to_arrow(array)))) == "5 * ?float64" assert str( awkward1.type(awkward1.from_arrow(awkward1.to_arrow( array[:-1])))) == "4 * ?float64"
def test_arrow_nonnullable_table(): x = pyarrow.array([1, 2, 3]) y = pyarrow.array([1.1, 2.2, 3.3]) table = pyarrow.Table.from_arrays([x], ["x"]) if hasattr(pyarrow, "column"): table2 = table.add_column(1, pyarrow.column(pyarrow.field("y", y.type, False), numpy.array([1.1, 2.2, 3.3]))) else: table2 = table.add_column(1, "y", y) assert awkward1.to_list(awkward1.from_arrow(table2, highlevel=False)) == [{"x": 1, "y": 1.1}, {"x": 2, "y": 2.2}, {"x": 3, "y": 3.3}]
def test_arrow_batch(): a = pyarrow.RecordBatch.from_arrays( [pyarrow.array([1.1, 2.2, 3.3, None, 5.5]), pyarrow.array([[1, 2, 3], [], [4, 5], [None], [6]]), pyarrow.array([{"x": 1, "y": 1.1}, {"x": 2, "y": 2.2}, {"x": 3, "y": 3.3}, {"x": 4, "y": None}, {"x": 5, "y": 5.5}]), pyarrow.array([{"x": 1, "y": 1.1}, None, None, {"x": 4, "y": None}, {"x": 5, "y": 5.5}]), pyarrow.array([[{"x": 1, "y": 1.1}, {"x": 2, "y": 2.2}, {"x": 3, "y": 3.3}], [], [{"x": 4, "y": None}, {"x": 5, "y": 5.5}], [None], [{"x": 6, "y": 6.6}]])], ["a", "b", "c", "d", "e"]) assert awkward1.to_list(awkward1.from_arrow(a, highlevel=False)) == [{"a": 1.1, "b": [1, 2, 3], "c": {"x": 1, "y": 1.1}, "d": {"x": 1, "y": 1.1}, "e": [{"x": 1, "y": 1.1}, {"x": 2, "y": 2.2}, {"x": 3, "y": 3.3}]}, {"a": 2.2, "b": [], "c": {"x": 2, "y": 2.2}, "d": None, "e": []}, {"a": 3.3, "b": [4, 5], "c": {"x": 3, "y": 3.3}, "d": None, "e": [{"x": 4, "y": None}, {"x": 5, "y": 5.5}]}, {"a": None, "b": [None], "c": {"x": 4, "y": None}, "d":{"x": 4, "y": None}, "e": [None]}, {"a": 5.5, "b": [6], "c": {"x": 5, "y": 5.5}, "d": {"x": 5, "y": 5.5}, "e": [{"x": 6, "y": 6.6}]}]
def __call__(self, version=1): df = self._obj if version == 0: return awkward0.Table( dict((c, df[c].ak(version=version)) for c in df.columns) ) elif version == 1: return awkward1.from_arrow(pyarrow.Table.from_pandas(df)) else: raise RuntimeError( "What version of awkward do you want? Specify `version=0` or `1`." )
def test_fromarrow(): boolarray = awkward1.layout.NumpyArray(numpy.array([True, True, True, False, False, True, False, True, False, True])) assert awkward1.to_list(awkward1.from_arrow(awkward1.to_arrow(boolarray), highlevel=False)) == awkward1.to_list(boolarray) content = awkward1.layout.NumpyArray( numpy.array([0.0, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9, 10.10])) assert awkward1.to_list(awkward1.from_arrow(awkward1.to_arrow(content), highlevel=False)) == awkward1.to_list(content) offsets = awkward1.layout.Index64(numpy.array([0, 3, 3, 5, 6, 10, 10])) listoffsetarray = awkward1.layout.ListOffsetArray64(offsets, content) assert awkward1.to_list(awkward1.from_arrow(awkward1.to_arrow(listoffsetarray), highlevel=False)) == awkward1.to_list(listoffsetarray) regulararray = awkward1.layout.RegularArray(listoffsetarray, 2) assert awkward1.to_list(awkward1.from_arrow(awkward1.to_arrow(regulararray), highlevel=False)) == awkward1.to_list(regulararray) content1 = awkward1.layout.NumpyArray(numpy.array([1, 2, 3, 4, 5])) content2 = awkward1.layout.NumpyArray( numpy.array([1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9])) offsets = awkward1.layout.Index32(numpy.array([0, 3, 3, 5, 6, 9])) recordarray = awkward1.layout.RecordArray( [content1, listoffsetarray, content2, content1], keys=["one", "chonks", "2", "wonky"]) assert awkward1.to_list(awkward1.from_arrow(awkward1.to_arrow(recordarray), highlevel=False)) == awkward1.to_list(recordarray) content0 = awkward1.Array([[1.1, 2.2, 3.3], [], [4.4, 5.5]]).layout content = awkward1.Array( ["one", "two", "three", "four", "five", "six", "seven", "eight", "nine"]).layout tags = awkward1.layout.Index8( numpy.array([1, 1, 0, 0, 1, 0, 1, 1], dtype=numpy.int8)) index = awkward1.layout.Index32( numpy.array([0, 1, 0, 1, 2, 2, 4, 3], dtype=numpy.int32)) array = awkward1.layout.UnionArray8_32(tags, index, [content0, content]) assert awkward1.to_list(awkward1.from_arrow(awkward1.to_arrow(array), highlevel=False)) == awkward1.to_list(array)
def test_arrow_toarrow_string(): a = awkward1.from_iter(["one", "two", "three"], highlevel=False) assert awkward1.to_list(awkward1.from_arrow(awkward1.to_arrow(a), highlevel=False)) == awkward1.to_list(a) a = awkward1.from_iter([["one", "two", "three"], [], ["four", "five"]], highlevel=False) assert awkward1.to_list(awkward1.from_arrow(awkward1.to_arrow(a), highlevel=False)) == awkward1.to_list(a) if hasattr(pyarrow.BinaryArray, 'from_buffers'): a = awkward1.from_iter([b"one", b"two", b"three"], highlevel=False) assert awkward1.to_list(awkward1.from_arrow(awkward1.to_arrow(a), highlevel=False)) == [b"one", b"two", b"three"] a = awkward1.from_iter([[b"one", b"two", b"three"], [], [b"four", b"five"]], highlevel=False) assert awkward1.to_list(awkward1.from_arrow(awkward1.to_arrow(a), highlevel=False)) == [[b"one", b"two", b"three"], [], [b"four", b"five"]] else: a = awkward1.from_iter([b"one", b"two", b"three"], highlevel=False) assert awkward1.to_list(awkward1.from_arrow(awkward1.to_arrow(a), highlevel=False)) == ["one", "two", "three"] a = awkward1.from_iter([[b"one", b"two", b"three"], [], [b"four", b"five"]], highlevel=False) assert awkward1.to_list(awkward1.from_arrow(awkward1.to_arrow(a), highlevel=False)) == [["one", "two", "three"], [], ["four", "five"]]
def pandas_series_to_awkward(series, version=1): values = series.values if "fletcher" not in str(values.dtype).lower(): if version == 1: return awkward1.from_numpy(values) else: return np.array(values, copy=False) array_arrow = values.data if version == 0: array = awkward0.fromarrow(array_arrow) if "MaskedArray" in str(type(array)): array = array._content[array.boolmask()] elif version == 1: array = awkward1.from_arrow(array_arrow) else: raise RuntimeError( "What version of awkward do you want? Specify `version=0` or `1`." ) return array
def test_arrow_null_dictarray(): a = pyarrow.DictionaryArray.from_arrays(pyarrow.array([0, 0, 2, 2, 1, 0, 2, 1, 1]), pyarrow.array(["one", None, "three"])) assert awkward1.to_list(awkward1.from_arrow(a, highlevel=False)) == ["one", "one", "three", "three", None, "one", "three", None, None]
def test_arrow_union_sparse(): a = pyarrow.UnionArray.from_sparse(pyarrow.array([0, 1, 0, 0, 1], type=pyarrow.int8()), [pyarrow.array([0.0, 1.1, 2.2, 3.3, 4.4]), pyarrow.array([True, True, False, True, False])]) assert awkward1.to_list(awkward1.from_arrow(a, highlevel=False)) == [0.0, True, 2.2, 3.3, False]
def test_arrow_union_dense_null_null(): a = pyarrow.UnionArray.from_dense(pyarrow.array([0, 1, 0, 0, 0, 1, 1], type=pyarrow.int8()), pyarrow.array([0, 0, 1, 2, 3, 1, 2], type=pyarrow.int32()), [pyarrow.array([0.0, 1.1, None, 3.3]), pyarrow.array([True, None, False])]) assert awkward1.to_list(awkward1.from_arrow(a, highlevel=False)) == [0.0, True, 1.1, None, 3.3, None, False]
def test_arrow_chunked_strings(): a = pyarrow.chunked_array([pyarrow.array(["one", "two", "three", "four", "five"]), pyarrow.array(["six", "seven", "eight"])]) assert awkward1.to_list(awkward1.from_arrow(a, highlevel=False)) == ["one", "two", "three", "four", "five", "six", "seven", "eight"]
def test_arrow_null_nested_strings_null(): a = pyarrow.array([["one", "two", None], [], None, ["four", "five"]]) assert awkward1.to_list(awkward1.from_arrow(a, highlevel=False)) == [["one", "two", None], [], None, ["four", "five"]]
def test_arrow_nested_nested_array(): a = pyarrow.array([[[1.1, 2.2], [3.3], []], [], [[4.4, 5.5]]]) assert awkward1.to_list(awkward1.from_arrow(a, highlevel=False)) == [[[1.1, 2.2], [3.3], []], [], [[4.4, 5.5]]]
def test_arrow_binary_null(): a = pyarrow.array([b"one", b"two", None, b"four", b"five"]) assert awkward1.to_list(awkward1.from_arrow(a, highlevel=False)) == [b"one", b"two", None, b"four", b"five"]
def test_arrow_struct_nested_null(): a = pyarrow.array([{"x": [], "y": 1.1}, {"x": [2], "y": 2.2}, {"x": [None, 3], "y": 3.3}]) assert awkward1.to_list(awkward1.from_arrow(a, highlevel=False)) == [{"x": [], "y": 1.1}, {"x": [2], "y": 2.2}, {"x": [None, 3], "y": 3.3}]
def test_arrow_null_nested_struct_nested_null(): a = pyarrow.array([[{"x": [], "y": 1.1}, {"x": [2], "y": 2.2}, {"x": [None, 3], "y": 3.3}], None, [], [{"x": [4, 4, 4], "y": 4.4}, {"x": [5, 5, 5, 5], "y": 5.5}]]) assert awkward1.to_list(awkward1.from_arrow(a, highlevel=False)) == [[{"x": [], "y": 1.1}, {"x": [2], "y": 2.2}, {"x": [None, 3], "y": 3.3}], None, [], [{"x": [4, 4, 4], "y": 4.4}, {"x": [5, 5, 5, 5], "y": 5.5}]]
def test_arrow_coverage100(): a = awkward1.from_iter([True, True, False, False, True, False, True, False], highlevel=False) assert awkward1.to_arrow(a).to_pylist() == awkward1.to_list(a) a = awkward1.layout.ListOffsetArray32(awkward1.layout.Index32(numpy.array([0, 5, 10], "i4")), awkward1.layout.NumpyArray(numpy.frombuffer(b"hellothere", "u1"), parameters={"__array__": "bytes"}), parameters={"__array__": "bytestring"}) assert [x for x in awkward1.to_arrow(a)] == [b"hello", b"there"] a = awkward1.layout.ByteMaskedArray(awkward1.layout.Index8(numpy.array([False, True, False, False, True, True])), awkward1.layout.ListOffsetArray32(awkward1.layout.Index32(numpy.array([0, 5, 10, 15, 20, 25, 30], "i4")), awkward1.layout.NumpyArray(numpy.frombuffer(b"hellotherehellotherehellothere", "u1"), parameters={"__array__": "bytes"}), parameters={"__array__": "bytestring"}), valid_when=False) assert [x for x in awkward1.to_arrow(a)] == [b"hello", None, b"hello", b"there", None, None] a = awkward1.layout.ByteMaskedArray(awkward1.layout.Index8(numpy.array([False, True])), awkward1.layout.ListOffsetArray32(awkward1.layout.Index32(numpy.array([0, 5, 10], "i4")), awkward1.layout.NumpyArray(numpy.frombuffer(b"hellothere", "u1"), parameters={"__array__": "bytes"}), parameters={"__array__": "bytestring"}), valid_when=False) assert [x for x in awkward1.to_arrow(a)] == [b"hello", None] a = awkward1.layout.IndexedOptionArray32(awkward1.layout.Index32(numpy.array([-1, 1, -1, 0, 0, -1], "i4")), awkward1.layout.ListOffsetArray32(awkward1.layout.Index32(numpy.array([0, 5, 10], "i4")), awkward1.layout.NumpyArray(numpy.frombuffer(b"hellothere", "u1"), parameters={"__array__": "bytes"}), parameters={"__array__": "bytestring"})) assert [x for x in awkward1.to_arrow(a)] == [None, b"there", None, b"hello", b"hello", None] a = awkward1.layout.ListOffsetArray32(awkward1.layout.Index32(numpy.array([0, 5, 10], "i4")), awkward1.layout.NumpyArray(numpy.frombuffer(b"hellothere", "u1"), parameters={"__array__": "chars"}), parameters={"__array__": "string"}) assert [x for x in awkward1.to_arrow(a)] == ["hello", "there"] a = awkward1.layout.ByteMaskedArray(awkward1.layout.Index8(numpy.array([False, True, False, False, True, True])), awkward1.layout.ListOffsetArray32(awkward1.layout.Index32(numpy.array([0, 5, 10, 15, 20, 25, 30], "i4")), awkward1.layout.NumpyArray(numpy.frombuffer(b"hellotherehellotherehellothere", "u1"), parameters={"__array__": "chars"}), parameters={"__array__": "string"}), valid_when=False) assert [x for x in awkward1.to_arrow(a)] == ["hello", None, "hello", "there", None, None] assert awkward1.to_list(awkward1.from_arrow(awkward1.to_arrow(a))) == ["hello", None, "hello", "there", None, None] a = awkward1.layout.ByteMaskedArray(awkward1.layout.Index8(numpy.array([False, True, False, False, True, True])), awkward1.layout.ListOffsetArray64(awkward1.layout.Index64(numpy.array([0, 5, 10, 15, 20, 25, 30], "i8")), awkward1.layout.NumpyArray(numpy.frombuffer(b"hellotherehellotherehellothere", "u1"), parameters={"__array__": "chars"}), parameters={"__array__": "string"}), valid_when=False) assert [x for x in awkward1.to_arrow(a)] == ["hello", None, "hello", "there", None, None] assert awkward1.to_list(awkward1.from_arrow(awkward1.to_arrow(a))) == ["hello", None, "hello", "there", None, None] a = awkward1.layout.ByteMaskedArray(awkward1.layout.Index8(numpy.array([False, True, False, False, True, True])), awkward1.layout.ListOffsetArray64(awkward1.layout.Index64(numpy.array([0, 5, 10, 15, 20, 25, 30], "i8")), awkward1.layout.NumpyArray(numpy.frombuffer(b"hellotherehellotherehellothere", "u1"), parameters={"__array__": "bytes"}), parameters={"__array__": "bytestring"}), valid_when=False) assert [x for x in awkward1.to_arrow(a)] == [b"hello", None, b"hello", b"there", None, None] assert awkward1.to_list(awkward1.from_arrow(awkward1.to_arrow(a))) == [b"hello", None, b"hello", b"there", None, None] a = awkward1.layout.ByteMaskedArray(awkward1.layout.Index8(numpy.array([False, True])), awkward1.layout.ListOffsetArray32(awkward1.layout.Index32(numpy.array([0, 5, 10], "i4")), awkward1.layout.NumpyArray(numpy.frombuffer(b"hellothere", "u1"), parameters={"__array__": "chars"}), parameters={"__array__": "string"}), valid_when=False) assert [x for x in awkward1.to_arrow(a)] == ["hello", None] a = awkward1.layout.IndexedOptionArray32(awkward1.layout.Index32(numpy.array([-1, 1, -1, 0, 0, -1], "i4")), awkward1.layout.ListOffsetArray32(awkward1.layout.Index32(numpy.array([0, 5, 10], "i4")), awkward1.layout.NumpyArray(numpy.frombuffer(b"hellothere", "u1"), parameters={"__array__": "chars"}), parameters={"__array__": "string"})) assert [x for x in awkward1.to_arrow(a)] == [None, "there", None, "hello", "hello", None] a = awkward1.layout.ListOffsetArray32(awkward1.layout.Index32(numpy.array([0, 5, 10], "i4")), awkward1.layout.NumpyArray(numpy.frombuffer(b"hellothere", "u1"))) assert [x for x in awkward1.to_arrow(a)] == [[104, 101, 108, 108, 111], [116, 104, 101, 114, 101]] a = awkward1.layout.ByteMaskedArray(awkward1.layout.Index8(numpy.array([False, True, False, False, True, True])), awkward1.layout.ListOffsetArray32(awkward1.layout.Index32(numpy.array([0, 5, 10, 15, 20, 25, 30], "i4")), awkward1.layout.NumpyArray(numpy.frombuffer(b"hellotherehellotherehellothere", "u1"))), valid_when=False) assert [x for x in awkward1.to_arrow(a)] == [[104, 101, 108, 108, 111], None, [104, 101, 108, 108, 111], [116, 104, 101, 114, 101], None, None] a = awkward1.layout.ByteMaskedArray(awkward1.layout.Index8(numpy.array([False, True])), awkward1.layout.ListOffsetArray32(awkward1.layout.Index32(numpy.array([0, 5, 10], "i4")), awkward1.layout.NumpyArray(numpy.frombuffer(b"hellothere", "u1"))), valid_when=False) assert [x for x in awkward1.to_arrow(a)] == [[104, 101, 108, 108, 111], None] a = awkward1.layout.IndexedOptionArray32(awkward1.layout.Index32(numpy.array([-1, 1, -1, 0, 0, -1], "i4")), awkward1.layout.ListOffsetArray32(awkward1.layout.Index32(numpy.array([0, 5, 10], "i4")), awkward1.layout.NumpyArray(numpy.frombuffer(b"hellothere", "u1")))) assert [x for x in awkward1.to_arrow(a)] == [None, [116, 104, 101, 114, 101], None, [104, 101, 108, 108, 111], [104, 101, 108, 108, 111], None] a = awkward1.layout.IndexedOptionArray32(awkward1.layout.Index32(numpy.array([-1, 1, -1, 0, 0, -1], "i4")), awkward1.layout.RegularArray(awkward1.layout.NumpyArray(numpy.array([1.1, 2.2, 3.3, 4.4, 5.5, 6.6])), 3)) assert awkward1.to_arrow(a).to_pylist() == [None, [4.4, 5.5, 6.6], None, [1.1, 2.2, 3.3], [1.1, 2.2, 3.3], None] a = awkward1.layout.IndexedOptionArray32(awkward1.layout.Index32(numpy.array([-1, 1, -1, 0, 0, -1, 1, -1], "i4")), awkward1.layout.RegularArray(awkward1.layout.NumpyArray(numpy.array([1.1, 2.2, 3.3, 4.4, 5.5, 6.6])), 3)) assert awkward1.to_arrow(a).to_pylist() == [None, [4.4, 5.5, 6.6], None, [1.1, 2.2, 3.3], [1.1, 2.2, 3.3], None, [4.4, 5.5, 6.6], None] a = awkward1.layout.IndexedOptionArray64(awkward1.layout.Index64(numpy.array([-1, 1, -1, 0, 0, -1, 1, -1], "i8")), awkward1.layout.RegularArray(awkward1.layout.NumpyArray(numpy.array([1.1, 2.2, 3.3, 4.4, 5.5, 6.6])), 3)) assert awkward1.to_arrow(a).to_pylist() == [None, [4.4, 5.5, 6.6], None, [1.1, 2.2, 3.3], [1.1, 2.2, 3.3], None, [4.4, 5.5, 6.6], None] a = awkward1.layout.ByteMaskedArray(awkward1.layout.Index8(numpy.array([True, True, True, True, False, False])), awkward1.layout.IndexedOptionArray32(awkward1.layout.Index32(numpy.array([-1, 1, -1, 0, 0, -1], "i4")), awkward1.layout.RegularArray(awkward1.layout.NumpyArray(numpy.array([1.1, 2.2, 3.3, 4.4, 5.5, 6.6])), 3)), valid_when=True) assert awkward1.to_arrow(a).to_pylist() == [None, [4.4, 5.5, 6.6], None, [1.1, 2.2, 3.3], None, None] a = awkward1.layout.UnmaskedArray(awkward1.layout.ListOffsetArray32(awkward1.layout.Index32(numpy.array([0, 5, 10], "i4")), awkward1.layout.NumpyArray(numpy.frombuffer(b"hellothere", "u1")))) assert [x for x in awkward1.to_arrow(a)] == [[104, 101, 108, 108, 111], [116, 104, 101, 114, 101]] a = pyarrow.array(["one", "two", "three", "two", "two", "one", "three", "one"]).dictionary_encode() b = awkward1.from_arrow(a, highlevel=False) assert isinstance(b, awkward1._util.indexedtypes) assert awkward1.to_list(b) == ["one", "two", "three", "two", "two", "one", "three", "one"] a = awkward1.Array([[1.1, 2.2, 3.3], [], None, [4.4, 5.5]]) assert awkward1.to_list(awkward1.from_arrow(awkward1.to_arrow(a))) == [[1.1, 2.2, 3.3], [], None, [4.4, 5.5]] a = awkward1.layout.ByteMaskedArray(awkward1.layout.Index8(numpy.array([False, False, False, True, True, False, False])), awkward1.layout.NumpyArray(numpy.array([1.1, 2.2, 3.3, 999, 314, 4.4, 5.5])), valid_when=False) assert awkward1.to_arrow(a).to_pylist() == [1.1, 2.2, 3.3, None, None, 4.4, 5.5] a = awkward1.layout.ByteMaskedArray(awkward1.layout.Index8(numpy.array([False, False, False, True, True, False, False])), awkward1.from_iter([b"hello", b"", b"there", b"yuk", b"", b"o", b"hellothere"], highlevel=False), valid_when=False) assert awkward1.to_arrow(a).to_pylist() == [b"hello", b"", b"there", None, None, b"o", b"hellothere"] a = awkward1.layout.ByteMaskedArray(awkward1.layout.Index8([True, True, False, True]), awkward1.from_iter([[1.1, 2.2, 3.3], [], [999], [4.4, 5.5]], highlevel=False), valid_when=True) assert awkward1.to_list(awkward1.from_arrow(awkward1.to_arrow(a))) == [[1.1, 2.2, 3.3], [], None, [4.4, 5.5]] a = awkward1.from_iter([[1, 2, 3], [], [4, 5], 999, 123], highlevel=False) assert awkward1.to_arrow(a).to_pylist() == [[1, 2, 3], [], [4, 5], 999, 123] assert awkward1.to_list(awkward1.from_arrow(awkward1.to_arrow(a))) == [[1, 2, 3], [], [4, 5], 999, 123] b = awkward1.layout.ByteMaskedArray(awkward1.layout.Index8(numpy.array([True, True, False, False, True])), a, valid_when=True) assert awkward1.to_arrow(b).to_pylist() == [[1, 2, 3], [], None, None, 123] assert awkward1.to_list(awkward1.from_arrow(awkward1.to_arrow(b))) == [[1, 2, 3], [], None, None, 123] content1 = awkward1.from_iter([1.1, 2.2, 3.3, 4.4, 5.5], highlevel=False) content2 = awkward1.layout.NumpyArray(numpy.array([], dtype=numpy.int32)) a = awkward1.layout.UnionArray8_32(awkward1.layout.Index8(numpy.array([0, 0, 0, 0, 0], "i1")), awkward1.layout.Index32(numpy.array([0, 1, 2, 3, 4], "i4")), [content1, content2]) assert awkward1.to_list(a) == [1.1, 2.2, 3.3, 4.4, 5.5] assert awkward1.to_arrow(a).to_pylist() == [1.1, 2.2, 3.3, 4.4, 5.5] assert awkward1.to_list(awkward1.from_arrow(awkward1.to_arrow(a))) == [1.1, 2.2, 3.3, 4.4, 5.5] a = pyarrow.UnionArray.from_sparse(pyarrow.array([0, 0, 0, 0, 0], type=pyarrow.int8()), [pyarrow.array([0.0, 1.1, None, 3.3, 4.4]), pyarrow.array([True, None, False, True, False])]) assert awkward1.to_list(awkward1.from_arrow(a, highlevel=False)) == [0.0, 1.1, None, 3.3, 4.4] uniontype = pyarrow.union([pyarrow.field("0", pyarrow.list_(pyarrow.float64())), pyarrow.field("1", pyarrow.float64())], "sparse", [0, 1]) a = pyarrow.Array.from_buffers( uniontype, 5, [pyarrow.py_buffer(numpy.array([3], "u1")), pyarrow.py_buffer(numpy.array([0, 1, 0, 1, 1], "i1")), None], children=[pyarrow.array([[0.0, 1.1, 2.2], [], [3.3, 4.4], [5.5], [6.6, 7.7, 8.8, 9.9]]), pyarrow.array([0.0, 1.1, 2.2, 3.3, 4.4])]) assert a.to_pylist() == [[0.0, 1.1, 2.2], 1.1, None, None, None] assert awkward1.to_list(awkward1.from_arrow(a)) == [[0.0, 1.1, 2.2], 1.1, None, None, None] a = pyarrow.chunked_array([pyarrow.array([1.1, 2.2, 3.3, 4.4, 5.5])]) assert awkward1.to_list(awkward1.from_arrow(a, highlevel=False)) == [1.1, 2.2, 3.3, 4.4, 5.5]
def __from_arrow__(cls, arrow_array): from .series import AwkwardSeries ak_arr = ak.from_arrow(arrow_array) return AwkwardSeries(ak_arr)
def test_arrow_null_struct_null(): a = pyarrow.array([{"x": 1, "y": 1.1}, None, {"x": 2, "y": None}, {"x": 3, "y": 3.3}]) assert awkward1.to_list(awkward1.from_arrow(a, highlevel=False)) == [{"x": 1, "y": 1.1}, None, {"x": 2, "y": None}, {"x": 3, "y": 3.3}]
def test_arrow_chunked_array(): a = pyarrow.chunked_array([pyarrow.array([1.1, 2.2, 3.3, 4.4, 5.5]), pyarrow.array([], pyarrow.float64()), pyarrow.array([6.6, 7.7, 8.8])]) assert awkward1.to_list(awkward1.from_arrow(a, highlevel=False)) == [1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8]
def test_arrow_null_nested_array_null(): a = pyarrow.array([[1.1, 2.2, None], [], None, [4.4, 5.5]]) assert awkward1.to_list(awkward1.from_arrow(a, highlevel=False)) == [[1.1, 2.2, None], [], None, [4.4, 5.5]]
def test_arrow_boolean(): a = pyarrow.array([True, True, False, False, True]) assert awkward1.to_list(awkward1.from_arrow(a, highlevel=False)) == [True, True, False, False, True]
def test_arrow_strings_null(): if not awkward1._util.py27: a = pyarrow.array(["one", "two", None, u"fo\u2014ur", "five"]) assert awkward1.to_list(awkward1.from_arrow(a, highlevel=False)) == ["one", "two", None, u"fo\u2014ur", "five"]
def test_arrow_chunked_struct(): t = pyarrow.struct({"x": pyarrow.int64(), "y": pyarrow.float64()}) a = pyarrow.chunked_array([pyarrow.array([{"x": 1, "y": 1.1}, {"x": 2, "y": 2.2}, {"x": 3, "y": 3.3}], t), pyarrow.array([], t), pyarrow.array([{"x": 4, "y": 4.4}, {"x": 5, "y": 5.5}], t)]) assert awkward1.to_list(awkward1.from_arrow(a, highlevel=False)) == [{"x": 1, "y": 1.1}, {"x": 2, "y": 2.2}, {"x": 3, "y": 3.3}, {"x": 4, "y": 4.4}, {"x": 5, "y": 5.5}]
def test_arrow_array_null(): a = pyarrow.array([1.1, 2.2, 3.3, None, 4.4, 5.5]) assert awkward1.to_list(awkward1.from_arrow(a, highlevel=False)) == [1.1, 2.2, 3.3, None, 4.4, 5.5]
def test_arrow_null_nested_struct_null(): a = pyarrow.array([[{"x": 1, "y": 1.1}, {"x": 2, "y": None}, {"x": 3, "y": 3.3}], None, [], [{"x": 4, "y": 4.4}, {"x": 5, "y": 5.5}]]) assert awkward1.to_list(awkward1.from_arrow(a, highlevel=False)) == [[{"x": 1, "y": 1.1}, {"x": 2, "y": None}, {"x": 3, "y": 3.3}], None, [], [{"x": 4, "y": 4.4}, {"x": 5, "y": 5.5}]]
def __from_arrow__(cls, arr): ak_arr = ak.from_arrow(arr) return cls(ak_arr)