def test_arrow_all_null_text_column(self):
dataframe, columns = arrow_table_to_dataframe(
arrow_table(
{"A": pyarrow.array(["a", "b", None, "c"])},
columns=[atypes.Column("A", atypes.ColumnType.Text())],
)
)
assert_frame_equal(dataframe, pd.DataFrame({"A": ["a", "b", np.nan, "c"]}))
self.assertEqual(columns, [Column("A", ColumnType.TEXT())])
def test_arrow_uint8_column(self):
dataframe, columns = arrow_table_to_dataframe(
arrow_table(
{"A": pyarrow.array([1, 2, 3, 253], type=pyarrow.uint8())},
columns=[atypes.Column("A", ColumnType.Number("{:,d}"))],
))
assert_frame_equal(dataframe,
pd.DataFrame({"A": [1, 2, 3, 253]}, dtype=np.uint8))
self.assertEqual(columns, [Column("A", ColumnType.Number("{:,d}"))])
def test_arrow_category_column(self):
atable = arrow_table({
"A":
pyarrow.DictionaryArray.from_arrays(
pyarrow.array([0, 1, None, 0], type=pyarrow.int8()),
pyarrow.array(["A", "B"], type=pyarrow.string()),
)
})
dataframe, columns = arrow_table_to_dataframe(atable)
self.assertEqual(columns, [Column("A", ColumnType.Text())])
assert_frame_equal(
dataframe,
pd.DataFrame({"A": ["A", "B", None, "A"]}, dtype="category"))
def test_arrow_all_null_category_column(self):
atable = arrow_table(
{
"A": pyarrow.DictionaryArray.from_arrays(
pyarrow.array([None], type=pyarrow.int8()),
pyarrow.array([], type=pyarrow.string()),
)
}
)
dataframe, columns = arrow_table_to_dataframe(atable)
self.assertEqual(columns, [Column("A", ColumnType.TEXT())])
assert_frame_equal(
dataframe, pd.DataFrame({"A": [None]}, dtype=str).astype("category")
)
def test_arrow_timestamp_column(self):
dataframe, columns = arrow_table_to_dataframe(
arrow_table(
{
"A":
pyarrow.array(
[dt.fromisoformat("2019-09-17T21:21:00.123456"), None],
type=pyarrow.timestamp(unit="ns", tz=None),
)
},
[atypes.Column("A", ColumnType.Timestamp())],
))
assert_frame_equal(
dataframe,
pd.DataFrame({"A": ["2019-09-17T21:21:00.123456Z", None]},
dtype="datetime64[ns]"),
)
self.assertEqual(columns, [Column("A", ColumnType.Timestamp())])
def test_arrow_table_reuse_string_memory(self):
dataframe, _ = arrow_table_to_dataframe(arrow_table({"A": ["x", "x"]}))
self.assertIs(dataframe["A"][0], dataframe["A"][1])