def test_arrow_all_null_text_column(self): dataframe, columns = arrow_table_to_dataframe( arrow_table( {"A": pyarrow.array(["a", "b", None, "c"])}, columns=[atypes.Column("A", atypes.ColumnType.Text())], ) ) assert_frame_equal(dataframe, pd.DataFrame({"A": ["a", "b", np.nan, "c"]})) self.assertEqual(columns, [Column("A", ColumnType.TEXT())])
def test_arrow_uint8_column(self): dataframe, columns = arrow_table_to_dataframe( arrow_table( {"A": pyarrow.array([1, 2, 3, 253], type=pyarrow.uint8())}, columns=[atypes.Column("A", ColumnType.Number("{:,d}"))], )) assert_frame_equal(dataframe, pd.DataFrame({"A": [1, 2, 3, 253]}, dtype=np.uint8)) self.assertEqual(columns, [Column("A", ColumnType.Number("{:,d}"))])
def test_arrow_category_column(self): atable = arrow_table({ "A": pyarrow.DictionaryArray.from_arrays( pyarrow.array([0, 1, None, 0], type=pyarrow.int8()), pyarrow.array(["A", "B"], type=pyarrow.string()), ) }) dataframe, columns = arrow_table_to_dataframe(atable) self.assertEqual(columns, [Column("A", ColumnType.Text())]) assert_frame_equal( dataframe, pd.DataFrame({"A": ["A", "B", None, "A"]}, dtype="category"))
def test_arrow_all_null_category_column(self): atable = arrow_table( { "A": pyarrow.DictionaryArray.from_arrays( pyarrow.array([None], type=pyarrow.int8()), pyarrow.array([], type=pyarrow.string()), ) } ) dataframe, columns = arrow_table_to_dataframe(atable) self.assertEqual(columns, [Column("A", ColumnType.TEXT())]) assert_frame_equal( dataframe, pd.DataFrame({"A": [None]}, dtype=str).astype("category") )
def test_arrow_timestamp_column(self): dataframe, columns = arrow_table_to_dataframe( arrow_table( { "A": pyarrow.array( [dt.fromisoformat("2019-09-17T21:21:00.123456"), None], type=pyarrow.timestamp(unit="ns", tz=None), ) }, [atypes.Column("A", ColumnType.Timestamp())], )) assert_frame_equal( dataframe, pd.DataFrame({"A": ["2019-09-17T21:21:00.123456Z", None]}, dtype="datetime64[ns]"), ) self.assertEqual(columns, [Column("A", ColumnType.Timestamp())])
def test_arrow_table_reuse_string_memory(self): dataframe, _ = arrow_table_to_dataframe(arrow_table({"A": ["x", "x"]})) self.assertIs(dataframe["A"][0], dataframe["A"][1])