def process_pandas(self, source: pd.DataFrame): self.protocol = "pandas" col_names = list(source.columns.values) col_types = [ utils._from_numpy_dtype(dtype) for dtype in source.dtypes.values ] table = pa.Table.from_pandas(source, preserve_index=False) sink = pa.BufferOutputStream() with pa.ipc.new_stream(sink, table.schema) as writer: writer.write_table(table) buf = sink.getvalue() self.deduced_properties = list(zip(col_names, col_types)) self.source = bytes(memoryview(buf))
def process_pandas(self, source: pd.DataFrame): self.protocol = "pandas" check_argument(len(source.shape) == 2) self.row_num = source.shape[0] self.column_num = source.shape[1] # Only support a subset of data types. check_argument(source.dtypes.values[0] in (np.dtype("int64"), np.dtype("long"))) for dtype in source.dtypes.values: check_argument( dtype in (np.dtype("int64"), np.dtype("long"), np.dtype("float64")) ) col_names = list(source.columns.values) col_types = [utils._from_numpy_dtype(dtype) for dtype in source.dtypes.values] self.deduced_properties = list(zip(col_names, col_types)) self.property_bytes = [source[name].values.tobytes("F") for name in col_names]
def process_numpy(self, source: Sequence[np.ndarray]): self.protocol = "numpy" self.row_num = source[0].shape[0] self.column_num = len(source) # Only support a subset of data types. check_argument(source[0].dtype in (np.dtype("int64"), np.dtype("long"))) for col in source: check_argument(col.dtype in ( np.dtype("int64"), np.dtype("long"), np.dtype("float64"), )) col_names = ["f%s" % i for i in range(self.column_num)] col_types = [utils._from_numpy_dtype(col.dtype) for col in source] self.deduced_properties = list(zip(col_names, col_types)) self.property_bytes = [col.tobytes("F") for col in source]