def col_to_format(c): return format if dtypes[c] == 'f' and format else array_default_format( dtypes[c])
def dataframe_to_thrift_struct(df, name, roffset, coffset, rows, cols, format): """ :type df: pandas.core.frame.DataFrame :type name: str :type coffset: int :type roffset: int :type rows: int :type cols: int :type format: str """ original_df = df dim = len(df.axes) num_rows = df.shape[0] num_cols = df.shape[1] if dim > 1 else 1 array_chunk = GetArrayResponse() array_chunk.slice = name array_chunk.rows = num_rows array_chunk.cols = num_cols array_chunk.type = "" array_chunk.max = "0" array_chunk.min = "0" format = format.replace("%", "") if not format: if num_rows > 0 and num_cols == 1: # series or data frame with one column try: kind = df.dtype.kind except AttributeError: try: kind = df.dtypes[0].kind except (IndexError, KeyError): kind = "O" format = array_default_format(kind) else: format = array_default_format(DEFAULT_DF_FORMAT) array_chunk.format = "%" + format if (rows, cols) == (-1, -1): rows, cols = num_rows, num_cols elif (rows, cols) == (0, 0): # return header only r = min(num_rows, DATAFRAME_HEADER_LOAD_MAX_SIZE) c = min(num_cols, DATAFRAME_HEADER_LOAD_MAX_SIZE) array_chunk.headers = header_data_to_thrift_struct(r, c, [""] * num_cols, [(0, 0)] * num_cols, lambda x: DEFAULT_DF_FORMAT, original_df, dim) array_chunk.data = array_data_to_thrift_struct(rows, cols, None, format) return array_chunk rows = min(rows, MAXIMUM_ARRAY_SIZE) cols = min(cols, MAXIMUM_ARRAY_SIZE, num_cols) # need to precompute column bounds here before slicing! col_bounds = [None] * cols dtypes = [None] * cols if dim > 1: for col in range(cols): dtype = df.dtypes.iloc[coffset + col].kind dtypes[col] = dtype if dtype in NUMPY_NUMERIC_TYPES and df.size != 0: cvalues = df.iloc[:, coffset + col] bounds = (cvalues.min(), cvalues.max()) else: bounds = (0, 0) col_bounds[col] = bounds else: dtype = df.dtype.kind dtypes[0] = dtype col_bounds[0] = (df.min(), df.max()) if dtype in NUMPY_NUMERIC_TYPES and df.size != 0 else (0, 0) df = df.iloc[roffset: roffset + rows, coffset: coffset + cols] if dim > 1 else df.iloc[roffset: roffset + rows] rows = df.shape[0] cols = df.shape[1] if dim > 1 else 1 def col_to_format(c): return get_column_formatter_by_type(format, dtypes[c]) iat = df.iat if dim == 1 or len(df.columns.unique()) == len(df.columns) else df.iloc def formatted_row_elements(row): return get_formatted_row_elements(row, iat, dim, cols, format, dtypes) array_chunk.headers = header_data_to_thrift_struct(rows, cols, dtypes, col_bounds, col_to_format, df, dim) array_chunk.data = array_data_to_thrift_struct(rows, cols, formatted_row_elements, format) return array_chunk
def dataframe_to_thrift_struct(df, name, roffset, coffset, rows, cols, format): """ :type df: pandas.core.frame.DataFrame :type name: str :type coffset: int :type roffset: int :type rows: int :type cols: int :type format: str """ dim = len(df.axes) num_rows = df.shape[0] num_cols = df.shape[1] if dim > 1 else 1 array_chunk = GetArrayResponse() array_chunk.slice = name array_chunk.rows = num_rows array_chunk.cols = num_cols array_chunk.type = "" array_chunk.max = "0" array_chunk.min = "0" format = format.replace("%", "") if not format: if num_rows > 0 and num_cols == 1: # series or data frame with one column try: kind = df.dtype.kind except AttributeError: try: kind = df.dtypes[0].kind except IndexError: kind = "O" format = array_default_format(kind) else: format = array_default_format("f") array_chunk.format = "%" + format if (rows, cols) == (-1, -1): rows, cols = num_rows, num_cols rows = min(rows, MAXIMUM_ARRAY_SIZE) cols = min(cols, MAXIMUM_ARRAY_SIZE, num_cols) # need to precompute column bounds here before slicing! col_bounds = [None] * cols dtypes = [None] * cols if dim > 1: for col in range(cols): dtype = df.dtypes.iloc[coffset + col].kind dtypes[col] = dtype if dtype in NUMPY_NUMERIC_TYPES: cvalues = df.iloc[:, coffset + col] bounds = (cvalues.min(), cvalues.max()) else: bounds = (0, 0) col_bounds[col] = bounds else: dtype = df.dtype.kind dtypes[0] = dtype col_bounds[0] = (df.min(), df.max()) if dtype in NUMPY_NUMERIC_TYPES else (0, 0) df = df.iloc[roffset: roffset + rows, coffset: coffset + cols] if dim > 1 else df.iloc[roffset: roffset + rows] rows = df.shape[0] cols = df.shape[1] if dim > 1 else 1 def col_to_format(c): return format if dtypes[c] in NUMPY_NUMERIC_TYPES and format else array_default_format(dtypes[c]) iat = df.iat if dim == 1 or len(df.columns.unique()) == len(df.columns) else df.iloc array_chunk.headers = header_data_to_thrift_struct(rows, cols, dtypes, col_bounds, col_to_format, df, dim) array_chunk.data = array_data_to_thrift_struct(rows, cols, lambda r: (("%" + col_to_format(c)) % (iat[r, c] if dim > 1 else iat[r]) for c in range(cols)), format) return array_chunk
def col_to_format(c): return format if dtypes[c] in NUMPY_NUMERIC_TYPES and format else array_default_format(dtypes[c])
def col_to_format(c): return format if dtypes[c] == 'f' and format else array_default_format(dtypes[c])