def combine_dtypes(cls, list_of_dtypes, column_names): # Compute dtypes by getting collecting and combining all of the partitions. The # reported dtypes from differing rows can be different based on the inference in # the limited data seen by each worker. We use pandas to compute the exact dtype # over the whole column for each column. dtypes = (pandas.concat(ray.get(list_of_dtypes), axis=1).apply( lambda row: find_common_type(row.values), axis=1).squeeze(axis=0)) dtypes.index = column_names return dtypes
def combine_dtypes(cls, list_of_dtypes, column_names): """Describes how data types should be combined when they do not match. Args: list_of_dtypes: A list of pandas Series with the data types. column_names: The names of the columns that the data types map to. Returns: A pandas Series containing the finalized data types. """ # Compute dtypes by getting collecting and combining all of the partitions. The # reported dtypes from differing rows can be different based on the inference in # the limited data seen by each worker. We use pandas to compute the exact dtype # over the whole column for each column. dtypes = (pandas.concat(list_of_dtypes, axis=1).apply( lambda row: find_common_type(row.values), axis=1).squeeze(axis=0)) dtypes.index = column_names return dtypes
def transpose(self): """Transpose the index and columns of this dataframe. Returns: A new dataframe. """ new_partitions = self._frame_mgr_cls.lazy_map_partitions( self._partitions, lambda df: df.T).T new_dtypes = pandas.Series( np.full(len(self.index), find_common_type(self.dtypes.values)), index=self.index, ) return self.__constructor__( new_partitions, self.columns, self.index, self._column_widths, self._row_lengths, dtypes=new_dtypes, )
def _compute_map_reduce_metadata(self, axis, new_parts): if axis == 0: columns = self.columns index = ["__reduced__"] new_lengths = [1] new_widths = self._column_widths new_dtypes = self._dtypes else: columns = ["__reduced__"] index = self.index new_lengths = self._row_lengths new_widths = [1] if self._dtypes is not None: new_dtypes = pandas.Series( np.full(1, find_common_type(self.dtypes.values)), index=["__reduced__"], ) else: new_dtypes = self._dtypes return self.__constructor__(new_parts, index, columns, new_lengths, new_widths, new_dtypes)
def dtype_builder(df): return df.apply(lambda col: find_common_type(col.values), axis=0)