def combine_dtypes(cls, list_of_dtypes, column_names):
# Compute dtypes by getting collecting and combining all of the partitions. The
# reported dtypes from differing rows can be different based on the inference in
# the limited data seen by each worker. We use pandas to compute the exact dtype
# over the whole column for each column.
dtypes = (pandas.concat(ray.get(list_of_dtypes), axis=1).apply(
lambda row: find_common_type(row.values), axis=1).squeeze(axis=0))
dtypes.index = column_names
return dtypes
def combine_dtypes(cls, list_of_dtypes, column_names):
"""Describes how data types should be combined when they do not match.
Args:
list_of_dtypes: A list of pandas Series with the data types.
column_names: The names of the columns that the data types map to.
Returns:
A pandas Series containing the finalized data types.
"""
# Compute dtypes by getting collecting and combining all of the partitions. The
# reported dtypes from differing rows can be different based on the inference in
# the limited data seen by each worker. We use pandas to compute the exact dtype
# over the whole column for each column.
dtypes = (pandas.concat(list_of_dtypes, axis=1).apply(
lambda row: find_common_type(row.values), axis=1).squeeze(axis=0))
dtypes.index = column_names
return dtypes
def transpose(self):
"""Transpose the index and columns of this dataframe.
Returns:
A new dataframe.
"""
new_partitions = self._frame_mgr_cls.lazy_map_partitions(
self._partitions, lambda df: df.T).T
new_dtypes = pandas.Series(
np.full(len(self.index), find_common_type(self.dtypes.values)),
index=self.index,
)
return self.__constructor__(
new_partitions,
self.columns,
self.index,
self._column_widths,
self._row_lengths,
dtypes=new_dtypes,
)
def _compute_map_reduce_metadata(self, axis, new_parts):
if axis == 0:
columns = self.columns
index = ["__reduced__"]
new_lengths = [1]
new_widths = self._column_widths
new_dtypes = self._dtypes
else:
columns = ["__reduced__"]
index = self.index
new_lengths = self._row_lengths
new_widths = [1]
if self._dtypes is not None:
new_dtypes = pandas.Series(
np.full(1, find_common_type(self.dtypes.values)),
index=["__reduced__"],
)
else:
new_dtypes = self._dtypes
return self.__constructor__(new_parts, index, columns, new_lengths,
new_widths, new_dtypes)
def dtype_builder(df):
return df.apply(lambda col: find_common_type(col.values), axis=0)