def transform(self, col_selector: ColumnSelector,
df: DataFrameType) -> DataFrameType:
new_df = type(df)()
tmp = "__tmp__" # Temporary column for sorting
df[tmp] = _arange(len(df), like_df=df, dtype="int32")
cat_names, multi_col_group = nvt_cat._get_multicolumn_names(
col_selector, df.columns, self.name_sep)
_read_pq_func = _read_parquet_dispatch(df)
for name in cat_names:
new_part = type(df)()
storage_name = self.storage_name.get(name, name)
name = multi_col_group.get(name, name)
path = self.categories[storage_name]
selection_l = list(name) if isinstance(name, tuple) else [name]
selection_r = list(name) if isinstance(name,
tuple) else [storage_name]
stat_df = nvt_cat._read_groupby_stat_df(path, storage_name,
self.cat_cache,
_read_pq_func)
tran_df = df[selection_l + [tmp]].merge(stat_df,
left_on=selection_l,
right_on=selection_r,
how="left")
tran_df = tran_df.sort_values(tmp)
tran_df.drop(columns=selection_l + [tmp], inplace=True)
new_cols = [c for c in tran_df.columns if c not in new_df.columns]
new_part = tran_df[new_cols].reset_index(drop=True)
new_df = _concat_columns([new_df, new_part])
df.drop(columns=[tmp], inplace=True)
return new_df
def _op_group_logic(self, cat_group, gdf, y_mean, fit_folds, group_ind):
# Define name of new TE column
if isinstance(self.out_col, list):
if group_ind >= len(self.out_col):
raise ValueError("out_col and cat_groups are different sizes.")
out_col = self.out_col[group_ind]
out_col = [out_col] if isinstance(out_col, str) else out_col
# ToDo Test
if len(out_col) != len(self.target):
raise ValueError("out_col and target are different sizes.")
else:
out_col = self._make_te_name(cat_group)
# Initialize new data
_read_pq_func = _read_parquet_dispatch(gdf)
new_gdf = cudf.DataFrame()
tmp = "__tmp__"
if fit_folds:
# Groupby Aggregation for each fold
cols = ["__fold__"] + cat_group
storage_name_folds = nvt_cat._make_name(*cols, sep=self.name_sep)
path_folds = self.stats[storage_name_folds]
agg_each_fold = nvt_cat._read_groupby_stat_df(
path_folds, storage_name_folds, self.cat_cache, _read_pq_func)
agg_each_fold.columns = cols + ["count_y"] + [
x + "_sum_y" for x in self.target
]
else:
cols = cat_group
# Groupby Aggregation for all data
storage_name_all = nvt_cat._make_name(*cat_group, sep=self.name_sep)
path_all = self.stats[storage_name_all]
agg_all = nvt_cat._read_groupby_stat_df(path_all, storage_name_all,
self.cat_cache, _read_pq_func)
agg_all.columns = cat_group + ["count_y_all"] + [
x + "_sum_y_all" for x in self.target
]
if fit_folds:
agg_each_fold = agg_each_fold.merge(agg_all,
on=cat_group,
how="left")
agg_each_fold["count_y_all"] = agg_each_fold[
"count_y_all"] - agg_each_fold["count_y"]
for i, x in enumerate(self.target):
agg_each_fold[x + "_sum_y_all"] = (
agg_each_fold[x + "_sum_y_all"] -
agg_each_fold[x + "_sum_y"])
agg_each_fold[out_col[i]] = (
agg_each_fold[x + "_sum_y_all"] + self.p_smooth *
y_mean[x]) / (agg_each_fold["count_y_all"] + self.p_smooth)
agg_each_fold = agg_each_fold.drop(
["count_y_all", "count_y"] +
[x + "_sum_y" for x in self.target] +
[x + "_sum_y_all" for x in self.target],
axis=1,
)
tran_gdf = gdf[cols + [tmp]].merge(agg_each_fold,
on=cols,
how="left")
del agg_each_fold
else:
for i, x in enumerate(self.target):
agg_all[out_col[i]] = (
agg_all[x + "_sum_y_all"] + self.p_smooth * y_mean[x]) / (
agg_all["count_y_all"] + self.p_smooth)
agg_all = agg_all.drop(["count_y_all"] +
[x + "_sum_y_all" for x in self.target],
axis=1)
tran_gdf = gdf[cols + [tmp]].merge(agg_all, on=cols, how="left")
del agg_all
# TODO: There is no need to perform the `agg_each_fold.merge(agg_all, ...)` merge
# for every partition. We can/should cache the result for better performance.
for i, x in enumerate(self.target):
tran_gdf[out_col[i]] = tran_gdf[out_col[i]].fillna(y_mean[x])
if self.out_dtype is not None:
tran_gdf[out_col] = tran_gdf[out_col].astype(self.out_dtype)
tran_gdf = tran_gdf.sort_values(tmp, ignore_index=True)
tran_gdf.drop(columns=cols + [tmp], inplace=True)
new_cols = [c for c in tran_gdf.columns if c not in new_gdf.columns]
new_gdf[new_cols] = tran_gdf[new_cols]
# Make sure we are preserving the index of gdf
new_gdf.index = gdf.index
return new_gdf
def _encode(
name,
storage_name,
path,
df,
cat_cache,
na_sentinel=-1,
freq_threshold=0,
search_sorted=False,
buckets=None,
encode_type="joint",
cat_names=None,
max_size=0,
):
if isinstance(buckets, int):
buckets = {name: buckets for name in cat_names}
# this is to apply freq_hashing logic
if max_size:
freq_threshold = 1
value = None
selection_l = name if isinstance(name, list) else [name]
selection_r = name if isinstance(name, list) else [storage_name]
list_col = _is_list_col(selection_l, df)
if path:
read_pq_func = _read_parquet_dispatch(df)
if cat_cache is not None:
cat_cache = (
cat_cache if isinstance(cat_cache, str) else cat_cache.get(storage_name, "disk")
)
if len(df):
with get_worker_cache("cats") as cache:
value = fetch_table_data(
cache,
path,
columns=selection_r,
cache=cat_cache,
cats_only=True,
reader=read_pq_func,
)
else:
value = read_pq_func(path, columns=selection_r)
value.index.name = "labels"
value.reset_index(drop=False, inplace=True)
if value is None:
value = type(df)()
for c in selection_r:
typ = df[selection_l[0]].dtype if len(selection_l) == 1 else df[c].dtype
value[c] = df._constructor_sliced([None], dtype=typ)
value.index.name = "labels"
value.reset_index(drop=False, inplace=True)
if not search_sorted:
if list_col:
codes = _flatten_list_column(df[selection_l[0]])
codes["order"] = _arange(len(codes), like_df=df)
else:
codes = type(df)({"order": _arange(len(df), like_df=df)}, index=df.index)
for c in selection_l:
codes[c] = df[c].copy()
if buckets and storage_name in buckets:
na_sentinel = _hash_bucket(df, buckets, selection_l, encode_type=encode_type)
# apply frequency hashing
if freq_threshold and buckets and storage_name in buckets:
merged_df = codes.merge(
value, left_on=selection_l, right_on=selection_r, how="left"
).sort_values("order")
merged_df.reset_index(drop=True, inplace=True)
max_id = merged_df["labels"].max()
merged_df["labels"].fillna(
df._constructor_sliced(na_sentinel + max_id + 1), inplace=True
)
labels = merged_df["labels"].values
# only do hashing
elif buckets and storage_name in buckets:
labels = na_sentinel
# no hashing
else:
na_sentinel = 0
labels = codes.merge(
value, left_on=selection_l, right_on=selection_r, how="left"
).sort_values("order")["labels"]
labels.fillna(na_sentinel, inplace=True)
labels = labels.values
else:
# Use `searchsorted` if we are using a "full" encoding
if list_col:
labels = value[selection_r].searchsorted(
df[selection_l[0]].list.leaves, side="left", na_position="first"
)
else:
labels = value[selection_r].searchsorted(
df[selection_l], side="left", na_position="first"
)
labels[labels >= len(value[selection_r])] = na_sentinel
if list_col:
labels = _encode_list_column(df[selection_l[0]], labels)
return labels