def transform(self, col_selector: ColumnSelector,
df: DataFrameType) -> DataFrameType:
for name in col_selector.names:
column = df[name]
if _is_list_dtype(column):
transformed = np.log(
_flatten_list_column_values(column).astype(np.float32) + 1)
df[name] = _encode_list_column(column, transformed)
else:
df[name] = np.log(column.astype(np.float32) + 1)
return df
def _encode(
name,
storage_name,
path,
df,
cat_cache,
na_sentinel=-1,
freq_threshold=0,
search_sorted=False,
buckets=None,
encode_type="joint",
cat_names=None,
max_size=0,
):
if isinstance(buckets, int):
buckets = {name: buckets for name in cat_names}
# this is to apply freq_hashing logic
if max_size:
freq_threshold = 1
value = None
selection_l = name if isinstance(name, list) else [name]
selection_r = name if isinstance(name, list) else [storage_name]
list_col = _is_list_col(selection_l, df)
if path:
read_pq_func = _read_parquet_dispatch(df)
if cat_cache is not None:
cat_cache = (
cat_cache if isinstance(cat_cache, str) else cat_cache.get(storage_name, "disk")
)
if len(df):
with get_worker_cache("cats") as cache:
value = fetch_table_data(
cache,
path,
columns=selection_r,
cache=cat_cache,
cats_only=True,
reader=read_pq_func,
)
else:
value = read_pq_func(path, columns=selection_r)
value.index.name = "labels"
value.reset_index(drop=False, inplace=True)
if value is None:
value = type(df)()
for c in selection_r:
typ = df[selection_l[0]].dtype if len(selection_l) == 1 else df[c].dtype
value[c] = df._constructor_sliced([None], dtype=typ)
value.index.name = "labels"
value.reset_index(drop=False, inplace=True)
if not search_sorted:
if list_col:
codes = _flatten_list_column(df[selection_l[0]])
codes["order"] = _arange(len(codes), like_df=df)
else:
codes = type(df)({"order": _arange(len(df), like_df=df)}, index=df.index)
for c in selection_l:
codes[c] = df[c].copy()
if buckets and storage_name in buckets:
na_sentinel = _hash_bucket(df, buckets, selection_l, encode_type=encode_type)
# apply frequency hashing
if freq_threshold and buckets and storage_name in buckets:
merged_df = codes.merge(
value, left_on=selection_l, right_on=selection_r, how="left"
).sort_values("order")
merged_df.reset_index(drop=True, inplace=True)
max_id = merged_df["labels"].max()
merged_df["labels"].fillna(
df._constructor_sliced(na_sentinel + max_id + 1), inplace=True
)
labels = merged_df["labels"].values
# only do hashing
elif buckets and storage_name in buckets:
labels = na_sentinel
# no hashing
else:
na_sentinel = 0
labels = codes.merge(
value, left_on=selection_l, right_on=selection_r, how="left"
).sort_values("order")["labels"]
labels.fillna(na_sentinel, inplace=True)
labels = labels.values
else:
# Use `searchsorted` if we are using a "full" encoding
if list_col:
labels = value[selection_r].searchsorted(
df[selection_l[0]].list.leaves, side="left", na_position="first"
)
else:
labels = value[selection_r].searchsorted(
df[selection_l], side="left", na_position="first"
)
labels[labels >= len(value[selection_r])] = na_sentinel
if list_col:
labels = _encode_list_column(df[selection_l[0]], labels)
return labels