def test_standard_scaler():
"""Tests basic StandardScaler functionality."""
col_a = [-1, 0, 1, 2]
col_b = [1, 1, 5, 5]
col_c = [1, 1, 1, None]
in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b, "C": col_c})
ds = ray.data.from_pandas(in_df)
scaler = StandardScaler(["B", "C"])
# Transform with unfitted preprocessor.
with pytest.raises(PreprocessorNotFittedException):
scaler.transform(ds)
# Fit data.
scaler.fit(ds)
assert scaler.stats_ == {
"mean(B)": 3.0,
"mean(C)": 1.0,
"std(B)": 2.0,
"std(C)": 0.0,
}
# Transform data.
transformed = scaler.transform(ds)
out_df = transformed.to_pandas()
processed_col_a = col_a
processed_col_b = [-1.0, -1.0, 1.0, 1.0]
processed_col_c = [0.0, 0.0, 0.0, None]
expected_df = pd.DataFrame.from_dict({
"A": processed_col_a,
"B": processed_col_b,
"C": processed_col_c
})
assert out_df.equals(expected_df)
# Transform batch.
pred_col_a = [1, 2, 3]
pred_col_b = [3, 5, 7]
pred_col_c = [0, 1, 2]
pred_in_df = pd.DataFrame.from_dict({
"A": pred_col_a,
"B": pred_col_b,
"C": pred_col_c
})
pred_out_df = scaler.transform_batch(pred_in_df)
pred_processed_col_a = pred_col_a
pred_processed_col_b = [0.0, 1.0, 2.0]
pred_processed_col_c = [-1.0, 0.0, 1.0]
pred_expected_df = pd.DataFrame.from_dict({
"A": pred_processed_col_a,
"B": pred_processed_col_b,
"C": pred_processed_col_c,
})
assert pred_out_df.equals(pred_expected_df)
import pandas as pd
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
data_raw = load_breast_cancer()
dataset_df = pd.DataFrame(data_raw["data"], columns=data_raw["feature_names"])
dataset_df["target"] = data_raw["target"]
train_df, test_df = train_test_split(dataset_df, test_size=0.3)
train_dataset = ray.data.from_pandas(train_df)
valid_dataset = ray.data.from_pandas(test_df)
test_dataset = ray.data.from_pandas(test_df.drop("target", axis=1))
# Create a preprocessor to scale some columns
columns_to_scale = ["mean radius", "mean texture"]
preprocessor = StandardScaler(columns=columns_to_scale)
# __air_xgb_preprocess_end__
# __air_xgb_train_start__
from ray.train.xgboost import XGBoostTrainer
# XGBoost specific params
params = {
"tree_method": "approx",
"objective": "binary:logistic",
"eval_metric": ["logloss", "error"],
}
num_workers = 2
use_gpu = False # use GPUs if detected.
def test_chain():
"""Tests basic Chain functionality."""
col_a = [-1, -1, 1, 1]
col_b = [1, 1, 1, None]
col_c = ["sunday", "monday", "tuesday", "tuesday"]
in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b, "C": col_c})
ds = ray.data.from_pandas(in_df)
def udf(df):
df["A"] *= 2
return df
batch_mapper = BatchMapper(fn=udf)
imputer = SimpleImputer(["B"])
scaler = StandardScaler(["A", "B"])
encoder = LabelEncoder("C")
chain = Chain(scaler, imputer, encoder, batch_mapper)
# Fit data.
chain.fit(ds)
assert imputer.stats_ == {
"mean(B)": 0.0,
}
assert scaler.stats_ == {
"mean(A)": 0.0,
"mean(B)": 1.0,
"std(A)": 1.0,
"std(B)": 0.0,
}
assert encoder.stats_ == {
"unique_values(C)": {
"monday": 0,
"sunday": 1,
"tuesday": 2
}
}
# Transform data.
transformed = chain.transform(ds)
out_df = transformed.to_pandas()
processed_col_a = [-2.0, -2.0, 2.0, 2.0]
processed_col_b = [0.0, 0.0, 0.0, 0.0]
processed_col_c = [1, 0, 2, 2]
expected_df = pd.DataFrame.from_dict({
"A": processed_col_a,
"B": processed_col_b,
"C": processed_col_c
})
assert out_df.equals(expected_df)
# Transform batch.
pred_col_a = [1, 2, None]
pred_col_b = [0, None, 2]
pred_col_c = ["monday", "tuesday", "wednesday"]
pred_in_df = pd.DataFrame.from_dict({
"A": pred_col_a,
"B": pred_col_b,
"C": pred_col_c
})
pred_out_df = chain.transform_batch(pred_in_df)
pred_processed_col_a = [2, 4, None]
pred_processed_col_b = [-1.0, 0.0, 1.0]
pred_processed_col_c = [0, 2, None]
pred_expected_df = pd.DataFrame.from_dict({
"A": pred_processed_col_a,
"B": pred_processed_col_b,
"C": pred_processed_col_c,
})
assert pred_out_df.equals(pred_expected_df)