def test_calders():
"""Test calders."""
data = DataTuple(
x=pd.DataFrame(np.linspace(0, 1, 100), columns=["x"]),
s=pd.DataFrame([1] * 75 + [0] * 25, columns=["s"]),
y=pd.DataFrame([1] * 50 + [0] * 25 + [1] * 10 + [0] * 15,
columns=["y"]),
name="TestData",
)
data, _ = em.train_test_split(data, train_percentage=1.0)
assert len(em.query_dt(data, "s == 0 & y == 0")) == 15
assert len(em.query_dt(data, "s == 0 & y == 1")) == 10
assert len(em.query_dt(data, "s == 1 & y == 0")) == 25
assert len(em.query_dt(data, "s == 1 & y == 1")) == 50
assert em.query_dt(data,
"s == 1 & y == 0").x.min().min() == approx(0.50,
abs=0.01)
calders: PreAlgorithm = Calders(preferable_class=1, disadvantaged_group=0)
new_train, new_test = calders.run(data, data.remove_y())
pd.testing.assert_frame_equal(new_test.x, data.x)
pd.testing.assert_frame_equal(new_test.s, data.s)
assert len(em.query_dt(new_train, "s == 0 & y == 0")) == 10
assert len(em.query_dt(new_train, "s == 0 & y == 1")) == 15
assert len(em.query_dt(new_train, "s == 1 & y == 0")) == 30
assert len(em.query_dt(new_train, "s == 1 & y == 1")) == 45
assert len(data) == len(new_train)
assert em.query_dt(new_train, "s == 1 & y == 1").x.min().min() == 0
assert em.query_dt(new_train,
"s == 1 & y == 0").x.min().min() == approx(0.45,
abs=0.01)
def test_apply_to_joined_df() -> None:
"""Tests apply_to_joined_df_function."""
datatup = DataTuple(
x=pd.DataFrame([3.0], columns=["a1"]),
s=pd.DataFrame([4.0], columns=["b2"]),
y=pd.DataFrame([6.0], columns=["c3"]),
name=None,
)
def _identity(x: pd.DataFrame):
return x
result = datatup.apply_to_joined_df(_identity)
pd.testing.assert_frame_equal(datatup.x, result.x)
pd.testing.assert_frame_equal(datatup.s, result.s)
pd.testing.assert_frame_equal(datatup.y, result.y)
def test_biased_split_sizes():
"""Test biased split sizes."""
data = DataTuple(
x=pd.DataFrame([0] * 1000, columns=["feat1-"]),
s=pd.DataFrame([1] * 750 + [0] * 250, columns=["sens="]),
y=pd.DataFrame([1] * 500 + [0] * 250 + [1] * 125 + [0] * 125,
columns=["label<"]),
name="TestData",
)
biased1, subset = em.get_biased_subset(data,
mixing_factor=0.0,
unbiased_pcnt=0.8)
assert len(biased1) < len(subset)
biased1, subset = em.get_biased_subset(data,
mixing_factor=0.0,
unbiased_pcnt=0.2)
assert len(biased1) > len(subset)
biased2, debiased = em.get_biased_and_debiased_subsets(data,
mixing_factor=0.0,
unbiased_pcnt=0.8,
fixed_unbiased=True)
assert len(biased2) < len(debiased)
biased2, debiased = em.get_biased_and_debiased_subsets(data,
mixing_factor=0.0,
unbiased_pcnt=0.2,
fixed_unbiased=True)
assert len(biased2) > len(debiased)
def test_concat():
"""Test concat."""
x: pd.DataFrame = pd.DataFrame(columns=["a"], data=[[1]])
s: pd.DataFrame = pd.DataFrame(columns=["b"], data=[[2]])
y: pd.DataFrame = pd.DataFrame(columns=["c"], data=[[3]])
data1 = DataTuple(x=x, s=s, y=y, name="test_data")
x = pd.DataFrame(columns=["a"], data=[[4]])
s = pd.DataFrame(columns=["b"], data=[[5]])
y = pd.DataFrame(columns=["c"], data=[[6]])
data2 = DataTuple(x=x, s=s, y=y, name="test_tuple")
data3 = em.concat_dt([data1, data2], axis="index", ignore_index=True)
pd.testing.assert_frame_equal(data3.x,
pd.DataFrame(columns=["a"], data=[[1], [4]]))
pd.testing.assert_frame_equal(data3.s,
pd.DataFrame(columns=["b"], data=[[2], [5]]))
pd.testing.assert_frame_equal(data3.y,
pd.DataFrame(columns=["c"], data=[[3], [6]]))
def test_dataset_name_none() -> None:
"""Tests that a DataTuple can be saved without the name property."""
datatup = DataTuple(
x=pd.DataFrame([3.0], columns=["a1"]),
s=pd.DataFrame([4.0], columns=["b2"]),
y=pd.DataFrame([6.0], columns=["c3"]),
name=None,
)
with TemporaryDirectory() as tmpdir:
tmp_path = Path(tmpdir)
path = tmp_path / "pytest.npz"
datatup.to_npz(path)
# reload from feather file
reloaded = DataTuple.from_npz(path)
assert reloaded.name is None
pd.testing.assert_frame_equal(datatup.x, reloaded.x)
pd.testing.assert_frame_equal(datatup.s, reloaded.s)
pd.testing.assert_frame_equal(datatup.y, reloaded.y)
def test_data_tuple_len() -> None:
"""Test DataTuple len property."""
datatup_unequal_len = DataTuple(
x=pd.DataFrame([3.0, 2.0], columns=["a1"]),
s=pd.DataFrame([4.0], columns=["b2"]),
y=pd.DataFrame([6.0], columns=["c3"]),
name=None,
)
with pytest.raises(AssertionError):
len(datatup_unequal_len)
datatup_equal_len = DataTuple(
x=pd.DataFrame([3.0, 2.0, 1.0], columns=["a1"]),
s=pd.DataFrame([4.0, 5.0, 9.0], columns=["b2"]),
y=pd.DataFrame([6.0, 4.2, 6.7], columns=["c3"]),
name=None,
)
assert len(datatup_equal_len) == 3
def _run_script_command(self, train_path, test_path, pred_path):
"""Check if the dataframes loaded from the files are the same as the original ones."""
del test_path
loaded = DataTuple.from_npz(train_path)
pd.testing.assert_frame_equal(data_tuple.x, loaded.x)
pd.testing.assert_frame_equal(data_tuple.s, loaded.s)
pd.testing.assert_frame_equal(data_tuple.y, loaded.y)
# write a file for the predictions
np.savez(pred_path, hard=np.load(train_path)["x"])
return ["-c", "pass"]
def test_query():
"""Test query."""
x: pd.DataFrame = pd.DataFrame(columns=["0a", "b"],
data=[[0, 1], [2, 3], [4, 5]])
s: pd.DataFrame = pd.DataFrame(columns=["c="], data=[[6], [7], [8]])
y: pd.DataFrame = pd.DataFrame(columns=["d"], data=[[9], [10], [11]])
data = DataTuple(x=x, s=s, y=y, name="test_data")
selected = em.query_dt(data, "`0a` == 0 & `c=` == 6 & d == 9")
pd.testing.assert_frame_equal(selected.x, x.head(1))
pd.testing.assert_frame_equal(selected.s, s.head(1))
pd.testing.assert_frame_equal(selected.y, y.head(1))
def simple_data() -> DataTuple:
"""Simple data for testing splitting methods."""
# visual representation of the data:
# s: ...111111111111111111111111111111111111111111111111111111111111110000000000000000000000000
# y: ...111111111111111111111111111111111111110000000000000000000000001111111111000000000000000
return DataTuple(
x=pd.DataFrame([0] * 1000, columns=["x"]),
s=pd.DataFrame([1] * 750 + [0] * 250, columns=["s"]),
y=pd.DataFrame([1] * 500 + [0] * 250 + [1] * 100 + [0] * 150,
columns=["y"]),
name="TestData",
)
def test_issue_431():
"""This issue highlighted that error would be raised due to not all values existing in subsets of the data."""
x = pd.DataFrame(np.random.randn(100), columns=["x"])
s = pd.DataFrame(np.random.randn(100), columns=["s"])
y = pd.DataFrame(np.random.randint(0, 5, 100), columns=["y"])
data = DataTuple(x=x, s=s, y=y)
train_test: Tuple[DataTuple, DataTuple] = train_test_split(data)
train, test = train_test
model: InAlgorithm = LR()
predictions: Prediction = model.run(train, test)
acc_per_sens = metric_per_sensitive_attribute(
predictions, test, TPR(pos_class=1,
labels=list(range(y.nunique()[0]))))
print(acc_per_sens)
def test_biased_split_nonbinary():
"""Test biased split nonbinary."""
# generate data that uses -1 and 1 instead of 0 and 1 for s and y
data = DataTuple(
x=pd.DataFrame([0] * 1000, columns=["feat1-"]),
s=pd.DataFrame([1] * 750 + [-1] * 250, columns=["sens="]),
y=pd.DataFrame([1] * 500 + [-1] * 250 + [1] * 125 + [-1] * 125,
columns=["label<"]),
name="TestData",
)
biased1, subset = em.get_biased_subset(data,
mixing_factor=0.5,
unbiased_pcnt=0.5)
assert len(biased1) == approx(len(subset), abs=4)
def test_simple_saving() -> None:
"""Tests that a DataTuple can be saved."""
data_tuple = DataTuple(
x=pd.DataFrame({"a1": np.array([3.2, 9.4, np.nan, 0.0])}),
s=pd.DataFrame({
"b1": np.array([18, -3, int(1e10)]),
"b2": np.array([1, 1, -1]),
"b3": np.array([0, 1, 0]),
}),
y=pd.DataFrame({
"c1": np.array([-2.0, -3.0, np.nan]),
"c3": np.array([0.0, 1.0, 0.0])
}),
name="test data",
)
class CheckEquality(InAlgorithmAsync):
"""Dummy algorithm class for testing whether writing and reading feather files works."""
def __init__(self) -> None:
super().__init__(name="Check equality", seed=-1)
def _run_script_command(self, train_path, test_path, pred_path):
"""Check if the dataframes loaded from the files are the same as the original ones."""
del test_path
loaded = DataTuple.from_npz(train_path)
pd.testing.assert_frame_equal(data_tuple.x, loaded.x)
pd.testing.assert_frame_equal(data_tuple.s, loaded.s)
pd.testing.assert_frame_equal(data_tuple.y, loaded.y)
# write a file for the predictions
np.savez(pred_path, hard=np.load(train_path)["x"])
return ["-c", "pass"]
def _fit_script_command(self, train_path, model_path):
"""Check if the dataframes loaded from the files are the same as the original ones."""
def _predict_script_command(self, model_path, test_path, pred_path):
"""Check if the dataframes loaded from the files are the same as the original ones."""
data_x = CheckEquality().run(data_tuple, data_tuple)
pd.testing.assert_series_equal( # type: ignore[call-arg]
data_tuple.x["a1"],
data_x.hard,
check_names=False)
def test_plot_no_tsne(toy_train_test: TrainTestPair):
"""Test plot."""
train, _ = toy_train_test
train = DataTuple(x=train.x[train.x.columns[:2]], s=train.s,
y=train.y) # type: ignore[arg-type]
save_2d_plot(train, "./plots/test.png")
def test_biased_split():
"""Test biased split."""
data = DataTuple(
x=pd.DataFrame([0] * 1000, columns=["feat1-"]),
s=pd.DataFrame([1] * 750 + [0] * 250, columns=["sens="]),
y=pd.DataFrame([1] * 500 + [0] * 250 + [1] * 125 + [0] * 125,
columns=["label<"]),
name="TestData",
)
# =================================== mixing factor = 0 =======================================
biased1, subset = em.get_biased_subset(data,
mixing_factor=0.0,
unbiased_pcnt=0.5)
# expected behavior: in biased1, s=y everywhere; `subset` is just a subset of `data`
assert biased1.s.shape == (313, 1)
assert biased1.y.shape == (313, 1)
assert (biased1.s.to_numpy() == biased1.y.to_numpy()).all()
assert biased1.name == "TestData - Biased (tm=0.0)"
# size is exactly 0.5 of the total
assert subset.s.shape[0] == approx(
500, abs=4) # can be off by 4 because of proportional split
assert subset.y.shape[0] == approx(500, abs=4)
assert subset.name == "TestData - Subset (tm=0.0)"
# the fraction of `data`points where y=s should be the same in the data and the subset
subset_mean = (subset.s.to_numpy() == subset.y.to_numpy()).mean()
data_mean = (data.s.to_numpy() == data.y.to_numpy()).mean()
assert subset_mean == approx(data_mean, abs=0.01)
biased2, debiased = em.get_biased_and_debiased_subsets(data,
mixing_factor=0.0,
unbiased_pcnt=0.5,
fixed_unbiased=True)
# expected behavior: in biased2, s=y everywhere; in debiased, 50% s=y and 50% s!=y
assert biased2.s.shape == (313, 1)
assert biased2.y.shape == (313, 1)
assert (biased2.s.to_numpy() == biased2.y.to_numpy()).all()
assert debiased.s.shape == (374, 1)
assert debiased.y.shape == (374, 1)
# for the debiased subset, s=y half of the time
count = np.count_nonzero(debiased.s.to_numpy() == debiased.y.to_numpy())
assert count == 374 // 2
# ================================= mixing factor = 0.5 =======================================
biased1, subset = em.get_biased_subset(data,
mixing_factor=0.5,
unbiased_pcnt=0.5)
# expected behavior: biased1 and `subset` are both just subsets of `data`
assert biased1.s.shape[0] == approx(500, abs=4)
assert biased1.y.shape[0] == approx(500, abs=4)
data_mean = (data.s.to_numpy() == data.y.to_numpy()).mean()
biased1_mean = (biased1.s.to_numpy() == biased1.y.to_numpy()).mean()
assert biased1_mean == approx(data_mean, abs=0.01)
assert subset.s.shape[0] == approx(500, abs=4)
assert subset.y.shape[0] == approx(500, abs=4)
# the fraction of `data`points where y=s should be the same in the data and the subset
subset_mean = (subset.s.to_numpy() == subset.y.to_numpy()).mean()
assert subset_mean == approx(data_mean, abs=0.01)
biased2, debiased = em.get_biased_and_debiased_subsets(data,
mixing_factor=0.5,
unbiased_pcnt=0.5,
fixed_unbiased=True)
# expected behavior: biased2 is just a subset of `data`; in debiased, 50% s=y and 50% s!=y
assert biased2.s.shape[0] == approx(500, abs=4)
assert biased2.y.shape[0] == approx(500, abs=4)
data_mean = (data.s.to_numpy() == data.y.to_numpy()).mean()
biased2_mean = (biased2.s.to_numpy() == biased2.y.to_numpy()).mean()
assert biased2_mean == approx(data_mean, abs=0.01)
assert debiased.s.shape == (374, 1)
assert debiased.y.shape == (374, 1)
# for the debiased subset, s=y half of the time
count = np.count_nonzero(debiased.s.to_numpy() == debiased.y.to_numpy())
assert count == 374 // 2
# ================================== mixing factor = 1 ========================================
biased1, subset = em.get_biased_subset(data,
mixing_factor=1.0,
unbiased_pcnt=0.5)
# expected behavior: in biased1, s!=y everywhere; `subset` is just a subset of `data`
assert biased1.s.shape == (188, 1)
assert biased1.y.shape == (188, 1)
assert (biased1.s.to_numpy() != biased1.y.to_numpy()).all()
assert subset.s.shape[0] == approx(500, abs=4)
assert subset.y.shape[0] == approx(500, abs=4)
# the fraction of `data`points where y=s should be the same in the data and the subset
subset_mean = (subset.s.to_numpy() == subset.y.to_numpy()).mean()
assert subset_mean == approx(data_mean, abs=0.01)
biased2, debiased = em.get_biased_and_debiased_subsets(data,
mixing_factor=1.0,
unbiased_pcnt=0.5,
fixed_unbiased=True)
# expected behavior: in biased2, s!=y everywhere; in debiased, 50% s=y and 50% s!=y
assert biased2.s.shape == (188, 1)
assert biased2.y.shape == (188, 1)
assert (biased2.s.to_numpy() != biased2.y.to_numpy()).all()
assert debiased.s.shape == (374, 1)
assert debiased.y.shape == (374, 1)
# for the debiased subset, s=y half of the time
count = np.count_nonzero(debiased.s.to_numpy() == debiased.y.to_numpy())
assert count == 374 // 2