def slicing_evaluation(df_train, df_test, train_model=None):
if train_model is None:
train_model = "mlp"
sfs = [
SlicingFunction.short_comment, SlicingFunction.ind_keyword,
SlicingFunction.cmp_re, SlicingFunction.industry_keyword
]
slice_names = [sf.name for sf in sfs]
scorer = Scorer(metrics=["f1"])
ft = FT.load(f"{WORK_PATH}/snorkel_flow/sources/fasttext_name_model.bin")
def get_ftr(text):
return ft.get_sentence_vector(' '.join(
[w for w in jieba.lcut(text.strip())]))
X_train = np.array(list(df_train.text.apply(get_ftr).values))
X_test = np.array(list(df_test.text.apply(get_ftr).values))
Y_train = df_train.label.values
Y_test = df_test.label.values
if train_model == "lr":
sklearn_model = LogisticRegression(C=0.001, solver="liblinear")
sklearn_model.fit(X=X_train, y=Y_train)
preds_test = sklearn_model.predict(X_test)
probs_test = preds_to_probs(
preds_test,
len([c for c in dir(Polarity) if not c.startswith("__")]))
print(f"Test set F1: {100 * f1_score(Y_test, preds_test):.1f}%")
applier = PandasSFApplier(sfs)
S_test = applier.apply(df_test)
analysis = scorer.score_slices(S=S_test,
golds=Y_test,
preds=preds_test,
probs=probs_test,
as_dataframe=True)
return analysis
if train_model == "mlp":
# Define model architecture
bow_dim = X_train.shape[1]
hidden_dim = bow_dim
mlp = get_pytorch_mlp(hidden_dim=hidden_dim, num_layers=2)
# Initialize slice model
slice_model = SliceAwareClassifier(
base_architecture=mlp,
head_dim=hidden_dim,
slice_names=slice_names,
scorer=scorer,
)
# generate the remaining S matrices with the new set of slicing functions
applier = PandasSFApplier(sfs)
S_train = applier.apply(df_train)
S_test = applier.apply(df_test)
# add slice labels to an existing dataloader
BATCH_SIZE = 64
train_dl = create_dict_dataloader(X_train, Y_train, "train")
train_dl_slice = slice_model.make_slice_dataloader(
train_dl.dataset, S_train, shuffle=True, batch_size=BATCH_SIZE)
test_dl = create_dict_dataloader(X_test, Y_test, "train")
test_dl_slice = slice_model.make_slice_dataloader(
test_dl.dataset, S_test, shuffle=False, batch_size=BATCH_SIZE)
# fit our classifier with the training set dataloader
trainer = Trainer(n_epochs=2, lr=1e-4, progress_bar=True)
trainer.fit(slice_model, [train_dl_slice])
analysis = slice_model.score_slices([test_dl_slice], as_dataframe=True)
return analysis
# %% [markdown]
# Using Snorkel's [`Trainer`](https://snorkel.readthedocs.io/en/master/packages/_autosummary/classification/snorkel.classification.Trainer.html), we fit our classifier with the training set dataloader.
# %%
from snorkel.classification import Trainer
# For demonstration purposes, we set n_epochs=2
trainer = Trainer(n_epochs=2, lr=1e-4, progress_bar=True)
trainer.fit(slice_model, [train_dl_slice])
# %% [markdown]
# At inference time, the primary task head (`spam_task`) will make all final predictions.
# We'd like to evaluate all the slice heads on the original task head — [`score_slices`](https://snorkel.readthedocs.io/en/v0.9.3/packages/_autosummary/slicing/snorkel.slicing.SliceAwareClassifier.html#snorkel.slicing.SliceAwareClassifier.score_slices) remaps all slice-related labels, denoted `spam_task_slice:{slice_name}_pred`, to be evaluated on the `spam_task`.
# %%
slice_model.score_slices([test_dl_slice], as_dataframe=True)
# %% [markdown]
# *Note: in this toy dataset, we see high variance in slice performance, because our dataset is so small that (i) there are few data points in the train split, giving little signal to learn over, and (ii) there are few data points in the test split, making our evaluation metrics very noisy.
# For a demonstration of data slicing deployed in state-of-the-art models, please see our [SuperGLUE](https://github.com/HazyResearch/snorkel-superglue/tree/master/tutorials) tutorials.*
# %% [markdown]
# ---
# ## Recap
# %% [markdown]
# This tutorial walked through the process authoring slices, monitoring model performance on specific slices, and improving model performance using slice information.
# This programming abstraction provides a mechanism to heuristically identify critical data subsets.
# For more technical details about _Slice-based Learning,_ please see our [NeurIPS 2019 paper](https://arxiv.org/abs/1909.06349)!
class SliceCombinerTest(unittest.TestCase):
def setUp(self):
# Define S_matrix
data_points = [SimpleNamespace(num=num) for num in DATA]
applier = SFApplier([f, g])
self.S = applier.apply(data_points, progress_bar=False)
# Define base architecture
self.hidden_dim = 10
self.mlp = nn.Sequential(
nn.Linear(2, self.hidden_dim),
nn.Linear(self.hidden_dim, self.hidden_dim),
nn.ReLU(),
)
# Define model parameters
self.data_name = "test_data"
self.task_name = "test_task"
# Define datasets
# Repeated data value for [N x 2] dim Tensor
self.X = torch.FloatTensor([(x, x) for x in DATA])
# Alternating labels
self.Y = torch.LongTensor([int(i % 2 == 0) for i in range(len(DATA))])
dataset_name = "test_dataset"
splits = ["train", "valid"]
self.datasets = [
create_dataset(self.X, self.Y, split, dataset_name, self.data_name,
self.task_name) for split in splits
]
self.slice_model = SliceAwareClassifier(
base_architecture=self.mlp,
head_dim=self.hidden_dim,
slice_names=[sf.name for sf in sfs],
input_data_key=self.data_name,
task_name=self.task_name,
scorer=Scorer(metrics=["f1"]),
)
def test_slice_tasks(self):
"""Ensure that all the desired slice tasks are initialized."""
expected_tasks = {
# Base task
"test_task",
# Slice tasks for default base slice
"test_task_slice:base_pred",
"test_task_slice:base_ind",
# Slice Tasks
"test_task_slice:f_pred",
"test_task_slice:f_ind",
"test_task_slice:g_pred",
"test_task_slice:g_ind",
}
self.assertEqual(self.slice_model.task_names, expected_tasks)
def test_make_slice_dataloader(self):
# Test correct construction
dataloader = self.slice_model.make_slice_dataloader(
dataset=self.datasets[0], S=self.S)
Y_dict = dataloader.dataset.Y_dict
self.assertEqual(len(Y_dict), 7)
self.assertIn("test_task", Y_dict)
self.assertIn("test_task_slice:base_pred", Y_dict)
self.assertIn("test_task_slice:base_ind", Y_dict)
self.assertIn("test_task_slice:f_pred", Y_dict)
self.assertIn("test_task_slice:f_ind", Y_dict)
self.assertIn("test_task_slice:g_pred", Y_dict)
self.assertIn("test_task_slice:g_ind", Y_dict)
# Test bad data input
bad_data_dataset = DictDataset(
name="test_data",
split="train",
X_dict={self.data_name: self.X},
Y_dict={"bad_labels": self.Y},
)
with self.assertRaisesRegex(ValueError, "labels missing"):
self.slice_model.make_slice_dataloader(dataset=bad_data_dataset,
S=self.S)
def test_scores_pipeline(self):
"""Ensure that the appropriate scores are returned with .score and .score_slices."""
# Make valid dataloader
valid_dl = self.slice_model.make_slice_dataloader(
dataset=self.datasets[1], S=self.S, batch_size=4)
# Eval overall
scores = self.slice_model.score([valid_dl])
# All labels should appears in .score() output
self.assertIn("test_task/test_dataset/valid/f1", scores)
self.assertIn("test_task_slice:f_pred/test_dataset/valid/f1", scores)
self.assertIn("test_task_slice:f_pred/test_dataset/valid/f1", scores)
self.assertIn("test_task_slice:g_ind/test_dataset/valid/f1", scores)
self.assertIn("test_task_slice:g_ind/test_dataset/valid/f1", scores)
# Eval on slices
slice_scores = self.slice_model.score_slices([valid_dl])
# Check that we eval on 'pred' labels in .score_slices() output
self.assertIn("test_task/test_dataset/valid/f1", slice_scores)
self.assertIn("test_task_slice:f_pred/test_dataset/valid/f1",
slice_scores)
self.assertIn("test_task_slice:g_pred/test_dataset/valid/f1",
slice_scores)
# No 'ind' labels!
self.assertNotIn("test_task_slice:f_ind/test_dataset/valid/f1",
slice_scores)
self.assertNotIn("test_task_slice:g_ind/test_dataset/valid/f1",
slice_scores)