def test_majority_label_vote(self):
L = np.array([[0, 1, 0], [0, 1, 0], [1, 0, 0], [-1, -1, 1]])
ml_voter = MajorityLabelVoter()
Y_p = ml_voter.predict_proba(L)
Y_p_true = np.array([[1.0, 0.0], [1.0, 0.0], [1.0, 0.0], [0.0, 1.0]])
np.testing.assert_array_almost_equal(Y_p, Y_p_true)
# %% [markdown]
# So our `LabelModel` improves over the majority vote baseline!
# However, it is typically **not suitable as an inference-time model** to make predictions for unseen data points, due to (among other things) some data points having all abstain labels.
# In the next section, we will use the output of the label model as training labels to train a
# discriminative classifier to see if we can improve performance further.
# This classifier will only need the text of the comment to make predictions, making it much more suitable
# for inference over unseen comments.
# For more information on the properties of the label model and when to use it, see the [Snorkel guides]().
# %% [markdown]
# We can also run error analysis after the label model has been trained.
# For example, let's take a look at 5 random false negatives from the `dev` set, which might inspire some more LFs that vote `SPAM`.
# %%
probs_dev = majority_model.predict_proba(L=L_dev)
preds_dev = probs_dev >= 0.5
buckets = get_label_buckets(Y_dev, preds_dev[:, 1])
df_fn_dev = df_dev[["text", "label"]].iloc[buckets[(SPAM, HAM)]]
df_fn_dev["probability"] = probs_dev[buckets[(SPAM, HAM)], 1]
df_fn_dev.sample(5, random_state=3)
# %% [markdown] {"tags": ["md-exclude"]}
# Let's briefly confirm that the labels the `LabelModel` produces are probabilistic in nature.
# The following histogram shows the confidences we have that each data point has the label SPAM.
# The points we are least certain about will have labels close to 0.5.
# %% {"tags": ["md-exclude"]}
def get_role_probs(lf_train: pd.DataFrame,
filter_abstains: bool = False,
lfs: Optional[List[labeling_function]] = None,
lf_dev: pd.DataFrame = None,
seed: Optional[int] = None,
tmp_path: Union[str, Path] = None,
use_majority_label_voter=False) -> pd.DataFrame:
"""
Takes "raw" data frame, builds argument role examples, (trains LabelModel), calculates event_argument_probs
and returns merged argument role examples with event_argument_probs.
:param use_majority_label_voter: Whether to use a majority label voter instead of the snorkel label model
:param seed: Seed for use in label model (mu initialization)
:param filter_abstains: Filters rows where all labeling functions abstained
:param lf_train: Training dataset which will be labeled using Snorkel
:param lfs: List of labeling functions
:param lf_dev: Optional development dataset that can be used to set a prior for the class balance
:param tmp_path: Path to temporarily store variables that are shared during random repeats
:return: Labeled lf_train, labeling function applier, label model
"""
df_train, L_train = None, None
df_dev, Y_dev, L_dev = None, None, None
tmp_train_path, tmp_dev_path = None, None
# For random repeats try to load pickled variables from first run as they are shared
if tmp_path:
tmp_train_path = Path(tmp_path).joinpath("role_train.pkl")
os.makedirs(os.path.dirname(tmp_train_path), exist_ok=True)
if tmp_train_path.exists():
with open(tmp_train_path, 'rb') as pickled_train:
df_train, L_train = pickle.load(pickled_train)
if lf_dev is not None:
tmp_dev_path = Path(tmp_path).joinpath("role_dev.pkl")
os.makedirs(os.path.dirname(tmp_dev_path), exist_ok=True)
if tmp_dev_path.exists():
with open(tmp_dev_path, 'rb') as pickled_dev:
df_dev, Y_dev, L_dev = pickle.load(pickled_dev)
if lfs is None:
lfs = get_role_list_lfs()
applier = PandasLFApplier(lfs)
if L_train is None or df_train is None:
df_train, _ = build_event_role_examples(lf_train)
logger.info("Running Event Role Labeling Function Applier")
L_train = applier.apply(df_train)
if tmp_path:
with open(tmp_train_path, 'wb') as pickled_train:
pickle.dump((df_train, L_train), pickled_train)
if lf_dev is not None and any(element is None
for element in [df_dev, Y_dev, L_dev]):
df_dev, Y_dev = build_event_role_examples(lf_dev)
logger.info("Running Event Role Labeling Function Applier on dev set")
L_dev = applier.apply(df_dev)
if tmp_path:
with open(tmp_dev_path, 'wb') as pickled_dev:
pickle.dump((df_dev, Y_dev, L_dev), pickled_dev)
if use_majority_label_voter:
logger.info(
"Using MajorityLabelVoter to calculate role class probabilities")
label_model = MajorityLabelVoter(cardinality=11)
else:
label_model = LabelModel(cardinality=11, verbose=True)
logger.info(
"Fitting LabelModel on the data and predicting role class probabilities"
)
if seed:
label_model.fit(L_train=L_train,
n_epochs=5000,
log_freq=500,
seed=seed,
Y_dev=Y_dev)
else:
label_model.fit(L_train=L_train,
n_epochs=5000,
log_freq=500,
Y_dev=Y_dev)
# Evaluate label model on development data
if df_dev is not None and Y_dev is not None:
metrics = ["accuracy", "f1_micro", "f1_macro"]
logger.info("Evaluate on the dev set")
label_model_metrics = label_model.score(L=L_dev,
Y=Y_dev,
tie_break_policy="random",
metrics=metrics)
if use_majority_label_voter:
logger.info('Role Majority Label Voter Metrics')
else:
logger.info('Role Label Model Metrics')
logger.info(
f"{'Accuracy:':<25} {label_model_metrics['accuracy'] * 100:.1f}%")
logger.info(
f"{'F1 (micro averaged):':<25} {label_model_metrics['f1_micro'] * 100:.1f}%"
)
logger.info(
f"{'F1 (macro averaged):':<25} {label_model_metrics['f1_macro'] * 100:.1f}%"
)
event_role_probs = label_model.predict_proba(L_train)
if filter_abstains:
df_train_filtered, probs_train_filtered = filter_unlabeled_dataframe(
X=df_train, y=event_role_probs, L=L_train)
merged_event_role_examples = merge_event_role_examples(
df_train_filtered, probs_train_filtered)
else:
# Multiplies probabilities of abstains with zero so that the example is treated as padding in the end model
merged_event_role_examples = merge_event_role_examples(
df_train, utils.zero_out_abstains(event_role_probs, L_train))
return merged_event_role_examples