def test_score(self):
L = np.array([[1, 1, 0], [-1, -1, -1], [1, 0, 1]])
Y = np.array([1, 0, 1])
label_model = LabelModel(cardinality=2, verbose=False)
label_model.fit(L, n_epochs=100)
results = label_model.score(L, Y, metrics=["accuracy", "coverage"])
np.testing.assert_array_almost_equal(label_model.predict(L),
np.array([1, -1, 1]))
results_expected = dict(accuracy=1.0, coverage=2 / 3)
self.assertEqual(results, results_expected)
L = np.array([[1, 0, 1], [1, 0, 1]])
label_model = self._set_up_model(L)
label_model.mu = nn.Parameter(label_model.mu_init.clone().clamp(
0.01, 0.99))
results = label_model.score(L, Y=np.array([0, 1]))
results_expected = dict(accuracy=0.5)
self.assertEqual(results, results_expected)
results = label_model.score(L=L,
Y=np.array([1, 0]),
metrics=["accuracy", "f1"])
results_expected = dict(accuracy=0.5, f1=2 / 3)
self.assertEqual(results, results_expected)
def calculate_metrics(
label_model: LabelModel,
dataset_name: str,
true_labels: np.ndarray,
save_to: AbsolutePath,
) -> Dict[str, float]:
"""
>>> from collections import namedtuple; import tempfile
>>> def mocked_predictions(l,return_probs,tie_break_policy): return np.array([1, 0, 1]), np.array([[0.1, 0.9], [0.8, 0.2], [0.25, 0.75]])
>>> def mocked_scores(L,Y,tie_break_policy,metrics):
... return {"f1": 1.0} if metrics == ['f1'] else {"roc_auc": 0.78}
>>> lm = namedtuple('LM', ['predict', 'score'])(mocked_predictions, mocked_scores)
>>> with tempfile.TemporaryDirectory() as tmpdirname:
... np.ndarray([]).dump(f"{tmpdirname}/heuristic_matrix_test_set.pkl")
... calculate_metrics(lm, "test_set", np.array([1, 1, 0]), Path(tmpdirname))
{'label_model_accuracy_test_set': 0.333, 'label_model_auc_test_set': 0.78, 'label_model_f1_test_set': 1.0, 'label_model_mse_test_set': 0.404}
>>> with tempfile.TemporaryDirectory() as tmpdirname:
... np.ndarray([]).dump(f"{tmpdirname}/heuristic_matrix_test_set.pkl")
... calculate_metrics(lm, "test_set", np.array([0, 1, 0]), Path(tmpdirname))
{'label_model_accuracy_test_set': 0.0, 'label_model_auc_test_set': 0.78, 'label_model_f1_test_set': 1.0, 'label_model_mse_test_set': 0.671}
"""
lines = np.load(str(save_to / f"heuristic_matrix_{dataset_name}.pkl"),
allow_pickle=True)
tie_break_policy = "random"
Y_pred, Y_prob = label_model.predict(lines,
return_probs=True,
tie_break_policy=tie_break_policy)
try:
auc = label_model.score(L=lines,
Y=true_labels,
tie_break_policy="random",
metrics=["roc_auc"])["roc_auc"]
auc = round(auc, 3)
except ValueError:
auc = "n/a"
f1 = label_model.score(L=lines,
Y=true_labels,
tie_break_policy="random",
metrics=["f1"])["f1"]
accuracy = sum(Y_pred == true_labels) / float(len(Y_pred))
mse = np.mean((Y_prob[:, 1] - true_labels)**2)
return {
f"label_model_accuracy_{dataset_name}": round(accuracy, 3),
f"label_model_auc_{dataset_name}": auc,
f"label_model_f1_{dataset_name}": round(f1, 3),
f"label_model_mse_{dataset_name}": round(mse, 3),
}
def test_progress_bar(self):
L = np.array([[1, 1, 0], [-1, -1, -1], [1, 0, 1]])
Y = np.array([1, 0, 1])
label_model = LabelModel(cardinality=2, verbose=False)
label_model.fit(L, n_epochs=100, progress_bar=False)
results = label_model.score(L, Y, metrics=["accuracy", "coverage"])
np.testing.assert_array_almost_equal(
label_model.predict(L), np.array([1, -1, 1])
)
results_expected = dict(accuracy=1.0, coverage=2 / 3)
self.assertEqual(results, results_expected)
def test_label_model_basic(self) -> None:
"""Test the LabelModel's estimate of P and Y on a simple synthetic dataset."""
np.random.seed(123)
P, Y, L = generate_simple_label_matrix(self.n, self.m,
self.cardinality)
# Train LabelModel
label_model = LabelModel(cardinality=self.cardinality, verbose=False)
label_model.fit(L, n_epochs=200, lr=0.01, seed=123)
# Test estimated LF conditional probabilities
P_lm = label_model.get_conditional_probs()
np.testing.assert_array_almost_equal(P, P_lm, decimal=2)
# Test predicted labels
score = label_model.score(L, Y)
self.assertGreaterEqual(score["accuracy"], 0.9)
def test_label_model_basic(self) -> None:
"""Test the LabelModel's estimate of P and Y on a simple synthetic dataset."""
np.random.seed(123)
P, Y, L = generate_simple_label_matrix(self.n, self.m,
self.cardinality)
# Train LabelModel
label_model = LabelModel(cardinality=self.cardinality, verbose=False)
label_model.fit(L, n_epochs=200, lr=0.01, seed=123)
# Test estimated LF conditional probabilities
P_lm = label_model.get_conditional_probs()
conditional_probs_err = (
np.linalg.norm(P.flatten() - P_lm.flatten(), ord=1) / P.size)
self.assertLessEqual(conditional_probs_err, 0.01)
# Test predicted labels
score = label_model.score(L, Y)
self.assertGreaterEqual(score["accuracy"], 0.9)
def label_model_creator(df_dev, Y_dev, df_train, df_test, Y_test):
# Accumulate all the labeling_functions for supply
supply_lfs = [
lf_supply, lf_customer, lf_sales_to, lf_our_customer, lf_acquisition,
lf_people, lf_sold, lf_relation, lf_competition
]
# Apply the above labeling functions to the data in Pandas dataframe formats
applier = PandasLFApplier(supply_lfs)
# Use the applier of the labeling functions to both development set and train set
L_dev = applier.apply(df_dev)
L_train = applier.apply(df_train)
L_test = applier.apply(df_test)
# caridnality : 2 (True and False)
label_model = LabelModel(cardinality=2, verbose=True)
# Fit the label_model
label_model.fit(L_train, Y_dev, n_epochs=5000, log_freq=500)
# accuracy for the label model using the test set
label_model_acc = label_model.score(L=L_test,
Y=Y_test,
tie_break_policy="random")["accuracy"]
print(f"{'Label Model Accuracy:':<25} {label_model_acc * 100:.1f}%")
# check the F-1 score and ROC_AUC score
probs_dev = label_model.predict_proba(L_dev)
preds_dev = probs_to_preds(probs_dev)
print(
f"Label model f1 score: {metric_score(Y_dev, preds_dev, probs=probs_dev, metric='f1')}"
)
print(
f"Label model roc-auc: {metric_score(Y_dev, preds_dev, probs=probs_dev, metric='roc_auc')}"
)
return label_model, L_train
def labeling_evaluation(df_train, df_test, label_model):
lfs = [
LabelingFunction.lf_ind_keyword, LabelingFunction.lf_short,
LabelingFunction.lf_cmp_re, LabelingFunction.lf_industry_keyword,
LabelingFunction.lf_surname_re, LabelingFunction.industry_cls
]
applier = PandasLFApplier(lfs=lfs)
L_train = applier.apply(df=df_train)
L_test = applier.apply(df=df_test)
Y_test = df_test.label.values
analysis = LFAnalysis(L=L_train, lfs=lfs).lf_summary()
if label_model == "majority":
majority_model = MajorityLabelVoter()
preds_train = majority_model.predict(L=L_train)
majority_acc = majority_model.score(
L=L_test, Y=Y_test, tie_break_policy="random")["accuracy"]
print(f"{'Majority Vote Accuracy:':<25} {majority_acc * 100:.1f}%")
df_train_filtered, preds_train_filtered = filter_unlabeled_dataframe(
X=df_train, y=preds_train, L=L_train)
return df_train_filtered, preds_train_filtered, analysis
if label_model == "weighted":
label_model = LabelModel(cardinality=len(
[c for c in dir(Polarity) if not c.startswith("__")]),
verbose=True)
label_model.fit(L_train=L_train, n_epochs=500, log_freq=100, seed=123)
probs_train = label_model.predict_proba(L_train)
label_model_acc = label_model.score(
L=L_test, Y=Y_test, tie_break_policy="random")["accuracy"]
print(f"{'Label Model Accuracy:':<25} {label_model_acc * 100:.1f}%")
df_train_filtered, probs_train_filtered = filter_unlabeled_dataframe(
X=df_train, y=probs_train, L=L_train)
preds_train_filtered = probs_to_preds(probs_train_filtered)
return df_train_filtered, probs_train_filtered, preds_train_filtered, analysis
def model_analysis(label_model: LabelModel,
training_set: pd.DataFrame,
L_train: np.ndarray,
L_test: np.ndarray,
Y_test: np.ndarray,
lfs: list,
output_file="output") -> None:
# TODO: consider using **kwargs instead of this painful list of arguments
"""Output analysis for the label model to a file
:param label_model: The current label model which we want to output analysis for
:type label_model: LabelModel
:param training_set: A dataframe containing the training dataset
:type training_set: pd.DataFrame
:param L_train: The matrix of labels generated by the labeling functions on the training data
:type L_train: np.ndarray
:param L_test: The matrix of labels generated bt the labeling functions on the testing data
:type L_test: np.ndarray
:param Y_test: Gold labels associated with data points in L_test
:type Y_test: np.ndarray
:param lfs: List of labeling functions
:type lfs: list
:param output_file: A path where the output file should be writtent to, defaults to `PROJECT_ROOT/output`
:type output_file: str, optional
"""
Y_train = label_model.predict_proba(L=L_train)
Y_pred = label_model.predict(L=L_test, tie_break_policy="abstain")
lf_analysis_train = LFAnalysis(L=L_train, lfs=lfs).lf_summary()
# TODO: Write this df to a output file. Ask Jennifer about how to handle this
print(lf_analysis_train)
# build majority label voter model
majority_model = MajorityLabelVoter()
majority_acc = majority_model.score(L=L_test,
Y=Y_test,
tie_break_policy="abstain",
metrics=["f1", "accuracy"])
label_model_acc = label_model.score(L=L_test,
Y=Y_test,
tie_break_policy="abstain",
metrics=["f1", "accuracy"])
# get precision and recall scores
p_score = precision_score(y_true=Y_test, y_pred=Y_pred, average='weighted')
r_score = recall_score(y_true=Y_test,
y_pred=Y_pred,
average='weighted',
labels=np.unique(Y_pred))
# how many documents abstained
probs_train = majority_model.predict_proba(L=L_train)
df_train_filtered, probs_train_filtered = filter_unlabeled_dataframe(
X=training_set, y=probs_train, L=L_train)
# get number of false positives
buckets = get_label_buckets(Y_test, Y_pred)
true_positives, false_positives, true_negatives, false_negatives = (
buckets.get((1, 1)), buckets.get((1, 0)), buckets.get(
(0, 0)), buckets.get((0, 1)))
# write analysis to file
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
with open(f"{'../output/logs/'}{output_file}_run_{timestamp}.txt",
"w") as output_file:
output_file.write(
f"{'Majority Vote Accuracy:':<25} {majority_acc['accuracy'] * 100:.2f}%"
)
output_file.write(
f"\n{'Majority Vote F1 Score:':<25} {majority_acc['f1'] * 100:.2f}%"
)
output_file.write(
f"\n{'Label Model Accuracy:':<25} {label_model_acc['accuracy'] * 100:.2f}%"
)
output_file.write(
f"\n{'Label Model F1 Score:':<25} {label_model_acc['f1'] * 100:.2f}%"
)
output_file.write(f"\n{'Precision Score:':<25} {p_score * 100:.2f}%")
output_file.write(f"\n{'Recall Score:':<25} {r_score * 100:.2f}%")
output_file.write(
f"\n{'Abstained Data Points:':<25} {len(df_train_filtered)}")
output_file.write(
f"\n{'True Positives:':<25} {len(true_positives) if true_positives is not None else 0}"
)
output_file.write(
f"\n{'False Positives:':<25} {len(false_positives) if false_positives is not None else 0}"
)
output_file.write(
f"\n{'False Negatives:':<25} {len(false_negatives) if false_negatives is not None else 0}"
)
output_file.write(
f"\n{'True Negatives:':<25} {len(true_negatives) if true_negatives is not None else 0}"
)
output_file.write(
f"\n{'Abstained Positives:':<25} {len(buckets[(1, -1)])}")
output_file.write(
f"\n{'Abstained Negatives:':<25} {len(buckets[(0, -1)])}")
lfs = config['lfs']
print('reading in data...')
X_train, y_train, X_dev, y_dev = read_data_from_config(config)
print('applying labelling functions to data...')
applier = PandasLFApplier(lfs=lfs)
L_train = applier.apply(df=X_train)
L_dev = applier.apply(df=X_dev)
print('fitting Label Model')
label_model = LabelModel(cardinality=config['cardinality'], verbose=True)
label_model.fit(L_train=L_train, n_epochs=500, log_freq=100, seed=123)
label_model_acc = label_model.score(L=L_dev,
Y=y_dev,
tie_break_policy="random")["accuracy"]
print(f'label model acc: {label_model_acc}')
print('fitting Majority Label Voter model')
majority_model = MajorityLabelVoter(cardinality=config['cardinality'])
# preds_train = majority_model.predict(L=L_train)
majority_acc = majority_model.score(L=L_dev,
Y=np.array(y_dev).reshape(-1, 1),
tie_break_policy="random")["accuracy"]
print(f'majority_label_acc: {majority_acc}')
log_metric('majority_label_acc', majority_acc)
log_metric('label_model_acc', label_model_acc)
probs_train = label_model.predict_proba(L=L_train)
preds_train = majority_model.predict(L=L_train)
# use LabelModel to produce training labels
label_model = LabelModel(cardinality=2, verbose=True)
label_model.fit(L_train=L_train, n_epochs=500, log_freq=100, seed=123)
# result using majority-vote model
Y_test = data_test.label.values
majority_acc = majority_model.score(L=L_test,
Y=Y_test,
tie_break_policy="random")["accuracy"]
print(f"{'Majority Vote Accuracy:':<25} {majority_acc * 100:.1f}%")
# results using label model
label_model_acc = label_model.score(L=L_test,
Y=Y_test,
tie_break_policy="random")["accuracy"]
print(f"{'Label Model Accuracy:':<25} {label_model_acc * 100:.1f}%")
# representing each data point using "bag of n-gram" feature
probs_train = label_model.predict_proba(L=L_train)
vectorizer = CountVectorizer(ngram_range=(1, 5))
X_train = vectorizer.fit_transform(data_train.text.tolist())
X_test = vectorizer.transform(data_test.text.tolist())
# replace each label distribution with the label having maximum probability
preds_train = probs_to_preds(probs=probs_train)
# train a Scikit-Learn classifier
sklearn_model = LogisticRegression(C=1e3, solver="liblinear")
sklearn_model.fit(X=X_train, y=preds_train)
# %% [markdown] # ## 3. Train Label Model # We now train a multi-class `LabelModel` to assign training labels to the unalabeled training set. # %% from snorkel.labeling.model import LabelModel label_model = LabelModel(cardinality=3, verbose=True) label_model.fit(L_train, seed=123, lr=0.01, log_freq=10, n_epochs=100) # %% [markdown] # We use [F1](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.f1_score.html) Micro average for the multiclass setting, which calculates metrics globally across classes, by counting the total true positives, false negatives and false positives. # %% label_model.score(L_valid, Y_valid, metrics=["f1_micro"]) # %% [markdown] # ## 4. Train a Classifier # You can then use these training labels to train any standard discriminative model, such as [an off-the-shelf ResNet](https://github.com/KaimingHe/deep-residual-networks), which should learn to generalize beyond the LF's we've developed! # %% [markdown] # #### Create DataLoaders for Classifier # %% from snorkel.classification import DictDataLoader from model import SceneGraphDataset, create_model df_train["labels"] = label_model.predict(L_train) if sample:
label_model = LabelModel(cardinality=2, verbose=True)
label_model.fit(L_train=L_train, n_epochs=500, log_freq=100, seed=123)
L_test = applier.apply(test_df)
# to_numerical = lambda x: x=='leave'
# Y_test = [to_numerical(item) for item in test_df.label]
Y_test = []
for item in test_df.label:
if item == 'stay':
Y_test.append(STAY)
else:
Y_test.append(LEAVE)
Y_test = np.asarray(Y_test)
label_model_performance = label_model.score(L=L_test, Y=Y_test, tie_break_policy="random",
metrics=['accuracy', 'precision', 'recall', 'f1'])
print(f"Label Model Accuracy: {label_model_performance['accuracy'] * 100:.1f}%")
predict_probs = label_model.predict_proba(L_unlabeled)
preds = probs_to_preds(predict_probs)
pred_labels = []
for i in range(len(preds)):
if preds[i]:
pred_labels.append('leave')
else:
pred_labels.append('stay')
unlabeled_data['label'] = pred_labels
unlabeled_data.to_csv(os.path.join(data_dir, 'snorkel_labeled_data.csv'), sep=',', index=False)
