def test_score(self): L = np.array([[1, 1, 0], [-1, -1, -1], [1, 0, 1]]) Y = np.array([1, 0, 1]) label_model = LabelModel(cardinality=2, verbose=False) label_model.fit(L, n_epochs=100) results = label_model.score(L, Y, metrics=["accuracy", "coverage"]) np.testing.assert_array_almost_equal(label_model.predict(L), np.array([1, -1, 1])) results_expected = dict(accuracy=1.0, coverage=2 / 3) self.assertEqual(results, results_expected) L = np.array([[1, 0, 1], [1, 0, 1]]) label_model = self._set_up_model(L) label_model.mu = nn.Parameter(label_model.mu_init.clone().clamp( 0.01, 0.99)) results = label_model.score(L, Y=np.array([0, 1])) results_expected = dict(accuracy=0.5) self.assertEqual(results, results_expected) results = label_model.score(L=L, Y=np.array([1, 0]), metrics=["accuracy", "f1"]) results_expected = dict(accuracy=0.5, f1=2 / 3) self.assertEqual(results, results_expected)
def calculate_metrics( label_model: LabelModel, dataset_name: str, true_labels: np.ndarray, save_to: AbsolutePath, ) -> Dict[str, float]: """ >>> from collections import namedtuple; import tempfile >>> def mocked_predictions(l,return_probs,tie_break_policy): return np.array([1, 0, 1]), np.array([[0.1, 0.9], [0.8, 0.2], [0.25, 0.75]]) >>> def mocked_scores(L,Y,tie_break_policy,metrics): ... return {"f1": 1.0} if metrics == ['f1'] else {"roc_auc": 0.78} >>> lm = namedtuple('LM', ['predict', 'score'])(mocked_predictions, mocked_scores) >>> with tempfile.TemporaryDirectory() as tmpdirname: ... np.ndarray([]).dump(f"{tmpdirname}/heuristic_matrix_test_set.pkl") ... calculate_metrics(lm, "test_set", np.array([1, 1, 0]), Path(tmpdirname)) {'label_model_accuracy_test_set': 0.333, 'label_model_auc_test_set': 0.78, 'label_model_f1_test_set': 1.0, 'label_model_mse_test_set': 0.404} >>> with tempfile.TemporaryDirectory() as tmpdirname: ... np.ndarray([]).dump(f"{tmpdirname}/heuristic_matrix_test_set.pkl") ... calculate_metrics(lm, "test_set", np.array([0, 1, 0]), Path(tmpdirname)) {'label_model_accuracy_test_set': 0.0, 'label_model_auc_test_set': 0.78, 'label_model_f1_test_set': 1.0, 'label_model_mse_test_set': 0.671} """ lines = np.load(str(save_to / f"heuristic_matrix_{dataset_name}.pkl"), allow_pickle=True) tie_break_policy = "random" Y_pred, Y_prob = label_model.predict(lines, return_probs=True, tie_break_policy=tie_break_policy) try: auc = label_model.score(L=lines, Y=true_labels, tie_break_policy="random", metrics=["roc_auc"])["roc_auc"] auc = round(auc, 3) except ValueError: auc = "n/a" f1 = label_model.score(L=lines, Y=true_labels, tie_break_policy="random", metrics=["f1"])["f1"] accuracy = sum(Y_pred == true_labels) / float(len(Y_pred)) mse = np.mean((Y_prob[:, 1] - true_labels)**2) return { f"label_model_accuracy_{dataset_name}": round(accuracy, 3), f"label_model_auc_{dataset_name}": auc, f"label_model_f1_{dataset_name}": round(f1, 3), f"label_model_mse_{dataset_name}": round(mse, 3), }
def test_progress_bar(self): L = np.array([[1, 1, 0], [-1, -1, -1], [1, 0, 1]]) Y = np.array([1, 0, 1]) label_model = LabelModel(cardinality=2, verbose=False) label_model.fit(L, n_epochs=100, progress_bar=False) results = label_model.score(L, Y, metrics=["accuracy", "coverage"]) np.testing.assert_array_almost_equal( label_model.predict(L), np.array([1, -1, 1]) ) results_expected = dict(accuracy=1.0, coverage=2 / 3) self.assertEqual(results, results_expected)
def test_label_model_basic(self) -> None: """Test the LabelModel's estimate of P and Y on a simple synthetic dataset.""" np.random.seed(123) P, Y, L = generate_simple_label_matrix(self.n, self.m, self.cardinality) # Train LabelModel label_model = LabelModel(cardinality=self.cardinality, verbose=False) label_model.fit(L, n_epochs=200, lr=0.01, seed=123) # Test estimated LF conditional probabilities P_lm = label_model.get_conditional_probs() np.testing.assert_array_almost_equal(P, P_lm, decimal=2) # Test predicted labels score = label_model.score(L, Y) self.assertGreaterEqual(score["accuracy"], 0.9)
def test_label_model_basic(self) -> None: """Test the LabelModel's estimate of P and Y on a simple synthetic dataset.""" np.random.seed(123) P, Y, L = generate_simple_label_matrix(self.n, self.m, self.cardinality) # Train LabelModel label_model = LabelModel(cardinality=self.cardinality, verbose=False) label_model.fit(L, n_epochs=200, lr=0.01, seed=123) # Test estimated LF conditional probabilities P_lm = label_model.get_conditional_probs() conditional_probs_err = ( np.linalg.norm(P.flatten() - P_lm.flatten(), ord=1) / P.size) self.assertLessEqual(conditional_probs_err, 0.01) # Test predicted labels score = label_model.score(L, Y) self.assertGreaterEqual(score["accuracy"], 0.9)
def label_model_creator(df_dev, Y_dev, df_train, df_test, Y_test): # Accumulate all the labeling_functions for supply supply_lfs = [ lf_supply, lf_customer, lf_sales_to, lf_our_customer, lf_acquisition, lf_people, lf_sold, lf_relation, lf_competition ] # Apply the above labeling functions to the data in Pandas dataframe formats applier = PandasLFApplier(supply_lfs) # Use the applier of the labeling functions to both development set and train set L_dev = applier.apply(df_dev) L_train = applier.apply(df_train) L_test = applier.apply(df_test) # caridnality : 2 (True and False) label_model = LabelModel(cardinality=2, verbose=True) # Fit the label_model label_model.fit(L_train, Y_dev, n_epochs=5000, log_freq=500) # accuracy for the label model using the test set label_model_acc = label_model.score(L=L_test, Y=Y_test, tie_break_policy="random")["accuracy"] print(f"{'Label Model Accuracy:':<25} {label_model_acc * 100:.1f}%") # check the F-1 score and ROC_AUC score probs_dev = label_model.predict_proba(L_dev) preds_dev = probs_to_preds(probs_dev) print( f"Label model f1 score: {metric_score(Y_dev, preds_dev, probs=probs_dev, metric='f1')}" ) print( f"Label model roc-auc: {metric_score(Y_dev, preds_dev, probs=probs_dev, metric='roc_auc')}" ) return label_model, L_train
def labeling_evaluation(df_train, df_test, label_model): lfs = [ LabelingFunction.lf_ind_keyword, LabelingFunction.lf_short, LabelingFunction.lf_cmp_re, LabelingFunction.lf_industry_keyword, LabelingFunction.lf_surname_re, LabelingFunction.industry_cls ] applier = PandasLFApplier(lfs=lfs) L_train = applier.apply(df=df_train) L_test = applier.apply(df=df_test) Y_test = df_test.label.values analysis = LFAnalysis(L=L_train, lfs=lfs).lf_summary() if label_model == "majority": majority_model = MajorityLabelVoter() preds_train = majority_model.predict(L=L_train) majority_acc = majority_model.score( L=L_test, Y=Y_test, tie_break_policy="random")["accuracy"] print(f"{'Majority Vote Accuracy:':<25} {majority_acc * 100:.1f}%") df_train_filtered, preds_train_filtered = filter_unlabeled_dataframe( X=df_train, y=preds_train, L=L_train) return df_train_filtered, preds_train_filtered, analysis if label_model == "weighted": label_model = LabelModel(cardinality=len( [c for c in dir(Polarity) if not c.startswith("__")]), verbose=True) label_model.fit(L_train=L_train, n_epochs=500, log_freq=100, seed=123) probs_train = label_model.predict_proba(L_train) label_model_acc = label_model.score( L=L_test, Y=Y_test, tie_break_policy="random")["accuracy"] print(f"{'Label Model Accuracy:':<25} {label_model_acc * 100:.1f}%") df_train_filtered, probs_train_filtered = filter_unlabeled_dataframe( X=df_train, y=probs_train, L=L_train) preds_train_filtered = probs_to_preds(probs_train_filtered) return df_train_filtered, probs_train_filtered, preds_train_filtered, analysis
def model_analysis(label_model: LabelModel, training_set: pd.DataFrame, L_train: np.ndarray, L_test: np.ndarray, Y_test: np.ndarray, lfs: list, output_file="output") -> None: # TODO: consider using **kwargs instead of this painful list of arguments """Output analysis for the label model to a file :param label_model: The current label model which we want to output analysis for :type label_model: LabelModel :param training_set: A dataframe containing the training dataset :type training_set: pd.DataFrame :param L_train: The matrix of labels generated by the labeling functions on the training data :type L_train: np.ndarray :param L_test: The matrix of labels generated bt the labeling functions on the testing data :type L_test: np.ndarray :param Y_test: Gold labels associated with data points in L_test :type Y_test: np.ndarray :param lfs: List of labeling functions :type lfs: list :param output_file: A path where the output file should be writtent to, defaults to `PROJECT_ROOT/output` :type output_file: str, optional """ Y_train = label_model.predict_proba(L=L_train) Y_pred = label_model.predict(L=L_test, tie_break_policy="abstain") lf_analysis_train = LFAnalysis(L=L_train, lfs=lfs).lf_summary() # TODO: Write this df to a output file. Ask Jennifer about how to handle this print(lf_analysis_train) # build majority label voter model majority_model = MajorityLabelVoter() majority_acc = majority_model.score(L=L_test, Y=Y_test, tie_break_policy="abstain", metrics=["f1", "accuracy"]) label_model_acc = label_model.score(L=L_test, Y=Y_test, tie_break_policy="abstain", metrics=["f1", "accuracy"]) # get precision and recall scores p_score = precision_score(y_true=Y_test, y_pred=Y_pred, average='weighted') r_score = recall_score(y_true=Y_test, y_pred=Y_pred, average='weighted', labels=np.unique(Y_pred)) # how many documents abstained probs_train = majority_model.predict_proba(L=L_train) df_train_filtered, probs_train_filtered = filter_unlabeled_dataframe( X=training_set, y=probs_train, L=L_train) # get number of false positives buckets = get_label_buckets(Y_test, Y_pred) true_positives, false_positives, true_negatives, false_negatives = ( buckets.get((1, 1)), buckets.get((1, 0)), buckets.get( (0, 0)), buckets.get((0, 1))) # write analysis to file timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") with open(f"{'../output/logs/'}{output_file}_run_{timestamp}.txt", "w") as output_file: output_file.write( f"{'Majority Vote Accuracy:':<25} {majority_acc['accuracy'] * 100:.2f}%" ) output_file.write( f"\n{'Majority Vote F1 Score:':<25} {majority_acc['f1'] * 100:.2f}%" ) output_file.write( f"\n{'Label Model Accuracy:':<25} {label_model_acc['accuracy'] * 100:.2f}%" ) output_file.write( f"\n{'Label Model F1 Score:':<25} {label_model_acc['f1'] * 100:.2f}%" ) output_file.write(f"\n{'Precision Score:':<25} {p_score * 100:.2f}%") output_file.write(f"\n{'Recall Score:':<25} {r_score * 100:.2f}%") output_file.write( f"\n{'Abstained Data Points:':<25} {len(df_train_filtered)}") output_file.write( f"\n{'True Positives:':<25} {len(true_positives) if true_positives is not None else 0}" ) output_file.write( f"\n{'False Positives:':<25} {len(false_positives) if false_positives is not None else 0}" ) output_file.write( f"\n{'False Negatives:':<25} {len(false_negatives) if false_negatives is not None else 0}" ) output_file.write( f"\n{'True Negatives:':<25} {len(true_negatives) if true_negatives is not None else 0}" ) output_file.write( f"\n{'Abstained Positives:':<25} {len(buckets[(1, -1)])}") output_file.write( f"\n{'Abstained Negatives:':<25} {len(buckets[(0, -1)])}")
lfs = config['lfs'] print('reading in data...') X_train, y_train, X_dev, y_dev = read_data_from_config(config) print('applying labelling functions to data...') applier = PandasLFApplier(lfs=lfs) L_train = applier.apply(df=X_train) L_dev = applier.apply(df=X_dev) print('fitting Label Model') label_model = LabelModel(cardinality=config['cardinality'], verbose=True) label_model.fit(L_train=L_train, n_epochs=500, log_freq=100, seed=123) label_model_acc = label_model.score(L=L_dev, Y=y_dev, tie_break_policy="random")["accuracy"] print(f'label model acc: {label_model_acc}') print('fitting Majority Label Voter model') majority_model = MajorityLabelVoter(cardinality=config['cardinality']) # preds_train = majority_model.predict(L=L_train) majority_acc = majority_model.score(L=L_dev, Y=np.array(y_dev).reshape(-1, 1), tie_break_policy="random")["accuracy"] print(f'majority_label_acc: {majority_acc}') log_metric('majority_label_acc', majority_acc) log_metric('label_model_acc', label_model_acc) probs_train = label_model.predict_proba(L=L_train)
preds_train = majority_model.predict(L=L_train) # use LabelModel to produce training labels label_model = LabelModel(cardinality=2, verbose=True) label_model.fit(L_train=L_train, n_epochs=500, log_freq=100, seed=123) # result using majority-vote model Y_test = data_test.label.values majority_acc = majority_model.score(L=L_test, Y=Y_test, tie_break_policy="random")["accuracy"] print(f"{'Majority Vote Accuracy:':<25} {majority_acc * 100:.1f}%") # results using label model label_model_acc = label_model.score(L=L_test, Y=Y_test, tie_break_policy="random")["accuracy"] print(f"{'Label Model Accuracy:':<25} {label_model_acc * 100:.1f}%") # representing each data point using "bag of n-gram" feature probs_train = label_model.predict_proba(L=L_train) vectorizer = CountVectorizer(ngram_range=(1, 5)) X_train = vectorizer.fit_transform(data_train.text.tolist()) X_test = vectorizer.transform(data_test.text.tolist()) # replace each label distribution with the label having maximum probability preds_train = probs_to_preds(probs=probs_train) # train a Scikit-Learn classifier sklearn_model = LogisticRegression(C=1e3, solver="liblinear") sklearn_model.fit(X=X_train, y=preds_train)
# %% [markdown] # ## 3. Train Label Model # We now train a multi-class `LabelModel` to assign training labels to the unalabeled training set. # %% from snorkel.labeling.model import LabelModel label_model = LabelModel(cardinality=3, verbose=True) label_model.fit(L_train, seed=123, lr=0.01, log_freq=10, n_epochs=100) # %% [markdown] # We use [F1](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.f1_score.html) Micro average for the multiclass setting, which calculates metrics globally across classes, by counting the total true positives, false negatives and false positives. # %% label_model.score(L_valid, Y_valid, metrics=["f1_micro"]) # %% [markdown] # ## 4. Train a Classifier # You can then use these training labels to train any standard discriminative model, such as [an off-the-shelf ResNet](https://github.com/KaimingHe/deep-residual-networks), which should learn to generalize beyond the LF's we've developed! # %% [markdown] # #### Create DataLoaders for Classifier # %% from snorkel.classification import DictDataLoader from model import SceneGraphDataset, create_model df_train["labels"] = label_model.predict(L_train) if sample:
label_model = LabelModel(cardinality=2, verbose=True) label_model.fit(L_train=L_train, n_epochs=500, log_freq=100, seed=123) L_test = applier.apply(test_df) # to_numerical = lambda x: x=='leave' # Y_test = [to_numerical(item) for item in test_df.label] Y_test = [] for item in test_df.label: if item == 'stay': Y_test.append(STAY) else: Y_test.append(LEAVE) Y_test = np.asarray(Y_test) label_model_performance = label_model.score(L=L_test, Y=Y_test, tie_break_policy="random", metrics=['accuracy', 'precision', 'recall', 'f1']) print(f"Label Model Accuracy: {label_model_performance['accuracy'] * 100:.1f}%") predict_probs = label_model.predict_proba(L_unlabeled) preds = probs_to_preds(predict_probs) pred_labels = [] for i in range(len(preds)): if preds[i]: pred_labels.append('leave') else: pred_labels.append('stay') unlabeled_data['label'] = pred_labels unlabeled_data.to_csv(os.path.join(data_dir, 'snorkel_labeled_data.csv'), sep=',', index=False)