def test_save_and_load(self): L = np.array([[0, -1, 0], [0, 1, 0]]) label_model = LabelModel(cardinality=2, verbose=False) label_model.fit(L, n_epochs=1) dir_path = tempfile.mkdtemp() save_path = dir_path + "label_model" label_model.save(save_path) label_model.load(save_path) shutil.rmtree(dir_path)
def test_save_and_load(self): L = np.array([[0, -1, 0], [0, 1, 1]]) label_model = LabelModel(cardinality=2, verbose=False) label_model.fit(L, n_epochs=1) original_preds = label_model.predict(L) dir_path = tempfile.mkdtemp() save_path = dir_path + "label_model.pkl" label_model.save(save_path) label_model_new = LabelModel(cardinality=2, verbose=False) label_model_new.load(save_path) loaded_preds = label_model_new.predict(L) shutil.rmtree(dir_path) np.testing.assert_array_equal(loaded_preds, original_preds)
test_unfired_idx = [i for i,item in enumerate(test_m) if sum(item)==0] targets_test = test_L[test_fired_idx] #majority voting using snorkel's majority voting model maj_preds_test = majority_model.predict(L=test_lsnork[test_fired_idx]) maj_precision_test, maj_recall_test, maj_f1_score_test, maj_support_test = precision_recall_fscore_support(targets_test, maj_preds_test) maj_accuracy_test = compute_accuracy(maj_support_test, maj_recall_test) print("precision on *** RULE COVERD TEST SET *** of MAJORITY VOTING: {}".format(maj_precision_test)) print("recall on *** RULE COVERED TEST SET *** of MAJORITY VOTING: {}".format(maj_recall_test)) print("f1_score on *** RULE COVERED TEST SET *** of MAJORITY VOTING: {}".format(maj_f1_score_test)) print("support on *** RULE COVERED TEST SET *** of MAJORITY VOTING: {}".format(maj_support_test)) print("accuracy on *** RULE COVERED TEST SET *** of MAJORITY VOTING: {}".format(maj_accuracy_test)) #Now train snorkels label model print("Training Snorkel's LabelModel") label_model = LabelModel(cardinality=num_classes, verbose=True) label_model.fit(L_train=U_lsnork, n_epochs=1000, lr=0.001, log_freq=100, seed=123) label_model.save(os.path.join(path_dir,"saved_label_model")) snork_preds_test = label_model.predict(L=test_lsnork[test_fired_idx]) snork_precision_test, snork_recall_test, snork_f1_score_test, snork_support_test = precision_recall_fscore_support(targets_test, snork_preds_test) snork_accuracy_test = compute_accuracy(snork_support_test, snork_recall_test) print("precision on *** RULE COVERED TEST SET *** of SNORKEL VOTING: {}".format(snork_precision_test)) print("recall on *** RULE COVERED TEST SET *** of SNORKEL VOTING: {}".format(snork_recall_test)) print("f1_score on *** RULE COVERED TEST SET *** of SNORKEL VOTING: {}".format(snork_f1_score_test)) print("support on *** RULE COVERED TEST SET *** of SNORKEL VOTING: {}".format(snork_support_test)) print("accuracy on *** RULE COVERED TEST SET *** of SNORKEL VOTING: {}".format(snork_accuracy_test))
'd_typ_inpo', # injury or poisoning 'd_typ_elii', # element, ion or isotope 'bm25_relevant', 'bm25_score'] ''' df_train = df Y_train = df_train.bm25_relevant.values lfs = [ lf.has_type_diap_medd_or_bhvr, lf.is_doctor_reply, lf.has_votes, lf.enity_overlap_jacc, lf.same_author, lf.number_relations_total, lf.entity_types ] applier = PandasLFApplier(lfs) L_train = applier.apply(df=df_train) label_model = LabelModel(cardinality=2, verbose=True) label_model.fit(L_train=L_train, n_epochs=20000, lr=0.0001, log_freq=10, seed=2345) label_model.save("trained_model_ehf.lbm") print("Finished,")
class Modeler: def __init__(self, df_train, df_dev, df_valid, df_test, df_heldout, lfs={}, label_model=None): df_train["seen"] = 0 self.df_train = df_train.reset_index() self.df_dev = df_dev self.df_valid = df_valid self.df_test = df_test self.df_heldout = df_heldout #self.Y_train = df_train.label.values self.Y_dev = df_dev.label.values self.Y_valid = df_valid.label.values self.Y_test = df_test.label.values self.Y_heldout = df_heldout.label.values self.lfs = lfs self.L_train = None self.L_dev = None self.L_valid = None self.L_heldout = None cardinality = len(df_valid.label.unique()) # for DEMOing purposes self.first_text_indices = [ 1262, #"check out" "youtube" 1892, # I love 1117, # url concept 1706, # emoji concept 952, # "nice" 971, # positive concept 958, # actually use emoji concept ] self.count = 0 if label_model is None: self.label_model = LabelModel(cardinality=cardinality, verbose=True) else: self.label_model = label_model self.vectorizer = CountVectorizer(ngram_range=(1, 2)) self.vectorizer.fit(df_train.text.tolist()) def get_lfs(self): return list(self.lfs.values()) def add_lfs(self, new_lfs: dict): self.lfs.update(new_lfs) def remove_lfs(self, old_lf_ids: list): for lf_id in old_lf_ids: del self.lfs[lf_id] return len(self.lfs) def apply_lfs(self): applier = PandasLFApplier(lfs=self.get_lfs()) self.L_train = applier.apply(df=self.df_train) self.L_dev = applier.apply(df=self.df_dev) self.L_heldout = applier.apply(df=self.df_heldout) #self.L_valid = applier.apply(df=self.df_valid) def find_duplicate_signature(self): label_matrix = np.vstack([self.L_train, self.L_dev]) seen_signatures = {} dupes = {} lfs = self.get_lfs() signatures = [ hash(label_matrix[:, i].tostring()) for i in range(len(lfs)) ] for i, s in enumerate(signatures): lf = lfs[i] if s in seen_signatures: dupes[lf.name] = seen_signatures[s] else: seen_signatures[s] = lf.name return dupes def lf_examples(self, lf_id, n=5): lf = self.lfs[lf_id] applier = PandasLFApplier(lfs=[lf]) L_train = applier.apply(df=self.df_train) labeled_examples = self.df_train[L_train != -1] samples = labeled_examples.sample(min(n, len(labeled_examples)), random_state=13) return [{"text": t} for t in samples["text"].values] def lf_mistakes(self, lf_id, n=5): lf = self.lfs[lf_id] applier = PandasLFApplier(lfs=[lf]) L_dev = applier.apply(df=self.df_dev).squeeze() labeled_examples = self.df_dev[(L_dev != -1) & (L_dev != self.df_dev["label"])] samples = labeled_examples.sample(min(n, len(labeled_examples)), random_state=13) return [{"text": t} for t in samples["text"].values] def fit_label_model(self): assert self.L_train is not None self.label_model.fit(L_train=self.L_train, n_epochs=1000, lr=0.001, log_freq=100, seed=123) def analyze_lfs(self): if len(self.lfs) > 0: df = LFAnalysis(L=self.L_train, lfs=self.get_lfs()).lf_summary() dev_df = LFAnalysis(L=self.L_dev, lfs=self.get_lfs()).lf_summary(Y=self.Y_dev) df = df.merge(dev_df, how="outer", suffixes=(" Training", " Dev."), left_index=True, right_index=True) df["Weight"] = self.label_model.get_weights() df["Duplicate"] = None for dupe, OG in self.find_duplicate_signature().items(): print("Duplicate labeling signature detected") print(dupe, OG) df.at[dupe, "Duplicate"] = OG return df return None def get_label_model_stats(self): result = self.label_model.score(L=self.L_dev, Y=self.Y_dev, metrics=["f1", "precision", "recall"]) probs_train = self.label_model.predict_proba(L=self.L_train) df_train_filtered, probs_train_filtered = filter_unlabeled_dataframe( X=self.df_train, y=probs_train, L=self.L_train) result["training_label_coverage"] = len(probs_train_filtered) / len( probs_train) result["class_0_ratio"] = (probs_train_filtered[:, 0] > 0.5).sum() / len(probs_train_filtered) if len(probs_train_filtered) == 0: result["class_0_ratio"] = 0 return result def get_heldout_stats(self): if self.L_heldout is not None: return self.label_model.score( L=self.L_heldout, Y=self.Y_heldout, metrics=["f1", "precision", "recall"]) return {} def train(self): probs_train = self.label_model.predict_proba(L=self.L_train) df_train_filtered, probs_train_filtered = filter_unlabeled_dataframe( X=self.df_train, y=probs_train, L=self.L_train) if len(df_train_filtered) == 0: print("Labeling functions cover none of the training examples!", file=sys.stderr) return {"micro_f1": 0} #from tensorflow.keras.utils import to_categorical #df_train_filtered, probs_train_filtered = self.df_dev, to_categorical(self.df_dev["label"].values) vectorizer = self.vectorizer X_train = vectorizer.transform(df_train_filtered.text.tolist()) X_dev = vectorizer.transform(self.df_dev.text.tolist()) X_valid = vectorizer.transform(self.df_valid.text.tolist()) X_test = vectorizer.transform(self.df_test.text.tolist()) self.keras_model = get_keras_logreg(input_dim=X_train.shape[1]) self.keras_model.fit( x=X_train, y=probs_train_filtered, validation_data=(X_valid, preds_to_probs(self.Y_valid, 2)), callbacks=[get_keras_early_stopping()], epochs=20, verbose=0, ) preds_test = self.keras_model.predict(x=X_test).argmax(axis=1) #return preds_test return self.get_stats(self.Y_test, preds_test) def get_heldout_lr_stats(self): X_heldout = self.vectorizer.transform(self.df_heldout.text.tolist()) preds_test = self.keras_model.predict(x=X_heldout).argmax(axis=1) return self.get_stats(self.Y_heldout, preds_test) def get_stats(self, Y_test, preds_test): label_classes = np.unique(self.Y_test) accuracy = metrics.accuracy_score(Y_test, preds_test) precision_0, precision_1 = metrics.precision_score( Y_test, preds_test, labels=label_classes, average=None) recall_0, recall_1 = metrics.recall_score(Y_test, preds_test, labels=label_classes, average=None) test_f1 = metrics.f1_score(Y_test, preds_test, labels=label_classes) #recall_0, recall_1 = metrics.precision_recall_fscore_support(self.Y_test, preds_test, labels=label_classes)["recall"] return { "micro_f1": test_f1, "recall_0": recall_0, "precision_0": precision_0, "accuracy": accuracy, "recall_1": recall_1, "precision_1": precision_1 } def entropy(self, prob_dist): #return(-(L_row_i==-1).sum()) return (-sum([x * log(x) for x in prob_dist])) def save(self, dir_name): self.label_model.save(os.path.join(dir_name, 'label_model.pkl')) with open(os.path.join(dir_name, 'model_lfs.pkl'), "wb+") as file: pickle.dump(self.lfs, file) def load(self, dir_name): with open(os.path.join(dir_name, 'model_lfs.pkl'), "rb") as file: lfs = pickle.load(file) label_model = LabelModel.load( os.path.join(dir_name, 'label_model.pkl')) self.lfs = lfs self.label_model = label_model
cui2vec = Cui2Vec().cui2vec X_gold_sent, X_gold_shortest_path, X_gold_src, X_gold_tgt, X_gold_src_txt, X_gold_tgt_txt, y_gold = data_handler.get_test_data() X_val_sent, X_val_shortest_path, X_val_src, X_val_tgt, X_val_src_txt, X_val_tgt_txt, y_val = data_handler.get_validation_data() applier = PandasLFApplier(label_functions.lfs) df_train = pd.DataFrame(list(zip(*data_handler.get_training_data())), columns=['shortest_path', 'sent', 'src', 'tgt', 'src_txt', 'tgt_txt']) L_train = applier.apply(df_train) label_model = LabelModel(cardinality=len(rel_names.rels_txt_to_int), verbose=True) label_model.fit(L_train, n_epochs=1000, lr=0.01, log_freq=100, seed=123) label_model.save('./models/LabelModel.model') train_probs = label_model.predict_proba(L_train) train_preds = probs_to_preds(train_probs, tie_break_policy='abstain') df_train = df_train.join(pd.DataFrame({'preds': train_preds, 'probs': list(map(max, train_probs))})) # -1 to otherwiseRelated df_train.loc[df_train.preds == -1, 'preds'] = rel_names.rels_txt_to_int['otherwiseRelated'] # Downsample otherwiseRelated dropNum = len(df_train[df_train.preds == rel_names.rels_txt_to_int['otherwiseRelated']]) - int(df_train['preds'].value_counts().mean()) df_train = df_train.drop(df_train[df_train.preds == rel_names.rels_txt_to_int['otherwiseRelated']].sample(dropNum).index) cnts = {} for x in df_train['preds']: