def test_score(self):
L = np.array([[1, 1, 0], [-1, -1, -1], [1, 0, 1]])
Y = np.array([1, 0, 1])
label_model = LabelModel(cardinality=2, verbose=False)
label_model.fit(L, n_epochs=100)
results = label_model.score(L, Y, metrics=["accuracy", "coverage"])
np.testing.assert_array_almost_equal(label_model.predict(L),
np.array([1, -1, 1]))
results_expected = dict(accuracy=1.0, coverage=2 / 3)
self.assertEqual(results, results_expected)
L = np.array([[1, 0, 1], [1, 0, 1]])
label_model = self._set_up_model(L)
label_model.mu = nn.Parameter(label_model.mu_init.clone().clamp(
0.01, 0.99))
results = label_model.score(L, Y=np.array([0, 1]))
results_expected = dict(accuracy=0.5)
self.assertEqual(results, results_expected)
results = label_model.score(L=L,
Y=np.array([1, 0]),
metrics=["accuracy", "f1"])
results_expected = dict(accuracy=0.5, f1=2 / 3)
self.assertEqual(results, results_expected)
def test_label_model_basic(self) -> None:
"""Test the LabelModel's estimate of P and Y on a simple synthetic dataset."""
np.random.seed(123)
P, Y, L = generate_simple_label_matrix(self.n, self.m, self.cardinality)
# Train LabelModel
label_model = LabelModel(cardinality=self.cardinality, verbose=False)
label_model.fit(L, n_epochs=200, lr=0.01, seed=123)
# Test estimated LF conditional probabilities
P_lm = label_model.get_conditional_probs()
np.testing.assert_array_almost_equal(P, P_lm, decimal=2)
# Test predicted labels
score = label_model.score(L, Y)
self.assertGreaterEqual(score["accuracy"], 0.9)
# The only information we need is the label matrix, which contains the output of the LFs on our training set.
# The `LabelModel` is able to learn weights for the labeling functions using only the label matrix as input.
# We also specify the `cardinality`, or number of classes.
# The `LabelModel` trains much more quickly than typical discriminative models since we only need the label matrix as input.
# %% {"tags": ["md-exclude-output"]}
from snorkel.labeling import LabelModel
label_model = LabelModel(cardinality=2, verbose=True)
label_model.fit(L_train=L_train, n_epochs=500, lr=0.001, log_freq=100, seed=123)
# %%
majority_acc = majority_model.score(L=L_valid, Y=Y_valid)["accuracy"]
print(f"{'Majority Vote Accuracy:':<25} {majority_acc * 100:.1f}%")
label_model_acc = label_model.score(L=L_valid, Y=Y_valid)["accuracy"]
print(f"{'Label Model Accuracy:':<25} {label_model_acc * 100:.1f}%")
# %% [markdown]
# So our `LabelModel` improves over the majority vote baseline!
# However, it is typically **not suitable as an inference-time model** to make predictions for unseen data points, due to (among other things) some data points having all abstain labels.
# In the next section, we will use the output of the label model as training labels to train a
# discriminative classifier to see if we can improve performance further.
# This classifier will only need the text of the comment to make predictions, making it much more suitable
# for inference over unseen comments.
# For more information on the properties of the label model and when to use it, see the [Snorkel guides]().
# %% [markdown]
# We can also run error analysis after the label model has been trained.
# For example, let's take a look at 5 random false negatives from the `dev` set, which might inspire some more LFs that vote `SPAM`.
class SklearnLabelModel(BaseEstimator, ClassifierMixin):
"""A sklearn wrapper for LabelModel for using sklearn GridSearch, RandomSearch.
Uses output of create_param_search_data.
Note that all hyperparameters for the fit and score functions are accepted at the time the class is defined.
Examples
--------
>>> L_train = np.array([[1, 1, 1, 0], [0, 1, 1, 0], [1, 0, 0, 1], [0, 1, 0, 1], [0, 0, 0, 1]])
>>> L_dev = np.array([[1, 1, 1, 0], [0, 1, 1, 0], [1, 0, 0, 1], [0, 1, 0, 1]])
>>> Y_dev = np.array([1, 1, 0, 1])
>>> from sklearn.model_selection import GridSearchCV
>>> param_grid = [{"lr": [0.01, 0.0000001],
... "l2": [0.001, 0.5], "metric": ["accuracy"]}]
>>> label_model = SklearnLabelModel()
>>> L, Y, cv_split = label_model.create_param_search_data(L_train, L_dev, Y_dev)
>>> clf = GridSearchCV(label_model, param_grid, cv=cv_split)
>>> clf = clf.fit(L, Y)
>>> clf.best_score_
0.75
>>> from sklearn.model_selection import RandomizedSearchCV
>>> param_dist = {"lr": np.random.uniform(low=1e-7, high=1e-2, size=(50,)),
... "l2": np.random.uniform(low=1e-2, high=0.5, size=(50,))}
>>> clf = RandomizedSearchCV(label_model, param_distributions=param_dist, n_iter=4, cv=cv_split, iid=False)
>>> clf = clf.fit(L, Y)
>>> np.around(clf.best_score_)
1.0
Parameters
----------
cardinality
Number of classes, by default 2
verbose
Whether to include print statements
device
What device to place the model on ('cpu' or 'cuda:0', for example)
metric
The metric to report with score()
tie_break_policy
Policy to break ties when converting probabilistic labels to predictions
n_epochs
The number of epochs to train (where each epoch is a single optimization step)
lr
Base learning rate (will also be affected by lr_scheduler choice and settings)
l2
Centered L2 regularization strength
optimizer
Which optimizer to use (one of ["sgd", "adam", "adamax"])
optimizer_config
Settings for the optimizer
lr_scheduler
Which lr_scheduler to use (one of ["constant", "linear", "exponential", "step"])
lr_scheduler_config
Settings for the LRScheduler
prec_init
LF precision initializations / priors
seed
A random seed to initialize the random number generator with
log_freq
Report loss every this many epochs (steps)
mu_eps
Restrict the learned conditional probabilities to [mu_eps, 1-mu_eps]
"""
def __init__(
self,
cardinality: int = 2,
verbose: bool = True,
device: str = "cpu",
metric: str = "accuracy",
tie_break_policy: str = "abstain",
n_epochs: int = 100,
lr: float = 0.01,
l2: float = 0.0,
optimizer: str = "sgd",
optimizer_config: Optional[OptimizerConfig] = None,
lr_scheduler: str = "constant",
lr_scheduler_config: Optional[LRSchedulerConfig] = None,
prec_init: float = 0.7,
seed: int = np.random.randint(1e6),
log_freq: int = 10,
mu_eps: Optional[float] = None,
class_balance: Optional[List[float]] = None,
**kwargs: Any,
) -> None:
self.cardinality = cardinality
self.verbose = verbose
self.device = device
self.metric = metric
self.tie_break_policy = tie_break_policy
self.n_epochs = n_epochs
self.lr = lr
self.l2 = l2
self.optimizer = optimizer
self.optimizer_config = (
optimizer_config if optimizer_config is not None else
OptimizerConfig() # type: ignore
)
self.lr_scheduler = lr_scheduler
self.lr_scheduler_config = (
lr_scheduler_config if lr_scheduler_config is not None else
LRSchedulerConfig() # type: ignore
)
self.prec_init = prec_init
self.seed = seed
self.log_freq = log_freq
self.mu_eps = mu_eps
self.class_balance = class_balance
self.label_model = LabelModel(cardinality=self.cardinality,
verbose=self.verbose,
device=self.device)
def fit(self,
L: np.ndarray,
Y: Optional[np.ndarray] = None) -> "SklearnLabelModel":
"""
Train label model.
Parameters
----------
L
An [n,m] matrix with values in {-1,0,1,...,k-1}
Y
Placeholder, not used for training model.
Returns
-------
SklearnLabelModel
"""
self.label_model.fit(
L_train=L,
class_balance=self.class_balance,
n_epochs=self.n_epochs,
lr=self.lr,
l2=self.l2,
optimizer=self.optimizer,
optimizer_config=self.optimizer_config,
lr_scheduler=self.lr_scheduler,
lr_scheduler_config=self.lr_scheduler_config,
prec_init=self.prec_init,
seed=self.seed,
log_freq=self.log_freq,
mu_eps=self.mu_eps,
)
return self
def score(self, L: np.ndarray, Y: np.ndarray) -> float:
"""Calculate score using self.metric.
Parameters
----------
L
An [n,m] matrix with values in {-1,0,1,...,k-1}
Y
Gold labels associated with data points in L
Returns
-------
float
Score for metric speficied in self.metric
"""
results = self.label_model.score(L, Y, [self.metric],
self.tie_break_policy)
return results[self.metric]
@staticmethod
def create_param_search_data(L_train: np.ndarray, L_dev: np.ndarray,
Y_dev: np.ndarray) -> PredefinedSplit:
"""
Create predefined cross validation split for SklearnLabelModel wrapper.
Returns combined L, Y matrix with corresponding CV split object with two splits.
Parameters
----------
L_train
An [n,m] matrix with values in {-1,0,1,...,k-1} used for training
L_dev
An [n,m] matrix with values in {-1,o0,1,...,k-1} used for scoring
Y_dev
Gold labels associated with L_dev
Returns
-------
L
Combined L_train and L_dev matrix
Y
Combined Y_train (all -1s) and Y_dev array
cv_split
PredefinedSplit object with two splits: L_train in train and *_dev in test
"""
n_train = np.shape(L_train)[0]
n_dev = np.shape(L_dev)[0]
if n_dev != np.shape(Y_dev)[0]:
raise ValueError(
"Num. datapoints in Y_dev and L_dev do not match.")
# combine train and dev L and Y
L_all = np.append(L_train, L_dev, axis=0)
Y_all = np.append(-1 * np.ones(n_train, dtype=int), Y_dev)
# create cv split array with one predefined split (train, dev)
test_fold = np.append(-1 * np.ones(n_train, dtype=int),
np.ones(n_dev, dtype=int))
cv_split = PredefinedSplit(test_fold=test_fold)
return L_all, Y_all, cv_split
def __repr__(self) -> str:
"""Pretty print."""
return str(vars(self))
def run_snorkel_labelling_classification(labeling_functions, file, l_train,
l_valid):
lfs = labeling_functions
# lfs = [lf.is_same_thread, lf.has_entities, lf.enity_overlap_jacc, lf.entity_type_overlap_jacc]
# lfs = [is_same_thread, enity_overlap, entity_types, entity_type_overlap]
# lfs = [is_long, has_votes, is_doctor_reply, is_same_thread, enity_overlap, has_type_dsyn, has_type_patf, has_type_sosy,
# has_type_dora, has_type_fndg, has_type_menp, has_type_chem, has_type_orch, has_type_horm, has_type_phsu,
# has_type_medd, has_type_bhvr, has_type_diap, has_type_bacs, has_type_enzy, has_type_inpo, has_type_elii]
# lfs = [has_votes, is_doctor_reply, is_same_thread, enity_overlap]
# lfs = [is_same_thread, enity_overlap, is_doctor_reply]
# analysis = LFAnalysis(L=l_train, lfs=lfs).lf_summary(Y=Y_train)
# print(analysis)
# print(analysis['Conflicts'])
# print(analysis['Overlaps'])
label_model = LabelModel(cardinality=2, verbose=True)
label_model.fit(L_train=l_train,
n_epochs=20000,
lr=0.0001,
log_freq=10,
seed=2345)
# label_model.fit(L_train=L_train, n_epochs=20, lr=0.0001, log_freq=10, seed=81794)
print("Model weights: " + str(label_model.get_weights()))
valid_probabilities = label_model.predict_proba(L=l_valid)
if 'predicted_prob' in df_valid:
# df_valid.drop(columns=['predicted_prob'], axis=1)
del df_valid['predicted_prob']
df_valid.insert(50, 'predicted_prob', valid_probabilities[:, 1])
# df_valid.to_csv("/container/filip/json/ehealthforum/trac/validation_df2.txt", sep="\t", header=True)
# df_valid = pd.read_csv("/filip/json/ehealthforum/trac/validation_df.txt", sep="\t")
def compute_precision_at_k(l, k):
l = l[:k]
return sum(l) / k
PROBABILITY_CUTOFF = 0.5
df_valid[
'predicted_label'] = df_valid['predicted_prob'] >= PROBABILITY_CUTOFF
true_positive_ratio = df_valid[df_valid.bm25_relevant == 1].count()['bm25_relevant'] / \
df_valid[df_valid.predicted_label == 1].count()['predicted_label']
print("Number of True relevant: " +
str(df_valid[df_valid.bm25_relevant == 1].count()['bm25_relevant']))
print("Number of Predicted relevant: " + str(df_valid[
df_valid.predicted_label == 1].count()['predicted_label']) + '\n')
print('True positive ratio: ' + str(true_positive_ratio) + '\n')
df_tru = df_valid.groupby(['query_thread']).head(10)['bm25_relevant']
df_pred = df_valid.groupby(['query_thread']).head(10)['predicted_label']
overall_precision = []
for query, group in df_valid.groupby(['query_thread']):
precision = compute_precision_at_k(
group['predicted_label'].head(10).tolist(), 10)
overall_precision.append(precision)
print('Overall precision: ' +
str(sum(overall_precision) / len(overall_precision)))
print("Accuracy: " + str(accuracy_score(df_tru, df_pred)))
label_model_acc = label_model.score(L=l_valid, Y=Y_valid)["accuracy"]
print(f"{'Label Model Accuracy:':<25} {label_model_acc * 100:.1f}%")
def main(train_path, output_dir, label_dir):
# Get all data
df = pd.read_csv(train_path)
# Get human labels
human_labels = read_human_labels(label_dir)
# df_test and lab_test: the set of all human-labeled notes, and their labels
df_test = df.merge(human_labels, on=['record_number'])
lab_test = df_test.human_label
del df_test['human_label']
# df_train: formed by removing all patients from df with a human-labeled note
df_train = df.merge(df_test.mr, indicator=True, how='left', on = ['mr'])
df_train = df_train.query('_merge=="left_only"').drop('_merge', axis=1)
# Generate label matrix
L_train = PandasLFApplier(lfs=lfs).apply(df=df_train)
L_test = PandasLFApplier(lfs=lfs).apply(df=df_test)
# Summarize LFs
output_train = LFAnalysis(L=L_train, lfs=lfs).lf_summary()
#print(output_train)
output_test = LFAnalysis(L=L_test, lfs=lfs).lf_summary(Y = lab_test.values)
#print(output_test)
# Save LF analysis
path = os.path.join(output_dir, 'LF_analysis_train.csv')
output_train.to_csv(path, index = True)
path = os.path.join(output_dir, 'LF_analysis_test.csv')
output_test.to_csv(path, index = True)
# Create label model
label_model = LabelModel(cardinality=2, verbose=True)
label_model.fit(L_train=L_train, n_epochs=500, log_freq=100, seed=123, class_balance = [0.3, 0.7])
# Evaluate the label model using labeled test set
for metric in ['recall', 'precision', 'f1', 'accuracy']:
label_model_acc = label_model.score(L=L_test, Y=lab_test, metrics=[metric], tie_break_policy="random")[metric]
print("%-15s %.2f%%" % (metric+":", label_model_acc * 100))
null_f1 = f1_score(lab_test.values, np.ones((df_test.shape[0],)))
print("%-15s %.2f%%" % ("null f1:", null_f1 * 100))
print("%-15s %.2f%%" % ("null accuracy:", np.maximum(1-np.mean(lab_test), np.mean(lab_test)) * 100))
# Save error analysis
preds = label_model.predict_proba(L_test)
error_analysis(df_test, L_test, lfs, preds[:,1], lab_test, output_dir)
# Get labels on train
probs_train = label_model.predict_proba(L_train)
# Filter out unlabeled data points
df_train_filtered, probs_train_filtered = filter_unlabeled_dataframe(X=df_train, y=probs_train, L=L_train)
# Save filtered training set
df_train_filtered['prob'] = probs_train_filtered[:,1]
path = os.path.join(output_dir, 'df_train_filtered.csv')
df_train_filtered.to_csv(path, index = False)
# Save label probs
path = os.path.join(output_dir, 'probs_train_filtered')
np.save(path, probs_train_filtered[:,1])
# Save training data set and labels
assert len(df_test) == len(lab_test)
df_test['human_label'] = lab_test
path = os.path.join(output_dir, 'df_test.csv')
df_test.to_csv(path, index = False)
path = os.path.join(output_dir, 'lab_test')
np.save(path, lab_test)
# %% [markdown] # ## 3. Train Label Model # We now train a multi-class `LabelModel` to assign training labels to the unalabeled training set. # %% from snorkel.labeling import LabelModel label_model = LabelModel(cardinality=3, verbose=True) label_model.fit(L_train, seed=123, lr=0.01, log_freq=10, n_epochs=100) # %% [markdown] # We use [F1](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.f1_score.html) Micro average for the multiclass setting, which calculates metrics globally across classes, by counting the total true positives, false negatives and false positives. # %% label_model.score(L_valid, Y_valid, metrics=["f1_micro"]) # %% [markdown] # ## 4. Train a Classifier # You can then use these training labels to train any standard discriminative model, such as [an off-the-shelf ResNet](https://github.com/KaimingHe/deep-residual-networks), which should learn to generalize beyond the LF's we've developed! # %% [markdown] # #### Create DataLoaders for Classifier # %% from snorkel.classification import DictDataLoader from model import SceneGraphDataset, create_model df_train["labels"] = label_model.predict(L_train) if sample:
def label_user(inp_path, prefix=""):
df_train = pd.read_pickle(inp_path)
########## threshold on word similarity
take_first = 100
overall_first = 10000
global thresh_by_value, overall_thresh
df_train['root_value'] = df_train['value'].swifter.set_dask_threshold(
dask_threshold=0.001).allow_dask_on_strings().apply(
lambda x: syn_to_hob[x])
thresh_by_value = df_train.groupby(
["root_value"]).apply(lambda x: np.partition(
x['lexicon_counts'], max(len(x['lexicon_counts']) - take_first, 0)
)[max(len(x['lexicon_counts']) - take_first, 0)]).to_dict()
overall_thresh = np.partition(df_train["lexicon_counts"].to_numpy(),
max(len(df_train) - overall_first, 0))[max(
len(df_train) - overall_first, 0)]
print(overall_thresh)
#############################
# separately loose - strict, pos - neg, period - without
names_pool = [
"context:2_count_pos", "context:3_count_pos", "context:100_count_pos",
"context:2_period_count_pos", "context:3_period_count_pos",
"context:100_period_count_pos", "context:2_count_neg",
"context:3_count_neg", "context:100_count_neg",
"context:2_period_count_neg", "context:3_period_count_neg",
"context:100_period_count_neg"
]
for f_name in names_pool:
curr_cols = [x for x in df_train.columns if f_name in x]
df_train['total_' + f_name] = df_train[curr_cols].swifter.apply(sum,
axis=1)
df_train = df_train.drop(curr_cols, axis=1)
for p in ["pos", "neg"]:
df_train["new_total_context:100_count_" + p] = df_train[[
"total_context:100_count_" + p, "total_context:3_count_" + p
]].swifter.apply(lambda x: max(
0, x["total_context:100_count_" + p] - x["total_context:3_count_" +
p]),
axis=1)
df_train["new_total_context:3_count_" + p] = df_train[[
"total_context:3_count_" + p, "total_context:2_count_" + p
]].swifter.apply(lambda x: max(
0, x["total_context:3_count_" + p] - x["total_context:2_count_" + p
]),
axis=1)
df_train["new_total_context:100_period_count_" + p] = df_train[[
"total_context:3_period_count_" + p,
"total_context:100_period_count_" + p
]].swifter.apply(lambda x: max(
0, x["total_context:100_period_count_" + p] - x[
"total_context:3_period_count_" + p]),
axis=1)
df_train["new_total_context:3_period_count_" + p] = df_train[[
"total_context:3_period_count_" + p,
"total_context:2_period_count_" + p
]].swifter.apply(lambda x: max(
0, x["total_context:3_period_count_" + p] - x[
"total_context:2_period_count_" + p]),
axis=1)
df_train["new_total_context:2_count_" + p] = df_train[[
"total_context:100_period_count_" + p, "total_context:2_count_" + p
]].swifter.apply(lambda x: max(
0, x["total_context:2_count_" + p] - x[
"total_context:100_period_count_" + p]),
axis=1)
df_train = df_train.drop(
["total_" + x for x in names_pool if "2_period_count" not in x],
axis=1)
lfs = [val_in_name, positive_lexicon_overall, positive_lexicon_pervalue]
num_of_thesholds = 3
step = 100 // num_of_thesholds
for col in df_train:
if col not in ["author", "value", "idd", "root_value"]:
if col not in [
"pos_prob_mean", "neg_prob_mean", "num_good_posts"
]: # , "lexicon_counts", "subreddit_counts", "name_in_subr_count"]:
thresholds = [0]
if "lexicon" in col and "unique" not in col:
continue
if True: # col in ["lexicon_counts", "unique_lexicon_counts"]:
vals = df_train[col].to_numpy()
thresholds = np.percentile(
vals, list(range(0 + step, 99 + step,
step))).astype(int)
thresholds = sorted(list(set(thresholds)))
if len(thresholds) > 1:
thresholds = thresholds[:-1]
if "lexicon" in col:
thresholds = [3]
# max_val = max(vals)
# thresholds = list(range(0, int(max_val), int(max_val/5) + 1))
# elif col == "pos_prob_mean":
# thresholds = [0.5 + 0.1 * x for x in range(5)]
for i in range(len(thresholds)):
thresh = thresholds[i]
next_threshold = sys.maxsize if i == len(
thresholds) - 1 else thresholds[i + 1]
previous_threshold = -sys.maxsize if i == 0 else thresholds[
i - 1]
if "lexicon_counts" not in col:
lfs.append(
make_thresold_lf(thresh=thresh,
col_name=col,
next_threshold=next_threshold))
else:
lfs.append(
make_lexicon_lf(
thresh=thresh,
pref=col,
previous_threshold=previous_threshold))
num_annotators = 0
if num_annotators > 0:
for i in range(1, num_annotators + 1):
lfs.append(make_annotator_lf(worker_index=i))
lfs = [
x for x in lfs
if any(y in str(x) for y in ["less", "context:2", "worker", "lexicon"])
]
print("created lfs their number", len(lfs))
print("\n".join(str(x) for x in lfs))
#### validation #####
do_val = False
if do_val:
df_golden = pd.read_csv(
"/home/tigunova/PycharmProjects/snorkel_labels/data/profession/gold_dev.csv"
)
name_val = list(df_golden["auth_val"])
# df_train['root_value'] = df_train['value'].swifter.apply(lambda x: syn_to_hob[x])
df_train["auth_val"] = df_train[["author", "value"]].swifter.apply(
lambda x: x["author"] + "+++" + x["value"], axis=1)
df_val = df_train[df_train.auth_val.isin(name_val)]
df_dev = df_train[~df_train.auth_val.isin(name_val)]
print("Number val", df_val.shape)
print("Number dev", df_dev.shape)
df_val = df_val.merge(df_golden, on="auth_val")
y_val = np.array(df_val["final"])
df_val = df_val.drop(labels="final", axis=1)
# create test set as well
with TQDMDaskProgressBar(desc="Dask Apply"):
applier = PandasParallelLFApplier(lfs=lfs)
L_val = applier.apply(df=df_val, n_parallel=num_cpu)
L_dev = applier.apply(df=df_dev, n_parallel=num_cpu)
dev_analysis = LFAnalysis(L=L_dev, lfs=lfs).lf_summary()
analysis = LFAnalysis(L=L_val, lfs=lfs).lf_summary(y_val)
analysis.to_csv("/home/tigunova/val_analysis.csv")
dev_analysis.to_csv("/home/tigunova/dev_analysis.csv")
print(analysis)
label_model = LabelModel(cardinality=2, verbose=True)
label_model.fit(L_dev) #, Y_dev=y_val)
model_stat = label_model.score(L=L_val, Y=y_val)
print(model_stat)
exit(0)
###########
#### picking threshold #####
do_threshold = False
if do_threshold:
df_golden = pd.read_csv(
"/home/tigunova/PycharmProjects/snorkel_labels/data/profession/gold_validation.csv"
)
name_val = list(df_golden["auth_val"])
# df_train['root_value'] = df_train['value'].swifter.apply(lambda x: syn_to_hob[x])
df_train["auth_val"] = df_train[["author", "value"]].swifter.apply(
lambda x: x["author"] + "+++" + x["value"], axis=1)
df_val = df_train[df_train.auth_val.isin(name_val)]
df_dev = df_train[~df_train.auth_val.isin(name_val)]
pop_size = df_dev.shape[0]
print("Number val", df_val.shape)
print("Number dev", df_dev.shape)
applier = PandasParallelLFApplier(lfs=lfs)
df_val = df_val.merge(df_golden, on="auth_val")
L_val = applier.apply(df=df_val, n_parallel=num_cpu)
val_thresholds = [0.01 * x for x in range(100)]
label_model = LabelModel(cardinality=2, verbose=True)
with TQDMDaskProgressBar(desc="Dask Apply"):
L_dev = applier.apply(df=df_dev, n_parallel=num_cpu)
label_model.fit(L_dev, class_balance=[0.5, 0.5]) # , Y_dev=y_val)
wghts = label_model.get_weights()
print("\n".join(str(x) for x in zip(lfs, wghts)))
probs_val = label_model.predict_proba(L=L_val)
probs_df = pd.DataFrame(probs_val,
columns=["neg_prob", "pos_prob"])
df_val = pd.concat([df_val.reset_index(), probs_df], axis=1)
probs_dev = label_model.predict_proba(L=L_dev)
probs_df = pd.DataFrame(probs_dev,
columns=["neg_prob", "pos_prob"])
df_dev = pd.concat([df_dev.reset_index(), probs_df], axis=1)
y_true = np.array(df_val["final"])
for th in val_thresholds:
y_pred = np.array(
df_val["pos_prob"].apply(lambda x: 1 if x > th else 0))
#print("true negatives")
#print(df_val[df_val["final"] == 1][df_val["pos_prob"] <= th][["auth_val", "text"]])
prec = precision_score(y_true, y_pred)
pred_labels = y_pred
true_labels = y_true
# True Positive (TP): we predict a label of 1 (positive), and the true label is 1.
TP = np.sum(np.logical_and(pred_labels == 1, true_labels == 1))
# True Negative (TN): we predict a label of 0 (negative), and the true label is 0.
TN = np.sum(np.logical_and(pred_labels == 0, true_labels == 0))
# False Positive (FP): we predict a label of 1 (positive), but the true label is 0.
FP = np.sum(np.logical_and(pred_labels == 1, true_labels == 0))
# False Negative (FN): we predict a label of 0 (negative), but the true label is 1.
FN = np.sum(np.logical_and(pred_labels == 0, true_labels == 1))
print('TP: %i, FP: %i, TN: %i, FN: %i' % (TP, FP, TN, FN))
# print(list(zip(label_model.predict(L=L_val_curr), y_val_curr)))
# print("******************************")
print("threshold %s, proportion population %.4f, precision %s" %
(str(th), df_dev[df_dev["pos_prob"] > th].shape[0] /
pop_size, str(prec)))
exit(0)
###########
with TQDMDaskProgressBar(desc="Dask Apply"):
applier = PandasParallelLFApplier(lfs=lfs)
L_train = applier.apply(df=df_train, n_parallel=num_cpu)
analysis = LFAnalysis(L=L_train, lfs=lfs).lf_summary()
print(analysis)
df_l_train = pd.DataFrame(
L_train, columns=["llf_" + str(x).split(",")[0] for x in lfs])
print(df_train.shape)
print(df_l_train.shape)
df_train = pd.concat([df_train.reset_index(), df_l_train], axis=1)
print(df_train.shape)
print("********************************************")
t4 = time.time()
label_model = LabelModel(cardinality=2, verbose=True)
label_model.fit(L_train=L_train,
n_epochs=1000,
lr=0.001,
log_freq=100,
seed=123,
class_balance=[0.3, 0.7])
probs_train = label_model.predict_proba(L=L_train)
print("labeling model work ", (time.time() - t4) / 60)
df_train_filtered, probs_train_filtered = filter_unlabeled_dataframe(
X=df_train, y=probs_train, L=L_train)
probs_df = pd.DataFrame(probs_train_filtered,
columns=["neg_prob", "pos_prob"])
print(df_train_filtered.shape)
print(probs_df.shape)
result_filtered = pd.concat([
df_train_filtered[['author', 'value', 'idd']].reset_index(), probs_df
],
axis=1)
print(result_filtered.shape)
print("****************************************************")
result_filtered.to_csv("/home/tigunova/some_result_" + prefix + ".csv")
print(df_train_filtered.shape)
print(probs_df.shape)
df_train_filtered = pd.concat([df_train_filtered.reset_index(), probs_df],
axis=1)
df_train_filtered = df_train_filtered.drop(["index"], axis=1)
print(df_train_filtered.shape)
df_train_filtered.to_pickle(
"/home/tigunova/PycharmProjects/snorkel_labels/data/profession/user_" +
prefix + ".pkl")
df_train_filtered.to_csv(
"/home/tigunova/PycharmProjects/snorkel_labels/data/profession/user_" +
prefix + ".csv")
# df_train.iloc[L_train[:, 1] == POS].to_csv("/home/tigunova/PycharmProjects/snorkel_labels/data/user_" + prefix + ".csv")
### write dict
output_threshold = 0.63
output_dict = defaultdict(list)
auth_hobby_dict = defaultdict(list)
for index, row in result_filtered.iterrows():
if row.value == row.value and row.author == row.author:
auth_hobby_dict[row.author].append([row.value, row.pos_prob])
allowed_labels = []
for index, row in df_train_filtered.iterrows():
if row.value == row.value and row.author == row.author:
if row.pos_prob > output_threshold:
output_dict[row.author].append([row.value] + row.idd +
[row.pos_prob])
allowed_labels.append(syn_to_hob[row.value])
print("\n".join([
str(y) for y in sorted(dict(Counter(allowed_labels)).items(),
key=lambda x: x[1])
]))
print(
"After cropping",
sum([
x if x < 500 else 500
for x in dict(Counter(allowed_labels)).values()
]))
print("users in total", len(output_dict))
for auth, stuffs in output_dict.items():
prof = ":::".join(set([x[0] for x in stuffs]))
prob = ":::".join([str(x[-1]) for x in stuffs])
msgs = set([x for l in stuffs for x in l[1:-1]])
output_dict[auth] = [prof] + list(msgs) + [prob]
with open(
"/home/tigunova/PycharmProjects/snorkel_labels/data/profession/sources/final_author_dict_"
+ prefix + ".txt", "w") as f_out:
f_out.write(repr(dict(auth_hobby_dict)))
with open("/home/tigunova/users_profession1.txt", "w") as f_out:
f_out.write(repr(dict(output_dict)))
class Modeler:
def __init__(self,
df_train,
df_dev,
df_valid,
df_test,
df_heldout,
lfs={},
label_model=None):
df_train["seen"] = 0
self.df_train = df_train.reset_index()
self.df_dev = df_dev
self.df_valid = df_valid
self.df_test = df_test
self.df_heldout = df_heldout
#self.Y_train = df_train.label.values
self.Y_dev = df_dev.label.values
self.Y_valid = df_valid.label.values
self.Y_test = df_test.label.values
self.Y_heldout = df_heldout.label.values
self.lfs = lfs
self.L_train = None
self.L_dev = None
self.L_valid = None
self.L_heldout = None
cardinality = len(df_valid.label.unique())
# for DEMOing purposes
self.first_text_indices = [
1262, #"check out" "youtube"
1892, # I love
1117, # url concept
1706, # emoji concept
952, # "nice"
971, # positive concept
958, # actually use emoji concept
]
self.count = 0
if label_model is None:
self.label_model = LabelModel(cardinality=cardinality,
verbose=True)
else:
self.label_model = label_model
self.vectorizer = CountVectorizer(ngram_range=(1, 2))
self.vectorizer.fit(df_train.text.tolist())
def get_lfs(self):
return list(self.lfs.values())
def add_lfs(self, new_lfs: dict):
self.lfs.update(new_lfs)
def remove_lfs(self, old_lf_ids: list):
for lf_id in old_lf_ids:
del self.lfs[lf_id]
return len(self.lfs)
def apply_lfs(self):
applier = PandasLFApplier(lfs=self.get_lfs())
self.L_train = applier.apply(df=self.df_train)
self.L_dev = applier.apply(df=self.df_dev)
self.L_heldout = applier.apply(df=self.df_heldout)
#self.L_valid = applier.apply(df=self.df_valid)
def find_duplicate_signature(self):
label_matrix = np.vstack([self.L_train, self.L_dev])
seen_signatures = {}
dupes = {}
lfs = self.get_lfs()
signatures = [
hash(label_matrix[:, i].tostring()) for i in range(len(lfs))
]
for i, s in enumerate(signatures):
lf = lfs[i]
if s in seen_signatures:
dupes[lf.name] = seen_signatures[s]
else:
seen_signatures[s] = lf.name
return dupes
def lf_examples(self, lf_id, n=5):
lf = self.lfs[lf_id]
applier = PandasLFApplier(lfs=[lf])
L_train = applier.apply(df=self.df_train)
labeled_examples = self.df_train[L_train != -1]
samples = labeled_examples.sample(min(n, len(labeled_examples)),
random_state=13)
return [{"text": t} for t in samples["text"].values]
def lf_mistakes(self, lf_id, n=5):
lf = self.lfs[lf_id]
applier = PandasLFApplier(lfs=[lf])
L_dev = applier.apply(df=self.df_dev).squeeze()
labeled_examples = self.df_dev[(L_dev != -1)
& (L_dev != self.df_dev["label"])]
samples = labeled_examples.sample(min(n, len(labeled_examples)),
random_state=13)
return [{"text": t} for t in samples["text"].values]
def fit_label_model(self):
assert self.L_train is not None
self.label_model.fit(L_train=self.L_train,
n_epochs=1000,
lr=0.001,
log_freq=100,
seed=123)
def analyze_lfs(self):
if len(self.lfs) > 0:
df = LFAnalysis(L=self.L_train, lfs=self.get_lfs()).lf_summary()
dev_df = LFAnalysis(L=self.L_dev,
lfs=self.get_lfs()).lf_summary(Y=self.Y_dev)
df = df.merge(dev_df,
how="outer",
suffixes=(" Training", " Dev."),
left_index=True,
right_index=True)
df["Weight"] = self.label_model.get_weights()
df["Duplicate"] = None
for dupe, OG in self.find_duplicate_signature().items():
print("Duplicate labeling signature detected")
print(dupe, OG)
df.at[dupe, "Duplicate"] = OG
return df
return None
def get_label_model_stats(self):
result = self.label_model.score(L=self.L_dev,
Y=self.Y_dev,
metrics=["f1", "precision", "recall"])
probs_train = self.label_model.predict_proba(L=self.L_train)
df_train_filtered, probs_train_filtered = filter_unlabeled_dataframe(
X=self.df_train, y=probs_train, L=self.L_train)
result["training_label_coverage"] = len(probs_train_filtered) / len(
probs_train)
result["class_0_ratio"] = (probs_train_filtered[:, 0] >
0.5).sum() / len(probs_train_filtered)
if len(probs_train_filtered) == 0:
result["class_0_ratio"] = 0
return result
def get_heldout_stats(self):
if self.L_heldout is not None:
return self.label_model.score(
L=self.L_heldout,
Y=self.Y_heldout,
metrics=["f1", "precision", "recall"])
return {}
def train(self):
probs_train = self.label_model.predict_proba(L=self.L_train)
df_train_filtered, probs_train_filtered = filter_unlabeled_dataframe(
X=self.df_train, y=probs_train, L=self.L_train)
if len(df_train_filtered) == 0:
print("Labeling functions cover none of the training examples!",
file=sys.stderr)
return {"micro_f1": 0}
#from tensorflow.keras.utils import to_categorical
#df_train_filtered, probs_train_filtered = self.df_dev, to_categorical(self.df_dev["label"].values)
vectorizer = self.vectorizer
X_train = vectorizer.transform(df_train_filtered.text.tolist())
X_dev = vectorizer.transform(self.df_dev.text.tolist())
X_valid = vectorizer.transform(self.df_valid.text.tolist())
X_test = vectorizer.transform(self.df_test.text.tolist())
self.keras_model = get_keras_logreg(input_dim=X_train.shape[1])
self.keras_model.fit(
x=X_train,
y=probs_train_filtered,
validation_data=(X_valid, preds_to_probs(self.Y_valid, 2)),
callbacks=[get_keras_early_stopping()],
epochs=20,
verbose=0,
)
preds_test = self.keras_model.predict(x=X_test).argmax(axis=1)
#return preds_test
return self.get_stats(self.Y_test, preds_test)
def get_heldout_lr_stats(self):
X_heldout = self.vectorizer.transform(self.df_heldout.text.tolist())
preds_test = self.keras_model.predict(x=X_heldout).argmax(axis=1)
return self.get_stats(self.Y_heldout, preds_test)
def get_stats(self, Y_test, preds_test):
label_classes = np.unique(self.Y_test)
accuracy = metrics.accuracy_score(Y_test, preds_test)
precision_0, precision_1 = metrics.precision_score(
Y_test, preds_test, labels=label_classes, average=None)
recall_0, recall_1 = metrics.recall_score(Y_test,
preds_test,
labels=label_classes,
average=None)
test_f1 = metrics.f1_score(Y_test, preds_test, labels=label_classes)
#recall_0, recall_1 = metrics.precision_recall_fscore_support(self.Y_test, preds_test, labels=label_classes)["recall"]
return {
"micro_f1": test_f1,
"recall_0": recall_0,
"precision_0": precision_0,
"accuracy": accuracy,
"recall_1": recall_1,
"precision_1": precision_1
}
def entropy(self, prob_dist):
#return(-(L_row_i==-1).sum())
return (-sum([x * log(x) for x in prob_dist]))
def save(self, dir_name):
self.label_model.save(os.path.join(dir_name, 'label_model.pkl'))
with open(os.path.join(dir_name, 'model_lfs.pkl'), "wb+") as file:
pickle.dump(self.lfs, file)
def load(self, dir_name):
with open(os.path.join(dir_name, 'model_lfs.pkl'), "rb") as file:
lfs = pickle.load(file)
label_model = LabelModel.load(
os.path.join(dir_name, 'label_model.pkl'))
self.lfs = lfs
self.label_model = label_model
label_model = LabelModel(cardinality=2, verbose=True)
label_model.fit(L_train=L_train,
n_epochs=500,
lr=0.001,
log_freq=100,
seed=123)
# %%
majority_acc = majority_model.score(L=L_valid,
Y=Y_valid,
tie_break_policy="random")["accuracy"]
print(f"{'Majority Vote Accuracy:':<25} {majority_acc * 100:.1f}%")
label_model_acc = label_model.score(L=L_valid,
Y=Y_valid,
tie_break_policy="random")["accuracy"]
print(f"{'Label Model Accuracy:':<25} {label_model_acc * 100:.1f}%")
# %% [markdown]
# So our `LabelModel` improves over the majority vote baseline!
# However, it is typically **not suitable as an inference-time model** to make predictions for unseen data points, due to (among other things) some data points having all abstain labels.
# In the next section, we will use the output of the label model as training labels to train a
# discriminative classifier to see if we can improve performance further.
# This classifier will only need the text of the comment to make predictions, making it much more suitable
# for inference over unseen comments.
# For more information on the properties of the label model and when to use it, see the [Snorkel guides]().
# %% [markdown]
# We can also run error analysis after the label model has been trained.
# For example, let's take a look at 5 random false negatives from the `dev` set, which might inspire some more LFs that vote `SPAM`.
df_train1["LF2"] = L_train[:,1]
df_train1["LF3"] = L_train[:,2]
df_train1["LF4"] = L_train[:,3]
df_train1["LF5"] = L_train[:,4]
df_train1["LF6"] = L_train[:,5]
df_train1["LF7"] = L_train[:,6]
df_train1["LF8"] = L_train[:,7]
df_train1.to_csv("yelp_LF.csv")
#--------train label model-------
label_model = LabelModel(cardinality=2, verbose=True)
label_model.fit(L_train=L_train, n_epochs=100, lr=0.01, seed=123)
probs_train = label_model.predict_proba(L=L_train)
preds_train = probs_to_preds(probs=probs_train)
print(label_model.score(L=L_train, Y=Y_train))
df_result = df[['tag']]
df_result['pred'] = preds_train
df_result.to_csv("yelp_snorkel.csv")
df_train_filtered, probs_train_filtered = filter_unlabeled_dataframe(
X=df_train, y=probs_train, L=L_train
)
preds_train_filtered = probs_to_preds(probs=probs_train_filtered)
print(len(preds_train_filtered))
from google.colab import files
files.download('yelp_LF.csv')
files.download('yelp_snorkel.csv')
# We also specify the `cardinality`, or number of classes.
# %% {"tags": ["md-exclude-output"]}
from snorkel.labeling import LabelModel
label_model = LabelModel(cardinality=2, verbose=True)
label_model.fit(L_train=L_train, n_epochs=500, log_freq=100, seed=123)
# %%
majority_acc = majority_model.score(L=L_test, Y=Y_test, tie_break_policy="random")[
"accuracy"
]
print(f"{'Majority Vote Accuracy:':<25} {majority_acc * 100:.1f}%")
label_model_acc = label_model.score(L=L_test, Y=Y_test, tie_break_policy="random")[
"accuracy"
]
print(f"{'Label Model Accuracy:':<25} {label_model_acc * 100:.1f}%")
# %% [markdown]
# The majority vote model or more sophisticated `LabelModel` could in principle be used directly as a classifier if the outputs of our labeling functions were made available at test time.
# However, these models (i.e. these re-weighted combinations of our labeling function's votes) will abstain on the data points that our labeling functions don't cover (and additionally, may require slow or unavailable features to execute at test time).
# In the next section, we will instead use the outputs of the `LabelModel` as training labels to train a discriminative classifier **which can generalize beyond the labeling function outputs** to see if we can improve performance further.
# This classifier will also only need the text of the comment to make predictions, making it much more suitable for inference over unseen comments.
# For more information on the properties of the label model, see the [Snorkel documentation](https://snorkel.readthedocs.io/en/master/packages/_autosummary/labeling/snorkel.labeling.LabelModel.html#snorkel.labeling.LabelModel).
# %% [markdown] {"tags": ["md-exclude"]}
# Let's briefly confirm that the labels the `LabelModel` produces are indeed probabilistic in nature.
# The following histogram shows the confidences we have that each data point has the label SPAM.
# The points we are least certain about will have labels close to 0.5.