def libact_first_try_second_run(self, enriched_train_df, extractor,
ideal_df, lbr, quota, validation_data_df,
return_dict):
trn_ds = TextDataset(enriched_train_df, cn.col_names, extractor)
qs = UncertaintySampling(trn_ds,
method='lc',
model=LogisticRegression())
E_out1 = []
E_out1 = np.append(
E_out1,
run_classifier(trn_ds.extract_labeled_dataframe(),
validation_data_df).f1)
for i in range(quota):
if len(trn_ds.get_unlabeled_entries()) == 0:
break # finished labeling all examples
ask_id = qs.make_query()
lb = lbr.label(trn_ds.extract_sentence(ask_id))
self.assertEqual(lb, ideal_df[cn.tag_col][ask_id])
trn_ds.update(ask_id, lb)
# model.train(trn_ds)
E_out1 = np.append(
E_out1,
run_classifier(trn_ds.extract_labeled_dataframe(),
validation_data_df).f1)
return_dict[2] = E_out1
def score_per_add_al(labeled_pool_df, base_training_df, validation_data_df):
# type: (DataFrame, DataFrame, DataFrame) -> tuple
gen_pool_df = labeled_pool_df.copy(deep=True)
gen_pool_df[cn.col_names.tag] = [np.NaN] * len(
gen_pool_df) # clear all tags
enriched_train_df = pd.concat([base_training_df, gen_pool_df],
ignore_index=True)
extractor = cn.Feature_Extractor(
enriched_train_df, cn.col_names) # build the feature extractor
trn_ds = TextDataset(enriched_train_df, cn.col_names, extractor)
qs = UncertaintySampling(trn_ds, method='lc', model=LogisticRegression())
ideal_df = pd.concat([base_training_df, labeled_pool_df],
ignore_index=True)
lbr = IdealTextLabeler(TextDataset(ideal_df, cn.col_names, extractor))
scoring_fun = lambda ds: run_classifier(ds.extract_labeled_dataframe(),
validation_data_df)
ex_added_list, res_list = run_active_learning(
trn_ds, scoring_fun, lbr, qs, len(enriched_train_df)) # label all df
return ex_added_list, res_list
def libact_first_try_first_run(self, enriched_train_df, extractor, lbr,
quota, validation_data_df, return_dict):
trn_ds = TextDataset(enriched_train_df, cn.col_names, extractor)
qs = UncertaintySampling(trn_ds,
method='lc',
model=LogisticRegression())
scoring_fun = lambda ds: run_classifier(ds.extract_labeled_dataframe(),
validation_data_df).f1
query_num, E_out1 = run_active_learning(trn_ds, scoring_fun, lbr, qs,
quota)
return_dict[1] = E_out1
def scores_per_add_default(labeled_pool_df, base_training_df,
validation_data_df):
# type: (pd.DataFrame, pd.DataFrame, callable, int) -> (list, list)
"""
get result from adding examples from $labeled_pool_df to $base_train_df, training them and seeing their results
:param labeled_pool_df: df containing the new examples we asked the expert for
:param base_train_df: the original training examples we had
:param validation_data_df: df containing the validation data we test our model's performance on
:return: (ex_added_list, res_list), a list containing the number of examples added
and a list containing its corresponding result
"""
# return score_per_add_al(labeled_pool_df, base_training_df, validation_data_df)
return score_per_addition_results(
labeled_pool_df, base_training_df,
lambda en_df: run_classifier(en_df, validation_data_df))
def test_heuristic_test_data_gain_works(self):
cn.load_codementor_sentiment_analysis_parameters()
kb_helper.load_WordNet_model()
base_train_df, pool_df, validation_data_df = prepare_pool_based_dataset(
)
all_sents = list(base_train_df[cn.col_names.text])
tns = 2
pool_name, prep_pools = (
"orig pool", lambda *_:
(pool_df.iloc[:tns + 5], cn.labeled_pool_df.iloc[:tns + 5]))
train_with_pool_df = pd.concat([base_train_df, pool_df],
ignore_index=True)
generated_pool_df, labeled_pool_df = prep_pools(
all_sents, train_with_pool_df, cn.col_names, tns)
cn.experiment_purpose += pool_name + " "
trn_ds, lbr, extractor = prepare_trn_ds(base_train_df,
generated_pool_df,
labeled_pool_df)
final_scoring_fun = partial(
lambda en_df: run_classifier(en_df, validation_data_df).acc)
table_headers = ['#added examples']
data = [range(0, tns + 1)]
compared_heuristics = [("test-data-gain",
lambda: SynStateTestDataGain),
("random", lambda: "random")]
for (heuristic_name, prepare_usage) in compared_heuristics:
ss_type = prepare_usage()
table_headers.append(heuristic_name)
print heuristic_name
_, heur_scores = insert_in_AL_fashion(trn_ds,
final_scoring_fun,
lbr,
ss_type,
labeled_pool_df,
quota=tns)
data.append(heur_scores)
print data[1]
print data[2]
self.assertEqual(data[1][0], data[2][0], "starts are same")
self.assertGreater(data[1][-1], data[2][-1],
"test-data-gain should be better than random")
def effect_of_size_of_semantic_environment():
experiment_name = 'effect_of_size_of_semantic_environment'
print experiment_name
# prepare the different splits of $data_df
balanced_train_df, validation_data_df = prepare_balanced_dataset(
print_expert_acc=False)
all_sents = list(balanced_train_df[cn.col_names.text])
tns = 50
prepare_pools_funcs = list()
prepare_pools_funcs.append(
local_search_gen_template("uncertainty lc LogReg", 5))
prepare_pools_funcs.append(local_search_gen_template("random-score", 5))
prepare_pools_funcs.append(generate_pool_using_random_synthesis())
final_scoring_fun = partial(
lambda en_df: run_classifier(en_df, validation_data_df).acc)
insertion_order_heuristic = find_heuristic("random-score")()
print cn.experiment_purpose
table_headers = ['size of semantic environment']
data = [[]]
for env_size in range(1, 5, 1) + range(5, 35, 5):
data[0].append(env_size)
for i, (pool_name, prepare_pool_fun) in enumerate(prepare_pools_funcs):
if pool_name not in table_headers:
table_headers.append(pool_name)
data.append([])
assert len(data) == i + 2, "meaning i+1 is our index"
print pool_name
cn.distance_measure = env_size
gen_pool_df, labeled_pool_df = prepare_pool_fun(
all_sents, balanced_train_df, cn.col_names, tns)
trn_ds, lbr, extractor = prepare_trn_ds(balanced_train_df,
gen_pool_df,
labeled_pool_df)
query_num, pool_insr_scores = insert_in_AL_fashion(
trn_ds,
final_scoring_fun,
lbr,
insertion_order_heuristic,
labeled_pool_df,
quota=tns)
data[i + 1].append(pool_insr_scores[-1])
return experiment_name, table_headers, data, plot_effect_of_size_of_semantic_environment
def effect_of_num_of_operators():
experiment_name = 'effect_of_num_of_operators'
print experiment_name
# prepare the different splits of $data_df
balanced_train_df, validation_data_df = prepare_balanced_dataset(
print_expert_acc=False)
all_sents = list(balanced_train_df[cn.col_names.text])
tns = 50
final_scoring_fun = partial(
lambda en_df: run_classifier(en_df, validation_data_df).acc)
insertion_order_heuristic = find_heuristic("random-score")()
print cn.experiment_purpose
table_headers = ['num of operators']
data = [[1] + range(2, 12, 2)]
for i, heur in enumerate(["uncertainty lc LogReg",
"random-score"]): # "uncertainty lc LogReg",
table_headers.append(heur)
data.append(list())
prepare_pools_funcs = list()
prepare_pools_funcs.append(local_search_gen_template(heur, 1))
for j in range(2, 12, 2):
prepare_pools_funcs.append(local_search_gen_template(heur, j))
for pool_name, prepare_pool_fun in prepare_pools_funcs:
gen_pool_df, labeled_pool_df = prepare_pool_fun(
all_sents, balanced_train_df, cn.col_names, tns)
trn_ds, lbr, extractor = prepare_trn_ds(balanced_train_df,
gen_pool_df,
labeled_pool_df)
print pool_name
query_num, pool_insr_scores = insert_in_batch_AL(
trn_ds,
final_scoring_fun,
lbr,
insertion_order_heuristic,
labeled_pool_df,
batch_num=len(labeled_pool_df))
data[1 + i].append(pool_insr_scores[-1])
return experiment_name, table_headers, data, plot_effect_of_num_of_operators