def predictWithThreshold(datadir, threshold, penalty_type='l2'):
maxent = LogisticRegression(penalty=penalty_type)
scores = defaultdict(list)
for dir in sorted(os.listdir(datadir), reverse=True):
trainfeatures, trainlabels, vec = feats_and_classify.collect_features(datadir+dir+'/train.conll')
TrainIndices=np.array(range(len(trainfeatures)))
features, labels, vec = feats_and_classify.collect_features(datadir+dir+'/all.conll')
TestIndices=np.array(range(len(trainfeatures),len(features)))
# print('\r'+dir, end="")
# print(dir)
TrainX_i = features[TrainIndices]
Trainy_i = labels[TrainIndices]
TestX_i = features[TestIndices]
Testy_i = labels[TestIndices]
maxent.fit(TrainX_i,Trainy_i)
# print('Finished fitting')
ypred_i, score=pred_for_threshold(maxent,TestX_i,Testy_i, threshold)
# print('Predicting')
scores["F1"].append(score[0])
scores["Recall"].append(score[1])
scores["Accuracy"].append(score[2])
scores["Precision"].append(score[3])
#scores = cross_validation.cross_val_score(maxent, features, labels, cv=10)
print("\n--")
for key in sorted(scores.keys()):
currentmetric = np.array(scores[key])
print("%s : %0.2f (+/- %0.2f)" % (key,currentmetric.mean(), currentmetric.std()))
print("--")
def main():
scriptdir = os.path.dirname(os.path.realpath(__file__))
default_pool = scriptdir+"/../data/cwi_training/cwi_training.txt.lbl.conll"
parser = argparse.ArgumentParser(description="Skeleton for features and classifier for CWI-2016--optimisation of threshhold")
parser.add_argument('--pooled_annotators', help="parsed-and-label input format", default=default_pool)
parser.add_argument('--threshold_matrix_file', help="location/name of the threshold matrix", default='annotator_threshold_matrix')
args = parser.parse_args()
threshold_dict = {}
threshold_list = []
current_label_list = []
features, labels_pooled, vec = feats_and_classify.collect_features(args.pooled_annotators)
print("total len of f, labels",len(features), len(labels_pooled))
for idx in "01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20".split(" "):
# for idx in "01".split(" "):
current_single_ann = scriptdir+"/../data/cwi_training/cwi_training_"+idx+".lbl.conll"
f_current, labels_current, v_current = feats_and_classify.collect_features(current_single_ann)
print(idx,"len of f, labels",len(f_current), len(labels_current) )
for idx in "01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20".split(" "):
# for idx in "01".split(" "):
current_single_ann = scriptdir+"/../data/cwi_training/cwi_training_"+idx+".lbl.conll"
_, labels_current, _ = feats_and_classify.collect_features(current_single_ann)
current_label_list.append(labels_current)
maxent, thresholds=getBestThreshold(features, labels_pooled,labels_current)
threshold_list.extend(thresholds)
print(thresholds)
predictAcrossThresholds(features, labels_pooled,labels_current, maxent, thresholds, average=True, median=True)
pooled_score_dict_ave = defaultdict(list)
pooled_score_dict_med = defaultdict(list)
count=0
for labels_current in current_label_list:
count+=1
print('ANNOTATOR COUNT '+str(count))
score_dict_ave, score_dict_med = predictAcrossThresholds(features, labels_pooled,labels_current, maxent, np.array(threshold_list), average=True, median=True)
for k in score_dict_ave:
pooled_score_dict_ave[k].append(score_dict_ave[k][0])
pooled_score_dict_med[k].append(score_dict_med[k][0])
#finding dimensions of matrix to print out
cols=20
rows=len(pooled_score_dict_ave[k])/20
mat=np.ndarray(shape=(rows,cols), buffer=np.array(pooled_score_dict_ave[k]), dtype=float)
print(mat)
pickle.dump( mat, open( args.threshold_matrix_file+"_ave.p", "wb" ) )
mat=np.ndarray(shape=(rows,cols), buffer=np.array(pooled_score_dict_med[k]), dtype=float)
print(mat)
pickle.dump( mat, open( args.threshold_matrix_file+"_med.p", "wb" ) )
for k in pooled_score_dict_ave:
print(k,":",np.array(pooled_score_dict_ave[k]).mean(),":",pooled_score_dict_ave[k])
print(k,":",np.median(pooled_score_dict_med[k]),":",pooled_score_dict_med[k])
sys.exit(0)
def main():
scriptdir = os.path.dirname(os.path.realpath(__file__))
parser = argparse.ArgumentParser(description="Skeleton for features and classifier for CWI-2016--optimisation of threshhold")
parser.add_argument('--threshold',type=float,default=0.5)
parser.add_argument('--annotator',type=str,default="03")
parser.add_argument('--penalty',type=str,choices=["l1","l2"],default="l1")
args = parser.parse_args()
current_single_ann = scriptdir+"/../data/cwi_training/cwi_training_"+args.annotator+".lbl.conll"
testfile = scriptdir+"/../data/cwi_testing/cwi_testing.txt.lbl.conll"
X__dict_train, y_train, v_train = feats_and_classify.collect_features(current_single_ann,vectorize=False)
X_dict_test, y_test, v_test = feats_and_classify.collect_features(testfile,vectorize=False)
featdicts = list([x for x in X__dict_train + X_dict_test])
vect = DictVectorizer()
X = vect.fit_transform(featdicts).toarray()
X_train=X[:len(y_train)]
X_test=X[len(y_train):]
maxent = LogisticRegression(penalty=args.penalty)
maxent.fit(X_train,y_train)
y_pred_proba = maxent.predict_proba(X_test)
ypred_i=["1" if pair[1]>=args.threshold else "0" for pair in y_pred_proba]
fout = open(args.annotator+".pred",mode="w")
print("\n".join(ypred_i),file=fout)
fout.close()
sys.exit(0)
def main():
scriptdir = os.path.dirname(os.path.realpath(__file__))
default_pool = scriptdir+"/../data/cwi_training/cwi_training.txt.lbl.conll"
parser = argparse.ArgumentParser(description="Skeleton for features and classifier for CWI-2016--optimisation of threshhold")
parser.add_argument('--iterations',type=int,default=5)
args = parser.parse_args()
all_feats = []
all_labels = defaultdict(list)
scores = defaultdict(list)
for idx in "01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20".split(" "):
# for idx in "01".split(" "):
current_single_ann = scriptdir+"/../data/cwi_training/cwi_training_"+idx+".lbl.conll"
f_current, labels_current, v_current = feats_and_classify.collect_features(current_single_ann,vectorize=False,generateFeatures=False)
for instance_index,l in enumerate(labels_current):
all_labels[instance_index].append(l)
current_single_ann = scriptdir+"/../data/cwi_training/cwi_training_01.lbl.conll"
feats, labels_current, v_current = feats_and_classify.collect_features(current_single_ann,vectorize=True,generateFeatures=True)
for it in range(args.iterations):
for TrainIndices, TestIndices in cross_validation.KFold(n=feats.shape[0], n_folds=10, shuffle=True, random_state=None):
maxent = LogisticRegression(penalty='l2')
TrainX_i = feats[TrainIndices]
Trainy_i = [all_labels[x][random.randrange(0,20)] for x in TrainIndices]
TestX_i = feats[TestIndices]
Testy_i = [all_labels[x][random.randrange(0,20)] for x in TestIndices]
maxent.fit(TrainX_i,Trainy_i)
ypred_i = maxent.predict(TestX_i)
acc = accuracy_score(ypred_i, Testy_i)
pre = precision_score(ypred_i, Testy_i)
rec = recall_score(ypred_i, Testy_i)
# shared task uses f1 of *accuracy* and recall!
f1 = 2 * acc * rec / (acc + rec)
scores["Accuracy"].append(acc)
scores["F1"].append(f1)
scores["Precision"].append(pre)
scores["Recall"].append(rec)
#scores = cross_validation.cross_val_score(maxent, features, labels, cv=10)
print("--")
for key in sorted(scores.keys()):
currentmetric = np.array(scores[key])
print("%s : %0.2f (+/- %0.2f)" % (key,currentmetric.mean(), currentmetric.std()))
print("--")
sys.exit(0)
def getBestThreshold(datadir, penalty_type='l2'):
maxent = LogisticRegression(penalty=penalty_type)
scores = {"F1": [], "Recall": [], "Accuracy": [], "Precision": []}
thresholds = []
print('Finding best thresholds...')
for dir in os.listdir(datadir):
trainfeatures, trainlabels, vec = feats_and_classify.collect_features(
datadir + dir + '/train.conll')
TrainIndices = np.array(range(len(trainfeatures)))
features, labels, vec = feats_and_classify.collect_features(
datadir + dir + '/all.conll')
TestIndices = np.array(range(len(trainfeatures), len(features)))
print('\r' + dir, end="")
# print(dir)
TrainX_i = features[TrainIndices]
Trainy_i = labels[TrainIndices]
TestX_i = features[TestIndices]
Testy_i = labels[TestIndices]
maxent.fit(TrainX_i, Trainy_i)
# print('Finished fitting')
#get prediction
thresh_i, ypred_i, score = optimize_threshold(maxent, TestX_i, Testy_i)
# print('Optimising')
thresholds.append(thresh_i)
scores["F1"].append(score[0])
scores["Recall"].append(score[1])
scores["Accuracy"].append(score[2])
scores["Precision"].append(score[3])
#scores = cross_validation.cross_val_score(maxent, features, labels, cv=10)
print("\n--")
for key in sorted(scores.keys()):
currentmetric = np.array(scores[key])
print("%s : %0.2f (+/- %0.2f)" %
(key, currentmetric.mean(), currentmetric.std()))
print("--")
return np.array(thresholds)
def getBestThreshold(datadir, penalty_type='l2'):
maxent = LogisticRegression(penalty=penalty_type)
scores = {"F1":[], "Recall":[], "Accuracy":[], "Precision":[]}
thresholds=[]
print('Finding best thresholds...')
for dir in os.listdir(datadir):
trainfeatures, trainlabels, vec = feats_and_classify.collect_features(datadir+dir+'/train.conll')
TrainIndices=np.array(range(len(trainfeatures)))
features, labels, vec = feats_and_classify.collect_features(datadir+dir+'/all.conll')
TestIndices=np.array(range(len(trainfeatures),len(features)))
print('\r'+dir, end="")
# print(dir)
TrainX_i = features[TrainIndices]
Trainy_i = labels[TrainIndices]
TestX_i = features[TestIndices]
Testy_i = labels[TestIndices]
maxent.fit(TrainX_i,Trainy_i)
# print('Finished fitting')
#get prediction
thresh_i, ypred_i, score=optimize_threshold(maxent,TestX_i,Testy_i)
# print('Optimising')
thresholds.append(thresh_i)
scores["F1"].append(score[0])
scores["Recall"].append(score[1])
scores["Accuracy"].append(score[2])
scores["Precision"].append(score[3])
#scores = cross_validation.cross_val_score(maxent, features, labels, cv=10)
print("\n--")
for key in sorted(scores.keys()):
currentmetric = np.array(scores[key])
print("%s : %0.2f (+/- %0.2f)" % (key,currentmetric.mean(), currentmetric.std()))
print("--")
return np.array(thresholds)
def main():
scriptdir = os.path.dirname(os.path.realpath(__file__))
defaultdata = scriptdir+"/../data/cwi_training/cwi_training.txt.lbl.conll"
parser = argparse.ArgumentParser(description="Skeleton for features and classifier for CWI-2016--optimisation of threshhold")
parser.add_argument('--train', help="parsed-and-label input format", default=defaultdata)
args = parser.parse_args()
features, labels, vec = feats_and_classify.collect_features(args.train)
maxent, thresholds=getBestThreshold(features, labels)
print(thresholds)
predictAcrossThresholds(features, labels, maxent, thresholds, average=True, median=True)
sys.exit(0)
def main():
scriptdir = os.path.dirname(os.path.realpath(__file__))
default_pool = scriptdir+"/../data/cwi_training/cwi_training.txt.lbl.conll"
parser = argparse.ArgumentParser(description="Skeleton for features and classifier for CWI-2016--optimisation of threshhold")
args = parser.parse_args()
all_feats = []
all_labels = []
scores = defaultdict(list)
for idx in "01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20".split(" "):
# for idx in "01".split(" "):
current_single_ann = scriptdir+"/../data/cwi_training/cwi_training_"+idx+".lbl.conll"
f_current, labels_current, v_current = feats_and_classify.collect_features(current_single_ann,vectorize=False)
all_feats.extend(f_current)
all_labels.extend(labels_current)
feats = DictVectorizer().fit_transform(all_feats).toarray()
all_labels = np.asarray(all_labels)
for TrainIndices, TestIndices in cross_validation.KFold(n=feats.shape[0], n_folds=10, shuffle=True, random_state=None):
maxent = LogisticRegression(penalty='l2')
TrainX_i = feats[TrainIndices]
Trainy_i = all_labels[TrainIndices]
TestX_i = feats[TestIndices]
Testy_i = all_labels[TestIndices]
maxent.fit(TrainX_i,Trainy_i)
ypred_i = maxent.predict(TestX_i)
acc = accuracy_score(ypred_i, Testy_i)
pre = precision_score(ypred_i, Testy_i)
rec = recall_score(ypred_i, Testy_i)
# shared task uses f1 of *accuracy* and recall!
f1 = 2 * acc * rec / (acc + rec)
scores["Accuracy"].append(acc)
scores["F1"].append(f1)
scores["Precision"].append(pre)
scores["Recall"].append(rec)
#scores = cross_validation.cross_val_score(maxent, features, labels, cv=10)
print("--")
for key in sorted(scores.keys()):
currentmetric = np.array(scores[key])
print("%s : %0.2f (+/- %0.2f)" % (key,currentmetric.mean(), currentmetric.std()))
print("--")
sys.exit(0)
def predictWithThreshold(datadir, threshold, penalty_type='l2'):
maxent = LogisticRegression(penalty=penalty_type)
scores = defaultdict(list)
for dir in sorted(os.listdir(datadir), reverse=True):
trainfeatures, trainlabels, vec = feats_and_classify.collect_features(
datadir + dir + '/train.conll')
TrainIndices = np.array(range(len(trainfeatures)))
features, labels, vec = feats_and_classify.collect_features(
datadir + dir + '/all.conll')
TestIndices = np.array(range(len(trainfeatures), len(features)))
# print('\r'+dir, end="")
# print(dir)
TrainX_i = features[TrainIndices]
Trainy_i = labels[TrainIndices]
TestX_i = features[TestIndices]
Testy_i = labels[TestIndices]
maxent.fit(TrainX_i, Trainy_i)
# print('Finished fitting')
ypred_i, score = pred_for_threshold(maxent, TestX_i, Testy_i,
threshold)
# print('Predicting')
scores["F1"].append(score[0])
scores["Recall"].append(score[1])
scores["Accuracy"].append(score[2])
scores["Precision"].append(score[3])
#scores = cross_validation.cross_val_score(maxent, features, labels, cv=10)
print("\n--")
for key in sorted(scores.keys()):
currentmetric = np.array(scores[key])
print("%s : %0.2f (+/- %0.2f)" %
(key, currentmetric.mean(), currentmetric.std()))
print("--")
def main():
scriptdir = os.path.dirname(os.path.realpath(__file__))
parser = argparse.ArgumentParser(
description=
"Skeleton for features and classifier for CWI-2016--optimisation of threshhold"
)
parser.add_argument('--threshold', type=float, default=0.5)
parser.add_argument('--annotator', type=str, default="03")
parser.add_argument('--penalty',
type=str,
choices=["l1", "l2"],
default="l1")
args = parser.parse_args()
current_single_ann = scriptdir + "/../data/cwi_training/cwi_training_" + args.annotator + ".lbl.conll"
testfile = scriptdir + "/../data/cwi_testing/cwi_testing.txt.lbl.conll"
X__dict_train, y_train, v_train = feats_and_classify.collect_features(
current_single_ann, vectorize=False)
X_dict_test, y_test, v_test = feats_and_classify.collect_features(
testfile, vectorize=False)
featdicts = list([x for x in X__dict_train + X_dict_test])
vect = DictVectorizer()
X = vect.fit_transform(featdicts).toarray()
X_train = X[:len(y_train)]
X_test = X[len(y_train):]
maxent = LogisticRegression(penalty=args.penalty)
maxent.fit(X_train, y_train)
y_pred_proba = maxent.predict_proba(X_test)
ypred_i = [
"1" if pair[1] >= args.threshold else "0" for pair in y_pred_proba
]
fout = open(args.annotator + ".pred", mode="w")
print("\n".join(ypred_i), file=fout)
fout.close()
sys.exit(0)
def main():
scriptdir = os.path.dirname(os.path.realpath(__file__))
default_data_parsed = scriptdir + "/../data/cwi_training/cwi_training.txt.lbl.conll"
default_data_allannotations = scriptdir + "/../data/cwi_training/cwi_training_allannotations.txt"
parser = argparse.ArgumentParser(
description=
"Skeleton for features and classifier for CWI-2016--optimisation of threshhold"
)
parser.add_argument('--all_annotations_file',
help="parsed-and-label input format",
default=default_data_allannotations)
parser.add_argument('--parsed_file',
help="parsed-and-label input format",
default=default_data_parsed)
parser.add_argument('--threshold_matrix_file',
help="location/name of the threshold matrix",
default='annotator_threshold_matrix')
parser.add_argument('--regularization',
help="regularizer, may be l1 or l2",
default='l2')
args = parser.parse_args()
print(args.regularization)
# f1_matrix holds for every training annotator: the list of tuples of
# avg/med f1_row based on avg/med threshold
f1_matrix = []
# holds for every training annotator: the list of tuples of avg/med threshold
t_matrix = []
current_label_list = []
f1_final = [
] # holds 4-tuples of avgs over (f1_avg_avg, f1_avg_med, f1_med_avg, f1_med_med) f.e. tr
t_final = [
] # holds 4-tuples of (t_avg_avg, t_avg_med, t_med_avg, t_med_med) f.e. tr
X, _, v = feats_and_classify.collect_features(args.parsed_file)
# train for every annotator...
#for vote_threshold in range(1,21):
for vote_threshold in range(3, 10):
#current_single_ann = scriptdir+"/../data/cwi_training/cwi_training_"+vote_threshold+".lbl.conll"
y_current_tr = feats_and_classify.collect_labels_positive_threshold(
args.all_annotations_file, vote_threshold)
print(
"Training, setting positive labels for examples with at least {} positive votes. "
.format(vote_threshold))
print("Training data has {} positive labels out of {}".format(
sum(y_current_tr), len(y_current_tr)))
f1_row = [
] # holds 4-tuples of (f1_avg_avg, f1_avg_med, f1_med_avg, f1_med_med) f.e. tr/te
t_row = [] # holds 2-tuples of (t_avg, t_med) f.e. tr/te
f1_matrix.append(f1_row)
t_matrix.append(t_row)
# optimize t for every annotator (except training annotator), yields avg/med t
#for idx in "03 04 05".split(" "):
#for idx in "01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20".split(" "):
# 02, 09, 17 are the annotators with the least/average/most positive votes
for idx in "02 09 17".split(" "):
print(" Testing on annotator " + idx)
#current_single_ann = scriptdir+"/../data/cwi_training/cwi_training_"+idx+".lbl.conll"
#y_current_te = feats_and_classify.collect_labels(current_single_ann)
y_current_te = feats_and_classify.collect_labels(
args.all_annotations_file,
int(idx) - 1)
current_label_list.append(y_current_te)
maxent, thresholds = getBestThreshold(
X,
y_current_tr,
y_current_te,
regularization=args.regularization)
t_avg = np.average(thresholds)
t_med = np.median(thresholds)
t_row.append((t_avg, t_med))
# calculate avg of avg t's, avg of med t's, ... for the current training annotator
t_avg_avg = np.average([t[0] for t in t_row])
t_avg_med = np.average([t[1] for t in t_row])
t_med_avg = np.median([t[0] for t in t_row])
t_med_med = np.median([t[1] for t in t_row])
t_final.append((t_avg_avg, t_avg_med, t_med_avg, t_med_med))
print(
"Computed optimal t's... Now running a new phase of CV experiments with these t's on test annotators."
)
#maxent = LogisticRegression(penalty=args.regularization)
#maxent.fit(X, y_current_tr)
#for idx in "01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20".split(" "):
# 02, 09, 17 are the annotators with the least/average/most positive votes
for idx in "02 09 17".split(" "):
f1_avg_avg = 0
f1_med_avg = 0
f1_avg_med = 0
f1_med_med = 0
print("Testing globally optimal t's for annotator {}".format(idx))
#y_current_te = feats_and_classify.collect_labels(current_single_ann)
y_current_te = feats_and_classify.collect_labels(
args.all_annotations_file,
int(idx) - 1)
#f1_avg_avg = cvWithThreshold(X, y_current_tr, y_current_te, t_avg_avg, args.regularization)['F1'][0]
f1_avg_med = cvWithThreshold(X, y_current_tr, y_current_te,
t_avg_med,
args.regularization)['F1'][0]
#f1_med_avg = cvWithThreshold(X, y_current_tr, y_current_te, t_med_avg, args.regularization)['F1'][0]
#f1_med_med = cvWithThreshold(X, y_current_tr, y_current_te, t_med_med, args.regularization)['F1'][0]
f1_row.append((f1_avg_avg, f1_avg_med, f1_med_avg, f1_med_med))
f1_final.append(tuple(map(np.average, zip(*f1_row))))
print(tuple(map(np.average, zip(*f1_row))))
print(f1_final)
# get the index (NB: array index!) of the max avg/med F1 (i.e. computed on avg/med threshold)
best_vote_threshold_avg_avg = np.argmax([f1[0] for f1 in f1_final])
best_vote_threshold_avg_med = np.argmax([f1[1] for f1 in f1_final])
best_vote_threshold_med_avg = np.argmax([f1[2] for f1 in f1_final])
best_vote_threshold_med_med = np.argmax([f1[3] for f1 in f1_final])
print(t_final)
sys.exit(0)
def main():
scriptdir = os.path.dirname(os.path.realpath(__file__))
default_data_parsed = scriptdir+"/../data/cwi_training/cwi_training.txt.lbl.conll"
default_data_allannotations = scriptdir+"/../data/cwi_training/cwi_training_allannotations.txt"
parser = argparse.ArgumentParser(description="Skeleton for features and classifier for CWI-2016--optimisation of threshhold")
parser.add_argument('--all_annotations_file', help="parsed-and-label input format", default=default_data_allannotations)
parser.add_argument('--parsed_file', help="parsed-and-label input format", default=default_data_parsed)
parser.add_argument('--threshold_matrix_file', help="location/name of the threshold matrix", default='annotator_threshold_matrix')
parser.add_argument('--regularization', help="regularizer, may be l1 or l2", default='l2')
args = parser.parse_args()
print(args.regularization)
# f1_matrix holds for every training annotator: the list of tuples of
# avg/med f1_row based on avg/med threshold
f1_matrix = []
# holds for every training annotator: the list of tuples of avg/med threshold
t_matrix = []
current_label_list = []
f1_final = [] # holds 4-tuples of avgs over (f1_avg_avg, f1_avg_med, f1_med_avg, f1_med_med) f.e. tr
t_final = [] # holds 4-tuples of (t_avg_avg, t_avg_med, t_med_avg, t_med_med) f.e. tr
X, _, v = feats_and_classify.collect_features(args.parsed_file)
# train for every annotator...
#for vote_threshold in range(1,21):
for vote_threshold in range(3,10):
#current_single_ann = scriptdir+"/../data/cwi_training/cwi_training_"+vote_threshold+".lbl.conll"
y_current_tr = feats_and_classify.collect_labels_positive_threshold(args.all_annotations_file, vote_threshold)
print("Training, setting positive labels for examples with at least {} positive votes. ".format(vote_threshold))
print("Training data has {} positive labels out of {}".format(sum(y_current_tr), len(y_current_tr)))
f1_row = [] # holds 4-tuples of (f1_avg_avg, f1_avg_med, f1_med_avg, f1_med_med) f.e. tr/te
t_row = [] # holds 2-tuples of (t_avg, t_med) f.e. tr/te
f1_matrix.append(f1_row)
t_matrix.append(t_row)
# optimize t for every annotator (except training annotator), yields avg/med t
#for idx in "03 04 05".split(" "):
#for idx in "01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20".split(" "):
# 02, 09, 17 are the annotators with the least/average/most positive votes
for idx in "02 09 17".split(" "):
print(" Testing on annotator "+idx)
#current_single_ann = scriptdir+"/../data/cwi_training/cwi_training_"+idx+".lbl.conll"
#y_current_te = feats_and_classify.collect_labels(current_single_ann)
y_current_te = feats_and_classify.collect_labels(args.all_annotations_file, int(idx)-1)
current_label_list.append(y_current_te)
maxent, thresholds=getBestThreshold(X, y_current_tr, y_current_te, regularization=args.regularization)
t_avg = np.average(thresholds)
t_med = np.median(thresholds)
t_row.append((t_avg, t_med))
# calculate avg of avg t's, avg of med t's, ... for the current training annotator
t_avg_avg = np.average([t[0] for t in t_row])
t_avg_med = np.average([t[1] for t in t_row])
t_med_avg = np.median([t[0] for t in t_row])
t_med_med = np.median([t[1] for t in t_row])
t_final.append((t_avg_avg, t_avg_med, t_med_avg, t_med_med))
print("Computed optimal t's... Now running a new phase of CV experiments with these t's on test annotators.")
#maxent = LogisticRegression(penalty=args.regularization)
#maxent.fit(X, y_current_tr)
#for idx in "01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20".split(" "):
# 02, 09, 17 are the annotators with the least/average/most positive votes
for idx in "02 09 17".split(" "):
f1_avg_avg = 0
f1_med_avg = 0
f1_avg_med = 0
f1_med_med = 0
print("Testing globally optimal t's for annotator {}".format(idx))
#y_current_te = feats_and_classify.collect_labels(current_single_ann)
y_current_te = feats_and_classify.collect_labels(args.all_annotations_file, int(idx)-1)
#f1_avg_avg = cvWithThreshold(X, y_current_tr, y_current_te, t_avg_avg, args.regularization)['F1'][0]
f1_avg_med = cvWithThreshold(X, y_current_tr, y_current_te, t_avg_med, args.regularization)['F1'][0]
#f1_med_avg = cvWithThreshold(X, y_current_tr, y_current_te, t_med_avg, args.regularization)['F1'][0]
#f1_med_med = cvWithThreshold(X, y_current_tr, y_current_te, t_med_med, args.regularization)['F1'][0]
f1_row.append((f1_avg_avg, f1_avg_med, f1_med_avg, f1_med_med))
f1_final.append(tuple(map(np.average, zip(*f1_row))))
print(tuple(map(np.average, zip(*f1_row))))
print(f1_final)
# get the index (NB: array index!) of the max avg/med F1 (i.e. computed on avg/med threshold)
best_vote_threshold_avg_avg = np.argmax([f1[0] for f1 in f1_final])
best_vote_threshold_avg_med = np.argmax([f1[1] for f1 in f1_final])
best_vote_threshold_med_avg = np.argmax([f1[2] for f1 in f1_final])
best_vote_threshold_med_med = np.argmax([f1[3] for f1 in f1_final])
print(t_final)
sys.exit(0)
def main():
scriptdir = os.path.dirname(os.path.realpath(__file__))
default_pool = scriptdir + "/../data/cwi_training/cwi_training.txt.lbl.conll"
parser = argparse.ArgumentParser(
description=
"Skeleton for features and classifier for CWI-2016--optimisation of threshhold"
)
parser.add_argument('--pooled_annotators',
help="parsed-and-label input format",
default=default_pool)
parser.add_argument('--threshold_matrix_file',
help="location/name of the threshold matrix",
default='annotator_threshold_matrix')
args = parser.parse_args()
threshold_dict = {}
threshold_list = []
current_label_list = []
features, labels_pooled, vec = feats_and_classify.collect_features(
args.pooled_annotators)
print("total len of f, labels", len(features), len(labels_pooled))
for idx in "01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20".split(
" "):
# for idx in "01".split(" "):
current_single_ann = scriptdir + "/../data/cwi_training/cwi_training_" + idx + ".lbl.conll"
f_current, labels_current, v_current = feats_and_classify.collect_features(
current_single_ann)
print(idx, "len of f, labels", len(f_current), len(labels_current))
for idx in "01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20".split(
" "):
# for idx in "01".split(" "):
current_single_ann = scriptdir + "/../data/cwi_training/cwi_training_" + idx + ".lbl.conll"
_, labels_current, _ = feats_and_classify.collect_features(
current_single_ann)
current_label_list.append(labels_current)
maxent, thresholds = getBestThreshold(features, labels_pooled,
labels_current)
threshold_list.extend(thresholds)
print(thresholds)
predictAcrossThresholds(features,
labels_pooled,
labels_current,
maxent,
thresholds,
average=True,
median=True)
pooled_score_dict_ave = defaultdict(list)
pooled_score_dict_med = defaultdict(list)
count = 0
for labels_current in current_label_list:
count += 1
print('ANNOTATOR COUNT ' + str(count))
score_dict_ave, score_dict_med = predictAcrossThresholds(
features,
labels_pooled,
labels_current,
maxent,
np.array(threshold_list),
average=True,
median=True)
for k in score_dict_ave:
pooled_score_dict_ave[k].append(score_dict_ave[k][0])
pooled_score_dict_med[k].append(score_dict_med[k][0])
#finding dimensions of matrix to print out
cols = 20
rows = len(pooled_score_dict_ave[k]) / 20
mat = np.ndarray(shape=(rows, cols),
buffer=np.array(pooled_score_dict_ave[k]),
dtype=float)
print(mat)
pickle.dump(mat, open(args.threshold_matrix_file + "_ave.p", "wb"))
mat = np.ndarray(shape=(rows, cols),
buffer=np.array(pooled_score_dict_med[k]),
dtype=float)
print(mat)
pickle.dump(mat, open(args.threshold_matrix_file + "_med.p", "wb"))
for k in pooled_score_dict_ave:
print(k, ":",
np.array(pooled_score_dict_ave[k]).mean(), ":",
pooled_score_dict_ave[k])
print(k, ":", np.median(pooled_score_dict_med[k]), ":",
pooled_score_dict_med[k])
sys.exit(0)
