def main():
arquivo = read_ground_truth("../../../../annotations/arquivo.tsv")
publico = read_ground_truth("../../../../annotations/publico_politica.tsv")
tp_total = 0
fp_total = 0
fn_total = 0
for x in arquivo:
if x['label'] == '':
continue
"""
print(x['title'])
print(x['ent1'])
print(x['ent2'])
print()
"""
tp, fp, fn = evaluate_ner(clean_title_re(x['title']), [x['ent1'], x['ent2']])
tp_total += tp
fp_total += fp
fn_total += fn
"""
for x in publico:
tp, fp, fn = evaluate_ner(x['title'], [x['ent1'], x['ent2']])
tp_total += tp
fp_total += fp
fn_total += fn
"""
print("Precision: ", tp_total / (tp_total + fp_total))
print("Recall : ", tp_total / (tp_total + fn_total))
def main():
all_data = read_ground_truth("../politiquices_data_v1.0.csv")
direction_clf = DirectionClassifier()
true_direction = []
pred_direction = []
wrong_patterns = defaultdict(int)
correct_patterns = defaultdict(int)
for idx, d in enumerate(all_data):
if "supports" in d["label"] or "opposes" in d["label"]:
clean_title = clean_title_quotes(clean_title_re(d['title']))
ent1 = d["ent1"]
ent2 = d["ent2"]
if ent1 not in clean_title or ent2 not in clean_title:
print("skipped: ", clean_title)
continue
true = "ent2_rel_ent1" if d["label"].endswith("ent1") else "ent1_rel_ent2"
true_direction.append(true)
pred, pattern, context, pos_tags = direction_clf.detect_direction(clean_title, ent1, ent2)
pred_direction.append(pred)
if true != pred:
wrong_patterns[pattern] += 1
"""
if pattern == "default":
print("true: ", true)
print("pred: ", pred)
print(d["title"])
print(context)
# print(pos_tags)
print("\n-----------------------------")
"""
elif true == pred:
correct_patterns[pattern] += 1
"""
if true == "ent2_rel_ent1":
print("true: ", true)
print("pred: ", pred)
print(d["title"])
print(pattern)
print()
print(context)
print("\n-----------------------------")
"""
print(classification_report(true_direction, pred_direction))
print("\nPATTERNS WRONG PREDICTION")
print("----------------------------")
for k, v in wrong_patterns.items():
print(k, v)
print("\nPATTERNS CORRECT PREDICTION")
print("----------------------------")
for k, v in correct_patterns.items():
print(k, v)
def main():
all_data = read_ground_truth("../politiquices_data_v1.0.csv")
labels = remap_y_target([s['label'] for s in all_data])
skf = StratifiedKFold(n_splits=4, random_state=42, shuffle=True)
print("Loading embeddings...")
word2embedding, word2index = get_embeddings()
all_data_shuffled = []
all_preds = []
all_trues = []
fold_n = 0
for train_index, test_index in skf.split(all_data, labels):
x_train = [
doc for idx, doc in enumerate(all_data) if idx in train_index
]
x_test = [doc for idx, doc in enumerate(all_data) if idx in test_index]
y_train = [
label for idx, label in enumerate(labels) if idx in train_index
]
y_test = [
label for idx, label in enumerate(labels) if idx in test_index
]
# get textual contexts
train_textual_context = get_text_tokens(x_train, tokenized=True)
test_textual_context = get_text_tokens(x_test, tokenized=True)
model = RelationshipClassifier(epochs=10)
model.train(train_textual_context,
y_train,
word2index,
word2embedding,
x_val_tks=test_textual_context,
y_val=y_test)
report_str, misclassifications, correct, pred_labels = model.evaluate(
x_test, y_test)
all_data_shuffled.extend(x_train)
all_trues.extend(y_test)
all_preds.extend(pred_labels)
fold_n += 1
print("\n\nFINAL REPORT")
print(classification_report(all_trues, all_preds, zero_division=0.00))
cm = confusion_matrix(all_trues,
all_preds,
labels=['opposes', 'other', 'supports'])
print_cm(cm, labels=['opposes', 'other', 'supports'])
print()
"""
def main():
data_publico = read_ground_truth(
"../../../../annotations/publico_politica.tsv")
data_arquivo = read_ground_truth("../../../../annotations/arquivo.tsv")
docs, labels = pre_process_train_data(data_arquivo + data_publico)
skf = StratifiedKFold(n_splits=2, random_state=42, shuffle=True)
fold_n = 0
for train_index, test_index in skf.split(docs, labels):
x_train = [doc for idx, doc in enumerate(docs) if idx in train_index]
x_test = [doc for idx, doc in enumerate(docs) if idx in test_index]
y_train = [
label for idx, label in enumerate(labels) if idx in train_index
]
y_test = [
label for idx, label in enumerate(labels) if idx in test_index
]
max_length = max([len(x) for x in x_train])
kclf = KerasTextClassifier(input_length=max_length,
n_classes=len(set(labels)),
max_words=15000,
emb_dim=50)
kclf.fit(x_train,
y_train,
X_val=x_test,
y_val=y_test,
epochs=15,
batch_size=16)
predictions = kclf.encoder.inverse_transform(kclf.predict(x_test))
print(classification_report(y_test, predictions))
fold_n += 1
max_length = max([len(x) for x in docs])
kclf = KerasTextClassifier(input_length=max_length,
n_classes=len(set(labels)),
max_words=150000,
emb_dim=50)
kclf.fit(docs, labels, epochs=15, batch_size=8)
kclf.save(path="trained_models/relationship_clf")
def get_wiki_ids_from_annotations():
training_data = read_ground_truth("../politiquices_data_v1.0.csv")
annotated_wiki_ids = set()
for entry in training_data:
p1_id = entry["ent1_id"]
p2_id = entry["ent2_id"]
if p1_id == 'None' or p2_id == 'None':
continue
annotated_wiki_ids.add(p1_id.split("/")[-1])
annotated_wiki_ids.add(p2_id.split("/")[-1])
print(f"{len(list(annotated_wiki_ids))} entities from annotations")
return list(annotated_wiki_ids)
def main():
args = parse_args()
arquivo = []
publico = []
chave = []
rels_gold = []
arquivo_publico_urls = []
gold_articles = []
gold_urls = None
gold_persons = None
if args.annotations:
training_data = read_ground_truth(
"../classifiers/politiquices_data_v1.0.csv")
gold_articles, gold_persons, rels_gold, urls_to_ignore = process_gold(
training_data)
print("ground truth : ", len(gold_articles))
gold_urls = [article.url for article in gold_articles] + urls_to_ignore
if args.arquivo:
# remove duplicates: keep only unique urls
arquivo_unique_url = remove_duplicates_with_same_url(args.arquivo)
# remove duplicates: same crawled url but different crawl date, keep oldest version
unique = remove_url_crawled_diff_dates_duplicates(arquivo_unique_url)
# remove duplicates: title + crawl date same and URL overlaps, e.g.: with/without params
arquivo = remove_duplicates_same_domain(unique)
# gather all publico.pt URLs
arquivo_publico_urls = get_publico_urls_in_arquivo(unique)
print("arquivo.pt : ", len(arquivo))
if args.publico:
publico = remove_duplicates_with_same_url(args.publico)
print("publico.pt : ", len(publico))
if args.chave:
chave = [entry for entry in processed_titles(args.chave)]
print("CHAVE : ", len(chave))
articles, persons, relationships = process_data(
arquivo + publico + chave,
persons=gold_persons,
publico_urls_in_arquivo=arquivo_publico_urls,
gold_urls=gold_urls,
)
populate_graph(articles + gold_articles, persons,
relationships + rels_gold)
def annotations():
# data = read_ground_truth("../../../nlp/classifiers/politiquices_training_data.tsv")
# data_webapp = read_ground_truth("../../../nlp/api_annotations/annotations_from_webapp.tsv")
dataset = read_ground_truth(
"../../../nlp/classifiers/politiquices_data_v1.0.csv")
webapp_data = read_ground_truth(
"../../../nlp/api_annotations/annotations_from_webapp.tsv")
training_data = [d['title'] for d in dataset + webapp_data]
all_other = get_relationships_to_annotate()
to_annotate = []
skipped = 0
for doc in all_other:
if doc['title'] in training_data:
skipped += 1
continue
to_annotate.append(doc)
print(f"Skipped {skipped} titles, already in annotated data")
for idx, r in enumerate(to_annotate):
link_one = r["title"].replace(
r["ent1_str"],
'<a id="ent_1" href="entity?q=' + r["ent1"].split("/")[-1] + '">' +
r["ent1_str"] + "</a>",
)
title_link = link_one.replace(
r["ent2_str"],
'<a id="ent_2" href="entity?q=' + r["ent2"].split("/")[-1] + '">' +
r["ent2_str"] + "</a>",
)
r["title_clickable"] = title_link
r["id"] = idx
return render_template("annotate_other.html", items=to_annotate)
def main():
# Defining DistilBERT tokonizer
tokenizer = DistilBertTokenizer.from_pretrained(distil_bert,
do_lower_case=True,
add_special_tokens=True,
max_length=21,
pad_to_max_length=True)
model_name = 'unicamp-dl/ptt5-base-portuguese-vocab'
tokenizer = T5Tokenizer.from_pretrained(model_name)
f_name = "../../../../data/annotations/publico_politica.tsv"
data = read_ground_truth(f_name, only_label=True)
docs, labels = pre_process_train_data(data)
y_test = [re.sub(r"_?ent[1-2]_?", "", y_sample) for y_sample in labels]
# tokenize
train_data = tokenize(docs[0:500], tokenizer)
train_label = y_test[0:500]
le = LabelEncoder()
y_train_encoded = le.fit_transform(train_label)
y_train_vec = to_categorical(y_train_encoded, num_classes=None)
model = fine_tuning_pretrained_transformer_model()
model.compile(loss={"output": categorical_crossentropy},
optimizer="adam",
metrics=["accuracy"])
model.fit(train_data, y_train_vec, epochs=10)
f_name = "model_test_10"
model.save_weights(f_name)
test_data = tokenize(docs[500:1000], tokenizer)
test_label = y_test[500:1000]
x_predicted_probs = model.predict(test_data)
labels_idx = np.argmax(x_predicted_probs, axis=1)
pred_labels = le.inverse_transform(labels_idx)
print("\n" + classification_report(test_label, pred_labels))
cm = confusion_matrix(test_label, pred_labels, labels=le.classes_)
print_cm(cm, labels=le.classes_)
print()
def main():
all_data = read_ground_truth("../politiquices_data_v1.0.csv")
labels = remap_y_target([s['label'] for s in all_data])
skf = StratifiedKFold(n_splits=4, random_state=42, shuffle=True)
all_data_shuffled = []
all_predictions = []
all_trues = []
fold_n = 0
for train_index, test_index in skf.split(all_data, labels):
print(f"fold: {fold_n}")
x_train = [
doc for idx, doc in enumerate(all_data) if idx in train_index
]
x_test = [doc for idx, doc in enumerate(all_data) if idx in test_index]
y_train = [
label for idx, label in enumerate(labels) if idx in train_index
]
y_test = [
label for idx, label in enumerate(labels) if idx in test_index
]
all_data_test = [
label for idx, label in enumerate(all_data) if idx in test_index
]
# target vector
le = LabelEncoder()
y_train_encoded = le.fit_transform(y_train)
# get textual contexts
train_textual_context = get_text_tokens(x_train, tokenized=True)
test_textual_context = get_text_tokens(x_test, tokenized=True)
# get other features
# train_other_features = get_features(train_textual_context)
# test_other_features = get_features(test_textual_context)
# no tokenization
tfidf = TfidfVectorizer(
tokenizer=dummy_fun,
preprocessor=dummy_fun,
# ngram_range=(1, 2)
)
# n-grams
# tfidf = TfidfVectorizer(ngram_range=(1, 2))
tf_idf_weights = tfidf.fit_transform(train_textual_context)
# clf = LogisticRegression(multi_class='multinomial', class_weight='balanced')
# clf = SGDClassifier(max_iter=15000, class_weight='balanced')
# clf = LinearSVC(class_weight='balanced', max_iter=2000)
# clf = MultinomialNB(alpha=0.3, fit_prior=True, class_prior=None)
clf = SVC(C=1.0,
class_weight='balanced',
decision_function_shape='ovo',
probability=True)
clf.fit(tf_idf_weights, y_train_encoded)
test_tf_idf_weights = tfidf.transform(test_textual_context)
predicted = clf.predict(test_tf_idf_weights)
prob_predictions = clf.predict_proba(test_tf_idf_weights)
y_pred_label = [le.classes_[pred] for pred in predicted]
# ToDo: imprimir isto para as classificações erradas
"""
for prob, pred, pred_label, contexts, label, sample in zip(
prob_predictions, predicted, y_pred_label, test_textual_context, y_train, x_test
):
print(sample['title'])
print("prob : ", prob)
print("pred : ", pred)
print("class: ", pred_label)
print()
"""
all_data_shuffled.extend(x_train)
all_trues.extend(y_test)
all_predictions.extend(y_pred_label)
error_analysis(y_pred_label, predicted, prob_predictions, y_test,
all_data_test)
fold_n += 1
# correct and not 'other'
"""
count = 0
for sample in supports_correct + opposes_correct:
doc = nlp(sample['title'])
related = are_entities_related(doc, sample)
if related:
continue
else:
count+=1
print(sample['title'])
print("ent1: ", sample['ent1'])
print("ent2: ", sample['ent2'])
if sample in supports_correct:
print("pred: supports")
elif sample in opposes_correct:
print("pred: opposes")
print("related: ", are_entities_related(doc, sample))
for token in doc:
print(f"{token.text:<10} \t {token.pos_:>10} \t {token.dep_:>20} \t {list(token.children)}")
print()
print(count)
"""
# other classified as opposes
"""
for sample in other_classified_as_opposes + other_classified_as_supports:
print("\n"+sample['title'])
print("ent1: ", sample['ent1'])
print("ent2: ", sample['ent2'])
print("true: other")
if sample in other_classified_as_supports:
print("pred: supports")
elif sample in other_classified_as_opposes:
print("pred: opposes")
print()
doc = nlp(sample['title'])
pos_tags = [t for t in doc]
before, between, after = get_contexts(pos_tags, sample['ent1'], sample['ent2'])
print("BEF: ", before)
print("BET: ", between)
print("AFT: ", after)
print("features used: ", get_text_tokens([sample]))
for token in doc:
print(f"{token.text:<10} \t {token.dep_:>10} \t {list(token.children)}")
print()
print("\n\n\n--------------")
print(
"""
# supports classified as opposes
"""
for sample in supports_classified_as_opposes:
print(sample['title'])
print("true: supports")
print("pred: opposes")
doc = nlp(sample['title'])
pos_tags = [t for t in doc]
before, between, after = get_contexts(pos_tags, sample['ent1'], sample['ent2'])
print("BEF: ", before)
print("BET: ", between)
print("AFT: ", after)
print("features used: ", get_text_tokens([sample]))
# print("features tags: ", [(t.text, t.lemma_, t.pos_) for t in pos_tags])
print("\n\n--------------")
"""
# opposes classified as supports
"""
for sample in opposes_classified_as_supports:
print(sample['title'])
print(sample['label'])
print("true: opposes")
print("pred: supports")
doc = nlp(sample['title'])
pos_tags = [t for t in doc]
before, between, after = get_contexts(pos_tags, sample['ent1'], sample['ent2'])
print("BEF: ", before)
print("BET: ", between)
print("AFT: ", after)
print("features used: ", get_text_tokens([sample]))
# print("features tags: ", [(t.text, t.lemma_, t.pos_) for t in pos_tags])
print("\n\n--------------")
"""
# opposes classified as other
for sample, prob_pred in opposes_classified_as_other:
print(sample['title'])
print("true: opposes")
print("pred: other")
print(prob_pred)
"""
doc = nlp(sample['title'])
pos_tags = [t for t in doc]
before, between, after = get_contexts(pos_tags, sample['ent1'], sample['ent2'])
print("BEF: ", before)
print("BET: ", between)
print("AFT: ", after)
print("features used: ", get_text_tokens([sample]))
"""
print("\n\n--------------")
# apply direction classifier to those that were correct
"""
for sample in supports_correct + opposes_correct:
title = sample['title']
ent1 = sample['ent1']
ent2 = sample['ent2']
pred_direction, pattern, context, tags = direction_clf.detect_direction(title, ent1, ent2)
print(title)
print(sample['label'])
print(pred_direction)
print("\n\n--------------")
"""
print("\n\nFINAL REPORT")
print(classification_report(all_trues, all_predictions,
zero_division=0.00))
cm = confusion_matrix(all_trues,
all_predictions,
labels=['opposes', 'other', 'supports'])
print_cm(cm, labels=['opposes', 'other', 'supports'])
print()
train_all_data(all_data, labels)
def main():
all_data = read_ground_truth("../politiquices_data_v1.0.csv")
labels = remap_y_target([s['label'] for s in all_data])
relevant_labels = ['opposes', 'supports']
relevant = [
'relevant' if label in relevant_labels else 'non-relevant'
for label in labels
]
predicted = []
relevant_not_related = 0
relevant_related = 0
other_related = 0
other_not_related = 0
for sample, label in zip(all_data, labels):
doc = nlp(sample["title"])
if label in ['opposes', 'supports']:
continue
if not are_entities_related(doc, sample):
other_not_related += 1
continue
other_related += 1
print(sample['title'])
print("ent1: ", sample['ent1'])
print("ent2: ", sample['ent2'])
print(label)
for token in doc:
print(
f"{token.text:<10} \t {token.pos_:>10} \t {token.dep_:>10} \t {list(token.children)}"
)
print()
# print("relevant_related: ", relevant_related)
# print("relevant_not_related: ", relevant_not_related)
# print()
print("other_related: ", other_related)
print("other_not_related: ", other_not_related)
print()
"""
# total = relevant_not_related + relevant_not_related
# print(total-relevant_not_related/total)
# print(classification_report(relevant, predicted))
"""
"""
verb = "<ADV>?<AUX|VERB><PART>?<ADV>?"
word = "<NOUN|ADJ|ADV|DET|ADP>"
preposition = "<ADP|ADJ>"
rel_pattern = "( %s (%s* (%s)+ )? )+ " % (verb, word, preposition)
reverb_mark = '''REVERB_PATTERN: {%s}''' % rel_pattern
reverb_pattern = nltk.RegexpParser(reverb_mark)
word_sentiment = WordSentiment()
contexts = defaultdict(lambda: defaultdict(list))
verbs = defaultdict(int)
verbs_lemma = defaultdict(int)
"""
"""
for sample in all_data:
title = sample["title"]
ent1 = sample["ent1"]
ent2 = sample["ent2"]
pos_tags = get_pos_tags(title)
context = get_context(pos_tags, ent1, ent2)
# to catch errors in training data
if context is None or len(context) == 0:
continue
label = sample['label']
if label in other_labels:
label = 'other'
else:
label = re.sub(r"_?ent[1-2]_?", "", label)
contexts['_'.join([t.pos_ for t in context])][label].append(' '.join([t.text for t in context]))
"""
"""
for x in sorted(verbs_lemma, key=lambda x: verbs_lemma[x], reverse=True):
print(x, verbs_lemma[x], word_sentiment.get_sentiment(x))
print(len(verbs_lemma))
"""
"""
for pos_tags in sorted(contexts, key=lambda x: len(contexts[x]), reverse=True):
print(pos_tags, "labels->", len(set(contexts[pos_tags])))
for label in contexts[pos_tags]:
print(label)
print("-----")
for text in sorted(set(contexts[pos_tags][label])):
print(text)
print("\n")
print("\n\n")
print(len(contexts))
"""
"""
def main():
data = read_ground_truth("../politiquices_data_v1.0.csv")
for x in data:
entity_one_str = x["ent1"]
entity_one_id = x["ent1_id"]
entity_two_str = x["ent2"]
entity_two_id = x["ent2_id"]
url = x['url']
if url.startswith('http://www.publico.pt'):
news_id = url.split("/")[-1]
url = 'https://publico.pt/' + news_id
evaluate_one(entity_one_str, entity_one_id, url)
evaluate_one(entity_two_str, entity_two_id, url)
wiki_id = []
unique_ne = []
ambiguous_named_entities = defaultdict(list)
for k in sorted(freqs, key=lambda x: len(freqs[x]), reverse=True):
# print(k, '\t', len(freqs[k]), len(set(freqs[k])))
for ne in freqs[k]:
ambiguous_named_entities[ne].append(k)
unique_ne.append(len(set(freqs[k])))
if k != 'None':
wiki_id.append(k)
print()
print("#named-entities (surface strings): ", len(ent_true))
print("#unique ids: ", len(freqs.keys()))
print("#unique surface strings: ", len(set(all_ent_surface_string)))
print(" with Wikidata : ", len(set(ent_surface_string_with_wiki)))
print(" without Wikidata : ",
len(set(ent_surface_string_without_wiki)))
print()
print("mean (unique_ne): ", mean(unique_ne))
print("mode (unique_ne): ", mode(unique_ne))
print("median (unique_ne): ", median(unique_ne))
print("st. dev. (unique_ne): ", stdev(unique_ne))
"""
for k, v in ambiguous_named_entities.items():
print(k, '\t', len(set(v)))
"""
seaborn.boxplot(data=unique_ne)
plt.savefig('unique.png', dpi=300)
not_found = []
correct = []
wrong = []
for ent_string, true_id, pred_id in zip(all_ent_surface_string, ent_true,
ent_pred):
if true_id.split("/")[-1] == pred_id.split("/")[-1]:
correct.append((ent_string, true_id))
elif true_id != pred_id:
if pred_id == 'None' and true_id != 'None':
not_found.append((ent_string, true_id))
else:
wrong.append((ent_string, true_id, pred_id))
print("CORRECT : ", len(correct))
print("NOT FOUND: ", len(not_found))
print("WRONG : ", len(wrong))
print()
print("accuracy: ", accuracy_score(ent_true, ent_pred))
write_iterator_to_file(sorted(not_found),
"entity_linking_could_not_disambiguate.txt")
write_iterator_to_file(sorted(wrong), "entity_linking_wrong.txt")
def main():
training_data = read_ground_truth("../politiquices_training_data.tsv")
training_data_webapp = read_ground_truth("../../api_annotations/annotations_from_webapp.tsv")
all_data = training_data + training_data_webapp
word_sentiment = WordSentiment()
contexts = defaultdict(int)
other = ['e', ',']
supports = ['apoiar', 'convidar', 'elogiar', 'confiança', 'felicitar']
opposes = ['acusar', 'criticar', 'responsabilizar', 'desmentir', 'atacar', 'contrariar']
true_labels = []
pred_labels = []
for sample in all_data:
if sample['label'] in other_labels:
true_labels.append('other')
else:
true_labels.append(re.sub(r"_?ent[1-2]_?", "", sample['label']))
title = sample["title"]
ent1 = sample["ent1"]
ent2 = sample["ent2"]
pos_tags = get_pos_tags(title)
context = get_context(pos_tags, ent1, ent2)
if context is None:
exit(-1)
# print([(t.text, t.pos_, t.morph) for t in context])
# print([t.text for t in context if t.pos_ == 'ADJ'])
context_text = ' '.join([t.lemma_ for t in context])
contexts[context_text] += 1
pred_label = 'other'
if any(x == context for x in other):
pred_label = 'other'
elif any(x in context_text for x in supports):
pred_label = 'supports'
elif any(x in context_text for x in opposes):
pred_label = 'opposes'
pred_labels.append(pred_label)
"""
if ' '.join([t.text for t in context]) == '':
print(title)
print(ent1)
print(ent2)
print()
print(sample['label'])
print("\n\n--------------")
"""
"""
for t in context:
if t.pos_ in ['ADP']:
continue
print(t.text, t.pos_, t.lemma_, '\t', word_sentiment.get_sentiment(t.lemma_))
"""
for x in sorted(contexts, key=lambda x: contexts[x], reverse=True):
print(x, contexts[x])
print()
print(classification_report(true_labels, pred_labels, zero_division=0.00))
cm = confusion_matrix(true_labels, pred_labels, labels=['opposes', 'other', 'supports'])
print_cm(cm, labels=['opposes', 'other', 'supports'])
print()