def eval_desc_batch():
type_pattern = r"([a-z]*)_"
radio_pattern = r"^(\w*)(_b)(\d*)(_e)(\d*)"
print([item for item in request.form.items()])
batch_id = -1
de_id = -1
for tag_id, value in request.form.items():
str_match = [
t(s) for t, s in zip((str, ),
re.search(type_pattern, tag_id).groups())
][0]
try:
if str_match == "radio":
(_, _, batch_id, _, de_id) = [
t(s)
for t, s in zip((str, str, int, str, int),
re.search(radio_pattern, tag_id).groups())
]
de: DescEval = query_by_id(DescEval, de_id)
de.approve(value=value)
de.add_self()
elif str_match == "comment":
(_, _, batch_id, _, de_id) = [
t(s)
for t, s in zip((str, str, int, str, int),
re.search(radio_pattern, tag_id).groups())
]
de: DescEval = query_by_id(DescEval, de_id)
de.comments = value
de.add_self()
elif str_match == "better":
(_, _, batch_id, _, de_id) = [
t(s)
for t, s in zip((str, str, int, str, int),
re.search(radio_pattern, tag_id).groups())
]
de: DescEval = query_by_id(DescEval, de_id)
de.set_compare_baseline(value)
de.add_self()
except Exception as e:
PrintException()
return app.response_class(response=json.dumps({"error": str(e)}),
status=601,
mimetype="application/json")
try:
_commit_session()
except Exception as exc:
PrintException()
return app.response_class(response=json.dumps({"error": str(exc)}),
status=621,
mimetype="application/json")
return app.response_class(response=json.dumps(request.form),
status=200,
mimetype="application/json")
def _common_names(amr1, amr2):
"""
:type amr1: AMRModel
:type amr2: AMRModel
:param amr1:
:param amr2:
:return:
"""
try:
return_list = set()
names1 = amr1.get_triples(relation='instance', target='name')
names2 = amr2.get_triples(relation='instance', target='name')
names_string_1 = zip(names1,
[amr1.get_names(name.source) for name in names1])
names_string_2 = zip(names2,
[amr2.get_names(name.source) for name in names2])
for name1, string1 in names_string_1:
for name2, string2 in names_string_2:
if string1 == string2 or string1 in string2 or string2 in string1:
return_list.add(
(name1, name2, min([len(string1),
len(string2)])))
return InstrumentedList(return_list)
except Exception as exc:
PrintException()
def get_triple(self, src=None, relation=None, target=None):
try:
if src is None and relation is None and target is None:
return None
if target is not None:
triple_is_name = self.is_name(target)
if triple_is_name:
return triple_is_name
return_list = []
for triple in self.list_triples:
if _match_triple(src, relation, target, triple):
return_list.append(triple)
if return_list:
if relation is not None and relation == 'instance':
return [
triple for triple in return_list
if triple.target is not None
][0]
else:
return return_list[0]
else:
return None
except Exception as exc:
PrintException()
return None
def get_subgraph(self, top):
"""
:type top: Triple
:param top:
:return:
"""
try:
_span = self.span(top=top)
possible_children = InstrumentedList(set(_span))
not_instances = []
instances = []
for children in possible_children:
if children.relation != 'instance':
not_instances.append(children)
else:
instances.append(children)
for not_instance in not_instances:
if all(instance.source != not_instance.source and instance.
source != not_instance.target and not_instance != top
for instance in instances):
possible_children.remove(not_instance)
possible_children = organize_triples_list(
triples_list=possible_children, top=top)
new_subgraph = AMRModel(
top=top.source,
triples=copy_triples_from_list(
possible_children)) if possible_children else None
return new_subgraph
except Exception as exc:
PrintException()
def _common_persons(amr1, amr2):
try:
return_list = set()
persons_self = amr1.get_triples(relation='instance', target='person')
persons_other = amr2.get_triples(relation='instance', target='person')
person_names_self = zip(persons_self, [
amr1.get_triples(src=ps.source, relation='name')
for ps in persons_self
])
person_names_other = zip(persons_other, [
amr2.get_triples(src=po.source, relation='name')
for po in persons_other
])
for person1, name1 in person_names_self:
names1 = amr1.get_names(name1.source)
for person2, name2 in person_names_other:
names2 = amr2.get_names(name2.source)
if names1 == names2 or names1 in names2 or names2 in names1:
return_list.add((person1, person2))
return return_list
except Exception as exc:
PrintException()
def triples(triples_list, src=None, relation=None, target=None):
try:
if src is None and relation is None and target is None:
return triples_list
else:
return_list = InstrumentedList([])
for triple in triples_list:
if _match_triple(src, relation, target, triple):
return_list.append(triple)
return return_list
except Exception as exc:
PrintException()
def relate_amr(self):
try:
amr_list = parse_to_amr_list(snts=self.news_object.get_sentences(
return_type="str"))
zip_amr_sntobj = zip(amr_list, self.news_object.get_sentences())
sntobj: Sentence
for amr, sntobj in zip_amr_sntobj:
amr_model = AMRModel(object=amr.get_penman(
return_type='graph'))
sntobj.add_amr(amr_model)
except Exception as exc:
PrintException()
def describe():
try:
response = do_describe(request.form["link"], method='baseline4')
response.pop('news', None)
response.pop('grupo', None)
return app.response_class(response=json.dumps(response),
status=200,
mimetype='application/json',
direct_passthrough=True)
except Exception as exc:
PrintException()
return app.response_class(response=json.dumps({'error': 'erro'}),
status=530,
mimetype='application/json',
direct_passthrough=True)
def organize_triples_list(triples_list, top):
try:
if isinstance(top, Triple):
new_list = InstrumentedList([])
new_list.append(top)
for triple in triples_list:
if triple != top:
if triple.relation == 'instance':
new_list.insert(1, triple)
else:
new_list.append(triple)
return new_list
except Exception as exc:
PrintException()
def align_manual(self, legenda, titulo, texto, img_path, person_choose,
object_choose):
try:
"""Alinha a partir de uma url fornecida pela usuario"""
self.titulo_noticia = titulo
self.legenda = legenda
self.noticia = texto
self.path_imagem = img_path
self._set_manual_resources()
self._process_text_image()
except:
PrintException()
# Alinha as pessoas
return self.begin_alignment(person_choose=person_choose,
object_choose=object_choose)
def do_align(link):
try:
_experimento_pessoa = 2
_experimento_objeto = 1
if "folha" in link:
alinhador = AlignTool(crawler=crawler_folha)
elif "bbc" in link:
alinhador = AlignTool(crawler=crawler_bbc)
else: # invalid url format
return app.response_class(response=json.dumps(
{'message': 'Bad request'}),
status=400,
mimetype='application/json')
# try:
news_object: News
grupo: AlignmentGroup
try:
result_pessoas, result_objetos, img_url, titulo, legenda, texto, dic_avaliacao, grupo, news_object = \
alinhador.align_from_url(link, _experimento_pessoa, _experimento_objeto)
except Exception as exc:
result_pessoas, result_objetos, img_url, titulo, legenda, texto, dic_avaliacao, grupo, news_object = \
alinhador.align_backup(link, _experimento_pessoa, _experimento_objeto)
if img_url != '':
shutil.copy2(STATIC_REL + 'alinhamento2.jpg', img_url)
return dict(result_pessoas=result_pessoas,
result_objetos=result_objetos,
img_alinhamento=img_url.replace(STATIC_REL, 'static/'),
texto=texto,
legenda=legenda,
titulo=titulo,
grupo=grupo,
news=news_object,
dic_avaliacao=dic_avaliacao)
except Exception as exc:
PrintException()
def _match_triple(src=None, relation=None, target=None, triple=None):
"""
:param src:
:param relation:
:param target:
:param triple:
:return:
"""
def __eq_or_in(attr, triple_attr):
return attr is not None and (attr.upper() == triple_attr.upper()
or attr.upper() in triple_attr.upper()
or triple_attr.upper() in attr.upper())
try:
if triple is None:
return False
rule = (relation is None and target is None
and (src is not None and src == triple.source))
rule = rule or (relation is None and __eq_or_in(target, triple.target)
and src is None)
rule = rule or (relation is None and __eq_or_in(target, triple.target)
and (src is not None and src == triple.source))
rule = rule or (__eq_or_in(relation, triple.relation)
and target is None and src is None)
rule = rule or (__eq_or_in(relation, triple.relation)
and target is None and
(src is not None and src == triple.source))
rule = rule or (__eq_or_in(relation, triple.relation)
and __eq_or_in(target, triple.target) and src is None)
rule = rule or (__eq_or_in(relation, triple.relation)
and __eq_or_in(target, triple.target) and
(src is not None and src == triple.source))
return rule
except Exception as exc:
PrintException()
def describe_batch():
print([(k, v) for k, v in request.form.items()])
try:
links = list(set(request.form["batch_links"].split("\n")))
links.sort()
total = len(links)
batch_name = request.form["batch_name"]
restriction = request.form["restriction"]
failed = []
with open(f'{TMP_DIR}/test_description.csv', 'w') as csv_file:
writer = csv.DictWriter(csv_file, fieldnames=FIELDS)
writer.writeheader()
if request.form["all"] == "1":
for i, link in enumerate(links):
desc_batch = create_desc_batch(name=f'{batch_name}_{link}')
print(f"#####\n\tbegining [{i}/{total}]\n#####")
link = link.replace('\r', '')
for selection in [0, 1, 2]:
selection_string = _get_selection_string(selection)
for method in ['baseline4', 'baseline5']:
try:
print(
f"#####\n\tbegining [{i}/{total} | {method}-{selection_string}]\n#####"
)
response = do_describe(link=link,
method=method,
select=selection)
news_object = response["news"]
news_object.fantasy_id = HARD_CODED_LINKS.index(
link)
grupo = response["grupo"]
for alignment in grupo.get_all_alignments():
if restriction == "all" or \
(restriction == "ne" and alignment.get_is_ne()) or \
(restriction == "obj" and not alignment.get_is_ne()):
alignment.add_to_db()
desc = alignment.get_description()
if desc is not None:
desc_eval = create_desc_eval(
desc_model=desc)
desc_batch.add_desc_eval(desc_eval)
desc_eval.add_self()
desc_batch.add_self()
news_object.set_path(response["img_alinhamento"])
news_object.save()
_write_test_csv(news_object)
_commit_session()
print(
f"#####\n\tfinishing [{i+1}/{total} | {method}-{selection_string}]\n#####"
)
except Exception as exc:
PrintException()
failed.append((link, f'{str(exc)}'))
print(f"#####\n\tfinishing [{i + 1}/{TOTAL}]\n#####")
if len(desc_batch.desc_eval_list) != 6:
failed.append((
link,
f'len != 6: {", ".join([de.get_method() for de in desc_batch.desc_eval_list])}'
))
else:
method = request.form["method"]
selection = int(request.form["selection"])
selection_string = _get_selection_string(selection)
desc_batch = create_desc_batch(
name=f'{batch_name}_method_{method}_{selection_string}')
for i, link in enumerate(links):
try:
print(
f"#####\n\tbegining [{i}/{total} | {method}-{_get_selection_string(selection)}]\n#####"
)
link = link.replace('\r', '')
response = do_describe(link=link,
method=method,
select=selection)
news_object = response["news"]
news_object.fantasy_id = i
grupo = response["grupo"]
for alignment in grupo.get_all_alignments():
if restriction == "all" or \
(restriction == "ne" and alignment.get_is_ne()) or \
(restriction == "obj" and not alignment.get_is_ne()):
alignment.add_to_db()
desc = alignment.get_description()
if desc is not None:
desc_eval = create_desc_eval(desc_model=desc)
desc_batch.add_desc_eval(desc_eval)
desc_eval.add_self()
desc_batch.add_self()
news_object.set_path(response["img_alinhamento"])
news_object.save()
_write_test_csv(news_object)
_commit_session()
print(
f"#####\n\tfinishing [{i + 1}/{total} | {method}-{selection_string}]\n#####"
)
except Exception as exc:
PrintException()
failed.append((link, f'{str(exc)}'))
if failed:
with open(f'{TMP_DIR}/failed_links.txt', 'w') as link_file:
for link, exc in failed:
link_file.write(f'{link}, {exc}\n')
from app.align_module.models import News
print("Done!")
return app.response_class(response=json.dumps({"success": "success"}),
status=200,
mimetype="application/json",
direct_passthrough=True)
except Exception as exc:
PrintException()
return app.response_class(response=json.dumps({"error": "error"}),
status=200,
mimetype="application/json",
direct_passthrough=True)
def biggest_amr(amrmodel_list):
try:
return max([amr for amr in amrmodel_list],
key=methodcaller('get_size'))
except Exception as exc:
PrintException()
def add(self, other, tuple_ref):
"""
:type tuple_ref: tuple
:param tuple_ref:
:type other: AMRModel
:param other:
:return:
"""
old_penman = self.penman
top = self.get_triple(src=self.top, relation='instance')
if any(triple.target == top.target and triple.source != tuple_ref[1]
for triple in other.list_triples):
return self.list_triples
try:
other_top = other.get_top()
old_triples = other.get_triples()
change_variable_list = []
if old_triples:
new_triples = set()
for old_triple in old_triples:
if old_triple.source == tuple_ref[1]:
if old_triple.relation != 'instance':
new_triple = Triple(src=tuple_ref[0],
rel=old_triple.relation,
tgt=old_triple.target)
new_triples.add(new_triple)
elif old_triple.target == tuple_ref[1]:
new_triple = Triple(src=old_triple.source,
rel=old_triple.relation,
tgt=tuple_ref[0])
new_triples.add(new_triple)
elif old_triple.is_instance() and old_triple.target:
old_source = old_triple.source
new_source = increase_variable(
old_triple.source, self.get_variable_list())
new_triple = Triple(src=new_source,
rel=old_triple.relation,
tgt=old_triple.target)
new_triples.add(new_triple)
change_variable_list.append((old_source, new_source))
elif not old_triple.is_instance() or not old_triple.target and old_triple.target != tuple_ref[1] and \
old_triple.source != tuple_ref[1]:
new_triple = Triple(src=old_triple.source,
rel=old_triple.relation,
tgt=old_triple.target)
new_triples.add(new_triple)
newer_triples = set()
if change_variable_list:
for old, new in change_variable_list:
for triple in new_triples:
if triple.source == old:
triple.source = new
newer_triples.add(triple)
else:
newer_triples.add(triple)
else:
newer_triples = new_triples
self.list_triples = InstrumentedList(
newer_triples.union(set(self.list_triples)))
self.list_triples = organize_triples_list(
self.list_triples, top)
self.penman = triple_model_list_to_penman(
self.list_triples, self.top)
return self.list_triples
except Exception as exc:
print(old_penman)
print(self.list_triples)
PrintException()
def _generate_amr(self, method='baseline4'):
"""
:param method:
:return:
"""
try:
try:
amr_list = sntsmodel_to_amrmodel(
[source for source, focus in self.src_focus])
information = []
amr: AMRModel
print(self.src_focus)
for focus_term, amr in zip(
[focus for source, focus in self.src_focus], amr_list):
is_name = amr.is_name(focus_term)
if not is_name:
focus_triple = amr.get_triple(relation='instance',
target=focus_term)
else:
focus_triple = is_name
if focus_triple is not None and focus_triple:
focus_subgraph = amr.get_subgraph(top=focus_triple)
focus_parent = amr.get_parents(focus_triple)
information.append({
"triple":
create_triple(copy=focus_triple),
"subgraph":
create_amrmodel(copy=focus_subgraph),
"parent":
focus_parent,
"amr":
amr
})
if information:
base_info, information = self._choose_biggest(information)
# consider parents
if method == 'baseline4':
base_info = self._add_parent_to_base(info=base_info,
base=base_info)
appended_amr_list = []
before = base_info["subgraph"].get_penman(
return_type='str', indent=True)
if len(information) > 1:
for info in information:
base_info["subgraph"].add(
other=info["subgraph"],
tuple_ref=(base_info["triple"].source,
info["triple"].source))
# consider parents
if method == 'baseline4':
base_info = self._add_parent_to_base(
info=info, base=base_info)
if before != base_info["subgraph"].get_penman(
return_type='str', indent=True):
appended_amr_list.append(info)
return base_info["subgraph"], base_info["amr"], [info["amr"] for info in appended_amr_list], \
[str(info["triple"]) for info in appended_amr_list]
except Exception as exc:
PrintException()
print(f'Error on {method}: {str(exc)}')
except Exception as exc:
PrintException()
print(f'[{__file__}] Error while generating AMR: {str(exc)}')
def begin_alignment(self, person_choose=None, object_choose=None):
try:
self.group = AlignmentGroup(name="align_tool")
# Alinha as pessoas
person = AlignPersons(self.lst_legenda,
self.lst_top_nomeadas_texto,
self.list_boundingBoxOrganizada,
self.directory + "/img_original.jpg",
self.directory + "/",
self.index_cor_bounding_box,
self.colors_bounding_box,
align_group=self.group,
palette=self.palette)
persons_aligned, self.group = person.align(person_choose)
if persons_aligned is None:
persons_aligned = {}
# O indice das cores continua de onde parou o indice realizado no alinhamento de pessoas.
print("INDEX -- " + str(self.index_cor_bounding_box))
print(self.colors_bounding_box[2])
# Alinha os objetos
object = AlignObjects(
self.lst_legenda, self.lst_top_nomeadas_texto,
self.lst_top_substantivos_objects,
self.list_boundingBoxOrganizada, self.directory,
self.dict_lematizado, self.index_cor_bounding_box,
self.colors_bounding_box, self.group, self.palette)
object_aligned, self.group = object.align(object_choose)
self.titulo_noticia = self.titulo_noticia.replace("?", "")
img_url = STATIC_REL + f'{self.titulo_noticia}_{person_choose}_{object_choose}.jpg'
self.news_object.add_from_zip_list(
self.aplicador_pln.get_snt_tok())
self.news_object.add_from_coref_objects(
self.aplicador_pln.get_crefdoc().get_corefs())
crefs = self._mark_coref()
self.noticia += '\n\n<ul>'
for i, cref in enumerate(crefs, start=1):
self.noticia += f'<sub>[{i}] {" | ".join([mention.text for mention in cref])}</sub><br/>'
# reseta o indice
self.palette.reset_colors()
# Prepara o texto, legenda, titulo que serao destacados
# try:
[persons_aligned, object_aligned
] = self._show_align_text(persons_aligned=persons_aligned,
object_aligned=object_aligned)
dic_avaliacao = {}
# prepara o dicionario de avaliação
if persons_aligned:
for key, value in persons_aligned.items():
dic_avaliacao[key] = ''
if object_aligned:
for key, value in object_aligned.items():
dic_avaliacao[key] = ''
# except Exception as e:
# print(e)
self.palette.reset_colors()
# with open(f'{TMP_DIR}/newsobject.json', 'w', encoding='utf-8') as file:
# file.write(self.news_object.get_text())
# json.dump(obj=self.news_object.get_text(), fp=)
self.news_object.add_highlighted(title=self.titulo_noticia,
subtitle=self.legenda,
text=self.noticia)
print("PROCESSO FINALIZADO")
return persons_aligned, object_aligned, img_url, self.titulo_noticia, self.legenda, self.noticia, dic_avaliacao, self.group, self.news_object
except Exception as exc:
PrintException()
def generate(self, method='baseline5', select=1):
"""
:return:
"""
try:
all_alignments = self.news_object.alignments()
descr_to_generate = []
alignment: Alignment
for alignment in all_alignments:
self._source_sentences = self.sentence_selection(
alignment=alignment, select=select)
self._current_alignment = alignment
try:
response = self._generate_amr(method=method)
if response is not None:
generated_amr, main_ancestral, adjacent_ancestral, focus_list = response
descr_to_generate.append(
(alignment, generated_amr, main_ancestral,
adjacent_ancestral, focus_list))
except Exception as exc:
PrintException()
print(
f'[{__file__}] Error while generating AMR from {alignment}: {str(exc)}'
)
amr_to_generate_text = []
generated_text_list = []
for alignment, generated_amr, _, _, _ in descr_to_generate:
amr_to_generate_text.append(generated_amr)
generated_text_list = penman_to_text(
amr_list=amr_to_generate_text)
for ((alignment, generated_amr, main_ancestral, adjacent_ancestral, focus_list), _generated_text) \
in zip(descr_to_generate, generated_text_list):
_generated_text = self.post_process(text=_generated_text,
amr=generated_amr)
description = create_description(text=_generated_text,
method=method)
description.method = f'{method}|{str(select)}'
description.add_amr(generated_amr)
description.set_used_focus(focus_list)
description.save()
main_sentence = create_sentence(
copy=main_ancestral.get_sentence())
main_copy = create_amrmodel(copy=main_ancestral)
main_sentence.add_amr(main_copy)
main_sentence.save()
adj_copies = []
for aa in adjacent_ancestral:
aa_sentence = create_sentence(copy=aa.get_sentence())
aa_copy = create_amrmodel(copy=aa)
aa_sentence.add_amr(aa_copy)
aa_sentence.save()
adj_copies.append(aa_copy)
description.keep_amr(main_copy, adj_copies)
alignment.add_description(description)
except Exception as exc:
PrintException()
print(
f'[{__file__}] Error while generating descriptions: {str(exc)}'
)