def infer_resegmentation(unanno_doc, anno_doc, verbose=0):
"""Infer resegmentation of EDUs.
Parameters
----------
anno_doc : GlozzDocument
Document to filter
verbose : int
Verbosity level
Returns
-------
anno_doc : GlozzDocument
Filtered document, where the support of relations and schemas
has been rewritten.
"""
anno_map = dict()
cautious_map = dict()
new_cdus = []
turns = [x for x in unanno_doc.units if is_turn(x)]
for turn in turns:
# `unannotated` was the starting point for the annotation process
u_edus = [
x for x in unanno_doc.units
if is_edu(x) and turn.span.encloses(x.span)
]
u_ids = set(x.local_id() for x in u_edus)
# `annotated` is the result of the annotation process
# find conflicts, as pair-wise overlaps between annotations
# from `annotated`
a_edus = [
x for x in anno_doc.units
if is_edu(x) and turn.span.encloses(x.span)
]
# 1. map new segments to their original equivalent, backporting
# dialogue act annotation
dup_items = [(elt_a, elt_b) for elt_a, elt_b in itertools.combinations(
sorted(a_edus, key=lambda x:
(x.local_id() in u_ids, x.local_id())), 2)
if (span_eq(elt_a.text_span(), elt_b.text_span(), eps=1)
and elt_b.local_id() in u_ids)]
anno_map.update(dup_items)
# backport dialogue act annotation to original segment
for elt_a, elt_b in dup_items:
if elt_a.type in DIALOGUE_ACTS:
# backport annotation to original segment elt_b
elt_b.type = elt_a.type
elt_b.features = elt_a.features
for k in ['lastModifier', 'lastModificationDate']:
elt_b.metadata[k] = elt_a.metadata[k]
# (locally) update the list of EDUs in anno_doc, so conflicts
# are not computed on trivially mapped segments
a_edus = [x for x in a_edus if x not in anno_map]
# 2. list conflicts, then whitelist them progressively
# NB: we sort EDUs in reverse using their local_ids, so that
# conflict pairs are of the form (stac*, skar*) ; this is
# admittedly a cheap, ad-hoc, trick to simulate an ordering
# such that annotations already present in unannotated < annotations
# introduced in annotated
pw_conflicts = [(elt_a, elt_b)
for elt_a, elt_b in itertools.combinations(
sorted(a_edus,
key=lambda x:
(x.type in DIALOGUE_ACTS, x.local_id())), 2)
if elt_a.overlaps(elt_b)]
# * Two cases are very close: EDU merges, and CDUs
rels_support = set(
anno_map.get(x, x) for rel in anno_doc.relations
for x in [rel.source, rel.target])
edu_merges = [] # list of (list of elt_a, elt_b)
cdu_guess = [] # list of (list of elt_a, elt_b)
for elt_b, pairs in itertools.groupby(pw_conflicts,
key=lambda x: x[1]):
sorted_a = sorted((y[0] for y in pairs),
key=lambda z: z.text_span())
span_seq_a = Span(sorted_a[0].text_span().char_start,
sorted_a[-1].text_span().char_end)
# we approximately check that the sequence of EDUs elts_a
# fully covers the span of elt_b, from start to end, with
# no overlap or that the whole sequence is enclosed in
# the annotation from `annotated` (this happens when some but
# not all of the merged EDUs have been deleted)
if ((approximate_cover(sorted_a, elt_b)
or elt_b.text_span().encloses(span_seq_a))):
# then, it is either an EDU merge or a CDU ;
# if any element of the sequence supports a relation,
# we take this as indicating a CDU
if any(y in rels_support for y in sorted_a):
# broadcast type, features, metadata to the segments
for elt_a in sorted_a:
elt_a.type = _SPLIT_PREFIX + elt_b.type
elt_a.features = elt_b.features
for k in ['lastModifier', 'lastModificationDate']:
elt_a.metadata[k] = elt_b.metadata[k]
# transform elt_b into a CDU
sch_relid = elt_b.local_id()
sch_units = set(y.local_id() for y in sorted_a)
sch_relas = set()
sch_schms = set()
sch_stype = 'Complex_discourse_unit'
sch_feats = {}
sch_metad = elt_b.metadata
new_cdu = Schema(sch_relid,
sch_units,
sch_relas,
sch_schms,
sch_stype,
sch_feats,
metadata=sch_metad)
new_cdus.append(new_cdu)
# map former (bad) segment to its proper CDU version
anno_map[elt_b] = new_cdu
cdu_guess.append((sorted_a, elt_b))
if verbose > 1:
print('CDU {}\nwas {}, from\n {}'.format(
new_cdu, elt_b,
'\n '.join(str(z) for z in sorted_a)))
elif all(elt_a.local_id() in u_ids for elt_a in sorted_a):
edu_merges.append((sorted_a, elt_b))
if verbose > 1:
print('EDU merge {} from\n {}'.format(
elt_b, '\n '.join(str(z) for z in sorted_a)))
else:
err_msg = 'Weird approximate cover:\n{}\n{}'
raise ValueError(
err_msg.format(', '.join(str(y) for y in sorted_a),
elt_b))
# map each of the segments to its CDU, so these pairs can be
# removed from the list of conflicts later
cdu_map = dict()
for elts_a, elt_b in cdu_guess:
map_items = [(elt_a, elt_b) for elt_a in elts_a]
cdu_map.update(map_items)
cautious_map.update(map_items)
# map each of the merged segments to the new, bigger EDU + mark
for elts_a, elt_b in edu_merges:
map_items = [(elt_a, elt_b) for elt_a in elts_a]
anno_map.update(map_items)
cautious_map.update(map_items)
# update list of conflicts: remove pairs that contain a segment
# and its merged EDU, or a segment and its enclosing CDU
pw_conflicts = [(elt_a, elt_b) for elt_a, elt_b in pw_conflicts
if (anno_map.get(elt_a, elt_a) != elt_b
and cdu_map.get(elt_a, elt_a) != elt_b)]
# * EDU splits
edu_splits = dict() # elt_a -> list of elt_b
for elt_a, pairs in itertools.groupby(pw_conflicts,
key=lambda x: x[0]):
sorted_b = sorted((y[1] for y in pairs), key=lambda z: z.span)
# we approximately check that the sequence of new EDUs
# fully covers the span of elt_a, from start to end, with
# no overlap
if ((elt_a.local_id() in u_ids
and approximate_cover(sorted_b, elt_a))):
edu_splits[elt_a] = sorted_b
pw_conflicts = [(elt_a, elt_b) for elt_a, elt_b in pw_conflicts
if elt_a not in set(edu_splits.keys())]
# map the split segment to the first of the resulting EDUs + mark
for elt_a, elts_b in edu_splits.items():
map_items = [(elt_a, elts_b[0])]
anno_map.update(map_items)
cautious_map.update(map_items)
if verbose:
if pw_conflicts:
print('Conflict:')
print('\n'.join(' {}\t<>\t{}'.format(str(elt_a), str(elt_b))
for elt_a, elt_b in pw_conflicts))
# update anno_doc using the computed mapping
anno_map_id = {x.local_id(): y.local_id() for x, y in anno_map.items()}
cautious_map_id = {
x.local_id(): y.local_id()
for x, y in cautious_map.items()
}
# * forget mapped units and segments rewritten as CDUs
anno_doc.units = [
x for x in anno_doc.units
if (not is_edu(x) or x.local_id() not in anno_map_id)
]
# * add the new CDUs to the list of schemas
anno_doc.schemas.extend(new_cdus)
# rewrite the support of relations and schemas
objects = {
x.local_id(): x
for x in itertools.chain(anno_doc.units, anno_doc.relations,
anno_doc.schemas)
}
# * rewrite the support of relations
for rel in anno_doc.relations:
src = anno_map_id.get(rel.span.t1, rel.span.t1)
tgt = anno_map_id.get(rel.span.t2, rel.span.t2)
# update relation span, source, target
rel.span = RelSpan(src, tgt)
rel.source = objects[src]
rel.target = objects[tgt]
# if necessary, mark relation type for review
if src in cautious_map_id or tgt in cautious_map_id:
rel.type = _SPLIT_PREFIX + rel.type
# * rewrite the support of schemas
for sch in anno_doc.schemas:
# sch.id = sch.id
sch.units = set(anno_map_id.get(x, x) for x in sch.units)
sch.relations = set(anno_map_id.get(x, x) for x in sch.relations)
sch.schemas = set(anno_map_id.get(x, x) for x in sch.schemas)
sch.type = sch.type
# sch.features = sch.features
# sch.metadata = sch.metadata
sch.span = sch.units | sch.relations | sch.schemas
sch.fleshout(objects)
return anno_doc
def read_node(node, context=None):
def get_one(name, default, ctx=None):
f = lambda n: read_node(n, ctx)
return on_single_element(node, default, f, name)
def get_all(name):
return list(map(read_node, node.findall(name)))
if node.tag == 'annotations':
hashcode = get_one('metadata', '', 'annotations')
if hashcode is '':
hashcode = None
units = get_all('unit')
rels = get_all('relation')
schemas = get_all('schema')
return (hashcode, units, rels, schemas)
elif node.tag == 'characterisation':
fs = get_one('featureSet', {})
unit_type = get_one('type', None)
return (unit_type, fs)
elif node.tag == 'feature':
attr = node.attrib['name']
val = node.text.strip() if node.text else None
return (attr, val)
# TODO throw exception if we see more than one instance of a key
elif node.tag == 'featureSet':
return dict(get_all('feature'))
elif node.tag == 'metadata' and context == 'annotations':
return node.attrib['corpusHashcode']
elif node.tag == 'metadata':
return dict([(t.tag, t.text.strip()) for t in node])
elif node.tag == 'positioning' and context == 'unit':
start = get_one('start', None)
end = get_one('end', None)
return Span(start, end)
elif node.tag == 'positioning' and context == 'relation':
terms = get_all('term')
if len(terms) != 2:
raise GlozzException("Was expecting exactly 2 terms, but got %d" %
len(terms))
else:
return RelSpan(terms[0], terms[1])
elif node.tag == 'positioning' and context == 'schema':
units = frozenset(get_all('embedded-unit'))
relations = frozenset(get_all('embedded-relation'))
schemas = frozenset(get_all('embedded-schema'))
return units, relations, schemas
elif node.tag == 'relation':
rel_id = node.attrib['id']
(unit_type, fs) = get_one('characterisation', None)
span = get_one('positioning', None, 'relation')
metadata = get_one('metadata', {})
return Relation(rel_id, span, unit_type, fs, metadata=metadata)
if node.tag == 'schema':
anno_id = node.attrib['id']
(anno_type, fs) = get_one('characterisation', None)
units, rels, schemas = get_one('positioning', None, 'schema')
metadata = get_one('metadata', {})
return Schema(anno_id,
units,
rels,
schemas,
anno_type,
fs,
metadata=metadata)
elif node.tag == 'singlePosition':
return int(node.attrib['index'])
elif node.tag == 'start' or node.tag == 'end':
return get_one('singlePosition', None)
elif node.tag in [
'term', 'embedded-unit', 'embedded-relation', 'embedded-schema'
]:
return node.attrib['id']
elif node.tag == 'type':
return node.text.strip()
elif node.tag == 'unit':
unit_id = node.attrib['id']
(unit_type, fs) = get_one('characterisation', None)
span = get_one('positioning', None, 'unit')
metadata = get_one('metadata', {})
return Unit(unit_id, span, unit_type, fs, metadata=metadata)
def _mk_doc(self):
""" Create an educe.annotation.Document from this graph """
def start(name):
return ord(name) - ord('a')
def glozz_id(name):
return 'du_' + str(start(name))
def is_edu(name):
return name not in self.cdus
anno_units = list()
anno_cdus = list()
anno_rels = list()
for du_name, speaker_set in self.speakers.items():
# EDU loop
if not is_edu(du_name):
continue
du_start, du_glozz_id = start(du_name), glozz_id(du_name)
x_edu = Unit(du_glozz_id, Span(du_start, du_start + 1), 'Segment',
dict())
speaker = list(speaker_set)[0]
turn = Unit('t' + du_glozz_id, Span(du_start, du_start + 1),
'Turn', {
'Identifier': du_start,
'Emitter': speaker
})
self.anno_map[du_name] = x_edu
anno_units.append(x_edu)
anno_units.append(turn)
for du_name, sub_names in self.cdus.items():
x_cdu = Schema(
glozz_id(du_name),
set(glozz_id(x) for x in sub_names if is_edu(x)), set(),
set(glozz_id(x) for x in sub_names if not is_edu(x)),
'Complex_discourse_unit', dict())
self.anno_map[du_name] = x_cdu
anno_cdus.append(x_cdu)
rel_count = 0
for src_name in self.down:
for tgt_name, rel_tag in self.down[src_name]:
rel_glozz_id = 'rel_' + str(rel_count)
rel_count += 1
if rel_tag == 'S':
rel_name = 'Q-Elab'
elif rel_tag == 'C':
rel_name = 'Contrast'
else:
raise ValueError('Unknown tag {0}'.format(rel_tag))
rel = Relation(rel_glozz_id,
RelSpan(glozz_id(src_name), glozz_id(tgt_name)),
rel_name, dict())
self.anno_map[(src_name, tgt_name)] = rel
anno_rels.append(rel)
dialogue = Unit(
'dialogue_0',
Span(0, max(u.text_span().char_end for u in anno_units)),
'Dialogue', {})
anno_units.append(dialogue)
doc = Document(anno_units, anno_rels, anno_cdus,
string.ascii_lowercase)
return doc
def __init__(self, id, units, relations, schemas):
Schema.__init__(self, id, frozenset(units), frozenset(relations),
frozenset(schemas), '', {})
def __init__(self, id, units, relations, schemas):
Schema.__init__(self, id, frozenset(units), frozenset(relations),
frozenset(schemas), '', {})