def test_positive_tagging_with_bilou_unit(self):
# Given
tagging_scheme = TaggingScheme.BILOU
slot_name = "animal"
slot_size = 1
# When
tags = positive_tagging(tagging_scheme, slot_name, slot_size)
# Then
expected_tags = [UNIT_PREFIX + slot_name]
self.assertListEqual(tags, expected_tags)
def _augment_slots(self, text, tokens, tags, builtin_slots_names):
scope = set(self.slot_name_mapping[slot]
for slot in builtin_slots_names)
builtin_entities = [
be for entity_kind in scope
for be in self.builtin_entity_parser.parse(
text, scope=[entity_kind], use_cache=True)
]
# We remove builtin entities which conflicts with custom slots
# extracted by the CRF
builtin_entities = _filter_overlapping_builtins(
builtin_entities, tokens, tags, self.config.tagging_scheme)
# We resolve conflicts between builtin entities by keeping the longest
# matches. In case when two builtin entities span the same range, we
# keep both.
builtin_entities = _disambiguate_builtin_entities(builtin_entities)
# We group builtin entities based on their position
grouped_entities = (list(bes) for _, bes in groupby(
builtin_entities, key=lambda s: s[RES_MATCH_RANGE][START]))
grouped_entities = sorted(
grouped_entities,
key=lambda entities: entities[0][RES_MATCH_RANGE][START])
features = self.compute_features(tokens)
spans_ranges = [
entities[0][RES_MATCH_RANGE] for entities in grouped_entities
]
tokens_indexes = _spans_to_tokens_indexes(spans_ranges, tokens)
# We loop on all possible slots permutations and use the CRF to find
# the best one in terms of probability
slots_permutations = _get_slots_permutations(grouped_entities,
self.slot_name_mapping)
best_updated_tags = tags
best_permutation_score = -1
for slots in slots_permutations:
updated_tags = copy(tags)
for slot_index, slot in enumerate(slots):
indexes = tokens_indexes[slot_index]
sub_tags_sequence = positive_tagging(
self.config.tagging_scheme, slot, len(indexes))
updated_tags[indexes[0]:indexes[-1] + 1] = sub_tags_sequence
score = self._get_sequence_probability(features, updated_tags)
if score > best_permutation_score:
best_updated_tags = updated_tags
best_permutation_score = score
slots = tags_to_slots(text, tokens, best_updated_tags,
self.config.tagging_scheme,
self.slot_name_mapping)
return _reconciliate_builtin_slots(text, slots, builtin_entities)
def test_positive_tagging_with_bilou_unit(self):
# Given
tagging_scheme = TaggingScheme.BILOU
slot_name = "animal"
slot_size = 1
# When
tags = positive_tagging(tagging_scheme, slot_name, slot_size)
# Then
expected_tags = [UNIT_PREFIX + slot_name]
self.assertListEqual(tags, expected_tags)
def _augment_slots(self, text, tokens, tags, builtin_slots_names):
scope = set(self.slot_name_mapping[slot]
for slot in builtin_slots_names)
builtin_entities = [be for entity_kind in scope
for be in get_builtin_entities(text, self.language,
[entity_kind])]
# We remove builtin entities which conflicts with custom slots
# extracted by the CRF
builtin_entities = _filter_overlapping_builtins(
builtin_entities, tokens, tags, self.config.tagging_scheme)
# We resolve conflicts between builtin entities by keeping the longest
# matches. In case when two builtin entities span the same range, we
# keep both.
builtin_entities = _disambiguate_builtin_entities(builtin_entities)
# We group builtin entities based on their position
grouped_entities = (
list(bes)
for _, bes in groupby(builtin_entities,
key=lambda s: s[RES_MATCH_RANGE][START]))
grouped_entities = sorted(
grouped_entities,
key=lambda entities: entities[0][RES_MATCH_RANGE][START])
features = self.compute_features(tokens)
spans_ranges = [entities[0][RES_MATCH_RANGE]
for entities in grouped_entities]
tokens_indexes = _spans_to_tokens_indexes(spans_ranges, tokens)
# We loop on all possible slots permutations and use the CRF to find
# the best one in terms of probability
slots_permutations = _get_slots_permutations(
grouped_entities, self.slot_name_mapping)
best_updated_tags = tags
best_permutation_score = -1
for slots in slots_permutations:
updated_tags = copy(tags)
for slot_index, slot in enumerate(slots):
indexes = tokens_indexes[slot_index]
sub_tags_sequence = positive_tagging(
self.config.tagging_scheme, slot, len(indexes))
updated_tags[indexes[0]:indexes[-1] + 1] = sub_tags_sequence
score = self._get_sequence_probability(features, updated_tags)
if score > best_permutation_score:
best_updated_tags = updated_tags
best_permutation_score = score
slots = tags_to_slots(text, tokens, best_updated_tags,
self.config.tagging_scheme,
self.slot_name_mapping)
return _reconciliate_builtin_slots(text, slots, builtin_entities)
def test_positive_tagging_with_io(self):
# Given
tagging_scheme = TaggingScheme.IO
slot_name = "animal"
slot_size = 3
# When
tags = positive_tagging(tagging_scheme, slot_name, slot_size)
# Then
t = INSIDE_PREFIX + slot_name
expected_tags = [t, t, t]
self.assertListEqual(tags, expected_tags)
def test_positive_tagging_should_handle_zero_length(self):
# Given
slot_name = "animal"
slot_size = 0
# When
tags = []
for scheme in TaggingScheme:
tags.append(positive_tagging(scheme, slot_name, slot_size))
# Then
expected_tags = [[]] * len(TaggingScheme)
self.assertEqual(tags, expected_tags)
def test_positive_tagging_with_bilou(self):
# Given
tagging_scheme = TaggingScheme.BILOU
slot_name = "animal"
slot_size = 3
# When
tags = positive_tagging(tagging_scheme, slot_name, slot_size)
# Then
expected_tags = [BEGINNING_PREFIX + slot_name,
INSIDE_PREFIX + slot_name, LAST_PREFIX + slot_name]
self.assertListEqual(tags, expected_tags)
def test_positive_tagging_with_io(self):
# Given
tagging_scheme = TaggingScheme.IO
slot_name = "animal"
slot_size = 3
# When
tags = positive_tagging(tagging_scheme, slot_name, slot_size)
# Then
t = INSIDE_PREFIX + slot_name
expected_tags = [t, t, t]
self.assertListEqual(tags, expected_tags)
def test_positive_tagging_should_handle_zero_length(self):
# Given
slot_name = "animal"
slot_size = 0
# When
tags = []
for scheme in TaggingScheme:
tags.append(positive_tagging(scheme, slot_name, slot_size))
# Then
expected_tags = [[]] * len(TaggingScheme)
self.assertEqual(tags, expected_tags)
def test_positive_tagging_with_bilou(self):
# Given
tagging_scheme = TaggingScheme.BILOU
slot_name = "animal"
slot_size = 3
# When
tags = positive_tagging(tagging_scheme, slot_name, slot_size)
# Then
expected_tags = [
BEGINNING_PREFIX + slot_name, INSIDE_PREFIX + slot_name,
LAST_PREFIX + slot_name
]
self.assertListEqual(tags, expected_tags)
def _augment_slots(self, text, tokens, tags, builtin_slots_names):
augmented_tags = tags
scope = [self.slot_name_mapping[slot] for slot in builtin_slots_names]
builtin_entities = get_builtin_entities(text, self.language, scope)
builtin_entities = _filter_overlapping_builtins(
builtin_entities, tokens, tags, self.config.tagging_scheme)
grouped_entities = groupby(builtin_entities,
key=lambda s: s[ENTITY_KIND])
features = None
for entity, matches in grouped_entities:
spans_ranges = [match[RES_MATCH_RANGE] for match in matches]
num_possible_builtins = len(spans_ranges)
tokens_indexes = _spans_to_tokens_indexes(spans_ranges, tokens)
related_slots = list(
set(s for s in builtin_slots_names if
self.slot_name_mapping[s] == entity))
best_updated_tags = augmented_tags
best_permutation_score = -1
for slots in _generate_slots_permutations(
num_possible_builtins, related_slots,
self.config.exhaustive_permutations_threshold):
updated_tags = copy(augmented_tags)
for slot_index, slot in enumerate(slots):
if slot_index >= len(tokens_indexes):
break
indexes = tokens_indexes[slot_index]
sub_tags_sequence = positive_tagging(
self.config.tagging_scheme, slot, len(indexes))
updated_tags[indexes[0]:indexes[-1] + 1] = \
sub_tags_sequence
if features is None:
features = self.compute_features(tokens)
score = self._get_sequence_probability(features, updated_tags)
if score > best_permutation_score:
best_updated_tags = updated_tags
best_permutation_score = score
augmented_tags = best_updated_tags
slots = tags_to_slots(text, tokens, augmented_tags,
self.config.tagging_scheme,
self.slot_name_mapping)
return _reconciliate_builtin_slots(text, slots, builtin_entities)
