def extract_features_from_doc(self, doc: Document, start_token_idx,
end_token_idx):
ret = {"seq_len": end_token_idx - start_token_idx}
for name, (converter,
preprocessor) in self.we_converters_preprocessors.items():
tokens = doc.tokens[start_token_idx:end_token_idx]
tokens = [
preprocessor(token) if preprocessor is not None else token
for token in tokens
]
ret[name] = encode_sequence(tokens, converter)
for key in self.vectors_keys:
ret[key] = doc.token_features[key][start_token_idx:end_token_idx]
for name, converter in self.word_level_features.items():
ret[name] = encode_sequence(
doc.token_features[name][start_token_idx:end_token_idx],
converter)
for name, fe in self.gazetteer_feature_extractors.items():
ret[name] = fe.extract_features(doc, start_token_idx,
end_token_idx)
if 'chars' in self.char_level_features:
chars = TokenFeatureExtractor._get_chars_features(
doc, start_token_idx, end_token_idx,
self.char_level_features['chars']['padding_size'])
ret['chars'] = encode_sequence3d(
chars, self.char_level_features['chars']['converter'])
return ret
def extract_features_from_doc(self, doc: Document, start_token, end_token):
features = {}
if "ne" in self.converters:
features["ne"] = encode_sequence(
self._get_ne_types(doc, start_token, end_token),
self.converters["ne"])
return features
def extract_features(self, doc: Document, start_token_idx: int,
end_token_idx: int):
tokens = doc.token_features["lemmas"] if self._lemmatize else doc.tokens
token_slice = map(self._processor,
tokens[start_token_idx:end_token_idx])
return encode_sequence(map(self._gazetteer.__contains__, token_slice),
self._converter)
def _get_entities_encoder_features(self, doc, start_token: int,
end_token: int) -> dict:
features = {}
# currently we can have here only entitites types and depths, use common pattern
for feature, feature_converter in self.entities_encoder_features_converters.items(
):
features[feature] = encode_sequence(
doc.token_features[feature][start_token:end_token],
feature_converter)
return features
def extract_features_from_doc(self, doc: Document, include_labels=False):
samples = []
for sent_idx, sent in enumerate(doc.sentences):
sample = self.token_feature_extractor.extract_features_from_doc(doc, sent.start_token, sent.end_token)
sample.update(self.ne_feature_extractor.extract_features_from_doc(doc, sent.start_token, sent.end_token))
if include_labels:
labels = self.labelling_strategy.encode_labels(sent, doc.entities.contained_in(sent))
sample["labels"] = encode_sequence(labels, self.labels_converter)
samples.append(sample)
return samples
def extract_features_from_doc(self, doc: Document, start_token, end_token,
wrt_span):
features = {}
factories = [("token_position", _get_token_positions_to_span),
("token_log_position", _get_token_positions_to_span),
("sent_position", _get_sentence_positions_to_span),
("at_root_dt_path",
_get_tokens_on_path_from_root_to_span),
("root_dt_path_position", lambda d, f, l, s:
_get_tokens_on_path_from_root_to_span(d, f, l, s, True))]
for feature_name, feature_factory in factories:
converter = self.converters.get(feature_name, None)
if converter is not None:
features[feature_name] = encode_sequence(
feature_factory(doc, start_token, end_token, wrt_span),
converter)
return features
def encoded_labels_to_entities(self, sent: Sentence, encoded_labels: List[int]) -> List[Entity]:
return self.labelling_strategy.decode_labels(
sent, encode_sequence(encoded_labels, self.reversed_labels_converter))