def pack( self, data_pack: DataPack, output_dict: Optional[Dict[str, Dict[str, List[str]]]] = None, ): """ Write the prediction results back to datapack. by writing the predicted ner to the original tokens. """ if output_dict is None: return current_entity_mention: Tuple[int, str] = (-1, "None") for i in range(len(output_dict["Token"]["tid"])): # an instance for j in range(len(output_dict["Token"]["tid"][i])): tid: int = output_dict["Token"]["tid"][i][j] # type: ignore orig_token: Token = data_pack.get_entry(tid) # type: ignore ner_tag: str = output_dict["Token"]["ner"][i][j] orig_token.ner = ner_tag token = orig_token token_ner = token.ner assert isinstance(token_ner, str) if token_ner[0] == "B": current_entity_mention = (token.span.begin, token_ner[2:]) elif token_ner[0] == "I": continue elif token_ner[0] == "O": continue elif token_ner[0] == "E": if token_ner[2:] != current_entity_mention[1]: continue entity = EntityMention(data_pack, current_entity_mention[0], token.span.end) entity.ner_type = current_entity_mention[1] elif token_ner[0] == "S": current_entity_mention = (token.span.begin, token_ner[2:]) entity = EntityMention(data_pack, current_entity_mention[0], token.span.end) entity.ner_type = current_entity_mention[1]
def pack(self, data_pack: DataPack, output_dict: Optional[Dict[str, Dict[str, List[Any]]]] = None): """ Write the prediction results back to datapack. by writing the predicted ner to the original subwords and convert predictions to something that makes sense in a word-by-word segmentation """ if output_dict is None: return for i in range(len(output_dict["Subword"]["tid"])): tids = output_dict["Subword"]["tid"][i] labels = output_dict["Subword"]["ner"][i] # Filter to labels not in `self.ft_configs.ignore_labels` entities = [ dict(idx=idx, label=label, tid=tid) for idx, (label, tid) in enumerate(zip(labels, tids)) if label not in self.ft_configs.ignore_labels ] entity_groups = self._compose_entities(entities, data_pack, tids) # Add NER tags and create EntityMention ontologies. for first_idx, last_idx in entity_groups: first_token: Subword = data_pack.get_entry(tids[first_idx]) begin = first_token.span.begin last_token: Subword = data_pack.get_entry(tids[last_idx]) end = last_token.span.end entity = EntityMention(data_pack, begin, end) entity.ner_type = self.ft_configs.ner_type
def _parse_pack(self, data: dict) -> Iterator[DataPack]: r"""Extracts information from input `data` of one document output from Prodigy Annotator including the text, tokens and its annotations into a DataPack. Args: data: a dict that contains information for one document. Returns: DataPack containing information extracted from `data`. """ pack = DataPack() text = data["text"] pack.set_text(text, replace_func=self.text_replace_operation) Document(pack, 0, len(text)) tokens = data["tokens"] spans = data["spans"] for token in tokens: begin = token["start"] end = token["end"] Token(pack, begin, end) for span_items in spans: begin = span_items["start"] end = span_items["end"] annotation_entry = EntityMention(pack, begin, end) annotation_entry.ner_type = span_items["label"] pack.pack_name = data["meta"]["id"] yield pack
def _process_entity_annotations( self, pack: DataPack, label: Optional[str], word_begin: int, word_end: int, current_entity_mention: Optional[Tuple[int, str]], ) -> Optional[Tuple[int, str]]: if label is None: return None ner_type = label.strip("()*") if "(" in label: # Entering into a span for a particular ner. current_entity_mention = (word_begin, ner_type) if ")" in label: if current_entity_mention is None: raise ValueError( "current_entity_mention is None when meet right blanket.") # Exiting a span, add and then reset the current span. entity = EntityMention(pack, current_entity_mention[0], word_end) entity.ner_type = current_entity_mention[1] current_entity_mention = None return current_entity_mention
def _parse_pack(self, data: dict) -> Iterator[DataPack]: r"""Extracts information from input `data` of one document output from Prodigy Annotator including the text, tokens and its annotations into a DataPack. Args: data: a dict that contains information for one document. Returns: DataPack containing information extracted from `data`. """ pack = DataPack() text = data['text'] pack.set_text(text, replace_func=self.text_replace_operation) Document(pack, 0, len(text)) tokens = data['tokens'] spans = data['spans'] for token in tokens: begin = token['start'] end = token['end'] Token(pack, begin, end) for span_items in spans: begin = span_items['start'] end = span_items['end'] annotation_entry = EntityMention(pack, begin, end) annotation_entry.ner_type = span_items['label'] pack.meta.doc_id = data['meta']['id'] yield pack
def _process(self, input_pack: DataPack): for sentence in input_pack.get(Sentence): token_entries = list( input_pack.get( entry_type=Token, range_annotation=sentence, components=self.token_component, )) tokens = [(token.text, token.pos) for token in token_entries] ne_tree = ne_chunk(tokens) index = 0 for chunk in ne_tree: if hasattr(chunk, "label"): # For example: # chunk: Tree('GPE', [('New', 'NNP'), ('York', 'NNP')]) begin_pos = token_entries[index].span.begin end_pos = token_entries[index + len(chunk) - 1].span.end entity = EntityMention(input_pack, begin_pos, end_pos) entity.ner_type = chunk.label() index += len(chunk) else: # For example: # chunk: ('This', 'DT') index += 1
def add_to_pack(self, pack: DataPack, instance: Annotation, prediction: List[int]): r"""Add the prediction for attribute to the instance. We make following assumptions for prediction. 1. If we encounter "I" while its tag is different from the previous tag, we will consider this "I" as a "B" and start a new tag here. 2. We will truncate the prediction it according to the number of entry. If the prediction contains `<PAD>` element, this should remove them. Args: pack (DataPack): The datapack that contains the current instance. instance (Annotation): The instance to which the extractor add prediction. prediction (Iterable[Union[int, Any]]): This is the output of the model, which contains the index for attributes of one instance. """ instance_tagging_unit: List[Annotation] = \ list(pack.get(self.config.tagging_unit, instance)) prediction = prediction[:len(instance_tagging_unit)] tags = [self.id2element(x) for x in prediction] tag_start = None tag_end = None tag_type = None for entry, tag in zip(instance_tagging_unit, tags): if tag[1] == "O" or tag[1] == "B" or \ (tag[1] == "I" and tag[0] != tag_type): if tag_type: entity_mention = EntityMention(pack, tag_start, tag_end) entity_mention.ner_type = tag_type tag_start = entry.begin tag_end = entry.end tag_type = tag[0] else: tag_end = entry.end # Handle the final tag if tag_type is not None and \ tag_start is not None and \ tag_end is not None: entity_mention = EntityMention(pack, tag_start, tag_end) entity_mention.ner_type = tag_type
def _process_ner(self, result, input_pack: DataPack): """Perform spaCy's NER Pipeline on the document. Args: result: SpaCy results input_pack: Input pack to fill Returns: """ for item in result.ents: entity = EntityMention(input_pack, item.start_char, item.end_char) entity.ner_type = item.label_
def _parse_pack(self, collection: str) -> Iterator[DataPack]: with open(collection, "r", encoding="utf8") as doc: pack_id: int = 0 pack: DataPack = DataPack() text: str = "" offset: int = 0 has_rows: bool = False sentence_begin: int = 0 sentence_cnt: int = 0 # NER tag is either "O" or in the format "X-Y", # where X is one of B, I, # Y is a tag like ORG, PER etc prev_y = None prev_x = None start_index = -1 for line in doc: line = line.strip() if line.find("DOCSTART") != -1: # Skip the first DOCSTART. if offset == 0: continue # Add remaining sentence. if has_rows: # Add the last sentence if exists. Sentence(pack, sentence_begin, offset - 1) sentence_cnt += 1 pack.set_text(text, replace_func=self.text_replace_operation) Document(pack, 0, len(text)) pack.pack_name = collection + "_%d" % pack_id pack_id += 1 yield pack # Create a new datapack. pack = DataPack() text = "" offset = 0 has_rows = False sentence_begin = 0 sentence_cnt = 0 prev_y = None prev_x = None start_index = -1 elif line != "" and not line.startswith("#"): conll_components = line.split() word = conll_components[0] pos = conll_components[1] chunk_id = conll_components[2] ner_tag = conll_components[3] # A new ner tag occurs. if ner_tag == "O" or ner_tag.split("-")[0] == "B": # Add previous ner tag to sentence if it exists. if prev_y is not None: entity_mention = EntityMention( pack, start_index, offset - 1) entity_mention.ner_type = prev_y # Start process current ner tag. if ner_tag == "O": # Current ner tag is O, reset information. prev_x = None prev_y = None start_index = -1 else: # Current ner tag is B. prev_x = "B" prev_y = ner_tag.split("-")[1] start_index = offset # This ner tag is connected to previous one. else: x, y = ner_tag.split("-") assert x == "I", "Unseen tag %s in the file." % x assert y == prev_y, "Error in %s." % ner_tag assert prev_x in ("B", "I"), "Error in %s." % ner_tag prev_x = "I" word_begin = offset word_end = offset + len(word) # Add tokens. token = Token(pack, word_begin, word_end) token.pos = pos token.chunk = chunk_id text += word + " " offset = word_end + 1 has_rows = True else: if not has_rows: # Skip consecutive empty lines. continue # Add sentence Sentence(pack, sentence_begin, offset - 1) # Handle the last ner tag if exists. if prev_x is not None: entity_mention = EntityMention(pack, start_index, offset - 1) entity_mention.ner_type = prev_y # Reset information. sentence_cnt += 1 has_rows = False prev_y = None prev_x = None sentence_begin = offset if has_rows: # Add the last sentence if exists. Sentence(pack, sentence_begin, offset - 1) sentence_cnt += 1 pack.set_text(text, replace_func=self.text_replace_operation) Document(pack, 0, len(text)) pack.pack_name = os.path.basename(collection) yield pack