def create(self, rows): instance = NLPInstance() instance.add_token().add_property('Word', '-Root-') for row in rows: instance.add_token().\ add_property(name='Word', value=row[1]).\ add_property(name='Index', value=row[0]).\ add_property(name='Lemma', value=row[2]).\ add_property(name='CPos', value=row[3]).\ add_property(name='Pos', value=row[4]).\ add_property(name='Feats', value=row[5]) for row in rows: # dependency mod = int(row[0]) try: instance.add_dependency(start=int(row[6]), end=mod, label=row[7], edge_type='dep') except (ValueError, IndexError, KeyError): print('Can\'t parse dependency', file=sys.stderr) instance.tokens[mod].add_property('DepMissing', 'missing') # role return instance
def create(self, rows): instance = NLPInstance() sentence = rows[0] # Skip <s> and dep count for i in range(2, len(sentence)): w_t_c = sentence[i].split('|') instance.add_token().\ add_property(name='Word', value=w_t_c[0]).\ add_property(name='Tag', value=w_t_c[1]).\ add_property(name='Category', value=w_t_c[2]).\ add_property(name='Index', value=str(i - 1)) # instance.add_token().add_property('Word', '-Root-') mod = 1 for row in rows: if row[0] != '<s>' and row[0] != '<\s>': # dependency try: instance.add_dependency(start=int(row[1]), end=int(row[0]), label=row[2] + '_' + row[3], edge_type='dep') except (ValueError, IndexError, KeyError): print('Can\'t parse dependency', file=sys.stderr) instance.tokens[mod].add_property('DepMissing', 'missing') mod += 1 return instance
def create(self, rows): instance = NLPInstance() instance.add_token().add_property('Word', '-Root-') predicates = [] # ArrayList<Integer>() for row in rows: instance.add_token().\ add_property(name='Word', value=row[1]).\ add_property(name='Index', value=row[0]).\ add_property(name='Lemma', value=row[2]).\ add_property(name='Pos', value=row[3]).\ add_property(name='Split Form', value=row[5]).\ add_property(name='Split Lemma', value=row[6]).\ add_property(name='Split PoS', value=row[7]) if row[10] != '_': index = int(row[0]) predicates.append(index) instance.add_span(index, index, row[10], 'sense') for row in rows: # dependency if row[8] != '_': instance.add_dependency(int(row[8]), int(row[0]), row[9], 'dep') # role for col in range(11, len(row)): label = row[col] if label != '_': pred = predicates[col - 11] arg = int(row[0]) # if arg != pred instance.add_edge(start=pred, end=arg, label=label, edge_type='role') return instance
def create(self, rows): instance = NLPInstance() for index, row in enumerate(rows): instance.add_token().\ add_property(name='Word', value=row[0]).\ add_property(name='Index', value=str(index)) predicate_count = 0 for index, row in enumerate(rows): try: if row[9] != '-': sense = row[10] + '.' + row[9] instance.add_span(index, index, sense, 'sense') self._extract_span04_05(rows, 11 + predicate_count, 'role', sense + ':', instance) predicate_count += 1 except IndexError: print('Can\'t parse file: not enough (10) column in row {0}'. format(row), file=sys.stderr) sys.exit(1) return instance
def create(self, rows): instance = NLPInstance() for index, row in enumerate(rows): instance.add_token().\ add_property(name='Word', value=row[0]).\ add_property(name='Index', value=str(index)) instance.add_span(index, index, row[1], 'ner (BIO)') self._extract_span00_02(rows=rows, column=1, field_type='ner', instance=instance) return instance
def load(self, file_name: str, _, __): """ * Loads a corpus from a file, starting at instance <code>from</code> and ending at instance <code>to</code> * (exclusive). This method is required to call * {@link com.googlecode.whatswrong.io.CorpusFormat.Monitor#progressed(int)} * after each instance that was processed. * * @param file the file to load the corpus from. * @param from the starting instance index. * @param to the end instance index. * @return a list of NLP instances loaded from the given file in the given interval. * @throws java.io.IOException if I/O goes wrong. """ result = [] with open(file_name, encoding='UTF-8') as reader: instance = None source_length = -1 target_length = -1 for line in reader: line = line.strip() if line.startswith('<source>'): content = line.strip()[8: len(line) - 9] for token in content.split(): instance.add_token().add_property('word', token) source_length = len(instance.tokens) instance.split_point = source_length elif line.startswith('<seg'): instance = NLPInstance(render_type=RenderType.alignment) elif line.startswith('<translation>'): content = line.strip()[13: len(line) - 14] for token in content.split(): instance.add_token().add_property('word', token) target_length = len(instance.tokens) - source_length elif line.startswith('<matrix>'): check_eof(reader.readline()) for tgt in range(target_length): line = check_eof(reader.readline()).strip() col = line.split() for src in range(1, len(col)): if col[src] == '1': instance.add_edge(src - 1, tgt + source_length, 'align', 'align') result.append(instance) return result
def create_open(rows): instance = NLPInstance() instance.add_token() for row in rows: instance.add_token(). \ add_property('Named Entity', row[0], 10). \ add_property('NamedEntity BBN', row[1], 11). \ add_property('WordNet', row[2], 12) index = 1 for index, row in enumerate(rows, start=1): # dependency instance.add_edge(start=int(row[3]), end=index, label=row[4], edge_type='malt') return index
def create(self, rows): instance = NLPInstance() for index, row in enumerate(rows): instance.add_token().\ add_property(name='Word', value=row[0]).\ add_property(name='Index', value=str(index)) predicate_count = 0 for index, row in enumerate(rows): if row[1] != '-': sense = row[1] instance.add_span(index, index, sense, 'sense') self._extract_span04_05(rows, 2 + predicate_count, 'role', sense + ':', instance) predicate_count += 1 return instance
def create(self, rows): instance = NLPInstance() instance.add_token().add_property(name='Word', value='-Root-') predicates = [] for row in rows: instance.add_token().\ add_property(name='Word', value=row[1]).\ add_property(name='Index', value=row[0]).\ add_property(name='Lemma', value=row[2]).\ add_property(name='PLemma', value=row[3]).\ add_property(name='PoS', value=row[4]).\ add_property(name='PPoS', value=row[5]).\ add_property(name='Feat', value=row[6]).\ add_property(name='PFeat', value=row[7]) if row[13] != '_': index = int(row[0]) predicates.append(index) instance.add_span(index, index, row[13], 'sense') for row in rows: # dependency if row[8] != '_': instance.add_dependency(start=int(row[8]), end=int(row[0]), label=row[10], edge_type='dep') if row[9] != '_': instance.add_dependency(start=int(row[9]), end=int(row[0]), label=row[11], edge_type='pdep') # role for col in range(14, len(row)): label = row[col] if label != '_': pred = predicates[col - 14] arg = int(row[0]) # if arg != pred: instance.add_dependency(start=pred, end=arg, label=label, edge_type='role') return instance
def create(self, rows): instance = NLPInstance() instance.add_token().add_property(name='Word', value='-Root-') for index, row in enumerate(rows, start=1): instance.add_token().\ add_property(name='Word', value=row[0]).\ add_property(name='Index', value=str(index)).\ add_property(name='Pos', value=row[1]) for mod, row in enumerate(rows, start=1): # dependency try: instance.add_dependency(start=int(row[2]), end=mod, label=row[3], edge_type='dep') except (ValueError, IndexError, KeyError): print('Can\'t parse dependency', file=sys.stderr) instance.tokens[mod].add_property('DepMissing', 'missing') # role return instance
def update_canvas(self, curr_sent_index: int): """ Updates the canvas based on the current state of the navigator.""" if self._selected_gold is not None: if self._selected_guess is not None: instance = nlp_diff( self._gold_corpora[self._selected_gold][curr_sent_index], self._guess_corpora[self._selected_guess][curr_sent_index], 'eval_status_Match', 'eval_status_FN', 'eval_status_FP') else: instance = self._gold_corpora[ self._selected_gold][curr_sent_index] self.canvas.set_nlp_instance(instance) else: example = NLPInstance() example.add_token().add_property('Word', '[root]').add_property( 'Index', '0') example.add_token().add_property('Word', 'Add').add_property('Index', '1') example.add_token().add_property('Word', 'a').add_property('Index', '2') example.add_token().add_property('Word', 'gold').add_property( 'Index', '3') example.add_token().add_property('Word', 'corpus').add_property( 'Index', '4') example.add_token().add_property('Word', '!').add_property('Index', '5') example.add_dependency(0, 1, 'ROOT', 'dep') example.add_dependency(0, 5, 'PUNC', 'dep') example.add_dependency(1, 4, 'OBJ', 'dep') example.add_dependency(4, 2, 'DET', 'dep') example.add_dependency(4, 3, 'MOD', 'dep') example.add_dependency(1, 4, 'A1', 'role') self.canvas.set_nlp_instance(example) self.canvas.filter.allowed_edge_types = set() self.canvas.filter.allowed_edge_types.add('dep') self.canvas.filter.allowed_edge_types.add('role') self.canvas.filter.allowed_edge_types.add('sense') self.canvas.filter.allowed_edge_types.add('ner') self.canvas.filter.allowed_edge_types.add('chunk') self.canvas.filter.allowed_edge_types.add('pos') self.canvas.filter.allowed_edge_types.add('align') self.canvas.filter.allowed_edge_properties.add('eval_status_FP') self.canvas.filter.allowed_edge_properties.add('eval_status_FN') self.canvas.filter.allowed_edge_properties.add('eval_status_Match') self.canvas.renderer.params['span.orders'] = { 'pos': 0, 'chunk (BIO)': 1, 'chunk': 2, 'ner (BIO)': 2, 'ner': 3, 'sense': 4, 'role': 5, 'phase': 5 } self.canvas.fire_instance_changed()
def load(self, file_name: str, _, __): """ * Loads files from the given directory with the extensions specified by the text fields of the accessory. * * @param file the directory load the corpus from. * @param from the starting instance index. * @param to the end instance index. * @return a list of NLP instances loaded from the given file in the given interval. * @throws java.io.IOException if I/O goes wrong. """ result = [] for txt_file_name in glob.glob(os.path.join(file_name, '*.' + self.txtExtensionField.strip())): filename = os.path.abspath(txt_file_name) prefix = filename.rsplit('.', maxsplit=1)[0] protein_file_name = '{0}.{1}'.format(prefix, self.proteinExtensionField.strip()) event_file_name = '{0}.{1}'.format(prefix, self.eventExtensionField.strip()) if os.path.exists(protein_file_name) and os.path.exists(event_file_name): """ * Loads all NLPInstances in the specified files. Creates one instance. * * @param txt_file_name the text file * @param protein_file_name the file with protein annotations * @param event_file_name the file with event annotations * @return NLPInstance that represents the given text and annotations * @throws IOException if IO goes wrong. """ char_to_token = {} instance = NLPInstance() with open(txt_file_name, encoding='UTF-8') as reader: current_token = instance.add_token() current_token_content = '' for current_index, character in enumerate(iter(functools.partial(reader.read, 1), '')): char_to_token[current_index] = current_token if character == ' ' or character == '\n': if len(current_token_content) > 0: current_token.add_property('Word', current_token_content) current_token.add_property('Index', str(len(instance.tokens) - 1)) current_token_content = '' current_token = instance.add_token() else: current_token_content += character id2token = {} with open(protein_file_name, encoding='UTF-8') as reader: for line in reader.readlines(): split = line.strip().split() if split[0].startswith('T'): elem_id = split[0] elem_type = split[1] elem_from = int(split[2]) elem_to = int(split[3]) from_token = char_to_token[elem_from] to_token = char_to_token[elem_to] instance.add_edge(from_token.index, to_token.index, elem_type, 'protein', EdgeRenderType.span) id2token[elem_id] = to_token with open(event_file_name, encoding='UTF-8') as reader: # get event mentions and locations etc. for line in reader.readlines(): split = line.strip().split() elem_id = split[0] if elem_id.startswith('T'): elem_type = split[1] elem_from = int(split[2]) elem_to = int(split[3]) from_token = char_to_token[elem_from] to_token = char_to_token[elem_to] if elem_type == 'Entity': term_class = 'entity' else: term_class = 'event' instance.add_edge(from_token.index, to_token.index, elem_type, term_class, EdgeRenderType.span) id2token[elem_id] = to_token elif elem_id.startswith('E'): type_and_mention_id = split[1].split(':') even_token = id2token[type_and_mention_id[1]] id2token[elem_id] = even_token with open(event_file_name, encoding='UTF-8') as reader: # now create the event roles for line in reader.readlines(): split = line.split() elem_id = split[0] if elem_id.startswith('E'): even_token = id2token[elem_id] for elem in split[2:]: role_and_id = elem.split(':') arg_token = id2token.get(role_and_id[1]) if arg_token is None: raise RuntimeError( 'There seems to be no mention associated with id {0} for event {1} in' ' file {2}'.format(role_and_id[1], elem_id, event_file_name)) instance.add_edge(even_token.index, arg_token.index, role_and_id[0], 'role', EdgeRenderType.dependency, note=elem_id) result.append(instance) return result
def write_tokens(self, word_type: str, tag_type: str, instance: NLPInstance): instance.add_token().add_property(word_type, self.label).add_property('Index', str(self.index))