def parse(self, text): # Lazy load model file to speed up startup if not self.model: self.model = self.load_model(self.language) text = text.strip() # Adding a period improves detection on especially short sentences period_added = False last_character = text.strip()[-1] if re.match(r"\w", last_character, flags=re.UNICODE): text += "." period_added = True pipeline = Pipeline(self.model, "tokenize", Pipeline.DEFAULT, Pipeline.DEFAULT, "conllu") error = ProcessingError() processed = pipeline.process(text, error) if error.occurred(): raise ParserException(error.message) # Remove the period to make sure input corresponds to output if period_added: processed = "\n".join(processed.rstrip().split("\n")[:-1]) + "\n\n" return processed
def parse(self, text): # Lazy load model file to speed up startup if not self.model: self.model = self.load_model() text = text.strip() # Adding a period improves detection on especially short sentences period_added = False last_character = text.strip()[-1] if re.match(r"\w", last_character, flags=re.UNICODE): text += "." period_added = True pipeline = Pipeline( self.model, "tokenize", Pipeline.DEFAULT, Pipeline.DEFAULT, "conllu" ) error = ProcessingError() processed = pipeline.process(text, error) if error.occurred(): raise ParserException(error.message) # Remove the period to make sure input corresponds to output if period_added: processed = "\n".join(processed.rstrip().split("\n")[:-1]) + "\n\n" return processed
class UdPipe: def __init__(self): self.model = None self.tokenizer = None self.error = None def load(self, model_path: str): logger.info("Loading UdPipe model ...") self.model = Model.load(model_path) if not self.model: raise Exception("Cannot load model from file '%s'." % model_path) self.tokenizer = self.model.newTokenizer(self.model.DEFAULT) if not self.tokenizer: raise Exception("The model does not have a tokenizer") self.error = ProcessingError() def transform(self, string: str): self.tokenizer.setText(string) sentences = [] sentence = Sentence() while self.tokenizer.nextSentence(sentence, self.error): self.model.tag(sentence, self.model.DEFAULT) sentences.append(sentence) sentence = Sentence() if self.error.occurred(): raise Exception(self.error.message) words = functools.reduce(lambda x, y: x + y, [[(w.lemma, w.upostag) for w in s.words] for s in sentences], []) return [word for word in words if word[1] not in ['<root>']]
def udpipeS(pathmodel, sourcepath, pathdestination): model = Model.load(pathmodel) if not model: sys.stderr.write("Cannot load model from file '%s'\n" % pathmodel) sys.exit(1) sys.stderr.write('done\n') pipeline = Pipeline(model, "horizontal", Pipeline.DEFAULT, Pipeline.DEFAULT, "conllu") error = ProcessingError() i = 1 for filename in os.listdir(sourcepath): f = open(pathdestination + filename[:-3] + "conllu", "a") f.truncate(0) text = io.open(sourcepath + filename, "r", encoding="utf-8") string = "".join(text.readlines()) # Process data processed = pipeline.process(string, error) if error.occurred(): sys.stderr.write("An error occurred when running run_udpipe: ") sys.stderr.write(error.message) sys.stderr.write("\n") sys.exit(1) f.write(processed) f.close() print("File n ", i, " processed of ", len(os.listdir(sourcepath))) i += 1
def udpipe(sentences, model_name, verbose=False): """ Parse text to Universal Dependencies using UDPipe. :param sentences: iterable of iterables of strings (one string per line) :param model_name: filename containing UDPipe model to load :param verbose: print extra information :return: iterable of lines containing parsed output """ from ufal.udpipe import Model, Pipeline, ProcessingError model = Model.load(model_name) if not model: raise ValueError("Invalid model: '%s'" % model_name) pipeline = Pipeline(model, "conllu", Pipeline.DEFAULT, Pipeline.DEFAULT, "conllu") lines1, lines2 = tee(l for s in sentences for l in s) text = "\n".join(lines1) error = ProcessingError() num_tokens = sum(1 for l in lines2 if l) with ioutil.external_write_mode(): print("Running %s on %d tokens... " % (model_name, num_tokens), end="", flush=True) start = time() processed = pipeline.process(text, error) duration = time() - start with ioutil.external_write_mode(): print("Done (%.3fs, %.0f tokens/s)" % (duration, num_tokens / duration if duration else 0)) if verbose: print(processed) if error.occurred(): raise RuntimeError(error.message) return processed.splitlines()
def udpipeG(pathmodel): path = "/home/guido/Progetto Unitexto/textdata/cleanedTxt/" model = Model.load(pathmodel) if not model: sys.stderr.write("Cannot load model from file '%s'\n" % pathmodel) sys.exit(1) sys.stderr.write('done\n') pipeline = Pipeline(model, "horizontal", Pipeline.DEFAULT, Pipeline.DEFAULT, "conllu") error = ProcessingError() # corp = io.open("/home/guido/Progetto Unitexto/textdata/corpus.txt","r",encoding= "utf-8") # Read whole input # string="".join(corp.readlines()) # Process data # processed = pipeline.process(string, error) f = open("/home/guido/Progetto Unitexto/textdata/corpus.conllu", "a") f.truncate(0) i = 1 for filename in os.listdir(path): text = io.open(path + filename, "r", encoding="utf-8") string = "".join(text.readlines()) # Process data processed = pipeline.process(string, error) if error.occurred(): sys.stderr.write("An error occurred when running run_udpipe: ") sys.stderr.write(error.message) sys.stderr.write("\n") sys.exit(1) f.write(processed) print("File n ", i, " processed of ", len(os.listdir(path))) i += 1
def get_sintax(self, text): error = ProcessingError() processed = self.pipeline.process(text, error) if error.occurred(): print("An error occurred when running run_udpipe: ") print(error.message) print("\n") return processed
def extract_sentences(input_file: str, output_file: str, logger) -> None: logger.info( "==== Now performing sentence extraction from paragraphs file ====") # UDPipe initliazation lang_model = 'lang_models/czech-ud-2.0-170801.udpipe' model = Model.load(lang_model) if not model: logger.error('Could not load UDPipe language model: ' + lang_model) ud_pipeline = Pipeline(model, 'tokenize', Pipeline.DEFAULT, Pipeline.DEFAULT, '') ud_error = ProcessingError() sentences_file = open(output_file, "w") # reopen paragraphs for reading paragraphs_file = open(input_file, "r") sentences_count = 0 for p_line in paragraphs_file: page_first_sentence = "" page_first_paragraph = p_line.split( '\t', 1) # use the variable as temporary list # If there is a paragraph content if len(page_first_paragraph) == 2: page_uri = page_first_paragraph[0] page_first_paragraph = page_first_paragraph[1] # Extract first sentence form paragraph using UDPipe: ud_output = ud_pipeline.process(page_first_paragraph, ud_error) if ud_error.occurred(): logger.error( 'Error occured while extracting sentence using UDPipe: ' + ud_error.message) page_first_sentence = "" else: ud_output = ud_output.split('\n') if len(ud_output) >= 4: page_first_sentence = ud_output[3][ 9:] # assumption about the output format else: page_first_sentence = "" # Write sentence to the file sentences_file.write(page_uri + '\t' + page_first_sentence + '\n') sentences_count += 1 if sentences_count % 2000 == 0: logger.info("Extracted {} sentences.".format(sentences_count)) logger.info("Finished extraction of {} sentences.".format(sentences_count)) paragraphs_file.close() sentences_file.close()
def process_sentence(self, sen, field_names=None): pipeline = Pipeline(self._model, self._inp_format, self._pos_settings, self._parse_settings, 'conllu') error = ProcessingError() # For catching errors... inp_sen = ''.join(self._encode_sentence(sen, field_names)) # Do the processing... + Write the output in CoNLL-U processed = pipeline.process(inp_sen, error) if error.occurred(): raise UDPipeError(error.message) ret_sen = self._decode_sentence(processed, sen, field_names) return ret_sen
def process_text(path_to_file, input_format, model, out_file): """ Apply NLP processing to text (tokenize, tag, parse) and save output in CONLLU format. """ pipeline = Pipeline(model, input_format, Pipeline.DEFAULT, Pipeline.DEFAULT, 'conllu') error = ProcessingError() with codecs.open(path_to_file) as f: text = f.read() processed = pipeline.process(text, error) if error.occurred(): print("Error when running UDPipe: ") print(error.message) print("\n") sys.exit(1) with codecs.open(out_file, 'w', 'utf-8') as of: of.write(processed)
class PredPattArgumentExtractor(ArgumentExtractor): def __init__( self, path_to_udpipe: str, resolve_relcl: bool = True, resolve_appos: bool = True, resolve_amod: bool = True, resolve_conj: bool = True, resolve_poss: bool = True, ud=dep_v2.VERSION, ): super().__init__() self.model = Model.load(path_to_udpipe) self.pipeline = Pipeline( self.model, "tokenize", Pipeline.DEFAULT, Pipeline.DEFAULT, "conllu" ) self._error = ProcessingError() self._opts = PredPattOpts( resolve_relcl=resolve_relcl, resolve_appos=resolve_appos, resolve_amod=resolve_amod, resolve_conj=resolve_conj, resolve_poss=resolve_poss, ud=ud, ) @lru_cache(maxsize=100000) def extract(self, sentence: str) -> List[Dict[str, Any]]: processed = self.pipeline.process(sentence, self._error) if self._error.occurred(): print(f"=== Error occurred: {self._error.message}") self._error = ProcessingError() return None else: conll_example = [ud_parse for sent_id, ud_parse in load_conllu(processed)][ 0 ] ppatt = PredPatt(conll_example, opts=self._opts) result = [] for predicate in ppatt.instances: structure = { "predicate": predicate.tokens, "arguments": [x.tokens for x in predicate.arguments], } result.append(structure) return result
def parse_udpipe(passages, model_name, verbose=False): from ufal.udpipe import Model, Pipeline, ProcessingError model = Model.load(model_name) if not model: raise ValueError("Invalid model: '%s'" % model_name) pipeline = Pipeline(model, "conllu", Pipeline.DEFAULT, Pipeline.DEFAULT, "conllu") passages1, passages2 = tee(passages) text = "\n".join(l for p in passages1 for l in to_conllu(p, tree=True)) error = ProcessingError() print("Running UDPipe on %d characters... " % len(text), end="", flush=True) start = time() processed = pipeline.process(text, error) print("Done (%.3fs)" % (time() - start)) if verbose: print(processed) if error.occurred(): raise RuntimeError(error.message) return zip(passages2, from_conllu(processed.splitlines(), passage_id=None))
def _read(self, text: str, input_format: str) -> List[Sentence]: """Convert the text to an UDPipe representation. text: Input text. input_format: Desired input format. RETURNS: Processed sentences. """ input_format.setText(text) error = ProcessingError() sentences = [] sentence = Sentence() while input_format.nextSentence(sentence, error): sentences.append(sentence) sentence = Sentence() if error.occurred(): raise Exception(error.message) return sentences
def _read(self, text, input_format): """Convert the text to a UDPipe representation. text (unicode): Input text. input_format (unicode): Desired input format. RETURNS (list): Processed ufal.udpipe.Sentence-s. """ input_format.setText(text) error = ProcessingError() sentences = [] sentence = Sentence() while input_format.nextSentence(sentence, error): sentences.append(sentence) sentence = Sentence() if error.occurred(): raise Exception(error.message) return sentences
def udpipe(conllu_in, model_path): model = Model.load(model_path) if not model: sys.stderr.write("Cannot load model from file '%s'\n" % sys.argv[3]) sys.exit(1) pipeline = Pipeline(model, "conllu", Pipeline.DEFAULT, Pipeline.DEFAULT, "conllu") error = ProcessingError() # Process data processed = pipeline.process(conllu_in, error) if error.occurred(): sys.stderr.write("An error occurred when running run_udpipe: ") sys.stderr.write(error.message) sys.stderr.write("\n") sys.exit(1) return processed
def main(args): model = Model.load(args.model) if not model: raise ValueError("Invalid model: '%s'" % args.model) os.makedirs(args.out_dir, exist_ok=True) pipeline = Pipeline(model, "tokenize" if args.txt else "conllu", Pipeline.DEFAULT, Pipeline.DEFAULT, "conllu") for pattern in args.filenames: for in_file in glob(pattern) or [pattern]: basename = os.path.basename(in_file) out_file = os.path.join(args.out_dir, os.path.splitext(basename)[0] + ".conllu") error = ProcessingError() with open(in_file, encoding="utf-8") as f: processed = pipeline.process(f.read(), error) if error.occurred(): raise RuntimeError(error.message) with open(out_file, "w", encoding="utf-8") as f: f.write(processed) if not args.quiet: print("Wrote '%s'" % out_file)
def get_tokens(input, lang="en"): from io import StringIO from ufal.udpipe import ProcessingError error = ProcessingError() pipeline = Understanding._get_udpipe_pipeline(lang) processed = pipeline.process(input, error) if error.occurred(): raise Exception(error.message) sio = StringIO(processed) # pandas frame is a bit overkill I guess # import pandas as pd # first three rows has comments # df = pd.read_csv(sio, sep="\t", skiprows=3) # return df import csv rd = csv.reader(sio, delimiter="\t") nodes = [] for row in rd: if len(row) < 8: continue nodes.append(Token(row)) return nodes
class TextUtils(object): def __init__(self): self.clause_splitter = rutokenizer.Segmenter() self.tokenizer = Tokenizer() self.tokenizer.load() #self.lexicon = Word2Lemmas() self.language_resources = LanguageResources() self.postagger = rupostagger.RuPosTagger() self.chunker = ruchunker.Chunker() self.word2tags = ruword2tags.RuWord2Tags() self.flexer = ruword2tags.RuFlexer() self.syntan = None self.gg_dictionaries = GenerativeGrammarDictionaries() #self.known_words = set() #self.lemmatizer = Mystem() self.lemmatizer = rulemma.Lemmatizer() self.word_embeddings = None def load_embeddings(self, w2v_dir, wc2v_dir): # Загрузка векторных словарей self.word_embeddings = WordEmbeddings() self.word_embeddings.load_models(w2v_dir, wc2v_dir) if wc2v_dir: p = os.path.join(wc2v_dir, 'wc2v.kv') self.word_embeddings.load_wc2v_model(p) p = os.path.join(w2v_dir, 'w2v.kv') self.word_embeddings.load_w2v_model(p) def load_dictionaries(self, data_folder, models_folder): self.lemmatizer.load() # Общий словарь для генеративных грамматик #self.gg_dictionaries.load(os.path.join(models_folder, 'generative_grammar_dictionaries.bin')) #word2lemmas_path = os.path.join(data_folder, 'ru_word2lemma.tsv.gz') #self.lexicon.load(word2lemmas_path) #word2tags_path = os.path.join(data_folder, 'chatbot_word2tags.dat') #self.postagger.load(word2tags_path) self.postagger.load() self.word2tags.load() self.flexer.load() self.chunker.load() # Грузим dependency parser UDPipe и русскоязычную модель model_file = os.path.join(models_folder, 'udpipe_syntagrus.model') self.udpipe_model = Model.load(model_file) self.udpipe_pipeline = Pipeline(self.udpipe_model, 'tokenize', Pipeline.DEFAULT, Pipeline.DEFAULT, 'conllu') self.udpipe_error = ProcessingError() #self.syntan = rusyntax2.Tagger(self.word2tags, w2v, self.postagger) #self.syntan.load() #rules_path = os.path.join(data_folder, 'rules.yaml') #with io.open(rules_path, 'r', encoding='utf-8') as f: #data = yaml.safe_load(f) #self.no_info_replicas = data['no_relevant_information'] #self.unknown_order = data['unknown_order'] #self.language_resources.key2phrase[u'yes'] = data[u'answers'][u'yes'] #self.language_resources.key2phrase[u'not'] = data[u'answers'][u'not'] # Список "хороших слов" для генеративной грамматики #with io.open(os.path.join(models_folder, 'dataset_words.txt'), 'r', encoding='utf-8') as rdr: # for line in rdr: # word = line.strip() # self.known_words.add(word) def apply_word_function(self, func, constants, words): part_of_speech = None tag = None if func == '$chooseAdjByGender': part_of_speech = 'ПРИЛАГАТЕЛЬНОЕ' tag = ('РОД', constants['gender']) elif func == '$chooseVByGender': part_of_speech = 'ГЛАГОЛ' tag = ('РОД', constants['gender']) elif func == '$chooseNByGender': part_of_speech = 'СУЩЕСТВИТЕЛЬНОЕ' tag = ('РОД', constants['gender']) else: raise NotImplementedError() tag2 = tag[0] + '=' + tag[1] for word in words: #tagsets = self.gg_dictionaries.grdict.get_word_tagsets2(word.lower(), part_of_speech) for tagset in self.word2tags[word.lower()]: if part_of_speech in tagset and tag2 in tagset: return word msg = 'Could not choose a word among {}'.format(' '.join(words)) raise RuntimeError(msg) def tag(self, words, with_lemmas=False): """ Частеречная разметка для цепочки слов words """ if with_lemmas: return self.lemmatizer.lemmatize(self.postagger.tag(words)) else: return self.postagger.tag(words) def canonize_text(self, s): """ Удаляем два и более пробелов подряд, заменяя на один """ s = re.sub("(\\s{2,})", ' ', s.strip()) return s def remove_terminators(self, s): """ Убираем финальные пунктуаторы ! ? .""" return s[:-1].strip() if s[-1] in u'?!.' else s def wordize_text(self, s): return u' '.join(self.tokenize(s)) def ngrams(self, s, n): #return [u''.join(z) for z in itertools.izip(*[s[i:] for i in range(n)])] return [u''.join(z) for z in zip(*[s[i:] for i in range(n)])] def words2str(self, words): return u' '.join( itertools.chain([BEG_WORD], filter(lambda z: len(z) > 0, words), [END_WORD])) def split_clauses(self, s): return list(self.clause_splitter.split(s)) def tokenize(self, s): return self.tokenizer.tokenize(s) def extract_lemma(self, token): return token[0] if token[1] == 'PRON' else token[2] def lemmatize(self, s): words = self.tokenizer.tokenize(s) #wx = u' '.join(words) #return [l for l in self.lemmatizer.lemmatize(wx) if len(l.strip()) > 0] tokens = self.lemmatizer.lemmatize(self.postagger.tag(words)) return [self.extract_lemma(t) for t in tokens] def lemmatize2(self, s): words = self.tokenizer.tokenize(s) return self.lemmatizer.lemmatize(self.postagger.tag(words)) def lpad_wordseq(self, words, n): """ Слева добавляем пустые слова """ return list( itertools.chain(itertools.repeat(PAD_WORD, n - len(words)), words)) def rpad_wordseq(self, words, n): """ Справа добавляем пустые слова """ return list( itertools.chain(words, itertools.repeat(PAD_WORD, n - len(words)))) #def get_lexicon(self): # return self.lexicon def is_question_word(self, word): return word in u'насколько где кто что почему откуда куда зачем чего кого кем чем кому чему ком чем как сколько ли когда докуда какой какая какое какие какого какую каких каким какими какому какой каков какова каковы'.split( ) def build_output_phrase(self, words): s = u' '.join(words) s = s.replace(u' ?', u'?').replace(u' !', u'!').replace(u' ,', u',').replace(u' :', u',') \ .replace(u' .', u'.').replace(u'( ', u'(').replace(u' )', u')') s = s[0].upper() + s[1:] return s def detect_person0(self, words): if any((word in (u'ты', u'тебя', u'тебе')) for word in words): return 2 if any((word in (u'я', u'мне', u'меня')) for word in words): return 1 return -1 def extract_chunks(self, sample): tokens = self.tokenizer.tokenize(sample) tagsets = list(self.postagger.tag(tokens)) lemmas = self.lemmatizer.lemmatize(tagsets) #edges = syntan.parse(tokens, tagsets) phrase_tokens = [] for word_index, (token, tagset, lemma) in enumerate(zip(tokens, tagsets, lemmas)): t = PhraseToken() t.word = token t.norm_word = token.lower() t.lemma = lemma[2] t.tagset = tagset[1] t.word_index = word_index phrase_tokens.append(t) chunks = self.chunker.parse(tokens) for chunk_index, chunk in enumerate(chunks): phrase_tokens[chunk.tokens[0].index].is_chunk_starter = True for token in chunk.tokens: phrase_tokens[token.index].chunk_index = chunk_index return chunks def word_similarity(self, word1, word2): return self.word_embeddings.word_similarity(word1, word2) def parse_syntax(self, text_str): processed = self.udpipe_pipeline.process(text_str, self.udpipe_error) if self.udpipe_error.occurred(): logging.error("An error occurred when running run_udpipe: %s", self.udpipe_error.message) return None parsed_data = pyconll.load_from_string(processed)[0] return parsed_data def get_udpipe_attr(self, token, tag_name): if tag_name in token.feats: v = list(token.feats[tag_name])[0] return v return '' def change_verb_gender(self, verb_inf, new_gender): """ Изменение формы глагола в прошедшем времени единственном числе """ required_tags = [('ВРЕМЯ', 'ПРОШЕДШЕЕ'), ('ЧИСЛО', 'ЕД')] if new_gender == 'Fem': required_tags.append(('РОД', 'ЖЕН')) else: required_tags.append(('РОД', 'МУЖ')) forms = list(self.flexer.find_forms_by_tags(verb_inf, required_tags)) if forms: return forms[0] else: return None def change_adj_gender(self, adj_lemma, new_gender, variant): if adj_lemma == 'должен': if new_gender == 'Fem': return 'должна' else: return 'должен' required_tags = [('ЧИСЛО', 'ЕД')] if variant == 'Short': required_tags.append(('КРАТКИЙ', '1')) else: required_tags.append(('КРАТКИЙ', '0')) required_tags.append(('ПАДЕЖ', 'ИМ')) if new_gender == 'Fem': required_tags.append(('РОД', 'ЖЕН')) else: required_tags.append(('РОД', 'МУЖ')) forms = list(self.flexer.find_forms_by_tags(adj_lemma, required_tags)) if forms: return forms[0] else: return None def is_premise_suitable_as_answer(self, premise_text): # Можно ли текст предпосылки использовать в качестве ответа tx = self.tokenize(premise_text) if len(tx) > 5: return False if ',' in tx or 'и' in tx or 'или' in tx: return False return True
class UDpipeparser(): def __init__(self, modeldir): self.modeldir = os.path.abspath(modeldir) self.model = Model.load(self.modeldir) self.error = ProcessingError() if not self.model: sys.stderr.write("Udpipe language model loading failed:" + self.modeldir) sys.exit(1) self.pipeline = Pipeline(self.model, "tokenize", Pipeline.DEFAULT, Pipeline.DEFAULT, "conllu") def udpipe_parse_sent(self, text): logger = logging.getLogger('petr_log') processed = self.pipeline.process(text, self.error) if self.error.occurred(): raise ValueError(self.error.message) #print(processed) parsed = [] for line in processed.split("\n"): if line.startswith("#"): continue if "VERB\t" not in line: line = unidecode.unidecode(line) parsed.append(line) #print(("\n").join(parsed)) #input(" ") return ("\n").join(parsed) def udpipe_parse_events(self, events): logger = logging.getLogger('petr_log') #modeldir = os.path.abspath(PETRglobals.udpipe_modsel_dir) #self.model = Model.load(modeldir) #self.pipeline = Pipeline(model,"tokenize",Pipeline.DEFAULT,Pipeline.DEFAULT,"conllu") total = len(list(event_dict.keys())) logger.info("Starting parse of {} stories.'.format(total)") for i, key in enumerate(event_dict.keys()): if (i / float(total)) * 100 in [10.0, 25.0, 50, 75.0]: print('Parse is {}% complete...'.format( (i / float(total)) * 100)) for sent in event_dict[key]['sents']: logger.info('Udpipe parsing {}_{}...'.format(key, sent)) sent_dict = event_dict[key]['sents'][sent] processed = self.pipeline.process(sent_dict['content'], self.error) if self.error.occurred(): #sys.stderr.write("An error occurred when running run_udpipe: ") #sys.stderr.write(error.message) #sys.stderr.write("\n") raise ValueError(self.error.message) parsed = [] for line in processed.split("\n"): if line.startswith("#"): continue parsed.append(line) sent_dict['parsed'] = utilities._format_ud_parsed_str( ("\n").join(parsed)) print(sent_dict['parsed']) input("") #print('Done with UDpipe parse...\n\n') logger.info('Done with UDpipe parse.') return event_dict
class UDPipe: """Wrapper for UDPipe (more pythonic than ufal.udpipe).""" def __init__(self, model): """Create the UDPipe tool object.""" self.model = model path = require_file(model) self.tool = Model.load(path) if not self.tool: raise IOError("Cannot load model from file '%s'" % path) self.error = ProcessingError() self.conllu_reader = ConlluReader() self.tokenizer = self.tool.newTokenizer(Model.DEFAULT) def tag_parse_tree(self, root): """Tag (+lemmatize, fill FEATS) and parse a tree (already tokenized).""" pipeline = Pipeline(self.tool, 'horizontal', Pipeline.DEFAULT, Pipeline.DEFAULT, 'conllu') in_data = " ".join([n.form for n in root.descendants]) out_data = pipeline.process(in_data, self.error) if self.error.occurred(): raise IOError("UDPipe error " + self.error.message) self.conllu_reader.files.filehandle = io.StringIO(out_data) parsed_root = self.conllu_reader.read_tree() nodes = [root] + root.descendants for parsed_node in parsed_root.descendants: node = nodes[parsed_node.ord] node.parent = nodes[parsed_node.parent.ord] for attr in 'upos xpos lemma feats'.split(): setattr(node, attr, getattr(parsed_node, attr)) # TODO: benchmark which solution is the fastest one. E.g. we could also do # for node, parsed_node in zip(root.descendants, parsed_root.descendants): # parsed_node.misc = node.misc # pylint: disable=protected-access #root._children, root._descendants = parsed_root._children, parsed_root._descendants def tokenize_tag_parse_tree(self, root): """Tokenize, tag (+lemmatize, fill FEATS) and parse the text stored in `root.text`.""" if root.children: raise ValueError('Tree already contained nodes before tokenization') # tokenization (I cannot turn off segmenter, so I need to join the segments) self.tokenizer.setText(root.text) u_sentence = Sentence() is_another = self.tokenizer.nextSentence(u_sentence) u_words = u_sentence.words n_words = u_words.size() - 1 if is_another: u_sent_cont = Sentence() while self.tokenizer.nextSentence(u_sent_cont): n_cont = u_sent_cont.words.size() - 1 for i in range(1, n_cont + 1): u_w = u_sent_cont.words[i] n_words += 1 u_w.id = n_words u_words.append(u_w) # tagging and parsing self.tool.tag(u_sentence, Model.DEFAULT) self.tool.parse(u_sentence, Model.DEFAULT) # converting UDPipe nodes to Udapi nodes heads, nodes = [], [root] for i in range(1, u_words.size()): u_w = u_words[i] node = root.create_child( form=u_w.form, lemma=u_w.lemma, upos=u_w.upostag, xpos=u_w.xpostag, feats=u_w.feats, deprel=u_w.deprel, ) node.misc = u_w.misc heads.append(u_w.head) nodes.append(node) for node in nodes[1:]: head = heads.pop(0) node.parent = nodes[head]
class Omorfi: """ An object holding omorfi binariesfor all the functions of omorfi. The following functionalities use automata binaries that need to be loaded separately: * analysis * tokenisation * generation * lemmatisation * segmentation * lookup * guess There is python code to perform basic string munging controlled by following bool attributes: try_lowercase: to use `str.lower()` try_titlecase: to use `str[0].upper() + str[1:]` try_uppercase: to use `str.upper()` try_detitlecase: to use `str[0].lower + str[1:]` The annotations will be changed when transformation has been applied. """ ## magic number for penalty weights _penalty = 28021984 ## paths to search auto-detected models from _stdpaths = [ '/usr/local/share/hfst/fi/', '/usr/share/hfst/fi/', '/usr/local/share/omorfi/', '/usr/share/omorfi/', './', 'generated/', 'src/generated/', '../src/generated/' ] def __init__(self, verbosity=False): """Construct Omorfi with given verbosity for printouts.""" self._verbosity = verbosity ## analyser model self.analyser = None ## tokeniser self.tokeniser = None ## generator model self.generator = None ## lemmatising model self.lemmatiser = None ## hyphenating model self.hyphenator = None ## segmenting model self.segmenter = None ## label-segment model self.labelsegmenter = None ## acceptor self.acceptor = None ## guesser model self.guesser = None ## UDPipe model self.udpiper = None ## UDPipeline object :-( self.udpipeline = None ## UDError object :-( self.uderror = None ## database of lexical unigram probabilities self.lexlogprobs = dict() ## database of tag unigram probabilities self.taglogprobs = dict() ## whether to lowercase and re-analyse if needed self.try_lowercase = True ## whether to Titlecase and re-analyse if needed self.try_titlecase = True ## whether to dEtitlecase and re-analyse if needed self.try_detitlecase = True ## whether to UPPERCASE and re-analyse if needed self.try_uppercase = False ## whether accept model is loaded self.can_accept = False ## whether analyser model is loaded self.can_analyse = False ## whether tokenisr model is loaded self.can_tokenise = True ## whether generator model is loaded self.can_generate = False ## whether lemmatising model is loaded self.can_lemmatise = False ## whether hypenation model is loaded self.can_hyphenate = False ## whether segmentation model is loaded self.can_segment = False ## whether label segmentation model is loaded self.can_labelsegment = False ## whether guesser model is loaded self.can_guess = False ## whether UDPipe is loaded self.can_udpipe = False def load_hfst(self, f): """Load an automaton from file. @param f containing single hfst automaton binary. """ try: his = libhfst.HfstInputStream(f) return his.read() except libhfst.NotTransducerStreamException: raise IOError def load_labelsegmenter(self, f): """Load labeled segments model from a file. @param f containing single hfst automaton binary. @sa load_hfst(self, f) """ self.labelsegmenter = self.load_hfst(f) self.can_labelsegment = True def load_segmenter(self, f): """Load segmentation model from a file. @param f containing single hfst automaton binary. @sa load_hfst(self, f) """ self.segmenter = self.load_hfst(f) self.can_segment = True def load_analyser(self, f): """Load analysis model from a file. @param f containing single hfst automaton binary. @sa load_hfst(self, f) """ self.analyser = self.load_hfst(f) self.can_analyse = True self.can_accept = True self.can_lemmatise = True def load_generator(self, f): """Load generation model from a file. @param f containing single hfst automaton binary. @sa load_hfst(self, f) """ self.generator = self.load_hfst(f) self.can_generate = True def load_acceptor(self, f): """Load acceptor model from a file. @param f containing single hfst automaton binary. @sa load_hfst(self, f) """ self.acceptor = self.load_hfst(f) self.can_accept = True def load_tokeniser(self, f): """Load tokeniser model from a file. @param f containing single hfst automaton binary. @sa load_hfst(self, f) """ self.tokeniser = self.load_hfst(f) self.can_tokenise = True def load_lemmatiser(self, f): """Load lemmatiser model from a file. @param f containing single hfst automaton binary. @sa load_hfst(self, f) """ self.tokeniser = self.load_hfst(f) self.can_lemmatise = True def load_hyphenator(self, f): """Load hyphenator model from a file. @param f containing single hfst automaton binary. @sa load_hfst(self, f) """ self.hyphenator = self.load_hfst(f) self.can_hyphenate = True def load_guesser(self, f): """Load guesser model from a file. @param f containing single hfst automaton binary. @sa load_hfst(self, f) """ self.guesser = self.load_hfst(f) self.can_guess = True def load_filename(self, path, **include): """Load omorfi automaton from filename and guess its use. A file name should consist of three parts separated by full stop. The second part must be a keyword describing the use of the automaton, first part is parsed as an identifier typically starting with the word omorfi, followed by any extras, such as the tagset for analysis or generation. The named arguments can include a name of automaton type as name, and truth value as value, for types of automata allowed to load. By default, the names `analyse`, `generate` and `segment` are loaded. Names not included are defaulted to False. E.g., `omorfi.load_filename(fn, analyse=True)` will only load file named fn if it can be identified as omorfi analyser. This is best used in conjunction with omorfi.load_from_dir. """ if len(include) == 0: include['analyse'] = True include['generate'] = True include['segment'] = True include['accept'] = True for ttype in [ 'analyse', 'generate', 'accept', 'tokenise', 'lemmatise', 'hyphenate', 'segment', 'labelsegment', 'guesser', 'udpipe' ]: if ttype not in include: include[ttype] = False parts = path[path.rfind('/') + 1:path.rfind('.')].split('.') if len(parts) != 2: if self._verbosity: print('not loaded', path) elif not parts[0] == 'omorfi': if self._verbosity: print('not omorfi', path) elif parts[1] == 'analyse' and include['analyse']: if self._verbosity: print('analyser', parts[0]) self.load_analyser(path) elif parts[1] == 'generate' and include['generate']: if self._verbosity: print('generator', parts[0]) self.load_generator(path) elif parts[1] == 'accept' and include['accept']: if self._verbosity: print('acceptor', parts[0]) self.load_acceptor(path) elif parts[1] == 'tokenise' and include['tokenise']: if self._verbosity: print('tokeniser', parts[0]) self.load_tokeniser(path) elif parts[1] == 'lemmatise' and include['lemmatise']: if self._verbosity: print('lemmatiser', parts[0]) self.load_lemmatiser(path) elif parts[1] == 'hyphenate' and include['hyphenate']: if self._verbosity: print('hyphenator', parts[0]) self.load_hyphenator(path) elif parts[1] == 'segment' and include['segment']: if self._verbosity: print('segmenter', parts[0]) self.load_segmenter(path) elif parts[1] == 'guesser' and include['guesser']: if self._verbosity: print('guesser', parts[0]) self.load_guesser(path) elif parts[1] == 'labelsegment' and include['labelsegment']: if self._verbosity: print('labelsegmenter', parts[0]) self.load_labelsegmenter(path) elif self._verbosity: print('skipped', parts) def _maybe_str2token(self, s): if isinstance(s, str): return {"surf": s} elif isinstance(s, dict): return s else: return { "error": "not a string or dict", "location": "maybe_str2token", "data": s } def load_from_dir(self, path=None, **include): """Load omorfi automata from given or known locations. If path is given it should point to directory of automata, otherwise standard installation paths are tried. Currently standard linux install paths are all globbed in following order: * /usr/local/share/hfst/fi/*.hfst * /usr/share/hfst/fi/*.hfst * /usr/local/share/omorfi/*.hfst * /usr/share/omorfi/*.hfst * getenv('HOME') + /.hfst/fi/*.hfst * getenv('HOME') + /.omorfi/*.hfst Last two paths require getenv('HOME'). All automata matching glob *.hfst are loaded and stored in part of omorfi class appropriate for their usage. They keyword args can be used to limit loading of automata. The name is analyser type and value is True. """ homepaths = [] if getenv('HOME'): home = getenv('HOME') homepaths = [home + '/.hfst/fi/', home + '/.omorfi/'] loadable = [] if path: if self._verbosity: print('adding', path + '/*.hfst') loadable = glob(path + '/*.hfst') else: for sp in self._stdpaths + homepaths: if self._verbosity: print('adding', sp + '/*.hfst') loadable += glob(sp + '/*.hfst') for filename in loadable: try: self.load_filename(filename, **include) except: print("broken HFST", filename, file=stderr) def load_udpipe(self, filename): """Load UDPipe model for statistical parsing. UDPipe can be used as extra information source for OOV symbols or all tokens. It works best with sentence-based analysis, token based does not keep track of context. @param filename path to UDPipe model """ if not can_udpipe: print("importing udpipe failed, cannot load udpipe xxx") return self.udpiper = Model.load(filename) # use pipeline for now, ugly but workable self.udpipeline = Pipeline(self.udpiper, 'horizontal', Pipeline.DEFAULT, Pipeline.DEFAULT, 'conllu') self.uderror = ProcessingError() ## udpipe is loaded self.can_udpipe = True def load_lexical_frequencies(self, lexfile): """Load a frequency list for lemmas. Experimental. Currently in uniq -c format, subject to change. @param filename path to file with frequencies. """ lextotal = 0 lexcounts = dict() for line in lexfile: fields = line.split('\t') lexcount = int(fields[0]) lexcounts[fields[1]] = lexcount lextotal += lexcount for lex, freq in lexcounts.items(): if freq != 0: self.lexlogprobs[lex] = log(freq / lextotal) else: # XXX: hack hack, should just use LM count stuff with # discounts self.lexlogprobs[lex] = log(1 / (lextotal + 1)) def load_omortag_frequencies(self, omorfile): """Load a frequenc list for tags. Experimental. Currently in uniq -c format. Subject to change. @param filename path to file with frequencies. """ omortotal = 0 omorcounts = dict() for line in omorfile: fields = line.split('\t') omorcount = int(fields[0]) omorcounts[fields[1]] = omorcount omortotal += omorcount for omor, freq in omorcounts.items(): if freq != 0: self.taglogprobs[omor] = log(freq / omortotal) else: # XXX: hack hack, should just use LM count stuff with # discounts self.taglogprobs[omor] = log(1 / (omortotal + 1)) def _find_retoken_recase(self, token): """Turns a string into a recased non-OOV token.""" if self.accept(token): return {"surf": token, "analsurf": token, "recase": "ORIGINALCASE"} if self.try_lowercase and self.accept(token.lower()): return { "surf": token, "analsurf": token.lower(), "recase": "LOWERCASED" } if self.try_uppercase and self.accept(token.upper()): return { "surf": token, analsurf: "token.upper()", "recase": "UPPERCASED" } if len(token) > 1: if self.try_titlecase and \ self.accept(token[0].upper() + token[1:].lower()): return { "surf": token, "analsurf": token[0].upper() + token[1:].lower(), "recase": "TITLECASED" } if self.try_detitlecase and \ self.accept(token[0].lower() + token[1:]): return { "surf": token, "analsurf": token[0].lower() + token[1:], "recase": "DETITLECASED" } return False def _find_retokens(self, token): """Turns a string into a list of likely tokens.""" retoken = self._find_retoken_recase(token) if retoken: return [retoken] # Word. if token[-1] in fin_punct_trailing: retoken = self._find_retoken_recase(token[:-1]) if retoken: retoken['SpaceAfter'] = "No" return [retoken, {"surf": token[-1], "SpaceBefore": "No"}] # -Word if token[0] in fin_punct_leading: retoken = self._find_retoken_recase(token[1:]) if retoken: retoken['SpaceBefore'] = 'No' return [{"surf": token[0], "SpaceAfter": "No"}, retoken] # "Word" if token[0] in fin_punct_leading and token[-1] in fin_punct_trailing: retoken = self._find_retoken_recase(token[1:-1]) if retoken: retoken['SpaceBefore'] = 'No' retoken['SpaceAfter'] = 'No' return [{ "surf": token[0], "SpaceAfter": "No" }, retoken, { "surf": token[-1], "SpaceBefore": "No" }] # word." or word", if len(token) > 2 and token[-1] in fin_punct_trailing and token[ -2] in fin_punct_trailing: retoken = self._find_retoken_recase(token[:-2]) if retoken: retoken["SpaceAfter"] = "No" return [ retoken, { "surf": token[-2], "SpaceBefore": "No", "SpaceAfter": "No" }, { "surf": token[-1], "SpaceBefore": "No" } ] # word.", if len(token) > 3 and token[-1] in fin_punct_trailing and token[ -2] in fin_punct_trailing and token[-3] in fin_punct_trailing: retoken = self._find_retoken_recase(token[:-3]) if retoken: retoken["SpaceAfter"] = "No" return [ retoken, { "surf": token[-3], "SpaceBefore": "No", "SpaceAfter": "No" }, { "surf": token[-2], "SpaceBefore": "No", "SpaceAfter": "No" }, { "surf": token[-1], "SpaceBefore": "No" } ] # "word." if len(token) > 3 and token[-1] in fin_punct_trailing and token[ -2] in fin_punct_trailing and token[0] in fin_punct_leading: retoken = self._find_retoken_recase(token[1:-2]) if retoken: retoken["SpaceAfter"] = "No" retoken["SpaceBefore"] = "No" return [{ "surf": token[0], "SpaceAfter": "No" }, retoken, { "surf": token[-2], "SpaceBefore": "No", "SpaceAfter": "No" }, { "surf": token[-1], "SpaceBefore": "No" }] # "word.", if len(token) > 4 and token[-1] in fin_punct_trailing and token[ -2] in fin_punct_trailing and token[ -3] in fin_punct_trailing and token[0] in fin_punct_leading: retoken = self._find_retoken_recase(token[1:-3]) if retoken: retoken["SpaceAfter"] = "No" retoken["SpaceBefore"] = "No" return [{ "surf": token[0], "SpaceAfter": "No" }, retoken, { "surf": token[-3], "SpaceBefore": "No", "SpaceAfter": "No" }, { "surf": token[-2], "SpaceBefore": "No", "SpaceAfter": "No" }, { "surf": token[-1], "SpaceBefore": "No" }] # ...non-word... pretokens = [] posttokens = [] while len(token) > 1 and token[-1] in fin_punct_trailing: posttokens = ([{ "surf": token[-1], "SpaceBefore": "No" }] + posttokens) token = token[:-1] while len(token) > 1 and token[0] in fin_punct_leading: pretokens += [{"surf": token[0], "SpaceAfter": "No"}] token = token[1:] lastresort = {"surf": token} if len(pretokens) > 0: lastresort['SpaceBefore'] = 'No' if len(posttokens) > 0: lastresort['SpaceAfter'] = 'No' return pretokens + [lastresort] + posttokens def _retokenise(self, tokens): """Takes list of string from and produces list of tokens. May change number of tokens. Should be used with result of split(). """ retokens = [] for token in tokens: for retoken in self._find_retokens(token): retokens.append(retoken) return retokens def fsa_tokenise(self, line): """Tokenise with FSA. @param line string to tokenise """ return None def python_tokenise(self, line): """Tokenise with python's basic string functions. @param line string to tokenise """ return self._retokenise(line.split()) def tokenise(self, line): """Perform tokenisation with loaded tokeniser if any, or `split()`. If tokeniser is available, it is applied to input line and if result is achieved, it is split to tokens according to tokenisation strategy and returned as a list. If no tokeniser are present, or none give results, the line will be tokenised using python's basic string functions. If analyser is present, tokeniser will try harder to get some analyses for each token using hard-coded list of extra splits. """ tokens = None if self.tokeniser: tokens = self.fsa_tokenise(line) if not tokens: tokens = self.python_tokenise(line) return tokens def _analyse_str(self, s): """Legacy function if you really need to analyse a string. Turn it into a token and analyse a token instead. This is not even the standard string mangling. Do not touch.""" token = {"surf": s} res = self._analyse_token(token) if len(s) > 2 and s[0].islower() and self.try_titlecase: tcs = s[0].upper() + s[1:].lower() if s != tcs: tctoken = {"surf": s, "analsurf": tcs, "recase": 'Titlecased'} tcres = self._analyse_token(tctoken) for r in tcres: r['anal'] += '[CASECHANGE=TITLECASED]' res = res + tcres if len(token) > 2 and s[0].isupper() and self.try_detitlecase: dts = s[0].lower() + s[1:] if dts != s: dttoken = { "surf": s, "analsurf": dts, "recase": "dETITLECASED" } dtres = self._analyse_token(dttoken) for r in dtres: r['anal'] += '[CASECHANGE=DETITLECASED]' res = res + dtres if not s.isupper() and self.try_uppercase: ups = s.upper() if s != ups: uptoken = {"surf": s, "analsurf": ups, "recase": "UPPERCASED"} upres = self._analyse_token(uptoken) for r in upres: r['anal'] += '[CASECHANGE=UPPERCASED]' res = res + upres if not s.islower() and self.try_lowercase: lows = s.lower() if s != lows: lowtoken = { "surf": s, "analsurf": lows, "recase": "lowercased" } lowres = self._analyse_token(lowtoken) for r in lowres: r['anal'] += '[CASECHANGE=LOWERCASED]' res += lowres return res def _analyse_token(self, token): rv = [] if "analsurf_override" in token: # begin of sentence, etc. recasing extra res = self.analyser.lookup(token["analsurf_override"]) for r in res: rvtoken = token.copy() rvtoken['anal'] = r[0] + '[WEIGHT=%f]' % (r[1]) rvtoken['weight'] = r[1] rv.append(rvtoken) if "analsurf" in token: # surface from already determined res = self.analyser.lookup(token["analsurf"]) for r in res: rvtoken = token.copy() rvtoken['anal'] = r[0] + '[WEIGHT=%f]' % (r[1]) rvtoken["weight"] = r[1] rv.append(rvtoken) else: # use real surface case res = self.analyser.lookup(token["surf"]) for r in res: rvtoken = token.copy() rvtoken['analsurf'] = token["surf"] rvtoken['anal'] = r[0] + '[WEIGHT=%f]' % (r[1]) rvtoken["weight"] = r[1] rv.append(rvtoken) if not "analsurf_override" in token and not "analsurf" in token: # also guess other cases s = token['surf'] if len(s) > 2 and s[0].islower() and self.try_titlecase: tcs = s[0].upper() + s[1:].lower() if tcs != s: tcres = self.analyser.lookup(tcs) for r in tcres: tctoken = token.copy() tctoken['recase'] = 'Titlecased' tctoken['analsurf'] = tcs tctoken['anal'] = r[0] + \ '[CASECHANGE=TITLECASED][WEIGHT=%f]' % (r[1] + self._penalty) tctoken["weight"] = r[1] + self._penalty rv.append(tctoken) if len(token) > 2 and s[0].isupper() and self.try_detitlecase: dts = s[0].lower() + s[1:] if dts != s: dtres = self.analyser.lookup(dts) for r in dtres: dttoken = token.copy() dttoken['recase'] = 'dETITLECASED' dttoken['analsurf'] = dts dttoken['anal'] = r[0] + \ "[CASECHANGE=DETITLECASED][WEIGHT=%f]" % (r[1] + self._penalty) dttoken["weight"] = r[1] + self._penalty rv.append(dttoken) if not s.isupper() and self.try_uppercase: ups = s.upper() if ups != s: upres = self.analyser.lookup(ups) for r in upres: uptoken = token.copy() uptoken['recase'] = 'UPPERCASED' uptoken['analsurf'] = ups uptoken['anal'] = r[0] + \ "[CASECHANGE=UPPERCASED][WEIGHT=%f]" % (r[1] + self._penalty) uptoken["weight"] = r[1] + self._penalty rv.append(uptoken) if not s.islower() and self.try_lowercase: lows = s.lower() if lows != s: lowres = self.analyser.lookup(lows) for r in lowres: lowtoken = token.copy() lowtoken['recase'] = 'lowercased' lowtoken['analsurf'] = lows lowtoken['anal'] = r[0] +\ "[CASECHANGE=LOWERCASED][WEIGHT=%f]" %(r[1] + self._penalty) lowtoken["weight"] = r[1] + self._penalty rv.append(lowtoken) return rv def analyse(self, token): """Perform a simple morphological analysis lookup. If try_titlecase does not evaluate to False, the analysis will also be performed with first letter uppercased and rest lowercased. If try_uppercase evaluates to not False, the analysis will also be performed on all uppercase variant. If try_lowercase evaluates to not False, the analysis will also be performed on all lowercase variant. The analyses with case mangling will have an additional element to them identifying the casing. """ anals = None if isinstance(token, str): anals = self._analyse_str(token) elif isinstance(token, dict): anals = self._analyse_token(token) else: anals = [{ "error": "token is not str or dict", "token": token, "location": "analyse()" }] if not anals: if isinstance(token, str): anal = {"anal": '[WORD_ID=%s][GUESS=UNKNOWN][WEIGHT=inf]' \ % (token), "weight": float('inf'), "OOV": "Yes", "guess": "None"} elif isinstance(token, dict): anal = token.copy() anal["anal"] = '[WORD_ID=%s][GUESS=UNKNOWN][WEIGHT=inf]' \ % (token['surf']) anal["weight"] = float('inf') anal["OOV"] = "Yes" anal["guess"] = "None" else: anal = { "error": "token is not str or dict", "token": token, "location": "analyse()" } anals = [anal] return anals def analyse_sentence(self, s): """Analyse a full sentence with tokenisation and guessing. for details of tokenisation, see @c tokenise(self, s). for details of analysis, see @c analyse(self, token). If further models like udpipe are loaded, may fill in gaps with that. """ tokens = self.tokenise(s) if not tokens: tokens = [{ "error": "cannot tokenise sentence", "sentence": s, "location": "analyse_sentence" }] analysis_lists = [] i = 0 for token in tokens: i += 1 analysis_lists[i] += [self.analyse(token)] if self.can_udpipe: # N.B: I used the vertical input here udinput = '\n'.join([token["surf"] for token in tokens]) uds = self.udpipe(udinput) if len(uds) == len(analysis_lists): for i in range(len(uds)): analsysis_lists[i] += [uds[i]] return None def _guess_str(self, s): token = {"surf": s} return self._guess_token(token) def _guess_token(self, token): res = self.guesser.lookup(token['surf']) guesses = [] for r in res: guesstoken = token.copy() guesstoken['anal'] = r[0] + '[GUESS=FSA][WEIGHT=%f]' % (r[1]) guesstoken['weight'] = float(r[1]) guesstoken['guess'] = 'FSA' guesses += [guesstoken] return guesses def _guess_heuristic(self, token): '''Heuristic guessing function written fully in python. This should always be the most simple basic backoff, e.g. noun singular nominative for everything. ''' guesstoken = token.copy() # woo advanced heuristics!! if token['surf'][0].isupper() and len(token['surf']) > 1: guesstoken['anal'] = '[WORD_ID=' + token['surf'] +\ "][UPOS=PROPN][NUM=SG][CASE=NOM][GUESS=HEUR]" +\ "[WEIGHT=%f]" %( self._penalty ) guesstoken['weight'] = self._penalty guesstoken['guess'] = 'PYTHON0ISUPPER' else: guesstoken['anal'] = '[WORD_ID=' + token['surf'] +\ "][UPOS=NOUN][NUM=SG][CASE=NOM][GUESS=HEUR]" +\ "[WEIGHT=%f]" %( self._penalty ) guesstoken['weight'] = self._penalty guesstoken['guess'] = 'PYTHONELSE' return [guesstoken] def guess(self, token): '''Speculate morphological analyses of OOV token. This method may use multiple information sources, but not the actual analyser. Therefore a typical use of this is after the analyse(token) function has failed. Note that some information sources perform badly when guessing without context, for these the analyse_sentence(sent) is the only option. ''' realtoken = self._maybe_str2token(token) guesses = self._guess_heuristic(realtoken) if self.can_udpipe: guesses += [self._udpipe(realtoken['surf'])] if self.can_guess: guesses += self._guess_token(realtoken) return guesses def _lemmatise(self, token): res = self.lemmatiser.lookup(token['surf']) lemmas = [] for r in res: lemmatoken = token.copy() lemmatoken['lemma'] = r[0] lemmatoken['lemmaweight'] = float(r[1]) lemmas += [lemmatoken] return lemmas def lemmatise(self, token): '''Lemmatise a token, returning a dictionary ID. Like morphological analysis, can return more than one results, which are possible (combinations of) lexeme ids. If the token is not in the dictionary, the surface form is returned as most likely "lemma". ''' realtoken = self._maybe_str2token(token) lemmas = None lemmas = self._lemmatise(realtoken) if not lemmas or len(lemmas) < 1: lemmatoken = realtoken.copy() lemmatoken['lemma'] = lemmatoken['surf'] lemmatoken['lemmaweight'] = float('inf') lemmas = [lemmatoken] return lemmas def _segment(self, token): res = self.segmenter.lookup(token['surf']) segmenteds = [] for r in res: segmenttoken = token.copy() segmenttoken['segments'] = r[0] segmenttoken['segmentweight'] = float(r[1]) segmenteds += [segmenttoken] return segmenteds def segment(self, token): '''Segment token into morphs, words and other string pieces. The segments come separated by some internal markers for different segment boundaries. ''' realtoken = self._maybe_str2token(token) segments = None segments = self._segment(realtoken) if not segments or len(segments) < 1: segmenttoken = realtoken.copy() segmenttoken['segments'] = segmenttoken['surf'] segments = [segmenttoken] return segments def _labelsegment(self, token): res = self.labelsegmenter.lookup(token['surf']) lss = [] for r in res: lstoken = token.copy() lstoken['labelsegments'] = r[0] lstoken['lsweight'] = float(r[1]) lss += [lstoken] return lss def labelsegment(self, token): '''Segment token into labelled morphs, words and other string pieces. The segments are suffixed with their morphologically relevant informations, e.g. lexical classes for root lexemes and inflectional features for inflectional segments. This functionality is experimental due to hacky way it was patched together. ''' realtoken = self._maybe_str2token(token) labelsegments = None labelsegments = self._labelsegment(realtoken) if not labelsegments or len(labelsegments) < 1: lstoken = realtoken.copy() lstoken['labelsegments'] = lstoken['surf'] lstoken['lsweight'] = float('inf') labelsegments = [lstoken] return labelsegments def _accept(self, token): """Look up token from acceptor model.""" if self.acceptor: res = self.acceptor.lookup(token['surf']) elif self.analyser: res = self.analyser.lookup(token['surf']) else: res = None return res def accept(self, token): '''Check if the token is in the dictionary or not. Returns False for OOVs, True otherwise. Note, that this is not necessarily more efficient than analyse(token) ''' realtoken = self._maybe_str2token(token) accept = False accepts = None accepts = self._accept(realtoken) if accepts and len(accepts) > 0: accept = True else: accept = False return accept def _generate(self, token): res = self.generator.lookup(token['anal']) generations = [] for r in res: g = token.copy() g['surf'] = r[0] g['genweight'] = r[1] generations += [g] return generations def generate(self, omorstring): '''Generate surface forms corresponding given token description. Currently only supports very direct omor style analysis string generation. For round-tripping and API consistency you can also feed a token dict here. ''' gentoken = {} if isinstance(omorstring, str): gentoken['anal'] = omorstring elif isinstance(omorstring, dict): # for round-tripping gentoken = omorstring else: gentoken = { 'error': 'token not dict or string', 'location': 'generate()' } generateds = None if self.can_generate: generated = self._generate(gentoken) if not generated: gentoken['surf'] = gentoken['anal'] gentoken['genweight'] = float('inf') generated = [gentoken] return generated def _udpipe(self, udinput): conllus = self.udpipeline.process(udinput, self.uderror) if self.uderror.occurred(): return None tokens = [] for conllu in conllus.split('\n'): if conllu.startswith('#'): continue elif conllu.strip() == '': continue tokens += [self._conllu2token(conllu)] return tokens def _conllu2token(self, conllu): fields = conllu.split() if len(fields) != 10: print("conllu2token conllu fail", fields) upos = fields[3] wordid = fields[2] surf = fields[1] ufeats = fields[5] misc = fields[9] analysis = '[WORD_ID=%s][UPOS=%s]%s[GUESS=UDPIPE]' % ( wordid, upos, self._ufeats2omor(ufeats)) token = { 'anal': analysis, 'misc': misc, 'upos': upos, 'surf': surf, 'ufeats': ufeats } return token def _ufeats2omor(self, ufeats): return '[' + ufeats.replace('|', '][') + ']'
sys.stdout = codecs.getwriter(encoding)(sys.stdout) if len(sys.argv) < 4: sys.stderr.write( 'Usage: %s input_format(tokenize|conllu|horizontal|vertical) output_format(conllu) model_file\n' % sys.argv[0]) sys.exit(1) sys.stderr.write('Loading model: ') model = Model.load(sys.argv[3]) if not model: sys.stderr.write("Cannot load model from file '%s'\n" % sys.argv[3]) sys.exit(1) sys.stderr.write('done\n') pipeline = Pipeline(model, sys.argv[1], Pipeline.DEFAULT, Pipeline.DEFAULT, sys.argv[2]) error = ProcessingError() # Read whole input text = ''.join(sys.stdin.readlines()) # Process data processed = pipeline.process(text, error) if error.occurred(): sys.stderr.write("An error occurred when running run_udpipe: ") sys.stderr.write(error.message) sys.stderr.write("\n") sys.exit(1) sys.stdout.write(processed)
class Omorfi: """ An object holding omorfi binaries for all the functions of omorfi. The following functionalities use automata binaries that need to be loaded separately: * analysis * tokenisation * generation * lemmatisation * segmentation * lookup * guess There is python code to perform basic string munging controlled by following bool attributes: try_lowercase: to use `str.lower()` try_titlecase: to use `str[0].upper() + str[1:]` try_uppercase: to use `str.upper()` try_detitlecase: to use `str[0].lower + str[1:]` The annotations will be changed when transformation has been applied. """ #: magic number for penalty weights _penalty = 28021984 def __init__(self, verbosity=False): """Construct Omorfi with given verbosity for printouts.""" self._verbosity = verbosity ## analyser model self.analyser = None ## tokeniser self.tokeniser = None ## generator model self.generator = None ## lemmatising model self.lemmatiser = None ## hyphenating model self.hyphenator = None ## segmenting model self.segmenter = None ## label-segment model self.labelsegmenter = None ## acceptor self.acceptor = None ## guesser model self.guesser = None ## UDPipe model self.udpiper = None ## UDPipeline object :-( self.udpipeline = None ## UDError object :-( self.uderror = None ## database of lexical unigram probabilities self.lexlogprobs = dict() ## database of tag unigram probabilities self.taglogprobs = dict() ## whether to lowercase and re-analyse if needed self.try_lowercase = True ## whether to Titlecase and re-analyse if needed self.try_titlecase = True ## whether to dEtitlecase and re-analyse if needed self.try_detitlecase = True ## whether to dEtitlecase and re-analyse if needed self.try_detitle_firstinsent = True ## whether to UPPERCASE and re-analyse if needed self.try_uppercase = False ## whether accept model is loaded self.can_accept = False ## whether analyser model is loaded self.can_analyse = False ## whether tokenisr model is loaded self.can_tokenise = True ## whether generator model is loaded self.can_generate = False ## whether lemmatising model is loaded self.can_lemmatise = False ## whether hypenation model is loaded self.can_hyphenate = False ## whether segmentation model is loaded self.can_segment = False ## whether label segmentation model is loaded self.can_labelsegment = False ## whether guesser model is loaded self.can_guess = False ## whether UDPipe is loaded self.can_udpipe = False def _load_hfst(self, f): """Load an automaton from file. Args: f: containing single hfst automaton binary. Throws: FileNotFoundError if file is not found """ try: his = libhfst.HfstInputStream(f) return his.read() except libhfst.NotTransducerStreamException: raise IOError(2, f) from None def load_labelsegmenter(self, f): """Load labeled segments model from a file. Args: f: containing single hfst automaton binary. """ self.labelsegmenter = self._load_hfst(f) self.can_labelsegment = True def load_segmenter(self, f): """Load segmentation model from a file. Args: f: containing single hfst automaton binary. """ self.segmenter = self._load_hfst(f) self.can_segment = True def load_analyser(self, f): """Load analysis model from a file. Args f: containing single hfst automaton binary. """ self.analyser = self._load_hfst(f) self.can_analyse = True self.can_accept = True self.can_lemmatise = True def load_generator(self, f): """Load generation model from a file. Args: f: containing single hfst automaton binary. """ self.generator = self._load_hfst(f) self.can_generate = True def load_acceptor(self, f): """Load acceptor model from a file. Args: f: containing single hfst automaton binary. """ self.acceptor = self._load_hfst(f) self.can_accept = True def load_tokeniser(self, f): """Load tokeniser model from a file. Args: f: containing single hfst automaton binary. """ self.tokeniser = self._load_hfst(f) self.can_tokenise = True def load_lemmatiser(self, f): """Load lemmatiser model from a file. Args: f: containing single hfst automaton binary. """ self.tokeniser = self._load_hfst(f) self.can_lemmatise = True def load_hyphenator(self, f): """Load hyphenator model from a file. Args: f: containing single hfst automaton binary. """ self.hyphenator = self._load_hfst(f) self.can_hyphenate = True def load_guesser(self, f): """Load guesser model from a file. Args: f: containing single hfst automaton binary. """ self.guesser = self._load_hfst(f) self.can_guess = True def load_udpipe(self, filename: str): """Load UDPipe model for statistical parsing. UDPipe can be used as extra information source for OOV symbols or all tokens. It works best with sentence-based analysis, token based does not keep track of context. @param filename path to UDPipe model """ if not can_udpipe: print("importing udpipe failed, cannot load udpipe xxx") return self.udpiper = Model.load(filename) # use pipeline for now, ugly but workable self.udpipeline = Pipeline(self.udpiper, 'horizontal', Pipeline.DEFAULT, Pipeline.DEFAULT, 'conllu') self.uderror = ProcessingError() ## udpipe is loaded self.can_udpipe = True def load_lexical_frequencies(self, lexfile): """Load a frequency list for lemmas. Experimental. Currently in uniq -c format, subject to change. Args: lexfile: file with frequencies. """ lextotal = 0 lexcounts = dict() for line in lexfile: fields = line.split('\t') lexcount = int(fields[0]) lexcounts[fields[1]] = lexcount lextotal += lexcount for lex, freq in lexcounts.items(): if freq != 0: self.lexlogprobs[lex] = log(freq / lextotal) else: # XXX: hack hack, should just use LM count stuff with # discounts self.lexlogprobs[lex] = log(1 / (lextotal + 1)) def load_omortag_frequencies(self, omorfile): """Load a frequenc list for tags. Experimental. Currently in uniq -c format. Subject to change. Args: omorfile: path to file with frequencies. """ omortotal = 0 omorcounts = dict() for line in omorfile: fields = line.split('\t') omorcount = int(fields[0]) omorcounts[fields[1]] = omorcount omortotal += omorcount for omor, freq in omorcounts.items(): if freq != 0: self.taglogprobs[omor] = log(freq / omortotal) else: # XXX: hack hack, should just use LM count stuff with # discounts self.taglogprobs[omor] = log(1 / (omortotal + 1)) def _find_retoken_recase(self, token: Token): """Checks if token is acceptable when case is ignored. Used case-ignorations depend on the settings. Args: token: token to recase Returns: recased token or False if no retoken is possible """ if self.accept(token): return token if len(token.surf) > 1: # we test len to just use 1: slice... if self.try_titlecase and not token.surf[0].isupper(): if self.accept( Token(token.surf[0].upper() + token.surf[1:].lower())): return token if self.try_detitlecase and not token.surf[0].islower(): if self.accept(Token(token.surf[0].lower() + token.surf[1:])): return token if self.try_lowercase: if self.accept(Token(token.surf.lower())): return token if self.try_uppercase: if self.accept(Token(token.surf.upper())): return token return False def _find_retokens(self, token: Token): """Finds list of acceptable sub-tokens from a token. Tries to strip punct tokens from left and right. Args: token: token to retokenise Returns: list of tokens giving best retokenisation """ retoken = self._find_retoken_recase(token) if retoken: return [retoken] pretokens = [] posttokens = [] for i in range(4): for j in range(4): if len(token.surf) > (i + j): if j == 0: resurf = token.surf[i:] else: resurf = token.surf[i:-j] presurfs = token.surf[:i] postsurfs = token.surf[-j:] pretrailpuncts = True for c in presurfs: if c in fin_punct_leading: pretoken = Token(c) pretoken.spaceafter = 'No' pretokens.append(pretoken) else: pretrailpuncts = False break for c in postsurfs: if c in fin_punct_trailing: posttoken = Token(c) posttoken.spacebefore = 'No' posttokens.append(posttoken) else: pretrailpuncts = False break if not pretrailpuncts: continue retoken = Token(resurf) reretoken = self._find_retoken_recase(retoken) if reretoken: return pretokens + [reretoken] + posttokens # no acceptable substring inside, just strip puncts return [token] def _retokenise(self, tokens: list): """Takes list of string from and produces list of tokens. May change number of tokens. Should be used with result of split(). Args: tokens: list of tokens to retokenise Returns: list of tokens representing best tokenisations of tokens """ retokens = [] for s in tokens: token = Token(s) for retoken in self._find_retokens(token): retokens.append(retoken) return retokens def fsa_tokenise(self, line: str): """Tokenise with FSA. Args: line: string to tokenise Todo: Not implemented (needs pmatch python support) """ return None def python_tokenise(self, line: str): """Tokenise with python's basic string functions. Args: line: string to tokenise """ return self._retokenise(line.split()) def tokenise(self, line: str): """Perform tokenisation with loaded tokeniser if any, or `split()`. If tokeniser is available, it is applied to input line and if result is achieved, it is split to tokens according to tokenisation strategy and returned as a list. If no tokeniser are present, or none give results, the line will be tokenised using python's basic string functions. If analyser is present, tokeniser will try harder to get some analyses for each token using hard-coded list of extra splits. Args: line: a string to be tokenised, should contain a line of text or a sentence Returns: A list of tokens based on the line. List may include boundary non-tokens if e.g. sentence boundaries are recognised. For empty line a paragraph break non-token may be returned. """ tokens = None if self.tokeniser: tokens = self.fsa_tokenise(line) if not tokens: tokens = self.python_tokenise(line) return tokens def _analyse(self, token: Token): '''Analyse token using HFST and perform recasings. Args: token: token to analyse''' # use real surface case newanals = list() res = self.analyser.lookup(token.surf) for r in res: omor = r[0] + '[WEIGHT=%f]' % (r[1]) weight = r[1] newanals.append(Analysis.fromomor(omor, weight)) if token.pos == 1 and token.surf[0].isupper()\ and len(token.surf) > 1: res = self.analyser.lookup(token.surf[0].lower() + token.surf[1:]) for r in res: omor = r[0] + '[WEIGHT=%f]' % (r[1]) weight = r[1] newanals.append(Analysis.fromomor(omor, weight)) for a in newanals: token.analyses.append(a) return newanals def analyse(self, token: Token): """Perform a simple morphological analysis lookup. The analysis will be performed for re-cased variants based on the state of the member variables. The re-cased analyses will have more penalty weight and additional analyses indicating the changes. Side-Effects: The analyses are stored in the token, and only the new analyses are returned. Args: token: token to be analysed. Returns: An HFST structure of raw analyses, or None if there are no matches in the dictionary. """ if isinstance(token, str): token = Token(token) anals = self._analyse(token) if not anals: omor = '[WORD_ID=' + token.surf + '][UPOS=X]' +\ '[GUESS=UNKNOWN][WEIGHT=inf]' weight = float('inf') anal = Analysis.fromomor(omor, weight) anal.manglers.append("GUESSER=NONE") token.analyses.append(anal) return [anal] return anals def analyse_sentence(self, s): """Analyse a full sentence with tokenisation and guessing. for details of tokenisation, see @c tokenise(self, s). for details of analysis, see @c analyse(self, token). If further models like udpipe are loaded, may fill in gaps with that. """ tokens = self.tokenise_sentence(s) if not tokens: errortoken = Token() errortoken.nontoken = "error" errortoken.error = "cannot tokenise sentence" errortoken.comment = { "sentence": s, "location": "analyse_sentence" } return [errortoken] analysis_lists = [] i = 0 for token in tokens: i += 1 analysis_lists[i] += [self.analyse(token)] if self.can_udpipe: # N.B: I used the vertical input here udinput = '\n'.join([token.surf for token in tokens]) uds = self._udpipe(udinput) if len(uds) == len(analysis_lists): for i, ud in enumerate(uds): analysis_lists[i] += [ud] return None def _guess_token(self, token: Token): '''Guess token reading using language models. Args: token: token to guess''' res = self.guesser.lookup(token.surf) for r in res: anal = r[0] + '[GUESS=FSA][WEIGHT=%f]' % (r[1]) weight = float(r[1]) guess = Analysis.fromomor(anal, weight) guess.manglers.append("GUESSER=FSA") token.analyses.append(guess) return res def _guess_heuristic(self, token: Token): '''Heuristic guessing function written fully in python. This is kind of last resort, but has some basic heuristics that may be always useful. Args: token: token to guess Returns: list: new analyses guessed ''' # woo advanced heuristics!! newanals = list() s = token.surf trieds = {s} if len(s) > 2 and (s[0].islower() or s[1].isupper()) and \ self.try_titlecase: tcs = s[0].upper() + s[1:].lower() if tcs not in trieds: tcres = self.analyser.lookup(tcs) for r in tcres: mangler = 'Titlecased' omor = r[0] + \ '[CASECHANGE=TITLECASED]' + \ '[WEIGHT=%f]' % (r[1] + self._penalty) weight = r[1] + self._penalty anal = Analysis.fromomor(omor, weight) anal.manglers.append(mangler) anal.analsurf = tcs newanals.append(anal) trieds.add(tcs) if len(s) > 2 and s[0].isupper() and self.try_detitlecase: dts = s[0].lower() + s[1:] if dts not in trieds: dtres = self.analyser.lookup(dts) for r in dtres: mangler = 'dETITLECASED' omor = r[0] + \ "[CASECHANGE=DETITLECASED]" + \ "[WEIGHT=%f]" % (r[1] + self._penalty) weight = r[1] if token.pos != 1: weight += self._penalty anal = Analysis.fromomor(omor, weight) anal.manglers.append(mangler) anal.analsurf = dts newanals.append(anal) trieds.add(dts) if not s.isupper() and self.try_uppercase: ups = s.upper() if ups not in trieds: upres = self.analyser.lookup(ups) for r in upres: mangler = 'UPPERCASED' omor = r[0] + \ "[CASECHANGE=UPPERCASED]" + \ "[WEIGHT=%f]" % (r[1] + self._penalty) weight = r[1] + self._penalty anal = Analysis.fromomor(omor, weight) anal.manglers.append(mangler) anal.analsurf = ups newanals.append(anal) trieds.add(ups) if not s.islower() and self.try_lowercase: lows = s.lower() if lows not in trieds: lowres = self.analyser.lookup(lows) for r in lowres: mangler = 'lowercased' omor = r[0] +\ "[CASECHANGE=LOWERCASED]" + \ "[WEIGHT=%f]" % (r[1] + self._penalty) weight = r[1] + self._penalty anal = Analysis.fromomor(omor, weight) anal.manglers.append(mangler) anal.analsurf = lows newanals.append(anal) trieds.add(lows) if not newanals: if len(token.surf) == 1: omor = '[WORD_ID=' + token.surf +\ "][UPOS=SYM][GUESS=HEUR]" +\ "[WEIGHT=%f]" % (self._penalty) weight = self._penalty guess = Analysis.fromomor(omor, weight) guess.manglers.append('GUESSER=PYTHON_LEN1') newanals.append(guess) elif token.surf[0].isupper() and len(token.surf) > 1: omor = '[WORD_ID=' + token.surf +\ "][UPOS=PROPN][NUM=SG][CASE=NOM][GUESS=HEUR]" +\ "[WEIGHT=%f]" % (self._penalty) weight = self._penalty guess = Analysis.fromomor(omor, weight) guess.manglers.append('GUESSER=PYTHON_0ISUPPER') newanals.append(guess) else: omor = '[WORD_ID=' + token.surf +\ "][UPOS=NOUN][NUM=SG][CASE=NOM][GUESS=HEUR]" +\ "[WEIGHT=%f]" % (self._penalty) weight = self._penalty guess = Analysis.fromomor(omor, weight) guess.manglers.append('GUESSER=PYTHON_ELSE') newanals.append(guess) for anal in newanals: token.analyses.append(anal) return newanals def guess(self, token: Token): '''Speculate morphological analyses of OOV token. This method may use multiple information sources, but not the actual analyser. Therefore a typical use of this is after the analyse(token) function has failed. Note that some information sources perform badly when guessing without context, for these the analyse_sentence(sent) is the only option. Side-effect: This operation stores guesses in token for future use as well as returning them. Args: token: token to analyse with guessers. Returns: New guesses as a list of Analysis objects. ''' guesses = self._guess_heuristic(token) if self.can_udpipe: guesses += [self._udpipe(token.surf)] if self.can_guess: guesses += self._guess_token(token) return guesses def _lemmatise(self, token): res = self.lemmatiser.lookup(token.surf) newlemmas = list() for r in res: lemma = r[0] weight = float(r[1]) anal = Analysis() anal.raw = lemma anal.rawtype = "lemma" anal.weight = weight newlemmas.append(anal) for lemma in newlemmas: token.lemmatisations.append(lemma) return newlemmas def lemmatise(self, token: Token): '''Lemmatise token, splitting it into valid word id's from lexical db. Side-effect: This operation stores lemmas in the token for future use and only returns HFST structures. Use Token's method's to retrieve tokens in pythonic structures. Args: token: token to lemmatise Returns: New lemmas in analysis list ''' lemmas = None lemmas = self._lemmatise(token) if not lemmas or len(lemmas) < 1: lemma = token.surf weight = float('inf') guess = Analysis() guess.raw = lemma guess.rawtype = "lemma" guess.ewight = weight guess.manglers.append("GUESSER=SURFISLEMMA") token.lemmatisations.append(guess) lemmas = [guess] return lemmas def _segment(self, token: Token): '''Intenal segmenting using HFST automaton. Args: token: token to segment.''' res = self.segmenter.lookup(token.surf) newsegs = list() for r in res: segments = r[0] weight = float(r[1]) anal = Analysis() anal.raw = segments anal.weight = weight anal.rawtype = "segments" newsegs.append(anal) for ns in newsegs: token.segmentations.append(ns) return newsegs def segment(self, token: Token): '''Segment token into morphs, words and other string pieces. Side-effect: this operation stores segments in the token for future use and only returns the HFST structures. To get pythonic data use Token's methods afterwards. Args: token: token to segment Returns: New segmentations in analysis list ''' segments = None segments = self._segment(token) if not segments or len(segments) < 1: segments = token.surf weight = float('inf') guess = Analysis() guess.raw = segments guess.weight = weight guess.rawtype = "segments" guess.manglers.append("GUESSER=SURFISSEGMENT") token.segmentations.append(guess) segments = [guess] return segments def _labelsegment(self, token: Token): '''Internal implementation of segment label lookup with FSA. Args: token: token to analyse Returns: list of new labelsegment analyses.''' res = self.labelsegmenter.lookup(token.surf) newlabels = list() for r in res: labelsegments = r[0] weight = float(r[1]) anal = Analysis() anal.raw = labelsegments anal.weight = weight anal.rawtype = "labelsegments" newlabels.append(anal) for ls in newlabels: token.labelsegmentations.append(ls) return newlabels def labelsegment(self, token: Token): '''Segment token into labelled morphs, words and other string pieces. The segments are suffixed with their morphologically relevant informations, e.g. lexical classes for root lexemes and inflectional features for inflectional segments. This functionality is experimental due to hacky way it was patched together. Side-effect: Note that this operation stores the labelsegments in the token for future use, and only returns raw HFST structures. To get pythonic you can use Token's methods afterwards. Args: token: token to segment with labels Returns: New labeled segemntations in analysis list. ''' labelsegments = None labelsegments = self._labelsegment(token) if not labelsegments or len(labelsegments) < 1: labelsegments = token.surf + "|UNK" lsweight = float('inf') guess = Analysis() guess.raw = labelsegments guess.weight = lsweight guess.rawtype = "labelsegments" guess.manglers.append("GUESSER=SURFISLABELS") token.labelsegmentations.append(guess) labelsegments = [guess] return labelsegments def _accept(self, token: Token): """Look up token from acceptor model. Args: token: token to accept Returns: analyses of token""" if self.acceptor: res = self.acceptor.lookup(token.surf) elif self.analyser: res = self.analyser.lookup(token.surf) else: res = None return res def accept(self, token): '''Check if the token is in the dictionary or not. Returns: False for OOVs, True otherwise. Note, that this is not necessarily more efficient than bool(analyse(token)) ''' return bool(self._accept(token)) def _generate(self, s: str): '''Generate surface forms from string using FSA model. Args: s: string matching raw omor analysis Returns: string containing surface forms ''' res = self.generator.lookup(s) generations = [] for r in res: generations += [r[0]] return "/".join(generations) def generate(self, omorstring: str): '''Generate surface forms corresponding given token description. Currently only supports very direct omor style analysis string generation. Args: omorstring: Omorfi analysis string to generate Returns A surface string word-form, or the omorstring argument if generation fails. Or None if generator is not loaded. ''' generated = None if self.can_generate: generated = self._generate(omorstring) if not generated: return omorstring return generated def _udpipe(self, udinput: str): """Pipes input to udpipe model. Args: udinput: input for udpipe Returns: tokens with udpipe analyses """ conllus = self.udpipeline.process(udinput, self.uderror) if self.uderror.occurred(): return None tokens = [] for conllu in conllus.split('\n'): if conllu.startswith('#'): continue elif conllu.strip() == '': continue tokens += [Token.fromconllu(conllu)] return tokens def tokenise_sentence(self, sentence: str): '''tokenise a sentence. To be used when text is already sentence-splitted. If the text is plain text with sentence boundaries within lines, use Args: sentence: a string containing one sentence Returns: list of tokens in sentence ''' if not sentence or sentence == '': token = Token() token.nontoken = "separator" token.comment = '' return [token] tokens = self.tokenise(sentence) pos = 1 for token in tokens: token.pos = pos pos += 1 return tokens def tokenise_plaintext(self, f): '''tokenise a whole text. Args: f: filelike object with iterable strings Returns: list of tokens ''' tokens = list() for line in f: tokens = self.tokenise(line.strip()) pos = 1 for token in tokens: token.pos = pos pos += 1 sep = Token() sep.nontoken = "separator" tokens.append(sep) return tokens eoft = Token() eoft.nontoken = "eof" tokens.append(eoft) return tokens def tokenise_conllu(self, f): '''tokenise a conllu sentence or comment. Should be used a file-like iterable that has CONLL-U sentence or comment or empty block coming up. Args: f: filelike object with iterable strings Returns: list of tokens ''' tokens = list() for line in f: fields = line.strip().split('\t') token = Token() if len(fields) != 10: if line.startswith('#'): token.nontoken = "comment" token.comment = line.strip() tokens.append(token) return tokens elif line.strip() == '': token.nontoken = "separator" token.comment = '' tokens.append(token) return tokens else: token.nontoken = "error" token.error = line.strip() tokens = [token] return tokens token._conllu = fields try: index = int(fields[0]) except ValueError: if '-' in fields[0]: # MWE continue elif '.' in fields[0]: # a ghost continue else: print("Cannot figure out token index", fields[0], file=stderr) exit(1) token.pos = index token.surf = fields[1] if fields[9] != '_': miscs = fields[9].split('|') for misc in miscs: k, v = misc.split('=') if k == 'SpaceAfter': token.spaceafter = v elif k in [ 'Alt', 'FTB-PronType', 'FTB-Rel', 'Missed-Rel', 'FTB-rel', 'Join', 'Missed-SUBCAT', 'FTB-Sub', 'Prefix', 'FTB1-InfForm', 'Missed-POSITION' ]: # FTB stuff pass else: print("Unknown MISC", k, file=stderr) exit(1) tokens.append(token) eoft = Token() eoft.nontoken = "eof" tokens.append(eoft) return tokens def tokenise_vislcg(self, f): '''Tokenises a sentence from VISL-CG format data. Returns a list of tokens when it hits first non-token block, including a token representing this non-token block. Args: f: filelike object to itrate strings of vislcg data Returns: list of tokens ''' tokens = list() pos = 1 for line in f: token = Token() line = line.strip() if not line or line == '': token.nontoken = "separator" token.comment = '' tokens.append(token) return tokens elif line.startswith("#") or line.startswith("<"): # # comment, or # <TAG> </TAG> token.nontoken = "comment" token.comment = line.strip() tokens.append(token) return tokens elif line.startswith('"<') and line.endswith('>"'): # "<surf>" token = Token() token.surf = line[2:-2] tokens.append(token) pos += 1 elif line.startswith('\t"'): # \t"lemma" ANAL ANAL ANAL fields = line.strip().split() token.lemma = fields[0].strip('"') elif line.startswith(';\t"'): # ;\t"lemma" ANAL ANAL ANAL KEYWORD:rulename token.nontoken = "gold" token.comment = line.strip() else: token.nontoken = "error" token.error = 'vislcg: ' + line.strip() eoft = Token() eoft.nontoken = "eof" tokens.append(eoft) return tokens
class ProcessorUDPipe: """Wrapper around UDPipe - Trainable pipeline library. Performs: 1. Tokenization. 2. Tagging. 3. Lemmatizing. 4. Parsing. """ def __init__(self, model_path, tagger=True, parser=True, delay_init=False): self._model_path = model_path self._enable_tagger = tagger self._enable_parser = parser self.model = None if not delay_init: self.init() def init(self): if self.model is None: self.model = Model.load(self._model_path) if not self.model: sys.stderr.write('Cannot load model from file "%s"\n' % self._model_path) self.tagger = Pipeline.DEFAULT if self._enable_tagger else Pipeline.NONE self.parser = Pipeline.DEFAULT if self._enable_parser else Pipeline.NONE self.error = ProcessingError() self.converter_conll = ConverterConllUDV1() def __call__(self, *argv): """Performs tokenization, tagging, lemmatizing and parsing. Args: text(str): text. OR tokens(list): List of Token objects. sentences(list): List of Sentence objects. Returns: Dictionary that contains: 1. tokens - list of objects Token. 2. sentences - list of objects Sentence. 3. lemma - list of lists of strings that represent lemmas of words. 4. postag - list of lists of strings that represent postags of words. 5. morph - list of lists of strings that represent morphological features. 6. syntax_dep_tree - list of lists of objects WordSynt that represent a dependency tree. """ assert self.model if type(argv[0]) == str: self.TOKENIZER = 'generic_tokenizer' self.pipeline = Pipeline(self.model, self.TOKENIZER, self.tagger, self.parser, 'conllu') return self.process_text(argv[0]) self.TOKENIZER = 'horizontal' self.pipeline = Pipeline(self.model, self.TOKENIZER, self.tagger, self.parser, 'conllu') return self.process_tokenized(argv[0], argv[1]) def process_text(self, text): udpipe_result = self.pipeline.process(text, self.error) if self.error.occurred(): sys.stderr.write( 'An error occurred when calling ProcessorUDPipe: ') sys.stderr.write(self.error.message) return sys.stderr.write('\n') annotation = self.convert_conll(text, udpipe_result) return annotation def process_tokenized(self, tokens, sentences): raw_input = self.prepare_tokenized_input(tokens, sentences) annotation = self.process_text(raw_input) lemma_result = annotation['lemma'] postag_result = annotation['postag'] morph_result = annotation['morph'] synt_dep_tree_result = annotation['syntax_dep_tree'] return { 'lemma': lemma_result, 'postag': postag_result, 'morph': morph_result, 'syntax_dep_tree': synt_dep_tree_result } def prepare_tokenized_input(self, tokens, sentences): raw_input_s = '' for sent in sentences: line = ' '.join((e.text for e in CSentence(tokens, sent))) raw_input_s += line raw_input_s += '\n' return raw_input_s def convert_conll(self, text, udpipe_result): annotation = self.converter_conll(udpipe_result) if self.tagger == Pipeline.NONE: for key in ('lemma', 'postag'): annotation.pop(key, None) if self.parser == Pipeline.NONE: for key in ('syntax_dep_tree', 'postag'): annotation.pop(key, None) for sent_lemma in annotation['lemma']: for i in range(len(sent_lemma)): sent_lemma[i] = sent_lemma[i].lower() annotation['sentences'] = self.converter_conll.sentence_split( annotation['form']) annotation['tokens'] = self.converter_conll.get_tokens( text, annotation['form']) return annotation
class Omorfi: """ An object holding omorfi binariesfor all the functions of omorfi. The following functionalities use automata binaries that need to be loaded separately: * analysis * tokenisation * generation * lemmatisation * segmentation * lookup * guess There is python code to perform basic string munging controlled by following bool attributes: try_lowercase: to use `str.lower()` try_titlecase: to use `str[0].upper() + str[1:]` try_uppercase: to use `str.upper()` try_detitlecase: to use `str[0].lower + str[1:]` The annotations will be changed when transformation has been applied. """ analyser = None tokeniser = None generator = None lemmatiser = None hyphenator = None segmenter = None labelsegmenter = None acceptor = None guesser = None udpiper = None udpipeline = None uderror = None try_lowercase = True try_titlecase = True try_detitlecase = True try_uppercase = False can_analyse = False can_tokenise = True can_generate = False can_lemmatise = False can_hyphenate = False can_segment = False can_labelsegment = False can_guess = False can_udpipe = False _verbosity = False _stdpaths = [ '/usr/local/share/hfst/fi/', '/usr/share/hfst/fi/', '/usr/local/share/omorfi/', '/usr/share/omorfi/', './', 'generated/', 'src/generated/', '../src/generated/' ] def __init__(self, verbosity=False): """Construct Omorfi with given verbosity for printouts.""" self._verbosity = verbosity def load_filename(self, path, **include): """Load omorfi automaton from filename and guess its use. A file name should consist of three parts separated by full stop. The second part must be a keyword describing the use of the automaton, first part is parsed as an identifier typically starting with the word omorfi, followed by any extras, such as the tagset for analysis or generation. The named arguments can include a name of automaton type as name, and truth value as value, for types of automata allowed to load. By default, the names `analyse`, `generate` and `segment` are loaded. Names not included are defaulted to False. E.g., `omorfi.load_filename(fn, analyse=True)` will only load file named fn if it can be identified as omorfi analyser. This is best used in conjunction with omorfi.load_from_dir. """ if len(include) == 0: include['analyse'] = True include['generate'] = True include['segment'] = True include['accept'] = True for ttype in [ 'analyse', 'generate', 'accept', 'tokenise', 'lemmatise', 'hyphenate', 'segment', 'labelsegment', 'guesser', 'udpipe' ]: if ttype not in include: include[ttype] = False his = None if self._verbosity: print('Opening file', path) if access(path, F_OK): his = libhfst.HfstInputStream(path) else: # FIXME: should fail if self._verbosity: print('No access to ', path, file=stderr) pass parts = path[path.rfind('/') + 1:path.rfind('.')].split('.') if len(parts) != 2: if self._verbosity: print('not loaded', path) elif not parts[0] == 'omorfi': if self._verbosity: print('not omorfi', path) elif parts[1] == 'analyse' and include['analyse']: if self._verbosity: print('analyser', parts[0]) self.analyser = his.read() self.can_analyse = True self.can_accept = True self.can_lemmatise = True elif parts[1] == 'generate' and include['generate']: if self._verbosity: print('generator', parts[0]) self.generator = his.read() self.can_generate = True elif parts[1] == 'accept' and include['accept']: if self._verbosity: print('acceptor', parts[0]) self.acceptor = his.read() self.can_accept = True elif parts[1] == 'tokenise' and include['tokenise']: if self._verbosity: print('tokeniser', parts[0]) self.tokeniser = his.read() self.can_tokenise = True elif parts[1] == 'lemmatise' and include['lemmatise']: if self._verbosity: print('lemmatiser', parts[0]) self.lemmatiser = his.read() self.can_lemmatise = True elif parts[1] == 'hyphenate' and include['hyphenate']: if self._verbosity: print('hyphenator', parts[0]) self.hyphenator = his.read() self.can_hyphenate = True elif parts[1] == 'segment' and include['segment']: if self._verbosity: print('segmenter', parts[0]) self.segmenter = his.read() self.can_segment = True elif parts[1] == 'guesser' and include['guesser']: if self._verbosity: print('guesser', parts[0]) self.guesser = his.read() self.can_guess = True elif parts[1] == 'labelsegment' and include['labelsegment']: if self._verbosity: print('labelsegmenter', parts[0]) self.labelsegmenter = his.read() self.can_segment = True elif self._verbosity: print('skipped', parts) def _maybe_str2token(self, s): if isinstance(s, str): return (s, "") else: return s def load_from_dir(self, path=None, **include): """Load omorfi automata from given or known locations. If path is given it should point to directory of automata, otherwise standard installation paths are tried. Currently standard linux install paths are all globbed in following order: * /usr/local/share/hfst/fi/*.hfst * /usr/share/hfst/fi/*.hfst * /usr/local/share/omorfi/*.hfst * /usr/share/omorfi/*.hfst * getenv('HOME') + /.hfst/fi/*.hfst * getenv('HOME') + /.omorfi/*.hfst Last two paths require getenv('HOME'). All automata matching glob *.hfst are loaded and stored in part of omorfi class appropriate for their usage. They keyword args can be used to limit loading of automata. The name is analyser type and value is True. """ homepaths = [] if getenv('HOME'): home = getenv('HOME') homepaths = [home + '/.hfst/fi/', home + '/.omorfi/'] loadable = [] if path: if self._verbosity: print('adding', path + '/*.hfst') loadable = glob(path + '/*.hfst') else: for sp in self._stdpaths + homepaths: if self._verbosity: print('adding', sp + '/*.hfst') loadable += glob(sp + '/*.hfst') for filename in loadable: try: self.load_filename(filename, **include) except: print("broken HFST", filename, file=stderr) def load_udpipe(self, filename): if not can_udpipe: print("importing udpipe failed, cannot load udpipe xxx") return self.udpiper = Model.load(filename) # use pipeline for now, ugly but workable self.udpipeline = Pipeline(self.udpiper, 'horizontal', Pipeline.DEFAULT, Pipeline.DEFAULT, 'conllu') self.uderror = ProcessingError() self.can_udpipe = True def _find_retoken_recase(self, token): if self.accept(token): return (token, "ORIGINALCASE") if self.try_lowercase and self.accept(token.lower()): return (token.lower(), "LOWERCASED=" + token) if self.try_uppercase and self.accept(token.upper()): return (token.upper(), "UPPERCASED=" + token) if len(token) > 1: if self.try_titlecase and self.accept(token[0].upper() + token[1:]): return (token[0].upper() + token[1:], "TITLECASED=" + token) if self.try_detitlecase and self.accept(token[0].lower() + token[1:]): return (token[0].lower() + token[1:], "DETITLECASED=" + token) return False def _find_retokens(self, token): retoken = self._find_retoken_recase(token) if retoken: return [retoken] # Word. if token[-1] in fin_punct_trailing: retoken = self._find_retoken_recase(token[:-1]) if retoken: return [(retoken[0], retoken[1] + "|SpaceAfter=No"), (token[-1], "SpaceBefore=No")] # -Word if token[0] in fin_punct_leading: retoken = self._find_retoken_recase(token[1:]) if retoken: return [(token[0], "SpaceAfter=No"), (retoken[0], retoken[1] + "|SpaceBefore=No")] # "Word" if token[0] in fin_punct_leading and token[-1] in fin_punct_trailing: retoken = self._find_retoken_recase(token[1:-1]) if retoken: return [(token[0], "SpaceAfter=No"), (retoken[0], retoken[1] + "|SpaceBefore=No|SpaceAfter=No"), (token[-1], "SpaceBefore=No")] # word." or word", if len(token) > 2 and token[-1] in fin_punct_trailing and token[ -2] in fin_punct_trailing: retoken = self._find_retoken_recase(token[:-2]) if retoken: return [(retoken[0], retoken[1] + "|SpaceAfter=No"), (token[-2], "SpaceBefore=No|SpaceAfter=No"), (token[-1], "SpaceBefore=No")] # word.", if len(token) > 3 and token[-1] in fin_punct_trailing and token[ -2] in fin_punct_trailing and token[-3] in fin_punct_trailing: retoken = self._find_retoken_recase(token[:-3]) if retoken: return [(retoken[0], retoken[1] + "|SpaceAfter=No"), (token[-3], "SpaceBefore=No|SpaceAfter=No"), (token[-2], "SpaceBefore=No|SpaceAfter=No"), (token[-1], "SpaceBefore=No")] # "word." if len(token) > 3 and token[-1] in fin_punct_trailing and token[ -2] in fin_punct_trailing and token[0] in fin_punct_leading: retoken = self._find_retoken_recase(token[1:-2]) if retoken: return [(token[0], "SpaceAfter=No"), (retoken[0], retoken[1] + "|SpaceBefore=No|SpaceAfter=No"), (token[-2], "SpaceBefore=No|SpaceAfter=No"), (token[-1], "SpaceBefore=No")] # "word.", if len(token) > 4 and token[-1] in fin_punct_trailing and token[ -2] in fin_punct_trailing and token[ -3] in fin_punct_trailing and token[0] in fin_punct_leading: retoken = self._find_retoken_recase(token[1:-3]) if retoken: return [(token[0], "SpaceAfter=No"), (retoken[0], retoken[1] + "|SpaceBefore=No|SpaceAfter=No"), (token[-3], "SpaceBefore=No|SpaceAfter=No"), (token[-2], "SpaceBefore=No|SpaceAfter=No"), (token[-1], "SpaceBefore=No")] # ...non-word... pretokens = [] posttokens = [] while len(token) > 1 and token[-1] in fin_punct_trailing: posttokens += [(token[-1], "SpaceBefore=No")] token = token[:-1] while len(token) > 1 and token[0] in fin_punct_leading: pretokens += [(token[0], "SpaceAfter=No")] token = token[1:] return pretokens + \ [(token, "SpaceBefore=No|SpaceAfter=No")] + \ posttokens def _retokenise(self, tokens): retokens = [] for token in tokens: for retoken in self._find_retokens(token): retokens.append(retoken) return retokens def _tokenise(self, line): return None def tokenise(self, line): """Perform tokenisation with loaded tokeniser if any, or `split()`. If tokeniser is available, it is applied to input line and if result is achieved, it is split to tokens according to tokenisation strategy and returned as a list. If no tokeniser are present, or none give results, the line will be tokenised using python's basic string functions. If analyser is present, tokeniser will try harder to get some analyses for each token using hard-coded list of extra splits. """ tokens = None if self.tokeniser: tokens = self._tokenise(line) if not tokens: tokens = self._retokenise(line.split()) return tokens def _analyse_str(self, s): token = (s, "") res = self._analyse_token(token) if len(s) > 2 and s[0].islower() and self.try_titlecase: tcs = s[0].upper() + s[1:] if s != tcs: tctoken = (tcs, 'TitleCased=' + s) tcres = self._analyse_token(tctoken) for r in tcres: r = (r[0] + '[CASECHANGE=TITLECASED]', r[1]) res = res + tcres if len(token) > 2 and token[0].isupper() and self.try_detitlecase: dts = s[0].lower() + s[1:] if dts != s: dttoken = (dts, "DetitleCased=" + s) dtres = self._analyse_token(dttoken) for r in dtres: r = (r[0] + '[CASECHANGE=DETITLECASED]', r[1]) res = res + dtres if not s.isupper() and self.try_uppercase: ups = s.upper() if s != ups: uptoken = (ups, "UpperCased=" + s) upres = self._analyse_token(uptoken) for r in upres: r = (r[0] + '[CASECHANGE=UPPERCASED]', r[1]) res = res + upres if not s.islower() and self.try_lowercase: lows = s.lower() if s != lows: lowtoken = (lows, "LowerCased=" + s) lowres = self._analyse_token(lowtoken) for r in lowres: r = (r[0] + '[CASECHANGE=LOWERCASED]', r[1]) res += lowres return res def _analyse_token(self, token): res = self.analyser.lookup(token[0]) for r in res: r = (r[0] + '[WEIGHT=%f]' % (r[1]), r[1], token[1]) return res def analyse(self, token): """Perform a simple morphological analysis lookup. If try_titlecase does not evaluate to False, the analysis will also be performed with first letter uppercased and rest lowercased. If try_uppercase evaluates to not False, the analysis will also be performed on all uppercase variant. If try_lowercase evaluates to not False, the analysis will also be performed on all lowercase variant. The analyses with case mangling will have an additional element to them identifying the casing. """ anals = None if isinstance(token, str): anals = self._analyse_str(token) else: anals = self._analyse_token(token) if not anals: if isinstance(token, str): anal = ('[WORD_ID=%s][GUESS=UNKNOWN][WEIGHT=inf]' % (token), float('inf'), "Unknown") else: anal = ('[WORD_ID=%s][GUESS=UNKNOWN][WEIGHT=inf]' % (token[0]), float('inf'), "Unknown") anals = [anal] return anals def analyse_sentence(self, s): """Analyse a full sentence with tokenisation and guessing. for details of tokenisation, see @c tokenise(self, s). for details of analysis, see @c analyse(self, token). If further models like udpipe are loaded, may fill in gaps with that. """ tokens = self.tokenise(s) if not tokens: tokens = [(s, "ERRORS=analyse_sentence_1")] analyses = [] for token in tokens: analyses += [self.analyse(token)] if self.can_udpipe: udinput = '\n'.join([token[0] for token in tokens]) uds = self.udpipe(udinput) if len(uds) == len(analyses): for i in range(len(uds)): analsyses[i] += [uds[i]] return None def _guess_str(self, s): token = (s, "") return self._guess_token(token) def _guess_token(self, token): res = self.guesser.lookup(token[0]) for r in res: r = (r[0] + '[GUESS=FSA][WEIGHT=%f]' % (r[1]), r[1], token[1]) return res def _guess_heuristic(self, token): guess = (token[0], float('inf'), token[1]) if token[0][0].isupper() and len(token[0]) > 1: guess = (token[0] + "[UPOS=PROPN][NUM=SG][CASE=NOM][GUESS=HEUR]" + "[WEIGHT=28021984]", 28021984, token[1]) else: guess = (token[0] + "[UPOS=NOUN][NUM=SG][CASE=NOM][GUESS=HEUR]" + "[WEIGHT=28021984]", 28021984, token[1]) return [guess] def guess(self, token): if not self.can_guess: if self.can_udpipe: return self._udpipe(token[0]) else: return self._guess_heuristic(self._maybe_str2token(token)) guesses = None if isinstance(token, str): guesses = self._guess_str(token) else: guesses = self._guess_token(token) return guesses def _lemmatise(self, token): res = self.lemmatiser.lookup(token) return res def lemmatise(self, token): lemmas = None lemmas = self._lemmatise(token) if not lemmas: lemma = (token, float('inf')) lemmas = [lemma] return lemmas def _segment(self, token): res = self.segmenter.lookup(token) return res def segment(self, token): segments = None segments = self._segment(token) if not segments: segment = (token, float('inf')) segments = [segment] return segments def _labelsegment(self, token): res = self.labelsegmenter.lookup(token) return res def labelsegment(self, token): labelsegments = None labelsegments = self._labelsegment(token) if not labelsegments: labelsegment = (token, float('inf')) labelsegments = [labelsegment] return labelsegments def _accept(self, token): res = self.acceptor.lookup(token) return res def accept(self, token): accept = False accepts = None accepts = self._accept(token) if accepts: accept = True return accept def _generate(self, omorstring): res = self.generator.lookup(omorstring) return res def generate(self, omorstring): generated = None if self.can_generate: generated = self._generate(omorstring) if not generated: generated = [(omorstring, float('inf'))] return generated def _udpipe(self, udinput): conllus = self.udpipeline.process(udinput, self.uderror) if self.uderror.occurred(): return None tokens = [] for conllu in conllus.split('\n'): if conllu.startswith('#'): continue elif conllu.strip() == '': continue tokens += [self._conllu2token(conllu)] return tokens def _conllu2token(self, conllu): fields = conllu.split() if len(fields) != 10: print("conllu2token conllu fail", fields) upos = fields[3] wordid = fields[2] surf = fields[1] ufeats = fields[5] misc = fields[9] analysis = '[WORD_ID=%s][UPOS=%s]%s[GUESS=UDPIPE]' % ( wordid, upos, self._ufeats2omor(ufeats)) return (analysis, float('inf'), misc) def _ufeats2omor(self, ufeats): return '[' + ufeats.replace('|', '][') + ']'
class UDPipe: """Wrapper for UDPipe (more pythonic than ufal.udpipe).""" def __init__(self, model): """Create the UDPipe tool object.""" self.model = model path = require_file(model) self.tool = Model.load(path) if not self.tool: raise IOError("Cannot load model from file '%s'" % path) self.error = ProcessingError() self.conllu_reader = ConlluReader() self.tokenizer = self.tool.newTokenizer(Model.DEFAULT) def tag_parse_tree(self, root): """Tag (+lemmatize, fill FEATS) and parse a tree (already tokenized).""" descendants = root.descendants if not descendants: return pipeline = Pipeline(self.tool, 'horizontal', Pipeline.DEFAULT, Pipeline.DEFAULT, 'conllu') in_data = " ".join([n.form for n in descendants]) out_data = pipeline.process(in_data, self.error) if self.error.occurred(): raise IOError("UDPipe error " + self.error.message) self.conllu_reader.files.filehandle = io.StringIO(out_data) parsed_root = self.conllu_reader.read_tree() nodes = [root] + descendants for parsed_node in parsed_root.descendants: node = nodes[parsed_node.ord] node.parent = nodes[parsed_node.parent.ord] for attr in 'upos xpos lemma feats deprel'.split(): setattr(node, attr, getattr(parsed_node, attr)) # TODO: benchmark which solution is the fastest one. E.g. we could also do # for node, parsed_node in zip(root.descendants, parsed_root.descendants): # parsed_node.misc = node.misc # pylint: disable=protected-access #root._children, root._descendants = parsed_root._children, parsed_root._descendants def tokenize_tag_parse_tree(self, root, resegment=False, tag=True, parse=True): """Tokenize, tag (+lemmatize, fill FEATS) and parse the text stored in `root.text`. If resegment=True, the returned list of Udapi trees may contain multiple trees. """ if root.children: raise ValueError( 'Tree already contained nodes before tokenization') # Tokenize and segment the text (segmentation cannot be turned off in older UDPipe versions). self.tokenizer.setText(root.text) is_another = True u_sentences = [] while is_another: u_sentence = Sentence() is_another = self.tokenizer.nextSentence(u_sentence) if is_another: u_sentences.append(u_sentence) # If resegmentation was not required, we need to join the segments. if not resegment and len(u_sentences) > 1: first_sent = u_sentences[0] n_words = first_sent.words.size() - 1 for other_sent in u_sentences[1:]: other_words = other_sent.words.size() - 1 for i in range(1, other_words + 1): u_w = other_sent.words[i] n_words += 1 u_w.id = n_words first_sent.words.append(u_w) u_sentences = [first_sent] # tagging and parsing if tag: for u_sentence in u_sentences: self.tool.tag(u_sentence, Model.DEFAULT) if parse: self.tool.parse(u_sentence, Model.DEFAULT) elif parse: raise ValueError( 'Combination parse=True tag=False is not allowed.') # converting UDPipe nodes to Udapi nodes new_root = root trees = [] for u_sentence in u_sentences: if not new_root: new_root = Root() heads, nodes = [], [new_root] u_words = u_sentence.words for i in range(1, u_words.size()): u_w = u_words[i] node = new_root.create_child( form=u_w.form, lemma=u_w.lemma, upos=u_w.upostag, xpos=u_w.xpostag, feats=u_w.feats, deprel=u_w.deprel, misc=u_w.misc, ) if parse: heads.append(u_w.head) nodes.append(node) if parse: for node in nodes[1:]: head = heads.pop(0) node.parent = nodes[head] trees.append(new_root) new_root = None return trees
# Install this. # !pip install ufal.udpipe # Load this from ufal.udpipe import Model, Pipeline, ProcessingError # Download model: https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-2898 # Load this model = Model.load('hungarian-szeged-ud-2.3-181115.udpipe') pipeline = Pipeline(model, 'tokenize', Pipeline.DEFAULT, Pipeline.DEFAULT, 'conllu') # Full pipeline2 = Pipeline(model, 'vertical', Pipeline.DEFAULT, Pipeline.DEFAULT, 'conllu') # No tokenization, Vertical format as input... pipeline3 = Pipeline( model, 'vertical', Pipeline.DEFAULT, Pipeline.NONE, 'conllu') # No tokenization & No Parsing = Just POS tagging pipeline4 = Pipeline(model, 'conllu', Pipeline.NONE, Pipeline.DEFAULT, 'conllu') # Just Parsing... # Remark: pipeline4 runs even when no POS taging supplied... Maybe it do POS tagging in background... error = ProcessingError() # For catching errors... # Do the processing... processed = pipeline.process('Az alma szép piros volt.', error) assert error.occurred(), 'Error happened check documentation!' # Write the output in CoNLL-U print(processed)
class UDPipe: """Wrapper for UDPipe (more pythonic than ufal.udpipe).""" def __init__(self, model): """Create the UDPipe tool object.""" self.model = model path = require_file(model) self.tool = Model.load(path) if not self.tool: raise IOError("Cannot load model from file '%s'" % path) self.error = ProcessingError() self.conllu_reader = ConlluReader() self.tokenizer = self.tool.newTokenizer(Model.DEFAULT) def tag_parse_tree(self, root): """Tag (+lemmatize, fill FEATS) and parse a tree (already tokenized).""" descendants = root.descendants if not descendants: return pipeline = Pipeline(self.tool, 'horizontal', Pipeline.DEFAULT, Pipeline.DEFAULT, 'conllu') in_data = " ".join([n.form for n in descendants]) out_data = pipeline.process(in_data, self.error) if self.error.occurred(): raise IOError("UDPipe error " + self.error.message) self.conllu_reader.files.filehandle = io.StringIO(out_data) parsed_root = self.conllu_reader.read_tree() nodes = [root] + descendants for parsed_node in parsed_root.descendants: node = nodes[parsed_node.ord] node.parent = nodes[parsed_node.parent.ord] for attr in 'upos xpos lemma feats'.split(): setattr(node, attr, getattr(parsed_node, attr)) # TODO: benchmark which solution is the fastest one. E.g. we could also do # for node, parsed_node in zip(root.descendants, parsed_root.descendants): # parsed_node.misc = node.misc # pylint: disable=protected-access #root._children, root._descendants = parsed_root._children, parsed_root._descendants def tokenize_tag_parse_tree(self, root): """Tokenize, tag (+lemmatize, fill FEATS) and parse the text stored in `root.text`.""" if root.children: raise ValueError('Tree already contained nodes before tokenization') # tokenization (I cannot turn off segmenter, so I need to join the segments) self.tokenizer.setText(root.text) u_sentence = Sentence() is_another = self.tokenizer.nextSentence(u_sentence) u_words = u_sentence.words n_words = u_words.size() - 1 if is_another: u_sent_cont = Sentence() while self.tokenizer.nextSentence(u_sent_cont): n_cont = u_sent_cont.words.size() - 1 for i in range(1, n_cont + 1): u_w = u_sent_cont.words[i] n_words += 1 u_w.id = n_words u_words.append(u_w) # tagging and parsing self.tool.tag(u_sentence, Model.DEFAULT) self.tool.parse(u_sentence, Model.DEFAULT) # converting UDPipe nodes to Udapi nodes heads, nodes = [], [root] for i in range(1, u_words.size()): u_w = u_words[i] node = root.create_child( form=u_w.form, lemma=u_w.lemma, upos=u_w.upostag, xpos=u_w.xpostag, feats=u_w.feats, deprel=u_w.deprel, ) node.misc = u_w.misc heads.append(u_w.head) nodes.append(node) for node in nodes[1:]: head = heads.pop(0) node.parent = nodes[head]
class SRE(object): def __init__(self, udmodel, wordModel): self.__udmodel__ = Model.load(udmodel) if self.__udmodel__ is None: raise ValueError('Unknown UDPipe model') self.__pipeline__ = Pipeline( self.__udmodel__, 'horizontal', Pipeline.DEFAULT, Pipeline.DEFAULT, 'conllu') self.__uderror__ = ProcessingError() self.__srem__ = WordModel(wordModel) self.__result__ = Result() def __checkerThemes__(self, themes): if isinstance(themes, list): for theme in themes: if not isinstance(theme, str): raise TypeError('Themes list must contain only str objects') else: raise TypeError('Themes must be list of str') def __evalTreeSentence__(self, themes, sentenceRoot): templateParsers = [_class.__name__ for _class in AbstractTemplateParser.__subclasses__()] for className in templateParsers: _class = globals()[className] templateParser = _class(themes, self.__srem__) templateValue = _class.getTemplateName() parseSentenceResult = templateParser.parse(sentenceRoot) self.__result__.add({templateValue: parseSentenceResult}) conjRoots = [_ for _ in sentenceRoot.children if _.token['deprel'] == 'conj'] if conjRoots: for conjRoot in conjRoots: self.__evalTreeSentence__(themes, conjRoot) def analyze(self, themes, filename, encoding='utf8'): self.__checkerThemes__(themes) print('Updating model with text... ', end='') self.__srem__.trainFile(filename, encoding=encoding) print('[OK]') print('Parsing sentences... ', end='') with open(filename, 'r', encoding=encoding) as file: for index, line in enumerate(file, start=1): processed_conllu = self.__pipeline__.process(line, self.__uderror__) if self.__uderror__.occurred(): raise RuntimeError('UDPipe error: ' + self.__uderror__.message) sentence_root = parse_tree(processed_conllu)[0] self.__evalTreeSentence__(themes, sentence_root) print('[OK]') def getFullResult(self, outputMode=Output.TO_FILE, filename='result.txt'): print('Getting full result... ') out = Output(outputMode, filename) out.out(self.__result__.getData()) def getResultByType(self, type, concept=None, outputMode=Output.TO_FILE, filename='result.txt'): templateParsers = [_class.__name__ for _class in AbstractTemplateParser.__subclasses__()] availableTypes = [globals()[_].getTemplateName() for _ in templateParsers] if type in availableTypes: print('Getting result by type ' + type + '... ') out = Output(outputMode, filename) out.out(self.__result__.getDataByType(type, concept)) else: raise AttributeError('Unknown semantic type. Available types: ' + ', '.join(availableTypes))