def main() -> None: """Точка входа в приложение.""" corpus_root = Path('corpus') # Настроим логирование результатов global _logger setup_logger(_logger, corpus_root / 'collocations.log') # Загрузим стоп-слова nltk.download('stopwords', '.env/share/nltk_data') stop_words = set(stopwords.words('russian')) # Импортируем корпус tags_root = corpus_root / 'pos_tagging' reader = ConllCorpusReader( str(tags_root), [f.name for f in tags_root.glob('*.tags')], columntypes=['words', 'ignore', 'ignore', 'ignore', 'pos'], separator='\t') _logger.info('Документов: %d', len(reader.fileids())) _logger.info('Токенов в первом документе (%s): %d', reader.fileids()[0], len(reader.words(reader.fileids()[0]))) _logger.info('Загружаем предложения') sentences = reader.sents() # Строим таблицы сопряжённости для всех слов в корпусе _logger.info('Считаем таблицу сопряжённости по всем словам') bigram_finder = BigramCollocationFinder.from_documents( [w.lower() for w in sent] for sent in tqdm(sentences)) _logger.info('Всего биграм: %d', bigram_finder.N) print_samples(bigram_finder) # А теперь отфильтруем по частоте и удалим пунктуацию, стоп-слова _logger.info( 'Отфильтруем пунктуацию, стоп-слова и установим предел по частоте') bigram_finder.apply_freq_filter(5) bigram_finder.apply_word_filter(lambda w: len(w) < 3 or w in stop_words) _logger.info('Всего биграм: %d', bigram_finder.N) print_samples(bigram_finder)
def sents(self, fileids=None, categories=None): return ConllCorpusReader.sents(self, self._resolve(fileids, categories))
(prob, state) = max((V[t-1][y0] + trans_p.logprob((y0, y)), y0) for y0 in states) V[t][y] = prob newpath[y] = path[state] + [y] # Don't need to remember the old paths path = newpath # Return the most likely sequence over the given time frame n = len(obs) - 1 (prob, state) = max((V[n][y], y) for y in states) return path[state] Vit1 = [] Vit2 = [] Vit3 = [] Vit4 = [] for sent in conllreader.sents(): Vit1.append(zip(sent, viterbi(sent, states, Train.A0j, Train.Aij, Train.Biw))) Vit2.append(zip(sent, viterbi(sent, states, Train.A0jLap, Train.AijLap, Train.BiwLap))) Vit3.append(zip(sent, viterbi(sent, states, Train.A0jGT, Train.AijGT, Train.BiwGT))) Vit4.append(zip(sent, viterbi(sent, states, Train.A0jMLE, Train.AijMLE, Train.BiwMLE))) # function for writing tagged corpora to files in CoNLL format def write_conll(filename, tagged_corpus): with open(filename, 'w') as out_file: for tagged_sent in tagged_corpus: tagged_words = ('\t'.join(w_t) for w_t in tagged_sent) out_file.write('\n'.join(tagged_words) + '\n\n') write_conll('unsmoothed.tt', Vit1) write_conll('laplace.tt', Vit2)
V[t][y] = prob newpath[y] = path[state] + [y] # Don't need to remember the old paths path = newpath # Return the most likely sequence over the given time frame n = len(obs) - 1 (prob, state) = max((V[n][y], y) for y in states) return path[state] Vit1 = [] Vit2 = [] Vit3 = [] Vit4 = [] for sent in conllreader.sents(): Vit1.append( zip(sent, viterbi(sent, states, Train.A0j, Train.Aij, Train.Biw))) Vit2.append( zip(sent, viterbi(sent, states, Train.A0jLap, Train.AijLap, Train.BiwLap))) Vit3.append( zip(sent, viterbi(sent, states, Train.A0jGT, Train.AijGT, Train.BiwGT))) Vit4.append( zip(sent, viterbi(sent, states, Train.A0jMLE, Train.AijMLE, Train.BiwMLE))) # function for writing tagged corpora to files in CoNLL format def write_conll(filename, tagged_corpus):