def evaluate_model(file_name, date, n_iter, scope, lang, n_eval=5): # corpus = Corpus() corpus.add_files(file_name, encoding='utf8') # preproc = TMPreproc(corpus) dtm_bg = preproc.dtm # var_params = [{'n_topics': k} for k in range(5, int(n_eval*10), n_eval)] # const_params = { 'n_iter': n_iter, 'random_state': 20200713 # to make results reproducible } eval_results = evaluate_topic_models(dtm_bg, varying_parameters=var_params, constant_parameters=const_params, metric=['loglikelihood', 'cao_juan_2009', 'arun_2010']#, #return_models=True ) # eval_results_by_topics = results_by_parameter(eval_results, 'n_topics') # name = "evaluate_model_{}_{}iter_{}eval_{}_{}.png".format(date, n_iter, n_eval, scope, lang) plot_eval_results(eval_results_by_topics, figsize=(8, 6), metric_direction_font_size='x-small', title_fontsize='small', axes_title_fontsize='x-small') plt.tight_layout() plt.savefig('out/'+name) return
def test_corpus_replace_characters_simple(): c = Corpus({'doc1': 'ABC', 'doc2': 'abcDeF'}) c.replace_characters({'a': None, 'C': 'c', 'e': ord('X')}) assert c.docs == { 'doc1': 'ABc', 'doc2': 'bcDXF', } c.replace_characters({ord('A'): None}) assert c.docs == { 'doc1': 'Bc', 'doc2': 'bcDXF', } c.replace_characters(str.maketrans('DXFY', '1234')) assert c.docs == { 'doc1': 'Bc', 'doc2': 'bc123', } c.replace_characters({}) assert c.docs == { 'doc1': 'Bc', 'doc2': 'bc123', }
def test_corpus_unique_characters(texts): all_chars = set(''.join(texts)) c = Corpus({str(i): t for i, t in enumerate(texts)}) res_chars = c.unique_characters assert isinstance(res_chars, set) assert res_chars == all_chars
def test_corpus_from_pickle(): c1 = Corpus({'a': '1', 'b': '22', 'c': '333'}) with tempfile.TemporaryFile(suffix='.pickle') as f: c1.to_pickle(f) f.seek(0) c2 = Corpus.from_pickle(f) assert c1.docs == c2.docs
def test_corpus_pass_tmpreproc(): c = Corpus() c['doc1'] = 'A simple example in simple English.' c['doc2'] = 'It contains only three very simple documents.' c['doc3'] = 'Simply written documents are very brief.' preproc = TMPreproc(c) tok = preproc.tokenize().tokens assert set(tok.keys()) == set(c.keys()) assert len(tok['doc1']) == 7
def test_corpus_apply(texts): c = Corpus({str(i): t for i, t in enumerate(texts)}) c_orig = c.copy() orig_doc_labels = c.doc_labels orig_doc_lengths = c.doc_lengths assert isinstance(c.apply(str.upper), Corpus) assert c.doc_labels == orig_doc_labels assert c.doc_lengths == orig_doc_lengths for dl, dt in c.items(): assert c_orig[dl].upper() == dt
def test_corpus_copy(texts): c1 = Corpus({str(i): t for i, t in enumerate(texts)}) c2 = c1.copy() assert c1.docs is not c2.docs assert c1.docs == c2.docs assert c1.doc_paths is not c2.doc_paths assert c1.doc_paths == c2.doc_paths assert c1.doc_labels == c2.doc_labels assert c1.doc_lengths == c2.doc_lengths assert c1.unique_characters == c2.unique_characters
def test_corpus_add_doc(): c = Corpus() with pytest.raises(ValueError): c.add_doc('', 'x') with pytest.raises(ValueError): c.add_doc(123, 'x') with pytest.raises(ValueError): c.add_doc('d1', None) c.add_doc('d1', 'd1 text') with pytest.raises(ValueError): c.add_doc('d1', 'd1 text') c.add_doc('d2', '') assert set(c.keys()) == {'d1', 'd2'}
def test_corpus_from_files(): doc_path = 'examples/data/gutenberg/kafka_verwandlung.txt' c1 = Corpus.from_files([doc_path]) c2 = Corpus().add_files([doc_path]) assert len(c1.docs) == len(c1.doc_paths) == 1 assert len(c2.docs) == len(c2.doc_paths) == 1 assert c1.docs.keys() == c2.docs.keys() == c1.doc_paths.keys( ) == c2.doc_paths.keys() only_doc_label = next(iter(c1.docs.keys())) assert only_doc_label.endswith('kafka_verwandlung') only_doc = c1.docs[only_doc_label] assert len(only_doc) > 0 assert c1.doc_paths[only_doc_label] == doc_path
def test_corpus_dict_methods(): c = Corpus() assert len(c) == 0 with pytest.raises(KeyError): x = c['x'] with pytest.raises(KeyError): c[1] = 'abc' with pytest.raises(KeyError): c[''] = 'abc' with pytest.raises(ValueError): c['d1'] = None c['d1'] = 'd1 text' assert len(c) == 1 assert 'd1' in c assert set(c.keys()) == {'d1'} assert c['d1'] == 'd1 text' c['d2'] = 'd2 text' assert len(c) == 2 for dl in c: assert dl in {'d1', 'd2'} assert set(c.keys()) == {'d1', 'd2'} for dl, dt in c.items(): assert dl in {'d1', 'd2'} assert c[dl] == dt with pytest.raises(KeyError): del c['d3'] del c['d1'] assert len(c) == 1 assert set(c.keys()) == {'d2'} del c['d2'] assert len(c) == 0 assert set(c.keys()) == set()
def test_corpus_filter_characters(texts): c = Corpus({str(i): t for i, t in enumerate(texts)}) c_orig = c.copy() orig_doc_labels = c.doc_labels orig_doc_lengths = c.doc_lengths orig_uniq_chars = c.unique_characters assert isinstance(c.filter_characters(orig_uniq_chars), Corpus) assert c.doc_labels == orig_doc_labels assert c.doc_lengths == orig_doc_lengths assert c.unique_characters == orig_uniq_chars not_in_corpus_chars = set(string.printable) - orig_uniq_chars if len(not_in_corpus_chars) > 0: c.filter_characters(not_in_corpus_chars) assert c.doc_labels == orig_doc_labels assert c.doc_lengths == {dl: 0 for dl in c.doc_labels} assert c.unique_characters == set() c = c_orig.copy() c.filter_characters(set()) assert c.doc_labels == orig_doc_labels assert c.doc_lengths == {dl: 0 for dl in c.doc_labels} assert c.unique_characters == set() if len(orig_uniq_chars) > 3: c = c_orig.copy() only_chars = set(sample(list(orig_uniq_chars), 3)) c.filter_characters(only_chars) assert c.doc_labels == orig_doc_labels assert c.doc_lengths != orig_doc_lengths assert c.unique_characters == only_chars c = c_orig.copy() only_chars = set(sample(list(orig_uniq_chars), 3)) c.filter_characters(''.join(only_chars)) # as char sequence assert c.doc_labels == orig_doc_labels assert c.doc_lengths != orig_doc_lengths assert c.unique_characters == only_chars
# MAX_DOC_LEN = 5000 N_DOCS_EN = 7 N_DOCS_DE = 3 # given from corpus size all_docs_en = { f_id: nltk.corpus.gutenberg.raw(f_id) for f_id in nltk.corpus.gutenberg.fileids() } smaller_docs_en = [(dl, txt[:min(nchar, MAX_DOC_LEN)]) for dl, txt, nchar in map(lambda x: (x[0], x[1], len(x[1])), all_docs_en.items())] corpus_en = Corpus( dict( sample([(dl, txt) for dl, txt in smaller_docs_en if dl != u'melville-moby_dick.txt'], N_DOCS_EN - 2))) corpus_en.docs['empty_doc'] = '' # additionally test empty document corpus_en.docs[u'melville-moby_dick.txt'] = dict(smaller_docs_en)[ u'melville-moby_dick.txt'] # make sure we always have moby dick #corpus_en = Corpus(dict(smaller_docs_en)) corpus_de = Corpus.from_folder('examples/data/gutenberg', read_size=MAX_DOC_LEN) @pytest.fixture def tmpreproc_en(): return TMPreproc(corpus_en.docs, language='english') @pytest.fixture
def test_empty_corpora(): c1 = Corpus() c2 = Corpus.from_files([]) c3 = Corpus.from_files([]).add_files([]) assert c1.docs == c2.docs == c3.docs == {}
from tmtoolkit.utils import pickle_data from tmtoolkit.topicmod.evaluate import results_by_parameter from tmtoolkit.topicmod.visualize import plot_eval_results logging.basicConfig(level=logging.INFO) tmtoolkit_log = logging.getLogger('tmtoolkit') tmtoolkit_log.setLevel(logging.INFO) tmtoolkit_log.propagate = True print('loading data...') bt18 = pd.read_pickle('data/bt18_sample_1000.pickle') print('loaded %d documents' % len(bt18)) doc_labels = [u'%s_%s' % info for info in zip(bt18.sitzung, bt18.sequence)] print('preprocessing data...') bt18corp = Corpus(dict(zip(doc_labels, bt18.text))) preproc = TMPreproc(bt18corp, language='german') preproc.tokenize().stem().clean_tokens() doc_labels = list(preproc.tokens.keys()) texts = list(preproc.tokens.values()) print('creating gensim corpus...') gnsm_dict = gensim.corpora.Dictionary.from_documents(texts) gnsm_corpus = [gnsm_dict.doc2bow(text) for text in texts] # evaluate topic models with different parameters const_params = dict(update_every=0, passes=10) ks = list(range(10, 140, 10)) + list(range(140, 200, 20)) varying_params = [dict(num_topics=k, alpha=1.0 / k) for k in ks]
logging.basicConfig(level=logging.DEBUG) tmtoolkit_log = logging.getLogger('tmtoolkit') tmtoolkit_log.setLevel(logging.DEBUG) tmtoolkit_log.propagate = True if __name__ == '__main__': # this is necessary for multiprocessing on Windows! if os.path.exists(DTM_PICKLE): print("loading DTM data from pickle file '%s'..." % DTM_PICKLE) dtm, vocab, doc_labels = load_dtm_from_pickle(DTM_PICKLE) else: europarl = nltk.corpus.util.LazyCorpusLoader( 'europarl_raw', nltk.corpus.EuroparlCorpusReader, fileids=FILEIDS) corpus = Corpus( {f: europarl.raw(f_id) for f, f_id in zip(FILES, FILEIDS)}) print("all loaded documents:") for dl, text in corpus.docs.items(): print("%s: %d chars" % (dl, len(text))) print("-----") start_time = time.time() preproc = TMPreproc(corpus.docs, language=u'german') print('tokenizing...') preproc.tokenize() print('POS tagging...') preproc.pos_tag() print('lemmatization...') preproc.lemmatize()
del bt18pickle, bt18zip #%% Generate document labels # format of the document labels: <session_number>_<speech_number> bt18_data['doc_label'] = ['%s_%s' % (str(sitzung).zfill(3), str(seq).zfill(5)) for sitzung, seq in zip(bt18_data.sitzung, bt18_data.sequence)] print('loaded data frame with %d rows:' % bt18_data.shape[0]) print(bt18_data.head()) #%% Generate a Corpus object to preprocess the raw, untokenized text data # we use the column "doc_label" as document labels and "text" as raw text corpus = Corpus(dict(zip(bt18_data.doc_label, bt18_data.text))) print('created corpus') print('document lengths in number of characters:') pprint(corpus.doc_lengths) # we don't need this anymore, remove it to free memory del bt18_data #%% Investigate the set of characters used in the whole corpus # we can see that there are several "strange" characters and unprintable unicode characters which may later cause # trouble print('used set of characters used in the whole corpus:') pprint(corpus.unique_characters)
def test_corpus_doc_lengths(): c = Corpus({'a': '1', 'b': '22', 'c': '333'}) assert isinstance(c.doc_lengths, dict) assert c.doc_lengths == {'a': 1, 'b': 2, 'c': 3}
def test_corpus_doc_labels(): c = Corpus({'a': 'Doc A text', 'b': 'Doc B text', 'c': 'Doc C text'}) assert isinstance(c.doc_labels, list) assert list(c.doc_labels) == c.get_doc_labels(sort=True) == list('abc')
def test_corpus_n_docs(): c = Corpus({'a': 'Doc A text', 'b': 'Doc B text', 'c': 'Doc C text'}) assert c.n_docs == len(c) == 3
from tmtoolkit.corpus import Corpus from tmtoolkit.preprocess import TMPreproc logging.basicConfig(level=logging.INFO) tmtoolkit_log = logging.getLogger('tmtoolkit') tmtoolkit_log.setLevel(logging.INFO) tmtoolkit_log.propagate = True #%% use_paragraphs = len(sys.argv) > 1 and sys.argv[1] == 'paragraphs' #%% corpus = Corpus({ f_id: nltk.corpus.gutenberg.raw(f_id) for f_id in nltk.corpus.gutenberg.fileids() if f_id != 'bible-kjv.txt' }) if use_paragraphs: print('using paragraphs as documents') corpus.split_by_paragraphs() print('%d documents' % len(corpus)) #%% timings = [] timing_labels = [] def add_timing(label):