def evaluate_model(file_name, date, n_iter, scope, lang, n_eval=5):
#
corpus = Corpus()
corpus.add_files(file_name, encoding='utf8')
#
preproc = TMPreproc(corpus)
dtm_bg = preproc.dtm
#
var_params = [{'n_topics': k} for k in range(5, int(n_eval*10), n_eval)]
#
const_params = {
'n_iter': n_iter,
'random_state': 20200713 # to make results reproducible
}
eval_results = evaluate_topic_models(dtm_bg,
varying_parameters=var_params,
constant_parameters=const_params,
metric=['loglikelihood', 'cao_juan_2009', 'arun_2010']#,
#return_models=True
)
#
eval_results_by_topics = results_by_parameter(eval_results, 'n_topics')
#
name = "evaluate_model_{}_{}iter_{}eval_{}_{}.png".format(date, n_iter, n_eval, scope, lang)
plot_eval_results(eval_results_by_topics, figsize=(8, 6), metric_direction_font_size='x-small', title_fontsize='small', axes_title_fontsize='x-small')
plt.tight_layout()
plt.savefig('out/'+name)
return
def test_corpus_pass_tmpreproc():
c = Corpus()
c['doc1'] = 'A simple example in simple English.'
c['doc2'] = 'It contains only three very simple documents.'
c['doc3'] = 'Simply written documents are very brief.'
preproc = TMPreproc(c)
tok = preproc.tokenize().tokens
assert set(tok.keys()) == set(c.keys())
assert len(tok['doc1']) == 7
An example for preprocessing documents in German language and generating a document-term-matrix (DTM).
"""
from pprint import pprint
import pandas as pd
from tmtoolkit.preprocess import TMPreproc
from tmtoolkit.utils import pickle_data
if __name__ == '__main__': # this is necessary for multiprocessing on Windows!
corpus = {
u'doc1': u'Ein einfaches Beispiel in einfachem Deutsch.',
u'doc2': u'Es enthält nur drei sehr einfache Dokumente.',
u'doc3': u'Die Dokumente sind sehr kurz.',
}
preproc = TMPreproc(corpus, language='german')
print('tokenized:')
preproc.tokenize()
pprint(preproc.tokens)
# preproc.stem()
# pprint(preproc.tokens)
print('POS tagged:')
preproc.pos_tag()
pprint(preproc.tokens_with_pos_tags)
print('lemmatized:')
preproc.lemmatize()
pprint(preproc.tokens_with_pos_tags)
print(corpus.docs.keys())
print("-----")
corpus.split_by_paragraphs()
print("documents split into paragraphs")
print(corpus.docs.keys())
print("-----")
print("first 5 paragraphs of Werther:")
for par_num in range(1, 6):
doclabel = u'werther-goethe_werther1-%d' % par_num
print(u"par%d (document label '%s'):" % (par_num, doclabel))
print(corpus.docs[doclabel])
print("-----")
preproc = TMPreproc(corpus.docs, language=u'german')
preproc.tokenize().tokens_to_lowercase()
print("tokenized first 5 paragraphs of Werther:")
for par_num in range(1, 6):
doclabel = u'werther-goethe_werther1-%d' % par_num
print(u"par%d (document label '%s'):" % (par_num, doclabel))
print(preproc.tokens[doclabel])
preproc.generate_ngrams(2, join=False).use_ngrams_as_tokens(join=True)
print("bigrams from first 5 paragraphs of Werther:")
for par_num in range(1, 6):
doclabel = u'werther-goethe_werther1-%d' % par_num
print(u"par%d (document label '%s'):" % (par_num, doclabel))
print(preproc.tokens[doclabel])
#%%
timings = []
timing_labels = []
def add_timing(label):
timings.append(datetime.today())
timing_labels.append(label)
#%%
add_timing('start')
preproc = TMPreproc(corpus, n_max_processes=cpu_count())
add_timing('load')
preproc.tokenize()
add_timing('tokenize')
preproc.expand_compound_tokens()
add_timing('expand_compound_tokens')
preproc.pos_tag()
add_timing('pos_tag')
preproc.lemmatize()
add_timing('lemmatize')
preproc.remove_special_chars_in_tokens()
from tmtoolkit.topicmod.evaluate import results_by_parameter
from tmtoolkit.topicmod.visualize import plot_eval_results
logging.basicConfig(level=logging.INFO)
tmtoolkit_log = logging.getLogger('tmtoolkit')
tmtoolkit_log.setLevel(logging.INFO)
tmtoolkit_log.propagate = True
print('loading data...')
bt18 = pd.read_pickle('data/bt18_sample_1000.pickle')
print('loaded %d documents' % len(bt18))
doc_labels = [u'%s_%s' % info for info in zip(bt18.sitzung, bt18.sequence)]
print('preprocessing data...')
bt18corp = Corpus(dict(zip(doc_labels, bt18.text)))
preproc = TMPreproc(bt18corp, language='german')
preproc.tokenize().stem().clean_tokens()
doc_labels = list(preproc.tokens.keys())
texts = list(preproc.tokens.values())
print('creating gensim corpus...')
gnsm_dict = gensim.corpora.Dictionary.from_documents(texts)
gnsm_corpus = [gnsm_dict.doc2bow(text) for text in texts]
# evaluate topic models with different parameters
const_params = dict(update_every=0, passes=10)
ks = list(range(10, 140, 10)) + list(range(140, 200, 20))
varying_params = [dict(num_topics=k, alpha=1.0 / k) for k in ks]
print('evaluating %d topic models' % len(varying_params))
tmtoolkit_log.setLevel(logging.INFO)
tmtoolkit_log.propagate = True
random.seed(20200320)
#%%
corpus = Corpus.from_builtin_corpus('en-NewsArticles').sample(1000)
print('%d documents' % len(corpus))
#%%
add_timing('start')
preproc = TMPreproc(corpus, language='en', n_max_processes=cpu_count())
add_timing('load and tokenize')
preproc.expand_compound_tokens()
add_timing('expand_compound_tokens')
preproc.pos_tag()
add_timing('pos_tag')
preproc.lemmatize()
add_timing('lemmatize')
preproc_copy = preproc.copy()
preproc_copy.shutdown_workers()
del preproc_copy
add_timing('copy')
An example for preprocessing documents in German language and generating a document-term-matrix (DTM).
"""
from pprint import pprint
from tmtoolkit.preprocess import TMPreproc
import pandas as pd
if __name__ == '__main__': # this is necessary for multiprocessing on Windows!
corpus = {
'doc1': u'A simple example in simple English.',
'doc2': u'It contains only three very simple documents.',
'doc3': u'Simply written documents are very brief.',
}
preproc = TMPreproc(corpus, language='english')
print('input corpus:')
pprint(corpus)
print('running preprocessing pipeline...')
preproc.tokenize().pos_tag().lemmatize().tokens_to_lowercase().clean_tokens()
print('final tokens:')
pprint(preproc.tokens)
print('DTM:')
doc_labels, vocab, dtm = preproc.get_dtm()
# using pandas just for a nice tabular output
print(pd.DataFrame(dtm.todense(), columns=vocab, index=doc_labels))
def tmpreproc_de():
return TMPreproc(corpus_de.docs, language='german')
def tmpreproc_en():
return TMPreproc(corpus_en.docs, language='english')
from tmtoolkit.corpus import Corpus
from tmtoolkit.preprocess import TMPreproc
from examples._benchmarktools import add_timing, print_timings
logging.basicConfig(level=logging.INFO)
tmtoolkit_log = logging.getLogger('tmtoolkit')
tmtoolkit_log.setLevel(logging.INFO)
tmtoolkit_log.propagate = True
#%%
corpus = Corpus.from_builtin_corpus('en-NewsArticles')
print('%d documents' % len(corpus))
#%%
add_timing('start')
preproc = TMPreproc(corpus, language='en', n_max_processes=4)
add_timing('load and tokenize')
preproc.pos_tag()
add_timing('pos_tag')
preproc.lemmatize()
add_timing('lemmatize')
print_timings()