def frequent_words(corpus, normalize, include_pos, n_top=100):
if include_pos is None or include_pos == ('', ):
include_pos = []
return [
x[0] for x in textacy_utility.get_most_frequent_words(
corpus, n_top, normalize=normalize, include_pos=include_pos)
] + ['_gpe_']
def pos_change_handler(*args):
with gui.output:
gui.compute.disabled = True
selected = set(gui.stop_words.value)
frequent_words = [
x[0] for x in textacy_utility.get_most_frequent_words(
corpus,
100,
normalize=gui.normalize.value,
include_pos=gui.include_pos.value)
]
gui.stop_words.options = frequent_words
selected = selected & set(gui.stop_words.options)
gui.stop_words.value = list(selected)
gui.compute.disabled = False
def pos_change_handler(*args):
with self.model_widgets.output:
self.model_widgets.compute.disabled = True
selected = set(self.corpus_widgets.stop_words.value)
frequent_words = [
x[0] for x in textacy_utility.get_most_frequent_words(
corpus,
100,
normalize=self.corpus_widgets.normalize.value,
include_pos=self.corpus_widgets.include_pos.value)
]
self.corpus_widgets.stop_words.options = frequent_words
selected = selected & set(
self.corpus_widgets.stop_words.options)
self.corpus_widgets.stop_words.value = list(selected)
self.model_widgets.compute.disabled = False
def word_frequency_gui(wti_index, corpus):
treaty_time_groups = wti_index.get_treaty_time_groupings()
lw = lambda w: widgets.Layout(width=w)
include_pos_tags = ['ADJ', 'VERB', 'ADV', 'NOUN', 'PROPN']
weighting_options = {'Count': 'count', 'Frequency': 'freq'}
normalize_options = {'': False, 'Lemma': 'lemma', 'Lower': 'lower'}
pos_options = include_pos_tags
default_include_pos = ['NOUN', 'PROPN']
frequent_words = [
x[0].lower() for x in textacy_utility.get_most_frequent_words(
corpus, 100, include_pos=default_include_pos)
]
group_by_options = {
treaty_time_groups[k]['title']: k
for k in treaty_time_groups
}
output_type_options = [
('Sample', 'table'),
('Rank', 'rank'),
('Excel', 'excel'),
]
ngrams_options = {'-': None, '1': [1], '1,2': [1, 2], '1,2,3': [1, 2, 3]}
party_preset_options = wti_index.get_party_preset_options()
parties_options = [
x for x in wti_index.get_countries_list() if x != 'ALL OTHER'
]
gui = types.SimpleNamespace(
progress=widgets.IntProgress(value=0,
min=0,
max=5,
step=1,
description='',
layout=lw('98%')),
parties=widgets.SelectMultiple(description='Parties',
options=parties_options,
value=[],
rows=7,
layout=lw('200px')),
party_preset=widgets_config.dropdown('Presets',
party_preset_options,
None,
layout=lw('200px')),
ngrams=widgets.Dropdown(description='n-grams',
options=ngrams_options,
value=None,
layout=lw('200px')),
min_word=widgets.Dropdown(description='Min length',
options=[1, 2, 3, 4],
value=1,
layout=lw('200px')),
normalize=widgets.Dropdown(description='Normalize',
options=normalize_options,
value='lemma',
layout=lw('200px')),
weighting=widgets.Dropdown(description='Weighting',
options=weighting_options,
value='freq',
layout=lw('200px')),
include_pos=widgets.SelectMultiple(description='POS',
options=pos_options,
value=default_include_pos,
rows=7,
layout=lw('150px')),
stop_words=widgets.SelectMultiple(description='STOP',
options=frequent_words,
value=list([]),
rows=7,
layout=lw('200px')),
group_by_column=widgets.Dropdown(description='Group by',
value='signed_year',
options=group_by_options,
layout=lw('200px')),
output_type=widgets.Dropdown(description='Output',
value='rank',
options=output_type_options,
layout=lw('200px')),
compute=widgets.Button(description='Compute',
button_style='Success',
layout=lw('120px')),
display_score=widgets.ToggleButton(description='Display score',
icon='check',
value=False,
layout=lw('120px')),
output=widgets.Output(layout={'border': '1px solid black'}),
file_suffix=widgets.Text(
value='',
placeholder='(optional id)',
description='ID:',
disabled=False,
layout=lw('200px'),
tooltip="Optional plain text id that will be added to filename."))
boxes = widgets.VBox([
gui.progress,
widgets.HBox([
widgets.VBox([
gui.normalize,
gui.ngrams,
gui.weighting,
gui.group_by_column,
gui.output_type,
]),
widgets.VBox([
gui.parties,
gui.party_preset,
]),
gui.include_pos,
widgets.VBox([
gui.stop_words,
gui.file_suffix,
]),
widgets.VBox([
gui.display_score,
gui.compute,
],
layout=widgets.Layout(align_items='flex-end')),
]), gui.output
])
display(boxes)
def on_party_preset_change(change): # pylint: disable=W0613
if gui.party_preset.value is None:
return
gui.parties.value = gui.parties.options if 'ALL' in gui.party_preset.value else gui.party_preset.value
gui.party_preset.observe(on_party_preset_change, names='value')
def pos_change_handler(*args):
with gui.output:
gui.compute.disabled = True
selected = set(gui.stop_words.value)
frequent_words = [
x[0].lower() for x in textacy_utility.get_most_frequent_words(
corpus,
100,
normalize=gui.normalize.value,
include_pos=gui.include_pos.value,
#weighting=gui.weighting.value
)
]
gui.stop_words.options = frequent_words
selected = selected & set(gui.stop_words.options)
gui.stop_words.value = list(selected)
gui.compute.disabled = False
gui.include_pos.observe(pos_change_handler, 'value')
gui.weighting.observe(pos_change_handler, 'value')
def compute_callback_handler(*_args):
gui.output.clear_output()
with gui.output:
try:
gui.compute.disabled = True
df_counts = compute_list_of_most_frequent_words(
corpus=corpus,
gui=gui,
target=gui.normalize.value,
treaty_time_groups=treaty_time_groups,
group_by_column=gui.group_by_column.value,
parties=gui.parties.value,
weighting=gui.weighting.value,
include_pos=gui.include_pos.value,
stop_words=set(gui.stop_words.value),
display_score=gui.display_score.value)
display_list_of_most_frequent_words(gui, df_counts)
except Exception as ex:
logger.error(ex)
raise
finally:
gui.progress.value = 0
gui.compute.disabled = False
gui.compute.on_click(compute_callback_handler)
return gui
def display_topic_model_gui(data_folder, state, corpus, **opts):
def spinner_widget(filename="images/spinner-02.gif", width=40, height=40):
with open(filename, "rb") as image_file:
image = image_file.read()
return widgets.Image(value=image,
format='gif',
width=width,
height=height,
layout={'visibility': 'hidden'})
pos_options = [
x for x in opts['tagset'].POS.unique() if x not in [
'PUNCT', '', 'DET', 'X', 'SPACE', 'PART', 'CONJ', 'SYM', 'INTJ',
'PRON'
]
]
engine_options = [
('MALLET LDA', 'gensim_mallet-lda'),
('gensim LDA', 'gensim_lda'),
('gensim LSI', 'gensim_lsi'),
('gensim HDP', 'gensim_hdp'),
('gensim DTM', 'gensim_dtm'),
('scikit LDA', 'sklearn_lda'),
('scikit NMF', 'sklearn_nmf'),
('scikit LSA', 'sklearn_lsa'),
('STTM LDA', 'gensim_sttm-lda'),
('STTM BTM', 'gensim_sttm-btm'),
('STTM PTM', 'gensim_sttm-ptm'),
('STTM SATM', 'gensim_sttm-satm'),
('STTM DMM', 'gensim_sttm-dmm'),
('STTM WATM', 'gensim_sttm-watm'),
]
normalize_options = {
'None': False,
'Use lemma': 'lemma',
'Lowercase': 'lower'
}
ngrams_options = {'1': [1], '1, 2': [1, 2], '1,2,3': [1, 2, 3]}
default_include_pos = ['NOUN', 'PROPN']
frequent_words = [
x[0] for x in textacy_utility.get_most_frequent_words(
corpus, 100, normalize='lemma', include_pos=default_include_pos)
] + ['_gpe_']
named_entities_disabled = len(corpus) == 0 or len(
corpus[0].spacy_doc.ents) == 0
gui = types.SimpleNamespace(
progress=widgets.IntProgress(value=0,
min=0,
max=5,
step=1,
description='',
layout=widgets.Layout(width='90%')),
n_topics=widgets.IntSlider(description='#topics',
min=2,
max=100,
value=20,
step=1,
layout=widgets.Layout(width='240px')),
min_freq=widgets.IntSlider(description='Min word freq',
min=0,
max=10,
value=2,
step=1,
layout=widgets.Layout(width='240px')),
max_doc_freq=widgets.IntSlider(description='Min doc %',
min=75,
max=100,
value=100,
step=1,
layout=widgets.Layout(width='240px')),
max_iter=widgets.IntSlider(description='Max iterations',
min=100,
max=6000,
value=2000,
step=10,
layout=widgets.Layout(width='240px')),
ngrams=widgets.Dropdown(description='n-grams',
options=ngrams_options,
value=[1],
layout=widgets.Layout(width='200px')),
normalize=widgets.Dropdown(description='Normalize',
options=normalize_options,
value='lemma',
layout=widgets.Layout(width='200px')),
filter_stops=widgets.ToggleButton(value=True,
description='Remove stopword',
tooltip='Filter out stopwords',
icon='check'),
named_entities=widgets.ToggleButton(value=False,
description='Merge entities',
tooltip='Merge entities',
icon='check',
disabled=named_entities_disabled),
apply_idf=widgets.ToggleButton(
value=False,
description='Apply IDF',
tooltip='Apply IDF (skikit-learn) or TF-IDF (gensim)',
icon='check'),
mask_gpe=widgets.ToggleButton(value=False,
description='Mask GPE',
tooltip='Mask GPE',
icon='check'),
include_pos=widgets.SelectMultiple(
description='POS',
options=pos_options,
value=default_include_pos,
rows=7,
layout=widgets.Layout(width='160px')),
stop_words=widgets.SelectMultiple(
description='STOP',
options=frequent_words,
value=list([]),
rows=7,
layout=widgets.Layout(width='220px')),
method=widgets.Dropdown(description='Engine',
options=engine_options,
value='gensim_lda',
layout=widgets.Layout(width='200px')),
compute=widgets.Button(description='Compute',
button_style='Success',
layout=widgets.Layout(width='115px',
background_color='blue')),
show_trace=widgets.Checkbox(
value=False, description='Show trace',
disabled=False), #layout=widgets.Layout(width='80px')),
boxes=None,
output=widgets.Output(
layout={'border': '1px solid black'}), # , 'height': '400px'
spinner=spinner_widget())
gui.boxes = widgets.VBox([
gui.progress,
widgets.HBox([
widgets.VBox(
[gui.n_topics, gui.min_freq, gui.max_doc_freq, gui.max_iter]),
widgets.VBox([
gui.filter_stops, gui.named_entities, gui.apply_idf,
gui.mask_gpe
]), gui.include_pos, gui.stop_words,
widgets.VBox([
gui.normalize, gui.ngrams, gui.method,
widgets.HBox([gui.spinner, gui.compute],
layout=widgets.Layout(align_items='flex-end'))
],
layout=widgets.Layout(align_items='flex-end')),
gui.show_trace
]),
widgets.VBox([gui.output]),
])
def tick(x=None):
gui.progress.value = gui.progress.value + 1 if x is None else x
def buzy(is_buzy):
gui.compute.disabled = is_buzy
gui.spinner.layout.visibility = 'visible' if is_buzy else 'hidden'
def compute_topic_model_handler(*args):
gui.output.clear_output()
buzy(True)
with gui.output:
try:
state.data = compute_topic_model(data_folder,
corpus,
gui.method.value,
gui.n_topics.value,
gui,
tick=tick,
**opts)
topics = topic_model_utility.get_topics_unstacked(
state.topic_model,
n_tokens=100,
id2term=state.id2term,
topic_ids=state.relevant_topics)
display(topics)
except Exception as ex:
logger.error(ex)
state.data = None
raise
finally:
buzy(False)
gui.compute.on_click(compute_topic_model_handler)
def pos_change_handler(*args):
with gui.output:
gui.compute.disabled = True
selected = set(gui.stop_words.value)
frequent_words = [
x[0] for x in textacy_utility.get_most_frequent_words(
corpus,
100,
normalize=gui.normalize.value,
include_pos=gui.include_pos.value)
]
gui.stop_words.options = frequent_words
selected = selected & set(gui.stop_words.options)
gui.stop_words.value = list(selected)
gui.compute.disabled = False
gui.include_pos.observe(pos_change_handler, 'value')
def method_change_handler(*args):
with gui.output:
gui.compute.disabled = True
method = gui.method.value
gui.apply_idf.disabled = False
gui.apply_idf.description = 'Apply TF-IDF' if method.startswith(
'gensim') else 'Apply IDF'
gui.ngrams.disabled = False
if 'MALLET' in method:
gui.ngrams.value = [1]
gui.ngrams.disabled = True
gui.apply_idf.description = 'TF-IDF N/A'
gui.apply_idf.disabled = True
gui.n_topics.disabled = False
if 'HDP' in method:
gui.n_topics.value = gui.n_topics.max
gui.n_topics.disabled = True
gui.compute.disabled = False
gui.method.observe(method_change_handler, 'value')
method_change_handler()
display(gui.boxes)
return gui
