def process_data(self):
texts = []
settings = self.main.settings_custom['keywords']
ref_file = self.main.wordless_files.find_file_by_name(
settings['generation_settings']['ref_file'], selected_only=True)
files = [
file for file in self.main.wordless_files.get_selected_files()
if file != ref_file
]
# Frequency
for i, file in enumerate([ref_file] + files):
text = wordless_text.Wordless_Text(self.main, file)
tokens = wordless_token_processing.wordless_process_tokens_wordlist(
text, token_settings=settings['token_settings'])
self.keywords_freq_files.append(collections.Counter(tokens))
if i > 0:
texts.append(text)
else:
tokens_ref = text.tokens
len_tokens_ref = len(tokens_ref)
# Total
if len(files) > 1:
text_total = wordless_text.Wordless_Text_Blank()
text_total.tokens = [
token for text in texts for token in text.tokens
]
texts.append(text_total)
self.keywords_freq_files.append(
sum(self.keywords_freq_files, collections.Counter()))
self.keywords_freq_files[0] = {
token: freq
for token, freq in self.keywords_freq_files[0].items()
if token in text_total.tokens
}
else:
self.keywords_freq_files[0] = {
token: freq
for token, freq in self.keywords_freq_files[0].items()
if token in self.keywords_freq_files[1]
}
self.progress_updated.emit(self.tr('Processing data ...'))
# Keyness
text_test_significance = settings['generation_settings'][
'test_significance']
text_measure_effect_size = settings['generation_settings'][
'measure_effect_size']
test_significance = self.main.settings_global['tests_significance'][
'keywords'][text_test_significance]['func']
measure_effect_size = self.main.settings_global[
'measures_effect_size']['keywords'][text_measure_effect_size][
'func']
keywords_freq_file_observed = self.keywords_freq_files[-1]
keywords_freq_file_ref = self.keywords_freq_files[0]
for text in texts:
keywords_stats_file = {}
tokens_observed = text.tokens
len_tokens_observed = len(tokens_observed)
if text_test_significance in [
self.tr('Student\'s t-test (Two-sample)'),
self.tr('Mann-Whitney U Test')
]:
# Test Statistic, p-value & Bayes Factor
if text_test_significance == self.tr(
'Student\'s t-test (Two-sample)'):
number_sections = self.main.settings_custom['measures'][
'statistical_significance'][
'students_t_test_2_sample']['number_sections']
use_data = self.main.settings_custom['measures'][
'statistical_significance'][
'students_t_test_2_sample']['use_data']
elif text_test_significance == self.tr('Mann-Whitney U Test'):
number_sections = self.main.settings_custom['measures'][
'statistical_significance']['mann_whitney_u_test'][
'number_sections']
use_data = self.main.settings_custom['measures'][
'statistical_significance']['mann_whitney_u_test'][
'use_data']
sections_observed = wordless_text_utils.to_sections(
tokens_observed, number_sections)
sections_ref = wordless_text_utils.to_sections(
tokens_ref, number_sections)
sections_freq_observed = [
collections.Counter(section)
for section in sections_observed
]
sections_freq_ref = [
collections.Counter(section)
for section in sections_observed
]
len_sections_observed = [
len(section) for section in sections_observed
]
len_sections_ref = [len(section) for section in sections_ref]
if use_data == self.tr('Absolute Frequency'):
for token in keywords_freq_file_observed:
counts_observed = [
section_freq.get(token, 0)
for section_freq in sections_freq_observed
]
counts_ref = [
section_freq.get(token, 0)
for section_freq in sections_freq_ref
]
keywords_stats_file[token] = test_significance(
self.main, counts_observed, counts_ref)
elif use_data == self.tr('Relative Frequency'):
for token in keywords_freq_file_observed:
counts_observed = [
section_freq.get(token, 0) /
len_sections_observed[i]
for i, section_freq in enumerate(
sections_freq_observed)
]
counts_ref = [
section_freq.get(token, 0) / len_sections_ref[i]
for i, section_freq in enumerate(sections_freq_ref)
]
keywords_stats_file[token] = test_significance(
self.main, counts_observed, counts_ref)
# Effect Size
for token in keywords_freq_file_observed:
c11 = keywords_freq_file_observed.get(token, 0)
c12 = keywords_freq_file_ref.get(token, 0)
c21 = len_tokens_observed - c11
c22 = len_tokens_ref - c12
keywords_stats_file[token].append(
measure_effect_size(self.main, c11, c12, c21, c22))
else:
for token in keywords_freq_file_observed:
c11 = keywords_freq_file_observed.get(token, 0)
c12 = keywords_freq_file_ref.get(token, 0)
c21 = len_tokens_observed - c11
c22 = len_tokens_ref - c12
# Test Statistic, p-value & Bayes Factor
keywords_stats_file[token] = test_significance(
self.main, c11, c12, c21, c22)
# Effect Size
keywords_stats_file[token].append(
measure_effect_size(self.main, c11, c12, c21, c22))
self.keywords_stats_files.append(keywords_stats_file)
if len(files) == 1:
self.keywords_freq_files.append(self.keywords_freq_files[1])
self.keywords_stats_files *= 2
def process_data(self):
texts = []
settings = self.main.settings_custom['wordlist']
files = self.main.wordless_files.get_selected_files()
# Frequency
for file in files:
text = wordless_text.Wordless_Text(self.main, file)
tokens = wordless_token_processing.wordless_process_tokens_wordlist(
text, token_settings=settings['token_settings'])
texts.append(text)
self.tokens_freq_files.append(collections.Counter(tokens))
# Total
if len(files) > 1:
text_total = wordless_text.Wordless_Text_Blank()
text_total.tokens = [
token for text in texts for token in text.tokens
]
texts.append(text_total)
self.tokens_freq_files.append(
sum(self.tokens_freq_files, collections.Counter()))
self.progress_updated.emit(self.tr('Processing data ...'))
# Dispersion & Adjusted Frequency
text_measure_dispersion = settings['generation_settings'][
'measure_dispersion']
text_measure_adjusted_freq = settings['generation_settings'][
'measure_adjusted_freq']
measure_dispersion = self.main.settings_global['measures_dispersion'][
text_measure_dispersion]['func']
measure_adjusted_freq = self.main.settings_global[
'measures_adjusted_freq'][text_measure_adjusted_freq]['func']
tokens_total = self.tokens_freq_files[-1].keys()
for text in texts:
tokens_stats_file = {}
# Dispersion
number_sections = self.main.settings_custom['measures'][
'dispersion']['general']['number_sections']
sections_freq = [
collections.Counter(section)
for section in wordless_text_utils.to_sections(
text.tokens, number_sections)
]
for token in tokens_total:
counts = [
section_freq[token] for section_freq in sections_freq
]
tokens_stats_file[token] = [measure_dispersion(counts)]
# Adjusted Frequency
if not self.main.settings_custom['measures']['adjusted_freq'][
'general']['use_same_settings_dispersion']:
number_sections = self.main.settings_custom['measures'][
'adjusted_freq']['general']['number_sections']
sections_freq = [
collections.Counter(section)
for section in wordless_text_utils.to_sections(
text.tokens, number_sections)
]
for token in tokens_total:
counts = [
section_freq[token] for section_freq in sections_freq
]
tokens_stats_file[token].append(measure_adjusted_freq(counts))
self.tokens_stats_files.append(tokens_stats_file)
if len(files) == 1:
self.tokens_freq_files *= 2
self.tokens_stats_files *= 2
def test_to_sections():
tokens = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
token_sections = wordless_text_utils.to_sections(tokens, num_sections=5)
assert token_sections == [[1, 2, 3], [4, 5, 6], [7, 8], [9, 10], [11, 12]]