def do_text_preprocessing(self, text, language_shortening):
text = self.majka_lemmatizers[
language_shortening].lemmatization_and_loaded_stop_words_removal_in_array(
textPreprocessing.tokenize_text(text),
self.stop_words[language_shortening])
print(text)
return textPreprocessing.tokenize_text(text)
def find_word_pairs_extended(text1, text2, majka_lemmatizer_text1, majka_lammatizer_text2, stop_words_file1, stop_words_file2, alias_dict):
# text1 = majka_lemmatizer_text1.lemmatization_and_stop_words_removal_not_included(
# textPreprocessing.tokenize_text(text1), stop_words_file1)
arrays = []
word_pairs = dict()
array1 = textPreprocessing.tokenize_text(text1)
arrays.append(textPreprocessing.tokenize_text(text2))
if text2 in alias_dict:
for alias in alias_dict[text2]:
# alias = majka_lammatizer_text2.lemmatization_and_stop_words_removal_not_included(
# textPreprocessing.tokenize_text(alias), stop_words_file2)
arrays.append(textPreprocessing.tokenize_text(alias))
for sentence in arrays:
if len(array1) > len(sentence):
min_length = len(sentence)
else:
min_length = len(array1)
for i in range(0, min_length):
if array1[i] not in word_pairs.keys():
word_pairs[array1[i]] = []
word_pairs[array1[i]].append(sentence[i])
return word_pairs
def find_dictionary_docs(self, preprocessed_text):
found_dictionary_docs = None
for token in textPreprocessing.tokenize_text(preprocessed_text):
help_dictionary_docs = dict()
for score in self.tf_idf_scores:
if token in score:
for doc, score_value in score[token].items():
if doc not in help_dictionary_docs:
help_dictionary_docs[doc] = score_value
else:
help_dictionary_docs[
doc] = help_dictionary_docs[doc] + score_value
if found_dictionary_docs is None:
found_dictionary_docs = help_dictionary_docs.copy()
else:
found_dictionary_docs = {
key: value
for key, value in found_dictionary_docs.items()
if key in help_dictionary_docs.keys()
}
sorted_dictionary = {
key: value
for key, value in sorted(found_dictionary_docs.items(),
key=lambda item: item[1],
reverse=True)
}
return sorted_dictionary
def find_word_pairs(text1, text2, majka_lemmatizer_text1, majka_lammatizer_text2, stop_words_file1, stop_words_file2):
# text1 = majka_lemmatizer_text1.lemmatization_and_stop_words_removal_not_included(
# textPreprocessing.tokenize_text(text1), stop_words_file1)
# text2 = majka_lammatizer_text2.lemmatization_and_stop_words_removal_not_included(
# textPreprocessing.tokenize_text(text2), stop_words_file2)
array1 = textPreprocessing.tokenize_text(text1)
array2 = textPreprocessing.tokenize_text(text2)
word_pairs = dict()
if len(array1) > len(array2):
min_length = len(array2)
else:
min_length = len(array1)
for i in range(0, min_length):
word_pairs[array1[i]] = array2[i]
return word_pairs
def parse_lemmatizer(self, lifo_stack, dictionary_base,
language_array_shortenings):
for character, dictionary in dictionary_base.items():
if character in language_array_shortenings:
# print("lang: " + character + " " + dictionary )
word = ''.join(lifo_stack)
translation = self.translator.translate(
dictionary, src=character, dest=self.base_language)
# print(translation.text + " " + word)
translation_text = self.majka_lemmatizers[character].\
lemmatization_and_loaded_stop_words_removal_in_array(
textPreprocessing.tokenize_text(translation.text), self.stop_words[character]).lower()
word_text = self.majka_lemmatizers[
character].lemmatization_and_loaded_stop_words_removal_in_array(
textPreprocessing.tokenize_text(word),
self.stop_words[character]).lower()
if translation_text == "" or word_text == "":
self.statistics['missed'][
character] = self.statistics['missed'][character] + 1
continue
print(translation_text + " " + word_text)
if translation_text != word_text:
self.statistics['negative'][
character] = self.statistics['negative'][character] + 1
else:
self.statistics['positive'][
character] = self.statistics['positive'][character] + 1
# print(word + " " + dictionary)
else:
lifo_stack.append(character)
self.parse_lemmatizer(lifo_stack, dictionary,
language_array_shortenings)
lifo_stack.pop()
def characters(self, content):
if self.currentData == "title" and self.on_page_to_do and re.search(
r"^MediaWiki:", content) is None:
if content in self.titles_indexes[self.language_shortening]:
self.document_identifier = str(
self.titles_indexes[self.language_shortening][content])
self.lang_document_identifier = self.language_shortening + "-" + self.document_identifier
self.prepared_for_indexing = True
if self.currentData == 'text' and self.prepared_for_indexing:
self.page_content = self.lemmatizer.lemmatization_and_stop_words_removal_not_included(
textPreprocessing.tokenize_text(content),
'stop_words/en_stop_words.json')
# print(self.document_identifier+ "< >"+str(len(content)))
if self.page_content != "":
self.text_content = self.text_content + self.page_content
def index_words(index, check_duplicates, text, document_identifier, dest_lang_shortening):
for word in textPreprocessing.tokenize_text(text):
word = str(word)
if word in index[dest_lang_shortening]:
if document_identifier not in check_duplicates[dest_lang_shortening][word]['doc']:
index[dest_lang_shortening][str(word)]['doc'].append(document_identifier)
index[dest_lang_shortening][word]['df'] = len(index[dest_lang_shortening][word]['doc'])
else:
index[dest_lang_shortening][word] = {}
index[dest_lang_shortening][word]['doc'] = []
index[dest_lang_shortening][word]['df'] = 1
check_duplicates[dest_lang_shortening][word] = {}
check_duplicates[dest_lang_shortening][word]['doc'] = set()
check_duplicates[dest_lang_shortening][word]['doc'].add(document_identifier)
index[dest_lang_shortening][word]['doc'].append(document_identifier)
def index_words_term_freq_doc_freq(index, doc_freq_index, text, document_identifier, dest_lang_shortening):
tokenized_text = textPreprocessing.tokenize_text(text)
doc_freq_index[dest_lang_shortening][document_identifier] = len(tokenized_text)
index[dest_lang_shortening + "_docs"] = index[dest_lang_shortening + "_docs"] + 0
for word in tokenized_text:
word = str(word)
if word in index[dest_lang_shortening]:
if document_identifier not in index[dest_lang_shortening][word]['doc']:
index[dest_lang_shortening][word]['doc'][document_identifier] = 1
index[dest_lang_shortening][word]['df'] = len(index[dest_lang_shortening][word]['doc'])
else:
index[dest_lang_shortening][word]['doc'][document_identifier] = \
index[dest_lang_shortening][word]['doc'][document_identifier] + 1
else:
index[dest_lang_shortening][word] = {}
index[dest_lang_shortening][word]['doc'] = {}
index[dest_lang_shortening][word]['df'] = 1
index[dest_lang_shortening][word]['doc'][document_identifier] = 1
def find_statistics_information_connection(connection_file, base_language_shortening,
language_array_shortenings):
connection_data = load_as_json(connection_file)
max_tokens = 25
number_records = 0
statistic_for_languages = dict()
for language_shortening in language_array_shortenings:
statistic_for_languages[language_shortening] = dict()
for language_shortening in language_array_shortenings:
statistic_for_languages[language_shortening]['records'] = 0
statistic_for_languages[language_shortening]['tokens'] = dict()
for i in range(1, max_tokens):
statistic_for_languages[language_shortening]['tokens'][str(i)] = 0
for record in connection_data[base_language_shortening]:
if 'title' in record and record['title'] != "":
number_records = number_records + 1
for language_shortening in language_array_shortenings:
if language_shortening in record and record[language_shortening] != "":
statistic_for_languages[language_shortening]['records'] = \
statistic_for_languages[language_shortening]['records'] + 1
number_tokens = len(textPreprocessing.tokenize_text(record[language_shortening]))
if number_tokens < max_tokens:
statistic_for_languages[language_shortening]['tokens'][str(number_tokens)] = \
statistic_for_languages[language_shortening]['tokens'][str(number_tokens)] + 1
else:
print('Number tokens for ' + language_shortening + 'exceeds: ' + str(number_tokens))
print("Number records: " + str(number_records))
for language_shortening in language_array_shortenings:
print("Number connections in " + language_shortening + " is: " +
str(statistic_for_languages[language_shortening]['records']))
for i in range(1, max_tokens):
print(str(i) + " token frequencies is: " + str(statistic_for_languages[language_shortening]['tokens']
[str(i)]))
def find_foreign_equivalents(self, text, initial_language_shortening,
dest_language_shortenings):
results = dict()
for language_shortening in dest_language_shortenings:
results[language_shortening] = ""
tokens = textPreprocessing.tokenize_text(text)
#self.do_text_preprocessing(text, initial_language_shortening)
for token in tokens:
result_part_dict = self.character_trees[
initial_language_shortening].find_word_more_languages(
token, dest_language_shortenings)
if result_part_dict is not None:
for language_shortening, result_part in result_part_dict.items(
):
results[language_shortening] = results[
language_shortening] + " " + result_part
for language_shortening, result in results.items():
if result != "":
print(language_shortening + ": " + result)
def find_statistics_information_from_alias_files_extended(alias_files):
statistic_for_language = dict()
max_tokens_base = 10
max_tokens_aliases = 20
max_number_aliases = 40
for alias_file in alias_files:
language_shortening = alias_file.split('_')[0]
alias_data = load_as_json(alias_file)
statistic_for_language[language_shortening] = dict()
statistic_for_language[language_shortening]['records'] = 0
statistic_for_language[language_shortening]['tokens_base'] = dict()
statistic_for_language[language_shortening]['tokens_aliases'] = dict()
statistic_for_language[language_shortening]['number_aliases'] = dict()
for i in range(1, max_tokens_base):
statistic_for_language[language_shortening]['tokens_base'][str(i)] = 0
for i in range(1, max_tokens_aliases):
statistic_for_language[language_shortening]['tokens_aliases'][str(i)] = 0
for i in range(1, max_number_aliases):
statistic_for_language[language_shortening]['number_aliases'][str(i)] = 0
for base_name, array_aliases in alias_data.items():
if base_name != "":
if len(array_aliases) < max_number_aliases:
statistic_for_language[language_shortening]['number_aliases'][str(len(array_aliases))] =\
statistic_for_language[language_shortening]['number_aliases'][str(len(array_aliases))] + 1
else:
print("Number of aliases for " + language_shortening + " exceeds: " + str(len(array_aliases)))
statistic_for_language[language_shortening]['records'] = \
statistic_for_language[language_shortening]['records'] + 1
base_tokens = len(textPreprocessing.tokenize_text(base_name))
if base_tokens < max_tokens_base:
statistic_for_language[language_shortening]['tokens_base'][str(base_tokens)] = \
statistic_for_language[language_shortening]['tokens_base'][str(base_tokens)] + 1
else:
print("Number base tokens for " + language_shortening + " exceeds: " + base_tokens)
for alias in array_aliases:
alias_tokens = len(textPreprocessing.tokenize_text(alias))
if alias_tokens < max_tokens_aliases:
statistic_for_language[language_shortening]['tokens_aliases'][str(alias_tokens)] = \
statistic_for_language[language_shortening]['tokens_aliases'][str(alias_tokens)] + 1
else:
print("Number alias tokens for " + language_shortening + " exceeds: " + alias_tokens)
for language_shortening in statistic_for_language.keys():
print('\n')
print("Number records for " + language_shortening + " containing aliases: " +
str(statistic_for_language[language_shortening]['records']))
for i in range(1, max_tokens_base):
print(str(i) + " base token frequencies are: " +
str(statistic_for_language[language_shortening]['tokens_base'][str(i)]))
for i in range(1, max_tokens_aliases):
print(str(i) + " token of aliases frequencies are: " +
str(statistic_for_language[language_shortening]['tokens_aliases'][str(i)]))
for i in range(1, max_number_aliases):
print(str(i) + " aliases have frequency of: " +
str(statistic_for_language[language_shortening]['number_aliases'][str(i)]))
def parse_lemmatizer_ext_min(self, lifo_stack, dictionary_base,
language_array_shortenings, min_frequency):
for character, dictionary in dictionary_base.items():
if character in language_array_shortenings:
# print("lang: " + character + " " + dictionary )
word = ''.join(lifo_stack)
word_text = self.majka_lemmatizers[
character].lemmatization_and_loaded_stop_words_removal_in_array(
textPreprocessing.tokenize_text(word),
self.stop_words[character]).lower()
if word_text == "":
self.statistics['missed'][
character] = self.statistics['missed'][character] + 1
continue
max = 0
array_of_equal = list()
for foreign_word, frequency in dictionary.items():
if frequency > max:
max = frequency
array_of_equal = list()
array_of_equal.append(foreign_word)
elif frequency == max:
array_of_equal.append(foreign_word)
if frequency < min_frequency:
self.statistics['missed'][
character] = self.statistics['missed'][character] + 1
continue
missed = True
translation_text = ""
if len(array_of_equal) > 1:
self.statistics['negative'][
character] = self.statistics['negative'][character] + 1
continue
translation = self.translator.translate(
array_of_equal[0], src=character, dest=self.base_language)
# print(translation.text + " " + word)
translation_text = self.majka_lemmatizers[character]. \
lemmatization_and_loaded_stop_words_removal_in_array(
textPreprocessing.tokenize_text(translation.text), self.stop_words[character]).lower()
if missed and (translation_text == "" or word_text == ""):
self.statistics['missed'][
character] = self.statistics['missed'][character] + 1
continue
print(translation_text + " " + word_text)
if translation_text != word_text:
self.statistics['negative'][
character] = self.statistics['negative'][character] + 1
else:
self.statistics['positive'][
character] = self.statistics['positive'][character] + 1
# print(word + " " + dictionary)
else:
lifo_stack.append(character)
self.parse_lemmatizer_ext_min(lifo_stack, dictionary,
language_array_shortenings,
min_frequency)
lifo_stack.pop()
def text_preprocessing(self, text, stopwords):
return self.lemmatizer.lemmatization_and_stop_words_removal_in_array_all_majka(
textPreprocessing.tokenize_text(text), stopwords)