def preprocessing():


	# Get filenames for the thesaurus, the latin text, and greek text
	thesaurus_filename, Latin_filename, Greek_filename = xls.get_whole_text_comparison_Args()

	# Read the thesaurus CSV file into a dicitonary object for easy access
	transDict = ths.build_thesaurus(thesaurus_filename)

	# Read both the latin text and greek text into a dictionary for easy access
	Latin_word_num, Latin_search_dict, Latin_text = xls.build_search_dictionary(Latin_filename, "Latin", True)

	Greek_word_num, Greek_search_dict, Greek_text  = xls.build_search_dictionary = (Greek_filename, "Greek", True)

	return Latin_word_num, Latin_search_dict, Latin_text, Greek_word_num, Greek_search_dict, Greek_text
def preprocessing():
	latin_cltk_importer = CorpusImporter('latin')
	latin_cltk_importer.import_corpus('latin_models_cltk')

	greek_cltk_importer = CorpusImporter('greek')
	greek_cltk_importer.import_corpus('greek_models_cltk')

	# Get filenames for the thesaurus, the latin text, and greek text
	thesaurus_filename, Greek_filename = xls.get_search_by_phrase_Args()

	# Read the thesaurus CSV file into a dicitonary object for easy access
	transDict = ths.build_thesaurus(thesaurus_filename)
	
	# Read greek text into a dictionary for easy access
	Greek_word_num, Greek_search_dict, Greek_text = xls.build_search_dictionary(Greek_filename, "Greek", True)

	return transDict, Greek_word_num, Greek_search_dict, Greek_text
def test_build_search_dict(curr_test, filename, words_in_file , language, lemmatized_version = False):

	word_num, search_dict, indexed_corpus = xls.build_search_dictionary(filename, language ,lemmatized_version)

	if not (word_num == words_in_file):
		curr_test.passed = False
		curr_test.errors.append("Wrong number of words added (only " + str(word_num) + " out of " + str(words_in_file) + " words added) ")
	
	test_file = open(filename,'r') 
	
	i = 0

	curr_word = ""

	while 1:
		char = test_file.read(1)

		if not (re.sub("[\p{Z}\t\r\n\v\f\s]", "", char) == ""):
			curr_word += char
		else:
			curr_word = normalize_word(curr_word)
			if not (curr_word == ""):
				if curr_word in search_dict:

					if not( i in search_dict[curr_word]):
						curr_test.passed = False	
									
						error_message = curr_word + " did not have the proper index in the search dict"
						error_message += "\n\t\t\t word number: " +str(i) + " Indices: " + str(search_dict[curr_word]) 
						curr_test.errors.append(error_message) 
				else:
					curr_test.passed = False
					error_message = curr_word + " was not found in the search dict"
						
					curr_test.errors.append(error_message)
					
				i += 1
			curr_word = ""

		if char == None or char == "":
			break

	test_file.close() 

	return curr_test
	def process_corpus(self, filename, language, make_IndexedText=True, use_lemmatized_text=False):
		self.corpus_ready = False
		self.word_num, self.search_dict, self.indexed_corpus = xls.build_search_dictionary(filename, language, use_lemmatized_text)
		self.corpus_ready = True