def provide_corpus_functionwords(numOfFunctionwords): global top_corpus_functionwords top_corpus_functionwords={} synchronized_functionwords_file_path = Path("vectors_handling/vectors/synchronized_functionwords/synchronized_functionwords.txt") if not util.exists(synchronized_functionwords_file_path): # can't find the file in memory log('Cannot find synchronized_functionwords file') # redundant corpus_functionwords = {} for domain_dir in os.scandir(setup.database): if domain_dir.name == 'europe_data' and setup.domain == 'in': for country_dir in os.scandir(domain_dir): country_name = str.split(os.path.basename(country_dir), '.')[1] log('Counting function words in ' + country_name) for user_dir in os.scandir(country_dir): for file_dir in os.scandir(user_dir): file = open(file_dir, "r", encoding="utf-8") lines = file.readlines() for line in lines: words = line.split() for word in words: if word in function_words_map.keys(): if word not in corpus_functionwords.keys(): corpus_functionwords[word] = 1 else: corpus_functionwords[word] += 1 top_corpus_functionwords = heapq.nlargest(numOfFunctionwords, corpus_functionwords, key=corpus_functionwords.get) util.save_file(synchronized_functionwords_file_path, top_corpus_functionwords) top_corpus_functionwords = util.load_file(synchronized_functionwords_file_path) return top_corpus_functionwords
def provide_function_words_map(): function_words_file_path = Path("vectors_handling/vectors/functionwords/function_words.txt") function_words = util.load_file(function_words_file_path) global function_words_map function_words_map={} for index in range(len(function_words)): function_words_map[heappop(function_words)] = index
def provide_top_spelling_errors(): spelling_file_path = Path("vectors_handling/vectors/spelling_errors/top_spelling_errors.txt") if not util.exists(spelling_file_path): # can't find the file in memory log('Cannot find top bipos file') # redundant generate_top_spelling_errors(spelling_file_path) top_spelling_errors = util.load_file(spelling_file_path) return top_spelling_errors
def provide_top_unigram(): unigram_file_path = Path("vectors_handling/vectors/unigrams/top_unigrams.txt") if not util.exists(unigram_file_path): # can't find the file in memory log('Cannot find top unigrams file') # redundant generate_top_unigrams(unigram_file_path) top_unigrams = util.load_file(unigram_file_path) return top_unigrams
def provide_top_tripos(): tripos_file_path = Path("vectors_handling/vectors/pos/top_tripos.txt") if not util.exists(tripos_file_path): # can't find the file in memory log('Cannot find top tripos file') # redundant generate_top_tripos(tripos_file_path) top_tripos = util.load_file(tripos_file_path) return top_tripos
def get_database(): global database database = util.load_file( 'utilities/database_dir.txt')[0] + util.FeatureToDirectory[feature]