def load_year_index_infos(index_dir, years, word_file, num_words=-1): """ Returns dictionary mapping year to: "index": word->id index for that year. "list": word_list for that year "indices": set of valid indices corresponding to the word list Assumes that each year is indexed seperately. """ if "index.pkl" in os.listdir(index_dir): return load_year_index_infos_common(load_pickle(index_dir + "index.pkl"), years, word_file, num_words=num_words) year_index_infos = collections.defaultdict(dict) word_lists = load_year_words(word_file, years) for year, word_list in word_lists.iteritems(): year_index = load_pickle(index_dir + "/" + str(year) + "-index.pkl") year_index_infos[year]["index"] = year_index if num_words != -1: word_list = word_list[:num_words] word_list, word_indices = get_word_indices(word_list, year_index) year_index_infos[year]["list"] = word_list year_index_infos[year]["indices"] = word_indices return year_index_infos
def load_year_index_infos_common(common_index, years, word_file, num_words=-1): """ Returns dictionary mapping year to: "index": word->id index for that year. "list": word_list for that year "indices": set of valid indices corresponding to the word list Assumes that each year is indexed seperately. """ year_index_infos = collections.defaultdict(dict) word_lists = load_year_words(word_file, years) for year, word_list in word_lists.iteritems(): year_index = common_index year_index_infos[year]["index"] = year_index if num_words != -1: word_list = word_list[:num_words] word_list, word_indices = get_word_indices(word_list, year_index) year_index_infos[year]["list"] = word_list year_index_infos[year]["indices"] = word_indices return year_index_infos
def load_year_index_infos_common(common_index, years, word_file, num_words=-1): """ Returns dictionary mapping year to: "index": word->id index for that year. "list": word_list for that year "indices": set of valid indices corresponding to the word list Assumes that each year is indexed seperately. """ year_index_infos = collections.defaultdict(dict) word_lists = load_year_words(word_file, years) for year, word_list in word_lists.iteritems(): year_index = common_index year_index_infos[year]["index"] = year_index if num_words != -1: word_list = word_list[:num_words] word_list, word_indices = get_word_indices(word_list, year_index) year_index_infos[year]["list"] = word_list year_index_infos[year]["indices"] = word_indices return year_index_infos
def load_year_index_infos(index_dir, years, word_file, num_words=-1): """ Returns dictionary mapping year to: "index": word->id index for that year. "list": word_list for that year "indices": set of valid indices corresponding to the word list Assumes that each year is indexed seperately. """ if "index.pkl" in os.listdir(index_dir): return load_year_index_infos_common(load_pickle(index_dir + "index.pkl"), years, word_file, num_words=num_words) year_index_infos = collections.defaultdict(dict) word_lists = load_year_words(word_file, years) for year, word_list in word_lists.iteritems(): year_index = load_pickle(index_dir + "/" + str(year) + "-index.pkl") year_index_infos[year]["index"] = year_index if num_words != -1: word_list = word_list[:num_words] word_list, word_indices = get_word_indices(word_list, year_index) year_index_infos[year]["list"] = word_list year_index_infos[year]["indices"] = word_indices return year_index_infos