def _vectorise_ps(self, ps: int, convert_to_proportions: bool): # Override the function, returning only the LSS representation directory_path = f"{self.corpus_path}\\problem{ps:03d}" pzd_fpath = (f"{directory_path}\\BTM_{self.btm_dir_suffix}" f"\\k{self.t}.pz_d") btm_lss = pd.read_csv(filepath_or_buffer=pzd_fpath, delim_whitespace=True, header=None) if len(self.btm.doc_index) == 0: doc_index = [] # We will need to build the index with Tools.scan_directory(directory_path) as docs: for doc in docs: if doc.is_dir(): continue doc_index.append(Tools.get_filename(doc.path)) btm_lss.index = doc_index else: btm_lss.index = self.btm.doc_index if convert_to_proportions: tokenised_btmcorpus_filepath = ( f"{directory_path}\\BTM_{self.btm_dir_suffix}" f"\\vectorised\\tokenised_btmcorpus.txt") with open(tokenised_btmcorpus_filepath) as c: tcorpus = c.readlines() freqs = [len(self._doc_gen_biterms(tdoc)) for tdoc in tcorpus] btm_lss = btm_lss.mul(freqs, axis="index") return btm_lss
def generate_gibbs_states_plots(self, states_path: str, cat: str = "likelihood"): new_dir = Tools.get_path(states_path, f"{cat}_plots") if Tools.path_exists(new_dir): print("Plots found, skipping..") return Tools.initialise_directory(new_dir) with Tools.scan_directory(states_path) as outputs: for i, output in enumerate(outputs): try: state_file = Tools.get_path(output.path, "state.log") df = pd.read_csv(filepath_or_buffer=state_file, delim_whitespace=True, index_col="iter") ax = sns.lineplot(x=df.index, y=cat, data=df) ax.margins(x=0) name = output.name fig = ax.get_figure() fig.savefig(Tools.get_path(states_path, f"{cat}_plots", f"{name}.png"), dpi=300, bbox_incehs="tight", format="png") fig.clf() print(f"{i}") except FileNotFoundError: print(f"→ Skipping {output.name}")
def _convert_corpus_to_bow(self, file_ext: str = "txt"): """ Convert a directory of text files into a BoW model. Parameters ---------- word_grams : int (optional) The number of words to combine as features. 1 is the default value, and it denotes the usage of word unigrams. Returns ------- bow_corpus : gnesim corpus The bag-of-words model. dictionary : gensim dictionary The id2word mapping. plain_documents : list The list of plain documents, to serve as a reference point. """ # Read in the plain text files plain_documents = [] with Tools.scan_directory(self.input_docs_path) as docs: for doc in docs: if doc.is_dir() or Tools.split_path( doc.path)[1] != f".{file_ext}": continue try: f = open(doc.path, mode="r", encoding="utf8") plain_documents.append(f.read()) self.doc_index.append(Tools.get_filename(doc.path)) except PermissionError: # Raised when trying to open a directory print("Skipped while loading files: {}".format(doc.name)) pass # Collocation Detection can be applied here via gensim.models.phrases # Tokenise corpus and remove too short documents tokenised_corpus = [[ ' '.join(tkn) for tkn in ngrams(word_tokenize(d.lower()), self.word_grams) ] for d in plain_documents if len(d) > 3] if self.drop_uncommon: freq = defaultdict(int) for doc in tokenised_corpus: for word in doc: freq[word] += 1 tokenised_corpus = [[w for w in doc if freq[w] > self.freq_th] for doc in tokenised_corpus] # Form the word ids dictionary for vectorisation dictionary = Dictionary(tokenised_corpus) corpus = [dictionary.doc2bow(t_d) for t_d in tokenised_corpus] return (corpus, dictionary, pd.DataFrame(data=plain_documents, index=self.doc_index, columns=["content"]))
def _concatenate_docs_into_btmcorpus(self, remove_bgw: bool = False, drop_uncommon: bool = False, drop_punctuation: bool = False): # Read in the plain text files plain_documents = [] with Tools.scan_directory(self.directory_path) as docs: for doc in docs: if doc.is_dir(): continue try: f = open(doc.path, mode="r", encoding="utf8") plain_documents.append(f.read()) self.doc_index.append(Tools.get_filename(doc.path)) except PermissionError: # Raised when trying to open a directory print("Skipped while loading files: {}".format(doc.name)) pass finally: f.close() # lowercase and strip \n away plain_documents = [ str.replace(d, "\n", "").lower() for d in plain_documents ] # it was observed that the topics are composed of a lot of stop words # Following the BTM paper and the observation, we remove these if remove_bgw: # Detect the language lang = detect(" ".join(plain_documents)) if lang == "en": lang = "english" elif lang == "nl": lang = "dutch" else: lang = "greek" new_documents = [] for d in plain_documents: terms = [ w for w in word_tokenize(text=d, language=lang) if w not in set(stopwords.words(lang)) ] new_documents.append(" ".join(terms)) plain_documents = new_documents if drop_punctuation: plain_documents = [ sub(pattern=r"[^\w\s]", repl="", string=d) for d in plain_documents ] # save it to disk Tools.save_list_to_text(mylist=plain_documents, filepath=self.plain_corpus_path) return plain_documents
def load_pz_d_into_df(self, use_frequencies: bool = False): """ Parameters ---------- use_frequencies : bool, optional DESCRIPTION. The default is False. Returns ------- btm_lss : TYPE DESCRIPTION. """ # ??? This function is not used, should be used in tester._vectorise_ps # Load the lss into df pzd_fpath = f"{self.directory_path}k{self.t}.pz_d" try: btm_lss = pd.read_csv(filepath_or_buffer=pzd_fpath, delim_whitespace=True) if not self.doc_index: # We will need to build the index with Tools.scan_directory(self.directory_path) as docs: for doc in docs: if doc.is_dir(): continue self.doc_index.append(Tools.get_filename(doc.path)) btm_lss.index = self.doc_index if use_frequencies: # The saved documents are in p(z|d) values # We want to proportion them to frequencies so that we have the # frequency of terms belonging to a topic # Since sum_b is used, we will use the count of biterms # Treating each p(zi|dj) as a proportion, we will count biterms with open(self.tokenised_btmcorpus_filepath) as c: tcorpus = c.readlines() # How many biterms are there? # Analyzing the C++ code, a widnow of 15 is used # regenerate the biterms and count as statistics can detect # redundancies in unordered terms: freqs = [len(self._doc_gen_biterms(tdoc)) for tdoc in tcorpus] btm_lss = btm_lss.mul(freqs, axis="index") return btm_lss except FileNotFoundError: return None
def smart_optimisation(self, plot_cat: str = "likelihood", tail_prcnt: float = 0.80, skip_factor: int = 1, verbose: bool = False): # First generate the outputs to compare: words_counts = self._generate_hdps_outputs(skip_factor=skip_factor, verbose=verbose) ret = {} # Loop over the outputs of different etas master_folder = Tools.get_path(self.out_dir, "optimisation") log_likelihoods = [] avg_num_topics = [] std_num_topics = [] pw_ll = [] errors = [] with Tools.scan_directory(master_folder) as perms: for perm in perms: # generate plots if not Tools.is_path_dir(perm.path): continue self.generate_gibbs_states_plots(states_path=perm.path, cat=plot_cat) with Tools.scan_directory(perm.path) as problems: for problem in problems: try: n_words = words_counts[problem.name] path_state = Tools.get_path( problem.path, "state.log") df_state = pd.read_csv( filepath_or_buffer=path_state, delim_whitespace=True, index_col="iter", usecols=["iter", "likelihood", "num.topics"]) ll = df_state.likelihood.tail( round(len(df_state) * tail_prcnt)).mean() avg_topics = df_state["num.topics"].tail( round(len(df_state) * tail_prcnt)).mean() std_topics = df_state["num.topics"].tail( round(len(df_state) * tail_prcnt)).std() log_likelihoods.append(ll) pw_ll.append(ll / n_words) avg_num_topics.append(avg_topics) std_num_topics.append(std_topics) except FileNotFoundError as e: print(f"{e}") errors.append(f"{e}") continue except KeyError: # Plots folders are being queried for n_words continue ret.update({ f"{perm.name}": [ round(sum(log_likelihoods) / len(log_likelihoods), 4), round(sum(pw_ll) / len(pw_ll), 4), round(sum(avg_num_topics) / len(avg_num_topics), 4), round(sum(std_num_topics) / len(std_num_topics), 4) ] }) # Save any encountered errors to disk too Tools.save_list_to_text(mylist=errors, filepath=Tools.get_path( self.out_dir, "optimisation", "opt_errors.txt")) pd.DataFrame(data=ret, index=["Log-l", "PwLL", "T-Avg", "T-Std" ]).T.to_csv(Tools.get_path(self.out_dir, "optimisation", "optimisation.csv"), index=True) return ret
def _generate_hdps_outputs(self, skip_factor: int = 1, verbose: bool = False): st = time.perf_counter() ldac_path = Tools.get_path("lda_c_format_HyperFalse", "dummy_ldac_corpus.dat") words_nums = {} vocab_file = Tools.get_path("lda_c_format_HyperFalse", "dummy_ldac_corpus.dat.vocab") # size = ((60 // skip_factor) # * len(self.etas) # * len(self.gammas)**2 # * len(self.alphas)**2) # Since we fixed the scales of Gammas size = ((60 // skip_factor) * len(self.etas) * len(self.gammas) * len(self.alphas)) i = 0 with Tools.scan_directory(self.training_folder) as ps_folders: for c, folder in enumerate(ps_folders): if not folder.name[0:7] == "problem": if verbose: print(f"→ Skipping {folder.name}") continue # Implement the skipping factor if c % skip_factor != 0: continue t = time.perf_counter() # Fix the scale parameters for the Gamma priors g_r = 1 a_r = 1 for eta in self.etas: # for g_s, g_r in product(self.gammas, repeat=2): # for a_s, a_r in product(self.alphas, repeat=2): # Only switch the shape parameter of Gammas for g_s in self.gammas: for a_s in self.alphas: # Cache the number of words for later if folder.name not in words_nums: vocab_path = Tools.get_path( folder.path, vocab_file) n_words = self._get_number_words(vocab_path) words_nums.update({folder.name: n_words}) i = i + 1 percentage = f"{100 * i / size:06.02f}" suff = (f"{g_s:0.2f}_{g_r:0.2f}_" f"{a_s:0.2f}_{a_r:0.2f}") if verbose: print(f"► Applying HDP with " f"eta={eta:0.1f} " f"gamma({g_s:0.2f}, {g_r:0.2f}) " f"alpha({a_s:0.2f}, {a_r:0.2f}) " f"on {folder.name} [{percentage}%]") directory = Tools.get_path(self.out_dir, "optimisation", f"{eta:0.1f}__{suff}", folder.name) if (Tools.path_exists(directory)): if verbose: print("\tcached result found at " f"{directory}") continue path_executable = r"{}\hdp.exe".format( self.hdp_path) data = Tools.get_path(folder.path, ldac_path) # Prepare the output directory Tools.initialise_directories(directory) if self.seed is not None: s.run([ path_executable, "--algorithm", "train", "--data", data, "--directory", directory, "--max_iter", str(self.iters), "--sample_hyper", "no", "--save_lag", "-1", "--eta", str(eta), "--gamma_a", str(g_s), "--gamma_b", str(g_r), "--alpha_a", str(a_s), "--alpha_b", str(a_r), "--random_seed", str(self.seed) ], stdout=s.DEVNULL, check=True, capture_output=False, text=True) else: s.run([ path_executable, "--algorithm", "train", "--data", data, "--directory", directory, "--max_iter", str(self.iters), "--sample_hyper", "no", "--save_lag", "-1", "--eta", str(eta), "--gamma_a", str(g_s), "--gamma_b", str(g_r), "--alpha_a", str(a_s), "--alpha_b", str(a_r) ], stdout=s.DEVNULL, check=True, capture_output=False, text=True) if verbose: print(f"--- {folder.name} done in " f"{time.perf_counter() - t:0.1f} seconds ---") period = round(time.perf_counter() - st, 2) print(f"----- Vectorisation done in {period} seconds -----") return words_nums
def assess_hyper_sampling(self, tail_prcnt: float, verbose: bool = False): """ A function to measure the average per word log-likelihood after hyper-sampling the concentration parameters of the Dirichlet distributions. Caution: the hdp must have been run on the data with hyper sampling and without it, in order to load the two representations and compare. Returns ------- dct: dict A dictionary containing the per word log-likelihood of the train data with the two methods pertaining to sampling the concentration parameters: normal and hyper. """ path_normal = Tools.get_path(".", "hdp_lss_HyperFalse", "state.log") path_hyper = Tools.get_path(".", "hdp_lss_HyperTrue", "state.log") path_ldac = Tools.get_path(".", "lda_c_format_HyperTrue", "dummy_ldac_corpus.dat.vocab") per_word_ll_normal = [] per_word_ll_hyper = [] if verbose: print("------Concentration Parameters Optimisation------") with Tools.scan_directory(self.training_folder) as dirs: for d in dirs: if d.name[0:7] != "problem": continue if verbose: print(f"\t► Processing {d.name}") normal = Tools.get_path(d.path, path_normal) hyper = Tools.get_path(d.path, path_hyper) vocab = Tools.get_path(d.path, path_ldac) n_words = self._get_number_words(vocab) df_normal = pd.read_csv(filepath_or_buffer=normal, delim_whitespace=True, index_col="iter", usecols=["iter", "likelihood"], squeeze=True) ll_normal = df_normal.tail(round(len(df_normal) * tail_prcnt)).mean() per_word_ll_normal.append(ll_normal / n_words) df_hyper = pd.read_csv(filepath_or_buffer=hyper, delim_whitespace=True, index_col="iter", usecols=["iter", "likelihood"], squeeze=True) ll_hyper = df_hyper.tail(round(len(df_hyper) * tail_prcnt)).mean() per_word_ll_hyper.append(ll_hyper / n_words) dct = { "Normal_Sampling": round(sum(per_word_ll_normal) / len(per_word_ll_normal), 4), "Hyper_Sampling": round(sum(per_word_ll_hyper) / len(per_word_ll_hyper), 4) } if verbose: print("-------------------------------------------------") pd.DataFrame(data=dct, index=[0 ]).to_csv(f"{self.out_dir}/hyper_optimisation.csv", index=False) return dct