def evaluate(train_round_idx, ae_module, cluster_module): test_loader = torch.utils.data.DataLoader( torch.utils.data.TensorDataset(pt_data), batch_size=256) pred_labels = np.zeros(pt_data.shape[0], dtype=np.int) index = 0 n_batches = 0 for batch_data in test_loader: batch_data = batch_data[0].cuda() n_batches += 1 batch_size = batch_data.shape[0] embedded_data, reconstructed_data = ae_module.forward(batch_data) labels = cluster_module.prediction_hard_np(embedded_data) pred_labels[index:index + batch_size] = labels index = index + batch_size pred_tree = dendrogram_purity_tree_from_clusters( cluster_module, pred_labels, 'single') pred_tree2 = dendrogram_purity_tree_from_clusters( cluster_module, pred_labels, 'complete') lp = leaf_purity(pred_tree, gold_labels) leaf_purity_value = f"{lp[0]:1.3}\t({lp[1]:1.3})" dp_value_single = dendrogram_purity(pred_tree, gold_labels) dp_value_complete = dendrogram_purity(pred_tree2, gold_labels) logger.info( f"{train_round_idx} Evaluation: leaf_purity: {leaf_purity_value}, purity_single: {dp_value_single}, purity_complete: {dp_value_complete}" ) return dp_value_single, dp_value_complete, leaf_purity_value
def evaluate(train_round_idx, ae_module, cluster_module): test_loader = torch.utils.data.DataLoader(torch.utils.data.TensorDataset( pt_data, torch.tensor(range(pt_data.shape[0]))), batch_size=256) pred_labels = np.zeros(pt_data.shape[0], dtype=np.int) pred_tree = None index = 0 n_batches = 0 print("start evaluation") for batch_data_id in test_loader: batch_data, batch_ids = batch_data_id batch_data = batch_data.cuda() n_batches += 1 batch_size = batch_data.shape[0] embeded_data, reconstructed_data = ae_module.forward(batch_data) labels = cluster_module.prediction_labels_np( min(cluster_module.n_leaf_nodes, n_clusters), embeded_data)[0] pred_labels[index:index + batch_size] = labels new_pred_tree = cluster_module.predict_tree(embeded_data, batch_ids) if pred_tree is None: pred_tree = new_pred_tree else: pred_tree = combine_to_trees(pred_tree, new_pred_tree) index = index + batch_size lp = leaf_purity(pred_tree, gold_labels) nmi_value = nmi(gold_labels, pred_labels, average_method='arithmetic') acc_value = cluster_acc(gold_labels, pred_labels)[0] dp_value = dendrogram_purity(pred_tree, gold_labels) leaf_purity_value = f"{lp[0]:1.3}\t({lp[1]:1.3})" logger.info( f"{train_round_idx} leaf_purity: {leaf_purity_value}, purity: {dp_value}, NMI: {nmi_value} ACC: {acc_value}" ) return nmi_value, acc_value, dp_value, leaf_purity_value
def run_experiment(ae_model_path): logger.info( f"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" ) logger.info( f"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" ) logger.info(f"Working now on {ae_model_path.name}") logger.info( f"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" ) logger.info( f"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" ) new_seed = random.randint(0, 1000) logger.info(f"Seed value for this is: {new_seed}") set_random_seed(new_seed) ae_module = stacked_ae(pt_data.shape[1], [500, 500, 2000, 10], weight_initalizer=torch.nn.init.xavier_normal_, activation_fn=lambda x: F.relu(x), loss_fn=None, optimizer_fn=None) model_data = torch.load(ae_model_path, map_location='cpu') ae_module.load_state_dict(model_data) ae_module = ae_module.cuda() # Get embedded data embedded_data = None for batch_data in torch.utils.data.DataLoader(pt_data, batch_size=256, shuffle=False): embedded_batch_np = ae_module.forward( batch_data.cuda())[0].detach().cpu().numpy() if embedded_data is None: embedded_data = embedded_batch_np else: embedded_data = np.concatenate([embedded_data, embedded_batch_np], 0) del ae_module # bisecting k-means: tree = bisection(n_leaf_nodes_final, embedded_data) bisec_labels = predict_by_tree(tree, embedded_data, n_clusters) bisec_tree = predict_id_tree(tree, embedded_data) bisec_km_nmi = nmi(gold_labels, bisec_labels, average_method='arithmetic') bisec_km_acc = cluster_acc(bisec_labels, gold_labels)[0] bisec_km_purity = dendrogram_purity(bisec_tree, gold_labels) lp = leaf_purity(bisec_tree, gold_labels) leaf_purity_value = f"{lp[0]:1.3}\t({lp[1]:1.3})" result_file = Path(f"{result_dir}/results_ae_biseckm_{dataset_name}.txt") result_file_exists = result_file.exists() f = open(result_file, "a+") if not result_file_exists: f.write( "#\"ae_model_name\"\t\"NMI\"\t\"ACC\"\t\"Dendrogram_Purity\"\t\"Leaf_Purity\t(Std)\"\n" ) f.write( f"{ae_model_path.name}\t{bisec_km_nmi}\t{bisec_km_acc}\t{bisec_km_purity}\t{leaf_purity_value}\n" ) f.close()
def run_experiment(ae_model_path): logger.info( f"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" ) logger.info( f"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" ) logger.info(f"Working now on {ae_model_path.name}") logger.info( f"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" ) logger.info( f"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" ) new_seed = random.randint(0, 1000) logger.info(f"Seed value for this is: {new_seed}") set_random_seed(new_seed) ae_module = stacked_ae(pt_data.shape[1], [500, 500, 2000, 10], weight_initalizer=torch.nn.init.xavier_normal_, activation_fn=lambda x: F.relu(x), loss_fn=None, optimizer_fn=None) model_data = torch.load(ae_model_path, map_location='cpu') ae_module.load_state_dict(model_data) ae_module = ae_module.cuda() # Get embedded data embedded_data = None for batch_data in torch.utils.data.DataLoader(pt_data, batch_size=256, shuffle=False): embedded_batch_np = ae_module.forward( batch_data.cuda())[0].detach().cpu().numpy() if embedded_data is None: embedded_data = embedded_batch_np else: embedded_data = np.concatenate([embedded_data, embedded_batch_np], 0) del ae_module sl_cl = AgglomerativeClustering(compute_full_tree=True, n_clusters=n_clusters, linkage="single").fit(embedded_data) sl_labels = sl_cl.labels_ sl_purity_tree = prune_dendrogram_purity_tree( to_dendrogram_purity_tree(sl_cl.children_), n_leaf_nodes_final) sl_nmi = nmi(gold_labels, sl_labels, average_method='arithmetic') sl_acc = cluster_acc(sl_labels, gold_labels)[0] sl_purity = dendrogram_purity(sl_purity_tree, gold_labels) sl_lp = leaf_purity(sl_purity_tree, gold_labels) sl_leaf_purity_value = f"{sl_lp[0]:1.3}\t({sl_lp[1]:1.3})" result_file_sl = Path( f"{result_dir}/results_ae_agglo_single_{dataset_name}.txt") result_file_sl_exists = result_file_sl.exists() f = open(result_file_sl, "a+") if not result_file_sl_exists: f.write( "#\"ae_model_name\"\t\"NMI\"\t\"ACC\"\t\"Dendrogram_Purity\"\t\"Leaf_Purity\t(Std)\"\n" ) f.write( f"{ae_model_path.name}\t{sl_nmi}\t{sl_acc}\t{sl_purity}\t{sl_leaf_purity_value}\n" ) f.close() del sl_cl, sl_labels, sl_purity_tree cl_cl = AgglomerativeClustering(compute_full_tree=True, n_clusters=n_clusters, linkage="complete").fit(embedded_data) cl_labels = cl_cl.labels_ cl_purity_tree = prune_dendrogram_purity_tree( to_dendrogram_purity_tree(cl_cl.children_), n_leaf_nodes_final) cl_nmi = nmi(gold_labels, cl_labels, average_method='arithmetic') cl_acc = cluster_acc(cl_labels, gold_labels)[0] cl_purity = dendrogram_purity(cl_purity_tree, gold_labels) cl_lp = leaf_purity(cl_purity_tree, gold_labels) cl_leaf_purity_value = f"{cl_lp[0]:1.3}\t({cl_lp[1]:1.3})" result_file_cl = Path( f"{result_dir}/results_ae_agglo_complete_{dataset_name}.txt", ) result_file_cl_exists = result_file_cl.exists() f = open(result_file_cl, "a+") if not result_file_cl_exists: f.write( "#\"ae_model_name\"\t\"NMI\"\t\"ACC\"\t\"Dendrogram_Purity\"\t\"Leaf_Purity\t(Std)\"\n" ) f.write( f"{ae_model_path.name}\t{cl_nmi}\t{cl_acc}\t{cl_purity}\t{cl_leaf_purity_value}\n" ) f.close() del cl_cl, cl_labels, cl_purity_tree