def main(): # Load a configuration args = configs.arg_parse() fix_seed(args.seed) # GPU or CPU if args.gpu: print("CUDA") else: print("Using CPU") # Load dataset data = prepare_data(args.dataset, args.train_ratio, args.input_dim, args.seed) # Load model model_path = 'models/GCN_model_{}.pth'.format(args.dataset) model = torch.load(model_path) # Evaluate GraphSVX if args.dataset == 'Mutagenicity': data = selected_data(data, args.dataset) eval_Mutagenicity(data, model, args) elif args.dataset == 'syn6': eval_syn6(data, model, args) else: eval_syn(data, model, args)
def main(): hparams = AmHparams() parser = hparams.parser am_hp = parser.parse_args() rate = 1 out_path = Const.NoiseOutPath delete_files(out_path) train_data = prepare_data('train', am_hp, shuffle=True, length=None) pathlist = train_data.path_lst pylist = train_data.pny_lst hzlist = train_data.han_lst length = len(pathlist) rand_list = random.sample(range(length), int(rate * length)) pre_list = [] for i in rand_list: path = pathlist[i] pre_list.append(os.path.join(Const.SpeechDataPath, path)) _, filename_list = add_noise(pre_list, out_path=Const.NoiseOutPath, keep_bits=False) data = '' with open(Const.NoiseDataTxT, 'w') as f: for i in range(len(rand_list)): pinyin = pylist[rand_list[i]] hanzi = hzlist[rand_list[i]] data += filename_list[i] + '\t' + pinyin + '\t' + hanzi + '\n' f.writelines(data[:-1]) print('---------------噪声数据生成完毕------------')
def main(): hparams = TransformerHparams() parser = hparams.parser hp = parser.parse_args() # 数据准备工作 test_data = prepare_data('test', hp, shuffle=True, length=None) test_data.feature_dim = hp.feature_dim transformer_test(hp, test_data)
def main(): hparams = TransformerHparams() parser = hparams.parser hp = parser.parse_args() # 数据准备工作 train_data = prepare_data('train', hp, shuffle=True, length=None) transformer_train(hp, train_data)
def main(): args = configs.arg_parse() fix_seed(args.seed) # Load the dataset data = prepare_data(args.dataset, args.train_ratio, args.input_dim, args.seed) # Define and train the model if args.dataset in ['Cora', 'PubMed']: # Retrieve the model and training hyperparameters depending the data/model given as input hyperparam = ''.join(['hparams_', args.dataset, '_', args.model]) param = ''.join(['params_', args.dataset, '_', args.model]) model = eval(args.model)(input_dim=data.num_features, output_dim=data.num_classes, **eval(hyperparam)) train_and_val(model, data, **eval(param)) _, test_acc = evaluate(data, model, data.test_mask) print('Test accuracy is {:.4f}'.format(test_acc)) elif args.dataset in ['syn6', 'Mutagenicity']: input_dims = data.x.shape[-1] model = GcnEncoderGraph(input_dims, args.hidden_dim, args.output_dim, data.num_classes, args.num_gc_layers, bn=args.bn, dropout=args.dropout, args=args) train_gc(data, model, args) _, test_acc = evaluate(data, model, data.test_mask) print('Test accuracy is {:.4f}'.format(test_acc)) else: # For pytorch geometric model #model = GCNNet(args.input_dim, args.hidden_dim, # data.num_classes, args.num_gc_layers, args=args) input_dims = data.x.shape[-1] model = GcnEncoderNode(data.num_features, args.hidden_dim, args.output_dim, data.num_classes, args.num_gc_layers, bn=args.bn, dropout=args.dropout, args=args) train_syn(data, model, args) _, test_acc = evaluate(data, model, data.test_mask) print('Test accuracy is {:.4f}'.format(test_acc)) # Save model model_path = 'models/{}_model_{}.pth'.format(args.model, args.dataset) if not os.path.exists(model_path) or args.save == True: torch.save(model, model_path)
def main(): args = configs.arg_parse() fix_seed(args.seed) # Load the dataset data = prepare_data(args.dataset, args.train_ratio, args.input_dim, args.seed) # Load the model model_path = 'models/{}_model_{}.pth'.format(args.model, args.dataset) model = torch.load(model_path) # Evaluate the model if args.dataset in ['Cora', 'PubMed']: _, test_acc = evaluate(data, model, data.test_mask) else: test_acc = test(data, model, data.test_mask) print('Test accuracy is {:.4f}'.format(test_acc)) # Explain it with GraphSVX explainer = GraphSVX(data, model, args.gpu) # Distinguish graph classfication from node classification if args.dataset in ['Mutagenicity', 'syn6']: explanations = explainer.explain_graphs(args.indexes, args.hops, args.num_samples, args.info, args.multiclass, args.fullempty, args.S, 'graph_classification', args.feat, args.coal, args.g, args.regu, True) else: explanations = explainer.explain(args.indexes, args.hops, args.num_samples, args.info, args.multiclass, args.fullempty, args.S, args.hv, args.feat, args.coal, args.g, args.regu, True) print('Sum explanations: ', [np.sum(explanation) for explanation in explanations]) print('Base value: ', explainer.base_values)
def recognition(type='dfcnn'): file = '../wav_file/input1.wav' # 现场输入识别 receive_wav(file) if type == 'dfcnn': # 1.声学模型----------------------------------- hparams = AmHparams() parser = hparams.parser hp = parser.parse_args() am_model = CNNCTCModel(hp) print('loading acoustic model...') select_model_step = 'model_04-14.91' am_model.load_model(select_model_step) # 2.语言模型----------------------------------- hparams = LmHparams() parser = hparams.parser hp = parser.parse_args() hp.is_training = False print('loading language model...') lm_model = Language_Model(hp) gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.1) sess = tf.Session(graph=lm_model.graph, config=tf.ConfigProto(gpu_options=gpu_options)) with lm_model.graph.as_default(): saver = tf.train.Saver() with sess.as_default(): latest = tf.train.latest_checkpoint(Const.LmModelFolder) saver.restore(sess, latest) while (True): dfcnn_speech(sess, am_model, lm_model, file) if type == 'transformer': hparams = TransformerHparams() parser = hparams.parser hp = parser.parse_args() hp.is_training = False train_data = prepare_data('train', hp, shuffle=True, length=None) model = Transformer(hp) with model.graph.as_default(): saver = tf.train.Saver() with tf.Session(graph=model.graph) as sess: latest = tf.train.latest_checkpoint(Const.TransformerFolder) saver.restore(sess, latest) while (True): transformer_speech(sess, model, train_data, file)
def train_model(is_streamlit=False): #Manda pré-processar as informações print('Preparing data...') if is_streamlit: st.write('Preparing data...') X_full, y_full, X_train, y_train, X_test, y_test = prepare_data() print('Preparing data... OK') #Manda treinar print('Trainning data...') if is_streamlit: st.write('Trainning data...') response = train_breast_cancer(X_full, y_full, X_train, y_train, X_test, y_test, is_streamlit) print('Trainning data... OK') #Retorna sucesso print(response)
def load_data(self): ''' Load data for training/validation ''' self.tr_set, self.dv_set, _, self.audio_dim, msg = \ prepare_data(self.paras.njobs, self.paras.dev_njobs, self.paras.gpu, self.paras.pin_memory, **self.config['data']) self.verbose(msg)
default='GAT', help="Name of the GNN: GCN or GAT") parser.add_argument( "--dataset", type=str, default='PubMed', help="Name of the dataset among Cora, PubMed, Amazon, PPI, Reddit") parser.add_argument("--seed", type=int, default=10) parser.add_argument("--save", type=str, default=False, help="True to save the trained model obtained") args = parser.parse_args() # Load the dataset data = prepare_data(args.dataset, args.seed) # Train the model - specific case for PPI dataset if args.dataset == "PPI": model = main_ppi(type=args.model) # test_ppi shows how to compute predictions (model(), then positive values => predict this class) else: # Retrieve the model and training hyperparameters depending the data/model given as input hyperparam = ''.join(['hparams_', args.dataset, '_', args.model]) param = ''.join(['params_', args.dataset, '_', args.model]) # Define the model if args.model == 'GCN': model = GCN(input_dim=data.x.size(1), output_dim=data.num_classes,
return f1, precision, recall, dict_pt_verbs else: return f1 <<<<<<< HEAD def main(name_file='all_f1', train_dir='all', test_dir='test', dir_files='data/disambiguation/', dir_results='results_2/', max_length=120, cuda_id=0, cuda=True, n_epochs=9, seed=0, lr=0.0001): ======= def main(name_file='all_f1', train_dir='all', test_dir='test', dir_files='data/disambiguation/', dir_results='results/', max_length=120, cuda_id=0, cuda=True, n_epochs=9, seed=0): >>>>>>> b4396c75b27e3d4f8680ea6762d7f2c530382768 dir_train = os.path.join(dir_files, train_dir) dir_test = os.path.join(dir_files, test_dir) dir_results = os.path.join(dir_results, train_dir, name_file) os.makedirs(dir_results, exist_ok=True) input_lang, output_lang, pairs_train, pairs_test, senses_per_sentence = prepare_data(name_file, 'verbs_selected_lemma', max_length=max_length, dir_train=dir_train, dir_test=dir_test) selected_synsets = np.load(os.path.join(dir_files, 'selected_synsets.npy')) reader = Seq2SeqDatasetReader( source_tokenizer=WordTokenizer(), target_tokenizer=WordTokenizer(), <<<<<<< HEAD ======= delimiter=',', >>>>>>> b4396c75b27e3d4f8680ea6762d7f2c530382768 source_token_indexers={'tokens': SingleIdTokenIndexer()}, target_token_indexers={'tokens': SingleIdTokenIndexer(namespace='target_tokens')}) train_dataset = reader.read(os.path.join(dir_train, name_file + '.tsv')) validation_dataset = reader.read(os.path.join(dir_test, 'verbs_selected_lemma.tsv')) vocab = Vocabulary.from_instances(train_dataset + validation_dataset,
def filter_useless_nodes_multiclass(args_dataset, args_model, args_explainers, args_hops, args_num_samples, args_test_samples, args_prop_noise_nodes, args_connectedness, node_indices, args_K, info, args_hv, args_feat, args_coal, args_g, args_multiclass, args_regu, args_gpu, args_fullempty, args_S, seed): """ Add noisy neighbours to dataset and check how many are included in explanations The fewest, the better the explainer. Args: Arguments defined in argument parser of script_eval.py """ # Define dataset data = prepare_data(args_dataset, seed=10) args_num_noise_nodes = int(args_prop_noise_nodes * data.x.size(0)) args_c = eval('EVAL1_' + data.name)['args_c'] args_p = eval('EVAL1_' + data.name)['args_p'] args_binary = eval('EVAL1_' + data.name)['args_binary'] # Select a random subset of nodes to eval the explainer on. if not node_indices: node_indices = extract_test_nodes(data, args_test_samples, seed) # Add noisy neighbours to the graph, with random features data = add_noise_neighbours(data, args_num_noise_nodes, node_indices, binary=args_binary, p=args_p, connectedness=args_connectedness) # Define training parameters depending on (model-dataset) couple hyperparam = ''.join(['hparams_', args_dataset, '_', args_model]) param = ''.join(['params_', args_dataset, '_', args_model]) # Define the model if args_model == 'GCN': model = GCN(input_dim=data.x.size( 1), output_dim=data.num_classes, **eval(hyperparam)) else: model = GAT(input_dim=data.x.size( 1), output_dim=data.num_classes, **eval(hyperparam)) # Re-train the model on dataset with noisy features train_and_val(model, data, **eval(param)) model.eval() with torch.no_grad(): log_logits = model(x=data.x, edge_index=data.edge_index) # [2708, 7] test_acc = accuracy(log_logits[data.test_mask], data.y[data.test_mask]) print('Test accuracy is {:.4f}'.format(test_acc)) del log_logits # Study attention weights of noisy nodes in GAT model - compare attention with explanations if str(type(model)) == "<class 'src.models.GAT'>": study_attention_weights(data, model, args_test_samples) # Adaptable K - top k explanations we look at for each node # Depends on number of existing features/neighbours considered for GraphSVX # if 'GraphSVX' in args_explainers: # K = [] # else: # K = [5]*len(node_indices) # Do for several explainers for c, explainer_name in enumerate(args_explainers): print('EXPLAINER: ', explainer_name) # Define the explainer explainer = eval(explainer_name)(data, model, args_gpu) # Loop on each test sample and store how many times do noisy nodes appear among # K most influential features in our explanations # 1 el per test sample - count number of noisy nodes in explanations total_num_noise_neis = [] # 1 el per test sample - count number of noisy nodes in explanations for 1 class pred_class_num_noise_neis = [] # 1 el per test sample - count number of noisy nodes in subgraph total_num_noisy_nei = [] total_neigbours = [] # 1 el per test samples - number of neigbours of v in subgraph M = [] # 1 el per test sample - number of non zero features for node_idx in tqdm(node_indices, desc='explain node', leave=False): # Look only at coefficients for nodes (not node features) if explainer_name == 'Greedy': coefs = explainer.explain_nei(node_index=node_idx, hops=args_hops, num_samples=args_num_samples, info=False, multiclass=True) elif explainer_name == 'GNNExplainer': _ = explainer.explain(node_index=node_idx, hops=args_hops, num_samples=args_num_samples, info=False, multiclass=True) coefs = explainer.coefs else: # Explanations via GraphSVX coefs = explainer.explain([node_idx], args_hops, args_num_samples, info, args_multiclass, args_fullempty, args_S, args_hv, args_feat, args_coal, args_g, args_regu) coefs = coefs[0].T[explainer.F:] # if explainer.F > 50: # K.append(10) # else: # K.append(int(explainer.F * args_K)) # Check how many non zero features M.append(explainer.M) # Number of noisy nodes in the subgraph of node_idx num_noisy_nodes = len( [n_idx for n_idx in explainer.neighbours if n_idx >= data.x.size(0)-args_num_noise_nodes]) # Number of neighbours in the subgraph total_neigbours.append(len(explainer.neighbours)) # Multilabel classification - consider all classes instead of focusing on the # class that is predicted by our model num_noise_neis = [] # one element for each class of a test sample true_conf, predicted_class = model(x=data.x, edge_index=data.edge_index).exp()[ node_idx].max(dim=0) for i in range(data.num_classes): # Store indexes of K most important features, for each class nei_indices = np.abs(coefs[:, i]).argsort()[-args_K:].tolist() # Number of noisy features that appear in explanations - use index to spot them num_noise_nei = sum( idx >= (explainer.neighbours.shape[0] - num_noisy_nodes) for idx in nei_indices) num_noise_neis.append(num_noise_nei) if i == predicted_class: #nei_indices = coefs[:,i].argsort()[-args_K:].tolist() #num_noise_nei = sum(idx >= (explainer.neighbours.shape[0] - num_noisy_nodes) for idx in nei_indices) pred_class_num_noise_neis.append(num_noise_nei) # Return this number => number of times noisy neighbours are provided as explanations total_num_noise_neis.append(sum(num_noise_neis)) # Return number of noisy nodes adjacent to node of interest total_num_noisy_nei.append(num_noisy_nodes) if info: print('Noisy neighbours included in explanations: ', total_num_noise_neis) print('There are {} noise neighbours found in the explanations of {} test samples, an average of {} per sample' .format(sum(total_num_noise_neis), args_test_samples, sum(total_num_noise_neis)/args_test_samples)) print(np.sum(pred_class_num_noise_neis) / args_test_samples, 'for the predicted class only') print('Proportion of explanations showing noisy neighbours: {:.2f}%'.format( 100 * sum(total_num_noise_neis) / (args_K * args_test_samples * data.num_classes))) perc = 100 * sum(total_num_noise_neis) / (args_test_samples * args_num_noise_nodes * data.num_classes) perc2 = 100 * ((args_K * args_test_samples * data.num_classes) - sum(total_num_noise_neis)) / (np.sum(M) - sum(total_num_noisy_nei)) print('Proportion of noisy neighbours found in explanations vs normal features: {:.2f}% vs {:.2f}'.format( perc, perc2)) print('Proportion of nodes in subgraph that are noisy: {:.2f}%'.format( 100 * sum(total_num_noisy_nei) / sum(total_neigbours))) print('Proportion of noisy neighbours in subgraph found in explanations: {:.2f}%'.format( 100 * sum(total_num_noise_neis) / (sum(total_num_noisy_nei) * data.num_classes))) # Plot of kernel density estimates of number of noisy features included in explanation # Do for all benchmarks (with diff colors) and plt.show() to get on the same graph total_num_noise_neis = [item/data.num_classes for item in total_num_noise_neis] plot_dist(total_num_noise_neis, label=explainer_name, color=COLOURS[c]) # else: # consider only predicted class # plot_dist(pred_class_num_noise_neis, # label=explainer_name, color=COLOURS[c]) # Random explainer - plot estimated kernel density total_num_noise_neis = noise_nodes_for_random( data, model, args_K, args_num_noise_nodes, node_indices) total_num_noise_neis= [item/data.num_classes for item in total_num_noise_neis] plot_dist(total_num_noise_neis, label='Random', color='y') plt.savefig('results/eval1_node_{}'.format(data.name)) #plt.show() return total_num_noise_neis
def filter_useless_nodes(args_dataset, args_model, args_explainers, args_hops, args_num_samples, args_test_samples, args_K, args_prop_noise_nodes, args_connectedness, node_indices, info, args_hv, args_feat, args_coal, args_g, args_multiclass, args_regu, args_gpu, args_fullempty, args_S, seed): """ Add noisy neighbours to dataset and check how many are included in explanations The fewest, the better the explainer. Args: Arguments defined in argument parser of script_eval.py """ # Define dataset data = prepare_data(args_dataset, seed=seed) # Select a random subset of nodes to eval the explainer on. if not node_indices: node_indices = extract_test_nodes(data, args_test_samples, seed) # Define number of noisy nodes according to dataset size args_num_noise_nodes = int(args_prop_noise_nodes * data.x.size(0)) args_c = eval('EVAL1_' + data.name)['args_c'] args_p = eval('EVAL1_' + data.name)['args_p'] args_binary = eval('EVAL1_' + data.name)['args_binary'] # Add noisy neighbours to the graph, with random features data = add_noise_neighbours(data, args_num_noise_nodes, node_indices, binary=args_binary, p=args_p, connectedness=args_connectedness, c=args_c) # Define training parameters depending on (model-dataset) couple hyperparam = ''.join(['hparams_', args_dataset, '_', args_model]) param = ''.join(['params_', args_dataset, '_', args_model]) # Define the model if args_model == 'GCN': model = GCN(input_dim=data.x.size( 1), output_dim=data.num_classes, **eval(hyperparam)) else: model = GAT(input_dim=data.x.size( 1), output_dim=data.num_classes, **eval(hyperparam)) # Re-train the model on dataset with noisy features train_and_val(model, data, **eval(param)) # Evaluate model model.eval() with torch.no_grad(): log_logits = model(x=data.x, edge_index=data.edge_index) # [2708, 7] test_acc = accuracy(log_logits[data.test_mask], data.y[data.test_mask]) print('Test accuracy is {:.4f}'.format(test_acc)) # Derive predicted class for each test node with torch.no_grad(): true_confs, predicted_classes = log_logits.exp()[node_indices].max(dim=1) del log_logits if args_regu == 1: args_regu = 0 # Study attention weights of noisy nodes in GAT model - compare attention with explanations if str(type(model)) == "<class 'src.models.GAT'>": study_attention_weights(data, model, args_test_samples) # Do for several explainers for c, explainer_name in enumerate(args_explainers): print('EXPLAINER: ', explainer_name) # Define the explainer explainer = eval(explainer_name)(data, model, args_gpu) # Loop on each test sample and store how many times do noisy nodes appear among # K most influential features in our explanations # count number of noisy nodes in explanations pred_class_num_noise_neis = [] # count number of noisy nodes in subgraph total_num_noisy_nei = [] # Number of neigbours of v in subgraph total_neigbours = [] # Stores number of most important neighbours we look at, for each node K = [] # To retrieve the predicted class j = 0 for node_idx in tqdm(node_indices, desc='explain node', leave=False): # Look only at coefficients for nodes (not node features) if explainer_name == 'Greedy': coefs = explainer.explain_nei(node_idx, args_hops, args_num_samples) elif explainer_name == 'GNNExplainer': _ = explainer.explain(node_idx, args_hops, args_num_samples) coefs = explainer.coefs else: # Explanations via GraphSVX coefs = explainer.explain([node_idx], args_hops, args_num_samples, info, args_multiclass, args_fullempty, args_S, args_hv, args_feat, args_coal, args_g, args_regu) coefs = coefs[0].T[explainer.F:] # Number of noisy nodes in the subgraph of node_idx num_noisy_nodes = len( [n_idx for n_idx in explainer.neighbours if n_idx >= data.x.size(0)-args_num_noise_nodes]) # Number of neighbours in the subgraph total_neigbours.append(len(explainer.neighbours)) # Adaptable K - vary according to number of nodes in the subgraph if len(explainer.neighbours) > 100: K.append(int(args_K * 100)) else: K.append( max(1, int(args_K * len(explainer.neighbours))) ) # Store indexes of K most important features, for each class nei_indices = coefs.argsort()[-K[j]:].tolist() # Number of noisy features that appear in explanations - use index to spot them noise_nei = [idx for idx in nei_indices if idx > (explainer.neighbours.shape[0] - num_noisy_nodes)] # If node importance of top K neighbours is unsignificant, discard # Possible as we have importance informative measure, unlike others. if explainer_name == 'GraphSVX': explainable_part = true_confs[c] - \ explainer.base_values[c] noise_nei = [idx for idx in noise_nei if np.abs(coefs[idx]) > 0.05*np.abs(explainable_part)] num_noise_nei = len(noise_nei) pred_class_num_noise_neis.append(num_noise_nei) # Return number of noisy nodes adjacent to node of interest total_num_noisy_nei.append(num_noisy_nodes) j += 1 print('Noisy neighbours included in explanations: ', pred_class_num_noise_neis) print('There are {} noise neighbours found in the explanations of {} test samples, an average of {} per sample' .format(sum(pred_class_num_noise_neis), args_test_samples, sum(pred_class_num_noise_neis)/args_test_samples)) print('Proportion of explanations showing noisy neighbours: {:.2f}%'.format( 100 * sum(pred_class_num_noise_neis) / sum(K))) perc = 100 * sum(pred_class_num_noise_neis) / (sum(total_num_noisy_nei)) perc2 = 100 * (sum(K) - sum(pred_class_num_noise_neis)) \ / (sum(total_neigbours) - sum(total_num_noisy_nei)) print('Proportion of noisy neighbours found in explanations vs normal neighbours (in subgraph): {:.2f}% vs {:.2f}'.format( perc, perc2)) print('Proportion of nodes in subgraph that are noisy: {:.2f}%'.format( 100 * sum(total_num_noisy_nei) / sum(total_neigbours))) print('Proportion of noisy neighbours found in explanations (entire graph): {:.2f}%'.format( 100 * sum(pred_class_num_noise_neis) / (args_test_samples * args_num_noise_nodes))) print('------------------------------------') # Plot of kernel density estimates of number of noisy features included in explanation # Do for all benchmarks (with diff colors) and plt.show() to get on the same graph plot_dist(pred_class_num_noise_neis, label=explainer_name, color=COLOURS[c]) # Random explainer - plot estimated kernel density total_num_noise_neis = noise_nodes_for_random( data, model, K, node_indices, total_num_noisy_nei, total_neigbours) plot_dist(total_num_noise_neis, label='Random', color='y') # Store graph - with key params and time now = datetime.now() current_time = now.strftime("%H:%M:%S") plt.savefig('results/eval1_node_{}_{}_{}_{}_{}.pdf'.format(data.name, args_coal, args_feat, args_hv, current_time)) plt.close() #plt.show() return total_num_noise_neis
node_indices = [ 2332, 2101, 1769, 2546, 2595, 1913, 1804, 2419, 2530, 1872, 2629, 2272, 1739, 2394, 1770, 2030, 2123, 2176, 1999, 2608 ] #node_indices= [2420,2455,1783,2165,2628,1822,2682,2261,1896,1880,2137,2237,2313,2218,1822,1719,1763,2263,2020,1988] node_indices = [ 10, 18, 89, 178, 333, 356, 378, 456, 500, 2222, 1220, 1900, 1328, 189, 1111, 124, 666, 684, 1556, 1881 ] node_indices = [ 1834, 2512, 2591, 2101, 1848, 1853, 2326, 1987, 2359, 2453, 2230, 2267, 2399, 2150, 2400, 2546, 1825, 2529, 2559, 1883 ] # Define dataset - include noisy features data = prepare_data(args_dataset, seed=10) data, noise_feat = add_noise_features(data, num_noise=args_num_noise_feat, binary=args_binary, p=args_p) # Define training parameters depending on (model-dataset) couple hyperparam = ''.join(['hparams_', args_dataset, '_', args_model]) param = ''.join(['params_', args_dataset, '_', args_model]) # Define the model if args_model == 'GCN': model = GCN(input_dim=data.x.size(1), output_dim=data.num_classes, **eval(hyperparam)) else:
def filter_useless_nodes(args_model, args_dataset, args_hops, args_num_samples, args_test_samples, args_K, args_num_noise_nodes, args_p, args_binary, args_connectedness, node_indices=None, info=True): """ Arguments defined in argument parser in script_eval.py Add noisy features to dataset and check how many are included in explanations The fewest, the better the explainer. """ ''' ####### Input in script_eval file args_dataset = 'Cora' args_model = 'GAT' args_hops = 2 args_num_samples = 100 args_test_samples = 10 args_num_noise_nodes = 20 args_K= 5 # maybe def depending on M args_p = 0.013 args_connectedness = 'medium' args_binary=True ''' #### Create function from here. Maybe create training fct first, to avoid retraining the model. # Define dataset data = prepare_data(args_dataset, seed=10) # Select random subset of nodes to eval the explainer on. if not node_indices: node_indices = extract_test_nodes(data, args_test_samples) # Include noisy neighbours data = add_noise_neighbors(data, args_num_noise_nodes, node_indices, binary=args_binary, p=args_p, connectedness=args_connectedness) # data, noise_feat = add_noise_features(data, num_noise=args_num_noise_feat, binary=True) # Define training parameters depending on (model-dataset) couple hyperparam = ''.join(['hparams_', args_dataset, '_', args_model]) param = ''.join(['params_', args_dataset, '_', args_model]) # Define the model if args_model == 'GCN': model = GCN(input_dim=data.x.size(1), output_dim=data.num_classes, **eval(hyperparam)) else: model = GAT(input_dim=data.x.size(1), output_dim=data.num_classes, **eval(hyperparam)) # Re-train the model on dataset with noisy features train_and_val(model, data, **eval(param)) # Study attention weights of noisy nodes - for 20 new nodes def study_attention_weights(data, model): """ Studies the attention weights of the GAT model """ _, alpha, alpha_bis = model(data.x, data.edge_index, att=True) edges, alpha1 = alpha[0][:, :-(data.x.size(0) - 1)], alpha[1][:-( data.x.size(0) - 1), :] # remove self loops att alpha2 = alpha_bis[1][:-(data.x.size(0) - 1)] att1 = [] att2 = [] for i in range( data.x.size(0) - args_test_samples, (data.x.size(0) - 1)): ind = (edges == i).nonzero() for j in ind[:, 1]: att1.append(torch.mean(alpha1[j])) att2.append(alpha2[j][0]) print('shape attention noisy', len(att2)) # It looks like these noisy nodes are very important print('av attention', (torch.mean(alpha1) + torch.mean(alpha2)) / 2) # 0.18 (torch.mean(torch.stack(att1)) + torch.mean(torch.stack(att2))) / 2 # 0.32 # In fact, noisy nodes are slightly below average in terms of attention received # Importance of interest: look only at imp. of noisy nei for test nodes print('attention 1 av. for noisy nodes: ', torch.mean(torch.stack(att1[0::2]))) print('attention 2 av. for noisy nodes: ', torch.mean(torch.stack(att2[0::2]))) # Study attention weights if str(type(model)) == "<class 'src.models.GAT'>": study_attention_weights(data, model) # Define explainer graphshap = GraphSHAP(data, model) # Loop on each test sample and store how many times do noise features appear among # K most influential features in our explanations total_num_noise_neis = [ ] # 1 el per test sample - count number of noisy nodes in explanations pred_class_num_noise_neis = [ ] # 1 el per test sample - count number of noisy nodes in explanations for 1 class total_num_noisy_nei = [ ] # 1 el per test sample - count number of noisy nodes in subgraph total_neigbours = [ ] # 1 el per test samples - number of neigbours of v in subgraph M = [] # 1 el per test sample - number of non zero features for node_idx in tqdm(node_indices, desc='explain node', leave=False): # Explanations via GraphSHAP coefs = graphshap.explainer(node_index=node_idx, hops=args_hops, num_samples=args_num_samples, info=False) # Check how many non zero features M.append(graphshap.M) # Number of noisy nodes in the subgraph of node_idx num_noisy_nodes = len([ n_idx for n_idx in graphshap.neighbors if n_idx >= data.x.size(0) - args_num_noise_nodes ]) total_neigbours.append(len(graphshap.neighbors)) # Multilabel classification - consider all classes instead of focusing on the # class that is predicted by our model num_noise_neis = [] # one element for each class of a test sample true_conf, predicted_class = model( x=data.x, edge_index=data.edge_index).exp()[node_idx].max(dim=0) for i in range(data.num_classes): # Store indexes of K most important features, for each class nei_indices = np.abs(coefs[:, i]).argsort()[-args_K:].tolist() # Number of noisy features that appear in explanations - use index to spot them num_noise_nei = sum(idx >= graphshap.M - num_noisy_nodes for idx in nei_indices) num_noise_neis.append(num_noise_nei) if i == predicted_class: pred_class_num_noise_neis.append(num_noise_nei) # Return this number => number of times noisy neighbours are provided as explanations total_num_noise_neis.append(sum(num_noise_neis)) # Return number of noisy nodes adjacent to node of interest total_num_noisy_nei.append(num_noisy_nodes) if info: print('Noisy neighbours included in explanations: ', total_num_noise_neis) print('There are {} noise neighbours found in the explanations of {} test samples, an average of {} per sample'\ .format(sum(total_num_noise_neis),args_test_samples,sum(total_num_noise_neis)/args_test_samples) ) print( np.sum(pred_class_num_noise_neis) / args_test_samples, 'for the predicted class only') print('Proportion of explanations showing noisy neighbours: {:.2f}%'. format(100 * sum(total_num_noise_neis) / (args_K * args_test_samples * data.num_classes))) perc = 100 * sum(total_num_noise_neis) / ( args_test_samples * args_num_noise_nodes * data.num_classes) perc2 = 100 * ( (args_K * args_test_samples * data.num_classes) - sum(total_num_noise_neis)) / (np.sum(M) - sum(total_num_noisy_nei)) print( 'Proportion of noisy neighbours found in explanations vs normal features: {:.2f}% vs {:.2f}' .format(perc, perc2)) print('Proportion of nodes in subgraph that are noisy: {:.2f}%'.format( 100 * sum(total_num_noisy_nei) / sum(total_neigbours))) print('Proportion of noisy neighbours among features: {:.2f}%'.format( 100 * sum(total_num_noisy_nei) / np.sum(M))) # Plot of kernel density estimates of number of noisy features included in explanation # Do for all benchmarks (with diff colors) and plt.show() to get on the same graph plot_dist(total_num_noise_neis, label='GraphSHAP', color='g') #plt.show() return total_num_noise_neis
def filter_useless_features(args_dataset, args_model, args_explainers, args_hops, args_num_samples, args_test_samples, args_K, args_prop_noise_feat, node_indices, info, args_hv, args_feat, args_coal, args_g, args_multiclass, args_regu, args_gpu, args_fullempty, args_S, seed): """ Add noisy features to dataset and check how many are included in explanations The fewest, the better the explainer. Args: Arguments defined in argument parser of script_eval.py """ # Define dataset data = prepare_data(args_dataset, seed=seed) args_num_noise_feat = int(data.x.size(1) * args_prop_noise_feat) args_p = eval('EVAL1_' + data.name)['args_p'] args_binary = eval('EVAL1_' + data.name)['args_binary'] # Include noisy neighbours data, noise_feat = add_noise_features( data, num_noise=args_num_noise_feat, binary=args_binary, p=args_p) # Define training parameters depending on (model-dataset) couple hyperparam = ''.join(['hparams_', args_dataset, '_', args_model]) param = ''.join(['params_', args_dataset, '_', args_model]) # Define the model if args_model == 'GCN': model = GCN(input_dim=data.x.size( 1), output_dim=data.num_classes, **eval(hyperparam)) else: model = GAT(input_dim=data.x.size( 1), output_dim=data.num_classes, **eval(hyperparam)) # Re-train the model on dataset with noisy features train_and_val(model, data, **eval(param)) # Select random subset of nodes to eval the explainer on. if not node_indices: node_indices = extract_test_nodes(data, args_test_samples, seed) # Evaluate the model on test set model.eval() with torch.no_grad(): log_logits = model(x=data.x, edge_index=data.edge_index) test_acc = accuracy(log_logits[data.test_mask], data.y[data.test_mask]) print('Test accuracy is {:.4f}'.format(test_acc)) # Derive predicted class for each test sample with torch.no_grad(): true_confs, predicted_classes = log_logits.exp()[node_indices].max(dim=1) del log_logits # Adaptable K - top k explanations we look at for each node # Depends on number of existing features considered for GraphSVX if 'GraphSVX' in args_explainers: K = [] else: K = [10]*len(node_indices) #for node_idx in node_indices: # K.append(int(data.x[node_idx].nonzero().shape[0] * args_K)) if args_regu == 0: args_regu = 1 # Loop on the different explainers selected for c, explainer_name in enumerate(args_explainers): # Define explainer explainer = eval(explainer_name)(data, model, args_gpu) print('EXPLAINER: ', explainer_name) # count noisy features found in explanations pred_class_num_noise_feats = [] # count number of noisy features considered total_num_noise_feat_considered = [] # count number of features F = [] # Loop on each test sample and store how many times do noise features appear among # K most influential features in our explanations j=0 for node_idx in tqdm(node_indices, desc='explain node', leave=False): # Explanations via GraphSVX if explainer_name == 'GraphSVX': coefs = explainer.explain( [node_idx], args_hops, args_num_samples, info, args_multiclass, args_fullempty, args_S, args_hv, args_feat, args_coal, args_g, args_regu, ) # Look only at features coefficients # Neighbours are irrelevant here coefs = coefs[0][:explainer.F] # Adaptable K if explainer.F > 100: K.append(int(args_K * 100)) else: K.append( max(1, int(explainer.F * args_K)) ) # Num_features_considered if args_feat == 'Null': feat_idx = noise_feat[explainer.neighbours, :].mean(axis=0).nonzero() num_noise_feat_considered = feat_idx.size()[0] # Consider all features (+ use expectation like below) elif args_feat == 'All': num_noise_feat_considered = args_num_noise_feat # Consider only features whose aggregated value is different from expected one else: # Stats dataset var = noise_feat.std(axis=0) mean = noise_feat.mean(axis=0) # Feature intermediate rep mean_subgraph = noise_feat[explainer.neighbours, :].mean(axis=0) # Select relevant features only - (E-e,E+e) mean_subgraph = torch.where(mean_subgraph > mean - 0.25*var, mean_subgraph, torch.ones_like(mean_subgraph)*100) mean_subgraph = torch.where(mean_subgraph < mean + 0.25*var, mean_subgraph, torch.ones_like(mean_subgraph)*100) feat_idx = (mean_subgraph == 100).nonzero() num_noise_feat_considered = feat_idx.shape[0] del mean, mean_subgraph, var else: coefs = explainer.explain(node_idx, args_hops, args_num_samples, info=False, multiclass=False )[:explainer.F] # All features are considered num_noise_feat_considered = args_num_noise_feat # Features considered F.append(explainer.F) # Store indexes of K most important node features, for each class feat_indices = coefs.argsort()[-K[j]:].tolist() # Number of noisy features that appear in explanations - use index to spot them num_noise_feat = [idx for idx in feat_indices if idx > (explainer.F - num_noise_feat_considered)] # If node importance of top K features is unsignificant, discard # Possible as we have importance informative measure, unlike others. if explainer_name == 'GraphSVX': explainable_part = true_confs[c] - \ explainer.base_values[c] num_noise_feat = [idx for idx in num_noise_feat if np.abs(coefs[idx]) > 0.05*np.abs(explainable_part)] # Count number of noisy that appear in explanations num_noise_feat = len(num_noise_feat) pred_class_num_noise_feats.append(num_noise_feat) # Return number of noisy features considered in this test sample total_num_noise_feat_considered.append(num_noise_feat_considered) j+=1 print('Noisy features included in explanations: ', sum(pred_class_num_noise_feats) ) print('For the predicted class, there are {} noise features found in the explanations of {} test samples, an average of {} per sample' .format(sum(pred_class_num_noise_feats), args_test_samples, sum(pred_class_num_noise_feats)/args_test_samples)) print(pred_class_num_noise_feats) if sum(F) != 0: perc = 100 * sum(total_num_noise_feat_considered) / sum(F) print( 'Proportion of considered noisy features among features: {:.2f}%'.format(perc)) if sum(K) != 0: perc = 100 * sum(pred_class_num_noise_feats) / sum(K) print('Proportion of explanations showing noisy features: {:.2f}%'.format(perc)) if sum(total_num_noise_feat_considered) != 0: perc = 100 * sum(pred_class_num_noise_feats) / (sum(total_num_noise_feat_considered)) perc2 = 100 * (sum(K) - sum(pred_class_num_noise_feats)) / (sum(F) - sum(total_num_noise_feat_considered)) print('Proportion of noisy features found in explanations vs proportion of normal features (among considered ones): {:.2f}% vs {:.2f}%, over considered features only'.format( perc, perc2)) print('------------------------------------') # Plot of kernel density estimates of number of noisy features included in explanation # Do for all benchmarks (with diff colors) and plt.show() to get on the same graph plot_dist(pred_class_num_noise_feats, label=explainer_name, color=COLOURS[c]) # Random explainer - plot estimated kernel density total_num_noise_feats = noise_feats_for_random( data, model, K, args_num_noise_feat, node_indices) save_path = 'results/eval1_feat' plot_dist(total_num_noise_feats, label='Random', color='y') # Store graph - with key params and time now = datetime.now() current_time = now.strftime("%H:%M:%S") plt.savefig('results/eval1_feat_{}_{}_{}_{}_{}.pdf'.format(data.name, args_coal, args_feat, args_hv, current_time)) plt.close()
def filter_useless_features(args_model, args_dataset, args_explainers, args_hops, args_num_samples, args_test_samples, args_K, args_num_noise_feat, args_p, args_binary, node_indices, info=True): """ Arguments defined in argument parser of script_eval.py Add noisy features to dataset and check how many are included in explanations The fewest, the better the explainer. """ ''' ####### Input in script_eval file args_dataset = 'Cora' args_model = 'GCN' args_explainers = ['GraphSHAP', 'Greedy'] args_hops = 2 args_num_samples = 100 # size shap dataset args_test_samples = 20 # number of test samples args_num_noise_feat= 25 # number of noisy features args_K= 5 # maybe def depending on M args_binary = True args_p = 0.5 info=True node_indices= [2420,2455,1783,2165,2628,1822,2682,2261,1896,1880,2137,2237,2313,2218,1822,1719,1763,2263,2020,1988] node_indices = [10, 18, 89, 178, 333, 356, 378, 456, 500, 2222, 1220, 1900, 1328, 189, 1111] node_indices = [1834,2512,2591,2101,1848,1853,2326,1987,2359,2453,2230,2267,2399, 2150,2400] ''' #### Create function from here. Maybe create training fct first, to avoid retraining the model. # Define dataset - include noisy features data = prepare_data(args_dataset, seed=10) data, noise_feat = add_noise_features(data, num_noise=args_num_noise_feat, binary=args_binary, p=args_p) # Define training parameters depending on (model-dataset) couple hyperparam = ''.join(['hparams_', args_dataset, '_', args_model]) param = ''.join(['params_', args_dataset, '_', args_model]) # Define the model if args_model == 'GCN': model = GCN(input_dim=data.x.size(1), output_dim=data.num_classes, **eval(hyperparam)) else: model = GAT(input_dim=data.x.size(1), output_dim=data.num_classes, **eval(hyperparam)) # Re-train the model on dataset with noisy features train_and_val(model, data, **eval(param)) # Select random subset of nodes to eval the explainer on. if not node_indices: node_indices = extract_test_nodes(data, args_test_samples) for explainer_name in args_explainers: # Define explainer explainer = eval(explainer_name)(data, model) total_num_noise_feats = [ ] # count noisy features found in explanations for each test sample (for each class) pred_class_num_noise_feats = [ ] # count noisy features found in explanations for each test sample for class of interest total_num_non_zero_noise_feat = [ ] # count number of noisy features considered for each test sample M = [] # count number of non zero features for each test sample # Loop on each test sample and store how many times do noise features appear among # K most influential features in our explanations for node_idx in tqdm(node_indices, desc='explain node', leave=False): # Explanations via GraphSHAP coefs = explainer.explainer(node_index=node_idx, hops=args_hops, num_samples=args_num_samples, info=False) # Check how many non zero features M.append(explainer.M) # Number of non zero noisy features num_non_zero_noise_feat = len( [val for val in noise_feat[node_idx] if val != 0]) # Multilabel classification - consider all classes instead of focusing on the # class that is predicted by our model num_noise_feats = [] true_conf, predicted_class = model( x=data.x, edge_index=data.edge_index).exp()[node_idx].max(dim=0) for i in range(data.num_classes): # Store indexes of K most important features, for each class feat_indices = np.abs(coefs[:, i]).argsort()[-args_K:].tolist() # Number of noisy features that appear in explanations - use index to spot them num_noise_feat = sum(idx < num_non_zero_noise_feat for idx in feat_indices) num_noise_feats.append(num_noise_feat) if i == predicted_class: pred_class_num_noise_feats.append(num_noise_feat) # Return this number => number of times noisy features are provided as explanations total_num_noise_feats.append(sum(num_noise_feats)) # Return number of noisy features considered in this test sample total_num_non_zero_noise_feat.append(num_non_zero_noise_feat) if info: print('Noise features included in explanations: ', total_num_noise_feats) print('There are {} noise features found in the explanations of {} test samples, an average of {} per sample'\ .format(sum(total_num_noise_feats),args_test_samples,sum(total_num_noise_feats)/args_test_samples) ) # Number of noisy features found in explanation for the predicted class print( np.sum(pred_class_num_noise_feats) / args_test_samples, 'for the predicted class only') perc = 100 * sum(total_num_non_zero_noise_feat) / np.sum(M) print('Overall proportion of considered noisy features : {:.2f}%'. format(perc)) perc = 100 * sum(total_num_noise_feats) / ( args_K * args_test_samples * data.num_classes) print('Percentage of explanations showing noisy features: {:.2f}%'. format(perc)) if sum(total_num_non_zero_noise_feat) != 0: perc = 100 * sum(total_num_noise_feats) / ( sum(total_num_non_zero_noise_feat) * data.num_classes) perc2 = 100 * ( args_K * args_test_samples * data.num_classes - sum(total_num_noise_feats)) / ( data.num_classes * (sum(M) - sum(total_num_non_zero_noise_feat))) print( 'Proportion of noisy features found in explanations vs normal features: {:.2f}% vs {:.2f}%, over considered features only' .format(perc, perc2)) print('------------------------------------') # Plot of kernel density estimates of number of noisy features included in explanation # Do for all benchmarks (with diff colors) and plt.show() to get on the same graph plot_dist(total_num_noise_feats, label=explainer_name, color='g') plt.show() return sum(total_num_noise_feats)
def eval_shap(args_dataset, args_model, args_test_samples, args_hops, args_K, args_num_samples, node_indices, info, args_hv, args_feat, args_coal, args_g, args_multiclass, args_regu, args_gpu, args_fullempty, args_S, seed): """ Compares SHAP and GraphSVX on graph based datasets Check if they agree on features'contribution towards prediction for several test samples """ # Define dataset data = prepare_data(args_dataset, seed=10) # Select a random subset of nodes to eval the explainer on. if not node_indices: node_indices = extract_test_nodes(data, args_test_samples, seed) # Define training parameters depending on (model-dataset) couple hyperparam = ''.join(['hparams_', args_dataset, '_', args_model]) param = ''.join(['params_', args_dataset, '_', args_model]) # Define the model if args_model == 'GCN': model = GCN(input_dim=data.x.size( 1), output_dim=data.num_classes, **eval(hyperparam)) else: model = GAT(input_dim=data.x.size( 1), output_dim=data.num_classes, **eval(hyperparam)) # Re-train the model on dataset with noisy features train_and_val(model, data, **eval(param)) # Store metrics iou = [] prop_contrib_diff = [] # Iterate over test samples for node_idx in tqdm(node_indices, desc='explain node', leave=False): # Define explainers we would like to compare graphshap = GraphSVX(data, model, args_gpu) shap = SHAP(data, model, args_gpu) # Explanations via GraphSVX graphshap_coefs = graphshap.explain([node_idx], args_hops, args_num_samples, info, args_multiclass, args_fullempty, args_S, args_hv, args_feat, # All args_coal, # Random or SmarerSoftRegu args_g, # WLS args_regu) # 1 graphshap_coefs = graphshap_coefs[0].T[:graphshap.F] shap_coefs = shap.explain(node_idx, args_hops, args_num_samples, info=False, multiclass=False )[:shap.F] # Consider node features only - for predicted class only true_conf, predicted_class = model(x=data.x, edge_index=data.edge_index).exp()[ node_idx].max(dim=0) # Need to apply regularisation # Proportional contribution prop_contrib_diff.append(np.abs( graphshap_coefs.sum( ) / np.abs(graphshap_coefs).sum() - shap_coefs.sum() / np.abs(shap_coefs).sum())) #print('GraphSVX proportional contribution to pred: {:.2f}'.format(graphshap_coefs.sum() / np.abs(graphshap_coefs).sum() )) #print('SHAP proportional contribution to pred: {:.2f}'.format(shap_coefs.sum() / np.abs(shap_coefs).sum() )) # Important features graphshap_feat_indices = np.abs(graphshap_coefs).argsort()[-10:].tolist() shap_feat_indices = np.abs(shap_coefs).argsort()[-10:].tolist() iou.append(len(set(graphshap_feat_indices).intersection(set(shap_feat_indices)) ) / len(set(graphshap_feat_indices).union(set(shap_feat_indices)))) #print('Iou important features: ', iou) print('iou av:', np.mean(iou)) print('difference in contibutions towards pred: ', np.mean(prop_contrib_diff))
def filter_useless_features_multiclass(args_dataset, args_model, args_explainers, args_hops, args_num_samples, args_test_samples, args_prop_noise_nodes, args_connectedness, node_indices, args_K, info, args_hv, args_feat, args_coal, args_g, args_multiclass, args_regu, args_gpu, args_fullempty, args_S, seed): """ Add noisy features to dataset and check how many are included in explanations The fewest, the better the explainer. Args: Arguments defined in argument parser of script_eval.py """ # Define dataset - include noisy features data = prepare_data(args_dataset, seed=10) args_num_noise_feat = int(data.x.size(1) * args_prop_noise_feat) args_p = eval('EVAL1_' + data.name)['args_p'] args_binary = eval('EVAL1_' + data.name)['args_binary'] data, noise_feat = add_noise_features( data, num_noise=args_num_noise_feat, binary=args_binary, p=args_p) # Define training parameters depending on (model-dataset) couple hyperparam = ''.join(['hparams_', args_dataset, '_', args_model]) param = ''.join(['params_', args_dataset, '_', args_model]) # Define the model if args_model == 'GCN': model = GCN(input_dim=data.x.size( 1), output_dim=data.num_classes, **eval(hyperparam)) else: model = GAT(input_dim=data.x.size( 1), output_dim=data.num_classes, **eval(hyperparam)) # Re-train the model on dataset with noisy features train_and_val(model, data, **eval(param)) # Select random subset of nodes to eval the explainer on. if not node_indices: node_indices = extract_test_nodes(data, args_test_samples, seed) # Evaluate the model - test set model.eval() with torch.no_grad(): log_logits = model(x=data.x, edge_index=data.edge_index) # [2708, 7] test_acc = accuracy(log_logits[data.test_mask], data.y[data.test_mask]) print('Test accuracy is {:.4f}'.format(test_acc)) del log_logits # Loop on different explainers selected for c, explainer_name in enumerate(args_explainers): # Define explainer explainer = eval(explainer_name)(data, model, args_gpu) # count noisy features found in explanations for each test sample (for each class) total_num_noise_feats = [] # count noisy features found in explanations for each test sample for class of interest pred_class_num_noise_feats = [] # count number of noisy features considered for each test sample (for each class) total_num_noise_feat_considered = [] F = [] # count number of non zero features for each test sample # Loop on each test sample and store how many times do noise features appear among # K most influential features in our explanations for node_idx in tqdm(node_indices, desc='explain node', leave=False): if explainer_name == 'GraphSVX': coefs = explainer.explain( [node_idx], args_hops, args_num_samples, info, args_multiclass, args_fullempty, args_S, args_hv, args_feat, args_coal, args_g, args_regu) coefs = coefs[0].T[:explainer.F] # Explanations via GraphSVX else: coefs = explainer.explain(node_index=node_idx, hops=args_hops, num_samples=args_num_samples, info=False, multiclass=True) # Check how many non zero features F.append(explainer.F) # Number of non zero noisy features # Dfferent for explainers with all features considered vs non zero features only (shap,graphshap) # if explainer.F != data.x.size(1) if explainer_name == 'GraphSVX' or explainer_name == 'SHAP': num_noise_feat_considered = len( [val for val in noise_feat[node_idx] if val != 0]) else: num_noise_feat_considered = args_num_noise_feat # Multilabel classification - consider all classes instead of focusing on the # class that is predicted by our model num_noise_feats = [] true_conf, predicted_class = model(x=data.x, edge_index=data.edge_index).exp()[ node_idx].max(dim=0) for i in range(data.num_classes): # Store indexes of K most important node features, for each class feat_indices = np.abs( coefs[:explainer.F, i]).argsort()[-args_K:].tolist() # Number of noisy features that appear in explanations - use index to spot them num_noise_feat = sum( idx < num_noise_feat_considered for idx in feat_indices) num_noise_feats.append(num_noise_feat) # For predicted class only if i == predicted_class: pred_class_num_noise_feats.append(num_noise_feat) # Return number of times noisy features are provided as explanations total_num_noise_feats.append(sum(num_noise_feats)) # Return number of noisy features considered in this test sample total_num_noise_feat_considered.append(num_noise_feat_considered) if info: print('Noise features included in explanations: ', total_num_noise_feats) print('There are {} noise features found in the explanations of {} test samples, an average of {} per sample' .format(sum(total_num_noise_feats), args_test_samples, sum(total_num_noise_feats)/args_test_samples)) # Number of noisy features found in explanation for the predicted class print(np.sum(pred_class_num_noise_feats) / args_test_samples, 'for the predicted class only') perc = 100 * sum(total_num_noise_feat_considered) / np.sum(F) print( 'Proportion of non-zero noisy features among non-zero features: {:.2f}%'.format(perc)) perc = 100 * sum(total_num_noise_feats) / \ (args_K * args_test_samples * data.num_classes) print( 'Proportion of explanations showing noisy features: {:.2f}%'.format(perc)) if sum(total_num_noise_feat_considered) != 0: perc = 100 * sum(total_num_noise_feats) / \ (sum(total_num_noise_feat_considered)*data.num_classes) perc2 = 100 * (args_K * args_test_samples * data.num_classes - sum(total_num_noise_feats)) / ( data.num_classes * (sum(F) - sum(total_num_noise_feat_considered))) print('Proportion of noisy features found in explanations vs normal features (among considered ones): {:.2f}% vs {:.2f}%, over considered features only'.format( perc, perc2)) print('------------------------------------') # Plot of kernel density estimates of number of noisy features included in explanation # Do for all benchmarks (with diff colors) and plt.show() to get on the same graph # plot_dist(total_num_noise_feats, label=explainer_name, color=COLOURS[c]) plot_dist(total_num_noise_feats, label=explainer_name, color=COLOURS[c]) # Random explainer - plot estimated kernel density total_num_noise_feats = noise_feats_for_random( data, model, args_K, args_num_noise_feat, node_indices) save_path = 'results/eval1_feat' plot_dist(total_num_noise_feats, label='Random', color='y') plt.savefig('results/eval1_feat_{}'.format(data.name))