def _get_accuracy_scores(self, sess: tf.compat.v1.Session, edges_pos: Dict[Tuple[int, int], List[np.array]], edges_neg: Dict[Tuple[int, int], List[np.array]], edge_type: Tuple[int, int, int]): """ Calculate metrics (AUROC, AUPRC, AP@50) Parameters ---------- sess : tf.compat.v1.Session Initialized tf session. edges_pos : Dict[Tuple[int, int], List[np.array]] From edge type to np.arrays of real edges for every edge class in this type. edges_neg : Dict[Tuple[int, int], List[np.array]] From edge type to np.arrays of fake edges for every edge class in this type. edge_type : Tuple[int, int, int] Edge type with class. Two first elements --- edge type, last element --- class in this type. Returns ------- """ self.feed_dict.update({self.placeholders['dropout']: 0}) self.feed_dict.update({self.placeholders['batch_edge_type_idx']: self.minibatch.edge_type2idx[edge_type]}) self.feed_dict.update({self.placeholders['batch_row_edge_type']: edge_type[0]}) self.feed_dict.update({self.placeholders['batch_col_edge_type']: edge_type[1]}) rec = sess.run(self.opt.predictions, feed_dict=self.feed_dict) uv = edges_pos[edge_type[:2]][edge_type[2]] u = uv[:, 0] v = uv[:, 1] preds = expit(rec[u, v]) assert np.all(self.adj_mats[edge_type[:2]][edge_type[2]][u, v] == 1), \ 'Positive examples (real edges) are not exist' uv = edges_neg[edge_type[:2]][edge_type[2]] u = uv[:, 0] v = uv[:, 1] preds_neg = expit(rec[u, v]) assert np.all(self.adj_mats[edge_type[:2]][edge_type[2]][u, v] == 0), \ 'Negative examples (fake edges) are real' # Predicted probs preds_all = np.hstack([preds, preds_neg]) # preds_all = np.nan_to_num(preds_all) # Real probs: 1 for pos, 0 for neg labels_all = np.hstack([np.ones(len(preds)), np.zeros(len(preds_neg))]) roc_sc = metrics.roc_auc_score(labels_all, preds_all) aupr_sc = metrics.average_precision_score(labels_all, preds_all) # Real existing edges (local indexes) actual = range(len(preds)) # All local indexes with probability (sorted) predicted = sorted(range(len(preds_all)), reverse=True, key=lambda i: preds_all[i]) apk_sc = rank_metrics.apk(actual, predicted, k=50) return roc_sc, aupr_sc, apk_sc
def get_accuracy_scores(edges_pos, edges_neg, edge_type): feed_dict.update({placeholders['dropout']: 0}) feed_dict.update({ placeholders['batch_edge_type_idx']: minibatch.edge_type2idx[edge_type] }) feed_dict.update({placeholders['batch_row_edge_type']: edge_type[0]}) feed_dict.update({placeholders['batch_col_edge_type']: edge_type[1]}) rec = sess.run(opt.predictions, feed_dict=feed_dict) def sigmoid(x): return 1. / (1 + np.exp(-x)) # Predict on test set of edges preds = [] actual = [] predicted = [] edge_ind = 0 # pos for u, v in edges_pos[edge_type[:2]][edge_type[2]]: score = sigmoid(rec[u, v]) preds.append(score) assert adj_mats_orig[edge_type[:2]][edge_type[2]][ u, v] == 1, 'Problem 1' actual.append(edge_ind) predicted.append((score, edge_ind)) edge_ind += 1 preds_neg = [] # neg for u, v in edges_neg[edge_type[:2]][edge_type[2]]: score = sigmoid(rec[u, v]) preds_neg.append(score) assert adj_mats_orig[edge_type[:2]][edge_type[2]][ u, v] == 0, 'Problem 0' predicted.append((score, edge_ind)) edge_ind += 1 preds_all = np.hstack([preds, preds_neg]) preds_all = np.nan_to_num(preds_all) labels_all = np.hstack([np.ones(len(preds)), np.zeros(len(preds_neg))]) predicted = list( zip(*sorted(predicted, reverse=True, key=itemgetter(0))))[1] # evatalution..... roc_sc = metrics.roc_auc_score(labels_all, preds_all) aupr_sc = metrics.average_precision_score(labels_all, preds_all) apk_sc = rank_metrics.apk(actual, predicted, k=10) return roc_sc, aupr_sc, apk_sc
def get_accuracy_scores(edges_pos, edges_neg, edge_type, feed_dict): """ Measure AUROC, AUPRC, and AP@50 for the given edge type using the provided list of positive and negative examples. :param edges_pos: Positive examples to measure accuracy over (need not be only edge_type type edges) :param edges_neg: Negative examples to measure accuracy over (need not be only edge_type type edges) :param edge_type: Edge type to filter by :param feed_dict: feed dictionary which should contain placeholders etc. See EdgeMiniBatchIterator#update_feed_dict :return: auroc, auprc, apk@50 """ actual, predicted, all_scores, all_labels, _, _, _ = get_predictions( edge_type, edges_neg, edges_pos, feed_dict) predicted = list( zip(*sorted(predicted, reverse=True, key=itemgetter(0))))[1] auroc = metrics.roc_auc_score(all_labels, all_scores) auprc = metrics.average_precision_score(all_labels, all_scores) apk_score = rank_metrics.apk(actual, predicted, k=50) return auroc, auprc, apk_score
def get_accuracy_scores(edges_pos, edges_neg, edge_type): feed_dict.update({placeholders['dropout']: 0}) rec = sess.run(opt.preds, feed_dict=feed_dict) def sigmoid(x): return 1 / (1 + np.exp(-x)) # Predict on test set of edges preds = [] actual = [] predicted = [] edge_ind = 0 for u, v in edges_pos[edge_type[:2]][edge_type[2]]: score = sigmoid(rec[u, v]) preds.append(score) actual.append(edge_ind) predicted.append((score, edge_ind)) edge_ind += 1 preds_neg = [] for u, v in edges_neg[edge_type[:2]][edge_type[2]]: score = sigmoid(rec[u, v]) preds_neg.append(score) predicted.append((score, edge_ind)) edge_ind += 1 preds_all = np.hstack([preds, preds_neg]) preds_all = np.nan_to_num(preds_all) labels_all = np.hstack([np.ones(len(preds)), np.zeros(len(preds))]) predicted = zip(*sorted(predicted, reverse=True, key=itemgetter(0)))[1] roc_score = metrics.roc_auc_score(labels_all, preds_all) aupr_score = metrics.average_precision_score(labels_all, preds_all) apk_score = rank_metrics.apk(actual, predicted, k=50) return roc_score, aupr_score, apk_score
def get_final_accuracy_scores(edges_pos, edges_neg, edge_type): feed_dict.update({placeholders['dropout']: 0}) feed_dict.update({ placeholders['batch_edge_type_idx']: minibatch.edge_type2idx[edge_type] }) feed_dict.update({placeholders['batch_row_edge_type']: edge_type[0]}) feed_dict.update({placeholders['batch_col_edge_type']: edge_type[1]}) rec = sess.run(opt.predictions, feed_dict=feed_dict) def sigmoid(x): return 1. / (1 + np.exp(-x)) # Predict on test set of edges preds = [] actual = [] predicted = [] edge_ind = 0 # pos for u, v in edges_pos[edge_type[:2]][edge_type[2]]: score = sigmoid(rec[u, v]) preds.append(score) assert adj_mats_orig[edge_type[:2]][edge_type[2]][ u, v] == 1, 'Problem 1' actual.append(edge_ind) predicted.append((score, edge_ind)) edge_ind += 1 preds_neg = [] # neg for u, v in edges_neg[edge_type[:2]][edge_type[2]]: score = sigmoid(rec[u, v]) preds_neg.append(score) assert adj_mats_orig[edge_type[:2]][edge_type[2]][ u, v] == 0, 'Problem 0' predicted.append((score, edge_ind)) edge_ind += 1 # hstac preds_all = np.hstack([preds, preds_neg]) preds_all = np.nan_to_num(preds_all) labels_all = np.hstack([np.ones(len(preds)), np.zeros(len(preds_neg))]) predicted = list( zip(*sorted(predicted, reverse=True, key=itemgetter(0))))[1] # evatalution..... roc_sc = metrics.roc_auc_score(labels_all, preds_all) aupr_sc = metrics.average_precision_score(labels_all, preds_all) apk_sc = rank_metrics.apk(actual, predicted, k=10) FPR, TPR, thresholds = metrics.roc_curve(labels_all, preds_all) precision, recall, _ = metrics.precision_recall_curve( labels_all, preds_all) mse = metrics.mean_squared_error(labels_all, preds_all) mae = metrics.median_absolute_error(labels_all, preds_all) r2 = metrics.r2_score(labels_all, preds_all) np.savetxt('./log/' + data_set + str(nowTime) + '/' + str(seed) + '' + str(edge_type) + '_' + str(nowTime) + '_true.txt', labels_all, fmt='%d') np.savetxt('./log/' + data_set + str(nowTime) + '/' + str(seed) + '' + str(edge_type) + '_' + str(nowTime) + '_pred.txt', preds_all, fmt='%.3f') preds_all[preds_all >= 0.5] = 1 preds_all[preds_all < 0.5] = 0 acc = metrics.accuracy_score(labels_all, preds_all) # metrics.accuracy_score(labels_all,preds_all) f1 = metrics.f1_score(labels_all, preds_all, average='macro') return FPR, TPR, roc_sc, \ precision,recall,aupr_sc, \ apk_sc , thresholds ,mse, mae,r2,acc,f1