def _get_accuracy_scores(self, sess: tf.compat.v1.Session,
edges_pos: Dict[Tuple[int, int], List[np.array]],
edges_neg: Dict[Tuple[int, int], List[np.array]],
edge_type: Tuple[int, int, int]):
"""
Calculate metrics (AUROC, AUPRC, AP@50)
Parameters
----------
sess : tf.compat.v1.Session
Initialized tf session.
edges_pos : Dict[Tuple[int, int], List[np.array]]
From edge type to np.arrays of real edges for every edge class in this type.
edges_neg : Dict[Tuple[int, int], List[np.array]]
From edge type to np.arrays of fake edges for every edge class in this type.
edge_type : Tuple[int, int, int]
Edge type with class.
Two first elements --- edge type, last element --- class in this type.
Returns
-------
"""
self.feed_dict.update({self.placeholders['dropout']: 0})
self.feed_dict.update({self.placeholders['batch_edge_type_idx']:
self.minibatch.edge_type2idx[edge_type]})
self.feed_dict.update({self.placeholders['batch_row_edge_type']: edge_type[0]})
self.feed_dict.update({self.placeholders['batch_col_edge_type']: edge_type[1]})
rec = sess.run(self.opt.predictions, feed_dict=self.feed_dict)
uv = edges_pos[edge_type[:2]][edge_type[2]]
u = uv[:, 0]
v = uv[:, 1]
preds = expit(rec[u, v])
assert np.all(self.adj_mats[edge_type[:2]][edge_type[2]][u, v] == 1), \
'Positive examples (real edges) are not exist'
uv = edges_neg[edge_type[:2]][edge_type[2]]
u = uv[:, 0]
v = uv[:, 1]
preds_neg = expit(rec[u, v])
assert np.all(self.adj_mats[edge_type[:2]][edge_type[2]][u, v] == 0), \
'Negative examples (fake edges) are real'
# Predicted probs
preds_all = np.hstack([preds, preds_neg])
# preds_all = np.nan_to_num(preds_all)
# Real probs: 1 for pos, 0 for neg
labels_all = np.hstack([np.ones(len(preds)), np.zeros(len(preds_neg))])
roc_sc = metrics.roc_auc_score(labels_all, preds_all)
aupr_sc = metrics.average_precision_score(labels_all, preds_all)
# Real existing edges (local indexes)
actual = range(len(preds))
# All local indexes with probability (sorted)
predicted = sorted(range(len(preds_all)), reverse=True,
key=lambda i: preds_all[i])
apk_sc = rank_metrics.apk(actual, predicted, k=50)
return roc_sc, aupr_sc, apk_sc
def get_accuracy_scores(edges_pos, edges_neg, edge_type):
feed_dict.update({placeholders['dropout']: 0})
feed_dict.update({
placeholders['batch_edge_type_idx']:
minibatch.edge_type2idx[edge_type]
})
feed_dict.update({placeholders['batch_row_edge_type']: edge_type[0]})
feed_dict.update({placeholders['batch_col_edge_type']: edge_type[1]})
rec = sess.run(opt.predictions, feed_dict=feed_dict)
def sigmoid(x):
return 1. / (1 + np.exp(-x))
# Predict on test set of edges
preds = []
actual = []
predicted = []
edge_ind = 0
# pos
for u, v in edges_pos[edge_type[:2]][edge_type[2]]:
score = sigmoid(rec[u, v])
preds.append(score)
assert adj_mats_orig[edge_type[:2]][edge_type[2]][
u, v] == 1, 'Problem 1'
actual.append(edge_ind)
predicted.append((score, edge_ind))
edge_ind += 1
preds_neg = []
# neg
for u, v in edges_neg[edge_type[:2]][edge_type[2]]:
score = sigmoid(rec[u, v])
preds_neg.append(score)
assert adj_mats_orig[edge_type[:2]][edge_type[2]][
u, v] == 0, 'Problem 0'
predicted.append((score, edge_ind))
edge_ind += 1
preds_all = np.hstack([preds, preds_neg])
preds_all = np.nan_to_num(preds_all)
labels_all = np.hstack([np.ones(len(preds)), np.zeros(len(preds_neg))])
predicted = list(
zip(*sorted(predicted, reverse=True, key=itemgetter(0))))[1]
# evatalution.....
roc_sc = metrics.roc_auc_score(labels_all, preds_all)
aupr_sc = metrics.average_precision_score(labels_all, preds_all)
apk_sc = rank_metrics.apk(actual, predicted, k=10)
return roc_sc, aupr_sc, apk_sc
def get_accuracy_scores(edges_pos, edges_neg, edge_type, feed_dict):
"""
Measure AUROC, AUPRC, and AP@50 for the given edge type using the provided list of positive and negative examples.
:param edges_pos: Positive examples to measure accuracy over (need not be only edge_type type edges)
:param edges_neg: Negative examples to measure accuracy over (need not be only edge_type type edges)
:param edge_type: Edge type to filter by
:param feed_dict: feed dictionary which should contain placeholders etc. See EdgeMiniBatchIterator#update_feed_dict
:return: auroc, auprc, apk@50
"""
actual, predicted, all_scores, all_labels, _, _, _ = get_predictions(
edge_type, edges_neg, edges_pos, feed_dict)
predicted = list(
zip(*sorted(predicted, reverse=True, key=itemgetter(0))))[1]
auroc = metrics.roc_auc_score(all_labels, all_scores)
auprc = metrics.average_precision_score(all_labels, all_scores)
apk_score = rank_metrics.apk(actual, predicted, k=50)
return auroc, auprc, apk_score
def get_accuracy_scores(edges_pos, edges_neg, edge_type):
feed_dict.update({placeholders['dropout']: 0})
rec = sess.run(opt.preds, feed_dict=feed_dict)
def sigmoid(x):
return 1 / (1 + np.exp(-x))
# Predict on test set of edges
preds = []
actual = []
predicted = []
edge_ind = 0
for u, v in edges_pos[edge_type[:2]][edge_type[2]]:
score = sigmoid(rec[u, v])
preds.append(score)
actual.append(edge_ind)
predicted.append((score, edge_ind))
edge_ind += 1
preds_neg = []
for u, v in edges_neg[edge_type[:2]][edge_type[2]]:
score = sigmoid(rec[u, v])
preds_neg.append(score)
predicted.append((score, edge_ind))
edge_ind += 1
preds_all = np.hstack([preds, preds_neg])
preds_all = np.nan_to_num(preds_all)
labels_all = np.hstack([np.ones(len(preds)), np.zeros(len(preds))])
predicted = zip(*sorted(predicted, reverse=True, key=itemgetter(0)))[1]
roc_score = metrics.roc_auc_score(labels_all, preds_all)
aupr_score = metrics.average_precision_score(labels_all, preds_all)
apk_score = rank_metrics.apk(actual, predicted, k=50)
return roc_score, aupr_score, apk_score
def get_final_accuracy_scores(edges_pos, edges_neg, edge_type):
feed_dict.update({placeholders['dropout']: 0})
feed_dict.update({
placeholders['batch_edge_type_idx']:
minibatch.edge_type2idx[edge_type]
})
feed_dict.update({placeholders['batch_row_edge_type']: edge_type[0]})
feed_dict.update({placeholders['batch_col_edge_type']: edge_type[1]})
rec = sess.run(opt.predictions, feed_dict=feed_dict)
def sigmoid(x):
return 1. / (1 + np.exp(-x))
# Predict on test set of edges
preds = []
actual = []
predicted = []
edge_ind = 0
# pos
for u, v in edges_pos[edge_type[:2]][edge_type[2]]:
score = sigmoid(rec[u, v])
preds.append(score)
assert adj_mats_orig[edge_type[:2]][edge_type[2]][
u, v] == 1, 'Problem 1'
actual.append(edge_ind)
predicted.append((score, edge_ind))
edge_ind += 1
preds_neg = []
# neg
for u, v in edges_neg[edge_type[:2]][edge_type[2]]:
score = sigmoid(rec[u, v])
preds_neg.append(score)
assert adj_mats_orig[edge_type[:2]][edge_type[2]][
u, v] == 0, 'Problem 0'
predicted.append((score, edge_ind))
edge_ind += 1
# hstac
preds_all = np.hstack([preds, preds_neg])
preds_all = np.nan_to_num(preds_all)
labels_all = np.hstack([np.ones(len(preds)), np.zeros(len(preds_neg))])
predicted = list(
zip(*sorted(predicted, reverse=True, key=itemgetter(0))))[1]
# evatalution.....
roc_sc = metrics.roc_auc_score(labels_all, preds_all)
aupr_sc = metrics.average_precision_score(labels_all, preds_all)
apk_sc = rank_metrics.apk(actual, predicted, k=10)
FPR, TPR, thresholds = metrics.roc_curve(labels_all, preds_all)
precision, recall, _ = metrics.precision_recall_curve(
labels_all, preds_all)
mse = metrics.mean_squared_error(labels_all, preds_all)
mae = metrics.median_absolute_error(labels_all, preds_all)
r2 = metrics.r2_score(labels_all, preds_all)
np.savetxt('./log/' + data_set + str(nowTime) + '/' + str(seed) + '' +
str(edge_type) + '_' + str(nowTime) + '_true.txt',
labels_all,
fmt='%d')
np.savetxt('./log/' + data_set + str(nowTime) + '/' + str(seed) + '' +
str(edge_type) + '_' + str(nowTime) + '_pred.txt',
preds_all,
fmt='%.3f')
preds_all[preds_all >= 0.5] = 1
preds_all[preds_all < 0.5] = 0
acc = metrics.accuracy_score(labels_all, preds_all)
# metrics.accuracy_score(labels_all,preds_all)
f1 = metrics.f1_score(labels_all, preds_all, average='macro')
return FPR, TPR, roc_sc, \
precision,recall,aupr_sc, \
apk_sc , thresholds ,mse, mae,r2,acc,f1
