def metrics(self, y_sig, y):
y_pred = y_sig > 0.5
return {
'acc': metricsF.accuracy(y_pred, y),
'roc': metricsF.auroc(y_sig, y),
'iou': metricsF.iou(y_pred, y),
}
def test_v1_5_metric_auc_auroc():
AUC.__init__._warned = False
with pytest.deprecated_call(match='It will be removed in v1.5.0'):
AUC()
ROC.__init__._warned = False
with pytest.deprecated_call(match='It will be removed in v1.5.0'):
ROC()
AUROC.__init__._warned = False
with pytest.deprecated_call(match='It will be removed in v1.5.0'):
AUROC()
x = torch.tensor([0, 1, 2, 3])
y = torch.tensor([0, 1, 2, 2])
auc._warned = False
with pytest.deprecated_call(match='It will be removed in v1.5.0'):
assert auc(x, y) == torch.tensor(4.)
preds = torch.tensor([0, 1, 2, 3])
target = torch.tensor([0, 1, 1, 1])
roc._warned = False
with pytest.deprecated_call(match='It will be removed in v1.5.0'):
fpr, tpr, thrs = roc(preds, target, pos_label=1)
assert torch.equal(fpr, torch.tensor([0., 0., 0., 0., 1.]))
assert torch.allclose(tpr, torch.tensor([0.0000, 0.3333, 0.6667, 1.0000, 1.0000]), atol=1e-4)
assert torch.equal(thrs, torch.tensor([4, 3, 2, 1, 0]))
preds = torch.tensor([0.13, 0.26, 0.08, 0.19, 0.34])
target = torch.tensor([0, 0, 1, 1, 1])
auroc._warned = False
with pytest.deprecated_call(match='It will be removed in v1.5.0'):
assert auroc(preds, target) == torch.tensor(0.5)
def compute_metrics(self, preds, tensors, metrics):
nelbo, nll, kl = metrics
B, X, A, M, Y, CE = tensors
if self.hparams['eval_type'] == 'mse':
mse, r2, ci = calc_stats(preds, tensors)
return {
'val_loss': nelbo,
'nll': nll,
'kl': kl,
'mse': mse,
'r2': r2,
'ci': ci
}
elif self.hparams['eval_type'] == 'f1':
f1 = self.f1(preds.argmax(dim=1), Y)
p = self.precision(preds.argmax(dim=1), Y)
r = self.recall(preds.argmax(dim=1), Y)
return {
'val_loss': nelbo,
'nll': nll,
'kl': kl,
'F1': f1,
'precision': p,
'recall': r
}
elif self.hparams['eval_type'] == 'auc':
auc = auroc(preds.argmax(dim=1), Y)
return {'val_loss': nelbo, 'nll': nll, 'kl': kl, 'auc': auc}
else:
raise ValueError('bad metric specified...')
def validation_epoch_end(self, outputs):
logits = torch.cat([o["logits"] for o in outputs], dim=0)
labels = torch.cat([o["labels"] for o in outputs], dim=0)
if self.hparams["multilabel"]:
self.log(
"val_auroc",
np.mean(
[
auroc(
torch.sigmoid(logits[:, i]),
labels[:, i],
pos_label=1,
)
.detach()
.cpu()
.item()
for i in range(logits.shape[1])
]
),
prog_bar=True,
)
else:
self.log(
"val_f1",
f1_score(
torch.argmax(logits, dim=1),
labels,
num_classes=self.hparams["output_size"],
class_reduction="macro",
)
.detach()
.cpu()
.item(),
prog_bar=True,
)
self.log(
"val_acc",
accuracy(torch.argmax(logits, dim=1), labels)
.detach()
.cpu()
.item(),
prog_bar=True,
)
self.log(
"val_loss",
self.loss(
logits,
labels.type(
logits.dtype if self.hparams["multilabel"] else labels.dtype
),
)
.detach()
.cpu()
.item(),
prog_bar=True,
)
def cal_auc(self):
auc_list = []
for id, (pred, label) in enumerate(zip(self.predict, self.label)):
try:
auc = auroc(torch.Tensor(pred), torch.Tensor(label))
except:
auc = torch.tensor(0)
auc_list.append(auc)
return auc_list
def test(model, loader):
model.eval()
pred = []
y = []
with torch.no_grad():
for data in loader:
data = data.to(device)
y.append(data.y.detach().cpu())
pred.append(model(data).detach().cpu())
return auroc(torch.cat(pred), torch.cat(y))
def prediction_metrics(X_bin, X_cts, mean_cts, proba_bin):
mse = 0.
auc = 0.
if X_cts is not None:
which_cts = ~X_cts.isnan()
mse = mean_squared_error(mean_cts[which_cts],
X_cts[which_cts]).item()
if X_bin is not None:
which_bin = ~X_bin.isnan()
auc = auroc(proba_bin[which_bin], X_bin[which_bin]).item()
return auc, mse
def training_epoch_end(self, outputs: List):
train_loss = torch.cat(
[out["loss"].unsqueeze(dim=0) for out in outputs]).mean()
y_pred = torch.cat([out["y_pred"] for out in outputs], dim=0)
y_true = torch.cat([out["y_true"] for out in outputs], dim=0)
train_auc = auroc(y_pred, y_true)
logs = {"train_loss": train_loss, "train_auc": train_auc}
return {
"log": logs,
}
def validation_step(self, batch: MINDBatch, batch_idx):
loss, y_score = self.model.forward(batch)
y_true = batch['targets']
n_processed = batch['batch_cand'].max() + 1
for n in range(n_processed):
mask = batch['batch_cand'] == n
s, t = y_score[mask], y_true[mask]
s = torch.softmax(s, dim=0)
self.am.val_roc.update(auroc(s, t))
self.am.val_ndcg10.update(ndcg_score(s, t))
self.am.val_loss.update(loss, n_processed)
def training_step(self, batch, batch_idx):
"""训练过程"""
input, label = batch
y_hat = self.forward(input)
loss = self.loss_fn(y_hat, label)
acc = self.compute_metric(y_hat, label)
auc = auroc(y_hat.view(-1), label.view(-1))
tensorboard_logs = {
'train_loss': loss,
'train_acc': acc,
"train_auc": auc,
"lr": self.trainer.optimizers[0].param_groups[0]['lr']
}
return {'loss': loss, 'log': tensorboard_logs}
def validation_epoch_end(self, outputs: List):
val_loss = torch.cat(
[out["val_loss"].unsqueeze(dim=0) for out in outputs]).mean()
y_pred = torch.cat([out["y_pred"] for out in outputs], dim=0)
y_true = torch.cat([out["y_true"] for out in outputs], dim=0)
val_auc = auroc(y_pred, y_true)
logs = {"val_loss": val_loss, "val_auc": val_auc}
print(
f"Epoch {self.current_epoch} // val loss: {val_loss:.4f}, val auc: {val_auc:.4f}, pos: {y_true.sum()}, neg: {len(y_true) - y_true.sum()}"
)
return {
"log": logs,
}
def calc_categorical_metrics(self, categorical_feature_idcs, thr_idx):
# Define helper functions:
select_func = lambda x: x[:, categorical_feature_idcs].flatten()
# Get relevant timeframe for targets and preds:
thresholds_flat = torch.cat([self.categorical_thresholds[thr_idx][:len(target)] for target in self.all_targets]).cpu()
targets_flat = torch.cat([select_func(target) for target in self.all_targets]).cpu()
preds_flat = torch.cat([torch.sigmoid(select_func(pred)) for pred in self.all_preds]).cpu()
pred_tar_thr = [[_p, _t, _thr] for _p, _t, _thr in zip(preds_flat, targets_flat, thresholds_flat) if not torch.isnan(_t)]
preds_flat = torch.stack([_d[0] for _d in pred_tar_thr])
targets_flat = torch.stack([_d[1] for _d in pred_tar_thr]).int()
thresholds_flat = torch.stack([_d[2] for _d in pred_tar_thr])
preds_bool = (preds_flat > thresholds_flat).float()
auc = auroc(preds_flat, targets_flat)
acc = accuracy(preds_bool.int(), targets_flat)
return auc, acc
def validation_epoch_end(self, outputs: List):
val_loss = torch.cat(
[out["val_loss"].unsqueeze(dim=0) for out in outputs]).mean()
y_pred = torch.cat([out["y_pred"] for out in outputs], dim=0)
y_true = torch.cat([out["y_true"] for out in outputs], dim=0)
val_auc = auroc(y_pred, y_true)
logs = {
"val_loss": val_loss,
"val_auc": val_auc,
"model_name": self.model_name,
"fold": self.fold,
"sz": self.sz,
}
return {
"log": logs,
}
def calculate_metrics(self, pred, target, prvs_metrics=None):
# output.cpu().detach().numpy()
metrics = {
'accuracy': pl_metrics.accuracy(torch.round(pred),
torch.round(target)),
'aucroc': pl_metrics.auroc(pred, target),
'f1_score': pl_metrics.f1_score(torch.round(pred),
torch.round(target))
}
if prvs_metrics is not None:
for key in metrics.keys():
metrics[key] += prvs_metrics[key]
metrics['counter'] = prvs_metrics['counter'] + 1
else:
metrics['counter'] = 1
return metrics
def prediction_metrics(self, X_bin, X_cts, mean_cts, proba_bin):
n_sample = self.model.n_samples
mse = 0.
auc = 0.
if X_cts is not None:
X_cts = X_cts.unsqueeze(1)
which_cts = ~X_cts.isnan()
for i in range(n_sample):
mean_cts_tmp = mean_cts[:, [i], :]
mse += mean_squared_error(mean_cts_tmp[which_cts], X_cts[which_cts]).item()
mse = mse / n_sample
if X_bin is not None:
X_bin = X_bin.unsqueeze(1)
which_bin = ~X_bin.isnan()
for i in range(n_sample):
proba_bin_tmp = proba_bin[:, [i], :]
auc += auroc(proba_bin_tmp[which_bin], X_bin[which_bin]).item()
auc = auc / n_sample
return auc, mse
def validation_epoch_end(self, outputs: List):
val_loss = torch.cat(
[out["val_loss"].unsqueeze(dim=0) for out in outputs]).mean()
y_pred = torch.cat([out["y_pred"] for out in outputs], dim=0)
y_true = torch.cat([out["y_true"] for out in outputs], dim=0)
try:
val_auc = auroc(y_pred, y_true)
except ValueError as err:
print(f"ValueError: {err}")
val_auc = torch.Tensor([0])
logs = {
"val_loss": val_loss,
"val_auc": val_auc,
"model_name": self.model_name,
"fold": self.fold,
"sz": self.sz,
}
return {
"log": logs,
}
def eval_epoch(args, logger, g, dataloader, encoder, decoder, msg2mail,
loss_fcn, device, num_samples):
m_ap, m_auc, m_acc = [[], [], []] if 'LP' in args.tasks else [0, 0, 0]
labels_all = torch.zeros((num_samples)).long()
logits_all = torch.zeros((num_samples))
attn_weight_all = torch.zeros((num_samples, args.n_mail))
m_loss = []
m_infer_time = []
with torch.no_grad():
encoder.eval()
decoder.eval()
loss = torch.tensor(0)
for batch_idx, (input_nodes, pos_graph, neg_graph, blocks, frontier,
current_ts) in enumerate(dataloader):
n_sample = pos_graph.num_edges()
start_idx = batch_idx * n_sample
end_idx = min(num_samples, start_idx + n_sample)
pos_graph = pos_graph.to(device)
neg_graph = neg_graph.to(device) if neg_graph is not None else None
if not args.no_time or not args.no_pos:
current_ts, pos_ts, num_pos_nodes = get_current_ts(
args, pos_graph, neg_graph)
pos_graph.ndata['ts'] = current_ts
else:
current_ts, pos_ts, num_pos_nodes = None, None, None
_ = dgl.add_reverse_edges(
neg_graph) if neg_graph is not None else None
start = time.time()
emb, attn_weight = encoder(dgl.add_reverse_edges(pos_graph), _,
num_pos_nodes)
#attn_weight_all[start_idx:end_idx] = attn_weight[:n_sample]
logits, labels = decoder(emb, pos_graph, neg_graph)
end = time.time() - start
m_infer_time.append(end)
loss = loss_fcn(logits, labels)
m_loss.append(loss.item())
mail = msg2mail.gen_mail(args, emb, input_nodes, pos_graph,
frontier, 'val')
if not args.no_time:
g.ndata['last_update'][pos_graph.ndata[dgl.NID]
[:num_pos_nodes]] = pos_ts.to('cpu')
g.ndata['feat'][pos_graph.ndata[dgl.NID]] = emb.to('cpu')
g.ndata['mail'][input_nodes] = mail
labels = labels.long()
logits = logits.sigmoid()
if 'LP' in args.tasks:
pred = logits > 0.5
m_ap.append(average_precision(logits, labels).cpu().numpy())
m_auc.append(auroc(logits, labels).cpu().numpy())
m_acc.append(accuracy(pred, labels).cpu().numpy())
else:
labels_all[start_idx:end_idx] = labels
logits_all[start_idx:end_idx] = logits
if 'LP' in args.tasks:
ap, auc, acc = np.mean(m_ap), np.mean(m_auc), np.mean(m_acc)
else:
pred_all = logits_all > 0.5
ap = average_precision(logits_all, labels_all).cpu().item()
auc = auroc(logits_all, labels_all).cpu().item()
acc = accuracy(pred_all, labels_all).cpu().item()
fprs, tprs, thresholds = roc(logits_all, labels_all)
fpr_l, tpr_l, thres_l = get_TPR_FPR_metrics(fprs, tprs, thresholds)
print_tp_fp_thres(args.tasks, logger, fpr_l, tpr_l, thres_l)
print('总推理时间', np.sum(m_infer_time))
logger.info(attn_weight_all.mean(0))
encoder.train()
decoder.train()
return ap, auc, acc, np.mean(m_loss)
def validation_epoch_end(self, outputs):
avg_loss = torch.stack([x['valid_loss'] for x in outputs]).mean()
yb = torch.cat([x['yb'] for x in outputs], 0)
predictions = torch.cat[x['yb'] for x in outputs], 0)
score = auroc(predictions, yb)
return {'val_loss': avg_loss, 'score': score}
def compute(self) -> torch.Tensor:
preds, targets = self._get_preds_and_targets()
if torch.unique(targets).numel() == 1:
return torch.tensor(np.nan)
return auroc(preds, targets)
def eval_epoch(args, g, dataloader, attn, decoder, bandi_sampler, loss_fcn,
device, num_samples):
m_ap, m_auc, m_acc = [[], [], []] if 'LP' in args.tasks else [0, 0, 0]
labels_all = torch.zeros((num_samples))
logits_all = torch.zeros((num_samples))
m_loss = []
m_infer_time = []
with torch.no_grad():
attn.eval()
decoder.eval()
#loss = torch.tensor(0)
for batch_idx, (input_nodes, pos_graph, neg_graph, blocks, frontier,
current_ts) in enumerate(dataloader):
pos_graph = pos_graph.to(device)
neg_graph = neg_graph.to(device)
n_sample = pos_graph.num_edges()
start_idx = batch_idx * n_sample
end_idx = min(num_samples, start_idx + n_sample)
for j in range(args.n_layer):
blocks[j] = blocks[j].to(device)
current_ts, pos_ts, num_pos_nodes = get_current_ts(
pos_graph, neg_graph)
pos_graph.ndata['ts'] = current_ts
start = time.time()
blocks, att_map = attn.forward(blocks)
emb = blocks[-1].dstdata['h']
blocks = bandi_sampler.weight_update(blocks, att_map)
blocks = bandi_sampler.prob_update(blocks)
logits, labels = decoder(emb, pos_graph, neg_graph)
end = time.time() - start
m_infer_time.append(end)
loss = loss_fcn(logits, labels)
m_loss.append(loss.item())
g.ndata['last_update'][pos_graph.ndata[dgl.NID]
[:num_pos_nodes]] = pos_ts.to('cpu')
g.edata['q_ij'][
blocks[-1].edata['eid']] = blocks[-1].edata['q_ij'].cpu()
g.edata['weight'][
blocks[-1].edata['eid']] = blocks[-1].edata['weight'].cpu()
#g.ndata['h'][pos_graph.ndata[dgl.NID]] = emb.to('cpu')
if 'LP' in args.tasks:
pred = logits.sigmoid() > 0.5
m_ap.append(average_precision(logits, labels).cpu().numpy())
m_auc.append(auroc(logits, labels).cpu().numpy())
m_acc.append(accuracy(pred, labels).cpu().numpy())
else:
labels_all[start_idx:end_idx] = labels
logits_all[start_idx:end_idx] = logits
if 'LP' in args.tasks:
ap, auc, acc = np.mean(m_ap), np.mean(m_auc), np.mean(m_acc)
else:
pred_all = logits_all.sigmoid() > 0.5
ap = average_precision(logits_all, labels_all).cpu().item()
auc = auroc(logits_all, labels_all).cpu().item()
acc = accuracy(pred_all, labels_all).cpu().item()
attn.train()
decoder.train()
return ap, auc, acc, np.mean(m_loss), np.sum(m_infer_time)
def compute(self):
return auroc(self.pred, self.true, pos_label=self.pos_label)
def compute(self):
if self.binary_target.sum() == self.binary_target.numel():
return torch.tensor(float("nan"), device=self.pred_score.device)
return auroc(self.pred_score, self.binary_target)
if torch.cuda.is_available():
torch.cuda.manual_seed(seed)
path = osp.join(osp.dirname(osp.realpath(__file__)), dataset_directory_path, args.fingerprint_type)
dataset = TUDataset(path, name=args.dataset, pre_transform=PreTransform(args))
perm = torch.randperm(len(dataset), dtype=torch.long)
dataset = dataset[perm]
tenpercent = int(len(dataset) * 0.1)
test_dataset = dataset[:2 * tenpercent]
train_dataset = dataset[int(2 * tenpercent):]
train_loader = DataLoader(train_dataset)
test_loader = DataLoader(test_dataset)
X_train = []
X_test = []
Y_train = []
Y_test = []
for data in train_loader:
X_train.append(data.emb.tolist())
Y_train.append(data.y.tolist())
for data in test_loader:
X_test.append(data.emb.tolist())
Y_test.append(data.y.tolist())
X_train, X_test, Y_train, Y_test = np.array(X_train).reshape(-1, embedding_size[args.fingerprint_type]), np.array(
X_test).reshape(-1, embedding_size[args.fingerprint_type]), np.array(Y_train).reshape(-1), np.array(
Y_test).reshape(-1)
clf = LogisticRegression(random_state=seed).fit(X_train, Y_train)
Y_pred = clf.decision_function(X_test)
Y_pred = torch.from_numpy(Y_pred)
Y_test = torch.from_numpy(Y_test)
print(auroc(Y_pred, Y_test))
