class ModelRunner:
def __init__(self, conf, GS,logger, data_logger=None, is_nni=False):
self._logger = logger
self._data_logger = EmptyLogger() if data_logger is None else data_logger
self._conf = conf
self.bar = 0.5
self._lr = conf["lr"]
self._is_nni = is_nni
# choosing GPU device
self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if self._device != "cpu":
with torch.cuda.device("cuda:{}".format(CUDA_Device)):
torch.cuda.empty_cache()
if not self._is_nni:
self._device = torch.device("cuda:{}".format(CUDA_Device))
self._loss = self.graphSaintLoss if GS else self.regular_loss
self.accuracy = self.accuracy_GraphSaint if GS else self.accuracy_regular
self._ce_loss = torch.nn.CrossEntropyLoss(reduction="mean").to(self._device)
self._ce_loss2 = torch.nn.BCELoss(reduction='mean')
@property
def logger(self):
return self._logger
@property
def data_logger(self):
return self._data_logger
def graphSaintLoss(self, calcs, beta=None, gamma=None):
if beta is None:
beta = 1 / len(calcs["f_ns_out"]) if len(calcs["f_ns_out"])!=0 else 0
gamma = 1 / len(calcs["s_ns_out"]) if len(calcs["s_ns_out"])!=0 else 0
cn_loss = self._ce_loss2(calcs["cn_out"], calcs["cn_label"].float())
f_ns_loss = self._ce_loss2(calcs["f_ns_out"], calcs["f_ns_labels"].float()) *(beta) if len(calcs["f_ns_out"])!=0 else 0
s_ns_loss = self._ce_loss2(calcs["s_ns_out"], calcs["s_ns_labels"].float()) * (gamma) if len(calcs["s_ns_out"])!=0 else 0
return cn_loss+f_ns_loss+s_ns_loss
def regular_loss(self, calcs, beta=None, gamma=None):
if beta is None:
beta = 1 / len(calcs["f_ns_out"]) if len(calcs["f_ns_out"])!=0 else 0
gamma = 1 / len(calcs["s_ns_out"]) if len(calcs["s_ns_out"])!=0 else 0
cn_loss = self._ce_loss(calcs["cn_out"], calcs["cn_label"].long())
f_ns_loss = self._ce_loss(calcs["f_ns_out"], calcs["f_ns_labels"].long()) *(beta) if len(calcs["f_ns_out"])!=0 else 0
s_ns_loss = self._ce_loss(calcs["s_ns_out"], calcs["s_ns_labels"].long()) * (gamma) if len(calcs["s_ns_out"])!=0 else 0
return cn_loss+f_ns_loss+s_ns_loss
def _get_model(self):
model = GCN(in_features=self._conf["in_features"],
hid_features=self._conf["hid_features"], out_features= self._conf["out_features"],
activation=self._conf["activation"], dropout= self._conf["dropout"])
opt = self._conf["optimizer"](model.parameters(), lr=self._conf["lr"], weight_decay=self._conf["weight_decay"])
return {"model": model, "optimizer": opt,
"beta": self._conf["beta"],"gamma": self._conf["gamma"],
"labels": self._conf["labels"], "X": self._conf["X"], "ds_name": self._conf["ds_name"], "adj_tr": self._conf["adj_tr"], "adj_te": self._conf["adj_te"],
"train_ind": self._conf["train_ind"], "test_ind": self._conf["test_ind"], "testt": self._conf["testt"], "traint": self._conf["traint"]
}
# verbose = 0 - silent
# verbose = 1 - print test results
# verbose = 2 - print train for each epoch and test results
def run(self, verbose=2):
if self._is_nni:
verbose = 0
model = self._get_model()
##
loss_train, acc_train, intermediate_acc_test, losses_train, accs_train, accs_cn_train, accs_f_train, accs_s_train, test_results = self.train(
self._conf["epochs"],
model=model,
verbose=verbose)
##
# Testing . ## result is only the last one! do not use. same as 7 in last
result = self.test(model=model, verbose=verbose if not self._is_nni else 0, print_to_file=True)
test_results.append(result)
if self._is_nni:
self._logger.debug('Final loss train: %3.4f' % loss_train)
self._logger.debug('Final accuracy train: %3.4f' % acc_train)
final_results = result["acc"]
self._logger.debug('Final accuracy test: %3.4f' % final_results)
# _nni.report_final_result(test_auc)
if verbose != 0:
names = ""
vals = ()
for name, val in result.items():
names = names + name + ": %3.4f "
vals = vals + tuple([val])
self._data_logger.info(name, val)
parameters = { "lr": self._conf["lr"],
"weight_decay": self._conf["weight_decay"],
"dropout": self._conf["dropout"], "optimizer": self._conf["optim_name"]}
return loss_train, acc_train, intermediate_acc_test, result, losses_train, accs_train, accs_cn_train, accs_f_train, accs_s_train, test_results, parameters
def train(self, epochs, model=None, verbose=2):
loss_train = 0.
acc_train = 0.
losses_train = []
accs_train = []
accs_cn_train = []
accs_f_train = []
accs_s_train = []
test_results = []
intermediate_test_acc = []
for epoch in range(epochs):
loss_train, acc_train, acc_train_cn , acc_train_f, acc_train_s= self._train(epoch, model, verbose)
losses_train.append(loss_train)
accs_train.append(acc_train)
accs_cn_train.append(acc_train_cn)
accs_f_train.append(acc_train_f)
accs_s_train.append(acc_train_s)
##
# /---------------------- FOR NNI -------------------------
if epoch % 5 == 0:
test_res = self.test(model, verbose=verbose if not self._is_nni else 0)
test_results.append(test_res)
if self._is_nni:
test_acc = test_res["acc"]
intermediate_test_acc.append(test_acc)
return loss_train, acc_train, intermediate_test_acc, losses_train, \
accs_train, accs_cn_train, accs_f_train, accs_s_train, test_results
''' This function calculates the output and the labels for each node:
for each node we take as an input the nodels' output and the labels, and return the output and label of the central node, of it's
first neighbors and of it's second neighbors. NOTE: we take only those that are in train indices
'''
def calculate_labels_outputs(self,node, outputs , labels, indices, ego_graph):
f_neighbors = set(list(ego_graph.neighbors(node)))
s_neighbors = set()
#create second neighbors
for f_neighbor in f_neighbors:
for s_neighbor in ego_graph.neighbors(f_neighbor):
if s_neighbor not in f_neighbors and s_neighbor != node and s_neighbor not in s_neighbors:
s_neighbors.add(s_neighbor)
# notice we use the "index" in order to have correlation between the neihbors and the output's index (graph nodes are labeld with numbers from 0 to N (of the big graph) and the output's labels from 0 to n=len(ego graph). so using the "index" solves it (hopefully) ;)
cn_out= outputs[[list(ego_graph.nodes).index(node)]]
cn_label = labels[[node]]
#create vectors for the first neighbors outputs and labels. NOTE: we take only those that are in train indices
f_ns_out = outputs[[list(ego_graph.nodes).index(f_n) for f_n in f_neighbors if indices[f_n]]]
f_ns_labels = labels[[f_n for f_n in f_neighbors if indices[f_n]]]
#same for second neoghbors
s_ns_out = outputs[[list(ego_graph.nodes).index(s_n) for s_n in s_neighbors if indices[s_n]]]
s_ns_labels = labels[[s_n for s_n in s_neighbors if indices[s_n]]]
return { "cn_out": cn_out, "cn_label": cn_label, "f_ns_out": f_ns_out, "f_ns_labels": f_ns_labels, "s_ns_out": s_ns_out, "s_ns_labels": s_ns_labels }
def _train(self, epoch, model, verbose=2):
model_ = model["model"]
model_ = model_.to(self._device)
optimizer = model["optimizer"]
#train ind are the nodes to create subgraphs from. traint are nodes in train (that we can learn from)
train_indices = model["train_ind"]
model["labels"] = model["labels"].to(self._device)
labels = model["labels"]
beta = model["beta"]
gamma = model["gamma"]
model_.train()
optimizer.zero_grad()
loss_train = 0.
loss_train1 = 0.
calcs_batch = []
BATCH_SIZE= 30
# create subgraphs only for partial, but use labels of all train indices
for idx,node in enumerate(train_indices):
# adj = nx.ego_graph(model["adj_matrices"], node, radius=2)
adj = model["adj_tr"][node]
X_t = model["X"][list(adj.nodes)].to(device=self._device)
output = model_(X_t, nx.adjacency_matrix(adj).tocoo())
calcs = self.calculate_labels_outputs( node, output, labels, model["traint"], adj)
#no batches:
loss_train += self._loss(calcs, beta, gamma)
# # if we want to use batches
# loss_train1 += self._loss(calcs, beta, gamma)
# loss_train += self._loss(calcs, beta, gamma).data.item()
# if idx % BATCH_SIZE == 0 and idx > 0:
# loss_train1 /= BATCH_SIZE
# loss_train1.backward()
# optimizer.step()
# loss_train1 = 0.
calcs_batch.append(calcs)
acc_train, acc_train_cn, acc_train_f, acc_train_s = self.accuracy(calcs_batch)
#
loss_train /= len(train_indices)
#
loss_train.backward()
optimizer.step()
if verbose == 2:
# Evaluate validation set performance separately,
# deactivates dropout during validation run.
self._logger.debug('Epoch: {:04d} '.format(epoch + 1) +
'ce_loss_train: {:.4f} '.format(loss_train) +
'acc_train: {:.4f} '.format(acc_train))
return loss_train, acc_train, acc_train_cn , acc_train_f, acc_train_s
''' Accuracy function. For the graphSaint we use sigmoid on each index, then we use BCE loss, then we span the vectors(of each node's result) to one vector of all the centrals,
one vector of all the first neighbors, and one vector of all the second neghbors,
put 1 in the indexes that have value >= 0.5 and 0 otherwise, then calculate f1 score on the vector
'''
@staticmethod
def accuracy_GraphSaint(calcs):
#create one vector that will contain all the central's nodes outputs (for each node in the train/test). same for labels
out, labs = ([calcs[i]["cn_out"].data[0].tolist() for i in range(len(calcs))],
[calcs[i]["cn_label"].data[0].tolist() for i in range(len(calcs))])
out = np.array(out)
labs = np.array(labs)
out[out > 0.5] = 1
out[out <= 0.5] = 0
acc_cn = metrics.f1_score(labs, out, average="micro")
# create one vector that will contain all the first neighbors outputs (of each node in the train/test. each node has vector of first neighbors- put it all to one long vector)
out = []
labs = []
for i in range(len(calcs)):
out += calcs[i]["f_ns_out"].data.tolist()
labs += calcs[i]["f_ns_labels"].data.tolist()
out=np.array(out)
labs=np.array(labs)
out[out > 0.5] = 1
out[out <= 0.5] = 0
if len(out) != 0:
acc_f = metrics.f1_score(labs, out, average="micro")
else:
acc_f = np.nan
# same for second neighbors (same as first)
out = []
labs = []
for i in range(len(calcs)):
out += calcs[i]["s_ns_out"].data.tolist()
labs += calcs[i]["s_ns_labels"].data.tolist()
out = np.array(out)
labs = np.array(labs)
out[out > 0.5] = 1
out[out <= 0.5] = 0
if len(out) != 0:
# fpr, tpr, thresholds = metrics.roc_curve(labs2, out2)
# acc_s = metrics.auc(fpr, tpr)
acc_s = metrics.f1_score(labs, out, average="micro")
else:
acc_s = np.nan
return np.nanmean(np.array([acc_cn, acc_f, acc_s])), acc_cn, acc_f, acc_s
def accuracy_regular(self,calcs):
out, labs = ([calcs[i]["cn_out"].data[0].tolist() for i in range(len(calcs))],
[calcs[i]["cn_label"].data[0].tolist() for i in range(len(calcs))])
acc_cn = sum(np.argmax(np.array(out), axis=1) == labs) / len(labs)
out = []
labs = []
for i in range(len(calcs)):
out += calcs[i]["f_ns_out"].data.tolist()
labs += calcs[i]["f_ns_labels"].data.tolist()
if len(out) != 0:
acc_f = sum(np.argmax(np.array(out), axis=1) == labs) / len(labs)
else:
acc_f = np.nan
out = []
labs = []
for i in range(len(calcs)):
out += calcs[i]["s_ns_out"].data.tolist()
labs += calcs[i]["s_ns_labels"].data.tolist()
if len(out) != 0:
acc_s = sum(np.argmax(np.array(out), axis=1) == labs) / len(labs)
else:
acc_s = np.nan
return np.nanmean(np.array([acc_cn, acc_f, acc_s])), acc_cn, acc_f, acc_s
def test(self, model=None, verbose=2, print_to_file=False):
model_ = model["model"]
test_indices = model["test_ind"]
labels = model["labels"]
beta = model["beta"]
gamma = model["gamma"]
model_.eval()
test_loss = 0
calcs_batch=[]
with torch.no_grad():
for node in test_indices:
# adj = nx.ego_graph(model["adj_matrices"], node, radius=2)
adj = model["adj_te"][node]
X_t = model["X"][list(adj.nodes)].to(device=self._device)
output = model_(X_t, nx.adjacency_matrix(adj).tocoo())
calcs = self.calculate_labels_outputs(node, output, labels, model["testt"], adj)
test_loss += self._loss(calcs, beta, gamma).data.item()
calcs_batch.append(calcs)
test_loss /= len(test_indices)
test_acc, acc_test_cn, acc_test_f, acc_test_s = self.accuracy(calcs_batch)
if verbose != 0:
self._logger.info("Test: ce_loss= {:.4f} ".format(test_loss) + "acc= {:.4f}".format(test_acc))
result = {"loss": test_loss, "acc": test_acc, "acc_cn": acc_test_cn, "acc_f":acc_test_f, "acc_s":acc_test_s}
return result
class ModelRunner:
def __init__(self, conf, logger, data_logger=None, is_nni=False):
self._logger = logger
self._data_logger = EmptyLogger(
) if data_logger is None else data_logger
self._conf = conf
self.bar = 0.5
self._lr = conf["lr"]
self._is_nni = is_nni
self._device = torch.device(
'cuda:0') if torch.cuda.is_available() else torch.device('cpu')
self._loss = BCELoss()
@property
def logger(self):
return self._logger
@property
def data_logger(self):
return self._data_logger
def my_loss(self, output, target, weights=None):
output = torch.clamp(output, min=1e-8, max=1 - 1e-8)
if weights is not None:
assert len(weights) == 2
loss = weights[1] * (target * torch.log(output)) + \
weights[0] * ((1 - target) * torch.log(1 - output))
else:
loss = target * torch.log(output) + (
1 - target) * torch.log(1 - output)
ret = torch.neg(torch.mean(loss))
return ret
def accuracy(self, output, labels):
#output = torch.sigmoid(output) ##todo USE it only with BCEWithLogit
idxs_1_labeled = torch.where(labels == 1)
answers = output[idxs_1_labeled]
true_pos = torch.where(answers >= 0.5) # tuple (,)
return len(true_pos[0]) / len(idxs_1_labeled[0])
def _get_model(self):
model = Graphs_Rec(in_features=self._conf["train_data"][0].shape[0],
hid_features=self._conf["hid_features"],
out_features=1,
activation=self._conf["activation"],
dropout=self._conf["dropout"])
opt = self._conf["optimizer"](model.parameters(),
lr=self._conf["lr"],
weight_decay=self._conf["weight_decay"])
##checged : added "feature_matrices"
return {
"model": model,
"optimizer": opt,
"train_data": self._conf["train_data"],
"training_labels": self._conf["training_labels"],
"test_data": self._conf["test_data"],
"test_labels": self._conf["test_labels"]
}
# verbose = 0 - silent
# verbose = 1 - print test results
# verbose = 2 - print train for each epoch and test results
def run(self, verbose=2):
if self._is_nni:
verbose = 0
model = self._get_model()
##
loss_train, acc_train, intermediate_acc_test, losses_train, accs_train, test_results = self.train(
self._conf["epochs"], model=model, verbose=verbose)
##
# Testing
result = self.test(model=model,
verbose=verbose if not self._is_nni else 0,
print_to_file=False)
if self._is_nni:
self._logger.debug('Final loss train: %3.4f' % loss_train)
self._logger.debug('Final accuracy train: %3.4f' % acc_train)
final_results = result["acc"]
self._logger.debug('Final accuracy test: %3.4f' % final_results)
# _nni.report_final_result(test_auc)
if verbose != 0:
names = ""
vals = ()
for name, val in result.items():
names = names + name + ": %3.4f "
vals = vals + tuple([val])
self._data_logger.info(name, val)
parameters = {
"lr": self._conf["lr"],
"weight_decay": self._conf["weight_decay"],
"dropout": self._conf["dropout"],
"optimizer": self._conf["optim_name"]
}
return loss_train, acc_train, intermediate_acc_test, result, losses_train, accs_train, test_results, parameters
def train(self, epochs, model=None, verbose=2):
loss_train = 0.
acc_train = 0.
losses_train = []
accs_train = []
test_results = []
intermediate_test_acc = []
for epoch in range(epochs):
loss_train, acc_train = self._train(epoch, model, verbose)
##
losses_train.append(loss_train)
accs_train.append(acc_train)
##
# /---------------------- FOR NNI -------------------------
if epoch % 5 == 0:
test_res = self.test(
model, verbose=verbose if not self._is_nni else 0)
test_results.append(test_res)
if self._is_nni:
test_acc = test_res["acc"]
intermediate_test_acc.append(test_acc)
return loss_train, acc_train, intermediate_test_acc, losses_train, \
accs_train, test_results
def _train(self, epoch, model, verbose=2):
#self._loss = self._loss = BCEWithLogitsLoss(torch.ones([223653]).to(self._device))
model_ = model["model"]
model_ = model_.to(self._device)
optimizer = model["optimizer"]
###!
labels = torch.from_numpy(model["training_labels"]).to(
dtype=torch.float, device=self._device)
labels = torch.DoubleTensor(model["training_labels"]).to(
dtype=torch.float, device=self._device) ###todo
train = torch.from_numpy(model["train_data"]).to(dtype=torch.float,
device=self._device)
model_.train()
optimizer.zero_grad()
self._loss = self.my_loss
###send the model
output = model_(train)
###
loss_train = 0.
labeld_1_num = len([b for b, item in enumerate(labels) if item == 1])
output = output.view(output.shape[0]) ###todo!
# loss_train += self._loss(output, labels, weights=[1,(len(train)-78)/78]) ##weights=[19/len(train),(len(train)-19)/len(train)])
loss_train += self._loss(
output,
labels,
weights=[
len(train) / (len(train) - labeld_1_num),
len(train) / labeld_1_num
]) ##weights=[19/len(train),(len(train)-19)/len(train)])
# loss_train /= len(train)
loss_train.backward()
optimizer.step()
acc_train = self.accuracy(output, labels)
if verbose == 2:
# Evaluate validation set performance separately,
# deactivates dropout during validation run.
self._logger.debug(
'Epoch: {:04d} '.format(epoch + 1) +
'loss_train: {:.4f} '.format(loss_train.data.item()) +
'acc_train: {:.4f} '.format(acc_train))
return loss_train, acc_train
def test(self, model=None, verbose=2, print_to_file=False):
#self._loss=self._loss = BCEWithLogitsLoss(torch.ones([894618]).to(self._device))
model_ = model["model"]
model_ = model_.to(self._device)
labels = torch.from_numpy(model["test_labels"]).to(dtype=torch.float,
device=self._device)
labels = torch.DoubleTensor(model["test_labels"]).to(
dtype=torch.float, device=self._device) ###todo###
test = torch.from_numpy(model["test_data"]).to(dtype=torch.float,
device=self._device)
model_.eval()
'''self._loss = self.my_loss
pos_weight = torch.ones([len(test)]).to(self._device) # All weights are equal to 1
pos_weight *= 79 / (len(test) - 79)
self._loss = torch.nn.BCEWithLogitsLoss(pos_weight=pos_weight)'''
###send the model
output = model_(test)
###
output = output.view(output.shape[0]) ###todo!
self._loss = self.my_loss
loss_test = 0.
loss_test += self._loss(output,
labels) #, weights=[1, (len(test) - 20) / 20])
#loss_test += self._loss(output, labels)
#loss_test /= len(test)
acc_test = self.accuracy(output, labels)
if verbose != 0:
self._logger.info(
"Test: loss= {:.4f} ".format(loss_test.data.item()) +
"acc= {:.4f}".format(acc_test))
result = {"loss": loss_test.data.item(), "acc": acc_test}
return result
class ModelRunner:
def __init__(self, dataset_path, conf, logger, data_logger=None):
self._logger = logger
self._data_logger = EmptyLogger(
) if data_logger is None else data_logger
self._criterion = torch.nn.NLLLoss()
self._conf = conf
features_meta = get_features()
self.loader = GraphLoader(
dataset_path,
features_meta,
is_max_connected=False, # self._conf['dataset'] == "citeseer",
cuda_num=conf["cuda"],
logger=self._logger)
def _get_models(self):
bow_feat = self.loader.bow_mx
topo_feat = self.loader.topo_mx
model1 = GCN(nfeat=bow_feat.shape[1],
hlayers=[self._conf["kipf"]["hidden"]],
nclass=self.loader.num_labels,
dropout=self._conf["kipf"]["dropout"])
opt1 = optim.Adam(model1.parameters(),
lr=self._conf["kipf"]["lr"],
weight_decay=self._conf["kipf"]["weight_decay"])
model2 = GCNCombined(nbow=bow_feat.shape[1],
nfeat=topo_feat.shape[1],
hlayers=self._conf["hidden_layers"],
nclass=self.loader.num_labels,
dropout=self._conf["dropout"])
opt2 = optim.Adam(model2.parameters(),
lr=self._conf["lr"],
weight_decay=self._conf["weight_decay"])
model3 = GCN(nfeat=topo_feat.shape[1],
hlayers=self._conf["multi_hidden_layers"],
nclass=self.loader.num_labels,
dropout=self._conf["dropout"],
layer_type=None)
opt3 = optim.Adam(model3.parameters(),
lr=self._conf["lr"],
weight_decay=self._conf["weight_decay"])
model4 = GCN(nfeat=topo_feat.shape[1],
hlayers=self._conf["multi_hidden_layers"],
nclass=self.loader.num_labels,
dropout=self._conf["dropout"],
layer_type=AsymmetricGCN)
opt4 = optim.Adam(model4.parameters(),
lr=self._conf["lr"],
weight_decay=self._conf["weight_decay"])
return {
"kipf": {
"model": model1,
"optimizer": opt1,
"arguments": [self.loader.bow_mx, self.loader.adj_mx],
"labels": self.loader.labels,
},
"our_combined": {
"model":
model2,
"optimizer":
opt2,
"arguments": [
self.loader.bow_mx, self.loader.topo_mx,
self.loader.adj_rt_mx
],
"labels":
self.loader.labels,
},
# "our_topo_sym": {
# "model": model3, "optimizer": opt3,
# "arguments": [self.loader.topo_mx, self.loader.adj_mx],
# "labels": self.loader.labels,
# },
# "our_topo_asymm": {
# "model": model4, "optimizer": opt4,
# "arguments": [self.loader.topo_mx, self.loader.adj_rt_mx],
# "labels": self.loader.labels,
# },
}
def run(self, train_p):
self.loader.split_train(train_p)
models = self._get_models()
if self._conf["cuda"] is not None:
[
model["model"].cuda(self._conf["cuda"])
for model in models.values()
]
for model in models.values():
model["arguments"] = list(map(Variable, model["arguments"]))
model["labels"] = Variable(model["labels"])
# Train model
train_idx, val_idx = self.loader.train_idx, self.loader.val_idx
for epoch in range(self._conf["epochs"]):
for name, model_args in models.items():
self._train(epoch, name, model_args, train_idx, val_idx)
# Testing
test_idx = self.loader.test_idx
result = {
name: self._test(name, model_args, test_idx)
for name, model_args in models.items()
}
for name, val in sorted(result.items(), key=lambda x: x[0]):
self._data_logger.info(name, val["loss"], val["acc"],
(train_p / (2 - train_p)) * 100)
return result
def _train(self, epoch, model_name, model_args, idx_train, idx_val):
model, optimizer = model_args["model"], model_args["optimizer"]
arguments, labels = model_args["arguments"], model_args["labels"]
model.train()
optimizer.zero_grad()
output = model(*arguments)
loss_train = self._criterion(output[idx_train], labels[idx_train])
acc_train = accuracy(output[idx_train], labels[idx_train])
loss_train.backward()
optimizer.step()
if not self._conf["fastmode"]:
# Evaluate validation set performance separately,
# deactivates dropout during validation run.
model.eval()
output = model(*arguments)
loss_val = self._criterion(output[idx_val], labels[idx_val])
acc_val = accuracy(output[idx_val], labels[idx_val])
self._logger.debug(model_name + ": " +
'Epoch: {:04d} '.format(epoch + 1) +
'loss_train: {:.4f} '.format(loss_train.data[0]) +
'acc_train: {:.4f} '.format(acc_train.data[0]) +
'loss_val: {:.4f} '.format(loss_val.data[0]) +
'acc_val: {:.4f} '.format(acc_val.data[0]))
def _test(self, model_name, model_args, test_idx):
model, arguments, labels = model_args["model"], model_args[
"arguments"], model_args["labels"]
model.eval()
output = model(*arguments)
loss_test = functional.nll_loss(output[test_idx], labels[test_idx])
acc_test = accuracy(output[test_idx], labels[test_idx])
self._logger.info(model_name + " Test: " +
"loss= {:.4f} ".format(loss_test.data[0]) +
"accuracy= {:.4f}".format(acc_test.data[0]))
return {"loss": loss_test.data[0], "acc": acc_test.data[0]}
class ModelRunner:
def __init__(self, data_path, cuda, logger, data_logger=None):
self._logger = logger
self._cuda = cuda
self._data_logger = EmptyLogger(
) if data_logger is None else data_logger
self._data_path = data_path
# feature_meta = NEIGHBOR_FEATURES
feature_meta = NODE_FEATURES
# feature_meta = NODE_FEATURES.copy()
# feature_meta.update(NEIGHBOR_FEATURES)
self.loader = GraphLoader(feature_meta,
cuda_num=cuda,
logger=self._logger)
def get_gnx_paths(self):
for path in sorted(os.listdir(self._data_path), key=int):
yield os.path.join(self._data_path, path)
def run(self, ordered_config):
config = dict(ordered_config)
first_path = os.path.join(next(self.get_gnx_paths()), "features_1")
self.loader.split_data(config["test_p"], self.get_gnx_paths())
# Load data
adj_r_t, adj, feat_x, topo_x, labels, idx_train, idx_test = self.loader.load(
data_type="asymmetric", feature_path=first_path)
# Model and optimizer
model1 = RNNModel(
feat_x_n=feat_x.shape[1] if feat_x.shape else 0,
topo_x_n=topo_x.shape[1] if topo_x.shape else 0,
n_output=labels.max().data[0] + 1,
h_layers=config["hidden_layers"],
dropout=config["dropout"],
rnn_type="RNN_RELU",
)
# nbow=bow_feat.shape[1],
# nfeat=topo_feat.shape[1],
#
# nclass=labels.max() + 1,
optimizer1 = optim.Adam(model1.parameters(),
lr=config["learning_rate"],
weight_decay=config["weight_decay"])
models = {
"our": {
"model": model1,
"optimizer": optimizer1,
},
}
if self._cuda is not None:
[model["model"].cuda(self._cuda) for model in models.values()]
# for model in models.values():
# model["arguments"] = list(map(Variable, model["arguments"]))
# model["labels"] = Variable(model["labels"])
# Train model
t_total = time.time()
for epoch in range(config["epochs"]):
for name, model_args in models.items():
self._train(epoch, name, model_args)
self._logger.debug("Optimization Finished!")
self._logger.debug("Total time elapsed: {:.4f}s".format(time.time() -
t_total))
# Testing
# self._train(epoch, config["train_p"], name, model_args, temp_gnx)
return {
name: self._test(name, model_args, ordered_config)
for name, model_args in models.items()
}
# def _train(self, epoch, model, optimizer, model_args, idx_train, idx_val, labels):
def _train(self, epoch, name, model_args):
model, optimizer = model_args["model"], model_args["optimizer"]
# arguments, labels = model_args["arguments"], model_args["labels"]
t = time.time()
model.train()
optimizer.zero_grad()
hidden = model.init_hidden()
loss_train, acc_train = [], []
for gnx_path in self.get_gnx_paths():
adj_r_t, adj, feat_x, topo_x, labels, idx_train, idx_test = self.loader.load(
data_type="asymmetric", feature_path=gnx_path)
output = model(feat_x, topo_x, adj, hidden)
loss_train.append(
functional.nll_loss(output[idx_train], labels[idx_train]))
acc_train.append(accuracy(output[idx_train], labels[idx_train]))
loss_train[-1].backward()
optimizer.step()
# if not KIPF_BASE["fastmode"]:
# Evaluate validation set performance separately,
# deactivates dropout during validation run.
# model.eval()
# output = model(*arguments)
loss_train = loss_train[-1]
acc_train = acc_train[-1]
# loss_val = functional.nll_loss(output[idx_val], labels[idx_val])
# acc_val = accuracy(output[idx_val], labels[idx_val])
self._logger.debug(
name + ": " + 'Epoch: {:04d} '.format(epoch + 1) +
'loss_train: {:.4f} '.format(loss_train.data[0]) +
'acc_train: {:.4f} '.format(acc_train.data[0]) +
# 'loss_val: {:.4f} '.format(loss_val.data[0]) +
# 'acc_val: {:.4f} '.format(acc_val.data[0]) +
'time: {:.4f}s'.format(time.time() - t))
def _test(self, name, model_args, config):
model, arguments = model_args["model"], model_args["arguments"]
model.eval()
loss_test = []
acc_test = []
hidden = model.init_hidden()
for gnx_path in self.get_gnx_paths():
adj_r_t, adj, feat_x, topo_x, labels, idx_train, idx_val, idx_test = self.loader.load(
data_type="asymmetric", feature_path=gnx_path)
output = model(feat_x, topo_x, adj, hidden)
loss_test.append(
functional.nll_loss(output[idx_test], labels[idx_test]))
acc_test.append(accuracy(output[idx_test], labels[idx_test]))
# loss_train = functional.nll_loss(output[idx_train], labels[idx_train])
# acc_train = accuracy(output[idx_train], labels[idx_train])
# loss_train.backward()
# optimizer.step()
# output = model(*arguments)
# loss_test = functional.nll_loss(output[idx_test], labels[idx_test])
# acc_test = accuracy(output[idx_test], labels[idx_test])
loss_test = loss_test[-1]
acc_test = acc_test[-1]
self._logger.info(name + " " + "Test set results: " +
"loss= {:.4f} ".format(loss_test.data[0]) +
"accuracy= {:.4f}".format(acc_test.data[0]))
self._data_logger.info(name, loss_test.data[0], acc_test.data[0],
*list(map(at(1), config)))
return {"loss": loss_test.data[0], "acc": acc_test.data[0]}
class ModelRunner:
def __init__(self, conf, logger, data_logger=None, is_nni=False):
self._logger = logger
self._data_logger = EmptyLogger() if data_logger is None else data_logger
self._conf = conf
self.bar = 0.5
self._lr = conf["lr"]
self._is_nni = is_nni
self._device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu')
self._ce_loss = self._soft_ce_loss
self._temporal_loss = torch.nn.MSELoss(reduction='sum').to(self._device)
@property
def logger(self):
return self._logger
@property
def data_logger(self):
return self._data_logger
def _soft_ce_loss(self, predicted, target,weights = None):
predicted = torch.clamp(predicted, 1e-9, 1 - 1e-9)
if weights is None:
return -(target * torch.log(predicted)).sum(dim=1).sum().to(self._device)
weights = torch.FloatTensor(weights).to(device=self._device)
weights[weights==0] = 0.01
b = -(torch.sqrt((weights).sum()/weights) * target * torch.log(predicted)).sum(dim=1).sum().to(self._device)
return b
def _get_model(self):
model = GCN(in_features=self._conf["training_mat"][0].shape[1],
hid_features=self._conf["hid_features"], out_features=15,
activation=self._conf["activation"], dropout= self._conf["dropout"])
opt = self._conf["optimizer"](model.parameters(), lr=self._conf["lr"], weight_decay=self._conf["weight_decay"])
##checged : added "feature_matrices"
return {"model": model, "optimizer": opt,
"training_mats": self._conf["training_mat"],
"training_labels": self._conf["training_labels"],
"test_mats": self._conf["test_mat"],
"test_labels": self._conf["test_labels"],
"adj_matrices": self._conf["adj_matrices"],
"feature_matrices": self._conf["feature_matrices"]}
# verbose = 0 - silent
# verbose = 1 - print test results
# verbose = 2 - print train for each epoch and test results
def run(self, verbose=2):
if self._is_nni:
verbose = 0
model = self._get_model()
##
loss_train, acc_train, intermediate_acc_test, losses_train, losses_tempo, accs_train, test_results = self.train(self._conf["epochs"],
model=model,
verbose=verbose)
##
# Testing
result = self.test(model=model, verbose=verbose if not self._is_nni else 0, print_to_file=True)
if self._is_nni:
self._logger.debug('Final loss train: %3.4f' % loss_train)
self._logger.debug('Final accuracy train: %3.4f' % acc_train)
final_results = result["acc"]
self._logger.debug('Final accuracy test: %3.4f' % final_results)
# _nni.report_final_result(test_auc)
if verbose != 0:
names = ""
vals = ()
for name, val in result.items():
names = names + name + ": %3.4f "
vals = vals + tuple([val])
self._data_logger.info(name, val)
parameters = {"temporal_pen" : self._conf["temporal_pen"],"lr":self._conf["lr"], "weight_decay":self._conf["weight_decay"],
"dropout":self._conf["dropout"], "optimizer":self._conf["optim_name"]}
return loss_train, acc_train, intermediate_acc_test, result, losses_train,losses_tempo, accs_train, test_results, parameters
def train(self, epochs, model=None, verbose=2):
loss_train = 0.
acc_train = 0.
losses_train = []
tempo_losses_train = []
accs_train = []
test_results = []
intermediate_test_acc = []
for epoch in range(epochs):
loss_train, tempo_loss, acc_train = self._train(epoch, model, verbose)
##
losses_train.append(loss_train)
accs_train.append(acc_train)
tempo_losses_train.append(tempo_loss)
##
# /---------------------- FOR NNI -------------------------
if epoch % 5 == 0:
test_res = self.test(model, verbose=verbose if not self._is_nni else 0)
test_results.append(test_res)
if self._is_nni:
test_acc = test_res["acc"]
intermediate_test_acc.append(test_acc)
return loss_train, acc_train, intermediate_test_acc, losses_train, \
tempo_losses_train, accs_train, test_results
def _train(self, epoch, model, verbose=2):
model_ = model["model"]
model_ = model_.to(self._device)
optimizer = model["optimizer"]
z_vals = [[] for _ in range(len(model["adj_matrices"]))]
outputs = [[] for _ in range(len(model["adj_matrices"]))]
labeled_indices = [[i for i in range(len(model["training_labels"][t])) if model["training_labels"][t][i] != -1]# and not np.array_equal([1.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],model["training_labels"][t][i])]
for t in range(len(model["training_labels"]))]
labels = [torch.DoubleTensor([model["training_labels"][t][i]
for i in labeled_indices[t]]).to(self._device) for t in
range(len(model["training_labels"]))]
model_.train()
optimizer.zero_grad()
for idx, adj in enumerate(model["adj_matrices"]):
training_mat = torch.from_numpy(model["feature_matrices"][idx]).to(dtype=torch.float, device=self._device)
z, output = model_(training_mat, adj)
z_vals[idx].append(z) # After 1 GCN layer only
outputs[idx].append(output) # Final guesses
tempo_loss = 0.
for t in range(len(z_vals) - 1):
#tempo_loss += self._conf["temporal_pen"] * sum( [self._temporal_loss(z_vals[t + 1][0][j], z_vals[t][0][j]) for j in range(len(z_vals[t][0]))])
tempo_loss += self._conf["temporal_pen"] * self._temporal_loss(z_vals[t + 1][0], z_vals[t][0])
loss_train = 0.
for u in range(len(outputs)): # For all times
Nj_s = [sum([labels[u][t][j] for t in range(len(labels[u]))]) for j in range(15)]
out = outputs[u][0][labeled_indices[u], :]
loss_train += self._ce_loss(out, labels[u], Nj_s)
tempo_loss /= ((len(z_vals) - 1) * model["feature_matrices"][0].shape[0])
loss_train /= sum([len(labeled_indices[u]) for u in range(len(outputs))])
total_loss = loss_train + tempo_loss
total_loss.backward()
optimizer.step()
###acc_train = [self.accuracy(outputs[i][0][labeled_indices[i], :], labels[i]) for i in range(len(labels))]
acc_train = self.accuracy(outputs[-1][0][labeled_indices[-1], :], labels[-1])
#acc_train/= len(outputs[-1][0][labeled_indices[-1], :])
# TODO: Right now the training accuracy is only on the last time. Should change?
if verbose == 2:
# Evaluate validation set performance separately,
# deactivates dropout during validation run.
self._logger.debug('Epoch: {:04d} '.format(epoch + 1) +
'ce_loss_train: {:.4f} '.format(loss_train.data.item()) +
'temp_loss: {:.4f} '.format(tempo_loss.data.item()) +
'acc_train: {:.4f} '.format(acc_train))
return loss_train, tempo_loss, acc_train
def test(self, model=None, verbose=2, print_to_file=False):
model_ = model["model"]
z_vals = [[] for x in range(len(model["adj_matrices"]))]
outputs = [[] for x in range(len(model["adj_matrices"]))]
labeled_indices = [[i for i in range(len(model["test_labels"][t])) if model["test_labels"][t][i] != -1]# and not np.array_equal([1.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],model["test_labels"][t][i])]
for t in range(len(model["test_labels"]))]
labels = [torch.DoubleTensor([model["test_labels"][t][i]
for i in labeled_indices[t]]).to(self._device) for t in
range(len(model["test_labels"]))]
model_.eval()
for idx, adj in enumerate(model["adj_matrices"]):
test_mat = torch.from_numpy(model["feature_matrices"][idx]).to(self._device)
z, output = model_(*[test_mat, adj])
z_vals[idx].append(z)
outputs[idx].append(output)
if print_to_file:
self._logger.debug("\nprint to files")
for i in range(len(model["adj_matrices"])):
np_output = outputs[i][0].cpu().data.numpy()
with open(os.path.join("gcn_MSE_weightedLoss", "gcn_" + str(i) + ".pkl"), "wb") as f:
pickle.dump(np_output, f, protocol=pickle.HIGHEST_PROTOCOL)
tempo_loss = 0.
for t in range(len(z_vals) - 1):
tempo_loss += self._conf["temporal_pen"] * self._temporal_loss(z_vals[t + 1][0], z_vals[t][0])
loss_test = 0.
for u in range(len(outputs)): # For all times
out = outputs[u][0][labeled_indices[u], :]
loss_test += self._ce_loss(out, labels[u])
tempo_loss /= ((len(z_vals) - 1) * model["feature_matrices"][0].shape[0])
loss_test /= sum([len(labeled_indices[u]) for u in range(len(outputs))])
total_loss = loss_test + tempo_loss
##acc_test = self.accuracy(outputs[-1][labeled_indices[-1], :], labels[-1]) # TODO: Same for accuracy on train.
acc_test = self.accuracy(outputs[-1][0][labeled_indices[-1], :],
labels[-1]) # TODO: Same for accuracy on train.
if verbose != 0:
self._logger.info("Test: ce_loss= {:.4f} ".format(loss_test.data.item()) +
"temp_loss= {:.4f} ".format(tempo_loss.data.item()) + "acc= {:.4f}".format(acc_test))
result = {"loss": loss_test.data.item(), "acc": acc_test, "tempo_loss": tempo_loss.data.item()}
return result
@staticmethod
def accuracy(output, labels): # should be named kl_divergence and best at lowest value
labs = labels.cpu().data.numpy()
out = output.cpu().data.numpy()
d_per_sample = np.zeros(labels.size(0))
for i in range(labels.size(0)):
for j in range(labels.size(1)):
if labs[i, j] == 0:
continue
else:
d_per_sample[i] += labs[i, j] * np.log(labs[i, j] / out[i, j])
mean_d = np.mean(d_per_sample)
return mean_d
class ModelRunner:
def __init__(self,
conf,
logger,
weights,
graph_params,
data_logger=None,
is_nni=False):
self._logger = logger
self._data_logger = EmptyLogger(
) if data_logger is None else data_logger
self._conf = conf
self._weights_dict = weights
self._clique_size = graph_params['clique_size']
self._graph_params = graph_params
self.bar = 0.5
self._lr = conf["lr"]
self._is_nni = is_nni
self._device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
def _build_weighted_loss(self, labels):
weights_list = []
for i in range(labels.shape[0]):
weights_list.append(self._weights_dict[labels[i].data.item()])
weights_tensor = torch.tensor(weights_list,
dtype=torch.double,
device=self._device)
self._criterion = torch.nn.BCELoss(weight=weights_tensor).to(
self._device)
@property
def logger(self):
return self._logger
@property
def data_logger(self):
return self._data_logger
@property
def graph_params(self):
return self._graph_params
def _get_model(self):
model = RNNGCN(n_features=self._conf["training_mat"][0].shape[1],
iterations=self._conf["iterations"])
opt = self._conf["optimizer"](model.parameters(),
lr=self._conf["lr"],
weight_decay=self._conf["weight_decay"])
if self._is_nni:
self._logger.debug(
f"Model: \nlearning rate: {self._conf['lr']:.4f} \nL2 regularization: {self._conf['weight_decay']:.4f} \n "
f"class weights: {self._weights_dict}")
return {
"model": model,
"optimizer": opt,
"training_mats": self._conf["training_mat"],
"training_adjs": self._conf["training_adj"],
"training_labels": self._conf["training_labels"],
"test_mats": self._conf["test_mat"],
"test_adjs": self._conf["test_adj"],
"test_labels": self._conf["test_labels"]
}
# verbose = 0 - silent
# verbose = 1 - print test results
# verbose = 2 - print train for each epoch and test results
def run(self, verbose=2):
if self._is_nni:
verbose = 0
model = self._get_model()
train_output_labels, intermediate_results, final_train_auc, final_loss_train = \
self.train(self._conf["epochs"], model=model, verbose=verbose)
# Testing
result = self.test(model=model,
verbose=verbose if not self._is_nni else 0)
result["train_output_labels"] = train_output_labels
result["auc_train"] = final_train_auc
if self._is_nni:
self._logger.debug(f'Final loss train: {final_loss_train:.4f}')
self._logger.debug(f'Final AUC train: {final_train_auc:.4f}')
final_results = result["auc"]
self._logger.debug(f'Final AUC test: {final_results:.4f}')
# _nni.report_final_result(test_auc)
if verbose != 0:
for name, val in result.items():
self._data_logger.info(name, val)
return intermediate_results, result
def train(self, epochs, model=None, verbose=2):
auc_train = 0.
output = 0.
train_labels = 0.
loss = 0.
intermediate_test_auc = []
for epoch in range(epochs):
output, train_labels, auc_train, loss = self._train(
epoch, model, verbose)
# /---------------------- FOR NNI -------------------------
if epoch % 5 == 0:
test_res = self.test(
model, verbose=verbose if not self._is_nni else 0)
if self._is_nni:
test_auc = test_res["auc"]
intermediate_test_auc.append(test_auc)
return np.vstack(
(output, train_labels)), intermediate_test_auc, auc_train, loss
def _train(self, epoch, model, verbose=2):
model_ = model["model"]
model_ = model_.to(self._device)
optimizer = model["optimizer"]
graphs_order = np.arange(len(model["training_labels"]))
np.random.shuffle(graphs_order)
outputs = []
all_training_labels = []
for i, idx in enumerate(graphs_order):
training_mat = torch.from_numpy(model["training_mats"][idx]).to(
device=self._device, dtype=torch.float)
training_adj = torch.from_numpy(
model["training_adjs"][idx]).to(device=self._device)
labels = torch.tensor(model["training_labels"][idx],
dtype=torch.double,
device=self._device)
model_.train()
optimizer.zero_grad()
output = model_(*[training_mat, training_adj])
outputs.append(output)
all_training_labels.append(labels)
self._build_weighted_loss(labels)
loss_train = self._criterion(
output.view(output.shape[0]).type_as(labels), labels)
loss_train.backward()
optimizer.step()
out = torch.cat(outputs, dim=0)
lb = torch.cat(all_training_labels, dim=0)
out = out.view(out.shape[0]).type_as(lb)
self._build_weighted_loss(lb)
all_training_loss = self._criterion(out, lb)
auc_train = self.auc(out, lb)
if verbose == 2:
# Evaluate validation set performance separately,
# deactivates dropout during validation run.
self._logger.debug(
'Epoch: {:04d} '.format(epoch + 1) +
'loss_train: {:.4f} '.format(all_training_loss.data.item()) +
'auc_train: {:.4f} '.format(auc_train))
return out.tolist(), lb.tolist(), auc_train, all_training_loss
def test(self, model=None, verbose=2):
model_ = model["model"]
graphs_order = np.arange(len(model["test_labels"]))
np.random.shuffle(graphs_order)
outputs = []
all_test_labels = []
for i, idx in enumerate(graphs_order):
test_mat = torch.from_numpy(model["test_mats"][idx]).to(
device=self._device, dtype=torch.float)
test_adj = torch.from_numpy(
model["test_adjs"][idx]).to(device=self._device)
test_labels = torch.tensor(model["test_labels"][idx],
dtype=torch.double,
device=self._device)
model_.eval()
output = model_(*[test_mat, test_adj])
outputs.append(output)
all_test_labels.append(test_labels)
out = torch.cat(outputs, dim=0)
lb = torch.cat(all_test_labels, dim=0)
out = out.view(out.shape[0]).type_as(lb)
self._build_weighted_loss(lb)
loss_test = self._criterion(out, lb)
auc_test = self.auc(out, lb)
ranked_vertices = np.argsort(out.cpu().detach().numpy())
positives = [
ranked_vertices[-(i + 1)] for i in range(self._clique_size)
]
true_positives = [i for i in positives if lb.tolist()[i] > 0.5]
if verbose != 0:
self._logger.info(
f"Test: loss= {loss_test.data.item():.4f} auc= {auc_test:.4f}")
result = {
"loss": loss_test.data.item(),
"auc": auc_test,
"true_positives": len(true_positives),
"output_labels": np.vstack((out.tolist(), lb.tolist()))
}
return result
@staticmethod
def auc(output, labels):
preds = output.data.type_as(labels)
return roc_auc_score(labels.cpu(), preds.cpu())
@staticmethod
def roc_curve(output, labels):
preds = output.data.type_as(labels)
return roc_curve(labels.cpu(), preds.cpu())