def build_gcn(args, tasker):
gcn_args = u.Namespace(args.gcn_parameters)
gcn_args.feats_per_node = tasker.feats_per_node
if args.model == 'gcn':
return mls.Sp_GCN(gcn_args,
activation=torch.nn.RReLU()).to(args.device)
elif args.model == 'skipgcn':
return mls.Sp_Skip_GCN(gcn_args,
activation=torch.nn.RReLU()).to(args.device)
elif args.model == 'skipfeatsgcn':
return mls.Sp_Skip_NodeFeats_GCN(
gcn_args, activation=torch.nn.RReLU()).to(args.device)
else:
assert args.num_hist_steps > 0, 'more than one step is necessary to train LSTM'
if args.model == 'lstmA':
return mls.Sp_GCN_LSTM_A(gcn_args, activation=torch.nn.RReLU()).to(
args.device)
elif args.model == 'gruA':
return mls.Sp_GCN_GRU_A(gcn_args, activation=torch.nn.RReLU()).to(
args.device)
elif args.model == 'lstmB':
return mls.Sp_GCN_LSTM_B(gcn_args, activation=torch.nn.RReLU()).to(
args.device)
elif args.model == 'gruB':
return mls.Sp_GCN_GRU_B(gcn_args, activation=torch.nn.RReLU()).to(
args.device)
elif args.model == 'egcn':
return egcn.EGCN(gcn_args,
activation=torch.nn.RReLU()).to(args.device)
elif args.model == 'egcn_h':
return egcn_h.EGCN(gcn_args,
activation=torch.nn.RReLU(),
device=args.device)
elif args.model == 'skipfeatsegcn_h':
return egcn_h.EGCN(gcn_args,
activation=torch.nn.RReLU(),
device=args.device,
skipfeats=True)
elif args.model == 'egcn_o':
return egcn_o.EGCN(gcn_args,
activation=torch.nn.RReLU(),
device=args.device)
else:
raise NotImplementedError('need to finish modifying the models')
def __init__(self, args):
super().__init__()
self.args = args
cell_args = u.Namespace({})
cell_args.rows = args.in_feats
cell_args.cols = args.out_feats
# self.dropout = nn.Dropout(dropout)
self.device = args.device
self.aggr_weight = self_Atten(args)
self.evolve_weights = mat_GRU_cell(cell_args)
# self.V = Parameter(torch.randn(self.args.in_feats, self.args.in_feats, requires_grad=True))
self.activation = self.args.activation
self.GCN_init_weights = Parameter(
torch.randn(self.args.in_feats, self.args.out_feats))
# self.GCN_init_weights = torch.randn(self.args.in_feats,self.args.out_feats)
nn.init.orthogonal_(self.GCN_init_weights)
def deployApp(data, self):
env = utils.Namespace()
setattr(env, "machine", "")
setattr(env, "ipaddr", self.request.remote_ip)
setattr(env, "username", "")
setattr(env, "program", "webctl")
master_svn = data.get("master_svn")
config_svn = data.get("config_svn")
logging.debug("master_svn:" + str(master_svn))
logging.debug("config_svn:" + str(config_svn))
if not master_svn:
master_svn = "HEAD"
if not config_svn:
config_svn = "HEAD"
OpenAPI("openapi",
None,
"deployApp",
args=(data.get("brand").upper(), data.get("app").upper(),
master_svn, config_svn),
env=env).socketCall(self)
def load_edges(self, args, tar_archive):
data = u.load_data_from_tar(args.uc_irc_args.edges_file,
tar_archive,
starting_line=2,
sep=' ')
cols = u.Namespace({'source': 0,
'target': 1,
'weight': 2,
'time': 3})
data = data.long()
self.num_nodes = int(data[:, [cols.source, cols.target]].max())
# first id should be 0 (they are already contiguous)
data[:, [cols.source, cols.target]] -= 1
# add edges in the other direction (simmetric)
data = torch.cat([data,
data[:, [cols.target,
cols.source,
cols.weight,
cols.time]]],
dim=0)
data[:, cols.time] = u.aggregate_by_time(data[:, cols.time],
args.uc_irc_args.aggr_time)
ids = data[:, cols.source] * self.num_nodes + data[:, cols.target]
self.num_non_existing = float(self.num_nodes ** 2 - ids.unique().size(0))
idx = data[:, [cols.source,
cols.target,
cols.time]]
self.max_time = data[:, cols.time].max()
self.min_time = data[:, cols.time].min()
return {'idx': idx, 'vals': torch.ones(idx.size(0))}
def load_edges_from_file(self, edges_file, folder, ids_str_to_int):
edges = []
not_found = 0
file = edges_file
file = os.path.join(folder, file)
with open(file) as file:
file = file.read().splitlines()
cols = u.Namespace({'source': 0,
'target': 1,
'time': 3,
'label': 4})
base_time = datetime.strptime("19800101", '%Y%m%d')
for line in file[1:]:
fields = line.split('\t')
sr = fields[cols.source]
tg = fields[cols.target]
if sr in ids_str_to_int.keys() and tg in ids_str_to_int.keys():
sr = ids_str_to_int[sr]
tg = ids_str_to_int[tg]
time = fields[cols.time].split(' ')[0]
time = datetime.strptime(time, '%Y-%m-%d')
time = (time - base_time).days
label = int(fields[cols.label])
edges.append([sr, tg, time, label])
# add the other edge to make it undirected
edges.append([tg, sr, time, label])
else:
not_found += 1
return edges, not_found
def eval_predicitions_at_k(self, predictions, true_classes, num_classes,
k):
conf_mat_per_class = utils.Namespace({})
conf_mat_per_class.true_positives = {}
conf_mat_per_class.false_negatives = {}
conf_mat_per_class.false_positives = {}
if predictions.size(0) < k:
k = predictions.size(0)
for cl in range(num_classes):
# sort for prediction with higher score for target class (cl)
_, idx_preds_at_k = torch.topk(predictions[:, cl],
k,
dim=0,
largest=True,
sorted=True)
predictions_at_k = predictions[idx_preds_at_k]
predicted_classes = predictions_at_k.argmax(dim=1)
cl_indices_at_k = true_classes[idx_preds_at_k] == cl
cl_indices = true_classes == cl
pos = predicted_classes == cl
hits = (predicted_classes[cl_indices_at_k] ==
true_classes[idx_preds_at_k][cl_indices_at_k])
tp = hits.sum()
fn = true_classes[cl_indices].size(
0
) - tp # This only if we want to consider the size at K -> hits.size(0) - tp
fp = pos.sum() - tp
conf_mat_per_class.true_positives[cl] = tp
conf_mat_per_class.false_negatives[cl] = fn
conf_mat_per_class.false_positives[cl] = fp
return conf_mat_per_class
def eval_predicitions(self, predictions, true_classes, num_classes):
predicted_classes = predictions.argmax(dim=1)
failures = (predicted_classes != true_classes).sum(dtype=torch.float)
error = failures / predictions.size(0)
conf_mat_per_class = utils.Namespace({})
conf_mat_per_class.true_positives = {}
conf_mat_per_class.false_negatives = {}
conf_mat_per_class.false_positives = {}
for cl in range(num_classes):
cl_indices = true_classes == cl
pos = predicted_classes == cl
hits = (predicted_classes[cl_indices] == true_classes[cl_indices])
tp = hits.sum()
fn = hits.size(0) - tp
fp = pos.sum() - tp
conf_mat_per_class.true_positives[cl] = tp
conf_mat_per_class.false_negatives[cl] = fn
conf_mat_per_class.false_positives[cl] = fp
return error, conf_mat_per_class
import torch
import utils as u
import numpy as np
import time
ECOLS = u.Namespace({
'source': 0,
'target': 1,
'time': 2,
'label': 3
}) #--> added for edge_cls
# def get_2_hot_deg_feats(adj,max_deg_out,max_deg_in,num_nodes):
# #For now it'll just return a 2-hot vector
# adj['vals'] = torch.ones(adj['idx'].size(0))
# degs_out, degs_in = get_degree_vects(adj,num_nodes)
# degs_out = {'idx': torch.cat([torch.arange(num_nodes).view(-1,1),
# degs_out.view(-1,1)],dim=1),
# 'vals': torch.ones(num_nodes)}
# # print ('XXX degs_out',degs_out['idx'].size(),degs_out['vals'].size())
# degs_out = u.make_sparse_tensor(degs_out,'long',[num_nodes,max_deg_out])
# degs_in = {'idx': torch.cat([torch.arange(num_nodes).view(-1,1),
# degs_in.view(-1,1)],dim=1),
# 'vals': torch.ones(num_nodes)}
# degs_in = u.make_sparse_tensor(degs_in,'long',[num_nodes,max_deg_in])
# hot_2 = torch.cat([degs_out,degs_in],dim = 1)
# hot_2 = {'idx': hot_2._indices().t(),
def _stns(self):
if not hasattr(self, '__stns'):
self.__stns = utils.Namespace(self.serviceType)
return self.__stns
__copyright__ = 'Copyright (c) 2014 SFR (http://www.sfr.com)'
__license__ = 'GNU LESSER GENERAL PUBLIC LICENSE Version 2.1'
import time
import logging
try:
from xml.etree import cElementTree as ElementTree
except:
from xml.etree import ElementTree
import utils
import http, urlparse
from wsgiref.util import shift_path_info, guess_scheme
SQNS = utils.Namespace('http://schemas.xmlsoap.org/soap/envelope/', 'soap')
DNS = utils.Namespace('urn:schemas-upnp-org:device-1-0', 'device')
SNS = utils.Namespace('urn:schemas-upnp-org:service-1-0', 'service')
CNS = utils.Namespace('urn:schemas-upnp-org:control-1-0', 'control')
ENS = utils.Namespace('urn:schemas-upnp-org:event-1-0', 'event')
SES = "http://schemas.xmlsoap.org/soap/encoding/"
class BaseUPnPObject(object):
EXPIRY = 1800
def __init__(self, **kwargs):
for k, v in kwargs.items():
if k not in self._ATTRS:
raise TypeError('Unknown attribute %s' % k)
def __init__(self,args):
assert args.task in ['link_pred', 'edge_cls'], 'bitcoin only implements link_pred or edge_cls'
self.ecols = u.Namespace({'FromNodeId': 0,
'ToNodeId': 1,
'Weight': 2,
'TimeStep': 3
})
args.bitcoin_args = u.Namespace(args.bitcoin_args)
#build edge data structure
edges = self.load_edges(args.bitcoin_args)
edges = self.make_contigous_node_ids(edges)
num_nodes = edges[:,[self.ecols.FromNodeId,
self.ecols.ToNodeId]].unique().size(0)
timesteps = u.aggregate_by_time(edges[:,self.ecols.TimeStep],args.bitcoin_args.aggr_time)
self.max_time = timesteps.max()
self.min_time = timesteps.min()
edges[:,self.ecols.TimeStep] = timesteps
edges[:,self.ecols.Weight] = self.cluster_negs_and_positives(edges[:,self.ecols.Weight])
#add the reversed link to make the graph undirected
edges = torch.cat([edges,edges[:,[self.ecols.ToNodeId,
self.ecols.FromNodeId,
self.ecols.Weight,
self.ecols.TimeStep]]])
#separate classes
sp_indices = edges[:,[self.ecols.FromNodeId,
self.ecols.ToNodeId,
self.ecols.TimeStep]].t()
sp_values = edges[:,self.ecols.Weight]
neg_mask = sp_values == -1
neg_sp_indices = sp_indices[:,neg_mask]
neg_sp_values = sp_values[neg_mask]
neg_sp_edges = torch.sparse.LongTensor(neg_sp_indices
,neg_sp_values,
torch.Size([num_nodes,
num_nodes,
self.max_time+1])).coalesce()
pos_mask = sp_values == 1
pos_sp_indices = sp_indices[:,pos_mask]
pos_sp_values = sp_values[pos_mask]
pos_sp_edges = torch.sparse.LongTensor(pos_sp_indices
,pos_sp_values,
torch.Size([num_nodes,
num_nodes,
self.max_time+1])).coalesce()
#scale positive class to separate after adding
pos_sp_edges *= 1000
#we substract the neg_sp_edges to make the values positive
sp_edges = (pos_sp_edges - neg_sp_edges).coalesce()
#separating negs and positive edges per edge/timestamp
vals = sp_edges._values()
neg_vals = vals%1000
pos_vals = vals//1000
#We add the negative and positive scores and do majority voting
vals = pos_vals - neg_vals
#creating labels new_vals -> the label of the edges
new_vals = torch.zeros(vals.size(0),dtype=torch.long)
new_vals[vals>0] = 1
new_vals[vals<=0] = 0
indices_labels = torch.cat([sp_edges._indices().t(),new_vals.view(-1,1)],dim=1)
#the weight of the edges (vals), is simply the number of edges between two entities at each time_step
vals = pos_vals + neg_vals
self.edges = {'idx': indices_labels, 'vals': vals}
self.num_nodes = num_nodes
self.num_classes = 2
def __init__(self, args):
args.reddit_args = u.Namespace(args.reddit_args)
folder = args.reddit_args.folder
# load nodes
cols = u.Namespace({'id': 0,
'feats': 1})
file = args.reddit_args.nodes_file
file = os.path.join(folder, file)
with open(file) as file:
file = file.read().splitlines()
ids_str_to_int = {}
id_counter = 0
feats = []
for line in file:
line = line.split(',')
# node id
nd_id = line[0]
if nd_id not in ids_str_to_int.keys():
ids_str_to_int[nd_id] = id_counter
id_counter += 1
nd_feats = [float(r) for r in line[1:]]
feats.append(nd_feats)
else:
print('duplicate id', nd_id)
raise Exception('duplicate_id')
feats = torch.tensor(feats, dtype=torch.float)
num_nodes = feats.size(0)
edges = []
not_found = 0
# load edges in title
edges_tmp, not_found_tmp = self.load_edges_from_file(args.reddit_args.title_edges_file,
folder,
ids_str_to_int)
edges.extend(edges_tmp)
not_found += not_found_tmp
# load edges in bodies
edges_tmp, not_found_tmp = self.load_edges_from_file(args.reddit_args.body_edges_file,
folder,
ids_str_to_int)
edges.extend(edges_tmp)
not_found += not_found_tmp
# min time should be 0 and time aggregation
edges = torch.LongTensor(edges)
edges[:, 2] = u.aggregate_by_time(edges[:, 2], args.reddit_args.aggr_time)
max_time = edges[:, 2].max()
# separate classes
sp_indices = edges[:, :3].t()
sp_values = edges[:, 3]
# sp_edges = torch.sparse.LongTensor(sp_indices
# ,sp_values,
# torch.Size([num_nodes,
# num_nodes,
# max_time+1])).coalesce()
# vals = sp_edges._values()
# print(vals[vals>0].sum() + vals[vals<0].sum()*-1)
# asdf
pos_mask = sp_values == 1
neg_mask = sp_values == -1
neg_sp_indices = sp_indices[:, neg_mask]
neg_sp_values = sp_values[neg_mask]
neg_sp_edges = torch.sparse.LongTensor(neg_sp_indices
, neg_sp_values,
torch.Size([num_nodes,
num_nodes,
max_time + 1])).coalesce()
pos_sp_indices = sp_indices[:, pos_mask]
pos_sp_values = sp_values[pos_mask]
pos_sp_edges = torch.sparse.LongTensor(pos_sp_indices
, pos_sp_values,
torch.Size([num_nodes,
num_nodes,
max_time + 1])).coalesce()
# scale positive class to separate after adding
pos_sp_edges *= 1000
sp_edges = (pos_sp_edges - neg_sp_edges).coalesce()
# separating negs and positive edges per edge/timestamp
vals = sp_edges._values()
neg_vals = vals % 1000
pos_vals = vals // 1000
# vals is simply the number of edges between two nodes at the same time_step, regardless of the edge label
vals = pos_vals - neg_vals
# creating labels new_vals -> the label of the edges
new_vals = torch.zeros(vals.size(0), dtype=torch.long)
new_vals[vals > 0] = 1
new_vals[vals <= 0] = 0
vals = pos_vals + neg_vals
indices_labels = torch.cat([sp_edges._indices().t(), new_vals.view(-1, 1)], dim=1)
self.edges = {'idx': indices_labels, 'vals': vals}
self.num_classes = 2
self.feats_per_node = feats.size(1)
self.num_nodes = num_nodes
self.nodes_feats = feats
self.max_time = max_time
self.min_time = 0
__author__ = 'Antoine Monnet [email protected]' __copyright__ = 'Copyright (c) 2014 SFR (http://www.sfr.com)' __license__ = 'GNU LESSER GENERAL PUBLIC LICENSE Version 2.1' import logging from persist import DB import ussl import utils try: from xml.etree import cElementTree as ElementTree except: from xml.etree import ElementTree DPNS = utils.Namespace('urn:schemas-upnp-org:gw:DeviceProtection', 'dp') XSINS = utils.Namespace('http://www.w3.org/2001/XMLSchema-instance', 'xsi') DPLOC = "urn:schemas-upnp-org:gw:DeviceProtection http://www.upnp.org/schemas/gw/DeviceProtection-v1.xsd" def acl(roles=None, restricted_roles=None): def restricted_action(action): action.roles = Roles(roles.split()) if roles is not None else None action.restricted_roles = Roles( restricted_roles.split()) if restricted_roles is not None else None return action return restricted_action from collections import Set
@author: ┤зл█
'''
import argparse
import logging
import sys
import env
from openapi import OpenAPI
import utils
from db import AppHistory
if __name__ == '__main__':
machine, ipaddr = utils.getHostInfo()
e = utils.Namespace()
setattr(e, "machine", machine)
setattr(e, "ipaddr", ipaddr)
setattr(e, "username", utils.getCurrentUser())
setattr(e, "program", "fanctl")
def deployApp(args):
OpenAPI("openapi", None, "deployApp",
args=(args.brand.upper(), args.module.upper(), args.master_rev, args.config_rev),
env=e).call()
def rollbackApp(args):
OpenAPI("openapi", None, "rollbackApp",
args=(args.brand.upper(), args.module.upper()),env=e).call()
def stopServer(args):
def forward(self, logits, labels, weights_):
'''
logits is a matrix M by C where m is the number of classifications and C are the number of classes
labels is a integer tensor of size M where each element corresponds to the class that prediction i
should be matching to
'''
weights = torch.tensor(weights_).to('cuda')
labels = labels.view(-1, 1)
alpha = weights[labels].view(-1, 1)
loss = alpha * (-logits.gather(-1, labels) + self.logsumexp(logits))
return loss.mean()
if __name__ == '__main__':
dataset = u.Namespace({'num_non_existing': torch.tensor(10)})
args = u.Namespace({'class_weights': [1.0, 1.0], 'task': 'no_link_pred'})
labels = torch.tensor([1, 0])
ce_ref = torch.nn.CrossEntropyLoss(reduction='sum')
ce = Cross_Entropy(args, dataset)
# print(ce.weights(labels))
# print(ce.weights(labels))
logits = torch.tensor([[1.0, -1.0], [1.0, -1.0]])
logits = torch.rand((5, 2))
labels = torch.randint(0, 2, (5, ))
print(ce(logits, labels) - ce_ref(logits, labels))
exit()
ce.logsumexp(logits)
# print(labels)
# print(ce.weights(labels))
# print(ce.weights(labels)[labels])
