def build_gcn(args, tasker): gcn_args = u.Namespace(args.gcn_parameters) gcn_args.feats_per_node = tasker.feats_per_node if args.model == 'gcn': return mls.Sp_GCN(gcn_args, activation=torch.nn.RReLU()).to(args.device) elif args.model == 'skipgcn': return mls.Sp_Skip_GCN(gcn_args, activation=torch.nn.RReLU()).to(args.device) elif args.model == 'skipfeatsgcn': return mls.Sp_Skip_NodeFeats_GCN( gcn_args, activation=torch.nn.RReLU()).to(args.device) else: assert args.num_hist_steps > 0, 'more than one step is necessary to train LSTM' if args.model == 'lstmA': return mls.Sp_GCN_LSTM_A(gcn_args, activation=torch.nn.RReLU()).to( args.device) elif args.model == 'gruA': return mls.Sp_GCN_GRU_A(gcn_args, activation=torch.nn.RReLU()).to( args.device) elif args.model == 'lstmB': return mls.Sp_GCN_LSTM_B(gcn_args, activation=torch.nn.RReLU()).to( args.device) elif args.model == 'gruB': return mls.Sp_GCN_GRU_B(gcn_args, activation=torch.nn.RReLU()).to( args.device) elif args.model == 'egcn': return egcn.EGCN(gcn_args, activation=torch.nn.RReLU()).to(args.device) elif args.model == 'egcn_h': return egcn_h.EGCN(gcn_args, activation=torch.nn.RReLU(), device=args.device) elif args.model == 'skipfeatsegcn_h': return egcn_h.EGCN(gcn_args, activation=torch.nn.RReLU(), device=args.device, skipfeats=True) elif args.model == 'egcn_o': return egcn_o.EGCN(gcn_args, activation=torch.nn.RReLU(), device=args.device) else: raise NotImplementedError('need to finish modifying the models')
def __init__(self, args): super().__init__() self.args = args cell_args = u.Namespace({}) cell_args.rows = args.in_feats cell_args.cols = args.out_feats # self.dropout = nn.Dropout(dropout) self.device = args.device self.aggr_weight = self_Atten(args) self.evolve_weights = mat_GRU_cell(cell_args) # self.V = Parameter(torch.randn(self.args.in_feats, self.args.in_feats, requires_grad=True)) self.activation = self.args.activation self.GCN_init_weights = Parameter( torch.randn(self.args.in_feats, self.args.out_feats)) # self.GCN_init_weights = torch.randn(self.args.in_feats,self.args.out_feats) nn.init.orthogonal_(self.GCN_init_weights)
def deployApp(data, self): env = utils.Namespace() setattr(env, "machine", "") setattr(env, "ipaddr", self.request.remote_ip) setattr(env, "username", "") setattr(env, "program", "webctl") master_svn = data.get("master_svn") config_svn = data.get("config_svn") logging.debug("master_svn:" + str(master_svn)) logging.debug("config_svn:" + str(config_svn)) if not master_svn: master_svn = "HEAD" if not config_svn: config_svn = "HEAD" OpenAPI("openapi", None, "deployApp", args=(data.get("brand").upper(), data.get("app").upper(), master_svn, config_svn), env=env).socketCall(self)
def load_edges(self, args, tar_archive): data = u.load_data_from_tar(args.uc_irc_args.edges_file, tar_archive, starting_line=2, sep=' ') cols = u.Namespace({'source': 0, 'target': 1, 'weight': 2, 'time': 3}) data = data.long() self.num_nodes = int(data[:, [cols.source, cols.target]].max()) # first id should be 0 (they are already contiguous) data[:, [cols.source, cols.target]] -= 1 # add edges in the other direction (simmetric) data = torch.cat([data, data[:, [cols.target, cols.source, cols.weight, cols.time]]], dim=0) data[:, cols.time] = u.aggregate_by_time(data[:, cols.time], args.uc_irc_args.aggr_time) ids = data[:, cols.source] * self.num_nodes + data[:, cols.target] self.num_non_existing = float(self.num_nodes ** 2 - ids.unique().size(0)) idx = data[:, [cols.source, cols.target, cols.time]] self.max_time = data[:, cols.time].max() self.min_time = data[:, cols.time].min() return {'idx': idx, 'vals': torch.ones(idx.size(0))}
def load_edges_from_file(self, edges_file, folder, ids_str_to_int): edges = [] not_found = 0 file = edges_file file = os.path.join(folder, file) with open(file) as file: file = file.read().splitlines() cols = u.Namespace({'source': 0, 'target': 1, 'time': 3, 'label': 4}) base_time = datetime.strptime("19800101", '%Y%m%d') for line in file[1:]: fields = line.split('\t') sr = fields[cols.source] tg = fields[cols.target] if sr in ids_str_to_int.keys() and tg in ids_str_to_int.keys(): sr = ids_str_to_int[sr] tg = ids_str_to_int[tg] time = fields[cols.time].split(' ')[0] time = datetime.strptime(time, '%Y-%m-%d') time = (time - base_time).days label = int(fields[cols.label]) edges.append([sr, tg, time, label]) # add the other edge to make it undirected edges.append([tg, sr, time, label]) else: not_found += 1 return edges, not_found
def eval_predicitions_at_k(self, predictions, true_classes, num_classes, k): conf_mat_per_class = utils.Namespace({}) conf_mat_per_class.true_positives = {} conf_mat_per_class.false_negatives = {} conf_mat_per_class.false_positives = {} if predictions.size(0) < k: k = predictions.size(0) for cl in range(num_classes): # sort for prediction with higher score for target class (cl) _, idx_preds_at_k = torch.topk(predictions[:, cl], k, dim=0, largest=True, sorted=True) predictions_at_k = predictions[idx_preds_at_k] predicted_classes = predictions_at_k.argmax(dim=1) cl_indices_at_k = true_classes[idx_preds_at_k] == cl cl_indices = true_classes == cl pos = predicted_classes == cl hits = (predicted_classes[cl_indices_at_k] == true_classes[idx_preds_at_k][cl_indices_at_k]) tp = hits.sum() fn = true_classes[cl_indices].size( 0 ) - tp # This only if we want to consider the size at K -> hits.size(0) - tp fp = pos.sum() - tp conf_mat_per_class.true_positives[cl] = tp conf_mat_per_class.false_negatives[cl] = fn conf_mat_per_class.false_positives[cl] = fp return conf_mat_per_class
def eval_predicitions(self, predictions, true_classes, num_classes): predicted_classes = predictions.argmax(dim=1) failures = (predicted_classes != true_classes).sum(dtype=torch.float) error = failures / predictions.size(0) conf_mat_per_class = utils.Namespace({}) conf_mat_per_class.true_positives = {} conf_mat_per_class.false_negatives = {} conf_mat_per_class.false_positives = {} for cl in range(num_classes): cl_indices = true_classes == cl pos = predicted_classes == cl hits = (predicted_classes[cl_indices] == true_classes[cl_indices]) tp = hits.sum() fn = hits.size(0) - tp fp = pos.sum() - tp conf_mat_per_class.true_positives[cl] = tp conf_mat_per_class.false_negatives[cl] = fn conf_mat_per_class.false_positives[cl] = fp return error, conf_mat_per_class
import torch import utils as u import numpy as np import time ECOLS = u.Namespace({ 'source': 0, 'target': 1, 'time': 2, 'label': 3 }) #--> added for edge_cls # def get_2_hot_deg_feats(adj,max_deg_out,max_deg_in,num_nodes): # #For now it'll just return a 2-hot vector # adj['vals'] = torch.ones(adj['idx'].size(0)) # degs_out, degs_in = get_degree_vects(adj,num_nodes) # degs_out = {'idx': torch.cat([torch.arange(num_nodes).view(-1,1), # degs_out.view(-1,1)],dim=1), # 'vals': torch.ones(num_nodes)} # # print ('XXX degs_out',degs_out['idx'].size(),degs_out['vals'].size()) # degs_out = u.make_sparse_tensor(degs_out,'long',[num_nodes,max_deg_out]) # degs_in = {'idx': torch.cat([torch.arange(num_nodes).view(-1,1), # degs_in.view(-1,1)],dim=1), # 'vals': torch.ones(num_nodes)} # degs_in = u.make_sparse_tensor(degs_in,'long',[num_nodes,max_deg_in]) # hot_2 = torch.cat([degs_out,degs_in],dim = 1) # hot_2 = {'idx': hot_2._indices().t(),
def _stns(self): if not hasattr(self, '__stns'): self.__stns = utils.Namespace(self.serviceType) return self.__stns
__copyright__ = 'Copyright (c) 2014 SFR (http://www.sfr.com)' __license__ = 'GNU LESSER GENERAL PUBLIC LICENSE Version 2.1' import time import logging try: from xml.etree import cElementTree as ElementTree except: from xml.etree import ElementTree import utils import http, urlparse from wsgiref.util import shift_path_info, guess_scheme SQNS = utils.Namespace('http://schemas.xmlsoap.org/soap/envelope/', 'soap') DNS = utils.Namespace('urn:schemas-upnp-org:device-1-0', 'device') SNS = utils.Namespace('urn:schemas-upnp-org:service-1-0', 'service') CNS = utils.Namespace('urn:schemas-upnp-org:control-1-0', 'control') ENS = utils.Namespace('urn:schemas-upnp-org:event-1-0', 'event') SES = "http://schemas.xmlsoap.org/soap/encoding/" class BaseUPnPObject(object): EXPIRY = 1800 def __init__(self, **kwargs): for k, v in kwargs.items(): if k not in self._ATTRS: raise TypeError('Unknown attribute %s' % k)
def __init__(self,args): assert args.task in ['link_pred', 'edge_cls'], 'bitcoin only implements link_pred or edge_cls' self.ecols = u.Namespace({'FromNodeId': 0, 'ToNodeId': 1, 'Weight': 2, 'TimeStep': 3 }) args.bitcoin_args = u.Namespace(args.bitcoin_args) #build edge data structure edges = self.load_edges(args.bitcoin_args) edges = self.make_contigous_node_ids(edges) num_nodes = edges[:,[self.ecols.FromNodeId, self.ecols.ToNodeId]].unique().size(0) timesteps = u.aggregate_by_time(edges[:,self.ecols.TimeStep],args.bitcoin_args.aggr_time) self.max_time = timesteps.max() self.min_time = timesteps.min() edges[:,self.ecols.TimeStep] = timesteps edges[:,self.ecols.Weight] = self.cluster_negs_and_positives(edges[:,self.ecols.Weight]) #add the reversed link to make the graph undirected edges = torch.cat([edges,edges[:,[self.ecols.ToNodeId, self.ecols.FromNodeId, self.ecols.Weight, self.ecols.TimeStep]]]) #separate classes sp_indices = edges[:,[self.ecols.FromNodeId, self.ecols.ToNodeId, self.ecols.TimeStep]].t() sp_values = edges[:,self.ecols.Weight] neg_mask = sp_values == -1 neg_sp_indices = sp_indices[:,neg_mask] neg_sp_values = sp_values[neg_mask] neg_sp_edges = torch.sparse.LongTensor(neg_sp_indices ,neg_sp_values, torch.Size([num_nodes, num_nodes, self.max_time+1])).coalesce() pos_mask = sp_values == 1 pos_sp_indices = sp_indices[:,pos_mask] pos_sp_values = sp_values[pos_mask] pos_sp_edges = torch.sparse.LongTensor(pos_sp_indices ,pos_sp_values, torch.Size([num_nodes, num_nodes, self.max_time+1])).coalesce() #scale positive class to separate after adding pos_sp_edges *= 1000 #we substract the neg_sp_edges to make the values positive sp_edges = (pos_sp_edges - neg_sp_edges).coalesce() #separating negs and positive edges per edge/timestamp vals = sp_edges._values() neg_vals = vals%1000 pos_vals = vals//1000 #We add the negative and positive scores and do majority voting vals = pos_vals - neg_vals #creating labels new_vals -> the label of the edges new_vals = torch.zeros(vals.size(0),dtype=torch.long) new_vals[vals>0] = 1 new_vals[vals<=0] = 0 indices_labels = torch.cat([sp_edges._indices().t(),new_vals.view(-1,1)],dim=1) #the weight of the edges (vals), is simply the number of edges between two entities at each time_step vals = pos_vals + neg_vals self.edges = {'idx': indices_labels, 'vals': vals} self.num_nodes = num_nodes self.num_classes = 2
def __init__(self, args): args.reddit_args = u.Namespace(args.reddit_args) folder = args.reddit_args.folder # load nodes cols = u.Namespace({'id': 0, 'feats': 1}) file = args.reddit_args.nodes_file file = os.path.join(folder, file) with open(file) as file: file = file.read().splitlines() ids_str_to_int = {} id_counter = 0 feats = [] for line in file: line = line.split(',') # node id nd_id = line[0] if nd_id not in ids_str_to_int.keys(): ids_str_to_int[nd_id] = id_counter id_counter += 1 nd_feats = [float(r) for r in line[1:]] feats.append(nd_feats) else: print('duplicate id', nd_id) raise Exception('duplicate_id') feats = torch.tensor(feats, dtype=torch.float) num_nodes = feats.size(0) edges = [] not_found = 0 # load edges in title edges_tmp, not_found_tmp = self.load_edges_from_file(args.reddit_args.title_edges_file, folder, ids_str_to_int) edges.extend(edges_tmp) not_found += not_found_tmp # load edges in bodies edges_tmp, not_found_tmp = self.load_edges_from_file(args.reddit_args.body_edges_file, folder, ids_str_to_int) edges.extend(edges_tmp) not_found += not_found_tmp # min time should be 0 and time aggregation edges = torch.LongTensor(edges) edges[:, 2] = u.aggregate_by_time(edges[:, 2], args.reddit_args.aggr_time) max_time = edges[:, 2].max() # separate classes sp_indices = edges[:, :3].t() sp_values = edges[:, 3] # sp_edges = torch.sparse.LongTensor(sp_indices # ,sp_values, # torch.Size([num_nodes, # num_nodes, # max_time+1])).coalesce() # vals = sp_edges._values() # print(vals[vals>0].sum() + vals[vals<0].sum()*-1) # asdf pos_mask = sp_values == 1 neg_mask = sp_values == -1 neg_sp_indices = sp_indices[:, neg_mask] neg_sp_values = sp_values[neg_mask] neg_sp_edges = torch.sparse.LongTensor(neg_sp_indices , neg_sp_values, torch.Size([num_nodes, num_nodes, max_time + 1])).coalesce() pos_sp_indices = sp_indices[:, pos_mask] pos_sp_values = sp_values[pos_mask] pos_sp_edges = torch.sparse.LongTensor(pos_sp_indices , pos_sp_values, torch.Size([num_nodes, num_nodes, max_time + 1])).coalesce() # scale positive class to separate after adding pos_sp_edges *= 1000 sp_edges = (pos_sp_edges - neg_sp_edges).coalesce() # separating negs and positive edges per edge/timestamp vals = sp_edges._values() neg_vals = vals % 1000 pos_vals = vals // 1000 # vals is simply the number of edges between two nodes at the same time_step, regardless of the edge label vals = pos_vals - neg_vals # creating labels new_vals -> the label of the edges new_vals = torch.zeros(vals.size(0), dtype=torch.long) new_vals[vals > 0] = 1 new_vals[vals <= 0] = 0 vals = pos_vals + neg_vals indices_labels = torch.cat([sp_edges._indices().t(), new_vals.view(-1, 1)], dim=1) self.edges = {'idx': indices_labels, 'vals': vals} self.num_classes = 2 self.feats_per_node = feats.size(1) self.num_nodes = num_nodes self.nodes_feats = feats self.max_time = max_time self.min_time = 0
__author__ = 'Antoine Monnet [email protected]' __copyright__ = 'Copyright (c) 2014 SFR (http://www.sfr.com)' __license__ = 'GNU LESSER GENERAL PUBLIC LICENSE Version 2.1' import logging from persist import DB import ussl import utils try: from xml.etree import cElementTree as ElementTree except: from xml.etree import ElementTree DPNS = utils.Namespace('urn:schemas-upnp-org:gw:DeviceProtection', 'dp') XSINS = utils.Namespace('http://www.w3.org/2001/XMLSchema-instance', 'xsi') DPLOC = "urn:schemas-upnp-org:gw:DeviceProtection http://www.upnp.org/schemas/gw/DeviceProtection-v1.xsd" def acl(roles=None, restricted_roles=None): def restricted_action(action): action.roles = Roles(roles.split()) if roles is not None else None action.restricted_roles = Roles( restricted_roles.split()) if restricted_roles is not None else None return action return restricted_action from collections import Set
@author: ┤зл█ ''' import argparse import logging import sys import env from openapi import OpenAPI import utils from db import AppHistory if __name__ == '__main__': machine, ipaddr = utils.getHostInfo() e = utils.Namespace() setattr(e, "machine", machine) setattr(e, "ipaddr", ipaddr) setattr(e, "username", utils.getCurrentUser()) setattr(e, "program", "fanctl") def deployApp(args): OpenAPI("openapi", None, "deployApp", args=(args.brand.upper(), args.module.upper(), args.master_rev, args.config_rev), env=e).call() def rollbackApp(args): OpenAPI("openapi", None, "rollbackApp", args=(args.brand.upper(), args.module.upper()),env=e).call() def stopServer(args):
def forward(self, logits, labels, weights_): ''' logits is a matrix M by C where m is the number of classifications and C are the number of classes labels is a integer tensor of size M where each element corresponds to the class that prediction i should be matching to ''' weights = torch.tensor(weights_).to('cuda') labels = labels.view(-1, 1) alpha = weights[labels].view(-1, 1) loss = alpha * (-logits.gather(-1, labels) + self.logsumexp(logits)) return loss.mean() if __name__ == '__main__': dataset = u.Namespace({'num_non_existing': torch.tensor(10)}) args = u.Namespace({'class_weights': [1.0, 1.0], 'task': 'no_link_pred'}) labels = torch.tensor([1, 0]) ce_ref = torch.nn.CrossEntropyLoss(reduction='sum') ce = Cross_Entropy(args, dataset) # print(ce.weights(labels)) # print(ce.weights(labels)) logits = torch.tensor([[1.0, -1.0], [1.0, -1.0]]) logits = torch.rand((5, 2)) labels = torch.randint(0, 2, (5, )) print(ce(logits, labels) - ce_ref(logits, labels)) exit() ce.logsumexp(logits) # print(labels) # print(ce.weights(labels)) # print(ce.weights(labels)[labels])