def __init__(self, input_dim=7, initial_dim=8, latent_dim=[32, 48, 64], mlp_hidden=32, num_class=2): print('Initializing DisNets') super(DisNets, self).__init__() self.latent_dim = latent_dim self.input_dim = input_dim # self.input_mlp = nn.Linear(self.input_dim, initial_dim) self.gcns = nn.ModuleList() self.layer_num = len(latent_dim) self.gcns.append(GCN(input_dim, self.latent_dim[0])) for i in range(1, len(latent_dim)): self.gcns.append(GCN(self.latent_dim[i - 1], self.latent_dim[i])) self.dense_dim = latent_dim[-1] self.Softmax = nn.Softmax(dim=0) self.h1_weights = nn.Linear(self.dense_dim, mlp_hidden) self.h2_weights = nn.Linear(mlp_hidden, num_class) self.mlp_non_linear = nn.ELU() weights_init(self)
def run_with_dataset(directory: Union[str, 'pathlib.Path'], dataset: str, hidden: List[int] = [91], dropout: float = 0.6449297033170698, learning_rate: float = 0.011888866964052763, weight_decay: float = 0.0005959130002875904, epochs: int = 200, verbose: bool = True) -> None: """Runs training with a given dataset Args: directory: Path to datasets dataset: dataset to run on hidden: Hidden Layer sizes dropout: Dropout Rate learning_rate: Learning Rate weight_decay: Weight decay epochs: Number of epochs to train for verbose: If True, prints messages during training time. \ Defaults to true """ gcn = GCN(*load_data(directory, dataset)) gcn.train(hidden=hidden, dropout=dropout, learning_rate=learning_rate, weight_decay=weight_decay, epochs=epochs, verbose=verbose) return gcn
def build_model(model_key, dataset, g, in_feats, n_classes): """ Returns a model instance based on --model command-line arg and dataset """ if model_key == 'MLP': return MLP(in_feats, 64, n_classes, 1, F.relu, 0.5) elif model_key == 'GCN': return GCN(g, in_feats, 16, n_classes, 1, F.relu, 0.5) elif model_key == 'GCN-64': return GCN(g, in_feats, 64, n_classes, 1, F.relu, 0.5) elif model_key == 'GAT': # Default args from paper num_heads = 8 num_out_heads = 8 if dataset == 'pubmed' else 1 num_layers = 1 # one *hidden* layer heads = ([num_heads] * num_layers) + [num_out_heads] return GAT( g, num_layers, in_feats, 8, # hidden units per layer n_classes, heads, F.elu, # activation fun 0.6, # feat dropout 0.6, # attn dropout 0.2, # negative slope for leakyrelu False # Use residual connections ) elif model_key == 'GraphSAGE': return GraphSAGE(g, in_feats, 16, n_classes, 1, F.relu, 0.5, "mean") # Add more models here raise ValueError("Invalid model key")
def run(args): gl.set_tape_capacity(1) g = load_graph(args) if args.use_mp: gl.set_tracker_mode(0) thg.set_client_num(args.client_num) thg.launch_server(g) else: g.init(task_index=args.rank, task_count=args.world_size) # TODO(baole): This is an estimate and an accurate value will be needed from graphlearn. length_per_worker = args.train_length // args.train_batch_size // args.world_size print('length_per_worker being set to: ' + str(length_per_worker)) # data loader train_query = query(g, args, mask=gl.Mask.TRAIN) if args.use_mp: train_dataset = thg.Dataset(train_query, window=5, induce_func=induce_func, graph=g) else: train_dataset = thg.Dataset(train_query, window=5, induce_func=induce_func) train_loader = thg.PyGDataLoader(train_dataset, multi_process=args.use_mp, length=length_per_worker) test_query = query(g, args, mask=gl.Mask.TEST) if args.use_mp: test_dataset = thg.Dataset(test_query, window=5, induce_func=induce_func, graph=g) else: test_dataset = thg.Dataset(test_query, window=5, induce_func=induce_func) test_loader = thg.PyGDataLoader(test_dataset, multi_process=args.use_mp) # define model model = GCN(input_dim=args.features_num, hidden_dim=args.hidden_dim, output_dim=args.class_num, depth=args.depth, drop_rate=args.drop_rate).to(device) if dist.is_initialized(): model = torch.nn.parallel.DistributedDataParallel(model) optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate) # train and test for epoch in range(0, args.epoch): train(model, train_loader, optimizer, args) test_acc = test(model, test_loader, args) log = 'Epoch: {:03d}, Test: {:.4f}' print(log.format(epoch, test_acc)) if not args.use_mp: g.close()
def test(adj): ''' test on GCN ''' adj = normalize_adj_tensor(adj) gcn = GCN(nfeat=features.shape[1], nhid=args.hidden, nclass=labels.max().item() + 1, dropout=0.5) if device != 'cpu': gcn = gcn.to(device) optimizer = optim.Adam(gcn.parameters(), lr=args.lr, weight_decay=5e-4) gcn.train() for epoch in range(args.epochs): optimizer.zero_grad() output = gcn(features, adj) loss_train = F.nll_loss(output[idx_train], labels[idx_train]) acc_train = accuracy(output[idx_train], labels[idx_train]) loss_train.backward() optimizer.step() gcn.eval() output = gcn(features, adj) loss_test = F.nll_loss(output[idx_test], labels[idx_test]) acc_test = accuracy(output[idx_test], labels[idx_test]) # print("Test set results:", # "loss= {:.4f}".format(loss_test.item()), # "accuracy= {:.4f}".format(acc_test.item())) return acc_test.item()
def predict(config, model_name='-1'): data, label = data_read(config.train_path) adjacency = get_adj(config.adj_path) x, y = normalize(data, label) x = x[:, :config.nodes] adj = adjacency[:config.nodes, :config.nodes] ph_adj = tf.placeholder(tf.float32, [config.nodes, config.nodes], 'adj') ph_data = tf.placeholder(tf.float32, [None, config.nodes, config.ts], 'data') ph_label = tf.placeholder(tf.float32, [None, 1], 'label') model = GCN(config.ts, 1, config.nodes, config.gl, config.dl) out = model(ph_data, ph_adj) sess = tf.Session() saver = tf.train.Saver() saver.restore(sess, './model/' + model_name + '/best_model.ckpt') batch_data, batch_label = get_batch(x, y, config.bs, config.ts, 'test') _data = batch_data.transpose([0, 2, 1]) out = sess.run(out, feed_dict={ ph_adj: adj, ph_data: _data, ph_label: batch_label[0] }) pre = un_normalize(out, label) print(analysis(pre, label[:-config.ts])) #axis = list(range(len(pre))) pdb.set_trace() plt.plot(pre) plt.plot(label[:-config.ts]) plt.show()
def init_model(self, model_path=''): # Model and optimizer if self.mode in ('sgc-clean', 'sgc'): self.model = SGC(nfeat=self.worker.n_features, nclass=self.worker.n_classes) elif self.mode in ('degree_mlp', 'basic_mlp'): self.model = MLP(nfeat=self.worker.n_features, nhid=self.args.hidden, nclass=self.worker.n_classes, dropout=self.args.dropout, size=self.worker.n_nodes, args=self.args) elif self.mode in ('degcn-clean'): self.model = DeGCN(nfeat=self.worker.n_features, nhid=self.args.hidden, nclass=self.worker.n_classes, dropout=self.args.dropout) elif self.mode in ('vanilla-clean', 'vanilla') or not self.args.fnormalize: if self.args.n_layer == 2: self.model = GCN(nfeat=self.worker.n_features, nhid=self.args.hidden, nclass=self.worker.n_classes, dropout=self.args.dropout) elif self.args.n_layer == 3: self.model = GCN3(nfeat=self.worker.n_features, nhid1=self.args.hidden1, nhid2=self.args.hidden2, nclass=self.worker.n_classes, dropout=self.args.dropout) else: raise NotImplementedError( f'n_layer = {self.args.n_layer} not implemented!') elif self.mode in ('clusteradj-clean', 'clusteradj'): self.model = ProjectionGCN(nfeat=self.worker.n_features, nhid=self.args.hidden, nclass=self.worker.n_classes, dropout=self.args.dropout, projection=self.worker.prj, size=self.worker.n_nodes, args=self.args) else: raise NotImplementedError( 'mode = {} no corrsponding model!'.format(self.mode)) if model_path: self.model.load_state_dict(torch.load(model_path)) print('load model from {} done!'.format(model_path)) self.model_path = model_path else: self.optimizer = optim.Adam(self.model.parameters(), lr=self.args.lr, weight_decay=self.args.weight_decay) if torch.cuda.is_available(): self.model.cuda()
def init_model(self, model_path=''): # Model and optimizer if self.mode in ('vanilla-clean', 'vanilla') or self.args.algorithm in ('HRG', 'GraphSample'): self.model = GCN(nfeat=self.worker.n_features, nhid=self.args.hidden, nclass=self.worker.n_classes, dropout=self.args.dropout) elif self.mode in ('clusteradj-clean', 'clusteradj'): self.model = ProjectionGCN(nfeat=self.worker.n_features, nhid=self.args.hidden, nclass=self.worker.n_classes, dropout=self.args.dropout, projection=self.worker.prj, size=self.worker.n_nodes) else: raise NotImplementedError( 'mode = {} no corrsponding model!'.format(self.mode)) if model_path: self.model.load_state_dict(torch.load(model_path)) print('load model from {} done!'.format(model_path)) else: self.optimizer = optim.Adam(self.model.parameters(), lr=self.args.lr, weight_decay=self.args.weight_decay) if torch.cuda.is_available(): self.model.cuda()
def main(): model = GCN(34, 32, 2) # model.load_state_dict(torch.load('model_40.pth')) if torch.cuda.is_available(): model = model.cuda() # model._initialize() print(my_cfg) optimizer = torch.optim.Adam(model.parameters(), lr=my_cfg['lr']) #optimizer = torch.optim.RMSprop(model.parameters(), lr=my_cfg['lr']) lr_scheduler = torch.optim.lr_scheduler.MultiStepLR( optimizer, milestones=my_cfg['milestones'], gamma=0.1) train(model, optimizer, lr_scheduler) return
def write_test(model_name): test_dataset = Smiles(data_choice='test', pos_weight=my_cfg['pos_weight'], device=device) test_loader = DataLoader(test_dataset, batch_size=my_cfg['batch_size'], shuffle=False) best_model = GCN(34, 32, 2) best_model.load_state_dict(torch.load(model_name)) if torch.cuda.is_available(): best_model = best_model.cuda() best_model.eval() print('\nStarting test ...') results = [] for i, (names, adj, features) in enumerate(test_loader): res = best_model(features, adj).detach().cpu() res = res.reshape(-1) for name, my_res in zip(names, res): results.append({'name': name, 'res': my_res}) exp_num = my_cfg['exp_num'] model_name = model_name.split('.')[-2][1:] with open(f'./data/test/output_{model_name}.txt', "w") as f: f.write('Chemical,Label\n') assert len(results) == 610 for i in range(len(results)): my_name = results[i]['name'] my_res = results[i]['res'] my_res = my_res.detach().cpu().numpy() f.write(f'{my_name},{my_res}\n') return
def __init__(self, n_in, n_h, activation): super(DGI, self).__init__() self.dense = nn.Linear(n_h, n_h) self.read = AvgReadout() self.attention = Attention(n_h) self.sigm = nn.Sigmoid() self.gcn = GCN(n_in, n_h, activation) self.disc = Discriminator(n_h) self.act = nn.PReLU()
def main(args): # load and preprocess dataset if args.gpu > 0: cuda = True device = torch.device('cuda:{}'.format(args.gpu)) else: device = torch.device('cpu') cuda = False cora_data = NeptuneCoraDataset(device, valid_ratio=0.1, test_ratio=0.2) #cora_data = CoraDataset(device, valid_ratio=0.1, test_ratio=0.2) features = cora_data.features test_set = cora_data.test_set g = cora_data.g in_feats = features['h**o'].shape[1] n_edges = g.number_of_edges() # normalization degs = g.in_degrees().float() norm = torch.pow(degs, -0.5) norm[torch.isinf(norm)] = 0 if cuda: norm = norm.cuda() g.ndata['norm'] = norm.unsqueeze(1) # create GCN model model = GCN(g, in_feats, args.n_hidden, cora_data.n_class, args.n_layers, F.relu) model.load_state_dict(torch.load(args.model_path)) if cuda: model.cuda() print() acc = evaluate(model, features['h**o'], test_set) print("Test accuracy {:.2%}".format(acc))
def __init__(self, nfeat, hidden_sizes, nclass, nnodes, dropout, train_iters, attack_features, lambda_, device, with_bias=False, lr=0.01, with_relu=False): super(BaseMeta, self).__init__() self.hidden_sizes = hidden_sizes self.nfeat = nfeat self.nclass = nclass self.with_bias = with_bias self.with_relu = with_relu self.gcn = GCN(nfeat=nfeat, nhid=hidden_sizes[0], nclass=nclass, dropout=0.5, with_relu=False) self.train_iters = train_iters self.surrogate_optimizer = optim.Adam(self.gcn.parameters(), lr=lr, weight_decay=5e-4) self.attack_features = attack_features self.lambda_ = lambda_ self.device = device self.nnodes = nnodes self.adj_changes = Parameter(torch.FloatTensor(nnodes, nnodes)) self.adj_changes.data.fill_(0)
def model_fn(): return GCN(graph, config['class_num'], config['features_num'], config['batch_size'], val_batch_size=config['val_batch_size'], test_batch_size=config['test_batch_size'], categorical_attrs_desc=config['categorical_attrs_desc'], hidden_dim=config['hidden_dim'], in_drop_rate=config['in_drop_rate'], hops_num=config['hops_num'], neighs_num=config['neighs_num'], full_graph_mode = config['full_graph_mode'])
def main(args): # load and preprocess dataset if args.gpu > 0: cuda = True device = torch.device('cuda:{}'.format(args.gpu)) else: device = torch.device('cpu') cuda = False cora_data = NeptuneCoraDataset(device, valid_ratio=0.0, test_ratio=0.1) #cora_data = CoraDataset(device, valid_ratio=0.1, test_ratio=0.2) features = cora_data.features # we infer type of nodes in test_set test_set = cora_data.test_set g = cora_data.g in_feats = features['h**o'].shape[1] n_edges = g.number_of_edges() # normalization degs = g.in_degrees().float() norm = torch.pow(degs, -0.5) norm[torch.isinf(norm)] = 0 if cuda: norm = norm.cuda() g.ndata['norm'] = norm.unsqueeze(1) # create GCN model model = GCN(g, in_feats, args.n_hidden, cora_data.n_class, args.n_layers, F.relu) model.load_state_dict(torch.load(args.model_path)) if cuda: model.cuda() model.eval() with torch.no_grad(): logits = model(features['h**o']) logits = logits[test_set[0]] _, indices = torch.max(logits, dim=1) nodes = test_set[0].numpy().tolist() indices = indices.numpy().tolist() for idx, label in enumerate(indices): node_id = nodes[idx] truth_nid = cora_data.translate_node(node_id) truth_label = cora_data.translate_label(label) print( "{{\"gremlin\":\"g.V(\\\"{}\\\").property(\\\"category\\\", \\\"{}\\\")\"}}" .format(truth_nid, truth_label))
def __init__(self, input_dim, node_type_num, initial_dim=8, latent_dim=[16, 24, 32], max_node = 12): print('Initializing Policy Nets') super(PolicyNN, self).__init__() self.latent_dim = latent_dim self.input_dim = input_dim self.node_type_num =node_type_num self.initial_dim = initial_dim # self.stop_mlp_hidden = 16 self.start_mlp_hidden = 16 self.tail_mlp_hidden = 24 self.input_mlp = nn.Linear(self.input_dim, initial_dim) self.gcns = nn.ModuleList() self.layer_num = len(latent_dim) self.gcns.append(GCN(self.initial_dim, self.latent_dim[0])) for i in range(1, len(latent_dim)): self.gcns.append(GCN(self.latent_dim[i-1], self.latent_dim[i])) self.dense_dim = latent_dim[-1] # self.stop_mlp1 = nn.Linear(self.dense_dim, self.stop_mlp_hidden) # self.stop_mlp_non_linear= nn.ReLU6() # self.stop_mlp2 = nn.Linear(self.stop_mlp_hidden, 2) self.start_mlp1= nn.Linear(self.dense_dim, self.start_mlp_hidden) self.start_mlp_non_linear = nn.ReLU6() self.start_mlp2= nn.Linear(self.start_mlp_hidden, 1) self.tail_mlp1= nn.Linear(2*self.dense_dim, self.tail_mlp_hidden) self.tail_mlp_non_linear = nn.ReLU6() self.tail_mlp2= nn.Linear(self.tail_mlp_hidden, 1) weights_init(self)
def train_gcn(): dataset = 'Cora' path = osp.join(osp.dirname(osp.realpath(__file__)), '.', 'data', dataset) dataset = Planetoid(path, dataset, T.NormalizeFeatures()) data = dataset[0] from gcn import GCN num_nodes = data.x.size(0) input_dim = data.x.size(1) hidden_dim = 16 num_classes = 7 model = GCN(input_dim, hidden_dim, num_classes) optimizer = torch.optim.Adam(model.parameters(), lr=0.02, weight_decay=0) for epoch in range(1, 2000): optimizer.zero_grad() output = model.forward_(data.x, data.edge_index) loss = model.loss(output, data.y) loss.backward() optimizer.step() acc = output.max(1)[1].eq(data.y).sum().item() / num_nodes print('epoch=%d loss=%f acc=%f' % (epoch, loss.item(), acc))
def __init__(self, vid_encoder, qns_encoder, ans_decoder, max_len_v, max_len_q, device): """ Reasoning with Heterogeneous Graph Alignment for Video Question Answering (AAAI20) """ super(HGA, self).__init__() self.vid_encoder = vid_encoder self.qns_encoder = qns_encoder self.ans_decoder = ans_decoder self.max_len_v = max_len_v self.max_len_q = max_len_q self.device = device hidden_size = vid_encoder.dim_hidden input_dropout_p = vid_encoder.input_dropout_p self.q_input_ln = nn.LayerNorm(hidden_size, elementwise_affine=False) self.v_input_ln = nn.LayerNorm(hidden_size, elementwise_affine=False) self.co_attn = CoAttention(hidden_size, n_layers=vid_encoder.n_layers, dropout_p=input_dropout_p) self.adj_learner = AdjLearner(hidden_size, hidden_size, dropout=input_dropout_p) self.gcn = GCN(hidden_size, hidden_size, hidden_size, num_layers=2, dropout=input_dropout_p) self.gcn_atten_pool = nn.Sequential( nn.Linear(hidden_size, hidden_size // 2), nn.Tanh(), nn.Linear(hidden_size // 2, 1), nn.Softmax( dim=-1)) #dim=-2 for attention-pooling otherwise sum-pooling self.global_fusion = fusions.Block([hidden_size, hidden_size], hidden_size, dropout_input=input_dropout_p) self.fusion = fusions.Block([hidden_size, hidden_size], hidden_size)
def train(config, model_name='-1'): data, label = data_read(config.train_path) adjacency = get_adj(config.adj_path) x, y = normalize(data, label) x = x[:, :config.nodes] x, y = trans_gcn(x, y, config.ts) adj = adjacency[:config.nodes, :config.nodes] ph_adj = tf.placeholder(tf.float32, [config.nodes, config.nodes], 'adj') ph_data = tf.placeholder(tf.float32, [None, config.nodes, config.ts], 'data') ph_label = tf.placeholder(tf.float32, [None, 1], 'label') model = GCN(config.ts, 1, config.nodes, config.gl, config.dl) out = model(ph_data, ph_adj) loss_op = compute_loss(out, ph_label) tf.summary.scalar('loss', loss_op) train_op = tf.train.AdamOptimizer(config.lr).minimize(loss_op) merge_op = tf.summary.merge_all() sess = tf.Session() sess.run(tf.global_variables_initializer()) writer = tf.summary.FileWriter('./logdir/' + model_name) writer.add_graph(sess.graph) saver = tf.train.Saver() _loss = [] for i in range(config.epoch): batch_data, batch_label = get_batch(x, y, config.bs, config.ts) for j in range(len(batch_label)): _data = batch_data[j].transpose([0, 2, 1]) _, loss, summary = sess.run([train_op, loss_op, merge_op], feed_dict={ ph_adj: adj, ph_data: _data, ph_label: batch_label[j] }) _loss.append(loss) writer.add_summary(summary, i) if i % 100 == 0: print('epoch====={}\t\t\tloss======{}\n\n'.format( i, np.mean(_loss))) _loss = [] saver.save(sess, './model/' + model_name + '/best_model.ckpt')
def __init__(self, in_channel): super(AdaMatting, self).__init__() # Encoder self.encoder_conv = nn.Sequential( nn.Conv2d(in_channel, 64, kernel_size=3, stride=1, padding=1, bias=True), nn.BatchNorm2d(64), nn.ReLU(inplace=True) ) encoder_inplanes = 64 self.encoder_maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) self.encoder_resblock1, encoder_inplanes = make_resblock(encoder_inplanes, 64, blocks=3, stride=2, block=Bottleneck) self.encoder_resblock2, encoder_inplanes = make_resblock(encoder_inplanes, 128, blocks=4, stride=2, block=Bottleneck) self.encoder_resblock3, encoder_inplanes = make_resblock(encoder_inplanes, 256, blocks=6, stride=2, block=Bottleneck) for m in self.modules(): if isinstance(m, nn.Conv2d): n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels m.weight.data.normal_(0, math.sqrt(2. / n)) elif isinstance(m, nn.BatchNorm2d): m.weight.data.fill_(1) m.bias.data.zero_() # Shortcuts self.shortcut_shallow = GCN(64, 64) self.shortcut_middle = GCN(64 * Bottleneck.expansion, 64 * Bottleneck.expansion) self.shortcut_deep = GCN(128 * Bottleneck.expansion, 128 * Bottleneck.expansion) # T-decoder self.t_decoder_upscale1 = nn.Sequential( nn.Conv2d(256 * Bottleneck.expansion, 512 * 4, kernel_size=3, stride=1, padding=1, bias=True), nn.BatchNorm2d(512 * 4), nn.ReLU(inplace=True), nn.PixelShuffle(2) ) self.t_decoder_upscale2 = nn.Sequential( nn.Conv2d(512, 256 * 4, kernel_size=3, stride=1, padding=1, bias=True), nn.BatchNorm2d(256 * 4), nn.ReLU(inplace=True), nn.PixelShuffle(2) ) self.t_decoder_upscale3 = nn.Sequential( nn.Conv2d(256, 64 * 4, kernel_size=3, stride=1, padding=1, bias=True), nn.BatchNorm2d(64 * 4), nn.ReLU(inplace=True), nn.PixelShuffle(2) ) self.t_decoder_upscale4 = nn.Sequential( nn.Conv2d(64, 3 * (2 ** 2), kernel_size=3, stride=1, padding=1, bias=True), nn.BatchNorm2d(3 * (2 ** 2)), nn.ReLU(inplace=True), nn.PixelShuffle(2) ) # A-deocder self.a_decoder_upscale1 = nn.Sequential( nn.Conv2d(256 * Bottleneck.expansion, 512 * 4, kernel_size=3, stride=1, padding=1, bias=True), nn.BatchNorm2d(512 * 4), nn.ReLU(inplace=True), nn.PixelShuffle(2) ) self.a_decoder_upscale2 = nn.Sequential( nn.Conv2d(512, 256 * 4, kernel_size=3, stride=1, padding=1, bias=True), nn.BatchNorm2d(256 * 4), nn.ReLU(inplace=True), nn.PixelShuffle(2) ) self.a_decoder_upscale3 = nn.Sequential( nn.Conv2d(256, 64 * 4, kernel_size=3, stride=1, padding=1, bias=True), nn.BatchNorm2d(64 * 4), nn.ReLU(inplace=True), nn.PixelShuffle(2) ) self.a_decoder_upscale4 = nn.Sequential( nn.Conv2d(64, 1 * (2 ** 2), kernel_size=3, stride=1, padding=1, bias=True), nn.BatchNorm2d(1 * (2 ** 2)), nn.ReLU(inplace=True), nn.PixelShuffle(2) ) # Propagation unit # self.propunit = PropUnit( # input_dim=4 + 1 + 1, # hidden_dim=[1], # kernel_size=(3, 3), # num_layers=3, # seq_len=3, # bias=True) self.prop_unit = nn.Sequential( nn.Conv2d(4 + 3 + 1, 64, kernel_size=3, stride=1, padding=1, bias=True), nn.BatchNorm2d(64), nn.ReLU(inplace=True), nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1, bias=True), nn.BatchNorm2d(64), nn.ReLU(inplace=True), nn.Conv2d(64, 1, kernel_size=1, stride=1, padding=0, bias=True), ) # Task uncertainty loss parameters self.log_sigma_t_sqr = nn.Parameter(torch.log(torch.Tensor([16.0]))) self.log_sigma_a_sqr = nn.Parameter(torch.log(torch.Tensor([16.0])))
'support': tf.sparse_placeholder(tf.float32), 'features': tf.sparse_placeholder(tf.float32, shape=tf.constant(features[2], dtype=tf.int64)), 'labels': tf.placeholder(tf.float32, shape=(None, y_train.shape[1])), 'labels_mask': tf.placeholder(tf.int32), 'dropout': tf.placeholder_with_default(0., shape=()), 'num_features_nonzero': tf.placeholder_with_default(tf.int32) } model = GCN(placeholders, input_dim=features[2][1], logging=True) sess = tf.Session() def evaluate(features, support, labels, mask, placeholders): t = time.time() feed_dict_val = construct_feed_dict(features, support, labels, mask, placeholders) outs_val = sess.run([model.loss, model.accuracy], feed_dict=feed_dict_val) return outs_val[0], outs_val[1], time.time() - t sess.run(tf.global_variables_initializer()) cost_val = [] for epoch in range(FLAGS.epochs):
def main(args): # load and preprocess dataset data = load_data(args) features = torch.FloatTensor(data.features) labels = torch.LongTensor(data.labels) train_mask = torch.ByteTensor(data.train_mask) val_mask = torch.ByteTensor(data.val_mask) test_mask = torch.ByteTensor(data.test_mask) in_feats = features.shape[1] n_classes = data.num_labels n_edges = data.graph.number_of_edges() n_nodes = data.graph.number_of_nodes() print("""----Data statistics------' #Nodes %d #Edges %d #Feature %d #Classes %d #Train samples %d #Val samples %d #Test samples %d""" % (n_nodes, n_edges, in_feats, n_classes, train_mask.sum().item(), val_mask.sum().item(), test_mask.sum().item())) if args.gpu < 0: cuda = False else: cuda = True torch.cuda.set_device(args.gpu) features = features.cuda() labels = labels.cuda() train_mask = train_mask.cuda() val_mask = val_mask.cuda() test_mask = test_mask.cuda() # graph preprocess and calculate normalization factor g = data.graph # add self loop if args.self_loop: g.remove_edges_from(g.selfloop_edges()) g.add_edges_from(zip(g.nodes(), g.nodes())) g = DGLGraph(g) n_edges = g.number_of_edges() # normalization degs = g.in_degrees().float() norm = torch.pow(degs, -0.5) norm[torch.isinf(norm)] = 0 if cuda: norm = norm.cuda() g.ndata['norm'] = norm.unsqueeze(1) # create GCN model model = GCN(g, in_feats, args.n_hidden, n_classes, args.n_layers, F.relu, args.dropout) if cuda: model.cuda() loss_fcn = torch.nn.CrossEntropyLoss() # use optimizer optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) # initialize graph dur = [] for epoch in range(args.n_epochs): model.train() if epoch >= 3: t0 = time.time() # forward logits = model(features) if epoch >= 3: dur.append(time.time() - t0) loss = loss_fcn(logits[train_mask], labels[train_mask]) optimizer.zero_grad() loss.backward() optimizer.step() acc = evaluate(model, features, labels, val_mask) print("Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | Accuracy {:.4f} | " "ETputs(KTEPS) {:.2f}". format(epoch, np.mean(dur), loss.item(), acc, n_edges / np.mean(dur) / 1000)) print() acc = evaluate(model, features, labels, test_mask) print("Test Accuracy {:.4f}".format(acc))
def main(args): # convert boolean type for args assert args.self_loop in ['True', 'False'], [ "Only True or False for self_loop, get ", args.self_loop ] assert args.use_layernorm in ['True', 'False'], [ "Only True or False for use_layernorm, get ", args.use_layernorm ] self_loop = (args.self_loop == 'True') use_layernorm = (args.use_layernorm == 'True') global t0 if args.dataset in {'cora', 'citeseer', 'pubmed'}: data = load_data(args) else: raise NotImplementedError(f'{args.dataset} is not a valid dataset') features = torch.FloatTensor(data.features) labels = torch.LongTensor(data.labels) train_mask = torch.ByteTensor(data.train_mask) val_mask = torch.ByteTensor(data.val_mask) test_mask = torch.ByteTensor(data.test_mask) in_feats = features.shape[1] n_classes = data.num_labels n_edges = data.graph.number_of_edges() print("""----Data statistics------' #Edges %d #Classes %d #Train samples %d #Val samples %d #Test samples %d""" % (n_edges, n_classes, train_mask.sum().item(), val_mask.sum().item(), test_mask.sum().item())) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') features = features.to(device) labels = labels.to(device) train_mask = train_mask.to(device) val_mask = val_mask.to(device) test_mask = test_mask.to(device) # graph preprocess and calculate normalization factor g = data.graph # add self loop if self_loop: g.remove_edges_from(nx.selfloop_edges(g)) g.add_edges_from(zip(g.nodes(), g.nodes())) g = DGLGraph(g) g = g.to(device) n_edges = g.number_of_edges() # normalization degs = g.in_degrees().float() norm = torch.pow(degs, -0.5) norm[torch.isinf(norm)] = 0 norm = norm.to(device) g.ndata['norm'] = norm.unsqueeze(1) # create GCN model model = GCN(g, in_feats, args.n_hidden, n_classes, args.n_layers, F.relu, args.dropout, use_layernorm) model = model.to(device) loss_fcn = torch.nn.CrossEntropyLoss() # use optimizer optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) # initialize graph record = [] dur = [] for epoch in range(args.n_epochs): if args.lr_scheduler: if epoch == int(0.5 * args.n_epochs): for pg in optimizer.param_groups: pg['lr'] = pg['lr'] / 10 elif epoch == int(0.75 * args.n_epochs): for pg in optimizer.param_groups: pg['lr'] = pg['lr'] / 10 model.train() if epoch >= 3: t0 = time.time() # forward optimizer.zero_grad() logits = model(features) loss = loss_fcn(logits[train_mask], labels[train_mask]) loss.backward() optimizer.step() if epoch >= 3: dur.append(time.time() - t0) acc_val = evaluate(model, features, labels, val_mask) acc_test = evaluate(model, features, labels, test_mask) record.append([acc_val, acc_test]) all_test_acc = [v[1] for v in record] all_val_acc = [v[0] for v in record] acc = evaluate(model, features, labels, test_mask) print(f"Final Test Accuracy: {acc:.4f}") print(f"Best Val Accuracy: {max(all_val_acc):.4f}") print(f"Best Test Accuracy: {max(all_test_acc):.4f}")
def __init__(self, config, modelname, conv_data, arcs, pretrained_weight=None, ncf_weights=None): super(NCFGModel, self).__init__() self.user_num = config.user_num self.conv_num = config.conv_num self.vocab_num = config.vocab_num self.mf_factor_dim = config.factor_dim # 'replying factors' in paper self.text_factor_dim = config.text_factor_dim # 'conversation interaction' factors in paper self.kernal_num = config.kernal_num # kernal number for CNN encoder self.embed_dim = config.embedding_dim self.hidden_dim = config.hidden_dim self.modelname = modelname # NCFGCN or NCFGRN self.mlp_layers_num = config.mlp_layers_num self.conv_data = conv_data self.arc_in = arcs[0] self.arc_out = arcs[1] if self.modelname == "NCFGCN": self.gcn_layers_num = config.gcn_layers_num self.use_gates = config.use_gates self.use_lstm = config.use_lstm elif self.modelname == "NCFGRN": self.grn_states_num = config.grn_states_num else: print "Modelname wrong!" exit() # matrix factorization factors if self.mf_factor_dim: self.mf_user_embedding = nn.Embedding(self.user_num, self.mf_factor_dim) self.mf_conv_embedding = nn.Embedding(self.conv_num, self.mf_factor_dim) if ncf_weights is not None: self.mf_user_embedding.load_state_dict({'weight': ncf_weights[0]}) self.mf_conv_embedding.load_state_dict({'weight': ncf_weights[1]}) else: nn.init.xavier_normal_(self.mf_user_embedding.weight) nn.init.xavier_normal_(self.mf_conv_embedding.weight) # user text factors self.user_embedding = nn.Embedding(self.user_num, self.text_factor_dim) if ncf_weights is not None: self.user_embedding.load_state_dict({'weight': ncf_weights[2]}) else: nn.init.xavier_normal_(self.user_embedding.weight) # word embedding layer self.word_embedding = nn.Embedding(self.vocab_num, self.embed_dim, padding_idx=0) if pretrained_weight is not None: self.word_embedding.load_state_dict({'weight': pretrained_weight}) # turn modeling layer self.turn_modeling = CNNEncoder(self.embed_dim, self.kernal_num, self.kernal_kind, config.dropout) turn_hidden_dim = self.kernal_num * 3 # conv modeling layer if self.modelname == "NCFGCN": if self.use_lstm: self.lstm_modeling = nn.LSTM(turn_hidden_dim + self.text_factor_dim, self.hidden_dim // 2, bidirectional=True) self.conv_modeling = GCN(self.hidden_dim, self.hidden_dim, self.hidden_dim, self.gcn_layers_num, self.use_gates, config.dropout) else: self.conv_modeling = GCN(turn_hidden_dim + self.text_factor_dim, self.hidden_dim, self.hidden_dim, self.gcn_layers_num, self.use_gates, config.dropout) else: self.conv_modeling = GRN(turn_hidden_dim + self.text_factor_dim, self.hidden_dim, self.grn_states_num) self.h2f = nn.Linear(self.hidden_dim, self.text_factor_dim) # mlp layers and out layer if self.mlp_layers_num: self.mlp_layers = [self.text_factor_dim] for idx in xrange(self.mlp_layers_num - 1): self.mlp_layers.append(self.mlp_layers[idx] / 2) self.mlp_layers.append(self.text_factor_dim * 2) self.mlps = nn.ModuleList([nn.Linear(self.mlp_layers[idx - 1], self.mlp_layers[idx]) for idx in xrange(self.mlp_layers_num)]) self.out_layer = nn.Linear(self.mf_factor_dim + self.mlp_layers[self.mlp_layers_num - 1], 1) else: self.out_layer = nn.Linear(self.mf_factor_dim + self.text_factor_dim * 2, 1) # activation functions self.relu = nn.ReLU() self.tanh = nn.Tanh()
A, features, y_train, y_val, y_test, train_mask, val_mask, test_mask = load_data_v1( 'cora') A = preprocess_adj(A) features /= features.sum(axis=1, ).reshape(-1, 1) if FEATURE_LESS: X = np.arange(A.shape[-1]) feature_dim = A.shape[-1] else: X = features feature_dim = X.shape[-1] model_input = [X, A] # Compile model model = GCN(A.shape[-1], feature_dim, 16, y_train.shape[1], dropout_rate=0.5, l2_reg=2.5e-4, feature_less=FEATURE_LESS, ) model.compile(optimizer=Adam(0.01), loss='categorical_crossentropy', weighted_metrics=['categorical_crossentropy', 'acc']) NB_EPOCH = 200 PATIENCE = 200 # early stopping patience val_data = (model_input, y_val, val_mask) mc_callback = ModelCheckpoint('./best_model.h5', monitor='val_weighted_categorical_crossentropy', save_best_only=True, save_weights_only=True) # train print("start training") model.fit(model_input, y_train, sample_weight=train_mask, validation_data=val_data,
parser.add_argument('--dropout', type=float, default=0.5, help='Dropout rate (1 - keep probability).') args = parser.parse_args() # set device device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # load dataset dataset = Planetoid(root='/tmp/' + args.dataset, name=args.dataset) data = dataset[0].to(device) # generate model and optimizer with parameter if args.model == 'GCN': model = GCN(dataset.num_features, args.hidden, dataset.num_classes).to(device) else: model = GraphSAGE(dataset.num_features, args.hidden, dataset.num_classes).to(device) optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) # define two list for plot Accuracy_list = [] Loss_list = [] # train the model model.train() for epoch in range(args.epochs):
class BaseMeta(Module): def __init__(self, nfeat, hidden_sizes, nclass, nnodes, dropout, train_iters, attack_features, lambda_, device, with_bias=False, lr=0.01, with_relu=False): super(BaseMeta, self).__init__() self.hidden_sizes = hidden_sizes self.nfeat = nfeat self.nclass = nclass self.with_bias = with_bias self.with_relu = with_relu self.gcn = GCN(nfeat=nfeat, nhid=hidden_sizes[0], nclass=nclass, dropout=0.5, with_relu=False) self.train_iters = train_iters self.surrogate_optimizer = optim.Adam(self.gcn.parameters(), lr=lr, weight_decay=5e-4) self.attack_features = attack_features self.lambda_ = lambda_ self.device = device self.nnodes = nnodes self.adj_changes = Parameter(torch.FloatTensor(nnodes, nnodes)) self.adj_changes.data.fill_(0) def filter_potential_singletons(self, modified_adj): """ Computes a mask for entries potentially leading to singleton nodes, i.e. one of the two nodes corresponding to the entry have degree 1 and there is an edge between the two nodes. Returns ------- torch.Tensor shape [N, N], float with ones everywhere except the entries of potential singleton nodes, where the returned tensor has value 0. """ degrees = modified_adj.sum(0) degree_one = (degrees == 1) resh = degree_one.repeat(modified_adj.shape[0], 1).float() l_and = resh * modified_adj logical_and_symmetric = l_and + l_and.t() flat_mask = 1 - logical_and_symmetric return flat_mask def train_surrogate(self, features, adj, labels, idx_train, train_iters=200): print( '=== training surrogate model to predict unlabled data for self-training' ) surrogate = self.gcn surrogate.initialize() adj_norm = utils.normalize_adj_tensor(adj) surrogate.train() for i in range(train_iters): self.surrogate_optimizer.zero_grad() output = surrogate(features, adj_norm) loss_train = F.nll_loss(output[idx_train], labels[idx_train]) loss_train.backward() self.surrogate_optimizer.step() # Predict the labels of the unlabeled nodes to use them for self-training. surrogate.eval() output = surrogate(features, adj_norm) labels_self_training = output.argmax(1) labels_self_training[idx_train] = labels[idx_train] # reset parameters for later updating surrogate.initialize() return labels_self_training def log_likelihood_constraint(self, modified_adj, ori_adj, ll_cutoff): """ Computes a mask for entries that, if the edge corresponding to the entry is added/removed, would lead to the log likelihood constraint to be violated. """ t_d_min = torch.tensor(2.0).to(self.device) t_possible_edges = np.array( np.triu(np.ones((self.nnodes, self.nnodes)), k=1).nonzero()).T allowed_mask, current_ratio = utils.likelihood_ratio_filter( t_possible_edges, modified_adj, ori_adj, t_d_min, ll_cutoff) return allowed_mask, current_ratio
# compute normalizing constant all_train_features = [] for _, _, _, features, _, _ in gcn_train_data: all_train_features.append(features) all_train_features = torch.cat(all_train_features) train_mean = torch.mean(all_train_features, dim=0) train_std = torch.std(all_train_features, dim=0) # train_mean[-10:] = 0 # train_std[-10:] = 1 # training GCN # model initiailization device = 'cuda' lr = 0.0005 model = GCN(raw_feature_size=full_feature_size) model = model.to(device=device) optimizer = torch.optim.Adam(model.parameters(), lr=lr) # loss functions mse_loss = nn.MSELoss() mae_loss = nn.L1Loss() loss_fn = mse_loss f_train_result = open('results/gcn/train.csv', 'a') f_test_result = open('results/gcn/test.csv', 'a') # training for epoch in range(2001): train_r2_list, train_mae_list, train_mse_list = [], [], [] test_r2_list, test_mae_list, test_mse_list = [], [], []
def run(rank, world_size, args): print('Running DDP on rank', rank, 'world size', world_size) setup(rank, world_size, args) dev_id = ragdoll.device_id() if len(args.input_graph) > 0 or len(args.cached_dir) > 0: data = SynDataset(rank == 0, args) else: data = Dataset(rank == 0, args) feat_size = args.feat_size features = torch.FloatTensor(data.features).cuda() labels = torch.LongTensor(data.labels).cuda() train_mask = torch.BoolTensor(data.train_mask).cuda() val_mask = torch.BoolTensor(data.val_mask).cuda() test_mask = torch.BoolTensor(data.test_mask).cuda() n_classes = args.n_classes n_nodes = data.n_nodes local_n_nodes = data.local_n_nodes model = GCN(data.graph, n_nodes, local_n_nodes, True, feat_size, args.n_hidden, n_classes, args.n_layers, F.relu, args.dropout, comm_net=args.comm_net) model.cuda() model = DDP(model, device_ids=[dev_id]) loss_fcn = torch.nn.CrossEntropyLoss() # use optimizer optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) optimizer.zero_grad() dur = [] print("Start training... for {} epochs".format(args.n_epochs)) for epoch in range(args.n_epochs): print('Epoch {} -------------'.format(epoch)) model.train() torch.distributed.barrier() if epoch >= 3: t0 = time.time() logits = model(features) loss = loss_fcn(logits[train_mask], labels[train_mask]) optimizer.zero_grad() loss.backward() torch.cuda.synchronize() t1 = time.time() optimizer.step() torch.cuda.synchronize() t2 = time.time() if epoch >= 3: dur.append(time.time() - t0) # acc, _, _ = evaluate(model, features, labels, val_mask) # print('acc is {}, loss is {}, this epoch using time {}, avg time {}.'.format( # acc, loss.item(), dur[-1] if epoch >= 3 else 0, np.mean(dur))) print('Using time to synchronize model', t2 - t1) print('Peak memory is {} GB'.format( torch.cuda.max_memory_allocated(dev_id) / 1e9)) print('this epoch uses time {} s, avg time {} s.'.format( dur[-1] if epoch >= 3 else 0, np.mean(dur))) ##acc, corr, total = evaluate(model, features, labels, test_mask) ##print('my corr is', corr, 'my total is', total) ##corr = torch.Tensor([corr]).cuda(dev_id) ##total = torch.Tensor([total]).cuda(dev_id) ##corrs, totals = [], [] ##for i in range(world_size): ## corrs.append(torch.Tensor([0]).cuda(dev_id)) ## totals.append(torch.Tensor([0]).cuda(dev_id)) ##torch.distributed.all_gather(corrs, corr) ##torch.distributed.all_gather(totals, total) ##print('corrs is', corrs) ##print('totals is', totals) ##corr = torch.stack(corrs, dim=0).sum(dim=0).item() * 1.0 ##total = torch.stack(totals, dim=0).sum(dim=0).item() * 1.0 ##print('Test acc is', corr / total) cleanup()
def main(args): # load and preprocess dataset # data = load_data(args) g, graph_labels = load_graphs( '/yushi/dataset/Amazon2M/Amazon2M_dglgraph.bin') assert len(g) == 1 g = g[0] data = g.ndata features = torch.FloatTensor(data['feat']) labels = torch.LongTensor(data['label']) if hasattr(torch, 'BoolTensor'): train_mask = data['train_mask'].bool() val_mask = data['val_mask'].bool() test_mask = data['test_mask'].bool() # else: # train_mask = torch.ByteTensor(data.train_mask) # val_mask = torch.ByteTensor(data.val_mask) # test_mask = torch.ByteTensor(data.test_mask) in_feats = features.shape[1] n_classes = 47 n_edges = g.number_of_edges() print("""----Data statistics------' #Edges %d #Classes %d #Train samples %d #Val samples %d #Test samples %d""" % (n_edges, n_classes, train_mask.int().sum().item(), val_mask.int().sum().item(), test_mask.int().sum().item())) if args.gpu < 0: cuda = False else: cuda = True torch.cuda.set_device(args.gpu) features = features.cuda() labels = labels.cuda() train_mask = train_mask.cuda() val_mask = val_mask.cuda() test_mask = test_mask.cuda() # graph preprocess and calculate normalization factor # g = data.graph # add self loop # if args.self_loop: # g.remove_edges_from(nx.selfloop_edges(g)) # g.add_edges_from(zip(g.nodes(), g.nodes())) # g = DGLGraph(g) n_edges = g.number_of_edges() # normalization degs = g.in_degrees().float() norm = torch.pow(degs, -0.5) norm[torch.isinf(norm)] = 0 if cuda: norm = norm.cuda() g.ndata['norm'] = norm.unsqueeze(1) # create GCN model model = GCN(g, in_feats, args.n_hidden, n_classes, args.n_layers, F.relu, args.dropout) if cuda: model.cuda() print(model) loss_fcn = torch.nn.CrossEntropyLoss() # use optimizer optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) # initialize graph dur = [] start = time.time() for epoch in range(args.n_epochs): model.train() if epoch >= 3: t0 = time.time() # forward logits = model(features) loss = loss_fcn(logits[train_mask], labels[train_mask]) optimizer.zero_grad() loss.backward() optimizer.step() if epoch >= 3: dur.append(time.time() - t0) acc = evaluate(model, features, labels, val_mask) print( "Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | Accuracy {:.4f} | " "ETputs(KTEPS) {:.2f}".format(epoch, np.mean(dur), loss.item(), acc, n_edges / np.mean(dur) / 1000)) print() acc = evaluate(model, features, labels, val_mask) # no test_mask print("Test accuracy {:.2%}".format(acc)) print( f'Training Time Consuming: {np.sum(dur)}, all time cost: {time.time() - start}' )