def __init__(self,
input_dim=7,
initial_dim=8,
latent_dim=[32, 48, 64],
mlp_hidden=32,
num_class=2):
print('Initializing DisNets')
super(DisNets, self).__init__()
self.latent_dim = latent_dim
self.input_dim = input_dim
# self.input_mlp = nn.Linear(self.input_dim, initial_dim)
self.gcns = nn.ModuleList()
self.layer_num = len(latent_dim)
self.gcns.append(GCN(input_dim, self.latent_dim[0]))
for i in range(1, len(latent_dim)):
self.gcns.append(GCN(self.latent_dim[i - 1], self.latent_dim[i]))
self.dense_dim = latent_dim[-1]
self.Softmax = nn.Softmax(dim=0)
self.h1_weights = nn.Linear(self.dense_dim, mlp_hidden)
self.h2_weights = nn.Linear(mlp_hidden, num_class)
self.mlp_non_linear = nn.ELU()
weights_init(self)
def run_with_dataset(directory: Union[str, 'pathlib.Path'],
dataset: str,
hidden: List[int] = [91],
dropout: float = 0.6449297033170698,
learning_rate: float = 0.011888866964052763,
weight_decay: float = 0.0005959130002875904,
epochs: int = 200,
verbose: bool = True) -> None:
"""Runs training with a given dataset
Args:
directory: Path to datasets
dataset: dataset to run on
hidden: Hidden Layer sizes
dropout: Dropout Rate
learning_rate: Learning Rate
weight_decay: Weight decay
epochs: Number of epochs to train for
verbose: If True, prints messages during training time. \
Defaults to true
"""
gcn = GCN(*load_data(directory, dataset))
gcn.train(hidden=hidden,
dropout=dropout,
learning_rate=learning_rate,
weight_decay=weight_decay,
epochs=epochs,
verbose=verbose)
return gcn
def build_model(model_key, dataset, g, in_feats, n_classes):
"""
Returns a model instance based on --model command-line arg and dataset
"""
if model_key == 'MLP':
return MLP(in_feats, 64, n_classes, 1, F.relu, 0.5)
elif model_key == 'GCN':
return GCN(g, in_feats, 16, n_classes, 1, F.relu, 0.5)
elif model_key == 'GCN-64':
return GCN(g, in_feats, 64, n_classes, 1, F.relu, 0.5)
elif model_key == 'GAT':
# Default args from paper
num_heads = 8
num_out_heads = 8 if dataset == 'pubmed' else 1
num_layers = 1 # one *hidden* layer
heads = ([num_heads] * num_layers) + [num_out_heads]
return GAT(
g,
num_layers,
in_feats,
8, # hidden units per layer
n_classes,
heads,
F.elu, # activation fun
0.6, # feat dropout
0.6, # attn dropout
0.2, # negative slope for leakyrelu
False # Use residual connections
)
elif model_key == 'GraphSAGE':
return GraphSAGE(g, in_feats, 16, n_classes, 1, F.relu, 0.5, "mean")
# Add more models here
raise ValueError("Invalid model key")
def run(args):
gl.set_tape_capacity(1)
g = load_graph(args)
if args.use_mp:
gl.set_tracker_mode(0)
thg.set_client_num(args.client_num)
thg.launch_server(g)
else:
g.init(task_index=args.rank, task_count=args.world_size)
# TODO(baole): This is an estimate and an accurate value will be needed from graphlearn.
length_per_worker = args.train_length // args.train_batch_size // args.world_size
print('length_per_worker being set to: ' + str(length_per_worker))
# data loader
train_query = query(g, args, mask=gl.Mask.TRAIN)
if args.use_mp:
train_dataset = thg.Dataset(train_query,
window=5,
induce_func=induce_func,
graph=g)
else:
train_dataset = thg.Dataset(train_query,
window=5,
induce_func=induce_func)
train_loader = thg.PyGDataLoader(train_dataset,
multi_process=args.use_mp,
length=length_per_worker)
test_query = query(g, args, mask=gl.Mask.TEST)
if args.use_mp:
test_dataset = thg.Dataset(test_query,
window=5,
induce_func=induce_func,
graph=g)
else:
test_dataset = thg.Dataset(test_query,
window=5,
induce_func=induce_func)
test_loader = thg.PyGDataLoader(test_dataset, multi_process=args.use_mp)
# define model
model = GCN(input_dim=args.features_num,
hidden_dim=args.hidden_dim,
output_dim=args.class_num,
depth=args.depth,
drop_rate=args.drop_rate).to(device)
if dist.is_initialized():
model = torch.nn.parallel.DistributedDataParallel(model)
optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate)
# train and test
for epoch in range(0, args.epoch):
train(model, train_loader, optimizer, args)
test_acc = test(model, test_loader, args)
log = 'Epoch: {:03d}, Test: {:.4f}'
print(log.format(epoch, test_acc))
if not args.use_mp:
g.close()
def test(adj):
''' test on GCN '''
adj = normalize_adj_tensor(adj)
gcn = GCN(nfeat=features.shape[1],
nhid=args.hidden,
nclass=labels.max().item() + 1,
dropout=0.5)
if device != 'cpu':
gcn = gcn.to(device)
optimizer = optim.Adam(gcn.parameters(), lr=args.lr, weight_decay=5e-4)
gcn.train()
for epoch in range(args.epochs):
optimizer.zero_grad()
output = gcn(features, adj)
loss_train = F.nll_loss(output[idx_train], labels[idx_train])
acc_train = accuracy(output[idx_train], labels[idx_train])
loss_train.backward()
optimizer.step()
gcn.eval()
output = gcn(features, adj)
loss_test = F.nll_loss(output[idx_test], labels[idx_test])
acc_test = accuracy(output[idx_test], labels[idx_test])
# print("Test set results:",
# "loss= {:.4f}".format(loss_test.item()),
# "accuracy= {:.4f}".format(acc_test.item()))
return acc_test.item()
def predict(config, model_name='-1'):
data, label = data_read(config.train_path)
adjacency = get_adj(config.adj_path)
x, y = normalize(data, label)
x = x[:, :config.nodes]
adj = adjacency[:config.nodes, :config.nodes]
ph_adj = tf.placeholder(tf.float32, [config.nodes, config.nodes], 'adj')
ph_data = tf.placeholder(tf.float32, [None, config.nodes, config.ts],
'data')
ph_label = tf.placeholder(tf.float32, [None, 1], 'label')
model = GCN(config.ts, 1, config.nodes, config.gl, config.dl)
out = model(ph_data, ph_adj)
sess = tf.Session()
saver = tf.train.Saver()
saver.restore(sess, './model/' + model_name + '/best_model.ckpt')
batch_data, batch_label = get_batch(x, y, config.bs, config.ts, 'test')
_data = batch_data.transpose([0, 2, 1])
out = sess.run(out,
feed_dict={
ph_adj: adj,
ph_data: _data,
ph_label: batch_label[0]
})
pre = un_normalize(out, label)
print(analysis(pre, label[:-config.ts]))
#axis = list(range(len(pre)))
pdb.set_trace()
plt.plot(pre)
plt.plot(label[:-config.ts])
plt.show()
def init_model(self, model_path=''):
# Model and optimizer
if self.mode in ('sgc-clean', 'sgc'):
self.model = SGC(nfeat=self.worker.n_features,
nclass=self.worker.n_classes)
elif self.mode in ('degree_mlp', 'basic_mlp'):
self.model = MLP(nfeat=self.worker.n_features,
nhid=self.args.hidden,
nclass=self.worker.n_classes,
dropout=self.args.dropout,
size=self.worker.n_nodes,
args=self.args)
elif self.mode in ('degcn-clean'):
self.model = DeGCN(nfeat=self.worker.n_features,
nhid=self.args.hidden,
nclass=self.worker.n_classes,
dropout=self.args.dropout)
elif self.mode in ('vanilla-clean',
'vanilla') or not self.args.fnormalize:
if self.args.n_layer == 2:
self.model = GCN(nfeat=self.worker.n_features,
nhid=self.args.hidden,
nclass=self.worker.n_classes,
dropout=self.args.dropout)
elif self.args.n_layer == 3:
self.model = GCN3(nfeat=self.worker.n_features,
nhid1=self.args.hidden1,
nhid2=self.args.hidden2,
nclass=self.worker.n_classes,
dropout=self.args.dropout)
else:
raise NotImplementedError(
f'n_layer = {self.args.n_layer} not implemented!')
elif self.mode in ('clusteradj-clean', 'clusteradj'):
self.model = ProjectionGCN(nfeat=self.worker.n_features,
nhid=self.args.hidden,
nclass=self.worker.n_classes,
dropout=self.args.dropout,
projection=self.worker.prj,
size=self.worker.n_nodes,
args=self.args)
else:
raise NotImplementedError(
'mode = {} no corrsponding model!'.format(self.mode))
if model_path:
self.model.load_state_dict(torch.load(model_path))
print('load model from {} done!'.format(model_path))
self.model_path = model_path
else:
self.optimizer = optim.Adam(self.model.parameters(),
lr=self.args.lr,
weight_decay=self.args.weight_decay)
if torch.cuda.is_available():
self.model.cuda()
def init_model(self, model_path=''):
# Model and optimizer
if self.mode in ('vanilla-clean',
'vanilla') or self.args.algorithm in ('HRG',
'GraphSample'):
self.model = GCN(nfeat=self.worker.n_features,
nhid=self.args.hidden,
nclass=self.worker.n_classes,
dropout=self.args.dropout)
elif self.mode in ('clusteradj-clean', 'clusteradj'):
self.model = ProjectionGCN(nfeat=self.worker.n_features,
nhid=self.args.hidden,
nclass=self.worker.n_classes,
dropout=self.args.dropout,
projection=self.worker.prj,
size=self.worker.n_nodes)
else:
raise NotImplementedError(
'mode = {} no corrsponding model!'.format(self.mode))
if model_path:
self.model.load_state_dict(torch.load(model_path))
print('load model from {} done!'.format(model_path))
else:
self.optimizer = optim.Adam(self.model.parameters(),
lr=self.args.lr,
weight_decay=self.args.weight_decay)
if torch.cuda.is_available():
self.model.cuda()
def main():
model = GCN(34, 32, 2)
# model.load_state_dict(torch.load('model_40.pth'))
if torch.cuda.is_available():
model = model.cuda()
# model._initialize()
print(my_cfg)
optimizer = torch.optim.Adam(model.parameters(), lr=my_cfg['lr'])
#optimizer = torch.optim.RMSprop(model.parameters(), lr=my_cfg['lr'])
lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(
optimizer, milestones=my_cfg['milestones'], gamma=0.1)
train(model, optimizer, lr_scheduler)
return
def write_test(model_name):
test_dataset = Smiles(data_choice='test', pos_weight=my_cfg['pos_weight'], device=device)
test_loader = DataLoader(test_dataset, batch_size=my_cfg['batch_size'], shuffle=False)
best_model = GCN(34, 32, 2)
best_model.load_state_dict(torch.load(model_name))
if torch.cuda.is_available():
best_model = best_model.cuda()
best_model.eval()
print('\nStarting test ...')
results = []
for i, (names, adj, features) in enumerate(test_loader):
res = best_model(features, adj).detach().cpu()
res = res.reshape(-1)
for name, my_res in zip(names, res):
results.append({'name': name, 'res': my_res})
exp_num = my_cfg['exp_num']
model_name = model_name.split('.')[-2][1:]
with open(f'./data/test/output_{model_name}.txt', "w") as f:
f.write('Chemical,Label\n')
assert len(results) == 610
for i in range(len(results)):
my_name = results[i]['name']
my_res = results[i]['res']
my_res = my_res.detach().cpu().numpy()
f.write(f'{my_name},{my_res}\n')
return
def __init__(self, n_in, n_h, activation):
super(DGI, self).__init__()
self.dense = nn.Linear(n_h, n_h)
self.read = AvgReadout()
self.attention = Attention(n_h)
self.sigm = nn.Sigmoid()
self.gcn = GCN(n_in, n_h, activation)
self.disc = Discriminator(n_h)
self.act = nn.PReLU()
def main(args):
# load and preprocess dataset
if args.gpu > 0:
cuda = True
device = torch.device('cuda:{}'.format(args.gpu))
else:
device = torch.device('cpu')
cuda = False
cora_data = NeptuneCoraDataset(device, valid_ratio=0.1, test_ratio=0.2)
#cora_data = CoraDataset(device, valid_ratio=0.1, test_ratio=0.2)
features = cora_data.features
test_set = cora_data.test_set
g = cora_data.g
in_feats = features['h**o'].shape[1]
n_edges = g.number_of_edges()
# normalization
degs = g.in_degrees().float()
norm = torch.pow(degs, -0.5)
norm[torch.isinf(norm)] = 0
if cuda:
norm = norm.cuda()
g.ndata['norm'] = norm.unsqueeze(1)
# create GCN model
model = GCN(g, in_feats, args.n_hidden, cora_data.n_class, args.n_layers,
F.relu)
model.load_state_dict(torch.load(args.model_path))
if cuda:
model.cuda()
print()
acc = evaluate(model, features['h**o'], test_set)
print("Test accuracy {:.2%}".format(acc))
def __init__(self,
nfeat,
hidden_sizes,
nclass,
nnodes,
dropout,
train_iters,
attack_features,
lambda_,
device,
with_bias=False,
lr=0.01,
with_relu=False):
super(BaseMeta, self).__init__()
self.hidden_sizes = hidden_sizes
self.nfeat = nfeat
self.nclass = nclass
self.with_bias = with_bias
self.with_relu = with_relu
self.gcn = GCN(nfeat=nfeat,
nhid=hidden_sizes[0],
nclass=nclass,
dropout=0.5,
with_relu=False)
self.train_iters = train_iters
self.surrogate_optimizer = optim.Adam(self.gcn.parameters(),
lr=lr,
weight_decay=5e-4)
self.attack_features = attack_features
self.lambda_ = lambda_
self.device = device
self.nnodes = nnodes
self.adj_changes = Parameter(torch.FloatTensor(nnodes, nnodes))
self.adj_changes.data.fill_(0)
def model_fn():
return GCN(graph,
config['class_num'],
config['features_num'],
config['batch_size'],
val_batch_size=config['val_batch_size'],
test_batch_size=config['test_batch_size'],
categorical_attrs_desc=config['categorical_attrs_desc'],
hidden_dim=config['hidden_dim'],
in_drop_rate=config['in_drop_rate'],
hops_num=config['hops_num'],
neighs_num=config['neighs_num'],
full_graph_mode = config['full_graph_mode'])
def main(args):
# load and preprocess dataset
if args.gpu > 0:
cuda = True
device = torch.device('cuda:{}'.format(args.gpu))
else:
device = torch.device('cpu')
cuda = False
cora_data = NeptuneCoraDataset(device, valid_ratio=0.0, test_ratio=0.1)
#cora_data = CoraDataset(device, valid_ratio=0.1, test_ratio=0.2)
features = cora_data.features
# we infer type of nodes in test_set
test_set = cora_data.test_set
g = cora_data.g
in_feats = features['h**o'].shape[1]
n_edges = g.number_of_edges()
# normalization
degs = g.in_degrees().float()
norm = torch.pow(degs, -0.5)
norm[torch.isinf(norm)] = 0
if cuda:
norm = norm.cuda()
g.ndata['norm'] = norm.unsqueeze(1)
# create GCN model
model = GCN(g, in_feats, args.n_hidden, cora_data.n_class, args.n_layers,
F.relu)
model.load_state_dict(torch.load(args.model_path))
if cuda:
model.cuda()
model.eval()
with torch.no_grad():
logits = model(features['h**o'])
logits = logits[test_set[0]]
_, indices = torch.max(logits, dim=1)
nodes = test_set[0].numpy().tolist()
indices = indices.numpy().tolist()
for idx, label in enumerate(indices):
node_id = nodes[idx]
truth_nid = cora_data.translate_node(node_id)
truth_label = cora_data.translate_label(label)
print(
"{{\"gremlin\":\"g.V(\\\"{}\\\").property(\\\"category\\\", \\\"{}\\\")\"}}"
.format(truth_nid, truth_label))
def __init__(self, input_dim, node_type_num, initial_dim=8, latent_dim=[16, 24, 32], max_node = 12):
print('Initializing Policy Nets')
super(PolicyNN, self).__init__()
self.latent_dim = latent_dim
self.input_dim = input_dim
self.node_type_num =node_type_num
self.initial_dim = initial_dim
# self.stop_mlp_hidden = 16
self.start_mlp_hidden = 16
self.tail_mlp_hidden = 24
self.input_mlp = nn.Linear(self.input_dim, initial_dim)
self.gcns = nn.ModuleList()
self.layer_num = len(latent_dim)
self.gcns.append(GCN(self.initial_dim, self.latent_dim[0]))
for i in range(1, len(latent_dim)):
self.gcns.append(GCN(self.latent_dim[i-1], self.latent_dim[i]))
self.dense_dim = latent_dim[-1]
# self.stop_mlp1 = nn.Linear(self.dense_dim, self.stop_mlp_hidden)
# self.stop_mlp_non_linear= nn.ReLU6()
# self.stop_mlp2 = nn.Linear(self.stop_mlp_hidden, 2)
self.start_mlp1= nn.Linear(self.dense_dim, self.start_mlp_hidden)
self.start_mlp_non_linear = nn.ReLU6()
self.start_mlp2= nn.Linear(self.start_mlp_hidden, 1)
self.tail_mlp1= nn.Linear(2*self.dense_dim, self.tail_mlp_hidden)
self.tail_mlp_non_linear = nn.ReLU6()
self.tail_mlp2= nn.Linear(self.tail_mlp_hidden, 1)
weights_init(self)
def train_gcn():
dataset = 'Cora'
path = osp.join(osp.dirname(osp.realpath(__file__)), '.', 'data', dataset)
dataset = Planetoid(path, dataset, T.NormalizeFeatures())
data = dataset[0]
from gcn import GCN
num_nodes = data.x.size(0)
input_dim = data.x.size(1)
hidden_dim = 16
num_classes = 7
model = GCN(input_dim, hidden_dim, num_classes)
optimizer = torch.optim.Adam(model.parameters(), lr=0.02, weight_decay=0)
for epoch in range(1, 2000):
optimizer.zero_grad()
output = model.forward_(data.x, data.edge_index)
loss = model.loss(output, data.y)
loss.backward()
optimizer.step()
acc = output.max(1)[1].eq(data.y).sum().item() / num_nodes
print('epoch=%d loss=%f acc=%f' % (epoch, loss.item(), acc))
def __init__(self, vid_encoder, qns_encoder, ans_decoder, max_len_v,
max_len_q, device):
"""
Reasoning with Heterogeneous Graph Alignment for Video Question Answering (AAAI20)
"""
super(HGA, self).__init__()
self.vid_encoder = vid_encoder
self.qns_encoder = qns_encoder
self.ans_decoder = ans_decoder
self.max_len_v = max_len_v
self.max_len_q = max_len_q
self.device = device
hidden_size = vid_encoder.dim_hidden
input_dropout_p = vid_encoder.input_dropout_p
self.q_input_ln = nn.LayerNorm(hidden_size, elementwise_affine=False)
self.v_input_ln = nn.LayerNorm(hidden_size, elementwise_affine=False)
self.co_attn = CoAttention(hidden_size,
n_layers=vid_encoder.n_layers,
dropout_p=input_dropout_p)
self.adj_learner = AdjLearner(hidden_size,
hidden_size,
dropout=input_dropout_p)
self.gcn = GCN(hidden_size,
hidden_size,
hidden_size,
num_layers=2,
dropout=input_dropout_p)
self.gcn_atten_pool = nn.Sequential(
nn.Linear(hidden_size, hidden_size // 2), nn.Tanh(),
nn.Linear(hidden_size // 2, 1), nn.Softmax(
dim=-1)) #dim=-2 for attention-pooling otherwise sum-pooling
self.global_fusion = fusions.Block([hidden_size, hidden_size],
hidden_size,
dropout_input=input_dropout_p)
self.fusion = fusions.Block([hidden_size, hidden_size], hidden_size)
def train(config, model_name='-1'):
data, label = data_read(config.train_path)
adjacency = get_adj(config.adj_path)
x, y = normalize(data, label)
x = x[:, :config.nodes]
x, y = trans_gcn(x, y, config.ts)
adj = adjacency[:config.nodes, :config.nodes]
ph_adj = tf.placeholder(tf.float32, [config.nodes, config.nodes], 'adj')
ph_data = tf.placeholder(tf.float32, [None, config.nodes, config.ts],
'data')
ph_label = tf.placeholder(tf.float32, [None, 1], 'label')
model = GCN(config.ts, 1, config.nodes, config.gl, config.dl)
out = model(ph_data, ph_adj)
loss_op = compute_loss(out, ph_label)
tf.summary.scalar('loss', loss_op)
train_op = tf.train.AdamOptimizer(config.lr).minimize(loss_op)
merge_op = tf.summary.merge_all()
sess = tf.Session()
sess.run(tf.global_variables_initializer())
writer = tf.summary.FileWriter('./logdir/' + model_name)
writer.add_graph(sess.graph)
saver = tf.train.Saver()
_loss = []
for i in range(config.epoch):
batch_data, batch_label = get_batch(x, y, config.bs, config.ts)
for j in range(len(batch_label)):
_data = batch_data[j].transpose([0, 2, 1])
_, loss, summary = sess.run([train_op, loss_op, merge_op],
feed_dict={
ph_adj: adj,
ph_data: _data,
ph_label: batch_label[j]
})
_loss.append(loss)
writer.add_summary(summary, i)
if i % 100 == 0:
print('epoch====={}\t\t\tloss======{}\n\n'.format(
i, np.mean(_loss)))
_loss = []
saver.save(sess, './model/' + model_name + '/best_model.ckpt')
def __init__(self, in_channel):
super(AdaMatting, self).__init__()
# Encoder
self.encoder_conv = nn.Sequential(
nn.Conv2d(in_channel, 64, kernel_size=3, stride=1, padding=1, bias=True),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True)
)
encoder_inplanes = 64
self.encoder_maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.encoder_resblock1, encoder_inplanes = make_resblock(encoder_inplanes, 64, blocks=3, stride=2, block=Bottleneck)
self.encoder_resblock2, encoder_inplanes = make_resblock(encoder_inplanes, 128, blocks=4, stride=2, block=Bottleneck)
self.encoder_resblock3, encoder_inplanes = make_resblock(encoder_inplanes, 256, blocks=6, stride=2, block=Bottleneck)
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
# Shortcuts
self.shortcut_shallow = GCN(64, 64)
self.shortcut_middle = GCN(64 * Bottleneck.expansion, 64 * Bottleneck.expansion)
self.shortcut_deep = GCN(128 * Bottleneck.expansion, 128 * Bottleneck.expansion)
# T-decoder
self.t_decoder_upscale1 = nn.Sequential(
nn.Conv2d(256 * Bottleneck.expansion, 512 * 4, kernel_size=3, stride=1, padding=1, bias=True),
nn.BatchNorm2d(512 * 4),
nn.ReLU(inplace=True),
nn.PixelShuffle(2)
)
self.t_decoder_upscale2 = nn.Sequential(
nn.Conv2d(512, 256 * 4, kernel_size=3, stride=1, padding=1, bias=True),
nn.BatchNorm2d(256 * 4),
nn.ReLU(inplace=True),
nn.PixelShuffle(2)
)
self.t_decoder_upscale3 = nn.Sequential(
nn.Conv2d(256, 64 * 4, kernel_size=3, stride=1, padding=1, bias=True),
nn.BatchNorm2d(64 * 4),
nn.ReLU(inplace=True),
nn.PixelShuffle(2)
)
self.t_decoder_upscale4 = nn.Sequential(
nn.Conv2d(64, 3 * (2 ** 2), kernel_size=3, stride=1, padding=1, bias=True),
nn.BatchNorm2d(3 * (2 ** 2)),
nn.ReLU(inplace=True),
nn.PixelShuffle(2)
)
# A-deocder
self.a_decoder_upscale1 = nn.Sequential(
nn.Conv2d(256 * Bottleneck.expansion, 512 * 4, kernel_size=3, stride=1, padding=1, bias=True),
nn.BatchNorm2d(512 * 4),
nn.ReLU(inplace=True),
nn.PixelShuffle(2)
)
self.a_decoder_upscale2 = nn.Sequential(
nn.Conv2d(512, 256 * 4, kernel_size=3, stride=1, padding=1, bias=True),
nn.BatchNorm2d(256 * 4),
nn.ReLU(inplace=True),
nn.PixelShuffle(2)
)
self.a_decoder_upscale3 = nn.Sequential(
nn.Conv2d(256, 64 * 4, kernel_size=3, stride=1, padding=1, bias=True),
nn.BatchNorm2d(64 * 4),
nn.ReLU(inplace=True),
nn.PixelShuffle(2)
)
self.a_decoder_upscale4 = nn.Sequential(
nn.Conv2d(64, 1 * (2 ** 2), kernel_size=3, stride=1, padding=1, bias=True),
nn.BatchNorm2d(1 * (2 ** 2)),
nn.ReLU(inplace=True),
nn.PixelShuffle(2)
)
# Propagation unit
# self.propunit = PropUnit(
# input_dim=4 + 1 + 1,
# hidden_dim=[1],
# kernel_size=(3, 3),
# num_layers=3,
# seq_len=3,
# bias=True)
self.prop_unit = nn.Sequential(
nn.Conv2d(4 + 3 + 1, 64, kernel_size=3, stride=1, padding=1, bias=True),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True),
nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1, bias=True),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True),
nn.Conv2d(64, 1, kernel_size=1, stride=1, padding=0, bias=True),
)
# Task uncertainty loss parameters
self.log_sigma_t_sqr = nn.Parameter(torch.log(torch.Tensor([16.0])))
self.log_sigma_a_sqr = nn.Parameter(torch.log(torch.Tensor([16.0])))
'support':
tf.sparse_placeholder(tf.float32),
'features':
tf.sparse_placeholder(tf.float32,
shape=tf.constant(features[2], dtype=tf.int64)),
'labels':
tf.placeholder(tf.float32, shape=(None, y_train.shape[1])),
'labels_mask':
tf.placeholder(tf.int32),
'dropout':
tf.placeholder_with_default(0., shape=()),
'num_features_nonzero':
tf.placeholder_with_default(tf.int32)
}
model = GCN(placeholders, input_dim=features[2][1], logging=True)
sess = tf.Session()
def evaluate(features, support, labels, mask, placeholders):
t = time.time()
feed_dict_val = construct_feed_dict(features, support, labels, mask,
placeholders)
outs_val = sess.run([model.loss, model.accuracy], feed_dict=feed_dict_val)
return outs_val[0], outs_val[1], time.time() - t
sess.run(tf.global_variables_initializer())
cost_val = []
for epoch in range(FLAGS.epochs):
def main(args):
# load and preprocess dataset
data = load_data(args)
features = torch.FloatTensor(data.features)
labels = torch.LongTensor(data.labels)
train_mask = torch.ByteTensor(data.train_mask)
val_mask = torch.ByteTensor(data.val_mask)
test_mask = torch.ByteTensor(data.test_mask)
in_feats = features.shape[1]
n_classes = data.num_labels
n_edges = data.graph.number_of_edges()
n_nodes = data.graph.number_of_nodes()
print("""----Data statistics------'
#Nodes %d
#Edges %d
#Feature %d
#Classes %d
#Train samples %d
#Val samples %d
#Test samples %d""" %
(n_nodes, n_edges, in_feats, n_classes,
train_mask.sum().item(),
val_mask.sum().item(),
test_mask.sum().item()))
if args.gpu < 0:
cuda = False
else:
cuda = True
torch.cuda.set_device(args.gpu)
features = features.cuda()
labels = labels.cuda()
train_mask = train_mask.cuda()
val_mask = val_mask.cuda()
test_mask = test_mask.cuda()
# graph preprocess and calculate normalization factor
g = data.graph
# add self loop
if args.self_loop:
g.remove_edges_from(g.selfloop_edges())
g.add_edges_from(zip(g.nodes(), g.nodes()))
g = DGLGraph(g)
n_edges = g.number_of_edges()
# normalization
degs = g.in_degrees().float()
norm = torch.pow(degs, -0.5)
norm[torch.isinf(norm)] = 0
if cuda:
norm = norm.cuda()
g.ndata['norm'] = norm.unsqueeze(1)
# create GCN model
model = GCN(g,
in_feats,
args.n_hidden,
n_classes,
args.n_layers,
F.relu,
args.dropout)
if cuda:
model.cuda()
loss_fcn = torch.nn.CrossEntropyLoss()
# use optimizer
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
# initialize graph
dur = []
for epoch in range(args.n_epochs):
model.train()
if epoch >= 3:
t0 = time.time()
# forward
logits = model(features)
if epoch >= 3:
dur.append(time.time() - t0)
loss = loss_fcn(logits[train_mask], labels[train_mask])
optimizer.zero_grad()
loss.backward()
optimizer.step()
acc = evaluate(model, features, labels, val_mask)
print("Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | Accuracy {:.4f} | "
"ETputs(KTEPS) {:.2f}". format(epoch, np.mean(dur), loss.item(),
acc, n_edges / np.mean(dur) / 1000))
print()
acc = evaluate(model, features, labels, test_mask)
print("Test Accuracy {:.4f}".format(acc))
def main(args):
# convert boolean type for args
assert args.self_loop in ['True', 'False'], [
"Only True or False for self_loop, get ", args.self_loop
]
assert args.use_layernorm in ['True', 'False'], [
"Only True or False for use_layernorm, get ", args.use_layernorm
]
self_loop = (args.self_loop == 'True')
use_layernorm = (args.use_layernorm == 'True')
global t0
if args.dataset in {'cora', 'citeseer', 'pubmed'}:
data = load_data(args)
else:
raise NotImplementedError(f'{args.dataset} is not a valid dataset')
features = torch.FloatTensor(data.features)
labels = torch.LongTensor(data.labels)
train_mask = torch.ByteTensor(data.train_mask)
val_mask = torch.ByteTensor(data.val_mask)
test_mask = torch.ByteTensor(data.test_mask)
in_feats = features.shape[1]
n_classes = data.num_labels
n_edges = data.graph.number_of_edges()
print("""----Data statistics------'
#Edges %d
#Classes %d
#Train samples %d
#Val samples %d
#Test samples %d""" % (n_edges, n_classes, train_mask.sum().item(),
val_mask.sum().item(), test_mask.sum().item()))
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
features = features.to(device)
labels = labels.to(device)
train_mask = train_mask.to(device)
val_mask = val_mask.to(device)
test_mask = test_mask.to(device)
# graph preprocess and calculate normalization factor
g = data.graph
# add self loop
if self_loop:
g.remove_edges_from(nx.selfloop_edges(g))
g.add_edges_from(zip(g.nodes(), g.nodes()))
g = DGLGraph(g)
g = g.to(device)
n_edges = g.number_of_edges()
# normalization
degs = g.in_degrees().float()
norm = torch.pow(degs, -0.5)
norm[torch.isinf(norm)] = 0
norm = norm.to(device)
g.ndata['norm'] = norm.unsqueeze(1)
# create GCN model
model = GCN(g, in_feats, args.n_hidden, n_classes, args.n_layers, F.relu,
args.dropout, use_layernorm)
model = model.to(device)
loss_fcn = torch.nn.CrossEntropyLoss()
# use optimizer
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
# initialize graph
record = []
dur = []
for epoch in range(args.n_epochs):
if args.lr_scheduler:
if epoch == int(0.5 * args.n_epochs):
for pg in optimizer.param_groups:
pg['lr'] = pg['lr'] / 10
elif epoch == int(0.75 * args.n_epochs):
for pg in optimizer.param_groups:
pg['lr'] = pg['lr'] / 10
model.train()
if epoch >= 3:
t0 = time.time()
# forward
optimizer.zero_grad()
logits = model(features)
loss = loss_fcn(logits[train_mask], labels[train_mask])
loss.backward()
optimizer.step()
if epoch >= 3:
dur.append(time.time() - t0)
acc_val = evaluate(model, features, labels, val_mask)
acc_test = evaluate(model, features, labels, test_mask)
record.append([acc_val, acc_test])
all_test_acc = [v[1] for v in record]
all_val_acc = [v[0] for v in record]
acc = evaluate(model, features, labels, test_mask)
print(f"Final Test Accuracy: {acc:.4f}")
print(f"Best Val Accuracy: {max(all_val_acc):.4f}")
print(f"Best Test Accuracy: {max(all_test_acc):.4f}")
def __init__(self, config, modelname, conv_data, arcs, pretrained_weight=None, ncf_weights=None):
super(NCFGModel, self).__init__()
self.user_num = config.user_num
self.conv_num = config.conv_num
self.vocab_num = config.vocab_num
self.mf_factor_dim = config.factor_dim # 'replying factors' in paper
self.text_factor_dim = config.text_factor_dim # 'conversation interaction' factors in paper
self.kernal_num = config.kernal_num # kernal number for CNN encoder
self.embed_dim = config.embedding_dim
self.hidden_dim = config.hidden_dim
self.modelname = modelname # NCFGCN or NCFGRN
self.mlp_layers_num = config.mlp_layers_num
self.conv_data = conv_data
self.arc_in = arcs[0]
self.arc_out = arcs[1]
if self.modelname == "NCFGCN":
self.gcn_layers_num = config.gcn_layers_num
self.use_gates = config.use_gates
self.use_lstm = config.use_lstm
elif self.modelname == "NCFGRN":
self.grn_states_num = config.grn_states_num
else:
print "Modelname wrong!"
exit()
# matrix factorization factors
if self.mf_factor_dim:
self.mf_user_embedding = nn.Embedding(self.user_num, self.mf_factor_dim)
self.mf_conv_embedding = nn.Embedding(self.conv_num, self.mf_factor_dim)
if ncf_weights is not None:
self.mf_user_embedding.load_state_dict({'weight': ncf_weights[0]})
self.mf_conv_embedding.load_state_dict({'weight': ncf_weights[1]})
else:
nn.init.xavier_normal_(self.mf_user_embedding.weight)
nn.init.xavier_normal_(self.mf_conv_embedding.weight)
# user text factors
self.user_embedding = nn.Embedding(self.user_num, self.text_factor_dim)
if ncf_weights is not None:
self.user_embedding.load_state_dict({'weight': ncf_weights[2]})
else:
nn.init.xavier_normal_(self.user_embedding.weight)
# word embedding layer
self.word_embedding = nn.Embedding(self.vocab_num, self.embed_dim, padding_idx=0)
if pretrained_weight is not None:
self.word_embedding.load_state_dict({'weight': pretrained_weight})
# turn modeling layer
self.turn_modeling = CNNEncoder(self.embed_dim, self.kernal_num, self.kernal_kind, config.dropout)
turn_hidden_dim = self.kernal_num * 3
# conv modeling layer
if self.modelname == "NCFGCN":
if self.use_lstm:
self.lstm_modeling = nn.LSTM(turn_hidden_dim + self.text_factor_dim, self.hidden_dim // 2,
bidirectional=True)
self.conv_modeling = GCN(self.hidden_dim, self.hidden_dim, self.hidden_dim, self.gcn_layers_num,
self.use_gates, config.dropout)
else:
self.conv_modeling = GCN(turn_hidden_dim + self.text_factor_dim, self.hidden_dim, self.hidden_dim,
self.gcn_layers_num, self.use_gates, config.dropout)
else:
self.conv_modeling = GRN(turn_hidden_dim + self.text_factor_dim, self.hidden_dim, self.grn_states_num)
self.h2f = nn.Linear(self.hidden_dim, self.text_factor_dim)
# mlp layers and out layer
if self.mlp_layers_num:
self.mlp_layers = [self.text_factor_dim]
for idx in xrange(self.mlp_layers_num - 1):
self.mlp_layers.append(self.mlp_layers[idx] / 2)
self.mlp_layers.append(self.text_factor_dim * 2)
self.mlps = nn.ModuleList([nn.Linear(self.mlp_layers[idx - 1], self.mlp_layers[idx]) for idx in xrange(self.mlp_layers_num)])
self.out_layer = nn.Linear(self.mf_factor_dim + self.mlp_layers[self.mlp_layers_num - 1], 1)
else:
self.out_layer = nn.Linear(self.mf_factor_dim + self.text_factor_dim * 2, 1)
# activation functions
self.relu = nn.ReLU()
self.tanh = nn.Tanh()
A, features, y_train, y_val, y_test, train_mask, val_mask, test_mask = load_data_v1(
'cora')
A = preprocess_adj(A)
features /= features.sum(axis=1, ).reshape(-1, 1)
if FEATURE_LESS:
X = np.arange(A.shape[-1])
feature_dim = A.shape[-1]
else:
X = features
feature_dim = X.shape[-1]
model_input = [X, A]
# Compile model
model = GCN(A.shape[-1], feature_dim, 16, y_train.shape[1], dropout_rate=0.5, l2_reg=2.5e-4,
feature_less=FEATURE_LESS, )
model.compile(optimizer=Adam(0.01), loss='categorical_crossentropy',
weighted_metrics=['categorical_crossentropy', 'acc'])
NB_EPOCH = 200
PATIENCE = 200 # early stopping patience
val_data = (model_input, y_val, val_mask)
mc_callback = ModelCheckpoint('./best_model.h5',
monitor='val_weighted_categorical_crossentropy',
save_best_only=True,
save_weights_only=True)
# train
print("start training")
model.fit(model_input, y_train, sample_weight=train_mask, validation_data=val_data,
parser.add_argument('--dropout',
type=float,
default=0.5,
help='Dropout rate (1 - keep probability).')
args = parser.parse_args()
# set device
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# load dataset
dataset = Planetoid(root='/tmp/' + args.dataset, name=args.dataset)
data = dataset[0].to(device)
# generate model and optimizer with parameter
if args.model == 'GCN':
model = GCN(dataset.num_features, args.hidden,
dataset.num_classes).to(device)
else:
model = GraphSAGE(dataset.num_features, args.hidden,
dataset.num_classes).to(device)
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
# define two list for plot
Accuracy_list = []
Loss_list = []
# train the model
model.train()
for epoch in range(args.epochs):
class BaseMeta(Module):
def __init__(self,
nfeat,
hidden_sizes,
nclass,
nnodes,
dropout,
train_iters,
attack_features,
lambda_,
device,
with_bias=False,
lr=0.01,
with_relu=False):
super(BaseMeta, self).__init__()
self.hidden_sizes = hidden_sizes
self.nfeat = nfeat
self.nclass = nclass
self.with_bias = with_bias
self.with_relu = with_relu
self.gcn = GCN(nfeat=nfeat,
nhid=hidden_sizes[0],
nclass=nclass,
dropout=0.5,
with_relu=False)
self.train_iters = train_iters
self.surrogate_optimizer = optim.Adam(self.gcn.parameters(),
lr=lr,
weight_decay=5e-4)
self.attack_features = attack_features
self.lambda_ = lambda_
self.device = device
self.nnodes = nnodes
self.adj_changes = Parameter(torch.FloatTensor(nnodes, nnodes))
self.adj_changes.data.fill_(0)
def filter_potential_singletons(self, modified_adj):
"""
Computes a mask for entries potentially leading to singleton nodes, i.e. one of the two nodes corresponding to
the entry have degree 1 and there is an edge between the two nodes.
Returns
-------
torch.Tensor shape [N, N], float with ones everywhere except the entries of potential singleton nodes,
where the returned tensor has value 0.
"""
degrees = modified_adj.sum(0)
degree_one = (degrees == 1)
resh = degree_one.repeat(modified_adj.shape[0], 1).float()
l_and = resh * modified_adj
logical_and_symmetric = l_and + l_and.t()
flat_mask = 1 - logical_and_symmetric
return flat_mask
def train_surrogate(self,
features,
adj,
labels,
idx_train,
train_iters=200):
print(
'=== training surrogate model to predict unlabled data for self-training'
)
surrogate = self.gcn
surrogate.initialize()
adj_norm = utils.normalize_adj_tensor(adj)
surrogate.train()
for i in range(train_iters):
self.surrogate_optimizer.zero_grad()
output = surrogate(features, adj_norm)
loss_train = F.nll_loss(output[idx_train], labels[idx_train])
loss_train.backward()
self.surrogate_optimizer.step()
# Predict the labels of the unlabeled nodes to use them for self-training.
surrogate.eval()
output = surrogate(features, adj_norm)
labels_self_training = output.argmax(1)
labels_self_training[idx_train] = labels[idx_train]
# reset parameters for later updating
surrogate.initialize()
return labels_self_training
def log_likelihood_constraint(self, modified_adj, ori_adj, ll_cutoff):
"""
Computes a mask for entries that, if the edge corresponding to the entry is added/removed, would lead to the
log likelihood constraint to be violated.
"""
t_d_min = torch.tensor(2.0).to(self.device)
t_possible_edges = np.array(
np.triu(np.ones((self.nnodes, self.nnodes)), k=1).nonzero()).T
allowed_mask, current_ratio = utils.likelihood_ratio_filter(
t_possible_edges, modified_adj, ori_adj, t_d_min, ll_cutoff)
return allowed_mask, current_ratio
# compute normalizing constant
all_train_features = []
for _, _, _, features, _, _ in gcn_train_data:
all_train_features.append(features)
all_train_features = torch.cat(all_train_features)
train_mean = torch.mean(all_train_features, dim=0)
train_std = torch.std(all_train_features, dim=0)
# train_mean[-10:] = 0
# train_std[-10:] = 1
# training GCN
# model initiailization
device = 'cuda'
lr = 0.0005
model = GCN(raw_feature_size=full_feature_size)
model = model.to(device=device)
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
# loss functions
mse_loss = nn.MSELoss()
mae_loss = nn.L1Loss()
loss_fn = mse_loss
f_train_result = open('results/gcn/train.csv', 'a')
f_test_result = open('results/gcn/test.csv', 'a')
# training
for epoch in range(2001):
train_r2_list, train_mae_list, train_mse_list = [], [], []
test_r2_list, test_mae_list, test_mse_list = [], [], []
def run(rank, world_size, args):
print('Running DDP on rank', rank, 'world size', world_size)
setup(rank, world_size, args)
dev_id = ragdoll.device_id()
if len(args.input_graph) > 0 or len(args.cached_dir) > 0:
data = SynDataset(rank == 0, args)
else:
data = Dataset(rank == 0, args)
feat_size = args.feat_size
features = torch.FloatTensor(data.features).cuda()
labels = torch.LongTensor(data.labels).cuda()
train_mask = torch.BoolTensor(data.train_mask).cuda()
val_mask = torch.BoolTensor(data.val_mask).cuda()
test_mask = torch.BoolTensor(data.test_mask).cuda()
n_classes = args.n_classes
n_nodes = data.n_nodes
local_n_nodes = data.local_n_nodes
model = GCN(data.graph, n_nodes, local_n_nodes, True, feat_size, args.n_hidden, n_classes,
args.n_layers, F.relu, args.dropout, comm_net=args.comm_net)
model.cuda()
model = DDP(model, device_ids=[dev_id])
loss_fcn = torch.nn.CrossEntropyLoss()
# use optimizer
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
optimizer.zero_grad()
dur = []
print("Start training... for {} epochs".format(args.n_epochs))
for epoch in range(args.n_epochs):
print('Epoch {} -------------'.format(epoch))
model.train()
torch.distributed.barrier()
if epoch >= 3:
t0 = time.time()
logits = model(features)
loss = loss_fcn(logits[train_mask], labels[train_mask])
optimizer.zero_grad()
loss.backward()
torch.cuda.synchronize()
t1 = time.time()
optimizer.step()
torch.cuda.synchronize()
t2 = time.time()
if epoch >= 3:
dur.append(time.time() - t0)
# acc, _, _ = evaluate(model, features, labels, val_mask)
# print('acc is {}, loss is {}, this epoch using time {}, avg time {}.'.format(
# acc, loss.item(), dur[-1] if epoch >= 3 else 0, np.mean(dur)))
print('Using time to synchronize model', t2 - t1)
print('Peak memory is {} GB'.format(
torch.cuda.max_memory_allocated(dev_id) / 1e9))
print('this epoch uses time {} s, avg time {} s.'.format(
dur[-1] if epoch >= 3 else 0, np.mean(dur)))
##acc, corr, total = evaluate(model, features, labels, test_mask)
##print('my corr is', corr, 'my total is', total)
##corr = torch.Tensor([corr]).cuda(dev_id)
##total = torch.Tensor([total]).cuda(dev_id)
##corrs, totals = [], []
##for i in range(world_size):
## corrs.append(torch.Tensor([0]).cuda(dev_id))
## totals.append(torch.Tensor([0]).cuda(dev_id))
##torch.distributed.all_gather(corrs, corr)
##torch.distributed.all_gather(totals, total)
##print('corrs is', corrs)
##print('totals is', totals)
##corr = torch.stack(corrs, dim=0).sum(dim=0).item() * 1.0
##total = torch.stack(totals, dim=0).sum(dim=0).item() * 1.0
##print('Test acc is', corr / total)
cleanup()
def main(args):
# load and preprocess dataset
# data = load_data(args)
g, graph_labels = load_graphs(
'/yushi/dataset/Amazon2M/Amazon2M_dglgraph.bin')
assert len(g) == 1
g = g[0]
data = g.ndata
features = torch.FloatTensor(data['feat'])
labels = torch.LongTensor(data['label'])
if hasattr(torch, 'BoolTensor'):
train_mask = data['train_mask'].bool()
val_mask = data['val_mask'].bool()
test_mask = data['test_mask'].bool()
# else:
# train_mask = torch.ByteTensor(data.train_mask)
# val_mask = torch.ByteTensor(data.val_mask)
# test_mask = torch.ByteTensor(data.test_mask)
in_feats = features.shape[1]
n_classes = 47
n_edges = g.number_of_edges()
print("""----Data statistics------'
#Edges %d
#Classes %d
#Train samples %d
#Val samples %d
#Test samples %d""" %
(n_edges, n_classes, train_mask.int().sum().item(),
val_mask.int().sum().item(), test_mask.int().sum().item()))
if args.gpu < 0:
cuda = False
else:
cuda = True
torch.cuda.set_device(args.gpu)
features = features.cuda()
labels = labels.cuda()
train_mask = train_mask.cuda()
val_mask = val_mask.cuda()
test_mask = test_mask.cuda()
# graph preprocess and calculate normalization factor
# g = data.graph
# add self loop
# if args.self_loop:
# g.remove_edges_from(nx.selfloop_edges(g))
# g.add_edges_from(zip(g.nodes(), g.nodes()))
# g = DGLGraph(g)
n_edges = g.number_of_edges()
# normalization
degs = g.in_degrees().float()
norm = torch.pow(degs, -0.5)
norm[torch.isinf(norm)] = 0
if cuda:
norm = norm.cuda()
g.ndata['norm'] = norm.unsqueeze(1)
# create GCN model
model = GCN(g, in_feats, args.n_hidden, n_classes, args.n_layers, F.relu,
args.dropout)
if cuda:
model.cuda()
print(model)
loss_fcn = torch.nn.CrossEntropyLoss()
# use optimizer
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
# initialize graph
dur = []
start = time.time()
for epoch in range(args.n_epochs):
model.train()
if epoch >= 3:
t0 = time.time()
# forward
logits = model(features)
loss = loss_fcn(logits[train_mask], labels[train_mask])
optimizer.zero_grad()
loss.backward()
optimizer.step()
if epoch >= 3:
dur.append(time.time() - t0)
acc = evaluate(model, features, labels, val_mask)
print(
"Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | Accuracy {:.4f} | "
"ETputs(KTEPS) {:.2f}".format(epoch, np.mean(dur), loss.item(),
acc, n_edges / np.mean(dur) / 1000))
print()
acc = evaluate(model, features, labels, val_mask) # no test_mask
print("Test accuracy {:.2%}".format(acc))
print(
f'Training Time Consuming: {np.sum(dur)}, all time cost: {time.time() - start}'
)
