def main(argv):
# config the CPU/GPU in TF, assume only one GPU is in use.
# For multi-gpu setting, please refer to
# https://www.tensorflow.org/guide/gpu#using_multiple_gpus
gpus = tf.config.experimental.list_physical_devices('GPU')
if len(gpus) == 0 or FLAGS.gpu_id is None:
device_id = "/device:CPU:0"
else:
tf.config.experimental.set_visible_devices(gpus[FLAGS.gpu_id], 'GPU')
device_id = '/device:GPU:0'
A_mat, X_mat, z_vec, train_idx, val_idx, test_idx = load_data_planetoid(
FLAGS.dataset)
An_mat = preprocess_graph(A_mat)
# N = A_mat.shape[0]
K = z_vec.max() + 1
with tf.device(device_id):
gcn = GCN(An_mat, X_mat, [FLAGS.hidden1, K])
gcn.train(train_idx, z_vec[train_idx], val_idx, z_vec[val_idx])
test_res = gcn.evaluate(test_idx, z_vec[test_idx], training=False)
# gcn = GCN(An_mat_diag, X_mat_stack, [FLAGS.hidden1, K])
# gcn.train(train_idx_recal, z_vec[train_idx], val_idx_recal, z_vec[val_idx])
# test_res = gcn.evaluate(test_idx_recal, z_vec[test_idx], training=False)
print("Dataset {}".format(FLAGS.dataset),
"Test loss {:.4f}".format(test_res[0]),
"test acc {:.4f}".format(test_res[1]))
def get_model(self, dataset):
if self.args.model=="GCN":
model = GCN(dataset.num_classes, dataset.img_size, k=self.args.K).cuda()
elif self.args.model=="UNet":
model = UNet(dataset.num_classes).cuda()
elif self.args.model=="GCN_DENSENET":
model = GCN_DENSENET(dataset.num_classes, dataset.img_size, k=self.args.K).cuda()
elif self.args.model=="GCN_DECONV":
model = GCN_DECONV(dataset.num_classes, dataset.img_size, k=self.args.K).cuda()
elif self.args.model=="GCN_PSP":
model = GCN_PSP(dataset.num_classes, dataset.img_size, k=self.args.K).cuda()
elif self.args.model=="GCN_COMB":
model = GCN_COMBINED(dataset.num_classes, dataset.img_size, k=self.args.K).cuda()
elif self.args.model=="GCN_RESNEXT":
model = GCN_RESNEXT(dataset.num_classes, k=self.args.K).cuda()
else:
raise ValueError("Invalid model arg.")
start_epoch = 0
if self.args.resume:
setup.load_save(model, self.args)
start_epoch = self.args.resume_epoch
self.args.start_epoch = start_epoch
model.train()
return model
def main(argv=None):
opt = parse_args(argv)
dataset = datasets.TCGADataset()
dataset.df = dataset.df - dataset.df.mean(axis=0)
gene_graph = GeneManiaGraph()
search_num_genes = [50, 100, 200, 300, 500, 1000, 2000, 4000, 8000, 16300]
test_size = 300
cuda = torch.cuda.is_available()
exp = []
for num_genes in search_num_genes:
start_time = time.time()
gene = "RPL4"
model = GCN(cuda=cuda,
dropout=opt.dropout,
num_layer=opt.num_layer,
channels=opt.channels,
embedding=opt.embedding,
aggregation=opt.aggregation,
lr=opt.lr,
agg_reduce=opt.agg_reduce)
dataset.labels = dataset.df[gene].where(
dataset.df[gene] > 0).notnull().astype("int")
dataset.labels = dataset.labels.values if type(
dataset.labels) == pd.Series else dataset.labels
X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(
dataset.df,
dataset.labels,
stratify=dataset.labels,
train_size=opt.train_size,
test_size=opt.test_size,
random_state=opt.seed)
if num_genes == 16300:
neighbors = gene_graph.nx_graph
else:
neighbors = gene_graph.bfs_sample_neighbors(gene, num_genes)
X_train = X_train[list(neighbors.nodes)].copy()
X_test = X_test[list(neighbors.nodes)].copy()
X_train[gene] = 1
X_test[gene] = 1
adj = sparse.csr_matrix(nx.to_numpy_matrix(neighbors))
model.fit(X_train, y_train, adj=adj)
y_hat = model.predict(X_test)
y_hat = np.argmax(y_hat, axis=1)
auc = sklearn.metrics.roc_auc_score(y_test,
np.asarray(y_hat).flatten())
del model
exp.append(auc)
report_results([{
"name": "auc",
"type": "objective",
"value": np.array(exp).mean()
}])
def get_model(model_name, num_classes, weight=None):
print('====> load {}-classes segmentation model: {}'.format(
num_classes, model_name))
model = None
if model_name == 'fcn8':
model = FCN8s(num_classes=num_classes)
elif model_name == 'fcn16':
model = FCN16VGG(num_classes=num_classes, pretrained=False)
elif model_name == 'fcn32':
model = FCN32VGG(num_classes=num_classes, pretrained=False)
elif model_name == 'unet':
model = UNet(num_classes=num_classes)
elif model_name == 'duc':
model = ResNetDUC(num_classes=num_classes)
elif model_name == 'duc_hdc':
model = ResNetDUCHDC(num_classes=num_classes)
elif model_name == 'gcn':
model = GCN(num_classes=num_classes, input_size=512)
elif model_name == 'pspnet':
model = PSPNet(num_classes=num_classes)
elif model_name == 'segnet':
model = SegNet(num_classes=num_classes)
if weight is not None:
model.load_state_dict(torch.load(weight))
return model
def __init__(self,
in_features_dim,
out_features_dim,
scope_embedding,
hidden_dim=0,
dropout=0.0):
super(EL_GCN, self).__init__()
self.gcn = GCN(in_features_dim, hidden_dim, dropout=dropout)
self.linear = nn.Linear(in_features_dim, out_features_dim)
self.scope_embedding = scope_embedding
self.len_scope = scope_embedding.shape[0]
def load_model(self, model_path, args):
if args.model == 'NON_ADAPTIVE_A3C':
self.model = BaseModel(args)
elif args.model == 'GCN':
self.model = GCN(args)
else:
self.model = SAVN(args)
saved_state = torch.load(model_path,
map_location=lambda storage, loc: storage)
self.model.load_state_dict(saved_state)
self.model_options = ModelOptions()
self.model_options.params = get_params(self.model, args.gpu_id)
def __init__(self, max_n_colors=10, inner_dim=64, timesteps=32, attention=False, attention_version='pairwise_0'):
super().__init__()
self.max_n_colors = max(max_n_colors, 10)
self.timesteps = timesteps
self.inner_dim = inner_dim
self.rnn_v = 1
self.v_init = torch.nn.Parameter(Normal(0, 1).sample([self.inner_dim]) / torch.sqrt(torch.Tensor([self.inner_dim])))
self.preprocess_gnn = GCN(hidden_dim=inner_dim)
self.rec_gnn = RecGNN(inner_dim, timesteps, attention=attention, attention_version=attention_version)
self.v_vote_mlp = nn.Sequential(
nn.Linear(in_features=self.inner_dim, out_features=self.inner_dim // 4),
nn.Sigmoid(),
nn.Linear(in_features=self.inner_dim // 4, out_features=1)
)
def main(argv=None):
opt = parse_args(argv)
tasks = TCGAMeta(download=True, preload=True)
task = tasks[113]
# Setup the results dictionary
filename = "experiments/results/clinical-tasks.pkl"
try:
results = pickle.load(open(filename, "rb"), encoding='latin1')
print("Loaded Checkpointed Results")
except Exception as e:
print(e)
results = pd.DataFrame(columns=[
'task', 'acc_metric', 'model', 'graph', 'trial', 'train_size',
'time_elapsed'
])
print("Created a New Results Dictionary")
train_size = 50
trials = 3
cuda = True
exp = []
for trial in range(trials):
model = GCN(cuda=cuda,
dropout=opt.dropout,
num_layer=opt.num_layer,
channels=opt.channels,
embedding=opt.embedding,
aggregation=opt.aggregation,
lr=opt.lr,
agg_reduce=opt.agg_reduce,
seed=trial)
task._samples = task._samples - task._samples.mean(axis=0)
task._samples = task._samples / task._samples.var()
X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(
task._samples,
task._labels,
stratify=task._labels,
train_size=train_size,
test_size=len(task._labels) - train_size)
adj = sparse.csr_matrix(nx.to_numpy_matrix(GeneManiaGraph().nx_graph))
model.fit(X_train, y_train, adj=adj)
y_hat = []
for chunk in get_every_n(X_test, 10):
y_hat.extend(np.argmax(model.predict(chunk), axis=1).numpy())
exp.append(model.metric(y_test, y_hat))
print(exp)
report_results([{
"name": "acc_metric",
"type": "objective",
"value": np.array(exp).mean()
}])
def visualize():
adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask = load_data(
args.dataset)
model = GCN(features.shape[1], args.hidden_dim, y_train.shape[1], 0)
gcn_layer1 = model.gcn_layer1
gcn_layer1.register_forward_hook(hook_fn_forward)
if args.checkpoint:
model.load_state_dict(torch.load(args.checkpoint))
model.eval()
model(adj, features)
x_all = gcn_layer1_output[0]
y_train = np.argmax(y_train[train_mask, :].numpy(), axis=1)
y_val = np.argmax(y_val[val_mask, :].numpy(), axis=1)
y_test = np.argmax(y_test[test_mask, :].numpy(), axis=1)
tsne_vis(x_all[train_mask], y_train, classnames[args.dataset], 'train_set',
args.dataset)
tsne_vis(x_all[val_mask], y_val, classnames[args.dataset], 'val_set',
args.dataset)
tsne_vis(x_all[test_mask], y_test, classnames[args.dataset], 'test_set',
args.dataset)
# model = MLP(column_names=dataset.df.columns, num_layer=1, dropout=False,
# train_valid_split=0.5, cuda=cuda, metric=sklearn.metrics.roc_auc_score,
# channels=16, batch_size=10, lr=0.0007, weight_decay=0.00000001,
# verbose=False, patience=5, num_epochs=10, seed=seed,
# full_data_cuda=True, evaluate_train=False)
# monitor number of epochs by is_first_degree (worked well for genemania)
# if is_first_degree:
# num_epochs = 30
# else:
# num_epochs = 10
model = GCN(column_names=dataset.df.columns,
name="GCN_lay3_chan32_emb32_dropout",
cuda=True,
num_layer=num_layer,
channels=channels,
embedding=embedding,
batch_size=batch_size,
dropout=dropout)
experiment = {
"gene": gene,
"tumor_type": tumor_type,
"model": "GCN",
# "graph": graph,
# "num_layer": num_layer,
# "channels": channels,
"train_size": train_size,
# "embedding": embedding,
# "dropout": dropout,
# "batch": batch_size
models = [
#GCN(name="GCN_lay20_chan32_emb32_dropout_pool_kmeans", cuda=cuda, dropout=True, num_layer=4, channels=32, embedding=32, prepool_extralayers=5, pooling="kmeans"),
#GCN(name="GCN_lay20_chan32_emb32_dropout_pool_hierarchy", cuda=cuda, dropout=True, num_layer=4, channels=32, embedding=32, prepool_extralayers=5, pooling="hierarchy"),
#GCN(name="GCN_lay20_chan32_emb32_dropout_pool_random", cuda=cuda, dropout=True,num_layer=4, channels=32, embedding=32, prepool_extralayers=5, pooling="random"),
# GCN(name="GCN_lay3_chan64_emb32_dropout_agg_hierarchy_reduce", cuda=cuda, dropout=True, num_layer=3, channels=64, embedding=32, aggregation="hierarchy", lr=0.001),
# GCN(name="GCN_lay3_chan64_emb32_dropout_agg_hierarchy_reduce_1.2", cuda=cuda, dropout=True, num_layer=3, channels=64, embedding=32, aggregation="hierarchy", agg_reduce=1.2, lr=0.001),
# GCN(name="GCN_lay3_chan64_emb32_dropout_agg_hierarchy_reduce_1.5", cuda=cuda, dropout=True, num_layer=3, channels=64, embedding=32, aggregation="hierarchy", agg_reduce=1.5, lr=0.001),
# GCN(name="GCN_lay3_chan64_emb32_dropout_agg_hierarchy_reduce_2", cuda=cuda, dropout=True, num_layer=3, channels=64, embedding=32, aggregation="hierarchy", agg_reduce=2, lr=0.001),
# GCN(name="GCN_lay3_chan64_emb32_dropout_agg_hierarchy_reduce_3", cuda=cuda, dropout=True, num_layer=3, channels=64, embedding=32, aggregation="hierarchy", agg_reduce=3, lr=0.001),
# GCN(name="GCN_lay3_chan64_emb32_dropout_agg_hierarchy_reduce_5", cuda=cuda, dropout=True, num_layer=3, channels=64, embedding=32, aggregation="hierarchy", agg_reduce=5, lr=0.001),
# GCN(name="GCN_lay3_chan64_emb32_dropout_agg_hierarchy_reduce_10", cuda=cuda, dropout=True, num_layer=3, channels=64, embedding=32, aggregation="hierarchy", agg_reduce=10, lr=0.001),
# GCN(name="GCN_lay3_chan64_emb32_dropout_pool_hierarchy", cuda=cuda, dropout=True, num_layer=3, channels=64, embedding=32, pooling="hierarchy"),
GCN(name="GCN_lay3_chan64_emb32_dropout",
cuda=cuda,
dropout=True,
num_layer=3,
channels=64,
embedding=32,
agg_reduce=2,
lr=0.001),
#MLP(name="MLP_lay2_chan512_dropout", cuda=cuda, dropout=True, num_layer=2, channels=512),
#MLP(name="MLP_lay2_chan512", cuda=cuda, dropout=False, num_layer=2, channels=512),
#SLR(name="SLR_lambda1_l11", cuda=cuda)
]
# Create the set of all experiment ids and see which are left to do
columns = ["task", "graph", "model", "seed", "train_size"]
all_exp_ids = [
x
for x in itertools.product(["".join(task.id) for task in tasks],
graphs.keys(), [model.name for model in models],
range(trials), [train_size])
assert train_wnids == wnids[:len(train_wnids)]
fc_vectors = torch.tensor(fc_vectors).float().cuda()
fc_vectors = F.normalize(fc_vectors)
hidden_layers = args.hidden_layers
print('{} nodes, {} edges'.format(n, len(edges)))
print('word vectors:', word_vectors.shape)
print('fc vectors:', fc_vectors.shape)
print('hidden layers:', hidden_layers)
gcn = GCN(n,
edges,
weights,
word_vectors,
word_vectors.shape[1],
fc_vectors.shape[1],
hidden_layers,
residual=(not args.no_residual)).cuda()
optimizer = torch.optim.Adam(gcn.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
v_train, v_val = map(float, args.trainval.split(','))
n_trainval = len(fc_vectors)
n_train = round(n_trainval * (v_train / (v_train + v_val)))
print('num train: {}, num val: {}'.format(n_train, n_trainval - n_train))
tlist = list(range(len(fc_vectors)))
random.shuffle(tlist)
# full_data_cuda=True, evaluate_train=False)
graph = "funcoup"
# load graph and create adj matrix.
gene_graph = graph_dict[graph]()
# adj for GCN
adj = gene_graph.adj()
# monitor number of epochs by is_first_degree (worked well for genemania)
# if is_first_degree:
# num_epochs = 30
# else:
# num_epochs = 10
model = GCN(column_names=dataset.df.columns, name="GCN_lay3_chan32_emb32_dropout", cuda=True, aggregation="hierarchy",
agg_reduce=2, num_layer=num_layer, channels=channels, embedding=embedding, batch_size=batch_size,
dropout=dropout, verbose=False)
# model = GCN(name="GCN_lay20_chan32_emb32_dropout_agg_hierarchy",
# dropout=True,
# cuda=True,
# num_layer=20,
# channels=32,
# embedding=32,
# aggregation="hierarchy",
# agg_reduce=1,
# lr=0.01,
# num_epochs=200,
# patience=100,
# verbose=True
# )
def __init__(self, config):
super(KPFCNN, self).__init__()
############
# Parameters
############
# Current radius of convolution and feature dimension
layer = 0
r = config.first_subsampling_dl * config.conv_radius
in_dim = config.in_features_dim
out_dim = config.first_feats_dim
self.K = config.num_kernel_points
self.epsilon = torch.nn.Parameter(torch.tensor(-5.0))
self.final_feats_dim = config.final_feats_dim
#####################
# List Encoder blocks
#####################
# Save all block operations in a list of modules
self.encoder_blocks = nn.ModuleList()
self.encoder_skip_dims = []
self.encoder_skips = []
# Loop over consecutive blocks
for block_i, block in enumerate(config.architecture):
# Check equivariance
if ('equivariant' in block) and (not out_dim % 3 == 0):
raise ValueError(
'Equivariant block but features dimension is not a factor of 3'
)
# Detect change to next layer for skip connection
if np.any([
tmp in block
for tmp in ['pool', 'strided', 'upsample', 'global']
]):
self.encoder_skips.append(block_i)
self.encoder_skip_dims.append(in_dim)
# Detect upsampling block to stop
if 'upsample' in block:
break
# Apply the good block function defining tf ops
self.encoder_blocks.append(
block_decider(block, r, in_dim, out_dim, layer, config))
# Update dimension of input from output
if 'simple' in block:
in_dim = out_dim // 2
else:
in_dim = out_dim
# Detect change to a subsampled layer
if 'pool' in block or 'strided' in block:
# Update radius and feature dimension for next layer
layer += 1
r *= 2
out_dim *= 2
#####################
# bottleneck layer and GNN part
#####################
gnn_feats_dim = config.gnn_feats_dim
self.bottle = nn.Conv1d(in_dim,
gnn_feats_dim,
kernel_size=1,
bias=True)
k = config.dgcnn_k
num_head = config.num_head
self.gnn = GCN(num_head, gnn_feats_dim, k, eval(config.nets))
self.proj_gnn = nn.Conv1d(gnn_feats_dim,
gnn_feats_dim,
kernel_size=1,
bias=True)
self.proj_score = nn.Conv1d(gnn_feats_dim, 1, kernel_size=1, bias=True)
#####################
# List Decoder blocks
#####################
out_dim = gnn_feats_dim + 2
# Save all block operations in a list of modules
self.decoder_blocks = nn.ModuleList()
self.decoder_concats = []
# Find first upsampling block
start_i = 0
for block_i, block in enumerate(config.architecture):
if 'upsample' in block:
start_i = block_i
break
# Loop over consecutive blocks
for block_i, block in enumerate(config.architecture[start_i:]):
# Add dimension of skip connection concat
if block_i > 0 and 'upsample' in config.architecture[start_i +
block_i - 1]:
in_dim += self.encoder_skip_dims[layer]
self.decoder_concats.append(block_i)
# Apply the good block function defining tf ops
self.decoder_blocks.append(
block_decider(block, r, in_dim, out_dim, layer, config))
# Update dimension of input from output
in_dim = out_dim
# Detect change to a subsampled layer
if 'upsample' in block:
# Update radius and feature dimension for next layer
layer -= 1
r *= 0.5
out_dim = out_dim // 2
return
from models.utils import get_accuracy
parser = argparse.ArgumentParser()
parser.add_argument('--dataset',
type=str,
default='citeseer',
help='Dataset to train')
parser.add_argument('--hidden_dim',
type=list,
default=16,
help='Dimensions of hidden layers')
parser.add_argument('--checkpoint',
type=str,
help='Directory to save checkpoints')
args = parser.parse_args()
adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask = load_data(
args.dataset)
model = GCN(features.shape[1], args.hidden_dim, y_train.shape[1], 0)
def evaluate(checkpoint):
model.load_state_dict(torch.load(checkpoint))
model.eval()
outputs = model(adj, features)
accuracy = get_accuracy(outputs, y_test, test_mask)
print("Accuracy on test set is %f" % accuracy)
if __name__ == '__main__':
evaluate(args.checkpoint)
t_features = tf.SparseTensor(*features)
t_y_train = tf.convert_to_tensor(y_train)
t_y_val = tf.convert_to_tensor(y_val)
t_y_test = tf.convert_to_tensor(y_test)
tm_train_mask = tf.convert_to_tensor(train_mask)
tm_val_mask = tf.convert_to_tensor(val_mask)
tm_test_mask = tf.convert_to_tensor(test_mask)
t_support = []
for i in range(len(support)):
t_support.append(tf.cast(tf.SparseTensor(*support[i]), dtype=tf.float64))
# Create model
model = GCN(input_dim=features[2][1],
output_dim=y_train.shape[1],
num_features_nonzero=features[1].shape)
# Loss and optimizer
optimizer = optimizers.Adam(lr=cfg.learning_rate)
cost_val = []
for epoch in range(cfg.epochs):
t = time.time()
with tf.GradientTape() as tape:
_, loss, acc = model((t_features, t_y_train, tm_train_mask, t_support))
grads = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(grads, model.trainable_variables))
print("Created a New Results Dictionary")
gene_graph = GeneManiaGraph()
search_num_genes=[50, 100, 200, 300, 500, 1000, 2000, 4000, 8000, 16300]
test_size=300
search_train_size=[50]
cuda = torch.cuda.is_available()
trials=3
search_genes = ["RPL4"]
models = [
#GCN(name="GCN_lay20_chan32_emb32_dropout_pool_hierarchy", cuda=cuda, dropout=True, num_layer=4, channels=32, embedding=32, prepool_extralayers=5, aggregation="hierarchy"),
#GCN(name="GCN_lay3_chan16_emb32_dropout_agg_hierarchy_prepool_1", cuda=cuda, dropout=True, num_layer=3, channels=16, embedding=32, prepool_extralayers=1, aggregation="hierarchy"),
#GCN(name="GCN_lay3_chan64_emb32_dropout_agg_hierarchy_reduce_5_schedule", cuda=cuda, dropout=True, num_layer=3, channels=64, embedding=32, aggregation="hierarchy", agg_reduce=5, lr=0.001, patience=15, scheduler=True),
#GCN(name="GCN_lay20_chan32_emb32_dropout_pool_random", cuda=cuda, num_layer=4, channels=32, embedding=32, prepool_extralayers=5, aggregation="random"),
GCN(name="GCN_lay3_chan64_emb32_dropout_agg_hierarchy_reduce", cuda=cuda, dropout=True, num_layer=3, channels=64, embedding=32, aggregation="hierarchy", lr=0.001),
GCN(name="GCN_lay3_chan64_emb32_dropout_agg_hierarchy_reduce_1.2", cuda=cuda, dropout=True, num_layer=3, channels=64, embedding=32, aggregation="hierarchy", agg_reduce=1.2, lr=0.001),
GCN(name="GCN_lay3_chan64_emb32_dropout_agg_hierarchy_reduce_1.5", cuda=cuda, dropout=True, num_layer=3, channels=64, embedding=32, aggregation="hierarchy", agg_reduce=1.5, lr=0.001),
GCN(name="GCN_lay3_chan64_emb32_dropout_agg_hierarchy_reduce_2", cuda=cuda, dropout=True, num_layer=3, channels=64, embedding=32, aggregation="hierarchy", agg_reduce=2, lr=0.001),
GCN(name="GCN_lay3_chan64_emb32_dropout_agg_hierarchy_reduce_3", cuda=cuda, dropout=True, num_layer=3, channels=64, embedding=32, aggregation="hierarchy", agg_reduce=3, lr=0.001),
GCN(name="GCN_lay3_chan64_emb32_dropout_agg_hierarchy_reduce_5", cuda=cuda, dropout=True, num_layer=3, channels=64, embedding=32, aggregation="hierarchy", agg_reduce=5, lr=0.001),
GCN(name="GCN_lay3_chan64_emb32_dropout_agg_hierarchy_reduce_10", cuda=cuda, dropout=True, num_layer=3, channels=64, embedding=32, aggregation="hierarchy", agg_reduce=10, lr=0.001),
GCN(name="GCN_lay3_chan64_emb32_dropout", cuda=cuda, dropout=True, num_layer=3, channels=64, embedding=32, lr=0.001),
MLP(name="MLP_lay2_chan512_dropout", cuda=cuda, dropout=True, num_layer=2, channels=512),
MLP(name="MLP_lay2_chan512", cuda=cuda, dropout=False, num_layer=2, channels=512),
SLR(name="SNLR_lambda1_l11", cuda=cuda)
]
# Create the set of all experiment ids and see which are left to do
model_names = [model.name for model in models]
columns = ["gene", "model", "num_genes", "train_size", "seed"]
edges = edges + [(u, u) for u in range(n)]
word_vectors = torch.tensor(graph['vectors']).cuda()
word_vectors = F.normalize(word_vectors)
fcfile = json.load(open('materials/fc-weights.json', 'r'))
train_wnids = [x[0] for x in fcfile]
fc_vectors = [x[1] for x in fcfile]
assert train_wnids == wnids[:len(train_wnids)]
fc_vectors = torch.tensor(fc_vectors).cuda()
fc_vectors = F.normalize(fc_vectors)
hidden_layers = args.hidden_layers
gcn = GCN(n,
edges,
word_vectors.shape[1],
fc_vectors.shape[1],
hidden_layers,
ks=args.k).cuda()
logging.info(gcn)
logging.info('{} nodes, {} edges'.format(n, len(edges)))
logging.info('word vectors: {}'.format(word_vectors.shape))
logging.info('fc vectors: {}'.format(fc_vectors.shape))
logging.info('hidden layers: {}'.format(hidden_layers))
optimizer = torch.optim.Adam(gcn.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
if args.trainval == 'awa2':
awa2_split = json.load(open('materials/awa2-split.json', 'r'))
train_wnids = awa2_split['train']
def init_model(self):
self.get_feature_length()
self.encoder = GCN([self.in_feature_len, 30, 18])
self.decoder = GCN([18, 30, self.in_feature_len])
class MetaSim(nn.Module):
def __init__(self, opts):
super(MetaSim, self).__init__()
self.opts = opts
self.cfg = read_json(opts['config'])
g = get_generator(opts['dataset'])
self.generator = g(self.cfg)
self.init_model()
def init_model(self):
self.get_feature_length()
self.encoder = GCN([self.in_feature_len, 30, 18])
self.decoder = GCN([18, 30, self.in_feature_len])
def get_feature_length(self):
"""
Samples a graph and encodes it to get the
feature length
"""
x = self.generator.sample()
f = self.generator.encode(x)
self.num_classes = len(self.generator.features.classes)
self.in_feature_len = f[0][0].shape[-1]
# first 0 for batch, second 0 for features
# f[.][1] is the mask denoting mutability
def freeze_encoder(self):
for n,p in self.encoder.named_parameters():
p.requires_grad = False
def forward(self, x, adj, masks=None, sample=False):
enc = self.encoder(x, adj)
dec = self.decoder(enc, adj)
# apply sigmoid on all continuous params
# softmax on classes
dec_act = torch.cat((
torch.softmax(dec[..., :self.num_classes], dim=-1),
torch.sigmoid(dec[..., self.num_classes:])
), dim=-1)
if sample:
assert masks is not None, 'Need masks for sampling'
dec_act, log_probs = self.sample(dec_act, masks)
return dec, dec_act, log_probs
return dec, dec_act
def sample(self, features, masks, sigma=None):
"""
Takes in predicted features (dec_act), masks
for mutable nodes
Returns new features and a log_prob for
each batch element
"""
if sigma is None:
sigma = torch.tensor(0.02).to(self.opts['device'])
m = masks.cpu().numpy()
log_probs = torch.zeros(features.shape[0], device=features.device,
dtype=torch.float32)
for b in range(features.shape[0]):
lp = 0
idxs = np.array(np.nonzero(m[b])).transpose()
for idx in idxs:
mu = features[b, idx[0], idx[1]]
n = torch.distributions.normal.Normal(mu, sigma)
sample = n.rsample()
features[b, idx[0], idx[1]] = sample
lp += n.log_prob(sample)
log_probs[b] = lp
return features, log_probs
adj = sparse.csr_matrix(nx.to_numpy_matrix(neighbors))
print(adj.shape)
if model_name == 'GCN':
model = GCN(name="GCN_noemb_lay1_chan64_dropout_agg_hierarchical", #_lay3_chan64_emb32_dropout_agg_hierarchy",
dropout=True,
cuda=torch.cuda.is_available(),
num_layer=1,
prepool_extralayers=0,
channels=64,
embedding=64, # sdict["embedding"],
aggregation="hierarchical",
lr=0.001,
num_epochs=100,
patience=30,
verbose=True,
seed=seed,
train_valid_split=0.8
)
elif model_name == 'GCN2':
model = GCN(name="GCN_noemb_lay1_chan64_dropout_agg_hierarchical_lr=0.0005", #_lay3_chan64_emb32_dropout_agg_hierarchy",
dropout=True,
cuda=torch.cuda.is_available(),
num_layer=1,
prepool_extralayers=0,
channels=64,
embedding=64, # sdict["embedding"],
def train_GCN():
labels = torch.tensor(MultiLabelBinarizer().fit_transform(entry_ids),
dtype=torch.float32,
device=device)
labels_weight = get_weight(labels)
n_class = p.label_size - 1 # p.label_size-1
# labels = torch.empty(labels.shape[0], n_class).uniform_(0,1)
# labels = torch.bernoulli(labels).to(device)
# y = torch.empty(labels.shape[0],1, dtype=torch.long, device=device).random_() % n_class
# labels = torch.empty(labels.shape[0],n_class, dtype=torch.float32, device=device)
# labels.zero_()
# labels.scatter_(1,y,1)
# check if adjacancy matrix already calculated
if not os.path.exists(os.path.join(mesh_folder, 'a_hat_matrix')):
a_hat = get_adjacancy_matrix(mesh_graph, node_list[1:])
# save node_list and the calculated adjacancy matrix
with open(os.path.join(mesh_folder, 'node_list'), 'wb') as f:
pickle.dump(node_list, f)
data = sparse.coo_matrix(a_hat)
with open(os.path.join(mesh_folder, 'a_hat_matrix'), 'wb') as f:
pickle.dump(data, f)
else:
with open(os.path.join(mesh_folder, 'a_hat_matrix'), 'rb') as f:
data = pickle.load(f)
i = torch.tensor([data.row, data.col], dtype=torch.long, device=device)
v = torch.tensor(data.data, dtype=torch.float32, device=device)
a_hat = torch.sparse.FloatTensor(
i, v, torch.Size([len(node_list) - 1,
len(node_list) - 1])).cuda()
# summery_model = SummeryCNN(p.word_vocab_size, word_embeddings, embedding_size=word_embedding_size, output_size=word_embedding_size,
# padidx=p._word_vocab.vocab['<pad>'])
# summery_model = SummeryRNN(p.word_vocab_size, word_embeddings, embedding_size=word_embedding_size, output_size=word_embedding_size,
# padidx=p._word_vocab.vocab['<pad>'])
gcn_model = GCN(nfeat=word_embedding_size, nhid=512, dropout=0.0)
fc_model = FC(word_embedding_size, n_class, dropout=0.5) # p.label_size-1
# summery_model.to(device)
gcn_model.to(device)
fc_model.to(device)
optimizer = torch.optim.Adam(
list(gcn_model.parameters()) + list(fc_model.parameters()),
lr=0.001) # weight_decay=0.01; list(summery_model.parameters()) +
# optimizer = torch.optim.Adam(list(summery_model.parameters()) + list(fc_model.parameters()), lr=0.001)
if use_elmo:
ELMO_folder = r'/media/druv022/Data1/Masters/Thesis/Data/ELMO'
with open(os.path.join(ELMO_folder, 'elmo_scope_weights'), 'rb') as f:
elmo_scope_embeddings = pickle.load(f)
with open(os.path.join(ELMO_folder, 'id2idx_dict'), 'rb') as f:
id2idx_dict = pickle.load(f)
scope_embds = []
for i in node_list:
if i in id2idx_dict:
scope_embds.append(
torch.sum(elmo_scope_embeddings[id2idx_dict[i]], dim=0))
if i == '<unk>':
scope_embds.append(
torch.zeros(word_embedding_size, device=device))
scope_embds = torch.stack(scope_embds)
best_z = []
best_acc = 0
epochs = 100
batch_size = 256
for epoch in range(epochs):
start_time = time.time()
if not use_elmo:
optimizer.zero_grad()
idxs = np.arange(0, len(node_ids))
np.random.shuffle(idxs)
word_ids_shuffled = [word_ids[i] for i in list(idxs)]
embeddings = []
for x in minibatch(word_ids_shuffled, batch_size=batch_size):
x = x.to(device)
z = summery_model(x)
embeddings.extend(z)
scope_embds = list(np.arange(0, len(node_ids)))
for i, idx in enumerate(idxs):
scope_embds[idx] = embeddings[i]
scope_embds = torch.stack(scope_embds)
z_, x = gcn_model(scope_embds, a_hat)
z = fc_model(z_)
# z = fc_model(scope_embds)
# loss1 = nn.functional.binary_cross_entropy_with_logits(z[0:5000],labels[0:5000])
# loss2 = nn.functional.binary_cross_entropy_with_logits(z[5000:10000],labels[5000:10000])
# loss3 = nn.functional.binary_cross_entropy_with_logits(z[10000:],labels[10000:])
# loss = (loss1 + loss2 + loss3)/3
loss = nn.functional.binary_cross_entropy_with_logits(
z, labels, reduction='mean',
pos_weight=labels_weight) #, pos_weight=labels_weight
# loss = nn.functional.cross_entropy(z,y.view(-1), reduction='mean')
loss.backward()
optimizer.step()
writer.add_scalars('loss', {'loss': loss.item()}, epoch)
with torch.no_grad():
pred_labels = (torch.sigmoid(z.data) > 0.5)
# pred_labels = nn.functional.softmax(z,dim=-1).detach()
# _, pred_idx = torch.max(pred_labels,-1)
# pred_labels = torch.empty(labels.shape[0],n_class, dtype=torch.float32, device=device)
# pred_labels.zero_()
# pred_labels.scatter_(1,pred_idx.view(-1,1),1)
true_labels = labels.data
if epoch % 10 == 0:
print(
classification_report(true_labels.cpu().numpy(),
pred_labels.cpu().numpy()))
acc, fp = calculate_accuracy(pred_labels.cpu().numpy(),
true_labels.cpu().numpy())
print(
f'Epoch: {epoch}\t Loss: {loss.item()}\t time: {time.time() - start_time}\tAccuracy: {acc}\t False positive: {fp}'
)
if acc > best_acc:
best_acc = acc
# best_z = z_
# # torch.save(summery_model.state_dict(),os.path.join(mesh_folder,'summery_model'))
# torch.save(gcn_model.state_dict(),os.path.join(mesh_folder,'gcn_model'))
# with open(os.path.join(mesh_folder,'GCN_mesh_embedding'), 'wb') as f:
# pickle.dump(x, f)
# print('Saving Best model...')
print('Best ACC: ', best_acc)
embedding_txt = r'/media/druv022/Data1/Masters/Thesis/Data/MeSH/GCN/embedding.txt'
embedding_temp = r'/media/druv022/Data1/Masters/Thesis/Data/MeSH/GCN/embedding_temp.txt'
embedding = r'/media/druv022/Data1/Masters/Thesis/Data/MeSH/GCN/embedding'
with open(embedding_txt, 'w') as f:
for idx, i in enumerate(node_list[1:]):
f.write(i + ' ' + ' '.join(
[str(i)
for i in z_[idx].cpu().detach().numpy().tolist()]) + '\n')
glove_file = datapath(embedding_txt)
temp_file = get_tmpfile(embedding_temp)
_ = glove2word2vec(glove_file, temp_file)
wv = KeyedVectors.load_word2vec_format(temp_file)
wv.save(embedding)
print('Complete')
def main(training_file,
dev_file,
test_file,
epochs=None,
patience=None,
num_heads=None,
num_out_heads=None,
num_layers=None,
num_hidden=None,
residual=None,
in_drop=None,
attn_drop=None,
lr=None,
weight_decay=None,
alpha=None,
batch_size=None,
graph_type=None,
net=None,
freeze=None,
cuda=None,
fw=None):
# number of training epochs
if epochs is None:
epochs = 400
print('EPOCHS', epochs)
# used for early stop
if patience is None:
patience = 15
print('PATIENCE', patience)
# number of hidden attention heads
if num_heads is None:
num_heads_ch = [4, 5, 6, 7]
else:
num_heads_ch = flattenList(num_heads)
print('NUM HEADS', num_heads_ch)
# number of output attention heads
if num_out_heads is None:
num_out_heads_ch = [4, 5, 6, 7]
else:
num_out_heads_ch = flattenList(num_out_heads)
print('NUM OUT HEADS', num_out_heads_ch)
# number of hidden layers
if num_layers is None:
num_layers_ch = [2, 3, 4, 5, 6]
else:
num_layers_ch = flattenList(num_layers)
print('NUM LAYERS', num_layers_ch)
# number of hidden units
if num_hidden is None:
num_hidden_ch = [32, 64, 96, 128, 256, 350, 512]
else:
num_hidden_ch = flattenList(num_hidden)
print('NUM HIDDEN', num_hidden_ch)
# use residual connection
if residual is None:
residual_ch = [True, False]
else:
residual_ch = flattenList(residual)
print('RESIDUAL', residual_ch)
# input feature dropout
if in_drop is None:
in_drop_ch = [0., 0.001, 0.0001, 0.00001]
else:
in_drop_ch = flattenList(in_drop)
print('IN DROP', in_drop_ch)
# attention dropout
if attn_drop is None:
attn_drop_ch = [0., 0.001, 0.0001, 0.00001]
else:
attn_drop_ch = flattenList(attn_drop)
print('ATTENTION DROP', attn_drop_ch)
# learning rate
if lr is None:
lr_ch = [0.0000005, 0.0000015, 0.00001, 0.00005, 0.0001]
else:
lr_ch = flattenList(lr)
print('LEARNING RATE', lr_ch)
# weight decay
if weight_decay is None:
weight_decay_ch = [0.0001, 0.001, 0.005]
else:
weight_decay_ch = flattenList(weight_decay)
print('WEIGHT DECAY', weight_decay_ch)
# the negative slop of leaky relu
if alpha is None:
alpha_ch = [0.1, 0.15, 0.2]
else:
alpha_ch = flattenList(alpha)
print('ALPHA', alpha_ch)
# batch size used for training, validation and test
if batch_size is None:
batch_size_ch = [175, 256, 350, 450, 512, 800, 1600]
else:
batch_size_ch = flattenList(batch_size)
print('BATCH SIZE', batch_size_ch)
# net type
if net is None:
net_ch = [GCN, GAT, RGCN, PGCN, PRGCN, GGN, PGAT, Custom_Net]
else:
net_ch_raw = flattenList(net)
net_ch = []
for ch in net_ch_raw:
if ch.lower() == 'gcn':
if fw == 'dgl':
net_ch.append(GCN)
else:
net_ch.append(PGCN)
elif ch.lower() == 'gat':
if fw == 'dgl':
net_ch.append(GAT)
else:
net_ch.append(PGAT)
elif ch.lower() == 'rgcn':
if fw == 'dgl':
net_ch.append(RGCN)
else:
net_ch.append(PRGCN)
elif ch.lower() == 'ggn':
net_ch.append(GGN)
elif ch.lower() == 'rgat':
net_ch.append(PRGAT)
elif ch.lower() == 'custom_net':
net_ch.append(Custom_Net)
else:
print('Network type {} is not recognised.'.format(ch))
exit(1)
print('NET TYPE', net_ch)
# graph type
if net_ch in [GCN, GAT, PGCN, GGN, PGAT, Custom_Net]:
if graph_type is None:
graph_type_ch = ['raw', '1', '2', '3', '4', 'relational']
else:
graph_type_ch = flattenList(graph_type)
else:
if graph_type is None:
graph_type_ch = ['relational']
else:
graph_type_ch = flattenList(graph_type)
print('GRAPH TYPE', graph_type_ch)
# Freeze input neurons?
if freeze is None:
freeze_ch = [True, False]
else:
freeze_ch = flattenList(freeze)
print('FREEZE', freeze_ch)
# CUDA?
if cuda is None:
device = torch.device("cpu")
elif cuda:
device = torch.device("cuda")
else:
device = torch.device("cpu")
print('DEVICE', device)
if fw is None:
fw = ['dgl', 'pg']
# define loss function
loss_fcn = torch.nn.BCEWithLogitsLoss()
#loss_fcn = torch.nn.MSELoss()
for trial in range(10):
trial_s = str(trial).zfill(6)
num_heads = random.choice(num_heads_ch)
num_out_heads = random.choice(num_out_heads_ch)
num_layers = random.choice(num_layers_ch)
num_hidden = random.choice(num_hidden_ch)
residual = random.choice(residual_ch)
in_drop = random.choice(in_drop_ch)
attn_drop = random.choice(attn_drop_ch)
lr = random.choice(lr_ch)
weight_decay = random.choice(weight_decay_ch)
alpha = random.choice(alpha_ch)
batch_size = random.choice(batch_size_ch)
graph_type = random.choice(graph_type_ch)
net_class = random.choice(net_ch)
freeze = random.choice(freeze_ch)
fw = random.choice(fw)
if freeze == False:
freeze = 0
else:
if graph_type == 'raw' or graph_type == '1' or graph_type == '2':
freeze = 4
elif graph_type == '3' or graph_type == '4':
freeze = 6
elif graph_type == 'relational':
freeze = 5
else:
exit(1)
print('=========================')
print('TRIAL', trial_s)
print('HEADS', num_heads)
print('OUT_HEADS', num_out_heads)
print('LAYERS', num_layers)
print('HIDDEN', num_hidden)
print('RESIDUAL', residual)
print('inDROP', in_drop)
print('atDROP', attn_drop)
print('LR', lr)
print('DECAY', weight_decay)
print('ALPHA', alpha)
print('BATCH', batch_size)
print('GRAPH_ALT', graph_type)
print('ARCHITECTURE', net_class)
print('FREEZE', freeze)
print('FRAMEWORK', fw)
print('=========================')
# create the dataset
print('Loading training set...')
train_dataset = SocNavDataset(training_file,
mode='train',
alt=graph_type)
print('Loading dev set...')
valid_dataset = SocNavDataset(dev_file, mode='valid', alt=graph_type)
print('Loading test set...')
test_dataset = SocNavDataset(test_file, mode='test', alt=graph_type)
print('Done loading files')
train_dataloader = DataLoader(train_dataset,
batch_size=batch_size,
collate_fn=collate)
valid_dataloader = DataLoader(valid_dataset,
batch_size=batch_size,
collate_fn=collate)
test_dataloader = DataLoader(test_dataset,
batch_size=batch_size,
collate_fn=collate)
num_rels = train_dataset.data[0].num_rels
cur_step = 0
best_loss = -1
n_classes = train_dataset.labels.shape[1]
print('Number of classes: {}'.format(n_classes))
num_feats = train_dataset.features.shape[1]
print('Number of features: {}'.format(num_feats))
g = train_dataset.graph
heads = ([num_heads] * num_layers) + [num_out_heads]
# define the model
if fw == 'dgl':
if net_class in [GCN]:
model = GCN(g, num_feats, num_hidden, n_classes, num_layers,
F.elu, in_drop)
elif net_class in [GAT]:
model = net_class(g,
num_layers,
num_feats,
num_hidden,
n_classes,
heads,
F.elu,
in_drop,
attn_drop,
alpha,
residual,
freeze=freeze)
else:
# def __init__(self, g, in_dim, h_dim, out_dim, num_rels, num_hidden_layers=1):
model = RGCN(g,
in_dim=num_feats,
h_dim=num_hidden,
out_dim=n_classes,
num_rels=num_rels,
feat_drop=in_drop,
num_hidden_layers=num_layers,
freeze=freeze)
else:
if net_class in [PGCN]:
model = PGCN(
num_feats,
n_classes,
num_hidden,
num_layers,
in_drop,
F.relu,
improved=True, # Compute A-hat as A + 2I
bias=True)
elif net_class in [PRGCN]:
model = PRGCN(
num_feats,
n_classes,
num_rels,
num_rels, # num_rels? # TODO: Add variable
num_hidden,
num_layers,
in_drop,
F.relu,
bias=True)
elif net_class in [PGAT]:
model = PGAT(num_feats,
n_classes,
num_heads,
in_drop,
num_hidden,
num_layers,
F.relu,
concat=True,
neg_slope=alpha,
bias=True)
elif net_class in [PRGAT]:
model = PRGAT(
num_feats,
n_classes,
num_heads,
num_rels,
num_rels, # num_rels? # TODO: Add variable
num_hidden,
num_layers,
num_layers,
in_drop,
F.relu,
alpha,
bias=True)
elif net_class in [Custom_Net]:
model = Custom_Net(features=num_feats,
hidden=num_layers,
out_features=num_hidden,
classes=n_classes)
else:
model = GGN(num_feats, num_layers, aggr='mean', bias=True)
# Describe the model
# describe_model(model)
# define the optimizer
optimizer = torch.optim.Adam(model.parameters(),
lr=lr,
weight_decay=weight_decay)
# for name, param in model.named_parameters():
# if param.requires_grad:
# print(name, param.data.shape)
model = model.to(device)
for epoch in range(epochs):
model.train()
loss_list = []
for batch, data in enumerate(train_dataloader):
subgraph, feats, labels = data
subgraph.set_n_initializer(dgl.init.zero_initializer)
subgraph.set_e_initializer(dgl.init.zero_initializer)
feats = feats.to(device)
labels = labels.to(device)
if fw == 'dgl':
model.g = subgraph
for layer in model.layers:
layer.g = subgraph
logits = model(feats.float())
else:
if net_class in [PGCN, PGAT, GGN]:
data = Data(x=feats.float(),
edge_index=torch.stack(
subgraph.edges()).to(device))
else:
data = Data(
x=feats.float(),
edge_index=torch.stack(
subgraph.edges()).to(device),
edge_type=subgraph.edata['rel_type'].squeeze().to(
device))
logits = model(data)
loss = loss_fcn(logits[getMaskForBatch(subgraph)],
labels.float())
optimizer.zero_grad()
a = list(model.parameters())[0].clone()
loss.backward()
optimizer.step()
b = list(model.parameters())[0].clone()
not_learning = torch.equal(a.data, b.data)
if not_learning:
print('Not learning')
# sys.exit(1)
else:
pass
# print('Diff: ', (a.data-b.data).sum())
# print(loss.item())
loss_list.append(loss.item())
loss_data = np.array(loss_list).mean()
print('Loss: {}'.format(loss_data))
if epoch % 5 == 0:
if epoch % 5 == 0:
print(
"Epoch {:05d} | Loss: {:.4f} | Patience: {} | ".format(
epoch, loss_data, cur_step),
end='')
score_list = []
val_loss_list = []
for batch, valid_data in enumerate(valid_dataloader):
subgraph, feats, labels = valid_data
subgraph.set_n_initializer(dgl.init.zero_initializer)
subgraph.set_e_initializer(dgl.init.zero_initializer)
feats = feats.to(device)
labels = labels.to(device)
score, val_loss = evaluate(feats.float(), model, subgraph,
labels.float(), loss_fcn, fw,
net_class)
score_list.append(score)
val_loss_list.append(val_loss)
mean_score = np.array(score_list).mean()
mean_val_loss = np.array(val_loss_list).mean()
if epoch % 5 == 0:
print("Score: {:.4f} MEAN: {:.4f} BEST: {:.4f}".format(
mean_score, mean_val_loss, best_loss))
# early stop
if best_loss > mean_val_loss or best_loss < 0:
best_loss = mean_val_loss
# Save the model
# print('Writing to', trial_s)
torch.save(model.state_dict(), fw + str(net) + '.tch')
params = [
val_loss, graph_type,
str(type(net_class)), g, num_layers, num_feats,
num_hidden, n_classes, heads, F.elu, in_drop,
attn_drop, alpha, residual, num_rels, freeze
]
pickle.dump(params, open(fw + str(net) + '.prms', 'wb'))
cur_step = 0
else:
cur_step += 1
if cur_step >= patience:
break
torch.save(model, 'gattrial.pth')
test_score_list = []
for batch, test_data in enumerate(test_dataloader):
subgraph, feats, labels = test_data
subgraph.set_n_initializer(dgl.init.zero_initializer)
subgraph.set_e_initializer(dgl.init.zero_initializer)
feats = feats.to(device)
labels = labels.to(device)
test_score_list.append(
evaluate(feats, model, subgraph, labels.float(), loss_fcn, fw,
net_class)[0])
print("F1-Score: {:.4f}".format(np.array(test_score_list).mean()))
model.eval()
return best_loss
edges = edges + [(v, u) for (u, v) in edges]
edges = edges + [(u, u) for u in range(n)]
word_vectors = torch.tensor(graph['vectors']).cuda()
word_vectors = F.normalize(word_vectors)
fcfile = json.load(open('materials/fc-weights.json', 'r'))
train_wnids = [x[0] for x in fcfile]
fc_vectors = [x[1] for x in fcfile]
assert train_wnids == wnids[:len(train_wnids)]
fc_vectors = torch.tensor(fc_vectors).cuda()
fc_vectors = F.normalize(fc_vectors)
hidden_layers = args.layers #'d2048,d' #'2048,2048,1024,1024,d512,d'
gcn = GCN(n, edges, word_vectors.shape[1], fc_vectors.shape[1], hidden_layers, args.norm_method).cuda()
print('{} nodes, {} edges'.format(n, len(edges)))
print('word vectors:', word_vectors.shape)
print('fc vectors:', fc_vectors.shape)
print('hidden layers:', hidden_layers)
optimizer = torch.optim.Adam(gcn.parameters(), lr=args.lr, weight_decay=args.weight_decay)
v_train, v_val = map(float, args.trainval.split(','))
n_trainval = len(fc_vectors)
n_train = round(n_trainval * (v_train / (v_train + v_val)))
print('num train: {}, num val: {}'.format(n_train, n_trainval - n_train))
tlist = list(range(len(fc_vectors)))
random.shuffle(tlist)
# data loading:
dataset = VisDaDataset(im_size=args.img_size)
dataloader = data.DataLoader(dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=8)
# setup evaluator:
evaluator = Evaluator(mode="cityscapes",
samples=25,
metrics=["miou"],
crf=False)
# setup model:
if args.model == "GCN":
model = GCN(dataset.num_classes, dataset.img_size, k=args.K).cuda()
elif args.model == "UNet":
model = UNet(dataset.num_classes).cuda()
else:
raise ValueError("Invalid model arg.")
model.train()
if args.resume:
assert os.path.exists(os.path.join(paths["project_path"], "saves"))
resume_path = save_path.format(args.resume_epoch)
model.load_state_dict(torch.load(resume_path))
start_epoch = args.resume_epoch
else:
start_epoch = 0
# setup loss and optimizer
def train(G, train_data, valid_data, test_data, args):
# read data
nodes_all = args.user_num + args.item_num
id_map = construct_id_map(nodes_all)
feats = preprocess_features(nodes_all)
num_supports = 1
placeholders = {
'batch1': tf.placeholder(tf.int32, shape=(None), name='batch1'),
'batch2': tf.placeholder(tf.int32, shape=(None), name='batch2'),
'batch3': tf.placeholder(tf.int32, shape=(None), name='batch3'),
'batch4': tf.placeholder(tf.int32, shape=(None), name='batch4'),
'feats': tf.sparse_placeholder(tf.float32),
'support':
[tf.sparse_placeholder(tf.float32) for _ in range(num_supports)],
'dropout': tf.placeholder_with_default(0., shape=(), name='dropout'),
'batch_size': tf.placeholder(tf.int32, name='batch_size'),
}
context_pairs = load_walks(train_data)
minibatch = EdgeMinibatchIterator(G,
id_map,
placeholders,
batch_size=args.batch_size,
max_degree=args.max_degree,
context_pairs=context_pairs)
adjs = load_adj(train_data, nodes_all)
support = [preprocess_adj(adjs)]
adj_info_ph = tf.placeholder(tf.int32, shape=minibatch.adj.shape)
adj_info = tf.Variable(adj_info_ph, trainable=False, name="adj_info")
print("build model....")
if args.input == './data/ml-100k/':
model = GCN(placeholders,
input_dim=feats[2][1],
embedding_dim=args.dim,
lr=args.learning_rate,
args=args,
logging=True)
else:
raise Exception('Error: data cannot evaluated')
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
sess = tf.Session(config=config)
# tensorboard
summary_writer = tf.summary.FileWriter("logs/", sess.graph)
sess.run(init_op, feed_dict={adj_info_ph: minibatch.adj})
train_adj_info = tf.assign(adj_info, minibatch.adj)
# training
t1 = time.time()
q_1_dict, mask = load_item_pop(train_data)
# DFS for each node to generate markov chain
print("generating markov chain by DFS......")
tt = time.time()
candidates = candidate_choose(G, mask, args)
print("time for generating negative examples", time.time() - tt)
N_steps = 10
N_negs = 1
best_mrr = 0
for epoch in range(args.epochs):
print("epoch %d" % epoch)
data_loader = Data_Loader(args.batch_size)
data_loader.load_train_data(train_data)
data_loader.reset_pointer()
for it in range(data_loader.num_batch):
batch_data = data_loader.next_batch()
node1 = [x[0] for x in batch_data]
node2 = [x[1] for x in batch_data]
# generate negative examples with MCNS
t0 = time.time()
if it == 0:
start_given = None
else:
start_given = generate_examples
generate_examples = negative_sampling(model, sess, candidates,
start_given, q_1_dict,
N_steps, N_negs, node1,
node2, args, support, feats,
placeholders)
# update model params
feed_dict = {
model.inputs1: node1,
model.inputs2: node2,
model.neg_samples: generate_examples,
model.batch_size: args.batch_size,
model.number: args.batch_size,
model.inputs: (feats[0], feats[1], feats[2])
}
feed_dict.update({
placeholders['support'][i]: support[i]
for i in range(len(support))
})
outs = sess.run([
model.merged_loss, model.merged_mrr, model.loss, model.mrr,
model.opt_op, model.outputs1, model.outputs2, model.neg_outputs
],
feed_dict=feed_dict)
# add_summary for tensorboard show
if it % args.print_step == 0:
summary_writer.add_summary(outs[0],
epoch * data_loader.num_batch + it)
summary_writer.add_summary(outs[1],
epoch * data_loader.num_batch + it)
if it % args.validate_iter == 0:
t2 = time.time()
val_cost, ranks, val_mrr, duration = evaluate(
sess,
model,
placeholders,
minibatch,
candidates,
q_1_dict,
N_steps,
N_negs,
valid_data,
support,
feats,
size=args.validate_batch_size)
print("evaluate time", time.time() - t2)
if it % args.print_step == 0:
print("model model", "Iter:", '%04d' % it, "d_loss=",
"{:.5f}".format(outs[2]), "d_mrr=",
"{:.5f}".format(outs[3]))
print("validation model", "Iter:", '%04d' % it, "val_loss=",
"{:.5f}".format(val_cost), "val_mrr=",
"{:.5f}".format(val_mrr))
# validation for early stopping......
val_cost, ranks, val_mrr, duration = evaluate(
sess,
model,
placeholders,
minibatch,
candidates,
q_1_dict,
N_steps,
N_negs,
valid_data,
support,
feats,
size=args.validate_batch_size)
curr_mrr = val_mrr
if curr_mrr > best_mrr:
best_mrr = curr_mrr
patience = 0
else:
patience += 1
if patience > args.patience:
print("Early Stopping...")
break
# save model embeddings for downstream task
save_embeddings(sess, model, minibatch, args.validate_batch_size, support,
feats, placeholders, args.save_dir)
print("training complete......")
# test for recommendation......
# mrr, hit30 = recommend(test_data, args)
mrr, hit30 = recommend(train_data, valid_data, test_data, args)
print("test_mrr=", "{:.5f}".format(mrr), "test_hit30=",
"{:.5f}".format(hit30))
def __init__(self,config,D=3): ME.MinkowskiNetwork.__init__(self, D) NORM_TYPE = self.NORM_TYPE BLOCK_NORM_TYPE = self.BLOCK_NORM_TYPE CHANNELS = self.CHANNELS TR_CHANNELS = self.TR_CHANNELS bn_momentum = config.bn_momentum self.normalize_feature = config.normalize_feature self.voxel_size = config.voxel_size self.conv1 = ME.MinkowskiConvolution( in_channels=config.in_feats_dim, out_channels=CHANNELS[1], kernel_size=config.conv1_kernel_size, stride=1, dilation=1, bias=False, dimension=D) self.norm1 = get_norm(NORM_TYPE, CHANNELS[1], bn_momentum=bn_momentum, D=D) self.block1 = get_block( BLOCK_NORM_TYPE, CHANNELS[1], CHANNELS[1], bn_momentum=bn_momentum, D=D) self.conv2 = ME.MinkowskiConvolution( in_channels=CHANNELS[1], out_channels=CHANNELS[2], kernel_size=3, stride=2, dilation=1, bias=False, dimension=D) self.norm2 = get_norm(NORM_TYPE, CHANNELS[2], bn_momentum=bn_momentum, D=D) self.block2 = get_block( BLOCK_NORM_TYPE, CHANNELS[2], CHANNELS[2], bn_momentum=bn_momentum, D=D) self.conv3 = ME.MinkowskiConvolution( in_channels=CHANNELS[2], out_channels=CHANNELS[3], kernel_size=3, stride=2, dilation=1, bias=False, dimension=D) self.norm3 = get_norm(NORM_TYPE, CHANNELS[3], bn_momentum=bn_momentum, D=D) self.block3 = get_block( BLOCK_NORM_TYPE, CHANNELS[3], CHANNELS[3], bn_momentum=bn_momentum, D=D) self.conv4 = ME.MinkowskiConvolution( in_channels=CHANNELS[3], out_channels=CHANNELS[4], kernel_size=3, stride=2, dilation=1, bias=False, dimension=D) self.norm4 = get_norm(NORM_TYPE, CHANNELS[4], bn_momentum=bn_momentum, D=D) self.block4 = get_block( BLOCK_NORM_TYPE, CHANNELS[4], CHANNELS[4], bn_momentum=bn_momentum, D=D) # adapt input tensor here self.conv4_tr = ME.MinkowskiConvolutionTranspose( in_channels = config.gnn_feats_dim + 2, out_channels=TR_CHANNELS[4], kernel_size=3, stride=2, dilation=1, bias=False, dimension=D) self.norm4_tr = get_norm(NORM_TYPE, TR_CHANNELS[4], bn_momentum=bn_momentum, D=D) self.block4_tr = get_block( BLOCK_NORM_TYPE, TR_CHANNELS[4], TR_CHANNELS[4], bn_momentum=bn_momentum, D=D) self.conv3_tr = ME.MinkowskiConvolutionTranspose( in_channels=CHANNELS[3] + TR_CHANNELS[4], out_channels=TR_CHANNELS[3], kernel_size=3, stride=2, dilation=1, bias=False, dimension=D) self.norm3_tr = get_norm(NORM_TYPE, TR_CHANNELS[3], bn_momentum=bn_momentum, D=D) self.block3_tr = get_block( BLOCK_NORM_TYPE, TR_CHANNELS[3], TR_CHANNELS[3], bn_momentum=bn_momentum, D=D) self.conv2_tr = ME.MinkowskiConvolutionTranspose( in_channels=CHANNELS[2] + TR_CHANNELS[3], out_channels=TR_CHANNELS[2], kernel_size=3, stride=2, dilation=1, bias=False, dimension=D) self.norm2_tr = get_norm(NORM_TYPE, TR_CHANNELS[2], bn_momentum=bn_momentum, D=D) self.block2_tr = get_block( BLOCK_NORM_TYPE, TR_CHANNELS[2], TR_CHANNELS[2], bn_momentum=bn_momentum, D=D) self.conv1_tr = ME.MinkowskiConvolution( in_channels=CHANNELS[1] + TR_CHANNELS[2], out_channels=TR_CHANNELS[1], kernel_size=1, stride=1, dilation=1, bias=False, dimension=D) self.final = ME.MinkowskiConvolution( in_channels=TR_CHANNELS[1], out_channels=config.out_feats_dim + 2, kernel_size=1, stride=1, dilation=1, bias=True, dimension=D) ############# # Overlap attention module self.epsilon = torch.nn.Parameter(torch.tensor(-5.0)) self.bottle = nn.Conv1d(CHANNELS[4], config.gnn_feats_dim,kernel_size=1,bias=True) self.gnn = GCN(config.num_head,config.gnn_feats_dim, config.dgcnn_k, config.nets) self.proj_gnn = nn.Conv1d(config.gnn_feats_dim,config.gnn_feats_dim,kernel_size=1, bias=True) self.proj_score = nn.Conv1d(config.gnn_feats_dim,1,kernel_size=1,bias=True)
if __name__ == "__main__":
version = yaml.load(open("eval.yaml", 'r'))["version"]
args = load_args(os.path.join(os.getcwd(), "saves{}".format(version)),
eval_path=os.getcwd())
trained_epochs = args.eval["epochs"]
save_path = os.path.join(args.paths["project_path"],
"saves{}".format(args.eval["version"]),
"{}-{}.pth".format(args.model, trained_epochs))
dataset = TestDataset()
if args.model == "GCN":
model = GCN(cityscapes.num_classes, args.img_size, k=args.K).cuda()
elif args.model == "GCN_DENSENET":
model = GCN_DENSENET(cityscapes.num_classes, args.img_size,
k=args.K).cuda()
elif args.model == "GCN_DECONV":
model = GCN_DECONV(cityscapes.num_classes, args.img_size,
k=args.K).cuda()
elif args.model == "GCN_PSP":
model = GCN_PSP(cityscapes.num_classes, args.img_size, k=args.K).cuda()
elif args.model == "GCN_COMB":
model = GCN_COMBINED(cityscapes.num_classes, args.img_size).cuda()
elif args.model == "GCN_RESNEXT":
model = GCN_RESNEXT(dataset.num_classes, k=self.args.K).cuda()
elif args.model == "UNet":
model = UNet(cityscapes.num_classes).cuda()
elif args.model == "PSPNet":
def main():
global args
args = parser.parse_args()
cfg = load_config(args.config)
device = torch.device("cuda:0" if not args.no_cuda else "cpu")
if args.mode == 'train':
exp_name = 'gcn' + '_old_wc1_small_bs_4_lr_1e-2_test'
tb_writer = SummaryWriter(tb_log_dir + exp_name)
train_loader, val_loader = build_data_loader(args.mode)
model = GCN(cfg).to(device)
optimizer = optim.Adam(model.parameters(),
args.learning_rate, (0.9, 0.999),
eps=1e-08)
#optimizer = optim.SGD(model.parameters(), args.learning_rate, momentum=0.9)
scheduler = optim.lr_scheduler.StepLR(optimizer,
step_size=50,
gamma=0.5)
total_tb_it = 0
best_val_loss = 1000
for epoch in range(args.epochs):
scheduler.step(epoch)
total_tb_it = train(cfg, model, train_loader, device, optimizer,
epoch, tb_writer, total_tb_it)
val_loss = validate(cfg, model, val_loader, device, tb_writer,
total_tb_it)
if val_loss <= best_val_loss:
best_val_loss = val_loss
name = checkpoint_dir + exp_name + '_model.pkl'
state = {
'epoch': epoch,
'model_state': model.state_dict(),
'optimizer_state': optimizer.state_dict()
}
torch.save(state, name)
tb_writer.close()
elif args.mode == 'predict':
print('Load data...')
test_loader = build_data_loader(args.mode)
print('Start predicting...')
model = GCN(cfg).to(device)
model.load_state_dict(torch.load(args.resume_file)['model_state'])
#summary(model, [(200, 1), (200, 200)])
test(cfg, model, test_loader, device)
else:
raise Exception("Unrecognized mode.")
gpus = tf.config.experimental.list_physical_devices('GPU')
if len(gpus) == 0 or FLAGS.gpu_id is None:
device_id = "/device:CPU:0"
else:
tf.config.experimental.set_visible_devices(gpus[FLAGS.gpu_id], 'GPU')
device_id = '/device:GPU:0'
A_mat, X_mat, z_vec, train_idx, val_idx, test_idx = load_data_planetoid(
FLAGS.dataset)
An_mat = preprocess_graph(A_mat)
# N = A_mat.shape[0]
K = z_vec.max() + 1
with tf.device(device_id):
gcn = GCN(An_mat, X_mat, [FLAGS.hidden1, K])
train_stats = gcn.train(train_idx, z_vec[train_idx], val_idx,
z_vec[val_idx])
train_losses = train_stats[0]
val_losses = train_stats[1]
train_accuracies = train_stats[2]
val_accuracies = train_stats[3]
with open("learned_lapl.pkl", "rb") as pkl:
lrnd = pickle.load(pkl)
with open("fixed_lapl.pkl", "rb") as pkl:
fxd = pickle.load(pkl)
plt.figure()
smooth_plot(fxd['train_losses'], label='Fixed Train Loss')