def gcn_chebyshev(v_size, F, N, n_classes, support):
x_in = Input(shape=(F, ))
g = [Input((N, ), sparse=True) for i in range(support)]
# transformer
embedding = Embedding(v_size, emb_dim)
e = embedding(x_in)
h = Attention(8, 16)([e, e, e])
h = GlobalAveragePooling1D()(h)
# gcn, support=1
out = GraphConvolution(n_classes, support, activation='sigmoid')([h] + g)
model = Model(inputs=[x_in] + g, outputs=out)
model.compile(loss='binary_crossentropy', optimizer='adam')
return model
def gcn(N, ):
x_in = Input(shape=(maxlen, embed_dim))
a_in = Input(shape=(maxlen, embed_dim))
g = Input((batch_size, ), sparse=True)
x = x_in
a = a_in
h = Attention(8, 16)([a, x, x])
h = GlobalAveragePooling1D()(h)
# gcn, support=1
h = GraphConvolution(h_dim, 1, activation='relu')([h, g])
#h = Dropout(0.2)(h)
#out = GraphConvolution(3, 1, activation='softmax')([h,g])
out = Dense(3, activation='softmax')(h)
model = Model(inputs=[x_in, a_in, g], outputs=out)
model.compile(loss='categorical_crossentropy', optimizer='adam')
return model
G = [
Input(shape=(None, None), batch_shape=(None, None), sparse=True)
for _ in range(support)
]
else:
raise Exception('Invalid filter type.')
X_in = Input(shape=(X.shape[1], ))
# Define model architecture
# NOTE: We pass arguments for graph convolutional layers as a list of tensors.
# This is somewhat hacky, more elegant options would require rewriting the Layer base class.
H = Dropout(0.5)(X_in)
H = GraphConvolution(16,
support,
activation='relu',
kernel_regularizer=l2(5e-4))([H] + G)
H = Dropout(0.5)(H)
Y = GraphConvolution(y.shape[1], support, activation='softmax')([H] + G)
# Compile model
model = Model(inputs=[X_in] + G, outputs=Y)
model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.01))
# Helper variables for main training loop
wait = 0
preds = None
best_val_loss = 99999
# Fit
for epoch in range(1, NB_EPOCH + 1):
def main():
# Parse command line arguments
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('--dataset',
default=DATASET,
choices=['cora'],
help='the data set to load')
arg_parser.add_argument('--filter',
default=FILTER,
choices=['localpool', 'chebyshev'],
help='the filter to use')
arg_parser.add_argument('--no-norm-symmetry',
dest='sym_norm',
action='store_false',
help='disable symmetric normalization')
arg_parser.add_argument('-e',
'--epochs',
type=int,
default=NB_EPOCH,
help='the number of epochs to train for')
arg_parser.add_argument('--patience',
type=int,
default=PATIENCE,
help='the patience for early-stopping')
arg_parser.add_argument('--max-degree',
type=int,
default=MAX_DEGREE,
help='maximum polynomial degree')
args = arg_parser.parse_args()
# Get data
X, A, y = load_data(dataset=args.dataset)
y_train, y_val, y_test, train_mask, val_mask, test_mask = get_splits(y)
# Normalize X
X = np.diag(1 / np.array(X.sum(1)).flatten()).dot(X)
if FILTER == 'localpool':
# Local pooling filters
# (see 'renormalization trick' in Kipf & Welling, arXiv 2016)
print('Using local pooling filters...')
A_ = preprocess_adj(A, args.sym_norm)
support = 1
graph = [X, A_]
G = [Input(shape=tuple(A_.shape[1:]), sparse=True)]
elif FILTER == 'chebyshev':
# Chebyshev polynomial basis filters (Defferard et al., NIPS 2016)
print('Using Chebyshev polynomial basis filters...')
L = normalized_laplacian(A, args.sym_norm)
L_scaled = rescale_laplacian(L)
T_k = chebyshev_polynomial(L_scaled, args.max_degree)
support = args.max_degree + 1
graph = [X] + T_k
G = [
Input(shape=tuple(T_k.shape[i + 1]), sparse=True)
for i in range(support)
]
else:
raise ValueError('Invalid filter type.')
X_in = Input(shape=(X.shape[1], ))
# Define model architecture
H = Dropout(0.5)(X_in)
H = GraphConvolution(16,
support,
activation='relu',
W_regularizer=l2(5e-4))([H] + G)
H = Dropout(0.5)(H)
Y = GraphConvolution(y.shape[1], support, activation='softmax')([H] + G)
# Compile model
model = Model(inputs=[X_in] + G, outputs=Y)
model.compile(loss='categorical_crossentropy',
optimizer=Adam(lr=0.01),
metrics=[accuracy_metric])
model.fit(graph,
y_train,
sample_weight=train_mask,
validation_data=(graph, y_val, val_mask),
batch_size=A.shape[0],
epochs=args.epochs,
shuffle=False,
verbose=1)
# Testing
test_loss, test_acc = model.evaluate(graph,
y_test,
sample_weight=test_mask,
batch_size=A.shape[0])
print("Test set results:", "loss= {}".format(test_loss),
"accuracy= {}".format(test_acc))
def tkipf_training(X_, A_, y, y_train, y_val, y_test, idx_train, idx_val, idx_test, train_mask):
# Define model architecture
# NOTE: We pass arguments for graph convolutional layers as a list of tensors.
# This is somewhat hacky, more elegant options would require rewriting the Layer base class.
# Choosing filter
if FILTER == 'localpool':
""" Local pooling filters (see 'renormalization trick' in Kipf & Welling, arXiv 2016) """
if args.log:
print('\tUsing local pooling filters...')
support = 1
graph = [X_, A_]
G = [Input(shape=(None, None), batch_shape=(None, None), sparse=True)]
elif FILTER == 'chebyshev':
""" Chebyshev polynomial basis filters (Defferard et al., NIPS 2016) """
#ATTENZIONE A globale e non preprocessata
print('Using Chebyshev polynomial basis filters...')
L = normalized_laplacian(A, SYM_NORM)
L_scaled = rescale_laplacian(L)
T_k = chebyshev_polynomial(L_scaled, MAX_DEGREE)
support = MAX_DEGREE + 1
graph = [X_] + T_k
G = [Input(shape=(None, None), batch_shape=(None, None), sparse=True) for _ in range(support)]
else:
raise Exception('Invalid filter type.')
# Model architecure
X_in = Input(shape=(X_.shape[1],))
H = Dropout(0.5)(X_in)
H = GraphConvolution(16, support, activation='relu', kernel_regularizer=l2(5e-4))([H]+G)
H = Dropout(0.5)(H)
Y = GraphConvolution(y.shape[1], support, activation='softmax')([H]+G)
model = Model(inputs=[X_in] + G, outputs=Y)
model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.01))
# Helper variables for k_fold loop
wait = 0
preds = None
best_val_loss = 99999
t_fit = time.time()
for epoch in range(1, NB_EPOCH + 1):
# Log wall-clock time
t = time.time()
# Single training iteration (we mask nodes without labels for loss calculation)
model.fit(graph, y_train, sample_weight=train_mask,
batch_size=A_.shape[0], epochs=1, shuffle=False, verbose=0)
if epoch == 1 and args.log:
print("tempo per fit [1° epoch] = "+ str(time.time()-t))
print("Pesi iniziali [1° epoch]")
print(model.get_weights()[0][0])
# Predict on full dataset
preds = model.predict(graph, batch_size=A_.shape[0])
# Train / validation scores
train_val_loss, train_val_acc = evaluate_preds(preds, [y_train, y_val],
[idx_train, idx_val])
if args.log:
print("Epoch: {:04d}".format(epoch),
"train_loss= {:.4f}".format(train_val_loss[0]),
"train_acc= {:.4f}".format(train_val_acc[0]),
"val_loss= {:.4f}".format(train_val_loss[1]),
"val_acc= {:.4f}".format(train_val_acc[1]),
"time= {:.4f}".format(time.time() - t))
# Early stopping
if train_val_loss[1] < best_val_loss:
best_val_loss = train_val_loss[1]
wait = 0
else:
if wait >= PATIENCE :
if args.log:
print('\tEpoch {}: early stopping'.format(epoch))
break
wait += 1
# Testing
if args.log:
print("Pesi finali [1° epoch]")
print(model.get_weights()[0][0])
print("time to complete fit : " + str(time.time() - t_fit))
test_loss, test_acc = evaluate_preds(preds, [y_test], [idx_test])
if args.log:
print("\nTest set results:",
"loss= {:.4f}".format(test_loss[0]),
"accuracy= {:.4f}".format(test_acc[0]))
clear_session() #Pulisco la sessione tensorflow --> fix per il tempo incrementale sul fit.
return test_loss[0], test_acc[0]
def main(dataset, filt, max_degree, sym_norm, nb_epoch, patience):
# Get data
X, A, y = load_data(dataset=dataset)
y_train, y_val, y_test, idx_train, idx_val, idx_test, train_mask = get_splits(
y, 0.35, 0.45)
# Normalize X
X /= X.sum(1).reshape(-1, 1)
if filt == 'localpool':
""" Local pooling filters (see 'renormalization trick' in Kipf & Welling, arXiv 2016) """
print('Using local pooling filters...')
A_ = preprocess_adj(A, sym_norm)
support = 1
graph = [X, A_]
G = [Input(shape=(None, None), batch_shape=(None, None), sparse=True)]
elif filt == 'chebyshev':
""" Chebyshev polynomial basis filters (Defferard et al., NIPS 2016) """
print('Using Chebyshev polynomial basis filters...')
L = normalized_laplacian(A, sym_norm)
L_scaled = rescale_laplacian(L)
T_k = chebyshev_polynomial(L_scaled, max_degree)
support = max_degree + 1
graph = [X] + T_k
G = [
Input(shape=(None, None), batch_shape=(None, None), sparse=True)
for _ in range(support)
]
else:
raise Exception('Invalid filter type.')
X_in = Input(shape=(X.shape[1], ))
# Define model architecture
# NOTE: We pass arguments for graph convolutional layers as a list of tensors.
# This is somewhat hacky, more elegant options would require rewriting the Layer base class.
H = Dropout(0.5)(X_in)
H = GraphConvolution(16,
support,
activation='relu',
kernel_regularizer=l2(5e-4))([H] + G)
H = Dropout(0.5)(H)
Y = GraphConvolution(y.shape[1], support, activation='softmax')([H] + G)
# Compile model
model = Model(inputs=[X_in] + G, outputs=Y)
model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.01))
# Helper variables for main training loop
wait = 0
preds = None
best_val_loss = 99999
# Fit
for epoch in range(1, nb_epoch + 1):
# Log wall-clock time
t = time.time()
# Single training iteration (we mask nodes without labels for loss calculation)
model.fit(graph,
y_train,
sample_weight=train_mask,
batch_size=A.shape[0],
epochs=1,
shuffle=False,
verbose=0)
# Predict on full dataset
preds = model.predict(graph, batch_size=A.shape[0])
# Train / validation scores
train_val_loss, train_val_acc = evaluate_preds(preds, [y_train, y_val],
[idx_train, idx_val])
print("Epoch: {:04d}".format(epoch),
"train_loss= {:.4f}".format(train_val_loss[0]),
"train_acc= {:.4f}".format(train_val_acc[0]),
"val_loss= {:.4f}".format(train_val_loss[1]),
"val_acc= {:.4f}".format(train_val_acc[1]),
"time= {:.4f}".format(time.time() - t))
# Early stopping
if train_val_loss[1] < best_val_loss:
best_val_loss = train_val_loss[1]
wait = 0
else:
if wait >= patience:
print('Epoch {}: early stopping'.format(epoch))
break
wait += 1
# Testing
test_loss, test_acc = evaluate_preds(preds, [y_test], [idx_test])
print("Test set results:", "loss= {:.4f}".format(test_loss[0]),
"accuracy= {:.4f}".format(test_acc[0]))
def run(param_dict):
param_dict = keras_cmdline.fill_missing_defaults(augment_parser,
param_dict)
optimizer = keras_cmdline.return_optimizer(param_dict)
pprint(param_dict)
EPOCHS = param_dict['epochs']
FILTER = param_dict['filter']
MAX_DEGREE = param_dict['max_degree']
SYM_NORM = param_dict['sys_norm']
DROPOUT = param_dict['dropout']
NUNITS = param_dict['nunits']
ACTIVATION = param_dict['activation']
BATCH_SIZE = param_dict['batch_size']
TIMEOUT = param_dict['timeout']
#SHARE_WEIGHTS = param_dict['share_weights']
# Define parameters
DATASET = 'cora'
#FILTER = 'localpool' # 'chebyshev'
#MAX_DEGREE = 2 # maximum polynomial degree
#SYM_NORM = True # symmetric (True) vs. left-only (False) normalization
PATIENCE = 10 # early stopping patience
# Get data
timer.start('stage in')
if param_dict['data_source']:
data_source = param_dict['data_source']
else:
data_source = os.path.dirname(os.path.abspath(__file__))
data_source = os.path.join(data_source, 'data/cora')
paths = util.stage_in(['cora.content', 'cora.cites'],
source=data_source,
dest=param_dict['stage_in_destination'])
path_content = paths['cora.content']
path_cites = paths['cora.cites']
idx_features_labels = np.genfromtxt(path_content, dtype=np.dtype(str))
features = sp.csr_matrix(idx_features_labels[:, 1:-1], dtype=np.float32)
labels = encode_onehot(idx_features_labels[:, -1])
# build graph
idx = np.array(idx_features_labels[:, 0], dtype=np.int32)
idx_map = {j: i for i, j in enumerate(idx)}
edges_unordered = np.genfromtxt(path_cites, dtype=np.int32)
edges = np.array(list(map(idx_map.get, edges_unordered.flatten())),
dtype=np.int32).reshape(edges_unordered.shape)
adj = sp.coo_matrix((np.ones(edges.shape[0]), (edges[:, 0], edges[:, 1])),
shape=(labels.shape[0], labels.shape[0]),
dtype=np.float32)
# build symmetric adjacency matrix
adj = adj + adj.T.multiply(adj.T > adj) - adj.multiply(adj.T > adj)
print('Dataset has {} nodes, {} edges, {} features.'.format(
adj.shape[0], edges.shape[0], features.shape[1]))
X, A, y = features.todense(), adj, labels
timer.end()
timer.start('preprocessing')
y_train, y_val, y_test, idx_train, idx_val, idx_test, train_mask = get_splits(
y)
# Normalize X
X /= X.sum(1).reshape(-1, 1)
if FILTER == 'localpool':
""" Local pooling filters (see 'renormalization trick' in Kipf & Welling, arXiv 2016) """
print('Using local pooling filters...')
A_ = preprocess_adj(A, SYM_NORM)
support = 1
graph = [X, A_]
G = [Input(shape=(None, None), batch_shape=(None, None), sparse=True)]
elif FILTER == 'chebyshev':
""" Chebyshev polynomial basis filters (Defferard et al., NIPS 2016) """
print('Using Chebyshev polynomial basis filters...')
L = normalized_laplacian(A, SYM_NORM)
L_scaled = rescale_laplacian(L)
T_k = chebyshev_polynomial(L_scaled, MAX_DEGREE)
support = MAX_DEGREE + 1
graph = [X] + T_k
G = [
Input(shape=(None, None), batch_shape=(None, None), sparse=True)
for _ in range(support)
]
else:
raise Exception('Invalid filter type.')
model_path = param_dict['model_path']
model_mda_path = None
model = None
initial_epoch = 0
if model_path:
custom_objects = {'GraphConvolution': GraphConvolution}
savedModel = util.resume_from_disk(BNAME,
param_dict,
data_dir=model_path,
custom_objects=custom_objects)
model_mda_path = savedModel.model_mda_path
model_path = savedModel.model_path
model = savedModel.model
initial_epoch = savedModel.initial_epoch
if model is None:
X_in = Input(shape=(X.shape[1], ))
# Define model architecture
# NOTE: We pass arguments for graph convolutional layers as a list of tensors.
# This is somewhat hacky, more elegant options would require rewriting the Layer base class.
H = Dropout(DROPOUT)(X_in)
H = GraphConvolution(NUNITS,
support,
activation=ACTIVATION,
kernel_regularizer=l2(5e-4))([H] + G)
H = Dropout(DROPOUT)(H)
Y = GraphConvolution(y.shape[1], support,
activation='softmax')([H] + G)
# Compile model
model = Model(inputs=[X_in] + G, outputs=Y)
model.compile(loss='categorical_crossentropy', optimizer=optimizer)
# Helper variables for main training loop
wait = 0
preds = None
best_val_loss = 99999
timer.end()
#earlystop = EarlyStopping(monitor='val_acc', min_delta=0.0001, patience=50, verbose=1, mode='auto')
timeout_monitor = TerminateOnTimeOut(TIMEOUT)
callbacks_list = [timeout_monitor]
# Fit
training_timer = util.Timer()
training_timer.start('model training')
prev_val_acc = 0
count = 0
patience = 50
delta = 0.0001
for epoch in range(initial_epoch, EPOCHS):
# Log wall-clock time
timer.start(f'epoch {epoch}')
# Single training iteration (we mask nodes without labels for loss calculation)
model.fit(graph,
y_train,
sample_weight=train_mask,
batch_size=A.shape[0],
epochs=1,
shuffle=False,
verbose=0)
# Predict on full dataset
preds = model.predict(graph, batch_size=A.shape[0]) #
# Train / validation scores
train_val_loss, train_val_acc = evaluate_preds(preds, [y_train, y_val],
[idx_train, idx_val])
print("Epoch: {:04d}".format(epoch),
"train_loss= {:.4f}".format(train_val_loss[0]),
"train_acc= {:.4f}".format(train_val_acc[0]),
"val_loss= {:.4f}".format(train_val_loss[1]),
"val_acc= {:.4f}".format(train_val_acc[1]))
timer.end()
diff = abs(prev_val_acc - train_val_acc[1])
#print(diff)
if diff > delta:
prev_val_acc = train_val_acc[1]
count = 0
else:
count = count + 1
if count >= patience:
print('Early stopping')
break
elapsed = time.time() - training_timer.t0
if elapsed >= TIMEOUT * 60:
print(' - timeout: training time = %2.3fs/%2.3fs' %
(elapsed, TIMEOUT * 60))
break
training_timer.end()
# Testing
test_loss, test_acc = evaluate_preds(preds, [y_test], [idx_test])
print("Test set results:", "loss= {:.4f}".format(test_loss[0]),
"accuracy= {:.4f}".format(test_acc[0]))
print('===Validation accuracy:', test_acc[0])
print('OUTPUT:', -test_acc[0])
if model_path:
timer.start('model save')
model.save(model_path)
util.save_meta_data(param_dict, model_mda_path)
timer.end()
return -test_acc[0]