def _init_parameters(self):
# Init embeddings
self._initial_embeddings = {
v: tf.get_variable(initializer=tf.random_normal((1, d)),
dtype=self.float_dtype,
name='{v}_init'.format(v=v))
for (v, d) in self.var.items()
}
# Init LSTM cells
self._RNN_cells = {
v: tf.contrib.rnn.LayerNormBasicLSTMCell(
d, activation=self.Cell_activation)
for (v, d) in self.var.items()
}
# Init message-computing MLPs
self._msg_MLPs = {
msg: Mlp(layer_sizes=[self.var[vin]] * self.MLP_depth +
[self.var[vout]],
activations=[self.Msg_activation] * self.MLP_depth +
[self.Msg_last_activation],
kernel_initializer=self.MLP_weight_initializer(),
bias_initializer=self.MLP_weight_initializer(),
name=msg,
name_internal_layers=True)
for msg, (vin, vout) in self.msg.items()
}
def _init_parameters(self):
# Init embeddings
self._tf_inits = {}
for v, d in self.var.items():
self._tf_inits[v] = tf.get_variable(name="{}_init".format(v),
initializer=tf.random_normal(
[1, d]),
dtype=self.float_dtype)
#end for
# Init LSTM cells
self._tf_cells = {}
for v, d in self.var.items():
self._tf_cells[v] = tf.contrib.rnn.LayerNormBasicLSTMCell(
d, activation=self.Cell_activation)
#end for
# Init Messages
self._tf_msgs = {}
for msg, vs in self.msg.items():
vin, vout = vs
self._tf_msgs[msg] = Mlp(
layer_sizes=[self.var[vin]
for _ in range(2)] + [self.var[vout]],
activations=[self.Msg_activation
for _ in range(2)] + [self.Msg_last_activation],
name=msg,
name_internal_layers=True,
kernel_initializer=self.MLP_weight_initializer(),
bias_initializer=self.MLP_bias_initializer())
#end for
return
def act(agent_id, obs, avail_actions):
avail_actions_ind = np.nonzero(avail_actions)[0]
nn_model = Mlp(avail_actions_ind.size)
action_index = np.argmax(nn_model(np.expand_dims(np.array(obs), axis=0)))
action = avail_actions_ind[action_index]
actions[agent_id] = action
def _init_parameters(self):
# Init embeddings
self._tf_inits = {}
for v, d in self.var.items():
self._tf_inits[v] = tf.get_variable("{}_init".format(v), [1, d],
dtype=tf.float32)
#end for
# Init LSTM cells
self._tf_cells = {}
for v, d in self.var.items():
self._tf_cells[v] = tf.contrib.rnn.LayerNormBasicLSTMCell(
d, activation=self.Cell_activation)
#end for
# Init Messages
self._tf_msgs = {}
for msg, vs in self.msg.items():
vin, vout = vs
self._tf_msgs[msg] = Mlp(
layer_sizes=[self.var[vin]
for _ in range(2)] + [self.var[vout]],
activations=[self.Msg_activation
for _ in range(2)] + [self.Msg_last_activation],
name=msg,
name_internal_layers=True,
kernel_initializer=self.MLP_weight_initializer(),
bias_initializer=self.MLP_bias_initializer())
#end for
# Init matrix MLPs
self.mat_MLP = {}
for M, vs in self.mat.items():
if vs["compute?"]:
v1, v2 = vs["vars"]
self.mat_MLP[M] = Mlp(
layer_sizes=[
max(self.var[v1], self.var[v2]) for _ in range(2)
] + [1],
activations=[self.Msg_activation for _ in range(2)] +
[self.Msg_last_activation],
name="mat_MLP_{}".format(M),
name_internal_layers=True,
kernel_initializer=self.MLP_weight_initializer(),
bias_initializer=self.MLP_bias_initializer())
#end
#end
return
def main():
#read dataset and preprocess it
dataset = PreProcessing("seeds_dataset.txt", separator='\s+')
dataset.normalize()
dataset.normalize_class()
#divide dataset into training and test sets
train, test = training.holdout(0.7, dataset.normalized_dataframe)
nn = Rbf(7, 3)
nn.train(train, eta=0.5, max_iterations=500)
print("RBF:", training.accuracy(nn, test, 3))
mm = Mlp(7, 3, 3)
mm.backpropagation(train.values.tolist(), max_iterations=500)
print("MLP:", training.accuracy(mm, test, 3))
def learn_with_gradient_testing(fname):
matFileContent = scipy.io.loadmat(fname) # corresponding MAT file
x_train = np.array(matFileContent['Xtrain'].tolist())
t_train = np.array(matFileContent['Ytrain'].flatten())
d = x_train.shape[1]
hidden_layers_list = [10]
mlp = Mlp(hidden_layers_list, d, True)
stopping_criterion = Mlp.BasicStoppingCriterion(0.05, 10)
error_data = mlp.train_network(x_train, t_train, x_train, t_train,
stopping_criterion)
print "Error data:"
error_data = map(lambda x: repr(x), error_data)
print reduce(lambda x, y: x+'\n'+y, error_data)
print "Log error:"
print mlp.get_input_error(x_train, t_train)
try:
x_test = np.array(matFileContent['Xtest'].tolist())
t_test = np.array(matFileContent['Ytest'].flatten())
print "Test log error:"
print mlp.get_input_error(x_test, t_test)
except:
pass
class TestMlp(unittest.TestCase):
def setUp(self):
self.mlp = Mlp(np.array([[1], [2]]), np.array([[1], [2]]), 12)
def test_weighted_sum_with_one_input(self):
# shape(1, 2) -> (1, 3) with bias added
inputs = np.array([[4, -3]])
# shape (3, 2) with added bias
weights = np.array([
[-1, 5], # Added bias weights
[1, 3],
[2, 4],
])
np.testing.assert_array_equal(
self.mlp._weighted_sum(inputs, weights),
np.array([[(BIAS * -1) + (4 * 1) + (-3 * 2),
(BIAS * 5) + (4 * 3) + (-3 * 4)]]))
def test_weighted_sum_with_two_inputs(self):
# shape(2, 3) -> (2, 4) with bias added
inputs = np.array([[4, -3, 1], [2, -1, 2]])
# shape (4, 2) with added bias
weights = np.array([
[-1, 5], # Added bias weights
[1, 3],
[2, 4],
[1, 2],
])
np.testing.assert_array_equal(
self.mlp._weighted_sum(inputs, weights),
np.array([
[(BIAS * -1) + (4 * 1) + (-3 * 2) + (1 * 1),
(BIAS * 5) + (4 * 3) + (-3 * 4) + (1 * 2)],
[(BIAS * -1) + (2 * 1) + (-1 * 2) + (2 * 1),
(BIAS * 5) + (2 * 3) + (-1 * 4) + (2 * 2)],
]))
def run(self):
start_time = time.time()
# print(f'thread {threading.current_thread().name} starts')
# avail_actions = env.get_avail_agent_actions(self.agent_id)
avail_actions_ind = np.nonzero(self.avail_actions)[0]
# obs = env.get_obs_agent(self.agent_id)
nn_model = Mlp(avail_actions_ind.size)
action_index = np.argmax(
nn_model(np.expand_dims(np.array(self.obs), axis=0)))
self.action = avail_actions_ind[action_index]
# self.action = 4
run_time = time.time() - start_time
def xor():
# Create a MLP with 2 input, a hidden layer with 2 nodes a single output node
nn = Mlp(init_nodes=2)
nn.add_layer(2)
nn.add_layer(1)
print("Training the network...")
for i in range(20000):
data = random.choice(training_xor)
nn.train(data["input"], data["output"])
# nn.save("xor.mlp")
for i in range(2):
for j in range(2):
out_class, out_prob = nn.predict([i, j])
print(
"Predicting XOR between {} and {} gave {} and the real is {} (Output: {:.2f})"
.format(i, j, out_prob > .5,
bool(i) ^ bool(j), out_prob))
def _init_parameters(self):
# Init LSTM cells
self._RNN_cells = {
v: self.RNN_cell(d, activation=self.Cell_activation)
for (v, d) in self.var.items()
}
# Init message-computing MLPs
self._msg_MLPs = {
msg:
Mlp(layer_sizes=[self.var[vin] for _ in range(self.MLP_depth)],
output_size=self.var[vout],
activations=[
self.Msg_activation for _ in range(self.MLP_depth)
],
output_activation=self.Msg_last_activation,
kernel_initializer=self.MLP_weight_initializer(),
bias_initializer=self.MLP_weight_initializer(),
name=msg,
name_internal_layers=True)
for msg, (vin, vout) in self.msg.items()
}
def build_network(d):
# Hyperparameters
learning_rate = 2e-5
parameter_l2norm_scaling = 1e-10
global_norm_gradient_clipping_ratio = 0.65
# Define placeholder for satisfiability statuses (one per problem)
instance_is_graph = tf.placeholder(tf.float32, [None],
name="instance_is_graph")
time_steps = tf.placeholder(tf.int32, shape=(), name='time_steps')
matrix_placeholder = tf.placeholder(tf.float32, [None, None], name="VV")
num_vars_on_instance = tf.placeholder(tf.int32, [None], name="instance_n")
n_vertices = tf.placeholder(tf.int32, shape=(None, ), name='n_vertices')
# Compute number of problems
p = tf.shape(instance_is_graph)[0]
# All edges embeddings are initialized with the same value, which is a trained parameter learned by the network
total_n = tf.shape(matrix_placeholder)[1]
v_init = tf.get_variable(initializer=tf.random_normal((1, d)),
dtype=tf.float32,
name='V_init')
vertex_initial_embeddings = tf.tile(
tf.div(v_init, tf.sqrt(tf.cast(d, tf.float32))), [total_n, 1])
# Define Typed Graph Network
gnn = TGN({
'V': d,
}, {'VV': ('V', 'V')}, {
'V_msg_V': ('V', 'V'),
}, {
'V': [{
'mat': 'VV',
'msg': 'V_msg_V',
'var': 'V'
}],
},
name='SUBGRAPH')
# Define V_vote
V_vote_MLP = Mlp(layer_sizes=[d for _ in range(3)],
activations=[tf.nn.relu for _ in range(3)],
output_size=1,
name="V_vote",
name_internal_layers=True,
kernel_initializer=tf.contrib.layers.xavier_initializer(),
bias_initializer=tf.zeros_initializer())
# Get the last embeddings
V_n = gnn({"VV": matrix_placeholder}, {"V": vertex_initial_embeddings},
time_steps)["V"].h
V_vote = V_vote_MLP(V_n)
# Reorganize votes' result to obtain a prediction for each problem instance
def _vote_while_cond(i, p, n_acc, n, n_var_list, predicted_is_graph,
L_vote):
return tf.less(i, p)
#end _vote_while_cond
predicted_is_graph = tf.TensorArray(size=p, dtype=tf.float32)
predicted_is_graph = tf.while_loop(
lambda i, pred_logits: tf.less(i, p), lambda i, pred_logits: (
(i + 1),
pred_logits.write(
i,
tf.reduce_mean(V_vote[tf.reduce_sum(n_vertices[
0:i]):tf.reduce_sum(n_vertices[0:i]) + n_vertices[i]]))),
[0, tf.TensorArray(size=p, dtype=tf.float32)])
predicted_is_graph = predicted_is_graph[1].stack()
# Define loss, accuracy
predict_costs = tf.nn.sigmoid_cross_entropy_with_logits(
labels=instance_is_graph, logits=predicted_is_graph)
predict_cost = tf.reduce_mean(predict_costs)
vars_cost = tf.zeros([])
tvars = tf.trainable_variables()
for var in tvars:
vars_cost = tf.add(vars_cost, tf.nn.l2_loss(var))
#end for
loss = tf.add(predict_cost, tf.multiply(vars_cost,
parameter_l2norm_scaling))
optimizer = tf.train.AdamOptimizer(name="Adam",
learning_rate=learning_rate)
grads, _ = tf.clip_by_global_norm(tf.gradients(loss, tvars),
global_norm_gradient_clipping_ratio)
train_step = optimizer.apply_gradients(zip(grads, tvars))
accuracy = tf.reduce_mean(
tf.cast(
tf.equal(
tf.cast(instance_is_graph, tf.bool),
tf.cast(tf.round(tf.nn.sigmoid(predicted_is_graph)), tf.bool)),
tf.float32))
# Define neurosat dictionary
neurosat = {}
neurosat["M"] = matrix_placeholder
neurosat["time_steps"] = time_steps
neurosat["gnn"] = gnn
neurosat["instance_is_graph"] = instance_is_graph
neurosat["predicted_is_graph"] = predicted_is_graph
neurosat["num_vars_on_instance"] = num_vars_on_instance
neurosat["loss"] = loss
neurosat["accuracy"] = accuracy
neurosat["train_step"] = train_step
return neurosat
def build_network(d):
# Builds the model
# Hyperparameters
learning_rate = 2e-5
parameter_l2norm_scaling = 1e-10
global_norm_gradient_clipping_ratio = 0.65
# Define GNN dictionary
GNN = {}
# Define placeholder for result values (one per problem)
#instance_prob_matrix_degree = tf.placeholder( tf.float32, [ None, None ], name = "instance_prob_matrix_degree" )
labels = tf.placeholder(tf.float32, [None, None], name="labels")
nodes_n = tf.placeholder(tf.int32, [None], name="nodes_n")
# Define Graph neural network
gnn = GraphNN(
{"N": d},
{"M": ("N", "N")},
{
"Nsource": ("N", "N"),
"Ntarget": ("N", "N")
},
{
"N": [{
"mat": "M",
"var": "N",
"msg": "Nsource"
}, {
"mat": "M",
"transpose?": True,
"var": "N",
"msg": "Ntarget"
}]
},
#Cell_activation = tf.nn.sigmoid,
#Msg_last_activation = tf.nn.sigmoid,
#name="Centrality",
)
# Define votes
# prep_MLP = Mlp(
# layer_sizes = [ d for _ in range(2) ],
# activations = [ tf.nn.relu for _ in range(2) ],
# output_size = d,
# name = "prep_MLP",
# name_internal_layers = True,
# kernel_initializer = tf.contrib.layers.xavier_initializer(),
# bias_initializer = tf.zeros_initializer()
# )
comp_MLP = Mlp(layer_sizes=[2 * d for _ in range(2)],
activations=[tf.nn.relu for _ in range(2)],
output_size=1,
name="comp_MLP",
name_internal_layers=True,
kernel_initializer=tf.contrib.layers.xavier_initializer(),
bias_initializer=tf.zeros_initializer())
# Compute the number of variables
n = tf.shape(gnn.matrix_placeholders["M"])[0]
# Compute number of problems
p = tf.shape(nodes_n)[0]
# Get the last embeddings
N_n = gnn.last_states["N"].h
#print("N_n shape:" +str(N_n.shape))
# Reorganize votes' result to obtain a prediction for each problem instance
def _vote_while_cond(i, acc_arr, cost_arr, rank10_labels, rank10_predicted,
n_acc):
return tf.less(i, p)
#end _vote_while_cond
def _vote_while_body(i, acc_arr, cost_arr, rank10_labels, rank10_predicted,
n_acc):
# Gather the embeddings for that problem
#p_embeddings = tf.gather(N_n, tf.range( n_acc, tf.add(n_acc, nodes_n[i]) ))
p_embeddings = tf.slice(N_n, [n_acc, 0], [nodes_n[i], d])
N_expanded = tf.expand_dims(p_embeddings, 0)
N1 = tf.tile(N_expanded, (nodes_n[i], 1, 1))
N_transposed = tf.transpose(N_expanded, (1, 0, 2))
N2 = tf.tile(N_transposed, (1, nodes_n[i], 1))
N1N2 = tf.concat([N1, N2], 2)
prob_matrix = comp_MLP(N1N2)
problem_predicted_matrix = tf.squeeze(prob_matrix)
p_labels = tf.slice(labels, [n_acc, n_acc], [nodes_n[i], nodes_n[i]])
p_predicted = tf.round(tf.sigmoid(problem_predicted_matrix))
s_labels = tf.reduce_sum(p_labels, axis=1)
_, labels_top = tf.nn.top_k(s_labels, k=10, sorted=True)
s_predicted = tf.reduce_sum(p_predicted, axis=1)
_, predicted_top = tf.nn.top_k(s_predicted, k=10, sorted=True)
#Compare labels to predicted values
#p_error = p_labels[n_acc:n_acc+nodes_n[i],n_acc:n_acc+nodes_n[i]]#
p_acc = tf.reduce_mean(
tf.cast(tf.equal(p_predicted, p_labels), tf.float32))
#Calculate cost for this problem
p_cost = tf.losses.sigmoid_cross_entropy(
multi_class_labels=p_labels, logits=problem_predicted_matrix)
# Update TensorArray
acc_arr = acc_arr.write(i, p_acc)
cost_arr = cost_arr.write(i, p_cost)
rank10_labels = rank10_labels.write(i, labels_top)
rank10_predicted = rank10_predicted.write(i, predicted_top)
return tf.add(i, tf.constant(
1)), acc_arr, cost_arr, rank10_labels, rank10_predicted, tf.add(
n_acc, nodes_n[i])
#end _vote_while_body
acc_arr = tf.TensorArray(size=p, dtype=tf.float32)
cost_arr = tf.TensorArray(size=p, dtype=tf.float32)
rank10_labels = tf.TensorArray(size=p, dtype=tf.int32)
rank10_predicted = tf.TensorArray(size=p, dtype=tf.int32)
_, acc_arr, cost_arr, rank10_labels, rank10_predicted, _ = tf.while_loop(
_vote_while_cond, _vote_while_body, [
tf.constant(0, dtype=tf.int32), acc_arr, cost_arr, rank10_labels,
rank10_predicted,
tf.constant(0, dtype=tf.int32)
])
acc_arr = acc_arr.stack()
cost_arr = cost_arr.stack()
rank10_labels = rank10_labels.stack()
rank10_predicted = rank10_predicted.stack()
# Define loss, %error
prob_degree_predict_cost = tf.reduce_mean(cost_arr)
vars_cost = tf.zeros([])
tvars = tf.trainable_variables()
for var in tvars:
vars_cost = tf.add(vars_cost, tf.nn.l2_loss(var))
#end for
loss = tf.add_n([
prob_degree_predict_cost,
tf.multiply(vars_cost, parameter_l2norm_scaling)
])
optimizer = tf.train.AdamOptimizer(name="Adam",
learning_rate=learning_rate)
grads, _ = tf.clip_by_global_norm(tf.gradients(loss, tvars),
global_norm_gradient_clipping_ratio)
train_step = optimizer.apply_gradients(zip(
grads, tvars)) #optimizer.minimize(loss) #
prob_degree_predict_acc = tf.reduce_mean(
acc_arr
) #percent_error_prob( labels = instance_prob_degree, predictions = predicted_prob )
GNN["gnn"] = gnn
GNN["labels"] = labels
GNN["prob_degree_predict_cost"] = prob_degree_predict_cost
GNN["prob_degree_predict_acc"] = prob_degree_predict_acc
GNN["rank10_labels"] = rank10_labels
GNN["rank10_predicted"] = rank10_predicted
GNN["loss"] = loss
GNN["nodes_n"] = nodes_n
GNN["train_step"] = train_step
return GNN
__author__ = 'jellyzhang'
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from mlp import Mlp
# Importing the dataset
dataset = pd.read_csv('../Churn_Modelling.csv')
X = dataset.iloc[:, 3:13].values
Y = dataset.iloc[:, 13].values
# Encoding categorical data
# Encode before splitting because matrix X and independent variable Y must be already encoded
# Found two categorical data (country, gender)
# create dummy variables, avoid dummy variable trap
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
labelencoder_X_1 = LabelEncoder()
X[:, 1] = labelencoder_X_1.fit_transform(X[:, 1])
labelencoder_X_2 = LabelEncoder()
X[:, 2] = labelencoder_X_2.fit_transform(X[:, 2])
mlp = Mlp(len(X[0]))
mlp.train(X, Y, epochs=100, batch_size=100)
def build_network(d):
# Define hyperparameters
d = d
learning_rate = 2e-5
l2norm_scaling = 1e-10
global_norm_gradient_clipping_ratio = 0.65
# Placeholder for answers to the decision problems (one per problem)
cn_exists = tf.placeholder( tf.float32, shape = (None,), name = 'cn_exists' )
# Placeholders for the list of number of vertices and edges per instance
n_vertices = tf.placeholder( tf.int32, shape = (None,), name = 'n_vertices')
n_edges = tf.placeholder( tf.int32, shape = (None,), name = 'n_edges')
# Placeholder for the adjacency matrix connecting each vertex to its neighbors
M_matrix = tf.placeholder( tf.float32, shape = (None,None), name = "M" )
# Placeholder for the adjacency matrix connecting each vertex to its candidate colors
VC_matrix = tf.placeholder( tf.float32, shape = (None,None), name = "VC" )
# Placeholder for chromatic number (one per problem)
chrom_number = tf.placeholder( tf.float32, shape = (None,), name = "chrom_number" )
# Placeholder for the number of timesteps the GNN is to run for
time_steps = tf.placeholder( tf.int32, shape = (), name = "time_steps" )
#Placeholder for initial color embeddings for the given batch
colors_initial_embeddings = tf.placeholder( tf.float32, shape=(None,d), name= "colors_initial_embeddings")
# All vertex embeddings are initialized with the same value, which is a trained parameter learned by the network
total_n = tf.shape(M_matrix)[1]
v_init = tf.get_variable(initializer=tf.random_normal((1,d)), dtype=tf.float32, name='V_init')
vertex_initial_embeddings = tf.tile(
tf.div(v_init, tf.sqrt(tf.cast(d, tf.float32))),
[total_n, 1]
)
# Define GNN dictionary
GNN = {}
# Define Graph neural network
gnn = GraphNN(
{
# V is the set of vertex embeddings
'V': d,
# C is for color embeddings
'C': d
},
{
# M is a V×V adjacency matrix connecting each vertex to its neighbors
'M': ('V','V'),
# MC is a VxC adjacency matrix connecting each vertex to its candidate colors
'VC': ('V','C')
},
{
# V_msg_C is a MLP which computes messages from vertex embeddings to color embeddings
'V_msg_C': ('V','C'),
# C_msg_V is a MLP which computes messages from color embeddings to vertex embeddings
'C_msg_V': ('C','V')
},
{ # V(t+1) <- Vu( M x V, VC x CmsgV(C) )
'V': [
{
'mat': 'M',
'var': 'V'
},
{
'mat': 'VC',
'var': 'C',
'msg': 'C_msg_V'
}
],
# C(t+1) <- Cu( VC^T x VmsgC(V))
'C': [
{
'mat': 'VC',
'msg': 'V_msg_C',
'transpose?': True,
'var': 'V'
}
]
}
,
name='graph-coloring'
)
# Populate GNN dictionary
GNN['gnn'] = gnn
GNN['cn_exists'] = cn_exists
GNN['n_vertices'] = n_vertices
GNN['n_edges'] = n_edges
GNN["M"] = M_matrix
GNN["VC"] = VC_matrix
GNN["chrom_number"] = chrom_number
GNN["time_steps"] = time_steps
GNN["colors_initial_embeddings"] = colors_initial_embeddings
# Define V_vote, which will compute one logit for each vertex
V_vote_MLP = Mlp(
layer_sizes = [ d for _ in range(3) ],
activations = [ tf.nn.relu for _ in range(3) ],
output_size = 1,
name = 'V_vote',
name_internal_layers = True,
kernel_initializer = tf.contrib.layers.xavier_initializer(),
bias_initializer = tf.zeros_initializer()
)
# Get the last embeddings
last_states = gnn(
{ "M": M_matrix, "VC": VC_matrix, 'chrom_number': chrom_number },
{ "V": vertex_initial_embeddings, "C": colors_initial_embeddings },
time_steps = time_steps
)
GNN["last_states"] = last_states
V_n = last_states['V'].h
C_n = last_states['C'].h
# Compute a vote for each embedding
V_vote = tf.reshape(V_vote_MLP(V_n), [-1])
# Compute the number of problems in the batch
num_problems = tf.shape(n_vertices)[0]
# Compute a logit probability for each problem
pred_logits = tf.while_loop(
lambda i, pred_logits: tf.less(i, num_problems),
lambda i, pred_logits:
(
(i+1),
pred_logits.write(
i,
tf.reduce_mean(V_vote[tf.reduce_sum(n_vertices[0:i]):tf.reduce_sum(n_vertices[0:i])+n_vertices[i]])
)
),
[0, tf.TensorArray(size=num_problems, dtype=tf.float32)]
)[1].stack()
# Convert logits into probabilities
GNN['predictions'] = tf.sigmoid(pred_logits)
# Compute True Positives, False Positives, True Negatives, False Negatives, accuracy
GNN['TP'] = tf.reduce_sum(tf.multiply(cn_exists, tf.cast(tf.equal(cn_exists, tf.round(GNN['predictions'])), tf.float32)))
GNN['FP'] = tf.reduce_sum(tf.multiply(cn_exists, tf.cast(tf.not_equal(cn_exists, tf.round(GNN['predictions'])), tf.float32)))
GNN['TN'] = tf.reduce_sum(tf.multiply(tf.ones_like(cn_exists)-cn_exists, tf.cast(tf.equal(cn_exists, tf.round(GNN['predictions'])), tf.float32)))
GNN['FN'] = tf.reduce_sum(tf.multiply(tf.ones_like(cn_exists)-cn_exists, tf.cast(tf.not_equal(cn_exists, tf.round(GNN['predictions'])), tf.float32)))
GNN['acc'] = tf.reduce_mean(tf.cast(tf.equal(cn_exists, tf.round(GNN['predictions'])), tf.float32))
# Define loss
GNN['loss'] = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=cn_exists, logits=pred_logits))
# Define optimizer
optimizer = tf.train.AdamOptimizer(name='Adam', learning_rate=learning_rate)
# Compute cost relative to L2 normalization
vars_cost = tf.add_n([ tf.nn.l2_loss(var) for var in tf.trainable_variables() ])
# Define gradients and train step
grads, _ = tf.clip_by_global_norm(tf.gradients(GNN['loss'] + tf.multiply(vars_cost, l2norm_scaling),tf.trainable_variables()),global_norm_gradient_clipping_ratio)
GNN['train_step'] = optimizer.apply_gradients(zip(grads, tf.trainable_variables()))
GNN['C_n'] = C_n
# Return GNN dictionary
return GNN
def build_network(d):
# Define hyperparameters
d = d
learning_rate = 2e-5
l2norm_scaling = 1e-10
global_norm_gradient_clipping_ratio = 0.65
# Define a placeholder for the answers to the decision problems
route_exists = tf.placeholder( tf.float32, shape = (None,), name = 'route_exists' )
# Define a placeholder for the cost of each route
route_costs = tf.placeholder( tf.float32, shape=(None,1), name='route_costs')
# Define a placeholder for the edges mask
edges_mask = tf.placeholder( tf.float32, shape = (None,), name = 'edges_mask' )
# Define placeholders for the list of number of vertices and edges per instance
n_vertices = tf.placeholder( tf.int32, shape = (None,), name = 'n_vertices')
n_edges = tf.placeholder( tf.int32, shape = (None,), name = 'edges')
#
EV_matrix = tf.placeholder( tf.float32, shape = (None,None), name = "EV" )
edge_weight = tf.placeholder( tf.float32, shape = (None,1), name = "edge_weight" )
target_cost = tf.placeholder( tf.float32, shape = (None,1), name = "target_cost" )
time_steps = tf.placeholder( tf.int32, shape = (), name = "time_steps" )
initial_embedding_mlp = Mlp(
layer_sizes = [ d for _ in range(3) ],
activations = [ tf.nn.relu for _ in range(3) ],
output_size = d,
name = 'E_init_MLP',
name_internal_layers = True,
kernel_initializer = tf.contrib.layers.xavier_initializer(),
bias_initializer = tf.zeros_initializer()
)
edge_initial_embeddings = initial_embedding_mlp(
tf.concat(
[ edge_weight, target_cost ],
axis = 1
)
)
vertex_initial_embeddings = tf.get_variable(
initializer = tf.random_normal( (1,d) ),
dtype = tf.float32, name='V_init'
)
total_n = tf.shape( EV_matrix )[1]
tiled_and_normalized_vertex_initial_embeddings = tf.tile(
tf.div(
vertex_initial_embeddings,
tf.sqrt( tf.cast( total_n, tf.float32 ) )
),
[ total_n, 1 ]
)
# Define GNN dictionary
GNN = {}
# Define Graph neural network
gnn = GraphNN(
{
# V is the set of vertex embeddings
'V': d,
# E is the set of edge embeddings
'E': d
},
{
# M is a E×V adjacency matrix connecting each edge to the vertices it is connected to
'EV': ('E','V')
},
{
# V_msg_E is a MLP which computes messages from vertex embeddings to edge embeddings
'V_msg_E': ('V','E'),
# E_msg_V is a MLP which computes messages from edge embeddings to vertex embeddings
'E_msg_V': ('E','V')
},
{
# V(t+1) ← Vu( EVᵀ × E_msg_V(E(t)) )
'V': [
{
'mat': 'EV',
'msg': 'E_msg_V',
'transpose?': True,
'var': 'E'
}
],
# E(t+1) ← Eu( EV × V_msg_E(V(t)), W, C )
'E': [
{
'mat': 'EV',
'msg': 'V_msg_E',
'var': 'V'
}
]
},
name='TSP'
)
# Populate GNN dictionary
GNN['gnn'] = gnn
GNN['route_exists'] = route_exists
GNN['route_costs'] = route_costs
GNN['edges_mask'] = edges_mask
GNN['n_vertices'] = n_vertices
GNN['n_edges'] = n_edges
GNN["EV"] = EV_matrix
GNN["W"] = edge_weight
GNN["C"] = target_cost
GNN["time_steps"] = time_steps
# Define E_vote, which will compute one logit for each edge
E_vote_MLP = Mlp(
layer_sizes = [ d for _ in range(3) ],
activations = [ tf.nn.relu for _ in range(3) ],
output_size = 1,
name = 'E_vote',
name_internal_layers = True,
kernel_initializer = tf.contrib.layers.xavier_initializer(),
bias_initializer = tf.zeros_initializer()
)
vote_bias = tf.get_variable(initializer=tf.zeros_initializer(), shape=(), dtype=tf.float32, name='vote_bias')
# Get the last embeddings
last_states = gnn(
{ "EV": EV_matrix },
{ "V": tiled_and_normalized_vertex_initial_embeddings, "E": edge_initial_embeddings },
time_steps = time_steps
)
GNN["last_states"] = last_states
E_n = last_states['E'].h
# Compute a vote for each embedding
#E_vote = tf.reshape(E_vote_MLP(tf.concat([E_n,route_costs], axis=1)), [-1])
E_vote = tf.reshape(E_vote_MLP(E_n), [-1])
E_prob = tf.sigmoid(E_vote)
# Compute the number of problems in the batch
num_problems = tf.shape(n_vertices)[0]
# n_edges_acc[i] is the number of edges in the batch up to the i-th instance
n_edges_acc = tf.map_fn(lambda i: tf.reduce_sum(tf.gather(n_edges, tf.range(0,i))), tf.range(0,num_problems))
# Compute decision predictions (one per problem)
_, pred_logits = tf.while_loop(
lambda i, predictions: tf.less(i, num_problems),
lambda i, predictions:
(
(i+1),
predictions.write(
i,
#tf.reduce_mean(tf.gather(E_vote, tf.range(n_edges_acc[i], n_edges_acc[i] + n_edges[i])))
tf.reduce_mean( E_vote[n_edges_acc[i]:n_edges_acc[i]+n_edges[i]] )
)
),
[0, tf.TensorArray(size=num_problems, dtype=tf.float32)]
)
pred_logits = pred_logits.stack() + vote_bias
GNN['predictions'] = tf.sigmoid(pred_logits)
# Count the number of edges that appear in the solution
pos_edges_n = tf.reduce_sum(edges_mask)
# Count the number of edges that do not appear in the solution
neg_edges_n = tf.reduce_sum(tf.subtract(tf.ones_like(edges_mask), edges_mask))
# Define edges loss
GNN['loss_edges'] = tf.losses.sigmoid_cross_entropy(
multi_class_labels = edges_mask,
logits = E_vote,
weights = tf.add(
tf.scalar_mul(
tf.divide(tf.add(pos_edges_n,neg_edges_n),pos_edges_n),
edges_mask),
tf.scalar_mul(
tf.divide(tf.add(pos_edges_n,neg_edges_n),neg_edges_n),
tf.subtract(tf.ones_like(edges_mask), edges_mask)
)
)
)
# Define decision loss
GNN['loss_decision'] = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=route_exists, logits=pred_logits))
# Compute true positives, false positives, true negatives, false negatives
GNN['true_pos'] = tf.reduce_sum(tf.multiply(route_exists, tf.cast(tf.equal(route_exists, tf.round(GNN['predictions'])), tf.float32)))
GNN['false_pos'] = tf.reduce_sum(tf.multiply(route_exists, tf.cast(tf.not_equal(route_exists, tf.round(GNN['predictions'])), tf.float32)))
GNN['true_neg'] = tf.reduce_sum(tf.multiply(1-route_exists, tf.cast(tf.equal(route_exists, tf.round(GNN['predictions'])), tf.float32)))
GNN['false_neg'] = tf.reduce_sum(tf.multiply(1-route_exists, tf.cast(tf.not_equal(route_exists, tf.round(GNN['predictions'])), tf.float32)))
# Define edges accuracy
GNN['acc_edges'] = tf.reduce_mean(tf.cast(tf.equal(edges_mask, tf.round(E_prob)), tf.float32))
# Define decision accuracy
GNN['acc_decision'] = tf.reduce_mean(tf.cast(tf.equal(route_exists, tf.round(GNN['predictions'])), tf.float32))
# Define optimizer
optimizer = tf.train.AdamOptimizer(name='Adam', learning_rate=learning_rate)
# Compute cost relative to L2 normalization
vars_cost = tf.add_n([ tf.nn.l2_loss(var) for var in tf.trainable_variables() ])
# Define gradients and train step
for loss_type in ['edges','decision']:
grads, _ = tf.clip_by_global_norm(tf.gradients(GNN['loss_' + loss_type] + tf.multiply(vars_cost, l2norm_scaling),tf.trainable_variables()),global_norm_gradient_clipping_ratio)
GNN['train_step_' + loss_type] = optimizer.apply_gradients(zip(grads, tf.trainable_variables()))
#end
# Return GNN dictionary
return GNN
def seed_test():
# Carregando e Normalizando os dados da base de vinhos
dataset = PreProcessing("seeds_dataset.txt", separator='\s+')
dataset.normalize()
dataset.normalize_class()
# Atributos a serem variados nos testes
n_layers = [1, 2]
hidden_layer = [3, [6, 6]]
momentums = [0.3, 0.5]
max_iterations = [100, 250, 500]
etas = [0.3, 0.5]
ps = [0.7, 0.9]
rbf_accuracy = 0
mlp_accuracy = 0
tests = 0
# Teste
for layer in n_layers:
for momentum in momentums:
for eta in etas:
for max_iteration in max_iterations:
for p in ps:
tests += 1
print("Test number", tests)
train, test = training.holdout(
p, dataset.normalized_dataframe)
print("INPUT NEURONS = 7 HIDDEN NEURONS = " +
str(int(6 / layer)) +
" OUTPUT NEURONS = 3 HIDDEN LAYER = " +
str(layer) + " ETA = " + str(eta) +
" MAX ITERATIONS = " + str(max_iteration) +
" MOMENTUM = " + str(momentum) + " P = " +
str(p))
print()
print("RBF")
nn = Rbf(7, 3)
nn.train(train, eta=0.5, max_iterations=max_iteration)
ac = training.accuracy(nn, test, 3)
rbf_accuracy += ac
print("ACCURACY =", ac)
print()
print("MLP")
example = test.values.tolist()
mm = Mlp(7,
hidden_layer[layer - 1],
3,
n_hidden_layers=layer)
mm.backpropagation(train.values.tolist(),
eta=eta,
max_iterations=max_iteration)
ac = training.accuracy(mm, test, n_classes=3)
mlp_accuracy += ac
print("ACCURACY =", ac)
print()
print("Rbf:")
nn.feed_forward(example[15][:(-1 * 3)])
print(example[15])
print("Result 1")
nn.show_class()
print()
print("Mlp")
print(example[15])
nn.feed_forward(example[15][:(-1 * 3)])
print("Result 2")
mm.show_class()
print()
print(
"******************************************************//******************************************************"
)
print()
print(tests, " tests executed. Rbf accuracy:", rbf_accuracy / tests,
" Mlp accuracy:", mlp_accuracy / tests)
def build_neurosat(d):
# Hyperparameters
learning_rate = 2e-5
parameter_l2norm_scaling = 1e-10
global_norm_gradient_clipping_ratio = 0.65
# Define placeholder for satisfiability statuses (one per problem)
instance_SAT = tf.placeholder(tf.float32, [None], name="instance_SAT")
time_steps = tf.placeholder(tf.int32, shape=(), name='time_steps')
matrix_placeholder = tf.placeholder(tf.float32, [None, None], name="M")
num_vars_on_instance = tf.placeholder(tf.int32, [None], name="instance_n")
# Literals
s = tf.shape(matrix_placeholder)
l = s[0]
m = s[1]
n = tf.floordiv(l, tf.constant(2))
# Compute number of problems
p = tf.shape(instance_SAT)[0]
# Define INV, a tf function to exchange positive and negative literal embeddings
def INV(Lh):
l = tf.shape(Lh)[0]
n = tf.div(l, tf.constant(2))
# Send messages from negated literals to positive ones, and vice-versa
Lh_pos = tf.gather(Lh, tf.range(tf.constant(0), n))
Lh_neg = tf.gather(Lh, tf.range(n, l))
Lh_inverted = tf.concat([Lh_neg, Lh_pos], axis=0)
return Lh_inverted
#end
var = {"L": d, "C": d}
s = tf.shape(matrix_placeholder)
num_vars = {"L": l, "C": m}
initial_embeddings = {
v: tf.get_variable(initializer=tf.random_normal((1, d)),
dtype=tf.float32,
name='{v}_init'.format(v=v))
for (v, d) in var.items()
}
tiled_and_normalized_initial_embeddings = {
v: tf.tile(tf.div(init, tf.sqrt(tf.cast(var[v], tf.float32))),
[num_vars[v], 1])
for v, init in initial_embeddings.items()
}
# Define Typed Graph Network
gnn = TGN(var, {"M": ("L", "C")}, {
"Lmsg": ("L", "C"),
"Cmsg": ("C", "L")
}, {
"L": [{
"fun": INV,
"var": "L"
}, {
"mat": "M",
"msg": "Cmsg",
"var": "C"
}],
"C": [{
"mat": "M",
"transpose?": True,
"msg": "Lmsg",
"var": "L"
}]
},
name="NeuroSAT")
# Define L_vote
L_vote_MLP = Mlp(layer_sizes=[d for _ in range(3)],
activations=[tf.nn.relu for _ in range(3)],
output_size=1,
name="L_vote",
name_internal_layers=True,
kernel_initializer=tf.contrib.layers.xavier_initializer(),
bias_initializer=tf.zeros_initializer())
# Get the last embeddings
L_n = gnn({"M": matrix_placeholder},
tiled_and_normalized_initial_embeddings, time_steps)["L"].h
L_vote = L_vote_MLP(L_n)
# Reorganize votes' result to obtain a prediction for each problem instance
def _vote_while_cond(i, p, n_acc, n, n_var_list, predicted_sat, L_vote):
return tf.less(i, p)
#end _vote_while_cond
def _vote_while_body(i, p, n_acc, n, n_var_list, predicted_SAT, L_vote):
# Helper for the amount of variables in this problem
i_n = n_var_list[i]
# Gather the positive and negative literals for that problem
pos_lits = tf.gather(L_vote, tf.range(n_acc, tf.add(n_acc, i_n)))
neg_lits = tf.gather(
L_vote, tf.range(tf.add(n, n_acc), tf.add(n, tf.add(n_acc, i_n))))
# Concatenate positive and negative literals and average their vote values
problem_predicted_SAT = tf.reduce_mean(
tf.concat([pos_lits, neg_lits], axis=1))
# Update TensorArray
predicted_SAT = predicted_SAT.write(i, problem_predicted_SAT)
return tf.add(i, tf.constant(1)), p, tf.add(
n_acc, i_n), n, n_var_list, predicted_SAT, L_vote
#end _vote_while_body
predicted_SAT = tf.TensorArray(size=p, dtype=tf.float32)
_, _, _, _, _, predicted_SAT, _ = tf.while_loop(
_vote_while_cond, _vote_while_body, [
tf.constant(0, dtype=tf.int32), p,
tf.constant(0, dtype=tf.int32), n, num_vars_on_instance,
predicted_SAT, L_vote
])
predicted_SAT = predicted_SAT.stack()
# Define loss, accuracy
predict_costs = tf.nn.sigmoid_cross_entropy_with_logits(
labels=instance_SAT, logits=predicted_SAT)
predict_cost = tf.reduce_mean(predict_costs)
vars_cost = tf.zeros([])
tvars = tf.trainable_variables()
for var in tvars:
vars_cost = tf.add(vars_cost, tf.nn.l2_loss(var))
#end for
loss = tf.add(predict_cost, tf.multiply(vars_cost,
parameter_l2norm_scaling))
optimizer = tf.train.AdamOptimizer(name="Adam",
learning_rate=learning_rate)
grads, _ = tf.clip_by_global_norm(tf.gradients(loss, tvars),
global_norm_gradient_clipping_ratio)
train_step = optimizer.apply_gradients(zip(grads, tvars))
accuracy = tf.reduce_mean(
tf.cast(
tf.equal(tf.cast(instance_SAT, tf.bool),
tf.cast(tf.round(tf.nn.sigmoid(predicted_SAT)), tf.bool)),
tf.float32))
# Define neurosat dictionary
neurosat = {}
neurosat["M"] = matrix_placeholder
neurosat["time_steps"] = time_steps
neurosat["gnn"] = gnn
neurosat["instance_SAT"] = instance_SAT
neurosat["predicted_SAT"] = predicted_SAT
neurosat["num_vars_on_instance"] = num_vars_on_instance
neurosat["loss"] = loss
neurosat["accuracy"] = accuracy
neurosat["train_step"] = train_step
return neurosat
def setUp(self):
self.mlp = Mlp(np.array([[1], [2]]), np.array([[1], [2]]), 12)
def learn(argv):
fname = argv[1]
if len(argv) == 3:
hidden_layers_list = eval(argv[2])
else:
hidden_layers_list = [10]
matFileContent = scipy.io.loadmat(fname) # corresponding MAT file
x_train = np.array(matFileContent['TrainSet'].tolist())
t_train = np.array(matFileContent['TrainClass'].tolist())
x_valid = np.array(matFileContent['ValidSet'].tolist())
t_valid = np.array(matFileContent['ValidClass'].tolist())
d = x_train.shape[1]
mlp = Mlp(hidden_layers_list, d)
stopping_criterion = Mlp.EarlyStoppingCriterion(5, 1e-5)
#stopping_criterion = Mlp.BasicStoppingCriterion(0.001, 100)
(error_data, best_epoch) = mlp.train_network(x_train, t_train,
x_valid, t_valid, stopping_criterion)
lrate = defaults.LEARNING_RATE_DEFAULT
mterm = defaults.MOMENTUM_TERM_DEFAULT
terms = str(lrate)[2:]+"_"+str(mterm)[2:]
arch_desc = reduce(lambda x, y:str(x)+"_"+str(y),
hidden_layers_list, "")
plt_file = 'plots/errors_' + terms + arch_desc + '.png'
plot_network_errors(error_data, best_epoch, plt_file)
print "Train log error and accuracy:"
print mlp.get_input_error(x_train, t_train), \
mlp.get_accuracy(x_train, t_train), "%"
print "Valid log error and accuracy:"
print mlp.get_input_error(x_valid, t_valid), \
mlp.get_accuracy(x_valid, t_valid), "%"
x_test = np.array(matFileContent['TestSet'].tolist())
t_test = np.array(matFileContent['TestClass'].tolist())
print "Test log error and accuracy:"
print mlp.get_input_error(x_test, t_test), \
mlp.get_accuracy(x_test, t_test), "%"
pickle.dump(mlp, open('trained_network.dat', 'wb'))
def build_network(d):
# Define hyperparameters
d = d
learning_rate = 2e-5
l2norm_scaling = 1e-10
global_norm_gradient_clipping_ratio = 0.65
# Placeholder for answers to the decision problems (one per problem)
route_exists = tf.placeholder(tf.float32,
shape=(None, ),
name='route_exists')
# Placeholders for the list of number of vertices and edges per instance
n_vertices = tf.placeholder(tf.int32, shape=(None, ), name='n_vertices')
n_edges = tf.placeholder(tf.int32, shape=(None, ), name='edges')
# Placeholder for the adjacency matrix connecting each edge to its source and target vertices
EV_matrix = tf.placeholder(tf.float32, shape=(None, None), name="EV")
# Placeholder for the column matrix of edge weights
edge_weight = tf.placeholder(tf.float32,
shape=(None, 1),
name="edge_weight")
# Placeholder for route target costs (one per problem)
target_cost = tf.placeholder(tf.float32,
shape=(None, 1),
name="target_cost")
# Placeholder for the number of timesteps the GNN is to run for
time_steps = tf.placeholder(tf.int32, shape=(), name="time_steps")
# Define a MLP to compute an initial embedding for each edge, given its
# weight and the target cost of the corresponding instance
edge_init_MLP = Mlp(
layer_sizes=[d / 8, d / 4, d / 2],
activations=[tf.nn.relu for _ in range(3)],
output_size=d,
name='E_init_MLP',
name_internal_layers=True,
kernel_initializer=tf.contrib.layers.xavier_initializer(),
bias_initializer=tf.zeros_initializer())
# Compute initial embeddings for edges
edge_initial_embeddings = edge_init_MLP(
tf.concat([edge_weight, target_cost], axis=1))
# All vertex embeddings are initialized with the same value, which is a trained parameter learned by the network
total_n = tf.shape(EV_matrix)[1]
v_init = tf.get_variable(initializer=tf.random_normal((1, d)),
dtype=tf.float32,
name='V_init')
vertex_initial_embeddings = tf.tile(
tf.div(v_init, tf.sqrt(tf.cast(d, tf.float32))), [total_n, 1])
# Define GNN dictionary
GNN = {}
# Define Graph neural network
gnn = GraphNN(
{
# V is the set of vertex embeddings
'V': d,
# E is the set of edge embeddings
'E': d
},
{
# M is a E×V adjacency matrix connecting each edge to the vertices it is connected to
'EV': ('E', 'V')
},
{
# V_msg_E is a MLP which computes messages from vertex embeddings to edge embeddings
'V_msg_E': ('V', 'E'),
# E_msg_V is a MLP which computes messages from edge embeddings to vertex embeddings
'E_msg_V': ('E', 'V')
},
{
# V(t+1) ← Vu( EVᵀ × E_msg_V(E(t)) )
'V': [{
'mat': 'EV',
'msg': 'E_msg_V',
'transpose?': True,
'var': 'E'
}],
# E(t+1) ← Eu( EV × V_msg_E(V(t)) )
'E': [{
'mat': 'EV',
'msg': 'V_msg_E',
'var': 'V'
}]
},
name='TSP')
# Populate GNN dictionary
GNN['gnn'] = gnn
GNN['route_exists'] = route_exists
GNN['n_vertices'] = n_vertices
GNN['n_edges'] = n_edges
GNN['EV'] = EV_matrix
GNN['W'] = edge_weight
GNN['C'] = target_cost
GNN['time_steps'] = time_steps
# Define E_vote, which will compute one logit for each edge
E_vote_MLP = Mlp(layer_sizes=[d for _ in range(3)],
activations=[tf.nn.relu for _ in range(3)],
output_size=1,
name='E_vote',
name_internal_layers=True,
kernel_initializer=tf.contrib.layers.xavier_initializer(),
bias_initializer=tf.zeros_initializer())
# Get the last embeddings
last_states = gnn({
"EV": EV_matrix,
'W': edge_weight,
'C': target_cost
}, {
"V": vertex_initial_embeddings,
"E": edge_initial_embeddings
},
time_steps=time_steps)
GNN["last_states"] = last_states
E_n = last_states['E'].h
# Compute a vote for each embedding
#E_vote = tf.reshape(E_vote_MLP( tf.concat([E_n,target_cost],axis=1) ), [-1])
E_vote = tf.reshape(E_vote_MLP(E_n), [-1])
# Compute the number of problems in the batch
num_problems = tf.shape(n_vertices)[0]
# Compute a logit probability for each problem
pred_logits = tf.while_loop(
lambda i, pred_logits: tf.less(i, num_problems), lambda i, pred_logits:
((i + 1),
pred_logits.write(
i,
tf.reduce_mean(E_vote[tf.reduce_sum(n_edges[0:i]):tf.reduce_sum(
n_edges[0:i]) + n_edges[i]]))),
[0, tf.TensorArray(size=num_problems, dtype=tf.float32)])[1].stack()
# Convert logits into probabilities
GNN['predictions'] = tf.sigmoid(pred_logits)
# Compute True Positives, False Positives, True Negatives, False Negatives, accuracy
GNN['TP'] = tf.reduce_sum(
tf.multiply(
route_exists,
tf.cast(tf.equal(route_exists, tf.round(GNN['predictions'])),
tf.float32)))
GNN['FP'] = tf.reduce_sum(
tf.multiply(
route_exists,
tf.cast(tf.not_equal(route_exists, tf.round(GNN['predictions'])),
tf.float32)))
GNN['TN'] = tf.reduce_sum(
tf.multiply(
tf.ones_like(route_exists) - route_exists,
tf.cast(tf.equal(route_exists, tf.round(GNN['predictions'])),
tf.float32)))
GNN['FN'] = tf.reduce_sum(
tf.multiply(
tf.ones_like(route_exists) - route_exists,
tf.cast(tf.not_equal(route_exists, tf.round(GNN['predictions'])),
tf.float32)))
GNN['acc'] = tf.reduce_mean(
tf.cast(tf.equal(route_exists, tf.round(GNN['predictions'])),
tf.float32))
# Define loss
GNN['loss'] = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(labels=route_exists,
logits=pred_logits))
# Define optimizer
optimizer = tf.train.AdamOptimizer(name='Adam',
learning_rate=learning_rate)
# Compute cost relative to L2 normalization
vars_cost = tf.add_n(
[tf.nn.l2_loss(var) for var in tf.trainable_variables()])
# Define gradients and train step
grads, _ = tf.clip_by_global_norm(
tf.gradients(GNN['loss'] + tf.multiply(vars_cost, l2norm_scaling),
tf.trainable_variables()),
global_norm_gradient_clipping_ratio)
GNN['train_step'] = optimizer.apply_gradients(
zip(grads, tf.trainable_variables()))
# Return GNN dictionary
return GNN
N_train = len(dataset['train']['target'])
N_test = len(dataset['test']['target'])
train_data_dict = {'data':dataset['train']['data'].reshape(N_train, dim).astype(np.float32),
'target':dataset['train']['target'].astype(np.int32)}
test_data_dict = {'data':dataset['test']['data'].reshape(N_test, dim).astype(np.float32),
'target':dataset['test']['target'].astype(np.int32)}
train_data = DataFeeder(train_data_dict, batchsize=args.batch)
test_data = DataFeeder(test_data_dict, batchsize=args.valbatch)
train_data.hook_preprocess(mnist_preprocess)
test_data.hook_preprocess(mnist_preprocess)
# Model Setup
h_units = 1200
model = models.ClassifierModel(Mlp(train_data['data'][0].size, h_units, h_units, np.max(train_data['target'])+1))
if args.gpu >= 0:
cuda.get_device(args.gpu).use()
model.to_gpu()
# Opimizer Setup
optimizer = optimizers.Adam()
optimizer.setup(model)
# Trainer Setup
updates = int(N_train / args.batch)
trainer = trainers.SupervisedTrainer(optimizer, logger, train_data, test_data, args.gpu)
# using EarlyStopping
patience = int(updates * args.epoch * 0.3)
def main(args):
if tf.test.is_gpu_available():
print(bc.OKGREEN + bc.BOLD + '#' * 9 + ' USING GPU ' + '#' * 9 +
bc.ENDC)
else:
print(bc.FAIL + bc.BOLD + '#' * 9 + ' NOT USING GPU ' + '#' * 9 +
bc.ENDC)
# Get agent name
tokens = args.p.split('/')
if args.p[-1] == '/':
assert (tokens.pop() == '')
dir_agent = '-'.join(tokens[-1].split('_')[:-3]) + '.save'
print(dir_agent)
# run this first to avoid failing after huge overhead
model_ok, initial_epoch = model_exists(args.m, dir_agent)
PATH_DIR_SAVE = os.path.join(args.m, dir_agent)
PATH_DIR_CKPT = os.path.join(PATH_DIR_SAVE, 'ckpts')
n_epoch = args.epochs
hypers = {
'lr': 0.00015,
'batch_size': 128,
'hl_activations': [ReLU, ReLU, ReLU, ReLU, ReLU, ReLU],
'hl_sizes': [1024, 1024, 512, 512, 512, 256],
'decay': 0.,
'bNorm': True,
'dropout': True,
'regularizer': None
}
# checking input data format.
if args.p.split('.')[-1] in ['hdf5', 'HDF5']:
f = h5py_cache.File(p,
'r',
chunk_cache_mem_size=1 * 1024**3,
swmr=True)
gen_tr = Gen4h5(f['X_tr'], f['Y_tr'], hypers['batch_size'], False)
gen_va = Gen4h5(f['X_va'], f['Y_va'], hypers['batch_size'], False)
else:
X, Y, mask = CV(args.p)
gen_tr = DataGenerator(X[mask], Y[mask], hypers['batch_size'])
gen_va = DataGenerator(X[~mask], Y[~mask], 1000)
os.makedirs(PATH_DIR_CKPT, exist_ok=True)
# Callbacks: save best & latest models.
callbacks = [
ModelCheckpoint(os.path.join(PATH_DIR_SAVE, 'best.h5'),
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=True,
mode='auto',
period=1),
ModelCheckpoint(os.path.join(PATH_DIR_CKPT,
'{epoch:02d}-{val_accuracy:.2f}.h5'),
monitor='val_loss',
verbose=1,
save_best_only=False,
save_weights_only=True,
mode='auto',
period=1),
CSVLogger(os.path.join(PATH_DIR_SAVE, 'training.log'), append=True)
]
m = Mlp(io_sizes=(glb.SIZE_OBS_VEC, glb.SIZE_ACT_VEC),
out_activation=Softmax,
loss='categorical_crossentropy',
metrics=['accuracy'],
**hypers,
verbose=1)
if model_ok:
# continue from previously saved
msg = "Saved model found. Resuming training."
print(bc.OKGREEN + bc.BOLD + msg + bc.ENDC)
h5s = os.listdir(PATH_DIR_CKPT)
h5s.sort()
saved_h5 = os.path.join(PATH_DIR_CKPT, h5s[-1])
m.construct_model(saved_h5, weights_only=True)
else:
# create new model
msg = "{} doesn't exist or is empty. Creating new model."
print(bc.WARNING + bc.BOLD + msg.format(PATH_DIR_SAVE) + bc.ENDC)
os.makedirs(PATH_DIR_CKPT, exist_ok=True)
m.construct_model()
m.train_model(gen_tr,
gen_va,
n_epoch=n_epoch,
callbacks=callbacks,
verbose=False,
workers=args.w,
use_mp=True,
max_q_size=args.q,
initial_epoch=initial_epoch)
def build_network(d, n):
"""
d: Embedding size
n: Sudoku board size (n² × n²)
"""
# Hyperparameters
learning_rate = 2e-5
parameter_l2norm_scaling = 1e-10
global_norm_gradient_clipping_ratio = 0.65
# Define GNN dictionary
GNN = {}
# Define givens matrix placeholder
is_given = tf.placeholder(tf.float32, [None], name="is_given")
# Count the number of problems
p = tf.div(tf.shape(is_given)[0], tf.constant(n**4))
# Define Graph neural network
gnn = GraphNN(
{"C": d},
{
"M_C": ("C", "C"),
"M_G": ("C", n**2)
},
{},
{"C": [{
"mat": "M_C",
"var": "C"
}, {
"mat": "M_G"
}]},
Cell_activation=tf.nn.sigmoid,
Msg_last_activation=tf.nn.sigmoid,
name="Sudoku",
)
# Define voting MLP
Cvote = Mlp(layer_sizes=[d for _ in range(2)],
activations=[tf.nn.relu for _ in range(2)],
output_size=n**2,
output_activation=tf.nn.sigmoid,
name="Cvote",
name_internal_layers=True,
kernel_initializer=tf.contrib.layers.xavier_initializer(),
bias_initializer=tf.zeros_initializer())
# Get the last embeddings
C_n = gnn.last_states["C"].h
# Get predictions
unscaled_preds = Cvote(C_n)
scaled_preds = tf.nn.softmax(unscaled_preds)
# Define losses, accuracies, optimizer, train steps
"""
The digit predicted for 'given' cells must be equal to the
corresponding digit in the puzzle. This loss penalizes cells deviating
from this pattern.
"""
loss_givens = tf.losses.softmax_cross_entropy(
logits=tf.multiply(
unscaled_preds,
tf.matmul(tf.expand_dims(is_given, 1), tf.ones((1, n**2)))),
onehot_labels=gnn.matrix_placeholders["M_G"],
)
# The conflicts loss is the expected number of conflicts in the grid
# Reduce sum along all cell pairs
loss_conflicts = tf.reduce_mean(
tf.reduce_sum(
# Perform element-wise multiplication with adjacency matrix between
# cells to only consider the conttribution of adjacent cells
tf.multiply(
gnn.matrix_placeholders["M_C"],
# Reduce sum along digit axis, computing for each pair of cells
# the probability that they are in conflict with each other
tf.reduce_sum(
# For each pair of cells i,j (among every board in the batch)
# and for each digit k compute
# P(cell(i)=k ∧ cell(j)=k) =
# P(cell(i)=k) × P(cell(j)=k)
tf.multiply(
tf.reshape(
tf.tile(
tf.reshape(scaled_preds, (p * n**4, 1, n**2)),
(p * n**4, 1, 1)), (p * n**4, p * n**4, n**2)),
tf.reshape(
tf.tile(
tf.reshape(scaled_preds, (p * n**4, 1, n**2)),
(1, p * n**4, 1)),
(p * n**4, p * n**4, n**2))),
axis=-1)),
axis=1))
# This accuracy measures how many 'given' cells were accurately marked
# Reduce mean along all cells from all boards in the batch
acc_givens = tf.subtract(
tf.constant(1.0),
tf.divide(
tf.reduce_sum(
# Perform element-wise multiplication with givens mask to only
# consider the conttribution of incorrectly marked cells in
# 'given' positions
tf.multiply(
is_given,
tf.cast(
# Compute a boolean 'diff' array indicating which cells are incorrectly marked
tf.not_equal(
tf.argmax(scaled_preds, axis=1),
tf.argmax(gnn.matrix_placeholders["M_G"], axis=1)),
tf.float32))),
tf.reduce_sum(is_given)))
# This metric is the average number of conflicts per cell
# Reduce mean along all cells from all boards in the batch
avg_conflicts = tf.reduce_mean(
tf.reduce_sum(
# Perform element-wise multiplication with adjacency matrix to only
# add the conttribution of adjacent cells with identical digits to the
# conflicts count
tf.multiply(
gnn.matrix_placeholders["M_C"],
tf.cast(
# Compute, for every cell pair, whether they are marked with the same digit
tf.equal(
tf.argmax(tf.reshape(
tf.tile(
tf.reshape(scaled_preds, (p * n**4, 1, n**2)),
(p * n**4, 1, 1)), (p * n**4, p * n**4, n**2)),
axis=-1),
tf.argmax(tf.reshape(
tf.tile(
tf.reshape(scaled_preds, (p * n**4, 1, n**2)),
(1, p * n**4, 1)), (p * n**4, p * n**4, n**2)),
axis=-1)),
tf.float32)),
axis=1))
vars_cost = tf.zeros([])
tvars = tf.trainable_variables()
for var in tvars:
vars_cost = tf.add(vars_cost, tf.nn.l2_loss(var))
#end for
optimizer = tf.train.AdamOptimizer(name="Adam",
learning_rate=learning_rate)
grads_givens, _ = tf.clip_by_global_norm(
tf.gradients(loss_givens, tvars), global_norm_gradient_clipping_ratio)
train_step_givens = optimizer.apply_gradients(zip(grads_givens, tvars))
grads_conflicts, _ = tf.clip_by_global_norm(
tf.gradients(loss_conflicts, tvars),
global_norm_gradient_clipping_ratio)
train_step_conflicts = optimizer.apply_gradients(
zip(grads_conflicts, tvars))
GNN["gnn"] = gnn
GNN["prediction"] = scaled_preds
GNN["error"] = 0
GNN["loss_givens"] = loss_givens
GNN["loss_conflicts"] = loss_conflicts
GNN["acc_givens"] = acc_givens
GNN["avg_conflicts"] = avg_conflicts
GNN["train_step_givens"] = train_step_givens
GNN["train_step_conflicts"] = train_step_conflicts
GNN["is_given"] = is_given
return GNN
def build_network(d):
# Hyperparameters
learning_rate = 2e-5
parameter_l2norm_scaling = 1e-10
global_norm_gradient_clipping_ratio = 0.65
# Define GNN dictionary
GNN = {}
# Define placeholder for result values (one per problem)
instance_colorability = tf.placeholder(tf.float32, [None],
name="instance_colorability")
# Define Graph neural network
gnn = GraphNN(
{
"V": d #,
#"C": d
},
{
"M_VV": ("V", "V") #,
#"M_VC": {"vars": ("V","C"), "compute?": False}
#"M_VC_mask": {"vars": ("V","C"), "compute?": False}
},
{
#"Cmsg": ("C","V"),
#"Vmsg": ("V","C")
},
{
"V": [{
"mat": "M_VV",
"var": "V"
} #,
#{
# "mat": "M_VC",
# "var": "C",
# "msg": "Cmsg"
#}
] #,
#"C": [
# {
# "var": "C"
# },
# {
# "mat": "M_VC",
# "transpose?": True,
# "var": "V",
# "msg": "Vmsg"
# }
#]
},
name="Coloring",
)
# Define V_vote
V_vote_MLP = Mlp(layer_sizes=[d for _ in range(2)],
activations=[tf.nn.relu for _ in range(2)],
output_size=1,
name="V_vote",
name_internal_layers=True,
kernel_initializer=tf.contrib.layers.xavier_initializer(),
bias_initializer=tf.zeros_initializer())
# Compute number of problems
p = tf.shape(instance_colorability)[0]
# Placeholder for the list of number of vertices per instance
n_vertices = tf.placeholder(tf.int32, shape=(None, ), name="n_vertices")
# Get the last embeddings
V_last = gnn.last_states["V"].h
V_vote = V_vote_MLP(V_last)
def _vote_while_body(i, n_acc, predicted_colorability):
# Helper for the amount of variables in this problem
i_n = n_vertices[i]
# Gather the vertices for that problem
vertices = tf.gather(V_vote, tf.range(n_acc, tf.add(n_acc, i_n)))
# Concatenate positive and negative literals and average their vote values
problem_predicted_colorability = tf.reduce_mean(vertices)
# Update TensorArray
predicted_colorability = predicted_colorability.write(
i, problem_predicted_colorability)
return tf.add(i, tf.constant(1)), tf.add(n_acc,
i_n), predicted_colorability
#end
def _vote_while_cond(i, n_acc, predicted_colorability):
return tf.less(i, p)
#end
predicted_colorability = tf.TensorArray(size=p, dtype=tf.float32)
_, _, predicted_colorability = tf.while_loop(
_vote_while_cond, _vote_while_body, [
tf.constant(0, dtype=tf.int32),
tf.constant(0, dtype=tf.int32), predicted_colorability
])
predicted_colorability = predicted_colorability.stack()
# Define loss, optimizer, train step
predict_costs = tf.nn.sigmoid_cross_entropy_with_logits(
labels=instance_colorability, logits=predicted_colorability)
predict_cost = tf.reduce_mean(predict_costs)
vars_cost = tf.zeros([])
tvars = tf.trainable_variables()
for var in tvars:
vars_cost = tf.add(vars_cost, tf.nn.l2_loss(var))
#end for
loss = tf.add(predict_cost, tf.multiply(vars_cost,
parameter_l2norm_scaling))
optimizer = tf.train.AdamOptimizer(name="Adam",
learning_rate=learning_rate)
grads, _ = tf.clip_by_global_norm(tf.gradients(loss, tvars),
global_norm_gradient_clipping_ratio)
train_step = optimizer.apply_gradients(zip(grads, tvars))
# Define accuracy
acc = tf.reduce_mean(
tf.cast(
tf.equal(
tf.cast(instance_colorability, tf.bool),
tf.cast(tf.round(tf.nn.sigmoid(predicted_colorability)),
tf.bool)), tf.float32))
GNN["gnn"] = gnn
#GNN["k"] = gnn.num_vars["C"]
GNN["n_vertices"] = n_vertices
GNN["instance_colorability"] = instance_colorability
GNN["predicted_colorability"] = tf.nn.sigmoid(predicted_colorability)
GNN["loss"] = loss
GNN["acc"] = acc
GNN["train_step"] = train_step
return GNN
def build_network(d):
# Builds the model
# Hyperparameters
learning_rate = 2e-5
parameter_l2norm_scaling = 1e-10
global_norm_gradient_clipping_ratio = 0.65
# Define GNN dictionary
GNN = {}
# Define placeholder for result values (one per problem)
instance_degree = tf.placeholder(tf.float32, [None],
name="instance_degree")
instance_betweenness = tf.placeholder(tf.float32, [None],
name="instance_betweenness")
instance_closeness = tf.placeholder(tf.float32, [None],
name="instance_closeness")
instance_eigenvector = tf.placeholder(tf.float32, [None],
name="instance_eigenvector")
instance_target = tf.placeholder(tf.int32, [None], name="instance_target")
# Define Graph neural network
gnn = GraphNN(
{"N": d},
{"M": ("N", "N")},
{
"Nsource": ("N", "N"),
"Ntarget": ("N", "N")
},
{
"N": [{
"mat": "M",
"var": "N",
"msg": "Nsource"
}, {
"mat": "M",
"transpose?": True,
"var": "N",
"msg": "Ntarget"
}]
},
Cell_activation=tf.nn.sigmoid,
Msg_last_activation=tf.nn.sigmoid,
name="Centrality",
)
# Define votes
degree_MLP = Mlp(layer_sizes=[d for _ in range(2)],
activations=[tf.nn.relu for _ in range(2)],
output_size=1,
name="degree_MLP",
name_internal_layers=True,
kernel_initializer=tf.contrib.layers.xavier_initializer(),
bias_initializer=tf.zeros_initializer())
betweenness_MLP = Mlp(
layer_sizes=[d for _ in range(2)],
activations=[tf.nn.relu for _ in range(2)],
output_size=1,
name="betweenness_MLP",
name_internal_layers=True,
kernel_initializer=tf.contrib.layers.xavier_initializer(),
bias_initializer=tf.zeros_initializer())
closeness_MLP = Mlp(
layer_sizes=[d for _ in range(2)],
activations=[tf.nn.relu for _ in range(2)],
output_size=1,
name="closeness_MLP",
name_internal_layers=True,
kernel_initializer=tf.contrib.layers.xavier_initializer(),
bias_initializer=tf.zeros_initializer())
eigenvector_MLP = Mlp(
layer_sizes=[d for _ in range(2)],
activations=[tf.nn.relu for _ in range(2)],
output_size=1,
name="eigenvector_MLP",
name_internal_layers=True,
kernel_initializer=tf.contrib.layers.xavier_initializer(),
bias_initializer=tf.zeros_initializer())
# Compute the number of variables
n = tf.shape(gnn.matrix_placeholders["M"])[0]
# Compute number of problems
p = tf.shape(instance_target)[0]
# Get the last embeddings
N_n = gnn.last_states["N"].h
degree_vote = degree_MLP(N_n)
betweenness_vote = betweenness_MLP(N_n)
closeness_vote = closeness_MLP(N_n)
eigenvector_vote = eigenvector_MLP(N_n)
# Reorganize votes' result to obtain a prediction for each problem instance
def _vote_while_cond(i, predicted_degree, predicted_betweenness,
predicted_closeness, predicted_eigenvector):
return tf.less(i, p)
#end _vote_while_cond
def _vote_while_body(i, predicted_degree, predicted_betweenness,
predicted_closeness, predicted_eigenvector):
# Gather the target nodes for that problem
final_node_degree = degree_vote[instance_target[i]]
problem_predicted_degree = tf.reshape(final_node_degree, shape=[])
final_node_betweenness = betweenness_vote[instance_target[i]]
problem_predicted_betweenness = tf.reshape(final_node_betweenness,
shape=[])
final_node_closeness = closeness_vote[instance_target[i]]
problem_predicted_closeness = tf.reshape(final_node_closeness,
shape=[])
final_node_eigenvector = eigenvector_vote[instance_target[i]]
problem_predicted_eigenvector = tf.reshape(final_node_eigenvector,
shape=[])
# Update TensorArray
predicted_degree = predicted_degree.write(i, problem_predicted_degree)
predicted_betweenness = predicted_betweenness.write(
i, problem_predicted_betweenness)
predicted_closeness = predicted_closeness.write(
i, problem_predicted_closeness)
predicted_eigenvector = predicted_eigenvector.write(
i, problem_predicted_eigenvector)
return tf.add(
i, tf.constant(1)
), predicted_degree, predicted_betweenness, predicted_closeness, predicted_eigenvector
#end _vote_while_body
predicted_degree = tf.TensorArray(size=p, dtype=tf.float32)
predicted_betweenness = tf.TensorArray(size=p, dtype=tf.float32)
predicted_closeness = tf.TensorArray(size=p, dtype=tf.float32)
predicted_eigenvector = tf.TensorArray(size=p, dtype=tf.float32)
_, predicted_degree, predicted_betweenness, predicted_closeness, predicted_eigenvector = tf.while_loop(
_vote_while_cond, _vote_while_body, [
tf.constant(0, dtype=tf.int32), predicted_degree,
predicted_betweenness, predicted_closeness, predicted_eigenvector
])
predicted_degree = predicted_degree.stack()
predicted_betweenness = predicted_betweenness.stack()
predicted_closeness = predicted_closeness.stack()
predicted_eigenvector = predicted_eigenvector.stack()
# Define loss, %error
betweenness_predict_costs = tf.losses.mean_squared_error(
labels=instance_betweenness, predictions=predicted_betweenness)
degree_predict_costs = tf.losses.mean_squared_error(
labels=instance_degree, predictions=predicted_degree)
closeness_predict_costs = tf.losses.mean_squared_error(
labels=instance_closeness, predictions=predicted_closeness)
eigenvector_predict_costs = tf.losses.mean_squared_error(
labels=instance_eigenvector, predictions=predicted_eigenvector)
betweenness_predict_cost = tf.reduce_mean(betweenness_predict_costs)
degree_predict_cost = tf.reduce_mean(degree_predict_costs)
closeness_predict_cost = tf.reduce_mean(closeness_predict_costs)
eigenvector_predict_cost = tf.reduce_mean(eigenvector_predict_costs)
vars_cost = tf.zeros([])
tvars = tf.trainable_variables()
for var in tvars:
vars_cost = tf.add(vars_cost, tf.nn.l2_loss(var))
#end for
loss = tf.add_n([
degree_predict_cost, betweenness_predict_cost, closeness_predict_cost,
eigenvector_predict_cost,
tf.multiply(vars_cost, parameter_l2norm_scaling)
])
optimizer = tf.train.AdamOptimizer(name="Adam",
learning_rate=learning_rate)
grads, _ = tf.clip_by_global_norm(tf.gradients(loss, tvars),
global_norm_gradient_clipping_ratio)
train_step = optimizer.apply_gradients(zip(grads, tvars))
degree_predict_error = percent_error(labels=instance_degree,
predictions=predicted_degree)
betweenness_predict_error = percent_error(
labels=instance_betweenness, predictions=predicted_betweenness)
closeness_predict_error = percent_error(labels=instance_closeness,
predictions=predicted_closeness)
eigenvector_predict_error = percent_error(
labels=instance_eigenvector, predictions=predicted_eigenvector)
error = tf.reduce_mean([
betweenness_predict_error, closeness_predict_error,
eigenvector_predict_error
])
GNN["gnn"] = gnn
GNN["instance_degree"] = instance_degree
GNN["instance_betweenness"] = instance_betweenness
GNN["instance_closeness"] = instance_closeness
GNN["instance_eigenvector"] = instance_eigenvector
GNN["instance_target"] = instance_target
GNN["predicted_degree"] = predicted_degree
GNN["predicted_betweenness"] = predicted_betweenness
GNN["predicted_closeness"] = predicted_closeness
GNN["predicted_eigenvector"] = predicted_eigenvector
GNN["degree_predict_cost"] = degree_predict_cost
GNN["betweenness_predict_cost"] = betweenness_predict_cost
GNN["closeness_predict_cost"] = closeness_predict_cost
GNN["eigenvector_predict_cost"] = eigenvector_predict_cost
GNN["error"] = error
GNN["degree_predict_error"] = degree_predict_error
GNN["betweenness_predict_error"] = betweenness_predict_error
GNN["closeness_predict_error"] = closeness_predict_error
GNN["eigenvector_predict_error"] = eigenvector_predict_error
GNN["loss"] = loss
GNN["train_step"] = train_step
return GNN
def ocr(training_population=5000, testing_population=1000):
print("Loading data...")
train = pd.read_csv("../datasets/mnist_train.csv")
train = process_df(train)
test_set = pd.read_csv("../datasets/mnist_test.csv")
test_set = process_df(test_set)
print("Loaded {} rows for training.".format(train.shape[0]))
print("Loaded {} rows for testing.".format(test_set.shape[0]))
nn = Mlp(init_nodes=784, learning_rate=.05)
nn.add_layer(300)
nn.add_layer(150, function="relu")
nn.add_layer(10)
print(
"Training the network with {} samples...".format(training_population))
for i in range(training_population):
data = train.sample(n=1)
label = data["label"].tolist()[0]
inputs = list(data.iloc[0, 1:])
outputs = [0] * 10
outputs[label] = 1
nn.train(inputs, outputs)
print("Trained successfully.")
# nn.save("ocr.mlp")
print("Testing with {} samples...".format(testing_population))
c_m = np.zeros(shape=(10, 10))
for i in range(testing_population):
data = test_set.sample(n=1)
inputs = list(data.iloc[0, 1:])
label = data["label"].tolist()[0]
out_class, out_prob = nn.predict(inputs)
c_m[label][out_class] += 1
print("Results:")
correct_guesses = np.sum(np.diagonal(c_m))
total_guesses = c_m.sum()
accuracy = correct_guesses / total_guesses
recall = 0
precision = 0
c_m_t = c_m.T
for i in range(10):
correct_guesses = c_m[i][i]
total_row = np.sum(c_m[i])
total_col = np.sum(c_m_t[i])
recall += (correct_guesses / total_row) if total_row > 0 else 0
precision += (correct_guesses / total_col) if total_col > 0 else 0
recall = recall / 10
precision = precision / 10
print(
"\tRecall: {0:.2f}\n\tPrecision: {0:.2f}\n\tAccuracy: {0:.2f}".format(
recall, precision, accuracy))
'target': dataset['train']['target'][test_idx].astype(np.int32)
}
#
labeled_data = datafeeders.SiameseFeeder(labeled_data_dict,
batchsize=args.lbatch)
unlabeled_data = DataFeeder(unlabeled_data_dict, batchsize=args.ubatch)
test_data = datafeeders.SiameseFeeder(test_data_dict, batchsize=args.valbatch)
labeled_data.hook_preprocess(mnist_preprocess)
unlabeled_data.hook_preprocess(mnist_preprocess_u)
test_data.hook_preprocess(mnist_preprocess)
# Model Setup
h_units = 1200
model = models.SiameseModel(
Mlp(labeled_data['data'][0].size, h_units, h_units,
np.max(labeled_data['target']) + 1))
if args.gpu >= 0:
cuda.get_device(args.gpu).use()
model.to_gpu()
# Opimizer Setup
alpha = 0.002
optimizer = optimizers.Adam(alpha=alpha)
optimizer.setup(model)
alpha_decay_interval = 500
alpha_decay_rate = 0.9
adam_alpha_scheduler = DecayOptimizerScheduler(optimizer, 'alpha',
alpha_decay_interval,
alpha_decay_rate)
def build_network(d):
# Builds the model
# Hyperparameters
learning_rate = 2e-5
parameter_l2norm_scaling = 1e-10
global_norm_gradient_clipping_ratio = 0.65
# Define GNN dictionary
GNN = {}
# Define placeholder for result values (one per problem)
#instance_prob_matrix_degree = tf.placeholder( tf.float32, [ None, None ], name = "instance_prob_matrix_degree" )
labels_deg = tf.placeholder( tf.float32, [ None, None ], name = "labels_deg" )
labels_bet = tf.placeholder( tf.float32, [ None, None ], name = "labels_bet" )
labels_clo = tf.placeholder( tf.float32, [ None, None ], name = "labels_clo" )
labels_eig = tf.placeholder( tf.float32, [ None, None ], name = "labels_eig" )
k = tf.placeholder( tf.int32, shape=() , name = "k" )
nodes_n = tf.placeholder( tf.int32, [ None ], name = "nodes_n" )
# Define Graph neural network
gnn = GraphNN(
{
"N": d
},
{
"M": ("N","N")
},
{
"Nsource": ("N","N"),
"Ntarget": ("N","N")
},
{
"N": [
{
"mat": "M",
"var": "N",
"msg": "Nsource"
},
{
"mat": "M",
"transpose?": True,
"var": "N",
"msg": "Ntarget"
}
]
},
)
# Define votes
# Define votes
degree_MLP = Mlp(
layer_sizes = [ 2*d for _ in range(2) ],
activations = [ tf.nn.relu for _ in range(2) ],
output_size = 1,
name = "degree_MLP",
name_internal_layers = True,
kernel_initializer = tf.contrib.layers.xavier_initializer(),
bias_initializer = tf.zeros_initializer()
)
betweenness_MLP = Mlp(
layer_sizes = [ 2*d for _ in range(2) ],
activations = [ tf.nn.relu for _ in range(2) ],
output_size = 1,
name = "betweenness_MLP",
name_internal_layers = True,
kernel_initializer = tf.contrib.layers.xavier_initializer(),
bias_initializer = tf.zeros_initializer()
)
closeness_MLP = Mlp(
layer_sizes = [ 2*d for _ in range(2) ],
activations = [ tf.nn.relu for _ in range(2) ],
output_size = 1,
name = "closeness_MLP",
name_internal_layers = True,
kernel_initializer = tf.contrib.layers.xavier_initializer(),
bias_initializer = tf.zeros_initializer()
)
eigenvector_MLP = Mlp(
layer_sizes = [ 2*d for _ in range(2) ],
activations = [ tf.nn.relu for _ in range(2) ],
output_size = 1,
name = "eigenvector_MLP",
name_internal_layers = True,
kernel_initializer = tf.contrib.layers.xavier_initializer(),
bias_initializer = tf.zeros_initializer()
)
# Compute the number of variables
n = tf.shape( gnn.matrix_placeholders["M"] )[0]
# Compute number of problems
p = tf.shape( nodes_n )[0]
# Get the last embeddings
N_n = gnn.last_states["N"].h
#print("N_n shape:" +str(N_n.shape))
# Reorganize votes' result to obtain a prediction for each problem instance
def _vote_while_cond(i, arr_acc_deg, arr_cost_deg, topk_deg, arr_acc_bet, arr_cost_bet, topk_bet, arr_acc_clo, arr_cost_clo, topk_clo, arr_acc_eig, arr_cost_eig, topk_eig, n_acc):
return tf.less( i, p )
#end _vote_while_cond
def _vote_while_body(i, arr_acc_deg, arr_cost_deg, topk_deg, arr_acc_bet, arr_cost_bet, topk_bet, arr_acc_clo, arr_cost_clo, topk_clo, arr_acc_eig, arr_cost_eig, topk_eig, n_acc):
# Gather the embeddings for that problem
#p_embeddings = tf.gather(N_n, tf.range( n_acc, tf.add(n_acc, nodes_n[i]) ))
p_embeddings = tf.slice( N_n, [n_acc, 0], [nodes_n[i], d])
N_expanded = tf.expand_dims(p_embeddings, 0)
N1 = tf.tile(N_expanded,(nodes_n[i],1,1))
N_transposed = tf.transpose(N_expanded, (1,0,2))
N2 = tf.tile(N_transposed, (1,nodes_n[i],1))
N1N2 = tf.concat([N1,N2], 2)
pred_deg_matrix = tf.squeeze( degree_MLP( N1N2 ) )
pred_bet_matrix = tf.squeeze( betweenness_MLP( N1N2 ) )
pred_clo_matrix = tf.squeeze( closeness_MLP( N1N2 ) )
pred_eig_matrix = tf.squeeze( eigenvector_MLP( N1N2 ) )
pred_deg_matrix_prob = tf.round( tf.sigmoid( pred_deg_matrix ) )
pred_bet_matrix_prob = tf.round( tf.sigmoid( pred_bet_matrix ) )
pred_clo_matrix_prob = tf.round( tf.sigmoid( pred_clo_matrix ) )
pred_eig_matrix_prob = tf.round( tf.sigmoid( pred_eig_matrix ) )
p_labels_deg = tf.slice( labels_deg, [n_acc, n_acc], [nodes_n[i], nodes_n[i]])
p_labels_bet = tf.slice( labels_bet, [n_acc, n_acc], [nodes_n[i], nodes_n[i]])
p_labels_clo = tf.slice( labels_clo, [n_acc, n_acc], [nodes_n[i], nodes_n[i]])
p_labels_eig = tf.slice( labels_eig, [n_acc, n_acc], [nodes_n[i], nodes_n[i]])
# s_labels = tf.reduce_sum( p_labels, axis=1 )
# _,labels_top = tf.nn.top_k( s_labels, k=10 , sorted=True )
#Get top k rankings for each centrality
s_predicted = tf.reduce_sum( pred_deg_matrix_prob, axis=1 )
_, predicted_topk_deg = tf.nn.top_k( s_predicted, k=k , sorted=True )
s_predicted = tf.reduce_sum( pred_bet_matrix_prob, axis=1 )
_, predicted_topk_bet = tf.nn.top_k( s_predicted, k=k , sorted=True )
s_predicted = tf.reduce_sum( pred_clo_matrix_prob, axis=1 )
_, predicted_topk_clo = tf.nn.top_k( s_predicted, k=k , sorted=True )
s_predicted = tf.reduce_sum( pred_eig_matrix_prob, axis=1 )
_, predicted_topk_eig = tf.nn.top_k( s_predicted, k=k , sorted=True )
#Compare labels to predicted values
#p_error = p_labels[n_acc:n_acc+nodes_n[i],n_acc:n_acc+nodes_n[i]]#
p_acc_deg = tf.reduce_mean(
tf.cast(
tf.equal(
pred_deg_matrix_prob, p_labels_deg
),
tf.float32
)
)
p_acc_bet = tf.reduce_mean(
tf.cast(
tf.equal(
pred_bet_matrix_prob, p_labels_bet
),
tf.float32
)
)
p_acc_clo = tf.reduce_mean(
tf.cast(
tf.equal(
pred_clo_matrix_prob, p_labels_clo
),
tf.float32
)
)
p_acc_eig = tf.reduce_mean(
tf.cast(
tf.equal(
pred_eig_matrix_prob, p_labels_eig
),
tf.float32
)
)
#Calculate cost for this problem
p_cost_deg = tf.losses.sigmoid_cross_entropy( multi_class_labels = p_labels_deg, logits = pred_deg_matrix)
p_cost_bet = tf.losses.sigmoid_cross_entropy( multi_class_labels = p_labels_bet, logits = pred_bet_matrix)
p_cost_clo = tf.losses.sigmoid_cross_entropy( multi_class_labels = p_labels_clo, logits = pred_clo_matrix)
p_cost_eig = tf.losses.sigmoid_cross_entropy( multi_class_labels = p_labels_eig, logits = pred_eig_matrix)
# Update TensorArray
arr_acc_deg = arr_acc_deg.write( i, p_acc_deg )
arr_cost_deg = arr_cost_deg.write(i, p_cost_deg )
topk_deg = topk_deg.write( i, predicted_topk_deg )
arr_acc_bet = arr_acc_bet.write( i, p_acc_bet )
arr_cost_bet = arr_cost_bet.write(i, p_cost_bet )
topk_bet = topk_bet.write( i, predicted_topk_bet )
arr_acc_clo = arr_acc_clo.write( i, p_acc_clo )
arr_cost_clo = arr_cost_clo.write(i, p_cost_clo )
topk_clo = topk_clo.write( i, predicted_topk_clo )
arr_acc_eig = arr_acc_eig.write( i, p_acc_eig )
arr_cost_eig = arr_cost_eig.write(i, p_cost_eig)
topk_eig = topk_eig.write( i, predicted_topk_eig )
return tf.add( i, tf.constant( 1 ) ), arr_acc_deg, arr_cost_deg, topk_deg, arr_acc_bet, arr_cost_bet, topk_bet, arr_acc_clo, arr_cost_clo, topk_clo, arr_acc_eig, arr_cost_eig, topk_eig, tf.add( n_acc, nodes_n[i] )
#end _vote_while_body
arr_acc_deg = tf.TensorArray( size = p, dtype = tf.float32 )
arr_cost_deg = tf.TensorArray( size = p, dtype = tf.float32 )
topk_deg = tf.TensorArray( size = p, dtype = tf.int32 )
arr_acc_bet = tf.TensorArray( size = p, dtype = tf.float32 )
arr_cost_bet = tf.TensorArray( size = p, dtype = tf.float32 )
topk_bet = tf.TensorArray( size = p, dtype = tf.int32 )
arr_acc_clo = tf.TensorArray( size = p, dtype = tf.float32 )
arr_cost_clo = tf.TensorArray( size = p, dtype = tf.float32 )
topk_clo = tf.TensorArray( size = p, dtype = tf.int32 )
arr_acc_eig = tf.TensorArray( size = p, dtype = tf.float32 )
arr_cost_eig = tf.TensorArray( size = p, dtype = tf.float32 )
topk_eig = tf.TensorArray( size = p, dtype = tf.int32 )
_, arr_acc_deg, arr_cost_deg, topk_deg, arr_acc_bet, arr_cost_bet, topk_bet, arr_acc_clo, arr_cost_clo, topk_clo, arr_acc_eig, arr_cost_eig , topk_eig, _ = tf.while_loop(
_vote_while_cond,
_vote_while_body,
[ tf.constant( 0, dtype = tf.int32 ), arr_acc_deg, arr_cost_deg, topk_deg, arr_acc_bet, arr_cost_bet, topk_bet, arr_acc_clo, arr_cost_clo, topk_clo, arr_acc_eig, arr_cost_eig, topk_eig, tf.constant( 0, dtype = tf.int32 ) ]
)
arr_acc_deg = arr_acc_deg.stack()
arr_cost_deg = arr_cost_deg.stack()
topk_deg = topk_deg.stack()
arr_acc_bet = arr_acc_bet.stack()
arr_cost_bet = arr_cost_bet.stack()
topk_bet = topk_bet.stack()
arr_acc_clo = arr_acc_clo.stack()
arr_cost_clo = arr_cost_clo.stack()
topk_clo = topk_clo.stack()
arr_acc_eig = arr_acc_eig.stack()
arr_cost_eig = arr_cost_eig.stack()
topk_eig = topk_eig.stack()
# Define batch loss
degree_predict_cost = tf.reduce_mean( arr_cost_deg )
betweenness_predict_cost = tf.reduce_mean( arr_cost_bet )
closeness_predict_cost = tf.reduce_mean( arr_cost_clo )
eigenvector_predict_cost = tf.reduce_mean( arr_cost_eig )
vars_cost = tf.zeros([])
tvars = tf.trainable_variables()
for var in tvars:
vars_cost = tf.add( vars_cost, tf.nn.l2_loss( var ) )
#end for
loss = tf.add_n( [ degree_predict_cost, betweenness_predict_cost, closeness_predict_cost, eigenvector_predict_cost, tf.multiply( vars_cost, parameter_l2norm_scaling ) ] )
optimizer = tf.train.AdamOptimizer( name = "Adam", learning_rate = learning_rate )
grads, _ = tf.clip_by_global_norm( tf.gradients( loss, tvars ), global_norm_gradient_clipping_ratio )
train_step = optimizer.apply_gradients( zip( grads, tvars ) ) #optimizer.minimize(loss) #
#Calculate the batch average accuracy
degree_predict_acc = tf.reduce_mean( arr_acc_deg )
closeness_predict_acc = tf.reduce_mean( arr_acc_clo )
betweenness_predict_acc = tf.reduce_mean( arr_acc_bet )
eigenvector_predict_acc = tf.reduce_mean( arr_acc_eig )
acc = tf.reduce_mean( [degree_predict_acc, closeness_predict_acc, betweenness_predict_acc, eigenvector_predict_acc] )
GNN["gnn"] = gnn
GNN["last_embeddings"] = N_n
GNN["labels_deg"] = labels_deg
GNN["labels_bet"] = labels_bet
GNN["labels_clo"] = labels_clo
GNN["labels_eig"] = labels_eig
GNN["k"] = k
GNN["degree_predict_cost"] = degree_predict_cost
GNN["degree_predict_acc"] = degree_predict_acc
GNN["topk_deg"] = topk_deg
GNN["betweenness_predict_cost"] = betweenness_predict_cost
GNN["betweenness_predict_acc"] = betweenness_predict_acc
GNN["topk_bet"] = topk_bet
GNN["closeness_predict_cost"] = closeness_predict_cost
GNN["closeness_predict_acc"] = closeness_predict_acc
GNN["topk_clo"] = topk_clo
GNN["eigenvector_predict_cost"] = eigenvector_predict_cost
GNN["eigenvector_predict_acc"] = eigenvector_predict_acc
GNN["topk_eig"] = topk_eig
GNN["acc"] = acc
GNN["loss"] = loss
GNN["nodes_n"] = nodes_n
GNN["train_step"] = train_step
return GNN
'target': dataset['train']['target'].astype(np.int32)
}
test_data_dict = {
'data': dataset['test']['data'].reshape(N_test, dim).astype(np.float32),
'target': dataset['test']['target'].astype(np.int32)
}
train_data = DataFeeder(train_data_dict, batchsize=args.batch)
test_data = DataFeeder(test_data_dict, batchsize=args.valbatch)
train_data.hook_preprocess(mnist_preprocess)
test_data.hook_preprocess(mnist_preprocess)
# Model Setup
h_units = 1200
model = models.ClassifierModel(
Mlp(train_data['data'][0].size, h_units, h_units,
np.max(train_data['target']) + 1))
if args.gpu >= 0:
cuda.get_device(args.gpu).use()
model.to_gpu()
# Opimizer Setup
optimizer = optimizers.Adam()
optimizer.setup(model)
trainer = trainers.SupervisedTrainer(optimizer, logger, (train_data, ),
test_data, args.gpu)
trainer.train(int(args.epoch * N_train / args.batch),
log_interval=1,
test_interval=N_train / args.batch,
test_nitr=N_test / args.valbatch)
def build_network(d):
# Hyperparameters
learning_rate = 2e-5
parameter_l2norm_scaling = 1e-10
global_norm_gradient_clipping_ratio = 0.65
# Define GNN dictionary
GNN = {}
# Define placeholder for result values (one per problem)
target_edges = tf.placeholder( tf.float32, [ None ], name = "target_edges" )
instance_val = tf.placeholder( tf.float32, [ None ], name = "instance_edge_num" )
instance_m_list = tf.placeholder( tf.int32, [ None ], name = "instance_edge_num" )
instance_target = tf.placeholder( tf.int32, [ None ], name = "instance_target" )
# Define INV, a tf function to exchange positive and negative literal embeddings
def INV(Lh):
l = tf.shape(Lh)[0]
n = tf.div(l,tf.constant(2))
# Send messages from negated literals to positive ones, and vice-versa
Lh_pos = tf.gather( Lh, tf.range( tf.constant( 0 ), n ) )
Lh_neg = tf.gather( Lh, tf.range( n, l ) )
Lh_inverted = tf.concat( [ Lh_neg, Lh_pos ], axis = 0 )
return Lh_inverted
#end
# Define Graph neural network
gnn = GraphNN(
{
"N": d, # Nodes
"E": d # Edges
},
{
"Ms": ("N","E"), # Matrix pointing from nodes to the edges they are sources
"Mt": ("N","E"), # Matrix pointing from nodes to the edges they are targets
"Mw": ("E","E"), # Matrix indicating an Edge weight
"S": ("N","N"), # Matrix indicating whether a node is the source
"T": ("N","N"), # Matrix indicating whether a node is the target
},
{
"NsmsgE": ("N","E"), # Message cast to convert messages from node sources to edges
"NtmsgE": ("N","E"), # Message cast to convert messages from node targets to edges
"EmsgNs": ("N","E"), # Message cast to convert messages from edges to node sources
"EmsgNt": ("N","E") # Message cast to convert messages from edges to node targets
},
{
"N": [
{
"mat": "Ms",
"msg": "EmsgNs",
"var": "E"
},
{
"mat": "Mt",
"msg": "EmsgNt",
"var": "E"
},
{
"mat": "S"
},
{
"mat": "T"
}
],
"E": [
{
"mat": "Ms",
"transpose?": True,
"msg": "NsmsgE",
"var": "N"
},
{
"mat": "Mt",
"transpose?": True,
"msg": "NtmsgE",
"var": "N"
},
{
"mat": "Mw"
}
]
},
name="Dijkstra_Quiver",
float_dtype = tf.float32
)
# Define L_vote
E_vote_MLP = Mlp(
layer_sizes = [ d for _ in range(2) ],
activations = [ tf.nn.relu for _ in range(2) ],
output_size = 1,
name = "E_vote",
name_internal_layers = True,
kernel_initializer = tf.contrib.layers.xavier_initializer(),
bias_initializer = tf.zeros_initializer()
)
# Compute the number of variables
m = tf.shape( gnn.matrix_placeholders["Mw"] )[0]
# Compute number of problems
p = tf.shape( instance_val )[0]
# Get the last embeddings
E_n = gnn.last_states["E"].h
E_vote_logits = E_vote_MLP( E_n )
E_vote = tf.nn.sigmoid( E_vote_logits )
E_objective = tf.sparse_tensor_dense_matmul( gnn.matrix_placeholders["Mw"], E_vote )
# Reorganize votes' result to obtain a prediction for each problem instance
def _vote_while_cond(i, m_acc, predicted_val):
return tf.less( i, p )
#end _vote_while_cond
def _vote_while_body(i, m_acc, predicted_val):
# Helper for the amount of edges in this problem
i_m = instance_m_list[i]
# Gather the edges of that problem
obj_vals = tf.gather( E_objective, tf.range( m_acc, tf.add( m_acc, i_m ) ) )
problem_predicted_val = tf.reduce_sum( obj_vals )
# Update TensorArray
predicted_val = predicted_val.write( i, problem_predicted_val )
return tf.add( i, tf.constant( 1 ) ), tf.add( m_acc, i_m ), predicted_val
#end _vote_while_body
predicted_val = tf.TensorArray( size = p, dtype = tf.float32 )
_, _, predicted_val = tf.while_loop(
_vote_while_cond,
_vote_while_body,
[ tf.constant( 0, dtype = tf.int32 ), tf.constant( 0, dtype = tf.int32 ), predicted_val ]
)
predicted_val = predicted_val.stack()
# Define loss and %error
predict_costs = tf.losses.sigmoid_cross_entropy(
multi_class_labels = target_edges,
logits = tf.reshape( E_vote_logits, [-1] )
)
predict_cost = tf.reduce_mean( predict_costs )
# %Error
abserror = tf.reduce_mean( tf.divide( tf.abs( tf.subtract( instance_val, predicted_val ) ), instance_val ) )
error = tf.reduce_mean( tf.divide( tf.subtract( instance_val, predicted_val ), instance_val ) )
vars_cost = tf.zeros([])
tvars = tf.trainable_variables()
for var in tvars:
vars_cost = tf.add( vars_cost, tf.nn.l2_loss( var ) )
#end for
loss = tf.add( predict_cost, tf.multiply( vars_cost, parameter_l2norm_scaling ) )
optimizer = tf.train.AdamOptimizer( name = "Adam", learning_rate = learning_rate )
grads, _ = tf.clip_by_global_norm( tf.gradients( loss, tvars ), global_norm_gradient_clipping_ratio )
train_step = optimizer.apply_gradients( zip( grads, tvars ) )
GNN["gnn"] = gnn
GNN["instance_val"] = instance_val
GNN["target_edges"] = target_edges
GNN["instance_target"] = instance_target
GNN["instance_m"] = instance_m_list
GNN["predicted_val"] = predicted_val
GNN["loss"] = loss
GNN["%error"] = error
GNN["%abserror"] = abserror
GNN["train_step"] = train_step
GNN["nop"] = tf.no_op()
return GNN
def build_network(d):
# Hyperparameters
learning_rate = 2e-5
parameter_l2norm_scaling = 1e-10
global_norm_gradient_clipping_ratio = 0.65
# Define GNN dictionary
GNN = {}
# Define placeholder for result values (one per problem)
instance_val = tf.placeholder( tf.float32, [ None ], name = "instance_val" )
instance_target = tf.placeholder( tf.int32, [ None ], name = "instance_target" )
# Define INV, a tf function to exchange positive and negative literal embeddings
def INV(Lh):
l = tf.shape(Lh)[0]
n = tf.div(l,tf.constant(2))
# Send messages from negated literals to positive ones, and vice-versa
Lh_pos = tf.gather( Lh, tf.range( tf.constant( 0 ), n ) )
Lh_neg = tf.gather( Lh, tf.range( n, l ) )
Lh_inverted = tf.concat( [ Lh_neg, Lh_pos ], axis = 0 )
return Lh_inverted
#end
# Define Graph neural network
gnn = GraphNN(
{
"N": d
},
{
"M": ("N","N"),
"S": ("N","N")
},
{
"Nmsg": ("N","N")
},
{
"N": [
{
"mat": "M",
"var": "N"
},
{
"mat": "M",
"transpose?": True,
"var": "N"
},
{
"mat": "S"
}
]
},
name="Dijkstra",
float_dtype = tf.float32
)
# Define L_vote
N_vote_MLP = Mlp(
layer_sizes = [ d for _ in range(2) ],
activations = [ tf.nn.relu for _ in range(2) ],
output_size = 1,
name = "N_vote",
name_internal_layers = True,
kernel_initializer = tf.contrib.layers.xavier_initializer(),
bias_initializer = tf.zeros_initializer()
)
# Compute the number of variables
n = tf.shape( gnn.matrix_placeholders["M"] )[0]
# Compute number of problems
p = tf.shape( instance_val )[0]
# Get the last embeddings
N_n = gnn.last_states["N"].h
N_vote = N_vote_MLP( N_n )
# Reorganize votes' result to obtain a prediction for each problem instance
def _vote_while_cond(i, predicted_val):
return tf.less( i, p )
#end _vote_while_cond
def _vote_while_body(i, predicted_val):
# Gather the target node for that problem
final_node = N_vote[ instance_target[i] ]
# Concatenate positive and negative literals and average their vote values
problem_predicted_val = tf.reshape( final_node, shape = [] )
# Update TensorArray
predicted_val = predicted_val.write( i, problem_predicted_val )
return tf.add( i, tf.constant( 1 ) ), predicted_val
#end _vote_while_body
predicted_val = tf.TensorArray( size = p, dtype = tf.float32 )
_, predicted_val = tf.while_loop(
_vote_while_cond,
_vote_while_body,
[ tf.constant( 0, dtype = tf.int32 ), predicted_val ]
)
predicted_val = predicted_val.stack()
# Define loss, %error
predict_costs = tf.losses.mean_squared_error( labels = instance_val, predictions = predicted_val )
predict_cost = tf.reduce_mean( predict_costs )
# %Error
abserror = tf.reduce_mean( tf.divide( tf.abs( tf.subtract( instance_val, predicted_val ) ), instance_val ) )
error = tf.reduce_mean( tf.divide( tf.subtract( instance_val, predicted_val ), instance_val ) )
vars_cost = tf.zeros([])
tvars = tf.trainable_variables()
for var in tvars:
vars_cost = tf.add( vars_cost, tf.nn.l2_loss( var ) )
#end for
loss = tf.add( predict_cost, tf.multiply( vars_cost, parameter_l2norm_scaling ) )
optimizer = tf.train.AdamOptimizer( name = "Adam", learning_rate = learning_rate )
grads, _ = tf.clip_by_global_norm( tf.gradients( loss, tvars ), global_norm_gradient_clipping_ratio )
train_step = optimizer.apply_gradients( zip( grads, tvars ) )
GNN["gnn"] = gnn
GNN["instance_val"] = instance_val
GNN["instance_target"] = instance_target
GNN["predicted_val"] = predicted_val
GNN["loss"] = loss
GNN["%error"] = error
GNN["%abserror"] = abserror
GNN["train_step"] = train_step
GNN["nop"] = tf.no_op()
return GNN
def main():
dataset = PreProcessing("wine_dataset.txt")
dataset.normalize(ignore_first_column=True)
dataset.switch_first_last_column()
dataset.normalize_class()
train, test = training.holdout(0.7, dataset.normalized_dataframe)
nn = Mlp(13, 10, 3, n_hidden_layers=1)
nn.backpropagation(train.values.tolist(), eta=0.5)
example = test.values.tolist()
print(len(example))
input()
#print(example)
#print(example[17])
#feed example
nn.feed_forward(example[0][:(-1 * 3)])
print(example[0])
nn.show_class()
nn.feed_forward(example[40][:(-1 * 3)])
print(example[40])
print(test.iloc[[40]].values.tolist())
input()
nn.show_class()
nn.feed_forward(example[31][:(-1 * 3)])
print(example[31])
nn.show_class()
print(training.accuracy(nn, test, n_classes=3))
"""
