def __init__(self, input_size, output_size, hidden_layer_sizes):
self.learning_rate = 0.1
self.input_layer = InputLayer(input_size)
self.output_layer = OutputLayer(output_size)
self.hidden_layers = [
HiddenLayer(hidden_layer_size)
for hidden_layer_size in hidden_layer_sizes
]
for i, hidden_layer in enumerate(self.hidden_layers):
if i == 0 and i == len(self.hidden_layers) - 1:
hidden_layer.initialize(self.input_layer, self.output_layer)
elif i == 0:
hidden_layer.initialize(self.input_layer,
self.hidden_layers[i + 1])
elif i == len(self.hidden_layers) - 1:
hidden_layer.initialize(self.hidden_layers[i - 1],
self.output_layer)
else:
hidden_layer.initialize(self.hidden_layers[i - 1],
self.hidden_layers[i + 1])
if (len(self.hidden_layers)):
self.output_layer.initialize(self.hidden_layers[-1])
else:
self.output_layer.initialize(self.input_layer)
def main():
print("{}; lr: {}; layer size: {}".format(conf["layer_name"],
conf["learning_rate"],
conf["num_classes"]))
output_layer = OutputLayer(conf["layer_name"],
conf["lower_layer"],
conf["lower_layer_size"],
conf["num_classes"],
conf["learning_rate"])
# init weights
output_layer.init_weights(None)
MAX_MESSAGE_LENGTH = 128 * 1024 *1024
server = grpc.server(futures.ThreadPoolExecutor(max_workers=4), options=
[('grpc.max_send_message_length', MAX_MESSAGE_LENGTH),
('grpc.max_receive_message_length', MAX_MESSAGE_LENGTH)])
nn_pb2_grpc.add_LayerDataExchangeServicer_to_server(output_layer,
server)
server.add_insecure_port(conf["listen_on"])
server.start()
# idle
try:
while True:
time.sleep(24*60*60)
except KeyboardInterrupt:
server.stop(0)
class FeedForwardNeuralNetwork:
def __init__(self, input_size, output_size, hidden_layer_sizes):
self.learning_rate = 0.1
self.input_layer = InputLayer(input_size)
self.output_layer = OutputLayer(output_size)
self.hidden_layers = [
HiddenLayer(hidden_layer_size)
for hidden_layer_size in hidden_layer_sizes
]
for i, hidden_layer in enumerate(self.hidden_layers):
if i == 0 and i == len(self.hidden_layers) - 1:
hidden_layer.initialize(self.input_layer, self.output_layer)
elif i == 0:
hidden_layer.initialize(self.input_layer,
self.hidden_layers[i + 1])
elif i == len(self.hidden_layers) - 1:
hidden_layer.initialize(self.hidden_layers[i - 1],
self.output_layer)
else:
hidden_layer.initialize(self.hidden_layers[i - 1],
self.hidden_layers[i + 1])
if (len(self.hidden_layers)):
self.output_layer.initialize(self.hidden_layers[-1])
else:
self.output_layer.initialize(self.input_layer)
def predict(self, input_arr):
self.input_layer.set_values(input_arr)
for hidden_layer in self.hidden_layers:
hidden_layer.feed_forward()
self.output_layer.feed_forward()
return self.output_layer.values
def train(self, input_arr, target_arr):
self.predict(input_arr)
self.output_layer.calculate_errors(target_arr)
for hidden_layer in reversed(self.hidden_layers):
hidden_layer.calculate_errors()
self.output_layer.adjust_parameters(self.learning_rate)
for hidden_layer in reversed(self.hidden_layers):
hidden_layer.adjust_parameters(self.learning_rate)
def build_symbolic_graph(self):
# allocate symbolic variables (defaults to floatX)
self.x = T.matrix('x')
if self.target_is_int:
self.y = T.ivector('y')
else:
self.y = T.matrix('y')
# assemble layers
self.hidden_layers = []
this_input = self.x
this_nin = self.n_in
for i in xrange(len(self.n_h)):
hidden_layer = HiddenLayer(self.rng,
input=this_input,
n_in=this_nin,
n_out=self.n_h[i],
activation=self.activations[i],
dropout=self.dropout,
params=self.params_init[i])
this_input = hidden_layer.output
this_nin = self.n_h[i]
self.hidden_layers.append(hidden_layer)
self.output_layer = OutputLayer(self.rng,
input=self.hidden_layers[-1].output,
n_in=this_nin,
n_out=self.n_out,
non_linearities=self.activations[-1],
params=self.params_init[-1])
self.layers = self.hidden_layers + [self.output_layer]
def __init__(self, args):
super(SIGN, self).__init__()
num_convs = args.num_convs
dense_dims = args.dense_dims
infeat_dim = args.infeat_dim
hidden_dim = args.hidden_dim
self.num_convs = num_convs
cut_dist = args.cut_dist
num_angle = args.num_angle
merge_b2b = args.merge_b2b
merge_b2a = args.merge_b2a
activation = args.activation
num_heads = args.num_heads
feat_drop = args.feat_drop
self.input_layer = SpatialInputLayer(hidden_dim,
cut_dist,
activation=F.relu)
self.atom2bond_layers = nn.LayerList()
self.bond2bond_layers = nn.LayerList()
self.bond2atom_layers = nn.LayerList()
for i in range(num_convs):
if i == 0:
atom_dim = infeat_dim
else:
atom_dim = hidden_dim * num_heads if 'cat' in merge_b2a else hidden_dim
bond_dim = hidden_dim * num_angle if 'cat' in merge_b2b else hidden_dim
self.atom2bond_layers.append(
Atom2BondLayer(atom_dim,
bond_dim=hidden_dim,
activation=activation))
self.bond2bond_layers.append(
Bond2BondLayer(hidden_dim,
hidden_dim,
num_angle,
feat_drop,
merge=merge_b2b,
activation=None))
self.bond2atom_layers.append(
Bond2AtomLayer(bond_dim,
atom_dim,
hidden_dim,
num_heads,
feat_drop,
merge=merge_b2a,
activation=activation))
self.pipool_layer = PiPoolLayer(hidden_dim, hidden_dim, num_angle)
self.output_layer = OutputLayer(hidden_dim, dense_dims)
def __init__(self, c, T, n, g, p, control_str):
super(STGCN, self).__init__()
self.control_str = control_str
self.num_layers = len(control_str)
self.num_nodes = n
self.layers = nn.ModuleList()
self.dropout = nn.Dropout(p)
# Temporal conv kernel size set to 3
self.Kt = 3
# c_index controls the change of channels
c_index = 0
num_temporal_layers = 0
# construct network based on 'control_str'
for i in range(self.num_layers):
layer_i = control_str[i]
# Temporal Layer
if layer_i == 'T':
self.layers.append(
TemporalConvLayer_Residual(c[c_index],
c[c_index + 1],
kernel=self.Kt))
c_index += 1
num_temporal_layers += 1
# Spatio Layer
if layer_i == 'S':
self.layers.append(
SpatialConvLayer(c[c_index], c[c_index + 1], g))
c_index += 1
# Norm Layer
if layer_i == 'N':
# TODO: The meaning of this layernorm
self.layers.append(nn.LayerNorm([n, c[c_index]]))
# c[c_index] is the last element in 'c'
# T - (self.Kt - 1) * num_temporal_layers returns the timesteps after previous temporal layer transformations cuz dialiation = 1
self.output = OutputLayer(c[c_index],
T - (self.Kt - 1) * num_temporal_layers,
self.num_nodes)
for layer in self.layers:
layer = layer.cuda()
def __init__(self, c, T, n, g, p, control_str):
super(STGCN_WAVE, self).__init__()
self.control_str = control_str
self.num_layers = len(control_str)
self.layers = nn.ModuleList()
self.dropout = nn.Dropout(p)
# c_index controls the change of channels
c_index = 0
# diapower controls the change of dilations in temporal CNNs where dilation = 2^diapower
diapower = 0
# construct network based on 'control_str'
for i in range(self.num_layers):
layer_i = control_str[i]
# Temporal Layer
if layer_i == 'T':
# Notice: dialation = 2^diapower (e.g. 1, 2, 4, 8) so that
# T_out = T_in - dialation * (kernel_size - 1) - 1 + 1
# if padding = 0 and stride = 1
self.layers.append(
TemporalConvLayer_Residual(c[c_index],
c[c_index + 1],
dia=2**diapower))
diapower += 1
c_index += 1
# Spatio Layer
if layer_i == 'S':
self.layers.append(SpatialConvLayer(c[c_index], c[c_index], g))
# Norm Layer
if layer_i == 'N':
# TODO: The meaning of this layernorm
self.layers.append(nn.LayerNorm([n, c[c_index]]))
# c[c_index] is the last element in 'c'
# T + 1 - 2**(diapower) returns the timesteps after previous temporal layer transformations
# 'n' will be needed by LayerNorm inside of the OutputLayer
self.output = OutputLayer(c[c_index], T + 1 - 2**(diapower), n)
for layer in self.layers:
layer = layer.cuda()
def add_output(self,
target_shape,
activation='sigmoid',
cost_function='sse',
init_type='random',
alpha_regularization=0):
layer_out = OutputLayer(None,
self.crt_layer,
self.bias,
shape=target_shape,
activation=activation,
init_type=init_type,
cost_function=cost_function,
regularization_type=self.regularization,
alpha_regularization=alpha_regularization)
self.crt_layer.next_layer = layer_out
self.crt_layer = layer_out
self.layers.append(layer_out)
self.cost_function = cost_function
return self
def build_symbolic_graph(self):
self.theano_rng = RandomStreams(self.rng.randint(100))
# allocate symbolic variables (defaults to floatX)
self.x = T.matrix('x')
if not self.denoising == None:
self.corrupted = self.get_corrupted_input(self.x, self.denoising)
else:
self.corrupted = self.x * T.ones_like(self.x)
# assemble layers
self.hidden_layer = HiddenLayer(self.rng,
input=self.corrupted,
n_in=self.n_in,
n_out=self.n_h,
activation=self.activations[0],
dropout=self.dropout,
params=self.params_init[0])
if self.tie_weights:
if self.params_init[1] == None:
self.params_init[1] = [
self.hidden_layer.params[0].T,
theano.shared(np.zeros(self.n_in,
dtype=theano.config.floatX),
name='b2')
]
else:
self.params_init[1] = [
self.hidden_layer.params[0].T, self.params_init[1][1]
]
self.output_layer = OutputLayer(self.rng,
input=self.hidden_layer.output,
n_in=self.n_h,
n_out=self.n_in,
non_linearities=self.activations[-1],
params=self.params_init[1])
self.layers = [self.hidden_layer, self.output_layer]
def build_symbolic_graph(self):
# allocate symbolic variables (defaults to floatX)
self.x1 = T.matrix('x1') # context
self.x2 = T.matrix('x2') # partial trace
if self.target_is_int:
self.y = T.ivector('y') # next pixel
else:
self.y = T.matrix('y')
self.z = T.matrix('z') #reconstruction of trace (full trace)
# make bad outputs
bad_outputs = make_bad_outs(self.n_trace, self.x2)
# assemble layers
self.hidden_layers = []
hidden_layer_1a = HiddenLayer(self.rng,
input=self.x1,
n_in=self.n_context,
n_out=self.n_h[0],
dropout=self.dropout,
activation=self.activations[0],
params=self.params_init[0])
hidden_layer_1b = HiddenLayer(self.rng,
input=self.x2,
n_in=self.n_trace,
n_out=self.n_h[1],
dropout=self.dropout,
activation=self.activations[1],
params=self.params_init[1])
input_to_h2 = T.concatenate(
[hidden_layer_1a.output, hidden_layer_1b.output], axis=1)
hidden_layer_2 = HiddenLayer(self.rng,
input=input_to_h2,
n_in=self.n_h[0] + self.n_h[1],
n_out=self.n_h[2],
dropout=self.dropout,
activation=self.activations[2],
params=self.params_init[2])
recon_layer = OutputLayer(self.rng,
input=hidden_layer_2.output,
n_in=self.n_h[2],
n_out=self.n_recon,
non_linearities=self.activations[3],
params=params_init[3])
if not self.n_out == 0:
pred_layer = OutputLayer(self.rng,
input=recon_layer.output,
n_in=self.n_recon,
n_out=self.n_out,
bad_output=bad_outputs,
non_linearities=self.activations[4],
params=self.params_init[4])
if self.n_out == 0:
self.layers = [
hidden_layer_1a, hidden_layer_1b, hidden_layer_2, recon_layer
]
else:
self.layers = [
hidden_layer_1a, hidden_layer_1b, hidden_layer_2, recon_layer,
pred_layer
]
fp = open(filepath, "w")
config = 0
for epoch in epochs:
for lr in learning_rates:
for reg in regularizations:
for alpha in momentums:
mean_loss = 0
mean_validation = 0
for i in range(k):
model = NeuralNetwork()
model.add(InputLayer(10))
model.add(DenseLayer(50, fanin=10))
model.add(DenseLayer(30, fanin=50))
model.add(OutputLayer(2, fanin=30))
model.compile(size, epoch, lr / size, None, reg, alpha,
"mean_squared_error")
(train, val) = data.kfolds(index=i, k=k)
mean_loss = mean_loss + model.fit(train[0], train[1])[-1]
mean_validation = mean_validation + model.evaluate(
val[0], val[1])
fp.write("{}, {}, {}, {}, {}, {}, {}\n".format(
config, epoch, lr, reg, alpha, mean_loss / k,
mean_validation / k))
config = config + 1
fp.close()
import numpy as np
import dataset as ds
from neural_networks import NeuralNetwork
from layers import InputLayer, OutputLayer, DenseLayer
import matplotlib.pyplot as plt
data = ds.MonksDataset()
my_model = NeuralNetwork()
my_model.add(InputLayer(17))
my_model.add(DenseLayer(10, fanin=17, activation="sigmoid"))
my_model.add(OutputLayer(1, fanin=10, activation="sigmoid"))
my_model.compile(122, 600, 0.075, None, 0.0001, 0, "mean_squared_error")
(loss, test_loss, accuracy, test_accuracy) = my_model.fit_monks(
data.train_data_patterns, data.train_data_targets, data.test_data_patterns,
data.test_data_targets)
print("Loss: {}".format(loss[-1]))
print("Test Loss: {}".format(test_loss[-1]))
print("Accuracy: {}".format(accuracy[-1]))
print("Test accuracy: {}".format(test_accuracy[-1]))
plot1 = plt.figure(1)
plt.plot(loss)
plt.plot(test_loss, "--")
plot2 = plt.figure(2)