def main():
# generate some dummy data for testing
manager = Data()
num_examples = 10**4
max_val = 1
train_batch_size = 16
train_size = int(num_examples/2)
eval_size = int(num_examples/2)
X, y = manager.create_data_set(num_of_examples=num_examples, max_val=max_val,
discriminator=lambda x: max_val*(1/(1+np.exp(-x)) + 1/(1+np.exp(x**2)))-max_val/2,
one_hot=False, plot_data=False, load_saved_data=True,
filename='dataset.npy')
train_examples, train_labels = X[0:train_size, :], y[0:train_size]
eval_examples, eval_labels = X[train_size:, :], y[train_size:]
print('train examples = {}, train labels = {}, eval examples = {}, eval labels = {}'.format(train_examples.shape,
train_labels.shape,
eval_examples.shape,
eval_labels.shape))
# get some small train batch
indices = np.random.randint(low=0,high=train_size,size=train_batch_size)
train_batch_examples, train_batch_labels = X[indices,:], y[indices]
# start by defining your default graph
graph = GRAPH()
graph.getDefaultGraph()
# declare your placeholders, to provide your inputs
# print(int(train_batch_examples.shape[1]))
input_features = placeholder(shape=(train_batch_size, int(train_batch_examples.shape[1])))
input_labels = placeholder(shape=(train_batch_size))
"""
Method #3
"""
# this is defined using layers
features = fully_connected(features=input_features, units=32)
features = relu(features)
features = fully_connected(features=features, units=64)
features = relu(features)
features = fully_connected(features=features, units=128)
features = relu(features)
features = fully_connected(features=features, units=64)
features = relu(features)
features = fully_connected(features=features, units=32)
features = relu(features)
features = fully_connected(features=features, units=2)
logits = softmax_classifier(features)
loss = CrossEntropyLoss(softmax_logits=logits, labels=input_labels)
# compile and run
graph.graph_compile(function=loss, verbose=True)
loss = graph.run(input_matrices={input_features: train_batch_examples, input_labels: train_batch_labels})
print(loss, logits.output.shape)
def main():
# generate some dummy data for testing
manager = Data()
num_examples = 10**4
max_val = 1
train_batch_size = 16
train_size = int(num_examples / 2)
eval_size = int(num_examples / 2)
X, y = manager.create_data_set(num_of_examples=num_examples,
max_val=max_val,
discriminator=lambda x: max_val *
(1 / (1 + np.exp(-x)) + 1 /
(1 + np.exp(x**2))) - max_val / 2,
one_hot=False,
plot_data=False,
load_saved_data=True,
filename='dataset.npy')
train_examples, train_labels = X[0:train_size, :], y[0:train_size]
eval_examples, eval_labels = X[train_size:, :], y[train_size:]
print(
'train examples = {}, train labels = {}, eval examples = {}, eval labels = {}'
.format(train_examples.shape, train_labels.shape, eval_examples.shape,
eval_labels.shape))
# get some small train batch
indices = np.random.randint(low=0, high=train_size, size=train_batch_size)
train_batch_examples, train_batch_labels = X[indices, :], y[indices]
# start by defining your default graph
graph = GRAPH()
graph.getDefaultGraph()
# declare your placeholders, to provide your inputs
# print(int(train_batch_examples.shape[1]))
input_features = placeholder(shape=(train_batch_size,
int(train_batch_examples.shape[1])))
input_labels = placeholder(shape=(train_batch_size))
"""
Method #2
"""
# define a small network
hidden_units = [32, 64, 128, 256, 128, 64, 32, 2]
all_weights, all_biases = [], []
prev_units = train_batch_examples[0, :].shape[
0] # this is the number of features in the inputs
# print(prev_units)
# declare all the weight and biases
for units in hidden_units:
all_weights.append(
Matrix(initial_value=np.random.uniform(
low=-0.1, high=0.1, size=(prev_units, units))))
all_biases.append(Matrix(initial_value=np.ones(shape=units)))
prev_units = units
# for weights in all_weights:
# print(weights.matrix.shape)
# calculate the features
features = input_features
for weights, bias in zip(all_weights, all_biases):
features = add(dot(features, weights), bias)
# calculate the logits
logits = softmax_classifier(features)
# and the loss
loss = CrossEntropyLoss(softmax_logits=logits, labels=input_labels)
# compile your graph
graph.graph_compile(function=loss, verbose=True)
# this is kind of sess.run()
loss = graph.run(input_matrices={
input_features: train_batch_examples,
input_labels: train_batch_labels
})
print(loss, logits.output.shape)
def main():
# generate some dummy data for testing
manager = Data()
num_examples = 10**4
max_val = 1
train_batch_size = 16
train_size = int(num_examples / 2)
eval_size = int(num_examples / 2)
X, y = manager.create_data_set(num_of_examples=num_examples,
max_val=max_val,
discriminator=lambda x: max_val *
(1 / (1 + np.exp(-x)) + 1 /
(1 + np.exp(x**2))) - max_val / 2,
one_hot=False,
plot_data=False,
load_saved_data=True,
filename='dataset.npy')
train_examples, train_labels = X[0:train_size, :], y[0:train_size]
eval_examples, eval_labels = X[train_size:, :], y[train_size:]
print(
'train examples = {}, train labels = {}, eval examples = {}, eval labels = {}'
.format(train_examples.shape, train_labels.shape, eval_examples.shape,
eval_labels.shape))
# get some small train batch
indices = np.random.randint(low=0, high=train_size, size=train_batch_size)
train_batch_examples, train_batch_labels = X[indices, :], y[indices]
# start by defining your default graph
graph = GRAPH()
graph.getDefaultGraph()
# declare your placeholders, to provide your inputs
# print(int(train_batch_examples.shape[1]))
input_features = placeholder(shape=(train_batch_size,
int(train_batch_examples.shape[1])))
input_labels = placeholder(shape=(train_batch_size))
"""
Method #1
"""
# declare all the weights and biases
weights1 = Matrix(
initial_value=np.random.uniform(low=-0.1, high=0.1, size=(2, 32)))
bias1 = Matrix(
initial_value=np.random.uniform(low=-0.1, high=0.1, size=(32)))
weights2 = Matrix(
initial_value=np.random.uniform(low=-0.1, high=0.1, size=(32, 64)))
bias2 = Matrix(
initial_value=np.random.uniform(low=-0.1, high=0.1, size=(64)))
weights3 = Matrix(
initial_value=np.random.uniform(low=-0.1, high=0.1, size=(64, 128)))
bias3 = Matrix(
initial_value=np.random.uniform(low=-0.1, high=0.1, size=(128)))
weights4 = Matrix(
initial_value=np.random.uniform(low=-0.1, high=0.1, size=(128, 64)))
bias4 = Matrix(
initial_value=np.random.uniform(low=-0.1, high=0.1, size=(64)))
weights5 = Matrix(
initial_value=np.random.uniform(low=-0.1, high=0.1, size=(64, 32)))
bias5 = Matrix(
initial_value=np.random.uniform(low=-0.1, high=0.1, size=(32)))
weights6 = Matrix(
initial_value=np.random.uniform(low=-0.1, high=0.1, size=(32, 2)))
bias6 = Matrix(
initial_value=np.random.uniform(low=-0.1, high=0.1, size=(2)))
# calculate some features
features = add(dot(input_features, weights1), bias1)
features = softmax_classifier(features)
features = add(dot(features, weights2), bias2)
features = softmax_classifier(features)
features = add(dot(features, weights3), bias3)
features = softmax_classifier(features)
features = add(dot(features, weights4), bias4)
features = softmax_classifier(features)
features = add(dot(features, weights5), bias5)
features = softmax_classifier(features)
features = add(dot(features, weights6), bias6)
logits = softmax_classifier(features)
loss = CrossEntropyLoss(softmax_logits=logits, labels=input_labels)
# compile and run
graph.graph_compile(function=loss, verbose=True)
loss = graph.run(input_matrices={input_features: train_batch_examples})
print(loss, logits.output.shape)
def main():
# generate some dummy data for testing
manager = Data()
num_examples = 10**4
max_val = 1
train_batch_size = 16
train_size = int(num_examples / 2)
eval_size = int(num_examples / 2)
X, y = manager.create_data_set(num_of_examples=num_examples,
max_val=max_val,
discriminator=lambda x: max_val *
(1 / (1 + np.exp(-x)) + 1 /
(1 + np.exp(x**2))) - max_val / 2,
one_hot=False,
plot_data=False,
load_saved_data=True,
filename='dataset.npy')
train_examples, train_labels = X[0:train_size, :], y[0:train_size]
eval_examples, eval_labels = X[train_size:, :], y[train_size:]
print(
'train examples = {}, train labels = {}, eval examples = {}, eval labels = {}'
.format(train_examples.shape, train_labels.shape, eval_examples.shape,
eval_labels.shape))
# get some small train batch
indices = np.random.randint(low=0, high=train_size, size=train_batch_size)
train_batch_examples, train_batch_labels = X[indices, :], y[indices]
# start by defining your default graph
graph = GRAPH()
graph.getDefaultGraph()
# declare your placeholders, to provide your inputs
# print(int(train_batch_examples.shape[1]))
input_features = placeholder(shape=(train_batch_size,
int(train_batch_examples.shape[1])))
input_labels = placeholder(shape=(train_batch_size))
# declare all the weights and biases
weights1 = Matrix(
initial_value=np.random.uniform(low=-0.1, high=0.1, size=(2, 32)))
bias1 = Matrix(initial_value=np.ones(shape=(train_batch_size, 32)))
weights2 = Matrix(
initial_value=np.random.uniform(low=-0.1, high=0.1, size=(32, 64)))
bias2 = Matrix(initial_value=np.ones(shape=(train_batch_size, 64)))
weights3 = Matrix(
initial_value=np.random.uniform(low=-0.1, high=0.1, size=(64, 128)))
bias3 = Matrix(initial_value=np.ones(shape=(train_batch_size, 128)))
weights4 = Matrix(
initial_value=np.random.uniform(low=-0.1, high=0.1, size=(128, 64)))
bias4 = Matrix(initial_value=np.ones(shape=(train_batch_size, 64)))
weights5 = Matrix(
initial_value=np.random.uniform(low=-0.1, high=0.1, size=(64, 32)))
bias5 = Matrix(initial_value=np.ones(shape=(train_batch_size, 32)))
weights6 = Matrix(
initial_value=np.random.uniform(low=-0.1, high=0.1, size=(32, 2)))
bias6 = Matrix(initial_value=np.ones(shape=(train_batch_size, 2)))
# calculate some features
features = add(dot(input_features, weights1), bias1)
features = relu(features)
features = add(dot(features, weights2), bias2)
features = relu(features)
features = add(dot(features, weights3), bias3)
features = relu(features)
features = add(dot(features, weights4), bias4)
features = relu(features)
features = add(dot(features, weights5), bias5)
features = relu(features)
logits = add(dot(features, weights6), bias6)
loss = Softmax_with_CrossEntropyLoss(logits=logits, labels=input_labels)
graph.graph_compile(function=loss, verbose=True)
for i in range(100000):
loss_value = graph.run(function=loss,
input_matrices={
input_features: train_batch_examples,
input_labels: train_batch_labels
})
if i % 1000 == 0:
print('iteration {}, batch loss_val = {}'.format(i, loss_value))
# calculate all the gradients in the network due to bad predictions
graph.gradients(function=loss)
graph.update(learn_rate=3e-3)
pass
def main():
# generate some dummy data for testing
manager = Data()
num_examples = 10**4
max_val = 4
train_batch_size = 32
train_size = int(num_examples / 2)
eval_size = int(num_examples / 2)
# discriminator = (1/(1+np.exp(-x)+1/(1+np.exp(x))))
X, y = manager.create_data_set(num_of_examples=num_examples,
max_val=max_val,
discriminator=lambda x: max_val *
(np.cos(np.sin(x**2))) - max_val / 2,
one_hot=False,
plot_data=True,
load_saved_data=False,
filename='dataset.npy')
train_examples, train_labels = X[0:train_size, :], y[0:train_size]
eval_examples, eval_labels = X[train_size:, :], y[train_size:]
print(
'train examples = {}, train labels = {}, eval examples = {}, eval labels = {}'
.format(train_examples.shape, train_labels.shape, eval_examples.shape,
eval_labels.shape))
# start by defining your default graph
graph = GRAPH()
graph.getDefaultGraph()
# declare your placeholders, to provide your inputs
# print(int(train_batch_examples.shape[1]))
input_features = placeholder(shape=(train_batch_size,
int(train_examples.shape[1])))
input_labels = placeholder(shape=(train_batch_size))
"""
Method #3
"""
# this is defined using layers
layer1 = fully_connected(features=input_features, units=32)
layer1 = relu(layer1)
layer2 = fully_connected(features=layer1, units=64)
layer2 = relu(layer2)
# layer2 = dropout(features=layer2, drop_rate=0.5)
layer2_1 = fully_connected(features=layer2, units=64)
layer2_1 = relu(layer2_1)
layer2_2 = fully_connected(features=layer2_1, units=64)
layer2_2 = relu(layer2_2)
# a recurrent connection
layer2_2 = add(layer2_2, layer2)
layer3 = fully_connected(features=layer2_2, units=128)
layer3 = relu(layer3)
layer4 = fully_connected(features=layer3, units=128)
layer4 = relu(layer4)
layer5 = fully_connected(features=layer4, units=128)
layer5 = relu(layer5)
# a recurrent connection
layer5 = add(layer5, layer3)
layer6 = fully_connected(features=layer5, units=64)
layer6 = relu(layer6)
# layer6 = dropout(features=layer6, drop_rate=0.2)
layer6_1 = fully_connected(features=layer6, units=64)
layer6_1 = relu(layer6_1)
layer6_2 = fully_connected(features=layer6_1, units=64)
layer6_2 = relu(layer6_2)
# a recurrent connection
layer6_2 = add(layer6_2, layer6)
layer7 = fully_connected(features=layer6_2, units=32)
layer7 = relu(layer7)
# layer7 = dropout(features=layer7, drop_rate=0.5)
logits = fully_connected(features=layer7, units=2)
loss = Softmax_with_CrossEntropyLoss(logits=logits, labels=input_labels)
# compile and run (always compile with the loss function, even if you don't use it!!!)
graph.graph_compile(function=loss, verbose=True)
# run a training loop
# all_W = []
# for layer in graph.forward_feed_order:
# if layer.is_trainable:
# all_W.append([layer.W, layer.bias])
def evaluate(batch_examples, batch_labels, mode='train'):
loss_val = graph.run(function=graph.loss,
input_matrices={
input_features: batch_examples,
input_labels: batch_labels
},
mode=mode)
accuracy = 100 / train_batch_size * np.sum(batch_labels == np.argmax(
np.exp(logits.output) /
np.sum(np.exp(logits.output), axis=1)[:, None],
axis=1))
return [loss_val, accuracy]
def training_loop(iterations):
for m in xrange(iterations):
# get some small train batch
indices = np.random.randint(low=0,
high=train_size,
size=train_batch_size)
train_batch_examples, train_batch_labels = X[
indices, :], y[indices]
# get the system outputs
[loss_val,
accuracy] = evaluate(batch_examples=train_batch_examples,
batch_labels=train_batch_labels)
# logits_val = graph.run(function=logits, input_matrices={input_features: train_batch_examples,
# input_labels: train_batch_labels})
# print(logits_val.shape)
if m % 1000 == 0:
# print('-----------')
print(
'log: at iteration #{}, train batch loss = {}, train batch accuracy = {}%'
.format(m, loss_val, accuracy)) #, logits.output.shape)
# print('batch accuracy = {}%'.format(m, accuracy)) #, logits.output.shape)
# print('-----------')
# calculate some evaluation accuracy
if m != 0 and m % 20000 == 0:
print('\n---------Evaluating Now-----------')
eval_loss, eval_accuracy = (0, 0)
steps = eval_size // train_batch_size
for k in range(steps):
eval_indices = range(k * train_batch_size,
(k + 1) * train_batch_size)
eval_batch_loss, eval_batch_accuracy = evaluate(
batch_examples=eval_examples[eval_indices, :],
batch_labels=eval_labels[eval_indices],
mode='test')
eval_loss += eval_batch_loss
eval_accuracy += eval_batch_accuracy
print('log: evaluation loss = {}, evaluation accuracy = {}%'.
format(eval_loss / steps, eval_accuracy / steps))
print('------------------------------------\n')
# do some testing
if m != 0 and m % 50000 == 0:
test_model()
# run and calculate the gradients w.r.t to the loss function
graph.gradients(function=loss)
# check something
# if m > 1:
# graph.gradients(function=logits)
# pass
# update the weights
graph.update(learn_rate=1e-2)
def test_model():
# now finally testing the model
x_ = np.linspace(start=-max_val, stop=max_val, num=64)
# we want to evaluate this thing
test_set = np.asarray([(x, y) for x in x_ for y in x_],
dtype=np.float64)
all_predictions = []
print('\n---------Testing Now-----------')
print('log: your test set is {}'.format(test_set.shape))
steps = 64 * 64 // train_batch_size
for k in range(steps):
test_indices = range(k * train_batch_size,
(k + 1) * train_batch_size)
test_logits = graph.run(
function=logits,
input_matrices={input_features: test_set[test_indices, :]},
mode='test')
test_logits -= np.max(test_logits)
exps = np.exp(test_logits)
softmaxed = exps / np.sum(exps, axis=1)[:, None]
# print(softmaxed)
predictions = np.argmax(softmaxed, axis=1)
# print(predictions)
# print(predictions)
# fill up predictions
all_predictions.append(predictions)
print('------------------------------------\n')
predictions = np.hstack(all_predictions)
# print(predictions.shape)
red = test_set[predictions == 0]
green = test_set[predictions == 1]
plot.scatter(green[:, 0], green[:, 1], color='g')
plot.scatter(red[:, 0], red[:, 1], color='r')
plot.show()
training_loop(iterations=1000000)
