def main():
def f(x):
return 1 / (1 + math.e**(-x))
def f_prime(x):
return x * (1 - x)
net = NeuralNetwork(f,
f_prime,
[[0.2, -0.3], [0.4, 0.1], [-0.5, 0.2]],
[[-0.3], [-0.2]],
bias_vals=[1, 1],
bias_weights=[[-0.4, 0.2], [0.1]])
net = NeuralNetwork(f,
f_prime,
[[0.1, 0, 0.3], [-0.20, 0.2, -0.4]],
[[-0.4, 0.2], [0.1, -0.1], [0.6, -0.2]],
bias_vals=[1, 1],
bias_weights=[[0.1, 0.2, 0.5], [-0.1, 0.6]])
print(net.weights, 'init')
net.train([0.6, 0.1], [1, 0], learning_rate=0.1)
net.train([0.6, 0.1], [1, 0], learning_rate=0.1)
# net.train([1, 0, 1], [1], learning_rate=0.9)
print(net.weights)
def next_generation(self, brain):
self.txt_generation += 1
self.g_label.config(text=f'Generation: {self.txt_generation}')
for p in self.players:
p.player.destroy()
self.players = []
for i in range(150):
new_brain = NeuralNetwork(8, 3, 1)
if brain is not None:
new_brain = NeuralNetwork.copy_from(brain)
player = Player(self.root, new_brain, self.obstacle, self.coin)
new_brain.mutate()
self.players.append(player)
print('\nBrain Clone:')
print(f'W_IH: {new_brain.weights_ih.data}')
print(f'W_HO: {new_brain.weights_ho.data}')
print(f'B_H: {new_brain.bias_h.data}')
print(f'B_O: {new_brain.bias_o.data}')
self.coin.reset()
self.obstacle.reset()
def __init__(self, nn=None):
super().__init__()
self.image = pygame.image.load(
"assets/sprites/redbird-midflap.png").convert_alpha()
self.rect = self.image.get_rect()
# Stores if player is jumping or not.
self.isjump = 0
self.score = 0
self.fitness = 0
if nn != None:
self.nn = nn
else:
self.nn = NeuralNetwork(5, 8, 2)
#bird position
self.rect.x = 144
self.rect.y = randint(120, 200)
self.falling = pygame.image.load(
"assets/sprites/redbird-downflap.png").convert_alpha()
self.jumping = pygame.image.load(
"assets/sprites/redbird-upflap.png").convert_alpha()
# Force (v) up and mass m.
self.v = V
self.m = 1
self.stopped = False
def breed(self, mother: NeuralNetwork, father: NeuralNetwork) -> list:
"""
Create two children based on parents hyper parameters
:param mother: mother network
:param father: father network
:return: list of children
"""
children = []
for _ in range(2):
child = {}
# Loop through the parameters and pick params for the kid
for param in self.nn_param_choices:
child[param] = random.choice(
[mother.network[param], father.network[param]])
# Now create a network object
network = NeuralNetwork(self.nn_param_choices)
network.set_network_parameters(child)
# Randomly mutate some of the children
if self.mutate_chance > random.random():
network = self.mutate(network)
children.append(network)
return children
def __init__(self, x, y, network=None):
self.birds = [
pygame.image.load('images/birdUp.png'),
pygame.image.load('images/bird.png'),
pygame.image.load('images/birdDown.png')
]
self.birdx = x // 4
self.birdy = y // 2
self.gravity = 0.8
self.speed = -18
self.velocity = 0
self.birdImage = 1
self.birdRadius = 32
self.birdRadiusY = 16
self.width = x
self.height = y
self.pipeHeight = 317
self.pipeWidth = 52
self.score = 0
self.fitness = 0
self.gameScore = 0
if network != None:
self.network = network.copy()
else:
self.network = NeuralNetwork(5, 8, 1)
class Meeple:
def __init__(self,
input_size: int,
hidden_size: tuple,
output_size: int,
isHallow=False):
self.fitness = float("-inf")
self.brain: NeuralNetwork
self.isAlive = True
if isHallow:
self.brain = NeuralNetwork(input_size,
hidden_size,
output_size,
isHollow=True)
else:
self.brain = NeuralNetwork(input_size, hidden_size, output_size)
def clone(self): #->Meeple:
temp: Meeple = Meeple(self.brain.input_size - 1,
tuple([x - 1 for x in self.brain.hidden_size]),
self.brain.output_size,
isHallow=True)
temp.brain = self.brain.clone()
return temp
def crossover(self, parent2): #parent1:Meeple, parent2:Meeple) -> Meeple:
temp: Meeple = Meeple(self.brain.input_size - 1,
tuple([x - 1 for x in self.brain.hidden_size]),
self.brain.output_size,
isHallow=True)
temp.brain = self.brain.crossover(parent2.brain)
return temp
def test_all_hidden_layer_sizes_must_be_positive_int(self):
with self.assertRaises(ValueError):
NeuralNetwork(0, 10, 10, [-1])
NeuralNetwork(0, 10, 10, [1.23])
NeuralNetwork(0, 10, 10, [1, 2, 3])
NeuralNetwork(0, 10, 10, (1, 42))
def __init__(self,
initial_position_x=0,
initial_position_y=0,
search_skill=2,
fuel_level=0,
max_fuel=0,
search_agent_type=SearchAgentsType.human,
ID=0):
self.__brain = NeuralNetwork(
) # the controller of the SearchAgent class
self.__ID = ID
# self.__type = search_agent_type
self.__empty_fuel = False
self.__max_fuel = max_fuel
self.__fuel_level = fuel_level
self.__path_taken = OrderedDict()
self.__path_taken[0] = (0, 0)
self.__steps_taken = 0
self.__turns_taken = 0
# self.__falsepos_list = list() # a list of the coordinates of false positives that the SearchAgent class found
# self.__search_skill = search_skill
self.__targets_found = 0
def get_neuralnetwork_weights(self, training_data, test_data):
neuralnetwork = NeuralNetwork(weights=10,
learning_rate=0.001,
epochs=100000)
neuralnetwork.neuralnetwork(training_data)
weights, bias = neuralnetwork.get_weights_and_bias()
return weights, bias
def __init__(self):
self.mapShots = {}
self.mapHits = {}
self.trained = False
self.currentState = np.array([list((0, 0, 0, 0))])
self.weights = []
self.id = randrange(0, 100)
# create model
self.model = NeuralNetwork([4, 6, 4, 4, 1])
# create model
self.keras = Sequential()
# Configure the Keras Model
self.keras.add(Dense(4, input_shape=(4, ), activation='relu'))
self.keras.add(Dense(6, activation='relu'))
self.keras.add(Dense(4, activation='relu'))
self.keras.add(Dense(4, activation='relu'))
self.keras.add(Dense(1, activation='sigmoid'))
self.keras.compile(loss='mean_squared_error',
optimizer='sgd',
metrics=['accuracy'])
# Initialize weights array
for layer in self.keras.layers:
self.weights.append(layer.get_weights()[0])
def __init__(self, space, configFile, numActions=500):
self.num_actions = numActions
self.config = configFile
self.space = space
self.colour = (np.random.uniform(0, 255), np.random.uniform(0, 255),
np.random.uniform(0, 255), 255)
self.objects = self.generateModel()
self.initial_amplitude = abs(self.angle())
self.maximum_amplitude = abs(self.angle())
self.fitness = 0
self.action_step = 0
self.num_reversals = 0
self.last_action = 0
self.prev_angle = self.angle()
self.prev_dir = 2
self.steps_to_done = 0
self.done = False
self.reached_max = False
self.reached_max_step = 0
self.movable = True
self.energydata = []
self.phasedata = []
self.pumpdata = []
self.neuralnetwork = NeuralNetwork(4, 2, *self.config['hiddenLayers'])
def test_run(self):
# Test correctness of run method
network = NeuralNetwork(3, 2, 1, 0.5)
network.weights_input_to_hidden = test_w_i_h.copy()
network.weights_hidden_to_output = test_w_h_o.copy()
self.assertTrue(np.allclose(network.run(inputs), 0.09998924))
def createAndTrainNN(H, L, K, ALPHA, NUMBER_OF_EIGENVALUES, ERROR_LIMIT,
HIDDEN_LAYER_SIZES, ACTIVATION, MAX_ITER, RANDOM_STATE,
TOL, _Q_START, _tVector, identifier, trainToo,
trainCoreTemp, trainSurfTemp, trainQ, testToo,
testCoreTemp, testSurfTemp, testQ):
'''
This section creates the Neural Network. The training data from the PID
controller is calculated in another sectoin.
'''
# Defining Training Inputs
trainingInputs = buildingInputs(trainSurfTemp, trainCoreTemp, _Q_START,
identifier)
# Defining Training Targets
trainingTargets = trainQ[_Q_START:][np.newaxis].T
# Initialzing Neural Network Object
NN = NeuralNetwork(HIDDEN_LAYER_SIZES, ACTIVATION, MAX_ITER, RANDOM_STATE,
TOL, _Q_START)
# Training Neural network
NN._trainNN(trainingInputs, trainingTargets)
testingCases = np.shape(testToo)[0]
_qError = np.full((testingCases, ), np.nan)
_TcoreError = np.full((testingCases, ), np.nan)
_TsurfError = np.full((testingCases, ), np.nan)
lengthVectors = np.size(trainToo)
qArray = np.full((testingCases, lengthVectors), np.nan)
TcoreArray = np.full((testingCases, lengthVectors), np.nan)
TsurfArray = np.full((testingCases, lengthVectors), np.nan)
for i in range(testingCases):
# Defining variables of alrady calculated PID Data
_Too = testToo[i, :]
_Tcore = testCoreTemp[i, :]
_Tsurf = testSurfTemp[i, :]
_Q = testQ[i, :]
# Intializing NNPredictAndError Class Object
nnPredict = NNPredictAndError(H, L, K, ALPHA, NUMBER_OF_EIGENVALUES,
ERROR_LIMIT, _Q_START, _Q, _Too,
_tVector, NN)
# Defining testing inputs
testInputs = buildingInputs(_Tsurf, _Tcore, _Q_START, identifier)
# Getting predicted Q's from Neural Network
nnPredict._predictQ(testInputs)
# Calling getTempeatures Function
nnPredict._calcuateTemperatrues()
# Calculating All Errors
_qError[i] = nnPredict.qRmse(_Q)
_TcoreError[i] = nnPredict.TCoreRmse(_Tcore)
_TsurfError[i] = nnPredict.TSurfRmse(_Tsurf)
# Saving Data for Plotting
qArray[i] = nnPredict._qVector
TcoreArray[i] = nnPredict._coreTemp
TsurfArray[i] = nnPredict._surfTemp
print(qArray)
return (_qError, _TcoreError, _TsurfError, qArray, TcoreArray, TsurfArray)
def load_network(self, fname):
try:
self.network = NeuralNetwork(self.training_x, self.training_y)
self.network.load(fname)
except ShapeError as e:
print('加载神经网络文件{}失败, 输入或输出维度不匹配!'.format(fname))
return False
return True
def crossValidation(attributes, attributes_types, target_class, folds, b, k):
config_file = './data/configs/network.txt'
initial_weights_file = './data/configs/initial_weights.txt'
dataset_file = './data/datasets/wine.txt'
accuracy_values = []
precision_values = []
recall_values = []
fmeasure_values = []
for i in range(k):
training_set_folds = list(folds)
training_set_folds.remove(folds[i])
training_set = transformToList(training_set_folds)
# bootstrap tem o tamanho do conjunto de treinamento
bootstrap_size = len(training_set)
test_set = folds[i]
forest = []
for j in range(b):
bootstrap = getBootstrap(training_set, bootstrap_size)
neurons_per_layer = [1, 2, 1]
network = NeuralNetwork(config_file=config_file,
dataset_file=dataset_file,
initial_weights_file=initial_weights_file,
neurons_per_layer=neurons_per_layer)
network.backpropagation()
forest.append(network)
# Usa o ensemble de B arvores para prever as instancias do fold i
# (fold de teste) e avaliar desempenho do algoritmo
true_positives, false_positives, false_negatives, true_negatives = evaluateForest(
forest, test_set, target_class)
accuracy_values.append(
calculateAccuracy(true_positives, true_negatives, false_positives,
false_negatives))
precision_value = calculatePrecision(true_positives, false_positives)
precision_values.append(precision_value)
recall_value = calculateRecall(true_positives, false_negatives)
recall_values.append(recall_value)
fmeasure_values.append(
calculateF1Measure(precision_value, recall_value))
accuracy = sum(accuracy_values) / len(accuracy_values)
precision = sum(precision_values) / len(precision_values)
recall = sum(recall_values) / len(recall_values)
fmeasure = sum(fmeasure_values) / len(fmeasure_values)
return accuracy, precision, recall, fmeasure
def set_brain(self, new_brain=list()):
self.__brain = NeuralNetwork(bias=0,
num_layers=3,
layers_info=dict({
0: (9, 5),
1: (5, 1)
}),
num_weights=50,
weights=new_brain)
def test_input_and_output_sizes_must_be_int(self):
with self.assertRaises(TypeError):
NeuralNetwork(0, "A", "B", [42])
with self.assertRaises(TypeError):
NeuralNetwork(0, 1, "B", [42])
with self.assertRaises(TypeError):
NeuralNetwork(0, "A", 1, [42])
nn = NeuralNetwork(0, 1, 1, [42])
def test_lambda_must_be_non_negative(self):
with self.assertRaises(TypeError):
NeuralNetwork(None, 1, 1, [42])
with self.assertRaises(TypeError):
NeuralNetwork("A", 1, 1, [42])
with self.assertRaises(ValueError):
NeuralNetwork(-0.5, 1, 1, [42])
NeuralNetwork(0, 1, 1, [42])
def run(stockName, interval_days):
nnetwork = NeuralNetwork()
nnetwork.setSupervisedDataSetSize(9, 1)
objStock = datahandler.getStock(stockName)
neural_network_input = objStock.getData()
if debug:
print '##################################################################'
print stockName
print '##################################################################\n'
total_tests = 0
total_hits = total_hits_highs = total_hits_lows = 0
total_highs = total_lows = 0
fout_highs = open(
'./results/' + stockName + '_' + str(interval_days) + '_highs', 'w')
fout_lows = open(
'./results/' + stockName + '_' + str(interval_days) + '_lows', 'w')
fout_all = open(
'./results/' + stockName + '_' + str(interval_days) + '_all', 'w')
try:
# Sliding Window
for i in range(len(neural_network_input) - 3 * interval_days):
ret = run_day(nnetwork, neural_network_input, i, interval_days,
total_tests, total_hits, total_hits_highs,
total_hits_lows, total_highs, total_lows)
total_tests = ret[0]
total_hits = ret[1]
total_hits_highs = ret[2]
total_hits_lows = ret[3]
total_highs = ret[4]
total_lows = ret[5]
rel_highs = rel_lows = ''
if total_highs == 0:
rel_highs = ',-1'
else:
rel_highs = ',' + str(float(total_hits_highs) / total_highs)
if total_lows == 0:
rel_lows = ',-1'
else:
rel_lows = ',' + str(float(total_hits_lows) / total_lows)
rel_all = ',' + str(float(total_hits) / total_tests)
fout_highs.write(rel_highs)
fout_lows.write(rel_lows)
fout_all.write(rel_all)
except KeyboardInterrupt:
fout_highs.close()
fout_lows.close()
fout_all.close()
def test_neuralNetwork_weights_initially_zero(self):
network = NeuralNetwork([
InputLayer(10),
DenseLayer(5, activation='linear'),
OutputLayer(1, activation='linear')
],
batch_size=50)
inputs = np.asarray([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
outputs = np.asarray([0])
np.testing.assert_array_equal(network.predict(inputs), outputs)
def train_nn(num_input, num_hid, num_out, data_list, num_iterations=1000):
#Neural Network with 2 inputs, 6 hidden, and 3 outputs
nn = NeuralNetwork(num_input, num_hid, num_out)
for i in range(num_iterations):
for data in data_list:
#print(data[0], data[1])
nn.train(data[0], data[1])
random.shuffle(data_list)
print("Training Data #", i)
return nn
def __init__(self, tr_data):
self.training_x, self.training_y = self._parse_training_data(tr_data)
net = [(self.training_x.shape[0], ''), (4, 'relu'), (4, 'relu'),
(self.training_y.shape[0], 'feedthrough')]
try:
self.network = NeuralNetwork(self.training_x,
self.training_y,
sizes=net)
except ShapeError as e:
print('初始化神经网络结构失败!')
def test2():
print('Test 2')
nn = NeuralNetwork([9, 2], [nnet.logsigmoid])
input_samples = [
[[0], [0], [0], [0], [1], [1], [0], [1], [0]],
[[0], [0], [0], [0], [1], [1], [0], [1], [1]],
[[0], [0], [0], [1], [1], [0], [0], [1], [0]],
[[0], [0], [0], [1], [1], [0], [1], [1], [0]],
[[0], [1], [0], [0], [1], [1], [0], [0], [0]],
[[0], [1], [1], [0], [1], [1], [0], [0], [0]],
[[0], [1], [0], [1], [1], [0], [0], [0], [0]],
[[1], [1], [0], [1], [1], [0], [0], [0], [0]],
[[1], [1], [1], [1], [1], [1], [1], [1], [1]],
[[0], [0], [0], [0], [0], [0], [0], [0], [0]],
[[1], [0], [0], [1], [0], [0], [1], [0], [0]],
[[1], [0], [1], [1], [0], [1], [1], [0], [1]],
[[1], [1], [1], [0], [0], [0], [1], [1], [1]],
[[0], [0], [0], [0], [0], [0], [1], [1], [1]],
[[1], [1], [1], [0], [0], [0], [0], [0], [0]],
[[1], [1], [1], [1], [1], [1], [1], [1], [1]],
[[1], [1], [1], [1], [0], [1], [1], [1], [1]],
[[0], [1], [0], [0], [1], [0], [0], [1], [0]],
[[0], [1], [0], [1], [1], [1], [0], [1], [0]],
[[0], [0], [0], [1], [1], [1], [0], [0], [0]],
[[0], [1], [0], [0], [1], [1], [0], [1], [0]],
[[0], [1], [0], [1], [1], [0], [0], [1], [0]],
[[0], [0], [0], [1], [1], [1], [0], [1], [0]],
[[0], [1], [0], [1], [1], [1], [0], [0], [0]],
[[1], [0], [1], [0], [1], [1], [1], [1], [0]],
[[0], [0], [1], [0], [1], [1], [1], [1], [1]],
[[1], [0], [0], [1], [1], [0], [0], [1], [1]],
[[1], [0], [0], [1], [1], [0], [1], [1], [1]],
[[1], [1], [0], [0], [1], [1], [0], [0], [1]],
[[1], [1], [1], [0], [1], [1], [0], [0], [1]],
[[0], [1], [1], [1], [1], [0], [1], [0], [0]],
[[1], [1], [1], [1], [1], [0], [1], [0], [0]],
]
targets = [[[1], [0]], [[1], [0]], [[1], [0]], [[1], [0]], [[1], [0]],
[[1], [0]], [[1], [0]], [[1], [0]], [[1], [0]], [[0], [1]],
[[0], [1]], [[0], [1]], [[0], [1]], [[0], [1]], [[0], [1]],
[[0], [1]], [[0], [1]], [[0], [1]], [[0], [1]], [[0], [1]],
[[0], [1]], [[0], [1]], [[0], [1]], [[0], [1]], [[0], [1]],
[[0], [1]], [[0], [1]], [[0], [1]], [[0], [1]], [[0], [1]],
[[0], [1]], [[0], [1]]]
perceptron.learn(nn, input_samples, targets)
print('Weights: ' + str(nn.get_weights()))
weights = nn.get_weights()
for w in weights:
for i in w:
print(i)
def run(stockName, interval_days):
nnetwork = NeuralNetwork()
nnetwork.setSupervisedDataSetSize(9, 1)
objStock = datahandler.getStock(stockName)
neural_network_input = objStock.getData()
if debug:
print '##################################################################'
print stockName
print '##################################################################\n'
total_tests=0
total_hits=total_hits_highs=total_hits_lows=0
total_highs=total_lows=0
fout_highs = open('./results/'+stockName+'_'+str(interval_days)+'_highs', 'w')
fout_lows = open('./results/'+stockName+'_'+str(interval_days)+'_lows', 'w')
fout_all = open('./results/'+stockName+'_'+str(interval_days)+'_all', 'w')
try:
# Sliding Window
for i in range(len(neural_network_input)-3*interval_days):
ret = run_day(nnetwork, neural_network_input, i, interval_days, total_tests, total_hits, total_hits_highs, total_hits_lows, total_highs, total_lows)
total_tests = ret[0]
total_hits = ret[1]
total_hits_highs = ret[2]
total_hits_lows = ret[3]
total_highs = ret[4]
total_lows = ret[5]
rel_highs=rel_lows=''
if total_highs==0:
rel_highs = ',-1'
else:
rel_highs = ','+str(float(total_hits_highs)/total_highs)
if total_lows==0:
rel_lows=',-1'
else:
rel_lows = ','+str(float(total_hits_lows)/total_lows)
rel_all = ','+str(float(total_hits)/total_tests)
fout_highs.write(rel_highs)
fout_lows.write(rel_lows)
fout_all.write(rel_all)
except KeyboardInterrupt:
fout_highs.close()
fout_lows.close()
fout_all.close()
def neuralnetwork(self, training_data, test_data):
"""
training_data: data used for training
test_data: data used for testing
"""
neuralnetwork = NeuralNetwork(weights=10,
learning_rate=0.001,
epochs=100000)
neuralnetwork.neuralnetwork(training_data)
predicted = neuralnetwork.predict(test_data)
loss = self.poisson_loss(predicted, test_data["coauthorship"])
return predicted, loss
def __init__(self, game):
self.x = 50
self.radius = 15
from game import Game
self.y = Game.height // 2
self.y_vel = 0
self.gravity = 1
self.score = 0
self.fitness = 0
self.distance = 0
self.brain = NeuralNetwork(8, 5, 2)
self.game = game
self.inputs = 8 * [0]
def __init__(self):
self.x = 400
self.y = 300
self.radius = 15
self.shootTimer = 0
self.points = [[0, 0], [0, 0], [0, 0], [0, 0]]
self.vel = 100
self.angle = 0
self.nn = NeuralNetwork(shape)
self.delete = False
def __init__(self,
hiddenLayers,
hiddenActivationFunction=f.relu,
hiddenActivationFunctionDerivative=f.reluDerivative,
outputActivationFunction=f.gaussian,
outputActivationFunctionDerivative=f.gaussianDerivative):
layers = [9]
for i in range(len(hiddenLayers)):
layers.append(hiddenLayers[i])
layers.append(9)
self.NN = NeuralNetwork(layers, hiddenActivationFunction,
hiddenActivationFunctionDerivative,
outputActivationFunction,
outputActivationFunctionDerivative)
self.totalGamesTrained = 0
def test_should_have_valid_size_for_thetas(self):
nn = NeuralNetwork(0, 100, 10, (25, 50))
self.assertEqual(len(nn._initial_thetas), 3)
self.assertEqual(nn._initial_thetas[0].shape, (25, 101))
self.assertEqual(nn._initial_thetas[1].shape, (50, 26))
self.assertEqual(nn._initial_thetas[2].shape, (10, 51))
def test_training_for_XOR_learns(self):
nn = NeuralNetwork.init(0, 2, 1, [2])
X = np.matrix([[0, 0], [0, 1], [1, 0], [1, 1]])
Y = np.matrix([[0], [1], [1], [0]])
model = nn.train(X, Y, maxiter=400, tolerance=np.finfo(float).eps)
prediction = model.predict_binary_classification(X)
np.testing.assert_array_equal(prediction, Y)
def __init__(self,
input_size: int,
hidden_size: tuple,
output_size: int,
isHallow=False):
self.fitness = float("-inf")
self.brain: NeuralNetwork
self.isAlive = True
if isHallow:
self.brain = NeuralNetwork(input_size,
hidden_size,
output_size,
isHollow=True)
else:
self.brain = NeuralNetwork(input_size, hidden_size, output_size)
def batchpropagate(self, start, network):
testInputs = self.inputSet[start]
targetOutputs = self.targetSet[start]
testOutputs = network.Predict(testInputs)
network.backpropagate(targetOutputs)
sumOfWeights = NeuralNetwork(network)
totalError = 0.0
for i in range(targetOutputs.shape[0]):
totalError += (testOutputs[i,0] - targetOutputs[i, 0]) ** 2
for i in range(start + 1, start + self.batchSize):
if i >= self.dataCount:
break
testInputs = self.inputSet[i]
targetOutputs = self.targetSet[i]
testOutputs = network.Predict(testInputs)
network.backpropagate(targetOutputs)
for j in range(len(network.weights)):
sumOfWeights.weights[j].dValue += network.weights[j].dValue
for i in range(targetOutputs.shape[0]):
totalError += (testOutputs[i,0] - targetOutputs[i, 0]) ** 2
for j in range(len(network.weights)):
sumOfWeights.weights[j].dValue /= self.batchSize
network.weights[j].dValue = sumOfWeights.weights[j].dValue
sumOfWeights = None
return totalError / self.batchSize
def test_should_persist_all_the_ctor_args(self):
nn = NeuralNetwork(0.5, 100, 42, (1, 2, 3, 4))
self.assertEqual(nn.input_layer_size, 100)
self.assertEqual(nn.output_layer_size, 42)
self.assertEqual(nn.hidden_layer_count, 4)
self.assertEqual(nn.hidden_layer_sizes, [1, 2, 3, 4])
self.assertEqual(nn.lambda_val, 0.5)
def test_training_a_nn_multiple_times_keeps_the_initial_theta_unchanged(self):
nn = NeuralNetwork.init(0.03, 5, 3, [10])
expected = nn._initial_thetas[:]
X = np.random.rand(10, 5)
Y = np.asmatrix((np.random.rand(10,3) > 0.5).astype(int))
nn.train(X, Y)
nn.train(X, Y)
self.assertEqual(expected, nn._initial_thetas)
def test_spoon_feeding_theta_should_work_when_learning_XNOR(self):
thetas = [np.matrix([[-7, 5, 5], [5, -10, -10]]), np.matrix([[-5, 10, 10]])]
nn = NeuralNetwork.init_with_theta(0, thetas)
X = np.matrix([[0, 0], [0, 1], [1, 0], [1, 1]])
Y = np.matrix([[1], [0], [0], [1]])
model = nn.train(X, Y)
prediction = model.predict_binary_classification(X)
np.testing.assert_array_equal(prediction, Y)
def run(stockName, interval_days, year):
nnetwork = NeuralNetwork()
nnetwork.setSupervisedDataSetSize(9, 1)
objStock = datahandler.getStock(stockName)
neural_network_input = objStock.getData()
yearLimits = objStock.getYearLimits(year)
yearLimits[0] = max(0, yearLimits[0]-2*interval_days)
#print '##################################################################'
#print stockName
#print '##################################################################\n'
total_tests=0
total_hits=total_hits_highs=total_hits_lows=0
total_highs=total_lows=0
# Sliding Window
#for i in range(len(neural_network_input)-3*interval_days):
for i in range(yearLimits[0], yearLimits[1]-3*interval_days):
ret = run_day(nnetwork, neural_network_input, i, interval_days, total_tests, total_hits, total_hits_highs, total_hits_lows, total_highs, total_lows)
total_tests = ret[0]
total_hits = ret[1]
total_hits_highs = ret[2]
total_hits_lows = ret[3]
total_highs = ret[4]
total_lows = ret[5]
rel_highs=rel_lows=''
if total_highs==0:
rel_highs = '-1'
else:
rel_highs = str(float(total_hits_highs)/total_highs)
if total_lows==0:
rel_lows='-1'
else:
rel_lows = str(float(total_hits_lows)/total_lows)
rel_all = str(float(total_hits)/total_tests)
#print 'All: ' + rel_all + ' High: ' + rel_highs + ' Low: ' + rel_lows
print 'Total:\n' + str(total_hits) + ' out of ' + str(total_tests)
print str(total_hits_highs) + ' out of ' + str(total_highs) + ' highs.'
print str(total_hits_lows) + ' out of ' + str(total_lows) + ' lows.' + '\n'
def test_gradient_calculation_should_not_throw(self):
nn = NeuralNetwork.init(0.03, 5, 1, [10])
X = np.random.rand(10, 5)
Y = np.matrix([[0, 1, 0, 1, 0, 1, 1, 1, 0, 1]]).T
unrolled_thetas = nn._unroll_matrices(nn._initial_thetas)
result = nn._calculate_cost_gradient(unrolled_thetas, X, Y)
# no idea what the cost would be, but i expect it to be greater than zero
# it is extremely unlikely to have a perfect model in just one step with random initialization
self.assertGreaterEqual(result[0], 0.)
def generate(self, indices = None):
m, n = self.X_train.shape
k = self.Y_train.shape[1]
m_cv = self.X_cv.shape[0]
nn = NeuralNetwork.init(self._lambda, n, k, self._hidden_layer_sizes)
indices = range(1, m+1) if indices is None else indices
for i in indices:
x_sub = self.X_train[:i,:]
y_sub = self.Y_train[:i,:]
model = nn.train(x_sub, y_sub)
cost_reg_train = nn.cost_regularization(model.thetas, i)
h_train = model.evaluate(x_sub)
error_train = self._cost(y_sub, h_train, i, cost_reg_train)
cost_reg_cv = nn.cost_regularization(model.thetas, m_cv)
h_cv = model.evaluate(self.X_cv)
error_cv = self._cost(self.Y_cv, h_cv, m_cv, cost_reg_cv)
yield (i, error_train, error_cv)
def findOptimaEpoch():
chunks = makeChunks("car.data")
tuning_set = chunks[0]
grow_set = []
for c in chunks[1:]:
grow_set += c
epochs = 80
g = []
r = 0.1
while r < 0.9:
g.append(r)
r += 0.1
# for lr in g: # vary the learning rate
# for blah in range(0,5): # repeat five times
# for n in range(1,16):
t0 = time()
error_rates = []
NN = NeuralNetwork(13, 3) # layers fixed at 3
NN.buildNetwork(0.65, 0.9, 0.1) # pass the learning rate
for i in range(epochs):
NN.trainTheNetwork(grow_set)
error_rate, tp, fp, tn, fn = NN.testExampleData(tuning_set)
error_rates.append(error_rate)
fp = open("finding-optima-epoch.txt", "a")
# layers: {0}, neurons: {1}
fp.write("{0} {1} {2}\n".format(*(NN.n_layers, NN.m_neurons, 0.65)))
# Number of run epochs: {0}\n
fp.write("{0}\n".format(epochs))
# error rates: {0}\n
fp.write("{0}\n".format(str(error_rates)))
# epoch with lowest error rate: {0} -> {1} (zero indexing)\n
fp.write("optima epoch: {0} @ {1}\n".format(error_rates.index(min(error_rates)), min(error_rates)))
# Total training time: {0}\n\n
fp.write("{0}\n\n".format(time() - t0))
fp.close()
print(error_rates.index(min(error_rates)))
del (NN)
def kfoldCrossValidation(epochs):
chunks = makeChunks("car.data")
err_list = []
tp_sum = 0
falsep_sum = 0
tn_sum = 0
fn_sum = 0
for fold in range(10):
t0 = time()
# defaulting to 10 chunks
validation_set = chunks[fold]
training_set = []
for chunk in chunks[:fold] + chunks[fold + 1 :]:
training_set += chunk
# best determined network size
NN = NeuralNetwork(13, 3)
# best determined learning rate, upper and lower classification limits
NN.buildNetwork(0.65, 0.9, 0.1)
for i in range(epochs): # train the network
NN.trainTheNetwork(training_set)
ttime = time() - t0
er, tp, falsep, tn, fn = NN.testExampleData(validation_set)
err_list.append(er)
tp_sum += tp
falsep_sum += falsep
tn_sum += tn
fn_sum += fn
fp = open("NN-kfoldcrossvalidation.txt", "a")
fp.write("{0} {1} {2}\n".format(*(NN.n_layers, NN.m_neurons, 0.65))) # layers: {0}, neurons: {1}
fp.write("{0}\n".format(epochs)) # Number of run epochs: {0}
fp.write("error rate: {0}\n".format(er)) # error rate: {0}
fp.write("T/F analysis: TP[{0}], FP[{1}], TN[{2}], FN[{3}]\n".format(tp, falsep, tn, fn))
fp.write("{0}\n\n".format(ttime)) # Total training time: {0}
fp.close()
del (NN)
return err_list, tp_sum, falsep_sum, tn_sum, fn_sum
def test_train_should_return_a_model(self):
nn = NeuralNetwork.init(0.03, 5, 3, [10])
X = np.random.rand(10, 5)
Y = np.asmatrix((np.random.rand(10,3) > 0.5).astype(int))
result = nn.train(X, Y)
self.assertIsInstance(result, Model)
print digits.images.shape
pl.matshow(digits.images[0])
X = digits.data
print ('X', X)
y = np.array([to_bit(x) for x in digits.target])
print ('y', y)
X -= X.min() ## 0 - 1
X /= X.max() ##
nn = NeuralNetwork([64, 100, 100], 'tanh')
nn.train(X, y, epochs= 10)
one = '''
....x...
...xx...
..x.x...
....x...
....x...
....x...
....x...
....x...
'''
one_pattern = to_pattern(one)
def test_roll_into_theta_should_work_properly(self):
nn = NeuralNetwork.init(0, 2, 2, [2])
unrolled_theta_vector = [1, 2, 3, 4, 5, 6, -1, -2, -3, -4, -5, -6]
actual = nn._roll_into_matrices(unrolled_theta_vector)
expected = np.asarray([[[1,2,3], [4,5,6]], [[-1, -2, -3], [-4, -5, -6]]])
np.testing.assert_array_equal(actual, expected)
# combining both data and applying random sort training_input = female_input + male_input training_target = female_target + male_target training_data = zip(training_input, training_target) random.shuffle(training_data) training_input[:], training_target[:] = zip(*training_data) training_target = np.asarray(training_target) # Network configuration input_units = len(training_input[0]) output_units = 1 n_hidden = 200 momentum = 0.9 neuralNetwork = NeuralNetwork(learning_rate=0.01,n_in=input_units,n_hidden=n_hidden,n_out=output_units, momentum = momentum, activation='sigmoid') neuralNetwork.initialize_weights() results_file = ''.join(['results_lr',str(neuralNetwork.learning_rate),'_m',str(momentum),'_',str(n_hidden),"hidden",'_gender_classification']+['.out']) neuralNetwork.backpropagation(training_input,training_target,maxIterations=500, batch= False,file_name=results_file) network_file = ''.join(['trained_lr',str(neuralNetwork.learning_rate),'_m',str(momentum),'_',str(n_hidden),"hidden",'_gender_classification']+['.nn']) pickle.dump(neuralNetwork, file(network_file,'wb')) nn2 = pickle.load(file(network_file,'rb')) # Testing network on directories test_dir = 'test' male_dir = 'male' female_dir = 'female' print 'Test for %s directory' % (test_dir) getTestData(test_dir,nn2) print 'Test for %s directory' % (male_dir)
def test_unroll_theta_should_work_properly(self):
thetas = np.asarray([[[1,2,3], [4,5,6]], [[-1, -2, -3], [-4, -5, -6]]])
actual = NeuralNetwork._unroll_matrices(thetas)
expected = np.array([1, 2, 3, 4, 5, 6, -1, -2, -3, -4, -5, -6])
np.testing.assert_array_equal(actual, expected)
def test_tehta_regularization_should_return_proper_value(self):
nn = NeuralNetwork.init(0.5, 2, 3, [2])
theta = np.array([[1,2,3], [4,5,6]])
actual = nn._theta_regularization(theta, 100)
expected = np.array([[0, 0.01, 0.015], [0, 0.025, 0.03]])
np.testing.assert_array_equal(actual, expected)
# initialize weights with U ~ [-sqrt(3)/sqrt(k), sqrt(3)/sqrt(k)]
#factor = np.sqrt(3.0) / np.sqrt(layers[0:-1])
factor = 0.05 *np.sqrt(layers[0:-1]) / np.sqrt(layers[0:-1])
#factor = 1.0*np.sqrt(layers[0:-1]) / np.sqrt(layers[0:-1])
w = []; b = []
for i_o_theta in range(len(layers)-1):
w.append(np.random.uniform(-factor[i_o_theta], factor[i_o_theta], layers[i_o_theta:i_o_theta+2][::-1]))
b.append(np.random.uniform(-factor[i_o_theta], factor[i_o_theta], (1,layers[i_o_theta + 1])))
"""
train both networks with back-prop
"""
nn1 = NeuralNetwork(layers, 'tanh', 'softmax', early_stop = 'off', reg = 0)
cost_trace1, valid_err1 = nn1.train(train_x, train_y, w, b, momentum, learningRate, batch_size, nepoch, valid_x, valid_y)
nn2 = NeuralNetwork(layers, 'tanh', 'softmax', early_stop = 'off', reg = regularizer)
cost_trace2, valid_err2 = nn2.train(train_x, train_y, w, b, momentum, learningRate, batch_size, nepoch, valid_x, valid_y)
test_err1 = nn1.predict(test_x, test_y)
test_err2 = nn2.predict(test_x, test_y)
# print 'test_err:%.2f%%'%(test_err*100)
"""
Draw comparison
"""
plt.figure(1)
plt.plot(valid_err1[0:100], 'b-', linewidth=2, label='Typical Learning Process', hold='on')
,[1]]) x = numpy.array([[0,0], [0,1], [1,0], [1,1]]) target = numpy.array([[0] ,[1] ,[1] ,[0]]) #setting number of inputs and number of outputs in the neural network _ , xColumns = x.shape _ , targetColumns = target.shape neuralNetwork = NeuralNetwork(learning_rate=0.1,n_in=xColumns,n_hidden=2,n_out=targetColumns,activation='tanh',momentum=0.9) neuralNetwork.initialize_weights() neuralNetwork.backpropagation(x,target,maxIterations=10000, batch=False) # Network result after training estimation = neuralNetwork .feed_forward(x) printSeparator = "----------------" print "Estimated values:" print estimation print printSeparator print "Target values:" print target print printSeparator
def setUp(self): self.neuralNetwork = NeuralNetwork(learning_rate=0.15,n_hidden=2,momentum=0.95,activation='tanh') self.acceptanceEpsilon = 0.05 self.seed = 1 self.maxIterations = 11000
class NeuralNetworkTest(unittest.TestCase): def setUp(self): self.neuralNetwork = NeuralNetwork(learning_rate=0.15,n_hidden=2,momentum=0.95,activation='tanh') self.acceptanceEpsilon = 0.05 self.seed = 1 self.maxIterations = 11000 def tearDown(self): del self.neuralNetwork del self.acceptanceEpsilon del self.seed del self.maxIterations def retrieveEstimationError(self,x,target): #setting number of inputs and number of outputs in the neural network _ , xColumns = x.shape _ , targetColumns = target.shape self.neuralNetwork.n_in = xColumns self.neuralNetwork.n_out = targetColumns self.neuralNetwork.initialize_weights() self.neuralNetwork.backpropagation(x,target,maxIterations=self.maxIterations) # Network result after training estimation = self.neuralNetwork.feed_forward(x) estimationError = EstimationError(estimatedValues=estimation,targetValues=target) estimationError.computeErrors() totalError = estimationError.getTotalError() return totalError def testXOR(self): numpy.random.seed(seed=self.seed) x = numpy.array([[0,0], [0,1], [1,0], [1,1]]) target = numpy.array([[0] ,[1] ,[1] ,[0]]) totalError = self.retrieveEstimationError(x,target) print 'Error XOR:',totalError self.assertTrue(totalError<=self.acceptanceEpsilon) def testOR(self): numpy.random.seed(seed=self.seed) x = numpy.array([[0,0], [0,1], [1,0], [1,1]]) target = numpy.array([[0] ,[1] ,[1] ,[1]]) totalError = self.retrieveEstimationError(x,target) print 'Error OR:',totalError self.assertTrue(totalError<=self.acceptanceEpsilon) def testAND(self): numpy.random.seed(seed=self.seed) x = numpy.array([[0,0], [0,1], [1,0], [1,1]]) target = numpy.array([[0] ,[0] ,[0] ,[1]]) totalError = self.retrieveEstimationError(x,target) print 'Error AND:',totalError self.assertTrue(totalError<=self.acceptanceEpsilon) #test (x1 or x2) and x3 def testORAND(self): numpy.random.seed(seed=self.seed) x = numpy.array([[0,0,0], [0,0,1], [0,1,0], [1,0,0], [0,1,1], [1,1,0], [1,0,1], [1,1,1]]) target = numpy.array([[0] ,[0] ,[0] ,[0] ,[1] ,[0] ,[1] ,[1]]) totalError = self.retrieveEstimationError(x,target) print 'Error ORAND:',totalError self.assertTrue(totalError<=self.acceptanceEpsilon) #test (x1 and x2) or x3 def testANDOR(self): numpy.random.seed(seed=self.seed) x = numpy.array([[0,0,0], [0,0,1], [0,1,0], [1,0,0], [0,1,1], [1,1,0], [1,0,1], [1,1,1]]) target = numpy.array([[0] ,[1] ,[0] ,[0] ,[1] ,[1] ,[1] ,[1]]) totalError = self.retrieveEstimationError(x,target) print 'Error ANDOR:',totalError self.assertTrue(totalError<=self.acceptanceEpsilon)
def test_any_output_other_than_zero_one_throws(self):
nn = NeuralNetwork.init(0.03, 5, 1, [10])
X = np.random.rand(10, 5)
Y = np.matrix([[0, 1, 0, 1, 0, 1, 1, 2, 0, 1]])
with self.assertRaises(ValueError):
nn.train(X, Y)
def run():
# Fetch data
f1 = file('../data/mldata/mnist_data.pkl','rb')
mnist = pickle.load(f1)
f1.close()
split = 60000
X_train = np.reshape(mnist.data[:split], (-1,1,28,28))/255.0
Y_train = mnist.target[:split]
X_test = np.reshape(mnist.data[split:], (-1,1,28,28))/255.0
Y_test = mnist.target[split:]
n_classes = np.unique(Y_train).size
# Downsample training data
n_train_samples = 3000
train_idxs = np.random.random_integers(0, split-1, n_train_samples)
X_train = X_train[train_idxs, ...]
Y_train = Y_train[train_idxs, ...]
Y_train_one_hot = one_hot(Y_train)
print ('number of train samples: %d')%(n_train_samples)
print ('number of test samples: %d')%(X_test.shape[0])
# setup network
nn = NeuralNetwork(
layers = [
Layers.Convolution(
n_feats=12,
filter_shape=(5,5),
strides=(1,1),
weight_scale=0.1,
weight_decay=0.001),
Layers.Activation('relu'),
Layers.Pool(
pool_shape=(2,2),
strides=(2,2),
mode='max'),
Layers.Convolution(
n_feats=16,
filter_shape=(5,5),
strides=(1,1),
weight_scale=0.1,
weight_decay=0.001),
Layers.Activation('relu'),
Layers.Flatten(),
Layers.Linear(
n_out=n_classes,
weight_scale=0.1,
weight_decay=0.02),
Layers.Softmax()
]
)
#check gradient
# nn.check_gradients(X_train[:10], Y_train_one_hot[:10])
# Train neural network
t0 = time.time()
nn.train(X_train, Y_train_one_hot, learning_rate=0.05, max_iter=3, batch_size=32)
t1 = time.time()
print('Duration: %.1fs' % (t1-t0))
# Evaluate on test data
# Y_test_one_hot = one_hot(Y_test)
error = nn.error(X_test, Y_test)
print('Test error rate: %.4f' % error)
def test_theta_regularization_should_return_zero_for_no_lambda(self):
nn = NeuralNetwork.init(0, 10, 2, [10, 10])
theta = np.array([[1,2,3], [4,5,6]])
actual = nn._theta_regularization(theta, 100)
expected = np.zeros(theta.shape)
np.testing.assert_array_equal(actual, expected)
y = eeg_data.iloc[:,-1].reshape(-1,1)
print X.shape
print y.shape
from sklearn.preprocessing import StandardScaler
X = StandardScaler().fit_transform(X)
# use my custom data splitter for training/test
X_train, X_test, y_train, y_test = train_test_split(X, y, seed=0, test_size = 0.25)
start_time = timeit.timeit()
print "starting fit"
nn = NeuralNetwork(n_iter = 2, n_print = 5, learning_rate = 1,\
num_hidden_units=24, seed=42, verbose = False)
nn.fit(X_train, y_train)
end_time = timeit.timeit()
print "Fitting time: {}".format(end_time - start_time)
print "W matrix (size = {} x {}) = {}".format(nn.W.shape[0],nn.W.shape[1],nn.W)
print "V matrix (size = {} x {}) = {}".format(nn.V.shape[0],nn.V.shape[1],nn.V)
np.set_printoptions(threshold=np.inf)
y_pred = nn.predict(X_train)
print "Training Accuracy score = {}".format(accuracy_score(y_train, y_pred))
y_pred = nn.predict(X_test)
print "Test Accuracy score = {}".format(accuracy_score(y_test, y_pred))
def _test():
dset = create_dataset('tests/lenses.mff')
dset.train.normalize_attributes()
for e in dset.train.examples:
print(e)
classifier = NeuralNetwork(trainset=dset.train, max_error=.2, debug=True)
evaluator = Evaluator(classifier)
evaluator.holdout(.5)
dset = create_dataset('tests/lenses.mff')
dset.train.nominal_to_linear()
print(dset)
classifier = NeuralNetwork(trainset=dset.train, debug=True, max_error=.1, j=10)
evaluator = Evaluator(classifier)
evaluator.holdout(.2)
dset = create_dataset('tests/test_data/iris-binary.mff')
classifier = NeuralNetwork(trainset=dset.train, debug=True, max_error=.1)
classifier.train(dset.train)
dset = create_dataset('tests/test_data/votes.mff')
classifier = NeuralNetwork(trainset=dset.train, debug=True, max_error=.1)
classifier.train(dset.train)
dset = create_dataset('tests/test_data/mushroom.mff')
classifier = NeuralNetwork(trainset=dset.train, debug=True)
classifier.train(dset.train)
dset = create_dataset('tests/test_data/soybean.mff')
classifier = NeuralNetwork(trainset=dset.train, debug=True)
classifier.train()
data = dataset.readlines()
for entry in data:
(x,y,area) = entry.split()
x = float(x)
y = float(y)
area = int(area)
if area==-1: area = 0
x_input.append([x,y])
target.append([area])
x_input = numpy.array(x_input)
target = numpy.array(target)
numpy.random.seed(seed=1) #using fixed seed for testing purposes
_ , xColumns = x_input.shape
_ , targetColumns = target.shape
n_hidden = 8
momentum = 0
neuralNetwork = NeuralNetwork(learning_rate=0.01,n_in=xColumns,n_hidden=n_hidden,n_out=targetColumns, momentum = momentum)
neuralNetwork.initialize_weights()
results_file = ''.join(['results_lr',str(neuralNetwork.learning_rate),'_m',str(momentum),'_',str(n_hidden),"hidden",file_name.rsplit('.', 1)[0]]+['.out'])
neuralNetwork.backpropagation(x_input,target,maxIterations=10000, batch= False,file_name=results_file)
network_file = ''.join(['trained_lr',str(neuralNetwork.learning_rate),'_m',str(momentum),'_',str(n_hidden),"hidden",file_name.rsplit('.', 1)[0]]+['.nn'])
pickle.dump(neuralNetwork, file(network_file,'wb'))
print neuralNetwork.feed_forward(x_input)
nn2 = pickle.load(file(network_file,'rb'))
print network_file
print nn2.feed_forward(x_input)
def test_cost_regularization_should_return_proper_value(self):
nn = NeuralNetwork.init(0.1, 2, 2, [2])
current_thetas = np.array([[[1,2,3], [4,5,6]], [[-1, -2, -3], [-4, -5, -6]]])
actual = nn.cost_regularization(current_thetas, 100)
expected = 0.074
self.assertAlmostEqual(actual, expected)
def test_cost_regularization_returns_zero_for_no_lambda(self):
nn = NeuralNetwork.init(0, 10, 2, [10, 10])
thetas = nn._initial_thetas
actual = nn.cost_regularization(thetas, 10)
self.assertEqual(actual, 0)
"""
train_x = util.MNIST_Scale(train_x)
valid_x = util.MNIST_Scale(valid_x)
test_x = util.MNIST_Scale(test_x)
"""
# determine network structure and hyperparameters
layers = np.array([784, 1000, 300, 10])
learningRate = 0.001
momentum = 0.00
batch_size = 10
num_of_batch = len(train_y)/batch_size
nepoch = 2000
# initialize weights with U ~ [-sqrt(3)/sqrt(k), sqrt(3)/sqrt(k)]
#factor = np.sqrt(3.0) / np.sqrt(layers[0:-1])
factor = 0.05 *np.sqrt(layers[0:-1]) / np.sqrt(layers[0:-1])
#factor = 1.0*np.sqrt(layers[0:-1]) / np.sqrt(layers[0:-1])
w = []; b = []
for i_o_theta in range(len(layers)-1):
w.append(np.random.uniform(-factor[i_o_theta], factor[i_o_theta], layers[i_o_theta:i_o_theta+2][::-1]))
b.append(np.random.uniform(-factor[i_o_theta], factor[i_o_theta], (1,layers[i_o_theta + 1])))
nn = NeuralNetwork(layers, 'tanh', 'softmax', 'on')
cost_trace, valid_err = nn.train(train_x, train_y, w, b, momentum, learningRate, batch_size, nepoch, valid_x, valid_y)
test_err = nn.predict(test_x, test_y)
print 'test_err:%.2f%%'%(test_err*100)
