def train_prediction(
net: Neural_network.NeuralNet,
inputs_train: Tensor,
targets_train: Tensor,
inputs_test: Tensor,
targets_test: Tensor,
loss: Loss.Loss = Loss.MeanSquareError(),
optimizer: OptimizerClass.Optimizer = OptimizerClass.SGD(),
num_epochs: int = 5000,
batch_size: int = 32):
Data = pd.DataFrame(columns=('MSE_train', 'MSE_test', 'error_round_train',
'error_round_test'))
size_training = inputs_train.shape[0]
for epoch in range(num_epochs):
Chi2_train = 0.0
error_round_train = 0.0
nbr_batch = 0
for i in range(0, size_training, batch_size):
nbr_batch += 1
# 1) feed forward
y_actual = net.forward(inputs_train[i:i + batch_size])
# 2) compute the loss and the gradients
Chi2_train += loss.loss(targets_train[i:i + batch_size], y_actual)
grad_ini = loss.grad(targets_train[i:i + batch_size], y_actual)
# 3)feed backwards
grad_fini = net.backward(grad_ini)
# 4) update the net
optimizer.step(net, n_epoch=epoch)
error_round_train += Error_round.error_round(
targets_train[i:i + batch_size], y_actual)
Chi2_train = Chi2_train / nbr_batch
error_round_train = error_round_train / nbr_batch
y_actual_test = net.forward(inputs_test)
Chi2_test = loss.loss(targets_test, y_actual_test)
error_round_test = Error_round.error_round(targets_test, y_actual_test)
if epoch % 100 == 0:
print('epoch : ' + str(epoch) + "/" + str(num_epochs) + "\r",
end="")
datanew = pd.DataFrame({
'MSE_train': [Chi2_train],
'MSE_test': [Chi2_test],
'error_round_train': [error_round_train],
'error_round_test': [error_round_test]
})
Data = Data.append(datanew)
os.chdir(path_ini)
Data.to_csv('Opt_num_epoch_backup.csv', index=False)
return Data
def initialize_new():
"""Initializes a new example neural net with one hidden layer."""
result = NeuralNet(2)
result.add_relu(100)
result.add_relu(3)
result.add_softmax()
result.add_cross_entropy_loss()
return result
def train(net: Neural_network.NeuralNet,
inputs: Tensor,
targets: Tensor,
loss: Loss = Loss.MeanSquareError(),
optimizer: OptimizerClass.Optimizer = OptimizerClass.SGD(),
num_epochs: int = 5000,
batch_size: int = 32) -> tuple:
chi2_list = []
round_error_list = []
size_training = inputs.shape[0]
for epoch in range(num_epochs):
chi2_loss = 0.0
round_error_loss = 0.0
nbr_batch = 0
for i in range(0, size_training, batch_size):
nbr_batch += 1
# 1) Feed forward
y_actual = net.forward(inputs[i:i + batch_size])
# 2) Compute the loss and the gradient
chi2_loss += loss.loss(targets[i:i + batch_size], y_actual)
round_error_loss += Error_round.error_round(
targets[i:i + batch_size], y_actual)
grad_ini = loss.grad(targets[i:i + batch_size], y_actual)
# 3) Feed backwards
grad_fini = net.backward(grad_ini)
# 4) Update the net
optimizer.step(net, n_epoch=epoch)
chi2_loss = chi2_loss / nbr_batch
round_error_loss = round_error_loss / nbr_batch
chi2_list.append(chi2_loss)
round_error_list.append(round_error_loss)
# Print status every 50 iterations
if epoch % 50 == 0:
print('\r epoch : ' + str(epoch) + "/" + str(num_epochs) +
", training mean squared error : " + str(chi2_loss) + "\r",
end="")
print('epoch : ' + str(epoch) + "/" + str(num_epochs) +
", training final mean squared error : " + str(chi2_loss) + '\n')
return chi2_list, round_error_list
def eliminate(self, bot, replace=False):
self.time_since_last_death = 0.0
self.bots.remove(bot)
color = bot.RGB
neural_net_example = NeuralNet((2, 3, 3), ("sigmoid", "softmax"))
if replace:
example_bot = Bot(neural_net_example, color, self)
self.bots.append(example_bot)
def __init__(self, size, mutation_rate, no_food):
assert (size >= 5)
assert (0 < mutation_rate < 1)
self.SIZE = size
self.mutation_rate = mutation_rate
self.bots = []
self.food = []
self.time_since_last_death = 0.0
# The neural network will have 1 neuron in the input layer, 1 hidden
# layer with 2 neurons, and 4 neurons in the output layer. The sigmoid
# activation function will be used on the hidden layer, and a softmax
# activation function will be used on the output layer. Input consists
# of the bot's direction and if there is or isn't food in the bots field
# of vision. Output consists of whether or not to move foward, turn
# left, turn right, or do nothing.
neural_net_example = NeuralNet((2, 3, 3), ("sigmoid", "softmax"))
colors = [(255, 0, 0), (0, 255, 0)]
# spawning equal no of carni_bots and herbi_bots
for i in range(size):
random_rgb = colors[0]
example_bot = Bot(neural_net_example, random_rgb, self)
self.bots.append(example_bot)
for i in range(size):
random_rgb = colors[1]
example_bot = Bot(neural_net_example, random_rgb, self)
self.bots.append(example_bot)
for i in range(no_food):
self.food.append(Food(self))
# append strongest parents from previous run
if os.path.isfile("modelc.h5"):
self.bots.append(
Bot(NeuralNet(model_file="modelc.h5"), (255, 0, 0), self))
self.bots.append(
Bot(NeuralNet(model_file="modelh.h5"), (0, 255, 0), self))
def prediction_calc(tic, data=None):
if data == None:
data = pd.read_csv(RAW_DATA_FOLDER + tic + '.csv').tail(30)
data.index = range(len(data))
data = normalize(data)
date = data.index[-1]
print(date)
inputs = getInputs(date, data)
weights = np.load(WEIGHTS_FILE_PATH, allow_pickle=True)
network = nn.NeuralNet([0, 0, 7], [41, 100, 50, 1])
network.weights = weights
prediction = network.calculateOutput(inputs, single_input=True)
if prediction > .5:
confidence = ((prediction - .5) / .4) * 100
return (1, confidence)
else:
confidence = np.sqrt((.5 - prediction) / .4) * 100
return (0, confidence)
def optim():
"""Determine best lambda parameter"""
#list of lambda values to try
lambdas = [
0, 0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.24
]
#lambdas = [2.56, 5.12, 10, 20, 40, 80, 160, 320, 640, 1280, 2560]
#list to record training parameters
histories = []
J_train = []
J_test = []
accuracies = []
for l in lambdas:
#define and train model
model = NeuralNet(alpha=0.003,
n_iter=40000,
lamb=l,
hidden_size=28,
n_labels=2)
Theta1, Theta2, cost_history = model.fit(X_train, Y_train)
#append parameters to lists
histories.append(cost_history)
J_train.append(cost_history[-1])
X_test_1 = model.add_intercept(X_test)
J_test.append(model.forward_prop(X_test_1, Y_test, Theta1, Theta2)[4])
accuracies.append(model.predict(Theta1, Theta2, X_test, Y_test)[1])
#plot costs VS lambda
fig, ax = plt.subplots()
ax.plot(lambdas, J_train, color="C0", label="training")
ax.plot(lambdas, J_test, color="C2", label="testing")
ax.set_xlabel("Lambda")
ax.set_ylabel("Cost")
ax.legend()
plt.show()
#plot accuracies VS lambdas
fig, ax = plt.subplots()
ax.plot(lambdas, accuracies)
ax.set_xlabel("Lambda")
ax.set_ylabel("Accuracy on test sample")
plt.show()
return accuracies
def train():
"""Train the neural network"""
model = NeuralNet(alpha=0.003,
n_iter=300000,
lamb=5,
hidden_size=28,
n_labels=2)
start = time.time()
Theta1, Theta2, cost_history = model.fit(X, Y)
stop = time.time()
elapsed = stop - start
pred, accu = model.predict(Theta1, Theta2, X_test, Y_test)
print("Time to train: {:.1f} seconds".format(elapsed))
print("Model accuracy: {:.3f}".format(accu))
model.plot_history(cost_history)
return Theta1, Theta2, pred, accu, cost_history
def feed(self, bot, food, is_bot=False):
bot.score = 1.0
if is_bot == False:
# # print "Bot-Food collision"
self.food.remove(food)
self.food.append(Food(self))
else: # bot-bot collision
# # print "Bot-Bot collision"
idx = self.bots.index(bot)
self.bots.pop(idx)
if len(self.bots) <= 7:
# # print "Creating new Bots!"
neural_net_example = NeuralNet((2, 3, 3),
("sigmoid", "softmax"))
colors = [(255, 0, 0), (0, 255, 0)]
for i in range(self.SIZE):
random_rgb = colors[np.random.randint(0, 2)]
example_bot = Bot(neural_net_example, random_rgb, self)
self.bots.append(example_bot)
num_to_replace = int(self.SIZE / 7 - 1)
if num_to_replace < 2:
num_to_replace = 2
for i in range(num_to_replace):
weakest = self.bots[0]
for other in self.bots:
if other.score < weakest.score:
weakest = other
self.eliminate(weakest)
colors = [(255, 0, 0), (0, 255, 0)]
# asexual reproduction - mutation
for i in range(num_to_replace):
if np.random.uniform(0, 1) <= self.mutation_rate:
new_rgb = colors[np.random.randint(0, 2)]
new_bot = Bot(bot.nnet, new_rgb, self)
# new_bot.x = bot.x + Bot.HITBOX_RADIUS * 4 * np.random.uniform(0, 1) * np.random.choice((-1, 1))
# new_bot.y = bot.y + Bot.HITBOX_RADIUS * 4 * np.random.uniform(0, 1) * np.random.choice((-1, 1))
new_bot.x = np.random.randint(
settings.
WINDOW_WIDTH) + Bot.HITBOX_RADIUS * 4 * np.random.uniform(
0, 1) * np.random.choice((-1, 1))
new_bot.y = np.random.randint(
settings.
WINDOW_HEIGHT) + Bot.HITBOX_RADIUS * 4 * np.random.uniform(
0, 1) * np.random.choice((-1, 1))
# nn
nb_c = new_bot.nnet.get_all_weights()
mutated = False
while not mutated:
# for k in range(len(nb_c)-1):
# # # print "len: ",len(nb_c)
# # # print "from: ",len(nb_c[k])
# # # print "to: ",len(nb_c[k][0]-1)
# for i in range(len(nb_c[k])-1):
# for j in range(len(nb_c[k][0])):
# if np.random.uniform(0, 1) <= self.mutation_rate:
# nb_c[k][i][j] = nb_c[k][i][j] * np.random.normal(1, 0.5) + np.random.standard_normal()
# mutated = True
for k in range(len(nb_c)):
if np.random.uniform(0, 1) <= self.mutation_rate:
i = np.random.randint(0, len(nb_c[k]))
j = np.random.randint(0, len(nb_c[k][0]))
nb_c[k][i][j] = nb_c[k][i][j] * np.random.normal(
1, 0.5) + np.random.standard_normal()
mutated = True
new_bot.nnet.set_all_weights(nb_c)
self.bots.append(new_bot)
self.bots[self.bots.index(new_bot)].RGB = (255, 255, 255)
# sexual - crossover
else:
# # print "Sexual! :P"
sorted_bots_by_score = sorted(self.bots,
key=lambda x: x.score,
reverse=True)
# get first 2 strongest bots
bot1, bot2 = sorted_bots_by_score[0], sorted_bots_by_score[1]
self.bots[self.bots.index(bot1)].change_color((255, 255, 255))
self.bots[self.bots.index(bot2)].change_color((255, 255, 255))
conn1 = bot1.nnet.get_all_weights()
conn2 = bot2.nnet.get_all_weights()
# get random weight to crossover
idx1 = np.random.randint(len(conn1))
idx2 = np.random.randint(len(conn1[idx1]))
idx3 = np.random.randint(len(conn1[idx1][idx2]))
conn3 = conn1
# print "booyah: ",conn3[idx1][idx2][idx3] , conn2[idx1][idx2][idx3]
# raw_input()
conn3[idx1][idx2][idx3] = conn2[idx1][idx2][idx3]
new_bot = Bot(bot.nnet, bot.RGB, self)
new_bot.nnet.set_all_weights(conn3)
new_bot.x = bot.x + Bot.HITBOX_RADIUS * 4 * np.random.uniform(
0, 1) * np.random.choice((-1, 1))
new_bot.y = bot.y + Bot.HITBOX_RADIUS * 4 * np.random.uniform(
0, 1) * np.random.choice((-1, 1))
self.bots.append(new_bot)
def run_classifier(folder_path,
optimizer_string,
workers,
device,
calculate_precision=False):
data_loader = load_data(250, workers, "train.csv", folder_path, 32)
data_load_time = 0
neural_net = NeuralNet().to(device)
criterion = nn.BCELoss()
optimizer = get_optimizer(optimizer_string, neural_net)
loss_outer_array = []
precision1_outer_array = []
precision3_outer_array = []
epoch_start_time = time.monotonic()
for epoch in range(5):
if calculate_precision:
loss_array = []
precision1_array = []
precision3_array = []
enumerator = enumerate(data_loader)
data_load_start_time = time.monotonic()
compute_start_time = data_load_start_time
data_load_time = 0
compute_time = 0
for i, data in enumerator:
data_load_end_time = time.monotonic()
data_load_time = data_load_time + (data_load_end_time -
data_load_start_time)
input_tensor, input_label_tensor = data['image_tensor'].to(
device), data['image_label_tensor'].to(device)
optimizer.zero_grad()
output_label_tensor = neural_net(input_tensor).float()
loss = criterion(output_label_tensor, input_label_tensor)
loss.backward()
optimizer.step()
compute_end_time = time.monotonic()
compute_time = compute_time + (compute_end_time -
compute_start_time)
if calculate_precision:
precision = accuracy(output_label_tensor, input_label_tensor)
precision1_array.append(round(precision[0], 2))
precision3_array.append(round(precision[1], 2))
loss_array.append(round(loss.item(), 2))
compute_start_time = time.monotonic()
data_load_start_time = time.monotonic()
if calculate_precision:
loss_outer_array.append(loss_array)
precision1_outer_array.append(precision1_array)
precision3_outer_array.append(precision3_array)
epoch_end_time = time.monotonic()
epoch_time = epoch_end_time - epoch_start_time
print("-----------------Training done------------------------")
if calculate_precision:
print(
"-----------------Printing precision stats------------------------"
)
for epoch in range(5):
for i, data in enumerate(data_loader, 0):
print(
"Epoch = {}, batch = {}, Loss = {}, Precision@1 {}, Precision@3 {}. \n"
.format(epoch, i, loss_outer_array[epoch][i],
precision1_outer_array[epoch][i],
precision3_outer_array[epoch][i]))
average_loss = sum(sum(x) for x in loss_outer_array) / 600.0
average_precision_1 = sum(sum(x)
for x in precision1_outer_array) / 600.0
average_precision_3 = sum(sum(x)
for x in precision3_outer_array) / 600.0
print("Average loss over 5 epoch = {}".format(average_loss))
print("Average precision@1 over 5 epoch = {}".format(
average_precision_1))
print("Average precision@3 over 5 epoch = {}".format(
average_precision_3))
print("-----------------Printing aggregate stats------------------------")
print("Aggregated time for data loading = {}".format(data_load_time))
print(
"Aggregated time for mini-batch computation = {}".format(compute_time))
print("Aggregated time for each epoch = {}".format(epoch_time))
print("-----------------Printing average stats------------------------")
print("Average time per mini batch for data loading = {}".format(
round(data_load_time / 600.0, 2)))
print("Average time per minibatch for computation = {}".format(
round(compute_time / 600.0, 2)))
print("Average time for each epoch = {}".format(round(epoch_time / 5.0,
2)))
if __name__ == "__main__":
learning_rate = 0.05
hidden_units = 7
num_epochs = 50
train = "datasets/heart_train.json"
test = "datasets/heart_test.json"
# parse the json file for data
X_train, y_train, meta_train = parse_json(train)
X_test, y_test, meta_test = parse_json(test)
print("model,epoch,F1_train,F1_test")
# Run logistic regression
for epoch in range(1,num_epochs+1):
logistic = NeuralNet(learning_rate=learning_rate, num_epoch=epoch,hidden_dim=())
logistic.fit(X_train, y_train, meta_train)
F1_train = logistic.predict(X_train, y_true=y_train, F1=True)
F1_test = logistic.predict(X_test, y_true=y_test, F1=True)
print("logistic-0.05-heart,{},{},{}".format(epoch,F1_train,F1_test))
# Run neural network
for epoch in range(1,num_epochs+1):
nnet = NeuralNet(learning_rate=learning_rate, num_epoch=epoch,hidden_dim=(hidden_units,))
nnet.fit(X_train, y_train, meta_train)
F1_train = nnet.predict(X_train, y_true=y_train, F1=True)
F1_test = nnet.predict(X_test, y_true=y_test, F1=True)
print("nnet-0.05-7-heart,{},{},{}".format(epoch,F1_train,F1_test))
from parse_json import parse_json
import sys
import json
import numpy as np
import pandas as pd
if __name__ == "__main__":
# <learning-rate> <#hidden-units> <#epochs> <train-set-file> <test-set-file>
if len(sys.argv) == 1:
learning_rate = 0.05
hidden_units = 7
num_epochs = 20
train = "datasets/heart_train.json"
test = "datasets/heart_test.json"
else:
tmp, learning_rate, hidden_units, num_epochs, train, test = sys.argv
learning_rate = float(learning_rate)
hidden_units = int(hidden_units)
num_epochs = int(num_epochs)
# parse the json file for data
X_train, y_train, meta_train = parse_json(train)
X_test, y_test, meta_test = parse_json(test)
nn = NeuralNet(learning_rate=learning_rate,
num_epoch=num_epochs,
hidden_dim=(hidden_units, ))
nn.fit(X_train, y_train, meta_train, verbose=True)
nn.predict(X_test, y_true=y_test, verbose=True)
def prediction(net: Neural_network.NeuralNet, inputs: Tensor) -> Tensor:
return net.forward(inputs)
return [j for j in results if type(0) != type(j)]
if __name__ == "__main__":
save_string = 'prediction threshold: ' + str(PREDICTION_THRESHOLD) + '\n'
save_string = save_string + 'gain threshold: ' + str(GAIN_THRESHOLD) + '\n'
save_string = save_string + 'minimum price: ' + str(MINIMUM_PRICE) + '\n'
save_string = save_string + 'days ahead: ' + str(DAYS_AHEAD) + '\n'
save_string = save_string + 'attributes: ' + str(ATTRIBUTES) + '\n'
save_string = save_string + '\nSTATS:\n\n'
#gets network and tickers
weights = np.load('test_results/Great+03/end_weights.npy',
allow_pickle=True)
network = nn.NeuralNet([0, 0, 7], [41, 100, 50, 1])
network.weights = weights
stock_tickers = pd.read_csv('data/stock_names.csv')['Ticker']
#removes tickers wihtout data
for tic in stock_tickers:
if not os.path.exists('data/normalized_data/' + tic + '.csv'):
stock_tickers = stock_tickers[stock_tickers != tic]
stock_tickers = stock_tickers.sample(frac=PERCENT_STOCKS)
#collects data
print('collecting data...')
pool = Pool()
results = list(
tqdm(pool.imap(testing_network, stock_tickers),
total=len(stock_tickers)))
pool.close()
countin += 1
flotar = ""
print 'complete.'
print scorecard
return scorecard
if __name__ == '__main__':
print 'Starting neural network to recognize audio... maybe....'
input_nodes = 80000 # 200x200 pixel image == 40,000 pixels, each px has red + green values to save so we need 80k
hidden_nodes = 185
output_nodes = 10 # 0 == hello, 1 == yes, 2 == no
learning_rate = 0.1
nn = NeuralNet(input_nodes, hidden_nodes, output_nodes, learning_rate)
# Train
train_the_neural_net(nn, epochs=1)
# Test
test_results = numpy.asarray(test_the_neural_net(nn))
# Print results
print('Neural network is {}% accurate at predicting audio.... maybe....'.
format(test_results.sum() / float(test_results.size) * 100.0))
thyme = time.time()
thyme -= ti
print str(thyme) + " seconds"
def testing_network(tic):
#gets data
weights = np.load('test_results/Great+03/end_weights.npy',
allow_pickle=True)
network = nn.NeuralNet([0, 0, 7], [41, 100, 50, 1])
network.weights = weights
stock_data_normalized = pd.read_csv('data/normalized_data/' + tic + '.csv',
index_col='date').tail(1520)
stock_data = pd.read_csv(
'data/historical_stock_data/' + tic + '.csv',
index_col='date',
).tail(1520)
stock_data = stock_data[stock_data_normalized.index[0]:]
results = [0] * len(stock_data_normalized.index[20:-DAYS_AHEAD])
i = 0
for date in stock_data_normalized.index[20:-DAYS_AHEAD]:
input_data = dp.getInputs(date, stock_data_normalized)
prediction = network.calculateOutput(input_data, single_input=True)
#disregards low priced stocks
if stock_data.loc[date, 'close'] > MINIMUM_PRICE:
percent_change = (
(stock_data.loc[stock_data.index[20 + i + DAYS_AHEAD], 'close']
- stock_data.loc[date, 'close']) /
stock_data.loc[date, 'close']) * 100
crossed_threshold, high = day_change_check(
stock_data,
20 + i,
ATTRIBUTES,
GAIN_THRESHOLD,
days_ahead=DAYS_AHEAD,
)
if crossed_threshold:
results[i] = (prediction, 1, percent_change, high)
else:
""" if (percent_change > .03) & (prediction >.8):
print('high')
print('5 day close price:',stock_data.loc[stock_data.index[i+5],'close'])
print('compare price:', stock_data.loc[date,'close'])
elif (percent_change < -.5)& (prediction >.8):
print('low')
print('5 day close price:',stock_data.loc[stock_data.index[i+5],'close'])
print('compare price:', stock_data.loc[date,'close']) """
if percent_change > 3:
print('date:', date)
print('five day date:',
stock_data.index[20 + i + DAYS_AHEAD])
print('current days price: ', stock_data.loc[date,
'close'])
print(
'five day close: ',
stock_data.loc[stock_data.index[20 + i + DAYS_AHEAD],
'close'])
print('day change high:', high)
""" if percent_change < -3:
print('date:', date)
print('five day date:', stock_data.index[20+i+DAYS_AHEAD])
print('current days price: ', stock_data.loc[date,'close'])
print('five day close: ', stock_data.loc[stock_data.index[20+i+DAYS_AHEAD],'close'])
print('day change high:',high)
print('percent change: ', percent_change) """
results[i] = (prediction, 0, percent_change, high)
i = i + 1
return [j for j in results if type(0) != type(j)]
print 'complete.'
return scorecard
if __name__ == '__main__':
print 'Starting neural network to recognize handwritten digits.'
input_nodes = 102
hidden_nodes = 30
output_nodes = 1
learning_rate = 0.1
weight_file = open('sd.txt', 'r')
weight_list = weight_file.readlines()
weight_list = numpy.asfarray(weight_list)
weight_file.close()
nn = NeuralNet(input_nodes, hidden_nodes, output_nodes, learning_rate,
weight_list)
# Train
train_the_neural_net(nn, epochs=100)
# Test
test_results = numpy.asarray(test_the_neural_net(nn))
# Print results
print('Neural network is {}% accurate at predicting handwritten digits.'.
format(test_results.sum() / float(test_results.size) * 100.0))
9 : Step
"""
if __name__ == "__main__":
#counters
numPredictions = 0
training_errors = [0] * NUM_ITERATIONS
testing_errors = None
start_weights = None
end_weights = None
time_per_stock = 0
time_per_iteration = 0
overview_string = ''
#create neural network
network = nn.NeuralNet(ACTIVATION_FUNCTIONS, NODES_PER_LAYER)
#get tickers
stock_tickers = pd.read_csv('data/stock_names.csv')['Ticker']
#removes tickers wihtout data
for tic in stock_tickers:
if not os.path.exists('data/training/' + tic + '.csv'):
stock_tickers = stock_tickers[stock_tickers != tic]
if not os.path.exists('data/normalized_data/' + tic + '.csv'):
stock_tickers = stock_tickers[stock_tickers != tic]
print("loading training data")
#get input training dictionaries
loading_time = -time.time()
#spawns a process for each stock data that needs to be loaded
pool = Pool()
results = list(
tqdm.tqdm(pool.imap(inputsForBackProp, stock_tickers),
print("Sanity check")
fig, ax = plt.subplots(figsize=(5, 5))
circ = plt.Circle((0.5, 0.5), math.sqrt(1 / 2 / math.pi), fill=False)
plt.scatter(data_input[:, 0].cpu(), data_input[:, 1].cpu())
plt.xlabel('x0')
plt.ylabel('x1')
plt.title('Scatter of input')
ax.add_patch(circ)
plt.show()
print('Average of train_target==1 is', torch.mean(data_target * 1.0).item())
print(" ")
net1 = NeuralNet(layer_type='Sequential',
learning_rate=2e-3,
regularization=0.05,
iteration=1000,
batch_size=64,
dtype=dtype,
device=device)
net1.append(
Module(layer_type='Linear',
if_batchnorm=True,
activation_type='relu',
params_shape=(2, 100),
dtype=dtype,
device=device))
net1.append(
Module(layer_type='Linear',
if_batchnorm=True,
activation_type='relu',
params_shape=(100, 50),