train_t = train_t[:1000]
train_t = to_categorical(train_t)
# setting parameters
max_epochs = 20
train_size = X_train.shape[0]
batch_size = 200
learning_rate = 0.01
scale = 0.005
iter_per_epoch = max(train_size / batch_size, 1)
max_iter = int(max_epochs * iter_per_epoch)
# constructing model
model = Neural_net(n_input=784,
n_hidden=[50, 100, 70, 100],
n_output=10,
w_std=scale,
batch_norm=False)
bn_model = Neural_net(n_input=784,
n_hidden=[50, 100, 70, 100],
n_output=10,
w_std=scale,
batch_norm=True)
model_dict = {'batch_norm': bn_model, 'normal': model}
train_acc = {}
'''---- learning ----'''
for name, network_ in model_dict.items():
network_.add(['relu'] * 4)
optim = Adam(lr=learning_rate)
multi_func = [np.square, sinusoidal, np.abs, heaviside]
func_name = ['square', 'sinusoidal', 'abs', 'heaviside']
train_acc = {}
# test data
x = np.arange(-1, 1, 0.01).reshape(-1, 1)
fig = plt.figure(figsize=(15, 9))
for n, func in enumerate(multi_func, 1):
train_y = func(train_x)
# plotting training data
ax = fig.add_subplot(2, 2, n)
ax.scatter(train_x, train_y, s=12, color='blue')
# constructing NN
model = Neural_net(n_input=1, n_hidden=[10, 10, 10], n_output=1)
model.add(['tanh', 'tanh', 'sigmoid'])
routine = Adam(lr=0.1, beta_1=0.9, beta_2=0.95)
model.compile(loss='sum_squared_error', optimizer=routine)
#-----learning------
import time
start = time.time()
history = model.fit(train_x, train_y, n_iter=max_iter, history=True)
print(time.time() - start)
train_acc[func_name[n - 1]] = history['acc']
# prediction data
y = model(x)
ax.plot(x, y, 'r-')
plt.xticks([-1, 0, 1])
plt.yticks([0, 0.5, 1])
import matplotlib.pyplot as plt
from deepL_module.datasets.mnist import load_mnist
from deepL_module.nn.multi_layer_nn import Neural_net
from deepL_module.nn.optimizers import *
from deepL_module.nn.cost_functions import *
from deepL_module.base import *
'''#0 loading data '''
(X_train, train_t), (X_test, test_t) = load_mnist(normalize=True)
train_size = X_train.shape[0]
batch_size = 256
max_iter = 1e+4
'''#1 config for NN '''
model = Neural_net(784, [100, 100, 100, 100], 10, alpha=0.01)
model.add(['tanh', 'softsign', 'softplus', 'swish'])
model.set_loss('categorical_crossentropy')
train_loss = []
train_acc = []
'''#2 optimizer '''
optim = Adam(lr=0.01)
'''#3 learning '''
for i in range(int(max_iter)):
batch_mask = np.random.choice(train_size, batch_size)
x_batch = X_train[batch_mask]
t_batch = to_categorical(train_t[batch_mask], cls_num=10)
# update weight matrix
grads = model.gradient(x_batch, t_batch)
optim.update(model.params, grads)
# get loss data
def create_noise_data(sample: int = 400):
x, labels = make_moons(sample, noise=0.2)
x[:, 0] -= 0.5
return x, labels.reshape(-1, 1)
# training dataset
train_x, labels = create_noise_data()
# test dataset
X, Y = np.meshgrid(np.linspace(-3, 3, 100), np.linspace(-3, 3, 100))
test_x = np.array([X.ravel(), Y.ravel()]).reshape(2, -1).T
# constructing NN
model = Neural_net(2, [10, 10, 10], 1)
model.add(['tanh', 'swish', 'relu'])
model.set_loss('binary_crossentropy')
optimizer = Adam(lr=0.1)
score_acc = []
#---learning----
for _ in range(max_iter):
grads = model.gradient(train_x, labels)
optimizer.update(model.params, grads)
score = model.accuracy(train_x, labels)
score_acc.append(np.asarray(score))
# plot the training data
plt.scatter(train_x[labels.ravel() == 0, 0],
(X_train, train_t), (X_test, test_t) = load_mnist(normalize=True)
train_size = X_train.shape[0]
batch_size = 256
max_iter = 1000
# === layers ===
layers = ['tanh', 'relu', 'softplus', 'softsign', 'swish']
model = {}
optimizer = {}
train_loss = {}
train_acc = {}
for key in layers:
model[key] = Neural_net(n_input=784,
n_hidden=[5, 5, 5],
n_output=10,
alpha=0.01)
model[key].add([key] * 3)
model[key].set_loss('categorical_crossentropy')
optimizer[key] = Adam(lr=0.01)
train_loss[key] = []
train_acc[key] = []
#=== learning ===
for i in range(max_iter):
batch_mask = np.random.choice(train_size, batch_size)
x_batch = X_train[batch_mask]
t_batch = to_categorical(train_t[batch_mask], cls_num=10)
for key in layers:
grads = model[key].gradient(x_batch, t_batch)
def create_noise_data(sample: int = 400):
x, labels = make_moons(sample, noise=0.2)
x[:, 0] -= 0.5
return x, labels.reshape(-1, 1)
# training dataset
train_x, labels = create_noise_data()
# test dataset
X, Y = np.meshgrid(np.linspace(-3, 3, 100), np.linspace(-3, 3, 100))
test_x = np.array([X.ravel(), Y.ravel()]).reshape(2, -1).T
# constructing NN
model = Neural_net(2, [10, 10, 10], 1)
model.add(['tanh', 'swish', 'tanh'])
routine = Adam(lr=0.1)
model.compile(loss='binary_crossentropy', optimizer=routine)
#---learning----
history = model.fit(train_x, labels, n_iter=max_iter, history=True)
# plot the training data
plt.scatter(train_x[labels.ravel() == 0, 0],
train_x[labels.ravel() == 0, 1],
marker=".",
s=20,
color='b')
plt.scatter(train_x[labels.ravel() == 1, 0],
for name in func_name:
train_acc[name] = []
# test data
x = np.arange(-1,1,0.01).reshape(-1,1)
fig = plt.figure(figsize=(15, 9))
for n,func in enumerate(multi_func,1):
train_y = func(train_x)
# plotting training data
ax = fig.add_subplot(2,2,n)
ax.scatter(train_x, train_y, s=12, color='blue')
# constructing NN
model = Neural_net(n_input=1, n_hidden=[10,10,10], n_output=1)
model.add(['tanh', 'tanh', 'sigmoid'])
model.set_loss('sum_squared_error')
optimizer = Adam(lr=0.1, beta_1=0.9, beta_2=0.95)
#-----learning------
for _ in range(int(max_iter)):
grads = model.gradient(train_x,train_y)
optimizer.update(model.params, grads)
score = model.accuracy(train_x, train_y)
train_acc[func_name[n-1]].append(score)
# prediction data
y = model(x)
ax.plot(x, y, 'r-')
plt.xticks([-1,0,1])
plt.yticks([0,0.5,1])
import numpy as np
import matplotlib.pyplot as plt
from deepL_module.datasets.mnist import load_mnist
from deepL_module.nn.multi_layer_nn import Neural_net
from deepL_module.nn.optimizers import *
from deepL_module.base import *
from deepL_module.base import *
from deepL_module.nn import *
max_iter = 1e+4
'''#0 loading data '''
(X_train, train_t), (X_test, test_t) = load_mnist(normalize=True)
train_t = to_categorical(train_t)
'''#1 config for NN '''
model = Neural_net(784, [100, 100, 100, 100], 10, alpha=0.01)
model.add(['tanh', 'softsign', 'softplus', 'swish'])
# optimizer
routine = Adam(lr=0.01)
model.compile(loss='categorical_crossentropy', optimizer=routine)
'''#2 learning '''
import time
start = time.time()
hist = model.fit(X_train,
train_t,
batch_size=256,
n_iter=max_iter,
history=True)
print(time.time() - start)
'''#3 preparing test data '''
data, label = get_mini_batch(X_test, test_t, batch_size=1)
'''#4 showing image '''