def test_linear():
coef1 = 1002
coef2 = 13
bias = 4
# fonction linéaire que l'on apprend
def f(x1, x2):
return x1 * coef1 + coef2 * x2 + bias
# données d'entrainement avec bruit
def f_bruit(x1, x2):
bruit = np.random.normal(0, 1, len(x1)).reshape((-1, 1))
return f(x1, x2) + bruit
nb_data = 100
x1 = np.random.uniform(-10, 10, nb_data)
x2 = np.random.uniform(-10, 10, nb_data)
x1 = x1.reshape((-1, 1))
x2 = x2.reshape((-1, 1))
datay = f_bruit(x1, x2)
datax = np.concatenate((x1, x2), axis=1)
# Input and Output size of our NN
input_size = len(datax[0])
output_size = 1
# Initialize modules with respective size
iteration = 300
gradient_step = 1e-4
m_mse = MSE()
m_linear = Linear(input_size, output_size)
for _ in range(iteration):
hidden_l = m_linear.forward(datax)
loss = m_mse.forward(datay, hidden_l)
print("max loss:", np.max(loss))
loss_back = m_mse.backward(datay, hidden_l)
m_linear.backward_update_gradient(datax, loss_back)
m_linear.update_parameters(gradient_step=gradient_step)
m_linear.zero_grad()
x1 = np.random.uniform(-10, 10, nb_data)
x2 = np.random.uniform(-10, 10, nb_data)
x1 = x1.reshape((-1, 1))
x2 = x2.reshape((-1, 1))
testy = f(x1, x2)
testx = np.concatenate((x1, x2), axis=1)
hidden_l = m_linear.forward(testx)
print("max différence res:", np.max(hidden_l - testy))
print("parameters:", str(m_linear._parameters))
print("valeurs voulues:", str([[coef1], [coef2]]))
print("biais:", str(m_linear._bias_parameters))
print("valeur voulue:", str([bias]))
def test_auto_encodeur():
uspsdatatrain = "../data/USPS_train.txt"
uspsdatatest = "../data/USPS_test.txt"
alltrainx, alltrainy = load_usps(uspsdatatrain)
alltestx, alltesty = load_usps(uspsdatatest)
alltrainx /= 2
alltestx /= 2
TNSE(alltesty, alltestx)
return
# Initialize modules with respective size
iteration = 100
gradient_step = 1e-3
batch_size = 50 # len(alltrainx)
input_size = alltrainx.shape[1]
hidden_size = 100
compression_size = 2
m_linear = Linear(input_size, hidden_size)
m_tanh = TanH()
m_linear2 = Linear(hidden_size, compression_size)
m_linear3 = Linear(compression_size, hidden_size)
m_linear3._parameters = m_linear2._parameters.T
m_linear4 = Linear(hidden_size, input_size)
m_linear4._parameters = m_linear._parameters.T
m_sigmoid = Sigmoid()
m_loss = BCE()
seq = Sequential([
m_linear, m_tanh, m_linear2, m_tanh, m_linear3, m_tanh, m_linear4,
m_sigmoid
])
opt = Optim(seq, loss=m_loss, eps=gradient_step)
opt.SGD(alltrainx, alltrainx, batch_size, maxiter=iteration, verbose=2)
predict = opt.predict(alltestx)
compression_train = seq.forward(alltrainx)[-5]
compression = seq.forward(alltestx)[-5]
"""
# print
for i in range(6):
plt.imshow(alltestx[i].reshape((16,16)))
plt.show()
plt.imshow(predict[i].reshape((16,16)))
plt.show()
plt.imshow(compression[i].reshape((compression_size,1)))
plt.show()
"""
# if compression_size == 2:
# plot_2D(alltesty, compression)
TNSE(alltesty, alltestx)
def test_non_linear_SGD():
# data generation
datax, datay = tools.gen_arti(centerx=1,
centery=1,
sigma=0.1,
nbex=1000,
data_type=1,
epsilon=0.1)
testx, testy = tools.gen_arti(centerx=1,
centery=1,
sigma=0.1,
nbex=1000,
data_type=1,
epsilon=0.1)
datay = datay[..., None]
testy = testy[..., None]
datay = np.where(datay == -1, 0, 1)
testy = np.where(testy == -1, 0, 1)
# NN parameters
batch_size = len(datax)
input_size = len(datax[0])
hidden_size = 2
final_size = 1
iteration = 100
gradient_step = 10e-3
# Setup NN
net = Sequential([
Linear(input_size, hidden_size, bias=True),
TanH(),
Linear(hidden_size, final_size, bias=True),
Sigmoid()
])
opt = Optim(net=net, loss=MSE(), eps=gradient_step)
opt.SGD(datax, datay, batch_size, maxiter=iteration, verbose=2)
def yhat(x):
yhat = opt.predict(x)
return np.where(yhat >= 0.5, 1, -1)
tools.plot_frontiere(testx, yhat, step=100)
tools.plot_data(testx, testy.reshape(-1))
plt.plot()
plt.show()
def test_auto_encodeur():
mnist = fetch_openml('mnist_784')
alltrainx, alltrainy = mnist.data.to_numpy()[:10000, :], np.intc(
mnist.target.to_numpy()[:10000])
alltestx, alltesty = mnist.data.to_numpy()[10000:20000, :], np.intc(
mnist.target.to_numpy()[10000:20000])
alltestx, alltrainx = alltestx / 255, alltrainx / 255
# Initialize modules with respective size
iteration = 100
gradient_step = 1e-3
batch_size = 50 # len(alltrainx)
input_size = alltrainx.shape[1]
hidden_size = 100
compression_size = 10
m_linear = Linear(input_size, hidden_size)
m_tanh = TanH()
m_linear2 = Linear(hidden_size, compression_size)
m_linear3 = Linear(compression_size, hidden_size)
m_linear3._parameters = m_linear2._parameters.T
m_linear4 = Linear(hidden_size, input_size)
m_linear4._parameters = m_linear._parameters.T
m_sigmoid = Sigmoid()
m_loss = BCE()
seq = Sequential([
m_linear, m_tanh, m_linear2, m_tanh, m_linear3, m_tanh, m_linear4,
m_sigmoid
])
opt = Optim(seq, loss=m_loss, eps=gradient_step)
opt.SGD(alltrainx, alltrainx, batch_size, maxiter=iteration, verbose=True)
predict = opt.predict(alltestx)
compression_train = seq.forward(alltrainx)[-5]
compression = seq.forward(alltestx)[-5]
# print
size = int(np.sqrt(alltestx.shape[1]))
for i in range(6):
plt.imshow(alltestx[i].reshape((size, size)))
plt.show()
plt.imshow(predict[i].reshape((size, size)))
plt.show()
plt.imshow(compression[i].reshape(-1, 1))
plt.show()
# TNSE(alltesty,compression)
return cluster(compression_train, compression, alltrainy, alltesty)
def test_linear_SGD():
coef1 = 1002
coef2 = 13
bias = 4
# fonction linéaire que l'on apprend
def f(x1, x2):
return x1 * coef1 + coef2 * x2 + bias
# données d'entrainement avec bruit
def f_bruit(x1, x2):
bruit = np.random.normal(0, 1, len(x1)).reshape((-1, 1))
return f(x1, x2) + bruit
nb_data = 100
x1 = np.random.uniform(-10, 10, nb_data)
x2 = np.random.uniform(-10, 10, nb_data)
x1 = x1.reshape((-1, 1))
x2 = x2.reshape((-1, 1))
datay = f_bruit(x1, x2)
datax = np.concatenate((x1, x2), axis=1)
# Input and Output size of our NN
input_size = len(datax[0])
output_size = 1
# Initialize modules with respective size
iteration = 50
gradient_step = 1e-4
m_linear = Linear(input_size, output_size)
m_mse = MSE()
seq = Sequential([m_linear])
opt = Optim(seq, loss=m_mse, eps=gradient_step)
opt.SGD(datax, datay, nb_data, maxiter=iteration, verbose=2)
x1 = np.random.uniform(-10, 10, nb_data)
x2 = np.random.uniform(-10, 10, nb_data)
x1 = x1.reshape((-1, 1))
x2 = x2.reshape((-1, 1))
testy = f(x1, x2)
testx = np.concatenate((x1, x2), axis=1)
hidden_l = opt.predict(testx)
print("max différence res:", np.max(hidden_l - testy))
print("parameters:", str(m_linear._parameters))
print("valeurs voulues:", str([[coef1], [coef2]]))
print("biais:", str(m_linear._bias_parameters))
print("valeur voulue:", str([bias]))
data_type=0,
epsilon=0.1)
datay_r = np.zeros((len(datay), 2))
# Re-arranging data to compute a probability
for y in range(len(datay)):
if datay[y] == -1:
datay_r[y][0] = 1
elif datay[y] == 1:
datay_r[y][1] = 1
# Input and Output size of our NN
input_size = len(datax[0])
output_size = len(np.unique(datay))
# Initialize modules with respective size
m_mse = MSE()
m_linear = Linear(input_size, output_size)
m_sigmoid = Sigmoid()
# Etape forward
hidden_l = m_linear.forward(datax)
assert (hidden_l.shape == (len(datax), output_size))
act_l = m_sigmoid.forward(hidden_l)
assert (act_l.shape == hidden_l.shape)
loss = m_mse.forward(hidden_l, datay_r)
assert (loss.shape == (len(datax), ))
# Etape Backward
loss_back = m_mse.backward(datay_r, hidden_l)
assert (loss_back.shape == (len(datax), output_size))
delta_funcact = m_sigmoid.backward_delta(hidden_l, loss_back)
assert (loss_back.shape == delta_funcact.shape)
delta_linear = m_linear.backward_delta(datax, delta_funcact)
# sortie de taille (nb entr�es, nombre de donn�es)
chan_input = 1
chan_output = 32
stride = 1
max_pool_stride = 2
max_pool_kernel = 2
# loss function
sftmax = CESoftMax()
# Network parameters
net = Sequential([
Conv1D(kernel_size, chan_input, chan_output, stride=stride),
MaxPool1D(max_pool_kernel, max_pool_stride),
Flatten(),
Linear(4064, 100),
ReLU(),
Linear(100, 10)
])
# Train networks
opt = Optim(net=net, loss=sftmax, eps=gradient_step)
opt.SGD(alltrainx,
alltrainy_proba,
batch_size,
X_val=allvalx,
Y_val=allvaly,
f_val=lambda x: np.argmax(Softmax().forward(x), axis=1),
maxiter=iterations,
verbose=2)
def test_non_linear():
# data generation
datax, datay = tools.gen_arti(centerx=1,
centery=1,
sigma=0.1,
nbex=1000,
data_type=1,
epsilon=0.1)
testx, testy = tools.gen_arti(centerx=1,
centery=1,
sigma=0.1,
nbex=1000,
data_type=1,
epsilon=0.1)
datay = datay[..., None]
testy = testy[..., None]
datay = np.where(datay == -1, 0, 1)
testy = np.where(testy == -1, 0, 1)
nb_data = len(datax)
# Input and Output size of our NN
input_size = len(datax[0])
hidden_size = 3
final_size = 1
# Initialize modules with respective size
iteration = 100
gradient_step = 10e-3
m_linear_first = Linear(input_size, hidden_size, bias=True)
m_linear_second = Linear(hidden_size, final_size, bias=True)
m_sig = Sigmoid()
m_tanh = TanH()
m_mse = MSE()
for _ in range(iteration):
hidden_l = m_linear_first.forward(datax)
hidden_l_tanh = m_tanh.forward(hidden_l)
hidden_l2 = m_linear_second.forward(hidden_l_tanh)
hidden_l2_sigmoid = m_sig.forward(hidden_l2)
loss = m_mse.forward(datay, hidden_l2_sigmoid)
print("max loss:", np.max(loss))
loss_back = m_mse.backward(datay, hidden_l2_sigmoid)
delta_sigmoid = m_sig.backward_delta(hidden_l2, loss_back)
delta_linear_2 = m_linear_second.backward_delta(
hidden_l_tanh, delta_sigmoid)
delta_tanh = m_tanh.backward_delta(hidden_l, delta_linear_2)
m_linear_second.backward_update_gradient(hidden_l_tanh, delta_sigmoid)
m_linear_first.backward_update_gradient(datax, delta_tanh)
m_linear_second.update_parameters(gradient_step=gradient_step)
m_linear_first.update_parameters(gradient_step=gradient_step)
m_linear_first.zero_grad()
m_linear_second.zero_grad()
def yhat(x):
hidden_l = m_linear_first.forward(x)
hidden_l_tanh = m_tanh.forward(hidden_l)
hidden_l2 = m_linear_second.forward(hidden_l_tanh)
yhat = m_sig.forward(hidden_l2)
return np.where(yhat >= 0.5, 1, -1)
tools.plot_frontiere(testx, yhat, step=100)
tools.plot_data(testx, testy.reshape(-1))
plt.show()
def test_multiclass():
"""
testx, testy = get_usps([neg, pos], alltestx, alltesty)
testy = np.where(testy == neg, -1, 1)
:return:
"""
uspsdatatrain = "../data/USPS_train.txt"
uspsdatatest = "../data/USPS_test.txt"
alltrainx, alltrainy = load_usps(uspsdatatrain)
alltestx, alltesty = load_usps(uspsdatatest)
input_size = len(alltrainx[0])
output_size = len(np.unique(alltesty))
alltrainy_proba = transform_numbers(alltrainy, output_size)
# Initialize modules with respective size
iteration = 100
gradient_step = 10e-5
arbitrary_neural = 128
m_linear = Linear(input_size, arbitrary_neural)
m_act1 = TanH()
m_linear2 = Linear(arbitrary_neural, output_size)
m_act2 = Softmax()
# m_loss = BCE()
m_loss = CESoftMax()
for _ in range(iteration):
# Etape forward
hidden_l1 = m_linear.forward(alltrainx)
act1 = m_act1.forward(hidden_l1)
hidden_l2 = m_linear2.forward(act1)
loss = m_loss.forward(alltrainy_proba, hidden_l2)
# print("max loss:", np.mean(loss, axis=0))
print("max loss:", np.mean(loss))
# Etape Backward
loss_back = m_loss.backward(alltrainy_proba, hidden_l2)
hidden_l2_back = m_linear2.backward_delta(act1, loss_back)
act1_back = m_act1.backward_delta(hidden_l1, hidden_l2_back)
# update gradient
m_linear2.backward_update_gradient(act1, loss_back)
m_linear.backward_update_gradient(alltrainx, act1_back)
# update parameters
m_linear2.update_parameters(gradient_step=gradient_step)
m_linear.update_parameters(gradient_step=gradient_step)
m_linear.zero_grad()
m_linear2.zero_grad()
hidden_l1 = m_linear.forward(alltestx)
act1 = m_act1.forward(hidden_l1)
hidden_l2 = m_linear2.forward(act1)
act2 = m_act2.forward(hidden_l2)
predict = np.argmax(act2, axis=1)
res = skt.confusion_matrix(predict, alltesty)
print(np.sum(np.where(predict == alltesty, 1, 0)) / len(predict))
plt.imshow(res)
from src.Activation.softmax import Softmax from src.Activation.tanH import TanH from src.Loss.BCE import BCE from src.Loss.CESoftMax import CE from src.Loss.MSE import MSE from src.Module.conv1D import Conv1D from src.Module.flatten import Flatten from src.Module.linear import Linear from src.Pooling.maxPool1D import MaxPool1D np.random.seed(1) datax = np.random.randn(20, 10) datay = np.random.choice([-1, 1], 20, replace=True) dataymulti = np.random.choice(range(10), 20, replace=True) linear = Linear(10, 1) sigmoide = Sigmoid() softmax = Softmax() tanh = TanH() relu = ReLU() conv1D = Conv1D(k_size=3, chan_in=1, chan_out=5, stride=2) maxpool1D = MaxPool1D(k_size=2, stride=2) flatten = Flatten() mse = MSE() bce = BCE() crossentr = CE() # cross entropy avec log softmax ## Lineaire et MSE linear.zero_grad() res_lin = linear.forward(datax)