def __init__(self, optimizer=Momentum(), D=8, T=2):
# aggregation層の深さ
self.T = T
# 訓練パラメータ
self.params = {}
self.params["W"] = SIGMA * np.random.randn(D, D)
self.params["A"] = SIGMA * np.random.randn(D)
self.params["b"] = np.array([0.0])
# 最適化手法
self.optimizer = optimizer
def get_opt():
opt = parameter.optimization.lower()
if (opt == "adam"):
return Adam()
elif (opt == "adadelta"):
return Adadelta()
elif (opt == "adagrad"):
return AdaGrad()
elif (opt == "rmsprop"):
return RMSProp()
elif (opt == "nesterov"):
return Nesterov()
elif (opt == "momentum"):
return Momentum()
else:
return None
label = np.zeros((dataNum, last_units))
for i in range(last_units):
# currently -1~0. 0~1 can be used due to some issue
dataset[dataNum/last_units*i:dataNum/last_units*(i+1),:,:] = np.random.uniform(i-1, i, (dataNum/last_units,channel,input_shape))
label[dataNum/last_units*i:dataNum/last_units*(i+1),i] = 1
net = Network(
[Conv2D(4, 3, input_shape=input_shape), # 1*12*12 -> 4*10*10
ReLU(),
MaxPooling2D(2), # 10*10*4 -> 9*9*4
FullyConnect(units=last_units, input_shape=324),
Softmax(),
],
#FullyConnect(units=last_units)],
# FullyConnect(units=last_units, input_shape=144)],
learning_rate = learning_rate,
optimizer=Momentum(0.9),
batch=batchSize,
dtype=np.float32
)
err_prev = 0
for e in range(epoch):
err = 0
for batchIdx in range(0, dataNum, batchSize):
batchData = dataset[batchIdx:batchIdx + batchSize,:,:]
batchLabel = label[batchIdx:batchIdx + batchSize,:]
err += net.train(batchData, batchLabel, loss=MSE())
print "epoch", e
print "\t", err
if abs(err_prev - err) < 10e-15:
print "early_stop"
import sys, os
import numpy as np
from dataset.mnist import load_mnist
from twolayer import TwoLayerNet
from optimizer import SGD,Momentum
(x_train,t_train),(x_test,t_test) = load_mnist(normalize=True , one_hot_label=True)
network = TwoLayerNet(input_size = 784,hidden_size = 50,output_size=10)
optimizer = Momentum()
iters_num = 1000000
train_size = x_train.shape[0]
batch_size = 1000
learning_rate = 0.1
train_loss_list = []
train_acc_list = []
test_acc_list = []
iter_per_epoch = max(train_size/batch_size,1)
for i in range(iters_num):
batch_mask = np.random.choice(train_size,batch_size)
x_batch = x_train[batch_mask]
t_batch = t_train[batch_mask]
grads = network.gradient(x_batch,t_batch)
def f(x, y):
return x**2 / 20.0 + y**2
def df(x, y):
return x / 10.0, 2.0 * y
if __name__ == "__main__":
init_pos = (-7.0, 2.0)
params = {}
grads = {}
optimizers = OrderedDict()
optimizers['SGD'] = SGD(lr=0.95)
optimizers['Momentum'] = Momentum(lr=0.1)
optimizers['AdaGrad'] = AdaGrad(lr=1.5)
optimizers['Adam'] = Adam(lr=0.3)
fig = plt.figure()
idx = 1
for key in optimizers.keys():
optimizer = optimizers[key]
x_history = []
y_history = []
params['x'], params['y'] = init_pos[0], init_pos[1]
grads['x'], grads['y'] = 0, 0
iters_times = 30
import time
import numpy as np
from pathlib import Path
from matplotlib import pyplot as plt
try:
sys.path.append(os.path.join(Path(os.getcwd()).parent, 'lib'))
from mnist import load_mnist
import multilayernet as network
from optimizer import SGD, Momentum, AdaGrad, Adam
except ImportError:
print('Library Module Can Not Found')
# 1. optimizer set-up
optimizers = {
'sgd': SGD(),
'momentum': Momentum(),
'adagrad': AdaGrad(),
'adam': Adam()
}
# 2. load training/test data
(train_x, train_t), (test_x, test_t) = load_mnist(normalize=True,
flatten=True,
one_hot_label=True)
# 3. hyperparameters
batch_size = 100
epochs = 30
# learning_rate = 0.1
# 4. Testing
from optimizer import SGD, Momentum, AdaGrad, Adam
from multi_layer_net import MultiLayerNet
from utils import smooth_curve
if __name__ == "__main__":
# 读取 mnist 数据
(train_img, train_label), (test_img,
test_label) = load_mnist(normalize=True)
train_size = train_img.shape[0]
batch_size = 128
iters_times = 2000
# 初始化
optimizers = {}
optimizers['SGD'] = SGD()
optimizers['Momentum'] = Momentum()
optimizers['AdaGrad'] = AdaGrad()
optimizers['Adam'] = Adam()
networks = {}
train_loss = {}
for key in optimizers.keys():
networks[key] = MultiLayerNet(input_size=784,
hidden_size_list=[100, 100, 100, 100],
output_size=10)
train_loss[key] = []
# 训练网络
for i in range(iters_times):
batch_mask = np.random.choice(train_size, batch_size)
train_img_batch = train_img[batch_mask]
def mnist_model():
initializer = parameter.GaussianInitializer(std=0.1)
bias_initializer = parameter.ConstantInitializer(0.1)
model = network.Network()
model.add(
FullyConnected(
name='fc1',
in_feature=784,
out_feature=512,
weight_initializer=initializer,
bias_initializer=bias_initializer))
model.add(
BatchNorm(name='bn1',
num_features=512)
)
model.add(ReLU(name='relu1'))
model.add(
FullyConnected(
name='fc2',
in_feature=512,
out_feature=256,
weight_initializer=initializer,
bias_initializer=bias_initializer))
model.add(ReLU(name='relu2'))
model.add(
FullyConnected(
name='fc3',
in_feature=256,
out_feature=10,
weight_initializer=initializer,
bias_initializer=bias_initializer))
model.add(Softmax())
model.add_loss(CrossEntropyLoss())
lr = 0.1
optimizer = Momentum(lr=lr)
print(model)
for k, v in model.parameters().items():
print(k, v)
traingset = fetch_traingset()
testset = fetch_testingset()
test_images, test_labels = testset['images'], testset['labels']
test_labels = np.array(test_labels)
images, labels = traingset['images'], traingset['labels']
batch_size = 512
training_size = len(images)
pbar = tqdm.tqdm(range(100))
for epoch in pbar:
losses = []
model.train_mode()
for i in range(int(training_size/batch_size)):
batch_images = np.array(images[i*batch_size:(i+1)*batch_size])
batch_labels = np.array(labels[i*batch_size:(i+1)*batch_size])
batch_labels = one_hot(batch_labels, 10)
_, loss = model.forward(batch_images, batch_labels)
losses.append(loss)
model.optimize(optimizer)
model.eval_mode()
predicts = np.zeros((len(test_labels)))
for i in range(int(len(test_labels)/1000)):
batch_images = np.array(test_images[i*1000:(i+1)*1000])
pred = model.forward(batch_images)
pred = np.argmax(pred, 1)
predicts[i*1000:(i+1)*1000] = pred
acc = np.sum(test_labels == predicts) * 100 / len(test_labels)
pbar.set_description('e:{} loss:{} acc:{}%'.format(epoch, float(np.mean(losses)), acc))