def __init__(self, optimizer=Momentum(), D=8, T=2): # aggregation層の深さ self.T = T # 訓練パラメータ self.params = {} self.params["W"] = SIGMA * np.random.randn(D, D) self.params["A"] = SIGMA * np.random.randn(D) self.params["b"] = np.array([0.0]) # 最適化手法 self.optimizer = optimizer
def get_opt(): opt = parameter.optimization.lower() if (opt == "adam"): return Adam() elif (opt == "adadelta"): return Adadelta() elif (opt == "adagrad"): return AdaGrad() elif (opt == "rmsprop"): return RMSProp() elif (opt == "nesterov"): return Nesterov() elif (opt == "momentum"): return Momentum() else: return None
label = np.zeros((dataNum, last_units)) for i in range(last_units): # currently -1~0. 0~1 can be used due to some issue dataset[dataNum/last_units*i:dataNum/last_units*(i+1),:,:] = np.random.uniform(i-1, i, (dataNum/last_units,channel,input_shape)) label[dataNum/last_units*i:dataNum/last_units*(i+1),i] = 1 net = Network( [Conv2D(4, 3, input_shape=input_shape), # 1*12*12 -> 4*10*10 ReLU(), MaxPooling2D(2), # 10*10*4 -> 9*9*4 FullyConnect(units=last_units, input_shape=324), Softmax(), ], #FullyConnect(units=last_units)], # FullyConnect(units=last_units, input_shape=144)], learning_rate = learning_rate, optimizer=Momentum(0.9), batch=batchSize, dtype=np.float32 ) err_prev = 0 for e in range(epoch): err = 0 for batchIdx in range(0, dataNum, batchSize): batchData = dataset[batchIdx:batchIdx + batchSize,:,:] batchLabel = label[batchIdx:batchIdx + batchSize,:] err += net.train(batchData, batchLabel, loss=MSE()) print "epoch", e print "\t", err if abs(err_prev - err) < 10e-15: print "early_stop"
import sys, os import numpy as np from dataset.mnist import load_mnist from twolayer import TwoLayerNet from optimizer import SGD,Momentum (x_train,t_train),(x_test,t_test) = load_mnist(normalize=True , one_hot_label=True) network = TwoLayerNet(input_size = 784,hidden_size = 50,output_size=10) optimizer = Momentum() iters_num = 1000000 train_size = x_train.shape[0] batch_size = 1000 learning_rate = 0.1 train_loss_list = [] train_acc_list = [] test_acc_list = [] iter_per_epoch = max(train_size/batch_size,1) for i in range(iters_num): batch_mask = np.random.choice(train_size,batch_size) x_batch = x_train[batch_mask] t_batch = t_train[batch_mask] grads = network.gradient(x_batch,t_batch)
def f(x, y): return x**2 / 20.0 + y**2 def df(x, y): return x / 10.0, 2.0 * y if __name__ == "__main__": init_pos = (-7.0, 2.0) params = {} grads = {} optimizers = OrderedDict() optimizers['SGD'] = SGD(lr=0.95) optimizers['Momentum'] = Momentum(lr=0.1) optimizers['AdaGrad'] = AdaGrad(lr=1.5) optimizers['Adam'] = Adam(lr=0.3) fig = plt.figure() idx = 1 for key in optimizers.keys(): optimizer = optimizers[key] x_history = [] y_history = [] params['x'], params['y'] = init_pos[0], init_pos[1] grads['x'], grads['y'] = 0, 0 iters_times = 30
import time import numpy as np from pathlib import Path from matplotlib import pyplot as plt try: sys.path.append(os.path.join(Path(os.getcwd()).parent, 'lib')) from mnist import load_mnist import multilayernet as network from optimizer import SGD, Momentum, AdaGrad, Adam except ImportError: print('Library Module Can Not Found') # 1. optimizer set-up optimizers = { 'sgd': SGD(), 'momentum': Momentum(), 'adagrad': AdaGrad(), 'adam': Adam() } # 2. load training/test data (train_x, train_t), (test_x, test_t) = load_mnist(normalize=True, flatten=True, one_hot_label=True) # 3. hyperparameters batch_size = 100 epochs = 30 # learning_rate = 0.1 # 4. Testing
from optimizer import SGD, Momentum, AdaGrad, Adam from multi_layer_net import MultiLayerNet from utils import smooth_curve if __name__ == "__main__": # 读取 mnist 数据 (train_img, train_label), (test_img, test_label) = load_mnist(normalize=True) train_size = train_img.shape[0] batch_size = 128 iters_times = 2000 # 初始化 optimizers = {} optimizers['SGD'] = SGD() optimizers['Momentum'] = Momentum() optimizers['AdaGrad'] = AdaGrad() optimizers['Adam'] = Adam() networks = {} train_loss = {} for key in optimizers.keys(): networks[key] = MultiLayerNet(input_size=784, hidden_size_list=[100, 100, 100, 100], output_size=10) train_loss[key] = [] # 训练网络 for i in range(iters_times): batch_mask = np.random.choice(train_size, batch_size) train_img_batch = train_img[batch_mask]
def mnist_model(): initializer = parameter.GaussianInitializer(std=0.1) bias_initializer = parameter.ConstantInitializer(0.1) model = network.Network() model.add( FullyConnected( name='fc1', in_feature=784, out_feature=512, weight_initializer=initializer, bias_initializer=bias_initializer)) model.add( BatchNorm(name='bn1', num_features=512) ) model.add(ReLU(name='relu1')) model.add( FullyConnected( name='fc2', in_feature=512, out_feature=256, weight_initializer=initializer, bias_initializer=bias_initializer)) model.add(ReLU(name='relu2')) model.add( FullyConnected( name='fc3', in_feature=256, out_feature=10, weight_initializer=initializer, bias_initializer=bias_initializer)) model.add(Softmax()) model.add_loss(CrossEntropyLoss()) lr = 0.1 optimizer = Momentum(lr=lr) print(model) for k, v in model.parameters().items(): print(k, v) traingset = fetch_traingset() testset = fetch_testingset() test_images, test_labels = testset['images'], testset['labels'] test_labels = np.array(test_labels) images, labels = traingset['images'], traingset['labels'] batch_size = 512 training_size = len(images) pbar = tqdm.tqdm(range(100)) for epoch in pbar: losses = [] model.train_mode() for i in range(int(training_size/batch_size)): batch_images = np.array(images[i*batch_size:(i+1)*batch_size]) batch_labels = np.array(labels[i*batch_size:(i+1)*batch_size]) batch_labels = one_hot(batch_labels, 10) _, loss = model.forward(batch_images, batch_labels) losses.append(loss) model.optimize(optimizer) model.eval_mode() predicts = np.zeros((len(test_labels))) for i in range(int(len(test_labels)/1000)): batch_images = np.array(test_images[i*1000:(i+1)*1000]) pred = model.forward(batch_images) pred = np.argmax(pred, 1) predicts[i*1000:(i+1)*1000] = pred acc = np.sum(test_labels == predicts) * 100 / len(test_labels) pbar.set_description('e:{} loss:{} acc:{}%'.format(epoch, float(np.mean(losses)), acc))