if __name__ == '__main__': freeze_support() num_hidden_units = 500 num_hidden_layers = 5 num_passes = 30 # data = dataset.mnist_dataset.load('dataset/mnist') data = dataset.cifar10_dataset.load() initializers = [ weight_initializer.Fill(0), weight_initializer.Fill(1e-3), weight_initializer.Fill(1), weight_initializer.RandomUniform(-1, 1), weight_initializer.RandomUniform(-1/np.sqrt(num_hidden_units), 1/np.sqrt(num_hidden_units)), weight_initializer.RandomUniform(-1/num_hidden_units, 1/num_hidden_units), weight_initializer.RandomNormal(1, 0), weight_initializer.RandomNormal(1 / np.sqrt(num_hidden_units)) ] labels = [ 'Fill(0)', 'Fill(0.001)', 'Fill(1)', 'Uniform(low=-1, high=1)', 'Uniform(low=-1/sqrt(fan_out), high=1/sqrt(fan_out))', 'Uniform(low=-1/fan_out, high=1/fan_out)', 'Normal(sigma=1, mu=0)', 'Normal(sigma=1/sqrt(fan_out), mu=0)',
from network.optimizer import GDMomentumOptimizer if __name__ == '__main__': freeze_support() data = dataset.cifar10_dataset.load() num_passes = 30 initializers = [[], [], [], [], [], [], []] for i in [8*16*16, 16*8*8, 32*4*4]: initializers[0].append(weight_initializer.Fill(0)), initializers[1].append(weight_initializer.Fill(1e-3)), initializers[2].append(weight_initializer.Fill(1)), initializers[3].append(weight_initializer.RandomUniform(-1, 1)) initializers[4].append(weight_initializer.RandomUniform(-1/np.sqrt(i), 1/np.sqrt( i))) initializers[5].append(weight_initializer.RandomNormal()) initializers[6].append(weight_initializer.RandomNormal(1/np.sqrt(i))) labels = [ 'Fill(0)', 'Fill(0.001)', 'Fill(1)', 'Uniform(low=-1, high=1)', 'Uniform(low=-1/sqrt(fan_out), high=1/sqrt(fan_out))', 'Normal(sigma=1, mu=0)', 'Normal(sigma=1/sqrt(fan_out), mu=0)', ] statistics = []
from network.layers.conv_to_fully_connected import ConvToFullyConnected from network.layers.fully_connected import FullyConnected from network.model import Model from network.optimizer import GDMomentumOptimizer if __name__ == '__main__': """ Goal: Compare """ freeze_support() num_hidden_units = 240 initializers = [ weight_initializer.RandomUniform(-1, 1), weight_initializer.RandomUniform(-100, 100), ] lrs = [1e-2, 1e-4] statistics = [] for initializer, lr in zip(initializers, lrs): layers = [ ConvToFullyConnected(), FullyConnected(size=num_hidden_units, activation=activation.tanh, fb_weight_initializer=initializer), FullyConnected(size=num_hidden_units, activation=activation.tanh,