def main(): def epoch_status_function(epoch_time, epoch, average_loss, testing_error, is_best): if is_best: with open(os.path.join(base_path, 'model.pkl'), 'wb') as model_file: pickle.dump(network, model_file) with open(os.path.join(base_path, 'loss.txt'), 'at') as error_file: print('{} {:.4f} {:.10f}'.format(epoch, epoch_time, average_loss), file=error_file) with open(os.path.join(base_path, 'error.txt'), 'at') as error_file: print('{} {:.4f} {:.10f}'.format(epoch, epoch_time, testing_error * 100.0), file=error_file) print("Time: {:7.2f} sec, Epoch: {:4d}, Testing error: {:.5f}%".format( epoch_time, epoch, testing_error * 100.0)) def setup_base_path(runs): def format_float(f): return '{:f}'.format(f).rstrip('0').rstrip('.') base_path = os.path.join(args.output_directory, 'mnist_network_{}_layers_{}_activation_{}_learning_{}_minibatches{}{}{}'.format( '-'.join(args.hidden_layers), args.hidden_function, format_float(args.learning_rate), args.minibatch_size, ('_' + format_float(args.L1) + '_L1') if args.L1 else '', ('_' + format_float(args.L2) + '_L2') if args.L2 else '', ('_' + format_float(args.dropout) + '_dropout') if args.dropout else '')) if runs: run_index = 1 while True: base_path_run = os.path.join(base_path, str(run_index)) if os.path.isdir(base_path_run): run_index += 1 else: base_path = base_path_run break if not os.path.isdir(base_path): os.makedirs(base_path) existing_files = [ f for f in os.listdir(base_path) ] for existing_file in existing_files: try: os.remove(existing_file) except: pass return base_path parser = argparse.ArgumentParser() parser.add_argument('--L1', type=float) parser.add_argument('--L2', type=float) parser.add_argument('--R', type=int) parser.add_argument('--data') parser.add_argument('--dropout', type=float) parser.add_argument('--epochs', type=int, default=100) parser.add_argument('--hidden_function', default='relu') parser.add_argument('--hidden_layers', nargs='*') parser.add_argument('--learning_rate', type=float, default=0.005) parser.add_argument('--major_demo', action='store_true') parser.add_argument('--max_time', type=int) parser.add_argument('--minibatch_size', type=int, default=40) parser.add_argument('--minor_demo', action='store_true') parser.add_argument('--model') parser.add_argument('--output_directory', default='../data') parser.add_argument('--runs', action='store_true') parser.add_argument('--seed', type=int) parser.add_argument('--training_ratio', type=float) args = parser.parse_args() print(args) if args.seed: random.seed(args.seed) numpy.random.seed(random.randint(0, 2 ** 30)) if (args.major_demo or args.minor_demo) and not args.model: print("No model file provided.", file=sys.stderr) sys.exit(-1) if args.model: network = pickle.load(open(args.model, 'rb')) predict_function = theano.function( inputs=[network.inputs], outputs=network.classify()) if args.major_demo: if not args.R: print("No R parameter provided.", file=sys.stderr) sys.exit(-1) ann = Ann(predict_function) d = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../mnist/basics/') print('r = {} d = {}'.format(args.R, d)) mnistdemo.major_demo(ann, args.R, d) elif args.minor_demo: mnist_basics.minor_demo(Ann(predict_function, args.minibatch_size)) elif args.data: dataset, labelset = pickle.load(open(args.data, 'rb')) total = len(dataset) correct = sum(predict_function(dataset[i:i+1])[0] == labelset[i:i+1][0] for i in range(len(dataset))) print("{}/{} ({:.2f}%)".format(correct, total, 100.0 * correct / total)) else: print("No data file provided.", file=sys.stderr) sys.exit(-1) else: base_path = setup_base_path(args.runs) with open(os.path.join(base_path, 'log.txt'), 'at') as log_file: temp_stdout = sys.stdout sys.stdout = Tee(sys.stdout, log_file) layer_sizes = [28 * 28] + list(map(int, args.hidden_layers or [])) + [10] print("Loading dataset 'training' from file...") flat_training_data, flat_training_labels = mnist_basics.load_all_flat_cases('training', dir='../mnist/basics/') print("Loading dataset 'testing' from file...") flat_testing_data, flat_testing_labels = mnist_basics.load_all_flat_cases('testing', dir='../mnist/basics/') print("Creating shared Theano dataset variables...") num_training_examples = int(math.ceil(args.training_ratio * len(flat_training_data))) if args.training_ratio else len(flat_training_data) training_dataset = build_theano_dataset(flat_training_data[:num_training_examples], flat_training_labels[:num_training_examples]) testing_dataset = build_theano_dataset(flat_testing_data, flat_testing_labels) minibatch_index = T.lscalar() x = T.matrix('x') y = T.ivector('y') activation_functions = { "relu": theano.tensor.nnet.relu, "sigmoid": theano.tensor.nnet.sigmoid, "tanh": theano.tensor.tanh } network = vi.theano.Network( x, layer_sizes, activation_functions[args.hidden_function], args.dropout, args.L1, args.L2) training_minibatch_count = int(math.ceil(training_dataset.size / args.minibatch_size)) testing_minibatch_count = int(math.ceil(testing_dataset.size / args.minibatch_size)) training_function, testing_function = build_theano_functions( network, x, y, minibatch_index, args.minibatch_size, args.learning_rate, training_dataset, testing_dataset) print("Starting stochastic gradient descent. num_training_examples={}".format( num_training_examples, args.learning_rate, args.epochs, args.max_time)) training_time, training_epochs, testing_error = \ vi.theano.stochastic_gradient_descent( training_function, training_minibatch_count, testing_function, testing_minibatch_count, learning_rate=args.learning_rate, epochs=args.epochs, max_time=args.max_time, epoch_status_function=epoch_status_function) print(("Training completed after {:.2f} seconds. {} epochs at {:.2f} epochs / second. " + "Testing error: {:.5f}%").format( training_time, training_epochs, training_epochs / training_time, testing_error * 100.0)) time.sleep(1) sys.stdout.flush() sys.stdout = temp_stdout
# def main(): training_acc = 0 testing_acc = 0 total_time = 0 number_of_nets = 20 # for i in range(number_of_nets): training_time = time.time() print ("Network #",i) nn=ANN(0.05, [(784,10)]) nn.training(500) print ("One hidden layer-> 100 nodes") training_acc += nn.test_trainset() testing_acc += nn.test_testset() print("Test set accuracy: ", nn.test_testset()) print("Train set accuracy: ", nn.test_trainset()) total_time += (time.time() - training_time) print(" ") print("average accuracies: ") print("Training set: ", (training_acc/number_of_nets)) print("Testing set: ", (testing_acc/number_of_nets)) print("Average compute time: ", (total_time/number_of_nets)) # nn=ANN(0.1,[(784,100),(100,10)]) start=time.time() nn.training(50) print("det tok: ",time.time()-start," aa trene nettet.") MNIST.minor_demo(nn) #blind_cases = MNIST.read_demo_file("demo_prep") #nn.blind_test(blind_cases)
from ann import ann import mnist_basics as mnist EPOCHS_PER_GAME = 3 BATCH = 100 NEURONS_IN_HIDDEN_LAYERS = [784,500,500,10] LIST_OF_FUNCTIONS = ["rectify","rectify","softmax"] LEARNING_RATE = 0.001 MOMENTUM_RATE = 0.9 a = ann(neuronsInHiddenLayers=NEURONS_IN_HIDDEN_LAYERS, listOfFunctions=LIST_OF_FUNCTIONS, learningRate=LEARNING_RATE, momentumRate=MOMENTUM_RATE, errorFunc="RMSprop") a.run(BATCH,EPOCHS_PER_GAME) mnist.minor_demo(a)
from ann import ann import mnist_basics as mnist EPOCHS_PER_GAME = 3 BATCH = 100 NEURONS_IN_HIDDEN_LAYERS = [784, 500, 500, 10] LIST_OF_FUNCTIONS = ["rectify", "rectify", "softmax"] LEARNING_RATE = 0.001 MOMENTUM_RATE = 0.9 a = ann(neuronsInHiddenLayers=NEURONS_IN_HIDDEN_LAYERS, listOfFunctions=LIST_OF_FUNCTIONS, learningRate=LEARNING_RATE, momentumRate=MOMENTUM_RATE, errorFunc="RMSprop") a.run(BATCH, EPOCHS_PER_GAME) mnist.minor_demo(a)
def main(): def epoch_status_function(epoch_time, epoch, average_loss, testing_error, is_best): if is_best: with open(os.path.join(base_path, 'model.pkl'), 'wb') as model_file: pickle.dump(network, model_file) with open(os.path.join(base_path, 'loss.txt'), 'at') as error_file: print('{} {:.4f} {:.10f}'.format(epoch, epoch_time, average_loss), file=error_file) with open(os.path.join(base_path, 'error.txt'), 'at') as error_file: print('{} {:.4f} {:.10f}'.format(epoch, epoch_time, testing_error * 100.0), file=error_file) print("Time: {:7.2f} sec, Epoch: {:4d}, Testing error: {:.5f}%".format( epoch_time, epoch, testing_error * 100.0)) def setup_base_path(runs): def format_float(f): return '{:f}'.format(f).rstrip('0').rstrip('.') base_path = os.path.join( args.output_directory, 'mnist_network_{}_layers_{}_activation_{}_learning_{}_minibatches{}{}{}' .format('-'.join(args.hidden_layers), args.hidden_function, format_float(args.learning_rate), args.minibatch_size, ('_' + format_float(args.L1) + '_L1') if args.L1 else '', ('_' + format_float(args.L2) + '_L2') if args.L2 else '', ('_' + format_float(args.dropout) + '_dropout') if args.dropout else '')) if runs: run_index = 1 while True: base_path_run = os.path.join(base_path, str(run_index)) if os.path.isdir(base_path_run): run_index += 1 else: base_path = base_path_run break if not os.path.isdir(base_path): os.makedirs(base_path) existing_files = [f for f in os.listdir(base_path)] for existing_file in existing_files: try: os.remove(existing_file) except: pass return base_path parser = argparse.ArgumentParser() parser.add_argument('--L1', type=float) parser.add_argument('--L2', type=float) parser.add_argument('--R', type=int) parser.add_argument('--data') parser.add_argument('--dropout', type=float) parser.add_argument('--epochs', type=int, default=100) parser.add_argument('--hidden_function', default='relu') parser.add_argument('--hidden_layers', nargs='*') parser.add_argument('--learning_rate', type=float, default=0.005) parser.add_argument('--major_demo', action='store_true') parser.add_argument('--max_time', type=int) parser.add_argument('--minibatch_size', type=int, default=40) parser.add_argument('--minor_demo', action='store_true') parser.add_argument('--model') parser.add_argument('--output_directory', default='../data') parser.add_argument('--runs', action='store_true') parser.add_argument('--seed', type=int) parser.add_argument('--training_ratio', type=float) args = parser.parse_args() print(args) if args.seed: random.seed(args.seed) numpy.random.seed(random.randint(0, 2**30)) if (args.major_demo or args.minor_demo) and not args.model: print("No model file provided.", file=sys.stderr) sys.exit(-1) if args.model: network = pickle.load(open(args.model, 'rb')) predict_function = theano.function(inputs=[network.inputs], outputs=network.classify()) if args.major_demo: if not args.R: print("No R parameter provided.", file=sys.stderr) sys.exit(-1) ann = Ann(predict_function) d = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../mnist/basics/') print('r = {} d = {}'.format(args.R, d)) mnistdemo.major_demo(ann, args.R, d) elif args.minor_demo: mnist_basics.minor_demo(Ann(predict_function, args.minibatch_size)) elif args.data: dataset, labelset = pickle.load(open(args.data, 'rb')) total = len(dataset) correct = sum( predict_function(dataset[i:i + 1])[0] == labelset[i:i + 1][0] for i in range(len(dataset))) print("{}/{} ({:.2f}%)".format(correct, total, 100.0 * correct / total)) else: print("No data file provided.", file=sys.stderr) sys.exit(-1) else: base_path = setup_base_path(args.runs) with open(os.path.join(base_path, 'log.txt'), 'at') as log_file: temp_stdout = sys.stdout sys.stdout = Tee(sys.stdout, log_file) layer_sizes = [28 * 28] + list(map(int, args.hidden_layers or [])) + [10] print("Loading dataset 'training' from file...") flat_training_data, flat_training_labels = mnist_basics.load_all_flat_cases( 'training', dir='../mnist/basics/') print("Loading dataset 'testing' from file...") flat_testing_data, flat_testing_labels = mnist_basics.load_all_flat_cases( 'testing', dir='../mnist/basics/') print("Creating shared Theano dataset variables...") num_training_examples = int( math.ceil(args.training_ratio * len(flat_training_data)) ) if args.training_ratio else len(flat_training_data) training_dataset = build_theano_dataset( flat_training_data[:num_training_examples], flat_training_labels[:num_training_examples]) testing_dataset = build_theano_dataset(flat_testing_data, flat_testing_labels) minibatch_index = T.lscalar() x = T.matrix('x') y = T.ivector('y') activation_functions = { "relu": theano.tensor.nnet.relu, "sigmoid": theano.tensor.nnet.sigmoid, "tanh": theano.tensor.tanh } network = vi.theano.Network( x, layer_sizes, activation_functions[args.hidden_function], args.dropout, args.L1, args.L2) training_minibatch_count = int( math.ceil(training_dataset.size / args.minibatch_size)) testing_minibatch_count = int( math.ceil(testing_dataset.size / args.minibatch_size)) training_function, testing_function = build_theano_functions( network, x, y, minibatch_index, args.minibatch_size, args.learning_rate, training_dataset, testing_dataset) print( "Starting stochastic gradient descent. num_training_examples={}" .format(num_training_examples, args.learning_rate, args.epochs, args.max_time)) training_time, training_epochs, testing_error = \ vi.theano.stochastic_gradient_descent( training_function, training_minibatch_count, testing_function, testing_minibatch_count, learning_rate=args.learning_rate, epochs=args.epochs, max_time=args.max_time, epoch_status_function=epoch_status_function) print(( "Training completed after {:.2f} seconds. {} epochs at {:.2f} epochs / second. " + "Testing error: {:.5f}%").format( training_time, training_epochs, training_epochs / training_time, testing_error * 100.0)) time.sleep(1) sys.stdout.flush() sys.stdout = temp_stdout