def main():
def epoch_status_function(epoch_time, epoch, average_loss, testing_error, is_best):
if is_best:
with open(os.path.join(base_path, 'model.pkl'), 'wb') as model_file:
pickle.dump(network, model_file)
with open(os.path.join(base_path, 'loss.txt'), 'at') as error_file:
print('{} {:.4f} {:.10f}'.format(epoch, epoch_time, average_loss), file=error_file)
with open(os.path.join(base_path, 'error.txt'), 'at') as error_file:
print('{} {:.4f} {:.10f}'.format(epoch, epoch_time, testing_error * 100.0), file=error_file)
print("Time: {:7.2f} sec, Epoch: {:4d}, Testing error: {:.5f}%".format(
epoch_time, epoch, testing_error * 100.0))
def setup_base_path(runs):
def format_float(f):
return '{:f}'.format(f).rstrip('0').rstrip('.')
base_path = os.path.join(args.output_directory,
'mnist_network_{}_layers_{}_activation_{}_learning_{}_minibatches{}{}{}'.format(
'-'.join(args.hidden_layers),
args.hidden_function,
format_float(args.learning_rate),
args.minibatch_size,
('_' + format_float(args.L1) + '_L1') if args.L1 else '',
('_' + format_float(args.L2) + '_L2') if args.L2 else '',
('_' + format_float(args.dropout) + '_dropout') if args.dropout else ''))
if runs:
run_index = 1
while True:
base_path_run = os.path.join(base_path, str(run_index))
if os.path.isdir(base_path_run):
run_index += 1
else:
base_path = base_path_run
break
if not os.path.isdir(base_path):
os.makedirs(base_path)
existing_files = [ f for f in os.listdir(base_path) ]
for existing_file in existing_files:
try:
os.remove(existing_file)
except:
pass
return base_path
parser = argparse.ArgumentParser()
parser.add_argument('--L1', type=float)
parser.add_argument('--L2', type=float)
parser.add_argument('--R', type=int)
parser.add_argument('--data')
parser.add_argument('--dropout', type=float)
parser.add_argument('--epochs', type=int, default=100)
parser.add_argument('--hidden_function', default='relu')
parser.add_argument('--hidden_layers', nargs='*')
parser.add_argument('--learning_rate', type=float, default=0.005)
parser.add_argument('--major_demo', action='store_true')
parser.add_argument('--max_time', type=int)
parser.add_argument('--minibatch_size', type=int, default=40)
parser.add_argument('--minor_demo', action='store_true')
parser.add_argument('--model')
parser.add_argument('--output_directory', default='../data')
parser.add_argument('--runs', action='store_true')
parser.add_argument('--seed', type=int)
parser.add_argument('--training_ratio', type=float)
args = parser.parse_args()
print(args)
if args.seed:
random.seed(args.seed)
numpy.random.seed(random.randint(0, 2 ** 30))
if (args.major_demo or args.minor_demo) and not args.model:
print("No model file provided.", file=sys.stderr)
sys.exit(-1)
if args.model:
network = pickle.load(open(args.model, 'rb'))
predict_function = theano.function(
inputs=[network.inputs],
outputs=network.classify())
if args.major_demo:
if not args.R:
print("No R parameter provided.", file=sys.stderr)
sys.exit(-1)
ann = Ann(predict_function)
d = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../mnist/basics/')
print('r = {} d = {}'.format(args.R, d))
mnistdemo.major_demo(ann, args.R, d)
elif args.minor_demo:
mnist_basics.minor_demo(Ann(predict_function, args.minibatch_size))
elif args.data:
dataset, labelset = pickle.load(open(args.data, 'rb'))
total = len(dataset)
correct = sum(predict_function(dataset[i:i+1])[0] == labelset[i:i+1][0] for i in range(len(dataset)))
print("{}/{} ({:.2f}%)".format(correct, total, 100.0 * correct / total))
else:
print("No data file provided.", file=sys.stderr)
sys.exit(-1)
else:
base_path = setup_base_path(args.runs)
with open(os.path.join(base_path, 'log.txt'), 'at') as log_file:
temp_stdout = sys.stdout
sys.stdout = Tee(sys.stdout, log_file)
layer_sizes = [28 * 28] + list(map(int, args.hidden_layers or [])) + [10]
print("Loading dataset 'training' from file...")
flat_training_data, flat_training_labels = mnist_basics.load_all_flat_cases('training', dir='../mnist/basics/')
print("Loading dataset 'testing' from file...")
flat_testing_data, flat_testing_labels = mnist_basics.load_all_flat_cases('testing', dir='../mnist/basics/')
print("Creating shared Theano dataset variables...")
num_training_examples = int(math.ceil(args.training_ratio * len(flat_training_data))) if args.training_ratio else len(flat_training_data)
training_dataset = build_theano_dataset(flat_training_data[:num_training_examples], flat_training_labels[:num_training_examples])
testing_dataset = build_theano_dataset(flat_testing_data, flat_testing_labels)
minibatch_index = T.lscalar()
x = T.matrix('x')
y = T.ivector('y')
activation_functions = { "relu": theano.tensor.nnet.relu, "sigmoid": theano.tensor.nnet.sigmoid, "tanh": theano.tensor.tanh }
network = vi.theano.Network(
x, layer_sizes, activation_functions[args.hidden_function], args.dropout, args.L1, args.L2)
training_minibatch_count = int(math.ceil(training_dataset.size / args.minibatch_size))
testing_minibatch_count = int(math.ceil(testing_dataset.size / args.minibatch_size))
training_function, testing_function = build_theano_functions(
network, x, y, minibatch_index, args.minibatch_size, args.learning_rate,
training_dataset, testing_dataset)
print("Starting stochastic gradient descent. num_training_examples={}".format(
num_training_examples, args.learning_rate, args.epochs, args.max_time))
training_time, training_epochs, testing_error = \
vi.theano.stochastic_gradient_descent(
training_function,
training_minibatch_count,
testing_function,
testing_minibatch_count,
learning_rate=args.learning_rate,
epochs=args.epochs,
max_time=args.max_time,
epoch_status_function=epoch_status_function)
print(("Training completed after {:.2f} seconds. {} epochs at {:.2f} epochs / second. " +
"Testing error: {:.5f}%").format(
training_time,
training_epochs,
training_epochs / training_time,
testing_error * 100.0))
time.sleep(1)
sys.stdout.flush()
sys.stdout = temp_stdout
#
def main():
training_acc = 0
testing_acc = 0
total_time = 0
number_of_nets = 20
#
for i in range(number_of_nets):
training_time = time.time()
print ("Network #",i)
nn=ANN(0.05, [(784,10)])
nn.training(500)
print ("One hidden layer-> 100 nodes")
training_acc += nn.test_trainset()
testing_acc += nn.test_testset()
print("Test set accuracy: ", nn.test_testset())
print("Train set accuracy: ", nn.test_trainset())
total_time += (time.time() - training_time)
print(" ")
print("average accuracies: ")
print("Training set: ", (training_acc/number_of_nets))
print("Testing set: ", (testing_acc/number_of_nets))
print("Average compute time: ", (total_time/number_of_nets))
#
nn=ANN(0.1,[(784,100),(100,10)])
start=time.time()
nn.training(50)
print("det tok: ",time.time()-start," aa trene nettet.")
MNIST.minor_demo(nn)
#blind_cases = MNIST.read_demo_file("demo_prep")
#nn.blind_test(blind_cases)
from ann import ann import mnist_basics as mnist EPOCHS_PER_GAME = 3 BATCH = 100 NEURONS_IN_HIDDEN_LAYERS = [784,500,500,10] LIST_OF_FUNCTIONS = ["rectify","rectify","softmax"] LEARNING_RATE = 0.001 MOMENTUM_RATE = 0.9 a = ann(neuronsInHiddenLayers=NEURONS_IN_HIDDEN_LAYERS, listOfFunctions=LIST_OF_FUNCTIONS, learningRate=LEARNING_RATE, momentumRate=MOMENTUM_RATE, errorFunc="RMSprop") a.run(BATCH,EPOCHS_PER_GAME) mnist.minor_demo(a)
from ann import ann
import mnist_basics as mnist
EPOCHS_PER_GAME = 3
BATCH = 100
NEURONS_IN_HIDDEN_LAYERS = [784, 500, 500, 10]
LIST_OF_FUNCTIONS = ["rectify", "rectify", "softmax"]
LEARNING_RATE = 0.001
MOMENTUM_RATE = 0.9
a = ann(neuronsInHiddenLayers=NEURONS_IN_HIDDEN_LAYERS,
listOfFunctions=LIST_OF_FUNCTIONS,
learningRate=LEARNING_RATE,
momentumRate=MOMENTUM_RATE,
errorFunc="RMSprop")
a.run(BATCH, EPOCHS_PER_GAME)
mnist.minor_demo(a)
def main():
def epoch_status_function(epoch_time, epoch, average_loss, testing_error,
is_best):
if is_best:
with open(os.path.join(base_path, 'model.pkl'),
'wb') as model_file:
pickle.dump(network, model_file)
with open(os.path.join(base_path, 'loss.txt'), 'at') as error_file:
print('{} {:.4f} {:.10f}'.format(epoch, epoch_time, average_loss),
file=error_file)
with open(os.path.join(base_path, 'error.txt'), 'at') as error_file:
print('{} {:.4f} {:.10f}'.format(epoch, epoch_time,
testing_error * 100.0),
file=error_file)
print("Time: {:7.2f} sec, Epoch: {:4d}, Testing error: {:.5f}%".format(
epoch_time, epoch, testing_error * 100.0))
def setup_base_path(runs):
def format_float(f):
return '{:f}'.format(f).rstrip('0').rstrip('.')
base_path = os.path.join(
args.output_directory,
'mnist_network_{}_layers_{}_activation_{}_learning_{}_minibatches{}{}{}'
.format('-'.join(args.hidden_layers), args.hidden_function,
format_float(args.learning_rate), args.minibatch_size,
('_' + format_float(args.L1) + '_L1') if args.L1 else '',
('_' + format_float(args.L2) + '_L2') if args.L2 else '',
('_' + format_float(args.dropout) +
'_dropout') if args.dropout else ''))
if runs:
run_index = 1
while True:
base_path_run = os.path.join(base_path, str(run_index))
if os.path.isdir(base_path_run):
run_index += 1
else:
base_path = base_path_run
break
if not os.path.isdir(base_path):
os.makedirs(base_path)
existing_files = [f for f in os.listdir(base_path)]
for existing_file in existing_files:
try:
os.remove(existing_file)
except:
pass
return base_path
parser = argparse.ArgumentParser()
parser.add_argument('--L1', type=float)
parser.add_argument('--L2', type=float)
parser.add_argument('--R', type=int)
parser.add_argument('--data')
parser.add_argument('--dropout', type=float)
parser.add_argument('--epochs', type=int, default=100)
parser.add_argument('--hidden_function', default='relu')
parser.add_argument('--hidden_layers', nargs='*')
parser.add_argument('--learning_rate', type=float, default=0.005)
parser.add_argument('--major_demo', action='store_true')
parser.add_argument('--max_time', type=int)
parser.add_argument('--minibatch_size', type=int, default=40)
parser.add_argument('--minor_demo', action='store_true')
parser.add_argument('--model')
parser.add_argument('--output_directory', default='../data')
parser.add_argument('--runs', action='store_true')
parser.add_argument('--seed', type=int)
parser.add_argument('--training_ratio', type=float)
args = parser.parse_args()
print(args)
if args.seed:
random.seed(args.seed)
numpy.random.seed(random.randint(0, 2**30))
if (args.major_demo or args.minor_demo) and not args.model:
print("No model file provided.", file=sys.stderr)
sys.exit(-1)
if args.model:
network = pickle.load(open(args.model, 'rb'))
predict_function = theano.function(inputs=[network.inputs],
outputs=network.classify())
if args.major_demo:
if not args.R:
print("No R parameter provided.", file=sys.stderr)
sys.exit(-1)
ann = Ann(predict_function)
d = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'../mnist/basics/')
print('r = {} d = {}'.format(args.R, d))
mnistdemo.major_demo(ann, args.R, d)
elif args.minor_demo:
mnist_basics.minor_demo(Ann(predict_function, args.minibatch_size))
elif args.data:
dataset, labelset = pickle.load(open(args.data, 'rb'))
total = len(dataset)
correct = sum(
predict_function(dataset[i:i + 1])[0] == labelset[i:i + 1][0]
for i in range(len(dataset)))
print("{}/{} ({:.2f}%)".format(correct, total,
100.0 * correct / total))
else:
print("No data file provided.", file=sys.stderr)
sys.exit(-1)
else:
base_path = setup_base_path(args.runs)
with open(os.path.join(base_path, 'log.txt'), 'at') as log_file:
temp_stdout = sys.stdout
sys.stdout = Tee(sys.stdout, log_file)
layer_sizes = [28 * 28] + list(map(int, args.hidden_layers
or [])) + [10]
print("Loading dataset 'training' from file...")
flat_training_data, flat_training_labels = mnist_basics.load_all_flat_cases(
'training', dir='../mnist/basics/')
print("Loading dataset 'testing' from file...")
flat_testing_data, flat_testing_labels = mnist_basics.load_all_flat_cases(
'testing', dir='../mnist/basics/')
print("Creating shared Theano dataset variables...")
num_training_examples = int(
math.ceil(args.training_ratio * len(flat_training_data))
) if args.training_ratio else len(flat_training_data)
training_dataset = build_theano_dataset(
flat_training_data[:num_training_examples],
flat_training_labels[:num_training_examples])
testing_dataset = build_theano_dataset(flat_testing_data,
flat_testing_labels)
minibatch_index = T.lscalar()
x = T.matrix('x')
y = T.ivector('y')
activation_functions = {
"relu": theano.tensor.nnet.relu,
"sigmoid": theano.tensor.nnet.sigmoid,
"tanh": theano.tensor.tanh
}
network = vi.theano.Network(
x, layer_sizes, activation_functions[args.hidden_function],
args.dropout, args.L1, args.L2)
training_minibatch_count = int(
math.ceil(training_dataset.size / args.minibatch_size))
testing_minibatch_count = int(
math.ceil(testing_dataset.size / args.minibatch_size))
training_function, testing_function = build_theano_functions(
network, x, y, minibatch_index, args.minibatch_size,
args.learning_rate, training_dataset, testing_dataset)
print(
"Starting stochastic gradient descent. num_training_examples={}"
.format(num_training_examples, args.learning_rate, args.epochs,
args.max_time))
training_time, training_epochs, testing_error = \
vi.theano.stochastic_gradient_descent(
training_function,
training_minibatch_count,
testing_function,
testing_minibatch_count,
learning_rate=args.learning_rate,
epochs=args.epochs,
max_time=args.max_time,
epoch_status_function=epoch_status_function)
print((
"Training completed after {:.2f} seconds. {} epochs at {:.2f} epochs / second. "
+ "Testing error: {:.5f}%").format(
training_time, training_epochs,
training_epochs / training_time, testing_error * 100.0))
time.sleep(1)
sys.stdout.flush()
sys.stdout = temp_stdout
