def search(results_path, network_type, num_layers, num_neurons, batch_size,
num_epochs, training_method, regularization):
"""Search relevant files.
based on input arguments and return a list of filename
Parameters
----------
results_path : string
Destination of result files
network_type : string
feedforward or RNN
num_layers : string
either integer from 1-5 or "all"
num_neurons : string
either integer from 10-300 or "all"
batch_size : string
batch size for each mini-batch samples
num_epochs : string
total number of training epochs
training method : string
sgd, momentum, adagrad, rmsprop, all
regularization : string
l2, dropout or none, all
Returns
-------
result_list : Dictionary
a list of relevant result files
"""
all_layers = xrange(1, 6)
all_neurons = xrange(10, 305, 5)
all_methods = ["sgd", "momentum", "adagrad", "rmsprop"]
all_regularization = ["dropout", "l2"]
result_list = []
if num_layers != "all":
all_layers = [int(num_layers)]
if num_neurons != "all":
all_neurons = [int(num_neurons)]
if training_method != "all":
all_methods = [training_method]
if regularization != "all":
all_regularization = [regularization]
for n_layers in all_layers:
for n_neurons in all_neurons:
for method in all_methods:
for regular in all_regularization:
exp_id = ds.create_exp_id(network_type, n_layers,
n_neurons, batch_size,
num_epochs, method, regular)
file_path = os.path.join(results_path, exp_id + ".pkl")
with open(file_path, 'r') as f:
result_list.append(pickle.load(f))
return result_list
def search(results_path, network_type, batch_size, num_epochs):
"""Find all available results available in results destination.
Parameters
----------
results_path : string
destination of experiment results
network_type : string
feedforward or RNN
batch_size : string
batch size for each mini-batch samples
num_epochs : string
total number of training epochs
Returns
-------
result_lit : List
a list of relevant result files
"""
all_layers = xrange(1, 6)
all_neurons = xrange(10, 305, 5)
all_methods = ["sgd", "momentum", "adagrad", "rmsprop"]
all_regularization = ["dropout", "l2"]
result_list = []
for n_layers in all_layers:
for n_neurons in all_neurons:
for method in all_methods:
for regular in all_regularization:
exp_id = ds.create_exp_id(network_type, n_layers,
n_neurons, batch_size,
num_epochs, method, regular)
file_path = os.path.join(results_path, exp_id + ".pkl")
if os.path.exists(file_path):
with open(file_path, 'r') as f:
result_list.append(pickle.load(f))
return result_list
def rf_lstm_experiment(data_name, exp_network, in_dim, out_dim, num_layers,
start_neurons, num_neurons, batch_size, num_epochs):
"""LSTM Experiment."""
# load dataset
train_set = IterableDataset(
ds.transform_sequence(data_name, "train", batch_size))
test_set = IterableDataset(
ds.transform_sequence(data_name, "test", batch_size))
stream_train = DataStream(dataset=train_set)
stream_test = DataStream(dataset=test_set)
methods = ['sgd', 'momentum', 'adagrad', 'rmsprop']
for n_layers in xrange(1, num_layers + 1):
for n_neurons in xrange(start_neurons, num_neurons + 5, 5):
for method in methods:
X = T.tensor3("features")
y = T.matrix("targets")
x_to_h = Linear(in_dim,
n_neurons * 4,
name='x_to_h',
weights_init=IsotropicGaussian(),
biases_init=Constant(0.0))
lstm = LSTM(n_neurons,
name='lstm',
weights_init=IsotropicGaussian(),
biases_init=Constant(0.0))
h_to_o = nc.setup_ff_network(n_neurons, out_dim, n_layers - 1,
n_neurons)
X_trans = x_to_h.apply(X)
h, c = lstm.apply(X_trans)
y_hat = h_to_o.apply(h[-1])
cost, cg = nc.create_cg_and_cost(y, y_hat, "none")
lstm.initialize()
x_to_h.initialize()
h_to_o.initialize()
algorithm = nc.setup_algorithms(cost, cg, method, type="RNN")
test_monitor = DataStreamMonitoring(variables=[cost],
data_stream=stream_test,
prefix="test")
train_monitor = TrainingDataMonitoring(variables=[cost],
prefix="train",
after_epoch=True)
main_loop = MainLoop(
algorithm=algorithm,
data_stream=stream_train,
extensions=[
test_monitor, train_monitor,
FinishAfter(after_n_epochs=num_epochs),
Printing(),
ProgressBar()
])
main_loop.run()
# Saving results
exp_id = ds.create_exp_id(exp_network, n_layers, n_neurons,
batch_size, num_epochs, method,
"none")
# prepare related functions
predict = theano.function([X], y_hat)
# prepare related data
train_features, train_targets = ds.get_iter_data(train_set)
test_features, test_targets = ds.get_iter_data(test_set)
# Prediction of result
train_predicted = gen_prediction(predict, train_features)
test_predicted = gen_prediction(predict, test_features)
# Get cost
cost = ds.get_cost_data(test_monitor, train_set.num_examples,
num_epochs)
# logging
ds.save_experiment(train_targets, train_predicted,
test_targets, test_predicted, cost,
exp_network, n_layers, n_neurons,
batch_size, num_epochs, method, "none",
exp_id, "../results/")
def analysis(results_path, network_type, num_layers, num_neurons, batch_size,
num_epochs, training_method, regularization, mode):
"""An Analysis result from given arguments.
Parameters
----------
results_path : string
Destination of result files
network_type : string
feedforward or RNN
num_layers : string
either integer from 1-5 or "all"
num_neurons : string
either integer from 10-300 or "all"
batch_size : string
batch size for each mini-batch samples
num_epochs : string
total number of training epochs
training method : string
sgd, momentum, adagrad, rmsprop, all
regularization : string
l2, dropout or none, all
mode : string
output mode: targets-predicted, epochs-cost, cost-algorithm,
neurons-cost, cost-regular
"""
if mode == "targets-predicted":
assert num_layers != "all", "num-layers should be 1-5 \
in targets-predicted mode"
assert num_neurons != "all", "num-neurons should be 10-300 \
in targets-predicted mode"
assert training_method != "all", "training-method shouldn't \
be all in targets-predicted mode"
assert regularization != "all", "regularization shouldn't be all \
in targets-predicted mode"
results_list = search(results_path, network_type, num_layers,
num_neurons, batch_size, num_epochs,
training_method, regularization)
result = results_list[0]
train_fn = os.path.join(results_path, "images",
result["exp_id"] + "_targets-predicted-train")
test_fn = os.path.join(results_path, "images",
result['exp_id'] + "_targets-predicted-test")
draw.draw_target_predicted(result['train_targets'],
result['train_predicted'], train_fn)
draw.draw_target_predicted(result['test_targets'],
result['test_predicted'], test_fn)
elif mode == "epochs-cost":
assert num_layers != "all", "num-layers should be 1-5 \
in epochs-cost mode"
assert num_neurons != "all", "num-neurons should be 10-300 \
in epochs-cost mode"
assert training_method != "all", "training-method shouldn't be \
all in epochs-cost mode"
assert regularization != "all", "regularization shouldn't be \
all in epochs-cost mode"
results_list = search(results_path, network_type, num_layers,
num_neurons, batch_size, num_epochs,
training_method, regularization)
result = results_list[0]
fn = os.path.join(results_path, "images",
result['exp_id'] + "_epochs-cost")
draw.draw_epochs_cost(result['cost'], fn)
elif mode == "cost-algorithm":
assert num_layers != "all", "num-layers should be 1-5 \
in cost-algorithm mode"
assert num_neurons != "all", "num-neurons should be 10-300 \
in cost-algorithm mode"
assert training_method == "all", "training-method should be all \
in cost-algorithm mode"
assert regularization != "all", "regularization shouldn't be all \
in cost-algorithm mode"
results_list = search(results_path, network_type, num_layers,
num_neurons, batch_size, num_epochs,
training_method, regularization)
cost_arr = results_list[0]['cost'][1, :]
for k in xrange(1, len(results_list)):
cost_arr = np.vstack((cost_arr, results_list[k]['cost'][1, :]))
fn = ds.create_exp_id(network_type, num_layers, num_neurons,
batch_size, num_epochs, "all", regularization)
fn = os.path.join(results_path, "images", fn + "_cost-algorithm")
draw.draw_cost_algorithms(cost_arr, fn)
elif mode == "neurons-cost":
assert num_layers == "all", "num-layers should be all \
in neurons-cost mode"
assert num_neurons == "all", "num-neurons should be all \
in neurons-cost mode"
assert training_method != "all", "training-method shouldn't be \
all in neurons-cost mode"
assert regularization != "all", "regularization shouldn't be \
all in neurons-cost mode"
results_list = search(results_path, network_type, num_layers,
num_neurons, batch_size, num_epochs,
training_method, regularization)
cost_arr = np.zeros((5, 59))
for i in xrange(5):
for k in xrange(59):
cost_arr[i, k] = np.min(results_list[i * 59 + k]['cost'][1, :])
fn = ds.create_exp_id(network_type, "all", "all", batch_size,
num_epochs, training_method, regularization)
fn = os.path.join(results_path, "images", fn + "_neurons-cost")
draw.draw_neurons_layers_cost(cost_arr, fn)
elif mode == "cost-regular":
assert num_layers != "all", "num-layers should be 1-5 \
in cost-regular mode"
assert num_neurons != "all", "num-neurons should be 10-300 \
in cost-regular mode"
assert training_method != "all", "training-method shouldn't be \
all in cost-regular mode"
assert regularization == "all", "regularization should be all \
in cost-regular mode"
results_list = search(results_path, network_type, num_layers,
num_neurons, batch_size, num_epochs,
training_method, regularization)
cost_arr = results_list[0]['cost'][1, :]
cost_arr = np.vstack((cost_arr, results_list[1]['cost'][1, :]))
fn = ds.create_exp_id(network_type, num_layers, num_neurons,
batch_size, num_epochs, training_method, "all")
fn = os.path.join(results_path, "images", fn + "_cost-regular")
draw.draw_cost_dropout(cost_arr, fn)
else:
print "Error"
return
def rf_ff_experiment(data_name,
exp_network,
in_dim,
out_dim,
num_layers,
start_neurons,
num_neurons,
batch_size,
num_epochs):
"""Feedforward Experiment."""
# load dataset
train_set, stream_train, test_set, stream_test = ds.prepare_datastream(
data_name, batch_size)
methods = ['sgd', 'momentum', 'adagrad', 'rmsprop']
regularization = ['l2', 'dropout'] # regularization in list
for n_layers in xrange(1, num_layers + 1):
for n_neurons in xrange(start_neurons, num_neurons + 5, 5):
for method in methods:
for regular in regularization:
# setup network
X = T.matrix("features")
y = T.matrix("targets")
net = nc.setup_ff_network(
in_dim, out_dim, n_layers, n_neurons)
y_hat = net.apply(X)
cost, cg = nc.create_cg_and_cost(y, y_hat, regular)
net.initialize()
algorithm = nc.setup_algorithms(cost, cg, method)
test_monitor = DataStreamMonitoring(
variables=[cost],
data_stream=stream_test,
prefix="test")
train_monitor = TrainingDataMonitoring(variables=[cost],
prefix="train",
after_epoch=True)
main_loop = MainLoop(
algorithm=algorithm,
data_stream=stream_train,
extensions=[test_monitor,
train_monitor,
FinishAfter(
after_n_epochs=num_epochs),
Printing(), ProgressBar()])
main_loop.run()
# Saving results
exp_id = ds.create_exp_id(exp_network, n_layers, n_neurons,
batch_size, num_epochs, method,
regular)
# prepare related functions
predict = theano.function([X], y_hat)
# prepare related data
train_features, train_targets = ds.get_data(train_set)
test_features, test_targets = ds.get_data(test_set)
# Prediction of result
train_predicted = predict(train_features)
test_predicted = predict(test_features)
# Get cost
cost = ds.get_cost_data(
test_monitor, train_set.num_examples / batch_size,
num_epochs)
# logging
ds.save_experiment(train_targets, train_predicted,
test_targets, test_predicted,
cost, exp_network, n_layers, n_neurons,
batch_size, num_epochs, method, regular,
exp_id, "../results/")