def main():
configure_theano()
options = parse_options()
config_file = options['config']
config = ConfigParser.ConfigParser()
config.read(config_file)
print('CLI options: {}'.format(options.items()))
print('Reading Config File: {}...'.format(config_file))
print(config.items('stream1'))
print(config.items('lstm_classifier'))
print(config.items('training'))
print('preprocessing dataset...')
# stream 1
s1_data = load_mat_file(config.get('stream1', 'data'))
s1_imagesize = tuple([int(d) for d in config.get('stream1', 'imagesize').split(',')])
s1 = config.get('stream1', 'model')
s1_inputdim = config.getint('stream1', 'input_dimensions')
s1_shape = config.get('stream1', 'shape')
s1_nonlinearities = config.get('stream1', 'nonlinearities')
# lstm classifier
fusiontype = config.get('lstm_classifier', 'fusiontype')
weight_init = options['weight_init'] if 'weight_init' in options else config.get('lstm_classifier', 'weight_init')
use_peepholes = options['use_peepholes'] if 'use_peepholes' in options else config.getboolean('lstm_classifier',
'use_peepholes')
use_blstm = True if config.has_option('lstm_classifier', 'use_blstm') else False
windowsize = config.getint('lstm_classifier', 'windowsize')
output_classes = config.getint('lstm_classifier', 'output_classes')
output_classnames = config.get('lstm_classifier', 'output_classnames').split(',')
lstm_size = config.getint('lstm_classifier', 'lstm_size')
matlab_target_offset = config.getboolean('lstm_classifier', 'matlab_target_offset')
# capture training parameters
validation_window = int(options['validation_window']) \
if 'validation_window' in options else config.getint('training', 'validation_window')
num_epoch = int(options['num_epoch']) if 'num_epoch' in options else config.getint('training', 'num_epoch')
learning_rate = options['learning_rate'] if 'learning_rate' in options \
else config.getfloat('training', 'learning_rate')
epochsize = config.getint('training', 'epochsize')
batchsize = config.getint('training', 'batchsize')
weight_init_fn = las.init.GlorotUniform()
if weight_init == 'glorot':
weight_init_fn = las.init.GlorotUniform()
if weight_init == 'norm':
weight_init_fn = las.init.Normal(0.1)
if weight_init == 'uniform':
weight_init_fn = las.init.Uniform()
if weight_init == 'ortho':
weight_init_fn = las.init.Orthogonal()
s1_data_matrix = s1_data['dataMatrix'].astype('float32')
#visualize_images(s1_data_matrix[800:864])
targets_vec = s1_data['targetsVec'].reshape((-1,))
subjects_vec = s1_data['subjectsVec'].reshape((-1,))
vidlen_vec = s1_data['videoLengthVec'].reshape((-1,))
iter_vec = s1_data['iterVec'].reshape((-1,))
if matlab_target_offset:
targets_vec -= 1
s1_data_matrix = presplit_dataprocessing(s1_data_matrix, vidlen_vec, config, 'stream1', imagesize=s1_imagesize)
indexes = create_split_index(len(s1_data_matrix), vidlen_vec, iter_vec)
s1_train_vidlens, s1_val_vidlens = split_videolen(vidlen_vec, iter_vec)
# split the data
s1_train_X = s1_data_matrix[indexes == True]
s1_train_y = targets_vec[indexes == True].reshape((-1,))
s1_val_X = s1_data_matrix[indexes == False]
s1_val_y = targets_vec[indexes == False].reshape((-1,))
s1_train_X, s1_val_X = postsplit_datapreprocessing(s1_train_X, s1_val_X, config, 'stream1')
s1_has_encoder = config.getboolean('stream1', 'has_encoder')
# IMPT: the encoder was trained with fortan ordered images, so to visualize
# convert all the images to C order using reshape_images_order()
# output = dbn.predict(test_X)
# test_X = reshape_images_order(test_X, (26, 44))
# output = reshape_images_order(output, (26, 44))
# visualize_reconstruction(test_X[:36, :], output[:36, :], shape=(26, 44))
window = T.iscalar('theta')
inputs1 = T.tensor3('inputs1', dtype='float32')
mask = T.matrix('mask', dtype='uint8')
targets = T.imatrix('targets')
print('constructing end to end model...')
if s1_has_encoder:
ae1 = load_decoder(s1, s1_shape, s1_nonlinearities)
network = deltanet_majority_vote.create_model(ae1, (None, None, s1_inputdim), inputs1,
(None, None), mask,
lstm_size, window, output_classes, weight_init_fn,
use_peepholes, use_blstm)
else:
network = deltanet_v1.create_model((None, None, s1_inputdim), inputs1,
(None, None), mask, window,
lstm_size, output_classes, weight_init_fn, use_peepholes, use_blstm)
print_network(network)
# draw_to_file(las.layers.get_all_layers(network), 'network.png')
print('compiling model...')
predictions = las.layers.get_output(network, deterministic=False)
all_params = las.layers.get_all_params(network, trainable=True)
cost = temporal_softmax_loss(predictions, targets, mask)
updates = adam(cost, all_params, learning_rate=learning_rate)
train = theano.function(
[inputs1, targets, mask, window],
cost, updates=updates, allow_input_downcast=True)
compute_train_cost = theano.function([inputs1, targets, mask, window],
cost, allow_input_downcast=True)
test_predictions = las.layers.get_output(network, deterministic=True)
test_cost = temporal_softmax_loss(test_predictions, targets, mask)
compute_test_cost = theano.function(
[inputs1, targets, mask, window], test_cost, allow_input_downcast=True)
val_fn = theano.function([inputs1, mask, window], test_predictions, allow_input_downcast=True)
# We'll train the network with 10 epochs of 30 minibatches each
print('begin training...')
cost_train = []
cost_val = []
class_rate = []
STRIP_SIZE = 3
val_window = circular_list(validation_window)
train_strip = np.zeros((STRIP_SIZE,))
best_val = float('inf')
best_cr = 0.0
datagen = gen_lstm_batch_random(s1_train_X, s1_train_y, s1_train_vidlens, batchsize=batchsize)
val_datagen = gen_lstm_batch_random(s1_val_X, s1_val_y, s1_val_vidlens, batchsize=len(s1_val_vidlens))
# We'll use this "validation set" to periodically check progress
X_val, y_val, mask_val, idxs_val = next(val_datagen)
# reshape the targets for validation
y_val_evaluate = y_val
y_val = y_val.reshape((-1, 1)).repeat(mask_val.shape[-1], axis=-1)
for epoch in range(num_epoch):
time_start = time.time()
for i in range(epochsize):
X, y, m, batch_idxs = next(datagen)
# repeat targets based on max sequence len
y = y.reshape((-1, 1))
y = y.repeat(m.shape[-1], axis=-1)
print_str = 'Epoch {} batch {}/{}: {} examples using adam with learning rate = {}'.format(
epoch + 1, i + 1, epochsize, len(X), learning_rate)
print(print_str, end='')
sys.stdout.flush()
train(X, y, m, windowsize)
print('\r', end='')
cost = compute_train_cost(X, y, m, windowsize)
val_cost = compute_test_cost(X_val, y_val, mask_val, windowsize)
cost_train.append(cost)
cost_val.append(val_cost)
train_strip[epoch % STRIP_SIZE] = cost
val_window.push(val_cost)
gl = 100 * (cost_val[-1] / np.min(cost_val) - 1)
pk = 1000 * (np.sum(train_strip) / (STRIP_SIZE * np.min(train_strip)) - 1)
pq = gl / pk
cr, val_conf = evaluate_model2(X_val, y_val_evaluate, mask_val, windowsize, val_fn)
class_rate.append(cr)
print("Epoch {} train cost = {}, val cost = {}, "
"GL loss = {:.3f}, GQ = {:.3f}, CR = {:.3f} ({:.1f}sec)"
.format(epoch + 1, cost_train[-1], cost_val[-1], gl, pq, cr, time.time() - time_start))
if val_cost < best_val:
best_val = val_cost
best_cr = cr
best_conf = val_conf
best_params = las.layers.get_all_param_values(network)
if epoch >= validation_window and early_stop2(val_window, best_val, validation_window):
break
print('Final Model')
print('CR: {}, val loss: {}'.format(best_cr, best_val))
# plot confusion matrix
table_str = plot_confusion_matrix(best_conf, output_classnames, fmt='pipe')
print('confusion matrix: ')
print(table_str)
if 'save_plot' in options:
prefix = options['save_plot']
plot_validation_cost(cost_train, cost_val, savefilename='{}.validloss.png'.format(prefix))
with open('{}.confmat.txt'.format(prefix), mode='a') as f:
f.write(table_str)
f.write('\n\n')
if 'write_results' in options:
print('writing results to {}'.format(options['write_results']))
results_file = options['write_results']
with open(results_file, mode='a') as f:
f.write('{},{}\n'.format(best_cr, best_val))
if 'save_best' in options:
print('saving best model...')
las.layers.set_all_param_values(network, best_params)
save_model_params(network, options['save_best'])
print('best model saved to {}'.format(options['save_best']))
def main():
configure_theano()
options = parse_options()
config_file = options['config']
config = ConfigParser.ConfigParser()
config.read(config_file)
print('CLI options: {}'.format(options.items()))
print('Reading Config File: {}...'.format(config_file))
print(config.items('stream1'))
print(config.items('lstm_classifier'))
print(config.items('training'))
print('preprocessing dataset...')
# stream 1
s1_data = load_mat_file(config.get('stream1', 'data'))
s1_imagesize = tuple(
[int(d) for d in config.get('stream1', 'imagesize').split(',')])
s1 = config.get('stream1', 'model')
s1_inputdim = config.getint('stream1', 'input_dimensions')
s1_shape = config.get('stream1', 'shape')
s1_nonlinearities = config.get('stream1', 'nonlinearities')
# lstm classifier
fusiontype = config.get('lstm_classifier', 'fusiontype')
weight_init = options[
'weight_init'] if 'weight_init' in options else config.get(
'lstm_classifier', 'weight_init')
use_peepholes = options[
'use_peepholes'] if 'use_peepholes' in options else config.getboolean(
'lstm_classifier', 'use_peepholes')
use_blstm = True if config.has_option('lstm_classifier',
'use_blstm') else False
windowsize = config.getint('lstm_classifier', 'windowsize')
output_classes = config.getint('lstm_classifier', 'output_classes')
output_classnames = config.get('lstm_classifier',
'output_classnames').split(',')
lstm_size = config.getint('lstm_classifier', 'lstm_size')
matlab_target_offset = config.getboolean('lstm_classifier',
'matlab_target_offset')
# capture training parameters
validation_window = int(options['validation_window']) \
if 'validation_window' in options else config.getint('training', 'validation_window')
num_epoch = int(
options['num_epoch']) if 'num_epoch' in options else config.getint(
'training', 'num_epoch')
learning_rate = options['learning_rate'] if 'learning_rate' in options \
else config.getfloat('training', 'learning_rate')
epochsize = config.getint('training', 'epochsize')
batchsize = config.getint('training', 'batchsize')
weight_init_fn = las.init.GlorotUniform()
if weight_init == 'glorot':
weight_init_fn = las.init.GlorotUniform()
if weight_init == 'norm':
weight_init_fn = las.init.Normal(0.1)
if weight_init == 'uniform':
weight_init_fn = las.init.Uniform()
if weight_init == 'ortho':
weight_init_fn = las.init.Orthogonal()
s1_data_matrix = s1_data['dataMatrix'].astype('float32')
#visualize_images(s1_data_matrix[800:864])
targets_vec = s1_data['targetsVec'].reshape((-1, ))
subjects_vec = s1_data['subjectsVec'].reshape((-1, ))
vidlen_vec = s1_data['videoLengthVec'].reshape((-1, ))
iter_vec = s1_data['iterVec'].reshape((-1, ))
if matlab_target_offset:
targets_vec -= 1
s1_data_matrix = presplit_dataprocessing(s1_data_matrix,
vidlen_vec,
config,
'stream1',
imagesize=s1_imagesize)
indexes = create_split_index(len(s1_data_matrix), vidlen_vec, iter_vec)
s1_train_vidlens, s1_val_vidlens = split_videolen(vidlen_vec, iter_vec)
# split the data
s1_train_X = s1_data_matrix[indexes == True]
s1_train_y = targets_vec[indexes == True].reshape((-1, ))
s1_val_X = s1_data_matrix[indexes == False]
s1_val_y = targets_vec[indexes == False].reshape((-1, ))
s1_train_X, s1_val_X = postsplit_datapreprocessing(s1_train_X, s1_val_X,
config, 'stream1')
s1_has_encoder = config.getboolean('stream1', 'has_encoder')
# IMPT: the encoder was trained with fortan ordered images, so to visualize
# convert all the images to C order using reshape_images_order()
# output = dbn.predict(test_X)
# test_X = reshape_images_order(test_X, (26, 44))
# output = reshape_images_order(output, (26, 44))
# visualize_reconstruction(test_X[:36, :], output[:36, :], shape=(26, 44))
window = T.iscalar('theta')
inputs1 = T.tensor3('inputs1', dtype='float32')
mask = T.matrix('mask', dtype='uint8')
targets = T.imatrix('targets')
print('constructing end to end model...')
if s1_has_encoder:
ae1 = load_decoder(s1, s1_shape, s1_nonlinearities)
network = deltanet_majority_vote.create_model(
ae1, (None, None, s1_inputdim), inputs1, (None, None), mask,
lstm_size, window, output_classes, weight_init_fn, use_peepholes,
use_blstm)
else:
network = deltanet_v1.create_model(
(None, None, s1_inputdim), inputs1, (None, None), mask, window,
lstm_size, output_classes, weight_init_fn, use_peepholes,
use_blstm)
print_network(network)
# draw_to_file(las.layers.get_all_layers(network), 'network.png')
print('compiling model...')
predictions = las.layers.get_output(network, deterministic=False)
all_params = las.layers.get_all_params(network, trainable=True)
cost = temporal_softmax_loss(predictions, targets, mask)
updates = adam(cost, all_params, learning_rate=learning_rate)
train = theano.function([inputs1, targets, mask, window],
cost,
updates=updates,
allow_input_downcast=True)
compute_train_cost = theano.function([inputs1, targets, mask, window],
cost,
allow_input_downcast=True)
test_predictions = las.layers.get_output(network, deterministic=True)
test_cost = temporal_softmax_loss(test_predictions, targets, mask)
compute_test_cost = theano.function([inputs1, targets, mask, window],
test_cost,
allow_input_downcast=True)
val_fn = theano.function([inputs1, mask, window],
test_predictions,
allow_input_downcast=True)
# We'll train the network with 10 epochs of 30 minibatches each
print('begin training...')
cost_train = []
cost_val = []
class_rate = []
STRIP_SIZE = 3
val_window = circular_list(validation_window)
train_strip = np.zeros((STRIP_SIZE, ))
best_val = float('inf')
best_cr = 0.0
datagen = gen_lstm_batch_random(s1_train_X,
s1_train_y,
s1_train_vidlens,
batchsize=batchsize)
val_datagen = gen_lstm_batch_random(s1_val_X,
s1_val_y,
s1_val_vidlens,
batchsize=len(s1_val_vidlens))
# We'll use this "validation set" to periodically check progress
X_val, y_val, mask_val, idxs_val = next(val_datagen)
# reshape the targets for validation
y_val_evaluate = y_val
y_val = y_val.reshape((-1, 1)).repeat(mask_val.shape[-1], axis=-1)
for epoch in range(num_epoch):
time_start = time.time()
for i in range(epochsize):
X, y, m, batch_idxs = next(datagen)
# repeat targets based on max sequence len
y = y.reshape((-1, 1))
y = y.repeat(m.shape[-1], axis=-1)
print_str = 'Epoch {} batch {}/{}: {} examples using adam with learning rate = {}'.format(
epoch + 1, i + 1, epochsize, len(X), learning_rate)
print(print_str, end='')
sys.stdout.flush()
train(X, y, m, windowsize)
print('\r', end='')
cost = compute_train_cost(X, y, m, windowsize)
val_cost = compute_test_cost(X_val, y_val, mask_val, windowsize)
cost_train.append(cost)
cost_val.append(val_cost)
train_strip[epoch % STRIP_SIZE] = cost
val_window.push(val_cost)
gl = 100 * (cost_val[-1] / np.min(cost_val) - 1)
pk = 1000 * (np.sum(train_strip) /
(STRIP_SIZE * np.min(train_strip)) - 1)
pq = gl / pk
cr, val_conf = evaluate_model2(X_val, y_val_evaluate, mask_val,
windowsize, val_fn)
class_rate.append(cr)
print("Epoch {} train cost = {}, val cost = {}, "
"GL loss = {:.3f}, GQ = {:.3f}, CR = {:.3f} ({:.1f}sec)".format(
epoch + 1, cost_train[-1], cost_val[-1], gl, pq, cr,
time.time() - time_start))
if val_cost < best_val:
best_val = val_cost
best_cr = cr
best_conf = val_conf
best_params = las.layers.get_all_param_values(network)
if epoch >= validation_window and early_stop2(val_window, best_val,
validation_window):
break
print('Final Model')
print('CR: {}, val loss: {}'.format(best_cr, best_val))
# plot confusion matrix
table_str = plot_confusion_matrix(best_conf, output_classnames, fmt='pipe')
print('confusion matrix: ')
print(table_str)
if 'save_plot' in options:
prefix = options['save_plot']
plot_validation_cost(cost_train,
cost_val,
savefilename='{}.validloss.png'.format(prefix))
with open('{}.confmat.txt'.format(prefix), mode='a') as f:
f.write(table_str)
f.write('\n\n')
if 'write_results' in options:
print('writing results to {}'.format(options['write_results']))
results_file = options['write_results']
with open(results_file, mode='a') as f:
f.write('{},{}\n'.format(best_cr, best_val))
if 'save_best' in options:
print('saving best model...')
las.layers.set_all_param_values(network, best_params)
save_model_params(network, options['save_best'])
print('best model saved to {}'.format(options['save_best']))
def main():
configure_theano()
options = parse_options()
config_file = options['config']
config = ConfigParser.ConfigParser()
config.read(config_file)
print('CLI options: {}'.format(options.items()))
print('Reading Config File: {}...'.format(config_file))
print(config.items('stream1'))
print(config.items('lstm_classifier'))
print(config.items('training'))
print('preprocessing dataset...')
data = load_mat_file(config.get('stream1', 'data'))
has_encoder = config.getboolean('stream1', 'has_encoder')
stream1_dim = config.getint('stream1', 'input_dimensions')
imagesize = tuple([int(d) for d in config.get('stream1', 'imagesize').split(',')])
if has_encoder:
stream1 = config.get('stream1', 'model')
stream1_shape = config.get('stream1', 'shape')
stream1_nonlinearities = config.get('stream1', 'nonlinearities')
# lstm classifier
output_classes = config.getint('lstm_classifier', 'output_classes')
output_classnames = config.get('lstm_classifier', 'output_classnames').split(',')
lstm_size = config.getint('lstm_classifier', 'lstm_size')
matlab_target_offset = config.getboolean('lstm_classifier', 'matlab_target_offset')
# lstm classifier configurations
weight_init = options['weight_init'] if 'weight_init' in options else config.get('lstm_classifier', 'weight_init')
use_peepholes = options['use_peepholes'] if 'use_peepholes' in options else config.getboolean('lstm_classifier',
'use_peepholes')
use_blstm = True if config.has_option('lstm_classifier', 'use_blstm') else False
windowsize = config.getint('lstm_classifier', 'windowsize')
# capture training parameters
validation_window = int(options['validation_window']) \
if 'validation_window' in options else config.getint('training', 'validation_window')
num_epoch = int(options['num_epoch']) if 'num_epoch' in options else config.getint('training', 'num_epoch')
learning_rate = options['learning_rate'] if 'learning_rate' in options \
else config.getfloat('training', 'learning_rate')
epochsize = config.getint('training', 'epochsize')
batchsize = config.getint('training', 'batchsize')
weight_init_fn = las.init.GlorotUniform()
if weight_init == 'glorot':
weight_init_fn = las.init.GlorotUniform()
if weight_init == 'norm':
weight_init_fn = las.init.Normal(0.1)
if weight_init == 'uniform':
weight_init_fn = las.init.Uniform()
if weight_init == 'ortho':
weight_init_fn = las.init.Orthogonal()
data_matrix = data['dataMatrix'].astype('float32')
targets_vec = data['targetsVec'].reshape((-1,))
subjects_vec = data['subjectsVec'].reshape((-1,))
vidlen_vec = data['videoLengthVec'].reshape((-1,))
iter_vec = data['iterVec'].reshape((-1,))
data_matrix = presplit_dataprocessing(data_matrix, vidlen_vec, config, 'stream1', imagesize=imagesize)
indexes = create_split_index(len(data_matrix), vidlen_vec, iter_vec)
train_vidlen_vec, test_vidlen_vec = split_videolen(vidlen_vec, iter_vec)
if matlab_target_offset:
targets_vec -= 1
# split the data
train_data = data_matrix[indexes == True]
train_targets = targets_vec[indexes == True]
train_targets = train_targets.reshape((len(train_targets),))
test_data = data_matrix[indexes == False]
test_targets = targets_vec[indexes == False]
test_targets = test_targets.reshape((len(test_targets),))
train_data, test_data = postsplit_datapreprocessing(train_data, test_data, config, 'stream1')
inputs = T.tensor3('inputs', dtype='float32')
window = T.iscalar('theta')
mask = T.matrix('mask', dtype='uint8')
targets = T.imatrix('targets')
print('constructing end to end model...')
if not has_encoder:
network = deltanet_v1.create_model((None, None, stream1_dim), inputs,
(None, None), mask, window,
lstm_size, output_classes, weight_init_fn, use_peepholes, use_blstm)
else:
ae1 = load_decoder(stream1, stream1_shape, stream1_nonlinearities)
network = deltanet_majority_vote.create_model(ae1, (None, None, stream1_dim), inputs,
(None, None), mask,
lstm_size, window, output_classes, weight_init_fn, use_peepholes)
print_network(network)
draw_to_file(las.layers.get_all_layers(network), 'network.png', verbose=True)
# exit()
print('compiling model...')
predictions = las.layers.get_output(network, deterministic=False)
all_params = las.layers.get_all_params(network, trainable=True)
cost = temporal_softmax_loss(predictions, targets, mask)
updates = las.updates.adam(cost, all_params, learning_rate)
train = theano.function(
[inputs, targets, mask, window],
cost, updates=updates, allow_input_downcast=True)
compute_train_cost = theano.function([inputs, targets, mask, window], cost, allow_input_downcast=True)
test_predictions = las.layers.get_output(network, deterministic=True)
test_cost = temporal_softmax_loss(test_predictions, targets, mask)
compute_test_cost = theano.function(
[inputs, targets, mask, window], test_cost, allow_input_downcast=True)
val_fn = theano.function([inputs, mask, window], test_predictions, allow_input_downcast=True)
# We'll train the network with 10 epochs of 30 minibatches each
print('begin training...')
cost_train = []
cost_val = []
class_rate = []
STRIP_SIZE = 3
val_window = circular_list(validation_window)
train_strip = np.zeros((STRIP_SIZE,))
best_val = float('inf')
best_conf = None
best_cr = 0.0
datagen = gen_lstm_batch_random(train_data, train_targets, train_vidlen_vec, batchsize=batchsize)
val_datagen = gen_lstm_batch_random(test_data, test_targets, test_vidlen_vec,
batchsize=len(test_vidlen_vec))
# We'll use this "validation set" to periodically check progress
X_val, y_val, mask_val, idxs_val = next(val_datagen)
# reshape the targets for validation
y_val_evaluate = y_val
y_val = y_val.reshape((-1, 1)).repeat(mask_val.shape[-1], axis=-1)
for epoch in range(num_epoch):
time_start = time.time()
for i in range(epochsize):
X, y, m, batch_idxs = next(datagen)
# repeat targets based on max sequence len
y = y.reshape((-1, 1))
y = y.repeat(m.shape[-1], axis=-1)
print_str = 'Epoch {} batch {}/{}: {} examples at learning rate = {:.4f}'.format(
epoch + 1, i + 1, epochsize, len(X), learning_rate)
print(print_str, end='')
sys.stdout.flush()
train(X, y, m, windowsize)
print('\r', end='')
cost = compute_train_cost(X, y, m, windowsize)
val_cost = compute_test_cost(X_val, y_val, mask_val, windowsize)
cost_train.append(cost)
cost_val.append(val_cost)
train_strip[epoch % STRIP_SIZE] = cost
val_window.push(val_cost)
gl = 100 * (cost_val[-1] / np.min(cost_val) - 1)
pk = 1000 * (np.sum(train_strip) / (STRIP_SIZE * np.min(train_strip)) - 1)
pq = gl / pk
cr, val_conf = evaluate_model2(X_val, y_val_evaluate, mask_val, windowsize, val_fn)
class_rate.append(cr)
print("Epoch {} train cost = {}, validation cost = {}, "
"generalization loss = {:.3f}, GQ = {:.3f}, classification rate = {:.3f} ({:.1f}sec)"
.format(epoch + 1, cost_train[-1], cost_val[-1], gl, pq, cr, time.time() - time_start))
if val_cost < best_val:
best_val = val_cost
best_conf = val_conf
best_cr = cr
if epoch >= validation_window and early_stop2(val_window, best_val, validation_window):
break
print('Best Model')
print('classification rate: {}, validation loss: {}'.format(best_cr, best_val))
print('confusion matrix: ')
plot_confusion_matrix(best_conf, output_classnames, fmt='latex')
plot_validation_cost(cost_train, cost_val, class_rate)
def main():
configure_theano()
options = parse_options()
config_file = options['config']
config = ConfigParser.ConfigParser()
config.read(config_file)
print('CLI options: {}'.format(options.items()))
print('Reading Config File: {}...'.format(config_file))
print(config.items('stream1'))
print(config.items('lstm_classifier'))
print(config.items('training'))
print('preprocessing dataset...')
data = load_mat_file(config.get('stream1', 'data'))
stream1 = config.get('stream1', 'model')
imagesize = tuple(
[int(d) for d in config.get('stream1', 'imagesize').split(',')])
stream1_dim = config.getint('stream1', 'input_dimensions')
stream1_shape = config.get('stream1', 'shape')
stream1_nonlinearities = config.get('stream1', 'nonlinearities')
# lstm classifier
output_classes = config.getint('lstm_classifier', 'output_classes')
output_classnames = config.get('lstm_classifier',
'output_classnames').split(',')
lstm_size = config.getint('lstm_classifier', 'lstm_size')
matlab_target_offset = config.getboolean('lstm_classifier',
'matlab_target_offset')
# data preprocessing options
reorderdata = config.getboolean('stream1', 'reorderdata')
diffimage = config.getboolean('stream1', 'diffimage')
meanremove = config.getboolean('stream1', 'meanremove')
samplewisenormalize = config.getboolean('stream1', 'samplewisenormalize')
featurewisenormalize = config.getboolean('stream1', 'featurewisenormalize')
# lstm classifier configurations
weight_init = options[
'weight_init'] if 'weight_init' in options else config.get(
'lstm_classifier', 'weight_init')
use_peepholes = options[
'use_peepholes'] if 'use_peepholes' in options else config.getboolean(
'lstm_classifier', 'use_peepholes')
windowsize = config.getint('lstm_classifier', 'windowsize')
# capture training parameters
validation_window = int(options['validation_window']) \
if 'validation_window' in options else config.getint('training', 'validation_window')
num_epoch = int(
options['num_epoch']) if 'num_epoch' in options else config.getint(
'training', 'num_epoch')
learning_rate = options['learning_rate'] if 'learning_rate' in options \
else config.getfloat('training', 'learning_rate')
epochsize = config.getint('training', 'epochsize')
batchsize = config.getint('training', 'batchsize')
weight_init_fn = las.init.GlorotUniform()
if weight_init == 'glorot':
weight_init_fn = las.init.GlorotUniform()
if weight_init == 'norm':
weight_init_fn = las.init.Normal(0.1)
if weight_init == 'uniform':
weight_init_fn = las.init.Uniform()
if weight_init == 'ortho':
weight_init_fn = las.init.Orthogonal()
train_subject_ids = read_data_split_file(
config.get('training', 'train_subjects_file'))
val_subject_ids = read_data_split_file(
config.get('training', 'val_subjects_file'))
test_subject_ids = read_data_split_file(
config.get('training', 'test_subjects_file'))
data_matrix = data['dataMatrix'].astype('float32')
targets_vec = data['targetsVec'].reshape((-1, ))
subjects_vec = data['subjectsVec'].reshape((-1, ))
vidlen_vec = data['videoLengthVec'].reshape((-1, ))
if reorderdata:
data_matrix = reorder_data(data_matrix, (imagesize[0], imagesize[1]))
train_X, train_y, train_vidlens, train_subjects, \
val_X, val_y, val_vidlens, val_subjects, \
test_X, test_y, test_vidlens, test_subjects = split_seq_data(data_matrix, targets_vec, subjects_vec, vidlen_vec,
train_subject_ids, val_subject_ids, test_subject_ids)
if matlab_target_offset:
train_y -= 1
val_y -= 1
test_y -= 1
if meanremove:
train_X = sequencewise_mean_image_subtraction(train_X, train_vidlens)
val_X = sequencewise_mean_image_subtraction(val_X, val_vidlens)
test_X = sequencewise_mean_image_subtraction(test_X, test_vidlens)
if diffimage:
train_X = compute_diff_images(train_X, train_vidlens)
val_X = compute_diff_images(val_X, val_vidlens)
test_X = compute_diff_images(test_X, test_vidlens)
if samplewisenormalize:
train_X = normalize_input(train_X)
val_X = normalize_input(val_X)
test_X = normalize_input(test_X)
if featurewisenormalize:
train_X, mean, std = featurewise_normalize_sequence(train_X)
val_X = (val_X - mean) / std
test_X = (test_X - mean) / std
ae1 = load_decoder(stream1, stream1_shape, stream1_nonlinearities)
# IMPT: the encoder was trained with fortan ordered images, so to visualize
# convert all the images to C order using reshape_images_order()
# output = dbn.predict(test_X)
# test_X = reshape_images_order(test_X, (26, 44))
# output = reshape_images_order(output, (26, 44))
# visualize_reconstruction(test_X[:36, :], output[:36, :], shape=(26, 44))
window = T.iscalar('theta')
inputs1 = T.tensor3('inputs1', dtype='float32')
mask = T.matrix('mask', dtype='uint8')
targets = T.imatrix('targets')
print('constructing end to end model...')
network = deltanet_majority_vote.create_model(
ae1, (None, None, stream1_dim), inputs1, (None, None), mask, lstm_size,
window, output_classes, weight_init_fn, use_peepholes)
print_network(network)
print('compiling model...')
predictions = las.layers.get_output(network, deterministic=False)
all_params = las.layers.get_all_params(network, trainable=True)
cost = temporal_softmax_loss(predictions, targets, mask)
default_learning_rate = theano.shared(las.utils.floatX(learning_rate),
'default_lr')
lr_config = {
'fc1': theano.shared(las.utils.floatX(0.001)),
'fc2': theano.shared(las.utils.floatX(0.001)),
'fc3': theano.shared(las.utils.floatX(0.001))
}
lr_map = custom.updates.generate_lr_map(all_params, lr_config,
default_learning_rate)
# updates = adam(cost, all_params, default_learning_rate)
updates = custom.updates.adam_vlr(cost, all_params, lr_map)
train = theano.function([inputs1, targets, mask, window],
cost,
updates=updates,
allow_input_downcast=True)
compute_train_cost = theano.function([inputs1, targets, mask, window],
cost,
allow_input_downcast=True)
test_predictions = las.layers.get_output(network, deterministic=True)
test_cost = temporal_softmax_loss(test_predictions, targets, mask)
compute_test_cost = theano.function([inputs1, targets, mask, window],
test_cost,
allow_input_downcast=True)
val_fn = theano.function([inputs1, mask, window],
test_predictions,
allow_input_downcast=True)
# We'll train the network with 10 epochs of 30 minibatches each
print('begin training...')
cost_train = []
cost_val = []
class_rate = []
STRIP_SIZE = 3
val_window = circular_list(validation_window)
train_strip = np.zeros((STRIP_SIZE, ))
best_val = float('inf')
best_cr = 0.0
datagen = gen_lstm_batch_random(train_X,
train_y,
train_vidlens,
batchsize=batchsize)
val_datagen = gen_lstm_batch_random(val_X,
val_y,
val_vidlens,
batchsize=len(val_vidlens))
test_datagen = gen_lstm_batch_random(test_X,
test_y,
test_vidlens,
batchsize=len(test_vidlens))
# We'll use this "validation set" to periodically check progress
X_val, y_val, mask_val, idxs_val = next(val_datagen)
# we use the test set to check final classification rate
X_test, y_test, mask_test, idxs_test = next(test_datagen)
# reshape the targets for validation
y_val_evaluate = y_val
y_val = y_val.reshape((-1, 1)).repeat(mask_val.shape[-1], axis=-1)
for epoch in range(num_epoch):
time_start = time.time()
for i in range(epochsize):
X, y, m, batch_idxs = next(datagen)
# repeat targets based on max sequence len
y = y.reshape((-1, 1))
y = y.repeat(m.shape[-1], axis=-1)
print_str = 'Epoch {} batch {}/{}: {} examples using adam with learning rate = {}'.format(
epoch + 1, i + 1, epochsize, len(X), learning_rate)
print(print_str, end='')
sys.stdout.flush()
train(X, y, m, windowsize)
print('\r', end='')
cost = compute_train_cost(X, y, m, windowsize)
val_cost = compute_test_cost(X_val, y_val, mask_val, windowsize)
cost_train.append(cost)
cost_val.append(val_cost)
train_strip[epoch % STRIP_SIZE] = cost
val_window.push(val_cost)
gl = 100 * (cost_val[-1] / np.min(cost_val) - 1)
pk = 1000 * (np.sum(train_strip) /
(STRIP_SIZE * np.min(train_strip)) - 1)
pq = gl / pk
cr, val_conf = evaluate_model2(X_val, y_val_evaluate, mask_val,
windowsize, val_fn)
class_rate.append(cr)
if val_cost < best_val:
best_val = val_cost
best_cr = cr
test_cr, test_conf = evaluate_model2(X_test, y_test, mask_test,
windowsize, val_fn)
print(
"Epoch {} train cost = {}, val cost = {}, "
"GL loss = {:.3f}, GQ = {:.3f}, CR = {:.3f}, Test CR= {:.3f} ({:.1f}sec)"
.format(epoch + 1, cost_train[-1], cost_val[-1], gl, pq, cr,
test_cr,
time.time() - time_start))
best_params = las.layers.get_all_param_values(network)
else:
print("Epoch {} train cost = {}, val cost = {}, "
"GL loss = {:.3f}, GQ = {:.3f}, CR = {:.3f} ({:.1f}sec)".
format(epoch + 1, cost_train[-1], cost_val[-1], gl, pq, cr,
time.time() - time_start))
if epoch >= validation_window and early_stop2(val_window, best_val,
validation_window):
break
# Show that learning rates are changed by exploding learning rates for encoder layers
# The training loss should increase dramatically and learning should diverge
if epoch + 1 == 4:
print('explode fc1,fc2,fc3 learning rates to 100.0')
lr_config['fc1'].set_value(100.0)
lr_config['fc2'].set_value(100.0)
lr_config['fc3'].set_value(100.0)
print('Final Model')
print('CR: {}, val loss: {}, Test CR: {}'.format(best_cr, best_val,
test_cr))
# plot confusion matrix
table_str = plot_confusion_matrix(test_conf, output_classnames, fmt='pipe')
print('confusion matrix: ')
print(table_str)
if 'save_plot' in options:
prefix = options['save_plot']
plot_validation_cost(cost_train,
cost_val,
savefilename='{}.validloss.png'.format(prefix))
with open('{}.confmat.txt'.format(prefix), mode='a') as f:
f.write(table_str)
f.write('\n\n')
if 'write_results' in options:
print('writing results to {}'.format(options['write_results']))
results_file = options['write_results']
with open(results_file, mode='a') as f:
f.write('{},{},{}\n'.format(test_cr, best_cr, best_val))
if 'save_best' in options:
print('saving best model...')
las.layers.set_all_param_values(network, best_params)
save_model_params(network, options['save_best'])
print('best model saved to {}'.format(options['save_best']))
def main():
configure_theano()
options = parse_options()
config_file = options['config']
config = ConfigParser.ConfigParser()
config.read(config_file)
print('CLI options: {}'.format(options.items()))
print('Reading Config File: {}...'.format(config_file))
print(config.items('stream1'))
print(config.items('lstm_classifier'))
print(config.items('training'))
print('preprocessing dataset...')
data = load_mat_file(config.get('stream1', 'data'))
stream1 = config.get('stream1', 'model')
imagesize = tuple([int(d) for d in config.get('stream1', 'imagesize').split(',')])
stream1_dim = config.getint('stream1', 'input_dimensions')
stream1_shape = config.get('stream1', 'shape')
stream1_nonlinearities = config.get('stream1', 'nonlinearities')
# lstm classifier
output_classes = config.getint('lstm_classifier', 'output_classes')
output_classnames = config.get('lstm_classifier', 'output_classnames').split(',')
lstm_size = config.getint('lstm_classifier', 'lstm_size')
matlab_target_offset = config.getboolean('lstm_classifier', 'matlab_target_offset')
# data preprocessing options
reorderdata = config.getboolean('stream1', 'reorderdata')
diffimage = config.getboolean('stream1', 'diffimage')
meanremove = config.getboolean('stream1', 'meanremove')
samplewisenormalize = config.getboolean('stream1', 'samplewisenormalize')
featurewisenormalize = config.getboolean('stream1', 'featurewisenormalize')
# lstm classifier configurations
weight_init = options['weight_init'] if 'weight_init' in options else config.get('lstm_classifier', 'weight_init')
use_peepholes = options['use_peepholes'] if 'use_peepholes' in options else config.getboolean('lstm_classifier',
'use_peepholes')
windowsize = config.getint('lstm_classifier', 'windowsize')
# capture training parameters
validation_window = int(options['validation_window']) \
if 'validation_window' in options else config.getint('training', 'validation_window')
num_epoch = int(options['num_epoch']) if 'num_epoch' in options else config.getint('training', 'num_epoch')
learning_rate = options['learning_rate'] if 'learning_rate' in options \
else config.getfloat('training', 'learning_rate')
epochsize = config.getint('training', 'epochsize')
batchsize = config.getint('training', 'batchsize')
weight_init_fn = las.init.GlorotUniform()
if weight_init == 'glorot':
weight_init_fn = las.init.GlorotUniform()
if weight_init == 'norm':
weight_init_fn = las.init.Normal(0.1)
if weight_init == 'uniform':
weight_init_fn = las.init.Uniform()
if weight_init == 'ortho':
weight_init_fn = las.init.Orthogonal()
train_subject_ids = read_data_split_file(config.get('training', 'train_subjects_file'))
val_subject_ids = read_data_split_file(config.get('training', 'val_subjects_file'))
test_subject_ids = read_data_split_file(config.get('training', 'test_subjects_file'))
data_matrix = data['dataMatrix'].astype('float32')
targets_vec = data['targetsVec'].reshape((-1,))
subjects_vec = data['subjectsVec'].reshape((-1,))
vidlen_vec = data['videoLengthVec'].reshape((-1,))
if reorderdata:
data_matrix = reorder_data(data_matrix, (imagesize[0], imagesize[1]))
train_X, train_y, train_vidlens, train_subjects, \
val_X, val_y, val_vidlens, val_subjects, \
test_X, test_y, test_vidlens, test_subjects = split_seq_data(data_matrix, targets_vec, subjects_vec, vidlen_vec,
train_subject_ids, val_subject_ids, test_subject_ids)
if matlab_target_offset:
train_y -= 1
val_y -= 1
test_y -= 1
if meanremove:
train_X = sequencewise_mean_image_subtraction(train_X, train_vidlens)
val_X = sequencewise_mean_image_subtraction(val_X, val_vidlens)
test_X = sequencewise_mean_image_subtraction(test_X, test_vidlens)
if diffimage:
train_X = compute_diff_images(train_X, train_vidlens)
val_X = compute_diff_images(val_X, val_vidlens)
test_X = compute_diff_images(test_X, test_vidlens)
if samplewisenormalize:
train_X = normalize_input(train_X)
val_X = normalize_input(val_X)
test_X = normalize_input(test_X)
if featurewisenormalize:
train_X, mean, std = featurewise_normalize_sequence(train_X)
val_X = (val_X - mean) / std
test_X = (test_X - mean) / std
ae1 = load_decoder(stream1, stream1_shape, stream1_nonlinearities)
# IMPT: the encoder was trained with fortan ordered images, so to visualize
# convert all the images to C order using reshape_images_order()
# output = dbn.predict(test_X)
# test_X = reshape_images_order(test_X, (26, 44))
# output = reshape_images_order(output, (26, 44))
# visualize_reconstruction(test_X[:36, :], output[:36, :], shape=(26, 44))
window = T.iscalar('theta')
inputs1 = T.tensor3('inputs1', dtype='float32')
mask = T.matrix('mask', dtype='uint8')
targets = T.imatrix('targets')
print('constructing end to end model...')
network = deltanet_majority_vote.create_model(ae1, (None, None, stream1_dim), inputs1,
(None, None), mask,
lstm_size, window, output_classes,
weight_init_fn, use_peepholes)
print_network(network)
print('compiling model...')
predictions = las.layers.get_output(network, deterministic=False)
all_params = las.layers.get_all_params(network, trainable=True)
cost = temporal_softmax_loss(predictions, targets, mask)
default_learning_rate = theano.shared(las.utils.floatX(learning_rate), 'default_lr')
lr_config = {
'fc1': theano.shared(las.utils.floatX(0.001)),
'fc2': theano.shared(las.utils.floatX(0.001)),
'fc3': theano.shared(las.utils.floatX(0.001))
}
lr_map = custom.updates.generate_lr_map(all_params, lr_config, default_learning_rate)
# updates = adam(cost, all_params, default_learning_rate)
updates = custom.updates.adam_vlr(cost, all_params, lr_map)
train = theano.function(
[inputs1, targets, mask, window],
cost, updates=updates, allow_input_downcast=True)
compute_train_cost = theano.function([inputs1, targets, mask, window],
cost, allow_input_downcast=True)
test_predictions = las.layers.get_output(network, deterministic=True)
test_cost = temporal_softmax_loss(test_predictions, targets, mask)
compute_test_cost = theano.function(
[inputs1, targets, mask, window], test_cost, allow_input_downcast=True)
val_fn = theano.function([inputs1, mask, window], test_predictions, allow_input_downcast=True)
# We'll train the network with 10 epochs of 30 minibatches each
print('begin training...')
cost_train = []
cost_val = []
class_rate = []
STRIP_SIZE = 3
val_window = circular_list(validation_window)
train_strip = np.zeros((STRIP_SIZE,))
best_val = float('inf')
best_cr = 0.0
datagen = gen_lstm_batch_random(train_X, train_y, train_vidlens, batchsize=batchsize)
val_datagen = gen_lstm_batch_random(val_X, val_y, val_vidlens, batchsize=len(val_vidlens))
test_datagen = gen_lstm_batch_random(test_X, test_y, test_vidlens, batchsize=len(test_vidlens))
# We'll use this "validation set" to periodically check progress
X_val, y_val, mask_val, idxs_val = next(val_datagen)
# we use the test set to check final classification rate
X_test, y_test, mask_test, idxs_test = next(test_datagen)
# reshape the targets for validation
y_val_evaluate = y_val
y_val = y_val.reshape((-1, 1)).repeat(mask_val.shape[-1], axis=-1)
for epoch in range(num_epoch):
time_start = time.time()
for i in range(epochsize):
X, y, m, batch_idxs = next(datagen)
# repeat targets based on max sequence len
y = y.reshape((-1, 1))
y = y.repeat(m.shape[-1], axis=-1)
print_str = 'Epoch {} batch {}/{}: {} examples using adam with learning rate = {}'.format(
epoch + 1, i + 1, epochsize, len(X), learning_rate)
print(print_str, end='')
sys.stdout.flush()
train(X, y, m, windowsize)
print('\r', end='')
cost = compute_train_cost(X, y, m, windowsize)
val_cost = compute_test_cost(X_val, y_val, mask_val, windowsize)
cost_train.append(cost)
cost_val.append(val_cost)
train_strip[epoch % STRIP_SIZE] = cost
val_window.push(val_cost)
gl = 100 * (cost_val[-1] / np.min(cost_val) - 1)
pk = 1000 * (np.sum(train_strip) / (STRIP_SIZE * np.min(train_strip)) - 1)
pq = gl / pk
cr, val_conf = evaluate_model2(X_val, y_val_evaluate, mask_val, windowsize, val_fn)
class_rate.append(cr)
if val_cost < best_val:
best_val = val_cost
best_cr = cr
test_cr, test_conf = evaluate_model2(X_test, y_test, mask_test, windowsize, val_fn)
print("Epoch {} train cost = {}, val cost = {}, "
"GL loss = {:.3f}, GQ = {:.3f}, CR = {:.3f}, Test CR= {:.3f} ({:.1f}sec)"
.format(epoch + 1, cost_train[-1], cost_val[-1], gl, pq, cr, test_cr, time.time() - time_start))
best_params = las.layers.get_all_param_values(network)
else:
print("Epoch {} train cost = {}, val cost = {}, "
"GL loss = {:.3f}, GQ = {:.3f}, CR = {:.3f} ({:.1f}sec)"
.format(epoch + 1, cost_train[-1], cost_val[-1], gl, pq, cr, time.time() - time_start))
if epoch >= validation_window and early_stop2(val_window, best_val, validation_window):
break
# Show that learning rates are changed by exploding learning rates for encoder layers
# The training loss should increase dramatically and learning should diverge
if epoch + 1 == 4:
print('explode fc1,fc2,fc3 learning rates to 100.0')
lr_config['fc1'].set_value(100.0)
lr_config['fc2'].set_value(100.0)
lr_config['fc3'].set_value(100.0)
print('Final Model')
print('CR: {}, val loss: {}, Test CR: {}'.format(best_cr, best_val, test_cr))
# plot confusion matrix
table_str = plot_confusion_matrix(test_conf, output_classnames, fmt='pipe')
print('confusion matrix: ')
print(table_str)
if 'save_plot' in options:
prefix = options['save_plot']
plot_validation_cost(cost_train, cost_val, savefilename='{}.validloss.png'.format(prefix))
with open('{}.confmat.txt'.format(prefix), mode='a') as f:
f.write(table_str)
f.write('\n\n')
if 'write_results' in options:
print('writing results to {}'.format(options['write_results']))
results_file = options['write_results']
with open(results_file, mode='a') as f:
f.write('{},{},{}\n'.format(test_cr, best_cr, best_val))
if 'save_best' in options:
print('saving best model...')
las.layers.set_all_param_values(network, best_params)
save_model_params(network, options['save_best'])
print('best model saved to {}'.format(options['save_best']))