"""

:description: training a recurrent neural network for classification task

:development:

:todo:

(1) adaptive learning rate!!!

:options:

(1) try to build a model that kind of works, which is to say, get this running on a ec2 instance gpu

(2) try to implement momentum (or some other model training improvement)

(a) stochastic grad instead of literal grad descent

(3) implement variable len sequences

(4) implement encoder-decoder

:reminders:

(1) the main function is currently main_theano_chest_accel, where the layers var determines the model and the task is classification

:works:

(1) main_theano_softmax works conclusively

:results:

NOTE: all these results so far are due to the fact that the 2nd index (3rd class) is more common than the others (it is the 'standing' category). As such, all these are pretty much meaningless; however, it seems that the model does seem to be working, it's just that this data is not well suited for analysis by rnn. So, get some other data.

:chest accel:

:1*recurrent + softmax:

:n_hid = *1, 1/5 data, n_epochs = 20, drop = .0:

:accuracy:

:final cost: 1.93

:n_hid = *2, all data, n_epochs = 20, drop = .0:

:accuracy:

:final cost: 1.97

:2*recurrent + softmax:

:n_hid = *1, all data, n_epochs = 20, drop = .0:

:accuracy:

:final cost: 1.85

:1*lstm + softmax:

:n_hid = *2, all data, n_epochs = 20, forget_gate_bias = 0.2, drop = .0:

:accuracy:

:final cost: 1.91

:n_hid = *1, all data, n_epochs = 20, forget_gate_bias = 0.05, drop = .0:

:accuracy:

:final cost: 1.88

:n_hid = *1, all data, n_epochs = 20, forget_gate_bias = 0.05, drop = .3:

:accuracy:

:final cost:

:3*lstm + softmax:

:n_hid = *1, all data, n_epochs = 20, forget_gate_bias = 0.05, drop = .0:

:accuracy:

:final cost: 1.85

:sign_lang:

:1*lstm + softmax:

:n_hid = *1, all data, n_epochs = 20, forget_gate_bias = 0.05, drop = .3:

:accuracy: 3.6%

:final cost: 4.02

"""

import sys

sys.path.insert(1,'/Library/Python/2.7/site-packages')

import numpy as np

import theano

import theano.tensor as T

# from pylearn2.sandbox.rnn.space import SequenceSpace, SequenceDataSpace

# from pylearn2.space import VectorSpace, CompositeSpace, VectorSequenceSpace

# from pylearn2.costs.mlp import Default

# from pylearn2.sandbox.rnn.models.rnn import Recurrent, RNN

# from pylearn2.models.mlp import Softmax, MLP

# from pylearn2.training_algorithms.sgd import SGD

# from pylearn2.termination_criteria import EpochCounter

# from pylearn2.blocks import StackedBlocks

# from pylearn2.datasets.transformer_dataset import TransformerDataset

# from pylearn2.train import Train

from toy_datasets import ToyVideoDataset, easy_data_generator, hard_data_generator_reconstruct, hard_data_generator_random_reconstruct, easy_data_generator_reconstruct, super_easy_data_generator_reconstruct, chest_accel_data_loader, easy_softmax_data_generator, get_sequence_lengths

from enc_dec_rnn import EncDecRNN, EncDecRecurrent, Softmax, LSTM

from file_utils import save_model, load_model

import encoding_recurrent

import encoding_recurrent_overlap

import sign_lang

import chest_accel

import variable_length_sequence_lstm

MAX_EPOCHS_SUPERVISED = 2

def get_dataset_toy(data_generator=None, n_examples=10, n_classes=101, n_features=4096, n_frames=1):

trainset = ToyVideoDataset(data_generator=data_generator, n_examples=n_examples, n_classes=n_classes, n_features=n_features, n_frames=n_frames)

testset = ToyVideoDataset(data_generator=data_generator, n_examples=n_examples, n_classes=n_classes, n_features=n_features, n_frames=n_frames)

return trainset, testset

def get_dataset_toy_raw(data_generator=None, n_examples=100, n_classes=10, n_features=20, n_frames=6, only_train=False):

trainset, testset = get_dataset_toy(data_generator, n_examples, n_classes, n_features, n_frames)

return trainset.get_data_as_shared(), testset.get_data_as_shared()

# def get_rnn(structure):

# recurrent_layer = Recurrent(dim=structure[0][1], layer_name='recurrent', irange=0.02, indices=[-1])

# softmax_layer = Softmax(n_classes=structure[1][1], layer_name='softmax', irange=0.02)

# layers = [recurrent_layer, softmax_layer]

# # return RNN(input_space=SequenceDataSpace(VectorSpace(dim=structure[0][0])), layers=layers)

# return RNN(input_space=SequenceSpace(VectorSpace(dim=structure[0][0])), layers=layers)

# def get_trainer(model, trainset):

# # 1. create the model config

# config = {'learning_rate': 0.1,

# 'cost' : Default(),

# 'batch_size': 1,

# 'monitoring_batches': 1,

# 'monitoring_dataset': trainset,

# 'termination_criterion': EpochCounter(max_epochs=MAX_EPOCHS_SUPERVISED),

# 'update_callbacks': None

# }

# # 2. create the training algo

# train_algo = SGD(**config)

# # 3. create and return the Train object

# return Train(model=model, dataset=trainset, algorithm=train_algo, extensions=None)

def get_random_indices(max_index, samples_per_epoch):

return np.random.randint(0, max_index, samples_per_epoch)

# def main_pylearn():

# """

# :description:

# :development: Instead of this layers approach, you may be able to use a MLP or RNN class to wrap the layers. This is probably what is being emmulated with this approach.

# """

# # create datasets

# print('loading dataset...')

# n_input_at_each_timestep = 5

# n_classes = 6

# n_frames = 5

# trainset, testset = get_dataset_toy(n_classes=n_classes, n_features=n_input_at_each_timestep, n_frames=n_frames)

# print('trainset.X.shape: {}'.format(trainset.X.shape))

# print('type(trainset.X): {}'.format(type(trainset.X)))

# iter = trainset.iterator()

# next = iter.next()

# print('iter.next(): {}'.format(next))

# print('iter.next()[0].shape: {}'.format(next[0].shape))

# print('type(iter.next()): {}'.format(type(next)))

# print('type(iter.next()[0]): {}'.format(type(next[0])))

# # determine network structure

# n_hidden = 5

# structure = [[n_input_at_each_timestep, n_hidden], [n_hidden, n_classes]]

# # create layers of network

# print('building model...')

# rnn = get_rnn(structure)

# # create the supervised trainer for the network

# print('building trainer...')

# layer_trainer = get_trainer(rnn, trainset)

# # run the main loop of the trainer

# print('training model...')

# layer_trainer.main_loop()

def main_theano():

print('{}: {}'.format(param.name, param.get_value()))

def main_theano_chest_accel():

print('{}: {}'.format(param.name, param.get_value()))

def main_theano_sign_lang():

print('{}: {}'.format(param.name, param.get_value()))

def main_theano_sign_lang_var_len():

print('{}: {}'.format(param.name, param.get_value()))

def main_theano_sign_lang_var_len_adadelta():

print('{}: {}'.format(param.name, param.get_value()))

def main_theano_softmax():

print('{}: {}'.format(param.name, param.get_value()))

if __name__ == '__main__':

main_theano_sign_lang_var_len()