def create_rnn(hidden_dim, vocab_dim, mode="rnn"):
# input
x = tensor.imatrix('inchar')
y = tensor.imatrix('outchar')
#
W = LookupTable(
name="W1",
#dim = hidden_dim*4,
dim=hidden_dim,
length=vocab_dim,
weights_init=initialization.IsotropicGaussian(0.01),
biases_init=initialization.Constant(0))
if mode == "lstm":
# Long Short Term Memory
H = LSTM(hidden_dim,
name='H',
weights_init=initialization.IsotropicGaussian(0.01),
biases_init=initialization.Constant(0.0))
else:
# recurrent history weight
H = SimpleRecurrent(
name="H",
dim=hidden_dim,
activation=Tanh(),
weights_init=initialization.IsotropicGaussian(0.01))
#
S = Linear(name="W2",
input_dim=hidden_dim,
output_dim=vocab_dim,
weights_init=initialization.IsotropicGaussian(0.01),
biases_init=initialization.Constant(0))
A = NDimensionalSoftmax(name="softmax")
initLayers([W, H, S])
activations = W.apply(x)
hiddens = H.apply(activations) #[0]
activations2 = S.apply(hiddens)
y_hat = A.apply(activations2, extra_ndim=1)
cost = A.categorical_cross_entropy(y, activations2, extra_ndim=1).mean()
cg = ComputationGraph(cost)
#print VariableFilter(roles=[WEIGHT])(cg.variables)
#W1,H,W2 = VariableFilter(roles=[WEIGHT])(cg.variables)
layers = (x, W, H, S, A, y)
return cg, layers, y_hat, cost
def get_presoft(h, args):
output_size = get_output_size(args.dataset)
# If args.skip_connections: dim = args.layers * args.state_dim
# else: dim = args.state_dim
use_all_states = args.skip_connections or args.skip_output or (
args.rnn_type in ["clockwork", "soft"])
output_layer = Linear(
input_dim=use_all_states * args.layers * args.state_dim +
(1 - use_all_states) * args.state_dim,
output_dim=output_size,
name="output_layer")
output_layer.weights_init = initialization.IsotropicGaussian(0.1)
output_layer.biases_init = initialization.Constant(0)
output_layer.initialize()
presoft = output_layer.apply(h)
if not has_indices(args.dataset):
presoft = Tanh().apply(presoft)
presoft.name = 'presoft'
return presoft
def build_fork_lookup(vocab_size, args):
x = tensor.lmatrix('features')
virtual_dim = 6
time_length = 5
mini_batch_size = 2
skip_connections = True
layers = 3
# Build the model
output_names = []
output_dims = []
for d in range(layers):
if d > 0:
suffix = '_' + str(d)
else:
suffix = ''
if d == 0 or skip_connections:
output_names.append("inputs" + suffix)
output_dims.append(virtual_dim)
print output_names
print output_dims
lookup = LookupTable(length=vocab_size, dim=virtual_dim)
lookup.weights_init = initialization.IsotropicGaussian(0.1)
lookup.biases_init = initialization.Constant(0)
fork = Fork(output_names=output_names,
input_dim=time_length,
output_dims=output_dims,
prototype=FeedforwardSequence([lookup.apply]))
# Return list of 3D Tensor, one for each layer
# (Batch X Time X embedding_dim)
pre_rnn = fork.apply(x)
fork.initialize()
f = theano.function([x], pre_rnn)
return f
def build_model_hard(vocab_size, args, dtype=floatX):
logger.info('Building model ...')
# Parameters for the model
context = args.context
state_dim = args.state_dim
layers = args.layers
skip_connections = args.skip_connections
# Symbolic variables
# In both cases: Time X Batch
x = tensor.lmatrix('features')
y = tensor.lmatrix('targets')
# Build the model
output_names = []
output_dims = []
for d in range(layers):
if d > 0:
suffix = '_' + str(d)
else:
suffix = ''
if d == 0 or skip_connections:
output_names.append("inputs" + suffix)
output_dims.append(state_dim)
lookup = LookupTable(length=vocab_size, dim=state_dim)
lookup.weights_init = initialization.IsotropicGaussian(0.1)
lookup.biases_init = initialization.Constant(0)
fork = Fork(output_names=output_names,
input_dim=args.mini_batch_size,
output_dims=output_dims,
prototype=FeedforwardSequence([lookup.apply]))
transitions = [SimpleRecurrent(dim=state_dim, activation=Tanh())]
for i in range(layers - 1):
mlp = MLP(activations=[Logistic()],
dims=[2 * state_dim, 1],
weights_init=initialization.IsotropicGaussian(0.1),
biases_init=initialization.Constant(0),
name="mlp_" + str(i))
transitions.append(
HardGatedRecurrent(dim=state_dim, mlp=mlp, activation=Tanh()))
rnn = RecurrentStack(transitions, skip_connections=skip_connections)
# dim = layers * state_dim
output_layer = Linear(input_dim=layers * state_dim,
output_dim=vocab_size,
name="output_layer")
# Return list of 3D Tensor, one for each layer
# (Time X Batch X embedding_dim)
pre_rnn = fork.apply(x)
# Give a name to the input of each layer
if skip_connections:
for t in range(len(pre_rnn)):
pre_rnn[t].name = "pre_rnn_" + str(t)
else:
pre_rnn.name = "pre_rnn"
# Prepare inputs for the RNN
kwargs = OrderedDict()
init_states = {}
for d in range(layers):
if d > 0:
suffix = '_' + str(d)
else:
suffix = ''
if skip_connections:
kwargs['inputs' + suffix] = pre_rnn[d]
elif d == 0:
kwargs['inputs' + suffix] = pre_rnn
init_states[d] = theano.shared(numpy.zeros(
(args.mini_batch_size, state_dim)).astype(floatX),
name='state0_%d' % d)
kwargs['states' + suffix] = init_states[d]
# Apply the RNN to the inputs
h = rnn.apply(low_memory=True, **kwargs)
# Now we have correctly:
# h = [state_1, state_2, state_3 ...]
# Save all the last states
last_states = {}
for d in range(layers):
last_states[d] = h[d][-1, :, :]
# Concatenate all the states
if layers > 1:
h = tensor.concatenate(h, axis=2)
h.name = "hidden_state"
# The updates of the hidden states
updates = []
for d in range(layers):
updates.append((init_states[d], last_states[d]))
presoft = output_layer.apply(h[context:, :, :])
# Define the cost
# Compute the probability distribution
time, batch, feat = presoft.shape
presoft.name = 'presoft'
cross_entropy = Softmax().categorical_cross_entropy(
y[context:, :].flatten(), presoft.reshape((batch * time, feat)))
cross_entropy = cross_entropy / tensor.log(2)
cross_entropy.name = "cross_entropy"
# TODO: add regularisation for the cost
# the log(1) is here in order to differentiate the two variables
# for monitoring
cost = cross_entropy + tensor.log(1)
cost.name = "regularized_cost"
# Initialize the model
logger.info('Initializing...')
fork.initialize()
rnn.weights_init = initialization.Orthogonal()
rnn.biases_init = initialization.Constant(0)
rnn.initialize()
output_layer.weights_init = initialization.IsotropicGaussian(0.1)
output_layer.biases_init = initialization.Constant(0)
output_layer.initialize()
return cost, cross_entropy, updates
def get_prernn(args):
# time x batch
x_mask = tensor.fmatrix('mask')
# Compute the state dim
if args.rnn_type == 'lstm':
state_dim = 4 * args.state_dim
else:
state_dim = args.state_dim
# Prepare the arguments for the fork
output_names = []
output_dims = []
for d in range(args.layers):
if d > 0:
suffix = RECURRENTSTACK_SEPARATOR + str(d)
else:
suffix = ''
if d == 0 or args.skip_connections:
output_names.append("inputs" + suffix)
output_dims.append(state_dim)
# Prepare the brick to be forked (LookupTable or Linear)
# Check if the dataset provides indices (in the case of a
# fixed vocabulary, x is 2D tensor) or if it gives raw values
# (x is 3D tensor)
if has_indices(args.dataset):
features = args.mini_batch_size
x = tensor.lmatrix('features')
vocab_size = get_output_size(args.dataset)
lookup = LookupTable(length=vocab_size, dim=state_dim)
lookup.weights_init = initialization.IsotropicGaussian(0.1)
lookup.biases_init = initialization.Constant(0)
forked = FeedforwardSequence([lookup.apply])
if not has_mask(args.dataset):
x_mask = tensor.ones_like(x, dtype=floatX)
else:
x = tensor.tensor3('features', dtype=floatX)
if args.used_inputs is not None:
x = tensor.set_subtensor(
x[args.used_inputs:, :, :],
tensor.zeros_like(x[args.used_inputs:, :, :], dtype=floatX))
features = get_output_size(args.dataset)
forked = Linear(input_dim=features, output_dim=state_dim)
forked.weights_init = initialization.IsotropicGaussian(0.1)
forked.biases_init = initialization.Constant(0)
if not has_mask(args.dataset):
x_mask = tensor.ones_like(x[:, :, 0], dtype=floatX)
# Define the fork
fork = Fork(output_names=output_names,
input_dim=features,
output_dims=output_dims,
prototype=forked)
fork.initialize()
# Apply the fork
prernn = fork.apply(x)
# Give a name to the input of each layer
if args.skip_connections:
for t in range(len(prernn)):
prernn[t].name = "pre_rnn_" + str(t)
else:
prernn.name = "pre_rnn"
return prernn, x_mask
def build_fork_lookup(vocab_size, time_length, args):
x = tensor.lmatrix('features')
virtual_dim = 6
state_dim = 6
skip_connections = False
layers = 1
# Build the model
output_names = []
output_dims = []
for d in range(layers):
if d > 0:
suffix = '_' + str(d)
else:
suffix = ''
if d == 0 or skip_connections:
output_names.append("inputs" + suffix)
output_dims.append(virtual_dim)
lookup = LookupTable(length=vocab_size, dim=virtual_dim)
lookup.weights_init = initialization.IsotropicGaussian(0.1)
lookup.biases_init = initialization.Constant(0)
fork = Fork(output_names=output_names, input_dim=time_length,
output_dims=output_dims,
prototype=FeedforwardSequence(
[lookup.apply]))
# Note that this order of the periods makes faster modules flow in slower
# ones with is the opposite of the original paper
transitions = [ClockworkBase(dim=state_dim, activation=Tanh(),
period=2 ** i) for i in range(layers)]
rnn = RecurrentStack(transitions, skip_connections=skip_connections)
# Return list of 3D Tensor, one for each layer
# (Batch X Time X embedding_dim)
pre_rnn = fork.apply(x)
# Give time as the first index for each element in the list:
# (Time X Batch X embedding_dim)
if layers > 1 and skip_connections:
for t in range(len(pre_rnn)):
pre_rnn[t] = pre_rnn[t].dimshuffle(1, 0, 2)
else:
pre_rnn = pre_rnn.dimshuffle(1, 0, 2)
f_pre_rnn = theano.function([x], pre_rnn)
# Prepare inputs for the RNN
kwargs = OrderedDict()
for d in range(layers):
if d > 0:
suffix = '_' + str(d)
else:
suffix = ''
if d == 0 or skip_connections:
if skip_connections:
kwargs['inputs' + suffix] = pre_rnn[d]
else:
kwargs['inputs' + suffix] = pre_rnn
print kwargs
# Apply the RNN to the inputs
h = rnn.apply(low_memory=True, **kwargs)
fork.initialize()
rnn.weights_init = initialization.Orthogonal()
rnn.biases_init = initialization.Constant(0)
rnn.initialize()
f_h = theano.function([x], h)
return f_pre_rnn, f_h
from extensions import LearningRateSchedule, Plot, TimedFinish
from datasets import parrot_stream
from model import Parrot
from utils import train_parse
args = train_parse()
exp_name = args.experiment_name
save_dir = args.save_dir
print "Saving config ..."
with open(os.path.join(save_dir, 'config', exp_name + '.pkl'), 'w') as f:
cPickle.dump(args, f)
print "Finished saving."
w_init = initialization.IsotropicGaussian(0.01)
b_init = initialization.Constant(0.)
train_stream = parrot_stream(
args.dataset, args.use_speaker, ('train',), args.batch_size,
noise_level=args.feedback_noise_level, labels_type=args.labels_type,
seq_size=args.seq_size, raw_data=args.raw_output)
if args.feedback_noise_level is None:
val_noise_level = None
else:
val_noise_level = 0.
valid_stream = parrot_stream(
args.dataset, args.use_speaker, ('valid',), args.batch_size,
noise_level=val_noise_level, labels_type=args.labels_type,
def build_model_lstm(vocab_size, args, dtype=floatX):
logger.info('Building model ...')
# Parameters for the model
context = args.context
state_dim = args.state_dim
layers = args.layers
skip_connections = args.skip_connections
virtual_dim = 4 * state_dim
# Symbolic variables
# In both cases: Time X Batch
x = tensor.lmatrix('features')
y = tensor.lmatrix('targets')
# Build the model
output_names = []
output_dims = []
for d in range(layers):
if d > 0:
suffix = '_' + str(d)
else:
suffix = ''
if d == 0 or skip_connections:
output_names.append("inputs" + suffix)
output_dims.append(virtual_dim)
lookup = LookupTable(length=vocab_size, dim=virtual_dim)
lookup.weights_init = initialization.IsotropicGaussian(0.1)
lookup.biases_init = initialization.Constant(0)
# Make sure time_length is what we need
fork = Fork(output_names=output_names,
input_dim=args.mini_batch_size,
output_dims=output_dims,
prototype=FeedforwardSequence([lookup.apply]))
transitions = [
LSTM(dim=state_dim, activation=Tanh()) for _ in range(layers)
]
rnn = RecurrentStack(transitions, skip_connections=skip_connections)
# If skip_connections: dim = layers * state_dim
# else: dim = state_dim
output_layer = Linear(input_dim=skip_connections * layers * state_dim +
(1 - skip_connections) * state_dim,
output_dim=vocab_size,
name="output_layer")
# Return list of 3D Tensor, one for each layer
# (Time X Batch X embedding_dim)
pre_rnn = fork.apply(x)
# Give a name to the input of each layer
if skip_connections:
for t in range(len(pre_rnn)):
pre_rnn[t].name = "pre_rnn_" + str(t)
else:
pre_rnn.name = "pre_rnn"
# Prepare inputs for the RNN
kwargs = OrderedDict()
init_states = {}
init_cells = {}
for d in range(layers):
if d > 0:
suffix = '_' + str(d)
else:
suffix = ''
if skip_connections:
kwargs['inputs' + suffix] = pre_rnn[d]
elif d == 0:
kwargs['inputs'] = pre_rnn
init_states[d] = theano.shared(numpy.zeros(
(args.mini_batch_size, state_dim)).astype(floatX),
name='state0_%d' % d)
init_cells[d] = theano.shared(numpy.zeros(
(args.mini_batch_size, state_dim)).astype(floatX),
name='cell0_%d' % d)
kwargs['states' + suffix] = init_states[d]
kwargs['cells' + suffix] = init_cells[d]
# Apply the RNN to the inputs
h = rnn.apply(low_memory=True, **kwargs)
# h = [state, cell, in, forget, out, state_1,
# cell_1, in_1, forget_1, out_1 ...]
last_states = {}
last_cells = {}
for d in range(layers):
last_states[d] = h[5 * d][-1, :, :]
last_cells[d] = h[5 * d + 1][-1, :, :]
# The updates of the hidden states
updates = []
for d in range(layers):
updates.append((init_states[d], last_states[d]))
updates.append((init_cells[d], last_states[d]))
# h = [state, cell, in, forget, out, state_1,
# cell_1, in_1, forget_1, out_1 ...]
# Extract the values
in_gates = h[2::5]
forget_gates = h[3::5]
out_gates = h[4::5]
gate_values = {
"in_gates": in_gates,
"forget_gates": forget_gates,
"out_gates": out_gates
}
h = h[::5]
# Now we have correctly:
# h = [state, state_1, state_2 ...] if layers > 1
# h = [state] if layers == 1
# If we have skip connections, concatenate all the states
# Else only consider the state of the highest layer
if layers > 1:
if skip_connections:
h = tensor.concatenate(h, axis=2)
else:
h = h[-1]
else:
h = h[0]
h.name = "hidden_state"
presoft = output_layer.apply(h[context:, :, :])
# Define the cost
# Compute the probability distribution
time, batch, feat = presoft.shape
presoft.name = 'presoft'
cross_entropy = Softmax().categorical_cross_entropy(
y[context:, :].flatten(), presoft.reshape((batch * time, feat)))
cross_entropy = cross_entropy / tensor.log(2)
cross_entropy.name = "cross_entropy"
# TODO: add regularisation for the cost
# the log(1) is here in order to differentiate the two variables
# for monitoring
cost = cross_entropy + tensor.log(1)
cost.name = "regularized_cost"
# Initialize the model
logger.info('Initializing...')
fork.initialize()
# Dont initialize as Orthogonal if we are about to load new parameters
if args.load_path is not None:
rnn.weights_init = initialization.Constant(0)
else:
rnn.weights_init = initialization.Orthogonal()
rnn.biases_init = initialization.Constant(0)
rnn.initialize()
output_layer.weights_init = initialization.IsotropicGaussian(0.1)
output_layer.biases_init = initialization.Constant(0)
output_layer.initialize()
return cost, cross_entropy, updates, gate_values
def build_model(args, dtype=floatX):
logger.info('Building model ...')
# Variables of the model
# the rubik's cube stickers
x = tensor.bmatrix("x")
# the action taken
action = tensor.bmatrix("action")
# y is the reward (Batch,)
y = tensor.fvector("y")
#####
# LookupTable
#####
lookup_x = LookupTable(length=6, dim=args.embed_dim)
lookup_action = LookupTable(length=6 + args.cube_size + 3,
dim=args.embed_dim)
lookup_x.name = "lookup_x"
lookup_x.weights_init = initialization.IsotropicGaussian(0.1)
lookup_x.biases_init = initialization.Constant(0)
lookup_action.name = "lookup_action"
lookup_action.weights_init = initialization.IsotropicGaussian(0.1)
lookup_action.biases_init = initialization.Constant(0)
lookup_x.initialize()
lookup_action.initialize()
x_embeded = lookup_x.apply(x)
action_embeded = lookup_action.apply(action)
#####
# MLP
#####
# Make x_embeded and action_embeded 2D
x_embeded = x_embeded.reshape(
(x_embeded.shape[0], x_embeded.shape[1] * x_embeded.shape[2]))
action_embeded = action_embeded.reshape(
(action_embeded.shape[0],
action_embeded.shape[1] * action_embeded.shape[2]))
# Concatenate inputs :
mlp_input = tensor.concatenate((x_embeded, action_embeded), axis=1)
# Bricks
l = args.layers
activations = []
# first layer dimension
dims = [args.embed_dim * (6 * (args.cube_size**2) + 3)]
# every hidden layer dimension and activation function
for _ in range(l):
activations.append(Rectifier())
dims.append(args.units_per_layer)
# last layer dimension
dims[-1] = 1
mlp = MLP(activations=activations, dims=dims)
y_hat = mlp.apply(mlp_input)
cost = SquaredError().apply(y.dimshuffle(0, "x"), y_hat)
cost.name = "mean_squared_error"
# Initialization
mlp.weights_init = initialization.IsotropicGaussian(0.1)
mlp.biases_init = initialization.Constant(0)
mlp.initialize()
# Q function
# Check if the parameters in this function will change through
# the updates of the gradient descent
Q = theano.function(inputs=[x, action],
outputs=y_hat,
allow_input_downcast=True)
# Cost, gradient and learning rate
lr = tensor.scalar('lr')
params = ComputationGraph(cost).parameters
gradients = tensor.grad(cost, params)
updates = OrderedDict((p, p - lr * g) for p, g in zip(params, gradients))
# Function to call to perfom a gradient descent on (y - Q)^2
gradient_descent_step = theano.function([x, action, y, lr],
cost,
updates=updates,
allow_input_downcast=True)
# Load the good parameters
if args.load_path is not None:
param_values = load_parameter_values(args.load_path)
model = Model(cost)
model.set_parameter_values(param_values)
return Q, gradient_descent_step, params
def build_model_soft(args, dtype=floatX):
logger.info('Building model ...')
# Return list of 3D Tensor, one for each layer
# (Time X Batch X embedding_dim)
pre_rnn, x_mask = get_prernn(args)
transitions = [SimpleRecurrent(dim=args.state_dim, activation=Tanh())]
# Build the MLP
dims = [2 * args.state_dim]
activations = []
for i in range(args.mlp_layers):
activations.append(Rectifier())
dims.append(args.state_dim)
# Activation of the last layer of the MLP
if args.mlp_activation == "logistic":
activations.append(Logistic())
elif args.mlp_activation == "rectifier":
activations.append(Rectifier())
elif args.mlp_activation == "hard_logistic":
activations.append(HardLogistic())
else:
assert False
# Output of MLP has dimension 1
dims.append(1)
for i in range(args.layers - 1):
mlp = MLP(activations=activations, dims=dims,
weights_init=initialization.IsotropicGaussian(0.1),
biases_init=initialization.Constant(0),
name="mlp_" + str(i))
transitions.append(
SoftGatedRecurrent(dim=args.state_dim,
mlp=mlp,
activation=Tanh()))
rnn = RecurrentStack(transitions, skip_connections=args.skip_connections)
initialize_rnn(rnn, args)
# Prepare inputs and initial states for the RNN
kwargs, inits = get_rnn_kwargs(pre_rnn, args)
# Apply the RNN to the inputs
h = rnn.apply(low_memory=True, mask=x_mask, **kwargs)
# Now we have:
# h = [state, state_1, gate_value_1, state_2, gate_value_2, state_3, ...]
# Extract gate_values
gate_values = h[2::2]
new_h = [h[0]]
new_h.extend(h[1::2])
h = new_h
# Now we have:
# h = [state, state_1, state_2, ...]
# gate_values = [gate_value_1, gate_value_2, gate_value_3]
for i, gate_value in enumerate(gate_values):
gate_value.name = "gate_value_" + str(i)
# Save all the last states
last_states = {}
hidden_states = []
for d in range(args.layers):
h[d] = h[d] * x_mask
last_states[d] = h[d][-1, :, :]
h[d].name = "hidden_state_" + str(d)
hidden_states.append(h[d])
# Concatenate all the states
if args.layers > 1:
h = tensor.concatenate(h, axis=2)
h.name = "hidden_state_all"
# The updates of the hidden states
updates = []
for d in range(args.layers):
updates.append((inits[0][d], last_states[d]))
presoft = get_presoft(h, args)
cost, cross_entropy = get_costs(presoft, args)
return cost, cross_entropy, updates, gate_values, hidden_states