def test_gru_rnn():
# random state so script is deterministic
random_state = np.random.RandomState(1999)
# home of the computational graph
graph = OrderedDict()
# number of hidden features
n_hid = 10
# number of output_features = input_features
n_out = X.shape[-1]
# input (where first dimension is time)
datasets_list = [X, X_mask, y, y_mask]
names_list = ["X", "X_mask", "y", "y_mask"]
test_values_list = [X, X_mask, y, y_mask]
X_sym, X_mask_sym, y_sym, y_mask_sym = add_datasets_to_graph(
datasets_list, names_list, graph, list_of_test_values=test_values_list)
# Setup weights
l1 = linear_layer([X_sym], graph, 'l1_proj', n_hid, random_state)
h = gru_recurrent_layer([l1], X_mask_sym, n_hid, graph, 'l1_rec',
random_state)
# linear output activation
y_hat = linear_layer([h], graph, 'l2_proj', n_out, random_state)
# error between output and target
cost = squared_error(y_hat, y_sym)
cost = masked_cost(cost, y_mask_sym).mean()
# Parameters of the model
params, grads = get_params_and_grads(graph, cost)
# Use stochastic gradient descent to optimize
opt = sgd(params)
learning_rate = 0.01
updates = opt.updates(params, grads, learning_rate)
fit_function = theano.function([X_sym, X_mask_sym, y_sym, y_mask_sym],
[cost],
updates=updates,
mode="FAST_COMPILE")
cost_function = theano.function([X_sym, X_mask_sym, y_sym, y_mask_sym],
[cost],
mode="FAST_COMPILE")
checkpoint_dict = {}
train_indices = np.arange(X.shape[1])
valid_indices = np.arange(X.shape[1])
early_stopping_trainer(fit_function,
cost_function,
checkpoint_dict, [X, y],
minibatch_size,
train_indices,
valid_indices,
fit_function_output_names=["cost"],
cost_function_output_name="valid_cost",
n_epochs=1)
def test_conditional_gru_recurrent():
random_state = np.random.RandomState(1999)
graph = OrderedDict()
n_hid = 5
n_out = n_chars
# input (where first dimension is time)
datasets_list = [X_mb, X_mask, y_mb, y_mask]
names_list = ["X", "X_mask", "y", "y_mask"]
X_sym, X_mask_sym, y_sym, y_mask_sym = add_datasets_to_graph(
datasets_list, names_list, graph)
h = gru_recurrent_layer([X_sym], X_mask_sym, n_hid, graph, 'l1_end',
random_state)
shifted_y_sym = shift_layer([y_sym], graph, 'shift')
h_dec, context = conditional_gru_recurrent_layer([y_sym], [h], y_mask_sym,
n_hid, graph, 'l2_dec',
random_state)
# linear output activation
y_hat = softmax_layer([h_dec, context, shifted_y_sym], graph, 'l2_proj',
n_out, random_state)
# error between output and target
cost = categorical_crossentropy(y_hat, y_sym)
cost = masked_cost(cost, y_mask_sym).mean()
# Parameters of the model
"""
params, grads = get_params_and_grads(graph, cost)
# Use stochastic gradient descent to optimize
opt = sgd(params)
learning_rate = 0.00000
updates = opt.updates(params, grads, learning_rate)
fit_function = theano.function([X_sym, X_mask_sym, y_sym, y_mask_sym],
[cost], updates=updates,
mode="FAST_COMPILE")
"""
cost_function = theano.function([X_sym, X_mask_sym, y_sym, y_mask_sym],
[cost], mode="FAST_COMPILE")
checkpoint_dict = {}
train_indices = np.arange(len(X))
valid_indices = np.arange(len(X))
early_stopping_trainer(cost_function, cost_function, checkpoint_dict,
[X, y],
minibatch_size, train_indices, valid_indices,
list_of_minibatch_functions=[text_minibatch_func],
fit_function_output_names=["cost"],
cost_function_output_name="valid_cost",
n_epochs=1)
valid_indices = train_indices
X_mb, X_mb_mask = make_masked_minibatch(X, slice(0, len(X)))
y_mb, y_mb_mask = make_masked_minibatch(y, slice(0, len(y)))
n_hid = 256
n_out = vocab_size + 1
datasets_list = [X_mb, X_mb_mask, y_mb, y_mb_mask]
names_list = ["X", "X_mask", "y", "y_mask"]
X_sym, X_mask_sym, y_sym, y_mask_sym = add_datasets_to_graph(
datasets_list, names_list, graph)
h = gru_recurrent_layer([X_sym],
X_mask_sym,
n_hid,
graph,
'l1_rec',
random_state=random_state)
y_pred = softmax_layer([h], graph, 'l2_proj', n_out, random_state=random_state)
cost = log_ctc_cost(y_sym, y_mask_sym, y_pred, X_mask_sym).mean()
params, grads = get_params_and_grads(graph, cost)
opt = adadelta(params)
updates = opt.updates(params, grads)
checkpoint_dict = {}
fit_function = theano.function([X_sym, X_mask_sym, y_sym, y_mask_sym], [cost],
updates=updates)
cost_function = theano.function([X_sym, X_mask_sym, y_sym, y_mask_sym], [cost])
X_query_mb, X_query_mask = make_embedding_minibatch(
X_query, slice(0, minibatch_size))
embedding_datasets = [X_story_mb, X_query_mb]
masks = [X_story_mask, X_query_mask]
r = add_embedding_datasets_to_graph(embedding_datasets, masks, "babi_data",
graph)
(X_story_syms, X_query_syms), (X_story_mask_sym, X_query_mask_sym) = r
y_sym = add_datasets_to_graph([y_answer], ["y"], graph)
l1_story = embedding_layer(X_story_syms, vocab_size, n_emb, graph, 'l1_story',
random_state=random_state)
masked_story = X_story_mask_sym.dimshuffle(0, 1, 'x') * l1_story
h_story = gru_recurrent_layer([masked_story], X_story_mask_sym, n_hid, graph,
'story_rec', random_state)
l1_query = embedding_layer(X_query_syms, vocab_size, n_emb, graph, 'l1_query',
random_state)
h_query = gru_recurrent_layer([l1_query], X_query_mask_sym, n_hid, graph,
'query_rec', random_state)
y_pred = softmax_layer([h_query[-1], h_story[-1]], graph, 'y_pred',
y_answer.shape[1], random_state=random_state)
cost = categorical_crossentropy(y_pred, y_sym).mean()
params, grads = get_params_and_grads(graph, cost)
opt = adadelta(params)
updates = opt.updates(params, grads)
print("Compiling fit...")
fit_function = theano.function(X_story_syms + [X_story_mask_sym] + X_query_syms
+ [X_query_mask_sym, y_sym], [cost],
