def test_gru_rnn(): # random state so script is deterministic random_state = np.random.RandomState(1999) # home of the computational graph graph = OrderedDict() # number of hidden features n_hid = 10 # number of output_features = input_features n_out = X.shape[-1] # input (where first dimension is time) datasets_list = [X, X_mask, y, y_mask] names_list = ["X", "X_mask", "y", "y_mask"] test_values_list = [X, X_mask, y, y_mask] X_sym, X_mask_sym, y_sym, y_mask_sym = add_datasets_to_graph( datasets_list, names_list, graph, list_of_test_values=test_values_list) # Setup weights l1 = linear_layer([X_sym], graph, 'l1_proj', n_hid, random_state) h = gru_recurrent_layer([l1], X_mask_sym, n_hid, graph, 'l1_rec', random_state) # linear output activation y_hat = linear_layer([h], graph, 'l2_proj', n_out, random_state) # error between output and target cost = squared_error(y_hat, y_sym) cost = masked_cost(cost, y_mask_sym).mean() # Parameters of the model params, grads = get_params_and_grads(graph, cost) # Use stochastic gradient descent to optimize opt = sgd(params) learning_rate = 0.01 updates = opt.updates(params, grads, learning_rate) fit_function = theano.function([X_sym, X_mask_sym, y_sym, y_mask_sym], [cost], updates=updates, mode="FAST_COMPILE") cost_function = theano.function([X_sym, X_mask_sym, y_sym, y_mask_sym], [cost], mode="FAST_COMPILE") checkpoint_dict = {} train_indices = np.arange(X.shape[1]) valid_indices = np.arange(X.shape[1]) early_stopping_trainer(fit_function, cost_function, checkpoint_dict, [X, y], minibatch_size, train_indices, valid_indices, fit_function_output_names=["cost"], cost_function_output_name="valid_cost", n_epochs=1)
def test_conditional_gru_recurrent(): random_state = np.random.RandomState(1999) graph = OrderedDict() n_hid = 5 n_out = n_chars # input (where first dimension is time) datasets_list = [X_mb, X_mask, y_mb, y_mask] names_list = ["X", "X_mask", "y", "y_mask"] X_sym, X_mask_sym, y_sym, y_mask_sym = add_datasets_to_graph( datasets_list, names_list, graph) h = gru_recurrent_layer([X_sym], X_mask_sym, n_hid, graph, 'l1_end', random_state) shifted_y_sym = shift_layer([y_sym], graph, 'shift') h_dec, context = conditional_gru_recurrent_layer([y_sym], [h], y_mask_sym, n_hid, graph, 'l2_dec', random_state) # linear output activation y_hat = softmax_layer([h_dec, context, shifted_y_sym], graph, 'l2_proj', n_out, random_state) # error between output and target cost = categorical_crossentropy(y_hat, y_sym) cost = masked_cost(cost, y_mask_sym).mean() # Parameters of the model """ params, grads = get_params_and_grads(graph, cost) # Use stochastic gradient descent to optimize opt = sgd(params) learning_rate = 0.00000 updates = opt.updates(params, grads, learning_rate) fit_function = theano.function([X_sym, X_mask_sym, y_sym, y_mask_sym], [cost], updates=updates, mode="FAST_COMPILE") """ cost_function = theano.function([X_sym, X_mask_sym, y_sym, y_mask_sym], [cost], mode="FAST_COMPILE") checkpoint_dict = {} train_indices = np.arange(len(X)) valid_indices = np.arange(len(X)) early_stopping_trainer(cost_function, cost_function, checkpoint_dict, [X, y], minibatch_size, train_indices, valid_indices, list_of_minibatch_functions=[text_minibatch_func], fit_function_output_names=["cost"], cost_function_output_name="valid_cost", n_epochs=1)
valid_indices = train_indices X_mb, X_mb_mask = make_masked_minibatch(X, slice(0, len(X))) y_mb, y_mb_mask = make_masked_minibatch(y, slice(0, len(y))) n_hid = 256 n_out = vocab_size + 1 datasets_list = [X_mb, X_mb_mask, y_mb, y_mb_mask] names_list = ["X", "X_mask", "y", "y_mask"] X_sym, X_mask_sym, y_sym, y_mask_sym = add_datasets_to_graph( datasets_list, names_list, graph) h = gru_recurrent_layer([X_sym], X_mask_sym, n_hid, graph, 'l1_rec', random_state=random_state) y_pred = softmax_layer([h], graph, 'l2_proj', n_out, random_state=random_state) cost = log_ctc_cost(y_sym, y_mask_sym, y_pred, X_mask_sym).mean() params, grads = get_params_and_grads(graph, cost) opt = adadelta(params) updates = opt.updates(params, grads) checkpoint_dict = {} fit_function = theano.function([X_sym, X_mask_sym, y_sym, y_mask_sym], [cost], updates=updates) cost_function = theano.function([X_sym, X_mask_sym, y_sym, y_mask_sym], [cost])
X_query_mb, X_query_mask = make_embedding_minibatch( X_query, slice(0, minibatch_size)) embedding_datasets = [X_story_mb, X_query_mb] masks = [X_story_mask, X_query_mask] r = add_embedding_datasets_to_graph(embedding_datasets, masks, "babi_data", graph) (X_story_syms, X_query_syms), (X_story_mask_sym, X_query_mask_sym) = r y_sym = add_datasets_to_graph([y_answer], ["y"], graph) l1_story = embedding_layer(X_story_syms, vocab_size, n_emb, graph, 'l1_story', random_state=random_state) masked_story = X_story_mask_sym.dimshuffle(0, 1, 'x') * l1_story h_story = gru_recurrent_layer([masked_story], X_story_mask_sym, n_hid, graph, 'story_rec', random_state) l1_query = embedding_layer(X_query_syms, vocab_size, n_emb, graph, 'l1_query', random_state) h_query = gru_recurrent_layer([l1_query], X_query_mask_sym, n_hid, graph, 'query_rec', random_state) y_pred = softmax_layer([h_query[-1], h_story[-1]], graph, 'y_pred', y_answer.shape[1], random_state=random_state) cost = categorical_crossentropy(y_pred, y_sym).mean() params, grads = get_params_and_grads(graph, cost) opt = adadelta(params) updates = opt.updates(params, grads) print("Compiling fit...") fit_function = theano.function(X_story_syms + [X_story_mask_sym] + X_query_syms + [X_query_mask_sym, y_sym], [cost],