def test_model_get_outputs_rnn(backend_default, data): data_path = load_ptb_test(path=data) data_set = Text(time_steps=50, path=data_path) # weight initialization init = Constant(0.08) # model initialization layers = [ Recurrent(150, init, activation=Logistic()), Affine(len(data_set.vocab), init, bias=init, activation=Rectlin()) ] model = Model(layers=layers) output = model.get_outputs(data_set) assert output.shape == (data_set.ndata, data_set.seq_length, data_set.nclass) # since the init are all constant and model is un-trained: # along the feature dim, the values should be all the same assert np.allclose(output[0, 0], output[0, 0, 0], rtol=0, atol=1e-5) assert np.allclose(output[0, 1], output[0, 1, 0], rtol=0, atol=1e-5) # along the time dim, the values should be increasing: assert np.alltrue(output[0, 2] > output[0, 1]) assert np.alltrue(output[0, 1] > output[0, 0])
def test_model_get_outputs_rnn(backend_default, data): data_path = load_text('ptb-valid', path=data) data_set = Text(time_steps=50, path=data_path) # weight initialization init = Constant(0.08) # model initialization layers = [ Recurrent(150, init, Logistic()), Affine(len(data_set.vocab), init, bias=init, activation=Rectlin()) ] model = Model(layers=layers) output = model.get_outputs(data_set) assert output.shape == ( data_set.ndata, data_set.seq_length, data_set.nclass)
batch_size=64) parser = NeonArgparser(__doc__, default_overrides=default_overrides) args = parser.parse_args() # hyperparameters time_steps = 64 hidden_size = 512 gradient_clip_value = 5 # download shakespeare text data_path = load_shakespeare(path=args.data_dir) train_path, valid_path = Text.create_valid_file(data_path) # load data and parse on character-level train_set = Text(time_steps, train_path) valid_set = Text(time_steps, valid_path, vocab=train_set.vocab) # weight initialization init = Uniform(low=-0.08, high=0.08) # model initialization layers = [ LSTM(hidden_size, init, activation=Logistic(), gate_activation=Tanh()), Affine(len(train_set.vocab), init, bias=init, activation=Softmax()) ] model = Model(layers=layers) cost = GeneralizedCost(costfunc=CrossEntropyMulti(usebits=True)) optimizer = RMSProp(gradient_clip_value=gradient_clip_value,
# download penn treebank train_path = load_text('ptb-train', path=args.data_dir) valid_path = load_text('ptb-valid', path=args.data_dir) # define a custom function to parse the input into individual tokens, which for # this data, splits into individual words. This can be passed into the Text # object during dataset creation as seen below. def tokenizer(s): return s.replace('\n', '<eos>').split() # load data and parse on word-level train_set = Text(time_steps, train_path, tokenizer=tokenizer, onehot_input=False) valid_set = Text(time_steps, valid_path, vocab=train_set.vocab, tokenizer=tokenizer, onehot_input=False) # weight initialization init = Uniform(low=-0.1, high=0.1) # model initialization rlayer_params = { "output_size": hidden_size, "init": init, "activation": Tanh(),
# setup backend be = gen_backend(backend=args.backend, batch_size=batch_size, rng_seed=args.rng_seed, device_id=args.device_id, default_dtype=args.datatype) # download penn treebank train_path = load_text('ptb-train', path=args.data_dir) valid_path = load_text('ptb-valid', path=args.data_dir) # load data and parse on word-level # a Text object can take a given tokenizer, for word-level parsing, it is str.split # a user can pass in a custom-defined tokenzier as well train_set = Text(time_steps, train_path, tokenizer=str.split) valid_set = Text(time_steps, valid_path, vocab=train_set.vocab, tokenizer=str.split) # weight initialization init = Uniform(low=-0.08, high=0.08) # model initialization if rlayer_type == 'lstm': rlayer1 = LSTM(hidden_size, init, Logistic(), Tanh()) rlayer2 = LSTM(hidden_size, init, Logistic(), Tanh()) elif rlayer_type == 'gru': rlayer1 = GRU(hidden_size, init, activation=Tanh(), gate_activation=Logistic()) rlayer2 = GRU(hidden_size, init, activation=Tanh(), gate_activation=Logistic()) else: raise NotImplementedError('%s layer not implemented' % rlayer_type)