def test_model_get_outputs_rnn(backend_default, data):
data_path = load_ptb_test(path=data)
data_set = Text(time_steps=50, path=data_path)
# weight initialization
init = Constant(0.08)
# model initialization
layers = [
Recurrent(150, init, activation=Logistic()),
Affine(len(data_set.vocab), init, bias=init, activation=Rectlin())
]
model = Model(layers=layers)
output = model.get_outputs(data_set)
assert output.shape == (data_set.ndata, data_set.seq_length,
data_set.nclass)
# since the init are all constant and model is un-trained:
# along the feature dim, the values should be all the same
assert np.allclose(output[0, 0], output[0, 0, 0], rtol=0, atol=1e-5)
assert np.allclose(output[0, 1], output[0, 1, 0], rtol=0, atol=1e-5)
# along the time dim, the values should be increasing:
assert np.alltrue(output[0, 2] > output[0, 1])
assert np.alltrue(output[0, 1] > output[0, 0])
def test_model_get_outputs_rnn(backend_default, data):
data_path = load_text('ptb-valid', path=data)
data_set = Text(time_steps=50, path=data_path)
# weight initialization
init = Constant(0.08)
# model initialization
layers = [
Recurrent(150, init, Logistic()),
Affine(len(data_set.vocab), init, bias=init, activation=Rectlin())
]
model = Model(layers=layers)
output = model.get_outputs(data_set)
assert output.shape == (
data_set.ndata, data_set.seq_length, data_set.nclass)
batch_size=64)
parser = NeonArgparser(__doc__, default_overrides=default_overrides)
args = parser.parse_args()
# hyperparameters
time_steps = 64
hidden_size = 512
gradient_clip_value = 5
# download shakespeare text
data_path = load_shakespeare(path=args.data_dir)
train_path, valid_path = Text.create_valid_file(data_path)
# load data and parse on character-level
train_set = Text(time_steps, train_path)
valid_set = Text(time_steps, valid_path, vocab=train_set.vocab)
# weight initialization
init = Uniform(low=-0.08, high=0.08)
# model initialization
layers = [
LSTM(hidden_size, init, activation=Logistic(), gate_activation=Tanh()),
Affine(len(train_set.vocab), init, bias=init, activation=Softmax())
]
model = Model(layers=layers)
cost = GeneralizedCost(costfunc=CrossEntropyMulti(usebits=True))
optimizer = RMSProp(gradient_clip_value=gradient_clip_value,
# download penn treebank
train_path = load_text('ptb-train', path=args.data_dir)
valid_path = load_text('ptb-valid', path=args.data_dir)
# define a custom function to parse the input into individual tokens, which for
# this data, splits into individual words. This can be passed into the Text
# object during dataset creation as seen below.
def tokenizer(s):
return s.replace('\n', '<eos>').split()
# load data and parse on word-level
train_set = Text(time_steps,
train_path,
tokenizer=tokenizer,
onehot_input=False)
valid_set = Text(time_steps,
valid_path,
vocab=train_set.vocab,
tokenizer=tokenizer,
onehot_input=False)
# weight initialization
init = Uniform(low=-0.1, high=0.1)
# model initialization
rlayer_params = {
"output_size": hidden_size,
"init": init,
"activation": Tanh(),
# setup backend
be = gen_backend(backend=args.backend,
batch_size=batch_size,
rng_seed=args.rng_seed,
device_id=args.device_id,
default_dtype=args.datatype)
# download penn treebank
train_path = load_text('ptb-train', path=args.data_dir)
valid_path = load_text('ptb-valid', path=args.data_dir)
# load data and parse on word-level
# a Text object can take a given tokenizer, for word-level parsing, it is str.split
# a user can pass in a custom-defined tokenzier as well
train_set = Text(time_steps, train_path, tokenizer=str.split)
valid_set = Text(time_steps, valid_path, vocab=train_set.vocab, tokenizer=str.split)
# weight initialization
init = Uniform(low=-0.08, high=0.08)
# model initialization
if rlayer_type == 'lstm':
rlayer1 = LSTM(hidden_size, init, Logistic(), Tanh())
rlayer2 = LSTM(hidden_size, init, Logistic(), Tanh())
elif rlayer_type == 'gru':
rlayer1 = GRU(hidden_size, init, activation=Tanh(), gate_activation=Logistic())
rlayer2 = GRU(hidden_size, init, activation=Tanh(), gate_activation=Logistic())
else:
raise NotImplementedError('%s layer not implemented' % rlayer_type)
