def test_model_predict_rnn(backend):
data_path = load_text('ptb-valid')
data_set = Text(time_steps=50, path=data_path)
# weight initialization
init = Constant(0.08)
# model initialization
layers = [
Recurrent(150, init, Logistic()),
Affine(len(data_set.vocab), init, bias=init, activation=Rectlin())
]
model = Model(layers=layers)
output = model.predict(data_set)
assert output.shape == (data_set.ndata, data_set.nclass)
def test_model_get_outputs_rnn(backend_default, data):
data_path = load_text('ptb-valid', path=data)
data_set = Text(time_steps=50, path=data_path)
# weight initialization
init = Constant(0.08)
# model initialization
layers = [
Recurrent(150, init, Logistic()),
Affine(len(data_set.vocab), init, bias=init, activation=Rectlin())
]
model = Model(layers=layers)
output = model.get_outputs(data_set)
assert output.shape == (
data_set.ndata, data_set.seq_length, data_set.nclass)
parser = NeonArgparser(__doc__)
parser.add_argument('--rlayer_type', default='lstm', choices=['gru', 'lstm'],
help='type of recurrent layer to use (gru or lstm)')
args = parser.parse_args(gen_be=False)
# hyperparameters from the reference
args.batch_size = 20
time_steps = 20
hidden_size = 200
gradient_clip_norm = 5
# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))
# download penn treebank
train_path = load_text('ptb-train', path=args.data_dir)
valid_path = load_text('ptb-valid', path=args.data_dir)
# define a custom function to parse the input into individual tokens, which for
# this data, splits into individual words. This can be passed into the Text
# object during dataset creation as seen below.
def tokenizer(s):
return s.replace('\n', '<eos>').split()
# load data and parse on word-level
train_set = Text(time_steps, train_path, tokenizer=tokenizer, onehot_input=False)
valid_set = Text(time_steps, valid_path, vocab=train_set.vocab, tokenizer=tokenizer,
onehot_input=False)
# weight initialization
gradient_limit = 15
vocab_size = 20000
sentence_length = 100
embedding_dim = 128
hidden_size = 128
reset_cells = True
# setup backend
be = gen_backend(backend=args.backend,
batch_size=batch_size,
rng_seed=args.rng_seed,
device_id=args.device_id,
default_dtype=args.datatype)
# make dataset
path = load_text('imdb', path=args.data_dir)
(X_train, y_train), (X_test, y_test), nclass = Text.pad_data(
path, vocab_size=vocab_size, sentence_length=sentence_length)
print "Vocab size - ", vocab_size
print "Sentence Length - ", sentence_length
print "# of train sentences", X_train.shape[0]
print "# of test sentence", X_test.shape[0]
train_set = DataIterator(X_train, y_train, nclass=2)
valid_set = DataIterator(X_test, y_test, nclass=2)
# weight initialization
init_emb = Uniform(low=-0.1/embedding_dim, high=0.1/embedding_dim)
init_glorot = GlorotUniform()
rlayer_type = "lstm"
# hyperparameters
time_steps = 50
hidden_size = 1000
clip_gradients = False
# setup backend
be = gen_backend(backend=args.backend,
batch_size=batch_size,
rng_seed=args.rng_seed,
device_id=args.device_id,
default_dtype=args.datatype)
# download penn treebank
train_path = load_text('ptb-train', path=args.data_dir)
valid_path = load_text('ptb-valid', path=args.data_dir)
# load data and parse on character-level
train_set = Text(time_steps, train_path)
valid_set = Text(time_steps, valid_path, vocab=train_set.vocab)
# weight initialization
init = Uniform(low=-0.08, high=0.08)
# model initialization
if rlayer_type == 'lstm':
rlayer = LSTM(hidden_size, init, Logistic(), Tanh())
elif rlayer_type == 'gru':
rlayer = GRU(hidden_size, init, activation=Tanh(), gate_activation=Logistic())
else:
args.save_path = 'rnn_text_gen.pickle'
# hyperparameters
time_steps = 64
hidden_size = 512
clip_gradients = True
# setup backend
be = gen_backend(backend=args.backend,
batch_size=batch_size,
rng_seed=args.rng_seed,
device_id=args.device_id,
default_dtype=args.datatype)
# download shakespeare text
data_path = load_text('shakespeare', path=args.data_dir)
train_path, valid_path = Text.create_valid_file(data_path)
# load data and parse on character-level
train_set = Text(time_steps, train_path)
valid_set = Text(time_steps, valid_path, vocab=train_set.vocab)
# weight initialization
init = Uniform(low=-0.08, high=0.08)
# model initialization
layers = [
LSTM(hidden_size, init, Logistic(), Tanh()),
Affine(len(train_set.vocab), init, bias=init, activation=Softmax())
]
model = Model(layers=layers)
gradient_limit = 15
vocab_size = 20000
sentence_length = 100
embedding_dim = 128
hidden_size = 128
reset_cells = True
# setup backend
be = gen_backend(backend=args.backend,
batch_size=batch_size,
rng_seed=args.rng_seed,
device_id=args.device_id,
default_dtype=args.datatype)
# make dataset
path = load_text('imdb', path=args.data_dir)
(X_train, y_train), (X_test, y_test), nclass = Text.pad_data(
path, vocab_size=vocab_size, sentence_length=sentence_length)
print "Vocab size - ", vocab_size
print "Sentence Length - ", sentence_length
print "# of train sentences", X_train.shape[0]
print "# of test sentence", X_test.shape[0]
import numpy as np
train_set = DataIterator(X_train, y_train, nclass=2)
test_set = DataIterator(X_test, y_test, nclass=2)
# weight initialization
init_emb = Uniform(low=-0.1/embedding_dim, high=0.1/embedding_dim)
init_glorot = GlorotUniform()
args.callback_args['save_path'] = args.save_path
if args.callback_args['serialize'] is None:
args.callback_args['serialize'] = 1
# hyperparameters
args.batch_size = 64
time_steps = 64
hidden_size = 512
gradient_clip_value = 5
# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))
# download shakespeare text
data_path = load_text('shakespeare', path=args.data_dir)
train_path, valid_path = Text.create_valid_file(data_path)
# load data and parse on character-level
train_set = Text(time_steps, train_path)
valid_set = Text(time_steps, valid_path, vocab=train_set.vocab)
# weight initialization
init = Uniform(low=-0.08, high=0.08)
# model initialization
layers = [
LSTM(hidden_size, init, activation=Logistic(), gate_activation=Tanh()),
Affine(len(train_set.vocab), init, bias=init, activation=Softmax())
]
model = Model(layers=layers)
parser.add_argument(
"--rlayer_type", default="lstm", choices=["gru", "lstm"], help="type of recurrent layer to use (gru or lstm)"
)
args = parser.parse_args(gen_be=False)
# hyperparameters from the reference
args.batch_size = 20
time_steps = 20
hidden_size = 200
gradient_clip_norm = 5
# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))
# download penn treebank
train_path = load_text("ptb-train", path=args.data_dir)
valid_path = load_text("ptb-valid", path=args.data_dir)
# load data and parse on word-level
# a Text object can take a given tokenizer, for word-level parsing, it is str.split
# a user can pass in a custom-defined tokenzier as well
tokenizer = lambda s: s.replace("\n", "<eos>").split()
train_set = Text(time_steps, train_path, tokenizer=tokenizer, onehot_input=False)
valid_set = Text(time_steps, valid_path, vocab=train_set.vocab, tokenizer=tokenizer, onehot_input=False)
# weight initialization
init = Uniform(low=-0.1, high=0.1)
# model initialization
rlayer_params = {"output_size": hidden_size, "init": init, "activation": Tanh(), "gate_activation": Logistic()}
if args.rlayer_type == "lstm":